├── ANN ├── DUDL_ANN_breadthVsDepth.ipynb ├── DUDL_ANN_classifyQwerties.ipynb ├── DUDL_ANN_codeChallengeQwerties.ipynb ├── DUDL_ANN_codeChallengeSeq2class.ipynb ├── DUDL_ANN_codeChallenge_regression.ipynb ├── DUDL_ANN_learningrates.ipynb ├── DUDL_ANN_multilayer.ipynb ├── DUDL_ANN_multioutput.ipynb ├── DUDL_ANN_nHiddenUnits.ipynb ├── DUDL_ANN_numParameters.ipynb ├── DUDL_ANN_regression.ipynb └── DUDL_ANN_seqVsClass.ipynb ├── CNN ├── DUDL_CNN_CNN4MNIST.ipynb ├── DUDL_CNN_CodeChallengeAEocclusion.ipynb ├── DUDL_CNN_CodeChallengeLinearUnits.ipynb ├── DUDL_CNN_EMNIST.ipynb ├── DUDL_CNN_GaussAE.ipynb ├── DUDL_CNN_GaussClass.ipynb ├── DUDL_CNN_GaussClassFeatureMaps.ipynb ├── DUDL_CNN_codeChallengeBeatThis.ipynb ├── DUDL_CNN_codeChallengeCustomLoss.ipynb ├── DUDL_CNN_codeChallengeNumChans.ipynb ├── DUDL_CNN_codeChallengeSoftcoding.ipynb ├── DUDL_CNN_findGauss.ipynb └── DUDL_CNN_shiftedMNIST.ipynb ├── CNNmilestone ├── DUDL_CNNmilestone_project1.ipynb ├── DUDL_CNNmilestone_project2.ipynb ├── DUDL_CNNmilestone_project3.ipynb └── DUDL_CNNmilestone_project4.ipynb ├── FFN ├── DUDL_FFN_CodeChallengeBreadthDepth.ipynb ├── DUDL_FFN_CodeChallenge_binMNIST.ipynb ├── DUDL_FFN_CodeChallenge_missing7.ipynb ├── DUDL_FFN_CodeChallenge_normalization.ipynb ├── DUDL_FFN_CodeChallenge_optimizers.ipynb ├── DUDL_FFN_FFNonMNIST.ipynb ├── DUDL_FFN_aboutMNIST.ipynb ├── DUDL_FFN_scrambledMNIST.ipynb ├── DUDL_FFN_shiftedMNIST.ipynb └── DUDL_FFN_weightHistograms.ipynb ├── FFNmilestone ├── DUDL_FFNmilestone_project1.ipynb ├── DUDL_FFNmilestone_project2.ipynb └── DUDL_FFNmilestone_project3.ipynb ├── GANs ├── DUDL_GAN_CNNganFMNIST.ipynb ├── DUDL_GAN_CNNganGaus.ipynb ├── DUDL_GAN_MNIST.ipynb ├── DUDL_GAN_codeChallengeCIFAR.ipynb ├── DUDL_GAN_codeChallengeFMNIST.ipynb ├── DUDL_GAN_codeChallengeFaces.ipynb └── DUDL_GAN_codeChallengeGaus.ipynb ├── GPU ├── DUDL_GPU_CodeChallenge2GPU.ipynb └── DUDL_GPU_implement.ipynb ├── LICENSE ├── README.md ├── RNN ├── DUDL_RNN_LSTMGRU.ipynb ├── DUDL_RNN_altSequences.ipynb ├── DUDL_RNN_codeChallenge_SineExtrapolate.ipynb ├── DUDL_RNN_intro2RNN.ipynb └── DUDL_RNN_loremipsum.ipynb ├── autoencoders ├── DUDL_autoenc_MNISTlatentCode.ipynb ├── DUDL_autoenc_codeChallenge_Nunits.ipynb ├── DUDL_autoenc_denoisingMNIST.ipynb ├── DUDL_autoenc_occlusion.ipynb └── DUDL_autoenc_tiedWeights.ipynb ├── convolution ├── DUDL_convolution_codeChallenge.ipynb ├── DUDL_convolution_conv2.ipynb ├── DUDL_convolution_conv2transpose.ipynb ├── DUDL_convolution_convInCode.ipynb ├── DUDL_convolution_customDataSet.ipynb ├── DUDL_convolution_meanMaxPool.ipynb └── DUDL_convolution_transforms.ipynb ├── data ├── DUDL_data_CodeChallengeUnbalanced.ipynb ├── DUDL_data_data2colab.ipynb ├── DUDL_data_dataVsDepth.ipynb ├── DUDL_data_datasetLoader.ipynb ├── DUDL_data_featureAugmentation.ipynb ├── DUDL_data_noiseAugmentation.ipynb ├── DUDL_data_oversampling.ipynb ├── DUDL_data_saveLoadModels.ipynb └── DUDL_data_saveTheBest.ipynb ├── gradientDescent ├── DUDL_GradientDescent_1D.ipynb ├── DUDL_GradientDescent_2D.ipynb ├── DUDL_GradientDescent_CodeChallengeStartValue.ipynb ├── DUDL_GradientDescent_codeChallenge_lr.ipynb └── DUDL_GradientDescent_experiment.ipynb ├── math ├── DUDL_math_argmin.ipynb ├── DUDL_math_derivatives1.ipynb ├── DUDL_math_derivatives2.ipynb ├── DUDL_math_dotproduct.ipynb ├── DUDL_math_entropy.ipynb ├── DUDL_math_log.ipynb ├── DUDL_math_matrixMult.ipynb ├── DUDL_math_meanvar.ipynb ├── DUDL_math_randomseed.ipynb ├── DUDL_math_sampling.ipynb ├── DUDL_math_softmax.ipynb ├── DUDL_math_transpose.ipynb └── DUDL_math_ttest.ipynb ├── measurePerformance ├── DUDL_measurePerformance_APRF.ipynb ├── DUDL_measurePerformance_APRFexample1.ipynb ├── DUDL_measurePerformance_codeChallenge_unequal.ipynb ├── DUDL_measurePerformance_example2.ipynb └── DUDL_measurePerformance_time.ipynb ├── metaparams ├── DUDL_metaparams_ActivationComparisons.ipynb ├── DUDL_metaparams_ActivationFuns.ipynb ├── DUDL_metaparams_CodeChallengeAdamL2.ipynb ├── DUDL_metaparams_CodeChallengeBatches.ipynb ├── DUDL_metaparams_CodeChallengeOptimizers.ipynb ├── DUDL_metaparams_CodeChallengeRelus.ipynb ├── DUDL_metaparams_CodeChallenge_sugar.ipynb ├── DUDL_metaparams_batchNorm.ipynb ├── DUDL_metaparams_codeChallengeDropout.ipynb ├── DUDL_metaparams_intro2winedata.ipynb ├── DUDL_metaparams_learningRateDecay.ipynb ├── DUDL_metaparams_loss.ipynb ├── DUDL_metaparams_momentum.ipynb ├── DUDL_metaparams_multioutput.ipynb └── DUDL_metaparams_optimizersComparison.ipynb ├── overfitting ├── DUDL_overfitting_dataLoader.ipynb ├── DUDL_overfitting_manual.ipynb ├── DUDL_overfitting_regression.ipynb ├── DUDL_overfitting_scikitlearn.ipynb └── DUDL_overfitting_trainDevsetTest.ipynb ├── regularization ├── DUDL_regular_L1regu.ipynb ├── DUDL_regular_L2regu.ipynb ├── DUDL_regular_codeChallenge_minibatch.ipynb ├── DUDL_regular_dropout.ipynb ├── DUDL_regular_dropoutInPytorch.ipynb ├── DUDL_regular_dropout_example2.ipynb ├── DUDL_regular_minibatch.ipynb └── DUDL_regular_testBatchT2.ipynb ├── styletransfer ├── DUDL_style_codeChallengeAlexNet.ipynb └── DUDL_style_screamingBathtub.ipynb ├── transferlearning ├── DUDL_transfer_MNISTtoFMNIST.ipynb ├── DUDL_transfer_PretrainCIFAR.ipynb ├── DUDL_transfer_codeChallengeVGG16.ipynb ├── DUDL_transfer_codeChallenge_letters2numbers.ipynb ├── DUDL_transfer_pretrainFMNIST.ipynb └── DUDL_transfer_resnet.ipynb └── weights ├── DUDL_weights_CodeChallenge_XavierKaiming.ipynb ├── DUDL_weights_XavierKaiming.ipynb ├── DUDL_weights_codeChallenge_identicalRandom.ipynb ├── DUDL_weights_codeChallenge_weightstd.ipynb ├── DUDL_weights_demoinits.ipynb ├── DUDL_weights_freezeWeights.ipynb ├── DUDL_weights_matrixsizes.ipynb └── DUDL_weights_weightchanges.ipynb /ANN/DUDL_ANN_classifyQwerties.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_ANN_classifyQwerties.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 10 | "timestamp": 1615877547147 11 | } 12 | ], 13 | "collapsed_sections": [], 14 | "authorship_tag": "ABX9TyO/QzjA/jBOs2J65rwY19ZO" 15 | }, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3" 19 | } 20 | }, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "bhWV8oes-wKR" 26 | }, 27 | "source": [ 28 | "# COURSE: A deep understanding of deep learning\n", 29 | "## SECTION: ANNs\n", 30 | "### LECTURE: ANN for classifying qwerties\n", 31 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 32 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "metadata": { 38 | "id": "j7-LiwqUMGYL" 39 | }, 40 | "source": [ 41 | "# import libraries\n", 42 | "import torch\n", 43 | "import torch.nn as nn\n", 44 | "import numpy as np\n", 45 | "\n", 46 | "import matplotlib.pyplot as plt\n", 47 | "import matplotlib_inline.backend_inline\n", 48 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 49 | ], 50 | "execution_count": null, 51 | "outputs": [] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "metadata": { 56 | "id": "j-SP8NPsMNRL" 57 | }, 58 | "source": [ 59 | "# create data\n", 60 | "\n", 61 | "nPerClust = 100\n", 62 | "blur = 1\n", 63 | "\n", 64 | "A = [ 1, 1 ]\n", 65 | "B = [ 5, 1 ]\n", 66 | "\n", 67 | "# generate data\n", 68 | "a = [ A[0]+np.random.randn(nPerClust)*blur , A[1]+np.random.randn(nPerClust)*blur ]\n", 69 | "b = [ B[0]+np.random.randn(nPerClust)*blur , B[1]+np.random.randn(nPerClust)*blur ]\n", 70 | "\n", 71 | "# true labels\n", 72 | "labels_np = np.vstack((np.zeros((nPerClust,1)),np.ones((nPerClust,1))))\n", 73 | "\n", 74 | "# concatanate into a matrix\n", 75 | "data_np = np.hstack((a,b)).T\n", 76 | "\n", 77 | "# convert to a pytorch tensor\n", 78 | "data = torch.tensor(data_np).float()\n", 79 | "labels = torch.tensor(labels_np).float()\n", 80 | "\n", 81 | "# show the data\n", 82 | "fig = plt.figure(figsize=(5,5))\n", 83 | "plt.plot(data[np.where(labels==0)[0],0],data[np.where(labels==0)[0],1],'bs')\n", 84 | "plt.plot(data[np.where(labels==1)[0],0],data[np.where(labels==1)[0],1],'ko')\n", 85 | "plt.title('The qwerties!')\n", 86 | "plt.xlabel('qwerty dimension 1')\n", 87 | "plt.ylabel('qwerty dimension 2')\n", 88 | "plt.show()" 89 | ], 90 | "execution_count": null, 91 | "outputs": [] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "metadata": { 96 | "id": "yfAFHG9a3MzF" 97 | }, 98 | "source": [ 99 | "# inspect types\n", 100 | "print(type(data_np))\n", 101 | "print(np.shape(data_np))\n", 102 | "print(' ')\n", 103 | "\n", 104 | "print(type(data))\n", 105 | "print(np.shape(data))" 106 | ], 107 | "execution_count": null, 108 | "outputs": [] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "metadata": { 113 | "id": "krQeh5wYMNla" 114 | }, 115 | "source": [ 116 | "# build the model\n", 117 | "ANNclassify = nn.Sequential(\n", 118 | " nn.Linear(2,1), # input layer\n", 119 | " nn.ReLU(), # activation unit\n", 120 | " nn.Linear(1,1), # output unit\n", 121 | " nn.Sigmoid(), # final activation unit (here for conceptual reasons; in practice, better to use BCEWithLogitsLoss)\n", 122 | " )\n", 123 | "\n", 124 | "ANNclassify" 125 | ], 126 | "execution_count": null, 127 | "outputs": [] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "metadata": { 132 | "id": "dmHh7GrvMNoy" 133 | }, 134 | "source": [ 135 | "# other model features\n", 136 | "\n", 137 | "learningRate = .01\n", 138 | "\n", 139 | "# loss function\n", 140 | "lossfun = nn.BCELoss()\n", 141 | "# Note: You'll learn in the \"Metaparameters\" section that it's better to use BCEWithLogitsLoss, but this is OK for now.\n", 142 | "\n", 143 | "# optimizer\n", 144 | "optimizer = torch.optim.SGD(ANNclassify.parameters(),lr=learningRate)\n" 145 | ], 146 | "execution_count": null, 147 | "outputs": [] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "of9E8ClxMNsD" 153 | }, 154 | "source": [ 155 | "# train the model\n", 156 | "numepochs = 1000\n", 157 | "losses = torch.zeros(numepochs)\n", 158 | "\n", 159 | "for epochi in range(numepochs):\n", 160 | "\n", 161 | " # forward pass\n", 162 | " yHat = ANNclassify(data)\n", 163 | "\n", 164 | " # compute loss\n", 165 | " loss = lossfun(yHat,labels)\n", 166 | " losses[epochi] = loss\n", 167 | "\n", 168 | " # backprop\n", 169 | " optimizer.zero_grad()\n", 170 | " loss.backward()\n", 171 | " optimizer.step()" 172 | ], 173 | "execution_count": null, 174 | "outputs": [] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "metadata": { 179 | "id": "zmX6K49WMNuy" 180 | }, 181 | "source": [ 182 | "# show the losses\n", 183 | "\n", 184 | "plt.plot(losses.detach(),'o',markerfacecolor='w',linewidth=.1)\n", 185 | "plt.xlabel('Epoch')\n", 186 | "plt.ylabel('Loss')\n", 187 | "plt.show()" 188 | ], 189 | "execution_count": null, 190 | "outputs": [] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "metadata": { 195 | "id": "i1TCt0mpMNxC" 196 | }, 197 | "source": [ 198 | "# compute the predictions\n", 199 | "\n", 200 | "# manually compute losses\n", 201 | "# final forward pass\n", 202 | "predictions = ANNclassify(data)\n", 203 | "\n", 204 | "predlabels = predictions>.5\n", 205 | "\n", 206 | "# find errors\n", 207 | "misclassified = np.where(predlabels != labels)[0]\n", 208 | "\n", 209 | "# total accuracy\n", 210 | "totalacc = 100-100*len(misclassified)/(2*nPerClust)\n", 211 | "\n", 212 | "print('Final accuracy: %g%%' %totalacc)\n" 213 | ], 214 | "execution_count": null, 215 | "outputs": [] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "metadata": { 220 | "id": "q0UnOnNVB8Xb" 221 | }, 222 | "source": [ 223 | "# plot the labeled data\n", 224 | "fig = plt.figure(figsize=(5,5))\n", 225 | "plt.plot(data[misclassified,0] ,data[misclassified,1],'rx',markersize=12,markeredgewidth=3)\n", 226 | "plt.plot(data[np.where(~predlabels)[0],0],data[np.where(~predlabels)[0],1],'bs')\n", 227 | "plt.plot(data[np.where(predlabels)[0],0] ,data[np.where(predlabels)[0],1] ,'ko')\n", 228 | "\n", 229 | "plt.legend(['Misclassified','blue','black'],bbox_to_anchor=(1,1))\n", 230 | "plt.title(f'{totalacc}% correct')\n", 231 | "plt.show()" 232 | ], 233 | "execution_count": null, 234 | "outputs": [] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "metadata": { 239 | "id": "YTHj0h7GEVMq" 240 | }, 241 | "source": [ 242 | "" 243 | ], 244 | "execution_count": null, 245 | "outputs": [] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": { 250 | "id": "JmraVzTcJ0x1" 251 | }, 252 | "source": [ 253 | "# Additional explorations" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "metadata": { 259 | "id": "pml6nCTcAMWC" 260 | }, 261 | "source": [ 262 | "# 1) It is common in DL to train the model for a specified number of epochs. But you can also train until\n", 263 | "# the model reaches a certain accuracy criterion. Re-write the code so that the model continues training\n", 264 | "# until it reaches 90% accuracy.\n", 265 | "# What would happen if the model falls into a local minimum and never reaches 90% accuracy? Yikes! You can\n", 266 | "# force-quit a process in google-colab by clicking on the top-left 'play' button of a code cell.\n", 267 | "# \n", 268 | "# 2) It is intuitive that the model can reach 100% accuracy if the qwerties are more separable. Modify the \n", 269 | "# qwerty-generating code to get the model to have 100% classification accuracy.\n", 270 | "# " 271 | ], 272 | "execution_count": null, 273 | "outputs": [] 274 | } 275 | ] 276 | } -------------------------------------------------------------------------------- /ANN/DUDL_ANN_nHiddenUnits.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 9 | "timestamp": 1615925514977 10 | } 11 | ] 12 | }, 13 | "kernelspec": { 14 | "name": "python3", 15 | "display_name": "Python 3" 16 | } 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "bhWV8oes-wKR" 23 | }, 24 | "source": [ 25 | "# COURSE: A deep understanding of deep learning\n", 26 | "## SECTION: ANNs\n", 27 | "### LECTURE: Comparing the number of hidden units\n", 28 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 29 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "metadata": { 35 | "id": "YeuAheYyhdZw" 36 | }, 37 | "source": [ 38 | "# import libraries\n", 39 | "import torch\n", 40 | "import torch.nn as nn\n", 41 | "\n", 42 | "import numpy as np\n", 43 | "import matplotlib.pyplot as plt" 44 | ], 45 | "execution_count": null, 46 | "outputs": [] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": { 51 | "id": "ia1C-oGK4uk1" 52 | }, 53 | "source": [ 54 | "# Import and organize the data" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "metadata": { 60 | "id": "MU7rvmWuhjud" 61 | }, 62 | "source": [ 63 | "# import dataset\n", 64 | "import pandas as pd\n", 65 | "iris = pd.read_csv('https://raw.githubusercontent.com/mwaskom/seaborn-data/master/iris.csv')\n", 66 | "print( iris.head() )\n", 67 | "\n", 68 | "# some plots to show the data\n", 69 | "sns.pairplot(iris, hue='species')\n", 70 | "plt.show()" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "metadata": { 78 | "id": "vJPkH6Bfh01_" 79 | }, 80 | "source": [ 81 | "# organize the data\n", 82 | "\n", 83 | "# convert from pandas dataframe to tensor\n", 84 | "data = torch.tensor( iris[iris.columns[0:4]].values ).float()\n", 85 | "\n", 86 | "# transform species to number\n", 87 | "labels = torch.zeros(len(data), dtype=torch.long)\n", 88 | "# labels[iris.species=='setosa'] = 0 # don't need!\n", 89 | "labels[iris.species=='versicolor'] = 1\n", 90 | "labels[iris.species=='virginica'] = 2\n", 91 | "\n", 92 | "labels" 93 | ], 94 | "execution_count": null, 95 | "outputs": [] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": { 100 | "id": "0CFW7ywE4w_u" 101 | }, 102 | "source": [ 103 | "# Functions to create and train the model" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "v0JMIGb1iV_9" 110 | }, 111 | "source": [ 112 | "# Note the input into the function!\n", 113 | "def createIrisModel(nHidden):\n", 114 | "\n", 115 | " # model architecture (with number of units soft-coded!)\n", 116 | " ANNiris = nn.Sequential(\n", 117 | " nn.Linear(4,nHidden), # input layer\n", 118 | " nn.ReLU(), # activation unit\n", 119 | " nn.Linear(nHidden,nHidden),# hidden layer\n", 120 | " nn.ReLU(), # activation unit\n", 121 | " nn.Linear(nHidden,3), # output unit\n", 122 | " #nn.Softmax(dim=1), # final activation unit (here for conceptual purposes, note the CEL function)\n", 123 | " )\n", 124 | "\n", 125 | " # loss function\n", 126 | " lossfun = nn.CrossEntropyLoss()\n", 127 | "\n", 128 | " # optimizer\n", 129 | " optimizer = torch.optim.SGD(ANNiris.parameters(),lr=.01)\n", 130 | "\n", 131 | " return ANNiris,lossfun,optimizer" 132 | ], 133 | "execution_count": null, 134 | "outputs": [] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "metadata": { 139 | "id": "cVD1nFTli7TO" 140 | }, 141 | "source": [ 142 | "# a function to train the model\n", 143 | "\n", 144 | "def trainTheModel(ANNiris):\n", 145 | "\n", 146 | " # initialize losses\n", 147 | " losses = torch.zeros(numepochs)\n", 148 | " ongoingAcc = []\n", 149 | "\n", 150 | " # loop over epochs\n", 151 | " for epochi in range(numepochs):\n", 152 | "\n", 153 | " # forward pass\n", 154 | " yHat = ANNiris(data)\n", 155 | "\n", 156 | " # compute loss\n", 157 | " loss = lossfun(yHat,labels)\n", 158 | " losses[epochi] = loss\n", 159 | "\n", 160 | " # backprop\n", 161 | " optimizer.zero_grad()\n", 162 | " loss.backward()\n", 163 | " optimizer.step()\n", 164 | "\n", 165 | " # final forward pass\n", 166 | " predictions = ANNiris(data)\n", 167 | "\n", 168 | " predlabels = torch.argmax(predictions,axis=1)\n", 169 | " return 100*torch.mean((predlabels==labels).float())" 170 | ], 171 | "execution_count": null, 172 | "outputs": [] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "metadata": { 177 | "id": "zm80pv3J48d1" 178 | }, 179 | "source": [ 180 | "# Run the experiment!" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "metadata": { 186 | "id": "hWOzwwHTrdxz" 187 | }, 188 | "source": [ 189 | "numepochs = 150\n", 190 | "numhiddens = np.arange(1,129)\n", 191 | "accuracies = []\n", 192 | "\n", 193 | "for nunits in numhiddens:\n", 194 | "\n", 195 | " # create a fresh model instance\n", 196 | " ANNiris,lossfun,optimizer = createIrisModel(nunits)\n", 197 | "\n", 198 | " # run the model\n", 199 | " acc = trainTheModel(ANNiris)\n", 200 | " accuracies.append( acc )\n" 201 | ], 202 | "execution_count": null, 203 | "outputs": [] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "metadata": { 208 | "id": "JYouZAY4i3jM" 209 | }, 210 | "source": [ 211 | "# report accuracy\n", 212 | "fig,ax = plt.subplots(1,figsize=(12,6))\n", 213 | "\n", 214 | "ax.plot(accuracies,'ko-',markerfacecolor='w',markersize=9)\n", 215 | "ax.plot(numhiddens[[0,-1]],[33,33],'--',color=[.8,.8,.8])\n", 216 | "ax.plot(numhiddens[[0,-1]],[67,67],'--',color=[.8,.8,.8])\n", 217 | "ax.set_ylabel('accuracy')\n", 218 | "ax.set_xlabel('Number of hidden units')\n", 219 | "ax.set_title('Accuracy')\n", 220 | "plt.show()\n" 221 | ], 222 | "execution_count": null, 223 | "outputs": [] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "metadata": { 228 | "id": "zvLH4h6ek5Ax" 229 | }, 230 | "source": [], 231 | "execution_count": null, 232 | "outputs": [] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": { 237 | "id": "T7MKj66sk5DY" 238 | }, 239 | "source": [ 240 | "# Additional explorations" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "metadata": { 246 | "id": "zIXi3fONIKVA" 247 | }, 248 | "source": [ 249 | "# 1) The results here show that models with fewer than ~50 hidden units have lackluster performance. Would these models\n", 250 | "# eventually learn if they were given more training epochs? Try this by re-running the experiment using 500 epochs.\n", 251 | "# Tip: Copy/paste the plotting code into a new cell to keep both plots. Or, take screenshots of the plots.\n", 252 | "#\n", 253 | "# 2) Going back to 150 epochs, explore the effect of changing the learning rate. This doesn't need to be a full parametric\n", 254 | "# experiment; you can simply try is again using learning rates of .1, .01 (what we used in the video), and .001.\n", 255 | "#\n", 256 | "# 3) With simple models and small datasets, it's possible to test many different parameter settings. However, larger\n", 257 | "# models take longer to train, and so running 128 tests is not always feasible. Modify the code to have the number of\n", 258 | "# hidden units range from 1 to 128 in steps of 14. Plot the results on top of the results using steps of 1 (that is,\n", 259 | "# show both results in the same graph). Does your interpretation change with fewer experiment runs?\n" 260 | ], 261 | "execution_count": null, 262 | "outputs": [] 263 | } 264 | ] 265 | } -------------------------------------------------------------------------------- /ANN/DUDL_ANN_numParameters.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_ANN_numParameters.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1Q_oDw0aMA4QFKDnLxuqJp62P8oPMtO1R", 10 | "timestamp": 1618255245074 11 | }, 12 | { 13 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 14 | "timestamp": 1615884593383 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyNp0/OYNZqgbUCz8Xrl2dyW" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | } 24 | }, 25 | "cells": [ 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "id": "bhWV8oes-wKR" 30 | }, 31 | "source": [ 32 | "# COURSE: A deep understanding of deep learning\n", 33 | "## SECTION: ANNs\n", 34 | "### LECTURE: Depth vs. breadth: number of parameters\n", 35 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 36 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "j7-LiwqUMGYL" 43 | }, 44 | "source": [ 45 | "# import libraries\n", 46 | "import numpy as np\n", 47 | "import torch\n", 48 | "import torch.nn as nn" 49 | ], 50 | "execution_count": null, 51 | "outputs": [] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "metadata": { 56 | "id": "krQeh5wYMNla" 57 | }, 58 | "source": [ 59 | "# build two models\n", 60 | "\n", 61 | "widenet = nn.Sequential(\n", 62 | " nn.Linear(2,4), # hidden layer\n", 63 | " nn.Linear(4,3), # output layer\n", 64 | " )\n", 65 | "\n", 66 | "\n", 67 | "deepnet = nn.Sequential(\n", 68 | " nn.Linear(2,2), # hidden layer\n", 69 | " nn.Linear(2,2), # hidden layer\n", 70 | " nn.Linear(2,3), # output layer\n", 71 | " )\n", 72 | "\n", 73 | "# print them out to have a look\n", 74 | "print(widenet)\n", 75 | "print(' ')\n", 76 | "print(deepnet)" 77 | ], 78 | "execution_count": null, 79 | "outputs": [] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "metadata": { 84 | "id": "Rv5g3ISypDNk" 85 | }, 86 | "source": [ 87 | "widenet." 88 | ], 89 | "execution_count": null, 90 | "outputs": [] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": { 95 | "id": "ni8L4jRgopMO" 96 | }, 97 | "source": [ 98 | "# Peeking inside the network" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "metadata": { 104 | "id": "lrKmii4Xmx-Z" 105 | }, 106 | "source": [ 107 | "# check out the parameters\n", 108 | "for p in deepnet.named_parameters():\n", 109 | " print(p)\n", 110 | " print(' ')" 111 | ], 112 | "execution_count": null, 113 | "outputs": [] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "metadata": { 118 | "id": "I811amwtouaY" 119 | }, 120 | "source": [ 121 | "# count the number of nodes ( = the number of biases)\n", 122 | "\n", 123 | "# named_parameters() is an iterable that returns the tuple (name,numbers)\n", 124 | "numNodesInWide = 0\n", 125 | "for p in widenet.named_parameters():\n", 126 | " if 'bias' in p[0]:\n", 127 | " numNodesInWide += len(p[1])\n", 128 | "\n", 129 | "numNodesInDeep = 0\n", 130 | "for paramName,paramVect in deepnet.named_parameters():\n", 131 | " if 'bias' in paramName:\n", 132 | " numNodesInDeep += len(paramVect)\n", 133 | "\n", 134 | "\n", 135 | "print('There are %s nodes in the wide network.' %numNodesInWide)\n", 136 | "print('There are %s nodes in the deep network.' %numNodesInDeep)" 137 | ], 138 | "execution_count": null, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "FVuYUMy7spW9" 145 | }, 146 | "source": [ 147 | "# just the parameters\n", 148 | "for p in widenet.parameters():\n", 149 | " print(p)\n", 150 | " print(' ')" 151 | ], 152 | "execution_count": null, 153 | "outputs": [] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "metadata": { 158 | "id": "xtTwxsVhirEq" 159 | }, 160 | "source": [ 161 | "# now count the total number of trainable parameters\n", 162 | "nparams = 0\n", 163 | "for p in widenet.parameters():\n", 164 | " if p.requires_grad:\n", 165 | " print('This piece has %s parameters' %p.numel())\n", 166 | " nparams += p.numel()\n", 167 | "\n", 168 | "print('\\n\\nTotal of %s parameters'%nparams)" 169 | ], 170 | "execution_count": null, 171 | "outputs": [] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "metadata": { 176 | "id": "PKr2ARdWivz8" 177 | }, 178 | "source": [ 179 | "# btw, can also use list comprehension\n", 180 | "\n", 181 | "nparams = np.sum([ p.numel() for p in widenet.parameters() if p.requires_grad ])\n", 182 | "print('Widenet has %s parameters'%nparams)\n", 183 | "\n", 184 | "nparams = np.sum([ p.numel() for p in deepnet.parameters() if p.requires_grad ])\n", 185 | "print('Deepnet has %s parameters'%nparams)" 186 | ], 187 | "execution_count": null, 188 | "outputs": [] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "metadata": { 193 | "id": "I9wsTcbrrYT7" 194 | }, 195 | "source": [ 196 | "" 197 | ], 198 | "execution_count": null, 199 | "outputs": [] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "metadata": { 204 | "id": "_6GzhyxLUrYy" 205 | }, 206 | "source": [ 207 | "# A nice simple way to print out the model info.\n", 208 | "from torchsummary import summary\n", 209 | "summary(widenet,(1,2))\n", 210 | "\n", 211 | "\n", 212 | "### NOTE ABOUT THE CODE IN THIS CELL:\n", 213 | "# torchsummary is being replaced by torchinfo.\n", 214 | "# If you are importing these libraries on your own (via pip), then see the following website:\n", 215 | "# https://pypi.org/project/torch-summary/\n", 216 | "# However, torchsummary will continue to be supported, so if the code in this cell works (meaning torchsummary is already installed), \n", 217 | "# then you don't need to do anything!" 218 | ], 219 | "execution_count": null, 220 | "outputs": [] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "metadata": { 225 | "id": "KHlWL3_drYhT" 226 | }, 227 | "source": [ 228 | "" 229 | ], 230 | "execution_count": null, 231 | "outputs": [] 232 | } 233 | ] 234 | } -------------------------------------------------------------------------------- /ANN/DUDL_ANN_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_ANN_regression.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 10 | "timestamp": 1615884593383 11 | } 12 | ], 13 | "collapsed_sections": [], 14 | "authorship_tag": "ABX9TyOfbw74CIER4uMxvaGa9RbX" 15 | }, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3" 19 | } 20 | }, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "bhWV8oes-wKR" 26 | }, 27 | "source": [ 28 | "# COURSE: A deep understanding of deep learning\n", 29 | "## SECTION: ANNs\n", 30 | "### LECTURE: ANN for regression\n", 31 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 32 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "metadata": { 38 | "id": "j7-LiwqUMGYL" 39 | }, 40 | "source": [ 41 | "# import libraries\n", 42 | "import numpy as np\n", 43 | "import torch\n", 44 | "import torch.nn as nn\n", 45 | "import matplotlib.pyplot as plt\n", 46 | "import matplotlib_inline.backend_inline\n", 47 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "metadata": { 55 | "id": "j-SP8NPsMNRL" 56 | }, 57 | "source": [ 58 | "# create data\n", 59 | "\n", 60 | "N = 30\n", 61 | "x = torch.randn(N,1)\n", 62 | "y = x + torch.randn(N,1)/2\n", 63 | "\n", 64 | "# and plot\n", 65 | "plt.plot(x,y,'s')\n", 66 | "plt.show()" 67 | ], 68 | "execution_count": null, 69 | "outputs": [] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "metadata": { 74 | "id": "krQeh5wYMNla" 75 | }, 76 | "source": [ 77 | "# build model\n", 78 | "ANNreg = nn.Sequential(\n", 79 | " nn.Linear(1,1), # input layer\n", 80 | " nn.ReLU(), # activation function\n", 81 | " nn.Linear(1,1) # output layer\n", 82 | " )\n", 83 | "\n", 84 | "ANNreg" 85 | ], 86 | "execution_count": null, 87 | "outputs": [] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "metadata": { 92 | "id": "dmHh7GrvMNoy" 93 | }, 94 | "source": [ 95 | "# learning rate\n", 96 | "learningRate = .05\n", 97 | "\n", 98 | "# loss function\n", 99 | "lossfun = nn.MSELoss()\n", 100 | "\n", 101 | "# optimizer (the flavor of gradient descent to implement)\n", 102 | "optimizer = torch.optim.SGD(ANNreg.parameters(),lr=learningRate)" 103 | ], 104 | "execution_count": null, 105 | "outputs": [] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "metadata": { 110 | "id": "of9E8ClxMNsD" 111 | }, 112 | "source": [ 113 | "# train the model\n", 114 | "numepochs = 500\n", 115 | "losses = torch.zeros(numepochs)\n", 116 | "\n", 117 | "\n", 118 | "## Train the model!\n", 119 | "for epochi in range(numepochs):\n", 120 | "\n", 121 | " # forward pass\n", 122 | " yHat = ANNreg(x)\n", 123 | "\n", 124 | " # compute loss\n", 125 | " loss = lossfun(yHat,y)\n", 126 | " losses[epochi] = loss\n", 127 | "\n", 128 | " # backprop\n", 129 | " optimizer.zero_grad()\n", 130 | " loss.backward()\n", 131 | " optimizer.step()" 132 | ], 133 | "execution_count": null, 134 | "outputs": [] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "metadata": { 139 | "id": "zmX6K49WMNuy" 140 | }, 141 | "source": [ 142 | "# show the losses\n", 143 | "\n", 144 | "# manually compute losses\n", 145 | "# final forward pass\n", 146 | "predictions = ANNreg(x)\n", 147 | "\n", 148 | "# final loss (MSE)\n", 149 | "testloss = (predictions-y).pow(2).mean()\n", 150 | "\n", 151 | "plt.plot(losses.detach(),'o',markerfacecolor='w',linewidth=.1)\n", 152 | "plt.plot(numepochs,testloss.detach(),'ro')\n", 153 | "plt.xlabel('Epoch')\n", 154 | "plt.ylabel('Loss')\n", 155 | "plt.title('Final loss = %g' %testloss.item())\n", 156 | "plt.show()" 157 | ], 158 | "execution_count": null, 159 | "outputs": [] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "metadata": { 164 | "id": "Lv_-DTq1LlDH" 165 | }, 166 | "source": [ 167 | "testloss.item()" 168 | ], 169 | "execution_count": null, 170 | "outputs": [] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "i1TCt0mpMNxC" 176 | }, 177 | "source": [ 178 | "# plot the data\n", 179 | "plt.plot(x,y,'bo',label='Real data')\n", 180 | "plt.plot(x,predictions.detach(),'rs',label='Predictions')\n", 181 | "plt.title(f'prediction-data r={np.corrcoef(y.T,predictions.detach().T)[0,1]:.2f}')\n", 182 | "plt.legend()\n", 183 | "plt.show()" 184 | ], 185 | "execution_count": null, 186 | "outputs": [] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "metadata": { 191 | "id": "vGkQUzqhUFpq" 192 | }, 193 | "source": [ 194 | "" 195 | ], 196 | "execution_count": null, 197 | "outputs": [] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": { 202 | "id": "JmraVzTcJ0x1" 203 | }, 204 | "source": [ 205 | "# Additional explorations" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "metadata": { 211 | "id": "pml6nCTcAMWC" 212 | }, 213 | "source": [ 214 | "# 1) How much data is \"enough\"? Try different values of N and see how low the loss gets. \n", 215 | "# Do you still get low loss (\"low\" is subjective, but let's say loss<.25) with N=10? N=5?\n", 216 | "# \n", 217 | "# 2) Does your conclusion above depend on the amount of noise in the data? Try changing the noise level\n", 218 | "# by changing the division (\"/2\") when creating y as x+randn.\n", 219 | "# \n", 220 | "# 3) Notice that the model doesn't always work well. Put the original code (that is, N=30 and /2 noise)\n", 221 | "# into a function or a for-loop and repeat the training 100 times (each time using a fresh model instance).\n", 222 | "# Then count the number of times the model had a loss>.25." 223 | ], 224 | "execution_count": null, 225 | "outputs": [] 226 | } 227 | ] 228 | } -------------------------------------------------------------------------------- /ANN/DUDL_ANN_seqVsClass.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_ANN_seqVsClass.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "10_geQnah5AvMsm8VDAQwNPhypOXradar", 10 | "timestamp": 1619718234949 11 | }, 12 | { 13 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 14 | "timestamp": 1615877547147 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyOAXrvptj+kO5+E69TV1nSi" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | } 24 | }, 25 | "cells": [ 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "id": "bhWV8oes-wKR" 30 | }, 31 | "source": [ 32 | "# COURSE: A deep understanding of deep learning\n", 33 | "## SECTION: ANNs\n", 34 | "### LECTURE: Defining models using sequential vs. class\n", 35 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 36 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "tyIa-7gXu76k" 43 | }, 44 | "source": [ 45 | "# NOTE: copied from notebook DUDL_ANN_classifyQwerties.ipynb" 46 | ], 47 | "execution_count": null, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "metadata": { 53 | "id": "j7-LiwqUMGYL" 54 | }, 55 | "source": [ 56 | "# import libraries\n", 57 | "import torch\n", 58 | "import torch.nn as nn\n", 59 | "import numpy as np\n", 60 | "\n", 61 | "# NEW!\n", 62 | "import torch.nn.functional as F\n", 63 | "\n", 64 | "import matplotlib.pyplot as plt\n", 65 | "import matplotlib_inline.backend_inline\n", 66 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 67 | ], 68 | "execution_count": null, 69 | "outputs": [] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "metadata": { 74 | "id": "j-SP8NPsMNRL" 75 | }, 76 | "source": [ 77 | "# create data\n", 78 | "\n", 79 | "nPerClust = 100\n", 80 | "blur = 1\n", 81 | "\n", 82 | "A = [ 1, 1 ]\n", 83 | "B = [ 5, 1 ]\n", 84 | "\n", 85 | "# generate data\n", 86 | "a = [ A[0]+np.random.randn(nPerClust)*blur , A[1]+np.random.randn(nPerClust)*blur ]\n", 87 | "b = [ B[0]+np.random.randn(nPerClust)*blur , B[1]+np.random.randn(nPerClust)*blur ]\n", 88 | "\n", 89 | "# true labels\n", 90 | "labels_np = np.vstack((np.zeros((nPerClust,1)),np.ones((nPerClust,1))))\n", 91 | "\n", 92 | "# concatanate into a matrix\n", 93 | "data_np = np.hstack((a,b)).T\n", 94 | "\n", 95 | "# convert to a pytorch tensor\n", 96 | "data = torch.tensor(data_np).float()\n", 97 | "labels = torch.tensor(labels_np).float()\n", 98 | "\n", 99 | "# show the data\n", 100 | "fig = plt.figure(figsize=(5,5))\n", 101 | "plt.plot(data[np.where(labels==0)[0],0],data[np.where(labels==0)[0],1],'bs')\n", 102 | "plt.plot(data[np.where(labels==1)[0],0],data[np.where(labels==1)[0],1],'ko')\n", 103 | "plt.title('The qwerties!')\n", 104 | "plt.xlabel('qwerty dimension 1')\n", 105 | "plt.ylabel('qwerty dimension 2')\n", 106 | "plt.show()" 107 | ], 108 | "execution_count": null, 109 | "outputs": [] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "metadata": { 114 | "id": "yfAFHG9a3MzF" 115 | }, 116 | "source": [ 117 | "# inspect types\n", 118 | "print(type(data_np))\n", 119 | "print(np.shape(data_np))\n", 120 | "print(' ')\n", 121 | "\n", 122 | "print(type(data))\n", 123 | "print(np.shape(data))" 124 | ], 125 | "execution_count": null, 126 | "outputs": [] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "metadata": { 131 | "id": "krQeh5wYMNla" 132 | }, 133 | "source": [ 134 | "# # build the model\n", 135 | "# ANNclassify = nn.Sequential(\n", 136 | "# nn.Linear(2,1), # input layer\n", 137 | "# nn.ReLU(), # activation unit\n", 138 | "# nn.Linear(1,1), # output unit\n", 139 | "# nn.Sigmoid(), # final activation unit (here for conceptual reasons; in practice, better to use BCEWithLogitsLoss)\n", 140 | "# )\n" 141 | ], 142 | "execution_count": null, 143 | "outputs": [] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "metadata": { 148 | "id": "XLyK-IlHvD2j" 149 | }, 150 | "source": [ 151 | "### define the class\n", 152 | "\n", 153 | "class theClass4ANN(nn.Module):\n", 154 | " def __init__(self):\n", 155 | " super().__init__()\n", 156 | "\n", 157 | " ### input layer\n", 158 | " self.input = nn.Linear(2,1)\n", 159 | " \n", 160 | " ### output layer\n", 161 | " self.output = nn.Linear(1,1)\n", 162 | "\n", 163 | " # forward pass\n", 164 | " def forward(self,x):\n", 165 | "\n", 166 | " # pass through the input layer\n", 167 | " x = self.input(x)\n", 168 | "\n", 169 | " # apply relu\n", 170 | " x = F.relu( x )\n", 171 | "\n", 172 | " # output layer\n", 173 | " x = self.output(x)\n", 174 | " x = torch.sigmoid(x)\n", 175 | "\n", 176 | " return x\n", 177 | "\n", 178 | "\n", 179 | "### create an instance of the class\n", 180 | "ANNclassify = theClass4ANN()" 181 | ], 182 | "execution_count": null, 183 | "outputs": [] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "metadata": { 188 | "id": "dmHh7GrvMNoy" 189 | }, 190 | "source": [ 191 | "# other model features\n", 192 | "\n", 193 | "learningRate = .01\n", 194 | "\n", 195 | "# loss function\n", 196 | "lossfun = nn.BCELoss()\n", 197 | "# Note: You'll learn in the \"Metaparameters\" section that it's better to use BCEWithLogitsLoss, but this is OK for now.\n", 198 | "\n", 199 | "# optimizer\n", 200 | "optimizer = torch.optim.SGD(ANNclassify.parameters(),lr=learningRate)\n" 201 | ], 202 | "execution_count": null, 203 | "outputs": [] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "metadata": { 208 | "id": "of9E8ClxMNsD" 209 | }, 210 | "source": [ 211 | "# train the model\n", 212 | "numepochs = 1000\n", 213 | "losses = torch.zeros(numepochs)\n", 214 | "\n", 215 | "for epochi in range(numepochs):\n", 216 | "\n", 217 | " # forward pass\n", 218 | " yHat = ANNclassify(data)\n", 219 | "\n", 220 | " # compute loss\n", 221 | " loss = lossfun(yHat,labels)\n", 222 | " losses[epochi] = loss\n", 223 | "\n", 224 | " # backprop\n", 225 | " optimizer.zero_grad()\n", 226 | " loss.backward()\n", 227 | " optimizer.step()" 228 | ], 229 | "execution_count": null, 230 | "outputs": [] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "metadata": { 235 | "id": "zmX6K49WMNuy" 236 | }, 237 | "source": [ 238 | "# show the losses\n", 239 | "\n", 240 | "plt.plot(losses.detach(),'o',markerfacecolor='w',linewidth=.1)\n", 241 | "plt.xlabel('Epoch')\n", 242 | "plt.ylabel('Loss')\n", 243 | "plt.show()" 244 | ], 245 | "execution_count": null, 246 | "outputs": [] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "metadata": { 251 | "id": "i1TCt0mpMNxC" 252 | }, 253 | "source": [ 254 | "# compute the predictions\n", 255 | "\n", 256 | "# manually compute losses\n", 257 | "# final forward pass\n", 258 | "predictions = ANNclassify(data)\n", 259 | "\n", 260 | "predlabels = predictions>.5\n", 261 | "\n", 262 | "# find errors\n", 263 | "misclassified = np.where(predlabels != labels)[0]\n", 264 | "\n", 265 | "# total accuracy\n", 266 | "totalacc = 100-100*len(misclassified)/(2*nPerClust)\n", 267 | "\n", 268 | "print('Final accuracy: %g%%' %totalacc)\n" 269 | ], 270 | "execution_count": null, 271 | "outputs": [] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "metadata": { 276 | "id": "q0UnOnNVB8Xb" 277 | }, 278 | "source": [ 279 | "# plot the labeled data\n", 280 | "fig = plt.figure(figsize=(5,5))\n", 281 | "plt.plot(data[misclassified,0] ,data[misclassified,1],'rx',markersize=12,markeredgewidth=3)\n", 282 | "plt.plot(data[np.where(~predlabels)[0],0],data[np.where(~predlabels)[0],1],'bs')\n", 283 | "plt.plot(data[np.where(predlabels)[0],0] ,data[np.where(predlabels)[0],1] ,'ko')\n", 284 | "\n", 285 | "plt.legend(['Misclassified','blue','black'],bbox_to_anchor=(1,1))\n", 286 | "plt.title(f'{totalacc}% correct')\n", 287 | "plt.show()" 288 | ], 289 | "execution_count": null, 290 | "outputs": [] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "metadata": { 295 | "id": "YTHj0h7GEVMq" 296 | }, 297 | "source": [ 298 | "" 299 | ], 300 | "execution_count": null, 301 | "outputs": [] 302 | } 303 | ] 304 | } -------------------------------------------------------------------------------- /FFN/DUDL_FFN_aboutMNIST.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_FFN_aboutMNIST.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 10 | "timestamp": 1617737766196 11 | }, 12 | { 13 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 14 | "timestamp": 1617734878578 15 | }, 16 | { 17 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 18 | "timestamp": 1617196833019 19 | }, 20 | { 21 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 22 | "timestamp": 1617124341706 23 | }, 24 | { 25 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 26 | "timestamp": 1616697516760 27 | }, 28 | { 29 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 30 | "timestamp": 1616615469755 31 | }, 32 | { 33 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 34 | "timestamp": 1616608248670 35 | } 36 | ], 37 | "collapsed_sections": [], 38 | "authorship_tag": "ABX9TyNNag92UMCy1GtgfJ+zFFU1" 39 | }, 40 | "kernelspec": { 41 | "name": "python3", 42 | "display_name": "Python 3" 43 | } 44 | }, 45 | "cells": [ 46 | { 47 | "cell_type": "markdown", 48 | "metadata": { 49 | "id": "bhWV8oes-wKR" 50 | }, 51 | "source": [ 52 | "# COURSE: A deep understanding of deep learning\n", 53 | "## SECTION: FFN\n", 54 | "### LECTURE: The MNIST dataset\n", 55 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 56 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "metadata": { 62 | "id": "YeuAheYyhdZw" 63 | }, 64 | "source": [ 65 | "# import libraries\n", 66 | "import numpy as np\n", 67 | "import matplotlib.pyplot as plt\n", 68 | "import matplotlib_inline.backend_inline\n", 69 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 70 | ], 71 | "execution_count": null, 72 | "outputs": [] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "id": "MU7rvmWuhjud" 78 | }, 79 | "source": [ 80 | "# import dataset (comes with colab!)\n", 81 | "data = np.loadtxt(open('sample_data/mnist_train_small.csv','rb'),delimiter=',')" 82 | ], 83 | "execution_count": null, 84 | "outputs": [] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "metadata": { 89 | "id": "8948Heuwth0l" 90 | }, 91 | "source": [ 92 | "# shape of the data matrix\n", 93 | "data.shape" 94 | ], 95 | "execution_count": null, 96 | "outputs": [] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "metadata": { 101 | "id": "Me1IOCLMuNPB" 102 | }, 103 | "source": [ 104 | "# extract labels (number IDs) and remove from data\n", 105 | "labels = data[:,0]\n", 106 | "data = data[:,1:]\n", 107 | "\n", 108 | "print(labels.shape)\n", 109 | "print(data.shape)" 110 | ], 111 | "execution_count": null, 112 | "outputs": [] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "metadata": { 117 | "id": "_COIDO8juVAl" 118 | }, 119 | "source": [ 120 | "# show a few random digits\n", 121 | "fig,axs = plt.subplots(3,4,figsize=(10,6))\n", 122 | "\n", 123 | "for ax in axs.flatten():\n", 124 | " # pick a random image\n", 125 | " randimg2show = np.random.randint(0,high=data.shape[0])\n", 126 | "\n", 127 | " # create the image (must be reshaped!)\n", 128 | " img = np.reshape(data[randimg2show,:],(28,28))\n", 129 | " ax.imshow(img,cmap='gray')\n", 130 | "\n", 131 | " # title\n", 132 | " ax.set_title('The number %i'%labels[randimg2show])\n", 133 | "\n", 134 | "plt.suptitle('How humans see the data',fontsize=20)\n", 135 | "plt.tight_layout(rect=[0,0,1,.95])\n", 136 | "plt.show()" 137 | ], 138 | "execution_count": null, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "vJPkH6Bfh01_" 145 | }, 146 | "source": [ 147 | "# show a few random digits\n", 148 | "fig,axs = plt.subplots(3,4,figsize=(10,6))\n", 149 | "\n", 150 | "for ax in axs.flatten():\n", 151 | " # pick a random image\n", 152 | " randimg2show = np.random.randint(0,high=data.shape[0])\n", 153 | "\n", 154 | " # create the image\n", 155 | " ax.plot(data[randimg2show,:],'ko')\n", 156 | "\n", 157 | " # title\n", 158 | " ax.set_title('The number %i'%labels[randimg2show])\n", 159 | "\n", 160 | "plt.suptitle('How the FFN model sees the data',fontsize=20)\n", 161 | "plt.tight_layout(rect=[0,0,1,.95])\n", 162 | "plt.show()" 163 | ], 164 | "execution_count": null, 165 | "outputs": [] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "metadata": { 170 | "id": "S6bXvj3ohX3R" 171 | }, 172 | "source": [ 173 | "# let's see some example 7s\n", 174 | "\n", 175 | "# find indices of all the 7's in the dataset\n", 176 | "the7s = np.where(labels==7)[0]\n", 177 | "\n", 178 | "# draw the first 12\n", 179 | "fig,axs = plt.subplots(2,6,figsize=(15,6))\n", 180 | "\n", 181 | "for i,ax in enumerate(axs.flatten()):\n", 182 | " img = np.reshape(data[the7s[i],:],(28,28))\n", 183 | " ax.imshow(img,cmap='gray')\n", 184 | " ax.axis('off')\n", 185 | "\n", 186 | "plt.suptitle(\"Example 7's\",fontsize=20)\n", 187 | "plt.tight_layout(rect=[0,0,1,.95])\n", 188 | "plt.show()" 189 | ], 190 | "execution_count": null, 191 | "outputs": [] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "metadata": { 196 | "id": "NeVhsO__fDKH" 197 | }, 198 | "source": [ 199 | "# how similar are all the 7's? \n", 200 | "\n", 201 | "# how many 7's are there?\n", 202 | "print(data[the7s,:].shape)\n", 203 | "\n", 204 | "\n", 205 | "# let's see how they relate to each other by computing spatial correlations\n", 206 | "C = np.corrcoef(data[the7s,:])\n", 207 | "\n", 208 | "# and visualize\n", 209 | "fig,ax = plt.subplots(1,3,figsize=(16,6))\n", 210 | "ax[0].imshow(C,vmin=0,vmax=1)\n", 211 | "ax[0].set_title(\"Correlation across all 7's\")\n", 212 | "\n", 213 | "# extract the unique correlations and show as a scatterplot\n", 214 | "uniqueCs = np.triu(C,k=1).flatten()\n", 215 | "ax[1].hist(uniqueCs[uniqueCs!=0],bins=100)\n", 216 | "ax[1].set_title('All unique correlations')\n", 217 | "ax[1].set_xlabel(\"Correlations of 7's\")\n", 218 | "ax[1].set_ylabel('Count')\n", 219 | "\n", 220 | "# show all 7's together\n", 221 | "aveAll7s = np.reshape( np.mean(data[the7s,:],axis=0) ,(28,28))\n", 222 | "ax[2].imshow(aveAll7s,cmap='gray')\n", 223 | "ax[2].set_title(\"All 7's averaged together\")\n", 224 | "\n", 225 | "plt.tight_layout()\n", 226 | "plt.show()" 227 | ], 228 | "execution_count": null, 229 | "outputs": [] 230 | } 231 | ] 232 | } -------------------------------------------------------------------------------- /FFNmilestone/DUDL_FFNmilestone_project1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1Q9LtmanyNt675-gO_kXRBKalCdP6xtvV", 9 | "timestamp": 1617253457100 10 | }, 11 | { 12 | "file_id": "1jeqKEJfI18GlAhSG8RO5aJ6Vrp4-nkTt", 13 | "timestamp": 1615909315432 14 | }, 15 | { 16 | "file_id": "10_geQnah5AvMsm8VDAQwNPhypOXradar", 17 | "timestamp": 1615893340208 18 | }, 19 | { 20 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 21 | "timestamp": 1615877547147 22 | } 23 | ] 24 | }, 25 | "kernelspec": { 26 | "name": "python3", 27 | "display_name": "Python 3" 28 | } 29 | }, 30 | "cells": [ 31 | { 32 | "cell_type": "markdown", 33 | "metadata": { 34 | "id": "bhWV8oes-wKR" 35 | }, 36 | "source": [ 37 | "# COURSE: A deep understanding of deep learning\n", 38 | "## SECTION: FFN milestone projects\n", 39 | "### LECTURE: Project 1: A gratuitously complex adding machine\n", 40 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 41 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "metadata": { 47 | "id": "j7-LiwqUMGYL" 48 | }, 49 | "source": [ 50 | "# import libraries\n", 51 | "import torch\n", 52 | "import torch.nn as nn\n", 53 | "import matplotlib.pyplot as plt\n", 54 | "import numpy as np" 55 | ], 56 | "execution_count": null, 57 | "outputs": [] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": { 62 | "id": "vRbiXgSFc8UC" 63 | }, 64 | "source": [ 65 | "# Create the training data" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "metadata": { 71 | "id": "j-SP8NPsMNRL" 72 | }, 73 | "source": [ 74 | "# create data\n", 75 | "\n", 76 | "N = 2000\n", 77 | "\n", 78 | "# generate data\n", 79 | "data = torch.randint(low=-10,high=11,size=(N,2)).float()\n", 80 | "\n", 81 | "# \"labels\"\n", 82 | "labels = torch.sum(data,axis=1,keepdim=True)\n", 83 | "\n", 84 | "# print the data\n", 85 | "for i in range(N):\n", 86 | " print(data[i],labels[i])" 87 | ], 88 | "execution_count": null, 89 | "outputs": [] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "metadata": { 94 | "id": "_l9dWwI23zgA" 95 | }, 96 | "source": [ 97 | "labels.shape" 98 | ], 99 | "execution_count": null, 100 | "outputs": [] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": { 105 | "id": "hVGH-eJNc-4z" 106 | }, 107 | "source": [ 108 | "# Design the model" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "metadata": { 114 | "id": "krQeh5wYMNla" 115 | }, 116 | "source": [ 117 | "# a function that builds the model\n", 118 | "\n", 119 | "def createADDmodel():\n", 120 | "\n", 121 | " # model architecture\n", 122 | " ADDclassify = nn.Sequential(\n", 123 | " nn.Linear(2,30), # input layer\n", 124 | " nn.ReLU(), # activation unit\n", 125 | " nn.Linear(30,1), # hidden layer\n", 126 | " nn.ReLU(), # activation unit\n", 127 | " nn.Linear(1,1), # output unit\n", 128 | " )\n", 129 | "\n", 130 | " # loss function\n", 131 | " lossfun = nn.MSELoss()\n", 132 | "\n", 133 | " # optimizer\n", 134 | " optimizer = torch.optim.Adam(ADDclassify.parameters(),lr=.01)\n", 135 | "\n", 136 | " # model output\n", 137 | " return ADDclassify,lossfun,optimizer" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "id": "JZWvPSy4dA2Z" 146 | }, 147 | "source": [ 148 | "# Train the model" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "metadata": { 154 | "id": "of9E8ClxMNsD" 155 | }, 156 | "source": [ 157 | "# a function that trains the model\n", 158 | "\n", 159 | "# a fixed parameter\n", 160 | "numepochs = 10\n", 161 | "\n", 162 | "def trainTheModel(ADDmodel):\n", 163 | "\n", 164 | " # initialize losses\n", 165 | " losses = torch.zeros(numepochs)\n", 166 | "\n", 167 | " # loop over epochs\n", 168 | " for epochi in range(numepochs):\n", 169 | "\n", 170 | " numloss = []\n", 171 | " # loop through \"minibatches\" of N=1\n", 172 | " for num,ans in zip(data,labels):\n", 173 | "\n", 174 | " # forward pass\n", 175 | " yHat = ADDmodel(num)\n", 176 | "\n", 177 | " # compute loss\n", 178 | " loss = lossfun(yHat,ans)\n", 179 | " numloss.append(loss.item())\n", 180 | "\n", 181 | " # backprop\n", 182 | " optimizer.zero_grad()\n", 183 | " loss.backward()\n", 184 | " optimizer.step()\n", 185 | "\n", 186 | " losses[epochi] = np.mean(numloss)\n", 187 | "\n", 188 | "\n", 189 | " # final forward pass\n", 190 | " with torch.no_grad():\n", 191 | " predictions = ADDmodel(data)\n", 192 | "\n", 193 | " # compute the predictions and report accuracy (is this really different from loss?)\n", 194 | " trainacc = 100*torch.mean( (np.abs(predictions-labels)<1).float() ) # <1 for rounding\n", 195 | "\n", 196 | "\n", 197 | " ### now test with new data!\n", 198 | " # create brand new test data\n", 199 | " TESTdata = torch.randint(low=-10,high=11,size=(N,2)).float()\n", 200 | " TESTlabels = torch.sum(TESTdata,axis=1,keepdim=True)\n", 201 | "\n", 202 | " # evaluate the model and compute accuracy\n", 203 | " with torch.no_grad():\n", 204 | " predictions = ADDmodel(TESTdata)\n", 205 | " testacc = 100*torch.mean( (np.abs(predictions-TESTlabels)<1).float() )\n", 206 | "\n", 207 | " # EOF\n", 208 | " return losses,trainacc,testacc,ADDmodel" 209 | ], 210 | "execution_count": null, 211 | "outputs": [] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "metadata": { 216 | "id": "SWQ4rttNbPiE" 217 | }, 218 | "source": [ 219 | "# test the model once to make sure it runs\n", 220 | "AddingMachine,lossfun,optimizer = createADDmodel()\n", 221 | "losses,trainacc,testacc,ADDmodel = trainTheModel(AddingMachine)\n", 222 | "\n", 223 | "# print the results\n", 224 | "trainacc,testacc\n", 225 | "# losses" 226 | ], 227 | "execution_count": null, 228 | "outputs": [] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": { 233 | "id": "BKpp_RMDdDeY" 234 | }, 235 | "source": [ 236 | "# Run the experiment!" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "metadata": { 242 | "id": "SatsWGs3x6Kg" 243 | }, 244 | "source": [ 245 | "# run the model 10 times to check reproducibility\n", 246 | "\n", 247 | "for i in range(10):\n", 248 | "\n", 249 | " # create a new 'machine' and train it\n", 250 | " AddingMachine,lossfun,optimizer = createADDmodel()\n", 251 | " _,trainacc,testacc,ADDmodel = trainTheModel(AddingMachine)\n", 252 | "\n", 253 | " # report accuracy\n", 254 | " print('Model instance %s, final TRAIN/TEST accuracies: %g%%, %g%%' %(i+1,trainacc,testacc))" 255 | ], 256 | "execution_count": null, 257 | "outputs": [] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "metadata": { 262 | "id": "aHpcfhym148x" 263 | }, 264 | "source": [ 265 | "# some plots from the final model\n", 266 | "\n", 267 | "data = torch.randint(low=-10,high=11,size=(N,2)).float()\n", 268 | "labels = torch.sum(data,axis=1,keepdim=True)\n", 269 | "\n", 270 | "# evaluate the model and compute accuracy\n", 271 | "with torch.no_grad():\n", 272 | " predictions = ADDmodel(data)\n", 273 | "\n", 274 | "\n", 275 | "# true and predicted values\n", 276 | "fig = plt.figure(figsize=(15,5))\n", 277 | "plt.plot(labels,'s')\n", 278 | "plt.plot(np.round(predictions.detach()),'rx')\n", 279 | "plt.legend(['True sum','Predicted sum'])\n", 280 | "plt.xlabel('Sample index')\n", 281 | "plt.ylabel('Sum')\n", 282 | "plt.title('Predicted vs. actual sum')\n", 283 | "plt.show()" 284 | ], 285 | "execution_count": null, 286 | "outputs": [] 287 | } 288 | ] 289 | } -------------------------------------------------------------------------------- /GPU/DUDL_GPU_implement.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_GPU_implement.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyO6Sbwn18acQD5H5eD06W+E" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "bhWV8oes-wKR" 25 | }, 26 | "source": [ 27 | "# COURSE: A deep understanding of deep learning\n", 28 | "## SECTION: Running models on a GPU\n", 29 | "### LECTURE: Implementation\n", 30 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 31 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "metadata": { 37 | "id": "YeuAheYyhdZw" 38 | }, 39 | "source": [ 40 | "# import libraries\n", 41 | "import torch\n", 42 | "import torch.nn as nn\n", 43 | "import matplotlib.pyplot as plt\n", 44 | "import time" 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "id": "g0O-zq4UiM0P" 53 | }, 54 | "source": [ 55 | "# Select the processor device" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "metadata": { 61 | "id": "03MycVnQiRqO" 62 | }, 63 | "source": [ 64 | "## Note: To run models on a GPU you must select from the menu:\n", 65 | "# -> Runtime\n", 66 | "# -> Change runtime type\n", 67 | "# -> Hardware accelerator\n", 68 | "# -> GPU" 69 | ], 70 | "execution_count": null, 71 | "outputs": [] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "metadata": { 76 | "id": "cThPRuj_iJA2" 77 | }, 78 | "source": [ 79 | "# use GPU\n", 80 | "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", 81 | "print(device)" 82 | ], 83 | "execution_count": null, 84 | "outputs": [] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": { 89 | "id": "rcumdem5jtiD" 90 | }, 91 | "source": [ 92 | "# Build a simple model" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "metadata": { 98 | "id": "AMi1lQWOj09S" 99 | }, 100 | "source": [ 101 | "net = nn.Sequential(\n", 102 | " nn.Linear(20,100),\n", 103 | " nn.ReLU(),\n", 104 | " nn.Linear(100,500),\n", 105 | " nn.ReLU(),\n", 106 | " nn.Linear(500,30),\n", 107 | " nn.ReLU(),\n", 108 | " nn.Linear(30,2)\n", 109 | " )" 110 | ], 111 | "execution_count": null, 112 | "outputs": [] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": { 117 | "id": "TTfiVpw5j6Mq" 118 | }, 119 | "source": [ 120 | "# Make some data" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "metadata": { 126 | "id": "A_AGHhXDj1UN" 127 | }, 128 | "source": [ 129 | "# just some random fluff\n", 130 | "data = torch.randn((1000,20)) # samples X features\n", 131 | "labels = torch.randint(low=0,high=2,size=(1,1000))" 132 | ], 133 | "execution_count": null, 134 | "outputs": [] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": { 139 | "id": "laT9cpQcmA1l" 140 | }, 141 | "source": [ 142 | "# Send the model and the data to the GPU" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "metadata": { 148 | "id": "nQzP_FXLj1QW" 149 | }, 150 | "source": [ 151 | "# model\n", 152 | "net.to(device)\n", 153 | "\n", 154 | "# data\n", 155 | "data = data.to(device)\n", 156 | "labels = labels.to(device)" 157 | ], 158 | "execution_count": null, 159 | "outputs": [] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "metadata": { 164 | "id": "b-uDroB7jPBe" 165 | }, 166 | "source": [ 167 | "data" 168 | ], 169 | "execution_count": null, 170 | "outputs": [] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "kCg4D6Hlfkgb" 176 | }, 177 | "source": [ 178 | "# you can also create data directly on the GPU\n", 179 | "dataG = torch.randn((1000,20),device=device)\n", 180 | "dataC = torch.randn((1000,20),device='cpu')\n", 181 | "\n", 182 | "print(data.device)\n", 183 | "print(dataG.device)\n", 184 | "print(dataC.device)" 185 | ], 186 | "execution_count": null, 187 | "outputs": [] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": { 192 | "id": "9yrwlRyQj1dF" 193 | }, 194 | "source": [ 195 | "# Getting results from the model" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "metadata": { 201 | "id": "tryFiNSej1gF" 202 | }, 203 | "source": [ 204 | "output = net(data)" 205 | ], 206 | "execution_count": null, 207 | "outputs": [] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "metadata": { 212 | "id": "_0-0zxQOj1it" 213 | }, 214 | "source": [ 215 | "output.device" 216 | ], 217 | "execution_count": null, 218 | "outputs": [] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "metadata": { 223 | "id": "DQVKoRmWmUm5" 224 | }, 225 | "source": [ 226 | "# try to plot the data\n", 227 | "plt.plot(output.detach());\n", 228 | "\n", 229 | "# plt.plot(output.detach().cpu());" 230 | ], 231 | "execution_count": null, 232 | "outputs": [] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": { 237 | "id": "VcNrG3HRnwms" 238 | }, 239 | "source": [ 240 | "# Experiment: Computation time" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "metadata": { 246 | "id": "4lJBOELHnyJf" 247 | }, 248 | "source": [ 249 | "# start the clock\n", 250 | "starttime = time.process_time()\n", 251 | "\n", 252 | "# move, run, retrieve\n", 253 | "device = 'cuda:0'\n", 254 | "net.to(device)\n", 255 | "data = data.to(device)\n", 256 | "labels = labels.to(device)\n", 257 | "output = net(data).detach().cpu()\n", 258 | "\n", 259 | "# stop the clock\n", 260 | "GPUtime = 1000*(time.process_time() - starttime)" 261 | ], 262 | "execution_count": null, 263 | "outputs": [] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "metadata": { 268 | "id": "a3o-SgmMnyMG" 269 | }, 270 | "source": [ 271 | "# start the clock\n", 272 | "starttime = time.process_time()\n", 273 | "\n", 274 | "# move, run, retrieve\n", 275 | "device = 'cpu'\n", 276 | "net.to(device)\n", 277 | "data = data.to(device)\n", 278 | "labels = labels.to(device)\n", 279 | "output = net(data).detach().cpu()\n", 280 | "\n", 281 | "# stop the clock\n", 282 | "CPUtime = 1000*(time.process_time() - starttime)" 283 | ], 284 | "execution_count": null, 285 | "outputs": [] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "metadata": { 290 | "id": "hzYccKwjnyOw" 291 | }, 292 | "source": [ 293 | "# time in ms\n", 294 | "[CPUtime,GPUtime]" 295 | ], 296 | "execution_count": null, 297 | "outputs": [] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "metadata": { 302 | "id": "Bo5dt4Nipprn" 303 | }, 304 | "source": [ 305 | "# but maybe \"sending\" it to the CPU takes too much overhead?" 306 | ], 307 | "execution_count": null, 308 | "outputs": [] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "metadata": { 313 | "id": "J-NvS5tvnyRX" 314 | }, 315 | "source": [ 316 | "# recreate network and data\n", 317 | "net = nn.Sequential(nn.Linear(20,100),nn.ReLU(),nn.Linear(100,500),nn.ReLU(),nn.Linear(500,30),nn.ReLU(),nn.Linear(30,2))\n", 318 | "data = torch.randn((1000,20))\n", 319 | "\n", 320 | "# rerun the experiment\n", 321 | "starttime = time.process_time()\n", 322 | "output = net(data).detach()\n", 323 | "CPUtime2 = 1000*(time.process_time() - starttime)\n", 324 | "\n", 325 | "# report the results\n", 326 | "[CPUtime2,CPUtime,GPUtime]" 327 | ], 328 | "execution_count": null, 329 | "outputs": [] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "metadata": { 334 | "id": "dJ1p2lXWkSdl" 335 | }, 336 | "source": [ 337 | "" 338 | ], 339 | "execution_count": null, 340 | "outputs": [] 341 | } 342 | ] 343 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Mike X Cohen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Understanding Of Deep Learning 2 | Python code accompanying the course "A deep understanding of deep learning (with Python intro)" 3 | 4 | Master deep learning in PyTorch using an experimental scientific approach, with lots of examples and practice problems. 5 | 6 | 7 | See https://www.udemy.com/course/deeplearning_x/?couponCode=202502 for more details, preview videos, and to enroll in the full course. 8 | 9 | -------------------------------------------------------------------------------- /convolution/DUDL_convolution_conv2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1GRajDS-VF5z8IslzZuMqbis3X6HDD-Uo", 9 | "timestamp": 1619468278654 10 | }, 11 | { 12 | "file_id": "1m0n2-UmB2tJiIDadlFkE6L5A4iZSqeBf", 13 | "timestamp": 1619459134813 14 | }, 15 | { 16 | "file_id": "19G9gTeBlYPQ-s3VS_3K2bVFtKTP344j6", 17 | "timestamp": 1619444797767 18 | }, 19 | { 20 | "file_id": "1FcEBC0NAESIlHQkv6_85R-XDUKGE8XbM", 21 | "timestamp": 1619155961717 22 | }, 23 | { 24 | "file_id": "1qKgZ8kVcqNgwtBzHbWq5yJH_HqI6DxWW", 25 | "timestamp": 1617803880910 26 | }, 27 | { 28 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 29 | "timestamp": 1617737766196 30 | }, 31 | { 32 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 33 | "timestamp": 1617734878578 34 | }, 35 | { 36 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 37 | "timestamp": 1617196833019 38 | }, 39 | { 40 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 41 | "timestamp": 1617124341706 42 | }, 43 | { 44 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 45 | "timestamp": 1616697516760 46 | }, 47 | { 48 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 49 | "timestamp": 1616615469755 50 | }, 51 | { 52 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 53 | "timestamp": 1616608248670 54 | } 55 | ] 56 | }, 57 | "kernelspec": { 58 | "name": "python3", 59 | "display_name": "Python 3" 60 | } 61 | }, 62 | "cells": [ 63 | { 64 | "cell_type": "markdown", 65 | "metadata": { 66 | "id": "bhWV8oes-wKR" 67 | }, 68 | "source": [ 69 | "# COURSE: A deep understanding of deep learning\n", 70 | "## SECTION: Convolution and transformations\n", 71 | "### LECTURE: The Conv2 class in PyTorch\n", 72 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 73 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "metadata": { 79 | "id": "YeuAheYyhdZw" 80 | }, 81 | "source": [ 82 | "# import libraries\n", 83 | "import numpy as np\n", 84 | "import torch\n", 85 | "import torch.nn as nn\n", 86 | "\n", 87 | "import matplotlib.pyplot as plt" 88 | ], 89 | "execution_count": null, 90 | "outputs": [] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": { 95 | "id": "0HOkOefftqyg" 96 | }, 97 | "source": [ 98 | "# Create a convolution instance" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "metadata": { 104 | "id": "VhIKo0_iaGz2" 105 | }, 106 | "source": [ 107 | "# create a conv2d class instance with parameters\n", 108 | "\n", 109 | "# parameters\n", 110 | "inChans = 3 # RGB\n", 111 | "outChans = 15\n", 112 | "krnSize = 5 # should be an odd number\n", 113 | "stride = 1\n", 114 | "padding = 0\n", 115 | "\n", 116 | "# create the instance (cf nn.Linear)\n", 117 | "c = nn.Conv2d(inChans,outChans,krnSize,stride,padding)\n", 118 | "\n", 119 | "# let's have a look at it\n", 120 | "print(c)\n", 121 | "print(' ')\n", 122 | "\n", 123 | "# check out its weight tensor; what are the dimensions?\n", 124 | "print( 'Size of weights: ' + str(c.weight.shape) )\n", 125 | "print( 'Size of bias: ' + str(c.bias.shape) )" 126 | ], 127 | "execution_count": null, 128 | "outputs": [] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "metadata": { 133 | "id": "N6p50RaP4kce" 134 | }, 135 | "source": [ 136 | "# What do these kernels look like?\n", 137 | "\n", 138 | "fig,axs = plt.subplots(3,5,figsize=(10,5))\n", 139 | "\n", 140 | "for i,ax in enumerate(axs.flatten()):\n", 141 | " ax.imshow(torch.squeeze(c.weight[i,0,:,:]).detach(),cmap='Purples')\n", 142 | " ax.set_title('L1(0)->L2(%s)'%i)\n", 143 | " ax.axis('off')\n", 144 | "\n", 145 | "plt.tight_layout()\n", 146 | "plt.show()" 147 | ], 148 | "execution_count": null, 149 | "outputs": [] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": { 154 | "id": "VAqVrcrGd98S" 155 | }, 156 | "source": [ 157 | "# Convolve with an image" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "metadata": { 163 | "id": "9SnUUHPm7xQE" 164 | }, 165 | "source": [ 166 | "# size of the image (N, RGB, height, width)\n", 167 | "imsize = (1,3,64,64)\n", 168 | "\n", 169 | "img = torch.rand(imsize)\n", 170 | "\n", 171 | "# pytorch wants channels first, but matplotlib wants channels last.\n", 172 | "# therefore, tensors must be permuted to visualize\n", 173 | "img2view = img.permute(2,3,1,0).numpy()\n", 174 | "print(img.shape)\n", 175 | "print(img2view.shape)\n", 176 | "\n", 177 | "plt.imshow(np.squeeze(img2view));" 178 | ], 179 | "execution_count": null, 180 | "outputs": [] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "metadata": { 185 | "id": "2WcTRmLu4F23" 186 | }, 187 | "source": [ 188 | "# convolve the image with the filter bank (set of 'outChans' kernels)\n", 189 | "convRes = c(img)\n", 190 | "\n", 191 | "print(img.shape)\n", 192 | "print(convRes.shape)" 193 | ], 194 | "execution_count": null, 195 | "outputs": [] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "metadata": { 200 | "id": "poDa9N6TwjPu" 201 | }, 202 | "source": [ 203 | "# What do the convolved images look like? (Hint: think of the bathtub picture.)\n", 204 | "\n", 205 | "fig,axs = plt.subplots(3,5,figsize=(10,5))\n", 206 | "\n", 207 | "for i,ax in enumerate(axs.flatten()):\n", 208 | "\n", 209 | " # extract this \"layer\" of the convolution result\n", 210 | " I = torch.squeeze(convRes[0,i,:,:]).detach()\n", 211 | "\n", 212 | " # and visualize it\n", 213 | " ax.imshow(I,cmap='Purples')\n", 214 | " ax.set_title('Conv. w/ filter %s'%i)\n", 215 | " ax.axis('off')\n", 216 | "\n", 217 | "plt.tight_layout()\n", 218 | "plt.show()" 219 | ], 220 | "execution_count": null, 221 | "outputs": [] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "metadata": { 226 | "id": "fCooGt9PiNb6" 227 | }, 228 | "source": [], 229 | "execution_count": null, 230 | "outputs": [] 231 | }, 232 | { 233 | "cell_type": "markdown", 234 | "metadata": { 235 | "id": "TVXQlW-svDUE" 236 | }, 237 | "source": [ 238 | "# Additional explorations" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "metadata": { 244 | "id": "nViqoQuOvbZB" 245 | }, 246 | "source": [ 247 | "# 1) Spend a few minutes changing the padding and stride parameters, and observe how those change the size of the result\n", 248 | "# of convolution (variable convRes). In the upcoming CodeChallenge, we'll look into this more rigorously, but it's\n", 249 | "# useful to have some initial familiarity.\n", 250 | "#" 251 | ], 252 | "execution_count": null, 253 | "outputs": [] 254 | } 255 | ] 256 | } -------------------------------------------------------------------------------- /convolution/DUDL_convolution_conv2transpose.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_convolution_conv2transpose.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "19imE5cZVTySOpFOe4Uvw97qMPMgs0MJ2", 10 | "timestamp": 1620678031999 11 | }, 12 | { 13 | "file_id": "1GRajDS-VF5z8IslzZuMqbis3X6HDD-Uo", 14 | "timestamp": 1619468278654 15 | }, 16 | { 17 | "file_id": "1m0n2-UmB2tJiIDadlFkE6L5A4iZSqeBf", 18 | "timestamp": 1619459134813 19 | }, 20 | { 21 | "file_id": "19G9gTeBlYPQ-s3VS_3K2bVFtKTP344j6", 22 | "timestamp": 1619444797767 23 | }, 24 | { 25 | "file_id": "1FcEBC0NAESIlHQkv6_85R-XDUKGE8XbM", 26 | "timestamp": 1619155961717 27 | }, 28 | { 29 | "file_id": "1qKgZ8kVcqNgwtBzHbWq5yJH_HqI6DxWW", 30 | "timestamp": 1617803880910 31 | }, 32 | { 33 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 34 | "timestamp": 1617737766196 35 | }, 36 | { 37 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 38 | "timestamp": 1617734878578 39 | }, 40 | { 41 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 42 | "timestamp": 1617196833019 43 | }, 44 | { 45 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 46 | "timestamp": 1617124341706 47 | }, 48 | { 49 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 50 | "timestamp": 1616697516760 51 | }, 52 | { 53 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 54 | "timestamp": 1616615469755 55 | }, 56 | { 57 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 58 | "timestamp": 1616608248670 59 | } 60 | ], 61 | "collapsed_sections": [], 62 | "authorship_tag": "ABX9TyMmwDcfzKJX666dqNPAElf7" 63 | }, 64 | "kernelspec": { 65 | "name": "python3", 66 | "display_name": "Python 3" 67 | } 68 | }, 69 | "cells": [ 70 | { 71 | "cell_type": "markdown", 72 | "metadata": { 73 | "id": "bhWV8oes-wKR" 74 | }, 75 | "source": [ 76 | "# COURSE: A deep understanding of deep learning\n", 77 | "## SECTION: Image convolution and transformations\n", 78 | "### LECTURE: Transpose convolution\n", 79 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 80 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "metadata": { 86 | "id": "YeuAheYyhdZw" 87 | }, 88 | "source": [ 89 | "# import libraries\n", 90 | "import numpy as np\n", 91 | "import torch\n", 92 | "import torch.nn as nn\n", 93 | "\n", 94 | "import matplotlib.pyplot as plt" 95 | ], 96 | "execution_count": null, 97 | "outputs": [] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": { 102 | "id": "0HOkOefftqyg" 103 | }, 104 | "source": [ 105 | "# Create a transpose convolution instance" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "metadata": { 111 | "id": "VhIKo0_iaGz2" 112 | }, 113 | "source": [ 114 | "# create a ConvTranspose2d class instance with parameters\n", 115 | "\n", 116 | "# parameters\n", 117 | "inChans = 3 # RGB\n", 118 | "outChans = 15\n", 119 | "krnSize = 5 # should be an odd number\n", 120 | "stride = 1\n", 121 | "padding = 0\n", 122 | "\n", 123 | "# create the instance\n", 124 | "c = nn.ConvTranspose2d(inChans,outChans,krnSize,stride,padding)\n", 125 | "\n", 126 | "# let's have a look at it\n", 127 | "print(c)\n", 128 | "print(' ')\n", 129 | "\n", 130 | "# check out its weight tensor; what are the dimensions?\n", 131 | "print( 'Size of weights: ' + str(c.weight.shape) )\n", 132 | "print( 'Size of bias: ' + str(c.bias.shape) )\n", 133 | "\n", 134 | "# tip: Compare the sizes of these weights with those of \"forward\" convolution (DUDL_convolution_conv2.ipynb)" 135 | ], 136 | "execution_count": null, 137 | "outputs": [] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "metadata": { 142 | "id": "N6p50RaP4kce" 143 | }, 144 | "source": [ 145 | "# What do these kernels look like?\n", 146 | "\n", 147 | "fig,axs = plt.subplots(3,5,figsize=(10,5))\n", 148 | "\n", 149 | "for i,ax in enumerate(axs.flatten()):\n", 150 | " ax.imshow(torch.squeeze(c.weight[0,i,:,:]).detach(),cmap='Purples')\n", 151 | " ax.set_title('L1(0)->L2(%s)'%i)\n", 152 | " ax.axis('off')\n", 153 | "\n", 154 | "plt.tight_layout()\n", 155 | "plt.show()" 156 | ], 157 | "execution_count": null, 158 | "outputs": [] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": { 163 | "id": "VAqVrcrGd98S" 164 | }, 165 | "source": [ 166 | "# Convolve with an image" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "metadata": { 172 | "id": "9SnUUHPm7xQE" 173 | }, 174 | "source": [ 175 | "# size of the image (N, RGB, width, height)\n", 176 | "imsize = (1,3,64,64)\n", 177 | "\n", 178 | "img = torch.rand(imsize)\n", 179 | "\n", 180 | "# pytorch wants channels first, but matplotlib wants channels last.\n", 181 | "# therefore, tensors must be permuted to visualize\n", 182 | "img2view = img.permute(2,3,1,0).numpy()\n", 183 | "print(img.shape)\n", 184 | "print(img2view.shape)\n", 185 | "\n", 186 | "plt.imshow(np.squeeze(img2view));" 187 | ], 188 | "execution_count": null, 189 | "outputs": [] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "metadata": { 194 | "id": "2WcTRmLu4F23" 195 | }, 196 | "source": [ 197 | "# convolve the image with the filter bank (set of 'outChans' kernels)\n", 198 | "convRes = c(img)\n", 199 | "\n", 200 | "print(img.shape)\n", 201 | "print(convRes.shape)" 202 | ], 203 | "execution_count": null, 204 | "outputs": [] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "metadata": { 209 | "id": "poDa9N6TwjPu" 210 | }, 211 | "source": [ 212 | "# What do the convolved images look like? (Hint: think of the bathtub picture.)\n", 213 | "\n", 214 | "fig,axs = plt.subplots(3,5,figsize=(10,5))\n", 215 | "\n", 216 | "for i,ax in enumerate(axs.flatten()):\n", 217 | "\n", 218 | " # extract this \"layer\" of the convolution result\n", 219 | " I = torch.squeeze(convRes[0,i,:,:]).detach()\n", 220 | "\n", 221 | " # and visualize it\n", 222 | " ax.imshow(I,cmap='Purples')\n", 223 | " ax.set_title('Conv. w/ filter %s'%i)\n", 224 | " ax.axis('off')\n", 225 | "\n", 226 | "plt.tight_layout()\n", 227 | "plt.show()" 228 | ], 229 | "execution_count": null, 230 | "outputs": [] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "metadata": { 235 | "id": "fCooGt9PiNb6" 236 | }, 237 | "source": [ 238 | "" 239 | ], 240 | "execution_count": null, 241 | "outputs": [] 242 | } 243 | ] 244 | } -------------------------------------------------------------------------------- /convolution/DUDL_convolution_meanMaxPool.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_convolution_meanMaxPool.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "19imE5cZVTySOpFOe4Uvw97qMPMgs0MJ2", 10 | "timestamp": 1619525236552 11 | }, 12 | { 13 | "file_id": "1GRajDS-VF5z8IslzZuMqbis3X6HDD-Uo", 14 | "timestamp": 1619468278654 15 | }, 16 | { 17 | "file_id": "1m0n2-UmB2tJiIDadlFkE6L5A4iZSqeBf", 18 | "timestamp": 1619459134813 19 | }, 20 | { 21 | "file_id": "19G9gTeBlYPQ-s3VS_3K2bVFtKTP344j6", 22 | "timestamp": 1619444797767 23 | }, 24 | { 25 | "file_id": "1FcEBC0NAESIlHQkv6_85R-XDUKGE8XbM", 26 | "timestamp": 1619155961717 27 | }, 28 | { 29 | "file_id": "1qKgZ8kVcqNgwtBzHbWq5yJH_HqI6DxWW", 30 | "timestamp": 1617803880910 31 | }, 32 | { 33 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 34 | "timestamp": 1617737766196 35 | }, 36 | { 37 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 38 | "timestamp": 1617734878578 39 | }, 40 | { 41 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 42 | "timestamp": 1617196833019 43 | }, 44 | { 45 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 46 | "timestamp": 1617124341706 47 | }, 48 | { 49 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 50 | "timestamp": 1616697516760 51 | }, 52 | { 53 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 54 | "timestamp": 1616615469755 55 | }, 56 | { 57 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 58 | "timestamp": 1616608248670 59 | } 60 | ], 61 | "collapsed_sections": [], 62 | "authorship_tag": "ABX9TyMIPmLlfMrWdk0Y8XK9tLYQ" 63 | }, 64 | "kernelspec": { 65 | "name": "python3", 66 | "display_name": "Python 3" 67 | } 68 | }, 69 | "cells": [ 70 | { 71 | "cell_type": "markdown", 72 | "metadata": { 73 | "id": "bhWV8oes-wKR" 74 | }, 75 | "source": [ 76 | "# COURSE: A deep understanding of deep learning\n", 77 | "## SECTION: Convolution and transformations\n", 78 | "### LECTURE: Max/mean pooling\n", 79 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 80 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "metadata": { 86 | "id": "YeuAheYyhdZw" 87 | }, 88 | "source": [ 89 | "# import libraries\n", 90 | "import torch\n", 91 | "import torch.nn as nn" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": { 99 | "id": "0HOkOefftqyg" 100 | }, 101 | "source": [ 102 | "# Create a maxpool instance" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "metadata": { 108 | "id": "VhIKo0_iaGz2" 109 | }, 110 | "source": [ 111 | "# create a pool class instance with parameters\n", 112 | "\n", 113 | "# parameters\n", 114 | "poolSize = 3\n", 115 | "stride = 3\n", 116 | "\n", 117 | "# create the instance\n", 118 | "p2 = nn.MaxPool2d(poolSize,stride=3)\n", 119 | "p3 = nn.MaxPool3d(poolSize,stride=3)\n", 120 | "\n", 121 | "# let's have a look at them\n", 122 | "print(p2)\n", 123 | "print(p3)" 124 | ], 125 | "execution_count": null, 126 | "outputs": [] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "metadata": { 131 | "id": "G669cetLBMFO" 132 | }, 133 | "source": [ 134 | "# Create image and apply maxpooling\n", 135 | "\n", 136 | "# create a 2D and a 3D image\n", 137 | "img2 = torch.randn(1,1,30,30)\n", 138 | "img3 = torch.randn(1,3,30,30)\n", 139 | "\n", 140 | "\n", 141 | "# all combinations of image and maxpool dimensionality\n", 142 | "img2Pool2 = p2(img2)\n", 143 | "print(f'2D image, 2D maxpool: {img2Pool2.shape}\\n' )\n", 144 | "\n", 145 | "# img2Pool3 = p3(img2)\n", 146 | "# print(f'2D image, 3D maxpool: {img2Pool3.shape}\\n' )\n", 147 | "\n", 148 | "img3Pool2 = p2(img3)\n", 149 | "print(f'3D image, 2D maxpool: {img3Pool2.shape}\\n' )\n", 150 | "\n", 151 | "img3Pool3 = p3(img3)\n", 152 | "print(f'3D image, 3D maxpool: {img3Pool3.shape}\\n' )\n" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": { 160 | "id": "VAqVrcrGd98S" 161 | }, 162 | "source": [ 163 | "# Create a simple CNN" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "metadata": { 169 | "id": "Bub-D2xxAiR3" 170 | }, 171 | "source": [ 172 | "littlenet = nn.Sequential(\n", 173 | " \n", 174 | " ## the conv-pool block\n", 175 | " nn.Conv2d(3,10,5,3,2), # convolution layer\n", 176 | " nn.ReLU(), # activation function\n", 177 | " nn.AvgPool3d(3,3), # average-pool\n", 178 | " \n", 179 | " ## the FFN block\n", 180 | " nn.Flatten(), # vectorize to get from image to linear\n", 181 | " nn.Linear(588,1), # FC linear layer\n", 182 | " nn.Sigmoid() # output activation\n", 183 | " )\n" 184 | ], 185 | "execution_count": null, 186 | "outputs": [] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "metadata": { 191 | "id": "9SnUUHPm7xQE" 192 | }, 193 | "source": [ 194 | "# test with a bit of data\n", 195 | "img = torch.rand(1,3,128,128)\n", 196 | "littlenet(img)" 197 | ], 198 | "execution_count": null, 199 | "outputs": [] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "metadata": { 204 | "id": "fCooGt9PiNb6" 205 | }, 206 | "source": [ 207 | "" 208 | ], 209 | "execution_count": null, 210 | "outputs": [] 211 | } 212 | ] 213 | } -------------------------------------------------------------------------------- /convolution/DUDL_convolution_transforms.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_convolution_transforms.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1m0n2-UmB2tJiIDadlFkE6L5A4iZSqeBf", 10 | "timestamp": 1619459134813 11 | }, 12 | { 13 | "file_id": "19G9gTeBlYPQ-s3VS_3K2bVFtKTP344j6", 14 | "timestamp": 1619444797767 15 | }, 16 | { 17 | "file_id": "1FcEBC0NAESIlHQkv6_85R-XDUKGE8XbM", 18 | "timestamp": 1619155961717 19 | }, 20 | { 21 | "file_id": "1qKgZ8kVcqNgwtBzHbWq5yJH_HqI6DxWW", 22 | "timestamp": 1617803880910 23 | }, 24 | { 25 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 26 | "timestamp": 1617737766196 27 | }, 28 | { 29 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 30 | "timestamp": 1617734878578 31 | }, 32 | { 33 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 34 | "timestamp": 1617196833019 35 | }, 36 | { 37 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 38 | "timestamp": 1617124341706 39 | }, 40 | { 41 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 42 | "timestamp": 1616697516760 43 | }, 44 | { 45 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 46 | "timestamp": 1616615469755 47 | }, 48 | { 49 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 50 | "timestamp": 1616608248670 51 | } 52 | ], 53 | "collapsed_sections": [], 54 | "authorship_tag": "ABX9TyNzAXVK67MmJWX6LzPABwTY" 55 | }, 56 | "kernelspec": { 57 | "name": "python3", 58 | "display_name": "Python 3" 59 | } 60 | }, 61 | "cells": [ 62 | { 63 | "cell_type": "markdown", 64 | "metadata": { 65 | "id": "bhWV8oes-wKR" 66 | }, 67 | "source": [ 68 | "# COURSE: A deep understanding of deep learning\n", 69 | "## SECTION: Convolution and transformations\n", 70 | "### LECTURE: Image transformations\n", 71 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 72 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "metadata": { 78 | "id": "YeuAheYyhdZw" 79 | }, 80 | "source": [ 81 | "# import libraries\n", 82 | "import numpy as np\n", 83 | "import torch \n", 84 | "\n", 85 | "# NEW!\n", 86 | "import torchvision\n", 87 | "import torchvision.transforms as T\n", 88 | "\n", 89 | "import matplotlib.pyplot as plt\n", 90 | "import matplotlib_inline.backend_inline\n", 91 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": { 99 | "id": "0HOkOefftqyg" 100 | }, 101 | "source": [ 102 | "# Import a dataset" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "metadata": { 108 | "id": "VhIKo0_iaGz2" 109 | }, 110 | "source": [ 111 | "# The list of datasets that come with torchvision: https://pytorch.org/vision/stable/index.html" 112 | ], 113 | "execution_count": null, 114 | "outputs": [] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "metadata": { 119 | "id": "MU7rvmWuhjud" 120 | }, 121 | "source": [ 122 | "# download the CIFAR10 dataset\n", 123 | "cdata = torchvision.datasets.CIFAR10(root='cifar10', download=True)\n", 124 | "\n", 125 | "print(cdata)" 126 | ], 127 | "execution_count": null, 128 | "outputs": [] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "metadata": { 133 | "id": "NhN7RYqYd915" 134 | }, 135 | "source": [ 136 | "# check out the shape of the dataset\n", 137 | "print( cdata.data.shape )\n", 138 | "\n", 139 | "# the unique categories\n", 140 | "print( cdata.classes )\n", 141 | "\n", 142 | "# .targets is a list of targets converted to ints\n", 143 | "print( len(cdata.targets) )" 144 | ], 145 | "execution_count": null, 146 | "outputs": [] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "metadata": { 151 | "id": "hMpyFzF-d95B" 152 | }, 153 | "source": [ 154 | "# inspect a few random images\n", 155 | "\n", 156 | "fig,axs = plt.subplots(5,5,figsize=(10,10))\n", 157 | "\n", 158 | "for ax in axs.flatten():\n", 159 | "\n", 160 | " # select a random picture\n", 161 | " randidx = np.random.choice(len(cdata.targets))\n", 162 | "\n", 163 | " # extract that image\n", 164 | " pic = cdata.data[randidx,:,:,:]\n", 165 | " # and its label\n", 166 | " label = cdata.classes[cdata.targets[randidx]]\n", 167 | "\n", 168 | " # and show!\n", 169 | " ax.imshow(pic)\n", 170 | " ax.text(16,0,label,ha='center',fontweight='bold',color='k',backgroundcolor='y')\n", 171 | " ax.axis('off')\n", 172 | "\n", 173 | "plt.tight_layout()\n", 174 | "plt.show()" 175 | ], 176 | "execution_count": null, 177 | "outputs": [] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": { 182 | "id": "VAqVrcrGd98S" 183 | }, 184 | "source": [ 185 | "# Apply some transformations" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "metadata": { 191 | "id": "9SnUUHPm7xQE" 192 | }, 193 | "source": [ 194 | "Ts = T.Compose([ T.ToTensor(),\n", 195 | " T.Resize(32*4),\n", 196 | " T.Grayscale(num_output_channels=1) ])\n", 197 | "\n", 198 | "# include the transform in the dataset\n", 199 | "cdata.transform = Ts\n", 200 | "\n", 201 | "# you can also apply the transforms immediately when loading in the data\n", 202 | "# cdata = torchvision.datasets.CIFAR10(root='cifar10', download=True, transform=Ts)\n", 203 | "\n", 204 | "\n", 205 | "# Important! Adding a transform doesn't change the image data:\n", 206 | "print(cdata.data[123,:,:,:].shape)" 207 | ], 208 | "execution_count": null, 209 | "outputs": [] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "metadata": { 214 | "id": "poDa9N6TwjPu" 215 | }, 216 | "source": [ 217 | "# apply the transform\n", 218 | "\n", 219 | "# option 1a: apply the transform \"externally\" to an image\n", 220 | "img1 = Ts( cdata.data[123,:,:,:] )\n", 221 | "\n", 222 | "# option 1b: use the embedded transform\n", 223 | "img2 = cdata.transform( cdata.data[123,:,:,:] )\n", 224 | "\n", 225 | "# let's see what we've done!\n", 226 | "fig,ax = plt.subplots(1,3,figsize=(10,3))\n", 227 | "ax[0].imshow(cdata.data[123,:,:,:])\n", 228 | "ax[1].imshow(torch.squeeze(img1))\n", 229 | "ax[2].imshow(torch.squeeze(img2),cmap='gray')\n", 230 | "\n", 231 | "plt.show()" 232 | ], 233 | "execution_count": null, 234 | "outputs": [] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "metadata": { 239 | "id": "Ze3BK_VJgBqd" 240 | }, 241 | "source": [ 242 | "# Note about ToTensor() and normalization:\n", 243 | "??T.ToTensor()" 244 | ], 245 | "execution_count": null, 246 | "outputs": [] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "metadata": { 251 | "id": "fCooGt9PiNb6" 252 | }, 253 | "source": [ 254 | "" 255 | ], 256 | "execution_count": null, 257 | "outputs": [] 258 | }, 259 | { 260 | "cell_type": "markdown", 261 | "metadata": { 262 | "id": "TVXQlW-svDUE" 263 | }, 264 | "source": [ 265 | "# Additional explorations" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "metadata": { 271 | "id": "nViqoQuOvbZB" 272 | }, 273 | "source": [ 274 | "# 1) There are many other transforms available in torchvision: https://pytorch.org/vision/stable/transforms.html\n", 275 | "# Many transformations are useful for data preparation and augmentation. We'll cover some of them later in the course,\n", 276 | "# but for now, read about RandomCrop(), RandomHorizontalFlip(), and CenterCrop(). Then implement them to understand \n", 277 | "# what they do to images.\n", 278 | "# Tip: It's probably best to test these transforms separately, and on one test image, as we did above.\n", 279 | "# " 280 | ], 281 | "execution_count": null, 282 | "outputs": [] 283 | } 284 | ] 285 | } -------------------------------------------------------------------------------- /data/DUDL_data_data2colab.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_data_data2colab.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1GRajDS-VF5z8IslzZuMqbis3X6HDD-Uo", 10 | "timestamp": 1619517605734 11 | }, 12 | { 13 | "file_id": "1m0n2-UmB2tJiIDadlFkE6L5A4iZSqeBf", 14 | "timestamp": 1619459134813 15 | }, 16 | { 17 | "file_id": "19G9gTeBlYPQ-s3VS_3K2bVFtKTP344j6", 18 | "timestamp": 1619444797767 19 | }, 20 | { 21 | "file_id": "1FcEBC0NAESIlHQkv6_85R-XDUKGE8XbM", 22 | "timestamp": 1619155961717 23 | }, 24 | { 25 | "file_id": "1qKgZ8kVcqNgwtBzHbWq5yJH_HqI6DxWW", 26 | "timestamp": 1617803880910 27 | }, 28 | { 29 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 30 | "timestamp": 1617737766196 31 | }, 32 | { 33 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 34 | "timestamp": 1617734878578 35 | }, 36 | { 37 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 38 | "timestamp": 1617196833019 39 | }, 40 | { 41 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 42 | "timestamp": 1617124341706 43 | }, 44 | { 45 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 46 | "timestamp": 1616697516760 47 | }, 48 | { 49 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 50 | "timestamp": 1616615469755 51 | }, 52 | { 53 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 54 | "timestamp": 1616608248670 55 | } 56 | ], 57 | "collapsed_sections": [], 58 | "authorship_tag": "ABX9TyPXUVN9Wfifyp37WAgF8YA+" 59 | }, 60 | "kernelspec": { 61 | "name": "python3", 62 | "display_name": "Python 3" 63 | } 64 | }, 65 | "cells": [ 66 | { 67 | "cell_type": "markdown", 68 | "metadata": { 69 | "id": "bhWV8oes-wKR" 70 | }, 71 | "source": [ 72 | "# COURSE: A deep understanding of deep learning\n", 73 | "## SECTION: More on data\n", 74 | "### LECTURE: Getting data into colab\n", 75 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 76 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "metadata": { 82 | "id": "YeuAheYyhdZw" 83 | }, 84 | "source": [ 85 | "" 86 | ], 87 | "execution_count": null, 88 | "outputs": [] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": { 93 | "id": "0HOkOefftqyg" 94 | }, 95 | "source": [ 96 | "# Import from torchvision" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "metadata": { 102 | "id": "MU7rvmWuhjud" 103 | }, 104 | "source": [ 105 | "import torchvision\n", 106 | "\n", 107 | "# download the CIFAR10 dataset\n", 108 | "cdata = torchvision.datasets.CIFAR10(root='cifar10', download=True)\n", 109 | "\n", 110 | "print(cdata)" 111 | ], 112 | "execution_count": null, 113 | "outputs": [] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "metadata": { 118 | "id": "VhIKo0_iaGz2" 119 | }, 120 | "source": [ 121 | "# Datasets that come with torchvision: https://pytorch.org/vision/stable/index.html" 122 | ], 123 | "execution_count": null, 124 | "outputs": [] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": { 129 | "id": "_dR4LoMZx8GN" 130 | }, 131 | "source": [ 132 | "# Download from the web" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "metadata": { 138 | "id": "NhN7RYqYd915" 139 | }, 140 | "source": [ 141 | "import pandas as pd\n", 142 | "\n", 143 | "# url\n", 144 | "marriage_url = 'https://www.cdc.gov/nchs/data/dvs/state-marriage-rates-90-95-99-19.xlsx'\n", 145 | "\n", 146 | "# import directly into pandas\n", 147 | "data = pd.read_excel(marriage_url,header=5)\n", 148 | "data" 149 | ], 150 | "execution_count": null, 151 | "outputs": [] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "metadata": { 156 | "id": "hMpyFzF-d95B" 157 | }, 158 | "source": [ 159 | "" 160 | ], 161 | "execution_count": null, 162 | "outputs": [] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": { 167 | "id": "VAqVrcrGd98S" 168 | }, 169 | "source": [ 170 | "# Upload from hard drive" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "metadata": { 176 | "id": "9SnUUHPm7xQE" 177 | }, 178 | "source": [ 179 | "from google.colab import files\n", 180 | "uploaded = files.upload()" 181 | ], 182 | "execution_count": null, 183 | "outputs": [] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "metadata": { 188 | "id": "fCooGt9PiNb6" 189 | }, 190 | "source": [ 191 | "" 192 | ], 193 | "execution_count": null, 194 | "outputs": [] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": { 199 | "id": "TVXQlW-svDUE" 200 | }, 201 | "source": [ 202 | "# Map your google-drive" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "metadata": { 208 | "id": "nViqoQuOvbZB" 209 | }, 210 | "source": [ 211 | "from google.colab import drive\n", 212 | "drive.mount('/content/gdrive')" 213 | ], 214 | "execution_count": null, 215 | "outputs": [] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "metadata": { 220 | "id": "8aa1MGVC1fMm" 221 | }, 222 | "source": [ 223 | "" 224 | ], 225 | "execution_count": null, 226 | "outputs": [] 227 | } 228 | ] 229 | } -------------------------------------------------------------------------------- /data/DUDL_data_datasetLoader.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_data_datasetLoader.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1t1AgKE-GpPPUcjGZyTF_Ie1Q9XWA3xn-", 10 | "timestamp": 1618172332728 11 | }, 12 | { 13 | "file_id": "1qKgZ8kVcqNgwtBzHbWq5yJH_HqI6DxWW", 14 | "timestamp": 1617803880910 15 | }, 16 | { 17 | "file_id": "15cpyHkJ435B4MqbyGjAH1poN4nCy_DE4", 18 | "timestamp": 1617737766196 19 | }, 20 | { 21 | "file_id": "1OLuWuaFu0hcFgkQ2hh5BqbRuqUZD7XcQ", 22 | "timestamp": 1617734878578 23 | }, 24 | { 25 | "file_id": "1XvzVGJPTJifVh8OpZVB7ykLxyUqYwQ1j", 26 | "timestamp": 1617196833019 27 | }, 28 | { 29 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 30 | "timestamp": 1617124341706 31 | }, 32 | { 33 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 34 | "timestamp": 1616697516760 35 | }, 36 | { 37 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 38 | "timestamp": 1616615469755 39 | }, 40 | { 41 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 42 | "timestamp": 1616608248670 43 | } 44 | ], 45 | "collapsed_sections": [], 46 | "authorship_tag": "ABX9TyMDTrf0kN4swdushiJqB9Kn" 47 | }, 48 | "kernelspec": { 49 | "name": "python3", 50 | "display_name": "Python 3" 51 | } 52 | }, 53 | "cells": [ 54 | { 55 | "cell_type": "markdown", 56 | "metadata": { 57 | "id": "bhWV8oes-wKR" 58 | }, 59 | "source": [ 60 | "# COURSE: A deep understanding of deep learning\n", 61 | "## SECTION: Data matrices and loaders\n", 62 | "### LECTURE: Anatomy of a torch dataset and dataloader\n", 63 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 64 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "metadata": { 70 | "id": "YeuAheYyhdZw" 71 | }, 72 | "source": [ 73 | "# import libraries\n", 74 | "import numpy as np\n", 75 | "import torch\n", 76 | "from torch.utils.data import DataLoader,TensorDataset" 77 | ], 78 | "execution_count": null, 79 | "outputs": [] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": { 84 | "id": "vFv_cmvApLb0" 85 | }, 86 | "source": [ 87 | "# Datasets" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "metadata": { 93 | "id": "aykKFgznOako" 94 | }, 95 | "source": [ 96 | "# create some data in numpy\n", 97 | "\n", 98 | "nObservations = 100\n", 99 | "nFeatures = 20\n", 100 | "\n", 101 | "data = np.random.randn(nObservations,nFeatures)" 102 | ], 103 | "execution_count": null, 104 | "outputs": [] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "Y_tZ1ymVp0Sf" 110 | }, 111 | "source": [ 112 | "# Convert to pytorch tensor\n", 113 | "dataT = torch.tensor( data ) \n", 114 | "\n", 115 | "# print out some information\n", 116 | "print('Numpy data:')\n", 117 | "print(type(data))\n", 118 | "print(data.shape) # numpy -> .shape\n", 119 | "print(data.dtype)\n", 120 | "print(' ')\n", 121 | "\n", 122 | "print('Tensor data:')\n", 123 | "print(type(dataT))\n", 124 | "print(dataT.size()) # torch -> .size()\n", 125 | "print(dataT.dtype)\n", 126 | "print(' ')" 127 | ], 128 | "execution_count": null, 129 | "outputs": [] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "metadata": { 134 | "id": "WlSTQeZ2nDDR" 135 | }, 136 | "source": [ 137 | "# Sometimes you need to convert data types\n", 138 | "\n", 139 | "dataT2 = torch.tensor( data ).float()\n", 140 | "print(dataT2.dtype)\n", 141 | "\n", 142 | "# \"long\" is for ints\n", 143 | "dataT3 = torch.tensor( data ).long()\n", 144 | "print(dataT3.dtype)\n" 145 | ], 146 | "execution_count": null, 147 | "outputs": [] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "u5fGd4h8mI8a" 153 | }, 154 | "source": [ 155 | "# Convert tensor into PyTorch Datasets\n", 156 | "\n", 157 | "# dataset = TensorDataset(data) # not a tensor!\n", 158 | "dataset = TensorDataset(dataT)\n", 159 | "\n", 160 | "# dataset is a two-element tuple comprising data,labels\n", 161 | "dataset.tensors[0]" 162 | ], 163 | "execution_count": null, 164 | "outputs": [] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "metadata": { 169 | "id": "wpvxvxBloej3" 170 | }, 171 | "source": [ 172 | "# Let's try again with labels\n", 173 | "labels = torch.ceil(torch.linspace(.01,4,nObservations))\n", 174 | "\n", 175 | "# transform to an actual matrix (column vector)\n", 176 | "labels = labels.reshape(( len(labels),1 ))\n", 177 | "# print( labels )\n", 178 | "\n", 179 | "# now make another dataset\n", 180 | "dataset = TensorDataset(dataT,labels)\n", 181 | "print( dataset.tensors[0].size() )\n", 182 | "print( dataset.tensors[1].size() )\n", 183 | "\n", 184 | "# for comparison\n", 185 | "print( np.shape(np.random.randint(5,size=nObservations)) )" 186 | ], 187 | "execution_count": null, 188 | "outputs": [] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": { 193 | "id": "wJ-JsNQGpIKT" 194 | }, 195 | "source": [ 196 | "# DataLoaders" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "metadata": { 202 | "id": "5_kahYcanxBg" 203 | }, 204 | "source": [ 205 | "# create a dataloader object\n", 206 | "batchsize = 25\n", 207 | "dataloader = DataLoader(dataset,batch_size=batchsize)#,shuffle=True,drop_last=True)\n", 208 | "\n", 209 | "dataloader.dataset.tensors[0].size()" 210 | ], 211 | "execution_count": null, 212 | "outputs": [] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "metadata": { 217 | "id": "xu7BF_OTqDj0" 218 | }, 219 | "source": [ 220 | "# sizes of each batch\n", 221 | "for dat,labs in dataloader:\n", 222 | " print('BATCH INFO:')\n", 223 | " print(dat.size())\n", 224 | " print(labs.size())\n", 225 | " print(' ')" 226 | ], 227 | "execution_count": null, 228 | "outputs": [] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "metadata": { 233 | "id": "oLkY18PZq-G3" 234 | }, 235 | "source": [ 236 | "# inspect the labels\n", 237 | "for dat,labs in dataloader:\n", 238 | " print(labs.T)\n", 239 | " print(' ')" 240 | ], 241 | "execution_count": null, 242 | "outputs": [] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "metadata": { 247 | "id": "zyeJ6mjjre6p" 248 | }, 249 | "source": [ 250 | "# try again with shuffling (shuffling happens during iterations)\n", 251 | "# dataloader = DataLoader(dataset,batch_size=batchsize,shuffle=True)\n", 252 | "\n", 253 | "for dat,labs in dataloader:\n", 254 | " print(labs.T)\n", 255 | " print(' ')" 256 | ], 257 | "execution_count": null, 258 | "outputs": [] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "metadata": { 263 | "id": "S236TLLury42" 264 | }, 265 | "source": [ 266 | "# To get only one batch (e.g., for testing)\n", 267 | "\n", 268 | "dat,labs = next(iter(dataloader))\n", 269 | "\n", 270 | "labs" 271 | ], 272 | "execution_count": null, 273 | "outputs": [] 274 | } 275 | ] 276 | } -------------------------------------------------------------------------------- /gradientDescent/DUDL_GradientDescent_1D.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1qFigOOWXcSNyA6hPpZnF-QqhFEB1e-hy", 9 | "timestamp": 1597210805829 10 | }, 11 | { 12 | "file_id": "1kGRo0g3UXxXpJuQSEtpKjEGA1Vxbaz8S", 13 | "timestamp": 1597128018290 14 | }, 15 | { 16 | "file_id": "1U4oG0A3DFC-XBWhvecYeA3YYReqHpShX", 17 | "timestamp": 1594575042741 18 | } 19 | ], 20 | "authorship_tag": "ABX9TyPYoewyzb58f8ctFslnMpqP" 21 | }, 22 | "kernelspec": { 23 | "name": "python3", 24 | "display_name": "Python 3" 25 | } 26 | }, 27 | "cells": [ 28 | { 29 | "cell_type": "markdown", 30 | "metadata": { 31 | "id": "bhWV8oes-wKR" 32 | }, 33 | "source": [ 34 | "# COURSE: A deep understanding of deep learning\n", 35 | "## SECTION: Gradient descent\n", 36 | "### LECTURE: Gradient descent in 1D\n", 37 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 38 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "metadata": { 44 | "id": "JL_0UKJOj1YP" 45 | }, 46 | "source": [ 47 | "# import all necessary modules\n", 48 | "import numpy as np\n", 49 | "import matplotlib.pyplot as plt\n", 50 | "\n", 51 | "import matplotlib_inline.backend_inline\n", 52 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 53 | ], 54 | "execution_count": null, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": { 60 | "id": "OeYMLgEvZY1X" 61 | }, 62 | "source": [ 63 | "# Gradient descent in 1D" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "metadata": { 69 | "id": "YwTBzVJsoKbg" 70 | }, 71 | "source": [ 72 | "# function (as a function)\n", 73 | "def fx(x):\n", 74 | " return 3*x**2 - 3*x + 4\n", 75 | "\n", 76 | "# derivative function\n", 77 | "def deriv(x):\n", 78 | " return 6*x - 3" 79 | ], 80 | "execution_count": null, 81 | "outputs": [] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "metadata": { 86 | "id": "_qRE_fHqUL6n" 87 | }, 88 | "source": [ 89 | "# plot the function and its derivative\n", 90 | "\n", 91 | "# define a range for x\n", 92 | "x = np.linspace(-2,2,2001)\n", 93 | "\n", 94 | "# plotting\n", 95 | "plt.plot(x,fx(x), x,deriv(x))\n", 96 | "plt.xlim(x[[0,-1]])\n", 97 | "plt.grid()\n", 98 | "plt.xlabel('x')\n", 99 | "plt.ylabel('f(x)')\n", 100 | "plt.legend(['y','dy'])\n", 101 | "plt.show()" 102 | ], 103 | "execution_count": null, 104 | "outputs": [] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "r-f27UUkZYpG" 110 | }, 111 | "source": [ 112 | "\n" 113 | ], 114 | "execution_count": null, 115 | "outputs": [] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "metadata": { 120 | "id": "-nzWuHfWVHyU" 121 | }, 122 | "source": [ 123 | "# random starting point\n", 124 | "localmin = np.random.choice(x,1)\n", 125 | "print(localmin)\n", 126 | "\n", 127 | "# learning parameters\n", 128 | "learning_rate = .01\n", 129 | "training_epochs = 100\n", 130 | "\n", 131 | "# run through training\n", 132 | "for i in range(training_epochs):\n", 133 | " grad = deriv(localmin)\n", 134 | " localmin = localmin - learning_rate*grad\n", 135 | "\n", 136 | "localmin" 137 | ], 138 | "execution_count": null, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "3hIord_pVIU8" 145 | }, 146 | "source": [ 147 | "# plot the results\n", 148 | "\n", 149 | "plt.plot(x,fx(x), x,deriv(x))\n", 150 | "plt.plot(localmin,deriv(localmin),'ro')\n", 151 | "plt.plot(localmin,fx(localmin),'ro')\n", 152 | "\n", 153 | "plt.xlim(x[[0,-1]])\n", 154 | "plt.grid()\n", 155 | "plt.xlabel('x')\n", 156 | "plt.ylabel('f(x)')\n", 157 | "plt.legend(['f(x)','df','f(x) min'])\n", 158 | "plt.title('Empirical local minimum: %s'%localmin[0])\n", 159 | "plt.show()" 160 | ], 161 | "execution_count": null, 162 | "outputs": [] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": { 167 | "id": "AKN5ul4YVIYb" 168 | }, 169 | "source": [ 170 | "# Store the model parameters and outputs on each iteration" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "metadata": { 176 | "id": "M22aVI6xVIbk" 177 | }, 178 | "source": [ 179 | "# random starting point\n", 180 | "localmin = np.random.choice(x,1)\n", 181 | "\n", 182 | "# learning parameters\n", 183 | "learning_rate = .01\n", 184 | "training_epochs = 100\n", 185 | "\n", 186 | "# run through training and store all the results\n", 187 | "modelparams = np.zeros((training_epochs,2))\n", 188 | "for i in range(training_epochs):\n", 189 | " grad = deriv(localmin)\n", 190 | " localmin = localmin - learning_rate*grad\n", 191 | " modelparams[i,0] = localmin[0]\n", 192 | " modelparams[i,1] = grad[0]\n" 193 | ], 194 | "execution_count": null, 195 | "outputs": [] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "metadata": { 200 | "id": "AInqnFtkVIeb" 201 | }, 202 | "source": [ 203 | "# plot the gradient over iterations\n", 204 | "\n", 205 | "fig,ax = plt.subplots(1,2,figsize=(12,4))\n", 206 | "\n", 207 | "for i in range(2):\n", 208 | " ax[i].plot(modelparams[:,i],'o-')\n", 209 | " ax[i].set_xlabel('Iteration')\n", 210 | " ax[i].set_title(f'Final estimated minimum: {localmin[0]:.5f}')\n", 211 | "\n", 212 | "ax[0].set_ylabel('Local minimum')\n", 213 | "ax[1].set_ylabel('Derivative')\n", 214 | "\n", 215 | "plt.show()" 216 | ], 217 | "execution_count": null, 218 | "outputs": [] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "metadata": { 223 | "id": "MAGG5fMMVIhM" 224 | }, 225 | "source": [], 226 | "execution_count": null, 227 | "outputs": [] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "metadata": { 232 | "id": "Uh28k_l29urR" 233 | }, 234 | "source": [ 235 | "# Additional explorations" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "metadata": { 241 | "id": "ib3uQtfv9wE2" 242 | }, 243 | "source": [ 244 | "# 1) Most often in DL, the model trains for a set number of iterations, which is what we do here. But there are other ways\n", 245 | "# of defining how long the training lasts. Modify the code so that training ends when the derivative is smaller than \n", 246 | "# some threshold, e.g., 0.1. Make sure your code is robust for negative derivatives.\n", 247 | "# \n", 248 | "# 2) Does this change to the code produce a more accurate result? What if you change the stopping threshold?\n", 249 | "# \n", 250 | "# 3) Can you think of any potential problems that might arise when the stopping criterion is based on the derivative \n", 251 | "# instead of a specified number of training epochs?\n", 252 | "# " 253 | ], 254 | "execution_count": null, 255 | "outputs": [] 256 | } 257 | ] 258 | } 259 | -------------------------------------------------------------------------------- /gradientDescent/DUDL_GradientDescent_2D.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1qFigOOWXcSNyA6hPpZnF-QqhFEB1e-hy", 9 | "timestamp": 1597210805829 10 | }, 11 | { 12 | "file_id": "1kGRo0g3UXxXpJuQSEtpKjEGA1Vxbaz8S", 13 | "timestamp": 1597128018290 14 | }, 15 | { 16 | "file_id": "1U4oG0A3DFC-XBWhvecYeA3YYReqHpShX", 17 | "timestamp": 1594575042741 18 | } 19 | ] 20 | }, 21 | "kernelspec": { 22 | "name": "python3", 23 | "display_name": "Python 3" 24 | } 25 | }, 26 | "cells": [ 27 | { 28 | "cell_type": "markdown", 29 | "metadata": { 30 | "id": "bhWV8oes-wKR" 31 | }, 32 | "source": [ 33 | "# COURSE: A deep understanding of deep learning\n", 34 | "## SECTION: Gradient descent\n", 35 | "### LECTURE: Gradient descent in 2D\n", 36 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 37 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "metadata": { 43 | "id": "JL_0UKJOj1YP" 44 | }, 45 | "source": [ 46 | "# import all necessary modules\n", 47 | "import numpy as np\n", 48 | "import matplotlib.pyplot as plt\n", 49 | "import sympy as sym # sympy to compute the partial derivatives\n", 50 | "\n", 51 | "import matplotlib_inline.backend_inline\n", 52 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 53 | ], 54 | "execution_count": null, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": { 60 | "id": "nuu117ieVIj0" 61 | }, 62 | "source": [ 63 | "# Gradient descent in 2D" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "metadata": { 69 | "id": "0w52qmG_bj1w" 70 | }, 71 | "source": [ 72 | "# the \"peaks\" function\n", 73 | "def peaks(x,y):\n", 74 | " # expand to a 2D mesh\n", 75 | " x,y = np.meshgrid(x,y)\n", 76 | " \n", 77 | " z = 3*(1-x)**2 * np.exp(-(x**2) - (y+1)**2) \\\n", 78 | " - 10*(x/5 - x**3 - y**5) * np.exp(-x**2-y**2) \\\n", 79 | " - 1/3*np.exp(-(x+1)**2 - y**2)\n", 80 | " return z" 81 | ], 82 | "execution_count": null, 83 | "outputs": [] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "metadata": { 88 | "id": "SFLOlXhVbj5A" 89 | }, 90 | "source": [ 91 | "# create the landscape\n", 92 | "x = np.linspace(-3,3,201)\n", 93 | "y = np.linspace(-3,3,201)\n", 94 | "\n", 95 | "Z = peaks(x,y)\n", 96 | "\n", 97 | "# let's have a look!\n", 98 | "plt.imshow(Z,extent=[x[0],x[-1],y[0],y[-1]],vmin=-5,vmax=5,origin='lower')\n", 99 | "plt.show()" 100 | ], 101 | "execution_count": null, 102 | "outputs": [] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "metadata": { 107 | "id": "guL7sYzwbj_q" 108 | }, 109 | "source": [ 110 | "# create derivative functions using sympy\n", 111 | "\n", 112 | "sx,sy = sym.symbols('sx,sy')\n", 113 | "\n", 114 | "sZ = 3*(1-sx)**2 * sym.exp(-(sx**2) - (sy+1)**2) \\\n", 115 | " - 10*(sx/5 - sx**3 - sy**5) * sym.exp(-sx**2-sy**2) \\\n", 116 | " - 1/3*sym.exp(-(sx+1)**2 - sy**2)\n", 117 | "\n", 118 | "\n", 119 | "# create functions from the sympy-computed derivatives\n", 120 | "df_x = sym.lambdify( (sx,sy),sym.diff(sZ,sx),'sympy' )\n", 121 | "df_y = sym.lambdify( (sx,sy),sym.diff(sZ,sy),'sympy' )\n", 122 | "\n", 123 | "df_x(1,1).evalf()" 124 | ], 125 | "execution_count": null, 126 | "outputs": [] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "metadata": { 131 | "id": "mo3H8RpgiUN9" 132 | }, 133 | "source": [ 134 | "# random starting point (uniform between -2 and +2)\n", 135 | "localmin = np.random.rand(2)*4-2 # also try specifying coordinates\n", 136 | "startpnt = localmin[:] # make a copy, not re-assign\n", 137 | "\n", 138 | "# learning parameters\n", 139 | "learning_rate = .01\n", 140 | "training_epochs = 1000\n", 141 | "\n", 142 | "# run through training\n", 143 | "trajectory = np.zeros((training_epochs,2))\n", 144 | "for i in range(training_epochs):\n", 145 | " grad = np.array([ df_x(localmin[0],localmin[1]).evalf(), \n", 146 | " df_y(localmin[0],localmin[1]).evalf() \n", 147 | " ])\n", 148 | " localmin = localmin - learning_rate*grad # add _ or [:] to change a variable in-place\n", 149 | " trajectory[i,:] = localmin\n", 150 | "\n", 151 | "\n", 152 | "print(localmin)\n", 153 | "print(startpnt)" 154 | ], 155 | "execution_count": null, 156 | "outputs": [] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "metadata": { 161 | "id": "sRrjCTqFbkCo" 162 | }, 163 | "source": [ 164 | "# let's have a look!\n", 165 | "plt.imshow(Z,extent=[x[0],x[-1],y[0],y[-1]],vmin=-5,vmax=5,origin='lower')\n", 166 | "plt.plot(startpnt[0],startpnt[1],'bs')\n", 167 | "plt.plot(localmin[0],localmin[1],'ro')\n", 168 | "plt.plot(trajectory[:,0],trajectory[:,1],'r')\n", 169 | "plt.legend(['rnd start','local min'])\n", 170 | "plt.colorbar()\n", 171 | "plt.show()" 172 | ], 173 | "execution_count": null, 174 | "outputs": [] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "metadata": { 179 | "id": "cY-P0o9vbkJD" 180 | }, 181 | "source": [], 182 | "execution_count": null, 183 | "outputs": [] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": { 188 | "id": "QvGE12fEiui7" 189 | }, 190 | "source": [ 191 | "# Additional explorations" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "metadata": { 197 | "id": "m54Y_kYXiwO9" 198 | }, 199 | "source": [ 200 | "# 1) Modify the code to force the initial guess to be [0,1.4]. Does the model reach a reasonable local minimum?\n", 201 | "# \n", 202 | "# 2) Using the same starting point, change the number of training epochs to 10,000. Does the final solution differ from\n", 203 | "# using 1000 epochs? \n", 204 | "# \n", 205 | "# 3) (Again with the same starting location) Change the learning to .1 (1000 epochs). What do you notice about the trajectory?\n", 206 | "# Try again with the learning rate set to .5, and then to .00001.\n", 207 | "# " 208 | ], 209 | "execution_count": null, 210 | "outputs": [] 211 | } 212 | ] 213 | } -------------------------------------------------------------------------------- /gradientDescent/DUDL_GradientDescent_CodeChallengeStartValue.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_GradientDescent_CodeChallengeStartValue.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1AKaKETDfNqVEFzv5BnYGSjHfaPtFpH4x", 10 | "timestamp": 1617033326873 11 | }, 12 | { 13 | "file_id": "1qFigOOWXcSNyA6hPpZnF-QqhFEB1e-hy", 14 | "timestamp": 1597210805829 15 | }, 16 | { 17 | "file_id": "1kGRo0g3UXxXpJuQSEtpKjEGA1Vxbaz8S", 18 | "timestamp": 1597128018290 19 | }, 20 | { 21 | "file_id": "1U4oG0A3DFC-XBWhvecYeA3YYReqHpShX", 22 | "timestamp": 1594575042741 23 | } 24 | ], 25 | "collapsed_sections": [], 26 | "authorship_tag": "ABX9TyNVWu+09BvHo8TdA+4YYlwi" 27 | }, 28 | "kernelspec": { 29 | "name": "python3", 30 | "display_name": "Python 3" 31 | } 32 | }, 33 | "cells": [ 34 | { 35 | "cell_type": "markdown", 36 | "metadata": { 37 | "id": "bhWV8oes-wKR" 38 | }, 39 | "source": [ 40 | "# COURSE: A deep understanding of deep learning\n", 41 | "## SECTION: Gradient descent\n", 42 | "### LECTURE: CodeChallenge: Unfortunate starting value\n", 43 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 44 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "metadata": { 50 | "id": "JL_0UKJOj1YP" 51 | }, 52 | "source": [ 53 | "# import all necessary modules\n", 54 | "import numpy as np\n", 55 | "import matplotlib.pyplot as plt" 56 | ], 57 | "execution_count": null, 58 | "outputs": [] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": { 63 | "id": "OeYMLgEvZY1X" 64 | }, 65 | "source": [ 66 | "# Gradient descent in 1D" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "metadata": { 72 | "id": "YwTBzVJsoKbg" 73 | }, 74 | "source": [ 75 | "# function (as a function)\n", 76 | "def fx(x):\n", 77 | " return np.cos(2*np.pi*x) + x**2\n", 78 | "\n", 79 | "# derivative function\n", 80 | "def deriv(x):\n", 81 | " return -2*np.pi*np.sin(2*np.pi*x) + 2*x" 82 | ], 83 | "execution_count": null, 84 | "outputs": [] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "metadata": { 89 | "id": "_qRE_fHqUL6n" 90 | }, 91 | "source": [ 92 | "# plot the function and its derivative\n", 93 | "\n", 94 | "# define a range for x\n", 95 | "x = np.linspace(-2,2,2001)\n", 96 | "\n", 97 | "# plotting\n", 98 | "plt.plot(x,fx(x), x,deriv(x))\n", 99 | "plt.xlim(x[[0,-1]])\n", 100 | "plt.grid()\n", 101 | "plt.xlabel('x')\n", 102 | "plt.ylabel('f(x)')\n", 103 | "plt.legend(['y','dy'])\n", 104 | "plt.show()" 105 | ], 106 | "execution_count": null, 107 | "outputs": [] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "metadata": { 112 | "id": "r-f27UUkZYpG" 113 | }, 114 | "source": [ 115 | "" 116 | ], 117 | "execution_count": null, 118 | "outputs": [] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "metadata": { 123 | "id": "-nzWuHfWVHyU" 124 | }, 125 | "source": [ 126 | "# random starting point\n", 127 | "localmin = np.random.choice(x,1) #np.array([0])#\n", 128 | "\n", 129 | "# learning parameters\n", 130 | "learning_rate = .01\n", 131 | "training_epochs = 100\n", 132 | "\n", 133 | "# run through training\n", 134 | "for i in range(training_epochs):\n", 135 | " grad = deriv(localmin)\n", 136 | " localmin = localmin - learning_rate*grad\n", 137 | "\n", 138 | "localmin" 139 | ], 140 | "execution_count": null, 141 | "outputs": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "metadata": { 146 | "id": "3hIord_pVIU8" 147 | }, 148 | "source": [ 149 | "# plot the results\n", 150 | "\n", 151 | "plt.plot(x,fx(x), x,deriv(x))\n", 152 | "plt.plot(localmin,deriv(localmin),'ro')\n", 153 | "plt.plot(localmin,fx(localmin),'ro')\n", 154 | "\n", 155 | "plt.xlim(x[[0,-1]])\n", 156 | "plt.grid()\n", 157 | "plt.xlabel('x')\n", 158 | "plt.ylabel('f(x)')\n", 159 | "plt.legend(['f(x)','df','f(x) min'])\n", 160 | "plt.title('Empirical local minimum: %s'%localmin[0])\n", 161 | "plt.show()" 162 | ], 163 | "execution_count": null, 164 | "outputs": [] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "metadata": { 169 | "id": "MAGG5fMMVIhM" 170 | }, 171 | "source": [ 172 | "" 173 | ], 174 | "execution_count": null, 175 | "outputs": [] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": { 180 | "id": "Uh28k_l29urR" 181 | }, 182 | "source": [ 183 | "# Additional explorations" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "metadata": { 189 | "id": "ib3uQtfv9wE2" 190 | }, 191 | "source": [ 192 | "# 1) The derivative has a multiplicative factor of 2 in it. Is that constant necessary for the accuracy of the g.d. result?\n", 193 | "# Try removing that '2' from the derivative and see whether the model can still find the minimum. Before running the\n", 194 | "# code, think about what you expect to happen. Does reality match your expectations? Why is (or isn't) that factor necessary?\n", 195 | "# \n", 196 | "# 2) What about the factor of '2' inside the np.sin() function? Is that important? Can you get an accurate result if you\n", 197 | "# remove it?\n", 198 | "# \n", 199 | "# 3) Try setting the initial value to a small but non-zero number, e.g., .0001 or -.0001. Does that help the solution?\n", 200 | "# " 201 | ], 202 | "execution_count": null, 203 | "outputs": [] 204 | } 205 | ] 206 | } -------------------------------------------------------------------------------- /gradientDescent/DUDL_GradientDescent_codeChallenge_lr.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [] 7 | }, 8 | "kernelspec": { 9 | "name": "python3", 10 | "display_name": "Python 3" 11 | } 12 | }, 13 | "cells": [ 14 | { 15 | "cell_type": "markdown", 16 | "metadata": { 17 | "id": "bhWV8oes-wKR" 18 | }, 19 | "source": [ 20 | "# COURSE: A deep understanding of deep learning\n", 21 | "## SECTION: Gradient descent\n", 22 | "### LECTURE: CodeChallenge: dynamic learning rates\n", 23 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 24 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "JL_0UKJOj1YP" 31 | }, 32 | "source": [ 33 | "# import all necessary modules\n", 34 | "import numpy as np\n", 35 | "import matplotlib.pyplot as plt\n", 36 | "\n", 37 | "import matplotlib_inline.backend_inline\n", 38 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 39 | ], 40 | "execution_count": null, 41 | "outputs": [] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": { 46 | "id": "OeYMLgEvZY1X" 47 | }, 48 | "source": [ 49 | "# Create the function and its derivative" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "metadata": { 55 | "id": "YwTBzVJsoKbg" 56 | }, 57 | "source": [ 58 | "# define a range for x\n", 59 | "x = np.linspace(-2,2,2001)\n", 60 | "\n", 61 | "# function (as a function)\n", 62 | "def fx(x):\n", 63 | " return 3*x**2 - 3*x + 4\n", 64 | "\n", 65 | "# derivative function\n", 66 | "def deriv(x):\n", 67 | " return 6*x - 3" 68 | ], 69 | "execution_count": null, 70 | "outputs": [] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": { 75 | "id": "nWC2T55ovJ-4" 76 | }, 77 | "source": [ 78 | "### G.D. using a fixed learning rate" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "metadata": { 84 | "id": "yB8dmH-nvJ_D" 85 | }, 86 | "source": [ 87 | "# random starting point\n", 88 | "localmin = np.random.choice(x,1)\n", 89 | "initval = localmin[:] # store the initial value\n", 90 | "\n", 91 | "# learning parameters\n", 92 | "learning_rate = .01\n", 93 | "training_epochs = 50\n", 94 | "\n", 95 | "# run through training and store all the results\n", 96 | "modelparamsFixed = np.zeros((training_epochs,3))\n", 97 | "for i in range(training_epochs):\n", 98 | "\n", 99 | " # compute gradient\n", 100 | " grad = deriv(localmin)\n", 101 | "\n", 102 | " # non-adaptive learning rate\n", 103 | " lr = learning_rate\n", 104 | "\n", 105 | " # update parameter according to g.d.\n", 106 | " localmin = localmin - lr*grad\n", 107 | "\n", 108 | " # store the parameters\n", 109 | " modelparamsFixed[i,0] = localmin[0]\n", 110 | " modelparamsFixed[i,1] = grad[0]\n", 111 | " modelparamsFixed[i,2] = lr" 112 | ], 113 | "execution_count": null, 114 | "outputs": [] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": { 119 | "id": "G24R3zyh6XWA" 120 | }, 121 | "source": [ 122 | "### G.D. using a gradient-based learning rate" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "metadata": { 128 | "id": "M22aVI6xVIbk" 129 | }, 130 | "source": [ 131 | "# random starting point\n", 132 | "localmin = np.random.choice(x,1)\n", 133 | "initval = localmin[:] # store the initial value\n", 134 | "\n", 135 | "# learning parameters\n", 136 | "learning_rate = .01\n", 137 | "\n", 138 | "# run through training and store all the results\n", 139 | "modelparamsGrad = np.zeros((training_epochs,3))\n", 140 | "for i in range(training_epochs):\n", 141 | "\n", 142 | " # compute gradient\n", 143 | " grad = deriv(localmin)\n", 144 | "\n", 145 | " # adapt the learning rate according to the gradient\n", 146 | " lr = learning_rate*np.abs(grad)\n", 147 | "\n", 148 | " # update parameter according to g.d.\n", 149 | " localmin = localmin - lr*grad\n", 150 | "\n", 151 | " # store the parameters\n", 152 | " modelparamsGrad[i,0] = localmin[0]\n", 153 | " modelparamsGrad[i,1] = grad[0]\n", 154 | " modelparamsGrad[i,2] = lr[0]" 155 | ], 156 | "execution_count": null, 157 | "outputs": [] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": { 162 | "id": "49VT4I5B6rhQ" 163 | }, 164 | "source": [ 165 | "### G.D. using a time-based learning rate" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "metadata": { 171 | "id": "X6OWQULR30oV" 172 | }, 173 | "source": [ 174 | "# redefine parameters\n", 175 | "learning_rate = .1\n", 176 | "localmin = initval\n", 177 | "\n", 178 | "# run through training and store all the results\n", 179 | "modelparamsTime = np.zeros((training_epochs,3))\n", 180 | "for i in range(training_epochs):\n", 181 | " grad = deriv(localmin)\n", 182 | " lr = learning_rate*(1-(i+1)/training_epochs)\n", 183 | " localmin = localmin - lr*grad\n", 184 | " modelparamsTime[i,0] = localmin[0]\n", 185 | " modelparamsTime[i,1] = grad[0]\n", 186 | " modelparamsTime[i,2] = lr" 187 | ], 188 | "execution_count": null, 189 | "outputs": [] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": { 194 | "id": "XEYWC-m36yYX" 195 | }, 196 | "source": [ 197 | "### Plot the results" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "metadata": { 203 | "id": "AInqnFtkVIeb" 204 | }, 205 | "source": [ 206 | "fig,ax = plt.subplots(1,3,figsize=(10,3))\n", 207 | "\n", 208 | "# generate the plots\n", 209 | "for i in range(3):\n", 210 | " ax[i].plot(modelparamsFixed[:,i],'o-',markerfacecolor='w')\n", 211 | " ax[i].plot(modelparamsGrad[:,i],'o-',markerfacecolor='w')\n", 212 | " ax[i].plot(modelparamsTime[:,i],'o-',markerfacecolor='w')\n", 213 | " ax[i].set_xlabel('Iteration')\n", 214 | "\n", 215 | "ax[0].set_ylabel('Local minimum')\n", 216 | "ax[1].set_ylabel('Derivative')\n", 217 | "ax[2].set_ylabel('Learning rate')\n", 218 | "ax[2].legend(['Fixed l.r.','Grad-based l.r.','Time-based l.r.'])\n", 219 | "\n", 220 | "plt.tight_layout()\n", 221 | "plt.show()" 222 | ], 223 | "execution_count": null, 224 | "outputs": [] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": { 229 | "id": "QvGE12fEiui7" 230 | }, 231 | "source": [ 232 | "# Additional explorations" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "metadata": { 238 | "id": "m54Y_kYXiwO9" 239 | }, 240 | "source": [ 241 | "# 1) Change the initial learning rate in the \"time\" experiment from .1 to .01. Do you still reach the same conclusion that\n", 242 | "# dynamic learning rates are better than a fixed learning rate?\n", 243 | "#\n", 244 | "# 2) Compute the average of all time-based learning rates (see variable 'modelparamsTime'). Next, replace the fixed\n", 245 | "# learning rate with the average over all dynamic learning rates. How does that affect the model's performance?\n", 246 | "#\n", 247 | "# 3) Going back to the original code (without the modifications above), you saw that the fixed learning rate model didn't\n", 248 | "# get to the correct local minimum. What happens if you increase the number of training epochs from 50 to 500? Does that\n", 249 | "# improve the situation, and what does that tell you about the relationship between learning rate and training epochs?\n", 250 | "#\n", 251 | "# 4) The code here initializes the starting value as a random number, which will differ for each learning rate method.\n", 252 | "# Is that appropriate or inappropriate for this experiment? Why? Change the code so that the starting value is the\n", 253 | "# same for all three learning rate models.\n", 254 | "#" 255 | ], 256 | "execution_count": null, 257 | "outputs": [] 258 | } 259 | ] 260 | } -------------------------------------------------------------------------------- /math/DUDL_math_argmin.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_argmin.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyOAnDMLXriVbV0NCcYRd0xe" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Math prerequisites\n", 28 | "### LECTURE: Min/max and argmin/max\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "wYjuZD1kPb3g" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import numpy as np\n", 41 | "import torch" 42 | ], 43 | "execution_count": null, 44 | "outputs": [] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": { 49 | "id": "8HsmHiC6Pf3t" 50 | }, 51 | "source": [ 52 | "# Using numpy" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "id": "eDMs2HT0Ph4r" 59 | }, 60 | "source": [ 61 | "# create a vector\n", 62 | "v = np.array([ 1,40,2,-3 ])\n", 63 | "\n", 64 | "# find and report the maximum and minimum values\n", 65 | "minval = np.min(v)\n", 66 | "maxval = np.max(v)\n", 67 | "\n", 68 | "print('Min,max: %g,%g' %(minval,maxval) )\n", 69 | "\n", 70 | "\n", 71 | "# now for the argmin/max\n", 72 | "minidx = np.argmin(v)\n", 73 | "maxidx = np.argmax(v)\n", 74 | "\n", 75 | "print('Min,max indices: %g,%g' %(minidx,maxidx) ), print(' ')\n", 76 | "\n", 77 | "# confirm\n", 78 | "print(f'Min val is { v[minidx] }, max val is { v[maxidx] }')" 79 | ], 80 | "execution_count": null, 81 | "outputs": [] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "metadata": { 86 | "id": "KVsOLZ66Ph7s" 87 | }, 88 | "source": [ 89 | "# repeat with matrix\n", 90 | "M = np.array([ [0,1,10], [20,8,5] ])\n", 91 | "print(M), print(' ')\n", 92 | "\n", 93 | "# various minima in this matrix!\n", 94 | "minvals1 = np.min(M) # minimum from ENTIRE matrix\n", 95 | "minvals2 = np.min(M,axis=0) # minimum of each column (across rows)\n", 96 | "minvals3 = np.min(M,axis=1) # minimum of each row (across columns)\n", 97 | "\n", 98 | "# print them out\n", 99 | "print(minvals1)\n", 100 | "print(minvals2)\n", 101 | "print(minvals3)" 102 | ], 103 | "execution_count": null, 104 | "outputs": [] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "KygYLWYoTIT4" 110 | }, 111 | "source": [ 112 | "# various minima in this matrix!\n", 113 | "minidx1 = np.argmin(M) # minimum from ENTIRE matrix\n", 114 | "minidx2 = np.argmin(M,axis=0) # minimum of each column (across rows)\n", 115 | "minidx3 = np.argmin(M,axis=1) # minimum of each row (across columns)\n", 116 | "\n", 117 | "# print them out\n", 118 | "print(M), print(' ') # reminder\n", 119 | "print(minidx1)\n", 120 | "print(minidx2)\n", 121 | "print(minidx3)" 122 | ], 123 | "execution_count": null, 124 | "outputs": [] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": { 129 | "id": "-M4TujzAR5vb" 130 | }, 131 | "source": [ 132 | "# Now using pytorch" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "metadata": { 138 | "id": "4wYM8pGRR76s" 139 | }, 140 | "source": [ 141 | "# create a vector\n", 142 | "v = torch.tensor([ 1,40,2,-3 ])\n", 143 | "\n", 144 | "# find and report the maximum and minimum values\n", 145 | "minval = torch.min(v)\n", 146 | "maxval = torch.max(v)\n", 147 | "\n", 148 | "print('Min,max: %g,%g' %(minval,maxval) )\n", 149 | "\n", 150 | "\n", 151 | "# now for the argmin/max\n", 152 | "minidx = torch.argmin(v)\n", 153 | "maxidx = torch.argmax(v)\n", 154 | "\n", 155 | "print('Min,max indices: %g,%g' %(minidx,maxidx) ), print(' ')\n", 156 | "\n", 157 | "# confirm\n", 158 | "print(f'Min val is { v[minidx] }, max val is { v[maxidx] }')" 159 | ], 160 | "execution_count": null, 161 | "outputs": [] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "metadata": { 166 | "id": "7AP93vgZR788" 167 | }, 168 | "source": [ 169 | "# repeat with matrix\n", 170 | "M = torch.tensor([ [0,1,10], [20,8,5] ])\n", 171 | "print(M), print(' ')\n", 172 | "\n", 173 | "# various minima in this matrix!\n", 174 | "min1 = torch.min(M) # minimum from ENTIRE matrix\n", 175 | "min2 = torch.min(M,axis=0) # minimum of each column (across rows)\n", 176 | "min3 = torch.min(M,axis=1) # minimum of each row (across columns)\n", 177 | "\n", 178 | "# print them out\n", 179 | "print(min1), print(' ')\n", 180 | "print(min2), print(' ')\n", 181 | "print(min2.values)\n", 182 | "print(min2.indices)" 183 | ], 184 | "execution_count": null, 185 | "outputs": [] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "metadata": { 190 | "id": "A5iVm1IWeP3_" 191 | }, 192 | "source": [ 193 | "min2.count" 194 | ], 195 | "execution_count": null, 196 | "outputs": [] 197 | } 198 | ] 199 | } -------------------------------------------------------------------------------- /math/DUDL_math_derivatives1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_derivatives1.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyO9sexGvS+Zgx1ajxmKpDH3" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Math prerequisites\n", 28 | "### LECTURE: Derivatives: intuition and polynomials\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "YvEj7AGdffUn" 37 | }, 38 | "source": [ 39 | "import numpy as np\n", 40 | "import matplotlib.pyplot as plt\n", 41 | "\n", 42 | "# sympy = symbolic math in Python\n", 43 | "import sympy as sym\n", 44 | "import sympy.plotting.plot as symplot" 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "2WUCrL06fngf" 53 | }, 54 | "source": [ 55 | "# create symbolic variables in sympy\n", 56 | "x = sym.symbols('x')\n", 57 | "\n", 58 | "# create a function\n", 59 | "fx = 2*x**2\n", 60 | "\n", 61 | "# compute its derivative\n", 62 | "df = sym.diff(fx,x)\n", 63 | "\n", 64 | "# print them\n", 65 | "print(fx)\n", 66 | "print(df)" 67 | ], 68 | "execution_count": null, 69 | "outputs": [] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "metadata": { 74 | "id": "MSxztQoagCVt" 75 | }, 76 | "source": [ 77 | "# plot them\n", 78 | "symplot(fx,(x,-4,4),title='The function')\n", 79 | "plt.show()\n", 80 | "\n", 81 | "symplot(df,(x,-4,4),title='Its derivative')\n", 82 | "plt.show()\n" 83 | ], 84 | "execution_count": null, 85 | "outputs": [] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "metadata": { 90 | "id": "BhMh2UGDhScO" 91 | }, 92 | "source": [ 93 | "# repeat with relu and sigmoid\n", 94 | "\n", 95 | "# create symbolic functions\n", 96 | "relu = sym.Max(0,x)\n", 97 | "sigmoid = 1 / (1+sym.exp(-x))\n", 98 | "\n", 99 | "# graph the functions\n", 100 | "p = symplot(relu,(x,-4,4),label='ReLU',show=False,line_color='blue')\n", 101 | "p.extend( symplot(sigmoid,(x,-4,4),label='Sigmoid',show=False,line_color='red') )\n", 102 | "p.legend = True\n", 103 | "p.title = 'The functions'\n", 104 | "p.show()\n", 105 | "\n", 106 | "\n", 107 | "# graph their derivatives\n", 108 | "p = symplot(sym.diff(relu),(x,-4,4),label='df(ReLU)',show=False,line_color='blue')\n", 109 | "p.extend( symplot(sym.diff(sigmoid),(x,-4,4),label='df(Sigmoid)',show=False,line_color='red') )\n", 110 | "p.legend = True\n", 111 | "p.title = 'The derivatives'\n", 112 | "p.show()\n" 113 | ], 114 | "execution_count": null, 115 | "outputs": [] 116 | } 117 | ] 118 | } -------------------------------------------------------------------------------- /math/DUDL_math_derivatives2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_derivatives2.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyOR72BPWb6K8zjZtQ9R8TZj" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Math prerequisites\n", 28 | "### LECTURE: Derivatives: product and chain rules\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "YvEj7AGdffUn" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import numpy as np\n", 41 | "import sympy as sym\n", 42 | "\n", 43 | "# make the equations look nicer\n", 44 | "from IPython.display import display" 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "2WUCrL06fngf" 53 | }, 54 | "source": [ 55 | "# create symbolic variables in sympy\n", 56 | "x = sym.symbols('x')\n", 57 | "\n", 58 | "# create two functions\n", 59 | "fx = 2*x**2\n", 60 | "gx = 4*x**3 - 3*x**4\n", 61 | "\n", 62 | "# compute their individual derivatives\n", 63 | "df = sym.diff(fx)\n", 64 | "dg = sym.diff(gx)\n", 65 | "\n", 66 | "# apply the product rule \"manually\"\n", 67 | "manual = df*gx + fx*dg\n", 68 | "thewrongway = df*dg\n", 69 | "\n", 70 | "# via sympy\n", 71 | "viasympy = sym.diff( fx*gx )\n", 72 | "\n", 73 | "\n", 74 | "# print everything\n", 75 | "print('The functions:')\n", 76 | "display(fx)\n", 77 | "display(gx)\n", 78 | "print(' ')\n", 79 | "\n", 80 | "print('Their derivatives:')\n", 81 | "display(df)\n", 82 | "display(dg)\n", 83 | "print(' ')\n", 84 | "\n", 85 | "print('Manual product rule:')\n", 86 | "display(manual)\n", 87 | "print(' ')\n", 88 | "\n", 89 | "print('Via sympy:')\n", 90 | "display(viasympy)\n", 91 | "print(' ')\n", 92 | "\n", 93 | "\n", 94 | "print('The wrong way:')\n", 95 | "display(thewrongway)" 96 | ], 97 | "execution_count": null, 98 | "outputs": [] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "metadata": { 103 | "id": "MSxztQoagCVt" 104 | }, 105 | "source": [ 106 | "# repeat with chain rule\n", 107 | "gx = x**2 + 4*x**3\n", 108 | "fx = ( gx )**5\n", 109 | "\n", 110 | "print('The function:')\n", 111 | "display(fx)\n", 112 | "print(' ')\n", 113 | "\n", 114 | "print('Its derivative:')\n", 115 | "display(sym.diff(fx))" 116 | ], 117 | "execution_count": null, 118 | "outputs": [] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "metadata": { 123 | "id": "BhMh2UGDhScO" 124 | }, 125 | "source": [ 126 | "" 127 | ], 128 | "execution_count": null, 129 | "outputs": [] 130 | } 131 | ] 132 | } -------------------------------------------------------------------------------- /math/DUDL_math_dotproduct.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_dotproduct.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1Cq_BWix2q_95q84lMxiEDIWrIkDR3XPL", 10 | "timestamp": 1616435155528 11 | } 12 | ], 13 | "collapsed_sections": [], 14 | "authorship_tag": "ABX9TyP/ls4M2Pg6e1QkL9G6yDZM" 15 | }, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3" 19 | }, 20 | "language_info": { 21 | "name": "python" 22 | } 23 | }, 24 | "cells": [ 25 | { 26 | "cell_type": "markdown", 27 | "metadata": { 28 | "id": "bhWV8oes-wKR" 29 | }, 30 | "source": [ 31 | "# COURSE: A deep understanding of deep learning\n", 32 | "## SECTION: Math prerequisites\n", 33 | "### LECTURE: OMG it's the dot product!\n", 34 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 35 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "metadata": { 41 | "id": "Yx_Rigz1AffS" 42 | }, 43 | "source": [ 44 | "# import libraries\n", 45 | "import numpy as np\n", 46 | "import torch" 47 | ], 48 | "execution_count": null, 49 | "outputs": [] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": { 54 | "id": "uakMyq7UBzCP" 55 | }, 56 | "source": [ 57 | "# Using numpy" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "metadata": { 63 | "id": "nG3ur0N-AneX" 64 | }, 65 | "source": [ 66 | "# create a vector\n", 67 | "nv1 = np.array([1,2,3,4])\n", 68 | "nv2 = np.array([0,1,0,-1])\n", 69 | "\n", 70 | "# dot product via function\n", 71 | "print(np.dot(nv1,nv2))\n", 72 | "\n", 73 | "# dot product via computation\n", 74 | "print(np.sum( nv1*nv2 ))" 75 | ], 76 | "execution_count": null, 77 | "outputs": [] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": { 82 | "id": "oRLFXHk9BP1r" 83 | }, 84 | "source": [ 85 | "# Using pytorch" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "metadata": { 91 | "id": "Kjq-oGoUBTXs" 92 | }, 93 | "source": [ 94 | "# create a vector\n", 95 | "tv1 = torch.tensor([1,2,3,4])\n", 96 | "tv2 = torch.tensor([0,1,0,-1])\n", 97 | "\n", 98 | "# dot product via function\n", 99 | "print(torch.dot(tv1,tv2))\n", 100 | "\n", 101 | "# dot product via computation\n", 102 | "print(torch.sum( tv1*tv2 ))" 103 | ], 104 | "execution_count": null, 105 | "outputs": [] 106 | } 107 | ] 108 | } -------------------------------------------------------------------------------- /math/DUDL_math_entropy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1oDaogKfz9gQYSAyQT-uF9xRI3EdrtokO", 9 | "timestamp": 1618209287295 10 | } 11 | ] 12 | }, 13 | "kernelspec": { 14 | "name": "python3", 15 | "display_name": "Python 3" 16 | }, 17 | "language_info": { 18 | "name": "python" 19 | } 20 | }, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "bhWV8oes-wKR" 26 | }, 27 | "source": [ 28 | "# COURSE: A deep understanding of deep learning\n", 29 | "## SECTION: Math prerequisites\n", 30 | "### LECTURE: Entropy and cross-entropy\n", 31 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 32 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "metadata": { 38 | "id": "2TD8IyfBGXiY" 39 | }, 40 | "source": [ 41 | "# import libraries\n", 42 | "import numpy as np\n", 43 | "import matplotlib.pyplot as plt" 44 | ], 45 | "execution_count": null, 46 | "outputs": [] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": { 51 | "id": "yo63BJPf0cau" 52 | }, 53 | "source": [ 54 | "# Reminder of entropy:\n", 55 | "\n", 56 | "$$H(p) = -\\sum_x p(x)\\log(p(x))$$" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "metadata": { 62 | "id": "vmjUxlEqGbDu" 63 | }, 64 | "source": [ 65 | "# probability of an event happening\n", 66 | "p = .25\n", 67 | "\n", 68 | "# NOT the correct formula!\n", 69 | "H = -( p*np.log(p) )\n", 70 | "print('Wrong entropy: ' + str(H))" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "metadata": { 78 | "id": "WHx3wf4Y0TJH" 79 | }, 80 | "source": [], 81 | "execution_count": null, 82 | "outputs": [] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "metadata": { 87 | "id": "HdZadwd12RGv" 88 | }, 89 | "source": [ 90 | "# the correct way to compute entropy\n", 91 | "x = [.25,.75]\n", 92 | "\n", 93 | "H = 0\n", 94 | "for p in x:\n", 95 | " H -= p*np.log(p)\n", 96 | "\n", 97 | "print('Correct entropy: ' + str(H))" 98 | ], 99 | "execution_count": null, 100 | "outputs": [] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "metadata": { 105 | "id": "R_bGT7kd2ipR" 106 | }, 107 | "source": [ 108 | "# also correct, written out for N=2 events\n", 109 | "H = -( p*np.log(p) + (1-p)*np.log(1-p) )\n", 110 | "print('Correct entropy: ' + str(H))" 111 | ], 112 | "execution_count": null, 113 | "outputs": [] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": { 118 | "id": "qFN5779d1ebD" 119 | }, 120 | "source": [ 121 | "# Cross-entropy" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "metadata": { 127 | "id": "sOug_tPzHY1y" 128 | }, 129 | "source": [ 130 | "# note: all probs must sum to 1!\n", 131 | "p = [ 1,0 ] # sum=1\n", 132 | "q = [ .25,.75 ] # sum=1\n", 133 | "\n", 134 | "H = 0\n", 135 | "for i in range(len(p)):\n", 136 | " H -= p[i]*np.log(q[i])\n", 137 | "\n", 138 | "print('Cross entropy: ' + str(H))" 139 | ], 140 | "execution_count": null, 141 | "outputs": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "metadata": { 146 | "id": "8H1p7JUr3Pn4" 147 | }, 148 | "source": [ 149 | "# also correct, written out for N=2 events\n", 150 | "H = -( p[0]*np.log(q[0]) + p[1]*np.log(q[1]) )\n", 151 | "print('Correct entropy: ' + str(H))\n", 152 | "\n", 153 | "# simplification\n", 154 | "H = -np.log(q[0])\n", 155 | "print('Manually simplified: ' + str(H))" 156 | ], 157 | "execution_count": null, 158 | "outputs": [] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "metadata": { 163 | "id": "fAHoba2V4QgO" 164 | }, 165 | "source": [ 166 | "# now using pytorch\n", 167 | "import torch\n", 168 | "import torch.nn.functional as F\n", 169 | "\n", 170 | "# note: inputs must be Tensors\n", 171 | "q_tensor = torch.Tensor(q)\n", 172 | "p_tensor = torch.Tensor(p)\n", 173 | "\n", 174 | "F.binary_cross_entropy(p,q)" 175 | ], 176 | "execution_count": null, 177 | "outputs": [] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "source": [], 182 | "metadata": { 183 | "colab": { 184 | "base_uri": "https://localhost:8080/", 185 | "height": 36 186 | }, 187 | "id": "QBQabnEEJisd", 188 | "executionInfo": { 189 | "status": "ok", 190 | "timestamp": 1692941200690, 191 | "user_tz": -180, 192 | "elapsed": 4, 193 | "user": { 194 | "displayName": "Mike X Cohen", 195 | "userId": "13901636194183843661" 196 | } 197 | }, 198 | "outputId": "c0f67256-1c5e-4929-86b8-ec36f2db4f1d" 199 | }, 200 | "execution_count": 14, 201 | "outputs": [ 202 | { 203 | "output_type": "execute_result", 204 | "data": { 205 | "text/plain": [ 206 | "'2.0.1+cu118'" 207 | ], 208 | "application/vnd.google.colaboratory.intrinsic+json": { 209 | "type": "string" 210 | } 211 | }, 212 | "metadata": {}, 213 | "execution_count": 14 214 | } 215 | ] 216 | } 217 | ] 218 | } -------------------------------------------------------------------------------- /math/DUDL_math_log.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_log.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyOTCYJ+mDz/ypupGyViltVn" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Math prerequisites\n", 28 | "### LECTURE: Logarithms\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "wYjuZD1kPb3g" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import numpy as np\n", 41 | "import matplotlib.pyplot as plt" 42 | ], 43 | "execution_count": null, 44 | "outputs": [] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "metadata": { 49 | "id": "tJ_A2ykHS5RY" 50 | }, 51 | "source": [ 52 | "# define a set of points to evaluate\n", 53 | "x = np.linspace(.0001,1,200)\n", 54 | "\n", 55 | "# compute their log\n", 56 | "logx = np.log(x)\n", 57 | "\n", 58 | "\n", 59 | "\n", 60 | "# plot!\n", 61 | "fig = plt.figure(figsize=(10,4))\n", 62 | "\n", 63 | "# increase font size. FYI\n", 64 | "plt.rcParams.update({'font.size':15})\n", 65 | "\n", 66 | "plt.plot(x,logx,'ks-',markerfacecolor='w')\n", 67 | "plt.xlabel('x')\n", 68 | "plt.ylabel('log(x)')\n", 69 | "plt.show()" 70 | ], 71 | "execution_count": null, 72 | "outputs": [] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "id": "QZaS568IThAr" 78 | }, 79 | "source": [ 80 | "# demonstration that log and exp are inverses\n", 81 | "\n", 82 | "# redefine with fewer points\n", 83 | "x = np.linspace(.0001,1,20)\n", 84 | "\n", 85 | "# log and exp\n", 86 | "logx = np.log(x)\n", 87 | "expx = np.exp(x)\n", 88 | "\n", 89 | "# the plot\n", 90 | "plt.plot(x,x,color=[.8,.8,.8])\n", 91 | "plt.plot(x,np.exp(logx),'o',markersize=8)\n", 92 | "plt.plot(x,np.log(expx),'x',markersize=8)\n", 93 | "plt.xlabel('x')\n", 94 | "plt.ylabel('f(g(x))')\n", 95 | "plt.legend(['unity','exp(log(x))','log(exp(x))'])\n", 96 | "plt.show()\n" 97 | ], 98 | "execution_count": null, 99 | "outputs": [] 100 | } 101 | ] 102 | } -------------------------------------------------------------------------------- /math/DUDL_math_matrixMult.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_matrixMult.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyN3xNttY0efRLiZkxgkBGe2" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Math prerequisites\n", 28 | "### LECTURE: Matrix multiplication\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "vRQNyNtlDoOb" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import numpy as np\n", 41 | "import torch" 42 | ], 43 | "execution_count": null, 44 | "outputs": [] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": { 49 | "id": "K0BSQG0fDpt_" 50 | }, 51 | "source": [ 52 | "# Using numpy" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "id": "T4_veEYqDpx9" 59 | }, 60 | "source": [ 61 | "# create some random matrices\n", 62 | "A = np.random.randn(3,4)\n", 63 | "B = np.random.randn(4,5)\n", 64 | "C = np.random.randn(3,7)\n", 65 | "\n", 66 | "# try some multiplications...\n", 67 | "print(np.round( A@B ,2)), print(' ')\n", 68 | "# print(np.round( A@C ,2)), print(' ')\n", 69 | "# print(np.round( B@C ,2)), print(' ')\n", 70 | "print(np.round( C.T@A ,2))" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": { 78 | "id": "89qsNy19Dp1U" 79 | }, 80 | "source": [ 81 | "# Using pytorch" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "metadata": { 87 | "id": "4Yrdf4WOEjxz" 88 | }, 89 | "source": [ 90 | "# create some random matrices\n", 91 | "A = torch.randn(3,4)\n", 92 | "B = torch.randn(4,5)\n", 93 | "C1 = np.random.randn(4,7)\n", 94 | "C2 = torch.tensor( C1,dtype=torch.float )\n", 95 | "\n", 96 | "# try some multiplications...\n", 97 | "# print(np.round( A@B ,2)), print(' ')\n", 98 | "# print(np.round( A@B.T ,2)), print(' ')\n", 99 | "print(np.round( A@C1 ,2)), print(' ')\n", 100 | "print(np.round( A@C2 ,2))" 101 | ], 102 | "execution_count": null, 103 | "outputs": [] 104 | } 105 | ] 106 | } -------------------------------------------------------------------------------- /math/DUDL_math_meanvar.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_meanvar.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1KX8lBi9Oey9iD2Aac8X6d-j3dMlyTodJ", 10 | "timestamp": 1617002781341 11 | } 12 | ], 13 | "collapsed_sections": [], 14 | "authorship_tag": "ABX9TyOq4j7MI+XdUF3fllmbwynp" 15 | }, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3" 19 | }, 20 | "language_info": { 21 | "name": "python" 22 | } 23 | }, 24 | "cells": [ 25 | { 26 | "cell_type": "markdown", 27 | "metadata": { 28 | "id": "bhWV8oes-wKR" 29 | }, 30 | "source": [ 31 | "# COURSE: A deep understanding of deep learning\n", 32 | "## SECTION: Math prerequisites\n", 33 | "### LECTURE: Mean and variance\n", 34 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 35 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "metadata": { 41 | "id": "wYjuZD1kPb3g" 42 | }, 43 | "source": [ 44 | "# import libraries\n", 45 | "import numpy as np" 46 | ], 47 | "execution_count": null, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "metadata": { 53 | "id": "tJ_A2ykHS5RY" 54 | }, 55 | "source": [ 56 | "# create a list of numbers to compute the mean and variance of\n", 57 | "x = [1,2,4,6,5,4,0]\n", 58 | "n = len(x)\n", 59 | "\n", 60 | "# compute the mean\n", 61 | "mean1 = np.mean(x)\n", 62 | "mean2 = np.sum(x) / n\n", 63 | "\n", 64 | "# print them\n", 65 | "print(mean1)\n", 66 | "print(mean2)" 67 | ], 68 | "execution_count": null, 69 | "outputs": [] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "metadata": { 74 | "id": "QZaS568IThAr" 75 | }, 76 | "source": [ 77 | "# variance\n", 78 | "\n", 79 | "var1 = np.var(x)\n", 80 | "var2 = (1/(n-1)) * np.sum( (x-mean1)**2 )\n", 81 | "\n", 82 | "print(var1)\n", 83 | "print(var2)" 84 | ], 85 | "execution_count": null, 86 | "outputs": [] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "metadata": { 91 | "id": "oAUTsBE646fN" 92 | }, 93 | "source": [ 94 | "# uh oh!\n", 95 | "\n", 96 | "var3 = np.var(x,ddof=1)\n", 97 | "\n", 98 | "print(var3)\n", 99 | "print(var2)" 100 | ], 101 | "execution_count": null, 102 | "outputs": [] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "metadata": { 107 | "id": "mdoWX2Fx5JIy" 108 | }, 109 | "source": [ 110 | "# does it matter for large N? \n", 111 | "\n", 112 | "N = 10000\n", 113 | "x = np.random.randint(0,high=20,size=N)\n", 114 | "\n", 115 | "var0 = np.var(x,ddof=0) # default\n", 116 | "var1 = np.var(x,ddof=1) # unbiased\n", 117 | "\n", 118 | "print(var0)\n", 119 | "print(var1)" 120 | ], 121 | "execution_count": null, 122 | "outputs": [] 123 | } 124 | ] 125 | } -------------------------------------------------------------------------------- /math/DUDL_math_randomseed.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "196i6MReeJLozrRYpZsltVy8BC6X1b9ik", 9 | "timestamp": 1619035090045 10 | }, 11 | { 12 | "file_id": "1KX8lBi9Oey9iD2Aac8X6d-j3dMlyTodJ", 13 | "timestamp": 1617002781341 14 | } 15 | ], 16 | "authorship_tag": "ABX9TyOJyuFc3I4KRRovgrF3zmPZ" 17 | }, 18 | "kernelspec": { 19 | "name": "python3", 20 | "display_name": "Python 3" 21 | }, 22 | "language_info": { 23 | "name": "python" 24 | } 25 | }, 26 | "cells": [ 27 | { 28 | "cell_type": "markdown", 29 | "metadata": { 30 | "id": "bhWV8oes-wKR" 31 | }, 32 | "source": [ 33 | "# COURSE: A deep understanding of deep learning\n", 34 | "## SECTION: Math prerequisites\n", 35 | "### LECTURE: Identical randomness via seeding\n", 36 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 37 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "metadata": { 43 | "id": "wYjuZD1kPb3g", 44 | "executionInfo": { 45 | "status": "ok", 46 | "timestamp": 1676841300342, 47 | "user_tz": -540, 48 | "elapsed": 4400, 49 | "user": { 50 | "displayName": "Mike X Cohen", 51 | "userId": "13901636194183843661" 52 | } 53 | } 54 | }, 55 | "source": [ 56 | "# import libraries\n", 57 | "import numpy as np\n", 58 | "import torch" 59 | ], 60 | "execution_count": 1, 61 | "outputs": [] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "metadata": { 66 | "id": "tJ_A2ykHS5RY" 67 | }, 68 | "source": [ 69 | "# generate a few random numbers\n", 70 | "np.random.randn(5)" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "metadata": { 78 | "id": "QZaS568IThAr" 79 | }, 80 | "source": [ 81 | "# repeat after fixing the seed (old-but-still-widely-used method)\n", 82 | "np.random.seed(17)\n", 83 | "print(np.random.randn(5))\n", 84 | "print(np.random.randn(5))\n", 85 | "\n", 86 | "# [ 0.27626589 -1.85462808 0.62390111 1.14531129 1.03719047]\n", 87 | "# [ 1.88663893 -0.11169829 -0.36210134 0.14867505 -0.43778315]" 88 | ], 89 | "execution_count": null, 90 | "outputs": [] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": { 95 | "id": "v5VZW0g-DxLl" 96 | }, 97 | "source": [ 98 | "# New seed mechanism in numpy" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "metadata": { 104 | "id": "7Ctn7a0hChbA", 105 | "colab": { 106 | "base_uri": "https://localhost:8080/" 107 | }, 108 | "executionInfo": { 109 | "status": "ok", 110 | "timestamp": 1676841327280, 111 | "user_tz": -540, 112 | "elapsed": 353, 113 | "user": { 114 | "displayName": "Mike X Cohen", 115 | "userId": "13901636194183843661" 116 | } 117 | }, 118 | "outputId": "8478bc3a-8ce0-456e-fa86-4235bad9c46d" 119 | }, 120 | "source": [ 121 | "randseed1 = np.random.RandomState(17)\n", 122 | "randseed2 = np.random.RandomState(20210530)\n", 123 | "\n", 124 | "print( randseed1.randn(5) ) # same sequence\n", 125 | "print( randseed2.randn(5) ) # different from above, but same each time\n", 126 | "print( randseed1.randn(5) ) # same as two up\n", 127 | "print( randseed2.randn(5) ) # same as two up\n", 128 | "print( np.random.randn(5) ) # different every time\n", 129 | "\n", 130 | "# [ 0.27626589 -1.85462808 0.62390111 1.14531129 1.03719047]\n", 131 | "# [-0.24972681 -1.01951826 2.23461339 0.72764703 1.2921122 ]\n", 132 | "# [ 1.88663893 -0.11169829 -0.36210134 0.14867505 -0.43778315]\n", 133 | "# [ 1.15494929 -0.0015467 -0.11196868 -1.08136725 0.10265891]\n", 134 | "# [ 2.171257 1.15231025 -1.81881234 -0.13804934 0.53983961]" 135 | ], 136 | "execution_count": 7, 137 | "outputs": [ 138 | { 139 | "output_type": "stream", 140 | "name": "stdout", 141 | "text": [ 142 | "[ 0.27626589 -1.85462808 0.62390111 1.14531129 1.03719047]\n", 143 | "[-0.24972681 -1.01951826 2.23461339 0.72764703 1.2921122 ]\n", 144 | "[ 1.88663893 -0.11169829 -0.36210134 0.14867505 -0.43778315]\n", 145 | "[ 1.15494929 -0.0015467 -0.11196868 -1.08136725 0.10265891]\n", 146 | "[ 0.85266105 -0.59833972 3.29199531 0.15720173 -1.3930384 ]\n" 147 | ] 148 | } 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "source": [], 154 | "metadata": { 155 | "colab": { 156 | "base_uri": "https://localhost:8080/" 157 | }, 158 | "id": "ZUniXETYh52I", 159 | "executionInfo": { 160 | "status": "ok", 161 | "timestamp": 1676841345103, 162 | "user_tz": -540, 163 | "elapsed": 363, 164 | "user": { 165 | "displayName": "Mike X Cohen", 166 | "userId": "13901636194183843661" 167 | } 168 | }, 169 | "outputId": "8f648038-068c-4f8c-da73-2794224daa3d" 170 | }, 171 | "execution_count": 9, 172 | "outputs": [ 173 | { 174 | "output_type": "stream", 175 | "name": "stdout", 176 | "text": [ 177 | "[-1.77528229 1.31487654 -0.47344805 -1.0922299 -0.25002744]\n" 178 | ] 179 | } 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": { 185 | "id": "Jy2WlM_oBjZy" 186 | }, 187 | "source": [ 188 | "# Now in pytorch" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "metadata": { 194 | "id": "oAUTsBE646fN" 195 | }, 196 | "source": [ 197 | "torch.randn(5)" 198 | ], 199 | "execution_count": null, 200 | "outputs": [] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "metadata": { 205 | "id": "mdoWX2Fx5JIy" 206 | }, 207 | "source": [ 208 | "torch.manual_seed(17)\n", 209 | "print( torch.randn(5) )\n", 210 | "\n", 211 | "# torch's seed doesn't spread to numpy\n", 212 | "print( np.random.randn(5) )" 213 | ], 214 | "execution_count": null, 215 | "outputs": [] 216 | } 217 | ] 218 | } -------------------------------------------------------------------------------- /math/DUDL_math_sampling.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_sampling.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "196i6MReeJLozrRYpZsltVy8BC6X1b9ik", 10 | "timestamp": 1617003240706 11 | }, 12 | { 13 | "file_id": "1KX8lBi9Oey9iD2Aac8X6d-j3dMlyTodJ", 14 | "timestamp": 1617002781341 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyOL7D04HBwew8NW+70XIQbD" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | }, 24 | "language_info": { 25 | "name": "python" 26 | } 27 | }, 28 | "cells": [ 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "id": "bhWV8oes-wKR" 33 | }, 34 | "source": [ 35 | "# COURSE: A deep understanding of deep learning\n", 36 | "## SECTION: Math prerequisites\n", 37 | "### LECTURE: Random sampling and sampling variability\n", 38 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 39 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "metadata": { 45 | "id": "wYjuZD1kPb3g" 46 | }, 47 | "source": [ 48 | "# import libraries\n", 49 | "import numpy as np\n", 50 | "import matplotlib.pyplot as plt" 51 | ], 52 | "execution_count": null, 53 | "outputs": [] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "id": "tJ_A2ykHS5RY" 59 | }, 60 | "source": [ 61 | "# create a list of numbers to compute the mean and variance of\n", 62 | "x = [1,2,4,6,5,4,0,-4,5,-2,6,10,-9,1,3,-6]\n", 63 | "n = len(x)\n", 64 | "\n", 65 | "# compute the population mean\n", 66 | "popmean = np.mean(x)\n", 67 | "\n", 68 | "# compute a sample mean\n", 69 | "sample = np.random.choice(x,size=5,replace=True)\n", 70 | "sampmean = np.mean(sample)\n", 71 | "\n", 72 | "# print them\n", 73 | "print(popmean)\n", 74 | "print(sampmean)" 75 | ], 76 | "execution_count": null, 77 | "outputs": [] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "metadata": { 82 | "id": "QZaS568IThAr" 83 | }, 84 | "source": [ 85 | "# compute lots of sample means\n", 86 | "\n", 87 | "# number of experiments to run\n", 88 | "nExpers = 10000\n", 89 | "\n", 90 | "# run the experiment!\n", 91 | "sampleMeans = np.zeros(nExpers)\n", 92 | "for i in range(nExpers):\n", 93 | "\n", 94 | " # step 1: draw a sample\n", 95 | " sample = np.random.choice(x,size=15,replace=True)\n", 96 | "\n", 97 | " # step 2: compute its mean\n", 98 | " sampleMeans[i] = np.mean(sample)\n", 99 | "\n", 100 | "\n", 101 | "\n", 102 | "# show the results as a histogram\n", 103 | "plt.hist(sampleMeans,bins=40,density=True)\n", 104 | "plt.plot([popmean,popmean],[0,.3],'m--')\n", 105 | "plt.ylabel('Count')\n", 106 | "plt.xlabel('Sample mean')\n", 107 | "plt.show()" 108 | ], 109 | "execution_count": null, 110 | "outputs": [] 111 | } 112 | ] 113 | } -------------------------------------------------------------------------------- /math/DUDL_math_transpose.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_transpose.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyMPFaV8b0XTO0Wj7WUXRZkF" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Math prerequisites\n", 28 | "### LECTURE: Vector and matrix transpose\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "Yx_Rigz1AffS" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import numpy as np\n", 41 | "import torch" 42 | ], 43 | "execution_count": null, 44 | "outputs": [] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": { 49 | "id": "uakMyq7UBzCP" 50 | }, 51 | "source": [ 52 | "# Using numpy" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "id": "nG3ur0N-AneX" 59 | }, 60 | "source": [ 61 | "# create a vector\n", 62 | "nv = np.array([ [1,2,3,4] ])\n", 63 | "print(nv), print(' ')\n", 64 | "\n", 65 | "# transpose it\n", 66 | "print(nv.T), print(' ')\n", 67 | "\n", 68 | "# transpose the transpose!\n", 69 | "nvT = nv.T\n", 70 | "print(nvT.T)" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "metadata": { 78 | "id": "Vmr-MkomA6K9" 79 | }, 80 | "source": [ 81 | "# repeat for a matrix\n", 82 | "nM = np.array([ [1,2,3,4],\n", 83 | " [5,6,7,8] \n", 84 | " ])\n", 85 | "print(nM), print(' ')\n", 86 | "\n", 87 | "# transpose it\n", 88 | "print(nM.T), print(' ')\n", 89 | "\n", 90 | "# transpose the transpose!\n", 91 | "nMT = nM.T\n", 92 | "print(nMT.T)" 93 | ], 94 | "execution_count": null, 95 | "outputs": [] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": { 100 | "id": "oRLFXHk9BP1r" 101 | }, 102 | "source": [ 103 | "# Using pytorch" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "Kjq-oGoUBTXs" 110 | }, 111 | "source": [ 112 | "# create a vector\n", 113 | "tv = torch.tensor([ [1,2,3,4] ])\n", 114 | "print(tv), print(' ')\n", 115 | "\n", 116 | "# transpose it\n", 117 | "print(tv.T), print(' ')\n", 118 | "\n", 119 | "# transpose the transpose!\n", 120 | "tvT = tv.T\n", 121 | "print(tvT.T)" 122 | ], 123 | "execution_count": null, 124 | "outputs": [] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "metadata": { 129 | "id": "qwzG0O_aBfnv" 130 | }, 131 | "source": [ 132 | "# repeat for a matrix\n", 133 | "tM = torch.tensor([ [1,2,3,4],[5,6,7,8] ])\n", 134 | "print(tM), print(' ')\n", 135 | "\n", 136 | "# transpose it\n", 137 | "print(tM.T), print(' ')\n", 138 | "\n", 139 | "# transpose the transpose!\n", 140 | "tMT = tM.T\n", 141 | "print(tMT.T)" 142 | ], 143 | "execution_count": null, 144 | "outputs": [] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "metadata": { 149 | "id": "cZJVgv8HBfwA" 150 | }, 151 | "source": [ 152 | "" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "metadata": { 160 | "id": "4J6zsIHoBfyp" 161 | }, 162 | "source": [ 163 | "# examine data types\n", 164 | "print(f'Variable nv is of type {type(nv)}')\n", 165 | "print(f'Variable nM is of type {type(nM)}')\n", 166 | "print(f'Variable tv is of type {type(tv)}')\n", 167 | "print(f'Variable tM is of type {type(tM)}')\n" 168 | ], 169 | "execution_count": null, 170 | "outputs": [] 171 | } 172 | ] 173 | } -------------------------------------------------------------------------------- /math/DUDL_math_ttest.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_math_ttest.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "196i6MReeJLozrRYpZsltVy8BC6X1b9ik", 10 | "timestamp": 1619030836877 11 | }, 12 | { 13 | "file_id": "1KX8lBi9Oey9iD2Aac8X6d-j3dMlyTodJ", 14 | "timestamp": 1617002781341 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyNT7nZ3dJB7ZvHntPAobaiK" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | }, 24 | "language_info": { 25 | "name": "python" 26 | } 27 | }, 28 | "cells": [ 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "id": "bhWV8oes-wKR" 33 | }, 34 | "source": [ 35 | "# COURSE: A deep understanding of deep learning\n", 36 | "## SECTION: Math prerequisites\n", 37 | "### LECTURE: The t-test\n", 38 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 39 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "metadata": { 45 | "id": "wYjuZD1kPb3g" 46 | }, 47 | "source": [ 48 | "# import libraries\n", 49 | "import numpy as np\n", 50 | "import matplotlib.pyplot as plt\n", 51 | "\n", 52 | "# NEW!\n", 53 | "import scipy.stats as stats" 54 | ], 55 | "execution_count": null, 56 | "outputs": [] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "metadata": { 61 | "id": "tJ_A2ykHS5RY" 62 | }, 63 | "source": [ 64 | "# parameters\n", 65 | "n1 = 30 # samples in dataset 1\n", 66 | "n2 = 40 # ...and 2\n", 67 | "mu1 = 1 # population mean in dataset 1\n", 68 | "mu2 = 2 # population mean in dataset 2\n", 69 | "\n", 70 | "# generate the data\n", 71 | "data1 = mu1 + np.random.randn(n1)\n", 72 | "data2 = mu2 + np.random.randn(n2)\n", 73 | "\n", 74 | "\n", 75 | "\n", 76 | "# plot them\n", 77 | "plt.plot(np.zeros(n1),data1,'ro',markerfacecolor='w',markersize=14)\n", 78 | "plt.plot(np.ones(n2), data2,'bs',markerfacecolor='w',markersize=14)\n", 79 | "plt.xlim([-1,2])\n", 80 | "plt.xticks([0,1],labels=['Group 1','Group 2'])\n", 81 | "plt.show()" 82 | ], 83 | "execution_count": null, 84 | "outputs": [] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "metadata": { 89 | "id": "oAUTsBE646fN" 90 | }, 91 | "source": [ 92 | "# t-test via stats package\n", 93 | "\n", 94 | "# _ind = independent samples\n", 95 | "t,p = stats.ttest_ind(data1,data2)\n", 96 | "print(t)\n", 97 | "print(p)" 98 | ], 99 | "execution_count": null, 100 | "outputs": [] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "metadata": { 105 | "id": "mdoWX2Fx5JIy" 106 | }, 107 | "source": [ 108 | "# common way to show t-test results in a plot\n", 109 | "fig = plt.figure(figsize=(10,4))\n", 110 | "plt.rcParams.update({'font.size':12}) # change the font size\n", 111 | "\n", 112 | "plt.plot(0+np.random.randn(n1)/15, data1, 'ro',markerfacecolor='w',markersize=14)\n", 113 | "plt.plot(1+np.random.randn(n2)/15, data2, 'bs',markerfacecolor='w',markersize=14)\n", 114 | "plt.xlim([-1,2])\n", 115 | "plt.xticks([0,1],labels=['Group 1','Group 2'])\n", 116 | "\n", 117 | "# set the title to include the t-value and p-value\n", 118 | "plt.title(f't = {t:.2f}, p={p:.3f}')\n", 119 | "\n", 120 | "plt.show()" 121 | ], 122 | "execution_count": null, 123 | "outputs": [] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "metadata": { 128 | "id": "O4iCYmR9yuA7" 129 | }, 130 | "source": [ 131 | "" 132 | ], 133 | "execution_count": null, 134 | "outputs": [] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "metadata": { 139 | "id": "_H15mEDryuOx" 140 | }, 141 | "source": [ 142 | "" 143 | ], 144 | "execution_count": null, 145 | "outputs": [] 146 | } 147 | ] 148 | } -------------------------------------------------------------------------------- /metaparams/DUDL_metaparams_loss.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_metaparams_loss.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyM8nF/yQRb+pqekb1hGILaP" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Metaparameters (activation, batch, optimizers)\n", 28 | "### LECTURE: Loss functions in Pytorch\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "oMl3PkezSPHA" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import torch\n", 41 | "import torch.nn as nn\n", 42 | "\n", 43 | "import numpy as np\n", 44 | "import matplotlib.pyplot as plt" 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "id": "I_fNhEIyU8-5" 53 | }, 54 | "source": [ 55 | "# Mean-squared error" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "metadata": { 61 | "id": "kww9GdqRS1_p" 62 | }, 63 | "source": [ 64 | "# loss function\n", 65 | "lossfunMSE = nn.MSELoss()\n", 66 | "\n", 67 | "# create predictions and real answer\n", 68 | "yHat = torch.linspace(-2,2,101)\n", 69 | "y = torch.tensor(.5)\n", 70 | "\n", 71 | "# compute MSE loss function\n", 72 | "L = np.zeros(101)\n", 73 | "for i,yy in enumerate(yHat):\n", 74 | " L[i] = lossfunMSE(yy,y)\n", 75 | "\n", 76 | "plt.plot(yHat,L,label='Loss')\n", 77 | "plt.plot([y,y],[0,np.max(L)],'r--',label='True value')\n", 78 | "plt.xlabel('Predicted value')\n", 79 | "plt.legend()\n", 80 | "plt.show()" 81 | ], 82 | "execution_count": null, 83 | "outputs": [] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": { 88 | "id": "JW_RBi6wU65j" 89 | }, 90 | "source": [ 91 | "# Binary cross-entropy" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "metadata": { 97 | "id": "HiN5ni12VEq0" 98 | }, 99 | "source": [ 100 | "# loss function\n", 101 | "lossfunBCE = nn.BCELoss()\n", 102 | "\n", 103 | "# create predictions and real answer\n", 104 | "yHat = torch.linspace(.001,.999,101)\n", 105 | "y1 = torch.tensor(0.)\n", 106 | "y2 = torch.tensor(1.)\n", 107 | "\n", 108 | "# compute MSE loss function\n", 109 | "L = np.zeros((101,2))\n", 110 | "for i,yy in enumerate(yHat):\n", 111 | " L[i,0] = lossfunBCE(yy,y1) # 0 is the correct answer\n", 112 | " L[i,1] = lossfunBCE(yy,y2) # 1 is the correct answer\n", 113 | "\n", 114 | "plt.plot(yHat,L)\n", 115 | "plt.xlabel('Predicted value')\n", 116 | "plt.ylabel('Loss')\n", 117 | "plt.legend(['correct=0','correct=1'])\n", 118 | "# plt.yscale('log')\n", 119 | "plt.show()" 120 | ], 121 | "execution_count": null, 122 | "outputs": [] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "metadata": { 127 | "id": "J_SJFL9CBVGA" 128 | }, 129 | "source": [ 130 | "# The example above shows data already in probabilities. Raw outputs will need to be converted to probabilities:\n", 131 | "\n", 132 | "# \"raw\" output of a model\n", 133 | "yHat = torch.tensor(2.)\n", 134 | "print(lossfunBCE(yHat,y2))\n", 135 | "\n", 136 | "# convert to prob via sigmoid\n", 137 | "sig = nn.Sigmoid()\n", 138 | "print(lossfunBCE( sig(yHat) ,y2))\n" 139 | ], 140 | "execution_count": null, 141 | "outputs": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "metadata": { 146 | "id": "wQmVOg-iCLGB" 147 | }, 148 | "source": [ 149 | "# However, PyTorch recommends using a single function that incorporates sigmoid+BCE due to increased numerical stability.\n", 150 | "# https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html?highlight=nn%20bcewithlogitsloss#torch.nn.BCEWithLogitsLoss\n", 151 | "\n", 152 | "\n", 153 | "# Thus, the recommended way to do it:\n", 154 | "lossfunBCE = nn.BCEWithLogitsLoss()\n", 155 | "yHat = torch.tensor(2.)\n", 156 | "print(lossfunBCE(yHat,y2))\n", 157 | "\n", 158 | "# In toy examples, numerical accuracy usually isn't a problem." 159 | ], 160 | "execution_count": null, 161 | "outputs": [] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": { 166 | "id": "TU_1uyOHWG7w" 167 | }, 168 | "source": [ 169 | "# Categorical cross-entropy" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "A868kRI3VEts" 176 | }, 177 | "source": [ 178 | "# loss function\n", 179 | "lossfunCCE = nn.CrossEntropyLoss()\n", 180 | "\n", 181 | "# vector of output layer (pre-softmax)\n", 182 | "yHat = torch.tensor([[1.,4,3]])\n", 183 | "\n", 184 | "for i in range(3):\n", 185 | " correctAnswer = torch.tensor([i])\n", 186 | " thisloss = lossfunCCE(yHat,correctAnswer).item()\n", 187 | " print( 'Loss when correct answer is %g: %g' %(i,thisloss) )" 188 | ], 189 | "execution_count": null, 190 | "outputs": [] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "metadata": { 195 | "id": "f5sW84OScqNu" 196 | }, 197 | "source": [ 198 | "# Repeat using pre-softmaxified output\n", 199 | "sm = nn.Softmax(dim=1)\n", 200 | "yHat_sm = sm(yHat)\n", 201 | "\n", 202 | "for i in range(3):\n", 203 | " correctAnswer = torch.tensor([i])\n", 204 | " thisloss = lossfunCCE(yHat_sm,correctAnswer).item()\n", 205 | " print( 'Loss when correct answer is %g: %g' %(i,thisloss) )" 206 | ], 207 | "execution_count": null, 208 | "outputs": [] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "metadata": { 213 | "id": "PskVgu-gK66H" 214 | }, 215 | "source": [ 216 | "# compare raw, softmax, and log-softmax outputs\n", 217 | "sm = nn.LogSoftmax(dim=1)\n", 218 | "yHat_logsm = sm(yHat)\n", 219 | "\n", 220 | "# print them\n", 221 | "print(yHat)\n", 222 | "print(yHat_sm)\n", 223 | "print(yHat_logsm)" 224 | ], 225 | "execution_count": null, 226 | "outputs": [] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": { 231 | "id": "qGtELAAfErbk" 232 | }, 233 | "source": [ 234 | "# Creating your own custom loss function" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "metadata": { 240 | "id": "DlCcAbitHhpK" 241 | }, 242 | "source": [ 243 | "class myLoss(nn.Module): # inherent info from nn.Module\n", 244 | " def __init__(self):\n", 245 | " super().__init__()\n", 246 | " \n", 247 | " def forward(self,x,y):\n", 248 | " loss = torch.abs(x-y)\n", 249 | " return loss\n", 250 | "\n", 251 | "# test it out!\n", 252 | "lfun = myLoss()\n", 253 | "lfun(torch.tensor(4),torch.tensor(5.2))" 254 | ], 255 | "execution_count": null, 256 | "outputs": [] 257 | } 258 | ] 259 | } -------------------------------------------------------------------------------- /overfitting/DUDL_overfitting_manual.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [ 7 | { 8 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 9 | "timestamp": 1616608248670 10 | } 11 | ] 12 | }, 13 | "kernelspec": { 14 | "name": "python3", 15 | "display_name": "Python 3" 16 | } 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "bhWV8oes-wKR" 23 | }, 24 | "source": [ 25 | "# COURSE: A deep understanding of deep learning\n", 26 | "## SECTION: Overfitting, cross-validation, regularization\n", 27 | "### LECTURE: Cross-validation -- manual separation\n", 28 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 29 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "metadata": { 35 | "id": "YeuAheYyhdZw" 36 | }, 37 | "source": [ 38 | "# import libraries\n", 39 | "import torch\n", 40 | "import torch.nn as nn\n", 41 | "import numpy as np" 42 | ], 43 | "execution_count": null, 44 | "outputs": [] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "metadata": { 49 | "id": "MU7rvmWuhjud" 50 | }, 51 | "source": [ 52 | "# import dataset\n", 53 | "import pandas as pd\n", 54 | "iris = pd.read_csv('https://raw.githubusercontent.com/mwaskom/seaborn-data/master/iris.csv')\n", 55 | "\n", 56 | "\n", 57 | "# convert from pandas dataframe to tensor\n", 58 | "data = torch.tensor( iris[iris.columns[0:4]].values ).float()\n", 59 | "\n", 60 | "# transform species to number\n", 61 | "labels = torch.zeros(len(data), dtype=torch.long)\n", 62 | "# labels[iris.species=='setosa'] = 0 # don't need!\n", 63 | "labels[iris.species=='versicolor'] = 1\n", 64 | "labels[iris.species=='virginica'] = 2" 65 | ], 66 | "execution_count": null, 67 | "outputs": [] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": { 72 | "id": "JiAFAHB20DQc" 73 | }, 74 | "source": [ 75 | "# Separate data into train and test" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "mwhgV43SXbCN" 82 | }, 83 | "source": [ 84 | "# (no devset here)\n", 85 | "\n", 86 | "# how many training examples\n", 87 | "propTraining = .8 # in proportion, not percent\n", 88 | "nTraining = int(len(labels)*propTraining)\n", 89 | "\n", 90 | "# initialize a boolean vector to select data and labels\n", 91 | "traintestBool = np.zeros(len(labels),dtype=bool)\n", 92 | "\n", 93 | "# is this the correct way to select samples?\n", 94 | "# traintestBool[range(nTraining)] = True\n", 95 | "\n", 96 | "# this is better, but why?\n", 97 | "items2use4train = np.random.choice(range(len(labels)),nTraining,replace=False)\n", 98 | "traintestBool[items2use4train] = True\n", 99 | "\n", 100 | "traintestBool" 101 | ], 102 | "execution_count": null, 103 | "outputs": [] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "metadata": { 108 | "id": "LPcj_f92bYs0" 109 | }, 110 | "source": [ 111 | "# test whether it's balanced\n", 112 | "print('Average of full data:')\n", 113 | "print( torch.mean(labels.float()) ) # =1 by definition\n", 114 | "print(' ')\n", 115 | "\n", 116 | "print('Average of training data:')\n", 117 | "print( torch.mean(labels[traintestBool].float()) ) # should be 1...\n", 118 | "print(' ')\n", 119 | "\n", 120 | "print('Average of test data:')\n", 121 | "print( torch.mean(labels[~traintestBool].float()) ) # should also be 1..." 122 | ], 123 | "execution_count": null, 124 | "outputs": [] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "metadata": { 129 | "id": "v0JMIGb1iV_9" 130 | }, 131 | "source": [ 132 | "# create the ANN model\n", 133 | "\n", 134 | "# model architecture\n", 135 | "ANNiris = nn.Sequential(\n", 136 | " nn.Linear(4,64), # input layer\n", 137 | " nn.ReLU(), # activation unit\n", 138 | " nn.Linear(64,64), # hidden layer\n", 139 | " nn.ReLU(), # activation unit\n", 140 | " nn.Linear(64,3), # output units\n", 141 | " )\n", 142 | "\n", 143 | "# loss function\n", 144 | "lossfun = nn.CrossEntropyLoss()\n", 145 | "\n", 146 | "# optimizer\n", 147 | "optimizer = torch.optim.SGD(ANNiris.parameters(),lr=.01)" 148 | ], 149 | "execution_count": null, 150 | "outputs": [] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "metadata": { 155 | "id": "iyxr6_P9b-x5" 156 | }, 157 | "source": [ 158 | "# entire dataset\n", 159 | "print( data.shape )\n", 160 | "\n", 161 | "# training set\n", 162 | "print( data[traintestBool,:].shape )\n", 163 | "\n", 164 | "# test set\n", 165 | "print( data[~traintestBool,:].shape )" 166 | ], 167 | "execution_count": null, 168 | "outputs": [] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": { 173 | "id": "bbx3Zkc_0UT8" 174 | }, 175 | "source": [ 176 | "# Train and test the model" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "metadata": { 182 | "id": "cVD1nFTli7TO" 183 | }, 184 | "source": [ 185 | "# train the model\n", 186 | "\n", 187 | "numepochs = 1000\n", 188 | "\n", 189 | "# initialize losses\n", 190 | "losses = torch.zeros(numepochs)\n", 191 | "ongoingAcc = []\n", 192 | "\n", 193 | "# loop over epochs\n", 194 | "for epochi in range(numepochs):\n", 195 | "\n", 196 | " # forward pass\n", 197 | " yHat = ANNiris(data[traintestBool,:])\n", 198 | "\n", 199 | " # compute accuracy (note: denser than previous code!)\n", 200 | " ongoingAcc.append( 100*torch.mean(\n", 201 | " (torch.argmax(yHat,axis=1) == labels[traintestBool]).float()) )\n", 202 | "\n", 203 | " # compute loss\n", 204 | " loss = lossfun(yHat,labels[traintestBool])\n", 205 | " losses[epochi] = loss\n", 206 | "\n", 207 | " # backprop\n", 208 | " optimizer.zero_grad()\n", 209 | " loss.backward()\n", 210 | " optimizer.step()" 211 | ], 212 | "execution_count": null, 213 | "outputs": [] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "metadata": { 218 | "id": "vXku7xIdcu7Y" 219 | }, 220 | "source": [ 221 | "# compute train and test accuracies\n", 222 | "\n", 223 | "# final forward pass USING TRAINING DATA\n", 224 | "predictions = ANNiris(data[traintestBool,:])\n", 225 | "trainacc = 100*torch.mean((torch.argmax(predictions,axis=1) == labels[traintestBool]).float())\n", 226 | "\n", 227 | "\n", 228 | "# final forward pass USING TEST DATA!\n", 229 | "predictions = ANNiris(data[~traintestBool,:])\n", 230 | "testacc = 100*torch.mean((torch.argmax(predictions,axis=1) == labels[~traintestBool]).float())" 231 | ], 232 | "execution_count": null, 233 | "outputs": [] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "metadata": { 238 | "id": "JYouZAY4i3jM" 239 | }, 240 | "source": [ 241 | "# report accuracies\n", 242 | "\n", 243 | "print('Final TRAIN accuracy: %g%%' %trainacc)\n", 244 | "print('Final TEST accuracy: %g%%' %testacc)" 245 | ], 246 | "execution_count": null, 247 | "outputs": [] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "metadata": { 252 | "id": "kcbD9nZmd9nu" 253 | }, 254 | "source": [ 255 | "# normally also inspect losses and accuracy by epoch, etc etc etc." 256 | ], 257 | "execution_count": null, 258 | "outputs": [] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "metadata": { 263 | "id": "MAzQqbq8fqSt" 264 | }, 265 | "source": [], 266 | "execution_count": null, 267 | "outputs": [] 268 | }, 269 | { 270 | "cell_type": "markdown", 271 | "metadata": { 272 | "id": "jwTbABK7fqzZ" 273 | }, 274 | "source": [ 275 | "# Additional explorations" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "metadata": { 281 | "id": "jWC_SDDCfrAo" 282 | }, 283 | "source": [ 284 | "# 1) Randomly assigning data samples to be in the train vs test phase produced a statistical balance, but it was\n", 285 | "# not perfect. Write an algorithm that will guarantee a balance of flower types while also randomly assigning\n", 286 | "# samples to be in train vs. test.\n", 287 | "#\n", 288 | "# 2) Revert the code to its original form -- with the strong imbalance in flower types. Then train the model. What are\n", 289 | "# the train and test accuracies? Compute the accuracy separately for each type of flower to see whether the model\n", 290 | "# learned some categories, or whether it performed equally on all three categories. Are you surprised at the results?\n", 291 | "#" 292 | ], 293 | "execution_count": null, 294 | "outputs": [] 295 | } 296 | ] 297 | } -------------------------------------------------------------------------------- /overfitting/DUDL_overfitting_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_overfitting_regression.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1Q_oDw0aMA4QFKDnLxuqJp62P8oPMtO1R", 10 | "timestamp": 1616705543155 11 | }, 12 | { 13 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 14 | "timestamp": 1615884593383 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyNSoqVKdJX7oDTOd7ULis2C" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | } 24 | }, 25 | "cells": [ 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "id": "bhWV8oes-wKR" 30 | }, 31 | "source": [ 32 | "# COURSE: A deep understanding of deep learning\n", 33 | "## SECTION: Overfitting, cross-validation, regularization\n", 34 | "### LECTURE: Cross-validation on regression\n", 35 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 36 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "j7-LiwqUMGYL" 43 | }, 44 | "source": [ 45 | "# import libraries\n", 46 | "import torch\n", 47 | "import torch.nn as nn\n", 48 | "import numpy as np\n", 49 | "import matplotlib.pyplot as plt" 50 | ], 51 | "execution_count": null, 52 | "outputs": [] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": { 57 | "id": "Wz6w7TtgQ6QF" 58 | }, 59 | "source": [ 60 | "# Create the data and the model" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "metadata": { 66 | "id": "j-SP8NPsMNRL" 67 | }, 68 | "source": [ 69 | "N = 100\n", 70 | "x = torch.randn(N,1)\n", 71 | "y = x + torch.randn(N,1)\n", 72 | "\n", 73 | "# and plot\n", 74 | "plt.plot(x,y,'s')\n", 75 | "plt.show()" 76 | ], 77 | "execution_count": null, 78 | "outputs": [] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "metadata": { 83 | "id": "krQeh5wYMNla" 84 | }, 85 | "source": [ 86 | "# build model\n", 87 | "ANNreg = nn.Sequential(\n", 88 | " nn.Linear(1,1), # input layer\n", 89 | " nn.ReLU(), # activation function\n", 90 | " nn.Linear(1,1) # output layer\n", 91 | " )" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "metadata": { 99 | "id": "dmHh7GrvMNoy" 100 | }, 101 | "source": [ 102 | "# model meta-parameters\n", 103 | "\n", 104 | "learningRate = .05\n", 105 | "\n", 106 | "# loss function\n", 107 | "lossfun = nn.MSELoss()\n", 108 | "\n", 109 | "# optimizer\n", 110 | "optimizer = torch.optim.SGD(ANNreg.parameters(),lr=learningRate)" 111 | ], 112 | "execution_count": null, 113 | "outputs": [] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": { 118 | "id": "vpZsJzRKQ-xM" 119 | }, 120 | "source": [ 121 | "# Select data for training" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "metadata": { 127 | "id": "of9E8ClxMNsD" 128 | }, 129 | "source": [ 130 | "# select training data (note the hard-coded N!)\n", 131 | "trainidx = np.random.choice(range(N),80,replace=False) # random indices\n", 132 | "trainBool = np.zeros(N,dtype=bool) # initialize vector of Falses'\n", 133 | "trainBool[trainidx] = True # set selected samples to True\n", 134 | "\n", 135 | "# show the sizes\n", 136 | "print(x[trainBool].shape)\n", 137 | "print(x[~trainBool].shape)" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "id": "lmUXAALTRPkL" 146 | }, 147 | "source": [ 148 | "# Train the model" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "metadata": { 154 | "id": "EY4ayy2VRGeZ" 155 | }, 156 | "source": [ 157 | "numepochs = 500\n", 158 | "for epochi in range(numepochs):\n", 159 | "\n", 160 | " # forward pass\n", 161 | " yHat = ANNreg(x[trainBool])\n", 162 | "\n", 163 | " # compute loss\n", 164 | " loss = lossfun(yHat,y[trainBool])\n", 165 | "\n", 166 | " # backprop\n", 167 | " optimizer.zero_grad()\n", 168 | " loss.backward()\n", 169 | " optimizer.step()" 170 | ], 171 | "execution_count": null, 172 | "outputs": [] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "metadata": { 177 | "id": "zmX6K49WMNuy" 178 | }, 179 | "source": [ 180 | "# report the losses\n", 181 | "\n", 182 | "# compute losses of the TEST set\n", 183 | "predYtest = ANNreg(x[~trainBool])\n", 184 | "testloss = (predYtest-y[~trainBool]).pow(2).mean()\n", 185 | "\n", 186 | "# print out final TRAIN loss and TEST loss\n", 187 | "print(f'Final TRAIN loss: {loss.detach():.2f}')\n", 188 | "print(f'Final TEST loss: {testloss.detach():.2f}')" 189 | ], 190 | "execution_count": null, 191 | "outputs": [] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "metadata": { 196 | "id": "i1TCt0mpMNxC" 197 | }, 198 | "source": [ 199 | "### plot the data\n", 200 | "\n", 201 | "# predictions for final training run\n", 202 | "predYtrain = ANNreg(x[trainBool]).detach().numpy()\n", 203 | "\n", 204 | "# now plot\n", 205 | "plt.plot(x,y,'k^',label='All data')\n", 206 | "plt.plot(x[trainBool], predYtrain,\n", 207 | " 'bs',markerfacecolor='w',label='Training pred.')\n", 208 | "plt.plot(x[~trainBool],predYtest.detach(),\n", 209 | " 'ro',markerfacecolor='w',label='Test pred.')\n", 210 | "plt.legend()\n", 211 | "plt.show()" 212 | ], 213 | "execution_count": null, 214 | "outputs": [] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "metadata": { 219 | "id": "ucAVWWYEbBE5" 220 | }, 221 | "source": [ 222 | "" 223 | ], 224 | "execution_count": null, 225 | "outputs": [] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": { 230 | "id": "jjvdKsrdZ4ka" 231 | }, 232 | "source": [ 233 | "# Additional explorations" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "metadata": { 239 | "id": "5EPeV1KZZ42b" 240 | }, 241 | "source": [ 242 | "# 1) The train/test split is currently hard-coded to be 80/20 (note the number \"80\"). This is bad coding style, because\n", 243 | "# if you change the number of datapoints from N=100 to N=10000, then we're still only training on 80 samples and testing\n", 244 | "# on 10000-80=9920 samples. Change how the variable trainBool is created so that it always trains on 80% of the data,\n", 245 | "# regardless of the dataset size.\n", 246 | "# \n", 247 | "# 2) Re-write this code to use scikitlearn and/or DataLoader instead of manually separating the data into train/test.\n", 248 | "# \n", 249 | "# 3) Do we really need 500 epochs to train the model? To find out, add code to the training loop to compute the MSEloss \n", 250 | "# for the train and test data on each iteration during training. Then plot the train and test error as a function of\n", 251 | "# training epoch. What is your evaluation of an appropriate amount of training for this model/dataset?\n", 252 | "# " 253 | ], 254 | "execution_count": null, 255 | "outputs": [] 256 | } 257 | ] 258 | } -------------------------------------------------------------------------------- /overfitting/DUDL_overfitting_trainDevsetTest.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_overfitting_trainDevsetTest.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 10 | "timestamp": 1616615469755 11 | }, 12 | { 13 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 14 | "timestamp": 1616608248670 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyORIyTWClz7YOKdhmp8/3fQ" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | } 24 | }, 25 | "cells": [ 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "id": "bhWV8oes-wKR" 30 | }, 31 | "source": [ 32 | "# COURSE: A deep understanding of deep learning\n", 33 | "## SECTION: Overfitting, cross-validation, regularization\n", 34 | "### LECTURE: Splitting data into train, devset, test\n", 35 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 36 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "YeuAheYyhdZw" 43 | }, 44 | "source": [ 45 | "# import libraries\n", 46 | "import numpy as np\n", 47 | "from sklearn.model_selection import train_test_split" 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "metadata": { 55 | "id": "q-YUb7pW19yy" 56 | }, 57 | "source": [ 58 | "### create fake dataset (same as in previous videos)\n", 59 | "\n", 60 | "fakedata = np.tile(np.array([1,2,3,4]),(10,1)) + np.tile(10*np.arange(1,11),(4,1)).T\n", 61 | "fakelabels = np.arange(10)>4\n", 62 | "print(fakedata), print(' ')\n", 63 | "print(fakelabels)" 64 | ], 65 | "execution_count": null, 66 | "outputs": [] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": { 71 | "id": "UhkvsJ6g6uXr" 72 | }, 73 | "source": [ 74 | "# Using train_test_split" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "metadata": { 80 | "id": "8bxbHGkP7JW3" 81 | }, 82 | "source": [ 83 | "# specify sizes of the partitions\n", 84 | "# order is train,devset,test\n", 85 | "partitions = [.8,.1,.1]\n", 86 | "\n", 87 | "# split the data (note the third input, and the TMP in the variable name)\n", 88 | "train_data,testTMP_data, train_labels,testTMP_labels = \\\n", 89 | " train_test_split(fakedata, fakelabels, train_size=partitions[0])\n", 90 | "\n", 91 | "# now split the TMP data\n", 92 | "split = partitions[1] / np.sum(partitions[1:])\n", 93 | "devset_data,test_data, devset_labels,test_labels = \\\n", 94 | " train_test_split(testTMP_data, testTMP_labels, train_size=partitions[1])\n", 95 | "\n", 96 | "\n", 97 | "\n", 98 | "\n", 99 | "# print out the sizes\n", 100 | "print('Training data size: ' + str(train_data.shape))\n", 101 | "print('Devset data size: ' + str(devset_data.shape))\n", 102 | "print('Test data size: ' + str(test_data.shape))\n", 103 | "print(' ')\n", 104 | "\n", 105 | "# print out the train/test data\n", 106 | "print('Training data: ')\n", 107 | "print(train_data)\n", 108 | "print(' ')\n", 109 | "\n", 110 | "print('Devset data: ')\n", 111 | "print(devset_data)\n", 112 | "print(' ')\n", 113 | "\n", 114 | "print('Test data: ')\n", 115 | "print(test_data)" 116 | ], 117 | "execution_count": null, 118 | "outputs": [] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": { 123 | "id": "EvUQFxSTV2SB" 124 | }, 125 | "source": [ 126 | "# Splitting the data manually using numpy" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "metadata": { 132 | "id": "XUZcKNWsXg00" 133 | }, 134 | "source": [ 135 | "# partition sizes in proportion\n", 136 | "partitions = np.array([.8,.1,.1])\n", 137 | "print('Partition proportions:')\n", 138 | "print(partitions)\n", 139 | "print(' ')\n", 140 | "\n", 141 | "# convert those into integers\n", 142 | "partitionBnd = np.cumsum(partitions*len(fakelabels)).astype(int)\n", 143 | "print('Partition boundaries:')\n", 144 | "print(partitionBnd)\n", 145 | "print(' ')\n", 146 | "\n", 147 | "\n", 148 | "# random indices\n", 149 | "randindices = np.random.permutation(range(len(fakelabels)))\n", 150 | "print('Randomized data indices:')\n", 151 | "print(randindices)\n", 152 | "print(' ')" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "metadata": { 160 | "id": "Vre4YiQBZmjy" 161 | }, 162 | "source": [ 163 | "# select rows for the training data\n", 164 | "train_dataN = fakedata[randindices[:partitionBnd[0]],:]\n", 165 | "train_labelsN = fakelabels[randindices[:partitionBnd[0]]]\n", 166 | "\n", 167 | "# select rows for the devset data\n", 168 | "devset_dataN = fakedata[randindices[partitionBnd[0]:partitionBnd[1]],:]\n", 169 | "devset_labelsN = fakelabels[randindices[partitionBnd[0]:partitionBnd[1]]]\n", 170 | "\n", 171 | "# select rows for the test data\n", 172 | "test_dataN = fakedata[randindices[partitionBnd[1]:],:]\n", 173 | "test_labelsN = fakelabels[randindices[partitionBnd[1]:]]" 174 | ], 175 | "execution_count": null, 176 | "outputs": [] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "metadata": { 181 | "id": "vbTLW0MkXg-V" 182 | }, 183 | "source": [ 184 | "# print out the sizes\n", 185 | "print('Training data size: ' + str(train_dataN.shape))\n", 186 | "print('Devset size: ' + str(devset_dataN.shape))\n", 187 | "print('Test data size: ' + str(test_dataN.shape))\n", 188 | "print(' ')\n", 189 | "\n", 190 | "# print out the train/test data\n", 191 | "print('Training data: ')\n", 192 | "print(train_dataN)\n", 193 | "print(' ')\n", 194 | "\n", 195 | "print('Devset data: ')\n", 196 | "print(devset_dataN)\n", 197 | "print(' ')\n", 198 | "\n", 199 | "print('Test data: ')\n", 200 | "print(test_dataN)" 201 | ], 202 | "execution_count": null, 203 | "outputs": [] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "metadata": { 208 | "id": "CaP1IQDrXhBc" 209 | }, 210 | "source": [ 211 | "" 212 | ], 213 | "execution_count": null, 214 | "outputs": [] 215 | } 216 | ] 217 | } -------------------------------------------------------------------------------- /regularization/DUDL_regular_dropoutInPytorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_regular_dropoutInPytorch.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "17KCLWj5_rIPAJqSQ1dJFAkSrJ7ctDwne", 10 | "timestamp": 1616945891013 11 | }, 12 | { 13 | "file_id": "1bv1_y32e3KEExFKKlPfC3rpw1JxmBr8H", 14 | "timestamp": 1616941708388 15 | }, 16 | { 17 | "file_id": "1GMq8u7KyHB2AE7Teyls9gK1T01OduQSn", 18 | "timestamp": 1616697516760 19 | }, 20 | { 21 | "file_id": "1Ui3kyHim-e0XLgDs2mkBxVlYg7TKYtcg", 22 | "timestamp": 1616615469755 23 | }, 24 | { 25 | "file_id": "1YpHocGI4rApOxIBb1ZghCU5L-hFnv4CK", 26 | "timestamp": 1616608248670 27 | } 28 | ], 29 | "collapsed_sections": [], 30 | "authorship_tag": "ABX9TyOLdTgViM3bQDslDQDqBhQV" 31 | }, 32 | "kernelspec": { 33 | "name": "python3", 34 | "display_name": "Python 3" 35 | } 36 | }, 37 | "cells": [ 38 | { 39 | "cell_type": "markdown", 40 | "metadata": { 41 | "id": "bhWV8oes-wKR" 42 | }, 43 | "source": [ 44 | "# COURSE: A deep understanding of deep learning\n", 45 | "## SECTION: Regularization\n", 46 | "### LECTURE: Dropout regularization in practice\n", 47 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 48 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "metadata": { 54 | "id": "YeuAheYyhdZw" 55 | }, 56 | "source": [ 57 | "# import libraries\n", 58 | "import torch\n", 59 | "import torch.nn as nn\n", 60 | "import torch.nn.functional as F" 61 | ], 62 | "execution_count": null, 63 | "outputs": [] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": { 68 | "id": "OAw7w-Is8quO" 69 | }, 70 | "source": [ 71 | "# Using dropout" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "id": "PakwZXu08s6B" 78 | }, 79 | "source": [ 80 | "# define a dropout instance and make some data\n", 81 | "prob = .5\n", 82 | "\n", 83 | "dropout = nn.Dropout(p=prob)\n", 84 | "x = torch.ones(10)\n", 85 | "\n", 86 | "# let's see what dropout returns\n", 87 | "y = dropout(x)\n", 88 | "print(x)\n", 89 | "print(y*(1-prob))\n", 90 | "print(torch.mean(y))" 91 | ], 92 | "execution_count": null, 93 | "outputs": [] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "metadata": { 98 | "id": "hnsHnDFjVQ6i" 99 | }, 100 | "source": [ 101 | "# dropout is turned off when evaluating the model\n", 102 | "dropout.eval()\n", 103 | "y = dropout(x)\n", 104 | "print(y)\n", 105 | "print(torch.mean(y))" 106 | ], 107 | "execution_count": null, 108 | "outputs": [] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "metadata": { 113 | "id": "5bY95mSjVYvA" 114 | }, 115 | "source": [ 116 | "# annoyingly, F.dropout() is not deactivated in eval mode:\n", 117 | "\n", 118 | "dropout.eval()\n", 119 | "y = F.dropout(x)\n", 120 | "print(y)\n", 121 | "print(torch.mean(y))" 122 | ], 123 | "execution_count": null, 124 | "outputs": [] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "metadata": { 129 | "id": "Aa-GvWSLXcH8" 130 | }, 131 | "source": [ 132 | "# but you can manually switch it off\n", 133 | "# dropout.eval()\n", 134 | "y = F.dropout(x,training=False)\n", 135 | "\n", 136 | "print(y)\n", 137 | "print(torch.mean(y))" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "metadata": { 145 | "id": "J_BFJYeTU_ak" 146 | }, 147 | "source": [ 148 | "# the model needs to be reset after toggling into eval mode\n", 149 | "\n", 150 | "dropout.train()\n", 151 | "y = dropout(x)\n", 152 | "print(y) # with dropout\n", 153 | "\n", 154 | "\n", 155 | "dropout.eval()\n", 156 | "y = dropout(x)\n", 157 | "print(y) # without dropout\n", 158 | "\n", 159 | "\n", 160 | "# dropout.train()\n", 161 | "y = dropout(x)\n", 162 | "print(y) # still w/o dropout ;)" 163 | ], 164 | "execution_count": null, 165 | "outputs": [] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "metadata": { 170 | "id": "eVXa_1zZkvbm" 171 | }, 172 | "source": [ 173 | "" 174 | ], 175 | "execution_count": null, 176 | "outputs": [] 177 | } 178 | ] 179 | } -------------------------------------------------------------------------------- /weights/DUDL_weights_codeChallenge_identicalRandom.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_weights_codeChallenge_identicalRandom.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyNVY6AaUJqtJZAGi+/ETocM" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "bhWV8oes-wKR" 24 | }, 25 | "source": [ 26 | "# COURSE: A deep understanding of deep learning\n", 27 | "## SECTION: Weight inits and investigations\n", 28 | "### LECTURE: CodeChallenge: Identically random weights\n", 29 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 30 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "metadata": { 36 | "id": "Vid8jNaLHNtQ" 37 | }, 38 | "source": [ 39 | "# import libraries\n", 40 | "import numpy as np\n", 41 | "import torch\n", 42 | "import torch.nn as nn\n", 43 | "import copy\n", 44 | "\n", 45 | "import matplotlib.pyplot as plt\n", 46 | "import matplotlib_inline.backend_inline\n", 47 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg')" 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "metadata": { 55 | "id": "PDWw3MdSHOV2" 56 | }, 57 | "source": [ 58 | "# create a model\n", 59 | "net = nn.Sequential(\n", 60 | " nn.Linear(2,8), # input layer\n", 61 | " nn.Linear(8,1), # hidden layer\n", 62 | " nn.Linear(1,1), # output unit\n", 63 | " )" 64 | ], 65 | "execution_count": null, 66 | "outputs": [] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": { 71 | "id": "k6dR60DFJDRR" 72 | }, 73 | "source": [ 74 | "# Create the four networks" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "metadata": { 80 | "id": "UL41jAxaHOZP" 81 | }, 82 | "source": [ 83 | "### NO RANDOM SEED\n", 84 | "\n", 85 | "# create a \"template\" network\n", 86 | "net_noseed = copy.deepcopy( net )\n", 87 | "\n", 88 | "# update the weights\n", 89 | "for p in net_noseed.named_parameters():\n", 90 | " if 'weight' in p[0]:\n", 91 | " nn.init.xavier_normal_(p[1].data)" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "metadata": { 99 | "id": "6ZUB_R8MHOcH" 100 | }, 101 | "source": [ 102 | "### RANDOM SEED 1a\n", 103 | "\n", 104 | "# create a \"template\" network\n", 105 | "net_rs1a = copy.deepcopy( net )\n", 106 | "\n", 107 | "# set the seed to 1\n", 108 | "torch.manual_seed(1)\n", 109 | "\n", 110 | "# update the weights\n", 111 | "for p in net_rs1a.named_parameters():\n", 112 | " if 'weight' in p[0]:\n", 113 | " nn.init.xavier_normal_(p[1].data)" 114 | ], 115 | "execution_count": null, 116 | "outputs": [] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "metadata": { 121 | "id": "nfN3OQBlHOfP" 122 | }, 123 | "source": [ 124 | "### RANDOM SEED 2\n", 125 | "\n", 126 | "# create a \"template\" network\n", 127 | "net_rs2 = copy.deepcopy( net )\n", 128 | "\n", 129 | "# set the seed to 2\n", 130 | "torch.manual_seed(2)\n", 131 | "\n", 132 | "# update the weights\n", 133 | "for p in net_rs2.named_parameters():\n", 134 | " if 'weight' in p[0]:\n", 135 | " nn.init.xavier_normal_(p[1].data)" 136 | ], 137 | "execution_count": null, 138 | "outputs": [] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "metadata": { 143 | "id": "orgfGEscHOiW" 144 | }, 145 | "source": [ 146 | "### RANDOM SEED 1b\n", 147 | "\n", 148 | "# create a \"template\" network\n", 149 | "net_rs1b = copy.deepcopy( net )\n", 150 | "\n", 151 | "# set the seed to 1\n", 152 | "torch.manual_seed(1)\n", 153 | "\n", 154 | "# update the weights\n", 155 | "for p in net_rs1b.named_parameters():\n", 156 | " if 'weight' in p[0]:\n", 157 | " nn.init.xavier_normal_(p[1].data)" 158 | ], 159 | "execution_count": null, 160 | "outputs": [] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": { 165 | "id": "RthL0VXjJMCh" 166 | }, 167 | "source": [ 168 | "# Extract all weights from all networks" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "metadata": { 174 | "id": "0CiwfIo2HOlW" 175 | }, 176 | "source": [ 177 | "# initialize empty arrays\n", 178 | "w_ns = np.array([])\n", 179 | "w_1a = np.array([])\n", 180 | "w_2 = np.array([])\n", 181 | "w_1b = np.array([])\n", 182 | "\n", 183 | "# loop over layers in the models\n", 184 | "for i in range(3):\n", 185 | "\n", 186 | " # extract the vectorized weights matrices\n", 187 | " w_ns = np.append(w_ns,net_noseed[i].weight.view(-1).detach().numpy())\n", 188 | " w_1a = np.append(w_1a, net_rs1a[i].weight.view(-1).detach().numpy())\n", 189 | " w_2 = np.append(w_2 , net_rs2[i].weight.view(-1).detach().numpy())\n", 190 | " w_1b = np.append(w_1b, net_rs1b[i].weight.view(-1).detach().numpy())\n", 191 | "\n", 192 | "# check the vector\n", 193 | "w_ns" 194 | ], 195 | "execution_count": null, 196 | "outputs": [] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "metadata": { 201 | "id": "yooMnVoxHOoe" 202 | }, 203 | "source": [ 204 | "# plotting\n", 205 | "\n", 206 | "fig = plt.figure(figsize=(15,5))\n", 207 | "\n", 208 | "plt.plot(w_ns,'ro',markersize=12,label='no seed')\n", 209 | "plt.plot(w_1a,'ks',markersize=12,label='rs1a')\n", 210 | "plt.plot(w_2 ,'m^',markersize=12,label='rs2')\n", 211 | "plt.plot(w_1b,'g+',markersize=15,label='rs1b',markeredgewidth=3)\n", 212 | "plt.legend()\n", 213 | "plt.xlabel('Vectorized weight index')\n", 214 | "plt.ylabel('Weight value')\n", 215 | "\n", 216 | "plt.show()" 217 | ], 218 | "execution_count": null, 219 | "outputs": [] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "metadata": { 224 | "id": "lY2_q4PjHOre" 225 | }, 226 | "source": [ 227 | "# the algebraic test\n", 228 | "\n", 229 | "print('1a-1b should be zeros:')\n", 230 | "print( w_1a-w_1b )\n", 231 | "print('\\n\\n')\n", 232 | "\n", 233 | "print('1a-2 should be non-zeros:')\n", 234 | "print( w_1a-w_2 )\n", 235 | "print('\\n\\n')\n", 236 | "\n", 237 | "print('1a-ns should be non-zeros:')\n", 238 | "print( w_1a-w_ns )\n", 239 | "print('\\n\\n')" 240 | ], 241 | "execution_count": null, 242 | "outputs": [] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "metadata": { 247 | "id": "PB27jHgSQBvr" 248 | }, 249 | "source": [ 250 | "" 251 | ], 252 | "execution_count": null, 253 | "outputs": [] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": { 258 | "id": "l9U2DpktHOuO" 259 | }, 260 | "source": [ 261 | "# Additional explorations" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "metadata": { 267 | "id": "E4XgsqVcM32M" 268 | }, 269 | "source": [ 270 | "# 1) The code here only fixed the *weights*, not the biases. Does that mean that net_rs1a and net_rs1b are actually\n", 271 | "# DIFFERENT networks??\n", 272 | "# \n", 273 | "# 2) Change the code to plot/subtract the biases instead of the weights. Are the results what you expected? If not,\n", 274 | "# figure out why!\n", 275 | "# " 276 | ], 277 | "execution_count": null, 278 | "outputs": [] 279 | } 280 | ] 281 | } -------------------------------------------------------------------------------- /weights/DUDL_weights_matrixsizes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "DUDL_weights_matrixsizes.ipynb", 7 | "provenance": [ 8 | { 9 | "file_id": "10_geQnah5AvMsm8VDAQwNPhypOXradar", 10 | "timestamp": 1619420451667 11 | }, 12 | { 13 | "file_id": "1FtQ99beHYcDFDywLdaPgFm-KjBeI8PvD", 14 | "timestamp": 1615877547147 15 | } 16 | ], 17 | "collapsed_sections": [], 18 | "authorship_tag": "ABX9TyP9SNH3ckSFTyepnrN3b2L4" 19 | }, 20 | "kernelspec": { 21 | "name": "python3", 22 | "display_name": "Python 3" 23 | } 24 | }, 25 | "cells": [ 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "id": "bhWV8oes-wKR" 30 | }, 31 | "source": [ 32 | "# COURSE: A deep understanding of deep learning\n", 33 | "## SECTION: Weight inits and investigations\n", 34 | "### LECTURE: Explanation of weight matrix sizes\n", 35 | "#### TEACHER: Mike X Cohen, sincxpress.com\n", 36 | "##### COURSE URL: udemy.com/course/deeplearning_x/?couponCode=202401" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "j7-LiwqUMGYL" 43 | }, 44 | "source": [ 45 | "# import libraries\n", 46 | "import torch\n", 47 | "import torch.nn as nn" 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "id": "y7sPkqG8ATnz" 56 | }, 57 | "source": [ 58 | "# Create a model and inspect its weight matrices sizes" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "metadata": { 64 | "id": "krQeh5wYMNla" 65 | }, 66 | "source": [ 67 | "# build a model\n", 68 | "aModel = nn.Sequential(\n", 69 | " nn.Linear(10,14), # input layer\n", 70 | " nn.Linear(14,19), # hidden layer\n", 71 | " nn.Linear(19,8), # output layer\n", 72 | " )\n", 73 | "\n", 74 | "aModel" 75 | ], 76 | "execution_count": null, 77 | "outputs": [] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "metadata": { 82 | "id": "YTHj0h7GEVMq" 83 | }, 84 | "source": [ 85 | "# print the sizes of the weights matrices in each layer\n", 86 | "for i in range(len(aModel)):\n", 87 | " print( aModel[i].weight.shape )" 88 | ], 89 | "execution_count": null, 90 | "outputs": [] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": { 95 | "id": "DGMwd5TCAZwr" 96 | }, 97 | "source": [ 98 | "# Build a model with inconsistent layer shapes" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "metadata": { 104 | "id": "sEMW2hT1_aTm" 105 | }, 106 | "source": [ 107 | "M2 = nn.Sequential(\n", 108 | " nn.Linear(10,14), # input layer\n", 109 | " nn.Linear(14,9), # hidden layer\n", 110 | " nn.Linear(19,8), # output layer\n", 111 | " )\n", 112 | "\n", 113 | "for i in range(len(M2)):\n", 114 | " print( M2[i].weight.shape )" 115 | ], 116 | "execution_count": null, 117 | "outputs": [] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": { 122 | "id": "wwhD7qwnAbUz" 123 | }, 124 | "source": [ 125 | "# Test both models with fake data" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "metadata": { 131 | "id": "Gux-Zj2NAA60" 132 | }, 133 | "source": [ 134 | "# generate the data\n", 135 | "nsamples = 5\n", 136 | "nfeatures = 10\n", 137 | "\n", 138 | "fakedata = torch.randn(nsamples,nfeatures)" 139 | ], 140 | "execution_count": null, 141 | "outputs": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "metadata": { 146 | "id": "HeYXIgQEAf03" 147 | }, 148 | "source": [ 149 | "# test the first model\n", 150 | "\n", 151 | "# does the size of the output make sense?\n", 152 | "aModel(fakedata).shape" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "metadata": { 160 | "id": "DutcYnW2AgA3" 161 | }, 162 | "source": [ 163 | "# test the second model\n", 164 | "\n", 165 | "# does the size of the output make sense?\n", 166 | "M2(fakedata).shape" 167 | ], 168 | "execution_count": null, 169 | "outputs": [] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "metadata": { 174 | "id": "_mSUBBkdAgGK" 175 | }, 176 | "source": [ 177 | "" 178 | ], 179 | "execution_count": null, 180 | "outputs": [] 181 | } 182 | ] 183 | } --------------------------------------------------------------------------------