├── .gitignore ├── Deep-Learning-Models ├── MCLR_Keras.ipynb ├── MCLR_PyTorch.ipynb ├── MCLR_TensorFlow.ipynb ├── README.md ├── autoencoder_Keras.ipynb ├── basics_PyTorch.ipynb ├── basics_TensorFlow.ipynb ├── convolutional_neural_network_Keras.ipynb ├── neural_network_deep_Keras.ipynb ├── neural_network_shallow_Keras.ipynb ├── perceptron_Keras.ipynb ├── perceptron_PyTorch.ipynb ├── perceptron_TensorFlow.ipynb └── sourceimages │ ├── MCLR.png │ ├── autoencoder_convolutional.png │ ├── autoencoder_deep.png │ ├── autoencoder_shallow.png │ ├── convolutional_neural_network.png │ ├── mandril.png │ ├── neural_network_deep.png │ ├── neural_network_shallow.png │ └── perceptron.png ├── LICENSE.md ├── Machine-Learning-Fundamentals ├── NN__utils.py ├── NN__visualizations.ipynb ├── NN_activation_functions.ipynb ├── NN_perceptron.ipynb ├── README.md ├── clustering__utils.py ├── clustering__visualizations.ipynb ├── clustering_dbscan.ipynb ├── clustering_k-means.ipynb ├── kNN__utils.py ├── kNN__visualizations.ipynb ├── kNN_classification.ipynb ├── kNN_regression.ipynb ├── output │ ├── NN_perceptron.gif │ ├── NN_perceptron_synthData.png │ ├── clustering_dbscan.png │ ├── clustering_k-means.gif │ ├── clustering_k-means_elbowMethod.png │ ├── clustering_synthetic_data.png │ ├── kNN_classificationA.gif │ ├── kNN_classificationB.png │ ├── kNN_introduction.png │ ├── kNN_regressionA.png │ ├── kNN_regressionB.png │ ├── regression_linear_anscombe_pred.png │ ├── regression_linear_anscombe_residual.png │ ├── regression_linear_correlation.png │ ├── regression_linear_gradDesc.gif │ ├── regression_linear_multipla_residual.png │ ├── regression_linear_multiple_pred.png │ ├── regression_linear_pred.png │ ├── regression_linear_residual.png │ ├── regression_logistic_data.png │ ├── regression_logistic_gradDesc.gif │ ├── regression_logistic_pred.png │ ├── regression_polynomial_linear.png │ └── regression_polynomial_pred.png ├── regression__utils.py ├── regression__visualizations.ipynb ├── regression_linear.ipynb ├── regression_logistic.ipynb └── regression_polynomial.ipynb ├── Mathematical-Foundations ├── README.md ├── calculus_fourier-series.ipynb ├── calculus_fourierSeries__utils.py ├── capital-pi-notation.ipynb ├── capital-sigma-notation.ipynb ├── dissimilarity__utils.py ├── dissimilarity_measures.ipynb ├── linear-algebra_matrices.ipynb ├── linear-algebra_vectors.ipynb └── sourceimages │ ├── linear-algebra_vectors.jpg │ ├── linear-algebra_vectors_addition.jpg │ ├── linear-algebra_vectors_angle.jpg │ ├── linear-algebra_vectors_orthogonal.jpg │ ├── linear-algebra_vectors_parallel.jpg │ ├── linear-algebra_vectors_scalar_multiplication.jpg │ ├── linear-algebra_vectors_subtraction.jpg │ └── linear-algebra_vectors_unit_vector.jpg ├── Natural-Language-Processing ├── README.md └── basics_NLTK.ipynb ├── Practical-Applications ├── README.md ├── _data │ └── woman01.png └── image_approximation_deepNN.ipynb ├── README.md ├── Time-Series └── README.md ├── Tips-and-Tricks ├── README.md ├── basics_Cython.ipynb ├── basics_Numba.ipynb ├── connect_openai_api.ipynb └── kaggle-data.ipynb └── docs ├── assets ├── icons │ ├── go_python_notebooks.svg │ ├── join_discussion.svg │ ├── support_this_project.svg │ └── visit_cookbook.svg └── images │ └── social-preview.png ├── index.html └── style.css /.gitignore: -------------------------------------------------------------------------------- 1 | ### Python ### 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | ### Jupyter Notebook ### 7 | .ipynb_checkpoints 8 | 9 | ################### 10 | ### Page Assets ### 11 | ################### 12 | 13 | ### NodeJS ### 14 | node_modules/ 15 | package-lock.json 16 | 17 | # live sass compiler 18 | *.css.map 19 | 20 | ###################### 21 | ### Tests and Temp ### 22 | ###################### 23 | 24 | ### Development files ### 25 | .env 26 | 27 | ### Image files ### 28 | _temp*.png 29 | _temp*.jpg 30 | _temp*.jpeg 31 | -------------------------------------------------------------------------------- /Deep-Learning-Models/MCLR_Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multi-class Logistic Regression [Keras]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [MCLR_Keras.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/MCLR_Keras.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Multi-class Logistic Regression* using Keras library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import tensorflow as tf" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "plt.rcParams['figure.figsize'] = (16, 8)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "mnist = tf.keras.datasets.mnist\n", 46 | "[X_train, Y_train],[X_test, Y_test] = mnist.load_data()\n", 47 | "\n", 48 | "# one hot for multi-class classification\n", 49 | "# MNIST = 10 classes [0-9]\n", 50 | "y_train_oh = np.zeros((Y_train.size, 10))\n", 51 | "y_train_oh[np.arange(Y_train.size), Y_train] = 1\n", 52 | "y_test_oh = np.zeros((Y_test.size, 10))\n", 53 | "y_test_oh[np.arange(Y_test.size), Y_test] = 1\n", 54 | "\n", 55 | "print('X_train:', X_train.shape)\n", 56 | "print('Y_train:', Y_train.shape)\n", 57 | "print('y_train_oh:', y_train_oh.shape)\n", 58 | "print('X_test:', X_test.shape)\n", 59 | "print('Y_test:', Y_test.shape)\n", 60 | "print('y_test_oh:', y_test_oh.shape)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 70 | "\n", 71 | "np.random.seed(1234)\n", 72 | "for ax in AX.ravel():\n", 73 | " rindex = np.random.randint(Y_train.size)\n", 74 | " ax.imshow(X_train[rindex])\n", 75 | " # title label + one-hot\n", 76 | " title = '{} :: '.format(Y_train[rindex]) \n", 77 | " title += ''.join([str(int(e)) for e in y_train_oh[rindex]]) \n", 78 | " ax.set_title(title)\n", 79 | "plt.grid(False)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 89 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 90 | "\n", 91 | "x_train = x_train.astype(np.float32)\n", 92 | "y_train = y_train.astype(np.float32)\n", 93 | "x_test = x_test.astype(np.float32)\n", 94 | "y_test = y_test.astype(np.float32)\n", 95 | "\n", 96 | "print('x_train:', x_train.shape)\n", 97 | "print('y_train:', y_train.shape)\n", 98 | "print('x_test:', x_test.shape)\n", 99 | "print('y_test:', y_test.shape)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "# reshape Xs\n", 109 | "x_train = x_train.reshape(-1, 28*28)\n", 110 | "x_test = x_test.reshape(-1, 28*28)\n", 111 | "\n", 112 | "print('x_train:', x_train.shape)\n", 113 | "print('x_test:', x_test.shape)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "## Algorithm\n", 121 | "---\n", 122 | "Multi-class logistic regression\n", 123 | "![mclr](sourceimages/MCLR.png \"Multi-Class Logistic Regression\")" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "EPOCHS = 5000 # epochs\n", 133 | "ALPHA = 0.005 # learning rate\n", 134 | "BATCH = 100 # batch size\n", 135 | "\n", 136 | "# m is the number of examples\n", 137 | "# n_x is the input size 28x28=784\n", 138 | "m, n_x = x_train.shape\n", 139 | "\n", 140 | "# model\n", 141 | "model = tf.keras.Sequential([\n", 142 | " tf.keras.layers.Input(n_x),\n", 143 | " tf.keras.layers.Dense(10, activation='softmax')\n", 144 | "])\n", 145 | "\n", 146 | "# Compile model\n", 147 | "model.compile(\n", 148 | " loss='categorical_crossentropy',\n", 149 | " optimizer='SGD',\n", 150 | " lr=ALPHA,\n", 151 | " metrics=['accuracy']\n", 152 | ")\n", 153 | "\n", 154 | "# loss and accuracy storage\n", 155 | "loss_plot = []; accA_plot = []\n", 156 | "\n", 157 | "for epoch in range(EPOCHS + 1):\n", 158 | " # randomic batch definition\n", 159 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 160 | " # training, metrics and storage\n", 161 | " model.fit(x_train[rbatch], y_train_oh[rbatch], epochs=1, verbose=0)\n", 162 | " loss_plot += [e*100 for e in model.history.history['loss']]\n", 163 | " accA_plot += [e*100 for e in model.history.history['accuracy']]\n", 164 | " if (not epoch % 1000) and (epoch != 0):\n", 165 | " print(f'epoch: {epoch:04d} | loss: {loss_plot[-1]:.3f} | accuracy: {accA_plot[-1]:06.2f} %')\n", 166 | "\n", 167 | "W_ = model.weights[0].numpy() # store W and B for visualization and test" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": null, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "fig, AX = plt.subplots(1, 10, sharey=True)\n", 177 | "\n", 178 | "for i in range(10):\n", 179 | " AX[i].imshow(W_.T[i].reshape(28, 28))\n", 180 | " AX[i].set_title(r'$W_{}$'.format(i))" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": null, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "fig, [axA, axB] = plt.subplots(2, 1, sharex=True)\n", 190 | "\n", 191 | "axA.plot(loss_plot)\n", 192 | "axA.set_ylabel('loss')\n", 193 | "axB.plot(accA_plot)\n", 194 | "axB.set_ylabel('accuracy')\n", 195 | "\n", 196 | "plt.xlabel('epochs')\n", 197 | "\n", 198 | "plt.show()" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 208 | "AX = [b for a in AX for b in a]\n", 209 | "\n", 210 | "pred = model.predict(x_test)\n", 211 | "\n", 212 | "np.random.seed(1)\n", 213 | "for ax in AX:\n", 214 | " index = np.random.randint(y_test.size)\n", 215 | " Z_ = pred[index]\n", 216 | " Y_ = np.argmax(Z_)\n", 217 | " if Y_ == y_test[index]:\n", 218 | " ax.imshow(x_test[index].reshape(28, 28))\n", 219 | " else:\n", 220 | " ax.imshow(1 - x_test[index].reshape(28, 28))\n", 221 | " ez = np.exp(Z_ - Z_.max())\n", 222 | " A_ = ez/ez.sum(); A_ = float(A_.T[Y_])\n", 223 | " ax.set_title(r'$\\hat{Y_i}$ = ' + str(Y_) + ' ; $A_i$ = {:.03f}'.format(A_), fontsize=20)" 224 | ] 225 | } 226 | ], 227 | "metadata": { 228 | "kernelspec": { 229 | "display_name": "Python [conda env:tfgpu]", 230 | "language": "python", 231 | "name": "conda-env-tfgpu-py" 232 | }, 233 | "language_info": { 234 | "codemirror_mode": { 235 | "name": "ipython", 236 | "version": 3 237 | }, 238 | "file_extension": ".py", 239 | "mimetype": "text/x-python", 240 | "name": "python", 241 | "nbconvert_exporter": "python", 242 | "pygments_lexer": "ipython3", 243 | "version": "3.7.7" 244 | } 245 | }, 246 | "nbformat": 4, 247 | "nbformat_minor": 2 248 | } 249 | -------------------------------------------------------------------------------- /Deep-Learning-Models/MCLR_PyTorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multi-class Logistic Regression [PyTorch]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [MCLR_PyTorch.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/MCLR_PyTorch.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Multi-class Logistic Regression* using PyTorch library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import torch\n", 28 | "from torchvision import datasets, transforms" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "plt.rcParams['figure.figsize'] = (16, 8)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "train_set = datasets.MNIST(root='./mnist', train=True, transform=None, download=True)\n", 47 | "test_set = datasets.MNIST(root='./mnist', train=False, transform=None, download=True)\n", 48 | "X_train, Y_train = train_set.train_data.numpy(), train_set.train_labels.numpy()\n", 49 | "X_test, Y_test = test_set.test_data.numpy(), test_set.test_labels.numpy()\n", 50 | "\n", 51 | "# one hot for multi-class classification\n", 52 | "# MNIST = 10 classes [0-9]\n", 53 | "y_train_oh = np.zeros((Y_train.size, 10))\n", 54 | "y_train_oh[np.arange(Y_train.size), Y_train] = 1\n", 55 | "y_test_oh = np.zeros((Y_test.size, 10))\n", 56 | "y_test_oh[np.arange(Y_test.size), Y_test] = 1\n", 57 | "\n", 58 | "print('X_train:', X_train.shape)\n", 59 | "print('Y_train:', Y_train.shape)\n", 60 | "print('y_train_oh:', y_train_oh.shape)\n", 61 | "print('X_test:', X_test.shape)\n", 62 | "print('Y_test:', Y_test.shape)\n", 63 | "print('y_test_oh:', y_test_oh.shape)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 73 | "\n", 74 | "np.random.seed(1234)\n", 75 | "for ax in AX.ravel():\n", 76 | " rindex = np.random.randint(Y_train.size)\n", 77 | " ax.imshow(X_train[rindex])\n", 78 | " # title label + one-hot\n", 79 | " title = '{} :: '.format(Y_train[rindex]) \n", 80 | " title += ''.join([str(int(e)) for e in y_train_oh[rindex]]) \n", 81 | " ax.set_title(title)\n", 82 | "plt.grid(False)" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 92 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 93 | "\n", 94 | "x_train = x_train.astype(np.float32)\n", 95 | "y_train = y_train.astype(np.float32)\n", 96 | "x_test = x_test.astype(np.float32)\n", 97 | "y_test = y_test.astype(np.float32)\n", 98 | "\n", 99 | "print('x_train:', x_train.shape)\n", 100 | "print('y_train:', y_train.shape)\n", 101 | "print('x_test:', x_test.shape)\n", 102 | "print('y_test:', y_test.shape)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "# reshape Xs\n", 112 | "x_train = x_train.reshape(-1, 28*28)\n", 113 | "x_test = x_test.reshape(-1, 28*28)\n", 114 | "\n", 115 | "print('x_train:', x_train.shape)\n", 116 | "print('x_test:', x_test.shape)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "## Algorithm\n", 124 | "---\n", 125 | "Multi-class logistic regression\n", 126 | "![mclr](sourceimages/MCLR.png \"Multi-Class Logistic Regression\")" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "def CrossEntropyLoss_oh(pred, labels):\n", 136 | " return -torch.mean(torch.sum(labels*torch.log(pred), dim=1))\n", 137 | "\n", 138 | "EPOCHS = 5000 # epochs\n", 139 | "ALPHA = 0.005 # learning rate\n", 140 | "BATCH = 100 # batch size\n", 141 | "\n", 142 | "# m is the number of examples\n", 143 | "# n_x is the input size 28x28=784\n", 144 | "m, n_x = x_train.shape\n", 145 | "\n", 146 | "# model\n", 147 | "Z = torch.nn.Linear(n_x, 10, bias=True)\n", 148 | "torch.nn.init.zeros_(Z.weight)\n", 149 | "A = torch.nn.Softmax(dim=1)\n", 150 | "\n", 151 | "# training graph and optimization\n", 152 | "optimizer = torch.optim.SGD(Z.parameters(), lr=ALPHA)\n", 153 | "\n", 154 | "# loss and accuracy storage\n", 155 | "loss_plot = []; accA_plot = []\n", 156 | "\n", 157 | "for epoch in range(EPOCHS + 1):\n", 158 | " # randomic batch definition\n", 159 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 160 | " # variables initialization\n", 161 | " X = torch.autograd.Variable(torch.FloatTensor(x_train[rbatch]))\n", 162 | " Y = torch.autograd.Variable(torch.FloatTensor(y_train_oh[rbatch]))\n", 163 | " # training, metrics and storage\n", 164 | " optimizer.zero_grad()\n", 165 | " L = CrossEntropyLoss_oh(A(Z(X)), Y)\n", 166 | " L.backward()\n", 167 | " optimizer.step()\n", 168 | "\n", 169 | " X_ = torch.autograd.Variable(torch.FloatTensor(x_test))\n", 170 | " Y_ = torch.autograd.Variable(torch.FloatTensor(y_test_oh))\n", 171 | "\n", 172 | " acc = torch.mean(1 - torch.abs(Y_ - A(Z(X_))))*100\n", 173 | " \n", 174 | " loss_plot += [L]; accA_plot += [acc]\n", 175 | " if (not epoch % 1000) and (epoch != 0):\n", 176 | " print('epoch: {0:04d} | loss: {1:.3f} | accuracy: {2:06.2f} %'.format(epoch, L, acc))\n", 177 | "W_ = Z.weight.detach().numpy()\n", 178 | "B_ = Z.bias.detach().numpy()" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "fig, AX = plt.subplots(1, 10, sharey=True)\n", 188 | "\n", 189 | "for i in range(10):\n", 190 | " AX[i].imshow(W_[i].reshape(28, 28))\n", 191 | " AX[i].set_title(r'$W_{}$'.format(i))" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "fig, [axA, axB] = plt.subplots(2, 1, sharex=True)\n", 201 | "\n", 202 | "axA.plot(loss_plot)\n", 203 | "axA.set_ylabel('loss')\n", 204 | "axB.plot(accA_plot)\n", 205 | "axB.set_ylabel('accuracy')\n", 206 | "\n", 207 | "plt.xlabel('epochs')\n", 208 | "\n", 209 | "plt.show()" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [ 218 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 219 | "AX = [b for a in AX for b in a]\n", 220 | "\n", 221 | "np.random.seed(1)\n", 222 | "for ax in AX:\n", 223 | " index = np.random.randint(y_test.size)\n", 224 | " Z_ = np.dot(W_, x_test[index]) + B_\n", 225 | " Y_ = np.argmax(Z_)\n", 226 | " if Y_ == y_test[index]:\n", 227 | " ax.imshow(x_test[index].reshape(28, 28))\n", 228 | " else:\n", 229 | " ax.imshow(1 - x_test[index].reshape(28, 28))\n", 230 | " ez = np.exp(Z_ - Z_.max())\n", 231 | " A_ = ez/ez.sum(); A_ = float(A_.T[Y_])\n", 232 | " ax.set_title(r'$\\hat{Y_i}$ = ' + str(Y_) + ' ; $A_i$ = {:.03f}'.format(A_), fontsize=20)" 233 | ] 234 | } 235 | ], 236 | "metadata": { 237 | "kernelspec": { 238 | "display_name": "Python 3", 239 | "language": "python", 240 | "name": "python3" 241 | }, 242 | "language_info": { 243 | "codemirror_mode": { 244 | "name": "ipython", 245 | "version": 3 246 | }, 247 | "file_extension": ".py", 248 | "mimetype": "text/x-python", 249 | "name": "python", 250 | "nbconvert_exporter": "python", 251 | "pygments_lexer": "ipython3", 252 | "version": "3.7.7" 253 | } 254 | }, 255 | "nbformat": 4, 256 | "nbformat_minor": 2 257 | } 258 | -------------------------------------------------------------------------------- /Deep-Learning-Models/MCLR_TensorFlow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multi-class Logistic Regression [TensorFlow]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [MCLR_TensorFlow.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/MCLR_TensorFlow.ipynb)\n", 12 | "---\n", 13 | "Implementation of Multi-class Logistic Regression using TensorFlow library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import tensorflow as tf" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "plt.rcParams['figure.figsize'] = (16, 8)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "mnist = tf.keras.datasets.mnist\n", 46 | "[X_train, Y_train],[X_test, Y_test] = mnist.load_data()\n", 47 | "\n", 48 | "# one hot for multi-class classification\n", 49 | "# MNIST = 10 classes [0-9]\n", 50 | "y_train_oh = np.zeros((Y_train.size, 10))\n", 51 | "y_train_oh[np.arange(Y_train.size), Y_train] = 1\n", 52 | "y_test_oh = np.zeros((Y_test.size, 10))\n", 53 | "y_test_oh[np.arange(Y_test.size), Y_test] = 1\n", 54 | "\n", 55 | "print('X_train:', X_train.shape)\n", 56 | "print('Y_train:', Y_train.shape)\n", 57 | "print('y_train_oh:', y_train_oh.shape)\n", 58 | "print('X_test:', X_test.shape)\n", 59 | "print('Y_test:', Y_test.shape)\n", 60 | "print('y_test_oh:', y_test_oh.shape)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 70 | "\n", 71 | "np.random.seed(1234)\n", 72 | "for ax in AX.ravel():\n", 73 | " rindex = np.random.randint(Y_train.size)\n", 74 | " ax.imshow(X_train[rindex])\n", 75 | " # title label + one-hot\n", 76 | " title = '{} :: '.format(Y_train[rindex]) \n", 77 | " title += ''.join([str(int(e)) for e in y_train_oh[rindex]]) \n", 78 | " ax.set_title(title)\n", 79 | "plt.grid(False)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 89 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 90 | "\n", 91 | "x_train = x_train.astype(np.float32)\n", 92 | "y_train = y_train.astype(np.float32)\n", 93 | "x_test = x_test.astype(np.float32)\n", 94 | "y_test = y_test.astype(np.float32)\n", 95 | "\n", 96 | "print('x_train:', x_train.shape)\n", 97 | "print('y_train:', y_train.shape)\n", 98 | "print('x_test:', x_test.shape)\n", 99 | "print('y_test:', y_test.shape)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "# reshape Xs\n", 109 | "x_train = x_train.reshape(-1, 28*28)\n", 110 | "x_test = x_test.reshape(-1, 28*28)\n", 111 | "\n", 112 | "print('x_train:', x_train.shape)\n", 113 | "print('x_test:', x_test.shape)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "## Algorithm\n", 121 | "---\n", 122 | "Multi-class logistic regression\n", 123 | "![mclr](sourceimages/MCLR.png \"Multi-Class Logistic Regression\")" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "EPOCHS = 5000 # epochs\n", 133 | "ALPHA = 0.005 # learning rate\n", 134 | "BATCH = 100 # batch size\n", 135 | "\n", 136 | "# m is the number of examples\n", 137 | "# n_x is the input size 28x28=784\n", 138 | "m, n_x = x_train.shape\n", 139 | "\n", 140 | "X = tf.placeholder(tf.float32, shape=[None, n_x], name='X')\n", 141 | "Y = tf.placeholder(tf.float32, shape=[None, 10], name='Y')\n", 142 | "\n", 143 | "# variables initialization\n", 144 | "W = tf.Variable(tf.zeros([n_x, 10]), tf.float32, name='W')\n", 145 | "B = tf.Variable(tf.zeros([1, 10]), tf.float32, name='B')\n", 146 | "\n", 147 | "init_variables = tf.global_variables_initializer()\n", 148 | "\n", 149 | "# model\n", 150 | "Z = tf.add(tf.matmul(X, W), B)\n", 151 | "A = tf.nn.softmax(Z)\n", 152 | "\n", 153 | "# training graph and optimization\n", 154 | "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=A, labels=Y))\n", 155 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=ALPHA).minimize(loss)\n", 156 | "accuracy = tf.reduce_mean(1 - tf.abs(Y - A))*100\n", 157 | "\n", 158 | "# loss and accuracy storage\n", 159 | "loss_plot = []; accA_plot = []\n", 160 | "\n", 161 | "with tf.Session() as sess:\n", 162 | " sess.run(init_variables)\n", 163 | " for epoch in range(EPOCHS + 1):\n", 164 | " # randomic batch definition\n", 165 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 166 | " # training, metrics and storage\n", 167 | " sess.run(optimizer, feed_dict={X: x_train[rbatch], Y: y_train_oh[rbatch]})\n", 168 | " L = sess.run(loss, feed_dict={X: x_train[rbatch], Y: y_train_oh[rbatch]})\n", 169 | " acc = sess.run(accuracy, feed_dict={X: x_test, Y: y_test_oh})\n", 170 | " loss_plot += [L]; accA_plot += [acc]\n", 171 | " if (not epoch % 1000) and (epoch != 0):\n", 172 | " print('epoch: {0:04d} | loss: {1:.3f} | accuracy: {2:06.2f} %'.format(epoch, L, acc))\n", 173 | " W_ = sess.run(W) # store W and B for visualization and test\n", 174 | " B_ = sess.run(B)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "fig, AX = plt.subplots(1, 10, sharey=True)\n", 184 | "\n", 185 | "for i in range(10):\n", 186 | " AX[i].imshow(W_.T[i].reshape(28, 28))\n", 187 | " AX[i].set_title(r'$W_{}$'.format(i))" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "fig, [axA, axB] = plt.subplots(2, 1, sharex=True)\n", 197 | "\n", 198 | "axA.plot(loss_plot)\n", 199 | "axA.set_ylabel('loss')\n", 200 | "axB.plot(accA_plot)\n", 201 | "axB.set_ylabel('accuracy')\n", 202 | "\n", 203 | "plt.xlabel('epochs')\n", 204 | "\n", 205 | "plt.show()" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [ 214 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 215 | "AX = [b for a in AX for b in a]\n", 216 | "\n", 217 | "np.random.seed(1)\n", 218 | "for ax in AX:\n", 219 | " index = np.random.randint(y_test.size)\n", 220 | " Z_ = np.dot(W_.T, x_test[index]) + B_\n", 221 | " Y_ = np.argmax(Z_)\n", 222 | " if Y_ == y_test[index]:\n", 223 | " ax.imshow(x_test[index].reshape(28, 28))\n", 224 | " else:\n", 225 | " ax.imshow(1 - x_test[index].reshape(28, 28))\n", 226 | " ez = np.exp(Z_ - Z_.max())\n", 227 | " A_ = ez/ez.sum(); A_ = float(A_.T[Y_])\n", 228 | " ax.set_title(r'$\\hat{Y_i}$ = ' + str(Y_) + ' ; $A_i$ = {:.03f}'.format(A_), fontsize=20)" 229 | ] 230 | } 231 | ], 232 | "metadata": { 233 | "kernelspec": { 234 | "display_name": "Python 3", 235 | "language": "python", 236 | "name": "python3" 237 | }, 238 | "language_info": { 239 | "codemirror_mode": { 240 | "name": "ipython", 241 | "version": 3 242 | }, 243 | "file_extension": ".py", 244 | "mimetype": "text/x-python", 245 | "name": "python", 246 | "nbconvert_exporter": "python", 247 | "pygments_lexer": "ipython3", 248 | "version": "3.7.7" 249 | } 250 | }, 251 | "nbformat": 4, 252 | "nbformat_minor": 2 253 | } 254 | -------------------------------------------------------------------------------- /Deep-Learning-Models/README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning Models 2 | 3 | Demonstration and practice of the most popular Deep Learning models, using *python* and *notebooks*. -------------------------------------------------------------------------------- /Deep-Learning-Models/basics_PyTorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Basics [PyTorch]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [MCLR_PyTorch.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/MCLR_PyTorch.ipynb)\n", 12 | "---\n", 13 | "Basic functions and operations using PyTorch library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Instalation\n", 30 | "---\n", 31 | "``` python\n", 32 | ">>> conda install -c pytorch pytorch\n", 33 | "```\n", 34 | "or\n", 35 | "``` python\n", 36 | ">>> pip install pytorch\n", 37 | "```" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "import torch" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "# un-initialized tensor\n", 56 | "a = torch.Tensor(3, 4)\n", 57 | "# initialized tensor\n", 58 | "b = torch.Tensor(np.arange(12).reshape(3, 4))\n", 59 | "\n", 60 | "print(a, a.shape, a.dtype)\n", 61 | "print(b, b.shape, b.dtype)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "## Sessions Objects and Output\n", 69 | "---" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "a = torch.Tensor(np.arange(4))\n", 79 | "b = torch.Tensor(np.arange(4)*10)\n", 80 | "c = torch.add(a, b)\n", 81 | "print(c)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "out = c.numpy()\n", 91 | "\n", 92 | "print(out)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "out = torch.Tensor(a.shape)\n", 102 | "torch.add(a, b, out=out)\n", 103 | "out = out.numpy()\n", 104 | "\n", 105 | "print(out)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "## Basic Operations\n", 113 | "---" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "s = 3 # scalar\n", 123 | "v = torch.Tensor(np.arange(5)) # vactor n=5\n", 124 | "m = torch.Tensor(np.arange(20).reshape(4, 5)) # matrix 4 x 5" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "# print S\n", 134 | "print(' S:', s)\n", 135 | "\n", 136 | "# V x S product\n", 137 | "vs = v*s\n", 138 | "print('V x S:', vs.numpy())\n", 139 | "\n", 140 | "# M x S product\n", 141 | "ms = m*s\n", 142 | "print('M x S:\\n', ms.numpy())" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "# print V\n", 152 | "print(' V:', v.numpy())\n", 153 | "\n", 154 | "# V x V > dot product\n", 155 | "vv = torch.dot(v, v)\n", 156 | "print('V x V:', vv.numpy())\n", 157 | "\n", 158 | "# M x V product\n", 159 | "mv = torch.mv(m, v)\n", 160 | "print('M x V:', mv.numpy())" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "# print M\n", 170 | "print(' M:\\n', m.numpy())\n", 171 | "\n", 172 | "# M x M product\n", 173 | "m1 = torch.Tensor(np.arange(15).reshape(5, 3)) # the M1's dimensions are 5 x 3\n", 174 | "mm = torch.mm(m, m1) # the result dimensions must be 4 x 3\n", 175 | "\n", 176 | "print('\\nM x M1:\\n', mm.numpy())" 177 | ] 178 | } 179 | ], 180 | "metadata": { 181 | "kernelspec": { 182 | "display_name": "Python 3", 183 | "language": "python", 184 | "name": "python3" 185 | }, 186 | "language_info": { 187 | "codemirror_mode": { 188 | "name": "ipython", 189 | "version": 3 190 | }, 191 | "file_extension": ".py", 192 | "mimetype": "text/x-python", 193 | "name": "python", 194 | "nbconvert_exporter": "python", 195 | "pygments_lexer": "ipython3", 196 | "version": "3.7.7" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 2 201 | } 202 | -------------------------------------------------------------------------------- /Deep-Learning-Models/basics_TensorFlow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Basics [TensorFlow]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [basics_TensorFlow.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/basics_TensorFlow.ipynb)\n", 12 | "---\n", 13 | "Basic functions and operations using TensorFlow library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Instalation\n", 30 | "---\n", 31 | "``` python\n", 32 | ">>> conda install -c conda-forge tensorflow\n", 33 | "```\n", 34 | "or\n", 35 | "``` python\n", 36 | ">>> pip install tensorflow\n", 37 | "```" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "import tensorflow as tf" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "# un-initialized tensor\n", 56 | "a = tf.placeholder(tf.float32, shape=(3, 4), name='a')\n", 57 | "# initialized tensor\n", 58 | "b = tf.constant(np.arange(12).reshape(3, 4), name='b')\n", 59 | "\n", 60 | "print(a, a.shape, a.dtype)\n", 61 | "print(b, b.shape, b.dtype)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "## Sessions Objects and Output\n", 69 | "---" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "a = tf.constant(np.arange(4), name='a')\n", 79 | "b = tf.constant(np.arange(4)*10, name='b')\n", 80 | "c = tf.add(a, b)\n", 81 | "print(c)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "# Iteractive session\n", 91 | "sess = tf.InteractiveSession()\n", 92 | "out = c.eval()\n", 93 | "sess.close()\n", 94 | "\n", 95 | "print(out)" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "# Non-Iteractive session\n", 105 | "sess = tf.Session()\n", 106 | "out = c.eval(session=sess)\n", 107 | "sess.close()\n", 108 | "\n", 109 | "print(out)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "# Non-Iteractive session using context manager\n", 119 | "with tf.Session() as sess:\n", 120 | " out1 = sess.run(c)\n", 121 | " out2 = c.eval()\n", 122 | "\n", 123 | "print(out1)\n", 124 | "print(out2)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "### Un-initialized tensor and feed dictionary" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "a = tf.placeholder(tf.int32, name='a')\n", 141 | "b = tf.placeholder(tf.int32, name='b')\n", 142 | "c = tf.add(a, b)\n", 143 | "\n", 144 | "with tf.Session() as sess:\n", 145 | " out = sess.run(c, feed_dict={a: 3, b: 4})\n", 146 | "\n", 147 | "print(out)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "## Basic Operations\n", 155 | "---" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "s = tf.constant(3) # scalar\n", 165 | "v = tf.constant(np.arange(5)) # vactor n=5\n", 166 | "m = tf.constant(np.arange(20).reshape(4, 5)) # matrix 4 x 5" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "# print S\n", 176 | "with tf.Session() as sess:\n", 177 | " print(' S:', sess.run(s))\n", 178 | "\n", 179 | "# V x S product\n", 180 | "vs = tf.multiply(v, s) # could be v*s as well\n", 181 | "with tf.Session() as sess:\n", 182 | " print('V x S:', sess.run(vs))\n", 183 | "\n", 184 | "# M x S product\n", 185 | "ms = tf.multiply(m, s) # could be m*s as well\n", 186 | "with tf.Session() as sess:\n", 187 | " print('M x S:\\n', sess.run(ms))" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "# print V\n", 197 | "with tf.Session() as sess:\n", 198 | " print(' V:', sess.run(v))\n", 199 | "\n", 200 | "# V x V > dot product\n", 201 | "vv = tf.tensordot(v, v, axes=1)\n", 202 | "with tf.Session() as sess:\n", 203 | " print('V x V:', sess.run(vv))\n", 204 | "\n", 205 | "# M x V product\n", 206 | "mv = tf.tensordot(m, v, axes=1)\n", 207 | "with tf.Session() as sess:\n", 208 | " print('M x V:', sess.run(mv))" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "# print M\n", 218 | "with tf.Session() as sess:\n", 219 | " print(' M:\\n', sess.run(m))\n", 220 | "\n", 221 | "# M x M product\n", 222 | "m1 = tf.constant(np.arange(15).reshape(5, 3)) # the M1's dimensions are 5 x 3\n", 223 | "mm = tf.matmul(m, m1) # the result dimensions must be 4 x 3\n", 224 | "\n", 225 | "with tf.Session() as sess:\n", 226 | " print('\\nM x M1:\\n', sess.run(mm))" 227 | ] 228 | } 229 | ], 230 | "metadata": { 231 | "kernelspec": { 232 | "display_name": "Python 3", 233 | "language": "python", 234 | "name": "python3" 235 | }, 236 | "language_info": { 237 | "codemirror_mode": { 238 | "name": "ipython", 239 | "version": 3 240 | }, 241 | "file_extension": ".py", 242 | "mimetype": "text/x-python", 243 | "name": "python", 244 | "nbconvert_exporter": "python", 245 | "pygments_lexer": "ipython3", 246 | "version": "3.7.7" 247 | } 248 | }, 249 | "nbformat": 4, 250 | "nbformat_minor": 2 251 | } 252 | -------------------------------------------------------------------------------- /Deep-Learning-Models/convolutional_neural_network_Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Convolutional Neural Network [Keras]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [convolutional_neural_network_Keras.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/convolutional_neural_network_Keras.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Convolutional Neural Network* using Keras library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "from PIL import Image\n", 27 | "\n", 28 | "import tensorflow as tf" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "plt.rcParams['figure.figsize'] = (20, 10)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "cifar10 = tf.keras.datasets.cifar10\n", 47 | "[X_train, Y_train],[X_test, Y_test] = cifar10.load_data()\n", 48 | "\n", 49 | "# Data preparation\n", 50 | "X_train, Y_train = X_train/255, Y_train\n", 51 | "X_test, Y_test = X_test/255, Y_test\n", 52 | "# Data augmentation [flip x]\n", 53 | "X_train = np.vstack([X_train, X_train[:,:,::-1,:]])\n", 54 | "Y_train = np.vstack([Y_train, Y_train])\n", 55 | "X_test = np.vstack([X_test, X_test[:,:,::-1,:]])\n", 56 | "Y_test = np.vstack([Y_test, Y_test])\n", 57 | "# one hot for multi-class classification\n", 58 | "# MNIST = 10 classes [0-9]\n", 59 | "Y_train_oh = np.zeros((Y_train.size, 10))\n", 60 | "Y_train_oh[np.arange(Y_train.size), Y_train.T] = 1\n", 61 | "Y_test_oh = np.zeros((Y_test.size, 10))\n", 62 | "Y_test_oh[np.arange(Y_test.size), Y_test.T] = 1\n", 63 | "\n", 64 | "print('X_train:', X_train.shape)\n", 65 | "print('Y_train:', Y_train.shape)\n", 66 | "print('Y_train_oh:', Y_train_oh.shape)\n", 67 | "print('X_test:', X_test.shape)\n", 68 | "print('Y_test:', Y_test.shape)\n", 69 | "print('Y_test_oh:', Y_test_oh.shape)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "# Label categories\n", 79 | "objects = ['Plane', 'Car', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n", 80 | "# Get dimensions\n", 81 | "N1, N2, C = X_train[0].shape" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 91 | "\n", 92 | "np.random.seed(1234)\n", 93 | "for ax in AX.ravel():\n", 94 | " rindex = np.random.randint(Y_train.size)\n", 95 | " ax.imshow(X_train[rindex])\n", 96 | " # title label + one-hot\n", 97 | " title = f'{objects[Y_train[rindex][0]]} {Y_train[rindex]} :: '\n", 98 | " title += ''.join([str(int(e)) for e in Y_train_oh[rindex]]) \n", 99 | " ax.set_title(title)\n", 100 | "plt.grid(False)" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "## Convolutional Neural Network\n", 108 | "---\n", 109 | "![convolutional_neural_network](sourceimages/convolutional_neural_network.png \"Convolutional Neural Network\")" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "# Model\n", 119 | "model = tf.keras.Sequential([\n", 120 | " tf.keras.layers.Input(X_train[0].shape),\n", 121 | " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", 122 | " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n", 123 | " tf.keras.layers.MaxPooling2D(pool_size=2),\n", 124 | " tf.keras.layers.Dropout(0.25),\n", 125 | " tf.keras.layers.Conv2D(128, 3, activation='relu'),\n", 126 | " tf.keras.layers.Conv2D(256, 3, activation='relu'),\n", 127 | " tf.keras.layers.MaxPooling2D(pool_size=2),\n", 128 | " tf.keras.layers.Dropout(0.25),\n", 129 | " tf.keras.layers.Flatten(),\n", 130 | " tf.keras.layers.Dense(N1*N2*C, activation='relu'),\n", 131 | " tf.keras.layers.Dropout(0.5),\n", 132 | " tf.keras.layers.Dense(10, activation='softmax')\n", 133 | "])\n", 134 | "\n", 135 | "model.summary()" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "model.compile(\n", 145 | " loss='categorical_crossentropy',\n", 146 | " optimizer='adam',\n", 147 | " metrics=['accuracy']\n", 148 | ")\n", 149 | "\n", 150 | "model.fit(X_train, Y_train_oh, epochs=10)" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "fig, AX = plt.subplots(4, 8, sharex=True, sharey=True)\n", 160 | "AX = [b for a in AX for b in a]\n", 161 | "\n", 162 | "pred = model.predict(X_test)\n", 163 | "\n", 164 | "np.random.seed(1234)\n", 165 | "for ax in AX:\n", 166 | " index = np.random.randint(Y_test.size)\n", 167 | " # Prediction\n", 168 | " A_ = pred[index]\n", 169 | " Y_ = np.argmax(A_)\n", 170 | " # Select sample image\n", 171 | " imshow = X_test[index]\n", 172 | " # Green square: predited correctly\n", 173 | " # Red square: predicted wrongly\n", 174 | " if Y_ == Y_test[index]:\n", 175 | " imshow[-4:, -4:] = (0, 1, 0)\n", 176 | " else:\n", 177 | " imshow[-4:, -4:] = (1, 0, 0)\n", 178 | " ax.imshow(imshow)\n", 179 | " ax.set_title(f'{objects[Y_]} ({Y_})' + r' ; $A_i$ = {:.02f}'.format(float(A_[Y_])), fontsize=12)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "print('Validation')\n", 189 | "pred1 = model.predict(X_test)\n", 190 | "print('Test:', np.sum((Y_test_oh - pred1)**2, axis=1).mean())\n", 191 | "pred2 = model.predict(X_train)\n", 192 | "print('Train:', np.sum((Y_train_oh - pred2)**2, axis=1).mean())" 193 | ] 194 | } 195 | ], 196 | "metadata": { 197 | "kernelspec": { 198 | "display_name": "Python [conda env:tfgpu]", 199 | "language": "python", 200 | "name": "conda-env-tfgpu-py" 201 | }, 202 | "language_info": { 203 | "codemirror_mode": { 204 | "name": "ipython", 205 | "version": 3 206 | }, 207 | "file_extension": ".py", 208 | "mimetype": "text/x-python", 209 | "name": "python", 210 | "nbconvert_exporter": "python", 211 | "pygments_lexer": "ipython3", 212 | "version": "3.7.7" 213 | } 214 | }, 215 | "nbformat": 4, 216 | "nbformat_minor": 2 217 | } 218 | -------------------------------------------------------------------------------- /Deep-Learning-Models/neural_network_deep_Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Neural Network [Keras]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [neural_network_deep_Keras.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/neural_network_deep_Keras.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Deep Neural Network* using Keras library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "from PIL import Image\n", 27 | "\n", 28 | "import tensorflow as tf" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "plt.rcParams['figure.figsize'] = (16, 8)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "n1, n2 = 128, 128\n", 47 | "# Read Image\n", 48 | "x = Image.open('sourceimages/mandril.png')\n", 49 | "# Rescale image to a lower resolution\n", 50 | "x = x.resize((n1, n2), Image.ANTIALIAS)\n", 51 | "x = np.asarray(x)/255\n", 52 | "n1, n2, c = x.shape\n", 53 | "\n", 54 | "### split channels ###\n", 55 | "r, g, b = x[:,:,0], x[:,:,1], x[:,:,2]" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "# Transform data and produce X_train\n", 65 | "Y_train = np.array([r.ravel(), g.ravel(), b.ravel()]).T\n", 66 | "t, s = np.mgrid[0:n1, 0:n2]\n", 67 | "s = (s - s.mean())/s.std()\n", 68 | "t = (t - t.mean())/t.std()\n", 69 | "# X_train is the normalized spatial coordinates\n", 70 | "X_train = np.array([s.ravel(), t.ravel()], dtype=np.float32).T" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "fig, [axA, axB] = plt.subplots(1, 2, figsize=(20, 10))\n", 80 | "\n", 81 | "st = np.stack([s, t, s*0], axis=2)\n", 82 | "st = (st - st.min())/(st.max() - st.min())\n", 83 | "st[:,:,2] = 0; axA.imshow(st); axA.axis('off')\n", 84 | "axA.text(5, 10, f'({s.min():.3f}, {t.min():.3f})', color='white', size=18)\n", 85 | "axA.text(86, 10, f'({s.max():.3f}, {t.min():.3f})', color='white', size=18)\n", 86 | "axA.text(5, 120, f'({s.min():.3f}, {t.max():.3f})', color='white', size=18)\n", 87 | "axA.text(86, 120, f'({s.max():.3f}, {t.max():.3f})', color='white', size=18)\n", 88 | "axA.set_title('X_Train')\n", 89 | "\n", 90 | "axB.imshow(x)\n", 91 | "axB.set_title('Y_Train')\n", 92 | "\n", 93 | "plt.show()" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "# Produce X_test to another scale\n", 103 | "# Upscale the image approximation\n", 104 | "N1 = N2 = 512\n", 105 | "t, s = np.mgrid[0:N1, 0:N2]\n", 106 | "s = (s - s.mean())/s.std()\n", 107 | "t = (t - t.mean())/t.std()\n", 108 | "X_test = np.array([s.ravel(), t.ravel()], dtype=np.float32).T\n", 109 | "\n", 110 | "print('X_train:', X_train.shape)\n", 111 | "print('Y_train:', Y_train.shape)\n", 112 | "print('X_test:', X_test.shape)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "## Deep Neural Network\n", 120 | "---\n", 121 | "![perceptron](sourceimages/neural_network_deep.png \"Deep Neural Network\")" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "EPOCHS = 100 # epochs\n", 131 | "\n", 132 | "# List of number of neurons for each hidden layer\n", 133 | "NEURONS = [32, 64, 128, 256, 512, 1024]\n", 134 | "\n", 135 | "# model \n", 136 | "# Input (2): x and y coordinates\n", 137 | "# Output (3): RGB\n", 138 | "model = tf.keras.Sequential([\n", 139 | " tf.keras.layers.Input(2),\n", 140 | " *[tf.keras.layers.Dense(n, activation='relu') for n in NEURONS],\n", 141 | " tf.keras.layers.Dense(3, activation='sigmoid')\n", 142 | "])\n", 143 | "\n", 144 | "model.compile(\n", 145 | " loss='mean_squared_error',\n", 146 | " optimizer='adam',\n", 147 | " metrics=['accuracy']\n", 148 | ")\n", 149 | "\n", 150 | "# loss and accuracy storage\n", 151 | "loss_plot = []; accA_plot = []\n", 152 | "\n", 153 | "for epoch in range(EPOCHS + 1):\n", 154 | " model.fit(X_train, Y_train, epochs=2, verbose=0)\n", 155 | " loss_plot += [e*100 for e in model.history.history['loss']]\n", 156 | " accA_plot += [e*100 for e in model.history.history['accuracy']]\n", 157 | " if (not epoch % 10) and (epoch != 0):\n", 158 | " print(f'epoch: {epoch:04d} | loss: {loss_plot[-1]:.3f} | accuracy: {accA_plot[-1]:06.2f} %')" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "fig, [axA, axB] = plt.subplots(2, 1, sharex=True)\n", 168 | "\n", 169 | "axA.plot(loss_plot)\n", 170 | "axA.set_ylabel('loss')\n", 171 | "axB.plot(accA_plot)\n", 172 | "axB.set_ylabel('accuracy')\n", 173 | "\n", 174 | "plt.xlabel('epochs')\n", 175 | "\n", 176 | "plt.show()" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [ 185 | "fig, [axA, axB] = plt.subplots(1, 2, figsize=(20, 10))\n", 186 | "\n", 187 | "axA.imshow(x)\n", 188 | "axA.set_title(r'$Y_{train}$', size=20)\n", 189 | "# Predict using X_train (128x128x3)\n", 190 | "Y_predA = model.predict(X_train)\n", 191 | "Y_predA = Y_predA.reshape(n1, n2, c)\n", 192 | "axB.imshow(Y_predA)\n", 193 | "axB.set_title(r'$\\hat{Y}_{train}$', size=20)\n", 194 | "\n", 195 | "plt.show()" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "fig, [axA, axB] = plt.subplots(1, 2, figsize=(20, 10))\n", 205 | "\n", 206 | "axA.imshow(x)\n", 207 | "axA.set_title(r'$Y_{train}$', size=20)\n", 208 | "# Predict using X_test (512x512x3)\n", 209 | "Y_predB = model.predict(X_test)\n", 210 | "Y_predB = Y_predB.reshape(N1, N2, c)\n", 211 | "axB.imshow(Y_predB)\n", 212 | "axB.set_title(r'$\\hat{Y}_{test}$', size=20)\n", 213 | "\n", 214 | "plt.show()" 215 | ] 216 | } 217 | ], 218 | "metadata": { 219 | "kernelspec": { 220 | "display_name": "Python 3", 221 | "language": "python", 222 | "name": "python3" 223 | }, 224 | "language_info": { 225 | "codemirror_mode": { 226 | "name": "ipython", 227 | "version": 3 228 | }, 229 | "file_extension": ".py", 230 | "mimetype": "text/x-python", 231 | "name": "python", 232 | "nbconvert_exporter": "python", 233 | "pygments_lexer": "ipython3", 234 | "version": "3.7.7" 235 | } 236 | }, 237 | "nbformat": 4, 238 | "nbformat_minor": 2 239 | } 240 | -------------------------------------------------------------------------------- /Deep-Learning-Models/neural_network_shallow_Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Shallow Neural Network [Keras]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [neural_network_shallow_Keras.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/neural_network_shallow_Keras.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Shallow Neural Network* using Keras library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import tensorflow as tf" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "plt.rcParams['figure.figsize'] = (16, 8)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "cifar10 = tf.keras.datasets.cifar10\n", 46 | "[X_train, Y_train],[X_test, Y_test] = cifar10.load_data()\n", 47 | "\n", 48 | "# data cleaning\n", 49 | "# using only zeros and ones\n", 50 | "X_train = X_train[Y_train.ravel() <= 1]\n", 51 | "Y_train = Y_train[Y_train <= 1]\n", 52 | "X_test = X_test[Y_test.ravel() <= 1]\n", 53 | "Y_test = Y_test[Y_test <= 1]\n", 54 | "\n", 55 | "print('X_train:', X_train.shape)\n", 56 | "print('Y_train:', Y_train.shape)\n", 57 | "print('X_test:', X_test.shape)\n", 58 | "print('Y_test:', Y_test.shape)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "# Label categories\n", 68 | "objects = ['Plane', 'Car']\n", 69 | "# Get dimensions\n", 70 | "N1, N2, C = X_train[0].shape" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 80 | "\n", 81 | "np.random.seed(1234)\n", 82 | "for ax in AX.ravel():\n", 83 | " rindex = np.random.randint(Y_train.size)\n", 84 | " ax.imshow(X_train[rindex])\n", 85 | " label = Y_train[rindex]\n", 86 | " ax.set_title(f'label: {objects[label]} ({label})')\n", 87 | "plt.grid(False)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "# data preparation\n", 97 | "# scales, dimensions and dtypes\n", 98 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 99 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 100 | "\n", 101 | "x_train = x_train.astype(np.float32).reshape(-1, N1*N2*C)\n", 102 | "y_train = y_train.astype(np.float32)\n", 103 | "x_test = x_test.astype(np.float32).reshape(-1, N1*N2*C)\n", 104 | "y_test = y_test.astype(np.float32)\n", 105 | "\n", 106 | "print('x_train:', x_train.shape)\n", 107 | "print('y_train:', y_train.shape)\n", 108 | "print('x_test:', x_test.shape)\n", 109 | "print('y_test:', y_test.shape)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "## Shallow Neural Network\n", 117 | "---\n", 118 | "![perceptron](sourceimages/neural_network_shallow.png \"Shallow Neural Network\")" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "EPOCHS = 500 # epochs\n", 128 | "BATCH = 1000 # batch size\n", 129 | "\n", 130 | "# Number of neurons on the hidden layer\n", 131 | "neurons = 32\n", 132 | "\n", 133 | "# m is the number of examples\n", 134 | "# n_x is the input size 28x28=784\n", 135 | "m, n_x = x_train.shape\n", 136 | "\n", 137 | "# model\n", 138 | "model = tf.keras.Sequential([\n", 139 | " tf.keras.layers.Input(n_x),\n", 140 | " tf.keras.layers.Dense(neurons, activation='relu'),\n", 141 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 142 | "])\n", 143 | "\n", 144 | "# Compile model\n", 145 | "model.compile(\n", 146 | " loss='mean_squared_error',\n", 147 | " optimizer='adam',\n", 148 | " metrics=['accuracy']\n", 149 | ")\n", 150 | "\n", 151 | "# loss and accuracy storage\n", 152 | "loss_plot = []; accA_plot = []\n", 153 | "\n", 154 | "for epoch in range(EPOCHS + 1):\n", 155 | " # randomic batch definition\n", 156 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 157 | " # training, metrics and storage\n", 158 | " model.fit(x_train[rbatch], y_train[rbatch], epochs=1, verbose=0)\n", 159 | " loss_plot += [e*100 for e in model.history.history['loss']]\n", 160 | " accA_plot += [e*100 for e in model.history.history['accuracy']]\n", 161 | " if (not epoch % 100) and (epoch != 0):\n", 162 | " print(f'epoch: {epoch:04d} | loss: {loss_plot[-1]:.3f} | accuracy: {accA_plot[-1]:06.2f} %')\n", 163 | "\n", 164 | "# store W and B for visualization and test\n", 165 | "w_ = model.weights[0].numpy()" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "fig, [axA, axB] = plt.subplots(2, 1, sharex=True)\n", 175 | "\n", 176 | "axA.plot(loss_plot)\n", 177 | "axA.set_ylabel('loss')\n", 178 | "axB.plot(accA_plot)\n", 179 | "axB.set_ylabel('accuracy')\n", 180 | "\n", 181 | "plt.xlabel('epochs')\n", 182 | "\n", 183 | "plt.show()" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 193 | "AX = [b for a in AX for b in a]\n", 194 | "\n", 195 | "pred = model.predict(x_test)\n", 196 | "\n", 197 | "np.random.seed(1)\n", 198 | "for ax in AX:\n", 199 | " index = np.random.randint(y_test.size)\n", 200 | " a_ = pred[index]\n", 201 | " y_ = 1 if a_ > 0.5 else 0\n", 202 | " imshow = x_test[index].reshape(N1, N2, C)\n", 203 | " if y_ == y_test[index]:\n", 204 | " imshow[-4:, -4:] = (0, 1, 0)\n", 205 | " ax.imshow(imshow)\n", 206 | " else:\n", 207 | " imshow[-4:, -4:] = (1, 0, 0)\n", 208 | " ax.imshow(imshow)\n", 209 | " ax.set_title(f'{objects[y_]} ({y_})' + r' ; $a_i$ = {:.02f}'.format(float(a_)), fontsize=20)" 210 | ] 211 | } 212 | ], 213 | "metadata": { 214 | "kernelspec": { 215 | "display_name": "Python 3", 216 | "language": "python", 217 | "name": "python3" 218 | }, 219 | "language_info": { 220 | "codemirror_mode": { 221 | "name": "ipython", 222 | "version": 3 223 | }, 224 | "file_extension": ".py", 225 | "mimetype": "text/x-python", 226 | "name": "python", 227 | "nbconvert_exporter": "python", 228 | "pygments_lexer": "ipython3", 229 | "version": "3.7.7" 230 | } 231 | }, 232 | "nbformat": 4, 233 | "nbformat_minor": 2 234 | } 235 | -------------------------------------------------------------------------------- /Deep-Learning-Models/perceptron_Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Perceptron [Keras]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [perceptron_Keras.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/perceptron_Keras.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Perceptron* model using using Keras library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import tensorflow as tf" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "plt.rcParams['figure.figsize'] = (16, 8)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "mnist = tf.keras.datasets.mnist\n", 46 | "[X_train, Y_train],[X_test, Y_test] = mnist.load_data()\n", 47 | "\n", 48 | "# data cleaning\n", 49 | "# using only zeros and ones\n", 50 | "X_train = X_train[Y_train <= 1]\n", 51 | "Y_train = Y_train[Y_train <= 1]\n", 52 | "X_test = X_test[Y_test <= 1]\n", 53 | "Y_test = Y_test[Y_test <= 1]\n", 54 | "\n", 55 | "print('X_train:', X_train.shape)\n", 56 | "print('Y_train:', Y_train.shape)\n", 57 | "print('X_test:', X_test.shape)\n", 58 | "print('Y_test:', Y_test.shape)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 68 | "\n", 69 | "np.random.seed(1234)\n", 70 | "for ax in AX.ravel():\n", 71 | " rindex = np.random.randint(Y_train.size)\n", 72 | " ax.imshow(X_train[rindex])\n", 73 | " ax.set_title('label: {}'.format(Y_train[rindex]))\n", 74 | "plt.grid(False)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "# data preparation\n", 84 | "# scales, dimensions and dtypes\n", 85 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 86 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 87 | "\n", 88 | "x_train = x_train.astype(np.float32).reshape(-1, 28*28)\n", 89 | "y_train = y_train.astype(np.float32)\n", 90 | "x_test = x_test.astype(np.float32).reshape(-1, 28*28)\n", 91 | "y_test = y_test.astype(np.float32)\n", 92 | "\n", 93 | "print('x_train:', x_train.shape)\n", 94 | "print('y_train:', y_train.shape)\n", 95 | "print('x_test:', x_test.shape)\n", 96 | "print('y_test:', y_test.shape)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "## Perceptron\n", 104 | "---\n", 105 | "![perceptron](sourceimages/perceptron.png \"Perceptron\")" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "EPOCHS = 500 # epochs\n", 115 | "ALPHA = 0.001 # learning rate\n", 116 | "BATCH = 100 # batch size\n", 117 | "\n", 118 | "# m is the number of examples\n", 119 | "# n_x is the input size 28x28=784\n", 120 | "m, n_x = x_train.shape\n", 121 | "\n", 122 | "# model\n", 123 | "model = tf.keras.Sequential([\n", 124 | " tf.keras.layers.Input(n_x),\n", 125 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 126 | "])\n", 127 | "\n", 128 | "# Compile model\n", 129 | "model.compile(\n", 130 | " loss='mean_squared_error',\n", 131 | " optimizer='SGD',\n", 132 | " lr=ALPHA,\n", 133 | " metrics=['accuracy']\n", 134 | ")\n", 135 | "\n", 136 | "# loss and accuracy storage\n", 137 | "loss_plot = []; accA_plot = []\n", 138 | "\n", 139 | "for epoch in range(EPOCHS + 1):\n", 140 | " # randomic batch definition\n", 141 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 142 | " # training, metrics and storage\n", 143 | " model.fit(x_train[rbatch], y_train[rbatch], epochs=1, verbose=0)\n", 144 | " loss_plot += [e*100 for e in model.history.history['loss']]\n", 145 | " accA_plot += [e*100 for e in model.history.history['accuracy']]\n", 146 | " if (not epoch % 100) and (epoch != 0):\n", 147 | " print(f'epoch: {epoch:04d} | loss: {loss_plot[-1]:.3f} | accuracy: {accA_plot[-1]:06.2f} %')\n", 148 | "\n", 149 | "w_ = model.weights[0].numpy() # store W and B for visualization and test" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "axA = plt.subplot(121)\n", 159 | "axA.imshow(w_.T.reshape(28, 28))\n", 160 | "cb = axA.set_title('W')\n", 161 | "\n", 162 | "axB = plt.subplot(222)\n", 163 | "axB.plot(loss_plot)\n", 164 | "axB.set_ylabel('loss')\n", 165 | "\n", 166 | "axC = plt.subplot(224)\n", 167 | "axC.plot(accA_plot)\n", 168 | "axC.set_ylabel('accuracy')\n", 169 | "\n", 170 | "plt.xlabel('epochs')\n", 171 | "\n", 172 | "plt.show()" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 182 | "AX = [b for a in AX for b in a]\n", 183 | "\n", 184 | "pred = model.predict(x_test)\n", 185 | "\n", 186 | "np.random.seed(1)\n", 187 | "for ax in AX:\n", 188 | " index = np.random.randint(y_test.size)\n", 189 | " a_ = pred[index]\n", 190 | " y_ = 1 if a_ > 0.5 else 0\n", 191 | " if y_ == y_test[index]:\n", 192 | " ax.imshow(x_test[index].reshape(28, 28))\n", 193 | " else:\n", 194 | " ax.imshow(1 - x_test[index].reshape(28, 28))\n", 195 | " ax.set_title(r'$\\hat{y_i}$ = ' + str(y_) + r' ; $a_i$ = {:.02f}'.format(float(a_)), fontsize=20)" 196 | ] 197 | } 198 | ], 199 | "metadata": { 200 | "kernelspec": { 201 | "display_name": "Python 3", 202 | "language": "python", 203 | "name": "python3" 204 | }, 205 | "language_info": { 206 | "codemirror_mode": { 207 | "name": "ipython", 208 | "version": 3 209 | }, 210 | "file_extension": ".py", 211 | "mimetype": "text/x-python", 212 | "name": "python", 213 | "nbconvert_exporter": "python", 214 | "pygments_lexer": "ipython3", 215 | "version": "3.7.7" 216 | } 217 | }, 218 | "nbformat": 4, 219 | "nbformat_minor": 2 220 | } 221 | -------------------------------------------------------------------------------- /Deep-Learning-Models/perceptron_PyTorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Perceptron [PyTorch]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [perceptron_PyTorch.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/perceptron_PyTorch.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Perceptron* model using using PyTorch library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import torch\n", 28 | "from torchvision import datasets, transforms" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "plt.rcParams['figure.figsize'] = (16, 8)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "train_set = datasets.MNIST(root='./mnist', train=True, transform=None, download=True)\n", 47 | "test_set = datasets.MNIST(root='./mnist', train=False, transform=None, download=True)\n", 48 | "X_train, Y_train = train_set.train_data.numpy(), train_set.train_labels.numpy()\n", 49 | "X_test, Y_test = test_set.test_data.numpy(), test_set.test_labels.numpy()\n", 50 | "\n", 51 | "# data cleaning\n", 52 | "# using only zeros and ones\n", 53 | "X_train = X_train[Y_train <= 1]\n", 54 | "Y_train = Y_train[Y_train <= 1]\n", 55 | "X_test = X_test[Y_test <= 1]\n", 56 | "Y_test = Y_test[Y_test <= 1]\n", 57 | "\n", 58 | "print('X_train:', X_train.shape)\n", 59 | "print('Y_train:', Y_train.shape)\n", 60 | "print('X_test:', X_test.shape)\n", 61 | "print('Y_test:', Y_test.shape)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 71 | "\n", 72 | "np.random.seed(1234)\n", 73 | "for ax in AX.ravel():\n", 74 | " rindex = np.random.randint(Y_train.size)\n", 75 | " ax.imshow(X_train[rindex])\n", 76 | " ax.set_title('label: {}'.format(Y_train[rindex]))\n", 77 | "plt.grid(False)" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "# data preparation\n", 87 | "# scales, dimensions and dtypes\n", 88 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 89 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 90 | "\n", 91 | "x_train = x_train.astype(np.float32).reshape(-1, 28*28)\n", 92 | "y_train = y_train.astype(np.float32)\n", 93 | "x_test = x_test.astype(np.float32).reshape(-1, 28*28)\n", 94 | "y_test = y_test.astype(np.float32)\n", 95 | "\n", 96 | "print('x_train:', x_train.shape)\n", 97 | "print('y_train:', y_train.shape)\n", 98 | "print('x_test:', x_test.shape)\n", 99 | "print('y_test:', y_test.shape)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "## Perceptron\n", 107 | "---\n", 108 | "![perceptron](sourceimages/perceptron.png \"Perceptron\")" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "EPOCHS = 500 # epochs\n", 118 | "ALPHA = 0.001 # learning rate\n", 119 | "BATCH = 100 # batch size\n", 120 | "\n", 121 | "# m is the number of examples\n", 122 | "# n_x is the input size 28x28=784\n", 123 | "m, n_x = x_train.shape\n", 124 | "\n", 125 | "# model\n", 126 | "Z = torch.nn.Linear(n_x, 1, bias=True)\n", 127 | "torch.nn.init.zeros_(Z.weight)\n", 128 | "\n", 129 | "# training graph and optimization\n", 130 | "loss = torch.nn.MSELoss()\n", 131 | "optimizer = torch.optim.SGD(Z.parameters(), lr=ALPHA)\n", 132 | "\n", 133 | "# loss and accuracy storage\n", 134 | "loss_plot = []; accA_plot = []\n", 135 | "\n", 136 | "for epoch in range(EPOCHS + 1):\n", 137 | " # randomic batch definition\n", 138 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 139 | " # variables initialization\n", 140 | " X = torch.autograd.Variable(torch.FloatTensor(x_train[rbatch]))\n", 141 | " Y = torch.autograd.Variable(torch.FloatTensor(y_train[rbatch]))\n", 142 | " # training, metrics and storage\n", 143 | " optimizer.zero_grad()\n", 144 | " L = loss(torch.sigmoid(Z(X)), Y)\n", 145 | " L.backward()\n", 146 | " optimizer.step()\n", 147 | " \n", 148 | " X_ = torch.autograd.Variable(torch.FloatTensor(x_test))\n", 149 | " Y_ = torch.autograd.Variable(torch.FloatTensor(y_test))\n", 150 | " prediction = torch.round(torch.sigmoid(Z(X_)))\n", 151 | " compare = torch.eq(prediction, Y_)\n", 152 | " cast = compare.float()\n", 153 | " acc = torch.mean(cast)*100\n", 154 | " \n", 155 | " loss_plot += [L.detach().numpy()]; accA_plot += [acc]\n", 156 | " if (not epoch % 100) and (epoch != 0):\n", 157 | " print('epoch: {0:04d} | loss: {1:.3f} | accuracy: {2:06.2f} %'.format(epoch, L, acc))\n", 158 | "w_ = Z.weight.detach().numpy()\n", 159 | "b_ = Z.bias.detach().numpy()" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "axA = plt.subplot(121)\n", 169 | "axA.imshow(w_.T.reshape(28, 28))\n", 170 | "cb = axA.set_title('W')\n", 171 | "\n", 172 | "axB = plt.subplot(222)\n", 173 | "axB.plot(loss_plot)\n", 174 | "axB.set_ylabel('loss')\n", 175 | "\n", 176 | "axC = plt.subplot(224)\n", 177 | "axC.plot(accA_plot)\n", 178 | "axC.set_ylabel('accuracy')\n", 179 | "\n", 180 | "plt.xlabel('epochs')\n", 181 | "\n", 182 | "plt.show()" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 192 | "AX = [b for a in AX for b in a]\n", 193 | "\n", 194 | "np.random.seed(1)\n", 195 | "for ax in AX:\n", 196 | " index = np.random.randint(y_test.size)\n", 197 | " z_ = np.dot(w_, x_test[index]) + b_\n", 198 | " a_ = 1/(1 + np.exp(-z_))\n", 199 | " y_ = 1 if a_ > 0.5 else 0\n", 200 | " if y_ == y_test[index]:\n", 201 | " ax.imshow(x_test[index].reshape(28, 28))\n", 202 | " else:\n", 203 | " ax.imshow(1 - x_test[index].reshape(28, 28))\n", 204 | " ax.set_title(r'$\\hat{y_i}$ = ' + str(y_) + r' ; $a_i$ = {:.03f}'.format(float(a_)), fontsize=20)" 205 | ] 206 | } 207 | ], 208 | "metadata": { 209 | "kernelspec": { 210 | "display_name": "Python 3", 211 | "language": "python", 212 | "name": "python3" 213 | }, 214 | "language_info": { 215 | "codemirror_mode": { 216 | "name": "ipython", 217 | "version": 3 218 | }, 219 | "file_extension": ".py", 220 | "mimetype": "text/x-python", 221 | "name": "python", 222 | "nbconvert_exporter": "python", 223 | "pygments_lexer": "ipython3", 224 | "version": "3.7.7" 225 | } 226 | }, 227 | "nbformat": 4, 228 | "nbformat_minor": 2 229 | } 230 | -------------------------------------------------------------------------------- /Deep-Learning-Models/perceptron_TensorFlow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Perceptron [TensorFlow]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [perceptron_TensorFlow.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/perceptron_TensorFlow.ipynb)\n", 12 | "---\n", 13 | "Implementation of *Perceptron* model using using TensorFlow library." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "import tensorflow as tf" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "plt.rcParams['figure.figsize'] = (16, 8)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "mnist = tf.keras.datasets.mnist\n", 46 | "[X_train, Y_train],[X_test, Y_test] = mnist.load_data()\n", 47 | "\n", 48 | "# data cleaning\n", 49 | "# using only zeros and ones\n", 50 | "X_train = X_train[Y_train <= 1]\n", 51 | "Y_train = Y_train[Y_train <= 1]\n", 52 | "X_test = X_test[Y_test <= 1]\n", 53 | "Y_test = Y_test[Y_test <= 1]\n", 54 | "\n", 55 | "print('X_train:', X_train.shape)\n", 56 | "print('Y_train:', Y_train.shape)\n", 57 | "print('X_test:', X_test.shape)\n", 58 | "print('Y_test:', Y_test.shape)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)\n", 68 | "\n", 69 | "np.random.seed(1234)\n", 70 | "for ax in AX.ravel():\n", 71 | " rindex = np.random.randint(Y_train.size)\n", 72 | " ax.imshow(X_train[rindex])\n", 73 | " ax.set_title('label: {}'.format(Y_train[rindex]))\n", 74 | "plt.grid(False)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "# data preparation\n", 84 | "# scales, dimensions and dtypes\n", 85 | "x_train, y_train = X_train/255, Y_train[np.newaxis].T\n", 86 | "x_test, y_test = X_test/255, Y_test[np.newaxis].T\n", 87 | "\n", 88 | "x_train = x_train.astype(np.float32).reshape(-1, 28*28)\n", 89 | "y_train = y_train.astype(np.float32)\n", 90 | "x_test = x_test.astype(np.float32).reshape(-1, 28*28)\n", 91 | "y_test = y_test.astype(np.float32)\n", 92 | "\n", 93 | "print('x_train:', x_train.shape)\n", 94 | "print('y_train:', y_train.shape)\n", 95 | "print('x_test:', x_test.shape)\n", 96 | "print('y_test:', y_test.shape)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "## Perceptron\n", 104 | "---\n", 105 | "![perceptron](sourceimages/perceptron.png \"Perceptron\")" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "EPOCHS = 500 # epochs\n", 115 | "ALPHA = 0.001 # learning rate\n", 116 | "BATCH = 100 # batch size\n", 117 | "\n", 118 | "# m is the number of examples\n", 119 | "# n_x is the input size 28x28=784\n", 120 | "m, n_x = x_train.shape\n", 121 | "\n", 122 | "X = tf.placeholder(tf.float32, shape=[None, n_x], name='X')\n", 123 | "Y = tf.placeholder(tf.float32, shape=[None, 1], name='Y')\n", 124 | "\n", 125 | "# variables initialization\n", 126 | "W = tf.Variable(tf.zeros([n_x, 1]), tf.float32, name='W')\n", 127 | "B = tf.Variable(tf.zeros([1, 1]), tf.float32, name='B')\n", 128 | "\n", 129 | "init_variables = tf.global_variables_initializer()\n", 130 | "\n", 131 | "# model\n", 132 | "Z = tf.add(tf.matmul(X, W), B)\n", 133 | "A = tf.nn.sigmoid(Z)\n", 134 | "\n", 135 | "# training graph and optimization\n", 136 | "loss = tf.reduce_mean(tf.losses.mean_squared_error(predictions=A, labels=Y))\n", 137 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=ALPHA).minimize(loss)\n", 138 | "\n", 139 | "# prediction graph\n", 140 | "prediction = tf.round(A) \n", 141 | "compare = tf.equal(prediction, Y)\n", 142 | "cast = tf.cast(compare, tf.float32)\n", 143 | "accuracy = tf.reduce_mean(cast)*100\n", 144 | "\n", 145 | "# loss and accuracy storage\n", 146 | "loss_plot = []; accA_plot = []\n", 147 | "\n", 148 | "with tf.Session() as sess:\n", 149 | " sess.run(init_variables)\n", 150 | " for epoch in range(EPOCHS + 1):\n", 151 | " # randomic batch definition\n", 152 | " rbatch = np.random.choice(Y_train.size, size=BATCH)\n", 153 | " # training, metrics and storage\n", 154 | " sess.run(optimizer, feed_dict={X: x_train[rbatch], Y: y_train[rbatch]})\n", 155 | " L = sess.run(loss, feed_dict={X: x_train[rbatch], Y: y_train[rbatch]})\n", 156 | " acc = sess.run(accuracy, feed_dict={X: x_test, Y: y_test})\n", 157 | " loss_plot += [L]; accA_plot += [acc]\n", 158 | " if (not epoch % 100) and (epoch != 0):\n", 159 | " print('epoch: {0:04d} | loss: {1:.3f} | accuracy: {2:06.2f} %'.format(epoch, L, acc))\n", 160 | " w_ = sess.run(W) # store W and B for visualization and test\n", 161 | " b_ = sess.run(B)" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "axA = plt.subplot(121)\n", 171 | "axA.imshow(w_.T.reshape(28, 28))\n", 172 | "cb = axA.set_title('W')\n", 173 | "\n", 174 | "axB = plt.subplot(222)\n", 175 | "axB.plot(loss_plot)\n", 176 | "axB.set_ylabel('loss')\n", 177 | "\n", 178 | "axC = plt.subplot(224)\n", 179 | "axC.plot(accA_plot)\n", 180 | "axC.set_ylabel('accuracy')\n", 181 | "\n", 182 | "plt.xlabel('epochs')\n", 183 | "\n", 184 | "plt.show()" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "fig, AX = plt.subplots(3, 6, figsize=(2048//72, 1024//72))\n", 194 | "AX = [b for a in AX for b in a]\n", 195 | "\n", 196 | "np.random.seed(1)\n", 197 | "for ax in AX:\n", 198 | " index = np.random.randint(y_test.size)\n", 199 | " z_ = np.dot(w_.T, x_test[index]) + b_\n", 200 | " a_ = 1/(1 + np.exp(-z_))\n", 201 | " y_ = 1 if a_ > 0.5 else 0\n", 202 | " if y_ == y_test[index]:\n", 203 | " ax.imshow(x_test[index].reshape(28, 28))\n", 204 | " else:\n", 205 | " ax.imshow(1 - x_test[index].reshape(28, 28))\n", 206 | " ax.set_title(r'$\\hat{y_i}$ = ' + str(y_) + r' ; $a_i$ = {:.02f}'.format(float(a_)), fontsize=20)" 207 | ] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": "Python 3", 213 | "language": "python", 214 | "name": "python3" 215 | }, 216 | "language_info": { 217 | "codemirror_mode": { 218 | "name": "ipython", 219 | "version": 3 220 | }, 221 | "file_extension": ".py", 222 | "mimetype": "text/x-python", 223 | "name": "python", 224 | "nbconvert_exporter": "python", 225 | "pygments_lexer": "ipython3", 226 | "version": "3.7.7" 227 | } 228 | }, 229 | "nbformat": 4, 230 | "nbformat_minor": 2 231 | } 232 | -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/MCLR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/MCLR.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/autoencoder_convolutional.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/autoencoder_convolutional.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/autoencoder_deep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/autoencoder_deep.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/autoencoder_shallow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/autoencoder_shallow.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/convolutional_neural_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/convolutional_neural_network.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/mandril.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/mandril.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/neural_network_deep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/neural_network_deep.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/neural_network_shallow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/neural_network_shallow.png -------------------------------------------------------------------------------- /Deep-Learning-Models/sourceimages/perceptron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Deep-Learning-Models/sourceimages/perceptron.png -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Diego Inácio 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/NN__utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ###################### 4 | ### Synthetic Data ### 5 | ###################### 6 | 7 | def synthData1(): 8 | ''' 9 | Returns synthetic data for perceptron 1 10 | ''' 11 | np.random.seed(sum([ord(c) for c in 'Neural Network'])) 12 | m1 = 200 13 | x1 = np.random.random((2, m1)) 14 | y1 = np.sum(x1.T*[2, 1], axis=1) > 1.5 15 | y1 = y1.reshape(1, m1) 16 | return [x1, y1] 17 | 18 | def synthData2(): 19 | ''' 20 | Returns synthetic data for perceptron 2 21 | ''' 22 | np.random.seed(sum([ord(c) for c in 'Neural Network'])) 23 | m2 = 100 24 | x2 = np.random.randn(2, m2) + [[0], [3]] 25 | x2 = np.concatenate([x2, np.random.randn(2, m2)], axis=1) 26 | y2 = np.array([[1]*m2 + [0]*m2]) 27 | return [x2, y2] -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/NN__visualizations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Neural Network - Visualizations\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "%matplotlib inline\n", 20 | "import matplotlib\n", 21 | "import matplotlib.pyplot as plt\n", 22 | "import matplotlib.animation as manim\n", 23 | "import numpy as np\n", 24 | "import imageio as iio\n", 25 | "\n", 26 | "from NN__utils import *" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "plt.rcParams['figure.figsize'] = (16, 8)\n", 36 | "plt.set_cmap('prism')" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "## 1. Perceptron\n", 44 | "---" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "%%time\n", 54 | "fig, (axA, axB) = plt.subplots(1, 2)\n", 55 | "\n", 56 | "# Synthetic data 1 and 2\n", 57 | "x1, y1 = synthData1()\n", 58 | "x2, y2 = synthData2()\n", 59 | "\n", 60 | "# vizualization\n", 61 | "axA.scatter(*x1, c=y1.ravel())\n", 62 | "axB.scatter(*x2, c=y2.ravel())\n", 63 | "fig.savefig('output/NN_perceptron_synthData.png', bbox_inches='tight')\n", 64 | "plt.show()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "class Perceptron(object):\n", 74 | " def __init__(self, dim=2, alpha=0.01, iters=16, seed=1):\n", 75 | " super(Perceptron, self).__init__()\n", 76 | " self._iters = iters\n", 77 | " self._alpha = alpha\n", 78 | " np.random.seed(seed)\n", 79 | " self._w = np.random.randn(dim, 1)\n", 80 | " self._b = np.random.random()\n", 81 | " self._cost = 0.0\n", 82 | " @property\n", 83 | " def iters(self):\n", 84 | " return self._iters\n", 85 | " @iters.setter\n", 86 | " def iters(self, iters):\n", 87 | " self._iters = int(iters)\n", 88 | " @property\n", 89 | " def alpha(self):\n", 90 | " return self._alpha\n", 91 | " @alpha.setter\n", 92 | " def alpha(self, alpha):\n", 93 | " self._alpha = int(alpha)\n", 94 | " @property\n", 95 | " def w(self):\n", 96 | " return self._w\n", 97 | " @property\n", 98 | " def b(self):\n", 99 | " return self._b\n", 100 | " @property\n", 101 | " def cost(self):\n", 102 | " return self._cost\n", 103 | " def fit(self, X, y):\n", 104 | " w, b = self._w, self._b\n", 105 | " alpha = self._alpha\n", 106 | " m = y.shape[1]\n", 107 | " for i in range(self._iters):\n", 108 | " Z = np.dot(w.T, X) + b\n", 109 | " A = 1/(1 + np.exp(-Z))\n", 110 | " dw = 1/m*np.dot(X, (A - y).T)\n", 111 | " db = 1/m*np.sum(A - y, axis=1)\n", 112 | " w = w - alpha*dw\n", 113 | " b = b - alpha*db\n", 114 | " self._w, self._b = w, b\n", 115 | " self._cost = -1/m*np.sum(y*np.log(A) + (1 - y)*np.log(1 - A), axis=1)\n", 116 | " self._cost = np.squeeze(self._cost)\n", 117 | " def pred(self, x, beta=0.5):\n", 118 | " w, b = self._w, self._b\n", 119 | " z = np.dot(w.T, x) + b\n", 120 | " a = 1/(1 + np.exp(-z))\n", 121 | " return a > beta" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "%%time\n", 131 | "fig, (axA, axB) = plt.subplots(1, 2)\n", 132 | "\n", 133 | "X = np.random.uniform(-3, 3, (2, 5000))\n", 134 | "np.random.seed(sum([ord(e) for e in 'perceptron']))\n", 135 | "p = Perceptron()\n", 136 | "\n", 137 | "K = 20\n", 138 | "def animation(frame):\n", 139 | " axA.cla(); axB.cla()\n", 140 | " p.iters = frame*1000 + 1\n", 141 | " p.fit(x1, y1)\n", 142 | " y1_hat = p.pred(X)\n", 143 | " p.fit(x2, y2)\n", 144 | " y2_hat = p.pred(X)\n", 145 | " axA.scatter(*X, c=y1_hat.ravel(), marker='*',\n", 146 | " alpha=0.05, edgecolors='none', s=512)\n", 147 | " axA.scatter(*x1, c=y1.ravel())\n", 148 | " axB.scatter(*X, c=y2_hat.ravel(), marker='*',\n", 149 | " alpha=0.05, edgecolors='none', s=512)\n", 150 | " axB.scatter(*x2, c=y2.ravel())\n", 151 | " fig.suptitle('learning rate = {0} | iter = {1} | cost = {2:.3f}'.format(p.alpha, p.iters - 1, p.cost))\n", 152 | " \n", 153 | " return fig.canvas.draw()\n", 154 | "\n", 155 | "anim = manim.FuncAnimation(fig, animation, frames=K, interval=500)\n", 156 | " \n", 157 | "anim.save('output/NN_perceptron.gif', writer=\"imagemagick\", extra_args=\"convert\")\n", 158 | "plt.close()\n", 159 | "\n", 160 | "# Solve repetition problem\n", 161 | "! magick convert output/NN_perceptron.gif -loop 0 output/NN_perceptron.gif\n", 162 | "! echo GIF exported and reconverted. Disregard any message above." 163 | ] 164 | } 165 | ], 166 | "metadata": { 167 | "kernelspec": { 168 | "display_name": "Python 3", 169 | "language": "python", 170 | "name": "python3" 171 | }, 172 | "language_info": { 173 | "codemirror_mode": { 174 | "name": "ipython", 175 | "version": 3 176 | }, 177 | "file_extension": ".py", 178 | "mimetype": "text/x-python", 179 | "name": "python", 180 | "nbconvert_exporter": "python", 181 | "pygments_lexer": "ipython3", 182 | "version": "3.7.10" 183 | } 184 | }, 185 | "nbformat": 4, 186 | "nbformat_minor": 2 187 | } 188 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/NN_perceptron.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Perceptron\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [NN_perceptron.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/NN_perceptron.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of the most fundamental *Neural Network* model." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from NN__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "# Synthetic data 1 and 2\n", 37 | "x1, y1 = synthData1()\n", 38 | "x2, y2 = synthData2()" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "![nn perceptron synthdata](output/NN_perceptron_synthData.png \"Neural Network Perceptron Synthetic Data\")" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## Perceptron\n", 53 | "---\n", 54 | "The *Perceptron* algorithm is very similar to the *Logistic Regression* one.\n", 55 | "### Foward propagation\n", 56 | "---\n", 57 | "$$ \\large\n", 58 | "Z=w^TX + b\n", 59 | "$$\n", 60 | "The activation function (*Sigmoid*) is:\n", 61 | "$$ \\large\n", 62 | "\\hat{y}=A=\\frac{1}{1 + e^{-Z}}\n", 63 | "$$\n", 64 | "The *cost* function is:\n", 65 | "$$ \\large\n", 66 | "J = -\\frac{1}{m}\\sum_{i=1}^{m}y^{(i)}\\log(a^{(i)})+(1-y^{(i)})\\log(1-a^{(i)})\n", 67 | "$$\n", 68 | "### Backward propagation\n", 69 | "---\n", 70 | "Gradients:\n", 71 | "$$ \\large \\frac{\\partial J}{\\partial w} = \\frac{1}{m}X(A-Y)^T$$\n", 72 | "$$ \\large \\frac{\\partial J}{\\partial b} = \\frac{1}{m} \\sum_{i=1}^m (a^{(i)}-y^{(i)})$$\n", 73 | "where:\n", 74 | "- $m$ is the number of examples in the dataset;\n", 75 | "- $a^{(i)}$ is the $i_{th}$ component of vector $A$.\n", 76 | "### Optimization\n", 77 | "---\n", 78 | "The optimization functions is:\n", 79 | "$$ \\large\n", 80 | "\\theta = \\theta - \\alpha d\\theta\n", 81 | "$$\n", 82 | "where $\\alpha$ is the *learning rage*." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "class Perceptron(object):\n", 92 | " def __init__(self, dim=2, alpha=0.01, iters=16, seed=1):\n", 93 | " super(Perceptron, self).__init__()\n", 94 | " self._iters = iters\n", 95 | " self._alpha = alpha\n", 96 | " np.random.seed(seed)\n", 97 | " self._w = np.random.randn(dim, 1)\n", 98 | " self._b = np.random.random()\n", 99 | " self._J = 0.0\n", 100 | " \n", 101 | " @property\n", 102 | " def J(self):\n", 103 | " return self._J\n", 104 | " \n", 105 | " def fit(self, X, y):\n", 106 | " w, b = self._w, self._b\n", 107 | " alpha = self._alpha\n", 108 | " m = y.shape[1]\n", 109 | " for i in range(self._iters):\n", 110 | " Z = np.dot(w.T, X) + b\n", 111 | " A = 1/(1 + np.exp(-Z))\n", 112 | " dw = 1/m*np.dot(X, (A - y).T)\n", 113 | " db = 1/m*np.sum(A - y, axis=1)\n", 114 | " w = w - alpha*dw\n", 115 | " b = b - alpha*db\n", 116 | " self._w, self._b = w, b\n", 117 | " self._J = -1/m*np.sum(y*np.log(A) + (1 - y)*np.log(1 - A), axis=1)\n", 118 | " self._J = np.squeeze(self._J)\n", 119 | " \n", 120 | " def pred(self, x, beta=0.5):\n", 121 | " w, b = self._w, self._b\n", 122 | " z = np.dot(w.T, x) + b\n", 123 | " a = 1/(1 + np.exp(-z))\n", 124 | " return a > beta" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "## Classification\n", 132 | "---" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "%%time\n", 142 | "# Perceptron model\n", 143 | "p = Perceptron(iters=20000)\n", 144 | "\n", 145 | "# Training and Prediction\n", 146 | "p.fit(x1, y1)\n", 147 | "y1_hat = p.pred(X)\n", 148 | "\n", 149 | "p.fit(x2, y2)\n", 150 | "y2_hat = p.pred(X)" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "![nn perceptron](output/NN_perceptron.gif \"Neural Network Perceptron\")" 158 | ] 159 | } 160 | ], 161 | "metadata": { 162 | "kernelspec": { 163 | "display_name": "Python 3", 164 | "language": "python", 165 | "name": "python3" 166 | }, 167 | "language_info": { 168 | "codemirror_mode": { 169 | "name": "ipython", 170 | "version": 3 171 | }, 172 | "file_extension": ".py", 173 | "mimetype": "text/x-python", 174 | "name": "python", 175 | "nbconvert_exporter": "python", 176 | "pygments_lexer": "ipython3", 177 | "version": "3.7.7" 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 2 182 | } 183 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning Fundamentals 2 | 3 | Walkthrough into the *Machine Learning*'s principles and implementation of the main algorithms from scratch. -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/clustering__utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ######################## 4 | ### Distance metrics ### 5 | ######################## 6 | 7 | ldist = ['euclidian', 'manhattan', 'chebyshev', 'canberra', 'cosine'] 8 | 9 | class Distance(object): 10 | def __init__(self, metrica='euclidian'): 11 | super(Distance, self).__init__() 12 | if metrica not in ldist: 13 | raise ValueError('Metric does not exist! Choose between: {}'.format(ldist)) 14 | self._metrica = metrica 15 | @property 16 | def metrica(self): 17 | return self._metrica 18 | @metrica.setter 19 | def metrica(self, m): 20 | if m not in ldist: 21 | raise ValueError('Metric does not exist! Choose between: {}'.format(ldist)) 22 | self._metrica = m 23 | def distance(self, p, q): 24 | if self._metrica == 'manhattan': 25 | return np.sum(np.absolute(p - q), axis=1) 26 | if self._metrica == 'chebyshev': 27 | return np.max(np.absolute(p - q), axis=1) 28 | if self._metrica == 'canberra': 29 | num = np.absolute(p - q) 30 | den = np.absolute(p) + np.absolute(q) 31 | return np.sum(num/den, axis=1) 32 | if self._metrica == 'cosine': 33 | if p.ndim == 1: 34 | p = p[np.newaxis] 35 | num = np.sum(p*q, axis=1) 36 | den = np.sum(p**2, axis=1)**0.5 37 | den = den*np.sum(q**2, axis=1)**0.5 38 | return 1 - num/den 39 | return np.sum((p - q)**2, axis=1)**0.5 40 | 41 | 42 | ###################### 43 | ### Synthetic Data ### 44 | ###################### 45 | 46 | def synthData(): 47 | ''' 48 | Returns synthetic data for clustering algorithms 49 | ''' 50 | np.random.seed(sum([ord(e) for e in 'clustering'])) 51 | 52 | # Gaussian mixture 53 | x1 = np.array([]) 54 | y1 = np.array([]) 55 | for i in range(3): 56 | mu1 = np.random.uniform(-2, 2) 57 | mu2 = np.random.uniform(-2, 2) 58 | sigma = np.random.uniform(0, 0.5) 59 | x1 = np.concatenate([x1, np.random.normal(mu1, sigma, 32)]) 60 | y1 = np.concatenate([y1, np.random.normal(mu2, sigma, 32)]) 61 | 62 | # Ring with center 63 | N = 1000 64 | t = np.random.uniform(0, 2*np.pi, N) 65 | x2 = np.cos(t)*10 + np.random.random(N) 66 | y2 = np.sin(t)*10 + np.random.random(N) 67 | x2 = np.concatenate([x2, np.random.uniform(-2.5, 2.5, N//4)]) 68 | y2 = np.concatenate([y2, np.random.uniform(-2.5, 2.5, N//4)]) 69 | 70 | return [x1, y1, x2, y2] -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/clustering_dbscan.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Clustering [DBSCAN]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [clustering_dbscan.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/clustering_dbscan.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of *clustering algorithm* using the *DBSCAN* technique." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from clustering__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "## Synthetic data\n", 35 | "---" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "x1, y1, x2, y2 = synthData()\n", 45 | "X1 = np.array([x1, y1]).T\n", 46 | "X2 = np.array([x2, y2]).T" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "![synthetic data](output/clustering_synthetic_data.png \"Synthetic data\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "## Algorithm\n", 61 | "---" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "class DBSCAN(Distance):\n", 71 | " def __init__(self, epsilon=1, minPts=4):\n", 72 | " super(DBSCAN, self).__init__()\n", 73 | " self._epsilon = epsilon\n", 74 | " self._minPts = minPts\n", 75 | "\n", 76 | " def pred(self, X):\n", 77 | " n = X.shape[0]\n", 78 | " C = 0\n", 79 | " label = np.zeros(n, int)\n", 80 | " for i, p in enumerate(X):\n", 81 | " if label[i]:\n", 82 | " continue\n", 83 | " d = self.distance(p, X)\n", 84 | " conj = d <= self._epsilon\n", 85 | " if np.sum(conj) - 1 < self._minPts:\n", 86 | " label[i] = -1\n", 87 | " continue\n", 88 | " C += 1\n", 89 | " argwhere = np.argwhere(conj)\n", 90 | " argwhere = list(np.ravel(argwhere))\n", 91 | " for q in argwhere:\n", 92 | " if label[q] == -1:\n", 93 | " label[q] = C\n", 94 | " if label[q] != 0:\n", 95 | " continue\n", 96 | " label[q] = C\n", 97 | " d = self.distance(X[q], np.delete(X, q, axis=0))\n", 98 | " conj2 = d <= self._epsilon\n", 99 | " if np.sum(conj2) >= self._minPts:\n", 100 | " argwhere2 = np.ravel(np.argwhere(conj2))\n", 101 | " argwhere += [a for a in argwhere2 if a not in argwhere]\n", 102 | " return label" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "### Application\n", 110 | "---" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "%%time\n", 120 | "\n", 121 | "epsilon1 = 0.5; minPts1 = 8\n", 122 | "dbscan1 = DBSCAN(epsilon1, minPts1)\n", 123 | "E1 = dbscan1.pred(X1)\n", 124 | "\n", 125 | "epsilon2 = 1; minPts2 = 8\n", 126 | "dbscan2 = DBSCAN(epsilon2, minPts2)\n", 127 | "E2 = dbscan2.pred(X2)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "![dbscan](output/clustering_dbscan.png \"DBSCAN\")" 135 | ] 136 | } 137 | ], 138 | "metadata": { 139 | "kernelspec": { 140 | "display_name": "Python 3", 141 | "language": "python", 142 | "name": "python3" 143 | }, 144 | "language_info": { 145 | "codemirror_mode": { 146 | "name": "ipython", 147 | "version": 3 148 | }, 149 | "file_extension": ".py", 150 | "mimetype": "text/x-python", 151 | "name": "python", 152 | "nbconvert_exporter": "python", 153 | "pygments_lexer": "ipython3", 154 | "version": "3.7.7" 155 | } 156 | }, 157 | "nbformat": 4, 158 | "nbformat_minor": 2 159 | } 160 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/clustering_k-means.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Clustering [k-means]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [clustering_k-means.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/clustering_k-means.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of *clustering algorithm* using the *k-means* technique." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from clustering__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "## Synthetic data\n", 35 | "---" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "x1, y1, x2, y2 = synthData()\n", 45 | "X1 = np.array([x1, y1]).T\n", 46 | "X2 = np.array([x2, y2]).T" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "![synthetic data](output/clustering_synthetic_data.png \"Synthetic data\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "## Algorithm\n", 61 | "---" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "class kMeans(Distance):\n", 71 | " def __init__(self, K=2, iters=16, seed=1):\n", 72 | " super(kMeans, self).__init__()\n", 73 | " self._K = K\n", 74 | " self._iters = iters\n", 75 | " self._seed = seed\n", 76 | " self._C = None\n", 77 | " \n", 78 | " def _FNC(self, x, c, n):\n", 79 | " # for each point,\n", 80 | " # find the nearest center\n", 81 | " cmp = np.ndarray(n, dtype=int)\n", 82 | " for i, p in enumerate(x):\n", 83 | " d = self.distance(p, self._C)\n", 84 | " cmp[i] = np.argmin(d)\n", 85 | " return cmp\n", 86 | " \n", 87 | " def pred(self, X):\n", 88 | " # prediction\n", 89 | " n, dim = X.shape\n", 90 | " np.random.seed(self._seed)\n", 91 | " sel = np.random.randint(0, n, self._K)\n", 92 | " self._C = X[sel]\n", 93 | " cmp = self._FNC(X, self._C, n)\n", 94 | " for _ in range(self._iters):\n", 95 | " # adjust position of centroids\n", 96 | " # to the mean value\n", 97 | " for i in range(sel.size):\n", 98 | " P = X[cmp == i]\n", 99 | " self._C[i] = np.mean(P, axis=0)\n", 100 | " cmp = self._FNC(X, self._C, n)\n", 101 | " return cmp, self._C" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "### Elbow method\n", 109 | "---\n", 110 | "Method to define the number of *K*." 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "%%time\n", 120 | "\n", 121 | "# elbow method\n", 122 | "Cs = 12\n", 123 | "V1 = np.zeros(Cs)\n", 124 | "V2 = np.zeros(Cs)\n", 125 | "D = Distance()\n", 126 | "for k in range(Cs):\n", 127 | " kmeans = kMeans(K=k + 1, seed=6)\n", 128 | " fnc1, C1 = kmeans.pred(X1)\n", 129 | " fnc2, C2 = kmeans.pred(X2)\n", 130 | " for i, [c1, c2] in enumerate(zip(C1, C2)):\n", 131 | " d1 = D.distance(c1, X1[fnc1 == i])**2\n", 132 | " d2 = D.distance(c2, X2[fnc2 == i])**2\n", 133 | " V1[k] += np.sum(d1)\n", 134 | " V2[k] += np.sum(d2)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "![elbow method](output/clustering_k-means_elbowMethod.png \"Elbow Method\")" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "### Application\n", 149 | "---\n", 150 | "Based on *elbow method* $C_1$ is 3 and $C_2$ is 6." 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "%%time\n", 160 | "\n", 161 | "iters = 20; seed = 6\n", 162 | "\n", 163 | "K1 = 3\n", 164 | "kmeans1 = kMeans(K1, iters, seed)\n", 165 | "fnc1, C1 = kmeans1.pred(X1)\n", 166 | "\n", 167 | "K2 = 6\n", 168 | "kmeans2 = kMeans(K2, iters, seed)\n", 169 | "fnc2, C2 = kmeans2.pred(X2)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "![k-means](output/clustering_k-means.gif \"k-Means\")" 177 | ] 178 | } 179 | ], 180 | "metadata": { 181 | "kernelspec": { 182 | "display_name": "Python 3", 183 | "language": "python", 184 | "name": "python3" 185 | }, 186 | "language_info": { 187 | "codemirror_mode": { 188 | "name": "ipython", 189 | "version": 3 190 | }, 191 | "file_extension": ".py", 192 | "mimetype": "text/x-python", 193 | "name": "python", 194 | "nbconvert_exporter": "python", 195 | "pygments_lexer": "ipython3", 196 | "version": "3.7.7" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 2 201 | } 202 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/kNN__utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ######################## 4 | ### Distance metrics ### 5 | ######################## 6 | 7 | ldist = ['euclidian', 'manhattan', 'chebyshev', 'canberra', 'cosine'] 8 | 9 | class Distance(object): 10 | def __init__(self, metrica='euclidian'): 11 | super(Distance, self).__init__() 12 | if metrica not in ldist: 13 | raise ValueError('Metric does not exist! Choose between: {}'.format(ldist)) 14 | self._metrica = metrica 15 | @property 16 | def metrica(self): 17 | return self._metrica 18 | @metrica.setter 19 | def metrica(self, m): 20 | if m not in ldist: 21 | raise ValueError('Metric does not exist! Choose between: {}'.format(ldist)) 22 | self._metrica = m 23 | def distance(self, p, q): 24 | if self._metrica == 'manhattan': 25 | return np.sum(np.absolute(p - q), axis=1) 26 | if self._metrica == 'chebyshev': 27 | return np.max(np.absolute(p - q), axis=1) 28 | if self._metrica == 'canberra': 29 | num = np.absolute(p - q) 30 | den = np.absolute(p) + np.absolute(q) 31 | return np.sum(num/den, axis=1) 32 | if self._metrica == 'cosine': 33 | if p.ndim == 1: 34 | p = p[np.newaxis] 35 | num = np.sum(p*q, axis=1) 36 | den = np.sum(p**2, axis=1)**0.5 37 | den = den*np.sum(q**2, axis=1)**0.5 38 | return 1 - num/den 39 | return np.sum((p - q)**2, axis=1)**0.5 40 | 41 | 42 | ###################### 43 | ### Synthetic Data ### 44 | ###################### 45 | 46 | def synthData1(): 47 | ''' 48 | Returns synthetic data for classification 1 49 | ''' 50 | np.random.seed(sum([ord(c) for c in 'k-nearest neighbors'])) 51 | N = 100 52 | Q1 = np.random.uniform(0, 1, N) 53 | Q2 = np.random.uniform(0, 1, N) 54 | CL = np.random.randint(0, 3, N) 55 | return [Q1, Q2, CL] 56 | 57 | def synthData2(): 58 | ''' 59 | Returns synthetic data for classification 2 60 | ''' 61 | np.random.seed(sum([ord(c) for c in 'k-nearest neighbors'])) 62 | N = 2048 63 | P1 = np.random.uniform(0, 1, N) 64 | P2 = np.random.uniform(0, 1, N) 65 | return [P1, P2] 66 | 67 | def synthData3(): 68 | ''' 69 | Returns synthetic data for regression 1 70 | ''' 71 | np.random.seed(sum([ord(c) for c in 'k-nearest neighbors'])) 72 | M, N = 32j, 32j 73 | X, Y = np.mgrid[-3:3:N*8, -3:3:M*8] 74 | Z = (1 - X + Y*X**3 + Y**5)*np.exp(-X**2 - Y**2) 75 | return [X, Y, Z] 76 | 77 | def synthData4(): 78 | ''' 79 | Returns synthetic data for regression 2 80 | ''' 81 | np.random.seed(sum([ord(c) for c in 'k-nearest neighbors'])) 82 | N = 512 83 | Q1 = np.random.uniform(-3, 3, N) 84 | Q2 = np.random.uniform(-3, 3, N) 85 | VL = (1 - Q1 + Q2*Q1**3 + Q2**5)*np.exp(-Q1**2 - Q2**2) 86 | return [Q1, Q2, VL] 87 | 88 | def synthData5(): 89 | ''' 90 | Returns synthetic data for regression 3 91 | ''' 92 | np.random.seed(sum([ord(c) for c in 'k-nearest neighbors'])) 93 | N = 2048 94 | P = np.random.uniform(-3, 3, (N, 2)) 95 | xi, yi = np.mgrid[-3:3:1024j, -3:3:1024j] 96 | return [P, xi, yi] -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/kNN_classification.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# k-NN Classification\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [kNN_classification.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/kNN_classification.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of *k-Nearest Neighbor Classification*." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from kNN__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "## Algorithm\n", 35 | "---" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "class kNNClass(Distance):\n", 45 | " def __init__(self, k=1):\n", 46 | " super(kNNClass, self).__init__()\n", 47 | " self._k = k\n", 48 | " self._q = None\n", 49 | " self._class = None\n", 50 | " \n", 51 | " def fit(self, X, y):\n", 52 | " self._q = X\n", 53 | " self._class = y\n", 54 | " \n", 55 | " def pred(self, P):\n", 56 | " y, NNs = [], []\n", 57 | " for i, p in enumerate(P):\n", 58 | " dist = self.distance(p, self._q)\n", 59 | " odist = np.argsort(dist)[:self._k]\n", 60 | " fdist = np.ravel(self._class[odist])\n", 61 | " hist = np.bincount(fdist)\n", 62 | " index = np.argmax(hist)\n", 63 | " y += [index]\n", 64 | " NNs += [odist]\n", 65 | " return np.array(y), np.array(NNs)" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "# Synthetic data 1\n", 75 | "Q1, Q2, CL = synthData1()\n", 76 | "Q = np.array([Q1, Q2]).T\n", 77 | "\n", 78 | "p = [[0.5, 0.5]]\n", 79 | "colors = ['#FF00AA', '#AAFF00', '#00AAFF']" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "k = 20\n", 89 | "knnc = kNNClass(k)\n", 90 | "knnc.fit(Q, y=CL)\n", 91 | "y_, NNs = knnc.pred(p)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "!['kNN classification A'](output/kNN_classificationA.gif \"k-Nearest Neighbor A\")" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "%%time \n", 108 | "\n", 109 | "# Synthetic data 2\n", 110 | "P1, P2 = synthData2()\n", 111 | "P = np.array([P1, P2]).T\n", 112 | "\n", 113 | "k = 128\n", 114 | "knnc = kNNClass(k)\n", 115 | "knnc.fit(Q, y=CL)\n", 116 | "y_, NNs = knnc.pred(P)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "!['kNN classification B'](output/kNN_classificationB.png \"k-Nearest Neighbor B\")" 124 | ] 125 | } 126 | ], 127 | "metadata": { 128 | "kernelspec": { 129 | "display_name": "Python 3", 130 | "language": "python", 131 | "name": "python3" 132 | }, 133 | "language_info": { 134 | "codemirror_mode": { 135 | "name": "ipython", 136 | "version": 3 137 | }, 138 | "file_extension": ".py", 139 | "mimetype": "text/x-python", 140 | "name": "python", 141 | "nbconvert_exporter": "python", 142 | "pygments_lexer": "ipython3", 143 | "version": "3.7.7" 144 | } 145 | }, 146 | "nbformat": 4, 147 | "nbformat_minor": 2 148 | } 149 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/kNN_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# k-NN Regression\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [kNN_regression.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/kNN_regression.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of *k-Nearest Neighbor Regression*." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from kNN__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "# Synthetic data 3 and 4\n", 37 | "X, Y, Z = synthData3()\n", 38 | "Q1, Q2, VL = synthData4()\n", 39 | "Q = np.array([Q1, Q2]).T" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "!['kNN regression A'](output/kNN_regressionA.png \"k-Nearest Neighbor Regression A\")" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "## Algorithm\n", 54 | "---" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "class kNNRegr(Distance):\n", 64 | " def __init__(self, k=1):\n", 65 | " super(kNNRegr, self).__init__()\n", 66 | " self._k = k\n", 67 | " self._q = None\n", 68 | " self._v = None\n", 69 | " \n", 70 | " def fit(self, X, y):\n", 71 | " self._q = X\n", 72 | " self._v = y\n", 73 | " \n", 74 | " def pred(self, P):\n", 75 | " y, NNs = [], []\n", 76 | " for i, p in enumerate(P):\n", 77 | " dist = self.distance(p, self._q)\n", 78 | " odist = np.argsort(dist)[:self._k]\n", 79 | " fdist = np.ravel(self._v[odist])\n", 80 | " ndist = dist[odist]\n", 81 | " ndist /= np.sum(ndist)\n", 82 | " y += [np.sum(fdist*np.flipud(ndist))]\n", 83 | " NNs += [odist]\n", 84 | " return np.array(y), np.array(NNs)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "%%time\n", 94 | "# Synthetic data 5\n", 95 | "P, xi, yi = synthData5()\n", 96 | "\n", 97 | "k = 128\n", 98 | "knnr = kNNRegr(k)\n", 99 | "knnr.fit(Q, y=VL)\n", 100 | "y_, NNs = knnr.pred(P)" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "!['kNN regression B'](output/kNN_regressionB.png \"k-Nearest Neighbor Regression B\")" 108 | ] 109 | } 110 | ], 111 | "metadata": { 112 | "kernelspec": { 113 | "display_name": "Python 3", 114 | "language": "python", 115 | "name": "python3" 116 | }, 117 | "language_info": { 118 | "codemirror_mode": { 119 | "name": "ipython", 120 | "version": 3 121 | }, 122 | "file_extension": ".py", 123 | "mimetype": "text/x-python", 124 | "name": "python", 125 | "nbconvert_exporter": "python", 126 | "pygments_lexer": "ipython3", 127 | "version": "3.7.7" 128 | } 129 | }, 130 | "nbformat": 4, 131 | "nbformat_minor": 2 132 | } 133 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/NN_perceptron.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/NN_perceptron.gif -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/NN_perceptron_synthData.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/NN_perceptron_synthData.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/clustering_dbscan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/clustering_dbscan.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/clustering_k-means.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/clustering_k-means.gif -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/clustering_k-means_elbowMethod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/clustering_k-means_elbowMethod.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/clustering_synthetic_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/clustering_synthetic_data.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/kNN_classificationA.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/kNN_classificationA.gif -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/kNN_classificationB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/kNN_classificationB.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/kNN_introduction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/kNN_introduction.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/kNN_regressionA.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/kNN_regressionA.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/kNN_regressionB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/kNN_regressionB.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_anscombe_pred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_anscombe_pred.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_anscombe_residual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_anscombe_residual.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_correlation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_correlation.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_gradDesc.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_gradDesc.gif -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_multipla_residual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_multipla_residual.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_multiple_pred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_multiple_pred.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_pred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_pred.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_linear_residual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_linear_residual.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_logistic_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_logistic_data.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_logistic_gradDesc.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_logistic_gradDesc.gif -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_logistic_pred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_logistic_pred.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_polynomial_linear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_polynomial_linear.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/output/regression_polynomial_pred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Machine-Learning-Fundamentals/output/regression_polynomial_pred.png -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/regression__utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def correlation(X, Y): 4 | ''' 5 | Correlation function 6 | ''' 7 | N = X.size 8 | mu_X = X.sum()/N 9 | mu_Y = Y.sum()/N 10 | cov = ((X - mu_X)*(Y - mu_Y)).sum()/N 11 | sigma_X = X.std() 12 | sigma_Y = Y.std() 13 | return cov/(sigma_X*sigma_Y) 14 | 15 | ###################### 16 | ### Synthetic Data ### 17 | ###################### 18 | 19 | def synthData1(): 20 | ''' 21 | Returns synthetic data for linear regression simple 22 | ''' 23 | np.random.seed(sum([ord(c) for c in 'Regression'])) 24 | N = 20 25 | x = np.linspace(0, 1, N) 26 | yA = x 27 | yB = x + (np.random.random(N)*2 - 1)*0.15 28 | yC = x + (np.random.random(N)*2 - 1)*0.5 29 | yD = np.random.random(N) 30 | return [x, yA, yB, yC, yD] 31 | 32 | def synthData2(M): 33 | ''' 34 | Returns synthetic data for linear regression multiple 35 | ''' 36 | np.random.seed(sum([ord(c) for c in 'Regression'])) 37 | N = complex(0, M) 38 | s, t = np.mgrid[-1:1:N, -1:1:N] 39 | x1 = s.reshape(1, -1)[0] 40 | x2 = t.reshape(1, -1)[0] 41 | y = (x1 + x2)*0.5 + (np.random.random(int(N.imag**2))*2 - 1)*0.75 42 | return [s, t, x1, x2, y] 43 | 44 | def synthData3(): 45 | ''' 46 | Returns synthetic data for linear regression gradient descent 47 | ''' 48 | np.random.seed(sum([ord(c) for c in 'Regression'])) 49 | N = 20 50 | x = np.linspace(0, 1, N) 51 | x_ = np.linspace(-5, 5, N) 52 | y = x + (np.random.random(N)*2 - 1)*0.25 53 | return [x, x_, y] 54 | 55 | def synthData4(): 56 | ''' 57 | Returns synthetic data for linear regression non-linear analysis 58 | Anscombes quartet 59 | ''' 60 | x1 = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5] 61 | y1 = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84,4.82, 5.68] 62 | x2 = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5] 63 | y2 = [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74] 64 | x3 = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5] 65 | y3 = [7.46, 6.77, 12.74,7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73] 66 | x4 = [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8] 67 | y4 = [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50,5.56, 7.91, 6.89] 68 | return np.array([x1, y1, x2, y2, x3, y3, x4, y4]) 69 | 70 | def synthData5(): 71 | ''' 72 | Returns synthetic data for logistic regression 73 | ''' 74 | np.random.seed(sum([ord(c) for c in 'Regression'])) 75 | N = 512 76 | x1 = np.random.normal(1, 0.3, N//2) 77 | x1 = np.concatenate([x1, np.random.normal(2, 0.3, N//2)]) 78 | x2 = np.random.normal(0, 0.3, N//2) 79 | x2 = np.concatenate([x2, np.random.normal(0.25, 0.3, N//2)]) 80 | y = np.zeros(N//2, np.int8) 81 | y = np.concatenate([y, np.ones(N//2, np.int8)]) 82 | return [x1, x2, y] 83 | 84 | def synthData6(): 85 | ''' 86 | Returns synthetic data for polynomial regression 87 | ''' 88 | np.random.seed(sum([ord(c) for c in 'Regression'])) 89 | N = 21 90 | x = np.random.uniform(-3, 3, N) 91 | y = x**3 - 3*x**2 + x + 1 + np.random.uniform(-3, 3, N) 92 | return [x, y] -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/regression_logistic.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Logistic Regression\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [regression_logistic.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/regression_logistic.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of *Logistic Regression* analysis." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from regression__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "$$ \\large\n", 35 | "h_{\\theta}(x)=g(\\theta^Tx)=\\frac{e^{\\theta^Tx}}{1+e^{\\theta^Tx}}=\\frac{1}{1+e^{-\\theta^Tx}}\n", 36 | "$$\n", 37 | "\n", 38 | "where:\n", 39 | "\n", 40 | "$$ \\large\n", 41 | "\\theta^Tx=\n", 42 | "\\begin{bmatrix}\n", 43 | " \\theta_0 \\\\\n", 44 | " \\theta_1 \\\\\n", 45 | " \\vdots \\\\\n", 46 | " \\theta_i\n", 47 | "\\end{bmatrix}\n", 48 | "\\begin{bmatrix}\n", 49 | " 1 & x_{11} & \\cdots & x_{1i} \\\\\n", 50 | " 1 & x_{21} & \\cdots & x_{2i} \\\\\n", 51 | " \\vdots & \\vdots & \\ddots & \\vdots \\\\\n", 52 | " 1 & x_{n1} & \\cdots & x_{ni}\n", 53 | "\\end{bmatrix}\n", 54 | "$$\n", 55 | "\n", 56 | "where:\n", 57 | "\n", 58 | "- $\\large h_\\theta(x)$ is the hypothesis;\n", 59 | "- $\\large g(z)$ is the logistic function or sigmoid;\n", 60 | "- $\\large \\theta_i$ is the parameters (or weights)." 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "def arraycast(f):\n", 70 | " '''\n", 71 | " Decorator for vectors and matrices cast\n", 72 | " '''\n", 73 | " def wrap(self, *X, y=[]):\n", 74 | " X = np.array(X)\n", 75 | " X = np.insert(X.T, 0, 1, 1)\n", 76 | " if list(y):\n", 77 | " y = np.array(y)[np.newaxis]\n", 78 | " return f(self, X, y)\n", 79 | " return f(self, X)\n", 80 | " return wrap\n", 81 | "\n", 82 | "class logisticRegression(object):\n", 83 | " def __init__(self, rate=0.001, iters=1024):\n", 84 | " self._rate = rate\n", 85 | " self._iters = iters\n", 86 | " self._theta = None\n", 87 | " @property\n", 88 | " def theta(self):\n", 89 | " return self._theta\n", 90 | " def _sigmoid(self, Z):\n", 91 | " return 1/(1 + np.exp(-Z))\n", 92 | " def _dsigmoid(self, Z):\n", 93 | " return self._sigmoid(Z)*(1 - self._sigmoid(Z))\n", 94 | " @arraycast\n", 95 | " def fit(self, X, y=[]):\n", 96 | " self._theta = np.ones((1, X.shape[-1]))\n", 97 | " for i in range(self._iters):\n", 98 | " thetaTx = np.dot(X, self._theta.T)\n", 99 | " h = self._sigmoid(thetaTx)\n", 100 | " delta = h - y.T\n", 101 | " grad = np.dot(X.T, delta).T\n", 102 | " self._theta -= grad*self._rate\n", 103 | " @arraycast\n", 104 | " def pred(self, x):\n", 105 | " return self._sigmoid(np.dot(x, self._theta.T)) > 0.5" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "# Synthetic data 5\n", 115 | "x1, x2, y = synthData5()" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "![logistic regression data](output/regression_logistic_data.png \"Logistic Regression Data\")" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "%%time\n", 132 | "# Training\n", 133 | "rlogb = logisticRegression(rate=0.001, iters=512)\n", 134 | "rlogb.fit(x1, x2, y=y)\n", 135 | "rlogb.pred(x1, x2)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "![logistic regression training](output/regression_logistic_gradDesc.gif \"Logistic Regression Training\")" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": {}, 148 | "source": [ 149 | "To find the boundary line components:\n", 150 | "\n", 151 | "$$ \\large\n", 152 | " \\theta_0+\\theta_1 x_1+\\theta_2 x_2=0\n", 153 | "$$\n", 154 | "\n", 155 | "Considering $\\large x_2$ as the dependent variable:\n", 156 | "\n", 157 | "$$ \\large\n", 158 | " x_2=-\\frac{\\theta_0+\\theta_1 x_1}{\\theta_2}\n", 159 | "$$" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "# Prediction\n", 169 | "w0, w1, w2 = rlogb.theta[0]\n", 170 | "f = lambda x: - (w0 + w1*x)/w2" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "![regressão logística prediction](output/regression_logistic_pred.png \"Logistic Regression Prediction\")" 178 | ] 179 | } 180 | ], 181 | "metadata": { 182 | "kernelspec": { 183 | "display_name": "Python 3", 184 | "language": "python", 185 | "name": "python3" 186 | }, 187 | "language_info": { 188 | "codemirror_mode": { 189 | "name": "ipython", 190 | "version": 3 191 | }, 192 | "file_extension": ".py", 193 | "mimetype": "text/x-python", 194 | "name": "python", 195 | "nbconvert_exporter": "python", 196 | "pygments_lexer": "ipython3", 197 | "version": "3.7.7" 198 | } 199 | }, 200 | "nbformat": 4, 201 | "nbformat_minor": 2 202 | } 203 | -------------------------------------------------------------------------------- /Machine-Learning-Fundamentals/regression_polynomial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Polynomial Regression\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [regression_polynomial.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Machine-Learning-Fundamentals/regression_polynomial.ipynb)\n", 12 | "---\n", 13 | "Overview and implementation of *Polynomial Regression* analysis." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "\n", 27 | "from regression__utils import *" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "Given the function:\n", 35 | "\n", 36 | "$$ \\large\n", 37 | " f(x)=x^3-3x^2+x+1+\\epsilon\n", 38 | "$$" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "# Synthetic data 6\n", 48 | "x, y = synthData6()\n", 49 | "\n", 50 | "# Predicting with Linear Regression\n", 51 | "# lrs = linearRegression_simple()\n", 52 | "# lrs.fit(x, y)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "![polynomial data and linear regression](output/regression_polynomial_linear.png \"Polynomial data and Linear Regression\")" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "## Algorithm\n", 67 | "---\n", 68 | "$$ \\large\n", 69 | "\\vec{y}=\\mathbf{X}\\vec{\\mathbf{\\beta}}+\\vec{\\epsilon}\n", 70 | "$$\n", 71 | "\n", 72 | "where $\\large \\mathbf{X}$ (or $\\large \\mathbf{V}$) is the *Vandermonde's matrix* of the independent variable, parametrised by the maximum degree $\\large m$, a response vector $\\large \\vec{y}$, a parameter vector $\\large \\vec{\\mathbf{\\beta}}$ and a random error vector $\\large \\vec{\\epsilon}$. In the form of a system of linear equations, we have:\n", 73 | "\n", 74 | "$$ \\large\n", 75 | "\\begin{bmatrix}\n", 76 | " y_1 \\\\\n", 77 | " y_2 \\\\\n", 78 | " y_3 \\\\\n", 79 | " \\vdots \\\\\n", 80 | " y_n\n", 81 | "\\end{bmatrix}\n", 82 | "=\n", 83 | "\\begin{bmatrix}\n", 84 | " 1 & x_1 & x_1^2 &\\cdots & x_1^m \\\\\n", 85 | " 1 & x_2 & x_2^2 & \\cdots & x_2^m \\\\\n", 86 | " 1 & x_3 & x_3^2 & \\cdots & x_3^m \\\\\n", 87 | " \\vdots & \\vdots & \\vdots & \\ddots & \\vdots \\\\\n", 88 | " 1 & x_n & x_n^2 & \\cdots & x_n^m\n", 89 | "\\end{bmatrix}\n", 90 | "\\begin{bmatrix}\n", 91 | " \\beta_1 \\\\\n", 92 | " \\beta_2 \\\\\n", 93 | " \\beta_3 \\\\\n", 94 | " \\vdots \\\\\n", 95 | " \\beta_m\n", 96 | "\\end{bmatrix}\n", 97 | "+\n", 98 | "\\begin{bmatrix}\n", 99 | " \\epsilon_1 \\\\\n", 100 | " \\epsilon_2 \\\\\n", 101 | " \\epsilon_3 \\\\\n", 102 | " \\vdots \\\\\n", 103 | " \\epsilon_n\n", 104 | "\\end{bmatrix}\n", 105 | "$$\n", 106 | "\n", 107 | "By means of the Least Squares Method, the estimated coefficient vector is given by:\n", 108 | "\n", 109 | "$$ \\large\n", 110 | "\\widehat{\\vec{\\mathbf{\\beta}}}=(\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^T\\vec{y}\n", 111 | "$$" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "def arraycast(f):\n", 121 | " '''\n", 122 | " Decorador para conversão de vetores e matrizes\n", 123 | " '''\n", 124 | " def wrap(self, X, y=[]):\n", 125 | " X = np.array(X)\n", 126 | " if list(y):\n", 127 | " y = np.array(y)\n", 128 | " return f(self, X, y)\n", 129 | " return f(self, X)\n", 130 | " return wrap\n", 131 | "\n", 132 | "class polynomialRegression(object):\n", 133 | " def __init__(self, degree=1):\n", 134 | " self._degree = degree\n", 135 | " self._beta = None\n", 136 | " @property\n", 137 | " def beta(self):\n", 138 | " return self._beta\n", 139 | " @arraycast\n", 140 | " def fit(self, X, y=[]):\n", 141 | " V = np.stack([X**i for i in range(self._degree + 1)], axis=0).T\n", 142 | " VTV = np.dot(V.T, V)\n", 143 | " VTV_i = np.linalg.inv(VTV)\n", 144 | " Vi = np.dot(VTV_i, V.T)\n", 145 | " self._beta = np.dot(Vi, y)\n", 146 | " @arraycast\n", 147 | " def pred(self, x):\n", 148 | " V = np.stack([x**i for i in range(self._degree + 1)], axis=0).T\n", 149 | " return np.dot(V, self._beta)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "Notice that our class has an attribute called degree which is the maximum degree of our function $\\large f(x)$. In our example it should be $\\large m=3$." 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "%%time\n", 166 | "polreg = polynomialRegression(3)\n", 167 | "polreg.fit(x, y=y)" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "![polynomial regression](output/regression_polynomial_pred.png \"Polynomial Regression\")" 175 | ] 176 | } 177 | ], 178 | "metadata": { 179 | "kernelspec": { 180 | "display_name": "Python 3", 181 | "language": "python", 182 | "name": "python3" 183 | }, 184 | "language_info": { 185 | "codemirror_mode": { 186 | "name": "ipython", 187 | "version": 3 188 | }, 189 | "file_extension": ".py", 190 | "mimetype": "text/x-python", 191 | "name": "python", 192 | "nbconvert_exporter": "python", 193 | "pygments_lexer": "ipython3", 194 | "version": "3.7.7" 195 | } 196 | }, 197 | "nbformat": 4, 198 | "nbformat_minor": 2 199 | } 200 | -------------------------------------------------------------------------------- /Mathematical-Foundations/README.md: -------------------------------------------------------------------------------- 1 | # Mathematical Foundations 2 | 3 | Main mathematical concepts applied to _Machine Learning_. 4 | -------------------------------------------------------------------------------- /Mathematical-Foundations/calculus_fourierSeries__utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | import matplotlib.animation as manim 5 | 6 | 7 | def plot_function1(f): 8 | t = np.linspace(- np.pi, np.pi, 10) 9 | 10 | # visualization 11 | fig, ax = plt.subplots(1, 1) 12 | ax.plot(t, f(t), c='#FF0044') 13 | ax.plot(t*2, f(t)*2, ':', c='#FF0044') 14 | 15 | ax.scatter([-np.pi, np.pi], [-1, 1], marker='o', s=100, c='#FF0044') 16 | ax.set_xlim([-4.2*np.pi, 4.2*np.pi]) 17 | ax.set_ylim([-1.2, 1.2]) 18 | ax.set_xticks([i*np.pi for i in range(-4, 5)]) 19 | ax.set_yticks((-1, 0, 1)) 20 | xlabels = [f'${i}\pi$' for i in range(-4, 5)] 21 | xlabels = list(map(lambda x: x.replace('1', ''), xlabels)) 22 | xlabels[len(xlabels)//2] = '0' 23 | ax.set_xticklabels(xlabels) 24 | ax.set_yticklabels(['$-1$', '0', '$1$']) 25 | ax.grid(alpha=0.75, linestyle=':') 26 | ax.axhline(y=0, color='k', alpha=0.5) 27 | ax.axvline(x=0, color='k', alpha=0.5) 28 | 29 | plt.show() 30 | 31 | 32 | def plot_function2(f): 33 | t = np.linspace(- np.pi, np.pi, 100) 34 | 35 | # visualization 36 | fig, ax = plt.subplots(1, 1) 37 | ax.plot(t, f(t), c='#44AA00') 38 | ax.plot(t*2, f(t*2), ':', c='#44AA00') 39 | 40 | ax.scatter([-np.pi, np.pi], [np.pi**2, np.pi**2], marker='o', s=100, c='#44AA00') 41 | ax.set_xlim([-4.2*np.pi, 4.2*np.pi]) 42 | ax.set_ylim([-0.5, np.pi**2 + 0.5]) 43 | ax.set_xticks([i*np.pi for i in range(-4, 5)]) 44 | ax.set_yticks([i*np.pi for i in range(4)]) 45 | xlabels = [f'${i}\pi$' for i in range(-4, 5)] 46 | xlabels = list(map(lambda x: x.replace('1', ''), xlabels)) 47 | xlabels[len(xlabels)//2] = '0' 48 | ax.set_xticklabels(xlabels) 49 | ylabels = [f'${i}\pi$' for i in range(4)] 50 | ylabels = list(map(lambda x: x.replace('1', ''), ylabels)) 51 | ylabels[0] = '0' 52 | ax.set_yticklabels(ylabels) 53 | ax.grid(alpha=0.75, linestyle=':') 54 | ax.axhline(y=0, color='k', alpha=0.5) 55 | ax.axvline(x=0, color='k', alpha=0.5) 56 | 57 | plt.show() 58 | 59 | 60 | def periodic_function1(f): 61 | t = np.linspace(- np.pi, np.pi, 10) 62 | 63 | # visualization 64 | fig, ax = plt.subplots(1, 1) 65 | 66 | for i in range(5): 67 | to = - 4*np.pi + 2*np.pi*i 68 | l = ax.plot(t + to, f(t), c='#FF0044', zorder=1)[0] 69 | ax.scatter([ 70 | to - np.pi, to + np.pi], [-1, 1], 71 | marker='o', s=100, zorder=2, 72 | facecolor="white", 73 | edgecolor=l.get_color(), 74 | linewidth=l.get_linewidth() 75 | ) 76 | 77 | ax.scatter([i*np.pi for i in range(-3, 4, 2)], [0]*4, marker='o', s=100, zorder=8, c='#FF0044') 78 | ax.set_xlim([-4.2*np.pi, 4.2*np.pi]) 79 | ax.set_ylim([-1.2, 1.2]) 80 | ax.set_xticks([i*np.pi for i in range(-4, 5)]) 81 | ax.set_yticks((-1, 0, 1)) 82 | xlabels = [f'${i}\pi$' for i in range(-4, 5)] 83 | xlabels = list(map(lambda x: x.replace('1', ''), xlabels)) 84 | xlabels[len(xlabels)//2] = '0' 85 | ax.set_xticklabels(xlabels) 86 | ax.set_yticklabels(['$-1$', '0', '$1$']) 87 | ax.grid(alpha=0.75, linestyle=':') 88 | ax.axhline(y=0, color='k', alpha=0.5) 89 | ax.axvline(x=0, color='k', alpha=0.5) 90 | 91 | plt.show() 92 | 93 | 94 | def periodic_function2(f): 95 | t = np.linspace(- np.pi, np.pi, 100) 96 | 97 | # visualization 98 | fig, ax = plt.subplots(1, 1) 99 | 100 | for i in range(5): 101 | to = - 4*np.pi + 2*np.pi*i 102 | ax.plot(t + to, f(t), c='#44AA00', zorder=1)[0] 103 | 104 | ax.set_xlim([-4.2*np.pi, 4.2*np.pi]) 105 | ax.set_ylim([-0.5, np.pi**2 + 0.5]) 106 | ax.set_xticks([i*np.pi for i in range(-4, 5)]) 107 | ax.set_yticks([i*np.pi for i in range(4)]) 108 | xlabels = [f'${i}\pi$' for i in range(-4, 5)] 109 | xlabels = list(map(lambda x: x.replace('1', ''), xlabels)) 110 | xlabels[len(xlabels)//2] = '0' 111 | ax.set_xticklabels(xlabels) 112 | ylabels = [f'${i}\pi$' for i in range(4)] 113 | ylabels = list(map(lambda x: x.replace('1', ''), ylabels)) 114 | ylabels[0] = '0' 115 | ax.set_yticklabels(ylabels) 116 | ax.grid(alpha=0.75, linestyle=':') 117 | ax.axhline(y=0, color='k', alpha=0.5) 118 | ax.axvline(x=0, color='k', alpha=0.5) 119 | 120 | plt.show() 121 | 122 | 123 | def serie_function1(s): 124 | t = np.linspace(- 5*np.pi, 5*np.pi, 4410) 125 | 126 | # visualization 127 | fig, ax = plt.subplots(1, 1) 128 | 129 | ax.plot(t, s(t), c='#FF0044') 130 | ax.set_xlim([-4.2*np.pi, 4.2*np.pi]) 131 | ax.set_ylim([-1.2, 1.2]) 132 | ax.set_xticks([i*np.pi for i in range(-4, 5)]) 133 | ax.set_yticks((-1, 0, 1)) 134 | xlabels = [f'${i}\pi$' for i in range(-4, 5)] 135 | xlabels = list(map(lambda x: x.replace('1', ''), xlabels)) 136 | xlabels[len(xlabels)//2] = '0' 137 | ax.set_xticklabels(xlabels) 138 | ax.set_yticklabels(['$-1$', '0', '$1$']) 139 | ax.grid(alpha=0.75, linestyle=':') 140 | ax.axhline(y=0, color='k', alpha=0.5) 141 | ax.axvline(x=0, color='k', alpha=0.5) 142 | 143 | plt.show() 144 | 145 | 146 | def serie_function2(s): 147 | t = np.linspace(- 5*np.pi, 5*np.pi, 4410) 148 | 149 | # visualization 150 | fig, ax = plt.subplots(1, 1) 151 | 152 | ax.plot(t, s(t), c='#44AA00') 153 | ax.set_xlim([-4.2*np.pi, 4.2*np.pi]) 154 | ax.set_ylim([-0.5, np.pi**2 + 0.5]) 155 | ax.set_xticks([i*np.pi for i in range(-4, 5)]) 156 | ax.set_yticks([i*np.pi for i in range(4)]) 157 | xlabels = [f'${i}\pi$' for i in range(-4, 5)] 158 | xlabels = list(map(lambda x: x.replace('1', ''), xlabels)) 159 | xlabels[len(xlabels)//2] = '0' 160 | ax.set_xticklabels(xlabels) 161 | ylabels = [f'${i}\pi$' for i in range(4)] 162 | ylabels = list(map(lambda x: x.replace('1', ''), ylabels)) 163 | ylabels[0] = '0' 164 | ax.set_yticklabels(ylabels) 165 | ax.grid(alpha=0.75, linestyle=':') 166 | ax.axhline(y=0, color='k', alpha=0.5) 167 | ax.axvline(x=0, color='k', alpha=0.5) 168 | 169 | plt.show() 170 | -------------------------------------------------------------------------------- /Mathematical-Foundations/capital-sigma-notation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "binary-cream", 6 | "metadata": {}, 7 | "source": [ 8 | "# Capital-sigma notation\n", 9 | "---\n", 10 | "- Author: Diego Inácio\n", 11 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 12 | "- Notebook: [capital-sigma-notation.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Mathematical-Foundations/capital-sigma-notation.ipynb)\n", 13 | "---\n", 14 | "Brief notes and practical examples with the *summation* operator." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "partial-hundred", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import numpy as np" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "fifteen-owner", 30 | "metadata": {}, 31 | "source": [ 32 | "## Summation notation\n", 33 | "---\n", 34 | "From the Greek alphabet, the capital Sigma letter, represented by the symbol $\\large \\Sigma$, is mathematical notation for *summation*. This operator has as function the sum or addition of multiples elements of a sequence. It is commonly associated to another mathematical symbol (greek capital Pi, $\\large \\Pi$), but its functionality is easily memorized by the sound of the word \"sum\": **Sss**um = **Sss**igma.\n", 35 | "\n", 36 | "$$ \\large\n", 37 | "\\sum_{i=m}^n x_i = x_m + x_{m+1} + x_{m+2} + \\cdots + x_{n-2} + x_{n-1} + x_{n}\n", 38 | "$$\n", 39 | "\n", 40 | "where:\n", 41 | "- $i$ is the index of summation;\n", 42 | "- $m$ is the *lower bound of summation*;\n", 43 | "- $n$ is the *upper bound of summation*;\n", 44 | "- $x_i$ is the indexed variable.\n", 45 | "\n", 46 | "For example, given the explicit sequence [1, 3, 7, 9, 13], the summation of all 5 elements is denoted as:\n", 47 | "\n", 48 | "$$ \\large\n", 49 | "\\begin{aligned}\n", 50 | "\\sum_{i=1}^5 x_i &= x_1 + x_2 + x_3 + x_4 + x_5 \\\\\n", 51 | "&= 1 + 3 + 7 + 9 + 13 \\\\\n", 52 | "&= 33\n", 53 | "\\end{aligned}\n", 54 | "$$" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "id": "administrative-overhead", 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "seq = [1, 3, 7, 9, 13]\n", 65 | "\n", 66 | "result = 0\n", 67 | "for x in seq:\n", 68 | " result += x\n", 69 | "\n", 70 | "print(result)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "id": "latter-jumping", 76 | "metadata": {}, 77 | "source": [ 78 | "Notice that in many algorithms the lower and the upper bounds are represented by the interval $[0,N-1]$, which denote the index bound of a digital array. Although, the main idea is still the same.\n", 79 | "\n", 80 | "$$ \\large\n", 81 | "\\sum_{k=0}^{N-1} = \\sum_{k}^N\n", 82 | "$$" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "id": "satisfied-sheet", 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "x = [1, 3, 7, 9, 13]\n", 93 | "\n", 94 | "result = x[0] + x[1] + x[2] + x[3] + x[4]\n", 95 | "\n", 96 | "print(result)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "id": "peaceful-experience", 102 | "metadata": {}, 103 | "source": [ 104 | "The library [Numpy](https://numpy.org/) has a very convenient method that makes the summing process easier and much more readable." 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "id": "unique-allen", 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "x = np.array([1, 3, 7, 9, 13])\n", 115 | "\n", 116 | "result = x.sum() # or np.sum(x)\n", 117 | "\n", 118 | "print(result)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "id": "pressed-samba", 124 | "metadata": {}, 125 | "source": [ 126 | "### Multidimensional\n", 127 | "---\n", 128 | "This operator can be nested:\n", 129 | "\n", 130 | "$$ \\large\n", 131 | "\\begin{aligned}\n", 132 | "\\sum_{i=2}^4 \\sum_{j=5}^6 (i \\cdot j) &= (2 \\cdot 5 + 2 \\cdot 6) + (3 \\cdot 5 + 3 \\cdot 6) + (4 \\cdot 5 + 4 \\cdot 6) \\\\\n", 133 | "&= (10 + 12) + (15 + 18) + (20 + 24) \\\\\n", 134 | "&= 22 + 33 + 44 \\\\\n", 135 | "&= 99\n", 136 | "\\end{aligned}\n", 137 | "$$" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "id": "intensive-fiber", 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "result = 0\n", 148 | "for i in range(2, 4+1):\n", 149 | " for j in range(5, 6+1):\n", 150 | " result += i*j\n", 151 | "\n", 152 | "print(result)" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "id": "liberal-headset", 158 | "metadata": {}, 159 | "source": [ 160 | "Nested summations are very commonly used to notate multidimensional isotropic or anisotropic objects. For example, given the matrix:\n", 161 | "\n", 162 | "$$ \\large\n", 163 | "M = \n", 164 | "\\begin{bmatrix}\n", 165 | "1 & 2 \\\\\n", 166 | "2 & 3 \\\\\n", 167 | "3 & 4\n", 168 | "\\end{bmatrix}\n", 169 | "$$\n", 170 | "\n", 171 | "The sum of all elements can be expressed as:\n", 172 | "\n", 173 | "$$ \\large\n", 174 | "\\begin{aligned}\n", 175 | "\\sum_i^M \\sum_j^N M_{ij} &= 1 + 2 + 2 + 3 + 3 + 4 \\\\\n", 176 | "&= 15\n", 177 | "\\end{aligned}\n", 178 | "$$" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "id": "palestinian-cleanup", 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "matrix = np.array([\n", 189 | " [1, 2], \n", 190 | " [2, 3], \n", 191 | " [3, 4]\n", 192 | "])\n", 193 | "\n", 194 | "M = len(matrix)\n", 195 | "N = len(matrix[0])\n", 196 | "\n", 197 | "result = 0\n", 198 | "for i in range(M):\n", 199 | " for j in range(N):\n", 200 | " result += matrix[i][j]\n", 201 | "\n", 202 | "print(result, np.sum(matrix))" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "id": "lesbian-advocate", 208 | "metadata": {}, 209 | "source": [ 210 | "## Properties\n", 211 | "---\n", 212 | "Follow some basic properties and identities." 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "id": "monthly-understanding", 218 | "metadata": {}, 219 | "source": [ 220 | "- The summation of a constant $\\large c$ is the same of its multiplication by the factor $N$:\n", 221 | "\n", 222 | "$$ \\large\n", 223 | "\\sum_i^N c = N \\cdot c\n", 224 | "$$" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "id": "dominant-stack", 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "N = 10\n", 235 | "c = 2\n", 236 | "\n", 237 | "print(np.sum([c for _ in range(N)]))\n", 238 | "print(N*c)\n", 239 | "print(np.sum([c]*N))" 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "id": "subject-connectivity", 245 | "metadata": {}, 246 | "source": [ 247 | "- The [commutativity](https://en.wikipedia.org/wiki/Commutative_property) and [associative](https://en.wikipedia.org/wiki/Associative_property) properties are denoted as:\n", 248 | "\n", 249 | "$$ \\large\n", 250 | "\\sum_i^N x_i \\pm \\sum_i^N y_i = \\sum_i^N (x_i \\pm y_i)\n", 251 | "$$" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "id": "brazilian-bearing", 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "x = np.array([1, 20, 3, 40])\n", 262 | "y = np.array([10, 2, 30, 4])\n", 263 | "\n", 264 | "print(np.sum(x) - np.sum(y))\n", 265 | "print(np.sum(x - y))" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "id": "cleared-techno", 271 | "metadata": {}, 272 | "source": [ 273 | "## Examples\n", 274 | "---\n", 275 | "Follow some example of summation operator applications." 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "id": "lasting-cleveland", 281 | "metadata": {}, 282 | "source": [ 283 | "- **Infinity serie**: An infinite serie is when the upper bound is infinity and its summation result converges to a known theoretical number. For example, the [Basel problem](https://en.wikipedia.org/wiki/Basel_problem):\n", 284 | "\n", 285 | "$$ \\large\n", 286 | "\\sum_{k=1}^\\infty \\frac{1}{k^2} = \\lim_{n \\rightarrow \\infty} \\sum_{k=1}^n \\frac{1}{k^2} = \\frac{\\pi^2}{6}\n", 287 | "$$" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "id": "filled-heath", 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "K = np.arange(1e6) + 1\n", 298 | "\n", 299 | "result = np.sum(1/K**2)\n", 300 | "\n", 301 | "print(result, np.pi**2/6)" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "id": "convenient-prescription", 307 | "metadata": {}, 308 | "source": [ 309 | "- **Alternating serie**: The signs of the general terms alternate between positive and negative and its infinite serie always converges.\n", 310 | "\n", 311 | "$$ \\large\n", 312 | "\\sum_{k=1}^n=\\frac{(-1)^{k+1}}{k^2}\n", 313 | "$$" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": null, 319 | "id": "divine-lunch", 320 | "metadata": {}, 321 | "outputs": [], 322 | "source": [ 323 | "K = np.arange(100) + 1\n", 324 | "\n", 325 | "result = np.sum((-1)**(K+1)/K**2)\n", 326 | "\n", 327 | "print(result)" 328 | ] 329 | } 330 | ], 331 | "metadata": { 332 | "kernelspec": { 333 | "display_name": "Python [conda env:root] *", 334 | "language": "python", 335 | "name": "conda-root-py" 336 | }, 337 | "language_info": { 338 | "codemirror_mode": { 339 | "name": "ipython", 340 | "version": 3 341 | }, 342 | "file_extension": ".py", 343 | "mimetype": "text/x-python", 344 | "name": "python", 345 | "nbconvert_exporter": "python", 346 | "pygments_lexer": "ipython3", 347 | "version": "3.7.10" 348 | } 349 | }, 350 | "nbformat": 4, 351 | "nbformat_minor": 5 352 | } 353 | -------------------------------------------------------------------------------- /Mathematical-Foundations/dissimilarity__utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from matplotlib import cm 4 | from matplotlib import transforms 5 | import numpy as np 6 | 7 | ################## 8 | # Synthetic data # 9 | ################## 10 | 11 | r = 1 12 | np.random.seed(123456) 13 | y1A = np.random.uniform(-r, r, 8) 14 | y2A = np.random.uniform(-r, r, 8) 15 | yA = np.array([y1A, y2A]).T 16 | M, N = 32j, 32j 17 | s, t = np.mgrid[-r:r:N*8, -r:r:M*8] 18 | yB = np.array([s.ravel(), t.ravel()]).T 19 | 20 | clni = ''' 21 | Um cabra de Lampião 22 | Por nome Pilão Deitado 23 | Que morreu numa trincheira 24 | Em certo tempo passado 25 | Agora pelo sertão 26 | Anda correndo visão 27 | Fazendo mal-assombrado 28 | 29 | E foi quem trouxe a notícia 30 | Que viu Lampião chegar 31 | O Inferno nesse dia 32 | Faltou pouco pra virar 33 | Incendiou-se o mercado 34 | Morreu tanto cão queimado 35 | Que faz pena até contar 36 | 37 | Morreu a mãe de Canguinha 38 | O pai de Forrobodó 39 | Três netos de Parafuso 40 | Um cão chamado Cotó 41 | Escapuliu Boca Ensossa 42 | E uma moleca moça 43 | Quase queimava o “totó” 44 | 45 | Morreram 100 negros velhos 46 | Que não trabalhavam mais 47 | Um cão chamado Trás-cá 48 | Vira-volta e Capataz 49 | Tromba Suja e Bigodeira 50 | Um cão chamado Goteira 51 | Cunhado de Satanás 52 | ''' 53 | 54 | #################### 55 | # Base conversions # 56 | #################### 57 | 58 | def conv_baseDecimal(N, b): 59 | S = {e:i for i, e in enumerate('0123456789ABCDEF')} 60 | N = N.upper() 61 | Ns = N.split('.' or ',') 62 | if len(Ns) == 1: 63 | n = len(Ns[0]) 64 | m = 0 65 | elif len(Ns) == 2: 66 | n = len(Ns[0]) 67 | m = len(Ns[1]) 68 | else: 69 | raise ValueError('The value must have just one fractional part!') 70 | N = ''.join(Ns)[::-1] 71 | if not all([S[d] <= b - 1 for d in N]): 72 | raise ValueError('The value of the digits must be less than the base!') 73 | return sum([S[d]*b**k for d, k in zip(N, range(-m, n))]) 74 | 75 | def conv_decimalBase(N, b, prec=32): 76 | S = {i:e for i, e in enumerate('0123456789ABCDEF')} 77 | i, f = divmod(N, 1) 78 | saida = '0' if not i else '' 79 | while(i): 80 | i, r = divmod(i, b) 81 | saida += S[r] 82 | saida = saida[::-1] + '.' 83 | 84 | while(f and len(saida) <= prec): 85 | f = round(f, 10) 86 | i, f = divmod(f*b, 1) 87 | saida += S[int(i)] 88 | return saida 89 | 90 | def ieee754_realBin(N, n=32): 91 | k = {16: 10, 32: 23, 64: 52, 128: 112, 256: 236} 92 | if n not in k: 93 | raise ValueError('Invalid accuracy value!') 94 | k = k[n] 95 | nq = n - k - 1 96 | s = '1' if N < 0 else '0' 97 | Nb = conv_decimalBase(abs(N), 2, n) 98 | a, b = Nb.find('1'), Nb.find('.') 99 | e = b - a if a > b else b - a - 1 100 | vies = 2**(nq)/2 - 1 101 | q = conv_decimalBase(vies + e, 2, nq) 102 | q = q[:-1].rjust(nq, '0') 103 | Nb = Nb.replace('.', '') 104 | p = Nb[a+1:a+k+1].ljust(k, '0') 105 | return s + q + p 106 | 107 | def ieee754_binReal(N): 108 | k = {16: 10, 32: 23, 64: 52, 128: 112, 256: 236} 109 | n = len(N) 110 | if n not in k: 111 | raise ValueError('Invalid IEEE 754 pattern!') 112 | k = k[n] 113 | nq = n - k - 1 114 | v = 2**(nq)/2 - 1 115 | s = eval(N[0]) 116 | q = N[1:nq+1] 117 | e = conv_baseDecimal(q, 2) - v 118 | p = '0.' + N[-k:] 119 | p = conv_baseDecimal(p, 2) 120 | return (-1)**s*(1 + p)*2**e 121 | 122 | ######### 123 | # Plots # 124 | ######### 125 | 126 | plt.rcParams['figure.figsize'] = (16, 8) 127 | cmap = cm.get_cmap('jet') 128 | 129 | def plotPoints(ax, x, d, dmax): 130 | xo, yo = x 131 | for i, [xi, yi] in enumerate(yA): 132 | color = cmap(d[i]/dmax) 133 | ax.plot([xo, xi], [yo, yi], '--', c=color, zorder=0) 134 | cx = (xi + xo)*0.5 135 | cy = (yi + yo)*0.5 136 | m = (yi - xo)/(xi - yo) 137 | theta = np.rad2deg(np.arctan(m)) 138 | ax.text(cx, cy, '{0:.02f}'.format(d[i]), 139 | rotation=theta, fontsize=12, 140 | horizontalalignment='center', 141 | verticalalignment='center', 142 | bbox=dict(boxstyle='round', 143 | color=color, 144 | alpha=0.5)) 145 | ax.scatter(*yA.T, zorder=2) 146 | ax.scatter(*x, c='k', zorder=2, lw=3) 147 | ax.annotate('x = ({0}, {1})'.format(*x), 148 | xy=x, xytext=(xo + 0.1, yo + 0.2), 149 | ha='left', va='bottom', 150 | fontsize=12, 151 | bbox=dict(boxstyle='round,pad=0.5', 152 | fc='k', 153 | alpha=0.25), 154 | arrowprops=dict(facecolor = 'black', 155 | alpha=0.5, 156 | width=1, 157 | headwidth=6, 158 | connectionstyle='arc3,rad=0.25')) 159 | ax.set_xlim([-r, r]) 160 | ax.set_ylim([-r, r]) 161 | 162 | def plotContour(ax, d, title='', fsize=12): 163 | mesh = ax.pcolormesh(s, t, d, cmap='jet') 164 | cont = ax.contour(s, t, d, 7, cmap = 'gray') 165 | ax.clabel(cont, fontsize=fsize, inline=True) 166 | ax.set_xlim([-r, r]) 167 | ax.set_ylim([-r, r]) 168 | ax.set_title(title) 169 | return mesh 170 | 171 | def plotDist(x, dA, dB, figname, ptitle='', ctitle='', save=False): 172 | fig, (axA, axB) = plt.subplots(1, 2) 173 | 174 | dmax = np.max([dA.max(), dB.max()]) 175 | 176 | plotPoints(axA, x, dA, dmax) 177 | mesh = plotContour(axB, dB, ctitle) 178 | 179 | cbar_ax = fig.add_axes([0.125, 0.025, 0.775, 0.05]) 180 | fig.colorbar(mesh, cax = cbar_ax, orientation = 'horizontal') 181 | fig.suptitle(' '.join([e.capitalize() for e in figname.split('_')])) 182 | if save: 183 | fig.savefig('_output/similarity_{}.png'.format(figname), bbox_inches='tight') 184 | 185 | def plotText(p, q, dT, figname, save=False): 186 | fig, axA = plt.subplots(1, 1) 187 | xo, yo = 0.05, 1 188 | for i, est in enumerate(q.split('\n\n')): 189 | yo -= 0.05 190 | if i == len(q.split('\n\n'))//2: 191 | xo = 0.55 192 | yo = 0.9 193 | for j, ver in enumerate(est.split('\n')): 194 | ax = plt.gca() 195 | t = ax.transData 196 | canvas = ax.figure.canvas 197 | yo -= 0.05 198 | for k, pal in enumerate(ver.split()): 199 | text = axA.text(xo, yo, pal + ' ',color=cmap(dT[pal]/max(dT.values())), 200 | fontsize=22, 201 | transform=t, 202 | horizontalalignment='left', 203 | verticalalignment='top') 204 | text.draw(canvas.get_renderer()) 205 | ex = text.get_window_extent() 206 | t = transforms.offset_copy(text._transform, x=ex.width, units='dots') 207 | cbar_ax = fig.add_axes([0.125, 0.025, 0.775, 0.05]) 208 | norm = matplotlib.colors.Normalize(vmin=0, vmax=max(dT.values())) 209 | matplotlib.colorbar.ColorbarBase(cbar_ax, 210 | cmap=cmap, 211 | norm=norm, 212 | orientation='horizontal') 213 | axA.set_title(' '.join([e.capitalize() for e in figname.split('_')])) 214 | axA.axis('off') 215 | if save: 216 | fig.savefig('_output/similarity_{}.png'.format(figname), bbox_inches='tight') -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_addition.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_addition.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_angle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_angle.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_orthogonal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_orthogonal.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_parallel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_parallel.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_scalar_multiplication.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_scalar_multiplication.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_subtraction.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_subtraction.jpg -------------------------------------------------------------------------------- /Mathematical-Foundations/sourceimages/linear-algebra_vectors_unit_vector.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Mathematical-Foundations/sourceimages/linear-algebra_vectors_unit_vector.jpg -------------------------------------------------------------------------------- /Natural-Language-Processing/README.md: -------------------------------------------------------------------------------- 1 | # Natural Language Processing 2 | 3 | Everything about Natural Language Processing, from fundamental concepts to practical applications. 4 | -------------------------------------------------------------------------------- /Natural-Language-Processing/basics_NLTK.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Basics [NLTK]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [basics_NLTK.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Natural-Language-Processing/basics_NLTK.ipynb)\n", 12 | "---\n", 13 | "Basics aspects of [Natural Language Toolkit](https://www.nltk.org/)." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Installation and download data files\n", 30 | "---\n", 31 | "\n", 32 | "[Installation](http://numba.pydata.org/numba-doc/latest/user/installing.html) command for *anaconda* and *pip*:\n", 33 | "\n", 34 | "```\n", 35 | "$ conda install --channel anaconda nltk\n", 36 | "```\n", 37 | "\n", 38 | "or\n", 39 | "\n", 40 | "```\n", 41 | "$ pip install nltk\n", 42 | "```" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "import nltk" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "To install all the data requirement for NLTK, first define the output directory and download it by running:\n" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "PATH = 'D:/GitHub/machine-learning-notebooks/Natural-Language-Processing/nltk_data'\n", 68 | "nltk.data.path.append(PATH)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "nltk.download(download_dir=PATH)" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "## Tokenize\n", 85 | "---\n", 86 | "*Tokenization* is the process of breaking up a text into pieces of text called *Token*. Tokenization can happen at several different levels, like: paragraphs, sentences, words, syllables, or phonemes.\n", 87 | "\n", 88 | "Given the example text:\n" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "text = 'Hello everyone, how are you all? This is an example of text, which will be tokenized in several ways. Thank you!'" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "### Tokenize sentences\n", 105 | "---" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "from nltk.tokenize import sent_tokenize" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "sentences = sent_tokenize(text)\n", 124 | "for sentence in sentences:\n", 125 | " print(sentence)" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "### Tokenize words\n", 133 | "---" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [ 142 | "from nltk.tokenize import word_tokenize" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "words = word_tokenize(text)\n", 152 | "for word in words:\n", 153 | " print(word)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "markdown", 158 | "metadata": {}, 159 | "source": [ 160 | "## Penn Part of Speech Tags\n", 161 | "---\n", 162 | "Alphabetical list of part-of-speech tags used in the Penn Treebank Project.\n", 163 | "\n", 164 | "number | tag | description\n", 165 | "-------|------|-------------\n", 166 | "1 | CC\t | Coordinating conjunction\n", 167 | "2 | CD\t | Cardinal number\n", 168 | "3 | DT\t | Determiner\n", 169 | "4 | EX\t | Existential there\n", 170 | "5 | FW\t | Foreign word\n", 171 | "6 | IN\t | Preposition or subordinating conjunction\n", 172 | "7 | JJ\t | Adjective\n", 173 | "8 | JJR | Adjective, comparative\n", 174 | "9 | JJS | Adjective, superlative\n", 175 | "10 | LS\t | List item marker\n", 176 | "11 | MD\t | Modal\n", 177 | "12 | NN\t | Noun, singular or mass\n", 178 | "13 | NNS | Noun, plural\n", 179 | "14 | NNP | Proper noun, singular\n", 180 | "15 | NNPS | Proper noun, plural\n", 181 | "16 | PDT | Predeterminer\n", 182 | "17 | POS | Possessive ending\n", 183 | "18 | PRP | Personal pronoun\n", 184 | "19 | PRP\\$| Possessive pronoun\n", 185 | "20 | RB\t | Adverb\n", 186 | "21 | RBR | Adverb, comparative\n", 187 | "22 | RBS | Adverb, superlative\n", 188 | "23 | RP\t | Particle\n", 189 | "24 | SYM | Symbol\n", 190 | "25 | TO\t | to\n", 191 | "26 | UH\t | Interjection\n", 192 | "27 | VB\t | Verb, base form\n", 193 | "28 | VBD | Verb, past tense\n", 194 | "29 | VBG | Verb, gerund or present participle\n", 195 | "30 | VBN | Verb, past participle\n", 196 | "31 | VBP | Verb, non-3rd person singular present\n", 197 | "32 | VBZ | Verb, 3rd person singular present\n", 198 | "33 | WDT | Wh-determiner\n", 199 | "34 | WP\t | Wh-pronoun\n", 200 | "35 | WP\\$ | Possessive wh-pronoun\n", 201 | "36 | WRB | Wh-adverb" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "tags = nltk.pos_tag(words)\n", 211 | "print(tags)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "## Chunking\n", 219 | "---\n", 220 | "[Chunking](http://www.nltk.org/howto/chunk.html) uses a special syntax for *regular expressions* rules that delimit the [chunks](https://www.nltk.org/api/nltk.chunk.html).\n", 221 | "\n", 222 | "For the following example, lets find any *noun* or *proper noun* (NN, NNS, NNP or NNPS) followed by a punctuation mark." 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "rule = r'Chunk: {+<.>}'\n", 232 | "parser = nltk.RegexpParser(rule)\n", 233 | "chunk = parser.parse(tags)\n", 234 | "\n", 235 | "chunk.draw()" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "![chunk tree](chunk_tree_temp.png)" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "## Stemming\n", 250 | "---\n", 251 | "[Stemming](https://www.nltk.org/howto/stem.html) removes all morphological affixes from words and leaves only the word stem." 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "metadata": {}, 258 | "outputs": [], 259 | "source": [ 260 | "from nltk.stem import PorterStemmer" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "words = ['tokenization', 'running', 'pythonic', 'understandable', 'avoidable', 'memorable']\n", 270 | "PS = PorterStemmer()\n", 271 | "for word in words:\n", 272 | " print(f'{word} -> {PS.stem(word)}')" 273 | ] 274 | }, 275 | { 276 | "cell_type": "markdown", 277 | "metadata": {}, 278 | "source": [ 279 | "## Lemmatization\n", 280 | "---\n", 281 | "Lemmatization is the process of converting a word to its meaningful base form." 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": null, 287 | "metadata": {}, 288 | "outputs": [], 289 | "source": [ 290 | "from nltk.stem import WordNetLemmatizer" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "words = ['children', 'feet', 'wolves', 'indices', 'leaves', 'mice', 'phenomena']\n", 300 | "WL = WordNetLemmatizer()\n", 301 | "for word in words:\n", 302 | " print(f'{word} -> {WL.lemmatize(word)}')" 303 | ] 304 | } 305 | ], 306 | "metadata": { 307 | "kernelspec": { 308 | "display_name": "Python 3", 309 | "language": "python", 310 | "name": "python3" 311 | }, 312 | "language_info": { 313 | "codemirror_mode": { 314 | "name": "ipython", 315 | "version": 3 316 | }, 317 | "file_extension": ".py", 318 | "mimetype": "text/x-python", 319 | "name": "python", 320 | "nbconvert_exporter": "python", 321 | "pygments_lexer": "ipython3", 322 | "version": "3.7.7" 323 | } 324 | }, 325 | "nbformat": 4, 326 | "nbformat_minor": 2 327 | } 328 | -------------------------------------------------------------------------------- /Practical-Applications/README.md: -------------------------------------------------------------------------------- 1 | # Practical Applications 2 | 3 | Practical experiments and creative applications in _machine learning_. 4 | -------------------------------------------------------------------------------- /Practical-Applications/_data/woman01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/Practical-Applications/_data/woman01.png -------------------------------------------------------------------------------- /Practical-Applications/image_approximation_deepNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Image Approximation\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [image_approximation_deepNN.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Practical-Applications/image_approximation_deepNN.ipynb)\n", 12 | "---\n", 13 | "*Image approximation* and *upscaling interpolation* using *deep Neural Network*." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np\n", 26 | "from PIL import Image\n", 27 | "\n", 28 | "import tensorflow as tf" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "plt.rcParams['figure.figsize'] = (16, 8)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Data exploration\n", 45 | "---" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "n1, n2 = 128, 128\n", 55 | "# Read Image\n", 56 | "x = Image.open('_data/woman01.png')\n", 57 | "# Rescale image to a lower resolution\n", 58 | "x = x.resize((n1, n2), Image.ANTIALIAS)\n", 59 | "x = np.asarray(x)/255\n", 60 | "n1, n2, c = x.shape\n", 61 | "\n", 62 | "### split channels ###\n", 63 | "r, g, b = x[:,:,0], x[:,:,1], x[:,:,2]" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# Transform data and produce X_train\n", 73 | "Y_train = np.array([r.ravel(), g.ravel(), b.ravel()]).T\n", 74 | "t, s = np.mgrid[0:n1, 0:n2]\n", 75 | "s = (s - s.mean())/s.std()\n", 76 | "t = (t - t.mean())/t.std()\n", 77 | "# X_train is the normalized spatial coordinates\n", 78 | "X_train = np.array([s.ravel(), t.ravel()], dtype=np.float32).T" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "fig, [axA, axB] = plt.subplots(1, 2, figsize=(20, 10))\n", 88 | "\n", 89 | "st = np.stack([s, t, s*0], axis=2)\n", 90 | "st = (st - st.min())/(st.max() - st.min())\n", 91 | "st[:,:,2] = 0; axA.imshow(st); axA.axis('off')\n", 92 | "axA.text(5, 10, f'({s.min():.3f}, {t.min():.3f})', color='white', size=18)\n", 93 | "axA.text(86, 10, f'({s.max():.3f}, {t.min():.3f})', color='white', size=18)\n", 94 | "axA.text(5, 120, f'({s.min():.3f}, {t.max():.3f})', color='white', size=18)\n", 95 | "axA.text(86, 120, f'({s.max():.3f}, {t.max():.3f})', color='white', size=18)\n", 96 | "axA.set_title(f'$X_{{train}}$ ({n1} x {n2})', size=20)\n", 97 | "\n", 98 | "axB.imshow(x)\n", 99 | "axB.set_title(f'$Y_{{train}}$ ({n1} x {n2})', size=20)\n", 100 | "\n", 101 | "plt.show()" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "# Produce X_test to another scale\n", 111 | "# Upscale the image approximation\n", 112 | "N1 = N2 = 512\n", 113 | "t, s = np.mgrid[0:N1, 0:N2]\n", 114 | "s = (s - s.mean())/s.std()\n", 115 | "t = (t - t.mean())/t.std()\n", 116 | "X_test = np.array([s.ravel(), t.ravel()], dtype=np.float32).T\n", 117 | "\n", 118 | "print('X_train:', X_train.shape)\n", 119 | "print('Y_train:', Y_train.shape)\n", 120 | "print('X_test:', X_test.shape)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "## Deep Neural Network\n", 128 | "---" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "# Avoid error: \"InternalError: Blas GEMM launch failed...\" (RTX card)\n", 138 | "physical_devices = tf.config.experimental.list_physical_devices('GPU')\n", 139 | "tf.config.experimental.set_memory_growth(physical_devices[0], True)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "metadata": {}, 146 | "outputs": [], 147 | "source": [ 148 | "# List of number of neurons for each hidden layer\n", 149 | "NEURONS = [2**i for i in range(5, 11)]\n", 150 | "\n", 151 | "# model \n", 152 | "# Input (2): x and y coordinates\n", 153 | "# Output (3): RGB\n", 154 | "model = tf.keras.Sequential([\n", 155 | " tf.keras.layers.Input(2),\n", 156 | " *[tf.keras.layers.Dense(n, activation='relu') for n in NEURONS],\n", 157 | " tf.keras.layers.Dense(3, activation='sigmoid')\n", 158 | "])\n", 159 | "\n", 160 | "model.summary()" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "model.compile(\n", 170 | " loss='mean_squared_error',\n", 171 | " optimizer='adam',\n", 172 | " metrics=['accuracy']\n", 173 | ")\n", 174 | "\n", 175 | "# Train model\n", 176 | "for i in range(5):\n", 177 | " print(f'\\nepochs: {i*100:04d} - {(i + 1)*100:04d}')\n", 178 | " model.fit(\n", 179 | " X_train, Y_train,\n", 180 | " epochs=99,\n", 181 | " verbose=0\n", 182 | " )\n", 183 | " model.fit(\n", 184 | " X_train, Y_train,\n", 185 | " epochs=1\n", 186 | " )" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": null, 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "fig, [axA, axB] = plt.subplots(1, 2, figsize=(20, 10))\n", 196 | "\n", 197 | "axA.imshow(x)\n", 198 | "axA.set_title(f'$Y_{{train}}$ ({n1} x {n2})', size=20)\n", 199 | "# Predict using X_train (128x128x3)\n", 200 | "Y_predA = model.predict(X_train)\n", 201 | "Y_predA = Y_predA.reshape(n1, n2, c)\n", 202 | "axB.imshow(Y_predA)\n", 203 | "axB.set_title(f'$\\hat{{Y}}_{{train}}$ ({n1} x {n2})', size=20)\n", 204 | "\n", 205 | "plt.show()" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [ 214 | "fig, axA = plt.subplots(1, 1, figsize=(20, 20))\n", 215 | "\n", 216 | "# Predict using X_test (512x512x3)\n", 217 | "Y_predB = model.predict(X_test)\n", 218 | "Y_predB = Y_predB.reshape(N1, N2, c)\n", 219 | "axA.imshow(Y_predB)\n", 220 | "axA.set_title(f'$\\hat{{Y}}_{{test}}$ ({N1} x {N2})', size=20)\n", 221 | "\n", 222 | "plt.show()" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "metadata": {}, 228 | "source": [ 229 | "## Learning process visualization\n", 230 | "---" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "from IPython.display import HTML\n", 240 | "\n", 241 | "HTML(\n", 242 | " ''\n", 243 | ")" 244 | ] 245 | } 246 | ], 247 | "metadata": { 248 | "kernelspec": { 249 | "display_name": "Python [conda env:tfgpu]", 250 | "language": "python", 251 | "name": "conda-env-tfgpu-py" 252 | }, 253 | "language_info": { 254 | "codemirror_mode": { 255 | "name": "ipython", 256 | "version": 3 257 | }, 258 | "file_extension": ".py", 259 | "mimetype": "text/x-python", 260 | "name": "python", 261 | "nbconvert_exporter": "python", 262 | "pygments_lexer": "ipython3", 263 | "version": "3.7.7" 264 | } 265 | }, 266 | "nbformat": 4, 267 | "nbformat_minor": 2 268 | } 269 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning Notebooks 2 | 3 | [![Machine Learning Notebooks](docs/assets/images/social-preview.png)](https://diegoinacio.github.io/machine-learning-notebooks/) 4 | 5 | This is an `authorial set` of fundamental _Python recipes_ in notebook format. I started this project with the sole purpose of helping people to understand **Machine Learning** and **Artificial Intelligence** concepts as clearly and succinctly as I could. 6 | 7 | In addition, I also developed a bit more appealing way to publish these notebooks, in order to make the experience a little more pleasant and productive. You can check it by visiting the [cookbook page](https://diegoinacio.github.io/machine-learning-notebooks/). 8 | 9 | [![visit cookbook](docs/assets/icons/visit_cookbook.svg)](https://diegoinacio.github.io/machine-learning-notebooks/) 10 | 11 | You can join the [discussion](https://github.com/diegoinacio/machine-learning-notebooks/discussions) section to ask questions and suggest new ideas. And to follow what lies ahead, check out my [python notebook content](https://github.com/users/diegoinacio/projects/6) board. 12 | 13 | [![join discussion](docs/assets/icons/join_discussion.svg)](https://github.com/diegoinacio/machine-learning-notebooks/discussions) 14 | 15 | This project is part of the collection [Python Notebooks](https://diegoinacio.github.io/python-notebooks/), which is a gathering of cookbooks on **Computer Science**, **Data Science**, **Machine Learning**, **Computer Vision** and more. 16 | 17 | [![go python notebooks](docs/assets/icons/go_python_notebooks.svg)](https://diegoinacio.github.io/python-notebooks/) 18 | 19 | If you feel that something here has helped you in somehow, please consider [supporting](https://ko-fi.com/diegoinacio/) this project: 20 | 21 | [![support this project](docs/assets/icons/support_this_project.svg)](https://ko-fi.com/diegoinacio/) 22 | 23 | Created with ❤️ by [Diego Inácio](https://diegoinacio.github.io/) 24 | -------------------------------------------------------------------------------- /Time-Series/README.md: -------------------------------------------------------------------------------- 1 | # Time Series 2 | 3 | Everything about Time Series Analysis and Forecasting, from fundamental concepts to practical applications. 4 | -------------------------------------------------------------------------------- /Tips-and-Tricks/README.md: -------------------------------------------------------------------------------- 1 | # Tips and Tricks 2 | 3 | A gathering of _Tips & Tricks_ involving any supporting information for _machine learning_ in general. 4 | -------------------------------------------------------------------------------- /Tips-and-Tricks/basics_Cython.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Basics [Cython]\n", 8 | "---\n", 9 | "- Author: Diego Inácio\n", 10 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 11 | "- Notebook: [basics_Cython.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Tips-and-Tricks/basics_Cython.ipynb)\n", 12 | "---\n", 13 | "Basic functions and operations using [Cython](https://cython.org/) and *Python*." 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## Installation\n", 21 | "---\n", 22 | "\n", 23 | "[Installation](http://docs.cython.org/en/latest/src/quickstart/install.html) command for *anaconda* and *pip*:\n", 24 | "\n", 25 | "```\n", 26 | "$ conda install -c anaconda cython\n", 27 | "```\n", 28 | "\n", 29 | "or\n", 30 | "\n", 31 | "```\n", 32 | "$ pip install Cython\n", 33 | "```" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "import cython" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "## Compilation\n", 50 | "---\n", 51 | "A *Cython* source file has the name of the module followed by the extension `.pyx`. For example, given the source file `examples_cy.pyx` with a simple function which returns a string.\n", 52 | "\n", 53 | "```python\n", 54 | "def hello_cython():\n", 55 | " return 'Hello, Cython!'\n", 56 | "```\n", 57 | "\n", 58 | "The following step consist of creating the `setup.py`, which will be responsible for the compilation process.\n", 59 | "\n", 60 | "```python\n", 61 | "from setuptools import setup\n", 62 | "from Cython.Build import cythonize\n", 63 | "\n", 64 | "setup(\n", 65 | " name=\"Examples Cython\",\n", 66 | " ext_modules=cythonize(\"examples_cy.pyx\")\n", 67 | ")\n", 68 | "```\n", 69 | "\n", 70 | "Given that, the compilation step is done by running the command:\n", 71 | "\n", 72 | "```\n", 73 | "$ python setup.py build_ext --inplace\n", 74 | "```" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "from examples_cy import hello_cython" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "print(hello_cython())" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "## Performance\n", 100 | "---\n", 101 | "The following example, we will try to approximate the value $\\large\\pi$ with the idea of $\\\\tan^{-1}1=\\frac{\\pi}{4}$ using the power series of *arctan*, defined by:\n", 102 | "\n", 103 | "$$\\large\n", 104 | "4 \\sum_{n=0}^{N}\\frac{(-1)^n}{2n+1} \\approx \\pi\n", 105 | "$$\n", 106 | "\n", 107 | "where $N$ tends to the infinity." 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "def pi_py(N):\n", 117 | " pi = 0\n", 118 | " for n in range(N):\n", 119 | " pi += (-1)**n/(2*n + 1)\n", 120 | " return 4*pi\n", 121 | "\n", 122 | "print(pi_py(1000000))" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "In the same *Cython* source file `examples_cy.pyx`, lets include the function and adapt it to be compiled.\n", 130 | "\n", 131 | "```python\n", 132 | "cdef double pi_cy(int N):\n", 133 | " cdef double pi = 0\n", 134 | " cdef int n\n", 135 | " for n in range(N):\n", 136 | " pi += (-1)**n/(2*n + 1)\n", 137 | " return 4*pi\n", 138 | "```\n", 139 | "\n", 140 | "*p.s.: compile it again running the command:*\n", 141 | "\n", 142 | "```\n", 143 | "$ python setup.py build_ext --inplace\n", 144 | "```" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "from examples_cy import pi_cy" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "# Time measurement over the situations\n", 163 | "print('[Python] pi_py |', end=' ')\n", 164 | "%timeit -n 5 -r 5 pi_py(1000000)\n", 165 | "print('[Cython] pi_cy |', end=' ')\n", 166 | "%timeit -n 5 -r 5 pi_cy(1000000)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "## Cython and Jupyter Notebook\n", 174 | "---\n", 175 | "To enable support for *Cython* compilation in *Jupyter Notebooks*, we have to run firstly the command:" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "%load_ext Cython" 185 | ] 186 | }, 187 | { 188 | "cell_type": "markdown", 189 | "metadata": {}, 190 | "source": [ 191 | "It will allow the *C functions* declaration inside cells, using the magic function `%%cython` for multiple lines.\n", 192 | "\n", 193 | "*p.s.: the function call must be within the same cell*" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "%%cython\n", 203 | "\n", 204 | "cdef int factorial(int x):\n", 205 | " if x <= 1:\n", 206 | " return 1\n", 207 | " return x*factorial(x - 1)\n", 208 | "\n", 209 | "print(factorial(10))" 210 | ] 211 | } 212 | ], 213 | "metadata": { 214 | "kernelspec": { 215 | "display_name": "Python 3", 216 | "language": "python", 217 | "name": "python3" 218 | }, 219 | "language_info": { 220 | "codemirror_mode": { 221 | "name": "ipython", 222 | "version": 3 223 | }, 224 | "file_extension": ".py", 225 | "mimetype": "text/x-python", 226 | "name": "python", 227 | "nbconvert_exporter": "python", 228 | "pygments_lexer": "ipython3", 229 | "version": "3.7.10" 230 | } 231 | }, 232 | "nbformat": 4, 233 | "nbformat_minor": 2 234 | } 235 | -------------------------------------------------------------------------------- /Tips-and-Tricks/connect_openai_api.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "attachments": {}, 5 | "cell_type": "markdown", 6 | "id": "selective-opinion", 7 | "metadata": {}, 8 | "source": [ 9 | "# Connect to OpenAI API\n", 10 | "---\n", 11 | "- Author: Diego Inácio\n", 12 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 13 | "- Notebook: [connect_openai_api.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Tips-and-Tricks/connect_openai_api.ipynb)\n", 14 | "---\n", 15 | "Step by step how to connect to *OpenAI API* and brief introduction.\n", 16 | "\n", 17 | "[OpenAI's API](https://openai.com/api/) is a set of tools and services that allow developers to train and deploy AI models. The API provides access to OpenAI's vast collection of resources, including datasets, algorithms, and pretrained models. The API also allows developers to create and train their own AI models." 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "id": "velvet-cooper", 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "from IPython.display import (\n", 28 | " HTML, display\n", 29 | ")\n", 30 | "\n", 31 | "import os\n", 32 | "import dotenv as de\n", 33 | "\n", 34 | "import openai" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "exceptional-debate", 40 | "metadata": {}, 41 | "source": [ 42 | "## Installation\n", 43 | "---\n", 44 | "To install the unofficial API and libraries, just the following command for pip or anaconda:\n", 45 | "\n", 46 | "``` python\n", 47 | ">>> pip install openai\n", 48 | "```\n", 49 | "\n", 50 | "or \n", 51 | "\n", 52 | "``` python\n", 53 | ">>> conda install -c conda-forge openai\n", 54 | "```" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "id": "modern-hundred", 60 | "metadata": {}, 61 | "source": [ 62 | "## Getting the API key\n", 63 | "---\n", 64 | "In order to connect to the *OpenAI's API* the developer needs access to an **API key**. To get one, first create an account at [beta.openai.com](https://beta.openai.com/) and [create a new secret key](https://beta.openai.com/account/api-keys) by accessing their user settings.\n", 65 | "\n", 66 | "After creating, save this secret key somewhere safe and accessible. You won't be able to view it again through your OpenAI account. If you lose this secret key, you'll need to generate a new one. The generated key must seem like:\n", 67 | "\n", 68 | "| SECRET KEY | CREATED | LAST USED |\n", 69 | "| :--------: | :----------: | :-------: |\n", 70 | "| sk-...Ksf8 | Jan 01, 2022 | Never |\n", 71 | "\n", 72 | "To configure the API with our generated secret key, use the python command:\n", 73 | "\n", 74 | "``` python\n", 75 | "openai.api_key = \n", 76 | "```\n", 77 | "\n", 78 | "In order to protect your key from explicit usage, I strongly recommend using some environment variable method or so. For this example, I'll be using [dotenv](https://pypi.org/project/python-dotenv/) and an external `.env` file." 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "id": "regional-polls", 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "de.load_dotenv(de.find_dotenv())\n", 89 | "\n", 90 | "OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")\n", 91 | "\n", 92 | "openai.api_key = OPENAI_API_KEY" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "id": "american-brunswick", 98 | "metadata": {}, 99 | "source": [ 100 | "## Connection test\n", 101 | "---\n", 102 | "To test the connection with OpenAI API, we will generate natural language text using a [GPT-3](https://beta.openai.com/docs/models/gpt-3) model." 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "id": "mental-student", 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "PROMPT = \"What are Artificial Intelligence and Machine Learning?\"\n", 113 | "\n", 114 | "response = openai.Completion.create(\n", 115 | " model=\"text-davinci-003\",\n", 116 | " prompt=PROMPT,\n", 117 | " max_tokens=100\n", 118 | ")\n", 119 | "\n", 120 | "print(response[\"choices\"][0][\"text\"])" 121 | ] 122 | } 123 | ], 124 | "metadata": { 125 | "kernelspec": { 126 | "display_name": "Python [conda env:root] *", 127 | "language": "python", 128 | "name": "conda-root-py" 129 | }, 130 | "language_info": { 131 | "codemirror_mode": { 132 | "name": "ipython", 133 | "version": 3 134 | }, 135 | "file_extension": ".py", 136 | "mimetype": "text/x-python", 137 | "name": "python", 138 | "nbconvert_exporter": "python", 139 | "pygments_lexer": "ipython3", 140 | "version": "3.7.10" 141 | } 142 | }, 143 | "nbformat": 4, 144 | "nbformat_minor": 5 145 | } 146 | -------------------------------------------------------------------------------- /Tips-and-Tricks/kaggle-data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "attachments": {}, 5 | "cell_type": "markdown", 6 | "id": "amended-avatar", 7 | "metadata": {}, 8 | "source": [ 9 | "# Kaggle data\n", 10 | "---\n", 11 | "- Author: Diego Inácio\n", 12 | "- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)\n", 13 | "- Notebook: [kaggle-data.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Tips-and-Tricks/kaggle-data.ipynb)\n", 14 | "---\n", 15 | "Methods to obtain datasets from the *Kaggle* platform.\n", 16 | "\n", 17 | "[Keggle](https://www.kaggle.com/) is basically an online community for *data scientists*. It is a platform that provide users ways to practice *machine learning*, *data analysis* or any kind of *data mining* projects. There is a lot of datasets available on Kaggle ready to be downloaded and explored. Here we will se how to get them." 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "id": "loaded-worship", 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "import os\n", 28 | "import sys\n", 29 | "import zipfile\n", 30 | "import pandas as pd" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "id": "coordinate-mouse", 36 | "metadata": {}, 37 | "source": [ 38 | "## Before getting the data\n", 39 | "---\n", 40 | "Before downloading any dataset, we have to set our credentials to explore Kaggle's API. The first step is to create an *API token* by doing the following steps below:\n", 41 | "\n", 42 | "- Visit [Kaggle](https://www.kaggle.com/) and go to your profile and click on account;\n", 43 | "- Roll down and click on **Create new API Token** button. \n", 44 | "\n", 45 | "```\n", 46 | "This will download a file called `kaggle.json`, which will provide us our **username** and **key** like:\n", 47 | "```\n", 48 | "\n", 49 | "``` json\n", 50 | "{\"username\": \"diegoinacio\", \"key\": \"abfj3......2q9b\"}\n", 51 | "```\n", 52 | "\n", 53 | "- Place this file in `$HOME/.kaggle/kaggle.json`" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "social-basics", 59 | "metadata": {}, 60 | "source": [ 61 | "## Kaggle API\n", 62 | "---\n", 63 | "To access Kaggle API functionality, first install the kaggle tools:\n", 64 | "\n", 65 | "``` shell\n", 66 | "pip install kaggle --upgrade\n", 67 | "```\n", 68 | "\n", 69 | "It will provide us with the possibility to access both the **CLI** functionality and Python interface. To know more, you can reach the [project repository](https://github.com/Kaggle/kaggle-api)." 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "id": "other-individual", 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "!kaggle -h" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "disciplinary-cursor", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "import kaggle" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "id": "wrapped-austin", 95 | "metadata": {}, 96 | "source": [ 97 | "### Datasets via CLI\n", 98 | "---" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "id": "organized-google", 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "!kaggle datasets list" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "id": "altered-musician", 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "!kaggle datasets download ahsan81/hotel-reservations-classification-dataset" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "id": "referenced-ticket", 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "zf = zipfile.ZipFile(\"./hotel-reservations-classification-dataset.zip\") \n", 129 | "df = pd.read_csv(zf.open(\"Hotel Reservations.csv\"))" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "id": "stunning-turkey", 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "df" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "id": "entitled-compound", 145 | "metadata": {}, 146 | "source": [ 147 | "### Datasets via Python interface\n", 148 | "---" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": null, 154 | "id": "adolescent-order", 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "from kaggle.api.kaggle_api_extended import KaggleApi" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "id": "boxed-reaction", 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "api = KaggleApi()\n", 169 | "api.authenticate()" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "id": "eight-occasions", 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "api.dataset_list()" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "id": "based-induction", 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "api.dataset_download_files(\"senapatirajesh/netflix-tv-shows-and-movies\")" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "id": "laughing-program", 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "zf = zipfile.ZipFile(\"./netflix-tv-shows-and-movies.zip\") \n", 200 | "df = pd.read_csv(zf.open(\"NetFlix.csv\"))" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "id": "indie-samuel", 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "df" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "id": "retired-medium", 216 | "metadata": {}, 217 | "source": [ 218 | "## opendatasets library\n", 219 | "---\n", 220 | "[opendatasets](https://github.com/JovianHQ/opendatasets/) is a Python library for downloading datasets not only from Kaggle but also from any online sources like Google Drive and others. To install that just run the following command:\n", 221 | "\n", 222 | "``` shell\n", 223 | "pip install opendatasets\n", 224 | "```" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "id": "embedded-oriental", 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "import opendatasets as ods" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "id": "above-little", 240 | "metadata": {}, 241 | "source": [ 242 | "To download from any source just provide an url. In the case of Kaggle, the command will interactively ask you for the **username** and **key** obtained from `kaggle.json` file. Copy and paste into the interactive widgets." 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "id": "mobile-dressing", 249 | "metadata": {}, 250 | "outputs": [], 251 | "source": [ 252 | "# This command will ask you for your username and key (from kaggle.json) interactively\n", 253 | "dataset_url = \"https://www.kaggle.com/rakkesharv/spotify-top-10000-streamed-songs\"\n", 254 | "ods.download(dataset_url)" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": null, 260 | "id": "seven-biology", 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "df = pd.read_csv(\"./spotify-top-10000-streamed-songs/Spotify_final_dataset.csv\")" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "id": "pointed-desert", 271 | "metadata": {}, 272 | "outputs": [], 273 | "source": [ 274 | "df" 275 | ] 276 | } 277 | ], 278 | "metadata": { 279 | "kernelspec": { 280 | "display_name": "Python [conda env:root] *", 281 | "language": "python", 282 | "name": "conda-root-py" 283 | }, 284 | "language_info": { 285 | "codemirror_mode": { 286 | "name": "ipython", 287 | "version": 3 288 | }, 289 | "file_extension": ".py", 290 | "mimetype": "text/x-python", 291 | "name": "python", 292 | "nbconvert_exporter": "python", 293 | "pygments_lexer": "ipython3", 294 | "version": "3.7.10" 295 | } 296 | }, 297 | "nbformat": 4, 298 | "nbformat_minor": 5 299 | } 300 | -------------------------------------------------------------------------------- /docs/assets/icons/go_python_notebooks.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 33 | 34 | visit_page 35 | 36 | 37 | 38 | > 39 | 41 | 42 | go to 43 | Python Notebooks 44 | 45 | -------------------------------------------------------------------------------- /docs/assets/icons/join_discussion.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 33 | 34 | visit_page 35 | 36 | 37 | 38 | > 39 | 41 | 42 | join 43 | Discussions 44 | 45 | -------------------------------------------------------------------------------- /docs/assets/icons/support_this_project.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 40 | 41 | visit_page 42 | 43 | 44 | 46 | 47 | 48 | ❤️ 49 | 51 | 52 | 53 | support 54 | this project 55 | 56 | -------------------------------------------------------------------------------- /docs/assets/icons/visit_cookbook.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 33 | 34 | visit_page 35 | 36 | 37 | 38 | > 39 | 41 | 42 | visit 43 | cookbook 44 | 45 | -------------------------------------------------------------------------------- /docs/assets/images/social-preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diegoinacio/machine-learning-notebooks/2df864632b8182e9cc83570db798254f5368f37b/docs/assets/images/social-preview.png -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 11 | Redirecting .. 12 | 13 | 14 | 15 | 16 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /docs/style.css: -------------------------------------------------------------------------------- 1 | @import url("fontawesome-all.min.css"); 2 | @import url("https://fonts.googleapis.com/css?family=Source+Sans+Pro:300,400|Passion+One:400"); 3 | 4 | * { 5 | margin: 0; 6 | padding: 0; 7 | border: 0; 8 | } 9 | 10 | body { 11 | margin: auto; 12 | width: 736px; 13 | font: 20px Helvetica, sans-serif; 14 | background-color: #252525; 15 | color: gray; 16 | } 17 | 18 | h1 { 19 | font-size: 2.5em; 20 | font-weight: 300; 21 | margin: 1em 0 1em 0; 22 | font-family: "Passion One", cursive; 23 | color: white; 24 | } 25 | 26 | h2 { 27 | margin: 1em 0 1em 0; 28 | } 29 | 30 | article { 31 | display: block; 32 | text-align: left; 33 | width: 650px; 34 | margin: 0 auto; 35 | } 36 | 37 | a { 38 | color: #26bbd9; 39 | text-decoration: none; 40 | } 41 | 42 | a:hover { 43 | color: white; 44 | text-decoration: none; 45 | } 46 | 47 | .signature { 48 | margin: 1em 0 1em 0; 49 | } 50 | --------------------------------------------------------------------------------