├── .gitattributes ├── README.md ├── exercises ├── exercise_1.py ├── exercise_1_answer.py ├── exercise_2.py ├── exercise_2_answer.py ├── exercise_3.py ├── exercise_3_answer.py ├── exercise_4.py ├── exercise_4_answer.py ├── exercise_5.py └── exercise_5_answer.py ├── oreilly_deep_learning_for_beginners_day_1.pptx ├── oreilly_deep_learning_for_beginners_day_2.pptx └── oreilly_deep_learning_for_beginners_day_3.pptx /.gitattributes: -------------------------------------------------------------------------------- 1 | *.pptx filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning for Beginners 2 | ## O'Reilly Online Training 3 | 4 | https://learning.oreilly.com/live-events/deep-learning-for-beginners-in-3-weeks/0636920079316/ 5 | -------------------------------------------------------------------------------- /exercises/exercise_1.py: -------------------------------------------------------------------------------- 1 | """ 2 | In the Python code below, replace the question marks "?" 3 | with the proper code to perform a neural network prediction 4 | on a maintenance dataset on whether a part needs replacement (1) or not (0). 5 | 6 | Use 3 nodes in the hidden layer, and ReLU as the activation function. 7 | 8 | Experiment with learning rate and iterations to optimize training. 9 | """ 10 | 11 | import pandas as pd 12 | import tensorflow as tf 13 | 14 | df = pd.read_csv('https://bit.ly/3wlFsb4') 15 | 16 | # Extract input variables (all rows, all columns but last column) 17 | # Note we should do some linear scaling here 18 | X = df.values[:, :-1] / 1000.0 19 | 20 | # Extract output column (all rows, last column) 21 | Y = df.values[:, -1] 22 | 23 | # Separate training and testing data 24 | 25 | # declare the model 26 | model = tf.keras.models.Sequential([ 27 | tf.keras.layers.Dense(?, activation=?), 28 | tf.keras.layers.Dense(?, activation=?) 29 | ]) 30 | 31 | loss_fn = tf.keras.losses.MeanSquaredError() 32 | 33 | # compile the model 34 | model.compile(optimizer='adam', 35 | loss=loss_fn, 36 | metrics=['accuracy']) 37 | 38 | # fit the model 39 | model.fit(?, ?, epochs=?, batch_size=?) 40 | 41 | 42 | # evaluate the model 43 | scores = model.evaluate(X, Y, verbose=0) 44 | print(f"Dataset Score: {scores[1]}") 45 | -------------------------------------------------------------------------------- /exercises/exercise_1_answer.py: -------------------------------------------------------------------------------- 1 | """ 2 | In the Python code below, replace the question marks "?" 3 | with the proper code to perform a neural network prediction 4 | on a maintenance dataset on whether a part needs replacement (1) or not (0). 5 | 6 | Use 3 nodes in the hidden layer, and ReLU as the activation function. 7 | 8 | Experiment with learning rate and iterations to optimize training. 9 | """ 10 | 11 | import pandas as pd 12 | import tensorflow as tf 13 | 14 | df = pd.read_csv('https://bit.ly/3wlFsb4') 15 | 16 | # Extract input variables (all rows, all columns but last column) 17 | # Note we should do some linear scaling here 18 | X = df.values[:, :-1] / 1000.0 19 | 20 | # Extract output column (all rows, last column) 21 | Y = df.values[:, -1] 22 | 23 | # Separate training and testing data 24 | 25 | # declare the model 26 | model = tf.keras.models.Sequential([ 27 | tf.keras.layers.Dense(3, activation='relu'), 28 | tf.keras.layers.Dense(1, activation='sigmoid') 29 | ]) 30 | 31 | loss_fn = tf.keras.losses.MeanSquaredError() 32 | 33 | # compile the model 34 | model.compile(optimizer='adam', 35 | loss=loss_fn, 36 | metrics=['accuracy']) 37 | 38 | # fit the model 39 | model.fit(X, Y, epochs=100, batch_size=32) 40 | 41 | 42 | # evaluate the model 43 | scores = model.evaluate(X, Y, verbose=0) 44 | print(f"Dataset Score: {scores[1]}") 45 | -------------------------------------------------------------------------------- /exercises/exercise_2.py: -------------------------------------------------------------------------------- 1 | """ 2 | COMPLETE THE CODE BELOW BY REPLACING THE QUESTION MARKS ?'s 3 | SO FORWARD PROPAGATION IS COMPLETE 4 | """ 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | all_data = pd.read_csv("https://bit.ly/3wlFsb4") 10 | 11 | # Extract the input columns, scale down 12 | X = (all_data.iloc[:, 0:3].values / 1000.0) 13 | Y = all_data.iloc[:, -1].values 14 | 15 | # Build neural network with weights and biases 16 | # with random initialization 17 | w_hidden = np.random.rand(3, 3) 18 | w_output = np.random.rand(1, 3) 19 | 20 | b_hidden = np.random.rand(3, 1) 21 | b_output = np.random.rand(1, 1) 22 | 23 | # Activation functions 24 | relu = lambda x: np.maximum(x, 0) 25 | logistic = lambda x: 1 / (1 + np.exp(-x)) 26 | 27 | # Runs inputs through the neural network to get predicted outputs 28 | def forward_prop(X): 29 | Z1 = ? @ X + ? 30 | A1 = relu(?) 31 | Z2 = ? @ ? + b_output 32 | A2 = logistic(?) 33 | return Z1, A1, Z2, A2 34 | 35 | # Calculate accuracy 36 | test_predictions = forward_prop(X.transpose())[3] # grab only A2 37 | test_comparisons = np.equal((test_predictions >= .5).flatten().astype(int), Y) 38 | accuracy = sum(test_comparisons.astype(int) / X.shape[0]) 39 | print("ACCURACY: ", accuracy) 40 | 41 | -------------------------------------------------------------------------------- /exercises/exercise_2_answer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | all_data = pd.read_csv("https://bit.ly/3wlFsb4") 5 | 6 | # Extract the input columns, scale down 7 | X = (all_data.iloc[:, 0:3].values / 1000.0) 8 | Y = all_data.iloc[:, -1].values 9 | 10 | # Build neural network with weights and biases 11 | # with random initialization 12 | w_hidden = np.random.rand(3, 3) 13 | w_output = np.random.rand(1, 3) 14 | 15 | b_hidden = np.random.rand(3, 1) 16 | b_output = np.random.rand(1, 1) 17 | 18 | # Activation functions 19 | relu = lambda x: np.maximum(x, 0) 20 | logistic = lambda x: 1 / (1 + np.exp(-x)) 21 | 22 | # Runs inputs through the neural network to get predicted outputs 23 | def forward_prop(X): 24 | Z1 = w_hidden @ X + b_hidden 25 | A1 = relu(Z1) 26 | Z2 = w_output @ A1 + b_output 27 | A2 = logistic(Z2) 28 | return Z1, A1, Z2, A2 29 | 30 | # Calculate accuracy 31 | test_predictions = forward_prop(X.transpose())[3] # grab only A2 32 | test_comparisons = np.equal((test_predictions >= .5).flatten().astype(int), Y) 33 | accuracy = sum(test_comparisons.astype(int) / X.shape[0]) 34 | print("ACCURACY: ", accuracy) 35 | 36 | -------------------------------------------------------------------------------- /exercises/exercise_3.py: -------------------------------------------------------------------------------- 1 | """ 2 | Complete the code below to perform stochastic gradient descent 3 | on a linear regression. 4 | 5 | Try to find a sufficient learning rate and number of iterations. 6 | """ 7 | 8 | import pandas as pd 9 | import numpy as np 10 | 11 | # Input data 12 | data = pd.read_csv('https://bit.ly/3BjgUSM', header=0) 13 | 14 | X = data.iloc[:, 0].values 15 | Y = data.iloc[:, 1].values 16 | 17 | n = data.shape[0] # rows 18 | 19 | # Building the model 20 | m = 0.0 21 | b = 0.0 22 | 23 | sample_size = 1 # sample size 24 | L = ? # The learning Rate 25 | epochs = ? # The number of iterations to perform gradient descent 26 | 27 | # Performing Stochastic Gradient Descent 28 | for i in range(epochs): 29 | idx = np.random.choice(n, sample_size, replace=False) 30 | x_sample = X[idx] 31 | y_sample = Y[idx] 32 | 33 | # The current predicted value of Y 34 | Y_pred = m * x_sample + b 35 | 36 | # d/dm derivative of loss function 37 | D_m = (-2 / sample_size) * sum(x_sample * (y_sample - Y_pred)) 38 | 39 | # d/db derivative of loss function 40 | D_b = (-2 / sample_size) * sum(y_sample - Y_pred) 41 | m -= ? * ? # Update m 42 | b -= L * ? # Update b 43 | 44 | # print progress 45 | if i % 10000 == 0: 46 | print(i, m, b) 47 | 48 | print("y = {0}x + {1}".format(m, b)) 49 | -------------------------------------------------------------------------------- /exercises/exercise_3_answer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Complete the code below to perform stochastic gradient descent 3 | on a linear regression. 4 | 5 | Try to find a sufficient learning rate and number of iterations. 6 | """ 7 | 8 | import pandas as pd 9 | import numpy as np 10 | 11 | # Input data 12 | data = pd.read_csv('https://bit.ly/3BjgUSM', header=0) 13 | 14 | X = data.iloc[:, 0].values 15 | Y = data.iloc[:, 1].values 16 | 17 | n = data.shape[0] # rows 18 | 19 | # Building the model 20 | m = 0.0 21 | b = 0.0 22 | 23 | sample_size = 1 # sample size 24 | L = .0001 # The learning Rate 25 | epochs = 1_000_000 # The number of iterations to perform gradient descent 26 | 27 | # Performing Stochastic Gradient Descent 28 | for i in range(epochs): 29 | idx = np.random.choice(n, sample_size, replace=False) 30 | x_sample = X[idx] 31 | y_sample = Y[idx] 32 | 33 | # The current predicted value of Y 34 | Y_pred = m * x_sample + b 35 | 36 | # d/dm derivative of loss function 37 | D_m = (-2 / sample_size) * sum(x_sample * (y_sample - Y_pred)) 38 | 39 | # d/db derivative of loss function 40 | D_b = (-2 / sample_size) * sum(y_sample - Y_pred) 41 | m = m - L * D_m # Update m 42 | b = b - L * D_b # Update b 43 | 44 | # print progress 45 | if i % 10000 == 0: 46 | print(i, m, b) 47 | 48 | print("y = {0}x + {1}".format(m, b)) 49 | -------------------------------------------------------------------------------- /exercises/exercise_4.py: -------------------------------------------------------------------------------- 1 | """ 2 | ON THE MAINTENANCE PREDICTION DATASET, 3 | COMPLETE THE CODE BELOW BY REPLACING THE QUESTION MARKS ?'s 4 | SO BACKWARD PROPAGATION AND STOCHASTIC GRADIENT DESCENT 5 | IS COMPLETE 6 | """ 7 | import numpy as np 8 | import pandas as pd 9 | from sklearn.model_selection import train_test_split 10 | 11 | all_data = pd.read_csv("https://bit.ly/3wlFsb4") 12 | 13 | # Extract the input columns, scale down by 1000 14 | X = (all_data.iloc[:, 0:3].values / 1000.0) 15 | Y = all_data.iloc[:, -1].values 16 | 17 | L = 0.05 18 | 19 | # Split train and test data sets 20 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size= 1/3) 21 | n = X_train.shape[0] 22 | 23 | # Build neural network with weights and biases 24 | # with random initialization 25 | w_hidden = np.random.rand(3, 3) 26 | w_output = np.random.rand(1, 3) 27 | 28 | b_hidden = np.random.rand(3, 1) 29 | b_output = np.random.rand(1, 1) 30 | 31 | # Activation functions 32 | relu = lambda x: np.maximum(x, 0) 33 | logistic = lambda x: 1 / (1 + np.exp(-x)) 34 | 35 | # Runs inputs through the neural network to get predicted outputs 36 | def forward_prop(X): 37 | Z1 = w_hidden @ X + b_hidden 38 | A1 = relu(Z1) 39 | Z2 = w_output @ A1 + b_output 40 | A2 = logistic(Z2) 41 | return Z1, A1, Z2, A2 42 | 43 | # Derivatives of Activation functions 44 | d_relu = lambda x: x > 0 45 | d_logistic = lambda x: np.exp(-x) / (1 + np.exp(-x)) ** 2 46 | 47 | # returns slopes for weights and biases 48 | # using chain rule 49 | def backward_prop(Z1, A1, Z2, A2, X, Y): 50 | dC_dA2 = 2 * A2 - 2 * Y 51 | dA2_dZ2 = d_logistic(Z2) 52 | dZ2_dA1 = w_output 53 | dZ2_dW2 = A1 54 | dZ2_dB2 = 1 55 | dA1_dZ1 = d_relu(Z1) 56 | dZ1_dW1 = X 57 | dZ1_dB1 = 1 58 | 59 | dC_dW2 = dC_dA2 @ ? @ dZ2_dW2.T 60 | 61 | dC_dB2 = dC_dA2 @ dA2_dZ2 * dZ2_dB2 62 | 63 | dC_dA1 = ? @ dA2_dZ2 @ dZ2_dA1 64 | 65 | dC_dW1 = dC_dA1 @ dA1_dZ1 @ dZ1_dW1.T 66 | 67 | dC_dB1 = dC_dA1 @ ? * dZ1_dB1 68 | 69 | return dC_dW1, dC_dB1, dC_dW2, dC_dB2 70 | 71 | # Execute gradient descent 72 | for i in range(100_000): 73 | # randomly select one of the training data 74 | idx = np.random.choice(n, 1, replace=False) 75 | X_sample = X_train[idx].transpose() 76 | Y_sample = Y_train[idx] 77 | 78 | # run randomly selected training data through neural network 79 | Z1, A1, Z2, A2 = forward_prop(X_sample) 80 | 81 | # distribute error through backpropagation 82 | # and return slopes for weights and biases 83 | dW1, dB1, dW2, dB2 = backward_prop(Z1, A1, Z2, A2, X_sample, Y_sample) 84 | 85 | # update weights and biases 86 | w_hidden -= ? * dW1 87 | b_hidden -= ? * ? 88 | w_output -= L * ? 89 | b_output -= L * dB2 90 | 91 | # Calculate accuracy 92 | test_predictions = forward_prop(X_test.transpose())[3] # grab only A2 93 | test_comparisons = np.equal((test_predictions >= .5).flatten().astype(int), Y_test) 94 | accuracy = sum(test_comparisons.astype(int) / X_test.shape[0]) 95 | print("ACCURACY: ", accuracy) 96 | 97 | -------------------------------------------------------------------------------- /exercises/exercise_4_answer.py: -------------------------------------------------------------------------------- 1 | """ 2 | ON THE MAINTENANCE PREDICTION DATASET, 3 | COMPLETE THE CODE BELOW BY REPLACING THE QUESTION MARKS ?'s 4 | SO BACKWARD PROPAGATION AND STOCHASTIC GRADIENT DESCENT 5 | IS COMPLETE 6 | """ 7 | import numpy as np 8 | import pandas as pd 9 | from sklearn.model_selection import train_test_split 10 | 11 | all_data = pd.read_csv("https://bit.ly/3wlFsb4") 12 | 13 | # Extract the input columns, scale down by 1000 14 | X = (all_data.iloc[:, 0:3].values / 1000.0) 15 | Y = all_data.iloc[:, -1].values 16 | 17 | L = 0.05 18 | 19 | # Split train and test data sets 20 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size= 1/3) 21 | n = X_train.shape[0] 22 | 23 | # Build neural network with weights and biases 24 | # with random initialization 25 | w_hidden = np.random.rand(3, 3) 26 | w_output = np.random.rand(1, 3) 27 | 28 | b_hidden = np.random.rand(3, 1) 29 | b_output = np.random.rand(1, 1) 30 | 31 | # Activation functions 32 | relu = lambda x: np.maximum(x, 0) 33 | logistic = lambda x: 1 / (1 + np.exp(-x)) 34 | 35 | # Runs inputs through the neural network to get predicted outputs 36 | def forward_prop(X): 37 | Z1 = w_hidden @ X + b_hidden 38 | A1 = relu(Z1) 39 | Z2 = w_output @ A1 + b_output 40 | A2 = logistic(Z2) 41 | return Z1, A1, Z2, A2 42 | 43 | # Derivatives of Activation functions 44 | d_relu = lambda x: x > 0 45 | d_logistic = lambda x: np.exp(-x) / (1 + np.exp(-x)) ** 2 46 | 47 | # returns slopes for weights and biases 48 | # using chain rule 49 | def backward_prop(Z1, A1, Z2, A2, X, Y): 50 | dC_dA2 = 2 * A2 - 2 * Y 51 | dA2_dZ2 = d_logistic(Z2) 52 | dZ2_dA1 = w_output 53 | dZ2_dW2 = A1 54 | dZ2_dB2 = 1 55 | dA1_dZ1 = d_relu(Z1) 56 | dZ1_dW1 = X 57 | dZ1_dB1 = 1 58 | 59 | dC_dW2 = dC_dA2 @ dA2_dZ2 @ dZ2_dW2.T 60 | 61 | dC_dB2 = dC_dA2 @ dA2_dZ2 * dZ2_dB2 62 | 63 | dC_dA1 = dC_dA2 @ dA2_dZ2 @ dZ2_dA1 64 | 65 | dC_dW1 = dC_dA1 @ dA1_dZ1 @ dZ1_dW1.T 66 | 67 | dC_dB1 = dC_dA1 @ dA1_dZ1 * dZ1_dB1 68 | 69 | return dC_dW1, dC_dB1, dC_dW2, dC_dB2 70 | 71 | # Execute gradient descent 72 | for i in range(100_000): 73 | # randomly select one of the training data 74 | idx = np.random.choice(n, 1, replace=False) 75 | X_sample = X_train[idx].transpose() 76 | Y_sample = Y_train[idx] 77 | 78 | # run randomly selected training data through neural network 79 | Z1, A1, Z2, A2 = forward_prop(X_sample) 80 | 81 | # distribute error through backpropagation 82 | # and return slopes for weights and biases 83 | dW1, dB1, dW2, dB2 = backward_prop(Z1, A1, Z2, A2, X_sample, Y_sample) 84 | 85 | # update weights and biases 86 | w_hidden -= L * dW1 87 | b_hidden -= L * dB1 88 | w_output -= L * dW2 89 | b_output -= L * dB2 90 | 91 | # Calculate accuracy 92 | test_predictions = forward_prop(X_test.transpose())[3] # grab only A2 93 | test_comparisons = np.equal((test_predictions >= .5).flatten().astype(int), Y_test) 94 | accuracy = sum(test_comparisons.astype(int) / X_test.shape[0]) 95 | print("ACCURACY: ", accuracy) 96 | 97 | -------------------------------------------------------------------------------- /exercises/exercise_5.py: -------------------------------------------------------------------------------- 1 | """ 2 | In the Python code below, replace the question marks "?" 3 | with the proper code to perform a neural network prediction 4 | on a maintenance dataset on whether a part needs replacement (1) or not (0). 5 | 6 | Use 3 nodes in the hidden layer, and ReLU as the activation function. 7 | Experiment with learning rate and iterations to optimize training. 8 | 9 | Set aside 1/3 of the data for testing. 10 | """ 11 | 12 | import pandas as pd 13 | from sklearn.model_selection import train_test_split 14 | import tensorflow as tf 15 | 16 | df = pd.read_csv('https://bit.ly/3wlFsb4') 17 | 18 | # Extract input variables (all rows, all columns but last column) 19 | # Note we should do some linear scaling here 20 | X = df.values[:, :-1] / 1000.0 21 | 22 | # Extract output column (all rows, last column) 23 | Y = df.values[:, -1] 24 | 25 | # Separate training and testing data 26 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=?, random_state=7) 27 | 28 | # declare the model 29 | model = tf.keras.models.Sequential([ 30 | tf.keras.layers.Dense(?, activation=?), 31 | tf.keras.layers.Dense(?, activation=?) 32 | ]) 33 | 34 | loss_fn = tf.keras.losses.MeanSquaredError() 35 | 36 | # compile the model 37 | model.compile(optimizer='adam', 38 | loss=loss_fn, 39 | metrics=['accuracy']) 40 | 41 | # fit the model 42 | model.fit(?, ?, epochs=?, batch_size=?) 43 | 44 | 45 | # evaluate the model 46 | scores = model.evaluate(?, ?, verbose=0) 47 | print(f"Test Dataset Score: {scores[1]}") 48 | -------------------------------------------------------------------------------- /exercises/exercise_5_answer.py: -------------------------------------------------------------------------------- 1 | """ 2 | In the Python code below, replace the question marks "?" 3 | with the proper code to perform a neural network prediction 4 | on a maintenance dataset on whether a part needs replacement (1) or not (0). 5 | 6 | Use 3 nodes in the hidden layer, and ReLU as the activation function. 7 | Experiment with learning rate and iterations to optimize training. 8 | 9 | Set aside 1/3 of the data for testing. 10 | """ 11 | 12 | import pandas as pd 13 | from sklearn.model_selection import train_test_split 14 | import tensorflow as tf 15 | 16 | df = pd.read_csv('https://bit.ly/3wlFsb4') 17 | 18 | # Extract input variables (all rows, all columns but last column) 19 | # Note we should do some linear scaling here 20 | X = df.values[:, :-1] / 1000.0 21 | 22 | # Extract output column (all rows, last column) 23 | Y = df.values[:, -1] 24 | 25 | # Separate training and testing data 26 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=1.0/3.0, random_state=7) 27 | 28 | # declare the model 29 | model = tf.keras.models.Sequential([ 30 | tf.keras.layers.Dense(3, activation='relu'), 31 | tf.keras.layers.Dense(1, activation='sigmoid') 32 | ]) 33 | 34 | loss_fn = tf.keras.losses.MeanSquaredError() 35 | 36 | # compile the model 37 | model.compile(optimizer='adam', 38 | loss=loss_fn, 39 | metrics=['accuracy']) 40 | 41 | # fit the model 42 | model.fit(X_train, Y_train, epochs=100, batch_size=32) 43 | 44 | 45 | # evaluate the model 46 | scores = model.evaluate(X_test, Y_test, verbose=0) 47 | print(f"Test Dataset Score: {scores[1]}") 48 | -------------------------------------------------------------------------------- /oreilly_deep_learning_for_beginners_day_1.pptx: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e2dad5f59804812e8db8407b107a8dd3772a051404ead63ef1811e934fe70569 3 | size 15512703 4 | -------------------------------------------------------------------------------- /oreilly_deep_learning_for_beginners_day_2.pptx: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:4c4d371ca2a56a85836079047c7122ad9e53381d4118930623e6c6b170dc3d99 3 | size 5027761 4 | -------------------------------------------------------------------------------- /oreilly_deep_learning_for_beginners_day_3.pptx: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3037c32b56d084659af2bc3f26588598e3f693083128c86b549ec9bf5866a4ce 3 | size 18871603 4 | --------------------------------------------------------------------------------