├── .gitignore
├── .DS_Store
├── static
├── .DS_Store
├── imgs
│ ├── 10
│ │ ├── step_0.png
│ │ ├── step_1.png
│ │ ├── step_2.png
│ │ └── convolutional_layer.png
│ ├── 11
│ │ ├── input_text.png
│ │ ├── one-hots.png
│ │ ├── dumb_network.png
│ │ ├── loss_matters.png
│ │ ├── embedding_layer.png
│ │ ├── word_analogies.png
│ │ ├── embedding_weights.png
│ │ └── domains_intersections.png
│ ├── 12
│ │ ├── LM.png
│ │ ├── ID_sum.png
│ │ ├── sketch_RNN.jpg
│ │ ├── sentence_avg.png
│ │ ├── standard_sum.png
│ │ ├── word_vector.png
│ │ ├── word_vectors.png
│ │ ├── Primitive_RNN.png
│ │ ├── bigger_stacking.png
│ │ ├── enhanced_RNNs.jpg
│ │ ├── RNN_Architecture.jpg
│ │ ├── storing_info_RNN.jpg
│ │ ├── stacked_embeddings.png
│ │ └── RNN_Classification_Example.jpg
│ ├── 13
│ │ ├── shared_node.png
│ │ ├── gradients_flow.jpg
│ │ ├── computation_graph.png
│ │ ├── FC-backpropagation.jpg
│ │ └── Multidimensional-derivatives.jpg
│ ├── 14
│ │ ├── LSTM_cell.png
│ │ ├── preprocessing.jpg
│ │ └── generated_sample.png
│ ├── .DS_Store
│ ├── 06
│ │ ├── 0.jpg
│ │ ├── 1.jpg
│ │ ├── 2.jpg
│ │ ├── 3.jpg
│ │ ├── 4'.jpg
│ │ ├── 5.jpg
│ │ ├── 6.jpg
│ │ ├── falsy.jpg
│ │ ├── backprop.jpg
│ │ ├── debug_mode.jpg
│ │ ├── Conversation.png
│ │ ├── Experiment_1.png
│ │ ├── Experiment_2.png
│ │ ├── Iteration_1.png
│ │ ├── Iteration_2.png
│ │ ├── Iteration_3.png
│ │ ├── MNIST_model.png
│ │ ├── hidden-layer.png
│ │ ├── many-to-many.png
│ │ ├── many-to-one.png
│ │ ├── optimization.jpg
│ │ ├── backpropagation.png
│ │ ├── model_and_loss.jpg
│ │ ├── output-pattern.png
│ │ ├── weight_update.png
│ │ ├── MNIST_clear_model.png
│ │ ├── Neural_Similarity.png
│ │ ├── introducing-relu.png
│ │ ├── linearity-problem.png
│ │ ├── more-explanation.jpg
│ │ ├── multiple_inputs_1.jpg
│ │ ├── multiple_inputs_2.jpg
│ │ ├── streetlight_data.png
│ │ ├── latent_correlation.png
│ │ ├── streetlight_problem.png
│ │ ├── symbols-to-numbers.png
│ │ ├── thought_understand.jpg
│ │ └── MNIST_weight_visualization.png
│ ├── Cover.png
│ ├── 02
│ │ ├── Knobs.png
│ │ ├── Learn.png
│ │ ├── Predict.png
│ │ ├── Clustering.png
│ │ ├── DL_ML_AI.png
│ │ └── Unsupervised_parametric_learning.png
│ ├── 07
│ │ ├── LEGO.png
│ │ ├── algebra.png
│ │ ├── all-in-all.png
│ │ ├── simplified.png
│ │ └── complicated.png
│ ├── 08
│ │ └── Dogs.png
│ ├── 09
│ │ ├── Tanh.png
│ │ ├── sigmoid.png
│ │ ├── weights.png
│ │ ├── Pariston.png
│ │ ├── monotonic.png
│ │ ├── similarity.png
│ │ ├── activations.png
│ │ ├── layer_delta.png
│ │ ├── non-linearity.png
│ │ ├── continuous-vs-non.png
│ │ └── .ipynb_checkpoints
│ │ │ └── non-linearity-checkpoint.png
│ ├── 03
│ │ ├── deep-NN.png
│ │ ├── one-to-one.png
│ │ ├── NN-1-to-many.png
│ │ ├── dot-product.png
│ │ ├── many-to-one.png
│ │ ├── 1-to-1-NN-Note.jpg
│ │ ├── NN-many-to-many.png
│ │ ├── first-network.png
│ │ ├── many-to-many-NN.png
│ │ ├── dot-product-analysis.png
│ │ └── Reminder-1-to-1-NN-Note.jpg
│ └── 04
│ │ ├── step-0.png
│ │ ├── step-1.png
│ │ ├── step-2.png
│ │ ├── step-3.png
│ │ ├── dependence.jpg
│ │ ├── divergence.png
│ │ ├── derivatives.jpg
│ │ ├── optimization.jpg
│ │ ├── Loss_function.jpg
│ │ ├── Loss_function.png
│ │ ├── derivatives_2.jpg
│ │ └── model_and_loss.jpg
└── data
│ ├── Spam
│ └── spam_0.txt
│ └── tasksv11
│ ├── README
│ └── LICENSE
├── .idea
├── misc.xml
├── vcs.xml
├── inspectionProfiles
│ └── profiles_settings.xml
├── grokking_deep_learning_notebooks.iml
├── modules.xml
└── workspace.xml
├── .ipynb_checkpoints
├── README-checkpoint.md
├── index-checkpoint.ipynb
├── 05.Generalizing_GD-checkpoint.ipynb
└── 09.Probabilities_Non-Linearities-checkpoint.ipynb
├── README.md
├── 05.Generalizing_GD.ipynb
├── 09.Probabilities_Non-Linearities.ipynb
└── 08.Regularization_Batching.ipynb
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/*
2 | *.pyc
3 |
4 |
--------------------------------------------------------------------------------
/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/.DS_Store
--------------------------------------------------------------------------------
/static/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/.DS_Store
--------------------------------------------------------------------------------
/static/imgs/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/.DS_Store
--------------------------------------------------------------------------------
/static/imgs/06/0.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/0.jpg
--------------------------------------------------------------------------------
/static/imgs/06/1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/1.jpg
--------------------------------------------------------------------------------
/static/imgs/06/2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/2.jpg
--------------------------------------------------------------------------------
/static/imgs/06/3.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/3.jpg
--------------------------------------------------------------------------------
/static/imgs/06/4'.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/4'.jpg
--------------------------------------------------------------------------------
/static/imgs/06/5.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/5.jpg
--------------------------------------------------------------------------------
/static/imgs/06/6.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/6.jpg
--------------------------------------------------------------------------------
/static/imgs/12/LM.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/LM.png
--------------------------------------------------------------------------------
/static/imgs/Cover.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/Cover.png
--------------------------------------------------------------------------------
/static/imgs/02/Knobs.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/02/Knobs.png
--------------------------------------------------------------------------------
/static/imgs/02/Learn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/02/Learn.png
--------------------------------------------------------------------------------
/static/imgs/06/falsy.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/falsy.jpg
--------------------------------------------------------------------------------
/static/imgs/07/LEGO.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/07/LEGO.png
--------------------------------------------------------------------------------
/static/imgs/08/Dogs.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/08/Dogs.png
--------------------------------------------------------------------------------
/static/imgs/09/Tanh.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/Tanh.png
--------------------------------------------------------------------------------
/static/imgs/02/Predict.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/02/Predict.png
--------------------------------------------------------------------------------
/static/imgs/03/deep-NN.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/deep-NN.png
--------------------------------------------------------------------------------
/static/imgs/04/step-0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/step-0.png
--------------------------------------------------------------------------------
/static/imgs/04/step-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/step-1.png
--------------------------------------------------------------------------------
/static/imgs/04/step-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/step-2.png
--------------------------------------------------------------------------------
/static/imgs/04/step-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/step-3.png
--------------------------------------------------------------------------------
/static/imgs/07/algebra.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/07/algebra.png
--------------------------------------------------------------------------------
/static/imgs/09/sigmoid.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/sigmoid.png
--------------------------------------------------------------------------------
/static/imgs/09/weights.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/weights.png
--------------------------------------------------------------------------------
/static/imgs/10/step_0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/10/step_0.png
--------------------------------------------------------------------------------
/static/imgs/10/step_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/10/step_1.png
--------------------------------------------------------------------------------
/static/imgs/10/step_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/10/step_2.png
--------------------------------------------------------------------------------
/static/imgs/12/ID_sum.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/ID_sum.png
--------------------------------------------------------------------------------
/static/data/Spam/spam_0.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/data/Spam/spam_0.txt
--------------------------------------------------------------------------------
/static/imgs/02/Clustering.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/02/Clustering.png
--------------------------------------------------------------------------------
/static/imgs/02/DL_ML_AI.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/02/DL_ML_AI.png
--------------------------------------------------------------------------------
/static/imgs/03/one-to-one.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/one-to-one.png
--------------------------------------------------------------------------------
/static/imgs/04/dependence.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/dependence.jpg
--------------------------------------------------------------------------------
/static/imgs/04/divergence.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/divergence.png
--------------------------------------------------------------------------------
/static/imgs/06/backprop.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/backprop.jpg
--------------------------------------------------------------------------------
/static/imgs/06/debug_mode.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/debug_mode.jpg
--------------------------------------------------------------------------------
/static/imgs/07/all-in-all.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/07/all-in-all.png
--------------------------------------------------------------------------------
/static/imgs/07/simplified.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/07/simplified.png
--------------------------------------------------------------------------------
/static/imgs/09/Pariston.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/Pariston.png
--------------------------------------------------------------------------------
/static/imgs/09/monotonic.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/monotonic.png
--------------------------------------------------------------------------------
/static/imgs/09/similarity.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/similarity.png
--------------------------------------------------------------------------------
/static/imgs/11/input_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/input_text.png
--------------------------------------------------------------------------------
/static/imgs/11/one-hots.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/one-hots.png
--------------------------------------------------------------------------------
/static/imgs/12/sketch_RNN.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/sketch_RNN.jpg
--------------------------------------------------------------------------------
/static/imgs/14/LSTM_cell.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/14/LSTM_cell.png
--------------------------------------------------------------------------------
/static/imgs/03/NN-1-to-many.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/NN-1-to-many.png
--------------------------------------------------------------------------------
/static/imgs/03/dot-product.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/dot-product.png
--------------------------------------------------------------------------------
/static/imgs/03/many-to-one.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/many-to-one.png
--------------------------------------------------------------------------------
/static/imgs/04/derivatives.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/derivatives.jpg
--------------------------------------------------------------------------------
/static/imgs/04/optimization.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/optimization.jpg
--------------------------------------------------------------------------------
/static/imgs/06/Conversation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Conversation.png
--------------------------------------------------------------------------------
/static/imgs/06/Experiment_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Experiment_1.png
--------------------------------------------------------------------------------
/static/imgs/06/Experiment_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Experiment_2.png
--------------------------------------------------------------------------------
/static/imgs/06/Iteration_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Iteration_1.png
--------------------------------------------------------------------------------
/static/imgs/06/Iteration_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Iteration_2.png
--------------------------------------------------------------------------------
/static/imgs/06/Iteration_3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Iteration_3.png
--------------------------------------------------------------------------------
/static/imgs/06/MNIST_model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/MNIST_model.png
--------------------------------------------------------------------------------
/static/imgs/06/hidden-layer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/hidden-layer.png
--------------------------------------------------------------------------------
/static/imgs/06/many-to-many.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/many-to-many.png
--------------------------------------------------------------------------------
/static/imgs/06/many-to-one.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/many-to-one.png
--------------------------------------------------------------------------------
/static/imgs/06/optimization.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/optimization.jpg
--------------------------------------------------------------------------------
/static/imgs/07/complicated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/07/complicated.png
--------------------------------------------------------------------------------
/static/imgs/09/activations.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/activations.png
--------------------------------------------------------------------------------
/static/imgs/09/layer_delta.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/layer_delta.png
--------------------------------------------------------------------------------
/static/imgs/11/dumb_network.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/dumb_network.png
--------------------------------------------------------------------------------
/static/imgs/11/loss_matters.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/loss_matters.png
--------------------------------------------------------------------------------
/static/imgs/12/sentence_avg.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/sentence_avg.png
--------------------------------------------------------------------------------
/static/imgs/12/standard_sum.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/standard_sum.png
--------------------------------------------------------------------------------
/static/imgs/12/word_vector.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/word_vector.png
--------------------------------------------------------------------------------
/static/imgs/12/word_vectors.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/word_vectors.png
--------------------------------------------------------------------------------
/static/imgs/13/shared_node.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/13/shared_node.png
--------------------------------------------------------------------------------
/static/imgs/03/1-to-1-NN-Note.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/1-to-1-NN-Note.jpg
--------------------------------------------------------------------------------
/static/imgs/03/NN-many-to-many.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/NN-many-to-many.png
--------------------------------------------------------------------------------
/static/imgs/03/first-network.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/first-network.png
--------------------------------------------------------------------------------
/static/imgs/03/many-to-many-NN.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/many-to-many-NN.png
--------------------------------------------------------------------------------
/static/imgs/04/Loss_function.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/Loss_function.jpg
--------------------------------------------------------------------------------
/static/imgs/04/Loss_function.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/Loss_function.png
--------------------------------------------------------------------------------
/static/imgs/04/derivatives_2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/derivatives_2.jpg
--------------------------------------------------------------------------------
/static/imgs/04/model_and_loss.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/04/model_and_loss.jpg
--------------------------------------------------------------------------------
/static/imgs/06/backpropagation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/backpropagation.png
--------------------------------------------------------------------------------
/static/imgs/06/model_and_loss.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/model_and_loss.jpg
--------------------------------------------------------------------------------
/static/imgs/06/output-pattern.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/output-pattern.png
--------------------------------------------------------------------------------
/static/imgs/06/weight_update.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/weight_update.png
--------------------------------------------------------------------------------
/static/imgs/09/non-linearity.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/non-linearity.png
--------------------------------------------------------------------------------
/static/imgs/11/embedding_layer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/embedding_layer.png
--------------------------------------------------------------------------------
/static/imgs/11/word_analogies.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/word_analogies.png
--------------------------------------------------------------------------------
/static/imgs/12/Primitive_RNN.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/Primitive_RNN.png
--------------------------------------------------------------------------------
/static/imgs/12/bigger_stacking.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/bigger_stacking.png
--------------------------------------------------------------------------------
/static/imgs/12/enhanced_RNNs.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/enhanced_RNNs.jpg
--------------------------------------------------------------------------------
/static/imgs/13/gradients_flow.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/13/gradients_flow.jpg
--------------------------------------------------------------------------------
/static/imgs/14/preprocessing.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/14/preprocessing.jpg
--------------------------------------------------------------------------------
/static/imgs/06/MNIST_clear_model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/MNIST_clear_model.png
--------------------------------------------------------------------------------
/static/imgs/06/Neural_Similarity.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/Neural_Similarity.png
--------------------------------------------------------------------------------
/static/imgs/06/introducing-relu.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/introducing-relu.png
--------------------------------------------------------------------------------
/static/imgs/06/linearity-problem.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/linearity-problem.png
--------------------------------------------------------------------------------
/static/imgs/06/more-explanation.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/more-explanation.jpg
--------------------------------------------------------------------------------
/static/imgs/06/multiple_inputs_1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/multiple_inputs_1.jpg
--------------------------------------------------------------------------------
/static/imgs/06/multiple_inputs_2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/multiple_inputs_2.jpg
--------------------------------------------------------------------------------
/static/imgs/06/streetlight_data.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/streetlight_data.png
--------------------------------------------------------------------------------
/static/imgs/09/continuous-vs-non.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/continuous-vs-non.png
--------------------------------------------------------------------------------
/static/imgs/11/embedding_weights.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/embedding_weights.png
--------------------------------------------------------------------------------
/static/imgs/12/RNN_Architecture.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/RNN_Architecture.jpg
--------------------------------------------------------------------------------
/static/imgs/12/storing_info_RNN.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/storing_info_RNN.jpg
--------------------------------------------------------------------------------
/static/imgs/13/computation_graph.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/13/computation_graph.png
--------------------------------------------------------------------------------
/static/imgs/14/generated_sample.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/14/generated_sample.png
--------------------------------------------------------------------------------
/static/imgs/03/dot-product-analysis.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/dot-product-analysis.png
--------------------------------------------------------------------------------
/static/imgs/06/latent_correlation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/latent_correlation.png
--------------------------------------------------------------------------------
/static/imgs/06/streetlight_problem.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/streetlight_problem.png
--------------------------------------------------------------------------------
/static/imgs/06/symbols-to-numbers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/symbols-to-numbers.png
--------------------------------------------------------------------------------
/static/imgs/06/thought_understand.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/thought_understand.jpg
--------------------------------------------------------------------------------
/static/imgs/10/convolutional_layer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/10/convolutional_layer.png
--------------------------------------------------------------------------------
/static/imgs/12/stacked_embeddings.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/stacked_embeddings.png
--------------------------------------------------------------------------------
/static/imgs/13/FC-backpropagation.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/13/FC-backpropagation.jpg
--------------------------------------------------------------------------------
/static/imgs/11/domains_intersections.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/11/domains_intersections.png
--------------------------------------------------------------------------------
/static/imgs/03/Reminder-1-to-1-NN-Note.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/03/Reminder-1-to-1-NN-Note.jpg
--------------------------------------------------------------------------------
/static/imgs/06/MNIST_weight_visualization.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/06/MNIST_weight_visualization.png
--------------------------------------------------------------------------------
/static/imgs/12/RNN_Classification_Example.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/12/RNN_Classification_Example.jpg
--------------------------------------------------------------------------------
/static/imgs/13/Multidimensional-derivatives.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/13/Multidimensional-derivatives.jpg
--------------------------------------------------------------------------------
/static/imgs/02/Unsupervised_parametric_learning.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/02/Unsupervised_parametric_learning.png
--------------------------------------------------------------------------------
/static/imgs/09/.ipynb_checkpoints/non-linearity-checkpoint.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Akramz/grokking-deep-learning-notebooks/HEAD/static/imgs/09/.ipynb_checkpoints/non-linearity-checkpoint.png
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/grokking_deep_learning_notebooks.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 | 1576497058595
48 |
49 |
50 | 1576497058595
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
--------------------------------------------------------------------------------
/static/data/tasksv11/README:
--------------------------------------------------------------------------------
1 | Towards AI Complete Question Answering: A Set of Prerequisite Toy Tasks
2 | -----------------------------------------------------------------------
3 | In this directory is the first set of 20 tasks for testing text understanding and reasoning in the bAbI project.
4 | The aim is that each task tests a unique aspect of text and reasoning, and hence test different capabilities of learning models. More tasks are planned in the future to capture more aspects.
5 |
6 | For each task, there are 1000 questions for training, and 1000 for testing.
7 | However, we emphasize that the goal is still to use as little data as possible to do well on the task (i.e. if you can use less than 1000 that's even better) -- and without resorting to engineering task-specific tricks that will not generalize to other tasks, as they may not be of much use subsequently. Note that the aim during evaluation is to use the _same_ learner across all tasks to evaluate its skills and capabilities.
8 | Further while the MemNN results in the paper use full supervision (including of the supporting facts) results with weak supervision would also be ultimately preferable as this kind of data is easier to collect. Hence results of that form are very welcome.
9 |
10 | For the reasons above there are currently two directories:
11 |
12 | 1) en/ -- the tasks in English, readable by humans.
13 | 2) shuffled/ -- the same tasks with shuffled letters so they are not readable by humans, and for existing parsers and taggers cannot be used in a straight-forward fashion to leverage extra resources-- in this case the learner is more forced to rely on the given training data. This mimics a learner being first presented a language and having to learn from scratch. We plan to add further languages in the future as well, e.g. German, French ..
14 |
15 | The file format for each task is as follows:
16 | ID text
17 | ID text
18 | ID text
19 | ID question[tab]answer[tab]supporting fact IDS.
20 | ...
21 |
22 | The IDs for a given "story" start at 1 and increase.
23 | When the IDs in a file reset back to 1 you can consider the following sentences as a new "story".
24 | Supporting fact IDs only ever reference the sentences within a "story".
25 |
26 | For Example:
27 | 1 Mary moved to the bathroom.
28 | 2 John went to the hallway.
29 | 3 Where is Mary? bathroom 1
30 | 4 Daniel went back to the hallway.
31 | 5 Sandra moved to the garden.
32 | 6 Where is Daniel? hallway 4
33 | 7 John moved to the office.
34 | 8 Sandra journeyed to the bathroom.
35 | 9 Where is Daniel? hallway 4
36 | 10 Mary moved to the hallway.
37 | 11 Daniel travelled to the office.
38 | 12 Where is Daniel? office 11
39 | 13 John went back to the garden.
40 | 14 John moved to the bedroom.
41 | 15 Where is Sandra? bathroom 8
42 | 1 Sandra travelled to the office.
43 | 2 Sandra went to the bathroom.
44 | 3 Where is Sandra? bathroom 2
45 |
46 | Changes between versions.
47 | =========================
48 | V1.1 (this version) - fixed some problems with task 3, and reduced the training set size available to 1000 as this matches the results in the paper cited above, in order to avoid confusion.
49 |
--------------------------------------------------------------------------------
/.ipynb_checkpoints/README-checkpoint.md:
--------------------------------------------------------------------------------
1 |
10 |
11 | # Notebooks — Grokking Deep Learning by Andrew W. Trask
12 |
13 | > https://www.manning.com/books/grokking-deep-learning
14 |
15 |
16 |
17 |
18 | - Grokking Deep Learning was written to help **give you a foundation in deep learning** so that you can **master a major deep learning framework**.
19 | - Requires no math background beyond basic arithmetic.
20 | - Doesn't rely on a high-level library that might hide what's going on.
21 | - Anyone can read this book and understand how deep learning really works.
22 | - You won't just read the theory, you'll discover it yourself.
23 | > (You can Buy the Book from [Manning Publications](https: //www.manning.com/books/grokking-deep-learning) or [Amazon](https://www.amazon.com/Grokking-Deep-Learning-Andrew-Trask/dp/1617293709)).
24 |
25 | # Roadmap
26 |
27 | "Grokking Deep Learning" has 16 Chapters:
28 |
29 | 1. Introducing Deep Learning: Why you should Learn It?
30 | 2. fundamental Concepts: How Do Machines Learn?
31 | 3. [Introduction to Neural Learning](https://akramz.github.io/grokking-deep-learning-notebooks/03.forward_propagation): Forward Propagation
32 | 4. [Introduction to Neural Learning](https://akramz.github.io/grokking-deep-learning-notebooks/04.gradient_descent): Gradient Descent
33 | 5. [Learning Multiple Weights at a Time](https://akramz.github.io/grokking-deep-learning-notebooks/05.Generalizing_GD): Generalizing Gradient Descent
34 | 6. [Building your first deep neural network](https://akramz.github.io/grokking-deep-learning-notebooks/06.Back_propagation): Introduction to Backpropagation
35 | 7. How to Picture Neural Networks: In your Head & on Paper
36 | 8. [Learning Signal & Ignoring Noise](https://akramz.github.io/grokking-deep-learning-notebooks/08.Regularization_Batching): Introduction to Regularization & Batching
37 | 9. [Modeling Probabilities & Non-Linearities](https://akramz.github.io/grokking-deep-learning-notebooks/09.Probabilities_Non-Linearities): Activation Functions
38 | 10. [Neural Learning about Edges & Corners](https://akramz.github.io/grokking-deep-learning-notebooks/10.Intro_to_CNNs): Introduction to Convolutional Neural Networks
39 | 11. [Neural Networks that Understand Language](https://akramz.github.io/grokking-deep-learning-notebooks/11.NNs_that_Understand_Language): King - Man + Woman == ?
40 | 12. [Neural Networks that write like Shakespeare](https://akramz.github.io/grokking-deep-learning-notebooks/12.RNNs): Recurrent Layers for Variable Length Data
41 | 13. [Introducing Automatic Optimization](https://akramz.github.io/grokking-deep-learning-notebooks/13.DL_framework): Let's build a deep learning framework
42 | 14. [Learning to Write like Shakespeare](https://akramz.github.io/grokking-deep-learning-notebooks/14.LSTMs_LM): Long Short-term Memory
43 | 15. Deep Learning on Unseen Data: Introducing Federated Learning
44 | 16. Where to Go from Here: A brief Guide
45 |
46 | > Missing Notebooks are mostly based on original content from the Book.
47 |
48 | ---
49 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Grokking Deep Learning by Andrew W. Trask
2 |
3 | > https://www.manning.com/books/grokking-deep-learning
4 |
5 |
6 |
7 | This repo is home to notes & the code that accompanies Andrew W. Trask's "Grokking Deep Learning" book. It provides a solid foundation in deep learning so that you can master any major deep learning framework.
8 |
9 | The book requires no math background beyond basic arithmetic. It doesn't rely on a high-level library that might hide what's going on. Anyone can read the book and understand how deep learning really works. You won't just read the theory, you'll discover it yourself.
10 |
11 | > (You can Buy the Book from [Manning Publications](https://www.manning.com/books/grokking-deep-learning) or [Amazon](https://www.amazon.com/Grokking-Deep-Learning-Andrew-Trask/dp/1617293709)).
12 |
13 | # Roadmap
14 |
15 | "Grokking Deep Learning" has 16 chapters. We provide links for the available notebooks:
16 |
17 | 1. Introducing Deep Learning: Why you should Learn It?
18 | 2. fundamental Concepts: How Do Machines Learn?
19 | 3. [Introduction to Neural Learning](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/03.forward_propagation.ipynb): Forward Propagation
20 | 4. [Introduction to Neural Learning](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/04.gradient_descent.ipynb): Gradient Descent
21 | 5. [Learning Multiple Weights at a Time](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/05.Generalizing_GD.ipynb): Generalizing Gradient Descent
22 | 6. [Building your first deep neural network](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/06.Back_propagation.ipynb): Introduction to Backpropagation
23 | 7. How to Picture Neural Networks: In your Head & on Paper
24 | 8. [Learning Signal & Ignoring Noise](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/08.Regularization_Batching.ipynb): Introduction to Regularization & Batching
25 | 9. [Modeling Probabilities & Non-Linearities](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/09.Probabilities_Non-Linearities.ipynb): Activation Functions
26 | 10. [Neural Learning about Edges & Corners](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/10.Intro_to_CNNs.ipynb): Introduction to Convolutional Neural Networks
27 | 11. [Neural Networks that Understand Language](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/11.NNs_that_Understand_Language.ipynb): King - Man + Woman == ?
28 | 12. [Neural Networks that write like Shakespeare](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/12.RNNs.ipynb): Recurrent Layers for Variable Length Data
29 | 13. [Introducing Automatic Optimization](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/13.DL_framework.ipynb): Let's build a deep learning framework
30 | 14. [Learning to Write like Shakespeare](https://nbviewer.jupyter.org/github/Akramz/grokking-deep-learning-notebooks/blob/master/14.LSTMs_LM.ipynb): Long Short-term Memory
31 | 15. Deep Learning on Unseen Data: Introducing Federated Learning
32 | 16. Where to Go from Here: A brief Guide
33 |
34 | > Hidden Notebooks are mostly based on original content from the book.
35 |
36 | ---
37 |
--------------------------------------------------------------------------------
/.ipynb_checkpoints/index-checkpoint.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Notebooks — Grokking Deep Learning by Andrew W. Trask\n",
8 | "> https://www.manning.com/books/grokking-deep-learning"
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "metadata": {},
14 | "source": [
15 | "
"
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {},
21 | "source": [
22 | "- Grokking Deep Learning was written to help **give you a foundation in deep learning** so that you can **master a major deep learning framework**.\n",
23 | " - Requires no math background beyond basic arithmetic.\n",
24 | " - Doesn't rely on a high-level library that might hide what's going on.\n",
25 | " - Anyone can read this book and understand how deep learning really works.\n",
26 | " - You won't just read the theory, you'll discover it yourself.\n",
27 | "\n",
28 | "> (You can Buy the Book from [Manning Publications](https://www.manning.com/books/grokking-deep-learning) or [Amazon](https://www.amazon.com/Grokking-Deep-Learning-Andrew-Trask/dp/1617293709))."
29 | ]
30 | },
31 | {
32 | "cell_type": "markdown",
33 | "metadata": {},
34 | "source": [
35 | "# Roadmap\n",
36 | "\n",
37 | "\"Grokking Deep Learning\" has 16 Chapters:"
38 | ]
39 | },
40 | {
41 | "cell_type": "markdown",
42 | "metadata": {},
43 | "source": [
44 | "1. Introducing Deep Learning: Why you should Learn It?\n",
45 | "2. fundamental Concepts: How Do Machines Learn?\n",
46 | "3. [Introduction to Neural Learning](https://akramz.github.io/grokking-deep-learning-notebooks/03.forward_propagation): Forward Propagation\n",
47 | "4. [Introduction to Neural Learning](https://akramz.github.io/grokking-deep-learning-notebooks/04.gradient_descent): Gradient Descent\n",
48 | "5. [Learning Multiple Weights at a Time](https://akramz.github.io/grokking-deep-learning-notebooks/05.Generalizing_GD): Generalizing Gradient Descent\n",
49 | "6. [Building your first deep neural network](https://akramz.github.io/grokking-deep-learning-notebooks/06.Back_propagation): Introduction to Backpropagation\n",
50 | "7. How to Picture Neural Networks: In your Head & on Paper\n",
51 | "8. [Learning Signal & Ignoring Noise](https://akramz.github.io/grokking-deep-learning-notebooks/08.Regularization_Batching): Introduction to Regularization & Batching\n",
52 | "9. [Modeling Probabilities & Non-Linearities](https://akramz.github.io/grokking-deep-learning-notebooks/09.Probabilities_Non-Linearities): Activation Functions\n",
53 | "10. [Neural Learning about Edges & Corners](https://akramz.github.io/grokking-deep-learning-notebooks/10.Intro_to_CNNs): Introduction to Convolutional Neural Networks\n",
54 | "11. [Neural Networks that Understand Language](https://akramz.github.io/grokking-deep-learning-notebooks/11.NNs_that_Understand_Language): King - Man + Woman == ?\n",
55 | "12. [Neural Networks that write like Shakespeare](https://akramz.github.io/grokking-deep-learning-notebooks/12.RNNs): Recurrent Layers for Variable Length Data\n",
56 | "13. [Introducing Automatic Optimization](https://akramz.github.io/grokking-deep-learning-notebooks/13.DL_framework): Let's build a deep learning framework\n",
57 | "14. [Learning to Write like Shakespeare](https://akramz.github.io/grokking-deep-learning-notebooks/14.LSTMs_LM): Long Short-term Memory\n",
58 | "15. Deep Learning on Unseen Data: Introducing Federated Learning\n",
59 | "16. Where to Go from Here: A brief Guide\n",
60 | "\n",
61 | "> Missing Notebooks are mostly based on original content from the Book."
62 | ]
63 | },
64 | {
65 | "cell_type": "markdown",
66 | "metadata": {},
67 | "source": [
68 | "---"
69 | ]
70 | }
71 | ],
72 | "metadata": {
73 | "kernelspec": {
74 | "display_name": "Python 3",
75 | "language": "python",
76 | "name": "python3"
77 | },
78 | "language_info": {
79 | "codemirror_mode": {
80 | "name": "ipython",
81 | "version": 3
82 | },
83 | "file_extension": ".py",
84 | "mimetype": "text/x-python",
85 | "name": "python",
86 | "nbconvert_exporter": "python",
87 | "pygments_lexer": "ipython3",
88 | "version": "3.7.5"
89 | }
90 | },
91 | "nbformat": 4,
92 | "nbformat_minor": 4
93 | }
94 |
--------------------------------------------------------------------------------
/static/data/tasksv11/LICENSE:
--------------------------------------------------------------------------------
1 | CC License
2 |
3 | bAbI tasks data
4 |
5 | Copyright (c) 2015-present, Facebook, Inc. All rights reserved.
6 |
7 | Creative Commons Legal Code
8 |
9 | Attribution 3.0 Unported
10 |
11 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
12 | LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN
13 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
14 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
15 | REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR
16 | DAMAGES RESULTING FROM ITS USE.
17 |
18 | License
19 |
20 | THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE
21 | COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY
22 | COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
23 | AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
24 |
25 | BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE
26 | TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY
27 | BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS
28 | CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND
29 | CONDITIONS.
30 |
31 | 1. Definitions
32 |
33 | a. "Adaptation" means a work based upon the Work, or upon the Work and
34 | other pre-existing works, such as a translation, adaptation,
35 | derivative work, arrangement of music or other alterations of a
36 | literary or artistic work, or phonogram or performance and includes
37 | cinematographic adaptations or any other form in which the Work may be
38 | recast, transformed, or adapted including in any form recognizably
39 | derived from the original, except that a work that constitutes a
40 | Collection will not be considered an Adaptation for the purpose of
41 | this License. For the avoidance of doubt, where the Work is a musical
42 | work, performance or phonogram, the synchronization of the Work in
43 | timed-relation with a moving image ("synching") will be considered an
44 | Adaptation for the purpose of this License.
45 | b. "Collection" means a collection of literary or artistic works, such as
46 | encyclopedias and anthologies, or performances, phonograms or
47 | broadcasts, or other works or subject matter other than works listed
48 | in Section 1(f) below, which, by reason of the selection and
49 | arrangement of their contents, constitute intellectual creations, in
50 | which the Work is included in its entirety in unmodified form along
51 | with one or more other contributions, each constituting separate and
52 | independent works in themselves, which together are assembled into a
53 | collective whole. A work that constitutes a Collection will not be
54 | considered an Adaptation (as defined above) for the purposes of this
55 | License.
56 | c. "Distribute" means to make available to the public the original and
57 | copies of the Work or Adaptation, as appropriate, through sale or
58 | other transfer of ownership.
59 | d. "Licensor" means the individual, individuals, entity or entities that
60 | offer(s) the Work under the terms of this License.
61 | e. "Original Author" means, in the case of a literary or artistic work,
62 | the individual, individuals, entity or entities who created the Work
63 | or if no individual or entity can be identified, the publisher; and in
64 | addition (i) in the case of a performance the actors, singers,
65 | musicians, dancers, and other persons who act, sing, deliver, declaim,
66 | play in, interpret or otherwise perform literary or artistic works or
67 | expressions of folklore; (ii) in the case of a phonogram the producer
68 | being the person or legal entity who first fixes the sounds of a
69 | performance or other sounds; and, (iii) in the case of broadcasts, the
70 | organization that transmits the broadcast.
71 | f. "Work" means the literary and/or artistic work offered under the terms
72 | of this License including without limitation any production in the
73 | literary, scientific and artistic domain, whatever may be the mode or
74 | form of its expression including digital form, such as a book,
75 | pamphlet and other writing; a lecture, address, sermon or other work
76 | of the same nature; a dramatic or dramatico-musical work; a
77 | choreographic work or entertainment in dumb show; a musical
78 | composition with or without words; a cinematographic work to which are
79 | assimilated works expressed by a process analogous to cinematography;
80 | a work of drawing, painting, architecture, sculpture, engraving or
81 | lithography; a photographic work to which are assimilated works
82 | expressed by a process analogous to photography; a work of applied
83 | art; an illustration, map, plan, sketch or three-dimensional work
84 | relative to geography, topography, architecture or science; a
85 | performance; a broadcast; a phonogram; a compilation of data to the
86 | extent it is protected as a copyrightable work; or a work performed by
87 | a variety or circus performer to the extent it is not otherwise
88 | considered a literary or artistic work.
89 | g. "You" means an individual or entity exercising rights under this
90 | License who has not previously violated the terms of this License with
91 | respect to the Work, or who has received express permission from the
92 | Licensor to exercise rights under this License despite a previous
93 | violation.
94 | h. "Publicly Perform" means to perform public recitations of the Work and
95 | to communicate to the public those public recitations, by any means or
96 | process, including by wire or wireless means or public digital
97 | performances; to make available to the public Works in such a way that
98 | members of the public may access these Works from a place and at a
99 | place individually chosen by them; to perform the Work to the public
100 | by any means or process and the communication to the public of the
101 | performances of the Work, including by public digital performance; to
102 | broadcast and rebroadcast the Work by any means including signs,
103 | sounds or images.
104 | i. "Reproduce" means to make copies of the Work by any means including
105 | without limitation by sound or visual recordings and the right of
106 | fixation and reproducing fixations of the Work, including storage of a
107 | protected performance or phonogram in digital form or other electronic
108 | medium.
109 |
110 | 2. Fair Dealing Rights. Nothing in this License is intended to reduce,
111 | limit, or restrict any uses free from copyright or rights arising from
112 | limitations or exceptions that are provided for in connection with the
113 | copyright protection under copyright law or other applicable laws.
114 |
115 | 3. License Grant. Subject to the terms and conditions of this License,
116 | Licensor hereby grants You a worldwide, royalty-free, non-exclusive,
117 | perpetual (for the duration of the applicable copyright) license to
118 | exercise the rights in the Work as stated below:
119 |
120 | a. to Reproduce the Work, to incorporate the Work into one or more
121 | Collections, and to Reproduce the Work as incorporated in the
122 | Collections;
123 | b. to create and Reproduce Adaptations provided that any such Adaptation,
124 | including any translation in any medium, takes reasonable steps to
125 | clearly label, demarcate or otherwise identify that changes were made
126 | to the original Work. For example, a translation could be marked "The
127 | original work was translated from English to Spanish," or a
128 | modification could indicate "The original work has been modified.";
129 | c. to Distribute and Publicly Perform the Work including as incorporated
130 | in Collections; and,
131 | d. to Distribute and Publicly Perform Adaptations.
132 | e. For the avoidance of doubt:
133 |
134 | i. Non-waivable Compulsory License Schemes. In those jurisdictions in
135 | which the right to collect royalties through any statutory or
136 | compulsory licensing scheme cannot be waived, the Licensor
137 | reserves the exclusive right to collect such royalties for any
138 | exercise by You of the rights granted under this License;
139 | ii. Waivable Compulsory License Schemes. In those jurisdictions in
140 | which the right to collect royalties through any statutory or
141 | compulsory licensing scheme can be waived, the Licensor waives the
142 | exclusive right to collect such royalties for any exercise by You
143 | of the rights granted under this License; and,
144 | iii. Voluntary License Schemes. The Licensor waives the right to
145 | collect royalties, whether individually or, in the event that the
146 | Licensor is a member of a collecting society that administers
147 | voluntary licensing schemes, via that society, from any exercise
148 | by You of the rights granted under this License.
149 |
150 | The above rights may be exercised in all media and formats whether now
151 | known or hereafter devised. The above rights include the right to make
152 | such modifications as are technically necessary to exercise the rights in
153 | other media and formats. Subject to Section 8(f), all rights not expressly
154 | granted by Licensor are hereby reserved.
155 |
156 | 4. Restrictions. The license granted in Section 3 above is expressly made
157 | subject to and limited by the following restrictions:
158 |
159 | a. You may Distribute or Publicly Perform the Work only under the terms
160 | of this License. You must include a copy of, or the Uniform Resource
161 | Identifier (URI) for, this License with every copy of the Work You
162 | Distribute or Publicly Perform. You may not offer or impose any terms
163 | on the Work that restrict the terms of this License or the ability of
164 | the recipient of the Work to exercise the rights granted to that
165 | recipient under the terms of the License. You may not sublicense the
166 | Work. You must keep intact all notices that refer to this License and
167 | to the disclaimer of warranties with every copy of the Work You
168 | Distribute or Publicly Perform. When You Distribute or Publicly
169 | Perform the Work, You may not impose any effective technological
170 | measures on the Work that restrict the ability of a recipient of the
171 | Work from You to exercise the rights granted to that recipient under
172 | the terms of the License. This Section 4(a) applies to the Work as
173 | incorporated in a Collection, but this does not require the Collection
174 | apart from the Work itself to be made subject to the terms of this
175 | License. If You create a Collection, upon notice from any Licensor You
176 | must, to the extent practicable, remove from the Collection any credit
177 | as required by Section 4(b), as requested. If You create an
178 | Adaptation, upon notice from any Licensor You must, to the extent
179 | practicable, remove from the Adaptation any credit as required by
180 | Section 4(b), as requested.
181 | b. If You Distribute, or Publicly Perform the Work or any Adaptations or
182 | Collections, You must, unless a request has been made pursuant to
183 | Section 4(a), keep intact all copyright notices for the Work and
184 | provide, reasonable to the medium or means You are utilizing: (i) the
185 | name of the Original Author (or pseudonym, if applicable) if supplied,
186 | and/or if the Original Author and/or Licensor designate another party
187 | or parties (e.g., a sponsor institute, publishing entity, journal) for
188 | attribution ("Attribution Parties") in Licensor's copyright notice,
189 | terms of service or by other reasonable means, the name of such party
190 | or parties; (ii) the title of the Work if supplied; (iii) to the
191 | extent reasonably practicable, the URI, if any, that Licensor
192 | specifies to be associated with the Work, unless such URI does not
193 | refer to the copyright notice or licensing information for the Work;
194 | and (iv) , consistent with Section 3(b), in the case of an Adaptation,
195 | a credit identifying the use of the Work in the Adaptation (e.g.,
196 | "French translation of the Work by Original Author," or "Screenplay
197 | based on original Work by Original Author"). The credit required by
198 | this Section 4 (b) may be implemented in any reasonable manner;
199 | provided, however, that in the case of a Adaptation or Collection, at
200 | a minimum such credit will appear, if a credit for all contributing
201 | authors of the Adaptation or Collection appears, then as part of these
202 | credits and in a manner at least as prominent as the credits for the
203 | other contributing authors. For the avoidance of doubt, You may only
204 | use the credit required by this Section for the purpose of attribution
205 | in the manner set out above and, by exercising Your rights under this
206 | License, You may not implicitly or explicitly assert or imply any
207 | connection with, sponsorship or endorsement by the Original Author,
208 | Licensor and/or Attribution Parties, as appropriate, of You or Your
209 | use of the Work, without the separate, express prior written
210 | permission of the Original Author, Licensor and/or Attribution
211 | Parties.
212 | c. Except as otherwise agreed in writing by the Licensor or as may be
213 | otherwise permitted by applicable law, if You Reproduce, Distribute or
214 | Publicly Perform the Work either by itself or as part of any
215 | Adaptations or Collections, You must not distort, mutilate, modify or
216 | take other derogatory action in relation to the Work which would be
217 | prejudicial to the Original Author's honor or reputation. Licensor
218 | agrees that in those jurisdictions (e.g. Japan), in which any exercise
219 | of the right granted in Section 3(b) of this License (the right to
220 | make Adaptations) would be deemed to be a distortion, mutilation,
221 | modification or other derogatory action prejudicial to the Original
222 | Author's honor and reputation, the Licensor will waive or not assert,
223 | as appropriate, this Section, to the fullest extent permitted by the
224 | applicable national law, to enable You to reasonably exercise Your
225 | right under Section 3(b) of this License (right to make Adaptations)
226 | but not otherwise.
227 |
228 | 5. Representations, Warranties and Disclaimer
229 |
230 | UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR
231 | OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY
232 | KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,
233 | INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,
234 | FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF
235 | LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS,
236 | WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION
237 | OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
238 |
239 | 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE
240 | LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR
241 | ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES
242 | ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS
243 | BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
244 |
245 | 7. Termination
246 |
247 | a. This License and the rights granted hereunder will terminate
248 | automatically upon any breach by You of the terms of this License.
249 | Individuals or entities who have received Adaptations or Collections
250 | from You under this License, however, will not have their licenses
251 | terminated provided such individuals or entities remain in full
252 | compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will
253 | survive any termination of this License.
254 | b. Subject to the above terms and conditions, the license granted here is
255 | perpetual (for the duration of the applicable copyright in the Work).
256 | Notwithstanding the above, Licensor reserves the right to release the
257 | Work under different license terms or to stop distributing the Work at
258 | any time; provided, however that any such election will not serve to
259 | withdraw this License (or any other license that has been, or is
260 | required to be, granted under the terms of this License), and this
261 | License will continue in full force and effect unless terminated as
262 | stated above.
263 |
264 | 8. Miscellaneous
265 |
266 | a. Each time You Distribute or Publicly Perform the Work or a Collection,
267 | the Licensor offers to the recipient a license to the Work on the same
268 | terms and conditions as the license granted to You under this License.
269 | b. Each time You Distribute or Publicly Perform an Adaptation, Licensor
270 | offers to the recipient a license to the original Work on the same
271 | terms and conditions as the license granted to You under this License.
272 | c. If any provision of this License is invalid or unenforceable under
273 | applicable law, it shall not affect the validity or enforceability of
274 | the remainder of the terms of this License, and without further action
275 | by the parties to this agreement, such provision shall be reformed to
276 | the minimum extent necessary to make such provision valid and
277 | enforceable.
278 | d. No term or provision of this License shall be deemed waived and no
279 | breach consented to unless such waiver or consent shall be in writing
280 | and signed by the party to be charged with such waiver or consent.
281 | e. This License constitutes the entire agreement between the parties with
282 | respect to the Work licensed here. There are no understandings,
283 | agreements or representations with respect to the Work not specified
284 | here. Licensor shall not be bound by any additional provisions that
285 | may appear in any communication from You. This License may not be
286 | modified without the mutual written agreement of the Licensor and You.
287 | f. The rights granted under, and the subject matter referenced, in this
288 | License were drafted utilizing the terminology of the Berne Convention
289 | for the Protection of Literary and Artistic Works (as amended on
290 | September 28, 1979), the Rome Convention of 1961, the WIPO Copyright
291 | Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996
292 | and the Universal Copyright Convention (as revised on July 24, 1971).
293 | These rights and subject matter take effect in the relevant
294 | jurisdiction in which the License terms are sought to be enforced
295 | according to the corresponding provisions of the implementation of
296 | those treaty provisions in the applicable national law. If the
297 | standard suite of rights granted under applicable copyright law
298 | includes additional rights not granted under this License, such
299 | additional rights are deemed to be included in the License; this
300 | License is not intended to restrict the license of any rights under
301 | applicable law.
302 |
303 |
304 | Creative Commons Notice
305 |
306 | Creative Commons is not a party to this License, and makes no warranty
307 | whatsoever in connection with the Work. Creative Commons will not be
308 | liable to You or any party on any legal theory for any damages
309 | whatsoever, including without limitation any general, special,
310 | incidental or consequential damages arising in connection to this
311 | license. Notwithstanding the foregoing two (2) sentences, if Creative
312 | Commons has expressly identified itself as the Licensor hereunder, it
313 | shall have all rights and obligations of Licensor.
314 |
315 | Except for the limited purpose of indicating to the public that the
316 | Work is licensed under the CCPL, Creative Commons does not authorize
317 | the use by either party of the trademark "Creative Commons" or any
318 | related trademark or logo of Creative Commons without the prior
319 | written consent of Creative Commons. Any permitted use will be in
320 | compliance with Creative Commons' then-current trademark usage
321 | guidelines, as may be published on its website or otherwise made
322 | available upon request from time to time. For the avoidance of doubt,
323 | this trademark restriction does not form part of this License.
324 |
325 | Creative Commons may be contacted at https://creativecommons.org/.
326 |
--------------------------------------------------------------------------------
/05.Generalizing_GD.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Learning Multiple Weights at a Time: Generalizing Gradient Descent\n",
8 | "\n",
9 | "In this chapter, we will:\n",
10 | "\n",
11 | "- Implement Gradient Descent with multiple inputs.\n",
12 | "- Analyze the effects of freezing one weight.\n",
13 | "- Implement gradient descent with multiple outputs.\n",
14 | "- Implement gradient descent with multiple inputs and outputs.\n",
15 | "- Visualize weight values. \n",
16 | "- Visual dot products.\n",
17 | "\n",
18 | "> [Richard Branson] \"You don't learn to walk by following rules. You learn by doing and by falling over.\""
19 | ]
20 | },
21 | {
22 | "cell_type": "markdown",
23 | "metadata": {},
24 | "source": [
25 | "## Gradient Descent Learning with Multiple Inputs"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "
\n",
172 | "\n",
173 | "`Delta` in this case is a measure of how much higher or lower we want the node's value to be, to predict perfectly given the currrent trraining example. `Weight Delta` is a derivative-based estimate of the direction and amount we should move a weight to reduce `node_delta`, accounting for scaling, negative reversal, and stopping."
174 | ]
175 | },
176 | {
177 | "cell_type": "markdown",
178 | "metadata": {},
179 | "source": [
180 | "## Let's Watch several steps of learning"
181 | ]
182 | },
183 | {
184 | "cell_type": "markdown",
185 | "metadata": {},
186 | "source": [
187 | "
"
188 | ]
189 | },
190 | {
191 | "cell_type": "markdown",
192 | "metadata": {},
193 | "source": [
194 | "In the above figure, we made three individual error/weight curves, one for each weight. The slope of each curve is reflected by the gradient values. The gradient is steeper for (a) than the others because (a) has an input value that is significantly higher than the others and thus, has a higher derivative."
195 | ]
196 | },
197 | {
198 | "cell_type": "markdown",
199 | "metadata": {},
200 | "source": [
201 | "
\n",
202 | "
"
203 | ]
204 | },
205 | {
206 | "cell_type": "markdown",
207 | "metadata": {},
208 | "source": [
209 | "We can notice that most of the learning was performed on the weight with the largest input because the input changes the slope significantly. This might be a good idea, in fact, there is a technique called `data normalization` that encourages learning across all weights despite data characteristics such as this. \n",
210 | "\n",
211 | "This significant difference in slope will force us to set a learning rate lower that the standard one (.01 instead of .1)."
212 | ]
213 | },
214 | {
215 | "cell_type": "markdown",
216 | "metadata": {},
217 | "source": [
218 | "## Freezing one weight: What does it do?"
219 | ]
220 | },
221 | {
222 | "cell_type": "markdown",
223 | "metadata": {},
224 | "source": [
225 | "Freezing weights is a great exercise to understand how the weights effect each other. We are going to train again, except weight (a) won't ever be adjusted, we will try to learn using only weights (b) and (c)."
226 | ]
227 | },
228 | {
229 | "cell_type": "code",
230 | "execution_count": 6,
231 | "metadata": {},
232 | "outputs": [
233 | {
234 | "name": "stdout",
235 | "output_type": "stream",
236 | "text": [
237 | "Iteration: 0\n",
238 | "Prediction: 0.96376\n",
239 | "Error: 0.00131\n",
240 | "Weights: [0.1119, 0.20091, -0.09832]\n",
241 | "- - - - - -\n",
242 | "Iteration: 1\n",
243 | "Prediction: 0.98401\n",
244 | "Error: 0.00026\n",
245 | "Weights: [0.1119, 0.20798, -0.08527]\n",
246 | "- - - - - -\n",
247 | "Iteration: 2\n",
248 | "Prediction: 0.99294\n",
249 | "Error: 5e-05\n",
250 | "Weights: [0.1119, 0.2111, -0.07952]\n",
251 | "- - - - - -\n"
252 | ]
253 | }
254 | ],
255 | "source": [
256 | "lr = .3\n",
257 | "\n",
258 | "for iter in range(3):\n",
259 | " pred = neural_network(x0, weights)\n",
260 | " \n",
261 | " error = (pred - true) ** 2\n",
262 | " delta = (pred - true)\n",
263 | " \n",
264 | " gradients = ele_mul(delta, x0)\n",
265 | " gradients[0] = 0\n",
266 | " \n",
267 | " print(f\"Iteration: {iter}\")\n",
268 | " print(f\"Prediction: {round(pred, 5)}\")\n",
269 | " print(f\"Error: {round(error, 5)}\")\n",
270 | " print(f\"Weights: {[round(w, 5) for w in weights]}\")\n",
271 | " print(f\"- - - - - -\")\n",
272 | " \n",
273 | " for i in range(len(weights)):\n",
274 | " weights[i] -= lr * gradients[i]"
275 | ]
276 | },
277 | {
278 | "cell_type": "markdown",
279 | "metadata": {},
280 | "source": [
281 | "
\n",
282 | " \n",
283 | " \n",
284 | "
\n",
285 | "\n",
286 | "It is somewhat surprising that the rest of the weights were learned despite removing the biggest contributing feature. However, the graph (a) still finds the bottom of the bowel because the whole gradient loss graph changes with error. The black dot can move horizontally only if the weight is updated, but because (a)'s weight is frozen, the dot must stay fixed. However, this does not stop the error to go to 0.\n",
287 | "\n",
288 | "This is an extremely important lesson. If we converge using only (b) and (c) then tried to train (a), it wouldn't move because the error already reached 0. In other words, (a) may be a powerful input with lots of predictive power, but if **the network accidentally figures out how to predict accurately on the training data without it, then it will never learn how to incorporate (a) into its prediction.**\n",
289 | "\n",
290 | "The 3 graphs representing the loss function according to each input element are. in reality **2D slices of a 4-dimensional space**. 3 of the dimensions arer. the weight values, and the forth is the error. The shape of the 4-D function represents the **error place** or the curvature of the loss function. The curvature is determined by the training data.\n",
291 | "\n",
292 | "Our goal is to find the weight configuration at the global minimum of the loss curvature."
293 | ]
294 | },
295 | {
296 | "cell_type": "markdown",
297 | "metadata": {},
298 | "source": [
299 | "## Gradient Descent Learning with Multiple Outputs"
300 | ]
301 | },
302 | {
303 | "cell_type": "markdown",
304 | "metadata": {},
305 | "source": [
306 | "Neural networks can also make multiple predictions using only a single input. Beyond that, we understand that a simple mechanism (stochastic gradient descent) is constantly being used to perform learning across a wide veriety of architectures."
307 | ]
308 | },
309 | {
310 | "cell_type": "code",
311 | "execution_count": 7,
312 | "metadata": {},
313 | "outputs": [],
314 | "source": [
315 | "# An empty network with multiple outputs.\n",
316 | "weights = [.3, .2, .9]\n",
317 | "\n",
318 | "def neural_network(X, W):\n",
319 | " prediction = ele_mul(X, W)\n",
320 | " return prediction"
321 | ]
322 | },
323 | {
324 | "cell_type": "code",
325 | "execution_count": 8,
326 | "metadata": {},
327 | "outputs": [],
328 | "source": [
329 | "# PREDICT: Making a prediction and calculating error and delta.\n",
330 | "wlrec = [.65, 1, 1, .9]\n",
331 | "hurt = [.1, 0, 0, .1]\n",
332 | "win = [1, 1, 0, 1]\n",
333 | "sad = [.1, 0, .1, .2]\n",
334 | "\n",
335 | "x0 = wlrec[0]\n",
336 | "target = [hurt[0], win[0], sad[0]]\n",
337 | "\n",
338 | "pred = neural_network(x0, weights)\n",
339 | "\n",
340 | "error = [0, 0, 0]\n",
341 | "pure_error = [0, 0, 0]\n",
342 | "\n",
343 | "for i in range(len(target)):\n",
344 | " error[i] = (pred[i] - target[i]) ** 2\n",
345 | " pure_error[i] = pred[i] - target[i]"
346 | ]
347 | },
348 | {
349 | "cell_type": "code",
350 | "execution_count": 9,
351 | "metadata": {},
352 | "outputs": [
353 | {
354 | "data": {
355 | "text/plain": [
356 | "([0.195, 0.13, 0.5850000000000001],\n",
357 | " [0.009025, 0.7569, 0.2352250000000001],\n",
358 | " [0.095, -0.87, 0.4850000000000001])"
359 | ]
360 | },
361 | "execution_count": 9,
362 | "metadata": {},
363 | "output_type": "execute_result"
364 | }
365 | ],
366 | "source": [
367 | "pred, error, pure_error"
368 | ]
369 | },
370 | {
371 | "cell_type": "code",
372 | "execution_count": 10,
373 | "metadata": {},
374 | "outputs": [],
375 | "source": [
376 | "# COMPARE: calculating each gradient.\n",
377 | "gradients = [x0 * pure_error[0], x0 * pure_error[1], x0 * pure_error[2]]"
378 | ]
379 | },
380 | {
381 | "cell_type": "code",
382 | "execution_count": 11,
383 | "metadata": {},
384 | "outputs": [],
385 | "source": [
386 | "# UPDATE: Updating the Weights.\n",
387 | "lr = 3\n",
388 | "for i in range(len(weights)):\n",
389 | " weights[i] -= lr * gradients[i]"
390 | ]
391 | },
392 | {
393 | "cell_type": "code",
394 | "execution_count": 12,
395 | "metadata": {},
396 | "outputs": [
397 | {
398 | "data": {
399 | "text/plain": [
400 | "[0.9753749999999997, 16.12025, -0.38887500000000247]"
401 | ]
402 | },
403 | "execution_count": 12,
404 | "metadata": {},
405 | "output_type": "execute_result"
406 | }
407 | ],
408 | "source": [
409 | "[input * w for w in weights]"
410 | ]
411 | },
412 | {
413 | "cell_type": "markdown",
414 | "metadata": {},
415 | "source": [
416 | "## Gradient Descent with Multiple Inputs & Outputs"
417 | ]
418 | },
419 | {
420 | "cell_type": "markdown",
421 | "metadata": {},
422 | "source": [
423 | "
\n",
424 | "\n",
425 | "SGD naturally generalize to arbitrary architectures. Let's implement SGD with multiple inputs and outputs:"
426 | ]
427 | },
428 | {
429 | "cell_type": "code",
430 | "execution_count": 13,
431 | "metadata": {},
432 | "outputs": [],
433 | "source": [
434 | "# 1. An empty Network with multiple inputs and outputs.\n",
435 | "weights = [[.1, .1, -.3], \n",
436 | " [.1, .2, 0], \n",
437 | " [0, 1.3, .1]]\n",
438 | "\n",
439 | "def vect_mat_mul(vect, matrix):\n",
440 | " assert(len(vect) == len(matrix))\n",
441 | " output = [0, 0, 0]\n",
442 | " for i in range(len(vect)):\n",
443 | " output[i] = w_sum(vect, matrix[i])\n",
444 | " return output\n",
445 | "\n",
446 | "def neural_network(X, W):\n",
447 | " prediction = vect_mat_mul(X, W)\n",
448 | " return prediction"
449 | ]
450 | },
451 | {
452 | "cell_type": "code",
453 | "execution_count": 14,
454 | "metadata": {},
455 | "outputs": [],
456 | "source": [
457 | "# 2. PREDICT: Making a Prediction & Calculating Error & Delta (Pure Error).\n",
458 | "\n",
459 | "# Inputs\n",
460 | "toes = [8.5, 9.5, 9.9, 9.0]\n",
461 | "wlrec = [.65, 0.8, 0.8, 0.9]\n",
462 | "nfans = [1.2, 1.3, 0.5, 1.0]\n",
463 | "\n",
464 | "# Outputs\n",
465 | "hurt = [.1, 0, 0,.1]\n",
466 | "win = [ 1, 1, 0, 1]\n",
467 | "sad = [.1, 0,.1,.2]\n",
468 | "\n",
469 | "# learning rate\n",
470 | "alpha = .01\n",
471 | "\n",
472 | "x0 = [toes[0], wlrec[0], nfans[0]]\n",
473 | "target = [hurt[0], win[0], sad[0]]\n",
474 | "\n",
475 | "prediction = neural_network(x0, weights)\n",
476 | "error, delta = [0, 0, 0], [0, 0, 0]\n",
477 | "for i in range(len(prediction)):\n",
478 | " error[i] = (prediction[i] - target[i]) ** 2\n",
479 | " delta[i] = prediction[i] - target[i]"
480 | ]
481 | },
482 | {
483 | "cell_type": "code",
484 | "execution_count": 15,
485 | "metadata": {},
486 | "outputs": [],
487 | "source": [
488 | "import numpy as np"
489 | ]
490 | },
491 | {
492 | "cell_type": "code",
493 | "execution_count": 16,
494 | "metadata": {},
495 | "outputs": [],
496 | "source": [
497 | "# 3. COMPARE: Calculating each weight_delta & Putting it on each weight.\n",
498 | "\n",
499 | "# we have 9 gradients, or weight deltas.\n",
500 | "def outer_prod(vect_a, vect_b):\n",
501 | " out = np.zeros((len(vect_a), len(vect_b)))\n",
502 | " for i in range(len(vect_a)):\n",
503 | " for j in range(len(vect_b)):\n",
504 | " out[i][j] = vect_a[i]*vect_b[j]\n",
505 | " return out\n",
506 | "\n",
507 | "gradients = outer_prod(x0, delta)"
508 | ]
509 | },
510 | {
511 | "cell_type": "code",
512 | "execution_count": 17,
513 | "metadata": {},
514 | "outputs": [],
515 | "source": [
516 | "# 4. LEARN: Updating the Weights\n",
517 | "for i in range(len(weights)):\n",
518 | " for j in range(len(weights[0])):\n",
519 | " weights[i][j] -= alpha * gradients[i][j]"
520 | ]
521 | },
522 | {
523 | "cell_type": "markdown",
524 | "metadata": {},
525 | "source": [
526 | "## What Do These weights Learn?"
527 | ]
528 | },
529 | {
530 | "cell_type": "markdown",
531 | "metadata": {},
532 | "source": [
533 | "Even though we understand \"how\" learning happens, another interesting question is \"**what do the weights store while learning?**\"\n",
534 | "\n",
535 | "To answer this question, we will move on to the first real-world dataset. It's called the modified national institure of standards and technology (MNIST) dataset. It consists of digits that high school students and employees of the US census bureau wrote some years ago. The interesting thing is that these images are black-and-white images of people's handwriting. Accompanying each digit is the actual number they were writing (0-9). Each image is 784 pixels (28 x 28). \n",
536 | "\n",
537 | "In this case, the neural network must have 784 input features. On the other end, we want to predict **10 probabilities**, one for each digit. The neural network will tell us which digit is most likely to be what was drawn.\n",
538 | "\n",
539 | "Let's take a look at the MNIST dataset:"
540 | ]
541 | },
542 | {
543 | "cell_type": "code",
544 | "execution_count": 25,
545 | "metadata": {},
546 | "outputs": [],
547 | "source": [
548 | "from tensorflow.keras.datasets import mnist"
549 | ]
550 | },
551 | {
552 | "cell_type": "code",
553 | "execution_count": 26,
554 | "metadata": {},
555 | "outputs": [
556 | {
557 | "name": "stdout",
558 | "output_type": "stream",
559 | "text": [
560 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
561 | "11493376/11490434 [==============================] - 9s 1us/step\n"
562 | ]
563 | }
564 | ],
565 | "source": [
566 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
567 | ]
568 | },
569 | {
570 | "cell_type": "code",
571 | "execution_count": 27,
572 | "metadata": {},
573 | "outputs": [],
574 | "source": [
575 | "# let's take a sample.\n",
576 | "images = X_train[0:1000]\n",
577 | "labels = y_train[0:1000]"
578 | ]
579 | },
580 | {
581 | "cell_type": "code",
582 | "execution_count": 28,
583 | "metadata": {},
584 | "outputs": [
585 | {
586 | "data": {
587 | "text/plain": [
588 | "((1000, 28, 28), (1000,))"
589 | ]
590 | },
591 | "execution_count": 28,
592 | "metadata": {},
593 | "output_type": "execute_result"
594 | }
595 | ],
596 | "source": [
597 | "images.shape, labels.shape"
598 | ]
599 | },
600 | {
601 | "cell_type": "markdown",
602 | "metadata": {},
603 | "source": [
604 | "
\n",
605 | " \n",
606 | "
\n",
607 | "\n",
608 | "This diagram represents the new MNIST classification neural network. If this network could perdict perfectly, It would take in an image's pixels (say, a 2) then predict `1` in the correct output position and `0` everywhere else. If it were able to do this correctly for all images in the dataset, it would have no error.\n",
609 | "\n",
610 | ".. But, **what does it mean to modify a bunch of weights to learn an aggregate pattern**?"
611 | ]
612 | },
613 | {
614 | "cell_type": "markdown",
615 | "metadata": {},
616 | "source": [
617 | "## Visualizing Weight Values"
618 | ]
619 | },
620 | {
621 | "cell_type": "markdown",
622 | "metadata": {},
623 | "source": [
624 | "
\n",
625 | " \n",
626 | "
\n",
627 | "\n",
628 | "An interesting and intuitive practice in neural network research is to visualize the weights as if they were an image. Each output node has a weight coming from each pixel, and so what is this relationship?\n",
629 | "- If the weight is high, it means the model believes there's a high degree of correlation between that pixel and the number 2.\n",
630 | "- If the weight is very low (even negative), the model believes there is a very low correlation between that pixel and the number 2.\n",
631 | "\n",
632 | "The network learned to construct artifacts of `2` and `1` in the above figure, which were created using the weights for `2` and `1`. The natural color (red) represents a `0`. The key to understanding why these artifacts were created, we should go back to the notion of the **dot product**."
633 | ]
634 | },
635 | {
636 | "cell_type": "markdown",
637 | "metadata": {},
638 | "source": [
639 | "## Visualizing Dot Products (Weighted Sums)"
640 | ]
641 | },
642 | {
643 | "cell_type": "code",
644 | "execution_count": 29,
645 | "metadata": {},
646 | "outputs": [],
647 | "source": [
648 | "# remember that the dot product is a mathematical measure of similarity.\n",
649 | "import numpy as np\n",
650 | "\n",
651 | "a = np.array([0, 1, 0, 1])\n",
652 | "b = np.array([1, 0, 1, 0])"
653 | ]
654 | },
655 | {
656 | "cell_type": "code",
657 | "execution_count": 30,
658 | "metadata": {},
659 | "outputs": [
660 | {
661 | "data": {
662 | "text/plain": [
663 | "0"
664 | ]
665 | },
666 | "execution_count": 30,
667 | "metadata": {},
668 | "output_type": "execute_result"
669 | }
670 | ],
671 | "source": [
672 | "a.dot(b)"
673 | ]
674 | },
675 | {
676 | "cell_type": "markdown",
677 | "metadata": {},
678 | "source": [
679 | "A dot product is a loose measurement of similarity between two vectors. \n",
680 | "\n",
681 | "
\n",
682 | " \n",
683 | "
\n",
684 | "\n",
685 | "Meaning, if the weight vector for `2` is similar to the input vector, then it will output a high score for `2` resulting in a higher probability. "
686 | ]
687 | },
688 | {
689 | "cell_type": "markdown",
690 | "metadata": {},
691 | "source": [
692 | "## Summary\n",
693 | "### Gradient Descent is a General Learning Algorithm"
694 | ]
695 | },
696 | {
697 | "cell_type": "markdown",
698 | "metadata": {},
699 | "source": [
700 | "Gradient Descent is a general learning algorithm. The most important subtext in this chapter is that gradient descent is a very flexible algorithm. If we combine. weights in a way that allows us to calculate the error function and a delta, then gradient descent can show us how to reduce error.\n",
701 | "\n",
702 | "We will spend the rest of this book exploring different types of weight combinations and error functions for which gradient descent is useful."
703 | ]
704 | },
705 | {
706 | "cell_type": "markdown",
707 | "metadata": {},
708 | "source": [
709 | "# Sketches"
710 | ]
711 | },
712 | {
713 | "cell_type": "markdown",
714 | "metadata": {},
715 | "source": [
716 | "
\n",
172 | "\n",
173 | "`Delta` in this case is a measure of how much higher or lower we want the node's value to be, to predict perfectly given the currrent trraining example. `Weight Delta` is a derivative-based estimate of the direction and amount we should move a weight to reduce `node_delta`, accounting for scaling, negative reversal, and stopping."
174 | ]
175 | },
176 | {
177 | "cell_type": "markdown",
178 | "metadata": {},
179 | "source": [
180 | "## Let's Watch several steps of learning"
181 | ]
182 | },
183 | {
184 | "cell_type": "markdown",
185 | "metadata": {},
186 | "source": [
187 | "
"
188 | ]
189 | },
190 | {
191 | "cell_type": "markdown",
192 | "metadata": {},
193 | "source": [
194 | "In the above figure, we made three individual error/weight curves, one for each weight. The slope of each curve is reflected by the gradient values. The gradient is steeper for (a) than the others because (a) has an input value that is significantly higher than the others and thus, has a higher derivative."
195 | ]
196 | },
197 | {
198 | "cell_type": "markdown",
199 | "metadata": {},
200 | "source": [
201 | "
\n",
202 | "
"
203 | ]
204 | },
205 | {
206 | "cell_type": "markdown",
207 | "metadata": {},
208 | "source": [
209 | "We can notice that most of the learning was performed on the weight with the largest input because the input changes the slope significantly. This might be a good idea, in fact, there is a technique called `data normalization` that encourages learning across all weights despite data characteristics such as this. \n",
210 | "\n",
211 | "This significant difference in slope will force us to set a learning rate lower that the standard one (.01 instead of .1)."
212 | ]
213 | },
214 | {
215 | "cell_type": "markdown",
216 | "metadata": {},
217 | "source": [
218 | "## Freezing one weight: What does it do?"
219 | ]
220 | },
221 | {
222 | "cell_type": "markdown",
223 | "metadata": {},
224 | "source": [
225 | "Freezing weights is a great exercise to understand how the weights effect each other. We are going to train again, except weight (a) won't ever be adjusted, we will try to learn using only weights (b) and (c)."
226 | ]
227 | },
228 | {
229 | "cell_type": "code",
230 | "execution_count": 6,
231 | "metadata": {},
232 | "outputs": [
233 | {
234 | "name": "stdout",
235 | "output_type": "stream",
236 | "text": [
237 | "Iteration: 0\n",
238 | "Prediction: 0.96376\n",
239 | "Error: 0.00131\n",
240 | "Weights: [0.1119, 0.20091, -0.09832]\n",
241 | "- - - - - -\n",
242 | "Iteration: 1\n",
243 | "Prediction: 0.98401\n",
244 | "Error: 0.00026\n",
245 | "Weights: [0.1119, 0.20798, -0.08527]\n",
246 | "- - - - - -\n",
247 | "Iteration: 2\n",
248 | "Prediction: 0.99294\n",
249 | "Error: 5e-05\n",
250 | "Weights: [0.1119, 0.2111, -0.07952]\n",
251 | "- - - - - -\n"
252 | ]
253 | }
254 | ],
255 | "source": [
256 | "lr = .3\n",
257 | "\n",
258 | "for iter in range(3):\n",
259 | " pred = neural_network(x0, weights)\n",
260 | " \n",
261 | " error = (pred - true) ** 2\n",
262 | " delta = (pred - true)\n",
263 | " \n",
264 | " gradients = ele_mul(delta, x0)\n",
265 | " gradients[0] = 0\n",
266 | " \n",
267 | " print(f\"Iteration: {iter}\")\n",
268 | " print(f\"Prediction: {round(pred, 5)}\")\n",
269 | " print(f\"Error: {round(error, 5)}\")\n",
270 | " print(f\"Weights: {[round(w, 5) for w in weights]}\")\n",
271 | " print(f\"- - - - - -\")\n",
272 | " \n",
273 | " for i in range(len(weights)):\n",
274 | " weights[i] -= lr * gradients[i]"
275 | ]
276 | },
277 | {
278 | "cell_type": "markdown",
279 | "metadata": {},
280 | "source": [
281 | "
\n",
282 | " \n",
283 | " \n",
284 | "
\n",
285 | "\n",
286 | "It is somewhat surprising that the rest of the weights were learned despite removing the biggest contributing feature. However, the graph (a) still finds the bottom of the bowel because the whole gradient loss graph changes with error. The black dot can move horizontally only if the weight is updated, but because (a)'s weight is frozen, the dot must stay fixed. However, this does not stop the error to go to 0.\n",
287 | "\n",
288 | "This is an extremely important lesson. If we converge using only (b) and (c) then tried to train (a), it wouldn't move because the error already reached 0. In other words, (a) may be a powerful input with lots of predictive power, but if **the network accidentally figures out how to predict accurately on the training data without it, then it will never learn how to incorporate (a) into its prediction.**\n",
289 | "\n",
290 | "The 3 graphs representing the loss function according to each input element are. in reality **2D slices of a 4-dimensional space**. 3 of the dimensions arer. the weight values, and the forth is the error. The shape of the 4-D function represents the **error place** or the curvature of the loss function. The curvature is determined by the training data.\n",
291 | "\n",
292 | "Our goal is to find the weight configuration at the global minimum of the loss curvature."
293 | ]
294 | },
295 | {
296 | "cell_type": "markdown",
297 | "metadata": {},
298 | "source": [
299 | "## Gradient Descent Learning with Multiple Outputs"
300 | ]
301 | },
302 | {
303 | "cell_type": "markdown",
304 | "metadata": {},
305 | "source": [
306 | "Neural networks can also make multiple predictions using only a single input. Beyond that, we understand that a simple mechanism (stochastic gradient descent) is constantly being used to perform learning across a wide veriety of architectures."
307 | ]
308 | },
309 | {
310 | "cell_type": "code",
311 | "execution_count": 7,
312 | "metadata": {},
313 | "outputs": [],
314 | "source": [
315 | "# An empty network with multiple outputs.\n",
316 | "weights = [.3, .2, .9]\n",
317 | "\n",
318 | "def neural_network(X, W):\n",
319 | " prediction = ele_mul(X, W)\n",
320 | " return prediction"
321 | ]
322 | },
323 | {
324 | "cell_type": "code",
325 | "execution_count": 8,
326 | "metadata": {},
327 | "outputs": [],
328 | "source": [
329 | "# PREDICT: Making a prediction and calculating error and delta.\n",
330 | "wlrec = [.65, 1, 1, .9]\n",
331 | "hurt = [.1, 0, 0, .1]\n",
332 | "win = [1, 1, 0, 1]\n",
333 | "sad = [.1, 0, .1, .2]\n",
334 | "\n",
335 | "x0 = wlrec[0]\n",
336 | "target = [hurt[0], win[0], sad[0]]\n",
337 | "\n",
338 | "pred = neural_network(x0, weights)\n",
339 | "\n",
340 | "error = [0, 0, 0]\n",
341 | "pure_error = [0, 0, 0]\n",
342 | "\n",
343 | "for i in range(len(target)):\n",
344 | " error[i] = (pred[i] - target[i]) ** 2\n",
345 | " pure_error[i] = pred[i] - target[i]"
346 | ]
347 | },
348 | {
349 | "cell_type": "code",
350 | "execution_count": 9,
351 | "metadata": {},
352 | "outputs": [
353 | {
354 | "data": {
355 | "text/plain": [
356 | "([0.195, 0.13, 0.5850000000000001],\n",
357 | " [0.009025, 0.7569, 0.2352250000000001],\n",
358 | " [0.095, -0.87, 0.4850000000000001])"
359 | ]
360 | },
361 | "execution_count": 9,
362 | "metadata": {},
363 | "output_type": "execute_result"
364 | }
365 | ],
366 | "source": [
367 | "pred, error, pure_error"
368 | ]
369 | },
370 | {
371 | "cell_type": "code",
372 | "execution_count": 10,
373 | "metadata": {},
374 | "outputs": [],
375 | "source": [
376 | "# COMPARE: calculating each gradient.\n",
377 | "gradients = [x0 * pure_error[0], x0 * pure_error[1], x0 * pure_error[2]]"
378 | ]
379 | },
380 | {
381 | "cell_type": "code",
382 | "execution_count": 11,
383 | "metadata": {},
384 | "outputs": [],
385 | "source": [
386 | "# UPDATE: Updating the Weights.\n",
387 | "lr = 3\n",
388 | "for i in range(len(weights)):\n",
389 | " weights[i] -= lr * gradients[i]"
390 | ]
391 | },
392 | {
393 | "cell_type": "code",
394 | "execution_count": 12,
395 | "metadata": {},
396 | "outputs": [
397 | {
398 | "data": {
399 | "text/plain": [
400 | "[0.9753749999999997, 16.12025, -0.38887500000000247]"
401 | ]
402 | },
403 | "execution_count": 12,
404 | "metadata": {},
405 | "output_type": "execute_result"
406 | }
407 | ],
408 | "source": [
409 | "[input * w for w in weights]"
410 | ]
411 | },
412 | {
413 | "cell_type": "markdown",
414 | "metadata": {},
415 | "source": [
416 | "## Gradient Descent with Multiple Inputs & Outputs"
417 | ]
418 | },
419 | {
420 | "cell_type": "markdown",
421 | "metadata": {},
422 | "source": [
423 | "
\n",
424 | "\n",
425 | "SGD naturally generalize to arbitrary architectures. Let's implement SGD with multiple inputs and outputs:"
426 | ]
427 | },
428 | {
429 | "cell_type": "code",
430 | "execution_count": 13,
431 | "metadata": {},
432 | "outputs": [],
433 | "source": [
434 | "# 1. An empty Network with multiple inputs and outputs.\n",
435 | "weights = [[.1, .1, -.3], \n",
436 | " [.1, .2, 0], \n",
437 | " [0, 1.3, .1]]\n",
438 | "\n",
439 | "def vect_mat_mul(vect, matrix):\n",
440 | " assert(len(vect) == len(matrix))\n",
441 | " output = [0, 0, 0]\n",
442 | " for i in range(len(vect)):\n",
443 | " output[i] = w_sum(vect, matrix[i])\n",
444 | " return output\n",
445 | "\n",
446 | "def neural_network(X, W):\n",
447 | " prediction = vect_mat_mul(X, W)\n",
448 | " return prediction"
449 | ]
450 | },
451 | {
452 | "cell_type": "code",
453 | "execution_count": 14,
454 | "metadata": {},
455 | "outputs": [],
456 | "source": [
457 | "# 2. PREDICT: Making a Prediction & Calculating Error & Delta (Pure Error).\n",
458 | "\n",
459 | "# Inputs\n",
460 | "toes = [8.5, 9.5, 9.9, 9.0]\n",
461 | "wlrec = [.65, 0.8, 0.8, 0.9]\n",
462 | "nfans = [1.2, 1.3, 0.5, 1.0]\n",
463 | "\n",
464 | "# Outputs\n",
465 | "hurt = [.1, 0, 0,.1]\n",
466 | "win = [ 1, 1, 0, 1]\n",
467 | "sad = [.1, 0,.1,.2]\n",
468 | "\n",
469 | "# learning rate\n",
470 | "alpha = .01\n",
471 | "\n",
472 | "x0 = [toes[0], wlrec[0], nfans[0]]\n",
473 | "target = [hurt[0], win[0], sad[0]]\n",
474 | "\n",
475 | "prediction = neural_network(x0, weights)\n",
476 | "error, delta = [0, 0, 0], [0, 0, 0]\n",
477 | "for i in range(len(prediction)):\n",
478 | " error[i] = (prediction[i] - target[i]) ** 2\n",
479 | " delta[i] = prediction[i] - target[i]"
480 | ]
481 | },
482 | {
483 | "cell_type": "code",
484 | "execution_count": 15,
485 | "metadata": {},
486 | "outputs": [],
487 | "source": [
488 | "import numpy as np"
489 | ]
490 | },
491 | {
492 | "cell_type": "code",
493 | "execution_count": 16,
494 | "metadata": {},
495 | "outputs": [],
496 | "source": [
497 | "# 3. COMPARE: Calculating each weight_delta & Putting it on each weight.\n",
498 | "\n",
499 | "# we have 9 gradients, or weight deltas.\n",
500 | "def outer_prod(vect_a, vect_b):\n",
501 | " out = np.zeros((len(vect_a), len(vect_b)))\n",
502 | " for i in range(len(vect_a)):\n",
503 | " for j in range(len(vect_b)):\n",
504 | " out[i][j] = vect_a[i]*vect_b[j]\n",
505 | " return out\n",
506 | "\n",
507 | "gradients = outer_prod(x0, delta)"
508 | ]
509 | },
510 | {
511 | "cell_type": "code",
512 | "execution_count": 17,
513 | "metadata": {},
514 | "outputs": [],
515 | "source": [
516 | "# 4. LEARN: Updating the Weights\n",
517 | "for i in range(len(weights)):\n",
518 | " for j in range(len(weights[0])):\n",
519 | " weights[i][j] -= alpha * gradients[i][j]"
520 | ]
521 | },
522 | {
523 | "cell_type": "markdown",
524 | "metadata": {},
525 | "source": [
526 | "## What Do These weights Learn?"
527 | ]
528 | },
529 | {
530 | "cell_type": "markdown",
531 | "metadata": {},
532 | "source": [
533 | "Even though we understand \"how\" learning happens, another interesting question is \"**what do the weights store while learning?**\"\n",
534 | "\n",
535 | "To answer this question, we will move on to the first real-world dataset. It's called the modified national institure of standards and technology (MNIST) dataset. It consists of digits that high school students and employees of the US census bureau wrote some years ago. The interesting thing is that these images are black-and-white images of people's handwriting. Accompanying each digit is the actual number they were writing (0-9). Each image is 784 pixels (28 x 28). \n",
536 | "\n",
537 | "In this case, the neural network must have 784 input features. On the other end, we want to predict **10 probabilities**, one for each digit. The neural network will tell us which digit is most likely to be what was drawn.\n",
538 | "\n",
539 | "Let's take a look at the MNIST dataset:"
540 | ]
541 | },
542 | {
543 | "cell_type": "code",
544 | "execution_count": 25,
545 | "metadata": {},
546 | "outputs": [],
547 | "source": [
548 | "from tensorflow.keras.datasets import mnist"
549 | ]
550 | },
551 | {
552 | "cell_type": "code",
553 | "execution_count": 26,
554 | "metadata": {},
555 | "outputs": [
556 | {
557 | "name": "stdout",
558 | "output_type": "stream",
559 | "text": [
560 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
561 | "11493376/11490434 [==============================] - 9s 1us/step\n"
562 | ]
563 | }
564 | ],
565 | "source": [
566 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
567 | ]
568 | },
569 | {
570 | "cell_type": "code",
571 | "execution_count": 27,
572 | "metadata": {},
573 | "outputs": [],
574 | "source": [
575 | "# let's take a sample.\n",
576 | "images = X_train[0:1000]\n",
577 | "labels = y_train[0:1000]"
578 | ]
579 | },
580 | {
581 | "cell_type": "code",
582 | "execution_count": 28,
583 | "metadata": {},
584 | "outputs": [
585 | {
586 | "data": {
587 | "text/plain": [
588 | "((1000, 28, 28), (1000,))"
589 | ]
590 | },
591 | "execution_count": 28,
592 | "metadata": {},
593 | "output_type": "execute_result"
594 | }
595 | ],
596 | "source": [
597 | "images.shape, labels.shape"
598 | ]
599 | },
600 | {
601 | "cell_type": "markdown",
602 | "metadata": {},
603 | "source": [
604 | "
\n",
605 | " \n",
606 | "
\n",
607 | "\n",
608 | "This diagram represents the new MNIST classification neural network. If this network could perdict perfectly, It would take in an image's pixels (say, a 2) then predict `1` in the correct output position and `0` everywhere else. If it were able to do this correctly for all images in the dataset, it would have no error.\n",
609 | "\n",
610 | ".. But, **what does it mean to modify a bunch of weights to learn an aggregate pattern**?"
611 | ]
612 | },
613 | {
614 | "cell_type": "markdown",
615 | "metadata": {},
616 | "source": [
617 | "## Visualizing Weight Values"
618 | ]
619 | },
620 | {
621 | "cell_type": "markdown",
622 | "metadata": {},
623 | "source": [
624 | "
\n",
625 | " \n",
626 | "
\n",
627 | "\n",
628 | "An interesting and intuitive practice in neural network research is to visualize the weights as if they were an image. Each output node has a weight coming from each pixel, and so what is this relationship?\n",
629 | "- If the weight is high, it means the model believes there's a high degree of correlation between that pixel and the number 2.\n",
630 | "- If the weight is very low (even negative), the model believes there is a very low correlation between that pixel and the number 2.\n",
631 | "\n",
632 | "The network learned to construct artifacts of `2` and `1` in the above figure, which were created using the weights for `2` and `1`. The natural color (red) represents a `0`. The key to understanding why these artifacts were created, we should go back to the notion of the **dot product**."
633 | ]
634 | },
635 | {
636 | "cell_type": "markdown",
637 | "metadata": {},
638 | "source": [
639 | "## Visualizing Dot Products (Weighted Sums)"
640 | ]
641 | },
642 | {
643 | "cell_type": "code",
644 | "execution_count": 29,
645 | "metadata": {},
646 | "outputs": [],
647 | "source": [
648 | "# remember that the dot product is a mathematical measure of similarity.\n",
649 | "import numpy as np\n",
650 | "\n",
651 | "a = np.array([0, 1, 0, 1])\n",
652 | "b = np.array([1, 0, 1, 0])"
653 | ]
654 | },
655 | {
656 | "cell_type": "code",
657 | "execution_count": 30,
658 | "metadata": {},
659 | "outputs": [
660 | {
661 | "data": {
662 | "text/plain": [
663 | "0"
664 | ]
665 | },
666 | "execution_count": 30,
667 | "metadata": {},
668 | "output_type": "execute_result"
669 | }
670 | ],
671 | "source": [
672 | "a.dot(b)"
673 | ]
674 | },
675 | {
676 | "cell_type": "markdown",
677 | "metadata": {},
678 | "source": [
679 | "A dot product is a loose measurement of similarity between two vectors. \n",
680 | "\n",
681 | "
\n",
682 | " \n",
683 | "
\n",
684 | "\n",
685 | "Meaning, if the weight vector for `2` is similar to the input vector, then it will output a high score for `2` resulting in a higher probability. "
686 | ]
687 | },
688 | {
689 | "cell_type": "markdown",
690 | "metadata": {},
691 | "source": [
692 | "## Summary\n",
693 | "### Gradient Descent is a General Learning Algorithm"
694 | ]
695 | },
696 | {
697 | "cell_type": "markdown",
698 | "metadata": {},
699 | "source": [
700 | "Gradient Descent is a general learning algorithm. The most important subtext in this chapter is that gradient descent is a very flexible algorithm. If we combine. weights in a way that allows us to calculate the error function and a delta, then gradient descent can show us how to reduce error.\n",
701 | "\n",
702 | "We will spend the rest of this book exploring different types of weight combinations and error functions for which gradient descent is useful."
703 | ]
704 | },
705 | {
706 | "cell_type": "markdown",
707 | "metadata": {},
708 | "source": [
709 | "# Sketches"
710 | ]
711 | },
712 | {
713 | "cell_type": "markdown",
714 | "metadata": {},
715 | "source": [
716 | "
\n",
725 | "\n",
726 | "---"
727 | ]
728 | }
729 | ],
730 | "metadata": {
731 | "kernelspec": {
732 | "display_name": "Python 3",
733 | "language": "python",
734 | "name": "python3"
735 | },
736 | "language_info": {
737 | "codemirror_mode": {
738 | "name": "ipython",
739 | "version": 3
740 | },
741 | "file_extension": ".py",
742 | "mimetype": "text/x-python",
743 | "name": "python",
744 | "nbconvert_exporter": "python",
745 | "pygments_lexer": "ipython3",
746 | "version": "3.8.6"
747 | }
748 | },
749 | "nbformat": 4,
750 | "nbformat_minor": 4
751 | }
752 |
--------------------------------------------------------------------------------
/09.Probabilities_Non-Linearities.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Modeling Probabilities & Non-Linearities: Activation Functions\n",
8 | "\n",
9 | "In this chapter, we will:\n",
10 | "\n",
11 | "- Learn what is an activation function.\n",
12 | "- Learn about the standard hidden activation functions\n",
13 | " - Sigmoid\n",
14 | " - Tanh\n",
15 | "- Learn about the standard output activation functions\n",
16 | " - Softmax\n",
17 | "\n",
18 | "> [George Gordon Byron] \"I Know that 2 and 2 make 4 –– & should be glad to prove it too if I could –– though I must say if by any sort of process I could convert 2 & 2 into 5 it would give me much greater pleasure\""
19 | ]
20 | },
21 | {
22 | "cell_type": "markdown",
23 | "metadata": {},
24 | "source": [
25 | "## What is an Activation Function?"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "An activation function is a function applied to the neurons in a layer during prediction. In that sense, an activation function is any function that can take one number and return another number. There are, however, an infinite number of functions in the universe, and not all of them are useful as activation functions. \n",
33 | "\n",
34 | "We've already used an activation function called `ReLU`. The `ReLU` function had the effect of turning all negative numbers to zero.\n",
35 | "\n",
36 | "There are many constraints on the nature of activation functions, we present them next."
37 | ]
38 | },
39 | {
40 | "cell_type": "markdown",
41 | "metadata": {},
42 | "source": [
43 | "### Constraint 1: The Function must be Continuous & Infinite in Domain\n",
44 | "\n",
45 | "
\n",
46 | "\n",
47 | "\n",
48 | "Meaning that the function must have an output number for any input."
49 | ]
50 | },
51 | {
52 | "cell_type": "markdown",
53 | "metadata": {},
54 | "source": [
55 | "### Constraint 2: Good Activation Functions are Monotonic (Increasing/Decreasing)\n",
56 | "\n",
57 | "
\n",
58 | "\n",
59 | "An activation function must never change direction (always increasing/decreasing). This particular constrraint is not technically a requirement but if we consider a function that map different input values to the same output, then that function may have multiple perfect configurations. As a result, we can't know the correct direction to go.\n",
60 | "\n",
61 | "For an advanced look into this subject, we should look more into convex versus non-convex optimization. "
62 | ]
63 | },
64 | {
65 | "cell_type": "markdown",
66 | "metadata": {},
67 | "source": [
68 | "### Constraint 3: Good Activation Functions are Non-Linear\n",
69 | "\n",
70 | "
\n",
71 | "\n",
72 | "Linear functions scale values, they do not effect how correlated a neuron is to various inputs. They just make the collective corrrelation that is represented louder or softer.\n",
73 | "\n",
74 | "What we want instead is **selective correlation**."
75 | ]
76 | },
77 | {
78 | "cell_type": "markdown",
79 | "metadata": {},
80 | "source": [
81 | "### Constraint 4: Good Activation Functions (& their derivatives) should be efficiently computable\n",
82 | "\n",
83 | "We will be using the chosen activation functions a lot. For this reason, we want them (and their derivatives) to be efficiently computable. As an example, `ReLU` has become very popular mostly because it's efficient to compute. "
84 | ]
85 | },
86 | {
87 | "cell_type": "markdown",
88 | "metadata": {},
89 | "source": [
90 | "## Standard Hidden-Layer Activation Functions\n",
91 | "### Which ones are most commonly used?"
92 | ]
93 | },
94 | {
95 | "cell_type": "markdown",
96 | "metadata": {},
97 | "source": [
98 | "### Sigmoid is the bread & butter Activation\n",
99 | "\n",
100 | "
\n",
101 | "\n",
102 | "Sigmoid is great because it smoothly squinshes the infinite amount of input to an output between $0$ and $1$. This lets us interpret the output of any neuron as a **probability**. We typically use this non-linearity both in hidden and outputs layers."
103 | ]
104 | },
105 | {
106 | "cell_type": "markdown",
107 | "metadata": {},
108 | "source": [
109 | "### Tanh is better than sigmoid for hidden layers\n",
110 | "\n",
111 | "
\n",
112 | "\n",
113 | "`Tanh` is the same as sigmoid except it's between `-1` and `1`. This means it can also throw in some **negative** correlation. This aspect of negative correlation is powerful for hidden layers. On many problems, `tanh` will outperform sigmoid in the hidden layers."
114 | ]
115 | },
116 | {
117 | "cell_type": "markdown",
118 | "metadata": {},
119 | "source": [
120 | "## Standard output layer activation functions\n",
121 | "\n",
122 | "For output layer activation functions, choosing the best one depends on what we're trying to predict. Broadly speaking, there are 3 major types of output layers:"
123 | ]
124 | },
125 | {
126 | "cell_type": "markdown",
127 | "metadata": {},
128 | "source": [
129 | "### Configuration 1: Predicting Raw Data Values (Regression) — No activation function\n",
130 | "\n",
131 | "One example might be predicting the average temperature in colorado given the average temperature in surrounding states."
132 | ]
133 | },
134 | {
135 | "cell_type": "markdown",
136 | "metadata": {},
137 | "source": [
138 | "### Configuration 2: Predicting Unrelated Yes/No Probabilities (Binary Classification) — Sigmoid\n",
139 | "\n",
140 | "It's best to use the sigmoid function, Because it models individual probabilities separately for each output node."
141 | ]
142 | },
143 | {
144 | "cell_type": "markdown",
145 | "metadata": {},
146 | "source": [
147 | "### Configuration 3: Predicting which-one probabilities (Categorical Classification) — Softmax\n",
148 | "\n",
149 | "This is by far the common use case in neural networks: predicting a single label out of many. In this case, it's better to have an activation function that models the idea that \"The more likely it's one label, The less likely it's any of the other labels\"."
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "## The Core Issue: Inputs have similarity\n",
157 | "### Different numbers share characteristics. It's good to let the network believe that\n",
158 | "\n",
159 | "
\n",
160 | " \n",
161 | " \n",
162 | "
"
163 | ]
164 | },
165 | {
166 | "cell_type": "markdown",
167 | "metadata": {},
168 | "source": [
169 | "As we can see in the above figure, the average `2` shares quite a bit with the average `3`. As a general rule, similar inputs create similar outputs, when we take some numbers and multiply them by a matrix, if the starting numbers are pretty similar, the ending numbers will be pretty similar.\n",
170 | "\n",
171 | "As a result, sigmoid will penelize the network for recognizing a `2` by anything other than features that are exclusively related to `2`s.\n",
172 | " \n",
173 | "Most Images share lots of pixels in the middle of images. Because of that **the network will start trying to focus on the edges instead**. As we can see in the weight Image, the light areas are probably the best individual indicators of a `2`. \n",
174 | "\n",
175 | "What we are really striving for, though, is a network that \"sees\" the entire shape of a digit before outputing a prediction."
176 | ]
177 | },
178 | {
179 | "cell_type": "markdown",
180 | "metadata": {},
181 | "source": [
182 | "## Softmax Computation\n",
183 | "### Softmax raises each input value exponentially and then divides by the layer's sum\n",
184 | "\n",
185 | "$$\\sigma(y)_{i} = \\frac{e^{y_i}}{\\sum_{j=1}^{K} e^{y_{j}}}$$\n",
186 | "\n",
187 | "Softmax raises each input value exponentially and then divides by the layer's sum. The nice thing about softmax is that the higher the network predicts one value, the lower it predicts all the others. It encourages the network to pick out class with a very high probability.\n",
188 | "\n",
189 | "To adjust how aggresively Softmax pushes the density to one class, use number slightly bigger or lower than $e$. Lower numbers will result in lower *attenuation* and higher numbers will result win bigger *attenuation*. For now, we stick to $e$."
190 | ]
191 | },
192 | {
193 | "cell_type": "markdown",
194 | "metadata": {},
195 | "source": [
196 | "## Activation Installation Instructions\n",
197 | "### How do we add our favorite activation function to any layer?\n",
198 | "\n",
199 | "We know the following:\n",
200 | "- The Slope of `ReLU` for positive numbers is exactly 1.\n",
201 | "- The Slope of `ReLU` for negative numbers is exactly 0.\n",
202 | "\n",
203 | "As a reminder, the slope is a measure of how much the output of `ReLU` will change given a change in the input. The Input to a layer refers to the value before the nonlinearity. Modifying the input to `ReLU` (by a tiny amount) will have a 1:1 effect if it was predicting positively and a 0:1 effect if it was predicting negatively.\n",
204 | "\n",
205 | "Thus, when we backpropagate, in order to generate `layer_1_delta`, multiply the backpropagated `delta` from `layer_2` (`layer_2_delta.dot(W_2.T)`) by the slope of `ReLU` at the point predicted in forward propagation. For some deltas the slope is $1$ (positive numbers) and for others it's $0$ (negative numbers).\n",
206 | "\n",
207 | "The important thing to remember is that the slope is an indicator of how much a tiny change to the input effects the output. The update effect encourages the network to leave weights alone if adjusting them will have little to no effect."
208 | ]
209 | },
210 | {
211 | "cell_type": "markdown",
212 | "metadata": {},
213 | "source": [
214 | "## Multiplying Delta by the Slope\n",
215 | "\n",
216 | "To compute `layer_delta`, we multiply the backpropagated delta by the layer's slope:\n",
217 | "\n",
218 | "
\n",
219 | " \n",
220 | "
\n",
221 | "\n",
222 | "`layer_1_delta[0]` represents how much higher or lower the first hidden node of layer 1 should be in order to reduce the error of the network. The end goal of delta on a neuron is to inform the weights whether they should move. If moving them would have no effect, they should be left alone.\n",
223 | "\n",
224 | "
\n",
225 | " \n",
226 | "
\n",
227 | "\n",
228 | "This condition is obvious for ReLU, which is either on or off, but Sigmoid's sensitivity to change in the input slowly increases as the input approaches 0 from either direction. However, **very positive** and **very negative** inputs approach a slope of very near 0. Thus, as the input becomes very positive/negative, small changes to the incoming weights become less relevant to the neuron's error for the specific training example."
229 | ]
230 | },
231 | {
232 | "cell_type": "markdown",
233 | "metadata": {},
234 | "source": [
235 | "## Converting Output to Slope (Derivative)\n",
236 | "\n",
237 | "For all of the previous activation functions, we can directly convert their output to their slope:\n",
238 | "\n",
239 | "
\n",
46 | "\n",
47 | "\n",
48 | "Meaning that the function must have an output number for any input."
49 | ]
50 | },
51 | {
52 | "cell_type": "markdown",
53 | "metadata": {},
54 | "source": [
55 | "### Constraint 2: Good Activation Functions are Monotonic (Increasing/Decreasing)\n",
56 | "\n",
57 | "
\n",
58 | "\n",
59 | "An activation function must never change direction (always increasing/decreasing). This particular constrraint is not technically a requirement but if we consider a function that map different input values to the same output, then that function may have multiple perfect configurations. As a result, we can't know the correct direction to go.\n",
60 | "\n",
61 | "For an advanced look into this subject, we should look more into convex versus non-convex optimization. "
62 | ]
63 | },
64 | {
65 | "cell_type": "markdown",
66 | "metadata": {},
67 | "source": [
68 | "### Constraint 3: Good Activation Functions are Non-Linear\n",
69 | "\n",
70 | "
\n",
71 | "\n",
72 | "Linear functions scale values, they do not effect how correlated a neuron is to various inputs. They just make the collective corrrelation that is represented louder or softer.\n",
73 | "\n",
74 | "What we want instead is **selective correlation**."
75 | ]
76 | },
77 | {
78 | "cell_type": "markdown",
79 | "metadata": {},
80 | "source": [
81 | "### Constraint 4: Good Activation Functions (& their derivatives) should be efficiently computable\n",
82 | "\n",
83 | "We will be using the chosen activation functions a lot. For this reason, we want them (and their derivatives) to be efficiently computable. As an example, `ReLU` has become very popular mostly because it's efficient to compute. "
84 | ]
85 | },
86 | {
87 | "cell_type": "markdown",
88 | "metadata": {},
89 | "source": [
90 | "## Standard Hidden-Layer Activation Functions\n",
91 | "### Which ones are most commonly used?"
92 | ]
93 | },
94 | {
95 | "cell_type": "markdown",
96 | "metadata": {},
97 | "source": [
98 | "### Sigmoid is the bread & butter Activation\n",
99 | "\n",
100 | "
\n",
101 | "\n",
102 | "Sigmoid is great because it smoothly squinshes the infinite amount of input to an output between $0$ and $1$. This lets us interpret the output of any neuron as a **probability**. We typically use this non-linearity both in hidden and outputs layers."
103 | ]
104 | },
105 | {
106 | "cell_type": "markdown",
107 | "metadata": {},
108 | "source": [
109 | "### Tanh is better than sigmoid for hidden layers\n",
110 | "\n",
111 | "
\n",
112 | "\n",
113 | "`Tanh` is the same as sigmoid except it's between `-1` and `1`. This means it can also throw in some **negative** correlation. This aspect of negative correlation is powerful for hidden layers. On many problems, `tanh` will outperform sigmoid in the hidden layers."
114 | ]
115 | },
116 | {
117 | "cell_type": "markdown",
118 | "metadata": {},
119 | "source": [
120 | "## Standard output layer activation functions\n",
121 | "\n",
122 | "For output layer activation functions, choosing the best one depends on what we're trying to predict. Broadly speaking, there are 3 major types of output layers:"
123 | ]
124 | },
125 | {
126 | "cell_type": "markdown",
127 | "metadata": {},
128 | "source": [
129 | "### Configuration 1: Predicting Raw Data Values (Regression) — No activation function\n",
130 | "\n",
131 | "One example might be predicting the average temperature in colorado given the average temperature in surrounding states."
132 | ]
133 | },
134 | {
135 | "cell_type": "markdown",
136 | "metadata": {},
137 | "source": [
138 | "### Configuration 2: Predicting Unrelated Yes/No Probabilities (Binary Classification) — Sigmoid\n",
139 | "\n",
140 | "It's best to use the sigmoid function, Because it models individual probabilities separately for each output node."
141 | ]
142 | },
143 | {
144 | "cell_type": "markdown",
145 | "metadata": {},
146 | "source": [
147 | "### Configuration 3: Predicting which-one probabilities (Categorical Classification) — Softmax\n",
148 | "\n",
149 | "This is by far the common use case in neural networks: predicting a single label out of many. In this case, it's better to have an activation function that models the idea that \"The more likely it's one label, The less likely it's any of the other labels\"."
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "metadata": {},
155 | "source": [
156 | "## The Core Issue: Inputs have similarity\n",
157 | "### Different numbers share characteristics. It's good to let the network believe that\n",
158 | "\n",
159 | "
\n",
160 | " \n",
161 | " \n",
162 | "
"
163 | ]
164 | },
165 | {
166 | "cell_type": "markdown",
167 | "metadata": {},
168 | "source": [
169 | "As we can see in the above figure, the average `2` shares quite a bit with the average `3`. As a general rule, similar inputs create similar outputs, when we take some numbers and multiply them by a matrix, if the starting numbers are pretty similar, the ending numbers will be pretty similar.\n",
170 | "\n",
171 | "As a result, sigmoid will penelize the network for recognizing a `2` by anything other than features that are exclusively related to `2`s.\n",
172 | " \n",
173 | "Most Images share lots of pixels in the middle of images. Because of that **the network will start trying to focus on the edges instead**. As we can see in the weight Image, the light areas are probably the best individual indicators of a `2`. \n",
174 | "\n",
175 | "What we are really striving for, though, is a network that \"sees\" the entire shape of a digit before outputing a prediction."
176 | ]
177 | },
178 | {
179 | "cell_type": "markdown",
180 | "metadata": {},
181 | "source": [
182 | "## Softmax Computation\n",
183 | "### Softmax raises each input value exponentially and then divides by the layer's sum\n",
184 | "\n",
185 | "$$\\sigma(y)_{i} = \\frac{e^{y_i}}{\\sum_{j=1}^{K} e^{y_{j}}}$$\n",
186 | "\n",
187 | "Softmax raises each input value exponentially and then divides by the layer's sum. The nice thing about softmax is that the higher the network predicts one value, the lower it predicts all the others. It encourages the network to pick out class with a very high probability.\n",
188 | "\n",
189 | "To adjust how aggresively Softmax pushes the density to one class, use number slightly bigger or lower than $e$. Lower numbers will result in lower *attenuation* and higher numbers will result win bigger *attenuation*. For now, we stick to $e$."
190 | ]
191 | },
192 | {
193 | "cell_type": "markdown",
194 | "metadata": {},
195 | "source": [
196 | "## Activation Installation Instructions\n",
197 | "### How do we add our favorite activation function to any layer?\n",
198 | "\n",
199 | "We know the following:\n",
200 | "- The Slope of `ReLU` for positive numbers is exactly 1.\n",
201 | "- The Slope of `ReLU` for negative numbers is exactly 0.\n",
202 | "\n",
203 | "As a reminder, the slope is a measure of how much the output of `ReLU` will change given a change in the input. The Input to a layer refers to the value before the nonlinearity. Modifying the input to `ReLU` (by a tiny amount) will have a 1:1 effect if it was predicting positively and a 0:1 effect if it was predicting negatively.\n",
204 | "\n",
205 | "Thus, when we backpropagate, in order to generate `layer_1_delta`, multiply the backpropagated `delta` from `layer_2` (`layer_2_delta.dot(W_2.T)`) by the slope of `ReLU` at the point predicted in forward propagation. For some deltas the slope is $1$ (positive numbers) and for others it's $0$ (negative numbers).\n",
206 | "\n",
207 | "The important thing to remember is that the slope is an indicator of how much a tiny change to the input effects the output. The update effect encourages the network to leave weights alone if adjusting them will have little to no effect."
208 | ]
209 | },
210 | {
211 | "cell_type": "markdown",
212 | "metadata": {},
213 | "source": [
214 | "## Multiplying Delta by the Slope\n",
215 | "\n",
216 | "To compute `layer_delta`, we multiply the backpropagated delta by the layer's slope:\n",
217 | "\n",
218 | "
\n",
219 | " \n",
220 | "
\n",
221 | "\n",
222 | "`layer_1_delta[0]` represents how much higher or lower the first hidden node of layer 1 should be in order to reduce the error of the network. The end goal of delta on a neuron is to inform the weights whether they should move. If moving them would have no effect, they should be left alone.\n",
223 | "\n",
224 | "
\n",
225 | " \n",
226 | "
\n",
227 | "\n",
228 | "This condition is obvious for ReLU, which is either on or off, but Sigmoid's sensitivity to change in the input slowly increases as the input approaches 0 from either direction. However, **very positive** and **very negative** inputs approach a slope of very near 0. Thus, as the input becomes very positive/negative, small changes to the incoming weights become less relevant to the neuron's error for the specific training example."
229 | ]
230 | },
231 | {
232 | "cell_type": "markdown",
233 | "metadata": {},
234 | "source": [
235 | "## Converting Output to Slope (Derivative)\n",
236 | "\n",
237 | "For all of the previous activation functions, we can directly convert their output to their slope:\n",
238 | "\n",
239 | "
![](\"static/imgs/Cover.png\")
![](\"static/imgs/06/many-to-one.png\")
![](\"static/imgs/06/Conversation.png\")
![](\"static/imgs/06/Iteration_1.png\")
![](\"static/imgs/06/Iteration_2.png\")
![](\"static/imgs/06/Iteration_3.png\")
![](\"static/imgs/06/Experiment_1.png\")
\n",
283 | " ![](\"static/imgs/06/Experiment_2.png\")
\n",
284 | "![](\"static/imgs/06/many-to-many.png\")
![](\"static/imgs/06/MNIST_clear_model.png\"){kind=spacer}
\n",
606 | "![](\"static/imgs/06/MNIST_weight_visualization.png\")
\n",
626 | "![](\"static/imgs/06/Neural_Similarity.png\")
\n",
683 | "![](\"static/imgs/06/0.jpg\")
\n",
718 | " ![](\"static/imgs/06/1.jpg\")
![](\"static/imgs/06/2.jpg\")
![](\"static/imgs/06/3.jpg\")
\n",
721 | " ![](\"static/imgs/06/4'.jpg\")
![](\"static/imgs/06/5.jpg\")
\n",
723 | " ![](\"static/imgs/06/6.jpg\")
\n",
724 | "![](\"static/imgs/09/continuous-vs-non.png\")
![](\"static/imgs/09/monotonic.png\")
![](\"static/imgs/09/non-linearity.png\")
![](\"static/imgs/09/sigmoid.png\")
![](\"static/imgs/09/tanh.png\")
![](\"static/imgs/09/similarity.png\")
\n",
161 | " ![](\"static/imgs/09/weights.png\")
\n",
162 | "![](\"static/imgs/09/layer_delta.png\")
\n",
220 | "![](\"static/imgs/09/activations.png\")
\n",
226 | "![](\"static/imgs/09/Pariston.png\")
\n",
241 | "![](\"static/imgs/08/Dogs.png\")