├── _config.yml ├── nbs ├── x2.png ├── imgs │ ├── knn.png │ ├── wb.jpeg │ ├── 8-1NN.jpg │ ├── Thumbs.db │ ├── arrays.png │ ├── digits.png │ ├── faces.png │ ├── img2.png │ ├── iris.png │ ├── loss.png │ ├── margin.jpg │ ├── spam1.jpeg │ ├── spam2.jpeg │ ├── Linear2.png │ ├── Linear3.png │ ├── Linear_4.png │ ├── cifar10.png │ ├── linear1.png │ ├── subplot.jpg │ ├── subplot.png │ ├── svm-loss.jpg │ ├── 8-4Dropout.jpg │ ├── Bell-Curve.png │ ├── cat-matrix.png │ ├── clustering.png │ ├── digits_2d.jpg │ ├── iris_setosa.jpg │ ├── ml_taxonomy.png │ ├── sup_unsup.png │ ├── broadcasting.png │ ├── classifydemo.jpeg │ ├── faces_approx.png │ ├── house_price.jpeg │ ├── faces_original.png │ ├── iris_versicolor.jpg │ ├── iris_virginica.jpg │ ├── numpy_plus_tab.png │ ├── backward_gradient.jpg │ ├── cluster_comparison.png │ ├── cross_entropy_loss.jpg │ ├── face_recognition.jpeg │ ├── 08-2-three-layer-NN.jpg │ ├── cross_entropy_formula.jpg │ ├── gradient_descent_fig.jpg │ ├── regularization_loss.jpg │ ├── svm_half_vectorized.jpg │ ├── 8-3Batch_normalization.jpg │ ├── 8-2Different_update_rules.gif │ ├── k-means_image_compression.png │ ├── gradient_descent_line_graph.gif │ ├── LinearClassification_prototypes.png │ └── unsupervised_workflow.svg ├── my_array.npy ├── utils │ ├── __pycache__ │ │ ├── layers.cpython-36.pyc │ │ └── data_utils.cpython-36.pyc │ ├── data_utils.py │ └── layers.py ├── fig_codes │ ├── __pycache__ │ │ ├── figures.cpython-36.pyc │ │ ├── plot_pca.cpython-36.pyc │ │ ├── plot_helpers.cpython-36.pyc │ │ ├── plot_scaling.cpython-36.pyc │ │ └── plot_2d_separator.cpython-36.pyc │ ├── plot_helpers.py │ ├── plot_2d_separator.py │ ├── plot_digits_dataset.py │ ├── plot_pca.py │ ├── plot_scaling.py │ └── figures.py ├── my_array.csv └── solutions │ ├── 03A_faces_plot.py │ ├── 06B_lin_with_sine.py │ ├── 04_wrong-predictions.py │ ├── 08B_digits_clustering.py │ ├── 06A_knn_vs_linreg.py │ ├── 07A_iris-pca.py │ └── 05A_knn_with_diff_k.py ├── MLiP-week01and02 ├── x2.png ├── my_array.npy ├── imgs │ ├── img2.png │ ├── arrays.png │ ├── cifar10.png │ ├── img2 (1).png │ ├── subplot.jpg │ ├── Bell-Curve.png │ ├── cat-matrix.png │ ├── broadcasting.png │ └── numpy_plus_tab.png ├── my_array.csv └── utils │ └── data_utils.py ├── MLiP-week10 └── imgs │ ├── CNN.png │ ├── Keras.png │ ├── convnet.jpeg │ ├── maxpool.jpeg │ ├── conv_layer.gif │ ├── keras-docs.jpg │ ├── DL-frameworks.png │ ├── cnn_flowchart.png │ ├── filter-3by3.png │ ├── neural_net2.jpeg │ ├── zero_padding.jpg │ ├── linear_classifier.jpg │ └── convolution_schematic.gif ├── MLiP-week05 ├── imgs │ ├── iris.png │ ├── clustering.png │ ├── digits_2d.jpg │ ├── sup_unsup.png │ ├── faces_approx.png │ ├── faces_original.png │ ├── cluster_comparison.png │ └── k-means_image_compression.png ├── figs │ ├── plot_helpers.py │ ├── plot_digits_dataset.py │ ├── plot_pca.py │ ├── plot_scaling.py │ └── figures.py └── solutions │ ├── 08B_digits_clustering.py │ └── 07A_iris-pca.py ├── MLiP-week08 ├── imgs │ ├── 8-1NN.jpg │ ├── 8-4Dropout.jpg │ ├── 08-2-three-layer-NN.jpg │ ├── 8-3Batch_normalization.jpg │ └── 8-2Different_update_rules.gif ├── data_utils.py └── layers.py ├── MLiP2017_Specification.pdf ├── 1-MLiP2017_Introduction.pdf ├── MLiP-week03and4 ├── imgs │ ├── iris.png │ ├── knn.png │ ├── digits.png │ ├── faces.png │ ├── spam1.jpeg │ ├── spam2.jpeg │ ├── iris_setosa.jpg │ ├── ml_taxonomy.png │ ├── house_price.jpeg │ ├── iris_virginica.jpg │ ├── iris_versicolor.jpg │ └── face_recognition.jpeg ├── solutions │ ├── 03A_faces_plot.py │ ├── 06B_lin_with_sine.py │ ├── 04_wrong-predictions.py │ ├── 06A_knn_vs_linreg.py │ └── 05A_knn_with_diff_k.py └── fig_codes │ └── plot_2d_separator.py ├── MLiP-week06-07 ├── imgs │ ├── loss.png │ ├── wb.jpeg │ ├── Linear2.png │ ├── Linear3.png │ ├── linear1.png │ ├── margin.jpg │ ├── Linear_4.png │ ├── svm-loss.jpg │ ├── classifydemo.jpeg │ ├── backward_gradient.jpg │ ├── cross_entropy_loss.jpg │ ├── gradient_descent_fig.jpg │ ├── regularization_loss.jpg │ ├── svm_half_vectorized.jpg │ ├── cross_entropy_formula.jpg │ ├── gradient_descent_line_graph.gif │ └── LinearClassification_prototypes.png ├── data_utils.py └── layers.py ├── MLiP-week09 ├── imgs │ ├── 09-crop.jpg │ ├── 09-flip.jpg │ ├── 09-Dropout.jpg │ ├── next_week.png │ ├── 09-momentum.jpg │ ├── 09_batch_norm.jpg │ ├── 09_batch_norm.png │ ├── 09-Augmentation.jpg │ ├── 09-SGD_problem1.jpg │ ├── 09-SGD_problem2.jpg │ ├── 09-color_jitter.jpg │ ├── 09-two-layer-NN.jpg │ ├── 09-three-layer-NN.jpg │ ├── algorithm-main-pic.png │ ├── 09-Batch_normalization.jpg │ └── 09-Different_update_rules.gif ├── data_utils.py ├── optim.py ├── fc_net.py ├── layers.py └── solver.py ├── python basics └── Python Basics.pdf ├── Assignments └── ‌excesizes │ └── MLiP-ex1-Neural_Networks_for_CIFAR10.pdf ├── .gitignore └── README.md /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /nbs/x2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/x2.png -------------------------------------------------------------------------------- /nbs/imgs/knn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/knn.png -------------------------------------------------------------------------------- /nbs/imgs/wb.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/wb.jpeg -------------------------------------------------------------------------------- /nbs/my_array.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/my_array.npy -------------------------------------------------------------------------------- /nbs/imgs/8-1NN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/8-1NN.jpg -------------------------------------------------------------------------------- /nbs/imgs/Thumbs.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/Thumbs.db -------------------------------------------------------------------------------- /nbs/imgs/arrays.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/arrays.png -------------------------------------------------------------------------------- /nbs/imgs/digits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/digits.png -------------------------------------------------------------------------------- /nbs/imgs/faces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/faces.png -------------------------------------------------------------------------------- /nbs/imgs/img2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/img2.png -------------------------------------------------------------------------------- /nbs/imgs/iris.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/iris.png -------------------------------------------------------------------------------- /nbs/imgs/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/loss.png -------------------------------------------------------------------------------- /nbs/imgs/margin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/margin.jpg -------------------------------------------------------------------------------- /nbs/imgs/spam1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/spam1.jpeg -------------------------------------------------------------------------------- /nbs/imgs/spam2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/spam2.jpeg -------------------------------------------------------------------------------- /nbs/imgs/Linear2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/Linear2.png -------------------------------------------------------------------------------- /nbs/imgs/Linear3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/Linear3.png -------------------------------------------------------------------------------- /nbs/imgs/Linear_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/Linear_4.png -------------------------------------------------------------------------------- /nbs/imgs/cifar10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/cifar10.png -------------------------------------------------------------------------------- /nbs/imgs/linear1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/linear1.png -------------------------------------------------------------------------------- /nbs/imgs/subplot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/subplot.jpg -------------------------------------------------------------------------------- /nbs/imgs/subplot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/subplot.png -------------------------------------------------------------------------------- /nbs/imgs/svm-loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/svm-loss.jpg -------------------------------------------------------------------------------- /MLiP-week01and02/x2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/x2.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/CNN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/CNN.png -------------------------------------------------------------------------------- /nbs/imgs/8-4Dropout.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/8-4Dropout.jpg -------------------------------------------------------------------------------- /nbs/imgs/Bell-Curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/Bell-Curve.png -------------------------------------------------------------------------------- /nbs/imgs/cat-matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/cat-matrix.png -------------------------------------------------------------------------------- /nbs/imgs/clustering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/clustering.png -------------------------------------------------------------------------------- /nbs/imgs/digits_2d.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/digits_2d.jpg -------------------------------------------------------------------------------- /nbs/imgs/iris_setosa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/iris_setosa.jpg -------------------------------------------------------------------------------- /nbs/imgs/ml_taxonomy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/ml_taxonomy.png -------------------------------------------------------------------------------- /nbs/imgs/sup_unsup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/sup_unsup.png -------------------------------------------------------------------------------- /MLiP-week05/imgs/iris.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/iris.png -------------------------------------------------------------------------------- /MLiP-week08/imgs/8-1NN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week08/imgs/8-1NN.jpg -------------------------------------------------------------------------------- /MLiP-week10/imgs/Keras.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/Keras.png -------------------------------------------------------------------------------- /MLiP2017_Specification.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP2017_Specification.pdf -------------------------------------------------------------------------------- /nbs/imgs/broadcasting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/broadcasting.png -------------------------------------------------------------------------------- /nbs/imgs/classifydemo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/classifydemo.jpeg -------------------------------------------------------------------------------- /nbs/imgs/faces_approx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/faces_approx.png -------------------------------------------------------------------------------- /nbs/imgs/house_price.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/house_price.jpeg -------------------------------------------------------------------------------- /1-MLiP2017_Introduction.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/1-MLiP2017_Introduction.pdf -------------------------------------------------------------------------------- /MLiP-week01and02/my_array.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/my_array.npy -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/iris.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/iris.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/knn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/knn.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/loss.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/wb.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/wb.jpeg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-crop.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-crop.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-flip.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-flip.jpg -------------------------------------------------------------------------------- /MLiP-week10/imgs/convnet.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/convnet.jpeg -------------------------------------------------------------------------------- /MLiP-week10/imgs/maxpool.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/maxpool.jpeg -------------------------------------------------------------------------------- /nbs/imgs/faces_original.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/faces_original.png -------------------------------------------------------------------------------- /nbs/imgs/iris_versicolor.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/iris_versicolor.jpg -------------------------------------------------------------------------------- /nbs/imgs/iris_virginica.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/iris_virginica.jpg -------------------------------------------------------------------------------- /nbs/imgs/numpy_plus_tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/numpy_plus_tab.png -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/img2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/img2.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/digits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/digits.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/faces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/faces.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/spam1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/spam1.jpeg -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/spam2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/spam2.jpeg -------------------------------------------------------------------------------- /MLiP-week05/imgs/clustering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/clustering.png -------------------------------------------------------------------------------- /MLiP-week05/imgs/digits_2d.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/digits_2d.jpg -------------------------------------------------------------------------------- /MLiP-week05/imgs/sup_unsup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/sup_unsup.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/Linear2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/Linear2.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/Linear3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/Linear3.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/linear1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/linear1.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/margin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/margin.jpg -------------------------------------------------------------------------------- /MLiP-week08/imgs/8-4Dropout.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week08/imgs/8-4Dropout.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-Dropout.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-Dropout.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/next_week.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/next_week.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/conv_layer.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/conv_layer.gif -------------------------------------------------------------------------------- /MLiP-week10/imgs/keras-docs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/keras-docs.jpg -------------------------------------------------------------------------------- /nbs/imgs/backward_gradient.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/backward_gradient.jpg -------------------------------------------------------------------------------- /nbs/imgs/cluster_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/cluster_comparison.png -------------------------------------------------------------------------------- /nbs/imgs/cross_entropy_loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/cross_entropy_loss.jpg -------------------------------------------------------------------------------- /nbs/imgs/face_recognition.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/face_recognition.jpeg -------------------------------------------------------------------------------- /python basics/Python Basics.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/python basics/Python Basics.pdf -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/arrays.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/arrays.png -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/cifar10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/cifar10.png -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/img2 (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/img2 (1).png -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/subplot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/subplot.jpg -------------------------------------------------------------------------------- /MLiP-week05/imgs/faces_approx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/faces_approx.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/Linear_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/Linear_4.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/svm-loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/svm-loss.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-momentum.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-momentum.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09_batch_norm.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09_batch_norm.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09_batch_norm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09_batch_norm.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/DL-frameworks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/DL-frameworks.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/cnn_flowchart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/cnn_flowchart.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/filter-3by3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/filter-3by3.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/neural_net2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/neural_net2.jpeg -------------------------------------------------------------------------------- /MLiP-week10/imgs/zero_padding.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/zero_padding.jpg -------------------------------------------------------------------------------- /nbs/imgs/08-2-three-layer-NN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/08-2-three-layer-NN.jpg -------------------------------------------------------------------------------- /nbs/imgs/cross_entropy_formula.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/cross_entropy_formula.jpg -------------------------------------------------------------------------------- /nbs/imgs/gradient_descent_fig.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/gradient_descent_fig.jpg -------------------------------------------------------------------------------- /nbs/imgs/regularization_loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/regularization_loss.jpg -------------------------------------------------------------------------------- /nbs/imgs/svm_half_vectorized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/svm_half_vectorized.jpg -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/Bell-Curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/Bell-Curve.png -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/cat-matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/cat-matrix.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/iris_setosa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/iris_setosa.jpg -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/ml_taxonomy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/ml_taxonomy.png -------------------------------------------------------------------------------- /MLiP-week05/imgs/faces_original.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/faces_original.png -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-Augmentation.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-Augmentation.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-SGD_problem1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-SGD_problem1.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-SGD_problem2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-SGD_problem2.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-color_jitter.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-color_jitter.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-two-layer-NN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-two-layer-NN.jpg -------------------------------------------------------------------------------- /nbs/imgs/8-3Batch_normalization.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/8-3Batch_normalization.jpg -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/broadcasting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/broadcasting.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/house_price.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/house_price.jpeg -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/iris_virginica.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/iris_virginica.jpg -------------------------------------------------------------------------------- /MLiP-week05/imgs/cluster_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/cluster_comparison.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/classifydemo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/classifydemo.jpeg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-three-layer-NN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-three-layer-NN.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/algorithm-main-pic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/algorithm-main-pic.png -------------------------------------------------------------------------------- /MLiP-week10/imgs/linear_classifier.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/linear_classifier.jpg -------------------------------------------------------------------------------- /nbs/imgs/8-2Different_update_rules.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/8-2Different_update_rules.gif -------------------------------------------------------------------------------- /nbs/imgs/k-means_image_compression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/k-means_image_compression.png -------------------------------------------------------------------------------- /MLiP-week01and02/imgs/numpy_plus_tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week01and02/imgs/numpy_plus_tab.png -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/iris_versicolor.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/iris_versicolor.jpg -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/backward_gradient.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/backward_gradient.jpg -------------------------------------------------------------------------------- /MLiP-week08/imgs/08-2-three-layer-NN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week08/imgs/08-2-three-layer-NN.jpg -------------------------------------------------------------------------------- /nbs/imgs/gradient_descent_line_graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/gradient_descent_line_graph.gif -------------------------------------------------------------------------------- /MLiP-week03and4/imgs/face_recognition.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week03and4/imgs/face_recognition.jpeg -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/cross_entropy_loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/cross_entropy_loss.jpg -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/gradient_descent_fig.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/gradient_descent_fig.jpg -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/regularization_loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/regularization_loss.jpg -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/svm_half_vectorized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/svm_half_vectorized.jpg -------------------------------------------------------------------------------- /MLiP-week08/imgs/8-3Batch_normalization.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week08/imgs/8-3Batch_normalization.jpg -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-Batch_normalization.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-Batch_normalization.jpg -------------------------------------------------------------------------------- /MLiP-week10/imgs/convolution_schematic.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week10/imgs/convolution_schematic.gif -------------------------------------------------------------------------------- /nbs/imgs/LinearClassification_prototypes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/imgs/LinearClassification_prototypes.png -------------------------------------------------------------------------------- /nbs/utils/__pycache__/layers.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/utils/__pycache__/layers.cpython-36.pyc -------------------------------------------------------------------------------- /MLiP-week05/imgs/k-means_image_compression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week05/imgs/k-means_image_compression.png -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/cross_entropy_formula.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/cross_entropy_formula.jpg -------------------------------------------------------------------------------- /MLiP-week08/imgs/8-2Different_update_rules.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week08/imgs/8-2Different_update_rules.gif -------------------------------------------------------------------------------- /MLiP-week09/imgs/09-Different_update_rules.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week09/imgs/09-Different_update_rules.gif -------------------------------------------------------------------------------- /nbs/fig_codes/__pycache__/figures.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/fig_codes/__pycache__/figures.cpython-36.pyc -------------------------------------------------------------------------------- /nbs/fig_codes/__pycache__/plot_pca.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/fig_codes/__pycache__/plot_pca.cpython-36.pyc -------------------------------------------------------------------------------- /nbs/my_array.csv: -------------------------------------------------------------------------------- 1 | 8.039213625487107162e-01,8.531978798142828557e-01,8.276900452937172581e-01 2 | 2.032233876496847280e-01,5.569927296454455501e-01,6.562618971496337883e-01 3 | -------------------------------------------------------------------------------- /nbs/utils/__pycache__/data_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/utils/__pycache__/data_utils.cpython-36.pyc -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/gradient_descent_line_graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/gradient_descent_line_graph.gif -------------------------------------------------------------------------------- /MLiP-week01and02/my_array.csv: -------------------------------------------------------------------------------- 1 | 2.047870966287423933e-02,4.652803246244485713e-01,1.418044583981390439e-01 2 | 1.575936388704680668e-01,1.492402393141355477e-01,5.979355554978926790e-01 3 | -------------------------------------------------------------------------------- /nbs/fig_codes/__pycache__/plot_helpers.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/fig_codes/__pycache__/plot_helpers.cpython-36.pyc -------------------------------------------------------------------------------- /nbs/fig_codes/__pycache__/plot_scaling.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/fig_codes/__pycache__/plot_scaling.cpython-36.pyc -------------------------------------------------------------------------------- /MLiP-week06-07/imgs/LinearClassification_prototypes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/MLiP-week06-07/imgs/LinearClassification_prototypes.png -------------------------------------------------------------------------------- /nbs/fig_codes/plot_helpers.py: -------------------------------------------------------------------------------- 1 | from matplotlib.colors import ListedColormap 2 | 3 | cm3 = ListedColormap(['#0000aa', '#ff2020', '#50ff50']) 4 | cm2 = ListedColormap(['#0000aa', '#ff2020']) 5 | -------------------------------------------------------------------------------- /MLiP-week05/figs/plot_helpers.py: -------------------------------------------------------------------------------- 1 | from matplotlib.colors import ListedColormap 2 | 3 | cm3 = ListedColormap(['#0000aa', '#ff2020', '#50ff50']) 4 | cm2 = ListedColormap(['#0000aa', '#ff2020']) 5 | -------------------------------------------------------------------------------- /nbs/fig_codes/__pycache__/plot_2d_separator.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/nbs/fig_codes/__pycache__/plot_2d_separator.cpython-36.pyc -------------------------------------------------------------------------------- /Assignments/‌excesizes/MLiP-ex1-Neural_Networks_for_CIFAR10.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snrazavi/Machine-Learning-in-Python-Workshop/HEAD/Assignments/‌excesizes/MLiP-ex1-Neural_Networks_for_CIFAR10.pdf -------------------------------------------------------------------------------- /nbs/solutions/03A_faces_plot.py: -------------------------------------------------------------------------------- 1 | faces = fetch_olivetti_faces() 2 | 3 | # set up the figure 4 | fig = plt.figure(figsize=(6, 6)) # figure size in inches 5 | fig.subplots_adjust(left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05) 6 | 7 | # plot the faces: 8 | for i in range(64): 9 | ax = fig.add_subplot(8, 8, i + 1, xticks=[], yticks=[]) 10 | ax.imshow(faces.images[i], cmap=plt.cm.bone, interpolation='nearest') 11 | -------------------------------------------------------------------------------- /MLiP-week03and4/solutions/03A_faces_plot.py: -------------------------------------------------------------------------------- 1 | faces = fetch_olivetti_faces() 2 | 3 | # set up the figure 4 | fig = plt.figure(figsize=(6, 6)) # figure size in inches 5 | fig.subplots_adjust(left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05) 6 | 7 | # plot the faces: 8 | for i in range(64): 9 | ax = fig.add_subplot(8, 8, i + 1, xticks=[], yticks=[]) 10 | ax.imshow(faces.images[i], cmap=plt.cm.bone, interpolation='nearest') 11 | -------------------------------------------------------------------------------- /nbs/solutions/06B_lin_with_sine.py: -------------------------------------------------------------------------------- 1 | XX_train = np.concatenate((X_train, np.sin(4 * X_train)), axis=1) 2 | XX_test = np.concatenate((X_test, np.sin(4 * X_test)), axis=1) 3 | regressor.fit(XX_train, y_train) 4 | y_pred_test_sine = regressor.predict(XX_test) 5 | 6 | plt.plot(X_test, y_test, 'o', label="data") 7 | plt.plot(X_test, y_pred_test_sine, 'o', label="prediction with sine") 8 | plt.plot(X_test, y_pred_test, label='prediction without sine') 9 | plt.legend(loc='best'); 10 | -------------------------------------------------------------------------------- /MLiP-week03and4/solutions/06B_lin_with_sine.py: -------------------------------------------------------------------------------- 1 | XX_train = np.concatenate((X_train, np.sin(4 * X_train)), axis=1) 2 | XX_test = np.concatenate((X_test, np.sin(4 * X_test)), axis=1) 3 | regressor.fit(XX_train, y_train) 4 | y_pred_test_sine = regressor.predict(XX_test) 5 | 6 | plt.plot(X_test, y_test, 'o', label="data") 7 | plt.plot(X_test, y_pred_test_sine, 'o', label="prediction with sine") 8 | plt.plot(X_test, y_pred_test, label='prediction without sine') 9 | plt.legend(loc='best'); 10 | -------------------------------------------------------------------------------- /nbs/solutions/04_wrong-predictions.py: -------------------------------------------------------------------------------- 1 | plt.figure(figsize=(10, 6)) 2 | 3 | 4 | for i in incorrect_idx: 5 | print('%d: Predicted %d | True label %d' % (i, y_pred[i], y_test[i])) 6 | 7 | # Plot two dimensions 8 | 9 | colors = ["darkblue", "darkgreen", "gray"] 10 | 11 | for n, color in enumerate(colors): 12 | idx = np.where(y_test == n)[0] 13 | plt.scatter(X_test[idx, 1], X_test[idx, 2], color=color, label="Class %s" % str(n)) 14 | 15 | for i, marker in zip(incorrect_idx, ['x', 's', 'v']): 16 | plt.scatter(X_test[i, 1], X_test[i, 2], 17 | color="darkred", 18 | marker=marker, 19 | s=60, 20 | label=i) 21 | 22 | plt.xlabel('sepal width [cm]') 23 | plt.ylabel('petal length [cm]') 24 | plt.legend(loc=1, scatterpoints=1) 25 | plt.title("Iris Classification results") 26 | plt.show() 27 | -------------------------------------------------------------------------------- /MLiP-week03and4/solutions/04_wrong-predictions.py: -------------------------------------------------------------------------------- 1 | plt.figure(figsize=(10, 6)) 2 | 3 | 4 | for i in incorrect_idx: 5 | print('%d: Predicted %d | True label %d' % (i, y_pred[i], y_test[i])) 6 | 7 | # Plot two dimensions 8 | 9 | colors = ["darkblue", "darkgreen", "gray"] 10 | 11 | for n, color in enumerate(colors): 12 | idx = np.where(y_test == n)[0] 13 | plt.scatter(X_test[idx, 1], X_test[idx, 2], color=color, label="Class %s" % str(n)) 14 | 15 | for i, marker in zip(incorrect_idx, ['x', 's', 'v']): 16 | plt.scatter(X_test[i, 1], X_test[i, 2], 17 | color="darkred", 18 | marker=marker, 19 | s=60, 20 | label=i) 21 | 22 | plt.xlabel('sepal width [cm]') 23 | plt.ylabel('petal length [cm]') 24 | plt.legend(loc=1, scatterpoints=1) 25 | plt.title("Iris Classification results") 26 | plt.show() 27 | -------------------------------------------------------------------------------- /nbs/solutions/08B_digits_clustering.py: -------------------------------------------------------------------------------- 1 | from sklearn.cluster import KMeans 2 | kmeans = KMeans(n_clusters=10) 3 | clusters = kmeans.fit_predict(digits.data) 4 | 5 | print(kmeans.cluster_centers_.shape) 6 | 7 | #------------------------------------------------------------ 8 | # visualize the cluster centers 9 | fig = plt.figure(figsize=(8, 3)) 10 | for i in range(10): 11 | ax = fig.add_subplot(2, 5, 1 + i) 12 | ax.imshow(kmeans.cluster_centers_[i].reshape((8, 8)), 13 | cmap=plt.cm.binary) 14 | from sklearn.manifold import Isomap 15 | X_iso = Isomap(n_neighbors=10).fit_transform(digits.data) 16 | 17 | #------------------------------------------------------------ 18 | # visualize the projected data 19 | fig, ax = plt.subplots(1, 2, figsize=(8, 4)) 20 | 21 | ax[0].scatter(X_iso[:, 0], X_iso[:, 1], c=clusters) 22 | ax[1].scatter(X_iso[:, 0], X_iso[:, 1], c=digits.target) 23 | -------------------------------------------------------------------------------- /MLiP-week05/solutions/08B_digits_clustering.py: -------------------------------------------------------------------------------- 1 | from sklearn.cluster import KMeans 2 | kmeans = KMeans(n_clusters=10) 3 | clusters = kmeans.fit_predict(digits.data) 4 | 5 | print(kmeans.cluster_centers_.shape) 6 | 7 | #------------------------------------------------------------ 8 | # visualize the cluster centers 9 | fig = plt.figure(figsize=(8, 3)) 10 | for i in range(10): 11 | ax = fig.add_subplot(2, 5, 1 + i) 12 | ax.imshow(kmeans.cluster_centers_[i].reshape((8, 8)), 13 | cmap=plt.cm.binary) 14 | from sklearn.manifold import Isomap 15 | X_iso = Isomap(n_neighbors=10).fit_transform(digits.data) 16 | 17 | #------------------------------------------------------------ 18 | # visualize the projected data 19 | fig, ax = plt.subplots(1, 2, figsize=(8, 4)) 20 | 21 | ax[0].scatter(X_iso[:, 0], X_iso[:, 1], c=clusters) 22 | ax[1].scatter(X_iso[:, 0], X_iso[:, 1], c=digits.target) 23 | -------------------------------------------------------------------------------- /MLiP-week01and02/utils/data_utils.py: -------------------------------------------------------------------------------- 1 | import cPickle as pickle 2 | import numpy as np 3 | import os 4 | from scipy.misc import imread 5 | 6 | def load_CIFAR_batch(filename): 7 | """ load single batch of cifar """ 8 | with open(filename, 'rb') as f: 9 | datadict = pickle.load(f) 10 | X = datadict['data'] 11 | Y = datadict['labels'] 12 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 13 | Y = np.array(Y) 14 | return X, Y 15 | 16 | def load_CIFAR10(ROOT): 17 | """ load all of cifar """ 18 | xs = [] 19 | ys = [] 20 | for b in range(1,6): 21 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 22 | X, Y = load_CIFAR_batch(f) 23 | xs.append(X) 24 | ys.append(Y) 25 | Xtr = np.concatenate(xs) 26 | Ytr = np.concatenate(ys) 27 | del X, Y 28 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 29 | return Xtr, Ytr, Xte, Yte 30 | -------------------------------------------------------------------------------- /nbs/solutions/06A_knn_vs_linreg.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_boston 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.linear_model import LinearRegression 4 | 5 | 6 | boston = load_boston() 7 | X = boston.data 8 | y = boston.target 9 | 10 | print('X.shape:', X.shape) 11 | X_train, X_test, y_train, y_test = train_test_split(X, y, 12 | test_size=0.25, 13 | random_state=42) 14 | 15 | linreg = LinearRegression() 16 | knnreg = KNeighborsRegressor(n_neighbors=1) 17 | 18 | linreg.fit(X_train, y_train) 19 | print('Linear Regression Train/Test: %.3f/%.3f' % 20 | (linreg.score(X_train, y_train), 21 | linreg.score(X_test, y_test))) 22 | 23 | knnreg.fit(X_train, y_train) 24 | print('KNeighborsRegressor Train/Test: %.3f/%.3f' % 25 | (knnreg.score(X_train, y_train), 26 | knnreg.score(X_test, y_test))) 27 | -------------------------------------------------------------------------------- /MLiP-week03and4/solutions/06A_knn_vs_linreg.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_boston 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.linear_model import LinearRegression 4 | 5 | 6 | boston = load_boston() 7 | X = boston.data 8 | y = boston.target 9 | 10 | print('X.shape:', X.shape) 11 | X_train, X_test, y_train, y_test = train_test_split(X, y, 12 | test_size=0.25, 13 | random_state=42) 14 | 15 | linreg = LinearRegression() 16 | knnreg = KNeighborsRegressor(n_neighbors=1) 17 | 18 | linreg.fit(X_train, y_train) 19 | print('Linear Regression Train/Test: %.3f/%.3f' % 20 | (linreg.score(X_train, y_train), 21 | linreg.score(X_test, y_test))) 22 | 23 | knnreg.fit(X_train, y_train) 24 | print('KNeighborsRegressor Train/Test: %.3f/%.3f' % 25 | (knnreg.score(X_train, y_train), 26 | knnreg.score(X_test, y_test))) 27 | -------------------------------------------------------------------------------- /nbs/solutions/07A_iris-pca.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_iris 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.decomposition import PCA 4 | from sklearn.preprocessing import StandardScaler 5 | 6 | iris = load_iris() 7 | 8 | X_train, X_test, y_train, y_test = train_test_split(iris.data, 9 | iris.target, 10 | random_state=0, 11 | stratify=iris.target) 12 | 13 | sc = StandardScaler() 14 | sc.fit(X_train) 15 | pca = PCA(n_components=2) 16 | 17 | X_train_pca = pca.fit_transform(sc.transform(X_train)) 18 | X_test_pca = pca.transform(sc.transform(X_test)) 19 | 20 | for X, y in zip((X_train_pca, X_test_pca), (y_train, y_test)): 21 | 22 | for i, annot in enumerate(zip(('Iris-setosa', 'Iris-versicolor', 'Iris-virginica'), 23 | ('blue', 'red', 'green'))): 24 | plt.scatter(X[y==i, 0], 25 | X[y==i, 1], 26 | label=annot[0], 27 | c=annot[1]) 28 | plt.xlabel('Principal Component 1') 29 | plt.ylabel('Principal Component 2') 30 | plt.legend(loc='best') 31 | plt.tight_layout() 32 | plt.show() 33 | -------------------------------------------------------------------------------- /nbs/solutions/05A_knn_with_diff_k.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_iris 2 | from sklearn.model_selection import train_test_split 3 | 4 | 5 | iris = load_iris() 6 | X = iris.data 7 | y = iris.target 8 | 9 | X_train, X_test, y_train, y_test = train_test_split(X, y, 10 | test_size=0.25, 11 | random_state=1234, 12 | stratify=y) 13 | 14 | X_trainsub, X_valid, y_trainsub, y_valid = train_test_split(X_train, y_train, 15 | test_size=0.5, 16 | random_state=1234, 17 | stratify=y_train) 18 | 19 | for k in range(1, 20): 20 | knn = KNeighborsClassifier(n_neighbors=k) 21 | train_score = knn.fit(X_trainsub, y_trainsub).\ 22 | score(X_trainsub, y_trainsub) 23 | valid_score = knn.score(X_valid, y_valid) 24 | print('k: %d, Train/Valid Acc: %.3f/%.3f' % 25 | (k, train_score, valid_score)) 26 | 27 | 28 | knn = KNeighborsClassifier(n_neighbors=9) 29 | knn.fit(X_train, y_train) 30 | print('k=9 Test Acc: %.3f' % knn.score(X_test, y_test)) 31 | -------------------------------------------------------------------------------- /MLiP-week05/solutions/07A_iris-pca.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_iris 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.decomposition import PCA 4 | from sklearn.preprocessing import StandardScaler 5 | 6 | iris = load_iris() 7 | 8 | X_train, X_test, y_train, y_test = train_test_split(iris.data, 9 | iris.target, 10 | random_state=0, 11 | stratify=iris.target) 12 | 13 | sc = StandardScaler() 14 | sc.fit(X_train) 15 | pca = PCA(n_components=2) 16 | 17 | X_train_pca = pca.fit_transform(sc.transform(X_train)) 18 | X_test_pca = pca.transform(sc.transform(X_test)) 19 | 20 | for X, y in zip((X_train_pca, X_test_pca), (y_train, y_test)): 21 | 22 | for i, annot in enumerate(zip(('Iris-setosa', 'Iris-versicolor', 'Iris-virginica'), 23 | ('blue', 'red', 'green'))): 24 | plt.scatter(X[y==i, 0], 25 | X[y==i, 1], 26 | label=annot[0], 27 | c=annot[1]) 28 | plt.xlabel('Principal Component 1') 29 | plt.ylabel('Principal Component 2') 30 | plt.legend(loc='best') 31 | plt.tight_layout() 32 | plt.show() 33 | -------------------------------------------------------------------------------- /MLiP-week03and4/solutions/05A_knn_with_diff_k.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_iris 2 | from sklearn.model_selection import train_test_split 3 | 4 | 5 | iris = load_iris() 6 | X = iris.data 7 | y = iris.target 8 | 9 | X_train, X_test, y_train, y_test = train_test_split(X, y, 10 | test_size=0.25, 11 | random_state=1234, 12 | stratify=y) 13 | 14 | X_trainsub, X_valid, y_trainsub, y_valid = train_test_split(X_train, y_train, 15 | test_size=0.5, 16 | random_state=1234, 17 | stratify=y_train) 18 | 19 | for k in range(1, 20): 20 | knn = KNeighborsClassifier(n_neighbors=k) 21 | train_score = knn.fit(X_trainsub, y_trainsub).\ 22 | score(X_trainsub, y_trainsub) 23 | valid_score = knn.score(X_valid, y_valid) 24 | print('k: %d, Train/Valid Acc: %.3f/%.3f' % 25 | (k, train_score, valid_score)) 26 | 27 | 28 | knn = KNeighborsClassifier(n_neighbors=9) 29 | knn.fit(X_train, y_train) 30 | print('k=9 Test Acc: %.3f' % knn.score(X_test, y_test)) 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /nbs/fig_codes/plot_2d_separator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | def plot_2d_separator(classifier, X, fill=False, ax=None, eps=None): 6 | if eps is None: 7 | eps = X.std() / 2. 8 | x_min, x_max = X[:, 0].min() - eps, X[:, 0].max() + eps 9 | y_min, y_max = X[:, 1].min() - eps, X[:, 1].max() + eps 10 | xx = np.linspace(x_min, x_max, 100) 11 | yy = np.linspace(y_min, y_max, 100) 12 | 13 | X1, X2 = np.meshgrid(xx, yy) 14 | X_grid = np.c_[X1.ravel(), X2.ravel()] 15 | try: 16 | decision_values = classifier.decision_function(X_grid) 17 | levels = [0] 18 | fill_levels = [decision_values.min(), 0, decision_values.max()] 19 | except AttributeError: 20 | # no decision_function 21 | decision_values = classifier.predict_proba(X_grid)[:, 1] 22 | levels = [.5] 23 | fill_levels = [0, .5, 1] 24 | 25 | if ax is None: 26 | ax = plt.gca() 27 | if fill: 28 | ax.contourf(X1, X2, decision_values.reshape(X1.shape), 29 | levels=fill_levels, colors=['blue', 'red']) 30 | else: 31 | ax.contour(X1, X2, decision_values.reshape(X1.shape), levels=levels, 32 | colors="black") 33 | ax.set_xlim(x_min, x_max) 34 | ax.set_ylim(y_min, y_max) 35 | ax.set_xticks(()) 36 | ax.set_yticks(()) 37 | 38 | 39 | if __name__ == '__main__': 40 | from sklearn.datasets import make_blobs 41 | from sklearn.linear_model import LogisticRegression 42 | X, y = make_blobs(centers=2, random_state=42) 43 | clf = LogisticRegression().fit(X, y) 44 | plot_2d_separator(clf, X, fill=True) 45 | plt.scatter(X[:, 0], X[:, 1], c=y) 46 | plt.show() 47 | -------------------------------------------------------------------------------- /MLiP-week03and4/fig_codes/plot_2d_separator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | def plot_2d_separator(classifier, X, fill=False, ax=None, eps=None): 6 | if eps is None: 7 | eps = X.std() / 2. 8 | x_min, x_max = X[:, 0].min() - eps, X[:, 0].max() + eps 9 | y_min, y_max = X[:, 1].min() - eps, X[:, 1].max() + eps 10 | xx = np.linspace(x_min, x_max, 100) 11 | yy = np.linspace(y_min, y_max, 100) 12 | 13 | X1, X2 = np.meshgrid(xx, yy) 14 | X_grid = np.c_[X1.ravel(), X2.ravel()] 15 | try: 16 | decision_values = classifier.decision_function(X_grid) 17 | levels = [0] 18 | fill_levels = [decision_values.min(), 0, decision_values.max()] 19 | except AttributeError: 20 | # no decision_function 21 | decision_values = classifier.predict_proba(X_grid)[:, 1] 22 | levels = [.5] 23 | fill_levels = [0, .5, 1] 24 | 25 | if ax is None: 26 | ax = plt.gca() 27 | if fill: 28 | ax.contourf(X1, X2, decision_values.reshape(X1.shape), 29 | levels=fill_levels, colors=['blue', 'red']) 30 | else: 31 | ax.contour(X1, X2, decision_values.reshape(X1.shape), levels=levels, 32 | colors="black") 33 | ax.set_xlim(x_min, x_max) 34 | ax.set_ylim(y_min, y_max) 35 | ax.set_xticks(()) 36 | ax.set_yticks(()) 37 | 38 | 39 | if __name__ == '__main__': 40 | from sklearn.datasets import make_blobs 41 | from sklearn.linear_model import LogisticRegression 42 | X, y = make_blobs(centers=2, random_state=42) 43 | clf = LogisticRegression().fit(X, y) 44 | plot_2d_separator(clf, X, fill=True) 45 | plt.scatter(X[:, 0], X[:, 1], c=y) 46 | plt.show() 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine-Learning-in-Python-Workshop 2 | My workshop on machine learning using python language to implement different algorithms (University of Tabriz, Iran, 2017). 3 | 4 | ## Contents 5 | ### Part 1: Using existing packages for machine learning (Week 1 to 5) 6 | - **Week 01 and 02:** Introduction to Numpy and Matplotlib packages 7 | - **Week 03 and 04**: Using Scikit Learn for Supervised Learning 8 | - **Week 05**: Using Scikit Learn for Unsupervised Learning 9 | ### Part 2: Implementing our machine Learning algorithms and models (Week 5 to 10) 10 | - **Week 06**: Linear classification 11 | - **Week 07**: Implementing Loss functions (Softmax loss and SVM loss) 12 | - **Week 08**: Implementing gradient descent, Backpropagation and Artifitial Neural Networks (MLP) 13 | - **Week 09:** Advanced topics including dropout, batch normalization, weight initialization and other optimization methods(Adam, RMSProp) 14 | - **Week 10:** Inroduction to Deep Learning and implementing a *Convolutional Neural Network (CNN)* for image classification. 15 | 16 | ## Prerequisites: 17 | - A basic knowledge of Python programming language. 18 | - A good understaning of Machine Learning. 19 | - Linear Algebra 20 | 21 | ## Videos in YouTube (in Persian): 22 | - http://www.youtube.com/c/SeyedNaserRazavi 23 | 24 | ## My website Address: 25 | - containing anything you need to learn and of course to use machine learning in real world applications: 26 | - http://wwww.snrazavi.ir/ 27 | 28 | ## The workshop page on my website: 29 | - http://www.snrazavi.ir/ml-with-python/ 30 | 31 | **Note:** The materials of this workshop are inspired from awesome lectures presented by Andrej Karpathy at Stanford, 2016. 32 | 33 | ### References: 34 | - Parts 6 to 8 are inspired from the wonderful course cs231n. 35 | - Parts 5 and 6 are heavily inspired from SciPy 2016 Scikit-learn Tutorial. 36 | -------------------------------------------------------------------------------- /nbs/utils/data_utils.py: -------------------------------------------------------------------------------- 1 | from six.moves import cPickle as pickle 2 | import numpy as np 3 | import os 4 | from scipy.misc import imread 5 | import platform 6 | 7 | def load_pickle(f): 8 | version = platform.python_version_tuple() 9 | if version[0] == '2': 10 | return pickle.load(f) 11 | elif version[0] == '3': 12 | return pickle.load(f, encoding='latin1') 13 | raise ValueError("invalid python version: {}".format(version)) 14 | 15 | def load_CIFAR_batch(filename): 16 | """ load single batch of cifar """ 17 | with open(filename, 'rb') as f: 18 | datadict = load_pickle(f) 19 | X = datadict['data'] 20 | Y = datadict['labels'] 21 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 22 | Y = np.array(Y) 23 | return X, Y 24 | 25 | def load_CIFAR10(ROOT): 26 | """ load all of cifar """ 27 | xs = [] 28 | ys = [] 29 | for b in range(1,6): 30 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 31 | X, Y = load_CIFAR_batch(f) 32 | xs.append(X) 33 | ys.append(Y) 34 | Xtr = np.concatenate(xs) 35 | Ytr = np.concatenate(ys) 36 | del X, Y 37 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 38 | return Xtr, Ytr, Xte, Yte 39 | 40 | 41 | def get_CIFAR10_data(cifar10_dir, num_training=49000, num_validation=1000, num_test=1000, 42 | subtract_mean=True): 43 | """ 44 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 45 | it for classifiers. These are the same steps as we used for the SVM, but 46 | condensed to a single function. 47 | """ 48 | # Load the raw CIFAR-10 data 49 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 50 | 51 | # Subsample the data 52 | mask = list(range(num_training, num_training + num_validation)) 53 | X_val = X_train[mask] 54 | y_val = y_train[mask] 55 | mask = list(range(num_training)) 56 | X_train = X_train[mask] 57 | y_train = y_train[mask] 58 | mask = list(range(num_test)) 59 | X_test = X_test[mask] 60 | y_test = y_test[mask] 61 | 62 | # Normalize the data: subtract the mean image 63 | if subtract_mean: 64 | mean_image = np.mean(X_train, axis=0) 65 | X_train -= mean_image 66 | X_val -= mean_image 67 | X_test -= mean_image 68 | 69 | # Transpose so that channels come first 70 | X_train = X_train.transpose(0, 3, 1, 2).copy() 71 | X_val = X_val.transpose(0, 3, 1, 2).copy() 72 | X_test = X_test.transpose(0, 3, 1, 2).copy() 73 | 74 | # Package data into a dictionary 75 | return { 76 | 'X_train': X_train, 'y_train': y_train, 77 | 'X_val': X_val, 'y_val': y_val, 78 | 'X_test': X_test, 'y_test': y_test, 79 | } 80 | -------------------------------------------------------------------------------- /MLiP-week06-07/data_utils.py: -------------------------------------------------------------------------------- 1 | from six.moves import cPickle as pickle 2 | import numpy as np 3 | import os 4 | from scipy.misc import imread 5 | import platform 6 | 7 | def load_pickle(f): 8 | version = platform.python_version_tuple() 9 | if version[0] == '2': 10 | return pickle.load(f) 11 | elif version[0] == '3': 12 | return pickle.load(f, encoding='latin1') 13 | raise ValueError("invalid python version: {}".format(version)) 14 | 15 | def load_CIFAR_batch(filename): 16 | """ load single batch of cifar """ 17 | with open(filename, 'rb') as f: 18 | datadict = load_pickle(f) 19 | X = datadict['data'] 20 | Y = datadict['labels'] 21 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 22 | Y = np.array(Y) 23 | return X, Y 24 | 25 | def load_CIFAR10(ROOT): 26 | """ load all of cifar """ 27 | xs = [] 28 | ys = [] 29 | for b in range(1,6): 30 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 31 | X, Y = load_CIFAR_batch(f) 32 | xs.append(X) 33 | ys.append(Y) 34 | Xtr = np.concatenate(xs) 35 | Ytr = np.concatenate(ys) 36 | del X, Y 37 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 38 | return Xtr, Ytr, Xte, Yte 39 | 40 | 41 | def get_CIFAR10_data(cifar10_dir, num_training=49000, num_validation=1000, num_test=1000, 42 | subtract_mean=True): 43 | """ 44 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 45 | it for classifiers. These are the same steps as we used for the SVM, but 46 | condensed to a single function. 47 | """ 48 | # Load the raw CIFAR-10 data 49 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 50 | 51 | # Subsample the data 52 | mask = list(range(num_training, num_training + num_validation)) 53 | X_val = X_train[mask] 54 | y_val = y_train[mask] 55 | mask = list(range(num_training)) 56 | X_train = X_train[mask] 57 | y_train = y_train[mask] 58 | mask = list(range(num_test)) 59 | X_test = X_test[mask] 60 | y_test = y_test[mask] 61 | 62 | # Normalize the data: subtract the mean image 63 | if subtract_mean: 64 | mean_image = np.mean(X_train, axis=0) 65 | X_train -= mean_image 66 | X_val -= mean_image 67 | X_test -= mean_image 68 | 69 | # Transpose so that channels come first 70 | X_train = X_train.transpose(0, 3, 1, 2).copy() 71 | X_val = X_val.transpose(0, 3, 1, 2).copy() 72 | X_test = X_test.transpose(0, 3, 1, 2).copy() 73 | 74 | # Package data into a dictionary 75 | return { 76 | 'X_train': X_train, 'y_train': y_train, 77 | 'X_val': X_val, 'y_val': y_val, 78 | 'X_test': X_test, 'y_test': y_test, 79 | } 80 | -------------------------------------------------------------------------------- /MLiP-week08/data_utils.py: -------------------------------------------------------------------------------- 1 | from six.moves import cPickle as pickle 2 | import numpy as np 3 | import os 4 | from scipy.misc import imread 5 | import platform 6 | 7 | def load_pickle(f): 8 | version = platform.python_version_tuple() 9 | if version[0] == '2': 10 | return pickle.load(f) 11 | elif version[0] == '3': 12 | return pickle.load(f, encoding='latin1') 13 | raise ValueError("invalid python version: {}".format(version)) 14 | 15 | def load_CIFAR_batch(filename): 16 | """ load single batch of cifar """ 17 | with open(filename, 'rb') as f: 18 | datadict = load_pickle(f) 19 | X = datadict['data'] 20 | Y = datadict['labels'] 21 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 22 | Y = np.array(Y) 23 | return X, Y 24 | 25 | def load_CIFAR10(ROOT): 26 | """ load all of cifar """ 27 | xs = [] 28 | ys = [] 29 | for b in range(1,6): 30 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 31 | X, Y = load_CIFAR_batch(f) 32 | xs.append(X) 33 | ys.append(Y) 34 | Xtr = np.concatenate(xs) 35 | Ytr = np.concatenate(ys) 36 | del X, Y 37 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 38 | return Xtr, Ytr, Xte, Yte 39 | 40 | 41 | def get_CIFAR10_data(cifar10_dir, num_training=49000, num_validation=1000, num_test=1000, 42 | subtract_mean=True): 43 | """ 44 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 45 | it for classifiers. These are the same steps as we used for the SVM, but 46 | condensed to a single function. 47 | """ 48 | # Load the raw CIFAR-10 data 49 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 50 | 51 | # Subsample the data 52 | mask = list(range(num_training, num_training + num_validation)) 53 | X_val = X_train[mask] 54 | y_val = y_train[mask] 55 | mask = list(range(num_training)) 56 | X_train = X_train[mask] 57 | y_train = y_train[mask] 58 | mask = list(range(num_test)) 59 | X_test = X_test[mask] 60 | y_test = y_test[mask] 61 | 62 | # Normalize the data: subtract the mean image 63 | if subtract_mean: 64 | mean_image = np.mean(X_train, axis=0) 65 | X_train -= mean_image 66 | X_val -= mean_image 67 | X_test -= mean_image 68 | 69 | # Transpose so that channels come first 70 | X_train = X_train.transpose(0, 3, 1, 2).copy() 71 | X_val = X_val.transpose(0, 3, 1, 2).copy() 72 | X_test = X_test.transpose(0, 3, 1, 2).copy() 73 | 74 | # Package data into a dictionary 75 | return { 76 | 'X_train': X_train, 'y_train': y_train, 77 | 'X_val': X_val, 'y_val': y_val, 78 | 'X_test': X_test, 'y_test': y_test, 79 | } 80 | -------------------------------------------------------------------------------- /MLiP-week09/data_utils.py: -------------------------------------------------------------------------------- 1 | from six.moves import cPickle as pickle 2 | import numpy as np 3 | import os 4 | from scipy.misc import imread 5 | import platform 6 | 7 | def load_pickle(f): 8 | version = platform.python_version_tuple() 9 | if version[0] == '2': 10 | return pickle.load(f) 11 | elif version[0] == '3': 12 | return pickle.load(f, encoding='latin1') 13 | raise ValueError("invalid python version: {}".format(version)) 14 | 15 | def load_CIFAR_batch(filename): 16 | """ load single batch of cifar """ 17 | with open(filename, 'rb') as f: 18 | datadict = load_pickle(f) 19 | X = datadict['data'] 20 | Y = datadict['labels'] 21 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 22 | Y = np.array(Y) 23 | return X, Y 24 | 25 | def load_CIFAR10(ROOT): 26 | """ load all of cifar """ 27 | xs = [] 28 | ys = [] 29 | for b in range(1,6): 30 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 31 | X, Y = load_CIFAR_batch(f) 32 | xs.append(X) 33 | ys.append(Y) 34 | Xtr = np.concatenate(xs) 35 | Ytr = np.concatenate(ys) 36 | del X, Y 37 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 38 | return Xtr, Ytr, Xte, Yte 39 | 40 | 41 | def get_CIFAR10_data(cifar10_dir, num_training=49000, num_validation=1000, num_test=1000, 42 | subtract_mean=True): 43 | """ 44 | Load the CIFAR-10 dataset from disk and perform preprocessing to prepare 45 | it for classifiers. These are the same steps as we used for the SVM, but 46 | condensed to a single function. 47 | """ 48 | # Load the raw CIFAR-10 data 49 | X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 50 | 51 | # Subsample the data 52 | mask = list(range(num_training, num_training + num_validation)) 53 | X_val = X_train[mask] 54 | y_val = y_train[mask] 55 | mask = list(range(num_training)) 56 | X_train = X_train[mask] 57 | y_train = y_train[mask] 58 | mask = list(range(num_test)) 59 | X_test = X_test[mask] 60 | y_test = y_test[mask] 61 | 62 | # Normalize the data: subtract the mean image 63 | if subtract_mean: 64 | mean_image = np.mean(X_train, axis=0) 65 | X_train -= mean_image 66 | X_val -= mean_image 67 | X_test -= mean_image 68 | 69 | # Transpose so that channels come first 70 | X_train = X_train.transpose(0, 3, 1, 2).copy() 71 | X_val = X_val.transpose(0, 3, 1, 2).copy() 72 | X_test = X_test.transpose(0, 3, 1, 2).copy() 73 | 74 | # Package data into a dictionary 75 | return { 76 | 'X_train': X_train, 'y_train': y_train, 77 | 'X_val': X_val, 'y_val': y_val, 78 | 'X_test': X_test, 'y_test': y_test, 79 | } 80 | -------------------------------------------------------------------------------- /MLiP-week05/figs/plot_digits_dataset.py: -------------------------------------------------------------------------------- 1 | # Taken from example in scikit-learn examples 2 | # Authors: Fabian Pedregosa 3 | # Olivier Grisel 4 | # Mathieu Blondel 5 | # Gael Varoquaux 6 | # License: BSD 3 clause (C) INRIA 2011 7 | 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | from matplotlib import offsetbox 11 | from sklearn import datasets, decomposition 12 | 13 | 14 | def digits_plot(): 15 | digits = datasets.load_digits(n_class=6) 16 | n_digits = 500 17 | X = digits.data[:n_digits] 18 | y = digits.target[:n_digits] 19 | n_samples, n_features = X.shape 20 | 21 | def plot_embedding(X, title=None): 22 | x_min, x_max = np.min(X, 0), np.max(X, 0) 23 | X = (X - x_min) / (x_max - x_min) 24 | 25 | plt.figure() 26 | ax = plt.subplot(111) 27 | for i in range(X.shape[0]): 28 | plt.text(X[i, 0], X[i, 1], str(digits.target[i]), 29 | color=plt.cm.Set1(y[i] / 10.), 30 | fontdict={'weight': 'bold', 'size': 9}) 31 | 32 | if hasattr(offsetbox, 'AnnotationBbox'): 33 | # only print thumbnails with matplotlib > 1.0 34 | shown_images = np.array([[1., 1.]]) # just something big 35 | for i in range(X.shape[0]): 36 | dist = np.sum((X[i] - shown_images) ** 2, 1) 37 | if np.min(dist) < 1e5: 38 | # don't show points that are too close 39 | # set a high threshold to basically turn this off 40 | continue 41 | shown_images = np.r_[shown_images, [X[i]]] 42 | imagebox = offsetbox.AnnotationBbox( 43 | offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r), 44 | X[i]) 45 | ax.add_artist(imagebox) 46 | plt.xticks([]), plt.yticks([]) 47 | if title is not None: 48 | plt.title(title) 49 | 50 | n_img_per_row = 10 51 | img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row)) 52 | for i in range(n_img_per_row): 53 | ix = 10 * i + 1 54 | for j in range(n_img_per_row): 55 | iy = 10 * j + 1 56 | img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8)) 57 | 58 | plt.imshow(img, cmap=plt.cm.binary) 59 | plt.xticks([]) 60 | plt.yticks([]) 61 | plt.title('A selection from the 64-dimensional digits dataset') 62 | print("Computing PCA projection") 63 | pca = decomposition.PCA(n_components=2).fit(X) 64 | X_pca = pca.transform(X) 65 | plot_embedding(X_pca, "Principal Components projection of the digits") 66 | plt.figure() 67 | plt.title("First Principal Component") 68 | plt.matshow(pca.components_[0, :].reshape(8, 8), cmap="gray") 69 | plt.axis('off') 70 | plt.figure() 71 | plt.title("Second Principal Component") 72 | plt.matshow(pca.components_[1, :].reshape(8, 8), cmap="gray") 73 | plt.axis('off') 74 | plt.show() 75 | -------------------------------------------------------------------------------- /nbs/fig_codes/plot_digits_dataset.py: -------------------------------------------------------------------------------- 1 | # Taken from example in scikit-learn examples 2 | # Authors: Fabian Pedregosa 3 | # Olivier Grisel 4 | # Mathieu Blondel 5 | # Gael Varoquaux 6 | # License: BSD 3 clause (C) INRIA 2011 7 | 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | from matplotlib import offsetbox 11 | from sklearn import datasets, decomposition 12 | 13 | 14 | def digits_plot(): 15 | digits = datasets.load_digits(n_class=6) 16 | n_digits = 500 17 | X = digits.data[:n_digits] 18 | y = digits.target[:n_digits] 19 | n_samples, n_features = X.shape 20 | 21 | def plot_embedding(X, title=None): 22 | x_min, x_max = np.min(X, 0), np.max(X, 0) 23 | X = (X - x_min) / (x_max - x_min) 24 | 25 | plt.figure() 26 | ax = plt.subplot(111) 27 | for i in range(X.shape[0]): 28 | plt.text(X[i, 0], X[i, 1], str(digits.target[i]), 29 | color=plt.cm.Set1(y[i] / 10.), 30 | fontdict={'weight': 'bold', 'size': 9}) 31 | 32 | if hasattr(offsetbox, 'AnnotationBbox'): 33 | # only print thumbnails with matplotlib > 1.0 34 | shown_images = np.array([[1., 1.]]) # just something big 35 | for i in range(X.shape[0]): 36 | dist = np.sum((X[i] - shown_images) ** 2, 1) 37 | if np.min(dist) < 1e5: 38 | # don't show points that are too close 39 | # set a high threshold to basically turn this off 40 | continue 41 | shown_images = np.r_[shown_images, [X[i]]] 42 | imagebox = offsetbox.AnnotationBbox( 43 | offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r), 44 | X[i]) 45 | ax.add_artist(imagebox) 46 | plt.xticks([]), plt.yticks([]) 47 | if title is not None: 48 | plt.title(title) 49 | 50 | n_img_per_row = 10 51 | img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row)) 52 | for i in range(n_img_per_row): 53 | ix = 10 * i + 1 54 | for j in range(n_img_per_row): 55 | iy = 10 * j + 1 56 | img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8)) 57 | 58 | plt.imshow(img, cmap=plt.cm.binary) 59 | plt.xticks([]) 60 | plt.yticks([]) 61 | plt.title('A selection from the 64-dimensional digits dataset') 62 | print("Computing PCA projection") 63 | pca = decomposition.PCA(n_components=2).fit(X) 64 | X_pca = pca.transform(X) 65 | plot_embedding(X_pca, "Principal Components projection of the digits") 66 | plt.figure() 67 | plt.title("First Principal Component") 68 | plt.matshow(pca.components_[0, :].reshape(8, 8), cmap="gray") 69 | plt.axis('off') 70 | plt.figure() 71 | plt.title("Second Principal Component") 72 | plt.matshow(pca.components_[1, :].reshape(8, 8), cmap="gray") 73 | plt.axis('off') 74 | plt.show() 75 | -------------------------------------------------------------------------------- /nbs/fig_codes/plot_pca.py: -------------------------------------------------------------------------------- 1 | from sklearn.decomposition import PCA 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | 5 | 6 | def plot_pca_illustration(): 7 | rnd = np.random.RandomState(5) 8 | X_ = rnd.normal(size=(300, 2)) 9 | X_blob = np.dot(X_, rnd.normal(size=(2, 2))) + rnd.normal(size=2) 10 | 11 | pca = PCA() 12 | pca.fit(X_blob) 13 | X_pca = pca.transform(X_blob) 14 | 15 | S = X_pca.std(axis=0) 16 | 17 | fig, axes = plt.subplots(2, 2, figsize=(10, 10)) 18 | axes = axes.ravel() 19 | 20 | axes[0].set_title("Original data") 21 | axes[0].scatter(X_blob[:, 0], X_blob[:, 1], c=X_pca[:, 0], linewidths=0, 22 | s=60, cmap='viridis') 23 | axes[0].set_xlabel("feature 1") 24 | axes[0].set_ylabel("feature 2") 25 | axes[0].arrow(pca.mean_[0], pca.mean_[1], S[0] * pca.components_[0, 0], 26 | S[0] * pca.components_[0, 1], width=.1, head_width=.3, 27 | color='k') 28 | axes[0].arrow(pca.mean_[0], pca.mean_[1], S[1] * pca.components_[1, 0], 29 | S[1] * pca.components_[1, 1], width=.1, head_width=.3, 30 | color='k') 31 | axes[0].text(-1.5, -.5, "Component 2", size=14) 32 | axes[0].text(-4, -4, "Component 1", size=14) 33 | axes[0].set_aspect('equal') 34 | 35 | axes[1].set_title("Transformed data") 36 | axes[1].scatter(X_pca[:, 0], X_pca[:, 1], c=X_pca[:, 0], linewidths=0, 37 | s=60, cmap='viridis') 38 | axes[1].set_xlabel("First principal component") 39 | axes[1].set_ylabel("Second principal component") 40 | axes[1].set_aspect('equal') 41 | axes[1].set_ylim(-8, 8) 42 | 43 | pca = PCA(n_components=1) 44 | pca.fit(X_blob) 45 | X_inverse = pca.inverse_transform(pca.transform(X_blob)) 46 | 47 | axes[2].set_title("Transformed data w/ second component dropped") 48 | axes[2].scatter(X_pca[:, 0], np.zeros(X_pca.shape[0]), c=X_pca[:, 0], 49 | linewidths=0, s=60, cmap='viridis') 50 | axes[2].set_xlabel("First principal component") 51 | axes[2].set_aspect('equal') 52 | axes[2].set_ylim(-8, 8) 53 | 54 | axes[3].set_title("Back-rotation using only first component") 55 | axes[3].scatter(X_inverse[:, 0], X_inverse[:, 1], c=X_pca[:, 0], 56 | linewidths=0, s=60, cmap='viridis') 57 | axes[3].set_xlabel("feature 1") 58 | axes[3].set_ylabel("feature 2") 59 | axes[3].set_aspect('equal') 60 | axes[3].set_xlim(-8, 4) 61 | axes[3].set_ylim(-8, 4) 62 | 63 | 64 | def plot_pca_whitening(): 65 | rnd = np.random.RandomState(5) 66 | X_ = rnd.normal(size=(300, 2)) 67 | X_blob = np.dot(X_, rnd.normal(size=(2, 2))) + rnd.normal(size=2) 68 | 69 | pca = PCA(whiten=True) 70 | pca.fit(X_blob) 71 | X_pca = pca.transform(X_blob) 72 | 73 | fig, axes = plt.subplots(1, 2, figsize=(10, 10)) 74 | axes = axes.ravel() 75 | 76 | axes[0].set_title("Original data") 77 | axes[0].scatter(X_blob[:, 0], X_blob[:, 1], c=X_pca[:, 0], linewidths=0, s=60, cmap='viridis') 78 | axes[0].set_xlabel("feature 1") 79 | axes[0].set_ylabel("feature 2") 80 | axes[0].set_aspect('equal') 81 | 82 | axes[1].set_title("Whitened data") 83 | axes[1].scatter(X_pca[:, 0], X_pca[:, 1], c=X_pca[:, 0], linewidths=0, s=60, cmap='viridis') 84 | axes[1].set_xlabel("First principal component") 85 | axes[1].set_ylabel("Second principal component") 86 | axes[1].set_aspect('equal') 87 | axes[1].set_xlim(-3, 4) 88 | -------------------------------------------------------------------------------- /MLiP-week05/figs/plot_pca.py: -------------------------------------------------------------------------------- 1 | from sklearn.decomposition import PCA 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | 5 | 6 | def plot_pca_illustration(): 7 | rnd = np.random.RandomState(5) 8 | X_ = rnd.normal(size=(300, 2)) 9 | X_blob = np.dot(X_, rnd.normal(size=(2, 2))) + rnd.normal(size=2) 10 | 11 | pca = PCA() 12 | pca.fit(X_blob) 13 | X_pca = pca.transform(X_blob) 14 | 15 | S = X_pca.std(axis=0) 16 | 17 | fig, axes = plt.subplots(2, 2, figsize=(10, 10)) 18 | axes = axes.ravel() 19 | 20 | axes[0].set_title("Original data") 21 | axes[0].scatter(X_blob[:, 0], X_blob[:, 1], c=X_pca[:, 0], linewidths=0, 22 | s=60, cmap='viridis') 23 | axes[0].set_xlabel("feature 1") 24 | axes[0].set_ylabel("feature 2") 25 | axes[0].arrow(pca.mean_[0], pca.mean_[1], S[0] * pca.components_[0, 0], 26 | S[0] * pca.components_[0, 1], width=.1, head_width=.3, 27 | color='k') 28 | axes[0].arrow(pca.mean_[0], pca.mean_[1], S[1] * pca.components_[1, 0], 29 | S[1] * pca.components_[1, 1], width=.1, head_width=.3, 30 | color='k') 31 | axes[0].text(-1.5, -.5, "Component 2", size=14) 32 | axes[0].text(-4, -4, "Component 1", size=14) 33 | axes[0].set_aspect('equal') 34 | 35 | axes[1].set_title("Transformed data") 36 | axes[1].scatter(X_pca[:, 0], X_pca[:, 1], c=X_pca[:, 0], linewidths=0, 37 | s=60, cmap='viridis') 38 | axes[1].set_xlabel("First principal component") 39 | axes[1].set_ylabel("Second principal component") 40 | axes[1].set_aspect('equal') 41 | axes[1].set_ylim(-8, 8) 42 | 43 | pca = PCA(n_components=1) 44 | pca.fit(X_blob) 45 | X_inverse = pca.inverse_transform(pca.transform(X_blob)) 46 | 47 | axes[2].set_title("Transformed data w/ second component dropped") 48 | axes[2].scatter(X_pca[:, 0], np.zeros(X_pca.shape[0]), c=X_pca[:, 0], 49 | linewidths=0, s=60, cmap='viridis') 50 | axes[2].set_xlabel("First principal component") 51 | axes[2].set_aspect('equal') 52 | axes[2].set_ylim(-8, 8) 53 | 54 | axes[3].set_title("Back-rotation using only first component") 55 | axes[3].scatter(X_inverse[:, 0], X_inverse[:, 1], c=X_pca[:, 0], 56 | linewidths=0, s=60, cmap='viridis') 57 | axes[3].set_xlabel("feature 1") 58 | axes[3].set_ylabel("feature 2") 59 | axes[3].set_aspect('equal') 60 | axes[3].set_xlim(-8, 4) 61 | axes[3].set_ylim(-8, 4) 62 | 63 | 64 | def plot_pca_whitening(): 65 | rnd = np.random.RandomState(5) 66 | X_ = rnd.normal(size=(300, 2)) 67 | X_blob = np.dot(X_, rnd.normal(size=(2, 2))) + rnd.normal(size=2) 68 | 69 | pca = PCA(whiten=True) 70 | pca.fit(X_blob) 71 | X_pca = pca.transform(X_blob) 72 | 73 | fig, axes = plt.subplots(1, 2, figsize=(10, 10)) 74 | axes = axes.ravel() 75 | 76 | axes[0].set_title("Original data") 77 | axes[0].scatter(X_blob[:, 0], X_blob[:, 1], c=X_pca[:, 0], linewidths=0, s=60, cmap='viridis') 78 | axes[0].set_xlabel("feature 1") 79 | axes[0].set_ylabel("feature 2") 80 | axes[0].set_aspect('equal') 81 | 82 | axes[1].set_title("Whitened data") 83 | axes[1].scatter(X_pca[:, 0], X_pca[:, 1], c=X_pca[:, 0], linewidths=0, s=60, cmap='viridis') 84 | axes[1].set_xlabel("First principal component") 85 | axes[1].set_ylabel("Second principal component") 86 | axes[1].set_aspect('equal') 87 | axes[1].set_xlim(-3, 4) 88 | -------------------------------------------------------------------------------- /nbs/fig_codes/plot_scaling.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from sklearn.datasets import make_blobs 4 | from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer, RobustScaler 5 | from sklearn.model_selection import train_test_split 6 | from .plot_helpers import cm2 7 | 8 | 9 | def plot_scaling(): 10 | X, y = make_blobs(n_samples=50, centers=2, random_state=4, cluster_std=1) 11 | X += 3 12 | 13 | plt.figure(figsize=(15, 8)) 14 | main_ax = plt.subplot2grid((2, 4), (0, 0), rowspan=2, colspan=2) 15 | 16 | main_ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cm2, s=60) 17 | maxx = np.abs(X[:, 0]).max() 18 | maxy = np.abs(X[:, 1]).max() 19 | 20 | main_ax.set_xlim(-maxx + 1, maxx + 1) 21 | main_ax.set_ylim(-maxy + 1, maxy + 1) 22 | main_ax.set_title("Original Data") 23 | other_axes = [plt.subplot2grid((2, 4), (i, j)) for j in range(2, 4) for i in range(2)] 24 | 25 | for ax, scaler in zip(other_axes, [StandardScaler(), RobustScaler(), 26 | MinMaxScaler(), Normalizer(norm='l2')]): 27 | X_ = scaler.fit_transform(X) 28 | ax.scatter(X_[:, 0], X_[:, 1], c=y, cmap=cm2, s=60) 29 | ax.set_xlim(-2, 2) 30 | ax.set_ylim(-2, 2) 31 | ax.set_title(type(scaler).__name__) 32 | 33 | other_axes.append(main_ax) 34 | 35 | for ax in other_axes: 36 | ax.spines['left'].set_position('center') 37 | ax.spines['right'].set_color('none') 38 | ax.spines['bottom'].set_position('center') 39 | ax.spines['top'].set_color('none') 40 | ax.xaxis.set_ticks_position('bottom') 41 | ax.yaxis.set_ticks_position('left') 42 | 43 | 44 | def plot_relative_scaling(): 45 | # make synthetic data 46 | X, _ = make_blobs(n_samples=50, centers=5, random_state=4, cluster_std=2) 47 | # split it into training and test set 48 | X_train, X_test = train_test_split(X, random_state=5, test_size=.1) 49 | # plot the training and test set 50 | fig, axes = plt.subplots(1, 3, figsize=(13, 4)) 51 | axes[0].scatter(X_train[:, 0], X_train[:, 1], 52 | c='b', label="training set", s=60, alpha=0.25) 53 | axes[0].scatter(X_test[:, 0], X_test[:, 1], marker='x', 54 | c='r', label="test set", s=60) 55 | axes[0].legend(loc='upper left') 56 | axes[0].set_title("original data") 57 | 58 | # scale the data using MinMaxScaler 59 | scaler = MinMaxScaler() 60 | scaler.fit(X_train) 61 | X_train_scaled = scaler.transform(X_train) 62 | X_test_scaled = scaler.transform(X_test) 63 | 64 | # visualize the properly scaled data 65 | axes[1].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1], 66 | c='b', label="training set", s=60, alpha=0.25) 67 | axes[1].scatter(X_test_scaled[:, 0], X_test_scaled[:, 1], marker='x', 68 | c='r', label="test set", s=60) 69 | axes[1].set_title("scaled data") 70 | 71 | # rescale the test set separately, so that test set min is 0 and test set max is 1 72 | # DO NOT DO THIS! For illustration purposes only 73 | test_scaler = MinMaxScaler() 74 | test_scaler.fit(X_test) 75 | X_test_scaled_badly = test_scaler.transform(X_test) 76 | 77 | # visualize wrongly scaled data 78 | axes[2].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1], 79 | c='b', label="training set", s=60, alpha=0.25) 80 | axes[2].scatter(X_test_scaled_badly[:, 0], X_test_scaled_badly[:, 1], marker='x', 81 | c='r', label="test set", s=60) 82 | axes[2].set_title("improperly scaled data") 83 | -------------------------------------------------------------------------------- /MLiP-week05/figs/plot_scaling.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from sklearn.datasets import make_blobs 4 | from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer, RobustScaler 5 | from sklearn.model_selection import train_test_split 6 | from .plot_helpers import cm2 7 | 8 | 9 | def plot_scaling(): 10 | X, y = make_blobs(n_samples=50, centers=2, random_state=4, cluster_std=1) 11 | X += 3 12 | 13 | plt.figure(figsize=(15, 8)) 14 | main_ax = plt.subplot2grid((2, 4), (0, 0), rowspan=2, colspan=2) 15 | 16 | main_ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cm2, s=60) 17 | maxx = np.abs(X[:, 0]).max() 18 | maxy = np.abs(X[:, 1]).max() 19 | 20 | main_ax.set_xlim(-maxx + 1, maxx + 1) 21 | main_ax.set_ylim(-maxy + 1, maxy + 1) 22 | main_ax.set_title("Original Data") 23 | other_axes = [plt.subplot2grid((2, 4), (i, j)) for j in range(2, 4) for i in range(2)] 24 | 25 | for ax, scaler in zip(other_axes, [StandardScaler(), RobustScaler(), 26 | MinMaxScaler(), Normalizer(norm='l2')]): 27 | X_ = scaler.fit_transform(X) 28 | ax.scatter(X_[:, 0], X_[:, 1], c=y, cmap=cm2, s=60) 29 | ax.set_xlim(-2, 2) 30 | ax.set_ylim(-2, 2) 31 | ax.set_title(type(scaler).__name__) 32 | 33 | other_axes.append(main_ax) 34 | 35 | for ax in other_axes: 36 | ax.spines['left'].set_position('center') 37 | ax.spines['right'].set_color('none') 38 | ax.spines['bottom'].set_position('center') 39 | ax.spines['top'].set_color('none') 40 | ax.xaxis.set_ticks_position('bottom') 41 | ax.yaxis.set_ticks_position('left') 42 | 43 | 44 | def plot_relative_scaling(): 45 | # make synthetic data 46 | X, _ = make_blobs(n_samples=50, centers=5, random_state=4, cluster_std=2) 47 | # split it into training and test set 48 | X_train, X_test = train_test_split(X, random_state=5, test_size=.1) 49 | # plot the training and test set 50 | fig, axes = plt.subplots(1, 3, figsize=(13, 4)) 51 | axes[0].scatter(X_train[:, 0], X_train[:, 1], 52 | c='b', label="training set", s=60, alpha=0.25) 53 | axes[0].scatter(X_test[:, 0], X_test[:, 1], marker='x', 54 | c='r', label="test set", s=60) 55 | axes[0].legend(loc='upper left') 56 | axes[0].set_title("original data") 57 | 58 | # scale the data using MinMaxScaler 59 | scaler = MinMaxScaler() 60 | scaler.fit(X_train) 61 | X_train_scaled = scaler.transform(X_train) 62 | X_test_scaled = scaler.transform(X_test) 63 | 64 | # visualize the properly scaled data 65 | axes[1].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1], 66 | c='b', label="training set", s=60, alpha=0.25) 67 | axes[1].scatter(X_test_scaled[:, 0], X_test_scaled[:, 1], marker='x', 68 | c='r', label="test set", s=60) 69 | axes[1].set_title("scaled data") 70 | 71 | # rescale the test set separately, so that test set min is 0 and test set max is 1 72 | # DO NOT DO THIS! For illustration purposes only 73 | test_scaler = MinMaxScaler() 74 | test_scaler.fit(X_test) 75 | X_test_scaled_badly = test_scaler.transform(X_test) 76 | 77 | # visualize wrongly scaled data 78 | axes[2].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1], 79 | c='b', label="training set", s=60, alpha=0.25) 80 | axes[2].scatter(X_test_scaled_badly[:, 0], X_test_scaled_badly[:, 1], marker='x', 81 | c='r', label="test set", s=60) 82 | axes[2].set_title("improperly scaled data") 83 | -------------------------------------------------------------------------------- /MLiP-week09/optim.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def sgd(w, dw, config=None): 5 | """ 6 | Performs vanilla stochastic gradient descent. 7 | 8 | config format: 9 | - learning_rate: Scalar learning rate. 10 | """ 11 | if config is None: config = {} 12 | config.setdefault('learning_rate', 1e-2) 13 | 14 | w -= config['learning_rate'] * dw 15 | return w, config 16 | 17 | 18 | def sgd_momentum(w, dw, config=None): 19 | """ 20 | Performs stochastic gradient descent with momentum. 21 | 22 | config format: 23 | - learning_rate: Scalar learning rate. 24 | - momentum: Scalar between 0 and 1 giving the momentum value. 25 | Setting momentum = 0 reduces to sgd. 26 | - velocity: A numpy array of the same shape as w and dw used to store a 27 | moving average of the gradients. 28 | """ 29 | if config is None: config = {} 30 | config.setdefault('learning_rate', 1e-2) 31 | config.setdefault('momentum', 0.9) 32 | v = config.get('velocity', np.zeros_like(w)) 33 | 34 | v = config['momentum'] * v - config['learning_rate'] * dw 35 | next_w = w + v 36 | config['velocity'] = v 37 | 38 | return next_w, config 39 | 40 | 41 | def rmsprop(x, dx, config=None): 42 | """ 43 | Uses the RMSProp update rule, which uses a moving average of squared 44 | gradient values to set adaptive per-parameter learning rates. 45 | 46 | config format: 47 | - learning_rate: Scalar learning rate. 48 | - decay_rate: Scalar between 0 and 1 giving the decay rate for the squared 49 | gradient cache. 50 | - epsilon: Small scalar used for smoothing to avoid dividing by zero. 51 | - cache: Moving average of second moments of gradients. 52 | """ 53 | if config is None: config = {} 54 | config.setdefault('learning_rate', 1e-2) 55 | config.setdefault('decay_rate', 0.99) 56 | config.setdefault('epsilon', 1e-8) 57 | config.setdefault('cache', np.zeros_like(x)) 58 | 59 | learning_rate = config['learning_rate'] 60 | decay_rate = config['decay_rate'] 61 | cache = config['cache'] 62 | eps = config['epsilon'] 63 | 64 | cache = decay_rate * cache + (1 - decay_rate) * dx ** 2 65 | next_x = x - learning_rate * dx / (np.sqrt(cache) + eps) 66 | 67 | config['cache'] = cache 68 | return next_x, config 69 | 70 | 71 | def adam(x, dx, config=None): 72 | """ 73 | Uses the Adam update rule, which incorporates moving averages of both the 74 | gradient and its square and a bias correction term. 75 | 76 | config format: 77 | - learning_rate: Scalar learning rate. 78 | - beta1: Decay rate for moving average of first moment of gradient. 79 | - beta2: Decay rate for moving average of second moment of gradient. 80 | - epsilon: Small scalar used for smoothing to avoid dividing by zero. 81 | - m: Moving average of gradient. 82 | - v: Moving average of squared gradient. 83 | - t: Iteration number. 84 | """ 85 | if config is None: config = {} 86 | config.setdefault('learning_rate', 1e-3) 87 | config.setdefault('beta1', 0.9) 88 | config.setdefault('beta2', 0.999) 89 | config.setdefault('epsilon', 1e-8) 90 | config.setdefault('m', np.zeros_like(x)) 91 | config.setdefault('v', np.zeros_like(x)) 92 | config.setdefault('t', 1) 93 | 94 | # read params from dictionary 95 | learning_rate = config['learning_rate'] 96 | beta1, beta2, eps = config['beta1'], config['beta2'], config['epsilon'] 97 | m, v, t = config['m'], config['v'], config['t'] 98 | 99 | # apply adam update rule 100 | t = t + 1 101 | m = beta1 * m + (1 - beta1) * dx 102 | v = beta2 * v + (1 - beta2) * dx ** 2 103 | mb = m / (1 - beta1 ** t) 104 | vb = v / (1 - beta2 ** t) 105 | next_x = x - learning_rate * mb / (np.sqrt(vb) + eps) 106 | 107 | # store new params in the dictionary 108 | config['m'], config['v'], config['t'] = m, v, t 109 | 110 | return next_x, config 111 | -------------------------------------------------------------------------------- /MLiP-week08/layers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def affine_forward(x, W, b): 5 | """ 6 | A linear mapping from inputs to scores. 7 | 8 | Inputs: 9 | - x: input matrix (N, d_1, ..., d_k) 10 | - W: weigh matrix (D, C) 11 | - b: bias vector (C, ) 12 | 13 | Outputs: 14 | - out: output of linear layer (N, C) 15 | """ 16 | x2d = np.reshape(x, (x.shape[0], -1)) # convert 4D input matrix to 2D 17 | out = np.dot(x2d, W) + b # linear transformation 18 | cache = (x, W, b) # keep for backward step (stay with us) 19 | return out, cache 20 | 21 | 22 | def affine_backward(dout, cache): 23 | """ 24 | Computes the backward pass for an affine layer. 25 | 26 | Inputs: 27 | - dout: Upstream derivative, of shape (N, C) 28 | - cache: Tuple of: 29 | - x: Input data, of shape (N, d_1, ... d_k) 30 | - w: Weights, of shape (D, C) 31 | - b: biases, of shape (C,) 32 | 33 | Outputs: 34 | - dx: Gradient with respect to x, of shape (N, d1, ..., d_k) 35 | - dw: Gradient with respect to w, of shape (D, C) 36 | - db: Gradient with respect to b, of shape (C,) 37 | """ 38 | x, w, b = cache 39 | x2d = np.reshape(x, (x.shape[0], -1)) 40 | 41 | # compute gradients 42 | db = np.sum(dout, axis=0) 43 | dw = np.dot(x2d.T, dout) 44 | dx = np.dot(dout, w.T) 45 | 46 | # reshape dx to match the size of x 47 | dx = dx.reshape(x.shape) 48 | 49 | return dx, dw, db 50 | 51 | def relu_forward(x): 52 | """Forward pass for a layer of rectified linear units. 53 | 54 | Inputs: 55 | - x: a numpy array of any shape 56 | 57 | Outputs: 58 | - out: output of relu, same shape as x 59 | - cache: x 60 | """ 61 | cache = x 62 | out = np.maximum(0, x) 63 | return out, cache 64 | 65 | def relu_backward(dout, cache): 66 | """Backward pass for a layer of rectified linear units. 67 | 68 | Inputs: 69 | - dout: upstream derevatives, of any shape 70 | - cache: x, same shape as dout 71 | 72 | Outputs: 73 | - dx: gradient of loss w.r.t x 74 | """ 75 | x = cache 76 | dx = dout * (x > 0) 77 | return dx 78 | 79 | def svm_loss(scores, y): 80 | """ 81 | Fully-vectorized implementation of SVM loss function. 82 | 83 | Inputs: 84 | - scores: scores for all training data (N, C) 85 | - y: correct labels for the training data of shape (N,) 86 | 87 | Outputs: 88 | - loss: data loss plus L2 regularization loss 89 | - grads: graidents of loss w.r.t scores 90 | """ 91 | 92 | N = scores.shape[0] 93 | 94 | # Compute svm data loss 95 | correct_class_scores = scores[range(N), y] 96 | margins = np.maximum(0.0, scores - correct_class_scores[:, None] + 1.0) 97 | margins[range(N), y] = 0.0 98 | loss = np.sum(margins) / N 99 | 100 | # Compute gradient off loss function w.r.t. scores 101 | num_pos = np.sum(margins > 0, axis=1) 102 | dscores = np.zeros(scores.shape) 103 | dscores[margins > 0] = 1 104 | dscores[range(N), y] -= num_pos 105 | dscores /= N 106 | 107 | return loss, dscores 108 | 109 | 110 | def softmax_loss(scores, y): 111 | """ 112 | Softmax loss function, fully vectorized implementation. 113 | 114 | Inputs have dimension D, there are C classes, and we operate on minibatches 115 | of N examples. 116 | 117 | Inputs: 118 | - scores: A numpy array of shape (N, C). 119 | - y: A numpy array of shape (N,) containing training labels; 120 | 121 | Outputs: 122 | - loss as single float 123 | - gradient with respect to scores 124 | """ 125 | N = scores.shape[0] # number of input data 126 | 127 | # compute data loss 128 | shifted_logits = scores - np.max(scores, axis=1, keepdims=True) 129 | Z = np.sum(np.exp(shifted_logits), axis=1, keepdims=True) 130 | log_probs = shifted_logits - np.log(Z) 131 | probs = np.exp(log_probs) 132 | loss = -np.sum(log_probs[range(N), y]) / N 133 | 134 | # Compute gradient of loss function w.r.t. scores 135 | dscores = probs.copy() 136 | dscores[range(N), y] -= 1 137 | dscores /= N 138 | 139 | return loss, dscores 140 | -------------------------------------------------------------------------------- /nbs/fig_codes/figures.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import warnings 4 | 5 | 6 | def plot_kmeans_interactive(min_clusters=1, max_clusters=6): 7 | #from IPython.html.widgets import interact 8 | from ipywidgets import interact 9 | from sklearn.metrics.pairwise import euclidean_distances 10 | from sklearn.datasets.samples_generator import make_blobs 11 | 12 | with warnings.catch_warnings(): 13 | warnings.filterwarnings('ignore') 14 | 15 | X, y = make_blobs(n_samples=300, centers=4, 16 | random_state=0, cluster_std=0.60) 17 | 18 | def _kmeans_step(frame=0, n_clusters=4): 19 | rng = np.random.RandomState(2) 20 | labels = np.zeros(X.shape[0]) 21 | centers = rng.randn(n_clusters, 2) 22 | 23 | nsteps = frame // 3 24 | 25 | for i in range(nsteps + 1): 26 | old_centers = centers 27 | if i < nsteps or frame % 3 > 0: 28 | dist = euclidean_distances(X, centers) 29 | labels = dist.argmin(1) 30 | 31 | if i < nsteps or frame % 3 > 1: 32 | centers = np.array([X[labels == j].mean(0) 33 | for j in range(n_clusters)]) 34 | nans = np.isnan(centers) 35 | centers[nans] = old_centers[nans] 36 | 37 | 38 | # plot the data and cluster centers 39 | plt.scatter(X[:, 0], X[:, 1], c=labels, s=50, cmap='rainbow', 40 | vmin=0, vmax=n_clusters - 1); 41 | plt.scatter(old_centers[:, 0], old_centers[:, 1], marker='o', 42 | c=np.arange(n_clusters), 43 | s=200, cmap='rainbow') 44 | plt.scatter(old_centers[:, 0], old_centers[:, 1], marker='o', 45 | c='black', s=50) 46 | 47 | # plot new centers if third frame 48 | if frame % 3 == 2: 49 | for i in range(n_clusters): 50 | plt.annotate('', centers[i], old_centers[i], 51 | arrowprops=dict(arrowstyle='->', linewidth=1)) 52 | plt.scatter(centers[:, 0], centers[:, 1], marker='o', 53 | c=np.arange(n_clusters), 54 | s=200, cmap='rainbow') 55 | plt.scatter(centers[:, 0], centers[:, 1], marker='o', 56 | c='black', s=50) 57 | 58 | plt.xlim(-4, 4) 59 | plt.ylim(-2, 10) 60 | 61 | if frame % 3 == 1: 62 | plt.text(3.8, 9.5, "1. Reassign points to nearest centroid", 63 | ha='right', va='top', size=14) 64 | elif frame % 3 == 2: 65 | plt.text(3.8, 9.5, "2. Update centroids to cluster means", 66 | ha='right', va='top', size=14) 67 | 68 | 69 | return interact(_kmeans_step, frame=np.arange(0, 50), 70 | n_clusters=np.arange(min_clusters, max_clusters)) 71 | 72 | 73 | def plot_image_components(x, coefficients=None, mean=0, components=None, 74 | imshape=(8, 8), n_components=6, fontsize=12): 75 | if coefficients is None: 76 | coefficients = x 77 | 78 | if components is None: 79 | components = np.eye(len(coefficients), len(x)) 80 | 81 | mean = np.zeros_like(x) + mean 82 | 83 | 84 | fig = plt.figure(figsize=(1.2 * (5 + n_components), 1.2 * 2)) 85 | g = plt.GridSpec(2, 5 + n_components, hspace=0.3) 86 | 87 | def show(i, j, x, title=None): 88 | ax = fig.add_subplot(g[i, j], xticks=[], yticks=[]) 89 | ax.imshow(x.reshape(imshape), interpolation='nearest') 90 | if title: 91 | ax.set_title(title, fontsize=fontsize) 92 | 93 | show(slice(2), slice(2), x, "True") 94 | 95 | approx = mean.copy() 96 | show(0, 2, np.zeros_like(x) + mean, r'$\mu$') 97 | show(1, 2, approx, r'$1 \cdot \mu$') 98 | 99 | for i in range(0, n_components): 100 | approx = approx + coefficients[i] * components[i] 101 | show(0, i + 3, components[i], r'$c_{0}$'.format(i + 1)) 102 | show(1, i + 3, approx, 103 | r"${0:.2f} \cdot c_{1}$".format(coefficients[i], i + 1)) 104 | plt.gca().text(0, 1.05, '$+$', ha='right', va='bottom', 105 | transform=plt.gca().transAxes, fontsize=fontsize) 106 | 107 | show(slice(2), slice(-2, None), approx, "Approx") 108 | 109 | 110 | def plot_pca_interactive(data, n_components=6): 111 | from sklearn.decomposition import PCA 112 | #from IPython.html.widgets import interact 113 | from ipywidgets import interact 114 | 115 | pca = PCA(n_components=n_components) 116 | Xproj = pca.fit_transform(data) 117 | 118 | def show_decomp(i=0): 119 | plot_image_components(data[i], Xproj[i], 120 | pca.mean_, pca.components_) 121 | 122 | interact(show_decomp, i=(0, data.shape[0] - 1)); 123 | -------------------------------------------------------------------------------- /MLiP-week05/figs/figures.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import warnings 4 | 5 | 6 | def plot_kmeans_interactive(min_clusters=1, max_clusters=6): 7 | #from IPython.html.widgets import interact 8 | from ipywidgets import interact 9 | from sklearn.metrics.pairwise import euclidean_distances 10 | from sklearn.datasets.samples_generator import make_blobs 11 | 12 | with warnings.catch_warnings(): 13 | warnings.filterwarnings('ignore') 14 | 15 | X, y = make_blobs(n_samples=300, centers=4, 16 | random_state=0, cluster_std=0.60) 17 | 18 | def _kmeans_step(frame=0, n_clusters=4): 19 | rng = np.random.RandomState(2) 20 | labels = np.zeros(X.shape[0]) 21 | centers = rng.randn(n_clusters, 2) 22 | 23 | nsteps = frame // 3 24 | 25 | for i in range(nsteps + 1): 26 | old_centers = centers 27 | if i < nsteps or frame % 3 > 0: 28 | dist = euclidean_distances(X, centers) 29 | labels = dist.argmin(1) 30 | 31 | if i < nsteps or frame % 3 > 1: 32 | centers = np.array([X[labels == j].mean(0) 33 | for j in range(n_clusters)]) 34 | nans = np.isnan(centers) 35 | centers[nans] = old_centers[nans] 36 | 37 | 38 | # plot the data and cluster centers 39 | plt.scatter(X[:, 0], X[:, 1], c=labels, s=50, cmap='rainbow', 40 | vmin=0, vmax=n_clusters - 1); 41 | plt.scatter(old_centers[:, 0], old_centers[:, 1], marker='o', 42 | c=np.arange(n_clusters), 43 | s=200, cmap='rainbow') 44 | plt.scatter(old_centers[:, 0], old_centers[:, 1], marker='o', 45 | c='black', s=50) 46 | 47 | # plot new centers if third frame 48 | if frame % 3 == 2: 49 | for i in range(n_clusters): 50 | plt.annotate('', centers[i], old_centers[i], 51 | arrowprops=dict(arrowstyle='->', linewidth=1)) 52 | plt.scatter(centers[:, 0], centers[:, 1], marker='o', 53 | c=np.arange(n_clusters), 54 | s=200, cmap='rainbow') 55 | plt.scatter(centers[:, 0], centers[:, 1], marker='o', 56 | c='black', s=50) 57 | 58 | plt.xlim(-4, 4) 59 | plt.ylim(-2, 10) 60 | 61 | if frame % 3 == 1: 62 | plt.text(3.8, 9.5, "1. Reassign points to nearest centroid", 63 | ha='right', va='top', size=14) 64 | elif frame % 3 == 2: 65 | plt.text(3.8, 9.5, "2. Update centroids to cluster means", 66 | ha='right', va='top', size=14) 67 | 68 | 69 | return interact(_kmeans_step, frame=np.arange(0, 50), 70 | n_clusters=np.arange(min_clusters, max_clusters)) 71 | 72 | 73 | def plot_image_components(x, coefficients=None, mean=0, components=None, 74 | imshape=(8, 8), n_components=6, fontsize=12): 75 | if coefficients is None: 76 | coefficients = x 77 | 78 | if components is None: 79 | components = np.eye(len(coefficients), len(x)) 80 | 81 | mean = np.zeros_like(x) + mean 82 | 83 | 84 | fig = plt.figure(figsize=(1.2 * (5 + n_components), 1.2 * 2)) 85 | g = plt.GridSpec(2, 5 + n_components, hspace=0.3) 86 | 87 | def show(i, j, x, title=None): 88 | ax = fig.add_subplot(g[i, j], xticks=[], yticks=[]) 89 | ax.imshow(x.reshape(imshape), interpolation='nearest') 90 | if title: 91 | ax.set_title(title, fontsize=fontsize) 92 | 93 | show(slice(2), slice(2), x, "True") 94 | 95 | approx = mean.copy() 96 | show(0, 2, np.zeros_like(x) + mean, r'$\mu$') 97 | show(1, 2, approx, r'$1 \cdot \mu$') 98 | 99 | for i in range(0, n_components): 100 | approx = approx + coefficients[i] * components[i] 101 | show(0, i + 3, components[i], r'$c_{0}$'.format(i + 1)) 102 | show(1, i + 3, approx, 103 | r"${0:.2f} \cdot c_{1}$".format(coefficients[i], i + 1)) 104 | plt.gca().text(0, 1.05, '$+$', ha='right', va='bottom', 105 | transform=plt.gca().transAxes, fontsize=fontsize) 106 | 107 | show(slice(2), slice(-2, None), approx, "Approx") 108 | 109 | 110 | def plot_pca_interactive(data, n_components=6): 111 | from sklearn.decomposition import PCA 112 | #from IPython.html.widgets import interact 113 | from ipywidgets import interact 114 | 115 | pca = PCA(n_components=n_components) 116 | Xproj = pca.fit_transform(data) 117 | 118 | def show_decomp(i=0): 119 | plot_image_components(data[i], Xproj[i], 120 | pca.mean_, pca.components_) 121 | 122 | interact(show_decomp, i=(0, data.shape[0] - 1)); 123 | -------------------------------------------------------------------------------- /MLiP-week09/fc_net.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from layers import * 3 | 4 | 5 | class FullyConnectedNet(object): 6 | """ 7 | A fully-connected neural network with an arbitrary number of hidden layers, 8 | ReLU nonlinearities, and a softmax loss function. This will also implement 9 | dropout and batch normalization as options. For a network with L layers, 10 | the architecture will be 11 | 12 | {affine - [batch norm] - relu - [dropout]} x (L - 1) - affine - softmax 13 | 14 | where batch normalization and dropout are optional, and the {...} block is 15 | repeated L - 1 times. 16 | 17 | Similar to the TwoLayerNet above, learn-able parameters are stored in the 18 | self.params dictionary and will be learned using the Solver class. 19 | """ 20 | 21 | def __init__(self, hidden_dims, input_dim=3*32*32, num_classes=10, 22 | dropout=0, use_batchnorm=False, reg=0.0, 23 | weight_scale=1e-2, dtype=np.float32, seed=None): 24 | """ 25 | Initialize a new FullyConnectedNet. 26 | 27 | Inputs: 28 | - hidden_dims: A list of integers giving the size of each hidden layer. 29 | - input_dim: An integer giving the size of the input. 30 | - num_classes: An integer giving the number of classes to classify. 31 | - dropout: Scalar between 0 and 1 giving dropout strength. If dropout=0 then 32 | the network should not use dropout at all. 33 | - use_batchnorm: Whether or not the network should use batch normalization. 34 | - reg: Scalar giving L2 regularization strength. 35 | - weight_scale: Scalar giving the standard deviation for random 36 | initialization of the weights. 37 | - dtype: A numpy data-type object; all computations will be performed using 38 | this data-type. float32 is faster but less accurate, so you should use 39 | float64 for numeric gradient checking. 40 | - seed: If not None, then pass this random seed to the dropout layers. This 41 | will make the dropout layers deterministic so we can gradient check the 42 | model. 43 | """ 44 | self.use_batchnorm = use_batchnorm 45 | self.use_dropout = dropout > 0 46 | self.reg = reg 47 | self.num_layers = 1 + len(hidden_dims) 48 | self.dtype = dtype 49 | self.params = {} 50 | 51 | dims = [input_dim] + hidden_dims + [num_classes] 52 | for i in range(1, self.num_layers + 1): 53 | self.params['W%d' %i] = weight_scale * np.random.randn(dims[i - 1], dims[i]) 54 | self.params['b%d' %i] = np.zeros(dims[i]) 55 | if i < self.num_layers and self.use_batchnorm: 56 | self.params['gamma%d' %i] = np.ones(dims[i]) 57 | self.params['beta%d' %i] = np.zeros(dims[i]) 58 | 59 | # When using dropout we need to pass a dropout_param dictionary to each 60 | # dropout layer so that the layer knows the dropout probability and the mode 61 | # (train / test). You can pass the same dropout_param to each dropout layer. 62 | self.dropout_param = {} 63 | if self.use_dropout: 64 | self.dropout_param = {'mode': 'train', 'p': dropout} 65 | if seed is not None: 66 | self.dropout_param['seed'] = seed 67 | 68 | # With batch normalization we need to keep track of running means and 69 | # variances, so we need to pass a special bn_param object to each batch 70 | # normalization layer. You should pass self.bn_params[0] to the forward pass 71 | # of the first batch normalization layer, self.bn_params[1] to the forward 72 | # pass of the second batch normalization layer, etc. 73 | self.bn_params = [] 74 | if self.use_batchnorm: 75 | self.bn_params = [{'mode': 'train'} for i in range(self.num_layers - 1)] 76 | 77 | # Cast all parameters to the correct data-type 78 | for k, v in self.params.items(): 79 | self.params[k] = v.astype(dtype) 80 | 81 | def loss(self, X, y=None): 82 | """ 83 | Compute loss and gradient for the fully-connected net. 84 | 85 | Input / output: Same as TwoLayerNet above. 86 | """ 87 | X = X.astype(self.dtype) 88 | mode = 'test' if y is None else 'train' 89 | 90 | # Set train/test mode for batchnorm params and dropout param since they 91 | # behave differently during training and testing. 92 | if self.use_dropout: 93 | self.dropout_param['mode'] = mode 94 | if self.use_batchnorm: 95 | for bn_param in self.bn_params: 96 | bn_param['mode'] = mode 97 | 98 | scores = None 99 | 100 | cache = {} 101 | a_cache, relu_cache, bn_cache, d_cache = {}, {}, {}, {} 102 | h = X 103 | for i in range(1, self.num_layers + 1): 104 | W, b = self.params['W%d' % i], self.params['b%d' % i] 105 | if i < self.num_layers: 106 | if self.use_batchnorm: 107 | gamma, beta = self.params['gamma%d' % i], self.params['beta%d' % i] 108 | h, a_cache[i] = affine_forward(h, W, b) 109 | h, bn_cache[i] = batchnorm_forward(h, gamma, beta, self.bn_params[i - 1]) 110 | h, relu_cache[i] = relu_forward(h) 111 | else: 112 | h, cache[i] = affine_relu_forward(h, W, b) 113 | if self.use_dropout: 114 | h, d_cache[i] = dropout_forward(h, self.dropout_param) 115 | else: 116 | scores, cache[i] = affine_forward(h, W, b) 117 | 118 | # If test mode return early 119 | if mode == 'test': 120 | return scores 121 | 122 | loss, grads = 0.0, {} 123 | 124 | loss, dscores = softmax_loss(scores, y) 125 | 126 | # backward pass 127 | dout = dscores 128 | for i in reversed(range(1, self.num_layers + 1)): 129 | if i < self.num_layers: 130 | if self.use_dropout: 131 | dout = dropout_backward(dout, d_cache[i]) 132 | if self.use_batchnorm: 133 | dout = relu_backward(dout, relu_cache[i]) 134 | dout, grads['gamma%d' % i], grads['beta%d' % i] = batchnorm_backward(dout, bn_cache[i]) 135 | dout, grads['W%d' % i], grads['b%d' % i] = affine_backward(dout, a_cache[i]) 136 | else: 137 | dout, grads['W%d' % i], grads['b%d' % i] = affine_relu_backward(dout, cache[i]) 138 | else: 139 | dout, grads['W%d' % i], grads['b%d' %i] = affine_backward(dout, cache[i]) 140 | 141 | for i in range(1, self.num_layers): 142 | W = self.params['W%d' % i] 143 | loss += 0.5 * self.reg * np.sum(W * W) 144 | grads['W%d' % i] += self.reg * W 145 | 146 | return loss, grads 147 | -------------------------------------------------------------------------------- /MLiP-week06-07/layers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def affine_forward(x, W, b): 5 | """ 6 | A linear mapping from inputs to scores. 7 | 8 | Inputs: 9 | - x: input matrix (N, d_1, ..., d_k) 10 | - W: weigh matrix (D, C) 11 | - b: bias vector (C, ) 12 | 13 | Outputs: 14 | - out: output of linear layer (N, C) 15 | """ 16 | x2d = np.reshape(x, (x.shape[0], -1)) # convert 4D input matrix to 2D 17 | out = np.dot(x2d, W) + b # linear transformation 18 | cache = (x, W, b) # keep for backward step (stay with us) 19 | return out, cache 20 | 21 | 22 | def affine_backward(dout, cache): 23 | """ 24 | Computes the backward pass for an affine layer. 25 | 26 | Inputs: 27 | - dout: Upstream derivative, of shape (N, C) 28 | - cache: Tuple of: 29 | - x: Input data, of shape (N, d_1, ... d_k) 30 | - w: Weights, of shape (D, C) 31 | - b: biases, of shape (C,) 32 | 33 | Outputs: 34 | - dx: Gradient with respect to x, of shape (N, d1, ..., d_k) 35 | - dw: Gradient with respect to w, of shape (D, C) 36 | - db: Gradient with respect to b, of shape (C,) 37 | """ 38 | x, w, b = cache 39 | x2d = np.reshape(x, (x.shape[0], -1)) 40 | 41 | # compute gradients 42 | db = np.sum(dout, axis=0) 43 | dw = np.dot(x2d.T, dout) 44 | dx = np.dot(dout, w.T) 45 | 46 | # reshape dx to match the size of x 47 | dx = dx.reshape(x.shape) 48 | 49 | return dx, dw, db 50 | 51 | def svm_loss_naive(scores, y, W, reg=1e-3): 52 | """ 53 | Naive implementation of SVM loss function. 54 | 55 | Inputs: 56 | - scores: scores for all training data (N, C) 57 | - y: correct labels for the training data 58 | - reg: regularization strength (lambd) 59 | 60 | Outputs: 61 | - loss: data loss plus L2 regularization loss 62 | - grads: graidents of loss wrt scores 63 | """ 64 | 65 | N, C = scores.shape 66 | 67 | # Compute svm data loss 68 | loss = 0.0 69 | for i in range(N): 70 | s = scores[i] # scores for the ith data 71 | correct_class = y[i] # correct class score 72 | 73 | for j in range(C): 74 | if j == y[i]: 75 | continue 76 | else: 77 | # loss += max(0, s[j] - s[correct_class] + 1.0) 78 | margin = s[j] - s[correct_class] + 1.0 79 | if margin > 0: 80 | loss += margin 81 | loss /= N 82 | 83 | # Adding L2-regularization loss 84 | loss += 0.5 * reg * np.sum(W * W) 85 | 86 | # Compute gradient off loss function w.r.t. scores 87 | # We will write this part later 88 | grads = {} 89 | 90 | return loss, grads 91 | 92 | def svm_loss_half_vectorized(scores, y, W, reg=1e-3): 93 | """ 94 | Half-vectorized implementation of SVM loss function. 95 | 96 | Inputs: 97 | - scores: scores for all training data (N, C) 98 | - y: correct labels for the training data 99 | - reg: regularization strength (lambd) 100 | 101 | Outputs: 102 | - loss: data loss plus L2 regularization loss 103 | - grads: graidents of loss wrt scores 104 | """ 105 | 106 | N, C = scores.shape 107 | 108 | # Compute svm data loss 109 | loss = 0.0 110 | for i in range(N): 111 | s = scores[i] # scores for the ith data 112 | correct_class = y[i] # correct class score 113 | 114 | margins = np.maximum(0.0, s - s[correct_class] + 1.0) 115 | margins[correct_class] = 0.0 116 | loss += np.sum(margins) 117 | 118 | loss /= N 119 | 120 | # Adding L2-regularization loss 121 | loss += 0.5 * reg * np.sum(W * W) 122 | 123 | # Compute gradient off loss function w.r.t. scores 124 | # We will write this part later 125 | grads = {} 126 | 127 | return loss, grads 128 | 129 | 130 | def svm_loss(scores, y, W, reg=1e-3): 131 | """ 132 | Fully-vectorized implementation of SVM loss function. 133 | 134 | Inputs: 135 | - scores: scores for all training data (N, C) 136 | - y: correct labels for the training data 137 | - reg: regularization strength (lambd) 138 | 139 | Outputs: 140 | - loss: data loss plus L2 regularization loss 141 | - grads: graidents of loss wrt scores 142 | """ 143 | 144 | N = scores.shape[0] 145 | 146 | # Compute svm data loss 147 | correct_class_scores = scores[range(N), y] 148 | margins = np.maximum(0.0, scores - correct_class_scores[:, None] + 1.0) 149 | margins[range(N), y] = 0.0 150 | loss = np.sum(margins) / N 151 | 152 | # Adding L2-regularization loss 153 | loss += 0.5 * reg * np.sum(W * W) 154 | 155 | # Compute gradient off loss function w.r.t. scores 156 | # We will write this part later 157 | grads = {} 158 | 159 | return loss, grads 160 | 161 | 162 | def softmax_loss_naive(scores, y, W, reg=1e-3): 163 | """ 164 | Softmax loss function, naive implementation (with loops) 165 | 166 | Inputs have dimension D, there are C classes, and we operate on minibatches 167 | of N examples. 168 | 169 | Inputs: 170 | - scores: A numpy array of shape (N, C). 171 | - y: A numpy array of shape (N,) containing training labels; 172 | - W: A numpy array of shape (D, C) containing weights. 173 | - reg: (float) regularization strength 174 | 175 | Outputs: 176 | - loss as single float 177 | - gradient with respect to weights W; an array of same shape as W 178 | """ 179 | N, C = scores.shape 180 | 181 | # compute data loss 182 | loss = 0.0 183 | for i in range(N): 184 | correct_class = y[i] 185 | score = scores[i] 186 | score -= np.max(scores) 187 | exp_score = np.exp(score) 188 | probs = exp_score / np.sum(exp_score) 189 | loss += -np.log(probs[correct_class]) 190 | 191 | loss /= N 192 | 193 | # compute regularization loss 194 | loss += 0.5 * reg * np.sum(W * W) 195 | 196 | # Compute gradient off loss function w.r.t. scores 197 | # We will write this part later 198 | grads = {} 199 | 200 | return loss, grads 201 | 202 | 203 | def softmax_loss(scores, y, W, reg=1e-3): 204 | """ 205 | Softmax loss function, naive implementation (with loops) 206 | 207 | Inputs have dimension D, there are C classes, and we operate on minibatches 208 | of N examples. 209 | 210 | Inputs: 211 | - scores: A numpy array of shape (N, C). 212 | - y: A numpy array of shape (N,) containing training labels; 213 | - W: A numpy array of shape (D, C) containing weights. 214 | - reg: (float) regularization strength 215 | 216 | Outputs: 217 | - loss as single float 218 | - gradient with respect to weights W; an array of same shape as W 219 | """ 220 | N = scores.shape[0] # number of input data 221 | 222 | # compute data loss 223 | scores -= np.max(scores, axis=1, keepdims=True) 224 | exp_scores = np.exp(scores) 225 | probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True) 226 | loss = -np.sum(np.log(probs[range(N), y])) / N 227 | 228 | # compute regularization loss 229 | loss += 0.5 * reg * np.sum(W * W) 230 | 231 | # Compute gradient off loss function w.r.t. scores 232 | # We will write this part later 233 | grads = {} 234 | 235 | return loss, grads 236 | -------------------------------------------------------------------------------- /nbs/utils/layers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def affine_forward(x, W, b): 5 | """ 6 | A linear mapping from inputs to scores. 7 | 8 | Inputs: 9 | - x: input matrix (N, d_1, ..., d_k) 10 | - W: weigh matrix (D, C) 11 | - b: bias vector (C, ) 12 | 13 | Outputs: 14 | - out: output of linear layer (N, C) 15 | """ 16 | x2d = np.reshape(x, (x.shape[0], -1)) # convert 4D input matrix to 2D 17 | out = np.dot(x2d, W) + b # linear transformation 18 | cache = (x, W, b) # keep for backward step (stay with us) 19 | return out, cache 20 | 21 | 22 | def affine_backward(dout, cache): 23 | """ 24 | Computes the backward pass for an affine layer. 25 | 26 | Inputs: 27 | - dout: Upstream derivative, of shape (N, C) 28 | - cache: Tuple of: 29 | - x: Input data, of shape (N, d_1, ... d_k) 30 | - w: Weights, of shape (D, C) 31 | - b: biases, of shape (C,) 32 | 33 | Outputs: 34 | - dx: Gradient with respect to x, of shape (N, d1, ..., d_k) 35 | - dw: Gradient with respect to w, of shape (D, C) 36 | - db: Gradient with respect to b, of shape (C,) 37 | """ 38 | x, w, b = cache 39 | x2d = np.reshape(x, (x.shape[0], -1)) 40 | 41 | # compute gradients 42 | db = np.sum(dout, axis=0) 43 | dw = np.dot(x2d.T, dout) 44 | dx = np.dot(dout, w.T) 45 | 46 | # reshape dx to match the size of x 47 | dx = dx.reshape(x.shape) 48 | 49 | return dx, dw, db 50 | 51 | def relu_forward(x): 52 | """Forward pass for a layer of rectified linear units. 53 | 54 | Inputs: 55 | - x: a numpy array of any shape 56 | 57 | Outputs: 58 | - out: output of relu, same shape as x 59 | - cache: x 60 | """ 61 | cache = x 62 | out = np.maximum(0, x) 63 | return out, cache 64 | 65 | def relu_backward(dout, cache): 66 | """Backward pass for a layer of rectified linear units. 67 | 68 | Inputs: 69 | - dout: upstream derevatives, of any shape 70 | - cache: x, same shape as dout 71 | 72 | Outputs: 73 | - dx: gradient of loss w.r.t x 74 | """ 75 | x = cache 76 | dx = dout * (x > 0) 77 | return dx 78 | 79 | def svm_loss_naive(scores, y): 80 | """ 81 | Naive implementation of SVM loss function. 82 | 83 | Inputs: 84 | - scores: scores for all training data (N, C) 85 | - y: correct labels for the training data 86 | 87 | Outputs: 88 | - loss: data loss plus L2 regularization loss 89 | - grads: graidents of loss w.r.t. scores 90 | """ 91 | 92 | N, C = scores.shape 93 | 94 | # Compute svm data loss 95 | loss = 0.0 96 | for i in range(N): 97 | s = scores[i] # scores for the ith data 98 | correct_class = y[i] # correct class score 99 | 100 | for j in range(C): 101 | if j == y[i]: 102 | continue 103 | else: 104 | # loss += max(0, s[j] - s[correct_class] + 1.0) 105 | margin = s[j] - s[correct_class] + 1.0 106 | if margin > 0: 107 | loss += margin 108 | loss /= N 109 | 110 | # Compute gradient off loss function w.r.t. scores 111 | # We will write this part later 112 | grads = {} 113 | 114 | return loss, grads 115 | 116 | def svm_loss_half_vectorized(scores, y): 117 | """ 118 | Half-vectorized implementation of SVM loss function. 119 | 120 | Inputs: 121 | - scores: scores for all training data (N, C) 122 | - y: correct labels for the training data 123 | 124 | Outputs: 125 | - loss: data loss plus L2 regularization loss 126 | - grads: graidents of loss wrt scores 127 | """ 128 | 129 | N, C = scores.shape 130 | 131 | # Compute svm data loss 132 | loss = 0.0 133 | for i in range(N): 134 | s = scores[i] # scores for the ith data 135 | correct_class = y[i] # correct class score 136 | 137 | margins = np.maximum(0.0, s - s[correct_class] + 1.0) 138 | margins[correct_class] = 0.0 139 | loss += np.sum(margins) 140 | 141 | loss /= N 142 | 143 | # Compute gradient off loss function w.r.t. scores 144 | # We will write this part later 145 | grads = {} 146 | 147 | return loss, grads 148 | 149 | def svm_loss(scores, y): 150 | """ 151 | Fully-vectorized implementation of SVM loss function. 152 | 153 | Inputs: 154 | - scores: scores for all training data (N, C) 155 | - y: correct labels for the training data of shape (N,) 156 | 157 | Outputs: 158 | - loss: data loss plus L2 regularization loss 159 | - grads: graidents of loss w.r.t scores 160 | """ 161 | 162 | N = scores.shape[0] 163 | 164 | # Compute svm data loss 165 | correct_class_scores = scores[range(N), y] 166 | margins = np.maximum(0.0, scores - correct_class_scores[:, None] + 1.0) 167 | margins[range(N), y] = 0.0 168 | loss = np.sum(margins) / N 169 | 170 | # Compute gradient off loss function w.r.t. scores 171 | num_pos = np.sum(margins > 0, axis=1) 172 | dscores = np.zeros(scores.shape) 173 | dscores[margins > 0] = 1 174 | dscores[range(N), y] -= num_pos 175 | dscores /= N 176 | 177 | return loss, dscores 178 | 179 | 180 | def softmax_loss_naive(scores, y): 181 | """ 182 | Softmax loss function, naive implementation (with loops) 183 | 184 | Inputs have dimension D, there are C classes, and we operate on minibatches 185 | of N examples. 186 | 187 | Inputs: 188 | - scores: A numpy array of shape (N, C). 189 | - y: A numpy array of shape (N,) containing training labels; 190 | 191 | Outputs: 192 | - loss: as single float 193 | - grads: gradient with respect to weights W; an array of same shape as W 194 | """ 195 | N, C = scores.shape 196 | 197 | # compute data loss 198 | loss = 0.0 199 | for i in range(N): 200 | correct_class = y[i] 201 | score = scores[i] 202 | score -= np.max(scores) 203 | exp_score = np.exp(score) 204 | probs = exp_score / np.sum(exp_score) 205 | loss += -np.log(probs[correct_class]) 206 | 207 | loss /= N 208 | 209 | # Compute gradient off loss function w.r.t. scores 210 | # We will write this part later 211 | grads = {} 212 | 213 | return loss, grads 214 | 215 | 216 | def softmax_loss(scores, y): 217 | """ 218 | Softmax loss function, fully vectorized implementation. 219 | 220 | Inputs have dimension D, there are C classes, and we operate on minibatches 221 | of N examples. 222 | 223 | Inputs: 224 | - scores: A numpy array of shape (N, C). 225 | - y: A numpy array of shape (N,) containing training labels; 226 | 227 | Outputs: 228 | - loss as single float 229 | - gradient with respect to scores 230 | """ 231 | N = scores.shape[0] # number of input data 232 | 233 | # compute data loss 234 | shifted_logits = scores - np.max(scores, axis=1, keepdims=True) 235 | Z = np.sum(np.exp(shifted_logits), axis=1, keepdims=True) 236 | log_probs = shifted_logits - np.log(Z) 237 | probs = np.exp(log_probs) 238 | loss = -np.sum(log_probs[range(N), y]) / N 239 | 240 | # Compute gradient of loss function w.r.t. scores 241 | dscores = probs.copy() 242 | dscores[range(N), y] -= 1 243 | dscores /= N 244 | 245 | return loss, dscores 246 | -------------------------------------------------------------------------------- /MLiP-week09/layers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def affine_forward(x, W, b): 5 | """ 6 | A linear mapping from inputs to scores. 7 | 8 | Inputs: 9 | - x: input matrix (N, d_1, ..., d_k) 10 | - W: weigh matrix (D, C) 11 | - b: bias vector (C, ) 12 | 13 | Outputs: 14 | - out: output of linear layer (N, C) 15 | """ 16 | x2d = np.reshape(x, (x.shape[0], -1)) # convert 4D input matrix to 2D 17 | out = np.dot(x2d, W) + b # linear transformation 18 | cache = (x, W, b) # keep for backward step (stay with us) 19 | return out, cache 20 | 21 | 22 | def affine_backward(dout, cache): 23 | """ 24 | Computes the backward pass for an affine layer. 25 | 26 | Inputs: 27 | - dout: Upstream derivative, of shape (N, C) 28 | - cache: Tuple of: 29 | - x: Input data, of shape (N, d_1, ... d_k) 30 | - w: Weights, of shape (D, C) 31 | - b: biases, of shape (C,) 32 | 33 | Outputs: 34 | - dx: Gradient with respect to x, of shape (N, d1, ..., d_k) 35 | - dw: Gradient with respect to w, of shape (D, C) 36 | - db: Gradient with respect to b, of shape (C,) 37 | """ 38 | x, w, b = cache 39 | x2d = np.reshape(x, (x.shape[0], -1)) 40 | 41 | # compute gradients 42 | db = np.sum(dout, axis=0) 43 | dw = np.dot(x2d.T, dout) 44 | dx = np.dot(dout, w.T) 45 | 46 | # reshape dx to match the size of x 47 | dx = dx.reshape(x.shape) 48 | 49 | return dx, dw, db 50 | 51 | def relu_forward(x): 52 | """Forward pass for a layer of rectified linear units. 53 | 54 | Inputs: 55 | - x: a numpy array of any shape 56 | 57 | Outputs: 58 | - out: output of relu, same shape as x 59 | - cache: x 60 | """ 61 | cache = x 62 | out = np.maximum(0, x) 63 | return out, cache 64 | 65 | def relu_backward(dout, cache): 66 | """Backward pass for a layer of rectified linear units. 67 | 68 | Inputs: 69 | - dout: upstream derevatives, of any shape 70 | - cache: x, same shape as dout 71 | 72 | Outputs: 73 | - dx: gradient of loss w.r.t x 74 | """ 75 | x = cache 76 | dx = dout * (x > 0) 77 | return dx 78 | 79 | 80 | def affine_relu_forward(x, w, b): 81 | out, cache_a = affine_forward(x, w, b) 82 | out, cache_r = relu_forward(out) 83 | return out, (cache_a, cache_r) 84 | 85 | 86 | def affine_relu_backward(dout, cache): 87 | cache_a, cache_r = cache 88 | dout = relu_backward(dout, cache_r) 89 | dx, dw, db = affine_backward(dout, cache_a) 90 | return dx, dw, db 91 | 92 | 93 | def dropout_forward(x, dropout_param): 94 | """ 95 | Performs the forward pass for (inverted) dropout. 96 | 97 | Inputs: 98 | - x: Input data, of any shape 99 | - dropout_param: A dictionary with the following keys: 100 | - p: Dropout parameter. We drop each neuron output with probability p. 101 | - mode: 'test' or 'train'. If the mode is train, then perform dropout; 102 | if the mode is test, then just return the input. 103 | - seed: Seed for the random number generator. Passing seed makes this 104 | function deterministic, which is needed for gradient checking but not 105 | in real networks. 106 | 107 | Outputs: 108 | - out: Array of the same shape as x. 109 | - cache: tuple (dropout_param, mask). In training mode, mask is the dropout 110 | mask that was used to multiply the input; in test mode, mask is None. 111 | """ 112 | p, mode = dropout_param['p'], dropout_param['mode'] 113 | if 'seed' in dropout_param: 114 | np.random.seed(dropout_param['seed']) 115 | 116 | mask = None 117 | 118 | if mode == 'train': 119 | mask = (np.random.rand(*x.shape) < (1 - p)) / (1 - p) 120 | out = x * mask 121 | elif mode == 'test': 122 | out = x 123 | 124 | cache = (dropout_param, mask) 125 | out = out.astype(x.dtype, copy=False) 126 | 127 | return out, cache 128 | 129 | 130 | def dropout_backward(dout, cache): 131 | """ 132 | Perform the backward pass for (inverted) dropout. 133 | 134 | Inputs: 135 | - dout: Upstream derivatives, of any shape 136 | - cache: (dropout_param, mask) from dropout_forward. 137 | """ 138 | dropout_param, mask = cache 139 | mode = dropout_param['mode'] 140 | 141 | if mode == 'train': 142 | dx = dout * mask 143 | elif mode == 'test': 144 | dx = dout 145 | return dx 146 | 147 | 148 | def batchnorm_forward(x, gamma, beta, bn_param): 149 | N, D = x.shape 150 | 151 | # get parameters 152 | mode = bn_param['mode'] # mode is train or test 153 | eps = bn_param.get('eps', 1e-5) 154 | momentum = bn_param.get('momentum', 0.9) 155 | running_mean = bn_param.get('running_mean', np.zeros(D, dtype=x.dtype)) 156 | running_var = bn_param.get('running_var', np.zeros(D, dtype=x.dtype)) 157 | cache = None 158 | 159 | if mode == 'train': 160 | 161 | # Normalize 162 | mu = np.mean(x, axis=0) 163 | xc = x - mu 164 | var = np.mean(xc ** 2, axis=0) 165 | std = (var + eps) ** 0.5 166 | xn = xc / std 167 | 168 | # Scale and Shift 169 | out = gamma * xn + beta 170 | 171 | cache = (x, xc, var, std, xn, gamma, eps) 172 | 173 | # update running mean and running average 174 | running_mean = momentum * running_mean + (1 - momentum) * mu 175 | running_var = momentum * running_var + (1 - momentum) * var 176 | 177 | bn_param['running_mean'] = running_mean 178 | bn_param['running_var' ] = running_var 179 | 180 | else: 181 | xn = (x - running_mean) / (np.sqrt(running_var + eps)) 182 | out = gamma * xn + beta 183 | 184 | return out, cache 185 | 186 | 187 | def batchnorm_backward(dout, cache): 188 | """ 189 | Backward pass for batch normalization. 190 | 191 | For this implementation, you should write out a computation graph for 192 | batch normalization on paper and propagate gradients backward through 193 | intermediate nodes. 194 | 195 | Inputs: 196 | - dout: Upstream derivatives, of shape (N, D) 197 | - cache: Variable of intermediates from batchnorm_forward. 198 | 199 | Returns a tuple of: 200 | - dx: Gradient with respect to inputs x, of shape (N, D) 201 | - dgamma: Gradient with respect to scale parameter gamma, of shape (D,) 202 | - dbeta: Gradient with respect to shift parameter beta, of shape (D,) 203 | """ 204 | dx, dgamma, dbeta = None, None, None 205 | 206 | x, xc, var, std, xn, gamma, eps = cache 207 | N = x.shape[0] 208 | 209 | dbeta = np.sum(dout, axis=0) 210 | dgamma = np.sum(dout * xn, axis=0) 211 | dxn = dout * gamma 212 | 213 | dxc = dxn / std 214 | dstd = np.sum(-(xc * dxn) / (std * std), axis=0) 215 | dvar = 0.5 * dstd / std 216 | 217 | dxc += (2.0 / N) * xc * dvar 218 | dmu = -np.sum(dxc, axis=0) 219 | dx = dxc + dmu / N 220 | 221 | return dx, dgamma, dbeta 222 | 223 | 224 | def svm_loss(scores, y): 225 | """ 226 | Fully-vectorized implementation of SVM loss function. 227 | 228 | Inputs: 229 | - scores: scores for all training data (N, C) 230 | - y: correct labels for the training data of shape (N,) 231 | 232 | Outputs: 233 | - loss: data loss plus L2 regularization loss 234 | - grads: graidents of loss w.r.t scores 235 | """ 236 | 237 | N = scores.shape[0] 238 | 239 | # Compute svm data loss 240 | correct_class_scores = scores[range(N), y] 241 | margins = np.maximum(0.0, scores - correct_class_scores[:, None] + 1.0) 242 | margins[range(N), y] = 0.0 243 | loss = np.sum(margins) / N 244 | 245 | # Compute gradient off loss function w.r.t. scores 246 | num_pos = np.sum(margins > 0, axis=1) 247 | dscores = np.zeros(scores.shape) 248 | dscores[margins > 0] = 1 249 | dscores[range(N), y] -= num_pos 250 | dscores /= N 251 | 252 | return loss, dscores 253 | 254 | 255 | def softmax_loss(scores, y): 256 | """ 257 | Softmax loss function, fully vectorized implementation. 258 | 259 | Inputs have dimension D, there are C classes, and we operate on minibatches 260 | of N examples. 261 | 262 | Inputs: 263 | - scores: A numpy array of shape (N, C). 264 | - y: A numpy array of shape (N,) containing training labels; 265 | 266 | Outputs: 267 | - loss as single float 268 | - gradient with respect to scores 269 | """ 270 | N = scores.shape[0] # number of input data 271 | 272 | # compute data loss 273 | shifted_logits = scores - np.max(scores, axis=1, keepdims=True) 274 | Z = np.sum(np.exp(shifted_logits), axis=1, keepdims=True) 275 | log_probs = shifted_logits - np.log(Z) 276 | probs = np.exp(log_probs) 277 | loss = -np.sum(log_probs[range(N), y]) / N 278 | 279 | # Compute gradient of loss function w.r.t. scores 280 | dscores = probs.copy() 281 | dscores[range(N), y] -= 1 282 | dscores /= N 283 | 284 | return loss, dscores 285 | -------------------------------------------------------------------------------- /MLiP-week09/solver.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle as pickle 3 | import numpy as np 4 | import optim 5 | 6 | class Solver(object): 7 | """ 8 | A Solver encapsulates all the logic necessary for training classification 9 | models. The Solver performs stochastic gradient descent using different 10 | update rules defined in optim.py. 11 | 12 | The solver accepts both training and validation data and labels so it can 13 | periodically check classification accuracy on both training and validation 14 | data to watch out for over-fitting. 15 | 16 | To train a model, you will first construct a Solver instance, passing the 17 | model, dataset, and various options (learning rate, batch size, etc) to the 18 | constructor. You will then call the train() method to run the optimization 19 | procedure and train the model. 20 | 21 | After the train() method returns, model.params will contain the parameters 22 | that performed best on the validation set over the course of training. 23 | In addition, the instance variable solver.loss_history will contain a list 24 | of all losses encountered during training and the instance variables 25 | solver.train_acc_history and solver.val_acc_history will be lists of the 26 | accuracies of the model on the training and validation set at each epoch. 27 | 28 | Example usage might look something like this: 29 | 30 | data = { 31 | 'X_train': # training data 32 | 'y_train': # training labels 33 | 'X_val': # validation data 34 | 'y_val': # validation labels 35 | } 36 | model = MyAwesomeModel(hidden_size=100, reg=10) 37 | solver = Solver(model, data, 38 | update_rule='sgd', 39 | optim_config={ 40 | 'learning_rate': 1e-3, 41 | }, 42 | lr_decay=0.95, 43 | num_epochs=10, batch_size=100, 44 | print_every=100) 45 | solver.train() 46 | 47 | 48 | A Solver works on a model object that must conform to the following API: 49 | 50 | - model.params must be a dictionary mapping string parameter names to numpy 51 | arrays containing parameter values. 52 | 53 | - model.loss(X, y) must be a function that computes training-time loss and 54 | gradients, and test-time classification scores, with the following inputs 55 | and outputs: 56 | 57 | Inputs: 58 | - X: Array giving a mini-batch of input data of shape (N, d_1, ..., d_k) 59 | - y: Array of labels, of shape (N,) giving labels for X where y[i] is the 60 | label for X[i]. 61 | 62 | Returns: 63 | If y is None, run a test-time forward pass and return: 64 | - scores: Array of shape (N, C) giving classification scores for X where 65 | scores[i, c] gives the score of class c for X[i]. 66 | 67 | If y is not None, run a training time forward and backward pass and 68 | return a tuple of: 69 | - loss: Scalar giving the loss 70 | - grads: Dictionary with the same keys as self.params mapping parameter 71 | names to gradients of the loss with respect to those parameters. 72 | """ 73 | 74 | def __init__(self, model, data, **kwargs): 75 | """ 76 | Construct a new Solver instance. 77 | 78 | Required arguments: 79 | - model: A model object conforming to the API described above 80 | - data: A dictionary of training and validation data containing: 81 | 'X_train': Array, shape (N_train, d_1, ..., d_k) of training images 82 | 'X_val': Array, shape (N_val, d_1, ..., d_k) of validation images 83 | 'y_train': Array, shape (N_train,) of labels for training images 84 | 'y_val': Array, shape (N_val,) of labels for validation images 85 | 86 | Optional arguments: 87 | - update_rule: A string giving the name of an update rule in optim.py. 88 | Default is 'sgd'. 89 | - optim_config: A dictionary containing hyper-parameters that will be 90 | passed to the chosen update rule. Each update rule requires different 91 | hyper-parameters (see optim.py) but all update rules require a 92 | 'learning_rate' parameter so that should always be present. 93 | - lr_decay: A scalar for learning rate decay; after each epoch the 94 | learning rate is multiplied by this value. 95 | - batch_size: Size of mini-batches used to compute loss and gradient 96 | during training. 97 | - num_epochs: The number of epochs to run for during training. 98 | - print_every: Integer; training losses will be printed every 99 | print_every iterations. 100 | - verbose: Boolean; if set to false then no output will be printed 101 | during training. 102 | - num_train_samples: Number of training samples used to check training 103 | accuracy; default is 1000; set to None to use entire training set. 104 | - num_val_samples: Number of validation samples to use to check val 105 | accuracy; default is None, which uses the entire validation set. 106 | - checkpoint_name: If not None, then save model checkpoints here every 107 | epoch. 108 | """ 109 | self.model = model 110 | self.X_train = data['X_train'] 111 | self.y_train = data['y_train'] 112 | self.X_val = data['X_val'] 113 | self.y_val = data['y_val'] 114 | 115 | # Unpack keyword arguments 116 | self.update_rule = kwargs.pop('update_rule', 'sgd') 117 | self.optim_config = kwargs.pop('optim_config', {}) 118 | self.lr_decay = kwargs.pop('lr_decay', 1.0) 119 | self.batch_size = kwargs.pop('batch_size', 100) 120 | self.num_epochs = kwargs.pop('num_epochs', 10) 121 | self.num_train_samples = kwargs.pop('num_train_samples', 1000) 122 | self.num_val_samples = kwargs.pop('num_val_samples', None) 123 | 124 | self.checkpoint_name = kwargs.pop('checkpoint_name', None) 125 | self.print_every = kwargs.pop('print_every', 10) 126 | self.verbose = kwargs.pop('verbose', True) 127 | 128 | # Throw an error if there are extra keyword arguments 129 | if len(kwargs) > 0: 130 | extra = ', '.join('"%s"' % k for k in list(kwargs.keys())) 131 | raise ValueError('Unrecognized arguments %s' % extra) 132 | 133 | # Make sure the update rule exists, then replace the string 134 | # name with the actual function 135 | if not hasattr(optim, self.update_rule): 136 | raise ValueError('Invalid update_rule "%s"' % self.update_rule) 137 | self.update_rule = getattr(optim, self.update_rule) 138 | 139 | self._reset() 140 | 141 | def _reset(self): 142 | """ 143 | Set up some book-keeping variables for optimization. Don't call this 144 | manually. 145 | """ 146 | # Set up some variables for book-keeping 147 | self.epoch = 0 148 | self.best_val_acc = 0 149 | self.best_params = {} 150 | self.loss_history = [] 151 | self.train_acc_history = [] 152 | self.val_acc_history = [] 153 | 154 | # Make a deep copy of the optim_config for each parameter 155 | self.optim_configs = {} 156 | for p in self.model.params: 157 | d = {k: v for k, v in self.optim_config.items()} 158 | self.optim_configs[p] = d 159 | 160 | def _step(self): 161 | """ 162 | Make a single gradient update. This is called by train() and should not 163 | be called manually. 164 | """ 165 | # Make a minibatch of training data 166 | num_train = self.X_train.shape[0] 167 | batch_mask = np.random.choice(num_train, self.batch_size) 168 | X_batch = self.X_train[batch_mask] 169 | y_batch = self.y_train[batch_mask] 170 | 171 | # Compute loss and gradient 172 | loss, grads = self.model.loss(X_batch, y_batch) 173 | self.loss_history.append(loss) 174 | 175 | # Perform a parameter update 176 | for p, w in self.model.params.items(): 177 | dw = grads[p] 178 | config = self.optim_configs[p] 179 | next_w, next_config = self.update_rule(w, dw, config) 180 | self.model.params[p] = next_w 181 | self.optim_configs[p] = next_config 182 | 183 | def _save_checkpoint(self): 184 | if self.checkpoint_name is None: return 185 | checkpoint = { 186 | 'model': self.model, 187 | 'update_rule': self.update_rule, 188 | 'lr_decay': self.lr_decay, 189 | 'optim_config': self.optim_config, 190 | 'batch_size': self.batch_size, 191 | 'num_train_samples': self.num_train_samples, 192 | 'num_val_samples': self.num_val_samples, 193 | 'epoch': self.epoch, 194 | 'loss_history': self.loss_history, 195 | 'train_acc_history': self.train_acc_history, 196 | 'val_acc_history': self.val_acc_history, 197 | } 198 | filename = '%s_epoch_%d.pkl' % (self.checkpoint_name, self.epoch) 199 | if self.verbose: 200 | print('Saving checkpoint to "%s"' % filename) 201 | with open(filename, 'wb') as f: 202 | pickle.dump(checkpoint, f) 203 | 204 | def check_accuracy(self, X, y, num_samples=None, batch_size=100): 205 | """ 206 | Check accuracy of the model on the provided data. 207 | 208 | Inputs: 209 | - X: Array of data, of shape (N, d_1, ..., d_k) 210 | - y: Array of labels, of shape (N,) 211 | - num_samples: If not None, subsample the data and only test the model 212 | on num_samples data points. 213 | - batch_size: Split X and y into batches of this size to avoid using 214 | too much memory. 215 | 216 | Returns: 217 | - acc: Scalar giving the fraction of instances that were correctly 218 | classified by the model. 219 | """ 220 | 221 | # Maybe subsample the data 222 | N = X.shape[0] 223 | if num_samples is not None and N > num_samples: 224 | mask = np.random.choice(N, num_samples) 225 | N = num_samples 226 | X = X[mask] 227 | y = y[mask] 228 | 229 | # Compute predictions in batches 230 | num_batches = N // batch_size 231 | if N % batch_size != 0: 232 | num_batches += 1 233 | y_pred = [] 234 | for i in range(num_batches): 235 | start = i * batch_size 236 | end = (i + 1) * batch_size 237 | scores = self.model.loss(X[start:end]) 238 | y_pred.append(np.argmax(scores, axis=1)) 239 | y_pred = np.hstack(y_pred) 240 | acc = np.mean(y_pred == y) 241 | 242 | return acc 243 | 244 | def train(self): 245 | """ 246 | Run optimization to train the model. 247 | """ 248 | num_train = self.X_train.shape[0] 249 | iterations_per_epoch = max(num_train // self.batch_size, 1) 250 | num_iterations = self.num_epochs * iterations_per_epoch 251 | 252 | for t in range(num_iterations): 253 | self._step() 254 | 255 | # Maybe print training loss 256 | if self.verbose and t % self.print_every == 0: 257 | print('(Iteration %4d / %4d) loss: %f' % ( 258 | t + 1, num_iterations, self.loss_history[-1])) 259 | 260 | # At the end of every epoch, increment the epoch counter and decay 261 | # the learning rate. 262 | epoch_end = (t + 1) % iterations_per_epoch == 0 263 | if epoch_end: 264 | self.epoch += 1 265 | for k in self.optim_configs: 266 | self.optim_configs[k]['learning_rate'] *= self.lr_decay 267 | 268 | # Check train and val accuracy on the first iteration, the last 269 | # iteration, and at the end of each epoch. 270 | first_it = (t == 0) 271 | last_it = (t == num_iterations - 1) 272 | if first_it or last_it or epoch_end: 273 | train_acc = self.check_accuracy(self.X_train, self.y_train, 274 | num_samples=self.num_train_samples) 275 | val_acc = self.check_accuracy(self.X_val, self.y_val, 276 | num_samples=self.num_val_samples) 277 | self.train_acc_history.append(train_acc) 278 | self.val_acc_history.append(val_acc) 279 | self._save_checkpoint() 280 | 281 | if self.verbose: 282 | print('(Epoch %2d / %2d) train acc: %f; val_acc: %f' % ( 283 | self.epoch, self.num_epochs, train_acc, val_acc)) 284 | 285 | # Keep track of the best model 286 | if val_acc > self.best_val_acc: 287 | self.best_val_acc = val_acc 288 | self.best_params = {} 289 | for k, v in self.model.params.items(): 290 | self.best_params[k] = v.copy() 291 | 292 | # At the end of training swap the best params into the model 293 | self.model.params = self.best_params 294 | -------------------------------------------------------------------------------- /nbs/imgs/unsupervised_workflow.svg: -------------------------------------------------------------------------------- 1 | 2 | image/svg+xmlTraining Data 365 | Test Data 391 | Model 413 | New View 435 | --------------------------------------------------------------------------------