├── README.md ├── ml-ex1 ├── computeCost.py ├── computeCostMulti.py ├── ex1.pdf ├── ex1.py ├── ex1_multi.py ├── ex1data1.txt ├── ex1data2.txt ├── featureNormalize.py ├── gradientDescent.py ├── gradientDescentMulti.py ├── normalEqn.py ├── plotData.py └── warmUpExercise.py ├── ml-ex2 ├── costFunction.py ├── costFunctionReg.py ├── ex2.pdf ├── ex2.py ├── ex2_reg.py ├── ex2data1.txt ├── ex2data2.txt ├── mapFeature.py ├── plotData.py ├── plotDecisionBoundary.py ├── predict.py └── sigmoid.py ├── ml-ex3 ├── displayData.py ├── ex3.pdf ├── ex3.py ├── ex3_nn.py ├── ex3data1.mat ├── ex3weights.mat ├── lrCostFunction.py ├── oneVsAll.py ├── predict.py ├── predictOneVsAll.py └── sigmoid.py ├── ml-ex4 ├── checkNNGradients.py ├── computeNumericalGradient.py ├── debugInitializeWeights.py ├── displayData.py ├── ex4.pdf ├── ex4.py ├── ex4data1.mat ├── ex4weights.mat ├── nnCostFunction.py ├── predict.py ├── randInitializeWeights.py ├── sigmoid.py └── sigmoidGradient.py ├── ml-ex5 ├── ex5.pdf ├── ex5.py ├── ex5data1.mat ├── featureNormalize.py ├── learningCurve.py ├── linearRegCostFunction.py ├── plotFit.py ├── polyFeatures.py ├── trainLinearReg.py └── validationCurve.py ├── ml-ex6 ├── cs229-notes3.pdf ├── dataset3Params.py ├── emailFeatures.py ├── emailSample1.txt ├── emailSample2.txt ├── ex6.pdf ├── ex6.py ├── ex6_spam.py ├── ex6data1.mat ├── ex6data2.mat ├── ex6data3.mat ├── gaussianKernel.py ├── getVocabList.py ├── linearKernel.py ├── plotData.py ├── processEmail.py ├── readFile.py ├── smo-book.pdf ├── smo.pdf ├── spamSample1.txt ├── spamSample2.txt ├── spamTest.mat ├── spamTrain.mat ├── stemmer.py ├── svmPredict.py ├── svmTrain.py ├── visualizeBoundary.py ├── visualizeBoundaryLinear.py └── vocab.txt ├── ml-ex7 ├── bird_small.mat ├── bird_small.png ├── computeCentroids.py ├── displayData.py ├── drawLine.py ├── ex7.pdf ├── ex7.py ├── ex7_pca.py ├── ex7data1.mat ├── ex7data2.mat ├── ex7faces.mat ├── featureNormalize.py ├── findClosestCentroids.py ├── kMeansInitCentroids.py ├── pca.py ├── plotDataPoints.py ├── plotProgresskMeans.py ├── projectData.py ├── recoverData.py └── runkMeans.py └── ml-ex8 ├── checkCostFunction.py ├── cofiCostFunc.py ├── computeNumericalGradient.py ├── estimateGaussian.py ├── ex8.pdf ├── ex8.py ├── ex8_cofi.py ├── ex8_movieParams.mat ├── ex8_movies.mat ├── ex8data1.mat ├── ex8data2.mat ├── loadMovieList.py ├── movie_ids.txt ├── multivariateGaussian.py ├── normalizeRatings.py ├── selectThreshold.py └── visualizeFit.py /README.md: -------------------------------------------------------------------------------- 1 | ![](http://makingagent.com/2018/03/24/coursera-ml-andrewng-linear-regression/coursera-ml-andrewng-cover.jpeg) 2 | 项目简介 3 | ------------ 4 | ![](https://img.shields.io/pypi/pyversions/Django.svg) 5 | 6 | 本项目是 吴恩达机器学习 课程的 笔记 和 作业。课程作业原先使用的是 Octave 和 MATLAB。不过笔者觉得无论是对 Machine Learning 的学习还是对未来工程项目的开发 Python 都更为合适。所以笔者就使用 Python 将课程作业重新实现了一遍。 7 | 希望这个项目能帮助大家理清课程的内容，理解算法背后的模型，掌握一些 Python 基本库的使用。 8 | 9 | Python 依赖包 10 | ------------ 11 | ```bash 12 | pip install numpy 13 | pip install matplotlib 14 | pip install scipy 15 | ``` 16 | 17 | 第一周 | 线性回归 18 | ------------ 19 | * [笔记](http://makingagent.com/2018/03/24/coursera-ml-andrewng-linear-regression/) 20 | * [作业说明](https://github.com/hertzcat/Coursera-ML-AndrewNg-Python/blob/master/ml-ex1/ex1.pdf) 21 | * 数据：`ex1data1.txt`，`ex1data2.txt` 22 | * 作业文件：`ex1.py`，`ex1_multi.py` 23 | 24 | ```bash 25 | python ex1.py 26 | python ex1_multi.py 27 | ``` 28 | 29 | 第二周 | 逻辑回归 30 | ------------ 31 | * [笔记](http://makingagent.com/2018/03/31/coursera-ml-andrewng-logistic-regression/) 32 | * [作业说明](https://github.com/hertzcat/Coursera-ML-AndrewNg-Python/blob/master/ml-ex2/ex2.pdf) 33 | * 数据：`ex2data2.txt`，`ex2data2.txt` 34 | * 作业文件：`ex2.py`，`ex2_reg.py` 35 | 36 | ```bash 37 | python ex2.py 38 | python ex2_reg.py 39 | ``` 40 | 41 | 第三周 | 神经网络 | 多分类问题 42 | ------------ 43 | * [笔记](http://makingagent.com/2018/04/07/coursera-ml-andrewng-nn-multi-class/) 44 | * [作业说明](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex3/ex3.pdf) 45 | * 数据：`ex3data1.mat`，`ex3weights.mat` 46 | * 作业文件：`ex3.py`，`ex3_nn.py` 47 | 48 | ```bash 49 | python ex3.py 50 | python ex3_nn.py 51 | ``` 52 | 53 | 第四周 | 神经网络 | 反向传播算法 54 | ------------ 55 | * [笔记](http://makingagent.com/2018/04/14/coursera-ml-andrewng-nn-back-propagation/) 56 | * [作业说明](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex4/ex4.pdf) 57 | * 数据：`ex4data1.mat`，`ex4weights.mat` 58 | * 作业文件：`ex4.py` 59 | 60 | ```bash 61 | python ex4.py 62 | ``` 63 | 64 | 第五周 | 方差与偏差 65 | ------------ 66 | * [笔记](http://makingagent.com/2018/04/21/coursera-ml-andrewng-bias-vs-variance/) 67 | * [作业说明](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex5/ex5.pdf) 68 | * 数据：`ex5data1.mat` 69 | * 作业文件：`ex5.py` 70 | 71 | ```bash 72 | python ex5.py 73 | ``` 74 | 75 | 第六周 | 支持向量机 76 | ------------ 77 | * [笔记](http://makingagent.com/2018/05/13/coursera-ml-andrewng-svm/) 78 | * [作业说明](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex6/ex6.pdf) 79 | * [cs229 讲义](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex6/cs229-notes3.pdf) 80 | * [cs299 SMO](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex6/smo.pdf) 81 | * [SMO 论文](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex6/smo-book.pdf) 82 | * 数据：`ex6data1.mat`，`ex6data2.mat`，`ex6data3.mat`，`spamTrain.mat`，`spamTest.mat` ... 83 | * 作业文件：`ex6.py`，`ex6_spam.py` 84 | 85 | ```bash 86 | python ex6.py 87 | python ex6_spam.py 88 | ``` 89 | 90 | 第七周 | 无监督学习算法 | k-means 与 PCA 91 | ------------ 92 | * [笔记](http://makingagent.com/2018/06/05/coursera-ml-andrewng-kmeans-and-pca/) 93 | * [作业说明](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex7/ex7.pdf) 94 | * 数据：`bird_small.png`，`ex7data1.mat`，`ex7data2.mat`，`ex7faces.mat` 95 | * 作业文件：`ex7.py`，`ex7_pca.py` 96 | 97 | ```bash 98 | python ex7.py 99 | python ex7_pca.py 100 | ``` 101 | 102 | 第八周 | 异常检测与协同过滤 103 | ------------ 104 | * [笔记](http://makingagent.com/2018/07/07/coursera-ml-andrewng-anomaly-detection-and-collaborative-filtering/) 105 | * [作业说明](https://github.com/hertzcat/Coursera-Machine-Learning/blob/master/ml-ex8/ex8.pdf) 106 | * 数据：`ex8data1.mat`，`ex8data2.mat`，`ex8_movies.mat`，`ex8_movieParams.mat`，`movie_ids.txt` 107 | * 作业文件：`ex8.py`，`ex8_cofi.py` 108 | 109 | ```bash 110 | python ex8.py 111 | python ex8_cofi.py 112 | ``` 113 | -------------------------------------------------------------------------------- /ml-ex1/computeCost.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #COMPUTECOST Compute cost for linear regression 4 | # J = COMPUTECOST(X, y, theta) computes the cost of using theta as the 5 | # parameter for linear regression to fit the data points in X and y 6 | def computeCost(X, y, theta): 7 | 8 | # Initialize some useful values 9 | m = len(y) # number of training examples 10 | 11 | error = np.dot(X,theta) - y 12 | J = np.dot(error.T, error) / m/2 13 | 14 | return J -------------------------------------------------------------------------------- /ml-ex1/computeCostMulti.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #COMPUTECOST Compute cost for linear regression 4 | # J = COMPUTECOST(X, y, theta) computes the cost of using theta as the 5 | # parameter for linear regression to fit the data points in X and y 6 | def computeCostMulti(X, y, theta): 7 | 8 | # Initialize some useful values 9 | m = len(y) # number of training examples 10 | 11 | error = np.dot(X,theta) - y 12 | J = np.dot(error.T, error) /m/2 13 | 14 | return J -------------------------------------------------------------------------------- /ml-ex1/ex1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex1/ex1.pdf -------------------------------------------------------------------------------- /ml-ex1/ex1.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class - Exercise 1: Linear Regression 2 | 3 | # Instructions 4 | # ------------ 5 | # 6 | # This file contains code that helps you get started on the 7 | # linear exercise. You will need to complete the following functions 8 | # in this exericse: 9 | # 10 | # warmUpExercise.py 11 | # plotData.py 12 | # gradientDescent.py 13 | # computeCost.py 14 | # gradientDescentMulti.py 15 | # computeCostMulti.py 16 | # featureNormalize.py 17 | # normalEqn.py 18 | # 19 | # For this exercise, you will not need to change any code in this file, 20 | # or any other files other than those mentioned above. 21 | # 22 | # x refers to the population size in 10,000s 23 | # y refers to the profit in $10,000s 24 | # 25 | 26 | ## Import 27 | import numpy as np 28 | import matplotlib.pyplot as plt 29 | from mpl_toolkits.mplot3d import Axes3D 30 | from warmUpExercise import warmUpExercise 31 | from plotData import plotData 32 | from computeCost import computeCost 33 | from gradientDescent import gradientDescent 34 | 35 | ## ==================== Part 1: Basic Function ==================== 36 | # Complete warmUpExercise.py 37 | print('Running warmUpExercise ... \n') 38 | print('5x5 Identity Matrix: \n') 39 | print(warmUpExercise()) 40 | print('\n') 41 | 42 | input('Program paused. Press enter to continue.\n') 43 | 44 | ## ======================= Part 2: Plotting ======================= 45 | print('Plotting Data ...\n') 46 | data = np.loadtxt('ex1data1.txt',delimiter=",") 47 | 48 | X = data[:, 0]; y = data[:, 1] 49 | 50 | m = len(y) # number of training examples 51 | 52 | # Plot Data 53 | # Note: You have to complete the code in plotData.py 54 | plotData(X, y) 55 | 56 | input('Program paused. Press enter to continue.\n') 57 | 58 | ## =================== Part 3: Gradient descent =================== 59 | print('Running Gradient Descent ...\n') 60 | 61 | X = np.vstack((np.ones(m), X)).T # Add a column of ones to x 62 | y = y.reshape(-1,1) 63 | theta = np.zeros((2, 1)) # initialize fitting parameters 64 | 65 | # Some gradient descent settings 66 | iterations = 1500 67 | alpha = 0.01 68 | 69 | # compute and display initial cost 70 | computeCost(X, y, theta) 71 | 72 | # run gradient descent 73 | theta, J_history = gradientDescent(X, y, theta, alpha, iterations) 74 | print(theta) 75 | 76 | # print theta to screen 77 | print('Theta found by gradient descent: ') 78 | print('%lf %lf \n'%(theta[0], theta[1])) 79 | 80 | # Plot the linear fit 81 | plt.plot(X[:,1], np.dot(X,theta), '-') 82 | plt.legend(['Training data', 'Linear regression']) 83 | 84 | # Predict values for population sizes of 35,000 and 70,000 85 | predict1 = np.dot(np.array([1, 3.5]),theta); 86 | print('For population = 35,000, we predict a profit of %f\n'%(predict1*10000)); 87 | predict2 = np.dot(np.array([1, 7]),theta); 88 | print('For population = 70,000, we predict a profit of %f\n'%(predict2*10000)); 89 | 90 | input('Program paused. Press enter to continue.\n') 91 | 92 | ## ============= Part 4: Visualizing J(theta_0, theta_1) ============= 93 | print('Visualizing J(theta_0, theta_1) ...\n') 94 | 95 | # Grid over which we will calculate J 96 | theta0_vals = np.linspace(-10, 10, 100); 97 | theta1_vals = np.linspace(-1, 4, 100); 98 | 99 | # initialize J_vals to a matrix of 0's 100 | J_vals = np.zeros((len(theta0_vals), len(theta1_vals))) 101 | 102 | # Fill out J_vals 103 | for i in range(len(theta0_vals)): 104 | for j in range(len(theta1_vals)): 105 | t = np.array([[theta0_vals[i]],[theta1_vals[j]]]) 106 | J_vals[i][j] = computeCost(X, y, t) 107 | 108 | # !!! important for plot 109 | theta0_vals, theta1_vals = np.meshgrid(theta0_vals, theta1_vals) 110 | 111 | # Because of the way meshgrids work in the surf command, we need to 112 | # transpose J_vals before calling surf, or else the axes will be flipped 113 | # Surface plo 114 | fig = plt.figure() 115 | ax = fig.gca(projection='3d') 116 | ax.plot_surface(theta0_vals, theta1_vals, J_vals.T,cmap='rainbow') 117 | ax.set_xlabel(r'$\theta_0$'); ax.set_ylabel(r'$\theta_1$') 118 | 119 | # Contour plot 120 | plt.figure() 121 | # Plot J_vals as 15 contours spaced logarithmically between 0.01 and 100 122 | plt.contour(theta0_vals, theta1_vals, J_vals.T, levels=np.logspace(-2,3,20)) 123 | plt.xlabel(r'$\theta_0$'); plt.ylabel(r'$\theta_1$') 124 | plt.plot(theta[0], theta[1], 'rx') 125 | 126 | input('Program paused. Press enter to continue.\n') 127 | plt.ioff() -------------------------------------------------------------------------------- /ml-ex1/ex1_multi.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 1: Linear regression with multiple variables 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # linear regression exercise. 9 | # 10 | # You will need to complete the following functions in this 11 | # exericse: 12 | # 13 | # warmUpExercise.py 14 | # plotData.py 15 | # gradientDescent.py 16 | # computeCost.py 17 | # gradientDescentMulti.py 18 | # computeCostMulti.py 19 | # featureNormalize.py 20 | # normalEqn.py 21 | # 22 | # For this part of the exercise, you will need to change some 23 | # parts of the code below for various experiments (e.g., changing 24 | # learning rates). 25 | # 26 | 27 | ## Import 28 | import numpy as np 29 | import matplotlib.pyplot as plt 30 | from mpl_toolkits.mplot3d import Axes3D 31 | from featureNormalize import featureNormalize 32 | from normalEqn import normalEqn 33 | from gradientDescentMulti import gradientDescentMulti 34 | 35 | ## ================ Part 1: Feature Normalization ================ 36 | 37 | print('Loading data ...\n') 38 | 39 | ## Load Data 40 | data = np.loadtxt('ex1data2.txt',delimiter=",") 41 | X = data[:, 0:2] 42 | y = data[:, 2].reshape(-1,1) 43 | m = len(y) 44 | 45 | # Print out some data points 46 | print('First 10 examples from the dataset: \n') 47 | # no find function for python 48 | # fprintf(' x = [%.0f %.0f], y = %.0f \n', [X(1:10,:) y(1:10,:)]'); 49 | print(X[0:10,:]) 50 | print(y[0:10]) 51 | 52 | input('Program paused. Press enter to continue.\n') 53 | 54 | # Scale features and set them to zero mean 55 | print('Normalizing Features ...\n') 56 | 57 | X, mu, sigma = featureNormalize(X) 58 | 59 | # Add intercept term to X 60 | X = np.vstack((np.ones(m), X.T)).T 61 | 62 | 63 | ## ================ Part 2: Gradient Descent ================ 64 | 65 | print('Running gradient descent ...\n') 66 | 67 | # Choose some alpha value 68 | alpha = 0.01 69 | num_iters = 400 70 | 71 | # Init Theta and Run Gradient Descent 72 | theta = np.zeros((3, 1)) 73 | theta, J_history = gradientDescentMulti(X, y, theta, alpha, num_iters) 74 | 75 | # Plot the convergence graph 76 | plt.ion() 77 | plt.figure() 78 | plt.plot(np.arange(0,J_history.size,1), J_history, '-g') 79 | plt.xlabel('Number of iterations') 80 | plt.ylabel('Cost J') 81 | 82 | # Display gradient descent's result 83 | print('Theta computed from gradient descent: \n') 84 | print(theta) 85 | print('\n') 86 | 87 | temp = np.array([[1.0, 1650.0, 3.0]]) 88 | temp[0,1:3] = (temp[0,1:3]-mu)/sigma 89 | price = np.dot(temp, theta) 90 | 91 | print('Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):\n $%f\n'%price) 92 | 93 | input('Program paused. Press enter to continue.\n') 94 | 95 | ## ================ Part 3: Normal Equations ================ 96 | 97 | print('Solving with normal equations...\n') 98 | 99 | ## Load Data 100 | data = np.loadtxt('ex1data2.txt',delimiter=",") 101 | X = data[:, 0:2] 102 | y = data[:, 2].reshape(-1,1) 103 | m = len(y) 104 | 105 | # Add intercept term to X 106 | X = np.vstack((np.ones(m), X.T)).T 107 | 108 | # Calculate the parameters from the normal equation 109 | theta = normalEqn(X, y) 110 | 111 | # Display normal equation's result 112 | print('Theta computed from the normal equations: \n') 113 | print(theta) 114 | print('\n') 115 | 116 | temp = np.array([[1.0, 1650.0, 3.0]]) 117 | price = np.dot(temp, theta) 118 | 119 | print('Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):\n $%f\n'%price) 120 | 121 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex1/ex1data1.txt: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /ml-ex1/ex1data2.txt: -------------------------------------------------------------------------------- 1 | 2104,3,399900 2 | 1600,3,329900 3 | 2400,3,369000 4 | 1416,2,232000 5 | 3000,4,539900 6 | 1985,4,299900 7 | 1534,3,314900 8 | 1427,3,198999 9 | 1380,3,212000 10 | 1494,3,242500 11 | 1940,4,239999 12 | 2000,3,347000 13 | 1890,3,329999 14 | 4478,5,699900 15 | 1268,3,259900 16 | 2300,4,449900 17 | 1320,2,299900 18 | 1236,3,199900 19 | 2609,4,499998 20 | 3031,4,599000 21 | 1767,3,252900 22 | 1888,2,255000 23 | 1604,3,242900 24 | 1962,4,259900 25 | 3890,3,573900 26 | 1100,3,249900 27 | 1458,3,464500 28 | 2526,3,469000 29 | 2200,3,475000 30 | 2637,3,299900 31 | 1839,2,349900 32 | 1000,1,169900 33 | 2040,4,314900 34 | 3137,3,579900 35 | 1811,4,285900 36 | 1437,3,249900 37 | 1239,3,229900 38 | 2132,4,345000 39 | 4215,4,549000 40 | 2162,4,287000 41 | 1664,2,368500 42 | 2238,3,329900 43 | 2567,4,314000 44 | 1200,3,299000 45 | 852,2,179900 46 | 1852,4,299900 47 | 1203,3,239500 48 | -------------------------------------------------------------------------------- /ml-ex1/featureNormalize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #FEATURENORMALIZE Normalizes the features in X 4 | # FEATURENORMALIZE(X) returns a normalized version of X where 5 | # the mean value of each feature is 0 and the standard deviation 6 | # is 1. This is often a good preprocessing step to do when 7 | # working with learning algorithms. 8 | def featureNormalize(X): 9 | 10 | mu = np.mean(X, axis=0) 11 | sigma = np.std(X, axis=0) 12 | 13 | return (X-mu)/sigma, mu, sigma -------------------------------------------------------------------------------- /ml-ex1/gradientDescent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from computeCost import computeCost 3 | 4 | #GRADIENTDESCENT Performs gradient descent to learn theta 5 | # theta = GRADIENTDESENT(X, y, theta, alpha, num_iters) updates theta by 6 | # taking num_iters gradient steps with learning rate alpha 7 | def gradientDescent(X, y, theta, alpha, num_iters): 8 | 9 | # Initialize some useful values 10 | m = len(y) # number of training examples 11 | J_history = np.zeros((num_iters, 1)) 12 | 13 | for iter in range(num_iters): 14 | 15 | error = np.dot(X,theta) - y 16 | theta = theta - alpha * np.dot(X.T,error) / m 17 | 18 | J_history[iter][0] = computeCost(X, y, theta) 19 | 20 | return theta, J_history -------------------------------------------------------------------------------- /ml-ex1/gradientDescentMulti.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from computeCostMulti import computeCostMulti 3 | 4 | #GRADIENTDESCENT Performs gradient descent to learn theta 5 | # theta = GRADIENTDESENT(X, y, theta, alpha, num_iters) updates theta by 6 | # taking num_iters gradient steps with learning rate alpha 7 | def gradientDescentMulti(X, y, theta, alpha, num_iters): 8 | 9 | # Initialize some useful values 10 | m = len(y) # number of training examples 11 | J_history = np.zeros((num_iters, 1)) 12 | 13 | for iter in range(num_iters): 14 | 15 | error = np.dot(X,theta) - y 16 | theta = theta - alpha * np.dot(X.T,error) / m 17 | 18 | J_history[iter][0] = computeCostMulti(X, y, theta) 19 | 20 | return theta, J_history -------------------------------------------------------------------------------- /ml-ex1/normalEqn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #NORMALEQN Computes the closed-form solution to linear regression 4 | # NORMALEQN(X,y) computes the closed-form solution to linear 5 | # regression using the normal equations. 6 | def normalEqn(X, y): 7 | 8 | B = np.linalg.pinv(np.dot(X.T,X)) 9 | theta = np.dot(np.dot(B,X.T), y) 10 | 11 | return theta -------------------------------------------------------------------------------- /ml-ex1/plotData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | 3 | 4 | #PLOTDATA Plots the data points x and y into a new figure 5 | # PLOTDATA(x,y) plots the data points and gives the figure axes labels of 6 | # population and profit. 7 | def plotData(x, y): 8 | 9 | plt.ion() 10 | plt.figure() 11 | plt.plot(x, y, 'x') 12 | plt.axis([4, 24, -5, 25]) 13 | plt.xlabel("Population of City in 10,000s") # setting the x label as population 14 | plt.ylabel("Profit in $10,000s") # setting the y label -------------------------------------------------------------------------------- /ml-ex1/warmUpExercise.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #WARMUPEXERCISE Example function in python 4 | # warmUpExercise() is an example function that returns the 5x5 identity matrix 5 | def warmUpExercise(): 6 | 7 | A = np.identity(5) 8 | 9 | return A -------------------------------------------------------------------------------- /ml-ex2/costFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #COSTFUNCTION Compute cost and gradient for logistic regression 5 | # J = COSTFUNCTION(theta, X, y) computes the cost of using theta as the 6 | # parameter for logistic regression and the gradient of the cost 7 | # w.r.t. to the parameters. 8 | def costFunction(theta, X, y): 9 | 10 | # Initialize some useful values 11 | m = len(y) # number of training examples 12 | 13 | theta = theta.reshape(-1,1) 14 | 15 | s = sigmoid( np.dot(X,theta) ) 16 | J = -( np.dot( y.T, np.log(s) ) + np.dot( (1-y).T, np.log(1-s) ) ) / m 17 | 18 | grad = np.dot(X.T, ( s - y )) / m 19 | 20 | return J[0], grad.reshape(1,-1)[0] -------------------------------------------------------------------------------- /ml-ex2/costFunctionReg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #COSTFUNCTIONREG Compute cost and gradient for logistic regression with regularization 5 | # J = COSTFUNCTIONREG(theta, X, y, lambda) computes the cost of using 6 | # theta as the parameter for regularized logistic regression and the 7 | # gradient of the cost w.r.t. to the parameters. 8 | def costFunctionReg(theta, X, y, _lambda): 9 | 10 | # Initialize some useful values 11 | m = len(y) # number of training examples 12 | 13 | theta = theta.reshape(-1,1) 14 | _theta = np.copy(theta) # !!! important, don't use reference 15 | _theta[0,0] = 0 16 | 17 | s = sigmoid( np.dot(X,theta) ) 18 | J = -( np.dot( y.T, np.log(s) ) + np.dot( (1-y).T, np.log(1-s) ) ) / m + _lambda * np.dot(_theta.T,_theta) / m/2 19 | 20 | grad = np.dot(X.T, ( s - y )) / m + _lambda * _theta / m 21 | 22 | return J[0], grad.reshape(1,-1)[0] -------------------------------------------------------------------------------- /ml-ex2/ex2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex2/ex2.pdf -------------------------------------------------------------------------------- /ml-ex2/ex2.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class - Exercise 2: Logistic Regression 2 | # 3 | # Instructions 4 | # ------------ 5 | # 6 | # This file contains code that helps you get started on the logistic 7 | # regression exercise. You will need to complete the following functions 8 | # in this exericse: 9 | # 10 | # sigmoid.py 11 | # costFunction.py 12 | # predict.py 13 | # costFunctionReg.py 14 | # 15 | # For this exercise, you will not need to change any code in this file, 16 | # or any other files other than those mentioned above. 17 | # 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | from scipy.optimize import minimize 23 | from plotData import plotData 24 | from costFunction import costFunction 25 | from plotDecisionBoundary import plotDecisionBoundary 26 | from sigmoid import sigmoid 27 | from predict import predict 28 | 29 | plt.ion() 30 | 31 | ## Load Data 32 | # The first two columns contains the exam scores and the third column 33 | # contains the label. 34 | 35 | data = np.loadtxt('ex2data1.txt', delimiter=',') 36 | X = data[:, 0:2] 37 | y = data[:, 2] 38 | 39 | ## ==================== Part 1: Plotting ==================== 40 | # We start the exercise by first plotting the data to understand the 41 | # the problem we are working with. 42 | 43 | print('Plotting data with + indicating (y = 1) examples and o indicating (y = 0) examples.\n') 44 | 45 | plotData(X, y) 46 | 47 | # Put some labels 48 | # Labels and Legend 49 | plt.xlabel('Exam 1 score') 50 | plt.ylabel('Exam 2 score') 51 | 52 | # Specified in plot order 53 | plt.legend(['Admitted', 'Not admitted']) 54 | 55 | input('\nProgram paused. Press enter to continue.\n') 56 | 57 | 58 | ## ============ Part 2: Compute Cost and Gradient ============ 59 | # In this part of the exercise, you will implement the cost and gradient 60 | # for logistic regression. You neeed to complete the code in 61 | # costFunction.py 62 | 63 | # Setup the data matrix appropriately, and add ones for the intercept term 64 | [m, n] = X.shape 65 | 66 | # Add intercept term to x and X_test 67 | X = np.vstack((np.ones(m), X.T)).T 68 | y = y.reshape(-1,1) 69 | 70 | # Initialize fitting parameters 71 | theta = np.zeros(n+1) 72 | # Compute and display initial cost and gradient 73 | cost, grad = costFunction(theta, X, y) 74 | 75 | print('Cost at initial theta (zeros): %f\n'%cost) 76 | print('Gradient at initial theta (zeros): \n') 77 | print(grad) 78 | 79 | input('\nProgram paused. Press enter to continue.\n') 80 | 81 | 82 | ## ============= Part 3: Optimizing using minimize ============= 83 | # In this exercise, you will use a scipy function (minimize) to find the 84 | # optimal parameters theta. 85 | 86 | # Set options for minimize 87 | res = minimize(costFunction, theta, method='BFGS', jac=True, options={'maxiter': 400}, args=(X, y)) 88 | 89 | # Print theta to screen 90 | print('Cost at theta found by fminunc: %f\n'%res.fun) 91 | print('theta: \n') 92 | print(res.x) 93 | 94 | # Plot Boundary 95 | theta = res.x.reshape(-1,1) 96 | plotDecisionBoundary(theta, X, y) 97 | 98 | # Put some labels 99 | # Labels and Legend 100 | plt.xlabel('Exam 1 score') 101 | plt.ylabel('Exam 2 score') 102 | 103 | input('\nProgram paused. Press enter to continue.\n') 104 | 105 | ## ============== Part 4: Predict and Accuracies ============== 106 | # After learning the parameters, you'll like to use it to predict the outcomes 107 | # on unseen data. In this part, you will use the logistic regression model 108 | # to predict the probability that a student with score 45 on exam 1 and 109 | # score 85 on exam 2 will be admitted. 110 | # 111 | # Furthermore, you will compute the training and test set accuracies of 112 | # our model. 113 | # 114 | # Your task is to complete the code in predict.m 115 | 116 | # Predict probability for a student with score 45 on exam 1 117 | # and score 85 on exam 2 118 | 119 | prob = sigmoid(np.dot([1, 45, 85], theta)) 120 | print('For a student with scores 45 and 85, we predict an admission probability of %f\n\n'%prob[0]) 121 | 122 | # Compute accuracy on our training set 123 | p = predict(theta, X) 124 | 125 | print('Train Accuracy: %.2f\n'%(np.mean(np.double(p == y)) * 100)) 126 | 127 | input('\nProgram paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex2/ex2_reg.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class - Exercise 2: Logistic Regression 2 | # 3 | # Instructions 4 | # ------------ 5 | # 6 | # This file contains code that helps you get started on the second part 7 | # of the exercise which covers regularization with logistic regression. 8 | # 9 | # You will need to complete the following functions in this exericse: 10 | # 11 | # sigmoid.py 12 | # costFunction.py 13 | # predict.py 14 | # costFunctionReg.py 15 | # 16 | # For this exercise, you will not need to change any code in this file, 17 | # or any other files other than those mentioned above. 18 | # 19 | 20 | ## Initialization 21 | import numpy as np 22 | import matplotlib.pyplot as plt 23 | from scipy.optimize import minimize 24 | from plotData import plotData 25 | from mapFeature import mapFeature 26 | from costFunctionReg import costFunctionReg 27 | from plotDecisionBoundary import plotDecisionBoundary 28 | from predict import predict 29 | 30 | plt.ion() 31 | 32 | ## Load Data 33 | # The first two columns contains the X values and the third column 34 | # contains the label (y). 35 | 36 | data = np.loadtxt('ex2data2.txt', delimiter=',') 37 | X = data[:, 0:2]; y = data[:, 2] 38 | 39 | plotData(X, y) 40 | 41 | # Labels and Legend 42 | plt.xlabel('Microchip Test 1') 43 | plt.ylabel('Microchip Test 2') 44 | 45 | # Specified in plot order 46 | plt.legend(['y = 1', 'y = 0']) 47 | 48 | 49 | ## =========== Part 1: Regularized Logistic Regression ============ 50 | # In this part, you are given a dataset with data points that are not 51 | # linearly separable. However, you would still like to use logistic 52 | # regression to classify the data points. 53 | # 54 | # To do so, you introduce more features to use -- in particular, you add 55 | # polynomial features to our data matrix (similar to polynomial 56 | # regression). 57 | # 58 | 59 | # Add Polynomial Features 60 | 61 | # Note that mapFeature also adds a column of ones for us, so the intercept 62 | # term is handled 63 | X = mapFeature(X[:,0], X[:,1]).T 64 | y = y.reshape(-1,1) 65 | 66 | # Initialize fitting parameters 67 | initial_theta = np.zeros(X.shape[1]) 68 | 69 | # Set regularization parameter lambda to 1 70 | _lambda = 1 71 | 72 | # Compute and display initial cost and gradient for regularized logistic 73 | # regression 74 | cost, grad = costFunctionReg(initial_theta, X, y, _lambda) 75 | 76 | print('Cost at initial theta (zeros): %f\n'%cost) 77 | 78 | input('\nProgram paused. Press enter to continue.\n') 79 | 80 | ## ============= Part 2: Regularization and Accuracies ============= 81 | # Optional Exercise: 82 | # In this part, you will get to try different values of lambda and 83 | # see how regularization affects the decision coundart 84 | # 85 | # Try the following values of lambda (0, 1, 10, 100). 86 | # 87 | # How does the decision boundary change when you vary lambda? How does 88 | # the training set accuracy vary? 89 | # 90 | 91 | # Initialize fitting parameters 92 | initial_theta = np.zeros(X.shape[1]) 93 | 94 | # Set regularization parameter lambda to 1 (you should vary this) 95 | _lambda = 1 96 | 97 | # Optimize 98 | res = minimize(costFunctionReg, initial_theta, method='BFGS', jac=True, options={'maxiter': 400}, args=(X, y, _lambda)) 99 | theta = res.x.reshape(-1,1) 100 | 101 | # Plot Boundary 102 | plotDecisionBoundary(theta, X, y) 103 | 104 | plt.title('lambda = %g'%_lambda) 105 | 106 | # Labels and Legend 107 | plt.xlabel('Microchip Test 1') 108 | plt.ylabel('Microchip Test 2') 109 | 110 | plt.legend(['y = 1', 'y = 0', 'Decision boundary']) 111 | 112 | # Compute accuracy on our training set 113 | p = predict(theta, X) 114 | 115 | print('Train Accuracy: %f\n'%(np.mean(np.double(p == y)) * 100)) 116 | 117 | input() -------------------------------------------------------------------------------- /ml-ex2/ex2data1.txt: -------------------------------------------------------------------------------- 1 | 34.62365962451697,78.0246928153624,0 2 | 30.28671076822607,43.89499752400101,0 3 | 35.84740876993872,72.90219802708364,0 4 | 60.18259938620976,86.30855209546826,1 5 | 79.0327360507101,75.3443764369103,1 6 | 45.08327747668339,56.3163717815305,0 7 | 61.10666453684766,96.51142588489624,1 8 | 75.02474556738889,46.55401354116538,1 9 | 76.09878670226257,87.42056971926803,1 10 | 84.43281996120035,43.53339331072109,1 11 | 95.86155507093572,38.22527805795094,0 12 | 75.01365838958247,30.60326323428011,0 13 | 82.30705337399482,76.48196330235604,1 14 | 69.36458875970939,97.71869196188608,1 15 | 39.53833914367223,76.03681085115882,0 16 | 53.9710521485623,89.20735013750205,1 17 | 69.07014406283025,52.74046973016765,1 18 | 67.94685547711617,46.67857410673128,0 19 | 70.66150955499435,92.92713789364831,1 20 | 76.97878372747498,47.57596364975532,1 21 | 67.37202754570876,42.83843832029179,0 22 | 89.67677575072079,65.79936592745237,1 23 | 50.534788289883,48.85581152764205,0 24 | 34.21206097786789,44.20952859866288,0 25 | 77.9240914545704,68.9723599933059,1 26 | 62.27101367004632,69.95445795447587,1 27 | 80.1901807509566,44.82162893218353,1 28 | 93.114388797442,38.80067033713209,0 29 | 61.83020602312595,50.25610789244621,0 30 | 38.78580379679423,64.99568095539578,0 31 | 61.379289447425,72.80788731317097,1 32 | 85.40451939411645,57.05198397627122,1 33 | 52.10797973193984,63.12762376881715,0 34 | 52.04540476831827,69.43286012045222,1 35 | 40.23689373545111,71.16774802184875,0 36 | 54.63510555424817,52.21388588061123,0 37 | 33.91550010906887,98.86943574220611,0 38 | 64.17698887494485,80.90806058670817,1 39 | 74.78925295941542,41.57341522824434,0 40 | 34.1836400264419,75.2377203360134,0 41 | 83.90239366249155,56.30804621605327,1 42 | 51.54772026906181,46.85629026349976,0 43 | 94.44336776917852,65.56892160559052,1 44 | 82.36875375713919,40.61825515970618,0 45 | 51.04775177128865,45.82270145776001,0 46 | 62.22267576120188,52.06099194836679,0 47 | 77.19303492601364,70.45820000180959,1 48 | 97.77159928000232,86.7278223300282,1 49 | 62.07306379667647,96.76882412413983,1 50 | 91.56497449807442,88.69629254546599,1 51 | 79.94481794066932,74.16311935043758,1 52 | 99.2725269292572,60.99903099844988,1 53 | 90.54671411399852,43.39060180650027,1 54 | 34.52451385320009,60.39634245837173,0 55 | 50.2864961189907,49.80453881323059,0 56 | 49.58667721632031,59.80895099453265,0 57 | 97.64563396007767,68.86157272420604,1 58 | 32.57720016809309,95.59854761387875,0 59 | 74.24869136721598,69.82457122657193,1 60 | 71.79646205863379,78.45356224515052,1 61 | 75.3956114656803,85.75993667331619,1 62 | 35.28611281526193,47.02051394723416,0 63 | 56.25381749711624,39.26147251058019,0 64 | 30.05882244669796,49.59297386723685,0 65 | 44.66826172480893,66.45008614558913,0 66 | 66.56089447242954,41.09209807936973,0 67 | 40.45755098375164,97.53518548909936,1 68 | 49.07256321908844,51.88321182073966,0 69 | 80.27957401466998,92.11606081344084,1 70 | 66.74671856944039,60.99139402740988,1 71 | 32.72283304060323,43.30717306430063,0 72 | 64.0393204150601,78.03168802018232,1 73 | 72.34649422579923,96.22759296761404,1 74 | 60.45788573918959,73.09499809758037,1 75 | 58.84095621726802,75.85844831279042,1 76 | 99.82785779692128,72.36925193383885,1 77 | 47.26426910848174,88.47586499559782,1 78 | 50.45815980285988,75.80985952982456,1 79 | 60.45555629271532,42.50840943572217,0 80 | 82.22666157785568,42.71987853716458,0 81 | 88.9138964166533,69.80378889835472,1 82 | 94.83450672430196,45.69430680250754,1 83 | 67.31925746917527,66.58935317747915,1 84 | 57.23870631569862,59.51428198012956,1 85 | 80.36675600171273,90.96014789746954,1 86 | 68.46852178591112,85.59430710452014,1 87 | 42.0754545384731,78.84478600148043,0 88 | 75.47770200533905,90.42453899753964,1 89 | 78.63542434898018,96.64742716885644,1 90 | 52.34800398794107,60.76950525602592,0 91 | 94.09433112516793,77.15910509073893,1 92 | 90.44855097096364,87.50879176484702,1 93 | 55.48216114069585,35.57070347228866,0 94 | 74.49269241843041,84.84513684930135,1 95 | 89.84580670720979,45.35828361091658,1 96 | 83.48916274498238,48.38028579728175,1 97 | 42.2617008099817,87.10385094025457,1 98 | 99.31500880510394,68.77540947206617,1 99 | 55.34001756003703,64.9319380069486,1 100 | 74.77589300092767,89.52981289513276,1 101 | -------------------------------------------------------------------------------- /ml-ex2/ex2data2.txt: -------------------------------------------------------------------------------- 1 | 0.051267,0.69956,1 2 | -0.092742,0.68494,1 3 | -0.21371,0.69225,1 4 | -0.375,0.50219,1 5 | -0.51325,0.46564,1 6 | -0.52477,0.2098,1 7 | -0.39804,0.034357,1 8 | -0.30588,-0.19225,1 9 | 0.016705,-0.40424,1 10 | 0.13191,-0.51389,1 11 | 0.38537,-0.56506,1 12 | 0.52938,-0.5212,1 13 | 0.63882,-0.24342,1 14 | 0.73675,-0.18494,1 15 | 0.54666,0.48757,1 16 | 0.322,0.5826,1 17 | 0.16647,0.53874,1 18 | -0.046659,0.81652,1 19 | -0.17339,0.69956,1 20 | -0.47869,0.63377,1 21 | -0.60541,0.59722,1 22 | -0.62846,0.33406,1 23 | -0.59389,0.005117,1 24 | -0.42108,-0.27266,1 25 | -0.11578,-0.39693,1 26 | 0.20104,-0.60161,1 27 | 0.46601,-0.53582,1 28 | 0.67339,-0.53582,1 29 | -0.13882,0.54605,1 30 | -0.29435,0.77997,1 31 | -0.26555,0.96272,1 32 | -0.16187,0.8019,1 33 | -0.17339,0.64839,1 34 | -0.28283,0.47295,1 35 | -0.36348,0.31213,1 36 | -0.30012,0.027047,1 37 | -0.23675,-0.21418,1 38 | -0.06394,-0.18494,1 39 | 0.062788,-0.16301,1 40 | 0.22984,-0.41155,1 41 | 0.2932,-0.2288,1 42 | 0.48329,-0.18494,1 43 | 0.64459,-0.14108,1 44 | 0.46025,0.012427,1 45 | 0.6273,0.15863,1 46 | 0.57546,0.26827,1 47 | 0.72523,0.44371,1 48 | 0.22408,0.52412,1 49 | 0.44297,0.67032,1 50 | 0.322,0.69225,1 51 | 0.13767,0.57529,1 52 | -0.0063364,0.39985,1 53 | -0.092742,0.55336,1 54 | -0.20795,0.35599,1 55 | -0.20795,0.17325,1 56 | -0.43836,0.21711,1 57 | -0.21947,-0.016813,1 58 | -0.13882,-0.27266,1 59 | 0.18376,0.93348,0 60 | 0.22408,0.77997,0 61 | 0.29896,0.61915,0 62 | 0.50634,0.75804,0 63 | 0.61578,0.7288,0 64 | 0.60426,0.59722,0 65 | 0.76555,0.50219,0 66 | 0.92684,0.3633,0 67 | 0.82316,0.27558,0 68 | 0.96141,0.085526,0 69 | 0.93836,0.012427,0 70 | 0.86348,-0.082602,0 71 | 0.89804,-0.20687,0 72 | 0.85196,-0.36769,0 73 | 0.82892,-0.5212,0 74 | 0.79435,-0.55775,0 75 | 0.59274,-0.7405,0 76 | 0.51786,-0.5943,0 77 | 0.46601,-0.41886,0 78 | 0.35081,-0.57968,0 79 | 0.28744,-0.76974,0 80 | 0.085829,-0.75512,0 81 | 0.14919,-0.57968,0 82 | -0.13306,-0.4481,0 83 | -0.40956,-0.41155,0 84 | -0.39228,-0.25804,0 85 | -0.74366,-0.25804,0 86 | -0.69758,0.041667,0 87 | -0.75518,0.2902,0 88 | -0.69758,0.68494,0 89 | -0.4038,0.70687,0 90 | -0.38076,0.91886,0 91 | -0.50749,0.90424,0 92 | -0.54781,0.70687,0 93 | 0.10311,0.77997,0 94 | 0.057028,0.91886,0 95 | -0.10426,0.99196,0 96 | -0.081221,1.1089,0 97 | 0.28744,1.087,0 98 | 0.39689,0.82383,0 99 | 0.63882,0.88962,0 100 | 0.82316,0.66301,0 101 | 0.67339,0.64108,0 102 | 1.0709,0.10015,0 103 | -0.046659,-0.57968,0 104 | -0.23675,-0.63816,0 105 | -0.15035,-0.36769,0 106 | -0.49021,-0.3019,0 107 | -0.46717,-0.13377,0 108 | -0.28859,-0.060673,0 109 | -0.61118,-0.067982,0 110 | -0.66302,-0.21418,0 111 | -0.59965,-0.41886,0 112 | -0.72638,-0.082602,0 113 | -0.83007,0.31213,0 114 | -0.72062,0.53874,0 115 | -0.59389,0.49488,0 116 | -0.48445,0.99927,0 117 | -0.0063364,0.99927,0 118 | 0.63265,-0.030612,0 119 | -------------------------------------------------------------------------------- /ml-ex2/mapFeature.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | # MAPFEATURE Feature mapping function to polynomial features 5 | # 6 | # MAPFEATURE(X1, X2) maps the two input features 7 | # to quadratic features used in the regularization exercise. 8 | # 9 | # Returns a new feature array with more features, comprising of 10 | # X1, X2, X1.^2, X2.^2, X1*X2, X1*X2.^2, etc.. 11 | # 12 | # Inputs X1, X2 must be the same size 13 | # 14 | def mapFeature(X1, X2): 15 | 16 | degree = 6 17 | out = np.ones(X1.shape) 18 | for i in range(1,degree+1): 19 | for j in range(0,i+1): 20 | out = np.row_stack((out, np.power(X1,(i-j))*np.power(X2,j))) 21 | 22 | return out -------------------------------------------------------------------------------- /ml-ex2/plotData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | #PLOTDATA Plots the data points X and y into a new figure 5 | # PLOTDATA(x,y) plots the data points with + for the positive examples 6 | # and o for the negative examples. X is assumed to be a Mx2 matrix. 7 | def plotData(X, y): 8 | 9 | # Create New Figure 10 | plt.figure() 11 | # Find Indices of Positive and Negative Examples 12 | pos = np.where(y==1); neg = np.where(y==0) 13 | plt.plot(X[pos][:,0],X[pos][:,1], 'k+', linewidth=2, markersize=7) 14 | plt.plot(X[neg][:,0],X[neg][:,1], 'ko', markerfacecolor='y', markersize=7) 15 | -------------------------------------------------------------------------------- /ml-ex2/plotDecisionBoundary.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from plotData import plotData 4 | from mapFeature import mapFeature 5 | 6 | #PLOTDECISIONBOUNDARY Plots the data points X and y into a new figure with 7 | #the decision boundary defined by theta 8 | # PLOTDECISIONBOUNDARY(theta, X,y) plots the data points with + for the 9 | # positive examples and o for the negative examples. X is assumed to be 10 | # a either 11 | # 1) Mx3 matrix, where the first column is an all-ones column for the 12 | # intercept. 13 | # 2) MxN, N>3 matrix, where the first column is all-ones 14 | def plotDecisionBoundary(theta, X, y): 15 | 16 | # Create New Figure 17 | plotData(X[:,1:3],y.T[0]) 18 | 19 | if X.shape[1] <= 3: 20 | # Only need 2 points to define a line, so choose two endpoints 21 | plot_x = np.array([np.min(X[:,1]), np.max(X[:,1])]) 22 | 23 | # Calculate the decision boundary line 24 | plot_y = (-1/theta[2])*(theta[1]*plot_x + theta[0]) 25 | 26 | # Plot, and adjust axes for better viewing 27 | plt.plot(plot_x,plot_y) 28 | 29 | # Legend, specific for the exercise 30 | plt.legend(['Admitted', 'Not admitted', 'Decision Boundary']) 31 | plt.axis([30, 100, 30, 100]) 32 | else: 33 | # Here is the grid range 34 | u = np.linspace(-1, 1.5, 50) 35 | v = np.linspace(-1, 1.5, 50) 36 | 37 | z = np.zeros((u.shape[0], v.shape[0])) 38 | # Evaluate z = theta*x over the grid 39 | for i in range(0,u.shape[0]): 40 | for j in range(0,v.shape[0]): 41 | z[i,j] = np.dot(theta.T, mapFeature(u[i],v[j])) 42 | 43 | # !!! important for plot 44 | u, v = np.meshgrid(u, v) 45 | 46 | # Plot z = 0 47 | # Notice you need to specify the range [0, 0] 48 | plt.contour(u, v, z.T, (0,), colors='g', linewidths=2) 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /ml-ex2/predict.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #PREDICT Predict whether the label is 0 or 1 using learned logistic 5 | #regression parameters theta 6 | # p = PREDICT(theta, X) computes the predictions for X using a 7 | # threshold at 0.5 (i.e., if sigmoid(theta'*x) >= 0.5, predict 1) 8 | def predict(theta, X): 9 | 10 | m = X.shape[0] # Number of training examples 11 | 12 | return np.round(sigmoid(np.dot(X, theta))) -------------------------------------------------------------------------------- /ml-ex2/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #SIGMOID Compute sigmoid functoon 4 | # J = SIGMOID(z) computes the sigmoid of z. 5 | def sigmoid(z): 6 | 7 | return 1 / ( 1 + np.exp(-z) ) -------------------------------------------------------------------------------- /ml-ex3/displayData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | #DISPLAYDATA Display 2D data in a nice grid 6 | # [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 7 | # stored in X in a nice grid. It returns the figure handle h and the 8 | # displayed array if requested. 9 | def displayData(X, example_width=None): 10 | 11 | # Set example_width automatically if not passed in 12 | if example_width == None: 13 | example_width = (int)(np.round(np.sqrt(X.shape[1]))) 14 | 15 | # Compute rows, cols 16 | m, n = X.shape 17 | example_height = (int)(n / example_width) 18 | 19 | # Compute number of items to display 20 | display_rows = (int)(np.floor(np.sqrt(m))) 21 | display_cols = (int)(np.ceil(m / display_rows)) 22 | 23 | # Between images padding 24 | pad = 1 25 | 26 | # Setup blank display 27 | display_array = - np.ones((pad + display_rows * (example_height + pad), \ 28 | pad + display_cols * (example_width + pad))) 29 | 30 | # Copy each example into a patch on the display array 31 | curr_ex = 0 32 | for j in range(display_rows): 33 | for i in range(display_cols): 34 | if curr_ex >= m: 35 | break 36 | 37 | # Copy the patch 38 | 39 | # Get the max value of the patch 40 | max_val = np.max(np.abs(X[curr_ex])) 41 | offset_height = pad + j * (example_height + pad) 42 | offset_width = pad + i * (example_width + pad) 43 | display_array[offset_height : offset_height+example_height,\ 44 | offset_width : offset_width+example_width] = \ 45 | X[curr_ex].reshape((example_height,example_width)).T / max_val 46 | curr_ex = curr_ex + 1 47 | 48 | if curr_ex >= m: 49 | break 50 | 51 | # Display Image 52 | plt.imshow(display_array, cmap='gray', vmin = -1, vmax = 1) 53 | 54 | # Do not show axis 55 | plt.axis('off') -------------------------------------------------------------------------------- /ml-ex3/ex3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex3/ex3.pdf -------------------------------------------------------------------------------- /ml-ex3/ex3.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class - Exercise 3 | Part 1: One-vs-all 2 | 3 | # Instructions 4 | # ------------ 5 | # 6 | # This file contains code that helps you get started on the 7 | # linear exercise. You will need to complete the following functions 8 | # in this exericse: 9 | # 10 | # lrCostFunction.py (logistic regression cost function) 11 | # oneVsAll.py 12 | # predictOneVsAll.py 13 | # predict.py 14 | # 15 | # For this exercise, you will not need to change any code in this file, 16 | # or any other files other than those mentioned above. 17 | # 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | import scipy.io as sio 23 | from scipy.optimize import minimize 24 | from displayData import displayData 25 | from oneVsAll import oneVsAll 26 | from predictOneVsAll import predictOneVsAll 27 | 28 | plt.ion() 29 | 30 | ## Setup the parameters you will use for this part of the exercise 31 | input_layer_size = 400 # 20x20 Input Images of Digits 32 | num_labels = 10 # 10 labels, from 1 to 10 33 | # (note that we have mapped "0" to label 10) 34 | 35 | ## =========== Part 1: Loading and Visualizing Data ============= 36 | # We start the exercise by first loading and visualizing the dataset. 37 | # You will be working with a dataset that contains handwritten digits. 38 | # 39 | 40 | # Load Training Data 41 | print('Loading and Visualizing Data ...\n') 42 | 43 | data = sio.loadmat('ex3data1.mat') # training data stored in arrays X, y 44 | X = data['X']; y = data['y']%10 45 | 46 | m = X.shape[0] 47 | 48 | # Randomly select 100 data points to display 49 | rand_indices = np.random.permutation(m) 50 | sel = X[rand_indices[0:100]] 51 | 52 | displayData(sel) 53 | 54 | input('Program paused. Press enter to continue.\n') 55 | 56 | ## ============ Part 2: Vectorize Logistic Regression ============ 57 | # In this part of the exercise, you will reuse your logistic regression 58 | # code from the last exercise. You task here is to make sure that your 59 | # regularized logistic regression implementation is vectorized. After 60 | # that, you will implement one-vs-all classification for the handwritten 61 | # digit dataset. 62 | # 63 | 64 | print('\nTraining One-vs-All Logistic Regression...\n') 65 | 66 | _lambda = 0.1 67 | all_theta = oneVsAll(X, y, num_labels, _lambda) 68 | 69 | 70 | input('Program paused. Press enter to continue.\n') 71 | 72 | ## ================ Part 3: Predict for One-Vs-All ================ 73 | # After ... 74 | pred = predictOneVsAll(all_theta, X) 75 | 76 | print('\nTraining Set Accuracy: %f\n'%(np.mean(np.double(pred == y.T)) * 100)) 77 | 78 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex3/ex3_nn.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class - Exercise 3 | Part 2: Neural Networks 2 | 3 | # Instructions 4 | # ------------ 5 | # 6 | # This file contains code that helps you get started on the 7 | # linear exercise. You will need to complete the following functions 8 | # in this exericse: 9 | # 10 | # lrCostFunction.py (logistic regression cost function) 11 | # oneVsAll.py 12 | # predictOneVsAll.py 13 | # predict.py 14 | # 15 | # For this exercise, you will not need to change any code in this file, 16 | # or any other files other than those mentioned above. 17 | # 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | import scipy.io as sio 23 | from scipy.optimize import minimize 24 | from displayData import displayData 25 | from oneVsAll import oneVsAll 26 | from predict import predict 27 | 28 | plt.ion() 29 | 30 | ## Setup the parameters you will use for this exercise 31 | input_layer_size = 400 # 20x20 Input Images of Digits 32 | hidden_layer_size = 25 # 25 hidden units 33 | num_labels = 10 # 10 labels, from 1 to 10 34 | # (note that we have mapped "0" to label 10) 35 | 36 | ## =========== Part 1: Loading and Visualizing Data ============= 37 | # We start the exercise by first loading and visualizing the dataset. 38 | # You will be working with a dataset that contains handwritten digits. 39 | # 40 | 41 | # Load Training Data 42 | print('Loading and Visualizing Data ...\n') 43 | 44 | data = sio.loadmat('ex3data1.mat') # training data stored in arrays X, y 45 | X = data['X']; y = data['y']%10 46 | 47 | m = X.shape[0] 48 | 49 | # Randomly select 100 data points to display 50 | rand_indices = np.random.permutation(m) 51 | sel = X[rand_indices[0:100]] 52 | 53 | displayData(sel) 54 | 55 | input('Program paused. Press enter to continue.\n') 56 | 57 | ## ================ Part 2: Loading Pameters ================ 58 | # In this part of the exercise, we load some pre-initialized 59 | # neural network parameters. 60 | 61 | print('\nLoading Saved Neural Network Parameters ...\n') 62 | 63 | # Load the weights into variables Theta1 and Theta2 64 | data = sio.loadmat('ex3weights.mat') 65 | Theta1 = data['Theta1']; Theta2 = data['Theta2'] 66 | 67 | ## ================= Part 3: Implement Predict ================= 68 | # After training the neural network, we would like to use it to predict 69 | # the labels. You will now implement the "predict" function to use the 70 | # neural network to predict the labels of the training set. This lets 71 | # you compute the training set accuracy. 72 | 73 | pred = (predict(Theta1, Theta2, X)+1)%10 74 | 75 | print('\nTraining Set Accuracy: %f\n'%(np.mean(np.double(pred == y.T)) * 100)) 76 | 77 | input('Program paused. Press enter to continue.\n') 78 | 79 | # To give you an idea of the network's output, you can also run 80 | # through the examples one at the a time to see what it is predicting. 81 | 82 | # Randomly permute examples 83 | rp = np.random.permutation(m) 84 | 85 | for i in range(m): 86 | # Display 87 | print('\nDisplaying Example Image\n') 88 | t = np.array([X[rp[i]]]) 89 | displayData(t) 90 | 91 | pred = predict(Theta1, Theta2, t) 92 | print('\nNeural Network Prediction: %d (digit %d)\n'%(pred, (pred+1)%10)) 93 | 94 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex3/ex3data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex3/ex3data1.mat -------------------------------------------------------------------------------- /ml-ex3/ex3weights.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex3/ex3weights.mat -------------------------------------------------------------------------------- /ml-ex3/lrCostFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #LRCOSTFUNCTION Compute cost and gradient for logistic regression with 5 | #regularization 6 | # J = LRCOSTFUNCTION(theta, X, y, lambda) computes the cost of using 7 | # theta as the parameter for regularized logistic regression and the 8 | # gradient of the cost w.r.t. to the parameters. 9 | def lrCostFunction(theta, X, y, _lambda): 10 | 11 | # Initialize some useful values 12 | m = len(y) # number of training examples 13 | 14 | theta = theta.reshape(-1,1) 15 | _theta = np.copy(theta) # !!! important, don't use reference 16 | _theta[0,0] = 0 17 | 18 | s = sigmoid( np.dot(X,theta) ) 19 | J = -( np.dot( y.T, np.log(s) ) + np.dot( (1-y).T, np.log(1-s) ) ) / m + _lambda * np.dot(_theta.T,_theta) / m/2 20 | 21 | grad = np.dot(X.T, ( s - y )) / m + _lambda * _theta / m 22 | 23 | return J[0], grad.reshape(1,-1)[0] -------------------------------------------------------------------------------- /ml-ex3/oneVsAll.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from lrCostFunction import lrCostFunction 4 | from scipy.optimize import minimize 5 | 6 | 7 | #ONEVSALL trains multiple logistic regression classifiers and returns all 8 | #the classifiers in a matrix all_theta, where the i-th row of all_theta 9 | #corresponds to the classifier for label i 10 | # [all_theta] = ONEVSALL(X, y, num_labels, lambda) trains num_labels 11 | # logisitc regression classifiers and returns each of these classifiers 12 | # in a matrix all_theta, where the i-th row of all_theta corresponds 13 | # to the classifier for label i 14 | def oneVsAll(X, y, num_labels, _lambda): 15 | 16 | # Some useful variables 17 | m, n = X.shape 18 | 19 | # You need to return the following variables correctly 20 | all_theta = np.zeros((num_labels, n + 1)) 21 | 22 | # Add ones to the X data matrix 23 | X = np.vstack((np.ones(m), X.T)).T 24 | y = y.reshape(-1,1) 25 | 26 | initial_theta = np.zeros(n + 1) 27 | 28 | for c in range(num_labels): 29 | 30 | res = minimize(lrCostFunction, initial_theta, method='CG', jac=True, options={'maxiter': 50}, args=(X, y==c, _lambda)) 31 | all_theta[c] = res.x 32 | 33 | return all_theta -------------------------------------------------------------------------------- /ml-ex3/predict.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #PREDICT Predict the label of an input given a trained neural network 5 | # p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the 6 | # trained weights of a neural network (Theta1, Theta2) 7 | def predict(Theta1, Theta2, X): 8 | 9 | # Useful values 10 | m = X.shape[0] 11 | num_labels = Theta2.shape[0] 12 | 13 | a1 = np.vstack((np.ones(m), X.T)).T 14 | 15 | z2 = np.dot(a1, Theta1.T) 16 | a2 = sigmoid(z2) 17 | 18 | a2 = np.vstack((np.ones(m), a2.T)).T 19 | 20 | z3 = np.dot(a2, Theta2.T) 21 | a3 = sigmoid(z3) 22 | 23 | return np.argmax(a3, axis=1) -------------------------------------------------------------------------------- /ml-ex3/predictOneVsAll.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #PREDICT Predict the label for a trained one-vs-all classifier. The labels 5 | #are in the range 1..K, where K = size(all_theta, 1). 6 | # p = PREDICTONEVSALL(all_theta, X) will return a vector of predictions 7 | # for each example in the matrix X. Note that X contains the examples in 8 | # rows. all_theta is a matrix where the i-th row is a trained logistic 9 | # regression theta vector for the i-th class. You should set p to a vector 10 | # of values from 1..K (e.g., p = [1; 3; 1; 2] predicts classes 1, 3, 1, 2 11 | # for 4 examples) 12 | def predictOneVsAll(all_theta, X): 13 | 14 | m = X.shape[0] # Number of training examples 15 | 16 | # Add ones to the X data matrix 17 | X = np.vstack((np.ones(m), X.T)).T 18 | 19 | return np.argmax(sigmoid(np.dot(all_theta, X.T)), axis=0) -------------------------------------------------------------------------------- /ml-ex3/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #SIGMOID Compute sigmoid functoon 4 | # J = SIGMOID(z) computes the sigmoid of z. 5 | def sigmoid(z): 6 | 7 | return 1 / ( 1 + np.exp(-z) ) -------------------------------------------------------------------------------- /ml-ex4/checkNNGradients.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from debugInitializeWeights import debugInitializeWeights 3 | from computeNumericalGradient import computeNumericalGradient 4 | from nnCostFunction import nnCostFunction 5 | 6 | #CHECKNNGRADIENTS Creates a small neural network to check the 7 | #backpropagation gradients 8 | # CHECKNNGRADIENTS(lambda) Creates a small neural network to check the 9 | # backpropagation gradients, it will output the analytical gradients 10 | # produced by your backprop code and the numerical gradients (computed 11 | # using computeNumericalGradient). These two gradient computations should 12 | # result in very similar values. 13 | # 14 | def checkNNGradients(_lambda=None): 15 | 16 | if _lambda == None: 17 | _lambda = 0 18 | 19 | input_layer_size = 3 20 | hidden_layer_size = 5 21 | num_labels = 3 22 | m = 5 23 | 24 | # We generate some 'random' test data 25 | Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size) 26 | Theta2 = debugInitializeWeights(num_labels, hidden_layer_size) 27 | # Reusing debugInitializeWeights to generate X 28 | X = debugInitializeWeights(m, input_layer_size - 1) 29 | y = (1 + np.arange(m)) % num_labels 30 | y = y.reshape(-1,1) 31 | 32 | # Unroll parameters 33 | nn_params = np.append(Theta1.flatten(),Theta2.flatten()) 34 | 35 | # Short hand for cost function 36 | costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size, \ 37 | num_labels, X, y, _lambda) 38 | 39 | cost, grad = costFunc(nn_params) 40 | numgrad = computeNumericalGradient(costFunc, nn_params) 41 | 42 | # Visually examine the two gradient computations. The two columns 43 | # you get should be very similar. 44 | print(grad) 45 | print(numgrad) 46 | print('The above two columns you get should be very similar.\n \ 47 | (Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n') 48 | 49 | # Evaluate the norm of the difference between two solutions. 50 | # If you have a correct implementation, and assuming you used EPSILON = 0.0001 51 | # in computeNumericalGradient.m, then diff below should be less than 1e-9 52 | diff = np.linalg.norm(numgrad-grad)/np.linalg.norm(numgrad+grad) 53 | 54 | print('If your backpropagation implementation is correct, then \n \ 55 | the relative difference will be small (less than 1e-9). \n \ 56 | \nRelative Difference: %g\n'%diff) -------------------------------------------------------------------------------- /ml-ex4/computeNumericalGradient.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #COMPUTENUMERICALGRADIENT Computes the gradient using "finite differences" 4 | #and gives us a numerical estimate of the gradient. 5 | # numgrad = COMPUTENUMERICALGRADIENT(J, theta) computes the numerical 6 | # gradient of the function J around theta. Calling y = J(theta) should 7 | # return the function value at theta. 8 | 9 | # Notes: The following code implements numerical gradient checking, and 10 | # returns the numerical gradient.It sets numgrad(i) to (a numerical 11 | # approximation of) the partial derivative of J with respect to the 12 | # i-th input argument, evaluated at theta. (i.e., numgrad(i) should 13 | # be the (approximately) the partial derivative of J with respect 14 | # to theta(i).) 15 | # 16 | def computeNumericalGradient(J, theta): 17 | 18 | numgrad = np.zeros(theta.size) 19 | perturb = np.zeros(theta.size) 20 | e = 1e-4 21 | for p in range(theta.size): 22 | # Set perturbation vector 23 | perturb[p] = e 24 | loss1, _ = J(theta - perturb) 25 | loss2, _ = J(theta + perturb) 26 | # Compute Numerical Gradient 27 | numgrad[p] = (loss2 - loss1) / (2*e) 28 | perturb[p] = 0 29 | 30 | return numgrad -------------------------------------------------------------------------------- /ml-ex4/debugInitializeWeights.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #DEBUGINITIALIZEWEIGHTS Initialize the weights of a layer with fan_in 4 | #incoming connections and fan_out outgoing connections using a fixed 5 | #strategy, this will help you later in debugging 6 | # W = DEBUGINITIALIZEWEIGHTS(fan_in, fan_out) initializes the weights 7 | # of a layer with fan_in incoming connections and fan_out outgoing 8 | # connections using a fix set of values 9 | # 10 | # Note that W should be set to a matrix of size(1 + fan_in, fan_out) as 11 | # the first row of W handles the "bias" terms 12 | # 13 | def debugInitializeWeights(fan_out, fan_in): 14 | 15 | # Set W to zeros 16 | W = np.zeros((fan_out, 1 + fan_in)) 17 | 18 | # Initialize W using "sin", this ensures that W is always of the same 19 | # values and will be useful for debugging 20 | return np.sin(np.arange(W.size)+1).reshape(W.shape) / 10 -------------------------------------------------------------------------------- /ml-ex4/displayData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | #DISPLAYDATA Display 2D data in a nice grid 6 | # [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 7 | # stored in X in a nice grid. It returns the figure handle h and the 8 | # displayed array if requested. 9 | def displayData(X, example_width=None): 10 | 11 | # Set example_width automatically if not passed in 12 | if example_width == None: 13 | example_width = (int)(np.round(np.sqrt(X.shape[1]))) 14 | 15 | # Compute rows, cols 16 | m, n = X.shape 17 | example_height = (int)(n / example_width) 18 | 19 | # Compute number of items to display 20 | display_rows = (int)(np.floor(np.sqrt(m))) 21 | display_cols = (int)(np.ceil(m / display_rows)) 22 | 23 | # Between images padding 24 | pad = 1 25 | 26 | # Setup blank display 27 | display_array = - np.ones((pad + display_rows * (example_height + pad), \ 28 | pad + display_cols * (example_width + pad))) 29 | 30 | # Copy each example into a patch on the display array 31 | curr_ex = 0 32 | for j in range(display_rows): 33 | for i in range(display_cols): 34 | if curr_ex >= m: 35 | break 36 | 37 | # Copy the patch 38 | 39 | # Get the max value of the patch 40 | max_val = np.max(np.abs(X[curr_ex])) 41 | offset_height = pad + j * (example_height + pad) 42 | offset_width = pad + i * (example_width + pad) 43 | display_array[offset_height : offset_height+example_height,\ 44 | offset_width : offset_width+example_width] = \ 45 | X[curr_ex].reshape((example_height,example_width)).T / max_val 46 | curr_ex = curr_ex + 1 47 | 48 | if curr_ex >= m: 49 | break 50 | 51 | # Display Image 52 | plt.imshow(display_array, cmap='gray', vmin = -1, vmax = 1) 53 | 54 | # Do not show axis 55 | plt.axis('off') -------------------------------------------------------------------------------- /ml-ex4/ex4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex4/ex4.pdf -------------------------------------------------------------------------------- /ml-ex4/ex4.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class - Exercise 4 Neural Network Learning 2 | 3 | # Instructions 4 | # ------------ 5 | # 6 | # This file contains code that helps you get started on the 7 | # linear exercise. You will need to complete the following functions 8 | # in this exericse: 9 | # 10 | # sigmoidGradient.m 11 | # randInitializeWeights.m 12 | # nnCostFunction.m 13 | # 14 | # For this exercise, you will not need to change any code in this file, 15 | # or any other files other than those mentioned above. 16 | # 17 | 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | import scipy.io as sio 23 | from displayData import displayData 24 | from nnCostFunction import nnCostFunction 25 | from sigmoidGradient import sigmoidGradient 26 | from randInitializeWeights import randInitializeWeights 27 | from checkNNGradients import checkNNGradients 28 | from scipy.optimize import minimize 29 | from predict import predict 30 | 31 | plt.ion() 32 | 33 | ## Setup the parameters you will use for this exercise 34 | input_layer_size = 400 # 20x20 Input Images of Digits 35 | hidden_layer_size = 25 # 25 hidden units 36 | num_labels = 10 # 10 labels, from 1 to 10 37 | # (note that we have mapped "0" to label 10) 38 | 39 | ## =========== Part 1: Loading and Visualizing Data ============= 40 | # We start the exercise by first loading and visualizing the dataset. 41 | # You will be working with a dataset that contains handwritten digits. 42 | # 43 | 44 | # Load Training Data 45 | print('Loading and Visualizing Data ...\n') 46 | 47 | data = sio.loadmat('ex4data1.mat') 48 | X = data['X']; y = data['y']%10 49 | 50 | m = X.shape[0] 51 | 52 | # Randomly select 100 data points to display 53 | rand_indices = np.random.permutation(m) 54 | sel = X[rand_indices[0:100]] 55 | 56 | displayData(sel) 57 | 58 | input('Program paused. Press enter to continue.\n') 59 | 60 | 61 | ## ================ Part 2: Loading Parameters ================ 62 | # In this part of the exercise, we load some pre-initialized 63 | # neural network parameters. 64 | 65 | print('\nLoading Saved Neural Network Parameters ...\n') 66 | 67 | # Load the weights into variables Theta1 and Theta2 68 | data = sio.loadmat('ex4weights.mat') 69 | Theta1 = data['Theta1']; Theta2 = data['Theta2'] 70 | 71 | # Unroll parameters 72 | nn_params = np.append(Theta1, Theta2) 73 | 74 | ## ================ Part 3: Compute Cost (Feedforward) ================ 75 | # To the neural network, you should first start by implementing the 76 | # feedforward part of the neural network that returns the cost only. You 77 | # should complete the code in nnCostFunction.m to return cost. After 78 | # implementing the feedforward to compute the cost, you can verify that 79 | # your implementation is correct by verifying that you get the same cost 80 | # as us for the fixed debugging parameters. 81 | # 82 | # We suggest implementing the feedforward cost *without* regularization 83 | # first so that it will be easier for you to debug. Later, in part 4, you 84 | # will get to implement the regularized cost. 85 | # 86 | print('\nFeedforward Using Neural Network ...\n') 87 | 88 | # Weight regularization parameter (we set this to 0 here). 89 | _lambda = 0 90 | 91 | J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, \ 92 | num_labels, X, y, _lambda) 93 | 94 | print('Cost at parameters (loaded from ex4weights): %f \ 95 | \n(this value should be about 0.287629)\n'%(J)) 96 | 97 | input('\nProgram paused. Press enter to continue.\n') 98 | 99 | ## =============== Part 4: Implement Regularization =============== 100 | # Once your cost function implementation is correct, you should now 101 | # continue to implement the regularization with the cost. 102 | # 103 | 104 | print('\nChecking Cost Function (w/ Regularization) ... \n') 105 | 106 | # Weight regularization parameter (we set this to 1 here). 107 | _lambda = 1 108 | 109 | J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, \ 110 | num_labels, X, y, _lambda) 111 | 112 | print('Cost at parameters (loaded from ex4weights): %f \ 113 | \n(this value should be about 0.383770)\n'%(J)) 114 | 115 | input('\nProgram paused. Press enter to continue.\n') 116 | 117 | ## ================ Part 5: Sigmoid Gradient ================ 118 | # Before you start implementing the neural network, you will first 119 | # implement the gradient for the sigmoid function. You should complete the 120 | # code in the sigmoidGradient.m file. 121 | # 122 | 123 | print('\nEvaluating sigmoid gradient...\n') 124 | 125 | g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1])) 126 | print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]:\n ') 127 | print(g) 128 | print('\n\n') 129 | 130 | input('\nProgram paused. Press enter to continue.\n') 131 | 132 | ## ================ Part 6: Initializing Pameters ================ 133 | # In this part of the exercise, you will be starting to implment a two 134 | # layer neural network that classifies digits. You will start by 135 | # implementing a function to initialize the weights of the neural network 136 | # (randInitializeWeights.m) 137 | 138 | print('\nInitializing Neural Network Parameters ...\n') 139 | 140 | initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size) 141 | initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels) 142 | 143 | # Unroll parameters 144 | initial_nn_params = np.append(initial_Theta1.flatten(), initial_Theta2.flatten()) 145 | 146 | 147 | ## =============== Part 7: Implement Backpropagation =============== 148 | # Once your cost matches up with ours, you should proceed to implement the 149 | # backpropagation algorithm for the neural network. You should add to the 150 | # code you've written in nnCostFunction.m to return the partial 151 | # derivatives of the parameters. 152 | # 153 | print('\nChecking Backpropagation... \n') 154 | 155 | # Check gradients by running checkNNGradients 156 | checkNNGradients() 157 | 158 | input('\nProgram paused. Press enter to continue.\n') 159 | 160 | 161 | ## =============== Part 8: Implement Regularization =============== 162 | # Once your backpropagation implementation is correct, you should now 163 | # continue to implement the regularization with the cost and gradient. 164 | # 165 | 166 | print('\nChecking Backpropagation (w/ Regularization) ... \n') 167 | 168 | # Check gradients by running checkNNGradients 169 | _lambda = 3 170 | checkNNGradients(_lambda) 171 | 172 | # Also output the costFunction debugging values 173 | debug_J, _ = nnCostFunction(nn_params, input_layer_size, \ 174 | hidden_layer_size, num_labels, X, y, _lambda) 175 | 176 | print('\n\nCost at (fixed) debugging parameters (w/ lambda = 10): %f \ 177 | \n(this value should be about 0.576051)\n\n'%debug_J) 178 | 179 | input('\nProgram paused. Press enter to continue.\n') 180 | 181 | 182 | ## =================== Part 8: Training NN =================== 183 | # You have now implemented all the code necessary to train a neural 184 | # network. To train your neural network, we will now use "fmincg", which 185 | # is a function which works similarly to "fminunc". Recall that these 186 | # advanced optimizers are able to train our cost functions efficiently as 187 | # long as we provide them with the gradient computations. 188 | # 189 | print('\nTraining Neural Network... \n') 190 | 191 | # You should also try different values of lambda 192 | _lambda = 1 193 | 194 | # Create "short hand" for the cost function to be minimized 195 | costFunction = lambda p: nnCostFunction(p, \ 196 | input_layer_size, \ 197 | hidden_layer_size, \ 198 | num_labels, X, y, _lambda) 199 | 200 | # Now, costFunction is a function that takes in only one argument (the 201 | # neural network parameters) 202 | res = minimize(costFunction, initial_nn_params, method='CG', jac=True, options={'maxiter': 200}) 203 | nn_params = res.x 204 | 205 | # Obtain Theta1 and Theta2 back from nn_params 206 | Theta1 = nn_params[0:hidden_layer_size * (input_layer_size + 1)].reshape(\ 207 | hidden_layer_size, input_layer_size + 1) 208 | 209 | Theta2 = nn_params[hidden_layer_size * (input_layer_size + 1):].reshape(\ 210 | num_labels, hidden_layer_size + 1) 211 | 212 | input('\nProgram paused. Press enter to continue.\n') 213 | 214 | ## ================= Part 9: Visualize Weights ================= 215 | # You can now "visualize" what the neural network is learning by 216 | # displaying the hidden units to see what features they are capturing in 217 | # the data. 218 | 219 | print('\nVisualizing Neural Network... \n') 220 | 221 | displayData(Theta1[:,1:]) 222 | 223 | input('\nProgram paused. Press enter to continue.\n') 224 | 225 | ## ================= Part 10: Implement Predict ================= 226 | # After training the neural network, we would like to use it to predict 227 | # the labels. You will now implement the "predict" function to use the 228 | # neural network to predict the labels of the training set. This lets 229 | # you compute the training set accuracy. 230 | 231 | pred = (predict(Theta1, Theta2, X)+1)%10 232 | 233 | print('\nTraining Set Accuracy: %f\n'%(np.mean(np.double(pred == y.T)) * 100)) 234 | 235 | rp = np.random.permutation(m) 236 | 237 | for i in range(m): 238 | # Display 239 | print('\nDisplaying Example Image\n') 240 | t = np.array([X[rp[i]]]) 241 | displayData(t) 242 | 243 | pred = predict(Theta1, Theta2, t) 244 | print('\nNeural Network Prediction: %d (digit %d)\n'%(pred, (pred+1)%10)) 245 | 246 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex4/ex4data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex4/ex4data1.mat -------------------------------------------------------------------------------- /ml-ex4/ex4weights.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex4/ex4weights.mat -------------------------------------------------------------------------------- /ml-ex4/nnCostFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | from sigmoidGradient import sigmoidGradient 4 | 5 | #NNCOSTFUNCTION Implements the neural network cost function for a two layer 6 | #neural network which performs classification 7 | # [J grad] = NNCOSTFUNCTON(nn_params, hidden_layer_size, num_labels, ... 8 | # X, y, lambda) computes the cost and gradient of the neural network. The 9 | # parameters for the neural network are "unrolled" into the vector 10 | # nn_params and need to be converted back into the weight matrices. 11 | # 12 | # The returned parameter grad should be a "unrolled" vector of the 13 | # partial derivatives of the neural network. 14 | # 15 | def nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, _lambda): 16 | 17 | # Reshape nn_params back into the parameters Theta1 and Theta2, the weight matrices 18 | # for our 2 layer neural network 19 | Theta1 = nn_params[0:hidden_layer_size * (input_layer_size + 1)].reshape(\ 20 | hidden_layer_size, input_layer_size + 1) 21 | 22 | Theta2 = nn_params[hidden_layer_size * (input_layer_size + 1):].reshape(\ 23 | num_labels, hidden_layer_size + 1) 24 | 25 | # Setup some useful variables 26 | m = len(y) # number of training examples 27 | 28 | # ====================== YOUR CODE HERE ====================== 29 | # Instructions: You should complete the code by working through the 30 | # following parts. 31 | # 32 | # Part 1: Feedforward the neural network and return the cost in the 33 | # variable J. After implementing Part 1, you can verify that your 34 | # cost function computation is correct by verifying the cost 35 | # computed in ex4.m 36 | # 37 | # Part 2: Implement the backpropagation algorithm to compute the gradients 38 | # Theta1_grad and Theta2_grad. You should return the partial derivatives of 39 | # the cost function with respect to Theta1 and Theta2 in Theta1_grad and 40 | # Theta2_grad, respectively. After implementing Part 2, you can check 41 | # that your implementation is correct by running checkNNGradients 42 | # 43 | # Note: The vector y passed into the function is a vector of labels 44 | # containing values from 1..K. You need to map this vector into a 45 | # binary vector of 1's and 0's to be used with the neural network 46 | # cost function. 47 | # 48 | # Hint: We recommend implementing backpropagation using a for-loop 49 | # over the training examples if you are implementing it for the 50 | # first time. 51 | # 52 | # Part 3: Implement regularization with the cost function and gradients. 53 | # 54 | # Hint: You can implement this around the code for 55 | # backpropagation. That is, you can compute the gradients for 56 | # the regularization separately and then add them to Theta1_grad 57 | # and Theta2_grad from Part 2. 58 | 59 | a1 = np.vstack((np.ones(m), X.T)).T 60 | a2 = sigmoid(np.dot(a1, Theta1.T)) 61 | a2 = np.vstack((np.ones(m), a2.T)).T 62 | a3 = sigmoid(np.dot(a2, Theta2.T)) 63 | y = np.tile((np.arange(num_labels)+1)%10,(m,1)) == np.tile(y,(1,num_labels)) 64 | 65 | regTheta1 = Theta1[:,1:] 66 | regTheta2 = Theta2[:,1:] 67 | 68 | J = -np.sum( y * np.log(a3) + (1-y) * np.log(1-a3) ) / m + \ 69 | _lambda * np.sum(regTheta1*regTheta1) / m/2 + \ 70 | _lambda * np.sum(regTheta2*regTheta2) / m/2 71 | 72 | delta1 = np.zeros(Theta1.shape) 73 | delta2 = np.zeros(Theta2.shape) 74 | for i in range(m): 75 | a1_ = a1[i]; a2_ = a2[i]; a3_ = a3[i] 76 | d3 = a3_ - y[i]; d2 = np.dot(d3,Theta2) * sigmoidGradient(np.append(1,np.dot(a1_, Theta1.T))) 77 | delta1 = delta1 + np.dot(d2[1:].reshape(-1,1),a1_.reshape(1,-1)); 78 | delta2 = delta2 + np.dot(d3.reshape(-1,1), a2_.reshape(1,-1)) 79 | 80 | regTheta1 = np.vstack((np.zeros(Theta1.shape[0]), regTheta1.T)).T 81 | regTheta2 = np.vstack((np.zeros(Theta2.shape[0]), regTheta2.T)).T 82 | Theta1_grad = delta1 / m + _lambda * regTheta1 / m 83 | Theta2_grad = delta2 / m + _lambda * regTheta2 / m 84 | 85 | grad = np.append(Theta1_grad.flatten(), Theta2_grad.flatten()) 86 | print('cost value: %lf'%J) 87 | 88 | return J, grad -------------------------------------------------------------------------------- /ml-ex4/predict.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #PREDICT Predict the label of an input given a trained neural network 5 | # p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the 6 | # trained weights of a neural network (Theta1, Theta2) 7 | def predict(Theta1, Theta2, X): 8 | 9 | # Useful values 10 | m = X.shape[0] 11 | num_labels = Theta2.shape[0] 12 | 13 | a1 = np.vstack((np.ones(m), X.T)).T 14 | a2 = sigmoid(np.dot(a1, Theta1.T)) 15 | a2 = np.vstack((np.ones(m), a2.T)).T 16 | a3 = sigmoid(np.dot(a2, Theta2.T)) 17 | 18 | return np.argmax(a3, axis=1) -------------------------------------------------------------------------------- /ml-ex4/randInitializeWeights.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #RANDINITIALIZEWEIGHTS Randomly initialize the weights of a layer with L_in 4 | #incoming connections and L_out outgoing connections 5 | # W = RANDINITIALIZEWEIGHTS(L_in, L_out) randomly initializes the weights 6 | # of a layer with L_in incoming connections and L_out outgoing 7 | # connections. 8 | # 9 | # Note that W should be set to a matrix of size(L_out, 1 + L_in) as 10 | # the column row of W handles the "bias" terms 11 | # 12 | def randInitializeWeights(L_in, L_out): 13 | 14 | epsilon = 0.12 15 | return np.random.rand(L_out,L_in+1) * 2 * epsilon - epsilon -------------------------------------------------------------------------------- /ml-ex4/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #SIGMOID Compute sigmoid functoon 4 | # J = SIGMOID(z) computes the sigmoid of z. 5 | def sigmoid(z): 6 | 7 | return 1 / ( 1 + np.exp(-z) ) -------------------------------------------------------------------------------- /ml-ex4/sigmoidGradient.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | #SIGMOIDGRADIENT returns the gradient of the sigmoid function 5 | #evaluated at z 6 | # g = SIGMOIDGRADIENT(z) computes the gradient of the sigmoid function 7 | # evaluated at z. This should work regardless if z is a matrix or a 8 | # vector. In particular, if z is a vector or matrix, you should return 9 | # the gradient for each element. 10 | def sigmoidGradient(z): 11 | 12 | return sigmoid(z) * (1 - sigmoid(z)) -------------------------------------------------------------------------------- /ml-ex5/ex5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex5/ex5.pdf -------------------------------------------------------------------------------- /ml-ex5/ex5.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 5 | Regularized Linear Regression and Bias-Variance 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # linearRegCostFunction.py 11 | # learningCurve.py 12 | # validationCurve.py 13 | # 14 | # For this exercise, you will not need to change any code in this file, 15 | # or any other files other than those mentioned above. 16 | # 17 | 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | import scipy.io as sio 23 | from linearRegCostFunction import linearRegCostFunction 24 | from trainLinearReg import trainLinearReg 25 | from learningCurve import learningCurve 26 | from polyFeatures import polyFeatures 27 | from featureNormalize import featureNormalize 28 | from plotFit import plotFit 29 | from validationCurve import validationCurve 30 | 31 | plt.ion() 32 | 33 | ## =========== Part 1: Loading and Visualizing Data ============= 34 | # We start the exercise by first loading and visualizing the dataset. 35 | # The following code will load the dataset into your environment and plot 36 | # the data. 37 | # 38 | 39 | # Load Training Data 40 | print('Loading and Visualizing Data ...\n') 41 | 42 | # Load from ex5data1: 43 | # You will have X, y, Xval, yval, Xtest, ytest in your environment 44 | data = sio.loadmat('ex5data1.mat') 45 | X = data['X']; y = data['y'] 46 | Xval = data['Xval']; yval = data['yval'] 47 | Xtest = data['Xtest']; ytest = data['ytest'] 48 | 49 | # m = Number of examples 50 | m = X.shape[0] 51 | _X = np.vstack((np.ones(m), X.T)).T 52 | _Xval = np.vstack((np.ones(Xval.shape[0]), Xval.T)).T 53 | _Xtest = np.vstack((np.ones(Xtest.shape[0]), Xtest.T)).T 54 | 55 | # Plot training data 56 | plt.figure() 57 | plt.plot(X, y, 'rx', ms=5, linewidth=1.5) 58 | plt.xlabel('Change in water level (x)') 59 | plt.ylabel('Water flowing out of the dam (y)') 60 | 61 | input('Program paused. Press enter to continue.\n') 62 | 63 | ## =========== Part 2: Regularized Linear Regression Cost ============= 64 | # You should now implement the cost function for regularized linear 65 | # regression. 66 | # 67 | 68 | theta = np.ones(2) 69 | J, _ = linearRegCostFunction(_X, y, theta, 1) 70 | 71 | print('Cost at theta = [1 ; 1]: %f ' \ 72 | '\n(this value should be about 303.993192)\n'%J) 73 | 74 | input('Program paused. Press enter to continue.\n') 75 | 76 | ## =========== Part 3: Regularized Linear Regression Gradient ============= 77 | # You should now implement the gradient for regularized linear 78 | # regression. 79 | # 80 | 81 | theta = np.ones(2) 82 | J, grad = linearRegCostFunction(_X, y, theta, 1) 83 | print(grad) 84 | 85 | print('Gradient at theta = [1 ; 1]: [%f; %f] ' \ 86 | '\n(this value should be about [-15.303016; 598.250744])\n'%(grad[0], grad[1])) 87 | 88 | input('Program paused. Press enter to continue.\n') 89 | 90 | 91 | ## =========== Part 4: Train Linear Regression ============= 92 | # Once you have implemented the cost and gradient correctly, the 93 | # trainLinearReg function will use your cost function to train 94 | # regularized linear regression. 95 | # 96 | # Write Up Note: The data is non-linear, so this will not give a great 97 | # fit. 98 | # 99 | 100 | # Train linear regression with lambda = 0 101 | _lambda = 0 102 | theta = trainLinearReg(_X, y, _lambda) 103 | 104 | # Plot fit over the data 105 | plt.plot(X, np.dot(_X, theta), '--', linewidth=2) 106 | 107 | input('Program paused. Press enter to continue.\n') 108 | 109 | 110 | ## =========== Part 5: Learning Curve for Linear Regression ============= 111 | # Next, you should implement the learningCurve function. 112 | # 113 | # Write Up Note: Since the model is underfitting the data, we expect to 114 | # see a graph with "high bias" -- slide 8 in ML-advice.pdf 115 | # 116 | 117 | _lambda = 0 118 | error_train, error_val = learningCurve(_X, y, _Xval, yval, _lambda) 119 | 120 | plt.figure() 121 | plt.plot(np.arange(m), error_train, np.arange(m), error_val) 122 | plt.title('Learning curve for linear regression') 123 | plt.legend(['Train', 'Cross Validation']) 124 | plt.xlabel('Number of training examples') 125 | plt.ylabel('Error') 126 | 127 | print('# Training Examples\tTrain Error\tCross Validation Error\n') 128 | for i in range(m): 129 | print(' \t%d\t\t%f\t%f\n'%(i, error_train[i], error_val[i])) 130 | 131 | input('Program paused. Press enter to continue.\n') 132 | 133 | ## =========== Part 6: Feature Mapping for Polynomial Regression ============= 134 | # One solution to this is to use polynomial regression. You should now 135 | # complete polyFeatures to map each example into its powers 136 | # 137 | 138 | p = 8 139 | 140 | # Map X onto Polynomial Features and Normalize 141 | X_poly = polyFeatures(X, p) 142 | X_poly, mu, sigma = featureNormalize(X_poly) # Normalize 143 | X_poly = np.vstack((np.ones(m), X_poly.T)).T # Add Ones 144 | 145 | # Map X_poly_test and normalize (using mu and sigma) 146 | X_poly_test = polyFeatures(Xtest, p) 147 | X_poly_test = X_poly_test - mu 148 | X_poly_test = X_poly_test / sigma 149 | X_poly_test = np.vstack((np.ones(X_poly_test.shape[0]), X_poly_test.T)).T 150 | 151 | # Map X_poly_val and normalize (using mu and sigma) 152 | X_poly_val = polyFeatures(Xval, p) 153 | X_poly_val = X_poly_val - mu 154 | X_poly_val = X_poly_val / sigma 155 | X_poly_val = np.vstack((np.ones(X_poly_val.shape[0]), X_poly_val.T)).T 156 | 157 | print('Normalized Training Example 1:\n') 158 | print(X_poly[0,:]) 159 | 160 | print('\nProgram paused. Press enter to continue.\n') 161 | 162 | ## =========== Part 7: Learning Curve for Polynomial Regression ============= 163 | # Now, you will get to experiment with polynomial regression with multiple 164 | # values of lambda. The code below runs polynomial regression with 165 | # lambda = 0. You should try running the code with different values of 166 | # lambda to see how the fit and learning curve change. 167 | # 168 | 169 | _lambda = 0 170 | theta = trainLinearReg(X_poly, y, _lambda) 171 | 172 | # Plot training data and fit 173 | plt.figure() 174 | plt.plot(X, y, 'rx', ms=5, linewidth=1.5) 175 | plotFit(np.min(X), np.max(X), mu, sigma, theta, p) 176 | plt.xlabel('Change in water level (x)') 177 | plt.ylabel('Water flowing out of the dam (y)') 178 | plt.title('Polynomial Regression Fit (lambda = %f)'%_lambda) 179 | 180 | plt.figure() 181 | error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, _lambda) 182 | plt.plot(np.arange(m), error_train, np.arange(m), error_val) 183 | 184 | plt.title('Polynomial Regression Learning Curve (lambda = %f)'%_lambda) 185 | plt.xlabel('Number of training examples') 186 | plt.ylabel('Error') 187 | plt.axis([0, 13, 0, 100]) 188 | plt.legend(['Train', 'Cross Validation']) 189 | 190 | print('Polynomial Regression (lambda = %f)\n\n'%_lambda) 191 | print('# Training Examples\tTrain Error\tCross Validation Error\n') 192 | for i in range(m): 193 | print(' \t%d\t\t%f\t%f\n'%(i, error_train[i], error_val[i])) 194 | 195 | input('Program paused. Press enter to continue.\n') 196 | 197 | ## =========== Part 8: Validation for Selecting Lambda ============= 198 | # You will now implement validationCurve to test various values of 199 | # lambda on a validation set. You will then use this to select the 200 | # "best" lambda value. 201 | # 202 | 203 | lambda_vec, error_train, error_val = validationCurve(X_poly, y, X_poly_val, yval) 204 | 205 | plt.figure() 206 | plt.plot(lambda_vec, error_train, lambda_vec, error_val) 207 | plt.legend(['Train', 'Cross Validation']) 208 | plt.xlabel('lambda') 209 | plt.ylabel('Error') 210 | 211 | print('lambda\t\tTrain Error\tValidation Error\n') 212 | for i in range(lambda_vec.shape[0]): 213 | print(' %f\t%f\t%f\n'%(lambda_vec[i], error_train[i], error_val[i])) 214 | 215 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex5/ex5data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex5/ex5data1.mat -------------------------------------------------------------------------------- /ml-ex5/featureNormalize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #FEATURENORMALIZE Normalizes the features in X 4 | # FEATURENORMALIZE(X) returns a normalized version of X where 5 | # the mean value of each feature is 0 and the standard deviation 6 | # is 1. This is often a good preprocessing step to do when 7 | # working with learning algorithms. 8 | def featureNormalize(X): 9 | 10 | mu = np.mean(X, axis=0) 11 | sigma = np.std(X, axis=0) 12 | 13 | return (X-mu)/sigma, mu, sigma -------------------------------------------------------------------------------- /ml-ex5/learningCurve.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from trainLinearReg import trainLinearReg 3 | from linearRegCostFunction import linearRegCostFunction 4 | 5 | #LEARNINGCURVE Generates the train and cross validation set errors needed 6 | #to plot a learning curve 7 | # [error_train, error_val] = ... 8 | # LEARNINGCURVE(X, y, Xval, yval, lambda) returns the train and 9 | # cross validation set errors for a learning curve. In particular, 10 | # it returns two vectors of the same length - error_train and 11 | # error_val. Then, error_train(i) contains the training error for 12 | # i examples (and similarly for error_val(i)). 13 | # 14 | # In this function, you will compute the train and test errors for 15 | # dataset sizes from 1 up to m. In practice, when working with larger 16 | # datasets, you might want to do this in larger intervals. 17 | # 18 | def learningCurve(X, y, Xval, yval, _lambda): 19 | 20 | # Number of training examples 21 | m = X.shape[0] 22 | 23 | # You need to return these values correctly 24 | error_train = np.zeros(m) 25 | error_val = np.zeros(m) 26 | 27 | for i in range(m): 28 | Xtrain = X[0:i+1] 29 | ytrain = y[0:i+1] 30 | theta = trainLinearReg(Xtrain, ytrain, _lambda) 31 | error_train[i], _ = linearRegCostFunction(Xtrain, ytrain, theta, 0) 32 | error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) 33 | 34 | return error_train, error_val 35 | -------------------------------------------------------------------------------- /ml-ex5/linearRegCostFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #LINEARREGCOSTFUNCTION Compute cost and gradient for regularized linear 4 | #regression with multiple variables 5 | # [J, grad] = LINEARREGCOSTFUNCTION(X, y, theta, lambda) computes the 6 | # cost of using theta as the parameter for linear regression to fit the 7 | # data points in X and y. Returns the cost in J and the gradient in grad 8 | def linearRegCostFunction(X, y, theta, _lambda): 9 | 10 | # Initialize some useful values 11 | m = len(y) # number of training examples 12 | 13 | theta = theta.reshape(-1,1) 14 | _theta = np.copy(theta) # !!! important, don't use reference 15 | _theta[0,0] = 0 16 | 17 | error = np.dot(X,theta) - y 18 | J = np.dot(error.T, error) / m/2 + _lambda * np.dot(_theta.T, _theta) /m/2 19 | grad = np.dot(X.T, error) / m + _lambda * _theta / m 20 | 21 | return J, grad.reshape(1,-1)[0] 22 | -------------------------------------------------------------------------------- /ml-ex5/plotFit.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from polyFeatures import polyFeatures 3 | import matplotlib.pyplot as plt 4 | 5 | #PLOTFIT Plots a learned polynomial regression fit over an existing figure. 6 | #Also works with linear regression. 7 | # PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial 8 | # fit with power p and feature normalization (mu, sigma). 9 | def plotFit(min_x, max_x, mu, sigma, theta, p): 10 | 11 | # We plot a range slightly bigger than the min and max values to get 12 | # an idea of how the fit will vary outside the range of the data points 13 | x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1,1) 14 | 15 | # Map the X values 16 | X_poly = polyFeatures(x, p) 17 | X_poly = X_poly - mu 18 | X_poly = X_poly / sigma 19 | 20 | # Add ones 21 | m = X_poly.shape[0] 22 | X_poly = np.vstack((np.ones(m), X_poly.T)).T 23 | 24 | # Plot 25 | theta = theta.reshape(-1,1) 26 | plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2) 27 | plt.axis([-70, 70, -50, 50]) -------------------------------------------------------------------------------- /ml-ex5/polyFeatures.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from linearRegCostFunction import linearRegCostFunction 3 | from scipy.optimize import minimize 4 | 5 | #POLYFEATURES Maps X (1D vector) into the p-th power 6 | # [X_poly] = POLYFEATURES(X, p) takes a data matrix X (size m x 1) and 7 | # maps each example into its polynomial features where 8 | # X_poly(i, :) = [X(i) X(i).^2 X(i).^3 ... X(i).^p]; 9 | # 10 | def polyFeatures(X, p): 11 | 12 | # You need to return the following variables correctly. 13 | X_poly = np.zeros((X.shape[0], p)) 14 | 15 | for i in range(p): 16 | X_poly[:,i] = np.power(X, i+1).reshape(1,-1) 17 | 18 | print(X_poly) 19 | 20 | return X_poly -------------------------------------------------------------------------------- /ml-ex5/trainLinearReg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from linearRegCostFunction import linearRegCostFunction 3 | from scipy.optimize import minimize 4 | 5 | #TRAINLINEARREG Trains linear regression given a dataset (X, y) and a 6 | #regularization parameter lambda 7 | # [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using 8 | # the dataset (X, y) and regularization parameter lambda. Returns the 9 | # trained parameters theta. 10 | # 11 | def trainLinearReg(X, y, _lambda): 12 | 13 | # Initialize Theta 14 | initial_theta = np.zeros(X.shape[1]) 15 | 16 | # Create "short hand" for the cost function to be minimized 17 | costFunction = lambda t: linearRegCostFunction(X, y, t, _lambda) 18 | 19 | res = minimize(costFunction, initial_theta, method='BFGS', jac=True, tol=1e-2) 20 | print(res) 21 | 22 | return res.x -------------------------------------------------------------------------------- /ml-ex5/validationCurve.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from trainLinearReg import trainLinearReg 3 | from linearRegCostFunction import linearRegCostFunction 4 | 5 | #VALIDATIONCURVE Generate the train and validation errors needed to 6 | #plot a validation curve that we can use to select lambda 7 | # [lambda_vec, error_train, error_val] = ... 8 | # VALIDATIONCURVE(X, y, Xval, yval) returns the train 9 | # and validation errors (in error_train, error_val) 10 | # for different values of lambda. You are given the training set (X, 11 | # y) and validation set (Xval, yval). 12 | # 13 | def validationCurve(X, y, Xval, yval): 14 | 15 | # Selected values of lambda (you should not change this) 16 | lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1,1) 17 | 18 | # You need to return these variables correctly. 19 | error_train = np.zeros(lambda_vec.shape[0]) 20 | error_val = np.zeros(lambda_vec.shape[0]) 21 | 22 | for i in range(lambda_vec.shape[0]): 23 | _lambda = lambda_vec[i] 24 | theta = trainLinearReg(X, y, _lambda) 25 | error_train[i], _ = linearRegCostFunction(X, y, theta, 0) 26 | error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) 27 | 28 | return lambda_vec, error_train, error_val 29 | -------------------------------------------------------------------------------- /ml-ex6/cs229-notes3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/cs229-notes3.pdf -------------------------------------------------------------------------------- /ml-ex6/dataset3Params.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from plotData import plotData 4 | from svmTrain import svmTrain 5 | from gaussianKernel import gaussianKernel 6 | from svmPredict import svmPredict 7 | 8 | #EX6PARAMS returns your choice of C and sigma for Part 3 of the exercise 9 | #where you select the optimal (C, sigma) learning parameters to use for SVM 10 | #with RBF kernel 11 | # [C, sigma] = EX6PARAMS(X, y, Xval, yval) returns your choice of C and 12 | # sigma. You should complete this function to return the optimal C and 13 | # sigma based on a cross-validation set. 14 | # 15 | def dataset3Params(X, y, Xval, yval): 16 | 17 | choice = np.array([0.01,0.03,0.1,0.3,1,3,10,30]).reshape(-1,1) 18 | minError = np.inf 19 | curC = np.inf 20 | cur_sigma = np.inf 21 | 22 | for i in range(choice.shape[0]): 23 | for j in range(choice.shape[0]): 24 | func = lambda a, b: gaussianKernel(a, b, choice[j]) 25 | func.__name__ = 'gaussianKernel' 26 | model = svmTrain(X, y, choice[i], func) 27 | predictions = svmPredict(model, Xval) 28 | error = np.mean(np.double(np.not_equal(predictions,yval))) 29 | if error < minError: 30 | minError = error 31 | curC = choice[i] 32 | cur_sigma = choice[j] 33 | 34 | 35 | C = curC 36 | sigma = cur_sigma 37 | 38 | return C, sigma 39 | -------------------------------------------------------------------------------- /ml-ex6/emailFeatures.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #EMAILFEATURES takes in a word_indices vector and produces a feature vector 4 | #from the word indices 5 | # x = EMAILFEATURES(word_indices) takes in a word_indices vector and 6 | # produces a feature vector from the word indices. 7 | def emailFeatures(word_indices): 8 | 9 | # Total number of words in the dictionary 10 | n = 1899 11 | 12 | x = np.zeros(n).reshape(-1,1) 13 | 14 | for i in word_indices: 15 | x[i] = 1 16 | 17 | return x -------------------------------------------------------------------------------- /ml-ex6/emailSample1.txt: -------------------------------------------------------------------------------- 1 | > Anyone knows how much it costs to host a web portal ? 2 | > 3 | Well, it depends on how many visitors you're expecting. 4 | This can be anywhere from less than 10 bucks a month to a couple of $100. 5 | You should checkout http://www.rackspace.com/ or perhaps Amazon EC2 6 | if youre running something big.. 7 | 8 | To unsubscribe yourself from this mailing list, send an email to: 9 | groupname-unsubscribe@egroups.com 10 | 11 | -------------------------------------------------------------------------------- /ml-ex6/emailSample2.txt: -------------------------------------------------------------------------------- 1 | Folks, 2 | 3 | my first time posting - have a bit of Unix experience, but am new to Linux. 4 | 5 | 6 | Just got a new PC at home - Dell box with Windows XP. Added a second hard disk 7 | for Linux. Partitioned the disk and have installed Suse 7.2 from CD, which went 8 | fine except it didn't pick up my monitor. 9 | 10 | I have a Dell branded E151FPp 15" LCD flat panel monitor and a nVidia GeForce4 11 | Ti4200 video card, both of which are probably too new to feature in Suse's default 12 | set. I downloaded a driver from the nVidia website and installed it using RPM. 13 | Then I ran Sax2 (as was recommended in some postings I found on the net), but 14 | it still doesn't feature my video card in the available list. What next? 15 | 16 | Another problem. I have a Dell branded keyboard and if I hit Caps-Lock twice, 17 | the whole machine crashes (in Linux, not Windows) - even the on/off switch is 18 | inactive, leaving me to reach for the power cable instead. 19 | 20 | If anyone can help me in any way with these probs., I'd be really grateful - 21 | I've searched the 'net but have run out of ideas. 22 | 23 | Or should I be going for a different version of Linux such as RedHat? Opinions 24 | welcome. 25 | 26 | Thanks a lot, 27 | Peter 28 | 29 | -- 30 | Irish Linux Users' Group: ilug@linux.ie 31 | http://www.linux.ie/mailman/listinfo/ilug for (un)subscription information. 32 | List maintainer: listmaster@linux.ie 33 | 34 | 35 | -------------------------------------------------------------------------------- /ml-ex6/ex6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/ex6.pdf -------------------------------------------------------------------------------- /ml-ex6/ex6.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 6 | Support Vector Machines 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # gaussianKernel.py 11 | # dataset3Params.py 12 | # processEmail.py 13 | # emailFeatures.py 14 | # 15 | # For this exercise, you will not need to change any code in this file, 16 | # or any other files other than those mentioned above. 17 | # 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | import scipy.io as sio 23 | from plotData import plotData 24 | from linearKernel import linearKernel 25 | from gaussianKernel import gaussianKernel 26 | from svmTrain import svmTrain 27 | from visualizeBoundaryLinear import visualizeBoundaryLinear 28 | from visualizeBoundary import visualizeBoundary 29 | from dataset3Params import dataset3Params 30 | 31 | plt.ion() 32 | 33 | ## =============== Part 1: Loading and Visualizing Data ================ 34 | # We start the exercise by first loading and visualizing the dataset. 35 | # The following code will load the dataset into your environment and plot 36 | # the data. 37 | # 38 | 39 | print('Loading and Visualizing Data ...\n') 40 | 41 | # Load from ex6data1: 42 | # You will have X, y in your environment 43 | data = sio.loadmat('ex6data1.mat') 44 | X = data['X']; y = data['y'] 45 | # Plot training data 46 | plotData(X, y.T[0]) 47 | 48 | input('\nProgram paused. Press enter to continue.\n') 49 | 50 | ## ==================== Part 2: Training Linear SVM ==================== 51 | # The following code will train a linear SVM on the dataset and plot the 52 | # decision boundary learned. 53 | # 54 | 55 | # Load from ex6data1: 56 | # You will have X, y in your environment 57 | data = sio.loadmat('ex6data1.mat') 58 | X = data['X']; y = data['y'] 59 | y = y.astype(int) 60 | 61 | print('\nTraining Linear SVM ...\n') 62 | 63 | # You should try to change the C value below and see how the decision 64 | # boundary varies (e.g., try C = 1000) 65 | C = 1 66 | model = svmTrain(X, y, C, linearKernel, 1e-3, 20) 67 | print(model) 68 | visualizeBoundaryLinear(X, y, model) 69 | 70 | input('Program paused. Press enter to continue.\n') 71 | 72 | ## =============== Part 3: Implementing Gaussian Kernel =============== 73 | # You will now implement the Gaussian kernel to use 74 | # with the SVM. You should complete the code in gaussianKernel.m 75 | # 76 | print('\nEvaluating the Gaussian Kernel ...\n') 77 | 78 | x1 = np.array([1,2,1]); x2 = np.array([0,4,-1]); sigma = 2 79 | sim = gaussianKernel(x1, x2, sigma) 80 | 81 | print('Gaussian Kernel between x1 = [1; 2; 1], x2 = [0; 4; -1], sigma = 0.5 :' \ 82 | '\n\t%lf\n(this value should be about 0.324652)\n', sim) 83 | 84 | input('Program paused. Press enter to continue.\n') 85 | 86 | ## =============== Part 4: Visualizing Dataset 2 ================ 87 | # The following code will load the next dataset into your environment and 88 | # plot the data. 89 | # 90 | 91 | print('Loading and Visualizing Data ...\n') 92 | 93 | # Load from ex6data2: 94 | # You will have X, y in your environment 95 | data = sio.loadmat('ex6data2.mat') 96 | X = data['X']; y = data['y'] 97 | 98 | # Plot training data 99 | plotData(X, y.T[0]) 100 | 101 | input('Program paused. Press enter to continue.\n') 102 | 103 | ## ========== Part 5: Training SVM with RBF Kernel (Dataset 2) ========== 104 | # After you have implemented the kernel, we can now use it to train the 105 | # SVM classifier. 106 | # 107 | print('\nTraining SVM with RBF Kernel (this may take 1 to 2 minutes) ...\n'); 108 | 109 | # Load from ex6data2: 110 | # You will have X, y in your environment 111 | data = sio.loadmat('ex6data2.mat') 112 | X = data['X']; y = data['y'] 113 | y = y.astype(int) 114 | 115 | # SVM Parameters 116 | C = 1; sigma = 0.1 117 | 118 | # We set the tolerance and max_passes lower here so that the code will run 119 | # faster. However, in practice, you will want to run the training to 120 | # convergence. 121 | func = lambda a, b: gaussianKernel(a, b, sigma) 122 | func.__name__ = 'gaussianKernel' 123 | model = svmTrain(X, y, C, func) 124 | visualizeBoundary(X, y, model) 125 | 126 | input('Program paused. Press enter to continue.\n') 127 | 128 | ## =============== Part 6: Visualizing Dataset 3 ================ 129 | # The following code will load the next dataset into your environment and 130 | # plot the data. 131 | # 132 | 133 | print('Loading and Visualizing Data ...\n') 134 | 135 | # Load from ex6data3: 136 | # You will have X, y in your environment 137 | data = sio.loadmat('ex6data3.mat') 138 | X = data['X']; y = data['y'] 139 | 140 | # Plot training data 141 | plotData(X, y.T[0]) 142 | 143 | input('Program paused. Press enter to continue.\n') 144 | 145 | ## ========== Part 7: Training SVM with RBF Kernel (Dataset 3) ========== 146 | 147 | # This is a different dataset that you can use to experiment with. Try 148 | # different values of C and sigma here. 149 | # 150 | 151 | # Load from ex6data3: 152 | # You will have X, y in your environment 153 | data = sio.loadmat('ex6data3.mat') 154 | X = data['X']; y = data['y'] 155 | Xval = data['Xval']; yval = data['yval'] 156 | y = y.astype(int) 157 | yval = yval.astype(int) 158 | 159 | # Try different SVM Parameters here 160 | C, sigma = dataset3Params(X, y, Xval, yval) 161 | 162 | # Train the SVM 163 | func = lambda a, b: gaussianKernel(a, b, sigma) 164 | model = svmTrain(X, y, C, func) 165 | visualizeBoundary(X, y, model) 166 | 167 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex6/ex6_spam.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 6 | Spam Classification with SVMs 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # gaussianKernel.py 11 | # dataset3Params.py 12 | # processEmail.py 13 | # emailFeatures.py 14 | # 15 | # For this exercise, you will not need to change any code in this file, 16 | # or any other files other than those mentioned above. 17 | # 18 | 19 | ## Initialization 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | import scipy.io as sio 23 | from readFile import readFile 24 | from processEmail import processEmail 25 | from emailFeatures import emailFeatures 26 | from svmTrain import svmTrain 27 | from svmPredict import svmPredict 28 | from linearKernel import linearKernel 29 | from getVocabList import getVocabList 30 | 31 | plt.ion() 32 | 33 | ## ==================== Part 1: Email Preprocessing ==================== 34 | # To use an SVM to classify emails into Spam v.s. Non-Spam, you first need 35 | # to convert each email into a vector of features. In this part, you will 36 | # implement the preprocessing steps for each email. You should 37 | # complete the code in processEmail.m to produce a word indices vector 38 | # for a given email. 39 | 40 | print('\nPreprocessing sample email (emailSample1.txt)\n') 41 | 42 | # Extract Features 43 | file_contents = readFile('emailSample1.txt') 44 | word_indices = processEmail(file_contents) 45 | 46 | # Print Stats 47 | print('Word Indices: \n') 48 | print(word_indices) 49 | print('\n\n') 50 | 51 | input('Program paused. Press enter to continue.\n') 52 | 53 | ## ==================== Part 2: Feature Extraction ==================== 54 | # Now, you will convert each email into a vector of features in R^n. 55 | # You should complete the code in emailFeatures.m to produce a feature 56 | # vector for a given email. 57 | 58 | print('\nExtracting features from sample email (emailSample1.txt)\n') 59 | 60 | # Extract Features 61 | file_contents = readFile('emailSample1.txt') 62 | word_indices = processEmail(file_contents) 63 | features = emailFeatures(word_indices) 64 | 65 | # Print Stats 66 | print('Length of feature vector: %d\n'%len(features)) 67 | print('Number of non-zero entries: %d\n'%sum(features > 0)) 68 | 69 | input('Program paused. Press enter to continue.\n') 70 | 71 | ## =========== Part 3: Train Linear SVM for Spam Classification ======== 72 | # In this section, you will train a linear classifier to determine if an 73 | # email is Spam or Not-Spam. 74 | 75 | # Load the Spam Email dataset 76 | # You will have X, y in your environment 77 | data = sio.loadmat('spamTrain.mat') 78 | X = data['X']; y = data['y'] 79 | y = y.astype(int) 80 | 81 | print('\nTraining Linear SVM (Spam Classification)\n') 82 | print('(this may take 1 to 2 minutes) ...\n') 83 | 84 | C = 0.1 85 | model = svmTrain(X, y, C, linearKernel) 86 | 87 | p = svmPredict(model, X) 88 | 89 | print('Training Accuracy: %f\n'%(np.mean(np.double(p == y)) * 100)) 90 | 91 | ## =================== Part 4: Test Spam Classification ================ 92 | # After training the classifier, we can evaluate it on a test set. We have 93 | # included a test set in spamTest.mat 94 | 95 | # Load the test dataset 96 | # You will have Xtest, ytest in your environment 97 | data = sio.loadmat('spamTest.mat') 98 | Xtest = data['Xtest']; ytest = data['ytest'] 99 | 100 | print('\nEvaluating the trained Linear SVM on a test set ...\n') 101 | 102 | p = svmPredict(model, Xtest) 103 | 104 | print('Test Accuracy: %f\n'%(np.mean(np.double(p == ytest)) * 100)) 105 | input('Program paused. Press enter to continue.\n') 106 | 107 | ## ================= Part 5: Top Predictors of Spam ==================== 108 | # Since the model we are training is a linear SVM, we can inspect the 109 | # weights learned by the model to understand better how it is determining 110 | # whether an email is spam or not. The following code finds the words with 111 | # the highest weights in the classifier. Informally, the classifier 112 | # 'thinks' that these words are the most likely indicators of spam. 113 | # 114 | 115 | # Sort the weights and obtin the vocabulary list 116 | idx = np.argsort(-model['w'], axis=0) 117 | vocabList = getVocabList() 118 | 119 | print('\nTop predictors of spam: \n') 120 | for i in range(15): 121 | print(' %-15s (%f) \n'%(vocabList[idx[i][0]], model['w'][idx[i][0]])) 122 | 123 | print('\n\n') 124 | print('\nProgram paused. Press enter to continue.\n') 125 | input('Program paused. Press enter to continue.\n') 126 | 127 | ## =================== Part 6: Try Your Own Emails ===================== 128 | # Now that you've trained the spam classifier, you can use it on your own 129 | # emails! In the starter code, we have included spamSample1.txt, 130 | # spamSample2.txt, emailSample1.txt and emailSample2.txt as examples. 131 | # The following code reads in one of these emails and then uses your 132 | # learned SVM classifier to determine whether the email is Spam or 133 | # Not Spam 134 | 135 | # Set the file to be read in (change this to spamSample2.txt, 136 | # emailSample1.txt or emailSample2.txt to see different predictions on 137 | # different emails types). Try your own emails as well! 138 | filename = 'spamSample1.txt' 139 | 140 | # Read and predict 141 | file_contents = readFile(filename) 142 | print(file_contents) 143 | word_indices = processEmail(file_contents) 144 | x = emailFeatures(word_indices) 145 | p = svmPredict(model, x) 146 | 147 | print('\nProcessed %s\n\nSpam Classification: %d\n'%(filename, p)) 148 | print('(1 indicates spam, 0 indicates not spam)\n\n') -------------------------------------------------------------------------------- /ml-ex6/ex6data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/ex6data1.mat -------------------------------------------------------------------------------- /ml-ex6/ex6data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/ex6data2.mat -------------------------------------------------------------------------------- /ml-ex6/ex6data3.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/ex6data3.mat -------------------------------------------------------------------------------- /ml-ex6/gaussianKernel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #RBFKERNEL returns a radial basis function kernel between x1 and x2 4 | # sim = gaussianKernel(x1, x2) returns a gaussian kernel between x1 and x2 5 | # and returns the value in sim 6 | def gaussianKernel(x1, x2, sigma): 7 | 8 | # Ensure that x1 and x2 are column vectors 9 | x1.reshape(-1,1) 10 | x2.reshape(-1,1) 11 | 12 | # Compute the kernel 13 | return np.exp(-np.sum((x1-x2)*(x1-x2))/2/sigma/sigma) 14 | -------------------------------------------------------------------------------- /ml-ex6/getVocabList.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #GETVOCABLIST reads the fixed vocabulary list in vocab.txt and returns a 4 | #cell array of the words 5 | # vocabList = GETVOCABLIST() reads the fixed vocabulary list in vocab.txt 6 | # and returns a cell array of the words in vocabList. 7 | def getVocabList(): 8 | 9 | ## Read the fixed vocabulary list 10 | f = open('vocab.txt', 'r') 11 | 12 | # Store all dictionary words in cell array vocab{} 13 | n = 1899 # Total number of words in the dictionary 14 | 15 | # For ease of implementation, we use a struct to map the strings => integers 16 | # In practice, you'll want to use some form of hashmap 17 | vocabList = [1]*n 18 | for i in range(n): 19 | # Word Index (can ignore since it will be = i) 20 | line = f.readline() 21 | vocabList[i] = line.split('\t')[1].strip() 22 | f.close() 23 | 24 | return vocabList -------------------------------------------------------------------------------- /ml-ex6/linearKernel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #LINEARKERNEL returns a linear kernel between x1 and x2 4 | # sim = linearKernel(x1, x2) returns a linear kernel between x1 and x2 5 | # and returns the value in sim 6 | def linearKernel(x1, x2): 7 | 8 | # Ensure that x1 and x2 are column vectors 9 | x1.reshape(1,-1) 10 | x2.reshape(1,-1) 11 | 12 | # Compute the kernel 13 | return np.dot(x1.T, x2) 14 | -------------------------------------------------------------------------------- /ml-ex6/plotData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | #PLOTDATA Plots the data points X and y into a new figure 5 | # PLOTDATA(x,y) plots the data points with + for the positive examples 6 | # and o for the negative examples. X is assumed to be a Mx2 matrix. 7 | def plotData(X, y): 8 | 9 | # Create New Figure 10 | plt.figure() 11 | # Find Indices of Positive and Negative Examples 12 | pos = np.where(y==1); neg = np.where(y==0) 13 | plt.plot(X[pos][:,0],X[pos][:,1], 'k+', linewidth=2, markersize=7) 14 | plt.plot(X[neg][:,0],X[neg][:,1], 'ko', markerfacecolor='y', markersize=7) 15 | -------------------------------------------------------------------------------- /ml-ex6/processEmail.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import re 3 | from getVocabList import getVocabList 4 | from stemmer import * 5 | 6 | #PROCESSEMAIL preprocesses a the body of an email and 7 | #returns a list of word_indices 8 | # word_indices = PROCESSEMAIL(email_contents) preprocesses 9 | # the body of an email and returns a list of indices of the 10 | # words contained in the email. 11 | # 12 | def processEmail(email_contents): 13 | 14 | # Load Vocabulary 15 | vocabList = getVocabList() 16 | 17 | # Init return value 18 | word_indices = [] 19 | 20 | # ========================== Preprocess Email =========================== 21 | 22 | # Find the Headers ( \n\n and remove ) 23 | # Uncomment the following lines if you are working with raw emails with the 24 | # full headers 25 | 26 | # Lower case 27 | email_contents = email_contents.lower() 28 | 29 | # Strip all HTML 30 | # Looks for any expression that starts with < and ends with > and replace 31 | # and does not have any < or > in the tag it with a space 32 | email_contents = re.sub(r"<[^<>]+>", " ", email_contents) 33 | 34 | # Handle Numbers 35 | # Look for one or more characters between 0-9 36 | email_contents = re.sub(r"[0-9]+", "number", email_contents) 37 | 38 | # Handle URLS 39 | # Look for strings starting with http:// or https:// 40 | email_contents = re.sub(r"(http|https)://[^\s]*", "httpaddr", email_contents) 41 | 42 | # Handle Email Addresses 43 | # Look for strings with @ in the middle 44 | email_contents = re.sub(r"[^\s]+@[^\s]+", "emailaddr", email_contents) 45 | 46 | # Handle $ sign 47 | email_contents = re.sub(r"[$]+", "dollar", email_contents) 48 | 49 | 50 | # ========================== Tokenize Email =========================== 51 | 52 | # Output the email to screen as well 53 | print('\n==== Processed Email ====\n\n') 54 | 55 | # Process file 56 | l = 0 57 | 58 | strs = re.split(r'[ `\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./<>?\n\t]', email_contents) 59 | 60 | p = PorterStemmer() 61 | for _str in strs: 62 | 63 | # Remove any non alphanumeric characters 64 | _str = re.sub(r"[^a-zA-Z0-9]", "", _str) 65 | 66 | # Stem the word 67 | # (the porterStemmer sometimes has issues, so we use a try catch block) 68 | _str = p.stem(_str) 69 | 70 | # Skip the word if it is too short 71 | if len(_str) < 1: 72 | continue 73 | 74 | for i in range(len(vocabList)): 75 | if _str == vocabList[i]: 76 | word_indices.append(i) 77 | break 78 | 79 | if l + len(_str) + 1 > 78: 80 | print('') 81 | l = 0 82 | 83 | print(_str+" ", end='') 84 | 85 | # Print footer 86 | print('\n\n=========================\n') 87 | return word_indices 88 | 89 | 90 | -------------------------------------------------------------------------------- /ml-ex6/readFile.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #READFILE reads a file and returns its entire contents 4 | # file_contents = READFILE(filename) reads a file and returns its entire 5 | # contents in file_contents 6 | # 7 | def readFile(filename): 8 | 9 | # Load File 10 | f = open(filename, 'r') 11 | if f: 12 | file_contents = f.read() 13 | f.close() 14 | else: 15 | file_contents = '' 16 | print('Unable to open %s\n', filename) 17 | 18 | return file_contents -------------------------------------------------------------------------------- /ml-ex6/smo-book.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/smo-book.pdf -------------------------------------------------------------------------------- /ml-ex6/smo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/smo.pdf -------------------------------------------------------------------------------- /ml-ex6/spamSample1.txt: -------------------------------------------------------------------------------- 1 | Do You Want To Make $1000 Or More Per Week? 2 | 3 | 4 | 5 | If you are a motivated and qualified individual - I 6 | will personally demonstrate to you a system that will 7 | make you $1,000 per week or more! This is NOT mlm. 8 | 9 | 10 | 11 | Call our 24 hour pre-recorded number to get the 12 | details. 13 | 14 | 15 | 16 | 000-456-789 17 | 18 | 19 | 20 | I need people who want to make serious money. Make 21 | the call and get the facts. 22 | 23 | Invest 2 minutes in yourself now! 24 | 25 | 26 | 27 | 000-456-789 28 | 29 | 30 | 31 | Looking forward to your call and I will introduce you 32 | to people like yourself who 33 | are currently making $10,000 plus per week! 34 | 35 | 36 | 37 | 000-456-789 38 | 39 | 40 | 41 | 3484lJGv6-241lEaN9080lRmS6-271WxHo7524qiyT5-438rjUv5615hQcf0-662eiDB9057dMtVl72 42 | 43 | -------------------------------------------------------------------------------- /ml-ex6/spamSample2.txt: -------------------------------------------------------------------------------- 1 | Best Buy Viagra Generic Online 2 | 3 | Viagra 100mg x 60 Pills $125, Free Pills & Reorder Discount, Top Selling 100% Quality & Satisfaction guaranteed! 4 | 5 | We accept VISA, Master & E-Check Payments, 90000+ Satisfied Customers! 6 | http://medphysitcstech.ru 7 | 8 | 9 | -------------------------------------------------------------------------------- /ml-ex6/spamTest.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/spamTest.mat -------------------------------------------------------------------------------- /ml-ex6/spamTrain.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex6/spamTrain.mat -------------------------------------------------------------------------------- /ml-ex6/stemmer.py: -------------------------------------------------------------------------------- 1 | class PorterStemmer: 2 | 3 | def isCons(self, letter): 4 | if letter == 'a' or letter == 'e' or letter == 'i' or letter == 'o' or letter == 'u': 5 | return False 6 | else: 7 | return True 8 | 9 | def isConsonant(self, word, i): 10 | letter = word[i] 11 | if self.isCons(letter): 12 | if letter == 'y' and self.isCons(word[i-1]): 13 | return False 14 | else: 15 | return True 16 | else: 17 | return False 18 | 19 | def isVowel(self, word, i): 20 | return not(self.isConsonant(word, i)) 21 | 22 | # *S 23 | def endsWith(self, stem, letter): 24 | if stem.endswith(letter): 25 | return True 26 | else: 27 | return False 28 | 29 | # *v* 30 | def containsVowel(self, stem): 31 | for i in stem: 32 | if not self.isCons(i): 33 | return True 34 | return False 35 | 36 | # *d 37 | def doubleCons(self, stem): 38 | if len(stem) >= 2: 39 | if self.isConsonant(stem, -1) and self.isConsonant(stem, -2): 40 | return True 41 | else: 42 | return False 43 | else: 44 | return False 45 | 46 | def getForm(self, word): 47 | form = [] 48 | formStr = '' 49 | for i in range(len(word)): 50 | if self.isConsonant(word, i): 51 | if i != 0: 52 | prev = form[-1] 53 | if prev != 'C': 54 | form.append('C') 55 | else: 56 | form.append('C') 57 | else: 58 | if i != 0: 59 | prev = form[-1] 60 | if prev != 'V': 61 | form.append('V') 62 | else: 63 | form.append('V') 64 | for j in form: 65 | formStr += j 66 | return formStr 67 | 68 | def getM(self, word): 69 | form = self.getForm(word) 70 | m = form.count('VC') 71 | return m 72 | 73 | # *o 74 | def cvc(self, word): 75 | if len(word) >= 3: 76 | f = -3 77 | s = -2 78 | t = -1 79 | third = word[t] 80 | if self.isConsonant(word, f) and self.isVowel(word, s) and self.isConsonant(word, t): 81 | if third != 'w' and third != 'x' and third != 'y': 82 | return True 83 | else: 84 | return False 85 | else: 86 | return False 87 | else: 88 | return False 89 | 90 | def replace(self, orig, rem, rep): 91 | result = orig.rfind(rem) 92 | base = orig[:result] 93 | replaced = base + rep 94 | return replaced 95 | 96 | def replaceM0(self, orig, rem, rep): 97 | result = orig.rfind(rem) 98 | base = orig[:result] 99 | if self.getM(base) > 0: 100 | replaced = base + rep 101 | return replaced 102 | else: 103 | return orig 104 | 105 | def replaceM1(self, orig, rem, rep): 106 | result = orig.rfind(rem) 107 | base = orig[:result] 108 | if self.getM(base) > 1: 109 | replaced = base + rep 110 | return replaced 111 | else: 112 | return orig 113 | 114 | def step1a(self, word): 115 | if word.endswith('sses'): 116 | word = self.replace(word, 'sses', 'ss') 117 | elif word.endswith('ies'): 118 | word = self.replace(word, 'ies', 'i') 119 | elif word.endswith('ss'): 120 | word = self.replace(word, 'ss', 'ss') 121 | elif word.endswith('s'): 122 | word = self.replace(word, 's', '') 123 | else: 124 | pass 125 | return word 126 | 127 | def step1b(self, word): 128 | flag = False 129 | if word.endswith('eed'): 130 | result = word.rfind('eed') 131 | base = word[:result] 132 | if self.getM(base) > 0: 133 | word = base 134 | word += 'ee' 135 | elif word.endswith('ed'): 136 | result = word.rfind('ed') 137 | base = word[:result] 138 | if self.containsVowel(base): 139 | word = base 140 | flag = True 141 | elif word.endswith('ing'): 142 | result = word.rfind('ing') 143 | base = word[:result] 144 | if self.containsVowel(base): 145 | word = base 146 | flag = True 147 | if flag: 148 | if word.endswith('at') or word.endswith('bl') or word.endswith('iz'): 149 | word += 'e' 150 | elif self.doubleCons(word) and not self.endsWith(word, 'l') and not self.endsWith(word, 's') and not self.endsWith(word, 'z'): 151 | word = word[:-1] 152 | elif self.getM(word) == 1 and self.cvc(word): 153 | word += 'e' 154 | else: 155 | pass 156 | else: 157 | pass 158 | return word 159 | 160 | def step1c(self, word): 161 | if word.endswith('y'): 162 | result = word.rfind('y') 163 | base = word[:result] 164 | if self.containsVowel(base): 165 | word = base 166 | word += 'i' 167 | return word 168 | 169 | def step2(self, word): 170 | if word.endswith('ational'): 171 | word = self.replaceM0(word, 'ational', 'ate') 172 | elif word.endswith('tional'): 173 | word = self.replaceM0(word, 'tional', 'tion') 174 | elif word.endswith('enci'): 175 | word = self.replaceM0(word, 'enci', 'ence') 176 | elif word.endswith('anci'): 177 | word = self.replaceM0(word, 'anci', 'ance') 178 | elif word.endswith('izer'): 179 | word = self.replaceM0(word, 'izer', 'ize') 180 | elif word.endswith('abli'): 181 | word = self.replaceM0(word, 'abli', 'able') 182 | elif word.endswith('alli'): 183 | word = self.replaceM0(word, 'alli', 'al') 184 | elif word.endswith('entli'): 185 | word = self.replaceM0(word, 'entli', 'ent') 186 | elif word.endswith('eli'): 187 | word = self.replaceM0(word, 'eli', 'e') 188 | elif word.endswith('ousli'): 189 | word = self.replaceM0(word, 'ousli', 'ous') 190 | elif word.endswith('ization'): 191 | word = self.replaceM0(word, 'ization', 'ize') 192 | elif word.endswith('ation'): 193 | word = self.replaceM0(word, 'ation', 'ate') 194 | elif word.endswith('ator'): 195 | word = self.replaceM0(word, 'ator', 'ate') 196 | elif word.endswith('alism'): 197 | word = self.replaceM0(word, 'alism', 'al') 198 | elif word.endswith('iveness'): 199 | word = self.replaceM0(word, 'iveness', 'ive') 200 | elif word.endswith('fulness'): 201 | word = self.replaceM0(word, 'fulness', 'ful') 202 | elif word.endswith('ousness'): 203 | word = self.replaceM0(word, 'ousness', 'ous') 204 | elif word.endswith('aliti'): 205 | word = self.replaceM0(word, 'aliti', 'al') 206 | elif word.endswith('iviti'): 207 | word = self.replaceM0(word, 'iviti', 'ive') 208 | elif word.endswith('biliti'): 209 | word = self.replaceM0(word, 'biliti', 'ble') 210 | return word 211 | 212 | def step3(self, word): 213 | if word.endswith('icate'): 214 | word = self.replaceM0(word, 'icate', 'ic') 215 | elif word.endswith('ative'): 216 | word = self.replaceM0(word, 'ative', '') 217 | elif word.endswith('alize'): 218 | word = self.replaceM0(word, 'alize', 'al') 219 | elif word.endswith('iciti'): 220 | word = self.replaceM0(word, 'iciti', 'ic') 221 | elif word.endswith('ful'): 222 | word = self.replaceM0(word, 'ful', '') 223 | elif word.endswith('ness'): 224 | word = self.replaceM0(word, 'ness', '') 225 | return word 226 | 227 | def step4(self, word): 228 | if word.endswith('al'): 229 | word = self.replaceM1(word, 'al', '') 230 | elif word.endswith('ance'): 231 | word = self.replaceM1(word, 'ance', '') 232 | elif word.endswith('ence'): 233 | word = self.replaceM1(word, 'ence', '') 234 | elif word.endswith('er'): 235 | word = self.replaceM1(word, 'er', '') 236 | elif word.endswith('ic'): 237 | word = self.replaceM1(word, 'ic', '') 238 | elif word.endswith('able'): 239 | word = self.replaceM1(word, 'able', '') 240 | elif word.endswith('ible'): 241 | word = self.replaceM1(word, 'ible', '') 242 | elif word.endswith('ant'): 243 | word = self.replaceM1(word, 'ant', '') 244 | elif word.endswith('ement'): 245 | word = self.replaceM1(word, 'ement', '') 246 | elif word.endswith('ment'): 247 | word = self.replaceM1(word, 'ment', '') 248 | elif word.endswith('ent'): 249 | word = self.replaceM1(word, 'ent', '') 250 | elif word.endswith('ou'): 251 | word = self.replaceM1(word, 'ou', '') 252 | elif word.endswith('ism'): 253 | word = self.replaceM1(word, 'ism', '') 254 | elif word.endswith('ate'): 255 | word = self.replaceM1(word, 'ate', '') 256 | elif word.endswith('iti'): 257 | word = self.replaceM1(word, 'iti', '') 258 | elif word.endswith('ous'): 259 | word = self.replaceM1(word, 'ous', '') 260 | elif word.endswith('ive'): 261 | word = self.replaceM1(word, 'ive', '') 262 | elif word.endswith('ize'): 263 | word = self.replaceM1(word, 'ize', '') 264 | elif word.endswith('ion'): 265 | result = word.rfind('ion') 266 | base = word[:result] 267 | if self.getM(base) > 1 and (self.endsWith(base, 's') or self.endsWith(base, 't')): 268 | word = base 269 | word = self.replaceM1(word, '', '') 270 | return word 271 | 272 | def step5a(self, word): 273 | if word.endswith('e'): 274 | base = word[:-1] 275 | if self.getM(base) > 1: 276 | word = base 277 | elif self.getM(base) == 1 and not self.cvc(base): 278 | word = base 279 | return word 280 | 281 | def step5b(self, word): 282 | if self.getM(word) > 1 and self.doubleCons(word) and self.endsWith(word, 'l'): 283 | word = word[:-1] 284 | return word 285 | 286 | def stem(self, word): 287 | word = self.step1a(word) 288 | word = self.step1b(word) 289 | word = self.step1c(word) 290 | word = self.step2(word) 291 | word = self.step3(word) 292 | word = self.step4(word) 293 | word = self.step5a(word) 294 | word = self.step5b(word) 295 | return word 296 | -------------------------------------------------------------------------------- /ml-ex6/svmPredict.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | #SVMPREDICT returns a vector of predictions using a trained SVM model 5 | #(svmTrain). 6 | # pred = SVMPREDICT(model, X) returns a vector of predictions using a 7 | # trained SVM model (svmTrain). X is a mxn matrix where there each 8 | # example is a row. model is a svm model returned from svmTrain. 9 | # predictions pred is a m x 1 column of predictions of {0, 1} values. 10 | # 11 | def svmPredict(model, X): 12 | 13 | # Check if we are getting a column vector, if so, then assume that we only 14 | # need to do prediction for a single example 15 | if X.shape[1] == 1: 16 | # Examples should be in rows 17 | X = X.T 18 | 19 | # Dataset 20 | m = X.shape[0] 21 | p = np.zeros(m).reshape(-1,1) 22 | pred = np.zeros(m).reshape(-1,1) 23 | 24 | if model['kernelFunction'].__name__ == 'linearKernel': 25 | # We can use the weights and bias directly if working with the 26 | # linear kernel 27 | p = np.dot(X, model['w']) + model['b'] 28 | elif model['kernelFunction'].__name__ == 'gaussianKernel': 29 | # Vectorized RBF Kernel 30 | # This is equivalent to computing the kernel on every pair of examples 31 | X1 = np.sum(X*X, axis=1).reshape(-1,1) 32 | X2 = np.sum(model['X']*model['X'], axis=1).reshape(1,-1) 33 | K = X1 + (X2 - 2 * np.dot(X, model['X'].T)) 34 | K = np.power(model['kernelFunction'](np.array(1), np.array(0)), K) 35 | K = model['y'].T * K 36 | K = model['alphas'].T * K 37 | p = np.sum(K, axis=1) 38 | else: 39 | # Other Non-linear kernel 40 | for i in range(m): 41 | prediction = 0 42 | for j in range(model['X'].shape[0]): 43 | prediction = prediction + \ 44 | np.dot(model['alphas'][j], model['y'][j]) * \ 45 | model['kernelFunction'](X[i,:].T, model['X'][j,:].T) 46 | p[i] = prediction + model['b'] 47 | 48 | # Convert predictions into 0 / 1 49 | pred[np.where(p >= 0)] = 1 50 | pred[np.where(p < 0)] = 0 51 | 52 | return pred 53 | 54 | -------------------------------------------------------------------------------- /ml-ex6/svmTrain.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #SVMTRAIN Trains an SVM classifier using a simplified version of the SMO 4 | #algorithm. 5 | # [model] = SVMTRAIN(X, Y, C, kernelFunction, tol, max_passes) trains an 6 | # SVM classifier and returns trained model. X is the matrix of training 7 | # examples. Each row is a training example, and the jth column holds the 8 | # jth feature. Y is a column matrix containing 1 for positive examples 9 | # and 0 for negative examples. C is the standard SVM regularization 10 | # parameter. tol is a tolerance value used for determining equality of 11 | # floating point numbers. max_passes controls the number of iterations 12 | # over the dataset (without changes to alpha) before the algorithm quits. 13 | # 14 | # Note: This is a simplified version of the SMO algorithm for training 15 | # SVMs. In practice, if you want to train an SVM classifier, we 16 | # recommend using an optimized package such as: 17 | # 18 | # LIBSVM (http://www.csie.ntu.edu.tw/~cjlin/libsvm/) 19 | # SVMLight (http://svmlight.joachims.org/) 20 | # 21 | # 22 | def svmTrain(X, Y, C, kernelFunction, tol=None, max_passes=None): 23 | 24 | if tol == None: 25 | tol = 1e-3 26 | 27 | if max_passes == None: 28 | max_passes = 5 29 | 30 | # Data parameters 31 | m = X.shape[0] 32 | n = X.shape[1] 33 | 34 | Y = Y.copy() 35 | Y[np.where(Y==0)] = -1 36 | 37 | # Variables 38 | alphas = np.zeros(m).reshape(-1,1) 39 | b = 0 40 | E = np.zeros(m).reshape(-1,1) 41 | passes = 0 42 | eta = 0 43 | L = 0 44 | H = 0 45 | 46 | # Pre-compute the Kernel Matrix since our dataset is small 47 | # (in practice, optimized SVM packages that handle large datasets 48 | # gracefully will _not_ do this) 49 | # 50 | # We have implemented optimized vectorized version of the Kernels here so 51 | # that the svm training will run faster. 52 | 53 | if kernelFunction.__name__ == 'linearKernel': 54 | # Vectorized computation for the Linear Kernel 55 | # This is equivalent to computing the kernel on every pair of examples 56 | K = np.dot(X, X.T) 57 | elif kernelFunction.__name__ == 'gaussianKernel': 58 | # Vectorized RBF Kernel 59 | # This is equivalent to computing the kernel on every pair of examples 60 | X2 = np.sum(X*X, axis=1).reshape(-1,1) 61 | K = X2 + (X2.T - 2 * np.dot(X, X.T)) 62 | K = np.power(kernelFunction(np.array(1), np.array(0)), K) 63 | else: 64 | # Pre-compute the Kernel Matrix 65 | # The following can be slow due to the lack of vectorization 66 | K = np.zeros((m, m)) 67 | for i in range(m): 68 | for j in range(m): 69 | K[i,j] = kernelFunction(X[i,:].T, X[j,:].T) 70 | K[j,i] = K[i,j] #the matrix is symmetric 71 | 72 | 73 | # Train 74 | print('\nTraining ...', end='') 75 | dots = 12 76 | while passes < max_passes: 77 | 78 | num_changed_alphas = 0 79 | for i in range(m): 80 | 81 | # Calculate Ei = f(x(i)) - y(i) using (2). 82 | # E(i) = b + sum (X(i, :) * (repmat(alphas.*Y,1,n).*X)') - Y(i); 83 | E[i] = b + np.sum(alphas * Y * K[:,i].reshape(-1,1)) - Y[i] 84 | 85 | # print(np.zeros(m).reshape(-1,1)) 86 | if (Y[i]*E[i] < -tol and alphas[i] < C) or (Y[i]*E[i] > tol and alphas[i] > 0): 87 | 88 | # In practice, there are many heuristics one can use to select 89 | # the i and j. In this simplified code, we select them randomly. 90 | j = np.int(np.ceil(m * np.random.rand()))-1 91 | while j == i: # Make sure i != j 92 | j = np.int(np.ceil(m * np.random.rand()))-1 93 | 94 | # Calculate Ej = f(x(j)) - y(j) using (2). 95 | E[j] = b + np.sum(alphas * Y * K[:,j].reshape(-1,1)) - Y[j] 96 | 97 | # Save old alphas 98 | alpha_i_old = alphas[i].copy() 99 | alpha_j_old = alphas[j].copy() 100 | 101 | # Compute L and H by (10) or (11). 102 | if Y[i] == Y[j]: 103 | L = np.maximum(0, alphas[j] + alphas[i] - C) 104 | H = np.minimum(C, alphas[j] + alphas[i]) 105 | else: 106 | L = np.maximum(0, alphas[j] - alphas[i]) 107 | H = np.minimum(C, C + alphas[j] - alphas[i]) 108 | 109 | if L == H: 110 | # continue to next i. 111 | continue 112 | 113 | # Compute eta by (14). 114 | eta = 2 * K[i, j] - K[i, i] - K[j, j] 115 | if eta >= 0: 116 | # continue to next i. 117 | continue 118 | 119 | # Compute and clip new value for alpha j using (12) and (15). 120 | alphas[j] = alphas[j] - (Y[j] * (E[i] - E[j])) / eta 121 | 122 | # Clip 123 | alphas[j] = np.minimum(H, alphas[j]) 124 | alphas[j] = np.maximum(L, alphas[j]) 125 | 126 | # Check if change in alpha is significant 127 | if abs(alphas[j] - alpha_j_old) < tol: 128 | # continue to next i. 129 | # replace anyway 130 | alphas[j] = alpha_j_old 131 | continue 132 | 133 | # Determine value for alpha i using (16). 134 | alphas[i] = alphas[i] + Y[i]*Y[j]*(alpha_j_old - alphas[j]) 135 | 136 | # Compute b1 and b2 using (17) and (18) respectively. 137 | b1 = b - E[i] \ 138 | - Y[i] * (alphas[i] - alpha_i_old) * K[i,j] \ 139 | - Y[j] * (alphas[j] - alpha_j_old) * K[i,j] 140 | b2 = b - E[j] \ 141 | - Y[i] * (alphas[i] - alpha_i_old) * K[i,j] \ 142 | - Y[j] * (alphas[j] - alpha_j_old) * K[j,j] 143 | 144 | # Compute b by (19). 145 | if 0 < alphas[i] and alphas[i] < C: 146 | b = b1 147 | elif 0 < alphas[j] and alphas[j] < C: 148 | b = b2 149 | else: 150 | b = (b1 + b2)/2 151 | 152 | num_changed_alphas = num_changed_alphas + 1 153 | 154 | if num_changed_alphas == 0: 155 | passes = passes + 1 156 | else: 157 | passes = 0 158 | 159 | print('.', end='') 160 | dots = dots + 1 161 | if dots > 78: 162 | dots = 0 163 | print('') 164 | 165 | print(' Done! \n\n') 166 | idx = np.where(alphas > 0) 167 | return { 168 | 'X': X[idx[0],:], 169 | 'y': Y[idx[0]], 170 | 'kernelFunction': kernelFunction, 171 | 'b': b, 172 | 'alphas': alphas[idx[0]], 173 | 'w': np.dot((alphas*Y).T,X).T 174 | } 175 | 176 | 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /ml-ex6/visualizeBoundary.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from plotData import plotData 4 | from svmPredict import svmPredict 5 | 6 | #VISUALIZEBOUNDARY plots a non-linear decision boundary learned by the SVM 7 | # VISUALIZEBOUNDARYLINEAR(X, y, model) plots a non-linear decision 8 | # boundary learned by the SVM and overlays the data on it 9 | def visualizeBoundary(X, y, model): 10 | 11 | # Plot the training data on top of the boundary 12 | plotData(X,y.T[0]) 13 | 14 | # Make classification predictions over a grid of values 15 | x1plot = np.linspace(np.min(X[:,0]), np.max(X[:,0]), 100).reshape(-1,1) 16 | x2plot = np.linspace(np.min(X[:,1]), np.max(X[:,1]), 100).reshape(-1,1) 17 | X1, X2 = np.meshgrid(x1plot, x2plot) 18 | vals = np.zeros(X1.shape) 19 | for i in range(X1.shape[1]): 20 | this_X = np.vstack((X1[:,i], X2[:,i])).T 21 | vals[:,i] = svmPredict(model, this_X).reshape(1,-1) 22 | 23 | # Plot the SVM boundary 24 | plt.contour(X1, X2, vals, colors='b', linewidths=1) -------------------------------------------------------------------------------- /ml-ex6/visualizeBoundaryLinear.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from plotData import plotData 4 | 5 | #VISUALIZEBOUNDARYLINEAR plots a linear decision boundary learned by the 6 | #SVM 7 | # VISUALIZEBOUNDARYLINEAR(X, y, model) plots a linear decision boundary 8 | # learned by the SVM and overlays the data on it 9 | def visualizeBoundaryLinear(X, y, model): 10 | 11 | w = model['w'] 12 | b = model['b'] 13 | xp = np.linspace(np.min(X[:,0]), np.max(X[:,0]), 100).reshape(-1,1) 14 | yp = -(w[0]*xp + b)/w[1] 15 | plotData(X, y.T[0]) 16 | plt.plot(xp, yp, 'b-') 17 | -------------------------------------------------------------------------------- /ml-ex7/bird_small.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex7/bird_small.mat -------------------------------------------------------------------------------- /ml-ex7/bird_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex7/bird_small.png -------------------------------------------------------------------------------- /ml-ex7/computeCentroids.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #COMPUTECENTROIDS returs the new centroids by computing the means of the 4 | #data points assigned to each centroid. 5 | # centroids = COMPUTECENTROIDS(X, idx, K) returns the new centroids by 6 | # computing the means of the data points assigned to each centroid. It is 7 | # given a dataset X where each row is a single data point, a vector 8 | # idx of centroid assignments (i.e. each entry in range [1..K]) for each 9 | # example, and K, the number of centroids. You should return a matrix 10 | # centroids, where each row of centroids is the mean of the data points 11 | # assigned to it. 12 | # 13 | def computeCentroids(X, idx, K): 14 | 15 | # Useful variables 16 | m = X.shape[0] 17 | n = X.shape[1] 18 | 19 | # % You need to return the following variables correctly. 20 | centroids = np.zeros((K, n)) 21 | 22 | num = np.zeros((K, 1)) 23 | sum = np.zeros((K, n)) 24 | 25 | for i in range(idx.shape[0]): 26 | z = idx[i] 27 | num[z] += 1 28 | sum[z,:] += X[i,:] 29 | 30 | centroids = sum / num 31 | 32 | return centroids -------------------------------------------------------------------------------- /ml-ex7/displayData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | #DISPLAYDATA Display 2D data in a nice grid 6 | # [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 7 | # stored in X in a nice grid. It returns the figure handle h and the 8 | # displayed array if requested. 9 | def displayData(X, example_width=None): 10 | 11 | # Set example_width automatically if not passed in 12 | if example_width == None: 13 | example_width = (int)(np.round(np.sqrt(X.shape[1]))) 14 | 15 | # Compute rows, cols 16 | m, n = X.shape 17 | example_height = (int)(n / example_width) 18 | 19 | # Compute number of items to display 20 | display_rows = (int)(np.floor(np.sqrt(m))) 21 | display_cols = (int)(np.ceil(m / display_rows)) 22 | 23 | # Between images padding 24 | pad = 1 25 | 26 | # Setup blank display 27 | display_array = - np.ones((pad + display_rows * (example_height + pad), \ 28 | pad + display_cols * (example_width + pad))) 29 | 30 | # Copy each example into a patch on the display array 31 | curr_ex = 0 32 | for j in range(display_rows): 33 | for i in range(display_cols): 34 | if curr_ex >= m: 35 | break 36 | 37 | # Copy the patch 38 | 39 | # Get the max value of the patch 40 | max_val = np.max(np.abs(X[curr_ex])) 41 | offset_height = pad + j * (example_height + pad) 42 | offset_width = pad + i * (example_width + pad) 43 | display_array[offset_height : offset_height+example_height,\ 44 | offset_width : offset_width+example_width] = \ 45 | X[curr_ex].reshape((example_height,example_width)).T / max_val 46 | curr_ex = curr_ex + 1 47 | 48 | if curr_ex >= m: 49 | break 50 | 51 | # Display Image 52 | plt.imshow(display_array, cmap='gray', vmin = -1, vmax = 1) 53 | 54 | # Do not show axis 55 | plt.axis('off') -------------------------------------------------------------------------------- /ml-ex7/drawLine.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from plotDataPoints import plotDataPoints 4 | 5 | #DRAWLINE Draws a line from point p1 to point p2 6 | # DRAWLINE(p1, p2) Draws a line from point p1 to point p2 and holds the 7 | # current figure 8 | def drawLine(p1, p2, *varargin): 9 | 10 | # Plot the examples 11 | plt.plot([p1[0], p2[0]], [p1[1], p2[1]], *varargin) 12 | -------------------------------------------------------------------------------- /ml-ex7/ex7.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex7/ex7.pdf -------------------------------------------------------------------------------- /ml-ex7/ex7.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 7 | Principle Component Analysis and K-Means Clustering 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # pca.py 11 | # projectData.py 12 | # recoverData.py 13 | # computeCentroids.py 14 | # findClosestCentroids.py 15 | # kMeansInitCentroids.py 16 | # 17 | # For this exercise, you will not need to change any code in this file, 18 | # or any other files other than those mentioned above. 19 | # 20 | 21 | ## Initialization 22 | import numpy as np 23 | import matplotlib.pyplot as plt 24 | import scipy.io as sio 25 | import matplotlib.image as mpimg 26 | from findClosestCentroids import findClosestCentroids 27 | from computeCentroids import computeCentroids 28 | from runkMeans import runkMeans 29 | from kMeansInitCentroids import kMeansInitCentroids 30 | 31 | plt.ion() 32 | 33 | ## ================= Part 1: Find Closest Centroids ==================== 34 | # To help you implement K-Means, we have divided the learning algorithm 35 | # into two functions -- findClosestCentroids and computeCentroids. In this 36 | # part, you shoudl complete the code in the findClosestCentroids function. 37 | # 38 | print('Finding closest centroids.\n\n') 39 | 40 | # Load an example dataset that we will be using 41 | data = sio.loadmat('ex7data2.mat') 42 | X = data['X'] 43 | 44 | # Select an initial set of centroids 45 | K = 3 # 3 Centroids 46 | initial_centroids = np.array([[3, 3], [6, 2], [8, 5]]) 47 | 48 | # Find the closest centroids for the examples using the 49 | # initial_centroids 50 | idx = findClosestCentroids(X, initial_centroids) 51 | 52 | print('Closest centroids for the first 3 examples: \n') 53 | print(idx[0:3]) 54 | print('\n(the closest centroids should be 1, 3, 2 respectively)\n') 55 | 56 | input('Program paused. Press enter to continue.\n') 57 | 58 | ## ===================== Part 2: Compute Means ========================= 59 | # After implementing the closest centroids function, you should now 60 | # complete the computeCentroids function. 61 | # 62 | print('\nComputing centroids means.\n\n') 63 | 64 | # Compute means based on the closest centroids found in the previous part. 65 | centroids = computeCentroids(X, idx, K) 66 | 67 | print('Centroids computed after initial finding of closest centroids: \n') 68 | print(centroids) 69 | print('\n(the centroids should be\n') 70 | print(' [ 2.428301 3.157924 ]\n') 71 | print(' [ 5.813503 2.633656 ]\n') 72 | print(' [ 7.119387 3.616684 ]\n\n') 73 | 74 | input('Program paused. Press enter to continue.\n') 75 | 76 | ## =================== Part 3: K-Means Clustering ====================== 77 | # After you have completed the two functions computeCentroids and 78 | # findClosestCentroids, you have all the necessary pieces to run the 79 | # kMeans algorithm. In this part, you will run the K-Means algorithm on 80 | # the example dataset we have provided. 81 | # 82 | print('\nRunning K-Means clustering on example dataset.\n\n') 83 | 84 | # Load an example dataset 85 | data = sio.loadmat('ex7data2.mat') 86 | X = data['X'] 87 | 88 | # Settings for running K-Means 89 | K = 3 90 | max_iters = 10 91 | 92 | # For consistency, here we set centroids to specific values 93 | # but in practice you want to generate them automatically, such as by 94 | # settings them to be random examples (as can be seen in 95 | # kMeansInitCentroids). 96 | initial_centroids = np.array([[3, 3], [6, 2], [8, 5]]) 97 | 98 | # Run K-Means algorithm. The 'true' at the end tells our function to plot 99 | # the progress of K-Means 100 | centroids, idx = runkMeans(X, initial_centroids, max_iters, True) 101 | print('\nK-Means Done.\n\n') 102 | 103 | input('Program paused. Press enter to continue.\n') 104 | 105 | ## ============= Part 4: K-Means Clustering on Pixels =============== 106 | # In this exercise, you will use K-Means to compress an image. To do this, 107 | # you will first run K-Means on the colors of the pixels in the image and 108 | # then you will map each pixel on to it's closest centroid. 109 | # 110 | # You should now complete the code in kMeansInitCentroids.m 111 | # 112 | 113 | print('\nRunning K-Means clustering on pixels from an image.\n\n') 114 | 115 | # Load an image of a bird 116 | A = mpimg.imread('bird_small.png') 117 | 118 | # Size of the image 119 | img_size = A.shape 120 | 121 | # Reshape the image into an Nx3 matrix where N = number of pixels. 122 | # Each row will contain the Red, Green and Blue pixel values 123 | # This gives us our dataset matrix X that we will use K-Means on. 124 | X = A.reshape(img_size[0]*img_size[1], 3) 125 | 126 | # Run your K-Means algorithm on this data 127 | # You should try different values of K and max_iters here 128 | K = 16 129 | max_iters = 10 130 | 131 | # When using K-Means, it is important the initialize the centroids 132 | # randomly. 133 | # You should complete the code in kMeansInitCentroids.m before proceeding 134 | initial_centroids = kMeansInitCentroids(X, K) 135 | 136 | # Run K-Means 137 | centroids, idx = runkMeans(X, initial_centroids, max_iters) 138 | 139 | input('Program paused. Press enter to continue.\n') 140 | 141 | ## ================= Part 5: Image Compression ====================== 142 | # In this part of the exercise, you will use the clusters of K-Means to 143 | # compress an image. To do this, we first find the closest clusters for 144 | # each example. After that, we 145 | 146 | print('\nApplying K-Means to compress an image.\n\n') 147 | 148 | # Find closest cluster members 149 | idx = findClosestCentroids(X, centroids) 150 | 151 | # Essentially, now we have represented the image X as in terms of the 152 | # indices in idx. 153 | 154 | # We can now recover the image from the indices (idx) by mapping each pixel 155 | # (specified by it's index in idx) to the centroid value 156 | X_recovered = centroids[idx,:] 157 | 158 | # Reshape the recovered image into proper dimensions 159 | X_recovered = X_recovered.reshape((img_size[0], img_size[1], 3)) 160 | 161 | # Display the original image 162 | plt.figure() 163 | plt.imshow(A) 164 | plt.axis('off') 165 | plt.title('Original') 166 | 167 | # Display compressed image side by side 168 | plt.figure() 169 | plt.imshow(X_recovered) 170 | plt.axis('off') 171 | plt.title('Compressed, with %d colors.'%K) 172 | 173 | 174 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex7/ex7_pca.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 7 | Principle Component Analysis and K-Means Clustering 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # pca.py 11 | # projectData.py 12 | # recoverData.py 13 | # computeCentroids.py 14 | # findClosestCentroids.py 15 | # kMeansInitCentroids.py 16 | # 17 | # For this exercise, you will not need to change any code in this file, 18 | # or any other files other than those mentioned above. 19 | # 20 | 21 | ## Initialization 22 | import numpy as np 23 | from mpl_toolkits.mplot3d import Axes3D 24 | import matplotlib.pyplot as plt 25 | import scipy.io as sio 26 | import matplotlib.image as mpimg 27 | from featureNormalize import featureNormalize 28 | from pca import pca 29 | from drawLine import drawLine 30 | from projectData import projectData 31 | from displayData import displayData 32 | from recoverData import recoverData 33 | from kMeansInitCentroids import kMeansInitCentroids 34 | from runkMeans import runkMeans 35 | from plotDataPoints import plotDataPoints 36 | 37 | plt.ion() 38 | 39 | ## ================== Part 1: Load Example Dataset =================== 40 | # We start this exercise by using a small dataset that is easily to 41 | # visualize 42 | # 43 | print('Visualizing example dataset for PCA.\n\n') 44 | 45 | # The following command loads the dataset. You should now have the 46 | # variable X in your environment 47 | data = sio.loadmat('ex7data1.mat') 48 | X = data['X'] 49 | 50 | # Visualize the example dataset 51 | plt.figure() 52 | plt.plot(X[:, 0], X[:, 1], 'bo', markerfacecolor="None") 53 | plt.axis([0.5, 6.5, 2, 8]) 54 | plt.axis('equal') 55 | 56 | input('Program paused. Press enter to continue.\n') 57 | 58 | 59 | ## =============== Part 2: Principal Component Analysis =============== 60 | # You should now implement PCA, a dimension reduction technique. You 61 | # should complete the code in pca.m 62 | # 63 | print('\nRunning PCA on example dataset.\n\n') 64 | 65 | # Before running PCA, it is important to first normalize X 66 | X_norm, mu, sigma = featureNormalize(X) 67 | 68 | # Run PCA 69 | U, S = pca(X_norm) 70 | 71 | # Compute mu, the mean of the each feature 72 | 73 | # Draw the eigenvectors centered at mean of data. These lines show the 74 | # directions of maximum variations in the dataset. 75 | drawLine(mu, mu + 1.5 * S[0] * U[:,0], 'k-') 76 | drawLine(mu, mu + 1.5 * S[1] * U[:,1], 'k-') 77 | 78 | 79 | print('Top eigenvector: \n') 80 | print(' U(:,1) = %f %f \n'%(U[0,0], U[1,0])) 81 | print('\n(you should expect to see -0.707107 -0.707107)\n') 82 | 83 | input('Program paused. Press enter to continue.\n') 84 | 85 | ## =================== Part 3: Dimension Reduction =================== 86 | # You should now implement the projection step to map the data onto the 87 | # first k eigenvectors. The code will then plot the data in this reduced 88 | # dimensional space. This will show you what the data looks like when 89 | # using only the corresponding eigenvectors to reconstruct it. 90 | # 91 | # You should complete the code in projectData.m 92 | # 93 | print('\nDimension reduction on example dataset.\n\n') 94 | 95 | # Plot the normalized dataset (returned from pca) 96 | plt.figure() 97 | plt.plot(X_norm[:, 0], X_norm[:, 1], 'bo', markerfacecolor="None") 98 | plt.axis([-4, 3, -4, 3]) 99 | plt.axis('equal') 100 | 101 | # Project the data onto K = 1 dimension 102 | K = 1 103 | Z = projectData(X_norm, U, K) 104 | print('Projection of the first example: %f\n'%Z[0]) 105 | print('\n(this value should be about 1.481274)\n\n') 106 | 107 | X_rec = recoverData(Z, U, K) 108 | print('Approximation of the first example: %f %f\n'%(X_rec[0, 0], X_rec[0, 1])) 109 | print('\n(this value should be about -1.047419 -1.047419)\n\n') 110 | 111 | # Draw lines connecting the projected points to the original points 112 | plt.plot(X_rec[:, 0], X_rec[:, 1], 'ro', markerfacecolor="None") 113 | for i in range(X_norm.shape[0]): 114 | drawLine(X_norm[i,:], X_rec[i,:], 'k--') 115 | 116 | input('Program paused. Press enter to continue.\n') 117 | 118 | ## =============== Part 4: Loading and Visualizing Face Data ============= 119 | # We start the exercise by first loading and visualizing the dataset. 120 | # The following code will load the dataset into your environment 121 | # 122 | print('\nLoading face dataset.\n\n') 123 | 124 | # Load Face dataset 125 | data = sio.loadmat('ex7faces.mat') 126 | X = data['X'] 127 | 128 | # Display the first 100 faces in the dataset 129 | plt.figure() 130 | displayData(X[0:100, :]) 131 | 132 | input('Program paused. Press enter to continue.\n') 133 | 134 | ## =========== Part 5: PCA on Face Data: Eigenfaces =================== 135 | # Run PCA and visualize the eigenvectors which are in this case eigenfaces 136 | # We display the first 36 eigenfaces. 137 | # 138 | print('\nRunning PCA on face dataset.\n (this mght take a minute or two ...)\n\n') 139 | 140 | # Before running PCA, it is important to first normalize X by subtracting 141 | # the mean value from each feature 142 | X_norm, mu, sigma = featureNormalize(X) 143 | 144 | # Run PCA 145 | U, S = pca(X_norm) 146 | 147 | # Visualize the top 36 eigenvectors found 148 | displayData(U[:, 0:36].T) 149 | 150 | input('Program paused. Press enter to continue.\n') 151 | 152 | 153 | ## ============= Part 6: Dimension Reduction for Faces ================= 154 | # Project images to the eigen space using the top k eigenvectors 155 | # If you are applying a machine learning algorithm 156 | print('\nDimension reduction for face dataset.\n\n') 157 | 158 | K = 100 159 | Z = projectData(X_norm, U, K) 160 | 161 | print('The projected data Z has a size of: ') 162 | print(Z.shape) 163 | 164 | input('\n\nProgram paused. Press enter to continue.\n') 165 | 166 | ## ==== Part 7: Visualization of Faces after PCA Dimension Reduction ==== 167 | # Project images to the eigen space using the top K eigen vectors and 168 | # visualize only using those K dimensions 169 | # Compare to the original input, which is also displayed 170 | 171 | print('\nVisualizing the projected (reduced dimension) faces.\n\n') 172 | 173 | K = 100 174 | X_rec = recoverData(Z, U, K) 175 | 176 | # Display normalized data 177 | plt.figure() 178 | plt.subplot(121) 179 | displayData(X_norm[0:100,:]) 180 | plt.title('Original faces') 181 | plt.axis('equal') 182 | 183 | # Display reconstructed data from only k eigenfaces 184 | plt.subplot(122) 185 | displayData(X_rec[0:100,:]) 186 | plt.title('Recovered faces') 187 | plt.axis('equal') 188 | 189 | input('Program paused. Press enter to continue.\n') 190 | 191 | ## === Part 8(a): Optional (ungraded) Exercise: PCA for Visualization === 192 | # One useful application of PCA is to use it to visualize high-dimensional 193 | # data. In the last K-Means exercise you ran K-Means on 3-dimensional 194 | # pixel colors of an image. We first visualize this output in 3D, and then 195 | # apply PCA to obtain a visualization in 2D. 196 | 197 | # Re-load the image from the previous exercise and run K-Means on it 198 | # For this to work, you need to complete the K-Means assignment first 199 | A = mpimg.imread('bird_small.png') 200 | 201 | # If imread does not work for you, you can try instead 202 | # load ('bird_small.mat'); 203 | 204 | img_size = A.shape 205 | X = A.reshape((img_size[0] * img_size[1], 3)) 206 | K = 16 207 | max_iters = 10 208 | initial_centroids = kMeansInitCentroids(X, K) 209 | centroids, idx = runkMeans(X, initial_centroids, max_iters) 210 | 211 | # Sample 1000 random indexes (since working with all the data is 212 | # too expensive. If you have a fast computer, you may increase this. 213 | sel = np.floor(np.random.rand(1000) * X.shape[0]) 214 | sel = sel.astype(int) 215 | 216 | # Setup Color Palette 217 | colors = idx[sel] / (K+1) 218 | 219 | # Visualize the data and centroid memberships in 3D 220 | fig = plt.figure() 221 | ax = Axes3D(fig) 222 | ax.scatter(X[sel,0], X[sel,1], X[sel,2], c=colors, cmap='hsv') 223 | plt.show() 224 | plt.title('Pixel dataset plotted in 3D. Color shows centroid memberships') 225 | input('Program paused. Press enter to continue.\n') 226 | 227 | ## === Part 8(b): Optional (ungraded) Exercise: PCA for Visualization === 228 | # Use PCA to project this cloud to 2D for visualization 229 | 230 | # Subtract the mean to use PCA 231 | X_norm, mu, sigma = featureNormalize(X) 232 | 233 | # PCA and project the data to 2D 234 | U, S = pca(X_norm) 235 | Z = projectData(X_norm, U, 2) 236 | 237 | # Plot in 2D 238 | fig = plt.figure() 239 | plotDataPoints(Z[sel, :], idx[sel], K) 240 | plt.title('Pixel dataset plotted in 2D, using PCA for dimensionality reduction') 241 | input('Program paused. Press enter to continue.\n') -------------------------------------------------------------------------------- /ml-ex7/ex7data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex7/ex7data1.mat -------------------------------------------------------------------------------- /ml-ex7/ex7data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex7/ex7data2.mat -------------------------------------------------------------------------------- /ml-ex7/ex7faces.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex7/ex7faces.mat -------------------------------------------------------------------------------- /ml-ex7/featureNormalize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #FEATURENORMALIZE Normalizes the features in X 4 | # FEATURENORMALIZE(X) returns a normalized version of X where 5 | # the mean value of each feature is 0 and the standard deviation 6 | # is 1. This is often a good preprocessing step to do when 7 | # working with learning algorithms. 8 | def featureNormalize(X): 9 | 10 | mu = np.mean(X, axis=0) 11 | sigma = np.std(X, axis=0) 12 | 13 | return (X-mu)/sigma, mu, sigma -------------------------------------------------------------------------------- /ml-ex7/findClosestCentroids.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #FINDCLOSESTCENTROIDS computes the centroid memberships for every example 4 | # idx = FINDCLOSESTCENTROIDS (X, centroids) returns the closest centroids 5 | # in idx for a dataset X where each row is a single example. idx = m x 1 6 | # vector of centroid assignments (i.e. each entry in range [1..K]) 7 | # 8 | def findClosestCentroids(X, centroids): 9 | 10 | # Set K 11 | K = centroids.shape[0] 12 | 13 | # You need to return the following variables correctly. 14 | idx = np.zeros(X.shape[0]) 15 | 16 | for i in range(X.shape[0]): 17 | min = np.inf 18 | for j in range(K): 19 | diff = np.sum(np.power(X[i,:] - centroids[j,:], 2)) 20 | if min > diff: 21 | min = diff 22 | idx[i] = j 23 | 24 | idx = idx.astype(int) 25 | 26 | return idx -------------------------------------------------------------------------------- /ml-ex7/kMeansInitCentroids.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #KMEANSINITCENTROIDS This function initializes K centroids that are to be 4 | #used in K-Means on the dataset X 5 | # centroids = KMEANSINITCENTROIDS(X, K) returns K initial centroids to be 6 | # used with the K-Means on the dataset X 7 | # 8 | def kMeansInitCentroids(X, K): 9 | 10 | randidx = np.random.permutation(X.shape[0]) 11 | centroids = X[randidx[0:K],:] 12 | 13 | return centroids -------------------------------------------------------------------------------- /ml-ex7/pca.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #PCA Run principal component analysis on the dataset X 4 | # [U, S, X] = pca(X) computes eigenvectors of the covariance matrix of X 5 | # Returns the eigenvectors U, the eigenvalues (on diagonal) in S 6 | # 7 | def pca(X): 8 | 9 | # Useful values 10 | m = X.shape[0] 11 | n = X.shape[1] 12 | 13 | sigma = np.dot(X.T, X) / m 14 | U, S, v = np.linalg.svd(sigma, full_matrices=True) 15 | 16 | return U, S -------------------------------------------------------------------------------- /ml-ex7/plotDataPoints.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | #PLOTDATAPOINTS plots data points in X, coloring them so that those with the same 5 | #index assignments in idx have the same color 6 | # PLOTDATAPOINTS(X, idx, K) plots data points in X, coloring them so that those 7 | # with the same index assignments in idx have the same color 8 | def plotDataPoints(X, idx, K): 9 | 10 | # Create palette 11 | colors = plt.cm.hsv(idx / K) 12 | 13 | # Plot the data 14 | plt.scatter(X[:,0], X[:,1], facecolors='none', edgecolors=colors, marker='o', cmap='hsv') -------------------------------------------------------------------------------- /ml-ex7/plotProgresskMeans.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from plotDataPoints import plotDataPoints 4 | from drawLine import drawLine 5 | 6 | #PLOTPROGRESSKMEANS is a helper function that displays the progress of 7 | #k-Means as it is running. It is intended for use only with 2D data. 8 | # PLOTPROGRESSKMEANS(X, centroids, previous, idx, K, i) plots the data 9 | # points with colors assigned to each centroid. With the previous 10 | # centroids, it also plots a line between the previous locations and 11 | # current locations of the centroids. 12 | # 13 | def plotProgresskMeans(X, centroids, previous, idx, K, i): 14 | 15 | # Plot the examples 16 | plotDataPoints(X, idx, K) 17 | 18 | # Plot the centroids as black x's 19 | plt.plot(centroids[:,0], centroids[:,1], 'kx', linewidth=5, markersize=7) 20 | 21 | # Plot the history of the centroids with lines 22 | for j in range(centroids.shape[0]): 23 | drawLine(centroids[j,:], previous[j,:], 'k-') 24 | 25 | plt.title('Iteration number %d'%i) -------------------------------------------------------------------------------- /ml-ex7/projectData.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #PROJECTDATA Computes the reduced data representation when projecting only 4 | #on to the top k eigenvectors 5 | # Z = projectData(X, U, K) computes the projection of 6 | # the normalized inputs X into the reduced dimensional space spanned by 7 | # the first K columns of U. It returns the projected examples in Z. 8 | # 9 | def projectData(X, U, K): 10 | 11 | # You need to return the following variables correctly. 12 | Z = np.zeros((X.shape[0], K)) 13 | 14 | for i in range(X.shape[0]): 15 | x = X[i, :] 16 | Z[i, :] = np.dot(x, U[:,0:K]) 17 | 18 | return Z -------------------------------------------------------------------------------- /ml-ex7/recoverData.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #RECOVERDATA Recovers an approximation of the original data when using the 4 | #projected data 5 | # X_rec = RECOVERDATA(Z, U, K) recovers an approximation the 6 | # original data that has been reduced to K dimensions. It returns the 7 | # approximate reconstruction in X_rec. 8 | # 9 | def recoverData(Z, U, K): 10 | 11 | # You need to return the following variables correctly. 12 | X_rec = np.zeros((Z.shape[0], U.shape[0])) 13 | 14 | for i in range(Z.shape[0]): 15 | v = Z[i, :] 16 | for j in range(U.shape[0]): 17 | X_rec[i, j] = np.dot(v, U[j, 0:K].T) 18 | 19 | 20 | return X_rec -------------------------------------------------------------------------------- /ml-ex7/runkMeans.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from plotProgresskMeans import plotProgresskMeans 4 | from findClosestCentroids import findClosestCentroids 5 | from computeCentroids import computeCentroids 6 | 7 | #RUNKMEANS runs the K-Means algorithm on data matrix X, where each row of X 8 | #is a single example 9 | # [centroids, idx] = RUNKMEANS(X, initial_centroids, max_iters, ... 10 | # plot_progress) runs the K-Means algorithm on data matrix X, where each 11 | # row of X is a single example. It uses initial_centroids used as the 12 | # initial centroids. max_iters specifies the total number of interactions 13 | # of K-Means to execute. plot_progress is a true/false flag that 14 | # indicates if the function should also plot its progress as the 15 | # learning happens. This is set to false by default. runkMeans returns 16 | # centroids, a Kxn matrix of the computed centroids and idx, a m x 1 17 | # vector of centroid assignments (i.e. each entry in range [1..K]) 18 | # 19 | def runkMeans(X, initial_centroids, max_iters, plot_progress=None): 20 | 21 | # Set default value for plot progress 22 | if plot_progress == None: 23 | plot_progress = False 24 | 25 | # Plot the data if we are plotting progress 26 | if plot_progress: 27 | plt.figure() 28 | 29 | # Initialize values 30 | m = X.shape[0] 31 | n = X.shape[1] 32 | K = initial_centroids.shape[0] 33 | centroids = initial_centroids 34 | previous_centroids = centroids 35 | idx = np.zeros(m) 36 | idx = idx.astype(int) 37 | 38 | # Run K-Means 39 | for i in range(max_iters): 40 | 41 | # Output progress 42 | print('K-Means iteration %d/%d...\n'%(i, max_iters)) 43 | 44 | # For each example in X, assign it to the closest centroid 45 | idx = findClosestCentroids(X, centroids) 46 | 47 | # For each example in X, assign it to the closest centroid 48 | if plot_progress: 49 | plotProgresskMeans(X, centroids, previous_centroids, idx, K, i) 50 | previous_centroids = centroids.copy() 51 | input('Press enter to continue.\n') 52 | 53 | # Given the memberships, compute new centroids 54 | centroids = computeCentroids(X, idx, K) 55 | 56 | return centroids, idx -------------------------------------------------------------------------------- /ml-ex8/checkCostFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from cofiCostFunc import cofiCostFunc 4 | from computeNumericalGradient import computeNumericalGradient 5 | 6 | #CHECKCOSTFUNCTION Creates a collaborative filering problem 7 | #to check your cost function and gradients 8 | # CHECKCOSTFUNCTION(lambda) Creates a collaborative filering problem 9 | # to check your cost function and gradients, it will output the 10 | # analytical gradients produced by your code and the numerical gradients 11 | # (computed using computeNumericalGradient). These two gradient 12 | # computations should result in very similar values. 13 | def checkCostFunction(_lambda=None): 14 | 15 | if _lambda == None: 16 | _lambda = 0 17 | 18 | ## Create small problem 19 | X_t = np.random.rand(4, 3) 20 | Theta_t = np.random.rand(5, 3) 21 | 22 | # Zap out most entries 23 | Y = np.dot(X_t, Theta_t.T) 24 | Y[np.where(np.random.rand(Y.shape[0], Y.shape[1]) > 0.5)] = 0 25 | R = np.zeros(Y.shape) 26 | R[np.where(Y != 0)] = 1 27 | R = R.astype(int) 28 | 29 | ## Run Gradient Checking 30 | X = np.random.randn(X_t.shape[0], X_t.shape[1]) 31 | Theta = np.random.randn(Theta_t.shape[0], Theta_t.shape[1]) 32 | num_users = Y.shape[1] 33 | num_movies = Y.shape[0] 34 | num_features = Theta_t.shape[1] 35 | 36 | func = lambda t: cofiCostFunc(t, Y, R, num_users, num_movies, num_features, _lambda) 37 | numgrad = computeNumericalGradient(func, np.append(X.flatten(), Theta.flatten())) 38 | 39 | cost, grad = cofiCostFunc( 40 | np.append(X.flatten(), Theta.flatten()), \ 41 | Y, R, num_users, num_movies, num_features, _lambda 42 | ) 43 | 44 | print(numgrad) 45 | print(grad) 46 | print("The above two columns you get should be very similar.\n \ 47 | (Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n") 48 | 49 | diff = np.linalg.norm(numgrad-grad)/np.linalg.norm(numgrad+grad) 50 | print('If your backpropagation implementation is correct, then \n \ 51 | the relative difference will be small (less than 1e-9). \n \ 52 | \nRelative Difference: %g\n'%diff) -------------------------------------------------------------------------------- /ml-ex8/cofiCostFunc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | #COFICOSTFUNC Collaborative filtering cost function 5 | # [J, grad] = COFICOSTFUNC(params, Y, R, num_users, num_movies, ... 6 | # num_features, lambda) returns the cost and gradient for the 7 | # collaborative filtering problem. 8 | # 9 | def cofiCostFunc(params, Y, R, num_users, num_movies, num_features, _lambda): 10 | 11 | # Unfold the U and W matrices from params 12 | X = params[0:num_movies*num_features].reshape(num_movies, num_features) 13 | Theta = params[num_movies*num_features:].reshape(num_users, num_features) 14 | 15 | X_grad = np.zeros(X.shape) 16 | Theta_grad = np.zeros(Theta.shape) 17 | 18 | J = np.sum(R * np.power(np.dot(X, Theta.T) - Y, 2)) / 2 + \ 19 | _lambda * np.sum(np.power(Theta, 2)) / 2 + \ 20 | _lambda * np.sum(np.power(X, 2)) / 2 21 | 22 | for i in range(num_movies): 23 | idx = np.where(R[i,:] == 1)[0] 24 | tempTheta = Theta[idx,:] 25 | tempY = Y[i,idx] 26 | X_grad[i,:] = np.dot(np.dot(X[i,:],tempTheta.T)-tempY, tempTheta) + _lambda * X[i,:] 27 | 28 | for i in range(num_users): 29 | idx = np.where(R[:,i] == 1)[0] 30 | tempX = X[idx,:] 31 | tempY = Y[idx,i] 32 | Theta_grad[i,:] = np.dot(np.dot(tempX, Theta[i,:].T).T-tempY, tempX) + _lambda * Theta[i,:] 33 | 34 | return J, np.append(X_grad.flatten(), Theta_grad.flatten()) 35 | -------------------------------------------------------------------------------- /ml-ex8/computeNumericalGradient.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | #COMPUTENUMERICALGRADIENT Computes the gradient using "finite differences" 5 | #and gives us a numerical estimate of the gradient. 6 | # numgrad = COMPUTENUMERICALGRADIENT(J, theta) computes the numerical 7 | # gradient of the function J around theta. Calling y = J(theta) should 8 | # return the function value at theta. 9 | 10 | # Notes: The following code implements numerical gradient checking, and 11 | # returns the numerical gradient.It sets numgrad(i) to (a numerical 12 | # approximation of) the partial derivative of J with respect to the 13 | # i-th input argument, evaluated at theta. (i.e., numgrad(i) should 14 | # be the (approximately) the partial derivative of J with respect 15 | # to theta(i).) 16 | # 17 | def computeNumericalGradient(J, theta): 18 | 19 | numgrad = np.zeros(theta.shape) 20 | perturb = np.zeros(theta.shape) 21 | e = 1e-4 22 | for p, _ in np.ndenumerate(theta): 23 | # Set perturbation vector 24 | perturb[p] = e 25 | loss1,_ = J(theta - perturb) 26 | loss2,_ = J(theta + perturb) 27 | # Compute Numerical Gradient 28 | numgrad[p] = (loss2 - loss1) / e / 2 29 | perturb[p] = 0 30 | 31 | return numgrad -------------------------------------------------------------------------------- /ml-ex8/estimateGaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #ESTIMATEGAUSSIAN This function estimates the parameters of a 4 | #Gaussian distribution using the data in X 5 | # [mu sigma2] = estimateGaussian(X), 6 | # The input X is the dataset with each n-dimensional data point in one row 7 | # The output is an n-dimensional vector mu, the mean of the data set 8 | # and the variances sigma^2, an n x 1 vector 9 | # 10 | def estimateGaussian(X): 11 | 12 | # Useful variables 13 | m, n = X.shape 14 | 15 | # You should return these values correctly 16 | mu = np.zeros(n) 17 | sigma = np.zeros(n) 18 | 19 | mu = np.sum(X, axis=0) / m 20 | sigma2 = np.sum(np.power(X-mu,2), axis=0) / m 21 | 22 | return mu, sigma2 23 | -------------------------------------------------------------------------------- /ml-ex8/ex8.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex8/ex8.pdf -------------------------------------------------------------------------------- /ml-ex8/ex8.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 8 | Anomaly Detection and Collaborative Filtering 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # estimateGaussian.py 11 | # selectThreshold.py 12 | # cofiCostFunc.py 13 | # 14 | # For this exercise, you will not need to change any code in this file, 15 | # or any other files other than those mentioned above. 16 | # 17 | 18 | ## Initialization 19 | import numpy as np 20 | import matplotlib.pyplot as plt 21 | import scipy.io as sio 22 | from estimateGaussian import estimateGaussian 23 | from multivariateGaussian import multivariateGaussian 24 | from visualizeFit import visualizeFit 25 | from selectThreshold import selectThreshold 26 | 27 | plt.ion() 28 | 29 | ## ================== Part 1: Load Example Dataset =================== 30 | # We start this exercise by using a small dataset that is easy to 31 | # visualize. 32 | # 33 | # Our example case consists of 2 network server statistics across 34 | # several machines: the latency and throughput of each machine. 35 | # This exercise will help us find possibly faulty (or very fast) machines. 36 | # 37 | 38 | print('Visualizing example dataset for outlier detection.\n\n') 39 | 40 | # The following command loads the dataset. You should now have the 41 | # variables X, Xval, yval in your environment 42 | data = sio.loadmat('ex8data1.mat') 43 | X = data['X'] 44 | Xval = data['Xval'] 45 | yval = data['yval'] 46 | 47 | # Visualize the example dataset 48 | plt.plot(X[:, 0], X[:, 1], 'bx', markersize=4) 49 | plt.axis([0, 30, 0, 30]) 50 | plt.xlabel('Latency (ms)') 51 | plt.ylabel('Throughput (mb/s)') 52 | 53 | input('Program paused. Press enter to continue.\n') 54 | 55 | 56 | ## ================== Part 2: Estimate the dataset statistics =================== 57 | # For this exercise, we assume a Gaussian distribution for the dataset. 58 | # 59 | # We first estimate the parameters of our assumed Gaussian distribution, 60 | # then compute the probabilities for each of the points and then visualize 61 | # both the overall distribution and where each of the points falls in 62 | # terms of that distribution. 63 | # 64 | print('Visualizing Gaussian fit.\n\n') 65 | 66 | # Estimate my and sigma2 67 | mu, sigma2 = estimateGaussian(X) 68 | print(mu) 69 | print(sigma2) 70 | 71 | # Returns the density of the multivariate normal at each data point (row) 72 | # of X 73 | p = multivariateGaussian(X, mu, sigma2) 74 | 75 | # Visualize the fit 76 | visualizeFit(X, mu, sigma2) 77 | # plt.xlabel('Latency (ms)') 78 | # plt.ylabel('Throughput (mb/s)') 79 | 80 | input('Program paused. Press enter to continue.\n') 81 | 82 | ## ================== Part 3: Find Outliers =================== 83 | # Now you will find a good epsilon threshold using a cross-validation set 84 | # probabilities given the estimated Gaussian distribution 85 | # 86 | 87 | pval = multivariateGaussian(Xval, mu, sigma2) 88 | 89 | epsilon, F1 = selectThreshold(yval, pval) 90 | print('Best epsilon found using cross-validation: %e\n'%epsilon) 91 | print('Best F1 on Cross Validation Set: %f\n'%F1) 92 | print(' (you should see a value epsilon of about 8.99e-05)\n\n') 93 | 94 | # Find the outliers in the training set and plot the 95 | outliers = np.where(p < epsilon) 96 | print(outliers) 97 | 98 | # Draw a red circle around those outliers 99 | plt.plot(X[outliers][:,0], X[outliers][:,1], mfc='none', mec='r', marker='o', linestyle='none', markersize=10) 100 | 101 | input('Program paused. Press enter to continue.\n') 102 | 103 | ## ================== Part 4: Multidimensional Outliers =================== 104 | # We will now use the code from the previous part and apply it to a 105 | # harder problem in which more features describe each datapoint and only 106 | # some features indicate whether a point is an outlier. 107 | # 108 | 109 | # Loads the second dataset. You should now have the 110 | # variables X, Xval, yval in your environment 111 | data = sio.loadmat('ex8data2.mat') 112 | X = data['X'] 113 | Xval = data['Xval'] 114 | yval = data['yval'] 115 | 116 | # Apply the same steps to the larger dataset 117 | mu, sigma2 = estimateGaussian(X) 118 | 119 | # Training set 120 | p = multivariateGaussian(X, mu, sigma2) 121 | 122 | # Cross-validation set 123 | pval = multivariateGaussian(Xval, mu, sigma2) 124 | 125 | # Find the best threshold 126 | epsilon, F1 = selectThreshold(yval, pval) 127 | 128 | print('Best epsilon found using cross-validation: %e\n'%epsilon) 129 | print('Best F1 on Cross Validation Set: %f\n'%F1) 130 | print('# Outliers found: %d\n'%np.sum(p < epsilon)) 131 | input(' (you should see a value epsilon of about 1.38e-18)\n\n') 132 | -------------------------------------------------------------------------------- /ml-ex8/ex8_cofi.py: -------------------------------------------------------------------------------- 1 | ## Machine Learning Online Class 2 | # Exercise 8 | Anomaly Detection and Collaborative Filtering 3 | # 4 | # Instructions 5 | # ------------ 6 | # 7 | # This file contains code that helps you get started on the 8 | # exercise. You will need to complete the following functions: 9 | # 10 | # estimateGaussian.m 11 | # selectThreshold.m 12 | # cofiCostFunc.m 13 | # 14 | # For this exercise, you will not need to change any code in this file, 15 | # or any other files other than those mentioned above. 16 | # 17 | 18 | ## Initialization 19 | import numpy as np 20 | import matplotlib.pyplot as plt 21 | import scipy.io as sio 22 | from cofiCostFunc import cofiCostFunc 23 | from checkCostFunction import checkCostFunction 24 | from loadMovieList import loadMovieList 25 | from normalizeRatings import normalizeRatings 26 | from scipy.optimize import minimize 27 | 28 | plt.ion() 29 | 30 | ## =============== Part 1: Loading movie ratings dataset ================ 31 | # You will start by loading the movie ratings dataset to understand the 32 | # structure of the data. 33 | # 34 | print('Loading movie ratings dataset.\n\n') 35 | 36 | # Load data 37 | data = sio.loadmat('ex8_movies.mat') 38 | Y = data['Y'] 39 | R = data['R'] 40 | 41 | # Y is a 1682x943 matrix, containing ratings (1-5) of 1682 movies on 42 | # 943 users 43 | # 44 | # R is a 1682x943 matrix, where R(i,j) = 1 if and only if user j gave a 45 | # rating to movie i 46 | 47 | # From the matrix, we can compute statistics like average rating. 48 | print('Average rating for movie 1 (Toy Story): %f / 5\n\n'%np.mean(Y[0, np.where(R[0, :]==1)])) 49 | 50 | # We can "visualize" the ratings matrix by plotting it with imagesc 51 | plt.imshow(Y) 52 | plt.ylabel('Movies') 53 | plt.xlabel('Users') 54 | 55 | input('\nProgram paused. Press enter to continue.\n') 56 | 57 | ## ============ Part 2: Collaborative Filtering Cost Function =========== 58 | # You will now implement the cost function for collaborative filtering. 59 | # To help you debug your cost function, we have included set of weights 60 | # that we trained on that. Specifically, you should complete the code in 61 | # cofiCostFunc.m to return J. 62 | 63 | # Load pre-trained weights (X, Theta, num_users, num_movies, num_features) 64 | data = sio.loadmat('ex8_movieParams.mat') 65 | X = data['X'] 66 | Theta = data['Theta'] 67 | 68 | # Reduce the data set size so that this runs faster 69 | num_users = 4; num_movies = 5; num_features = 3 70 | X = X[0:num_movies, 0:num_features] 71 | Theta = Theta[0:num_users, 0:num_features] 72 | Y = Y[0:num_movies, 0:num_users] 73 | R = R[0:num_movies, 0:num_users] 74 | 75 | # Evaluate cost function 76 | J, _ = cofiCostFunc(np.append(X.flatten(), Theta.flatten()), Y, R, num_users, num_movies, num_features, 0) 77 | 78 | print("Cost at loaded parameters: %f \n(this value should be about 22.22)\n"%J) 79 | 80 | input('\nProgram paused. Press enter to continue.\n') 81 | 82 | ## ============== Part 3: Collaborative Filtering Gradient ============== 83 | # Once your cost function matches up with ours, you should now implement 84 | # the collaborative filtering gradient function. Specifically, you should 85 | # complete the code in cofiCostFunc.m to return the grad argument. 86 | # 87 | print('\nChecking Gradients (without regularization) ... \n') 88 | 89 | # Check gradients by running checkNNGradients 90 | checkCostFunction() 91 | 92 | input('\nProgram paused. Press enter to continue.\n') 93 | 94 | ## ========= Part 4: Collaborative Filtering Cost Regularization ======== 95 | # Now, you should implement regularization for the cost function for 96 | # collaborative filtering. You can implement it by adding the cost of 97 | # regularization to the original cost computation. 98 | # 99 | 100 | # Evaluate cost function 101 | J,_ = cofiCostFunc(np.append(X.flatten(), Theta.flatten()), Y, R, num_users, num_movies, num_features, 1.5) 102 | 103 | print("Cost at loaded parameters (lambda = 1.5): %f \n(this value should be about 31.34)\n"%J) 104 | 105 | input('\nProgram paused. Press enter to continue.\n') 106 | 107 | ## ======= Part 5: Collaborative Filtering Gradient Regularization ====== 108 | # Once your cost matches up with ours, you should proceed to implement 109 | # regularization for the gradient. 110 | # 111 | 112 | # 113 | print('\nChecking Gradients (with regularization) ... \n') 114 | 115 | # Check gradients by running checkNNGradients 116 | checkCostFunction(1.5) 117 | 118 | input('\nProgram paused. Press enter to continue.\n') 119 | 120 | ## ============== Part 6: Entering ratings for a new user =============== 121 | # Before we will train the collaborative filtering model, we will first 122 | # add ratings that correspond to a new user that we just observed. This 123 | # part of the code will also allow you to put in your own ratings for the 124 | # movies in our dataset! 125 | # 126 | movieList = loadMovieList() 127 | 128 | # Initialize my ratings 129 | my_ratings = np.zeros(1682) 130 | 131 | # Check the file movie_idx.txt for id of each movie in our dataset 132 | # For example, Toy Story (1995) has ID 1, so to rate it "4", you can set 133 | my_ratings[0] = 4 134 | 135 | # Or suppose did not enjoy Silence of the Lambs (1991), you can set 136 | my_ratings[97] = 2 137 | 138 | # We have selected a few movies we liked / did not like and the ratings we 139 | # gave are as follows: 140 | my_ratings[6] = 3 141 | my_ratings[11]= 5 142 | my_ratings[53] = 4 143 | my_ratings[63]= 5 144 | my_ratings[65]= 3 145 | my_ratings[68] = 5 146 | my_ratings[182] = 4 147 | my_ratings[225] = 5 148 | my_ratings[354]= 5 149 | 150 | print('\n\nNew user ratings:\n') 151 | for i in range(np.size(my_ratings)): 152 | if my_ratings[i] > 0: 153 | print("Rated %d for %s"%(my_ratings[i], movieList[i])) 154 | 155 | input('\nProgram paused. Press enter to continue.\n') 156 | 157 | ## ================== Part 7: Learning Movie Ratings ==================== 158 | # Now, you will train the collaborative filtering model on a movie rating 159 | # dataset of 1682 movies and 943 users 160 | # 161 | 162 | print('\nTraining collaborative filtering...\n') 163 | 164 | # Load data 165 | data = sio.loadmat('ex8_movies.mat') 166 | Y = data['Y'] 167 | R = data['R'] 168 | 169 | # Y is a 1682x943 matrix, containing ratings (1-5) of 1682 movies by 170 | # 943 users 171 | # 172 | # R is a 1682x943 matrix, where R(i,j) = 1 if and only if user j gave a 173 | # rating to movie i 174 | 175 | # Add our own ratings to the data matrix 176 | Y = np.vstack((my_ratings, Y.T)).T 177 | Temp = np.zeros(R.shape[0]) 178 | Temp[np.where(my_ratings != 0)] = 1 179 | Temp = Temp.astype(int) 180 | R = np.vstack((Temp, R.T)).T 181 | 182 | # Normalize Ratings 183 | Ynorm, Ymean = normalizeRatings(Y, R) 184 | 185 | # Useful Values 186 | num_users = Y.shape[1] 187 | num_movies = Y.shape[0] 188 | num_features = 10 189 | 190 | # Set Initial Parameters (Theta, X) 191 | X = np.random.randn(num_movies, num_features) 192 | Theta = np.random.randn(num_users, num_features) 193 | 194 | initial_parameters = np.append(X.flatten(), Theta.flatten()) 195 | 196 | # Set Regularization 197 | _lambda = 10 198 | costFunction = lambda t: cofiCostFunc(t, \ 199 | Y, R, num_users, num_movies, \ 200 | num_features, _lambda) 201 | res = minimize(costFunction, initial_parameters, method='CG', jac=True, options={'maxiter': 100}) 202 | print(res) 203 | 204 | # Unfold the returned theta back into U and W 205 | # X = reshape(theta(1:num_movies*num_features), num_movies, num_features); 206 | # Theta = reshape(theta(num_movies*num_features+1:end), ... 207 | # num_users, num_features); 208 | X = res.x[0:num_movies*num_features].reshape(num_movies, num_features) 209 | Theta = res.x[num_movies*num_features:].reshape(num_users, num_features) 210 | 211 | print('Recommender system learning completed.\n') 212 | 213 | input('\nProgram paused. Press enter to continue.\n') 214 | 215 | ## ================== Part 8: Recommendation for you ==================== 216 | # After training the model, you can now make recommendations by computing 217 | # the predictions matrix. 218 | # 219 | 220 | p = np.dot(X, Theta.T) 221 | my_predictions = p[:,0] + Ymean 222 | 223 | movieList = loadMovieList() 224 | 225 | idx = np.argsort(-my_predictions) 226 | print("\nTop recommendations for you:\n") 227 | for i in range(10): 228 | j = idx[i] 229 | print("Predicting rating %.1f for movie %s"%(my_predictions[j], movieList[j])) 230 | 231 | print("\n\nOriginal ratings provided:\n") 232 | for i in range(np.size(my_ratings)): 233 | if my_ratings[i] > 0: 234 | print("Rated %d for %s"%(my_ratings[i], movieList[i])) 235 | -------------------------------------------------------------------------------- /ml-ex8/ex8_movieParams.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex8/ex8_movieParams.mat -------------------------------------------------------------------------------- /ml-ex8/ex8_movies.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex8/ex8_movies.mat -------------------------------------------------------------------------------- /ml-ex8/ex8data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex8/ex8data1.mat -------------------------------------------------------------------------------- /ml-ex8/ex8data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makingagent/Coursera-Machine-Learning/8eea2cf8914929fd3fca0a78e645693c781d4904/ml-ex8/ex8data2.mat -------------------------------------------------------------------------------- /ml-ex8/loadMovieList.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | #GETMOVIELIST reads the fixed movie list in movie.txt and returns a 5 | #cell array of the words 6 | # movieList = GETMOVIELIST() reads the fixed movie list in movie.txt 7 | # and returns a cell array of the words in movieList. 8 | def loadMovieList(): 9 | 10 | ## Read the fixed movieulary list 11 | f = open('movie_ids.txt', 'r') 12 | 13 | # Store all movies in cell array movie{} 14 | n = 1682 # Total number of movies 15 | 16 | movieList = [1]*n 17 | for i in range(n): 18 | line = f.readline() 19 | movieList[i] = line[line.find(' ')+1:] 20 | f.close() 21 | 22 | return movieList 23 | -------------------------------------------------------------------------------- /ml-ex8/multivariateGaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #MULTIVARIATEGAUSSIAN Computes the probability density function of the 4 | #multivariate gaussian distribution. 5 | # p = MULTIVARIATEGAUSSIAN(X, mu, Sigma2) Computes the probability 6 | # density function of the examples X under the multivariate gaussian 7 | # distribution with parameters mu and Sigma2. If Sigma2 is a matrix, it is 8 | # treated as the covariance matrix. If Sigma2 is a vector, it is treated 9 | # as the \sigma^2 values of the variances in each dimension (a diagonal 10 | # covariance matrix) 11 | # 12 | def multivariateGaussian(X, mu, Sigma2): 13 | 14 | k = mu.size 15 | 16 | if Sigma2.reshape(1, -1).shape[0] == 1 or Sigma2.reshape(1, -1).shape[1] == 1: 17 | Sigma2 = np.diag(Sigma2) 18 | 19 | X = X - mu 20 | p = np.power(2*np.pi, -k/2) * np.power(np.linalg.det(Sigma2), -0.5) 21 | p *= np.exp(-0.5 * np.sum(X.T * np.dot(np.linalg.pinv(Sigma2), X.T), axis=0)) 22 | 23 | return p 24 | -------------------------------------------------------------------------------- /ml-ex8/normalizeRatings.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | #NORMALIZERATINGS Preprocess data by subtracting mean rating for every 5 | #movie (every row) 6 | # [Ynorm, Ymean] = NORMALIZERATINGS(Y, R) normalized Y so that each movie 7 | # has a rating of 0 on average, and returns the mean rating in Ymean. 8 | # 9 | def normalizeRatings(Y, R): 10 | 11 | m, n = Y.shape 12 | Ymean = np.zeros(m) 13 | Ynorm = np.zeros(Y.shape) 14 | 15 | for i in range(m): 16 | idx = np.where(R[i,:] == 1) 17 | Ymean[i] = np.mean(Y[i, idx]) 18 | Ynorm[i, idx] = Y[i, idx] - Ymean[i] 19 | 20 | return Ynorm, Ymean 21 | -------------------------------------------------------------------------------- /ml-ex8/selectThreshold.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | #SELECTTHRESHOLD Find the best threshold (epsilon) to use for selecting 5 | #outliers 6 | # [bestEpsilon bestF1] = SELECTTHRESHOLD(yval, pval) finds the best 7 | # threshold to use for selecting outliers based on the results from a 8 | # validation set (pval) and the ground truth (yval). 9 | # 10 | def selectThreshold(yval, pval): 11 | 12 | bestEpsilon = 0 13 | bestF1 = 0 14 | F1 = 0 15 | 16 | arr = np.linspace(np.amin(pval), np.amax(pval), 1000) 17 | 18 | for i in range(arr.size-2): 19 | 20 | epsilon = arr[i+1] 21 | 22 | predictions = pval < epsilon 23 | 24 | yval = yval.reshape(1,-1)[0] 25 | 26 | tp = np.sum((predictions == 1) & (yval == 1)) 27 | fp = np.sum((predictions == 1) & (yval == 0)) 28 | fn = np.sum((predictions == 0) & (yval == 1)) 29 | prec = tp / (tp + fp) 30 | rec = tp / (tp + fn) 31 | F1 = 2 * prec * rec /(prec + rec) 32 | 33 | if F1 > bestF1: 34 | bestF1 = F1 35 | bestEpsilon = epsilon 36 | 37 | return bestEpsilon, bestF1 38 | -------------------------------------------------------------------------------- /ml-ex8/visualizeFit.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from multivariateGaussian import multivariateGaussian 4 | 5 | #VISUALIZEFIT Visualize the dataset and its estimated distribution. 6 | # VISUALIZEFIT(X, p, mu, sigma2) This visualization shows you the 7 | # probability density function of the Gaussian distribution. Each example 8 | # has a location (x1, x2) that depends on its feature values. 9 | # 10 | def visualizeFit(X, mu, Sigma2): 11 | 12 | t = np.linspace(0, 35, 71) 13 | X1, X2 = np.meshgrid(t, t) 14 | 15 | Z = multivariateGaussian(np.vstack((X1.reshape(1,-1), X2.reshape(1,-1))).T, mu, Sigma2) 16 | Z = Z.reshape(X1.shape[0], -1) 17 | 18 | plt.figure() 19 | plt.plot(X[:, 0], X[:, 1],'bx', markersize=4) 20 | plt.axis([0, 30, 0, 30]) 21 | 22 | # Do not plot if there are infinities 23 | if np.sum(np.isinf(Z)) == 0: 24 | plt.contour(X1, X2, Z, np.power(10, np.linspace(-20, 0, 7))) 25 | --------------------------------------------------------------------------------