├── .gitignore ├── README.md ├── algorithms_in_matlab ├── tutos │ └── week_2 │ │ ├── featuresX.dat │ │ ├── matrices_vectors.m │ │ ├── myPlot.png │ │ ├── out.mat │ │ ├── textfile.txt │ │ ├── tuto_1_basic_operations.m │ │ ├── tuto_2_moving_data_around.m │ │ ├── tuto_3_computing_on_data.m │ │ ├── tuto_4_ploting_data.m │ │ ├── tuto_5_for_if.m │ │ ├── tuto_5_functions.m │ │ └── tuto_6_vectorization.m ├── week_2 │ ├── ex1.pdf │ └── ex1 │ │ ├── computeCost.m │ │ ├── computeCostMulti.m │ │ ├── ex1.m │ │ ├── ex1_multi.m │ │ ├── ex1data1.txt │ │ ├── ex1data2.txt │ │ ├── featureNormalize.m │ │ ├── gradientDescent.m │ │ ├── gradientDescentMulti.m │ │ ├── normalEqn.m │ │ ├── plotData.m │ │ └── warmUpExercise.m ├── week_3 │ ├── ex2.pdf │ └── ex2 │ │ ├── costFunction.m │ │ ├── costFunctionReg.m │ │ ├── ex2.m │ │ ├── ex2_reg.m │ │ ├── ex2data1.txt │ │ ├── ex2data2.txt │ │ ├── mapFeature.m │ │ ├── plotData.m │ │ ├── plotDecisionBoundary.m │ │ ├── predict.m │ │ └── sigmoid.m ├── week_4 │ ├── ex3.pdf │ └── ex3 │ │ ├── displayData.m │ │ ├── ex3.m │ │ ├── ex3_nn.m │ │ ├── ex3data1.mat │ │ ├── ex3weights.mat │ │ ├── fmincg.m │ │ ├── lrCostFunction.m │ │ ├── oneVsAll.m │ │ ├── predict.m │ │ ├── predictOneVsAll.m │ │ └── sigmoid.m ├── week_5 │ ├── ex4.pdf │ └── ex4 │ │ ├── checkNNGradients.m │ │ ├── computeNumericalGradient.m │ │ ├── debugInitializeWeights.m │ │ ├── displayData.m │ │ ├── ex4.m │ │ ├── ex4data1.mat │ │ ├── ex4weights.mat │ │ ├── fmincg.m │ │ ├── nnCostFunction.m │ │ ├── predict.m │ │ ├── randInitializeWeights.m │ │ ├── sigmoid.m │ │ └── sigmoidGradient.m ├── week_6 │ ├── ex5.pdf │ └── ex5 │ │ ├── ex5.m │ │ ├── ex5data1.mat │ │ ├── featureNormalize.m │ │ ├── fmincg.m │ │ ├── learningCurve.m │ │ ├── linearRegCostFunction.m │ │ ├── plotFit.m │ │ ├── polyFeatures.m │ │ ├── trainLinearReg.m │ │ └── validationCurve.m ├── week_7 │ ├── ex6.pdf │ └── ex6 │ │ ├── dataset3Params.m │ │ ├── emailFeatures.m │ │ ├── emailSample1.txt │ │ ├── emailSample2.txt │ │ ├── ex6.m │ │ ├── ex6_spam.m │ │ ├── ex6data1.mat │ │ ├── ex6data2.mat │ │ ├── ex6data3.mat │ │ ├── gaussianKernel.m │ │ ├── getVocabList.m │ │ ├── linearKernel.m │ │ ├── plotData.m │ │ ├── porterStemmer.m │ │ ├── processEmail.m │ │ ├── readFile.m │ │ ├── spamSample1.txt │ │ ├── spamSample2.txt │ │ ├── spamTest.mat │ │ ├── spamTrain.mat │ │ ├── svmPredict.m │ │ ├── svmTrain.m │ │ ├── visualizeBoundary.m │ │ ├── visualizeBoundaryLinear.m │ │ └── vocab.txt ├── week_8 │ ├── ex7.pdf │ └── ex7 │ │ ├── bird_small.mat │ │ ├── bird_small.png │ │ ├── computeCentroids.m │ │ ├── displayData.m │ │ ├── drawLine.m │ │ ├── ex7.m │ │ ├── ex7_pca.m │ │ ├── ex7data1.mat │ │ ├── ex7data2.mat │ │ ├── ex7faces.mat │ │ ├── featureNormalize.m │ │ ├── findClosestCentroids.m │ │ ├── kMeansInitCentroids.m │ │ ├── me.jpg │ │ ├── me_compressed.PNG │ │ ├── pca.m │ │ ├── plotDataPoints.m │ │ ├── plotProgresskMeans.m │ │ ├── projectData.m │ │ ├── recoverData.m │ │ └── runkMeans.m └── week_9 │ ├── ex8.pdf │ └── ex8 │ ├── checkCostFunction.m │ ├── cofiCostFunc.m │ ├── computeNumericalGradient.m │ ├── estimateGaussian.m │ ├── ex8.m │ ├── ex8_cofi.m │ ├── ex8_movieParams.mat │ ├── ex8_movies.mat │ ├── ex8data1.mat │ ├── ex8data2.mat │ ├── fmincg.m │ ├── loadMovieList.m │ ├── movie_ids.txt │ ├── multivariateGaussian.m │ ├── normalizeRatings.m │ ├── originalGaussian.m │ ├── selectThreshold.m │ └── visualizeFit.m ├── algorithms_in_python ├── week_2 │ ├── ex1.pdf │ └── ex1 │ │ ├── computeCost.py │ │ ├── computeCostMulti.py │ │ ├── ex1.py │ │ ├── ex1MultiFeatures.py │ │ ├── ex1data1.txt │ │ ├── ex1data2.txt │ │ ├── featureNormalize.py │ │ ├── gradientDescent.py │ │ ├── gradientDescentMulti.py │ │ ├── normalEqn.py │ │ ├── plotData.py │ │ └── warmUpExercise.py ├── week_3 │ ├── ex2.pdf │ └── ex2 │ │ ├── costFunction.py │ │ ├── ex2.py │ │ ├── ex2_reg.py │ │ ├── ex2data1.txt │ │ ├── ex2data2.txt │ │ ├── gradient.py │ │ ├── mapFeature.py │ │ ├── plotData.py │ │ ├── plotDecisionBoundary.py │ │ ├── predict.py │ │ └── sigmoid.py ├── week_4 │ ├── ex3.pdf │ └── ex3 │ │ ├── MNIST_DATA.csv │ │ ├── MNIST_DATA_LABEL.csv │ │ ├── Theta1.csv │ │ ├── Theta2.csv │ │ ├── displayData.py │ │ ├── ex3.py │ │ ├── ex3_nn.py │ │ ├── lrCostFunction.py │ │ ├── lrGradient.py │ │ ├── oneVsAll.py │ │ ├── predict.py │ │ ├── predictOneVsAll.py │ │ └── sigmoid.py ├── week_5 │ ├── ex4.pdf │ └── ex4 │ │ ├── Theta1.csv │ │ ├── Theta2.csv │ │ ├── checkNNGradients.py │ │ ├── computeNumericalGradient.py │ │ ├── debugInitializeWeights.py │ │ ├── displayData.py │ │ ├── ex4.py │ │ ├── ex4_features.csv │ │ ├── ex4_labels.csv │ │ ├── nnCostFunction.py │ │ ├── predict.py │ │ ├── randInitializeWeights.py │ │ ├── sigmoid.py │ │ └── sigmoidGradient.py ├── week_6 │ ├── ex5.pdf │ └── ex5 │ │ ├── cross_validation_features.csv │ │ ├── cross_validation_labels.csv │ │ ├── ex5.py │ │ ├── featureNormalize.py │ │ ├── learningCurve.py │ │ ├── linearRegCostFunction.py │ │ ├── plotFit.py │ │ ├── polyFeatures.py │ │ ├── polynomialDegreeCurve.py │ │ ├── r2_score.py │ │ ├── test_features.csv │ │ ├── test_labels.csv │ │ ├── trainLinearReg.py │ │ ├── train_features.csv │ │ ├── train_labels.csv │ │ └── validationCurve.py ├── week_7 │ ├── ex6.pdf │ └── ex6 │ │ ├── dataset3Params.py │ │ ├── emailFeatures.py │ │ ├── emailSample1.txt │ │ ├── emailSample2.txt │ │ ├── ex6.py │ │ ├── ex6_spam.py │ │ ├── ex6data1.mat │ │ ├── ex6data2.mat │ │ ├── ex6data3.mat │ │ ├── getVocabList.py │ │ ├── plotData.py │ │ ├── processEmail.py │ │ ├── readFile.py │ │ ├── spamSample1.txt │ │ ├── spamSample2.txt │ │ ├── spamTest.mat │ │ ├── spamTrain.mat │ │ ├── svmModel.py │ │ ├── visualizeBoundary.py │ │ └── vocab.txt ├── week_8 │ ├── ex7.pdf │ └── ex7 │ │ ├── bird_small.mat │ │ ├── bird_small.png │ │ ├── computeCentroids.py │ │ ├── displayData.py │ │ ├── ex7.py │ │ ├── ex7_pca.py │ │ ├── ex7data1.mat │ │ ├── ex7data2.mat │ │ ├── ex7faces.mat │ │ ├── featureNormalize.py │ │ ├── findClosestCentroids.py │ │ ├── kMeansInitCentroids.py │ │ ├── pca.py │ │ ├── plotDataPoints.py │ │ ├── plotProgresskMeans.py │ │ ├── projectData.py │ │ ├── recoverData.py │ │ └── runKMeans.py └── week_9 │ ├── ex8.pdf │ └── ex8 │ ├── checkCostFunction.py │ ├── cofiCostFunc.py │ ├── computeNumericalGradient.py │ ├── estimateGaussian.py │ ├── ex8.py │ ├── ex8_cofi.py │ ├── ex8_movieParams.mat │ ├── ex8_movies.mat │ ├── ex8data1.mat │ ├── ex8data2.mat │ ├── loadMovieList.py │ ├── movie_ids.txt │ ├── multivariateGaussian.py │ ├── normalizeRatings.py │ ├── originalGaussian.py │ ├── selectThreshold.py │ └── visualizeFit.py ├── figures ├── 1_linear_regression.png ├── 1_linear_regression_3d.png ├── 2_logistic_regression.png ├── 3_one_vs_all_classification.png ├── 4_viz_nn.png ├── 5_learning_curves.png ├── 6_spam.png ├── 6_svms.png ├── 7_keams_image_compression.png ├── 7_kmeans.png ├── 8_pca_datasets_before.png ├── 8_pca_faces.png ├── 9_anomaly_detection.png └── 9_collaborative_filtering.png ├── jupyter_notebooks ├── week_2 │ ├── ex1.ipynb │ └── ex1MultiFeatures.ipynb ├── week_3 │ ├── ex2.ipynb │ └── ex2_reg.ipynb ├── week_4 │ ├── ex3.ipynb │ └── ex3_nn.ipynb ├── week_5 │ └── ex4.ipynb ├── week_6 │ └── ex5.ipynb ├── week_7 │ ├── ex6.ipynb │ └── ex6_spam.ipynb ├── week_8 │ ├── ex7.ipynb │ └── ex7_pca.ipynb └── week_9 │ ├── ex8.ipynb │ └── ex8_cofi.ipynb └── lectures ├── w10_Learning_with_Large_Datasets └── Lecture17.pdf ├── w11_photo_ocr_and_ml_pipline_and_getting_more_data_and_ceiling_analysis └── Lecture18.pdf ├── w1_intro_linear_regression_with_one_variable ├── Lecture1.pdf ├── Lecture2.pdf └── Lecture3.pdf ├── w2_linear_regression_with_multiple_variables ├── Lecture4.pdf └── Lecture5.pdf ├── w3_logitic_regression_and_regularization ├── Lecture6.pdf └── Lecture7.pdf ├── w4_multi_class_classification_and_neural_networks_representation └── Lecture8.pdf ├── w5_neural_networks_learning └── Lecture9.pdf ├── w6_evaluating_learning_algortihm_variance_bias_precision_recall ├── Lecture10.pdf └── Lecture11.pdf ├── w7_support_vector_machines ├── Lecture12.pdf └── svm-notes-long-08.pdf ├── w8_unsupervised_learning_clustering_kmeans_pca ├── Lecture13.pdf └── Lecture14.pdf └── w9_anomaly_detection_and_recommender_systems ├── Lecture15.pdf └── Lecture16.pdf /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .ipynb_checkpoints -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/featuresX.dat: -------------------------------------------------------------------------------- 1 | 41 5 2 | 55 6 3 | 56 6 4 | 85 6 5 | 4585 5 6 | 685 6 -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/matrices_vectors.m: -------------------------------------------------------------------------------- 1 | % The ; denotes we are going back to a new row. 2 | A = [1, 2, 3; 4, 5, 6; 7, 8, 9; 10, 11, 12]; 3 | 4 | % Initialize a vector 5 | v = [1;2;3]; 6 | 7 | % Get the dimension of the matrix A where m = rows and n = columns 8 | [m,n] = size(A); 9 | 10 | % You could also store it this way 11 | dim_A = size(A); 12 | 13 | % Get the dimension of the vector v 14 | dim_v = size(v); 15 | 16 | % Now let's index into the 2nd row 3rd column of matrix A 17 | A_23 = A(2,3); 18 | 19 | 20 | % Initialize matrix A and B 21 | A = [1, 2, 4; 5, 3, 2]; 22 | B = [1, 3, 4; 1, 1, 1]; 23 | 24 | % Initialize constant s 25 | s = 2; 26 | 27 | % See how element-wise addition works 28 | add_AB = A + B ; 29 | 30 | % See how element-wise subtraction works 31 | sub_AB = A - B; 32 | 33 | % See how scalar multiplication works 34 | mult_As = A * s; 35 | 36 | % Divide A by s 37 | div_As = A / s; 38 | 39 | % What happens if we have a Matrix + scalar? 40 | add_As = A + s; 41 | 42 | % Initialize matrix A 43 | A = [1, 2, 3; 4, 5, 6;7, 8, 9] ; 44 | 45 | % Initialize vector v 46 | v = [1; 1; 1] ; 47 | 48 | % Multiply A * v 49 | Av = A * v; 50 | 51 | % Initialize a 3 by 2 matrix 52 | A = [1, 2; 3, 4;5, 6]; 53 | 54 | % Initialize a 2 by 1 matrix 55 | B = [1; 2] ; 56 | 57 | % We expect a resulting matrix of (3 by 2)*(2 by 1) = (3 by 1) 58 | mult_AB = A*B; 59 | 60 | % Make sure you understand why we got that result 61 | 62 | % Initialize random matrices A and B 63 | A = [1,2;4,5]; 64 | B = [1,1;0,2]; 65 | 66 | % Initialize a 2 by 2 identity matrix 67 | I = eye(2); 68 | 69 | % The above notation is the same as I = [1,0;0,1] 70 | 71 | % What happens when we multiply I*A ? 72 | IA = I*A ; 73 | 74 | % How about A*I ? 75 | AI = A*I ; 76 | 77 | % Compute A*B 78 | AB = A*B ; 79 | 80 | % Is it equal to B*A? 81 | BA = B*A ; 82 | 83 | % Note that IA = AI but AB != BA 84 | 85 | 86 | % Initialize matrix A 87 | A = [1,2,0;0,5,6;7,0,9]; 88 | 89 | % Transpose A 90 | A_trans = A' ; 91 | 92 | % Take the inverse of A 93 | A_inv = inv(A); 94 | 95 | % What is A^(-1)*A? 96 | A_invA = inv(A)*A; 97 | 98 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/myPlot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/tutos/week_2/myPlot.png -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/out.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/tutos/week_2/out.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/textfile.txt: -------------------------------------------------------------------------------- 1 | 4.1000000e+01 5.0000000e+00 2 | 5.5000000e+01 6.0000000e+00 3 | 5.6000000e+01 6.0000000e+00 4 | 8.5000000e+01 6.0000000e+00 5 | 4.5850000e+03 5.0000000e+00 6 | 6.8500000e+02 6.0000000e+00 7 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_1_basic_operations.m: -------------------------------------------------------------------------------- 1 | %this is a comment 2 | my_var = pi; %semicolon supressing output 3 | disp(sprintf('2 decimals: %0.2f', my_var)); 4 | format short 5 | my_var 6 | format long 7 | my_var 8 | 9 | %Matrices 10 | A = [1 2;3 4; 5 6] % (3,2) matrix 11 | A = [1 2; 12 | 3 4; 13 | 5 6] 14 | %Vectors 15 | V = [1; 2; 3] % 3 dimensional vector 16 | V = 1:6 % generate (1, m) vector 17 | V = 0:2:8 % generate (1, m) vector min:step:max 18 | 19 | M = ones(2,3) % generates (2,3) 1-everywhere matrix 20 | 21 | W = zeros(1,3) % generates (1,3) 0-everywhere matrix 22 | 23 | W = rand(1,3) % generates (1,3) uniformal-rand(from 0 to 1)-everywhere matrix 24 | 25 | W = randn(1,3) % generates (1,3)normal-rand(from 0 to 1)-everywhere matrix 26 | 27 | 28 | 29 | W = -6 + sqrt(10) * (randn(1, 10000)); 30 | hist(W, 50); 31 | 32 | I = eye(4) % 4 x 4 identity matrix 33 | 34 | %help command 35 | help eye 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_2_moving_data_around.m: -------------------------------------------------------------------------------- 1 | A = [1 2;3 4; 5 6]; 2 | disp(A); 3 | sz = size(A); % size is matrix! 4 | disp(sz); 5 | number_of_rows = size(A, 1); 6 | disp(number_of_rows); 7 | number_of_columns = size(A, 2); 8 | disp(number_of_columns); 9 | 10 | %vectors 11 | V = [1 2 3 4]; 12 | size_longest_dimension = length(V); % 4 13 | length([1;2;3;4;5]); % gives 5 (longest dim is number of rows = 5 14 | %print working dir 15 | pwd 16 | %change dir 17 | cd . 18 | %list files 19 | ls 20 | %load data 21 | load featuresX.dat 22 | load('featuresX.dat') 23 | %see current variables 24 | who 25 | disp(featuresX) 26 | size(featuresX) 27 | %see current variables structured 28 | whos 29 | 30 | %get rid of variables 31 | clear V 32 | 33 | %save variabke to disk 34 | save out.mat featuresX 35 | clear % clears everything 36 | load out.mat %load variables to mem 37 | whos 38 | 39 | save textfile.txt featuresX -ascii %save as txt 40 | 41 | %fetch data from matrix 42 | A = [1 2; 3 4; 5 6]; 43 | A(3, 2) 44 | A(2, :) % ":" means elt along that row/colum 45 | A([1 3], :) % mean first and third row of A 46 | 47 | A(:, 2) 48 | A(:, 2) = [10; 11; 12] % replace the last column 49 | A = [A, [100; 101; 102]] % append a new column vector to right 50 | 51 | size(A) % new become 3 by 3 52 | 53 | A(:) % put all elemt into a single vector 54 | 55 | A = [1 2; 3 4; 5 6]; 56 | B = [11 12; 13 14; 15 16]; 57 | 58 | C = [A B] %concat the matrices horiz (same as [A, B] 59 | C = [A; B] % concat verti 60 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_3_computing_on_data.m: -------------------------------------------------------------------------------- 1 | A = [1 2; 3 4; 5 6]; 2 | B = [11 12; 13 14; 15 16]; 3 | C = [1 1; 2 2]; 4 | 5 | A * C 6 | A .* B %element wise op multi one by one 7 | 8 | V = [1; 2; 3] 9 | 1 ./ V 10 | log(V) 11 | exp(V) 12 | abs(V) 13 | -V % same as -1 * V 14 | V + ones(length(V), 1) % add 1 to each element 15 | %or directly 16 | V + 1 17 | 18 | A' %transpose (turn left) 19 | 20 | max_val = max(V) 21 | [max_val, max_index] = max(V) 22 | 23 | %do elemnt wise comparaison, compare each elmt to 2 24 | V < 2 25 | 26 | %find all eltms inf to 3 27 | find(V < 3) 28 | 29 | M = magic(3) 30 | 31 | [row, colum] = find(M >= 7) 32 | 33 | %sum 34 | sum(M) 35 | %sum of each column 36 | sum(M, 1) 37 | %sum of each row 38 | sum(M, 2) 39 | 40 | %sum diag 41 | sum(sum(M .* eye(3))) 42 | disp('****') 43 | sum(sum(M .* flipud(eye(3)))) 44 | 45 | 46 | %mult 47 | prod(M) 48 | 49 | W = [2.5 3.2 7.9] 50 | 51 | %floor 52 | floor(W) 53 | ceil(W) 54 | 55 | %max of each column 56 | max(M, [], 1) 57 | 58 | %max of each row 59 | max(M, [], 2) 60 | 61 | %max of matrix 62 | max(M) 63 | 64 | %max value 65 | max(max(M)) % or max(M(:)) 66 | max(M(:)) 67 | 68 | 69 | %pseudo inverse 70 | pinv(M) 71 | 72 | identity = M * pinv(M) 73 | 74 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_4_ploting_data.m: -------------------------------------------------------------------------------- 1 | t = 0:0.01:0.98; 2 | y1 = sin(2*pi*4*t); 3 | plot(t, y1); 4 | y2 = cos(2*pi*4*t); 5 | hold on % used to plot n functions in same window 6 | plot(t, y2, 'r'); 7 | xlabel('time') 8 | ylabel('value') 9 | legend('sin', 'cos') 10 | title('my plot') 11 | print -dpng 'myPlot.png' 12 | %see help plot 13 | close % close figure 14 | figure(1); plot(t, y1); 15 | figure(2); plot(t, y2); 16 | 17 | %divide plot into 1x2 grid, access first element 18 | subplot(1, 2, 1); 19 | plot(t, y1); 20 | subplot(1, 2, 2); 21 | plot(t, y2); 22 | 23 | %change axis scale 24 | axis([0.5 1 -1 1]) 25 | 26 | % clear figure 27 | clf; 28 | 29 | A = magic(5); 30 | imagesc(A), colorbar, colormap gray; % comma chaining commands 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_5_for_if.m: -------------------------------------------------------------------------------- 1 | V = zeros(10, 1) 2 | %for 3 | for i=1:10, 4 | V(i) = 2 ^ i; 5 | end; 6 | V 7 | %while 8 | i = 1; 9 | while i <= 5, 10 | V(i) = 100; 11 | i = i+1; 12 | end 13 | V 14 | %using break and continue 15 | i = 1; 16 | while true, 17 | V(i) = 999; 18 | i = i+ 1; 19 | if i == 6, 20 | break; 21 | end 22 | end 23 | V 24 | %general syntax of if statement 25 | V(1) = 2; 26 | if V(1) == 1, 27 | disp('The value is one'); 28 | elseif V(1) ==2, 29 | disp('The value is two'); 30 | else 31 | disp('The value is not one or two'); 32 | end 33 | 34 | %call local function 35 | disp(fn(5)); 36 | %call function defined in a seperate file 37 | myfuns = tuto_5_functions; 38 | disp(myfuns.square_function(5)); 39 | %multi return 40 | [a, b] = myfuns.squareAndCubeFunction(5); 41 | disp(a) 42 | disp(b) 43 | 44 | %calling cost function 45 | X = [1 1; 2 2; 1 3]; 46 | y = [1; 2; 3]; 47 | theta = [0; 1]; 48 | 49 | J = myfuns.costFunctionJ(X,y, theta) 50 | theta = [0; 0]; 51 | J = myfuns.costFunctionJ(X,y, theta) 52 | % which is 53 | J = (1^2 + 2^2 + 3^2) / (2*3) 54 | 55 | 56 | 57 | %Add lib dir to octave search path 58 | addpath('.'); 59 | 60 | 61 | 62 | 63 | %define function, in a script it has to be the last thing defined 64 | function y = fn(x) 65 | y = x ^ 2; 66 | end 67 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_5_functions.m: -------------------------------------------------------------------------------- 1 | function funs = tuto_5_functions 2 | funs.square_function=@square_function; 3 | funs.squareAndCubeFunction=@squareAndCubeFunction; 4 | funs.costFunctionJ = @costFunctionJ; 5 | end 6 | 7 | function J = costFunctionJ(X, y, theta) 8 | %X is the design matrix containing our training examples 9 | %y is the class labels 10 | 11 | m = size(X, 1); % number of training examples 12 | predictions = X * theta; %predictions of hypothesis on all m examples 13 | sqrErrors = (predictions - y) .^ 2; % squared erros 14 | 15 | J = 1/ (2 *m) * sum(sqrErrors); 16 | end 17 | 18 | function y = square_function(x) 19 | y = x ^ 2; 20 | end 21 | 22 | function [y1, y2] = squareAndCubeFunction(x) 23 | y1 = x ^ 2; 24 | y2 = x ^ 3; 25 | end 26 | -------------------------------------------------------------------------------- /algorithms_in_matlab/tutos/week_2/tuto_6_vectorization.m: -------------------------------------------------------------------------------- 1 | %Always use vectorization instead of a for loop 2 | %to compute gradient descent for linear regression 3 | %for j 4 | % wj = wj - alpha * 1/m * SUMi (h(xi) - yi) * xji 5 | %instead do 6 | % W = W - alpha * 1/M * SUMi (h(xi) - yi) * Xi 7 | 8 | A = magic(3) 9 | A(1:3, 1:2) -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_2/ex1.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/computeCost.m: -------------------------------------------------------------------------------- 1 | function J = computeCost(X, y, theta) 2 | %COMPUTECOST Compute cost for linear regression 3 | % J = COMPUTECOST(X, y, theta) computes the cost of using theta as the 4 | % parameter for linear regression to fit the data points in X and y 5 | 6 | % Initialize some useful values 7 | m = length(y); % number of training examples 8 | predictions = X * theta; %predictions of hypothesis on all m examples 9 | sqrErrors = (predictions - y) .^ 2; % squared erros 10 | J = 1/ (2 *m) * sum(sqrErrors); 11 | 12 | 13 | % ========================================================================= 14 | 15 | end 16 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/computeCostMulti.m: -------------------------------------------------------------------------------- 1 | function J = computeCostMulti(X, y, theta) 2 | %COMPUTECOSTMULTI Compute cost for linear regression with multiple variables 3 | % J = COMPUTECOSTMULTI(X, y, theta) computes the cost of using theta as the 4 | % parameter for linear regression to fit the data points in X and y 5 | 6 | % Initialize some useful values 7 | m = length(y); % number of training examples 8 | 9 | % You need to return the following variables correctly 10 | m = length(y); % number of training examples 11 | predictions = X * theta; %predictions of hypothesis on all m examples 12 | sqrErrors = (predictions - y) .^ 2; % squared erros 13 | J = 1/ (2 *m) * sum(sqrErrors); 14 | 15 | % ====================== YOUR CODE HERE ====================== 16 | % Instructions: Compute the cost of a particular choice of theta 17 | % You should set J to the cost. 18 | 19 | 20 | 21 | 22 | 23 | % ========================================================================= 24 | 25 | end 26 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/ex1_multi.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_2/ex1/ex1_multi.m -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/ex1data1.txt: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/ex1data2.txt: -------------------------------------------------------------------------------- 1 | 2104,3,399900 2 | 1600,3,329900 3 | 2400,3,369000 4 | 1416,2,232000 5 | 3000,4,539900 6 | 1985,4,299900 7 | 1534,3,314900 8 | 1427,3,198999 9 | 1380,3,212000 10 | 1494,3,242500 11 | 1940,4,239999 12 | 2000,3,347000 13 | 1890,3,329999 14 | 4478,5,699900 15 | 1268,3,259900 16 | 2300,4,449900 17 | 1320,2,299900 18 | 1236,3,199900 19 | 2609,4,499998 20 | 3031,4,599000 21 | 1767,3,252900 22 | 1888,2,255000 23 | 1604,3,242900 24 | 1962,4,259900 25 | 3890,3,573900 26 | 1100,3,249900 27 | 1458,3,464500 28 | 2526,3,469000 29 | 2200,3,475000 30 | 2637,3,299900 31 | 1839,2,349900 32 | 1000,1,169900 33 | 2040,4,314900 34 | 3137,3,579900 35 | 1811,4,285900 36 | 1437,3,249900 37 | 1239,3,229900 38 | 2132,4,345000 39 | 4215,4,549000 40 | 2162,4,287000 41 | 1664,2,368500 42 | 2238,3,329900 43 | 2567,4,314000 44 | 1200,3,299000 45 | 852,2,179900 46 | 1852,4,299900 47 | 1203,3,239500 48 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/featureNormalize.m: -------------------------------------------------------------------------------- 1 | function [X_norm, mu, sigma] = featureNormalize(X) 2 | %FEATURENORMALIZE Normalizes the features in X 3 | % FEATURENORMALIZE(X) returns a normalized version of X where 4 | % the mean value of each feature is 0 and the standard deviation 5 | % is 1. This is often a good preprocessing step to do when 6 | % working with learning algorithms. 7 | 8 | % You need to set these values correctly 9 | 10 | mu = mean(X); % mean of each column 11 | sigma = std(X); % Y vector containing standard deviation for each column 12 | X_norm = (X-mu) ./ sigma; %BE CARFUL USE WISE DIVIDE ./ AND NOT / since A /B = A * B' 13 | 14 | % ====================== YOUR CODE HERE ====================== 15 | % Instructions: First, for each feature dimension, compute the mean 16 | % of the feature and subtract it from the dataset, 17 | % storing the mean value in mu. Next, compute the 18 | % standard deviation of each feature and divide 19 | % each feature by it's standard deviation, storing 20 | % the standard deviation in sigma. 21 | % 22 | % Note that X is a matrix where each column is a 23 | % feature and each row is an example. You need 24 | % to perform the normalization separately for 25 | % each feature. 26 | % 27 | % Hint: You might find the 'mean' and 'std' functions useful. 28 | % 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | % ============================================================ 39 | 40 | end 41 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/gradientDescent.m: -------------------------------------------------------------------------------- 1 | function [theta, J_history] = gradientDescent(X, y, theta, alpha, num_iters) 2 | %GRADIENTDESCENT Performs gradient descent to learn theta 3 | % theta = GRADIENTDESCENT(X, y, theta, alpha, num_iters) updates theta by 4 | % taking num_iters gradient steps with learning rate alpha 5 | 6 | % Initialize some useful values 7 | m = length(y); % number of training examples 8 | J_history = zeros(num_iters, 1); 9 | 10 | for iter = 1:num_iters 11 | %we do linear regression with identity as an activation function f(x) = 12 | %x 13 | prediction = X * theta; 14 | delta = (1/m) * X' * (prediction - y); 15 | theta = theta - alpha * delta; 16 | % Save the cost J in every iteration 17 | J_history(iter) = computeCost(X, y, theta); 18 | end 19 | 20 | end 21 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/gradientDescentMulti.m: -------------------------------------------------------------------------------- 1 | function [theta, J_history] = gradientDescentMulti(X, y, theta, alpha, num_iters) 2 | %GRADIENTDESCENTMULTI Performs gradient descent to learn theta 3 | % theta = GRADIENTDESCENTMULTI(x, y, theta, alpha, num_iters) updates theta by 4 | % taking num_iters gradient steps with learning rate alpha 5 | 6 | % Initialize some useful values 7 | m = length(y); % number of training examples 8 | J_history = zeros(num_iters, 1); 9 | 10 | for iter = 1:num_iters 11 | 12 | prediction = X * theta; 13 | delta = (1/m) * X' * (prediction - y); 14 | theta = theta - alpha * delta; 15 | % Save the cost J in every iteration 16 | J_history(iter) = computeCostMulti(X, y, theta); 17 | 18 | end 19 | 20 | end 21 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/normalEqn.m: -------------------------------------------------------------------------------- 1 | function [theta] = normalEqn(X, y) 2 | %NORMALEQN Computes the closed-form solution to linear regression 3 | % NORMALEQN(X,y) computes the closed-form solution to linear 4 | % regression using the normal equations. 5 | 6 | theta = zeros(size(X, 2), 1); 7 | 8 | % ====================== YOUR CODE HERE ====================== 9 | % Instructions: Complete the code to compute the closed form solution 10 | % to linear regression and put the result in theta. 11 | % 12 | 13 | % ---------------------- Sample Solution ---------------------- 14 | 15 | theta = pinv(X' * X) * X' * y ; 16 | 17 | 18 | % ------------------------------------------------------------- 19 | 20 | 21 | % ============================================================ 22 | 23 | end 24 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/plotData.m: -------------------------------------------------------------------------------- 1 | function plotData(x, y) 2 | %PLOTDATA Plots the data points x and y into a new figure 3 | % PLOTDATA(x,y) plots the data points and gives the figure axes labels of 4 | % population and profit. 5 | 6 | %figure; % open a new figure window 7 | 8 | plot(x, y, 'rx', 'MarkerSize', 10); 9 | xlabel('Profit in $10,000'); 10 | ylabel('Population of city in 10,000s'); 11 | 12 | % ====================== YOUR CODE HERE ====================== 13 | % Instructions: Plot the training data into a figure using the 14 | % "figure" and "plot" commands. Set the axes labels using 15 | % the "xlabel" and "ylabel" commands. Assume the 16 | % population and revenue data have been passed in 17 | % as the x and y arguments of this function. 18 | % 19 | % Hint: You can use the 'rx' option with plot to have the markers 20 | % appear as red crosses. Furthermore, you can make the 21 | % markers larger by using plot(..., 'rx', 'MarkerSize', 10); 22 | 23 | 24 | 25 | 26 | 27 | % ============================================================ 28 | 29 | end 30 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_2/ex1/warmUpExercise.m: -------------------------------------------------------------------------------- 1 | function A = warmUpExercise() 2 | %WARMUPEXERCISE Example function in octave 3 | % A = WARMUPEXERCISE() is an example function that returns the 5x5 identity matrix 4 | 5 | A = eye(5); 6 | % ============= YOUR CODE HERE ============== 7 | % Instructions: Return the 5x5 identity matrix 8 | % In octave, we return values by defining which variables 9 | % represent the return values (at the top of the file) 10 | % and then set them accordingly. 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | % =========================================== 19 | 20 | 21 | end 22 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_3/ex2.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/costFunction.m: -------------------------------------------------------------------------------- 1 | function [J, grad] = costFunction(theta, X, y) 2 | %COSTFUNCTION Compute cost and gradient for logistic regression 3 | % J = COSTFUNCTION(theta, X, y) computes the cost of using theta as the 4 | % parameter for logistic regression and the gradient of the cost 5 | % w.r.t. to the parameters. 6 | 7 | % Initialize some useful values 8 | m = length(y); % number of training examples 9 | 10 | prediction = sigmoid(X * theta); 11 | 12 | cost_y_1 = (1 - y) .* log(1 - prediction); 13 | cost_y_0 = -1 .* y .* log(prediction); 14 | 15 | J = 1/m * sum(cost_y_0 - cost_y_1); 16 | grad = 1/m .* X' * (prediction - y); 17 | 18 | 19 | 20 | % ====================== YOUR CODE HERE ====================== 21 | % Instructions: Compute the cost of a particular choice of theta. 22 | % You should set J to the cost. 23 | % Compute the partial derivatives and set grad to the partial 24 | % derivatives of the cost w.r.t. each parameter in theta 25 | % 26 | % Note: grad should have the same dimensions as theta 27 | % 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | % ============================================================= 37 | 38 | end 39 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/costFunctionReg.m: -------------------------------------------------------------------------------- 1 | function [J, grad] = costFunctionReg(theta, X, y, lambda) 2 | %COSTFUNCTIONREG Compute cost and gradient for logistic regression with regularization 3 | % J = COSTFUNCTIONREG(theta, X, y, lambda) computes the cost of using 4 | % theta as the parameter for regularized logistic regression and the 5 | % gradient of the cost w.r.t. to the parameters. 6 | 7 | % Initialize some useful values 8 | m = length(y); % number of training examples 9 | prediction = sigmoid(X * theta); 10 | 11 | cost_y_1 = (1 - y) .* log(1 - prediction); 12 | cost_y_0 = -1 .* y .* log(prediction); 13 | 14 | grad_dim = size(X,2); 15 | 16 | J = 1/m * sum(cost_y_0 - cost_y_1) + (lambda/(2 * m)) * sum(theta(2:grad_dim) .* theta(2:grad_dim)); 17 | 18 | grad_without_regul = 1/m .* X' * (prediction - y) ; 19 | 20 | 21 | 22 | grad = [grad_without_regul(1); grad_without_regul(2:grad_dim) + (lambda/m) .* theta(2:grad_dim)]; 23 | 24 | 25 | 26 | 27 | 28 | 29 | % ============================================================= 30 | 31 | end 32 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/ex2data2.txt: -------------------------------------------------------------------------------- 1 | 0.051267,0.69956,1 2 | -0.092742,0.68494,1 3 | -0.21371,0.69225,1 4 | -0.375,0.50219,1 5 | -0.51325,0.46564,1 6 | -0.52477,0.2098,1 7 | -0.39804,0.034357,1 8 | -0.30588,-0.19225,1 9 | 0.016705,-0.40424,1 10 | 0.13191,-0.51389,1 11 | 0.38537,-0.56506,1 12 | 0.52938,-0.5212,1 13 | 0.63882,-0.24342,1 14 | 0.73675,-0.18494,1 15 | 0.54666,0.48757,1 16 | 0.322,0.5826,1 17 | 0.16647,0.53874,1 18 | -0.046659,0.81652,1 19 | -0.17339,0.69956,1 20 | -0.47869,0.63377,1 21 | -0.60541,0.59722,1 22 | -0.62846,0.33406,1 23 | -0.59389,0.005117,1 24 | -0.42108,-0.27266,1 25 | -0.11578,-0.39693,1 26 | 0.20104,-0.60161,1 27 | 0.46601,-0.53582,1 28 | 0.67339,-0.53582,1 29 | -0.13882,0.54605,1 30 | -0.29435,0.77997,1 31 | -0.26555,0.96272,1 32 | -0.16187,0.8019,1 33 | -0.17339,0.64839,1 34 | -0.28283,0.47295,1 35 | -0.36348,0.31213,1 36 | -0.30012,0.027047,1 37 | -0.23675,-0.21418,1 38 | -0.06394,-0.18494,1 39 | 0.062788,-0.16301,1 40 | 0.22984,-0.41155,1 41 | 0.2932,-0.2288,1 42 | 0.48329,-0.18494,1 43 | 0.64459,-0.14108,1 44 | 0.46025,0.012427,1 45 | 0.6273,0.15863,1 46 | 0.57546,0.26827,1 47 | 0.72523,0.44371,1 48 | 0.22408,0.52412,1 49 | 0.44297,0.67032,1 50 | 0.322,0.69225,1 51 | 0.13767,0.57529,1 52 | -0.0063364,0.39985,1 53 | -0.092742,0.55336,1 54 | -0.20795,0.35599,1 55 | -0.20795,0.17325,1 56 | -0.43836,0.21711,1 57 | -0.21947,-0.016813,1 58 | -0.13882,-0.27266,1 59 | 0.18376,0.93348,0 60 | 0.22408,0.77997,0 61 | 0.29896,0.61915,0 62 | 0.50634,0.75804,0 63 | 0.61578,0.7288,0 64 | 0.60426,0.59722,0 65 | 0.76555,0.50219,0 66 | 0.92684,0.3633,0 67 | 0.82316,0.27558,0 68 | 0.96141,0.085526,0 69 | 0.93836,0.012427,0 70 | 0.86348,-0.082602,0 71 | 0.89804,-0.20687,0 72 | 0.85196,-0.36769,0 73 | 0.82892,-0.5212,0 74 | 0.79435,-0.55775,0 75 | 0.59274,-0.7405,0 76 | 0.51786,-0.5943,0 77 | 0.46601,-0.41886,0 78 | 0.35081,-0.57968,0 79 | 0.28744,-0.76974,0 80 | 0.085829,-0.75512,0 81 | 0.14919,-0.57968,0 82 | -0.13306,-0.4481,0 83 | -0.40956,-0.41155,0 84 | -0.39228,-0.25804,0 85 | -0.74366,-0.25804,0 86 | -0.69758,0.041667,0 87 | -0.75518,0.2902,0 88 | -0.69758,0.68494,0 89 | -0.4038,0.70687,0 90 | -0.38076,0.91886,0 91 | -0.50749,0.90424,0 92 | -0.54781,0.70687,0 93 | 0.10311,0.77997,0 94 | 0.057028,0.91886,0 95 | -0.10426,0.99196,0 96 | -0.081221,1.1089,0 97 | 0.28744,1.087,0 98 | 0.39689,0.82383,0 99 | 0.63882,0.88962,0 100 | 0.82316,0.66301,0 101 | 0.67339,0.64108,0 102 | 1.0709,0.10015,0 103 | -0.046659,-0.57968,0 104 | -0.23675,-0.63816,0 105 | -0.15035,-0.36769,0 106 | -0.49021,-0.3019,0 107 | -0.46717,-0.13377,0 108 | -0.28859,-0.060673,0 109 | -0.61118,-0.067982,0 110 | -0.66302,-0.21418,0 111 | -0.59965,-0.41886,0 112 | -0.72638,-0.082602,0 113 | -0.83007,0.31213,0 114 | -0.72062,0.53874,0 115 | -0.59389,0.49488,0 116 | -0.48445,0.99927,0 117 | -0.0063364,0.99927,0 118 | 0.63265,-0.030612,0 119 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/mapFeature.m: -------------------------------------------------------------------------------- 1 | function out = mapFeature(X1, X2) 2 | % MAPFEATURE Feature mapping function to polynomial features 3 | % 4 | % MAPFEATURE(X1, X2) maps the two input features 5 | % to quadratic features used in the regularization exercise. 6 | % 7 | % Returns a new feature array with more features, comprising of 8 | % X1, X2, X1.^2, X2.^2, X1*X2, X1*X2.^2, etc.. 9 | % 10 | % Inputs X1, X2 must be the same size 11 | % 12 | 13 | degree = 6; 14 | out = ones(size(X1(:,1))); 15 | for i = 1:degree 16 | for j = 0:i 17 | out(:, end+1) = (X1.^(i-j)).*(X2.^j); 18 | end 19 | end 20 | 21 | end -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/plotData.m: -------------------------------------------------------------------------------- 1 | function plotData(X, y) 2 | %PLOTDATA Plots the data points X and y into a new figure 3 | % PLOTDATA(x,y) plots the data points with + for the positive examples 4 | % and o for the negative examples. X is assumed to be a Mx2 matrix. 5 | 6 | % Create New Figure 7 | figure; hold on; 8 | 9 | % Find Indices of Positive and Negative Examples 10 | pos = find(y==1); 11 | neg = find(y == 0); 12 | % Plot Examples 13 | plot(X(pos, 1), X(pos, 2), 'k+','LineWidth', 2, 'MarkerSize', 7); 14 | plot(X(neg, 1), X(neg, 2), 'ko', 'MarkerFaceColor', 'y', 'MarkerSize', 7); 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | % ========================================================================= 23 | 24 | 25 | 26 | hold off; 27 | 28 | end 29 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/plotDecisionBoundary.m: -------------------------------------------------------------------------------- 1 | function plotDecisionBoundary(theta, X, y) 2 | %PLOTDECISIONBOUNDARY Plots the data points X and y into a new figure with 3 | %the decision boundary defined by theta 4 | % PLOTDECISIONBOUNDARY(theta, X,y) plots the data points with + for the 5 | % positive examples and o for the negative examples. X is assumed to be 6 | % a either 7 | % 1) Mx3 matrix, where the first column is an all-ones column for the 8 | % intercept. 9 | % 2) MxN, N>3 matrix, where the first column is all-ones 10 | 11 | % Plot Data 12 | plotData(X(:,2:3), y); 13 | hold on 14 | 15 | if size(X, 2) <= 3 16 | % Only need 2 points to define a line, so choose two endpoints 17 | plot_x = [min(X(:,2))-2, max(X(:,2))+2]; 18 | 19 | % Calculate the decision boundary line 20 | plot_y = (-1./theta(3)).*(theta(2).*plot_x + theta(1)); 21 | 22 | % Plot, and adjust axes for better viewing 23 | plot(plot_x, plot_y) 24 | 25 | % Legend, specific for the exercise 26 | legend('Admitted', 'Not admitted', 'Decision Boundary') 27 | axis([30, 100, 30, 100]) 28 | else 29 | % Here is the grid range 30 | u = linspace(-1, 1.5, 50); 31 | v = linspace(-1, 1.5, 50); 32 | 33 | z = zeros(length(u), length(v)); 34 | % Evaluate z = theta*x over the grid 35 | for i = 1:length(u) 36 | for j = 1:length(v) 37 | z(i,j) = mapFeature(u(i), v(j))*theta; 38 | end 39 | end 40 | z = z'; % important to transpose z before calling contour 41 | 42 | % Plot z = 0 43 | % Notice you need to specify the range [0, 0] 44 | contour(u, v, z, [0, 0], 'LineWidth', 2) 45 | end 46 | hold off 47 | 48 | end 49 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/predict.m: -------------------------------------------------------------------------------- 1 | function p = predict(theta, X) 2 | %PREDICT Predict whether the label is 0 or 1 using learned logistic 3 | %regression parameters theta 4 | % p = PREDICT(theta, X) computes the predictions for X using a 5 | % threshold at 0.5 (i.e., if sigmoid(theta'*x) >= 0.5, predict 1) 6 | 7 | m = size(X, 1); % Number of training examples 8 | 9 | % You need to return the following variables correctly 10 | 11 | p = sigmoid(X *theta) >= 0.5; 12 | 13 | 14 | % ========================================================================= 15 | 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_3/ex2/sigmoid.m: -------------------------------------------------------------------------------- 1 | function g = sigmoid(z) 2 | %SIGMOID Compute sigmoid function 3 | % g = SIGMOID(z) computes the sigmoid of z. 4 | 5 | % ====================== YOUR CODE HERE ====================== 6 | % Instructions: Compute the sigmoid of each value of z (z can be a matrix, 7 | % vector or scalar). 8 | 9 | g = 1 ./ (1 + exp(-z)); 10 | 11 | 12 | 13 | 14 | 15 | % ============================================================= 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_4/ex3.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/displayData.m: -------------------------------------------------------------------------------- 1 | function [h, display_array] = displayData(X, example_width) 2 | %DISPLAYDATA Display 2D data in a nice grid 3 | % [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 4 | % stored in X in a nice grid. It returns the figure handle h and the 5 | % displayed array if requested. 6 | 7 | % Set example_width automatically if not passed in 8 | if ~exist('example_width', 'var') || isempty(example_width) 9 | example_width = round(sqrt(size(X, 2))); 10 | end 11 | 12 | % Gray Image 13 | colormap(gray); 14 | 15 | % Compute rows, cols 16 | [m n] = size(X); 17 | example_height = (n / example_width); 18 | 19 | % Compute number of items to display 20 | display_rows = floor(sqrt(m)); 21 | display_cols = ceil(m / display_rows); 22 | 23 | % Between images padding 24 | pad = 1; 25 | 26 | % Setup blank display 27 | display_array = - ones(pad + display_rows * (example_height + pad), ... 28 | pad + display_cols * (example_width + pad)); 29 | 30 | % Copy each example into a patch on the display array 31 | curr_ex = 1; 32 | for j = 1:display_rows 33 | for i = 1:display_cols 34 | if curr_ex > m, 35 | break; 36 | end 37 | % Copy the patch 38 | 39 | % Get the max value of the patch 40 | max_val = max(abs(X(curr_ex, :))); 41 | display_array(pad + (j - 1) * (example_height + pad) + (1:example_height), ... 42 | pad + (i - 1) * (example_width + pad) + (1:example_width)) = ... 43 | reshape(X(curr_ex, :), example_height, example_width) / max_val; 44 | curr_ex = curr_ex + 1; 45 | end 46 | if curr_ex > m, 47 | break; 48 | end 49 | end 50 | 51 | % Display Image 52 | h = imagesc(display_array, [-1 1]); 53 | 54 | % Do not show axis 55 | axis image off 56 | 57 | drawnow; 58 | 59 | end 60 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/ex3.m: -------------------------------------------------------------------------------- 1 | %% Machine Learning Online Class - Exercise 3 | Part 1: One-vs-all 2 | 3 | % Instructions 4 | % ------------ 5 | % 6 | % This file contains code that helps you get started on the 7 | % linear exercise. You will need to complete the following functions 8 | % in this exericse: 9 | % 10 | % lrCostFunction.m (logistic regression cost function) 11 | % oneVsAll.m 12 | % predictOneVsAll.m 13 | % predict.m 14 | % 15 | % For this exercise, you will not need to change any code in this file, 16 | % or any other files other than those mentioned above. 17 | % 18 | 19 | %% Initialization 20 | clear ; close all; clc 21 | 22 | %% Setup the parameters you will use for this part of the exercise 23 | input_layer_size = 400; % 20x20 Input Images of Digits 24 | num_labels = 10; % 10 labels, from 1 to 10 25 | % (note that we have mapped "0" to label 10) 26 | 27 | %% =========== Part 1: Loading and Visualizing Data ============= 28 | % We start the exercise by first loading and visualizing the dataset. 29 | % You will be working with a dataset that contains handwritten digits. 30 | % 31 | 32 | % Load Training Data 33 | fprintf('Loading and Visualizing Data ...\n') 34 | 35 | load('ex3data1.mat'); % training data stored in arrays X, y 36 | m = size(X, 1); 37 | 38 | %csvwrite("MNIST_DATA.csv",X) 39 | 40 | %csvwrite("MNIST_DATA_LABEL.csv", y) 41 | 42 | % Randomly select 100 data points to display 43 | rand_indices = randperm(m); 44 | sel = X(1:100, :); 45 | 46 | displayData(sel); 47 | 48 | fprintf('Program paused. Press enter to continue.\n'); 49 | pause; 50 | 51 | %% ============ Part 2a: Vectorize Logistic Regression ============ 52 | % In this part of the exercise, you will reuse your logistic regression 53 | % code from the last exercise. You task here is to make sure that your 54 | % regularized logistic regression implementation is vectorized. After 55 | % that, you will implement one-vs-all classification for the handwritten 56 | % digit dataset. 57 | % 58 | 59 | % Test case for lrCostFunction 60 | fprintf('\nTesting lrCostFunction() with regularization'); 61 | 62 | theta_t = [-2; -1; 1; 2]; 63 | X_t = [ones(5,1) reshape(1:15,5,3)/10]; 64 | y_t = ([1;0;1;0;1] >= 0.5); 65 | lambda_t = 3; 66 | [J grad] = lrCostFunction(theta_t, X_t, y_t, lambda_t); 67 | 68 | fprintf('\nCost: %f\n', J); 69 | fprintf('Expected cost: 2.534819\n'); 70 | fprintf('Gradients:\n'); 71 | fprintf(' %f \n', grad); 72 | fprintf('Expected gradients:\n'); 73 | fprintf(' 0.146561\n -0.548558\n 0.724722\n 1.398003\n'); 74 | 75 | fprintf('Program paused. Press enter to continue.\n'); 76 | pause; 77 | %% ============ Part 2b: One-vs-All Training ============ 78 | fprintf('\nTraining One-vs-All Logistic Regression...\n') 79 | 80 | lambda = 0.1; 81 | [all_theta] = oneVsAll(X, y, num_labels, lambda); 82 | 83 | fprintf('Program paused. Press enter to continue.\n'); 84 | pause; 85 | 86 | 87 | %% ================ Part 3: Predict for One-Vs-All ================ 88 | 89 | pred = predictOneVsAll(all_theta, X); 90 | 91 | fprintf('\nTraining Set Accuracy: %f\n', mean(double(pred == y)) * 100); 92 | 93 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/ex3_nn.m: -------------------------------------------------------------------------------- 1 | %% Machine Learning Online Class - Exercise 3 | Part 2: Neural Networks 2 | 3 | % Instructions 4 | % ------------ 5 | % 6 | % This file contains code that helps you get started on the 7 | % linear exercise. You will need to complete the following functions 8 | % in this exericse: 9 | % 10 | % lrCostFunction.m (logistic regression cost function) 11 | % oneVsAll.m 12 | % predictOneVsAll.m 13 | % predict.m 14 | % 15 | % For this exercise, you will not need to change any code in this file, 16 | % or any other files other than those mentioned above. 17 | % 18 | 19 | %% Initialization 20 | clear ; close all; clc 21 | 22 | %% Setup the parameters you will use for this exercise 23 | input_layer_size = 400; % 20x20 Input Images of Digits 24 | hidden_layer_size = 25; % 25 hidden units 25 | num_labels = 10; % 10 labels, from 1 to 10 26 | % (note that we have mapped "0" to label 10) 27 | 28 | %% =========== Part 1: Loading and Visualizing Data ============= 29 | % We start the exercise by first loading and visualizing the dataset. 30 | % You will be working with a dataset that contains handwritten digits. 31 | % 32 | 33 | % Load Training Data 34 | fprintf('Loading and Visualizing Data ...\n') 35 | 36 | load('ex3data1.mat'); 37 | m = size(X, 1); 38 | 39 | % Randomly select 100 data points to display 40 | sel = randperm(size(X, 1)); 41 | sel = sel(1:100); 42 | 43 | displayData(X(sel, :)); 44 | 45 | fprintf('Program paused. Press enter to continue.\n'); 46 | pause; 47 | 48 | %% ================ Part 2: Loading Pameters ================ 49 | % In this part of the exercise, we load some pre-initialized 50 | % neural network parameters. 51 | 52 | fprintf('\nLoading Saved Neural Network Parameters ...\n') 53 | 54 | % Load the weights into variables Theta1 and Theta2 55 | load('ex3weights.mat'); 56 | 57 | %% ================= Part 3: Implement Predict ================= 58 | % After training the neural network, we would like to use it to predict 59 | % the labels. You will now implement the "predict" function to use the 60 | % neural network to predict the labels of the training set. This lets 61 | % you compute the training set accuracy. 62 | 63 | pred = predict(Theta1, Theta2, X); 64 | 65 | fprintf('\nTraining Set Accuracy: %f\n', mean(double(pred == y)) * 100); 66 | 67 | fprintf('Program paused. Press enter to continue.\n'); 68 | pause; 69 | 70 | % To give you an idea of the network's output, you can also run 71 | % through the examples one at the a time to see what it is predicting. 72 | 73 | % Randomly permute examples 74 | rp = randperm(m); 75 | 76 | for i = 1:m 77 | % Display 78 | fprintf('\nDisplaying Example Image\n'); 79 | displayData(X(rp(i), :)); 80 | 81 | pred = predict(Theta1, Theta2, X(rp(i),:)); 82 | fprintf('\nNeural Network Prediction: %d (digit %d)\n', pred, mod(pred, 10)); 83 | 84 | % Pause with quit option 85 | s = input('Paused - press enter to continue, q to exit:','s'); 86 | if s == 'q' 87 | break 88 | end 89 | end 90 | 91 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/ex3data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_4/ex3/ex3data1.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/ex3weights.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_4/ex3/ex3weights.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/lrCostFunction.m: -------------------------------------------------------------------------------- 1 | function [J, grad] = lrCostFunction(theta, X, y, lambda) 2 | %LRCOSTFUNCTION Compute cost and gradient for logistic regression with 3 | %regularization 4 | % J = LRCOSTFUNCTION(theta, X, y, lambda) computes the cost of using 5 | % theta as the parameter for regularized logistic regression and the 6 | % gradient of the cost w.r.t. to the parameters. 7 | 8 | 9 | 10 | m = length(y); % number of training examples 11 | prediction = sigmoid(X * theta); 12 | 13 | cost_y_1 = (1 - y) .* log(1 - prediction); 14 | cost_y_0 = -1 .* y .* log(prediction); 15 | 16 | grad_dim = size(X,2); 17 | 18 | J = 1/m * sum(cost_y_0 - cost_y_1) + (lambda/(2 * m)) * sum(theta(2:grad_dim) .* theta(2:grad_dim)); 19 | 20 | grad_without_regul = 1/m .* X' * (prediction - y) ; 21 | 22 | grad = [grad_without_regul(1); grad_without_regul(2:grad_dim) + (lambda/m) .* theta(2:grad_dim)]; 23 | 24 | end 25 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/oneVsAll.m: -------------------------------------------------------------------------------- 1 | function [all_theta] = oneVsAll(X, y, num_labels, lambda) 2 | %ONEVSALL trains multiple logistic regression classifiers and returns all 3 | %the classifiers in a matrix all_theta, where the i-th row of all_theta 4 | %corresponds to the classifier for label i 5 | % [all_theta] = ONEVSALL(X, y, num_labels, lambda) trains num_labels 6 | % logistic regression classifiers and returns each of these classifiers 7 | % in a matrix all_theta, where the i-th row of all_theta corresponds 8 | % to the classifier for label i 9 | 10 | % Some useful variables 11 | m = size(X, 1); 12 | n = size(X, 2); 13 | 14 | % You need to return the following variables correctly 15 | all_theta = zeros(num_labels, n + 1); 16 | 17 | % Add ones to the X data matrix 18 | X = [ones(m, 1) X]; 19 | 20 | for c = 1:num_labels 21 | % Set Initial theta 22 | initial_theta = zeros(n + 1, 1); 23 | disp('optimizing theta ' + c) 24 | % 25 | % Set options for fminunc 26 | options = optimset('GradObj', 'on', 'MaxIter', 50,'Display','off'); 27 | % 28 | % Run fmincg to obtain the optimal theta 29 | % This function will return theta and the cost 30 | [theta] = ... 31 | fmincg(@(t)(lrCostFunction(t, X, (y == c), lambda)), ... 32 | initial_theta, options); 33 | all_theta(c,:) = theta; 34 | end 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | % ========================================================================= 51 | 52 | 53 | end 54 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/predict.m: -------------------------------------------------------------------------------- 1 | function p = predict(Theta1, Theta2, X) 2 | %PREDICT Predict the label of an input given a trained neural network 3 | % p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the 4 | % trained weights of a neural network (Theta1, Theta2) 5 | 6 | % Useful values 7 | m = size(X, 1); 8 | 9 | a1 = sigmoid([ones(m, 1) X] * Theta1'); 10 | 11 | a2 = sigmoid([ones(m, 1) a1] * Theta2'); 12 | 13 | [~, p] = max(a2, [], 2); 14 | 15 | % ========================================================================= 16 | 17 | 18 | end 19 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/predictOneVsAll.m: -------------------------------------------------------------------------------- 1 | function p = predictOneVsAll(all_theta, X) 2 | %PREDICT Predict the label for a trained one-vs-all classifier. The labels 3 | %are in the range 1..K, where K = size(all_theta, 1). 4 | % p = PREDICTONEVSALL(all_theta, X) will return a vector of predictions 5 | % for each example in the matrix X. Note that X contains the examples in 6 | % rows. all_theta is a matrix where the i-th row is a trained logistic 7 | % regression theta vector for the i-th class. You should set p to a vector 8 | % of values from 1..K (e.g., p = [1; 3; 1; 2] predicts classes 1, 3, 1, 2 9 | % for 4 examples) 10 | 11 | m = size(X, 1); 12 | 13 | % You need to return the following variables correctly 14 | 15 | % Add ones to the X data matrix 16 | X = [ones(m, 1) X]; 17 | 18 | [~, p] = max(sigmoid(X * all_theta'), [], 2); 19 | 20 | % ========================================================================= 21 | 22 | 23 | end 24 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_4/ex3/sigmoid.m: -------------------------------------------------------------------------------- 1 | function g = sigmoid(z) 2 | %SIGMOID Compute sigmoid functoon 3 | % J = SIGMOID(z) computes the sigmoid of z. 4 | 5 | g = 1.0 ./ (1.0 + exp(-z)); 6 | end 7 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_5/ex4.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/checkNNGradients.m: -------------------------------------------------------------------------------- 1 | function checkNNGradients(lambda) 2 | %CHECKNNGRADIENTS Creates a small neural network to check the 3 | %backpropagation gradients 4 | % CHECKNNGRADIENTS(lambda) Creates a small neural network to check the 5 | % backpropagation gradients, it will output the analytical gradients 6 | % produced by your backprop code and the numerical gradients (computed 7 | % using computeNumericalGradient). These two gradient computations should 8 | % result in very similar values. 9 | % 10 | 11 | if ~exist('lambda', 'var') || isempty(lambda) 12 | lambda = 0; 13 | end 14 | 15 | input_layer_size = 3; 16 | hidden_layer_size = 5; 17 | num_labels = 3; 18 | m = 5; 19 | 20 | % We generate some 'random' test data 21 | Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size); 22 | Theta2 = debugInitializeWeights(num_labels, hidden_layer_size); 23 | % Reusing debugInitializeWeights to generate X 24 | X = debugInitializeWeights(m, input_layer_size - 1); 25 | y = 1 + mod(1:m, num_labels)'; 26 | 27 | % Unroll parameters 28 | nn_params = [Theta1(:) ; Theta2(:)]; 29 | 30 | % Short hand for cost function 31 | costFunc = @(p) nnCostFunction(p, input_layer_size, hidden_layer_size, ... 32 | num_labels, X, y, lambda); 33 | 34 | [cost, grad] = costFunc(nn_params); 35 | numgrad = computeNumericalGradient(costFunc, nn_params); 36 | 37 | % Visually examine the two gradient computations. The two columns 38 | % you get should be very similar. 39 | disp([numgrad grad]); 40 | fprintf(['The above two columns you get should be very similar.\n' ... 41 | '(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n']); 42 | 43 | % Evaluate the norm of the difference between two solutions. 44 | % If you have a correct implementation, and assuming you used EPSILON = 0.0001 45 | % in computeNumericalGradient.m, then diff below should be less than 1e-9 46 | diff = norm(numgrad-grad)/norm(numgrad+grad); 47 | 48 | fprintf(['If your backpropagation implementation is correct, then \n' ... 49 | 'the relative difference will be small (less than 1e-9). \n' ... 50 | '\nRelative Difference: %g\n'], diff); 51 | 52 | end 53 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/computeNumericalGradient.m: -------------------------------------------------------------------------------- 1 | function numgrad = computeNumericalGradient(J, theta) 2 | %COMPUTENUMERICALGRADIENT Computes the gradient using "finite differences" 3 | %and gives us a numerical estimate of the gradient. 4 | % numgrad = COMPUTENUMERICALGRADIENT(J, theta) computes the numerical 5 | % gradient of the function J around theta. Calling y = J(theta) should 6 | % return the function value at theta. 7 | 8 | % Notes: The following code implements numerical gradient checking, and 9 | % returns the numerical gradient.It sets numgrad(i) to (a numerical 10 | % approximation of) the partial derivative of J with respect to the 11 | % i-th input argument, evaluated at theta. (i.e., numgrad(i) should 12 | % be the (approximately) the partial derivative of J with respect 13 | % to theta(i).) 14 | % 15 | 16 | numgrad = zeros(size(theta)); 17 | perturb = zeros(size(theta)); 18 | e = 1e-4; 19 | for p = 1:numel(theta) 20 | % Set perturbation vector 21 | perturb(p) = e; 22 | loss1 = J(theta - perturb); 23 | loss2 = J(theta + perturb); 24 | % Compute Numerical Gradient 25 | numgrad(p) = (loss2 - loss1) / (2*e); 26 | perturb(p) = 0; 27 | end 28 | 29 | end 30 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/debugInitializeWeights.m: -------------------------------------------------------------------------------- 1 | function W = debugInitializeWeights(fan_out, fan_in) 2 | %DEBUGINITIALIZEWEIGHTS Initialize the weights of a layer with fan_in 3 | %incoming connections and fan_out outgoing connections using a fixed 4 | %strategy, this will help you later in debugging 5 | % W = DEBUGINITIALIZEWEIGHTS(fan_in, fan_out) initializes the weights 6 | % of a layer with fan_in incoming connections and fan_out outgoing 7 | % connections using a fix set of values 8 | % 9 | % Note that W should be set to a matrix of size(1 + fan_in, fan_out) as 10 | % the first row of W handles the "bias" terms 11 | % 12 | 13 | % Set W to zeros 14 | W = zeros(fan_out, 1 + fan_in); 15 | 16 | % Initialize W using "sin", this ensures that W is always of the same 17 | % values and will be useful for debugging 18 | W = reshape(sin(1:numel(W)), size(W)) / 10; 19 | 20 | % ========================================================================= 21 | 22 | end 23 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/displayData.m: -------------------------------------------------------------------------------- 1 | function [h, display_array] = displayData(X, example_width) 2 | %DISPLAYDATA Display 2D data in a nice grid 3 | % [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 4 | % stored in X in a nice grid. It returns the figure handle h and the 5 | % displayed array if requested. 6 | 7 | % Set example_width automatically if not passed in 8 | if ~exist('example_width', 'var') || isempty(example_width) 9 | example_width = round(sqrt(size(X, 2))); 10 | end 11 | 12 | % Gray Image 13 | colormap(gray); 14 | 15 | % Compute rows, cols 16 | [m n] = size(X); 17 | example_height = (n / example_width); 18 | 19 | % Compute number of items to display 20 | display_rows = floor(sqrt(m)); 21 | display_cols = ceil(m / display_rows); 22 | 23 | % Between images padding 24 | pad = 1; 25 | 26 | % Setup blank display 27 | display_array = - ones(pad + display_rows * (example_height + pad), ... 28 | pad + display_cols * (example_width + pad)); 29 | 30 | % Copy each example into a patch on the display array 31 | curr_ex = 1; 32 | for j = 1:display_rows 33 | for i = 1:display_cols 34 | if curr_ex > m, 35 | break; 36 | end 37 | % Copy the patch 38 | 39 | % Get the max value of the patch 40 | max_val = max(abs(X(curr_ex, :))); 41 | display_array(pad + (j - 1) * (example_height + pad) + (1:example_height), ... 42 | pad + (i - 1) * (example_width + pad) + (1:example_width)) = ... 43 | reshape(X(curr_ex, :), example_height, example_width) / max_val; 44 | curr_ex = curr_ex + 1; 45 | end 46 | if curr_ex > m, 47 | break; 48 | end 49 | end 50 | 51 | % Display Image 52 | h = imagesc(display_array, [-1 1]); 53 | 54 | % Do not show axis 55 | axis image off 56 | 57 | drawnow; 58 | 59 | end 60 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/ex4data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_5/ex4/ex4data1.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/ex4weights.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_5/ex4/ex4weights.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/predict.m: -------------------------------------------------------------------------------- 1 | function p = predict(Theta1, Theta2, X) 2 | %PREDICT Predict the label of an input given a trained neural network 3 | % p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the 4 | % trained weights of a neural network (Theta1, Theta2) 5 | 6 | % Useful values 7 | m = size(X, 1); 8 | num_labels = size(Theta2, 1); 9 | 10 | % You need to return the following variables correctly 11 | p = zeros(size(X, 1), 1); 12 | 13 | h1 = sigmoid([ones(m, 1) X] * Theta1'); 14 | h2 = sigmoid([ones(m, 1) h1] * Theta2'); 15 | [dummy, p] = max(h2, [], 2); 16 | 17 | % ========================================================================= 18 | 19 | 20 | end 21 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/randInitializeWeights.m: -------------------------------------------------------------------------------- 1 | function W = randInitializeWeights(L_in, L_out) 2 | %RANDINITIALIZEWEIGHTS Randomly initialize the weights of a layer with L_in 3 | %incoming connections and L_out outgoing connections 4 | % W = RANDINITIALIZEWEIGHTS(L_in, L_out) randomly initializes the weights 5 | % of a layer with L_in incoming connections and L_out outgoing 6 | % connections. 7 | % 8 | % Note that W should be set to a matrix of size(L_out, 1 + L_in) as 9 | % the first row of W handles the "bias" terms 10 | % 11 | 12 | % ====================== YOUR CODE HERE ====================== 13 | % Instructions: Initialize W randomly so that we break the symmetry while 14 | % training the neural network. 15 | % 16 | % Note: The first row of W corresponds to the parameters for the bias units 17 | % 18 | 19 | 20 | epsilon_init = 0.12; 21 | W = rand(L_out, L_in + 1)* 2 * epsilon_init - epsilon_init; 22 | 23 | 24 | % ========================================================================= 25 | 26 | end 27 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/sigmoid.m: -------------------------------------------------------------------------------- 1 | function g = sigmoid(z) 2 | %SIGMOID Compute sigmoid functoon 3 | % J = SIGMOID(z) computes the sigmoid of z. 4 | 5 | g = 1.0 ./ (1.0 + exp(-z)); 6 | end 7 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_5/ex4/sigmoidGradient.m: -------------------------------------------------------------------------------- 1 | function g = sigmoidGradient(z) 2 | %SIGMOIDGRADIENT returns the gradient of the sigmoid function 3 | %evaluated at z 4 | % g = SIGMOIDGRADIENT(z) computes the gradient of the sigmoid function 5 | % evaluated at z. This should work regardless if z is a matrix or a 6 | % vector. In particular, if z is a vector or matrix, you should return 7 | % the gradient for each element. 8 | 9 | g = zeros(size(z)); 10 | 11 | % ====================== YOUR CODE HERE ====================== 12 | % Instructions: Compute the gradient of the sigmoid function evaluated at 13 | % each value of z (z can be a matrix, vector or scalar). 14 | 15 | g = sigmoid(z).*(1-sigmoid(z)); 16 | 17 | 18 | % ============================================================= 19 | 20 | 21 | 22 | end 23 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_6/ex5.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/ex5data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_6/ex5/ex5data1.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/featureNormalize.m: -------------------------------------------------------------------------------- 1 | function [X_norm, mu, sigma] = featureNormalize(X) 2 | %FEATURENORMALIZE Normalizes the features in X 3 | % FEATURENORMALIZE(X) returns a normalized version of X where 4 | % the mean value of each feature is 0 and the standard deviation 5 | % is 1. This is often a good preprocessing step to do when 6 | % working with learning algorithms. 7 | 8 | mu = mean(X); 9 | X_norm = bsxfun(@minus, X, mu); 10 | 11 | sigma = std(X_norm); 12 | X_norm = bsxfun(@rdivide, X_norm, sigma); 13 | 14 | 15 | % ============================================================ 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/learningCurve.m: -------------------------------------------------------------------------------- 1 | function [error_train, error_val] = ... 2 | learningCurve(X, y, Xval, yval, lambda) 3 | %LEARNINGCURVE Generates the train and cross validation set errors needed 4 | %to plot a learning curve 5 | % [error_train, error_val] = ... 6 | % LEARNINGCURVE(X, y, Xval, yval, lambda) returns the train and 7 | % cross validation set errors for a learning curve. In particular, 8 | % it returns two vectors of the same length - error_train and 9 | % error_val. Then, error_train(i) contains the training error for 10 | % i examples (and similarly for error_val(i)). 11 | % 12 | % In this function, you will compute the train and test errors for 13 | % dataset sizes from 1 up to m. In practice, when working with larger 14 | % datasets, you might want to do this in larger intervals. 15 | % 16 | 17 | % Number of training examples 18 | m = size(X, 1); 19 | 20 | error_train = zeros(m, 1); 21 | error_val = zeros(m,1); 22 | 23 | for i=1:m 24 | theta = trainLinearReg(X(1:i, :), y(1:i), lambda); 25 | [error_train(i), ~] = linearRegCostFunction(X(1:i, :), y(1:i), theta, 0); 26 | %You use the entire validation set to measure J_cv because you want to know how well 27 | %the theta values work on the validation set. You get a better (average) measurement 28 | %by using the entire CV set. 29 | [error_val(i), ~] = linearRegCostFunction(Xval, yval, theta, 0); 30 | end 31 | 32 | 33 | end 34 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/linearRegCostFunction.m: -------------------------------------------------------------------------------- 1 | function [J, grad] = linearRegCostFunction(X, y, theta, lambda) 2 | %LINEARREGCOSTFUNCTION Compute cost and gradient for regularized linear 3 | %regression with multiple variables 4 | % [J, grad] = LINEARREGCOSTFUNCTION(X, y, theta, lambda) computes the 5 | % cost of using theta as the parameter for linear regression to fit the 6 | % data points in X and y. Returns the cost in J and the gradient in grad 7 | 8 | % Initialize some useful values 9 | m = length(y); % number of training examples 10 | 11 | predictions = X * theta; 12 | 13 | errors = predictions - y; 14 | 15 | 16 | J = (1.0/(2 * m)) .* sum(errors .^2) + (lambda / (2 * m)) .* sum(theta(2:end) .^2); 17 | 18 | grad = (1.0/m) .* X' * errors; 19 | 20 | grad = [grad(1); grad(2:end) + (lambda /m) .* theta(2:end)]; 21 | 22 | 23 | end 24 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/plotFit.m: -------------------------------------------------------------------------------- 1 | function plotFit(min_x, max_x, mu, sigma, theta, p) 2 | %PLOTFIT Plots a learned polynomial regression fit over an existing figure. 3 | %Also works with linear regression. 4 | % PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial 5 | % fit with power p and feature normalization (mu, sigma). 6 | 7 | % Hold on to the current figure 8 | hold on; 9 | 10 | % We plot a range slightly bigger than the min and max values to get 11 | % an idea of how the fit will vary outside the range of the data points 12 | x = (min_x - 15: 0.05 : max_x + 25)'; 13 | 14 | % Map the X values 15 | X_poly = polyFeatures(x, p); 16 | X_poly = bsxfun(@minus, X_poly, mu); 17 | X_poly = bsxfun(@rdivide, X_poly, sigma); 18 | 19 | % Add ones 20 | X_poly = [ones(size(x, 1), 1) X_poly]; 21 | 22 | % Plot 23 | plot(x, X_poly * theta, '--', 'LineWidth', 2) 24 | 25 | % Hold off to the current figure 26 | hold off 27 | 28 | end 29 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/polyFeatures.m: -------------------------------------------------------------------------------- 1 | function [X_poly] = polyFeatures(X, p) 2 | %POLYFEATURES Maps X (1D vector) into the p-th power 3 | % [X_poly] = POLYFEATURES(X, p) takes a data matrix X (size m x 1) and 4 | % maps each example into its polynomial features where 5 | % X_poly(i, :) = [X(i) X(i).^2 X(i).^3 ... X(i).^p]; 6 | % 7 | 8 | 9 | % You need to return the following variables correctly. 10 | 11 | X_poly = zeros(numel(X), p); 12 | 13 | for j=1:p 14 | X_poly(:, j) = X(:,1) .^j; 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/trainLinearReg.m: -------------------------------------------------------------------------------- 1 | function [theta] = trainLinearReg(X, y, lambda) 2 | %TRAINLINEARREG Trains linear regression given a dataset (X, y) and a 3 | %regularization parameter lambda 4 | % [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using 5 | % the dataset (X, y) and regularization parameter lambda. Returns the 6 | % trained parameters theta. 7 | % 8 | 9 | % Initialize Theta 10 | initial_theta = zeros(size(X, 2), 1); 11 | 12 | % Create "short hand" for the cost function to be minimized 13 | costFunction = @(t) linearRegCostFunction(X, y, t, lambda); 14 | 15 | % Now, costFunction is a function that takes in only one argument 16 | options = optimset('MaxIter', 200, 'GradObj', 'on'); 17 | 18 | % Minimize using fmincg 19 | theta = fmincg(costFunction, initial_theta, options); 20 | 21 | end 22 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_6/ex5/validationCurve.m: -------------------------------------------------------------------------------- 1 | function [lambda_vec, error_train, error_val] = ... 2 | validationCurve(X, y, Xval, yval) 3 | %VALIDATIONCURVE Generate the train and validation errors needed to 4 | %plot a validation curve that we can use to select lambda 5 | % [lambda_vec, error_train, error_val] = ... 6 | % VALIDATIONCURVE(X, y, Xval, yval) returns the train 7 | % and validation errors (in error_train, error_val) 8 | % for different values of lambda. You are given the training set (X, 9 | % y) and validation set (Xval, yval). 10 | % 11 | 12 | % Selected values of lambda (you should not change this) 13 | lambda_vec = [0 0.001 0.003 0.01 0.03 0.1 0.3 1 3 10]'; 14 | 15 | % You need to return these variables correctly. 16 | error_train = zeros(length(lambda_vec), 1); 17 | error_val = zeros(length(lambda_vec), 1); 18 | 19 | % ====================== YOUR CODE HERE ====================== 20 | % Instructions: Fill in this function to return training errors in 21 | % error_train and the validation errors in error_val. The 22 | % vector lambda_vec contains the different lambda parameters 23 | % to use for each calculation of the errors, i.e, 24 | % error_train(i), and error_val(i) should give 25 | % you the errors obtained after training with 26 | % lambda = lambda_vec(i) 27 | % 28 | % Note: You can loop over lambda_vec with the following: 29 | % 30 | for i = 1:length(lambda_vec) 31 | lambda = lambda_vec(i); 32 | theta = trainLinearReg(X, y, lambda); 33 | [error_train(i), ~] = linearRegCostFunction(X, y, theta, 0); 34 | [error_val(i), ~] = linearRegCostFunction(Xval, yval, theta, 0); 35 | end 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | % ========================================================================= 49 | 50 | end 51 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_7/ex6.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/dataset3Params.m: -------------------------------------------------------------------------------- 1 | function [C, sigma] = dataset3Params(X, y, Xval, yval) 2 | %DATASET3PARAMS returns your choice of C and sigma for Part 3 of the exercise 3 | %where you select the optimal (C, sigma) learning parameters to use for SVM 4 | %with RBF kernel 5 | % [C, sigma] = DATASET3PARAMS(X, y, Xval, yval) returns your choice of C and 6 | % sigma. You should complete this function to return the optimal C and 7 | % sigma based on a cross-validation set. 8 | % 9 | 10 | % You need to return the following variables correctly. 11 | C = 1; 12 | sigma = 0.3; 13 | 14 | values = [0.01,0.03,0.1,0.3,1,3,10,30]; 15 | 16 | max_error = 1; 17 | 18 | for i=1:8 19 | for j=1:8 20 | current_c = values(i); 21 | current_sigma = values(j); 22 | 23 | %ALWAYS train the model on training sets (X and y) 24 | model= svmTrain(X, y, current_c, @(x1, x2) gaussianKernel(x1, x2, current_sigma)); 25 | 26 | %AND evaluate it on cross validation set 27 | predictions = svmPredict(model, Xval); 28 | error = mean(double(predictions ~= yval)); 29 | 30 | if error < max_error 31 | max_error = error; 32 | C = current_c; 33 | sigma = current_sigma; 34 | end 35 | 36 | end 37 | end 38 | 39 | 40 | 41 | 42 | 43 | % ========================================================================= 44 | 45 | end 46 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/emailFeatures.m: -------------------------------------------------------------------------------- 1 | function x = emailFeatures(word_indices) 2 | %EMAILFEATURES takes in a word_indices vector and produces a feature vector 3 | %from the word indices 4 | % x = EMAILFEATURES(word_indices) takes in a word_indices vector and 5 | % produces a feature vector from the word indices. 6 | 7 | % Total number of words in the dictionary 8 | n = 1899; 9 | 10 | x = ismember(reshape(1:1899,[n,1]), word_indices); 11 | 12 | 13 | end 14 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/emailSample1.txt: -------------------------------------------------------------------------------- 1 | > Anyone knows how much it costs to host a web portal ? 2 | > 3 | Well, it depends on how many visitors you're expecting. 4 | This can be anywhere from less than 10 bucks a month to a couple of $100. 5 | You should checkout http://www.rackspace.com/ or perhaps Amazon EC2 6 | if youre running something big.. 7 | 8 | To unsubscribe yourself from this mailing list, send an email to: 9 | groupname-unsubscribe@egroups.com 10 | 11 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/emailSample2.txt: -------------------------------------------------------------------------------- 1 | Folks, 2 | 3 | my first time posting - have a bit of Unix experience, but am new to Linux. 4 | 5 | 6 | Just got a new PC at home - Dell box with Windows XP. Added a second hard disk 7 | for Linux. Partitioned the disk and have installed Suse 7.2 from CD, which went 8 | fine except it didn't pick up my monitor. 9 | 10 | I have a Dell branded E151FPp 15" LCD flat panel monitor and a nVidia GeForce4 11 | Ti4200 video card, both of which are probably too new to feature in Suse's default 12 | set. I downloaded a driver from the nVidia website and installed it using RPM. 13 | Then I ran Sax2 (as was recommended in some postings I found on the net), but 14 | it still doesn't feature my video card in the available list. What next? 15 | 16 | Another problem. I have a Dell branded keyboard and if I hit Caps-Lock twice, 17 | the whole machine crashes (in Linux, not Windows) - even the on/off switch is 18 | inactive, leaving me to reach for the power cable instead. 19 | 20 | If anyone can help me in any way with these probs., I'd be really grateful - 21 | I've searched the 'net but have run out of ideas. 22 | 23 | Or should I be going for a different version of Linux such as RedHat? Opinions 24 | welcome. 25 | 26 | Thanks a lot, 27 | Peter 28 | 29 | -- 30 | Irish Linux Users' Group: ilug@linux.ie 31 | http://www.linux.ie/mailman/listinfo/ilug for (un)subscription information. 32 | List maintainer: listmaster@linux.ie 33 | 34 | 35 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/ex6data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_7/ex6/ex6data1.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/ex6data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_7/ex6/ex6data2.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/ex6data3.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_7/ex6/ex6data3.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/gaussianKernel.m: -------------------------------------------------------------------------------- 1 | function sim = gaussianKernel(x1, x2, sigma) 2 | %RBFKERNEL returns a radial basis function kernel between x1 and x2 3 | % sim = gaussianKernel(x1, x2) returns a gaussian kernel between x1 and x2 4 | % and returns the value in sim 5 | 6 | % Ensure that x1 and x2 are column vectors 7 | x1 = x1(:); x2 = x2(:); 8 | 9 | % You need to return the following variables correctly. 10 | sim = 0; 11 | 12 | % ====================== YOUR CODE HERE ====================== 13 | % Instructions: Fill in this function to return the similarity between x1 14 | % and x2 computed using a Gaussian kernel with bandwidth 15 | % sigma 16 | % 17 | % 18 | 19 | 20 | 21 | sim = exp(- sum((x1 - x2) .^ 2) / (2 * sigma ^ 2)); 22 | 23 | 24 | 25 | % ============================================================= 26 | 27 | end 28 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/getVocabList.m: -------------------------------------------------------------------------------- 1 | function vocabList = getVocabList() 2 | %GETVOCABLIST reads the fixed vocabulary list in vocab.txt and returns a 3 | %cell array of the words 4 | % vocabList = GETVOCABLIST() reads the fixed vocabulary list in vocab.txt 5 | % and returns a cell array of the words in vocabList. 6 | 7 | 8 | %% Read the fixed vocabulary list 9 | fid = fopen('vocab.txt'); 10 | 11 | % Store all dictionary words in cell array vocab{} 12 | n = 1899; % Total number of words in the dictionary 13 | 14 | % For ease of implementation, we use a struct to map the strings => integers 15 | % In practice, you'll want to use some form of hashmap 16 | vocabList = cell(n, 1); 17 | for i = 1:n 18 | % Word Index (can ignore since it will be = i) 19 | fscanf(fid, '%d', 1); 20 | % Actual Word 21 | vocabList{i} = fscanf(fid, '%s', 1); 22 | end 23 | fclose(fid); 24 | 25 | end 26 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/linearKernel.m: -------------------------------------------------------------------------------- 1 | function sim = linearKernel(x1, x2) 2 | %LINEARKERNEL returns a linear kernel between x1 and x2 3 | % sim = linearKernel(x1, x2) returns a linear kernel between x1 and x2 4 | % and returns the value in sim 5 | 6 | % Ensure that x1 and x2 are column vectors 7 | x1 = x1(:); x2 = x2(:); 8 | 9 | % Compute the kernel 10 | sim = x1' * x2; % dot product 11 | 12 | end -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/plotData.m: -------------------------------------------------------------------------------- 1 | function plotData(X, y) 2 | %PLOTDATA Plots the data points X and y into a new figure 3 | % PLOTDATA(x,y) plots the data points with + for the positive examples 4 | % and o for the negative examples. X is assumed to be a Mx2 matrix. 5 | % 6 | % Note: This was slightly modified such that it expects y = 1 or y = 0 7 | 8 | % Find Indices of Positive and Negative Examples 9 | pos = find(y == 1); neg = find(y == 0); 10 | 11 | % Plot Examples 12 | plot(X(pos, 1), X(pos, 2), 'k+','LineWidth', 1, 'MarkerSize', 7) 13 | hold on; 14 | plot(X(neg, 1), X(neg, 2), 'ko', 'MarkerFaceColor', 'y', 'MarkerSize', 7) 15 | hold off; 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/processEmail.m: -------------------------------------------------------------------------------- 1 | function word_indices = processEmail(email_contents) 2 | %PROCESSEMAIL preprocesses a the body of an email and 3 | %returns a list of word_indices 4 | % word_indices = PROCESSEMAIL(email_contents) preprocesses 5 | % the body of an email and returns a list of indices of the 6 | % words contained in the email. 7 | % 8 | 9 | % Load Vocabulary 10 | vocabList = getVocabList(); 11 | 12 | % Init return value 13 | word_indices = []; 14 | 15 | % ========================== Preprocess Email =========================== 16 | 17 | % Find the Headers ( \n\n and remove ) 18 | % Uncomment the following lines if you are working with raw emails with the 19 | % full headers 20 | 21 | % hdrstart = strfind(email_contents, ([char(10) char(10)])); 22 | % email_contents = email_contents(hdrstart(1):end); 23 | 24 | % Lower case 25 | email_contents = lower(email_contents); 26 | 27 | % Strip all HTML 28 | % Looks for any expression that starts with < and ends with > and replace 29 | % and does not have any < or > in the tag it with a space 30 | email_contents = regexprep(email_contents, '<[^<>]+>', ' '); 31 | 32 | % Handle Numbers 33 | % Look for one or more characters between 0-9 34 | email_contents = regexprep(email_contents, '[0-9]+', 'number'); 35 | 36 | % Handle URLS 37 | % Look for strings starting with http:// or https:// 38 | email_contents = regexprep(email_contents, ... 39 | '(http|https)://[^\s]*', 'httpaddr'); 40 | 41 | % Handle Email Addresses 42 | % Look for strings with @ in the middle 43 | email_contents = regexprep(email_contents, '[^\s]+@[^\s]+', 'emailaddr'); 44 | 45 | % Handle $ sign 46 | email_contents = regexprep(email_contents, '[$]+', 'dollar'); 47 | 48 | 49 | % ========================== Tokenize Email =========================== 50 | 51 | % Output the email to screen as well 52 | fprintf('\n==== Processed Email ====\n\n'); 53 | 54 | % Process file 55 | l = 0; 56 | 57 | while ~isempty(email_contents) 58 | 59 | % Tokenize and also get rid of any punctuation 60 | [str, email_contents] = ... 61 | strtok(email_contents, ... 62 | [' @$/#.-:&*+=[]?!(){},''">_<;%' char(10) char(13)]); 63 | 64 | % Remove any non alphanumeric characters 65 | str = regexprep(str, '[^a-zA-Z0-9]', ''); 66 | 67 | % Stem the word 68 | % (the porterStemmer sometimes has issues, so we use a try catch block) 69 | try str = porterStemmer(strtrim(str)); 70 | catch str = ''; continue; 71 | end; 72 | 73 | % Skip the word if it is too short 74 | if length(str) < 1 75 | continue; 76 | end 77 | 78 | for idx=1:length(vocabList) 79 | found = strcmp(str, vocabList{idx}); 80 | if found > 0 81 | word_indices = [word_indices ; idx]; 82 | end 83 | end 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | % ============================================================= 97 | 98 | 99 | % Print to screen, ensuring that the output lines are not too long 100 | if (l + length(str) + 1) > 78 101 | fprintf('\n'); 102 | l = 0; 103 | end 104 | fprintf('%s ', str); 105 | l = l + length(str) + 1; 106 | 107 | end 108 | 109 | % Print footer 110 | fprintf('\n\n=========================\n'); 111 | 112 | end 113 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/readFile.m: -------------------------------------------------------------------------------- 1 | function file_contents = readFile(filename) 2 | %READFILE reads a file and returns its entire contents 3 | % file_contents = READFILE(filename) reads a file and returns its entire 4 | % contents in file_contents 5 | % 6 | 7 | % Load File 8 | fid = fopen(filename); 9 | if fid 10 | file_contents = fscanf(fid, '%c', inf); 11 | fclose(fid); 12 | else 13 | file_contents = ''; 14 | fprintf('Unable to open %s\n', filename); 15 | end 16 | 17 | end 18 | 19 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/spamSample1.txt: -------------------------------------------------------------------------------- 1 | Hi @hzitoun, 2 | 3 | We've just launched the Recruit Holding's Restaurant Visitor Forecasting Competition! Here are the competition details. If you're interested in competing, or digging deeper into the specifics, click the 'Join the competition' button below. 4 | 5 | Host: 6 | Recruit Holdings, a large Japanese company that owns Hot Pepper Gourmet (a restaurant review service), AirREGI (a restaurant point of sales service), and Restaurant Board (reservation log management software company). 7 | 8 | Competition Description: 9 | In this competition, you're challenged to use reservation and visitation data to predict the total number of visitors to a restaurant for specified future dates. This information will help restaurants more efficiently schedule employees and order ingredients, hopefully allowing them to focus on creating an enjoyable dining experience for their customers. -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/spamSample2.txt: -------------------------------------------------------------------------------- 1 | Best Buy Viagra Generic Online 2 | 3 | Viagra 100mg x 60 Pills $125, Free Pills & Reorder Discount, Top Selling 100% Quality & Satisfaction guaranteed! 4 | 5 | We accept VISA, Master & E-Check Payments, 90000+ Satisfied Customers! 6 | http://medphysitcstech.ru 7 | 8 | 9 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/spamTest.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_7/ex6/spamTest.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/spamTrain.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_7/ex6/spamTrain.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/svmPredict.m: -------------------------------------------------------------------------------- 1 | function pred = svmPredict(model, X) 2 | %SVMPREDICT returns a vector of predictions using a trained SVM model 3 | %(svmTrain). 4 | % pred = SVMPREDICT(model, X) returns a vector of predictions using a 5 | % trained SVM model (svmTrain). X is a mxn matrix where there each 6 | % example is a row. model is a svm model returned from svmTrain. 7 | % predictions pred is a m x 1 column of predictions of {0, 1} values. 8 | % 9 | 10 | % Check if we are getting a column vector, if so, then assume that we only 11 | % need to do prediction for a single example 12 | if (size(X, 2) == 1) 13 | % Examples should be in rows 14 | X = X'; 15 | end 16 | 17 | % Dataset 18 | m = size(X, 1); 19 | p = zeros(m, 1); 20 | pred = zeros(m, 1); 21 | 22 | if strcmp(func2str(model.kernelFunction), 'linearKernel') 23 | % We can use the weights and bias directly if working with the 24 | % linear kernel 25 | p = X * model.w + model.b; 26 | elseif strfind(func2str(model.kernelFunction), 'gaussianKernel') 27 | % Vectorized RBF Kernel 28 | % This is equivalent to computing the kernel on every pair of examples 29 | X1 = sum(X.^2, 2); 30 | X2 = sum(model.X.^2, 2)'; 31 | K = bsxfun(@plus, X1, bsxfun(@plus, X2, - 2 * X * model.X')); 32 | K = model.kernelFunction(1, 0) .^ K; 33 | K = bsxfun(@times, model.y', K); 34 | K = bsxfun(@times, model.alphas', K); 35 | p = sum(K, 2); 36 | else 37 | % Other Non-linear kernel 38 | for i = 1:m 39 | prediction = 0; 40 | for j = 1:size(model.X, 1) 41 | prediction = prediction + ... 42 | model.alphas(j) * model.y(j) * ... 43 | model.kernelFunction(X(i,:)', model.X(j,:)'); 44 | end 45 | p(i) = prediction + model.b; 46 | end 47 | end 48 | 49 | % Convert predictions into 0 / 1 50 | pred(p >= 0) = 1; 51 | pred(p < 0) = 0; 52 | 53 | end 54 | 55 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/visualizeBoundary.m: -------------------------------------------------------------------------------- 1 | function visualizeBoundary(X, y, model, varargin) 2 | %VISUALIZEBOUNDARY plots a non-linear decision boundary learned by the SVM 3 | % VISUALIZEBOUNDARYLINEAR(X, y, model) plots a non-linear decision 4 | % boundary learned by the SVM and overlays the data on it 5 | 6 | % Plot the training data on top of the boundary 7 | plotData(X, y) 8 | 9 | % Make classification predictions over a grid of values 10 | x1plot = linspace(min(X(:,1)), max(X(:,1)), 100)'; 11 | x2plot = linspace(min(X(:,2)), max(X(:,2)), 100)'; 12 | [X1, X2] = meshgrid(x1plot, x2plot); 13 | vals = zeros(size(X1)); 14 | for i = 1:size(X1, 2) 15 | this_X = [X1(:, i), X2(:, i)]; 16 | vals(:, i) = svmPredict(model, this_X); 17 | end 18 | 19 | % Plot the SVM boundary 20 | hold on 21 | contour(X1, X2, vals, [0.5 0.5], 'b'); 22 | hold off; 23 | 24 | end 25 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_7/ex6/visualizeBoundaryLinear.m: -------------------------------------------------------------------------------- 1 | function visualizeBoundaryLinear(X, y, model) 2 | %VISUALIZEBOUNDARYLINEAR plots a linear decision boundary learned by the 3 | %SVM 4 | % VISUALIZEBOUNDARYLINEAR(X, y, model) plots a linear decision boundary 5 | % learned by the SVM and overlays the data on it 6 | 7 | w = model.w; 8 | b = model.b; 9 | xp = linspace(min(X(:,1)), max(X(:,1)), 100); 10 | yp = - (w(1)*xp + b)/w(2); 11 | plotData(X, y); 12 | hold on; 13 | plot(xp, yp, '-b'); 14 | hold off 15 | 16 | end 17 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/bird_small.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/bird_small.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/bird_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/bird_small.png -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/computeCentroids.m: -------------------------------------------------------------------------------- 1 | function centroids = computeCentroids(X, idx, K) 2 | %COMPUTECENTROIDS returns the new centroids by computing the means of the 3 | %data points assigned to each centroid. 4 | % centroids = COMPUTECENTROIDS(X, idx, K) returns the new centroids by 5 | % computing the means of the data points assigned to each centroid. It is 6 | % given a dataset X where each row is a single data point, a vector 7 | % idx of centroid assignments (i.e. each entry in range [1..K]) for each 8 | % example, and K, the number of centroids. You should return a matrix 9 | % centroids, where each row of centroids is the mean of the data points 10 | % assigned to it. 11 | % 12 | 13 | % Useful variables 14 | n = size(X, 2); 15 | 16 | centroids = zeros(K, n); 17 | 18 | [~, ~, ii] = unique(idx); 19 | for i=1:n 20 | centroids(:, i) = accumarray(ii, X(:,i),[], @mean); 21 | end 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | % ============================================================= 34 | 35 | 36 | end 37 | 38 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/displayData.m: -------------------------------------------------------------------------------- 1 | function [h, display_array] = displayData(X, example_width) 2 | %DISPLAYDATA Display 2D data in a nice grid 3 | % [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 4 | % stored in X in a nice grid. It returns the figure handle h and the 5 | % displayed array if requested. 6 | 7 | % Set example_width automatically if not passed in 8 | if ~exist('example_width', 'var') || isempty(example_width) 9 | example_width = round(sqrt(size(X, 2))); 10 | end 11 | 12 | % Gray Image 13 | colormap(gray); 14 | 15 | % Compute rows, cols 16 | [m n] = size(X); 17 | example_height = (n / example_width); 18 | 19 | % Compute number of items to display 20 | display_rows = floor(sqrt(m)); 21 | display_cols = ceil(m / display_rows); 22 | 23 | % Between images padding 24 | pad = 1; 25 | 26 | % Setup blank display 27 | display_array = - ones(pad + display_rows * (example_height + pad), ... 28 | pad + display_cols * (example_width + pad)); 29 | 30 | % Copy each example into a patch on the display array 31 | curr_ex = 1; 32 | for j = 1:display_rows 33 | for i = 1:display_cols 34 | if curr_ex > m, 35 | break; 36 | end 37 | % Copy the patch 38 | 39 | % Get the max value of the patch 40 | max_val = max(abs(X(curr_ex, :))); 41 | display_array(pad + (j - 1) * (example_height + pad) + (1:example_height), ... 42 | pad + (i - 1) * (example_width + pad) + (1:example_width)) = ... 43 | reshape(X(curr_ex, :), example_height, example_width) / max_val; 44 | curr_ex = curr_ex + 1; 45 | end 46 | if curr_ex > m, 47 | break; 48 | end 49 | end 50 | 51 | % Display Image 52 | h = imagesc(display_array, [-1 1]); 53 | 54 | % Do not show axis 55 | axis image off 56 | 57 | drawnow; 58 | 59 | end 60 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/drawLine.m: -------------------------------------------------------------------------------- 1 | function drawLine(p1, p2, varargin) 2 | %DRAWLINE Draws a line from point p1 to point p2 3 | % DRAWLINE(p1, p2) Draws a line from point p1 to point p2 and holds the 4 | % current figure 5 | 6 | plot([p1(1) p2(1)], [p1(2) p2(2)], varargin{:}); 7 | 8 | end -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/ex7data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/ex7data1.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/ex7data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/ex7data2.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/ex7faces.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/ex7faces.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/featureNormalize.m: -------------------------------------------------------------------------------- 1 | function [X_norm, mu, sigma] = featureNormalize(X) 2 | %FEATURENORMALIZE Normalizes the features in X 3 | % FEATURENORMALIZE(X) returns a normalized version of X where 4 | % the mean value of each feature is 0 and the standard deviation 5 | % is 1. This is often a good preprocessing step to do when 6 | % working with learning algorithms. 7 | 8 | mu = mean(X); 9 | X_norm = bsxfun(@minus, X, mu); 10 | 11 | sigma = std(X_norm); 12 | X_norm = bsxfun(@rdivide, X_norm, sigma); 13 | 14 | 15 | % ============================================================ 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/findClosestCentroids.m: -------------------------------------------------------------------------------- 1 | function idx = findClosestCentroids(X, centroids) 2 | %FINDCLOSESTCENTROIDS computes the centroid memberships for every example 3 | % idx = FINDCLOSESTCENTROIDS (X, centroids) returns the closest centroids 4 | % in idx for a dataset X where each row is a single example. idx = m x 1 5 | % vector of centroid assignments (i.e. each entry in range [1..K]) 6 | % 7 | m = size(X, 1); 8 | idx = zeros(m, 1); 9 | 10 | % ====================== YOUR CODE HERE ====================== 11 | % Instructions: Go over every example, find its closest centroid, and store 12 | % the index inside idx at the appropriate location. 13 | % Concretely, idx(i) should contain the index of the centroid 14 | % closest to example i. Hence, it should be a value in the 15 | % range 1..K 16 | for i=1:m 17 | V = centroids - X(i, :); 18 | D = vecnorm(V, 2, 2) .^ 2; 19 | [~, idx(i)] = min(D); 20 | end 21 | 22 | end 23 | 24 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/kMeansInitCentroids.m: -------------------------------------------------------------------------------- 1 | function centroids = kMeansInitCentroids(X, K) 2 | %KMEANSINITCENTROIDS This function initializes K centroids that are to be 3 | %used in K-Means on the dataset X 4 | % centroids = KMEANSINITCENTROIDS(X, K) returns K initial centroids to be 5 | % used with the K-Means on the dataset X 6 | % 7 | 8 | 9 | % ====================== YOUR CODE HERE ====================== 10 | % Instructions: You should set centroids to randomly chosen examples from 11 | % the dataset X 12 | % 13 | 14 | % Randomly reorder the indices of examples 15 | randidx = randperm(size(X, 1)); 16 | % Take the first K examples as centroids 17 | centroids = X(randidx(1:K), :); 18 | 19 | 20 | 21 | 22 | 23 | 24 | % ============================================================= 25 | 26 | end 27 | 28 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/me.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/me.jpg -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/me_compressed.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_8/ex7/me_compressed.PNG -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/pca.m: -------------------------------------------------------------------------------- 1 | function [U, S] = pca(X) 2 | %PCA Run principal component analysis on the dataset X 3 | % [U, S, X] = pca(X) computes eigenvectors of the covariance matrix of X 4 | % Returns the eigenvectors U, the eigenvalues (on diagonal) in S 5 | % 6 | 7 | % Useful values 8 | [m, n] = size(X); 9 | 10 | % You need to return the following variables correctly. 11 | 12 | 13 | % ====================== YOUR CODE HERE ====================== 14 | % Instructions: You should first compute the covariance matrix. Then, you 15 | % should use the "svd" function to compute the eigenvectors 16 | % and eigenvalues of the covariance matrix. 17 | % 18 | % Note: When computing the covariance matrix, remember to divide by m (the 19 | % number of examples). 20 | % 21 | Sigma = (1.0/m) * X' * X; 22 | 23 | 24 | [U, S, V] = svd(Sigma); 25 | 26 | 27 | 28 | 29 | % ========================================================================= 30 | 31 | end 32 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/plotDataPoints.m: -------------------------------------------------------------------------------- 1 | function plotDataPoints(X, idx, K) 2 | %PLOTDATAPOINTS plots data points in X, coloring them so that those with the same 3 | %index assignments in idx have the same color 4 | % PLOTDATAPOINTS(X, idx, K) plots data points in X, coloring them so that those 5 | % with the same index assignments in idx have the same color 6 | 7 | % Create palette 8 | palette = hsv(K + 1); 9 | colors = palette(idx, :); 10 | 11 | % Plot the data 12 | scatter(X(:,1), X(:,2), 15, colors); 13 | 14 | end 15 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/plotProgresskMeans.m: -------------------------------------------------------------------------------- 1 | function plotProgresskMeans(X, centroids, previous, idx, K, i) 2 | %PLOTPROGRESSKMEANS is a helper function that displays the progress of 3 | %k-Means as it is running. It is intended for use only with 2D data. 4 | % PLOTPROGRESSKMEANS(X, centroids, previous, idx, K, i) plots the data 5 | % points with colors assigned to each centroid. With the previous 6 | % centroids, it also plots a line between the previous locations and 7 | % current locations of the centroids. 8 | % 9 | 10 | % Plot the examples 11 | plotDataPoints(X, idx, K); 12 | 13 | % Plot the centroids as black x's 14 | plot(centroids(:,1), centroids(:,2), 'x', ... 15 | 'MarkerEdgeColor','k', ... 16 | 'MarkerSize', 10, 'LineWidth', 3); 17 | 18 | % Plot the history of the centroids with lines 19 | for j=1:size(centroids,1) 20 | drawLine(centroids(j, :), previous(j, :)); 21 | end 22 | 23 | % Title 24 | title(sprintf('Iteration number %d', i)) 25 | 26 | end 27 | 28 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/projectData.m: -------------------------------------------------------------------------------- 1 | function Z = projectData(X, U, K) 2 | %PROJECTDATA Computes the reduced data representation when projecting only 3 | %on to the top k eigenvectors 4 | % Z = projectData(X, U, K) computes the projection of 5 | % the normalized inputs X into the reduced dimensional space spanned by 6 | % the first K columns of U. It returns the projected examples in Z. 7 | % 8 | 9 | % ====================== YOUR CODE HERE ====================== 10 | % Instructions: Compute the projection of the data using only the top K 11 | % eigenvectors in U (first K columns). 12 | % For the i-th example X(i,:), the projection on to the k-th 13 | % eigenvector is given as follows: 14 | % x = X(i, :)'; 15 | % projection_k = x' * U(:, k); 16 | % 17 | 18 | Ureduce = U(:,1:K); % take the first k directions 19 | Z = X * Ureduce; % compute the projected data points 20 | 21 | 22 | 23 | % ============================================================= 24 | 25 | end 26 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/recoverData.m: -------------------------------------------------------------------------------- 1 | function X_rec = recoverData(Z, U, K) 2 | %RECOVERDATA Recovers an approximation of the original data when using the 3 | %projected data 4 | % X_rec = RECOVERDATA(Z, U, K) recovers an approximation the 5 | % original data that has been reduced to K dimensions. It returns the 6 | % approximate reconstruction in X_rec. 7 | % 8 | 9 | % ====================== YOUR CODE HERE ====================== 10 | % Instructions: Compute the approximation of the data by projecting back 11 | % onto the original space using the top K eigenvectors in U. 12 | % 13 | % For the i-th example Z(i,:), the (approximate) 14 | % recovered data for dimension j is given as follows: 15 | % v = Z(i, :)'; 16 | % recovered_j = v' * U(j, 1:K)'; 17 | % 18 | % Notice that U(j, 1:K) is a row vector. 19 | % 20 | Ureduce = U(:,1:K); % take the first k directions 21 | X_rec = Z * Ureduce'; % go back to our original number of features 22 | % ============================================================= 23 | 24 | end 25 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_8/ex7/runkMeans.m: -------------------------------------------------------------------------------- 1 | function [centroids, idx] = runkMeans(X, initial_centroids, ... 2 | max_iters, plot_progress) 3 | %RUNKMEANS runs the K-Means algorithm on data matrix X, where each row of X 4 | %is a single example 5 | % [centroids, idx] = RUNKMEANS(X, initial_centroids, max_iters, ... 6 | % plot_progress) runs the K-Means algorithm on data matrix X, where each 7 | % row of X is a single example. It uses initial_centroids used as the 8 | % initial centroids. max_iters specifies the total number of interactions 9 | % of K-Means to execute. plot_progress is a true/false flag that 10 | % indicates if the function should also plot its progress as the 11 | % learning happens. This is set to false by default. runkMeans returns 12 | % centroids, a Kxn matrix of the computed centroids and idx, a m x 1 13 | % vector of centroid assignments (i.e. each entry in range [1..K]) 14 | % 15 | 16 | % Set default value for plot progress 17 | if ~exist('plot_progress', 'var') || isempty(plot_progress) 18 | plot_progress = false; 19 | end 20 | 21 | % Plot the data if we are plotting progress 22 | if plot_progress 23 | figure; 24 | hold on; 25 | end 26 | 27 | % Initialize values 28 | [m n] = size(X); 29 | K = size(initial_centroids, 1); 30 | centroids = initial_centroids; 31 | previous_centroids = centroids; 32 | idx = zeros(m, 1); 33 | 34 | % Run K-Means 35 | for i=1:max_iters 36 | 37 | % Output progress 38 | fprintf('K-Means iteration %d/%d...\n', i, max_iters); 39 | if exist('OCTAVE_VERSION') 40 | fflush(stdout); 41 | end 42 | 43 | % For each example in X, assign it to the closest centroid 44 | idx = findClosestCentroids(X, centroids); 45 | 46 | % Optionally, plot progress here 47 | if plot_progress 48 | plotProgresskMeans(X, centroids, previous_centroids, idx, K, i); 49 | previous_centroids = centroids; 50 | fprintf('Press enter to continue.\n'); 51 | pause; 52 | end 53 | 54 | % Given the memberships, compute new centroids 55 | centroids = computeCentroids(X, idx, K); 56 | end 57 | 58 | % Hold off if we are plotting progress 59 | if plot_progress 60 | hold off; 61 | end 62 | 63 | end 64 | 65 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8.pdf -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/checkCostFunction.m: -------------------------------------------------------------------------------- 1 | function checkCostFunction(lambda) 2 | %CHECKCOSTFUNCTION Creates a collaborative filering problem 3 | %to check your cost function and gradients 4 | % CHECKCOSTFUNCTION(lambda) Creates a collaborative filering problem 5 | % to check your cost function and gradients, it will output the 6 | % analytical gradients produced by your code and the numerical gradients 7 | % (computed using computeNumericalGradient). These two gradient 8 | % computations should result in very similar values. 9 | 10 | % Set lambda 11 | if ~exist('lambda', 'var') || isempty(lambda) 12 | lambda = 0; 13 | end 14 | 15 | %% Create small problem 16 | X_t = rand(4, 3); 17 | Theta_t = rand(5, 3); 18 | 19 | % Zap out most entries 20 | Y = X_t * Theta_t'; 21 | Y(rand(size(Y)) > 0.5) = 0; 22 | R = zeros(size(Y)); 23 | R(Y ~= 0) = 1; 24 | 25 | %% Run Gradient Checking 26 | X = randn(size(X_t)); 27 | Theta = randn(size(Theta_t)); 28 | num_users = size(Y, 2); 29 | num_movies = size(Y, 1); 30 | num_features = size(Theta_t, 2); 31 | 32 | numgrad = computeNumericalGradient( ... 33 | @(t) cofiCostFunc(t, Y, R, num_users, num_movies, ... 34 | num_features, lambda), [X(:); Theta(:)]); 35 | 36 | [cost, grad] = cofiCostFunc([X(:); Theta(:)], Y, R, num_users, ... 37 | num_movies, num_features, lambda); 38 | 39 | disp([numgrad grad]); 40 | fprintf(['The above two columns you get should be very similar.\n' ... 41 | '(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n']); 42 | 43 | diff = norm(numgrad-grad)/norm(numgrad+grad); 44 | fprintf(['If your cost function implementation is correct, then \n' ... 45 | 'the relative difference will be small (less than 1e-9). \n' ... 46 | '\nRelative Difference: %g\n'], diff); 47 | 48 | end -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/cofiCostFunc.m: -------------------------------------------------------------------------------- 1 | function [J, grad] = cofiCostFunc(params, Y, R, num_users, num_movies, ... 2 | num_features, lambda) 3 | %COFICOSTFUNC Collaborative filtering cost function 4 | % [J, grad] = COFICOSTFUNC(params, Y, R, num_users, num_movies, ... 5 | % num_features, lambda) returns the cost and gradient for the 6 | % collaborative filtering problem. 7 | % 8 | 9 | % Unfold the U and W matrices from params 10 | X = reshape(params(1:num_movies*num_features), num_movies, num_features); 11 | Theta = reshape(params(num_movies*num_features+1:end), ... 12 | num_users, num_features); 13 | 14 | errors = (X * Theta' - Y) .* R; 15 | J = 1/2 * sum(sum(errors .^ 2)); 16 | 17 | penalty = (lambda/2) * (sum(sum(Theta .^2)) + sum(sum(X .^2))); 18 | J = J + penalty; 19 | 20 | 21 | X_grad = errors * Theta + lambda * X; 22 | Theta_grad = errors' * X + lambda * Theta; 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | % ============================================================= 41 | 42 | grad = [X_grad(:); Theta_grad(:)]; 43 | 44 | end 45 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/computeNumericalGradient.m: -------------------------------------------------------------------------------- 1 | function numgrad = computeNumericalGradient(J, theta) 2 | %COMPUTENUMERICALGRADIENT Computes the gradient using "finite differences" 3 | %and gives us a numerical estimate of the gradient. 4 | % numgrad = COMPUTENUMERICALGRADIENT(J, theta) computes the numerical 5 | % gradient of the function J around theta. Calling y = J(theta) should 6 | % return the function value at theta. 7 | 8 | % Notes: The following code implements numerical gradient checking, and 9 | % returns the numerical gradient.It sets numgrad(i) to (a numerical 10 | % approximation of) the partial derivative of J with respect to the 11 | % i-th input argument, evaluated at theta. (i.e., numgrad(i) should 12 | % be the (approximately) the partial derivative of J with respect 13 | % to theta(i).) 14 | % 15 | 16 | numgrad = zeros(size(theta)); 17 | perturb = zeros(size(theta)); 18 | e = 1e-4; 19 | for p = 1:numel(theta) 20 | % Set perturbation vector 21 | perturb(p) = e; 22 | loss1 = J(theta - perturb); 23 | loss2 = J(theta + perturb); 24 | % Compute Numerical Gradient 25 | numgrad(p) = (loss2 - loss1) / (2*e); 26 | perturb(p) = 0; 27 | end 28 | 29 | end 30 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/estimateGaussian.m: -------------------------------------------------------------------------------- 1 | function [mu sigma2] = estimateGaussian(X) 2 | %ESTIMATEGAUSSIAN This function estimates the parameters of a 3 | %Gaussian distribution using the data in X 4 | % [mu sigma2] = estimateGaussian(X), 5 | % The input X is the dataset with each n-dimensional data point in one row 6 | % The output is an n-dimensional vector mu, the mean of the data set 7 | % and the variances sigma^2, an n x 1 vector 8 | % 9 | 10 | % Useful variables 11 | [m, n] = size(X); 12 | 13 | 14 | % ====================== YOUR CODE HERE ====================== 15 | % Instructions: Compute the mean of the data and the variances 16 | % In particular, mu(i) should contain the mean of 17 | % the data for the i-th feature and sigma2(i) 18 | % should contain variance of the i-th feature. 19 | % 20 | 21 | mu = mean(X, 1); 22 | sigma2 = (1.0/m) * sum((X - mu) .^ 2); 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | % ============================================================= 34 | 35 | 36 | end 37 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/ex8_movieParams.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8/ex8_movieParams.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/ex8_movies.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8/ex8_movies.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/ex8data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8/ex8data1.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/ex8data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8/ex8data2.mat -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/loadMovieList.m: -------------------------------------------------------------------------------- 1 | function movieList = loadMovieList() 2 | %GETMOVIELIST reads the fixed movie list in movie.txt and returns a 3 | %cell array of the words 4 | % movieList = GETMOVIELIST() reads the fixed movie list in movie.txt 5 | % and returns a cell array of the words in movieList. 6 | 7 | 8 | %% Read the fixed movieulary list 9 | fid = fopen('movie_ids.txt'); 10 | 11 | % Store all movies in cell array movie{} 12 | n = 1682; % Total number of movies 13 | 14 | movieList = cell(n, 1); 15 | for i = 1:n 16 | % Read line 17 | line = fgets(fid); 18 | % Word Index (can ignore since it will be = i) 19 | [idx, movieName] = strtok(line, ' '); 20 | % Actual Word 21 | movieList{i} = strtrim(movieName); 22 | end 23 | fclose(fid); 24 | 25 | end 26 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/movie_ids.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8/movie_ids.txt -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/multivariateGaussian.m: -------------------------------------------------------------------------------- 1 | function p = multivariateGaussian(X, mu, Sigma2) 2 | %MULTIVARIATEGAUSSIAN Computes the probability density function of the 3 | %multivariate gaussian distribution. 4 | % p = MULTIVARIATEGAUSSIAN(X, mu, Sigma2) Computes the probability 5 | % density function of the examples X under the multivariate gaussian 6 | % distribution with parameters mu and Sigma2. If Sigma2 is a matrix, it is 7 | % treated as the covariance matrix. If Sigma2 is a vector, it is treated 8 | % as the \sigma^2 values of the variances in each dimension (a diagonal 9 | % covariance matrix) 10 | % 11 | 12 | k = length(mu); 13 | 14 | if (size(Sigma2, 2) == 1) || (size(Sigma2, 1) == 1) 15 | Sigma2 = diag(Sigma2); 16 | end 17 | 18 | X = bsxfun(@minus, X, mu(:)'); 19 | p = (2 * pi) ^ (- k / 2) * det(Sigma2) ^ (-0.5) * ... 20 | exp(-0.5 * sum(bsxfun(@times, X * pinv(Sigma2), X), 2)); 21 | 22 | end -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/normalizeRatings.m: -------------------------------------------------------------------------------- 1 | function [Ynorm, Ymean] = normalizeRatings(Y, R) 2 | %NORMALIZERATINGS Preprocess data by subtracting mean rating for every 3 | %movie (every row) 4 | % [Ynorm, Ymean] = NORMALIZERATINGS(Y, R) normalized Y so that each movie 5 | % has a rating of 0 on average, and returns the mean rating in Ymean. 6 | % 7 | 8 | [m, n] = size(Y); 9 | Ymean = zeros(m, 1); 10 | Ynorm = zeros(size(Y)); 11 | for i = 1:m 12 | idx = find(R(i, :) == 1); 13 | Ymean(i) = mean(Y(i, idx)); 14 | Ynorm(i, idx) = Y(i, idx) - Ymean(i); 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/originalGaussian.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_matlab/week_9/ex8/originalGaussian.m -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/selectThreshold.m: -------------------------------------------------------------------------------- 1 | function [bestEpsilon, bestF1] = selectThreshold(yval, pval) 2 | %SELECTTHRESHOLD Find the best threshold (epsilon) to use for selecting 3 | %outliers 4 | % [bestEpsilon bestF1] = SELECTTHRESHOLD(yval, pval) finds the best 5 | % threshold to use for selecting outliers based on the results from a 6 | % validation set (pval) and the ground truth (yval). 7 | % 8 | 9 | bestEpsilon = 0; 10 | bestF1 = 0; 11 | stepsize = (max(pval) - min(pval)) / 1000; 12 | 13 | for epsilon = min(pval):stepsize:max(pval) 14 | pred = (pval < epsilon); 15 | tp = sum((pred == 1) & (yval == 1)); 16 | fp = sum((pred == 1) & (yval == 0)); 17 | fn = sum((pred == 0) & (yval == 1)); 18 | prec = tp / (tp + fp); 19 | rec = tp / (tp + fn); 20 | F1 = 2 * (prec * rec) / (prec + rec); 21 | if F1 > bestF1 22 | bestF1 = F1; 23 | bestEpsilon = epsilon; 24 | end 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /algorithms_in_matlab/week_9/ex8/visualizeFit.m: -------------------------------------------------------------------------------- 1 | function visualizeFit(X, mu, sigma2) 2 | %VISUALIZEFIT Visualize the dataset and its estimated distribution. 3 | % VISUALIZEFIT(X, p, mu, sigma2) This visualization shows you the 4 | % probability density function of the Gaussian distribution. Each example 5 | % has a location (x1, x2) that depends on its feature values. 6 | % 7 | 8 | [X1,X2] = meshgrid(0:.5:35); 9 | Z = multivariateGaussian([X1(:) X2(:)],mu,sigma2); 10 | Z = reshape(Z,size(X1)); 11 | 12 | plot(X(:, 1), X(:, 2),'bx'); 13 | hold on; 14 | % Do not plot if there are infinities 15 | if (sum(isinf(Z)) == 0) 16 | contour(X1, X2, Z, 10.^(-20:3:0)'); 17 | end 18 | hold off; 19 | 20 | end -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_2/ex1.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/computeCost.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def computeCost(X,y, theta): 4 | """Computes cost for linear regression 5 | J = COMPUTECOST(X, y, theta) computes the cost of using theta as the 6 | parameter for linear regression to fit the data points in X and y""" 7 | 8 | m = y.size #number of training examples 9 | predictions = np.dot(X, theta) #predictions of hypothesis on all m examples 10 | errors = np.subtract(predictions, y) 11 | sqrErrors = np.power(errors, 2) #squared errors 12 | J = 1.0/ (2.0 * m) * sqrErrors.sum() # average squared error among predictions 13 | return J 14 | 15 | 16 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/computeCostMulti.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def computeCostMulti(X,y, theta): 4 | """Computes cost for linear regression 5 | J = COMPUTECOST(X, y, theta) computes the cost of using theta as the 6 | parameter for linear regression to fit the data points in X and y""" 7 | 8 | m = y.size #number of training examples 9 | predictions = np.dot(X, theta) #predictions of hypothesis on all m examples 10 | errors = np.subtract(predictions, y) 11 | sqrErrors = np.power(errors, 2) #squared errors 12 | J = 1.0 / (2 * m) * np.sum(sqrErrors) #cost 13 | return J 14 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/ex1MultiFeatures.py: -------------------------------------------------------------------------------- 1 | """ #Machine Learning Online Class 2 | Exercise 1: Linear regression with multiple variables 3 | """ 4 | import numpy as np 5 | 6 | 7 | import matplotlib.pyplot as plt 8 | from featureNormalize import featureNormalize 9 | from gradientDescentMulti import gradientDescentMulti 10 | from normalEqn import normalEqn 11 | 12 | 13 | def pause(): 14 | input("Press the key to continue...") 15 | 16 | # Initialization 17 | 18 | """## Part 1: Feature Normalization """ 19 | 20 | data = np.loadtxt('ex1data2.txt', delimiter =",") 21 | 22 | X = data[:, 0:2] 23 | y = data[:, 2] 24 | 25 | m = y.size #umber of training examples 26 | 27 | y = y.reshape((m,1)) #make y a matrix (m x 1) 28 | 29 | #Scale features and set them to zero mean 30 | print("Normalizing Features ...\n"); 31 | 32 | X, mu, sigma = featureNormalize(X) 33 | 34 | #Add intercept term to X 35 | X = np.c_[np.ones((m, 1)), X] # Add a column of ones to x 36 | 37 | 38 | """## Part 2: Gradient Descent """ 39 | 40 | print("Running gradient descent ...\n") 41 | 42 | #Choose some alpha value 43 | alpha = 0.01 44 | num_iters = 400 45 | 46 | #Init Theta and Run Gradient Descent 47 | theta = np.zeros((3, 1)) 48 | 49 | J_history, theta = gradientDescentMulti(X, y, theta, alpha, num_iters) 50 | 51 | 52 | #3D plot data 53 | from mpl_toolkits.mplot3d import Axes3D 54 | fig = plt.figure() 55 | ax = Axes3D(fig) 56 | 57 | A = X[:,0].reshape((m)) 58 | B = X[:,1].reshape((m)) 59 | C = y.reshape(m) 60 | 61 | training_data_plot, = ax.plot3D(A, B, zs=C, linestyle='None', color='red', marker='x', markersize=10, label="Training data") 62 | ax.set_xlabel("Size feet ^2") 63 | ax.set_ylabel('Number of bedrooms') 64 | ax.set_zlabel('Price') 65 | plt.show(block=False) 66 | 67 | C = np.dot(X, theta).reshape(m) 68 | 69 | #Plot the linear fit 70 | linea_regression_plot, = ax.plot3D(A, B, zs=C, color = 'blue', label="Linear regression") 71 | plt.show(block=False) 72 | plt.legend(handles=[training_data_plot, linea_regression_plot]) 73 | 74 | #Plot the convergence graph 75 | plt.figure(2) #new window 76 | plt.plot(np.arange(0,num_iters), J_history, linestyle='solid', color='blue') 77 | plt.xlabel("Number of iterations") 78 | plt.ylabel("Cost J") 79 | 80 | plt.show() 81 | 82 | 83 | #Display gradient descent's result 84 | print("Theta computed from gradient descent: \n") 85 | print("\n", theta) 86 | print("\n") 87 | 88 | 89 | 90 | # Estimate the price of a 1650 sq-ft, 3 br house 91 | 92 | normaliz_test_data = np.divide(np.array([[1650, 3]]) - mu, sigma) 93 | price = np.dot(np.c_[1, normaliz_test_data] , theta) 94 | 95 | print("Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):\n", price) 96 | 97 | print("Program paused. Press enter to continue.\n"); 98 | pause() 99 | 100 | """## Part 3: Normal Equations """ 101 | 102 | print("Solving with normal equations...\n") 103 | 104 | #Load Data 105 | data = np.loadtxt('ex1data2.txt', delimiter =",") 106 | X = data[:, 0:2] 107 | y = data[:, 2] 108 | m = y.size #umber of training examples 109 | y = y.reshape((m,1)) #make y a matrix (m x 1) 110 | 111 | 112 | #Add intercept term to X 113 | X = np.c_[np.ones((m, 1)), X] # Add a column of ones to x 114 | 115 | #Calculate the parameters from the normal equation 116 | theta = normalEqn(X, y) 117 | 118 | #Display normal equation's result 119 | print("Theta computed from the normal equations: \n") 120 | print("\n", theta) 121 | print("\n") 122 | 123 | 124 | #Estimate the price of a 1650 sq-ft, 3 br house 125 | 126 | price = np.dot(np.array([[1, 1650, 3]]), theta) 127 | 128 | 129 | print("Predicted price of a 1650 sq-ft, 3 br house (using normal equations):\n", price) 130 | 131 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/ex1data1.txt: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/ex1data2.txt: -------------------------------------------------------------------------------- 1 | 2104,3,399900 2 | 1600,3,329900 3 | 2400,3,369000 4 | 1416,2,232000 5 | 3000,4,539900 6 | 1985,4,299900 7 | 1534,3,314900 8 | 1427,3,198999 9 | 1380,3,212000 10 | 1494,3,242500 11 | 1940,4,239999 12 | 2000,3,347000 13 | 1890,3,329999 14 | 4478,5,699900 15 | 1268,3,259900 16 | 2300,4,449900 17 | 1320,2,299900 18 | 1236,3,199900 19 | 2609,4,499998 20 | 3031,4,599000 21 | 1767,3,252900 22 | 1888,2,255000 23 | 1604,3,242900 24 | 1962,4,259900 25 | 3890,3,573900 26 | 1100,3,249900 27 | 1458,3,464500 28 | 2526,3,469000 29 | 2200,3,475000 30 | 2637,3,299900 31 | 1839,2,349900 32 | 1000,1,169900 33 | 2040,4,314900 34 | 3137,3,579900 35 | 1811,4,285900 36 | 1437,3,249900 37 | 1239,3,229900 38 | 2132,4,345000 39 | 4215,4,549000 40 | 2162,4,287000 41 | 1664,2,368500 42 | 2238,3,329900 43 | 2567,4,314000 44 | 1200,3,299000 45 | 852,2,179900 46 | 1852,4,299900 47 | 1203,3,239500 48 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/featureNormalize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def featureNormalize(X): 4 | """Normalizes the features in X. It returns a normalized 5 | version of X where the mean value of each feature is 0 and 6 | the standard deviation is 1. 7 | This is often a good preprocessing step to do when 8 | working with learning algorithms. """ 9 | 10 | mu = np.mean(X) #mean of each column 11 | sigma = np.std(X) #standard deviation for each column 12 | X_norm = np.divide(np.subtract(X, mu), sigma) 13 | 14 | return X_norm, mu, sigma 15 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/gradientDescent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import computeCost as costModule 3 | 4 | def gradientDescent(X, y, theta, alpha, num_iters): 5 | """Performs gradient descent to learn theta 6 | theta = GRADIENTDESCENT(X, y, theta, alpha, num_iters) updates theta by 7 | taking num_iters gradient steps with learning rate alpha """ 8 | 9 | m = y.size #number of training examples 10 | J_history = np.zeros((num_iters, 1)) 11 | 12 | print("theta before for", theta.shape) 13 | 14 | #train 15 | for iter in range(num_iters): 16 | #do linear regression with identity (f(x) = x) as an activation function 17 | prediction = np.dot(X, theta) 18 | errors = np.subtract(prediction, y) 19 | delta = (1.0/m) * np.dot(X.T, errors) 20 | #update weight 21 | theta = theta - alpha * delta 22 | #save the cost J in every iteration 23 | J_history[iter] = costModule.computeCost(X, y, theta) 24 | 25 | return J_history, theta 26 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/gradientDescentMulti.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import computeCostMulti as costMultiModule 3 | 4 | def gradientDescentMulti(X, y, theta, alpha, num_iters): 5 | """Performs gradient descent to learn theta 6 | theta = GRADIENTDESCENT(X, y, theta, alpha, num_iters) updates theta by 7 | taking num_iters gradient steps with learning rate alpha """ 8 | 9 | m = y.size #number of training examples 10 | J_history = np.zeros((num_iters, 1)) 11 | 12 | for iter in range(1, num_iters): 13 | #do linear regression with identity (f(x) = x) as an activation function 14 | prediction = np.dot(X, theta) 15 | errors = prediction - y 16 | delta = (1.0/m) * np.dot(X.T, errors) 17 | #update weight 18 | theta = theta - alpha * delta 19 | #save the cost J in every iteration 20 | J_history[iter] = costMultiModule.computeCostMulti(X, y, theta) 21 | 22 | return J_history, theta 23 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/normalEqn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def normalEqn(X, y): 4 | """Computes the closed-form solution to linear regression 5 | using the normal equations.""" 6 | f = X.shape[1] #count features 7 | theta = np.zeros((f, 1)) 8 | print("shape theta", theta.shape) 9 | 10 | leftMul = np.linalg.pinv(np.dot(X.T, X)) 11 | rightMul = np.dot(X.T, y) 12 | 13 | theta = np.dot(leftMul, rightMul) 14 | 15 | return theta 16 | 17 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/plotData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | 3 | def plotData(x, y): 4 | """Plots the data points x and y into a new figure """ 5 | 6 | training_data_plot = plt.plot(x, y, linestyle='None', color='red', marker='x', markersize=10, label="Training data") 7 | plt.xlabel('Profit in $10,000') 8 | plt.ylabel('Population of city in 10,000s') 9 | 10 | return training_data_plot 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /algorithms_in_python/week_2/ex1/warmUpExercise.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def warmUpExercise(): 4 | """ an example function that returns the 5x5 identity matrix """ 5 | return np.identity(5) -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_3/ex2.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/costFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | def costFunction(theta, X, y, reg_lambda = None, flattenResult=False): 5 | """Compute cost and gradient for logistic regression 6 | J = COSTFUNCTION(theta, X, y) computes the cost of using theta as the 7 | parameter for logistic regression and the gradient of the cost 8 | w.r.t. to the parameters.""" 9 | 10 | m,n = X.shape 11 | theta = theta.reshape((n,1)) 12 | prediction = sigmoid(np.dot(X, theta)) #make predictions 13 | 14 | cost_y_1 = np.multiply(np.subtract(1,y), np.log(1 - prediction)) 15 | cost_y_0 = np.multiply(np.multiply(-1, y), np.log(prediction)) 16 | 17 | J = (1.0/m) * np.sum(np.subtract(cost_y_0, cost_y_1)) 18 | 19 | if reg_lambda: 20 | J = J + (reg_lambda/(2 * m)) * np.sum(np.power(theta[1:], 2)) #don't regularize theta[0, :] 21 | 22 | return J -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/ex2data2.txt: -------------------------------------------------------------------------------- 1 | 0.051267,0.69956,1 2 | -0.092742,0.68494,1 3 | -0.21371,0.69225,1 4 | -0.375,0.50219,1 5 | -0.51325,0.46564,1 6 | -0.52477,0.2098,1 7 | -0.39804,0.034357,1 8 | -0.30588,-0.19225,1 9 | 0.016705,-0.40424,1 10 | 0.13191,-0.51389,1 11 | 0.38537,-0.56506,1 12 | 0.52938,-0.5212,1 13 | 0.63882,-0.24342,1 14 | 0.73675,-0.18494,1 15 | 0.54666,0.48757,1 16 | 0.322,0.5826,1 17 | 0.16647,0.53874,1 18 | -0.046659,0.81652,1 19 | -0.17339,0.69956,1 20 | -0.47869,0.63377,1 21 | -0.60541,0.59722,1 22 | -0.62846,0.33406,1 23 | -0.59389,0.005117,1 24 | -0.42108,-0.27266,1 25 | -0.11578,-0.39693,1 26 | 0.20104,-0.60161,1 27 | 0.46601,-0.53582,1 28 | 0.67339,-0.53582,1 29 | -0.13882,0.54605,1 30 | -0.29435,0.77997,1 31 | -0.26555,0.96272,1 32 | -0.16187,0.8019,1 33 | -0.17339,0.64839,1 34 | -0.28283,0.47295,1 35 | -0.36348,0.31213,1 36 | -0.30012,0.027047,1 37 | -0.23675,-0.21418,1 38 | -0.06394,-0.18494,1 39 | 0.062788,-0.16301,1 40 | 0.22984,-0.41155,1 41 | 0.2932,-0.2288,1 42 | 0.48329,-0.18494,1 43 | 0.64459,-0.14108,1 44 | 0.46025,0.012427,1 45 | 0.6273,0.15863,1 46 | 0.57546,0.26827,1 47 | 0.72523,0.44371,1 48 | 0.22408,0.52412,1 49 | 0.44297,0.67032,1 50 | 0.322,0.69225,1 51 | 0.13767,0.57529,1 52 | -0.0063364,0.39985,1 53 | -0.092742,0.55336,1 54 | -0.20795,0.35599,1 55 | -0.20795,0.17325,1 56 | -0.43836,0.21711,1 57 | -0.21947,-0.016813,1 58 | -0.13882,-0.27266,1 59 | 0.18376,0.93348,0 60 | 0.22408,0.77997,0 61 | 0.29896,0.61915,0 62 | 0.50634,0.75804,0 63 | 0.61578,0.7288,0 64 | 0.60426,0.59722,0 65 | 0.76555,0.50219,0 66 | 0.92684,0.3633,0 67 | 0.82316,0.27558,0 68 | 0.96141,0.085526,0 69 | 0.93836,0.012427,0 70 | 0.86348,-0.082602,0 71 | 0.89804,-0.20687,0 72 | 0.85196,-0.36769,0 73 | 0.82892,-0.5212,0 74 | 0.79435,-0.55775,0 75 | 0.59274,-0.7405,0 76 | 0.51786,-0.5943,0 77 | 0.46601,-0.41886,0 78 | 0.35081,-0.57968,0 79 | 0.28744,-0.76974,0 80 | 0.085829,-0.75512,0 81 | 0.14919,-0.57968,0 82 | -0.13306,-0.4481,0 83 | -0.40956,-0.41155,0 84 | -0.39228,-0.25804,0 85 | -0.74366,-0.25804,0 86 | -0.69758,0.041667,0 87 | -0.75518,0.2902,0 88 | -0.69758,0.68494,0 89 | -0.4038,0.70687,0 90 | -0.38076,0.91886,0 91 | -0.50749,0.90424,0 92 | -0.54781,0.70687,0 93 | 0.10311,0.77997,0 94 | 0.057028,0.91886,0 95 | -0.10426,0.99196,0 96 | -0.081221,1.1089,0 97 | 0.28744,1.087,0 98 | 0.39689,0.82383,0 99 | 0.63882,0.88962,0 100 | 0.82316,0.66301,0 101 | 0.67339,0.64108,0 102 | 1.0709,0.10015,0 103 | -0.046659,-0.57968,0 104 | -0.23675,-0.63816,0 105 | -0.15035,-0.36769,0 106 | -0.49021,-0.3019,0 107 | -0.46717,-0.13377,0 108 | -0.28859,-0.060673,0 109 | -0.61118,-0.067982,0 110 | -0.66302,-0.21418,0 111 | -0.59965,-0.41886,0 112 | -0.72638,-0.082602,0 113 | -0.83007,0.31213,0 114 | -0.72062,0.53874,0 115 | -0.59389,0.49488,0 116 | -0.48445,0.99927,0 117 | -0.0063364,0.99927,0 118 | 0.63265,-0.030612,0 119 | -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/gradient.py: -------------------------------------------------------------------------------- 1 | from sigmoid import sigmoid 2 | import numpy as np 3 | 4 | def gradient(theta,X,y, reg_lambda=None, flattenResult=False): 5 | m,n = X.shape 6 | theta = theta.reshape((n,1)) 7 | prediction = sigmoid(np.dot(X, theta)) 8 | errors = np.subtract(prediction, y) 9 | grad = (1.0/m) * np.dot(X.T, errors) 10 | 11 | if reg_lambda: 12 | grad_with_regul = grad[1:] + (reg_lambda/m) * theta[1:] 13 | firstRow = grad[0, :].reshape((1,1)) 14 | grad = np.r_[firstRow, grad_with_regul] 15 | 16 | 17 | if flattenResult: 18 | return grad.flatten() 19 | 20 | return grad 21 | -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/mapFeature.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def mapFeature(X1, X2): 4 | """Feature mapping function to polynomial features 5 | MAPFEATURE(X1, X2) maps the two input features 6 | to quadratic features used in the regularization exercise. 7 | Returns a new feature array with more features, comprising of 8 | X1, X2, X1.^2, X2.^2, X1*X2, X1*X2.^2, etc.. 9 | Inputs X1, X2 must be the same size 10 | """ 11 | X1 = X1.reshape((X1.size, 1)) 12 | X2 = X2.reshape((X2.size, 1)) 13 | degree = 6 14 | out = np.ones(shape=(X1[:, 0].size, 1)) 15 | 16 | for i in range(1, degree + 1): 17 | for j in range(i + 1): 18 | r = (X1 ** (i - j)) * (X2 ** j) 19 | out = np.append(out, r, axis=1) 20 | 21 | return out -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/plotData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | def plotData(X, y, xlabel, ylabel, posLineLabel, negLineLabel): 5 | """PLOTDATA Plots the data points X and y into a new figure 6 | PLOTDATA(x,y) plots the data points with + for the positive examples 7 | and o for the negative examples. X is assumed to be a Mx2 matrix.""" 8 | 9 | #for large scale data set use plt.plot instead of scatter! 10 | #training_data_plot = plt.scatter(X[:,0], X[:,1], 30, marker='x', c=y, label="Training data") 11 | #cbar= plt.colorbar() 12 | #cbar.set_label("Admission") 13 | 14 | pos = np.where(y == 1) 15 | neg = np.where(y == 0) 16 | line_pos = plt.plot(X[pos, 0], X[pos, 1], marker='+', color='black', label=posLineLabel , linestyle='None')[0] 17 | line_neg = plt.plot(X[neg, 0], X[neg, 1], marker='o', color='yellow', label=negLineLabel, linestyle='None')[0] 18 | plt.xlabel(xlabel) 19 | plt.ylabel(ylabel) 20 | 21 | return line_pos, line_neg -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/plotDecisionBoundary.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from mapFeature import mapFeature 3 | import matplotlib.pyplot as plt 4 | 5 | def plotDecisionBoundary(theta, X, y): 6 | """PLOTDECISIONBOUNDARY Plots the data points X and y into a new figure with 7 | the decision boundary defined by theta 8 | PLOTDECISIONBOUNDARY(theta, X,y) plots the data points with + for the 9 | positive examples and o for the negative examples. X is assumed to be 10 | a either 11 | 1) Mx3 matrix, where the first column is an all-ones column for the 12 | intercept. 13 | 2) MxN, N>3 matrix, where the first column is all-ones""" 14 | 15 | if X.shape[1] <= 3: 16 | #Only need 2 points to define a line, so choose two endpoints 17 | plot_x = np.c_[np.min(X[:,1]) - 2, np.max(X[:,1]) + 2]; 18 | 19 | #Calculate the decision boundary line 20 | left = -1 / theta[2] 21 | right = theta[1] * plot_x + theta[0] 22 | plot_y = left * right 23 | 24 | #Plot, and adjust axes for better viewing 25 | line, = plt.plot(plot_x.flatten(), plot_y.flatten(), c = 'b', label="Decision Boundary", marker="*") 26 | 27 | plt.axis([30, 100, 30, 100]) 28 | 29 | return line 30 | else: 31 | #Here is the grid range 32 | u = np.linspace(-1, 1.5, num=50) 33 | v = np.linspace(-1, 1.5, num=50) 34 | 35 | A, B = np.meshgrid(u, v) 36 | 37 | z = np.zeros((u.size, v.size)) 38 | #Evaluate z = theta*x over the grid 39 | for i in range(u.size): 40 | for j in range(v.size): 41 | z[i,j] = np.dot(mapFeature(u[i], v[j]), theta) 42 | 43 | z = z.T # important to transpose z before calling contour 44 | #Plot z = 0 45 | #Notice you need to specify the range [0, 0] 46 | return plt.contour(A, B, z) 47 | 48 | -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/predict.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | def predict(theta, X): 5 | """PREDICT Predict whether the label is 0 or 1 using learned logistic 6 | regression parameters theta 7 | p = PREDICT(theta, X) computes the predictions for X using a 8 | threshold at 0.5 (i.e., if sigmoid(theta'*x) >= 0.5, predict 1) """ 9 | 10 | return sigmoid(np.dot(X, theta)) >=0.5 -------------------------------------------------------------------------------- /algorithms_in_python/week_3/ex2/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def sigmoid(z): 4 | """%SIGMOID Compute sigmoid function 5 | g = SIGMOID(z) computes the sigmoid of z. 6 | """ 7 | return 1.0 / (1 + np.exp(-z)) 8 | 9 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_4/ex3.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/Theta2.csv: -------------------------------------------------------------------------------- 1 | -0.761,-1.2124,-0.10187,-2.3685,-1.0578,-2.2082,0.56384,1.2111,2.2103,0.44456,-1.1824,1.0429,-1.6056,1.3042,1.3718,1.7483,-0.23366,-1.5201,1.1532,0.10368,-0.37208,-0.6153,-0.12568,-2.2719,-0.71836,-1.2969 2 | -0.61785,0.61559,-1.2655,1.8575,-0.91853,-0.055026,-0.3859,1.2952,-1.5684,-0.97026,-2.1833,-2.8503,-2.0773,1.6316,0.34902,1.8279,-2.4417,-0.8563,-0.29826,-2.0795,-1.2933,0.89982,0.28307,2.3118,-2.4644,1.4566 3 | -0.68934,-1.9454,2.0136,-3.1232,-0.23618,1.3868,0.90982,-1.5477,-0.79831,-0.656,0.73538,-2.5859,0.47211,0.55349,2.5126,-2.4167,-1.639,1.2027,-1.2025,-1.8345,-1.8801,-0.34056,0.23692,-1.0614,1.0276,-0.47691 4 | -0.67832,0.46299,0.58492,-0.16502,1.9326,-0.22966,-1.8473,0.49012,1.0715,-3.3191,1.5411,0.37372,-0.86485,-2.5827,0.97062,-0.51022,-0.68428,-1.6471,0.21153,-0.27422,1.726,1.3242,-2.6398,-0.080559,-2.0351,-1.4612 5 | -0.59664,-2.0448,2.057,1.951,0.17638,-2.1614,-0.40395,1.8016,-1.5628,-0.25253,0.23586,0.71657,1.0769,-0.35457,-1.6774,-0.12939,-0.67489,1.1407,1.3243,3.2116,-2.1589,-2.6016,-3.2226,-1.8961,-0.87488,2.5104 6 | -0.87795,0.43441,-0.93161,0.18391,-0.36078,0.61958,0.38625,-2.6515,2.2971,-2.0882,-1.8638,1.0606,0.77562,2.1347,-1.1497,-0.52081,0.99743,-1.4831,-2.3139,0.29517,-0.38705,-2.2061,0.30702,-1.1765,-1.6346,-0.82468 7 | -0.52747,1.2156,-1.501,-2.032,-1.5237,-2.4373,-2.3757,-1.3999,-0.88735,-0.63279,1.5045,-1.5808,0.58599,-0.7754,0.94257,2.1092,0.54479,0.43774,-1.2802,-0.04361,1.4775,-1.1328,-0.72847,0.047347,1.6575,1.6854 8 | -0.74902,-0.72249,-3.1523,0.36578,0.19811,-0.7306,1.6526,-2.3004,-1.8747,0.98095,-1.5883,1.3543,2.179,-1.9924,-2.0037,-0.38861,-2.3399,-2.9172,0.99399,-2.7048,-1.2714,1.8609,-1.2052,-0.38014,0.70872,-2.1101 9 | -0.66655,0.53602,1.3031,-1.0337,-4.0308,0.58173,-2.6572,0.8038,-1.0924,2.4991,0.36201,0.66195,-0.92161,-0.83124,-2.002,-2.949,0.64564,-1.1011,0.7451,0.58507,-1.9955,0.62591,1.806,-0.22309,-1.4044,-2.1319 10 | -0.46089,-1.4394,-1.2181,0.71093,0.45217,-0.35953,0.62285,-0.67005,-0.70691,0.063114,-1.232,-1.7465,-2.7196,-2.2144,-1.6931,-0.90927,0.87852,1.1866,-1.8704,0.39796,1.7211,-1.3693,0.85807,-0.2478,1.2801,-1.3275 11 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/displayData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from math import sqrt 4 | 5 | 6 | def displayData(X): 7 | 8 | """DISPLAYDATA Display 2D data in a nice grid 9 | [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 10 | stored in X in a nice grid. It returns the figure handle h and the 11 | displayed array if requested.""" 12 | 13 | 14 | print(X.shape) 15 | #Compute rows, cols 16 | m, n = X.shape 17 | 18 | 19 | nbImagesPerRow = int(sqrt(m)) 20 | columnsCount = (20 + 2) * nbImagesPerRow 21 | 22 | result = np.empty((0, columnsCount)) 23 | row = 0 24 | while row < m: 25 | new_row = np.empty((20, 0)) 26 | for col in range(nbImagesPerRow): 27 | new_row = np.c_[new_row, X[row].reshape(20, 20).T] 28 | new_row = np.c_[new_row, np.zeros((20,2))] 29 | row = row + 1 30 | result = np.r_[result, new_row] 31 | result = np.r_[result, np.zeros((1, columnsCount))] 32 | 33 | #Display Image 34 | plt.imshow(result, cmap='gray', interpolation='nearest') 35 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/ex3_nn.py: -------------------------------------------------------------------------------- 1 | """# Machine Learning Online Class - Exercise 3 | Part 2: Neural Networks 2 | """ 3 | 4 | from predict import predict 5 | from displayData import displayData 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | 9 | def pause(): 10 | input("") 11 | 12 | # Setup the parameters you will use for this exercise 13 | input_layer_size = 400 # 20x20 Input Images of Digits 14 | hidden_layer_size = 25 # 25 hidden units 15 | num_labels = 10 # 10 labels, from 0 to 9 16 | 17 | 18 | """## Part 1: Loading and Visualizing Data ============= 19 | We start the exercise by first loading and visualizing the dataset. 20 | You will be working with a dataset that contains handwritten digits. 21 | """ 22 | 23 | # Load Training Data 24 | print('Loading and Visualizing Data ...\n') 25 | 26 | X = np.loadtxt('MNIST_DATA.csv', delimiter =",") 27 | m,_ = X.shape 28 | 29 | y = np.loadtxt('MNIST_DATA_LABEL.csv', delimiter =",").reshape(m, 1) 30 | 31 | #since python indexes start with 0 and matlab's ones start with 10, we replace all 10 by 0 32 | y = np.where(y == 10, 0, y) 33 | 34 | #Randomly select 100 data points to display 35 | rand_indices = np.random.choice(m, 100) 36 | 37 | displayData(X[rand_indices]) 38 | 39 | plt.show() 40 | 41 | print('Program paused. Press enter to continue.\n') 42 | pause() 43 | 44 | """## Part 2: Loading Pameters ================ 45 | In this part of the exercise, we load some pre-initialized 46 | neural network parameters.""" 47 | 48 | print('\nLoading Saved Neural Network Parameters ...\n') 49 | 50 | # Load the weights into variables Theta1 and Theta2 51 | 52 | Theta1 = np.loadtxt('Theta1.csv', delimiter =",") 53 | Theta2 = np.loadtxt('Theta2.csv', delimiter =",") 54 | 55 | """## Part 3: Implement Predict ================= 56 | After training the neural network, we would like to use it to predict 57 | the labels. You will now implement the "predict" function to use the 58 | neural network to predict the labels of the training set. This lets 59 | you compute the training set accuracy.""" 60 | 61 | p = predict(Theta1, Theta2, X) 62 | y = y.reshape((m)) 63 | 64 | print(p[rand_indices]) 65 | 66 | print("training Set Accuracy:: ", np.multiply(np.mean((p == y).astype(int)), 100), "") 67 | 68 | print('Program paused. Press enter to continue.\n') 69 | pause() 70 | 71 | 72 | # To give you an idea of the network's output, you can also run 73 | # through the examples one at the a time to see what it is predicting. 74 | 75 | # Randomly permute examples 76 | rand_indices = np.random.choice(m, 600) 77 | 78 | for i in range(600): 79 | # Display 80 | print('\nDisplaying Example Image\n') 81 | imageData = X[rand_indices[i]].reshape(400, 1).T 82 | displayData(imageData) 83 | plt.show(block=False) 84 | pred = predict(Theta1, Theta2, imageData) 85 | print('\nNeural Network Prediction: ', pred) 86 | 87 | # Pause with quit option 88 | s = input('Paused - press enter to continue, q to exit:') 89 | if s == 'q': 90 | break -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/lrCostFunction.py: -------------------------------------------------------------------------------- 1 | from sigmoid import sigmoid 2 | import numpy as np 3 | 4 | def lrCostFunction(theta, X, y, reg_lambda): 5 | 6 | """LRCOSTFUNCTION Compute cost and gradient for logistic regression with 7 | regularization 8 | J = LRCOSTFUNCTION(theta, X, y, lambda) computes the cost of using 9 | theta as the parameter for regularized logistic regression and the 10 | gradient of the cost w.r.t. to the parameters. 11 | """ 12 | 13 | m, n = X.shape #number of training examples 14 | theta = theta.reshape((n,1)) 15 | 16 | prediction = sigmoid(X.dot(theta)) 17 | 18 | cost_y_1 = (1 - y) * np.log(1 - prediction) 19 | cost_y_0 = -1 * y * np.log(prediction) 20 | 21 | J = (1.0/m) * np.sum(cost_y_0 - cost_y_1) + (reg_lambda/(2.0 * m)) * np.sum(np.power(theta[1:], 2)) 22 | 23 | return J 24 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/lrGradient.py: -------------------------------------------------------------------------------- 1 | from sigmoid import sigmoid 2 | import numpy as np 3 | 4 | def lrGradient(theta, X,y, reg_lambda, flattenResult=True): 5 | m,n = X.shape 6 | theta = theta.reshape((n,1)) 7 | prediction = sigmoid(np.dot(X, theta)) 8 | errors = np.subtract(prediction, y) 9 | grad = (1.0/m) * np.dot(X.T, errors) 10 | 11 | grad_with_regul = grad[1:] + (reg_lambda/m) * theta[1:] 12 | firstRow = grad[0, :].reshape((1,1)) 13 | grad = np.r_[firstRow, grad_with_regul] 14 | 15 | if flattenResult: 16 | return grad.flatten() 17 | 18 | return grad 19 | 20 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/oneVsAll.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from lrGradient import lrGradient 3 | from lrCostFunction import lrCostFunction 4 | import scipy.optimize as op 5 | 6 | 7 | def oneVsAll(X, y, num_labels, reg_lambda): 8 | """ONEVSALL trains multiple logistic regression classifiers and returns all 9 | the classifiers in a matrix all_theta, where the i-th row of all_theta 10 | corresponds to the classifier for label i 11 | [all_theta] = ONEVSALL(X, y, num_labels, lambda) trains num_labels 12 | logistic regression classifiers and returns each of these classifiers 13 | in a matrix all_theta, where the i-th row of all_theta corresponds 14 | to the classifier for label i 15 | """ 16 | 17 | #Some useful variables 18 | m, n = X.shape 19 | 20 | #You need to return the following variables correctly 21 | all_theta = np.zeros((num_labels, n + 1)) 22 | 23 | #Add ones to the X data matrix 24 | X = np.c_[np.ones((m, 1)), X] 25 | 26 | for c in range(num_labels): 27 | 28 | print("optimizing theta", c) 29 | 30 | #Set Initial theta 31 | initial_theta = np.zeros((n + 1, 1)) 32 | 33 | # Run fmincg to obtain the optimal theta 34 | Result = op.minimize(fun = lrCostFunction, x0 = initial_theta, args = (X, (y == c) * 1, reg_lambda), method = 'TNC', jac = lrGradient, 35 | options={'maxiter' : 50}) 36 | optimal_theta = Result.x 37 | 38 | all_theta[c,:] = optimal_theta 39 | 40 | return all_theta 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/predict.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | def predict(Theta1, Theta2, X): 5 | """PREDICT Predict the label of an input given a trained neural network 6 | p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the 7 | trained weights of a neural network (Theta1, Theta2)""" 8 | 9 | #Useful values 10 | m = X.shape[0] 11 | 12 | a1 = sigmoid(np.c_[np.ones((m, 1)), X].dot(Theta1.T)) 13 | 14 | a2 = sigmoid(np.c_[np.ones((m, 1)), a1].dot(Theta2.T)) 15 | 16 | p = (np.argmax(a2, axis=1) + 1) 17 | 18 | p = np.where(p == 10, 0, p) 19 | 20 | return p 21 | 22 | 23 | -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/predictOneVsAll.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigmoid import sigmoid 3 | 4 | def predictOneVsAll(all_theta, X): 5 | """PREDICT Predict the label for a trained one-vs-all classifier. The labels 6 | are in the range 1..K, where K = size(all_theta, 1). 7 | p = PREDICTONEVSALL(all_theta, X) will return a vector of predictions 8 | for each example in the matrix X. Note that X contains the examples in 9 | rows. all_theta is a matrix where the i-th row is a trained logistic 10 | regression theta vector for the i-th class. You should set p to a vector 11 | of values from 1..K (e.g., p = [1; 3; 1; 2] predicts classes 1, 3, 1, 2 12 | for 4 examples)""" 13 | 14 | m = X.shape[0] 15 | 16 | #Add ones to the X data matrix 17 | X = np.c_[np.ones((m, 1)), X] 18 | 19 | predictions = sigmoid(np.dot(X, all_theta.T)) 20 | 21 | return np.argmax(predictions, axis=1) -------------------------------------------------------------------------------- /algorithms_in_python/week_4/ex3/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def sigmoid(z): 4 | return 1.0 / (1.0 + np.exp(-z)) 5 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_5/ex4.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/Theta2.csv: -------------------------------------------------------------------------------- 1 | -0.761,-1.2124,-0.10187,-2.3685,-1.0578,-2.2082,0.56384,1.2111,2.2103,0.44456,-1.1824,1.0429,-1.6056,1.3042,1.3718,1.7483,-0.23366,-1.5201,1.1532,0.10368,-0.37208,-0.6153,-0.12568,-2.2719,-0.71836,-1.2969 2 | -0.61785,0.61559,-1.2655,1.8575,-0.91853,-0.055026,-0.3859,1.2952,-1.5684,-0.97026,-2.1833,-2.8503,-2.0773,1.6316,0.34902,1.8279,-2.4417,-0.8563,-0.29826,-2.0795,-1.2933,0.89982,0.28307,2.3118,-2.4644,1.4566 3 | -0.68934,-1.9454,2.0136,-3.1232,-0.23618,1.3868,0.90982,-1.5477,-0.79831,-0.656,0.73538,-2.5859,0.47211,0.55349,2.5126,-2.4167,-1.639,1.2027,-1.2025,-1.8345,-1.8801,-0.34056,0.23692,-1.0614,1.0276,-0.47691 4 | -0.67832,0.46299,0.58492,-0.16502,1.9326,-0.22966,-1.8473,0.49012,1.0715,-3.3191,1.5411,0.37372,-0.86485,-2.5827,0.97062,-0.51022,-0.68428,-1.6471,0.21153,-0.27422,1.726,1.3242,-2.6398,-0.080559,-2.0351,-1.4612 5 | -0.59664,-2.0448,2.057,1.951,0.17638,-2.1614,-0.40395,1.8016,-1.5628,-0.25253,0.23586,0.71657,1.0769,-0.35457,-1.6774,-0.12939,-0.67489,1.1407,1.3243,3.2116,-2.1589,-2.6016,-3.2226,-1.8961,-0.87488,2.5104 6 | -0.87795,0.43441,-0.93161,0.18391,-0.36078,0.61958,0.38625,-2.6515,2.2971,-2.0882,-1.8638,1.0606,0.77562,2.1347,-1.1497,-0.52081,0.99743,-1.4831,-2.3139,0.29517,-0.38705,-2.2061,0.30702,-1.1765,-1.6346,-0.82468 7 | -0.52747,1.2156,-1.501,-2.032,-1.5237,-2.4373,-2.3757,-1.3999,-0.88735,-0.63279,1.5045,-1.5808,0.58599,-0.7754,0.94257,2.1092,0.54479,0.43774,-1.2802,-0.04361,1.4775,-1.1328,-0.72847,0.047347,1.6575,1.6854 8 | -0.74902,-0.72249,-3.1523,0.36578,0.19811,-0.7306,1.6526,-2.3004,-1.8747,0.98095,-1.5883,1.3543,2.179,-1.9924,-2.0037,-0.38861,-2.3399,-2.9172,0.99399,-2.7048,-1.2714,1.8609,-1.2052,-0.38014,0.70872,-2.1101 9 | -0.66655,0.53602,1.3031,-1.0337,-4.0308,0.58173,-2.6572,0.8038,-1.0924,2.4991,0.36201,0.66195,-0.92161,-0.83124,-2.002,-2.949,0.64564,-1.1011,0.7451,0.58507,-1.9955,0.62591,1.806,-0.22309,-1.4044,-2.1319 10 | -0.46089,-1.4394,-1.2181,0.71093,0.45217,-0.35953,0.62285,-0.67005,-0.70691,0.063114,-1.232,-1.7465,-2.7196,-2.2144,-1.6931,-0.90927,0.87852,1.1866,-1.8704,0.39796,1.7211,-1.3693,0.85807,-0.2478,1.2801,-1.3275 11 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/checkNNGradients.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from debugInitializeWeights import debugInitializeWeights 4 | from computeNumericalGradient import computeNumericalGradient 5 | from nnCostFunction import nnCostFunction 6 | 7 | def checkNNGradients(reg_lambda = 0): 8 | 9 | """ Creates a small neural network to check the backpropagation gradients 10 | CHECKNNGRADIENTS(reg_lambda) Creates a small neural network to check the 11 | backpropagation gradients, it will output the analytical gradients 12 | produced by your backprop code and the numerical gradients (computed 13 | using computeNumericalGradient). These two gradient computations should 14 | result in very similar values.""" 15 | 16 | input_layer_size = 3 17 | hidden_layer_size = 5 18 | num_labels = 3 19 | m = 5 20 | 21 | # We generate some 'random' test data 22 | Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size) 23 | Theta2 = debugInitializeWeights(num_labels, hidden_layer_size) 24 | 25 | # Reusing debugInitializeWeights to generate X 26 | X = debugInitializeWeights(m, input_layer_size - 1) 27 | y = np.mod(np.arange(m), num_labels).T.reshape(m, 1) 28 | 29 | # Unroll parameters 30 | nn_params = np.r_[Theta1.ravel(), Theta2.ravel()] 31 | 32 | 33 | # Short hand for cost function 34 | costFunc = lambda params: nnCostFunction(params, input_layer_size, hidden_layer_size, num_labels, X, y, reg_lambda) 35 | 36 | cost, grad = costFunc(nn_params) 37 | 38 | numgrad = computeNumericalGradient(costFunc, nn_params) 39 | 40 | # Visually examine the two gradient computations. The two columns 41 | # you get should be very similar. 42 | print(numgrad, grad) 43 | 44 | print('The above two columns you get should be very similar.\n', 45 | '(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n') 46 | 47 | # Evaluate the norm of the difference between two solutions. 48 | # If you have a correct implementation, and assuming you used EPSILON = 0.0001 49 | # in computeNumericalGradient.py, then diff below should be less than 1e-9 50 | diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad) 51 | 52 | print('If your backpropagation implementation is correct, then \n', 53 | 'the relative difference will be small (less than 1e-9). \n', 54 | '\nRelative Difference: \n', diff) 55 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/computeNumericalGradient.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def computeNumericalGradient(J, theta): 4 | """Computes the gradient using "finite differences" 5 | and gives us a numerical estimate of the gradient. 6 | numgrad = COMPUTENUMERICALGRADIENT(J, theta) computes the numerical 7 | gradient of the function J around theta. Calling y = J(theta) should 8 | return the function value at theta.""" 9 | 10 | # Notes: The following code implements numerical gradient checking, and 11 | # returns the numerical gradient.It sets numgrad(i) to (a numerical 12 | # approximation of) the partial derivative of J with respect to the 13 | # i-th input argument, evaluated at theta. (i.e., numgrad(i) should 14 | # be the (approximately) the partial derivative of J with respect 15 | # to theta(i).) 16 | # 17 | 18 | numgrad = np.zeros(theta.shape) 19 | perturb = np.zeros(theta.shape) 20 | length = theta.shape[0] 21 | 22 | e = 1e-4 23 | for p in range(length): 24 | # Set perturbation vector 25 | perturb[p] = e 26 | loss1, tmp = J(theta - perturb) 27 | loss2, tmp = J(theta + perturb) 28 | # Compute Numerical Gradient 29 | numgrad[p] = (loss2 - loss1) / (2*e) 30 | perturb[p] = 0 31 | 32 | return numgrad 33 | 34 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/debugInitializeWeights.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def debugInitializeWeights(fan_out, fan_in): 4 | """ Initialize the weights of a layer with fan_in 5 | incoming connections and fan_out outgoing connections using a fixed 6 | strategy, this will help you later in debugging 7 | W = debugInitializeWeights(fan_in, fan_out) initializes the weights 8 | of a layer with fan_in incoming connections and fan_out outgoing 9 | connections using a fix set of values""" 10 | 11 | # Set W to zeros 12 | W = np.zeros((fan_out, fan_in + 1)) 13 | 14 | # Initialize W using "sin", this ensures that W is always of the same 15 | # values and will be useful for debugging 16 | array = np.sin(np.arange(W.size)) 17 | 18 | W = np.reshape(array, W.shape) / 10 19 | 20 | return W 21 | 22 | 23 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/displayData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from math import sqrt 4 | 5 | 6 | def displayData(X): 7 | 8 | """DISPLAYDATA Display 2D data in a nice grid 9 | [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 10 | stored in X in a nice grid. It returns the figure handle h and the 11 | displayed array if requested.""" 12 | 13 | #Compute rows, cols 14 | m, n = X.shape 15 | 16 | nbImagesPerRow = int(sqrt(m)) 17 | columnsCount = (20 + 2) * nbImagesPerRow 18 | 19 | result = np.empty((0, columnsCount)) 20 | row = 0 21 | while row < m: 22 | new_row = np.empty((20, 0)) 23 | for col in range(nbImagesPerRow): 24 | new_row = np.c_[new_row, X[row].reshape(20, 20).T] 25 | new_row = np.c_[new_row, np.zeros((20,2))] 26 | row = row + 1 27 | result = np.r_[result, new_row] 28 | result = np.r_[result, np.zeros((1, columnsCount))] 29 | 30 | #Display Image 31 | plt.imshow(result, cmap='gray', interpolation='nearest') 32 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/predict.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sigmoid import sigmoid 4 | 5 | def predict(Theta1, Theta2, X): 6 | """Predicts the label of an input given a trained neural network 7 | p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the 8 | trained weights of a neural network (Theta1, Theta2)""" 9 | 10 | #Useful values 11 | m = X.shape[0] 12 | 13 | h1 = sigmoid(np.c_[np.ones((m, 1)), X].dot(Theta1.T)) 14 | 15 | h2 = sigmoid(np.c_[np.ones((m, 1)), h1].dot(Theta2.T)) 16 | 17 | p = np.argmax(h2, axis=1) 18 | 19 | return p 20 | 21 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/randInitializeWeights.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def randInitializeWeights(L_in, L_out): 4 | """Randomly initialize the weights of a layer with L_in 5 | incoming connections and L_out outgoing connections 6 | W = RANDINITIALIZEWEIGHTS(L_in, L_out) randomly initializes the weights 7 | of a layer with L_in incoming connections and L_out outgoing 8 | connections. 9 | Note that W should be set to a matrix of size(L_out, 1 + L_in) as 10 | the first row of W handles the "bias" terms 11 | """ 12 | 13 | epsilon_init = 0.12 14 | W = np.random.uniform(0, 1, (L_out, L_in + 1)) * 2 * epsilon_init - epsilon_init 15 | 16 | return W 17 | 18 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def sigmoid(z): 4 | """SIGMOID Compute sigmoid functoon 5 | J = SIGMOID(z) computes the sigmoid of z.""" 6 | 7 | return 1.0 / (1.0 + np.exp(-z)) 8 | -------------------------------------------------------------------------------- /algorithms_in_python/week_5/ex4/sigmoidGradient.py: -------------------------------------------------------------------------------- 1 | from sigmoid import sigmoid 2 | 3 | def sigmoidGradient(z): 4 | """returns the gradient of the sigmoid function evaluated at z 5 | g = SIGMOIDGRADIENT(z) computes the gradient of the sigmoid function 6 | evaluated at z. This should work regardless if z is a matrix or a 7 | vector. In particular, if z is a vector or matrix, you should return 8 | the gradient for each element.""" 9 | 10 | return sigmoid(z) * (1-sigmoid(z)) -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_6/ex5.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/cross_validation_features.csv: -------------------------------------------------------------------------------- 1 | -16.747 2 | -14.577 3 | 34.516 4 | -47.01 5 | 36.975 6 | -40.686 7 | -4.472 8 | 26.534 9 | -42.798 10 | 25.374 11 | -31.11 12 | 27.312 13 | -3.2639 14 | -1.8183 15 | -40.72 16 | -50.013 17 | -17.412 18 | 3.5882 19 | 7.0855 20 | 46.282 21 | 14.612 22 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/cross_validation_labels.csv: -------------------------------------------------------------------------------- 1 | 4.1702 2 | 4.0673 3 | 31.873 4 | 10.624 5 | 31.836 6 | 4.9594 7 | 4.4516 8 | 22.276 9 | -4.3874e-05 10 | 20.504 11 | 3.8583 12 | 19.365 13 | 4.8838 14 | 11.097 15 | 7.4617 16 | 1.4769 17 | 2.7192 18 | 10.927 19 | 8.3487 20 | 52.782 21 | 13.357 22 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/featureNormalize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def featureNormalize(X): 4 | """FEATURENORMALIZE Normalizes the features in X 5 | FEATURENORMALIZE(X) returns a normalized version of X where 6 | the mean value of each feature is 0 and the standard deviation 7 | is 1. This is often a good preprocessing step to do when 8 | working with learning algorithms.""" 9 | 10 | mu = np.mean(X, axis=0) #mean of each column 11 | X_norm = X - mu 12 | sigma = np.std(X_norm, axis=0) #standard deviation for each column 13 | X_norm = X_norm / sigma 14 | 15 | return X_norm, mu, sigma 16 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/learningCurve.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from trainLinearReg import trainLinearReg 3 | from linearRegCostFunction import linearRegCostFunction 4 | 5 | def learningCurve(X, y, Xval, yval, reg_lambda): 6 | """Generates the train and cross validation set errors needed 7 | to plot a learning curve. 8 | [error_train, error_val] = ... 9 | LEARNINGCURVE(X, y, Xval, yval, lambda) returns the train and 10 | cross validation set errors for a learning curve. In particular, 11 | it returns two vectors of the same length - error_train and 12 | error_val. Then, error_train(i) contains the training error for 13 | i examples (and similarly for error_val(i)). 14 | 15 | In this function, you will compute the train and test errors for 16 | dataset sizes from 1 up to m. In practice, when working with larger 17 | datasets, you might want to do this in larger intervals. 18 | """ 19 | 20 | # Number of training examples 21 | m = X.shape[0] 22 | 23 | error_train = np.zeros((m, 1)) 24 | error_val = np.zeros((m,1)) 25 | 26 | for i in range(1, m): 27 | theta = trainLinearReg(X[0:i, :], y[0:i], reg_lambda) 28 | error_train[i], tmp = linearRegCostFunction(X[0:i, :], y[0:i], theta, 0) 29 | #You use the entire validation set to measure J_cv because you want to know how well 30 | #the theta values work on the validation set. You get a better (average) measurement 31 | #by using the entire CV set. 32 | error_val[i], tmp = linearRegCostFunction(Xval, yval, theta, 0) 33 | 34 | return error_train, error_val 35 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/linearRegCostFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyGrad= None, returnOnlyCost = None, flattenResult = None ): 4 | """Computes cost and gradient for regularized linear 5 | regression with multiple variables 6 | [J, grad] = LINEARREGCOSTFUNCTION(X, y, theta, lambda) computes the 7 | cost of using theta as the parameter for linear regression to fit the 8 | data points in X and y. Returns the cost in J and the gradient in grad 9 | """ 10 | 11 | # Initialize some useful values 12 | m = len(y) # number of training examples 13 | 14 | 15 | theta_column_size = y.shape[1] 16 | theta_row_size = X.shape[1] 17 | 18 | theta = theta.reshape(theta_row_size, theta_column_size) 19 | 20 | predictions = X.dot(theta) 21 | 22 | errors = predictions - y 23 | 24 | #we dont regularize theta[0] 25 | J = (1.0/(2 * m)) * np.sum(errors ** 2) + (reg_lambda / (2 * m)) * np.sum(theta[1:] ** 2) 26 | 27 | grad = (1.0/m) * X.T.dot(errors) 28 | 29 | #we dont regularize theta[0] (bias) 30 | grad = np.r_[grad[0, :].reshape(1, theta_column_size), grad[1:, :] + (reg_lambda /m) * theta[1:, :]] 31 | 32 | if returnOnlyGrad: 33 | if flattenResult: 34 | return grad.flatten() 35 | return grad 36 | 37 | if returnOnlyCost: 38 | return J 39 | 40 | return J, grad 41 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/plotFit.py: -------------------------------------------------------------------------------- 1 | from polyFeatures import polyFeatures 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | def plotFit(min_x, max_x, mu, sigma, theta, p, label): 6 | """Plots a learned polynomial regression fit over an existing figure. 7 | Also works with linear regression. 8 | PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial 9 | fit with power p and feature normalization (mu, sigma). 10 | """ 11 | 12 | # We plot a range slightly bigger than the min and max values to get 13 | # an idea of how the fit will vary outside the range of the data points 14 | x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1) 15 | 16 | # Map the X values 17 | X_poly = polyFeatures(x, p) 18 | 19 | X_poly = X_poly - mu 20 | X_poly = X_poly / sigma 21 | 22 | # Add ones 23 | X_poly = np.c_[np.ones((x.shape[0], 1)), X_poly] 24 | 25 | curve, = plt.plot(x, X_poly.dot(theta), color='blue', label=label) 26 | 27 | return curve 28 | 29 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/polyFeatures.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def polyFeatures(X, p): 4 | """Maps X (1D vector) into the p-th power 5 | [X_poly] = POLYFEATURES(X, p) takes a data matrix X (size m x 1) and 6 | maps each example into its polynomial features where 7 | X_poly(i, :) = [X(i) X(i).^2 X(i).^3 ... X(i).^p]; 8 | """ 9 | X_poly = np.zeros((X.shape[0], p)) 10 | 11 | for j in range(1, p + 1): 12 | X_poly[:, j - 1] = X[:, 0] ** j 13 | #print("X_poly[:, {}]".format(j - 1), X_poly[:, j - 1]) 14 | 15 | return X_poly 16 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/polynomialDegreeCurve.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from linearRegCostFunction import linearRegCostFunction 3 | from trainLinearReg import trainLinearReg 4 | from polyFeatures import polyFeatures 5 | from featureNormalize import featureNormalize 6 | 7 | 8 | def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda): 9 | """Error cruve in function of degree of polynimal d 10 | """ 11 | 12 | dimensions = np.arange(1, 80).reshape(-1, 1) 13 | 14 | # You need to return these variables correctly. 15 | error_train = np.zeros((len(dimensions), 1)) 16 | error_val = np.zeros((len(dimensions), 1)) 17 | 18 | m_train_set = X.shape[0] 19 | m_val_set = Xval.shape[0] 20 | 21 | 22 | for i in range(len(dimensions)): 23 | dimension = dimensions[i] 24 | 25 | X_poly = polyFeatures(X, dimension) 26 | X_poly, mu, sigma = featureNormalize(X_poly) # Normalize 27 | X_poly = np.c_[np.ones((m_train_set, 1)), X_poly] 28 | 29 | X_poly_val = polyFeatures(Xval, dimension) 30 | X_poly_val = X_poly_val - mu 31 | X_poly_val = X_poly_val / sigma 32 | X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val] 33 | 34 | theta = trainLinearReg(X_poly, y, reg_lambda) 35 | error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0) 36 | error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0) 37 | 38 | 39 | return dimensions, error_train, error_val -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/r2_score.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | #https://en.wikipedia.org/wiki/Coefficient_of_determination 5 | def r2_score(y_true, y_pred): 6 | mean = np.mean(y_true) #mean of the observed dat 7 | SS_tot = np.sum((y_true - mean) ** 2) #The total sum of squares (proportional to the variance of the data) 8 | SS_res = np.sum((y_true - y_pred) ** 2) #The sum of squares of residuals, also called the residual sum of squares 9 | r2_score = 1 - (SS_res / SS_tot) #The most general definition of the coefficient of determination 10 | return r2_score 11 | 12 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/test_features.csv: -------------------------------------------------------------------------------- 1 | -33.318 2 | -37.912 3 | -51.207 4 | -6.1326 5 | 21.261 6 | -40.32 7 | -14.542 8 | 32.56 9 | 13.393 10 | 44.21 11 | -1.1427 12 | -12.767 13 | 34.055 14 | 39.224 15 | 1.9745 16 | 29.622 17 | -23.67 18 | -9.0118 19 | -55.941 20 | -35.709 21 | 9.5102 22 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/test_labels.csv: -------------------------------------------------------------------------------- 1 | 3.3169 2 | 5.3977 3 | 0.13043 4 | 6.1926 5 | 17.088 6 | 0.79951 7 | 2.8248 8 | 28.621 9 | 17.046 10 | 55.384 11 | 4.0794 12 | 8.2704 13 | 31.324 14 | 39.159 15 | 8.0873 16 | 24.111 17 | 2.4774 18 | 6.5661 19 | 6.0381 20 | 4.6927 21 | 10.83 22 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/trainLinearReg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.optimize as op 3 | from linearRegCostFunction import linearRegCostFunction 4 | 5 | def trainLinearReg(X, y, reg_lambda): 6 | """TRAINLINEARREG Trains linear regression given a dataset (X, y) and a 7 | regularization parameter lambda 8 | [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using 9 | the dataset (X, y) and regularization parameter lambda. Returns the 10 | trained parameters theta. 11 | """ 12 | 13 | #Initialize Theta 14 | initial_theta = np.zeros((X.shape[1], 1)) 15 | 16 | # Create "short hand" for the cost function to be minimized 17 | costFunction = lambda theta : linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyCost = True) 18 | 19 | gradFunc = lambda theta : linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyGrad = True, flattenResult = True) 20 | 21 | #should finish learning (reach local minima) in 4 iterations 22 | max_iter = 200 23 | 24 | # Run fmincg to obtain the optimal theta 25 | Result = op.minimize(fun = costFunction, x0 = initial_theta, method = 'TNC', jac = gradFunc, 26 | options={'maxiter' : max_iter, 'disp': True}) 27 | 28 | optimal_theta = Result.x 29 | 30 | return optimal_theta 31 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/train_features.csv: -------------------------------------------------------------------------------- 1 | -15.937 2 | -29.153 3 | 36.19 4 | 37.492 5 | -48.059 6 | -8.9415 7 | 15.308 8 | -34.706 9 | 1.3892 10 | -44.384 11 | 7.0135 12 | 22.763 13 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/train_labels.csv: -------------------------------------------------------------------------------- 1 | 2.1343 2 | 1.1733 3 | 34.359 4 | 36.838 5 | 2.809 6 | 2.1211 7 | 14.71 8 | 2.6142 9 | 3.7402 10 | 3.7317 11 | 7.6277 12 | 22.752 13 | -------------------------------------------------------------------------------- /algorithms_in_python/week_6/ex5/validationCurve.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from linearRegCostFunction import linearRegCostFunction 3 | from trainLinearReg import trainLinearReg 4 | 5 | def validationCurve(X, y, Xval, yval): 6 | """Generates the train and validation errors needed to 7 | plot a validation curve that we can use to select lambda 8 | VALIDATIONCURVE(X, y, Xval, yval) returns the train 9 | and validation errors (in error_train, error_val) 10 | for different values of lambda. You are given the training set (X, 11 | y) and validation set (Xval, yval). 12 | """ 13 | 14 | # Selected values of lambda (you should not change this) 15 | lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1, 1) 16 | 17 | # You need to return these variables correctly. 18 | error_train = np.zeros((len(lambda_vec), 1)) 19 | error_val = np.zeros((len(lambda_vec), 1)) 20 | 21 | 22 | for i in range(len(lambda_vec)): 23 | curr_lambda = lambda_vec[i] 24 | theta = trainLinearReg(X, y, curr_lambda) 25 | error_train[i], tmp = linearRegCostFunction(X, y, theta, 0) 26 | error_val[i], tmp = linearRegCostFunction(Xval, yval, theta, 0) 27 | 28 | 29 | return lambda_vec, error_train, error_val -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_7/ex6.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/dataset3Params.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from svmModel import SVMModel 3 | 4 | 5 | def dataset3_params(x, y, xval, yval): 6 | """ 7 | Returns your choice of C and sigma for Part 3 of the exercise 8 | where you select the optimal (C, sigma) learning parameters to use for SVM 9 | with RBF kernel 10 | [C, sigma] = DATASET3PARAMS(X, y, Xval, yval) returns your choice of C and 11 | sigma. You should complete this function to return the optimal C and 12 | sigma based on a cross-validation set. 13 | """ 14 | 15 | # You need to return the following variables correctly. 16 | c = 1 17 | sigma = 0.3 18 | 19 | values = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30] 20 | max_error = 1 21 | 22 | for current_c in values: 23 | for current_sigma in values: 24 | 25 | # ALWAYS train the model on training sets (X and y) 26 | model = SVMModel() 27 | model.train(x, y, current_c, kernel_type='rbf', tol=1e-3, max_passes=5, sigma=current_sigma) 28 | 29 | # AND evaluate it on cross validation set 30 | predictions = model.predict(xval) 31 | error = np.mean((predictions != yval).astype(float)) 32 | 33 | if error <= max_error: 34 | max_error = error 35 | c = current_c 36 | sigma = current_sigma 37 | 38 | return c, sigma 39 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/emailFeatures.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def email_features(word_indices): 5 | """Takes in a word_indices vector and produces a feature vector 6 | from the word indices 7 | """ 8 | 9 | # Total number of words in the dictionary 10 | n = 1899 11 | 12 | vector = np.arange(1, n + 1).reshape(-1, 1) 13 | 14 | return np.in1d(vector, word_indices) 15 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/emailSample1.txt: -------------------------------------------------------------------------------- 1 | > Anyone knows how much it costs to host a web portal ? 2 | > 3 | Well, it depends on how many visitors you're expecting. 4 | This can be anywhere from less than 10 bucks a month to a couple of $100. 5 | You should checkout http://www.rackspace.com/ or perhaps Amazon EC2 6 | if youre running something big.. 7 | 8 | To unsubscribe yourself from this mailing list, send an email to: 9 | groupname-unsubscribe@egroups.com 10 | 11 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/emailSample2.txt: -------------------------------------------------------------------------------- 1 | Folks, 2 | 3 | my first time posting - have a bit of Unix experience, but am new to Linux. 4 | 5 | 6 | Just got a new PC at home - Dell box with Windows XP. Added a second hard disk 7 | for Linux. Partitioned the disk and have installed Suse 7.2 from CD, which went 8 | fine except it didn't pick up my monitor. 9 | 10 | I have a Dell branded E151FPp 15" LCD flat panel monitor and a nVidia GeForce4 11 | Ti4200 video card, both of which are probably too new to feature in Suse's default 12 | set. I downloaded a driver from the nVidia website and installed it using RPM. 13 | Then I ran Sax2 (as was recommended in some postings I found on the net), but 14 | it still doesn't feature my video card in the available list. What next? 15 | 16 | Another problem. I have a Dell branded keyboard and if I hit Caps-Lock twice, 17 | the whole machine crashes (in Linux, not Windows) - even the on/off switch is 18 | inactive, leaving me to reach for the power cable instead. 19 | 20 | If anyone can help me in any way with these probs., I'd be really grateful - 21 | I've searched the 'net but have run out of ideas. 22 | 23 | Or should I be going for a different version of Linux such as RedHat? Opinions 24 | welcome. 25 | 26 | Thanks a lot, 27 | Peter 28 | 29 | -- 30 | Irish Linux Users' Group: ilug@linux.ie 31 | http://www.linux.ie/mailman/listinfo/ilug for (un)subscription information. 32 | List maintainer: listmaster@linux.ie 33 | 34 | 35 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/ex6data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_7/ex6/ex6data1.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/ex6data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_7/ex6/ex6data2.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/ex6data3.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_7/ex6/ex6data3.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/getVocabList.py: -------------------------------------------------------------------------------- 1 | def get_vocab_list(file_name): 2 | """"GETVOCABLIST reads the fixed vocabulary list in vocab.txt and returns a 3 | cell array of the words 4 | vocabList = GETVOCABLIST() reads the fixed vocabulary list in vocab.txt 5 | and returns a cell array of the words in vocabList. 6 | """ 7 | 8 | # Read the fixed vocabulary list 9 | try: 10 | with open(file_name, 'r') as file: 11 | lines = file.readlines() 12 | lines = [line.split('\t')[1].strip() for line in lines] 13 | return lines 14 | except IOError: 15 | print("can't open file", file_name) 16 | return [] 17 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/plotData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | def plotData(X, y): 5 | """ PLOTDATA Plots the data points X and y into a new figure 6 | PLOTDATA(x,y) plots the data points with + for the positive examples 7 | and o for the negative examples. X is assumed to be a Mx2 matrix. 8 | """ 9 | # Find Indices of Positive and Negative Examples 10 | pos = np.where(y == 1) 11 | neg = np.where(y == 0) 12 | 13 | 14 | plt.plot(X[pos, 0], X[pos, 1], marker='*', color='black', markersize = 10 , linestyle='None') 15 | plt.plot(X[neg, 0], X[neg, 1], marker='o', color='yellow',markersize = 10 , linestyle='None') 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/processEmail.py: -------------------------------------------------------------------------------- 1 | from nltk.stem import PorterStemmer 2 | from nltk.tokenize import word_tokenize 3 | from getVocabList import get_vocab_list 4 | 5 | import re 6 | 7 | # uncomment if you want to check fo newer version 8 | # import nltk 9 | # nltk.download('punkt') 10 | 11 | 12 | def regexprep(contents, regex, replace_value): 13 | return re.sub(regex, replace_value, contents) 14 | 15 | 16 | def process_email(email_contents, vocab_file_name): 17 | """Pre-processes a the body of an email and 18 | returns a list of word_indices 19 | word_indices = PROCESSEMAIL(email_contents) preprocesses 20 | the body of an email and returns a list of indices of the 21 | words contained in the email. 22 | """ 23 | 24 | # Load Vocabulary 25 | vocab_list = get_vocab_list(vocab_file_name) 26 | 27 | # Init return value 28 | word_indices = [] 29 | 30 | # ========================== Pre-process Email =========================== 31 | 32 | # Lower case 33 | email_contents = email_contents.lower() 34 | 35 | # Strip all HTML 36 | # Looks for any expression that starts with < and ends with > and replace 37 | # and does not have any < or > in the tag it with a space 38 | email_contents = regexprep(email_contents, r'<[^<>]+>', ' ') 39 | 40 | # Handle Numbers 41 | # Look for one or more characters between 0-9 42 | email_contents = regexprep(email_contents, r'[0-9]+', 'number') 43 | 44 | # Handle URLS 45 | # Look for strings starting with http:// or https:// 46 | email_contents = regexprep(email_contents, r'(http|https)://[^\s]*', 'httpaddr') 47 | 48 | # Handle Email Addresses 49 | # Look for strings with @ in the middle 50 | email_contents = regexprep(email_contents, r'[^\s]+@[^\s]+', 'emailaddr') 51 | 52 | # Handle $ sign 53 | email_contents = regexprep(email_contents, r'[$]+', 'dollar') 54 | 55 | # get rid of any punctuation 56 | email_contents = regexprep(email_contents, r'[^\w\s]', '') 57 | 58 | # remove \n 59 | email_contents = regexprep(email_contents, r'\n', '') 60 | 61 | # ========================== Tokenize Email =========================== 62 | 63 | # Output the email to screen as well 64 | print('\n==== Processed Email ====\n\n', email_contents) 65 | 66 | stemmer = PorterStemmer() 67 | 68 | # Tokenize 69 | for token in word_tokenize(email_contents): 70 | # Remove any non alphanumeric characters 71 | word = regexprep(token.strip(), '[^a-zA-Z0-9]', '') 72 | 73 | # Stem the word 74 | word = stemmer.stem(word) 75 | 76 | # Skip the word if it is too short 77 | if len(word) < 1: 78 | continue 79 | 80 | # append index 81 | try: 82 | word_indices.append(vocab_list.index(word) + 1) # add one because training set indexes start from 1 83 | except ValueError: 84 | continue 85 | 86 | return word_indices 87 | 88 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/readFile.py: -------------------------------------------------------------------------------- 1 | def read_file(filename): 2 | """READFILE reads a file and returns its entire contents 3 | file_contents = READFILE(filename) reads a file and returns its entire 4 | contents in file_contents 5 | """ 6 | try: 7 | with open(filename, 'r') as file: 8 | return file.read() 9 | except IOError: 10 | print("can't open file", filename) 11 | return '' 12 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/spamSample1.txt: -------------------------------------------------------------------------------- 1 | Click here to gain money. Visit this website http://ksdlksd.fr to gain many $ below guarantee now -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/spamSample2.txt: -------------------------------------------------------------------------------- 1 | Best Buy Viagra Generic Online 2 | 3 | Viagra 100mg x 60 Pills $125, Free Pills & Reorder Discount, Top Selling 100% Quality & Satisfaction guaranteed! 4 | 5 | We accept VISA, Master & E-Check Payments, 90000+ Satisfied Customers! 6 | http://medphysitcstech.ru 7 | 8 | 9 | -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/spamTest.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_7/ex6/spamTest.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/spamTrain.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_7/ex6/spamTrain.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_7/ex6/visualizeBoundary.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | def visualizeBoundary(X, y, model, h=0.02, pad=0.25): 6 | """ Plots a decision boundary learned by the SVM 7 | """ 8 | x_min, x_max = X[:, 0].min() - pad, X[:, 0].max() + pad 9 | y_min, y_max = X[:, 1].min() - pad, X[:, 1].max() + pad 10 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 11 | z = model.predict(np.c_[xx.ravel(), yy.ravel()]) 12 | z = z.reshape(xx.shape) 13 | plt.contourf(xx, yy, z, cmap=plt.cm.Paired, alpha=0.2) 14 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_8/ex7.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/bird_small.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_8/ex7/bird_small.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/bird_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_8/ex7/bird_small.png -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/computeCentroids.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def compute_centroids(X, idx, K): 5 | """Returns the new centroids by computing the means of the 6 | data points assigned to each centroid. 7 | centroids = compute_centroids(X, idx, K) returns the new centroids by 8 | computing the means of the data points assigned to each centroid. It is 9 | given a dataset X where each row is a single data point, a vector 10 | idx of centroid assignments (i.e. each entry in range [1..K]) for each 11 | example, and K, the number of centroids. You should return a matrix 12 | centroids, where each row of centroids is the mean of the data points 13 | assigned to it. 14 | """ 15 | 16 | # Useful variables 17 | m, n = X.shape 18 | 19 | centroids = np.zeros((K, n)) 20 | 21 | unique_indexes = np.unique(idx) 22 | 23 | for i, val in enumerate(unique_indexes): 24 | Ci_indexes = np.argwhere(idx == val)[:, 0] 25 | Ci = X[Ci_indexes] 26 | centroids[i, :] = np.mean(Ci, axis=0) 27 | 28 | return centroids 29 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/displayData.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | def displayData(X, nrows=10, ncols=10): 6 | # set up array 7 | fig, axarr = plt.subplots(nrows=nrows, ncols=ncols, 8 | figsize=(nrows, ncols)) 9 | 10 | nblock = int(np.sqrt(X.shape[1])) 11 | 12 | # loop over randomly drawn numbers 13 | ct = 0 14 | for ii in range(nrows): 15 | for jj in range(ncols): 16 | # ind = np.random.randint(X.shape[0]) 17 | tmp = X[ct, :].reshape(nblock, nblock, order='F') 18 | axarr[ii, jj].imshow(tmp, cmap='gray') 19 | plt.setp(axarr[ii, jj].get_xticklabels(), visible=False) 20 | plt.setp(axarr[ii, jj].get_yticklabels(), visible=False) 21 | plt.minorticks_off() 22 | ct += 1 23 | 24 | fig.subplots_adjust(hspace=0, wspace=0) 25 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/ex7data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_8/ex7/ex7data1.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/ex7data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_8/ex7/ex7data2.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/ex7faces.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_8/ex7/ex7faces.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/featureNormalize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def featureNormalize(X): 5 | """Normalizes the features in X 6 | featureNormalize(X) returns a normalized version of X where 7 | the mean value of each feature is 0 and the standard deviation 8 | is 1. This is often a good preprocessing step to do when 9 | working with learning algorithms.""" 10 | 11 | mu = np.mean(X, axis=0) # mean of each column 12 | X_norm = X - mu 13 | sigma = np.std(X_norm, axis=0) # standard deviation for each column 14 | X_norm = X_norm / sigma 15 | 16 | return X_norm, mu, sigma 17 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/findClosestCentroids.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def find_closest_centroids(x, centroids): 5 | """ Computes the centroid memberships for every example 6 | idx = FINDCLOSESTCENTROIDS (X, centroids) returns the closest centroids 7 | in idx for a dataset X where each row is a single example. idx = m x 1 8 | vector of centroid assignments (i.e. each entry in range [1..K]) 9 | """ 10 | m, n = x.shape 11 | idx = np.zeros((m, 1)) 12 | 13 | # The algorithm assigns every training example x[i] to its closest centroid 14 | for i in range(m): 15 | V = centroids - x[i, :] 16 | D = np.linalg.norm(V, axis=1, ord=2) ** 2 17 | idx[i] = np.argmin(D) 18 | 19 | return idx 20 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/kMeansInitCentroids.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def kmeans_init_centroids(X, K): 5 | """This function initializes K centroids that are to be 6 | used in K-Means on the dataset X 7 | centroids = kmeans_init_centroids(X, K) returns K initial centroids to be 8 | used with the K-Means on the dataset X 9 | """ 10 | 11 | # Randomly reorder the indices of examples 12 | np.random.seed(0) 13 | randidx = np.random.permutation(X.shape[0]) 14 | # Take the first K examples as centroids 15 | centroids = X[randidx[0:K], :] 16 | 17 | return centroids 18 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/pca.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def pca(X): 5 | """PCA Run principal component analysis on the dataset X 6 | [U, S, X] = pca(X) computes eigenvectors of the covariance matrix of X 7 | Returns the eigenvectors U, the eigenvalues (on diagonal) in S 8 | """ 9 | # Useful values 10 | m, n = X.shape 11 | 12 | Sigma = (1.0 / m) * X.T.dot(X) 13 | 14 | U, S, V = np.linalg.svd(Sigma) 15 | 16 | return U, S 17 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/plotDataPoints.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | 3 | 4 | def plot_data_points(x, idx, K): 5 | """ 6 | Plots data points in X, coloring them so that those with the same 7 | index assignments in idx have the same color 8 | """ 9 | plt.scatter(x[:, 0], x[:, 1], s=40, c=idx.astype(float), cmap=plt.cm.prism) 10 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/plotProgresskMeans.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | from plotDataPoints import plot_data_points 6 | 7 | 8 | def plot_progress_kmeans(X, centroids, previous, idx, K, i): 9 | """A helper function that displays the progress of 10 | #k-Means as it is running. It is intended for use only with 2D data. 11 | # plot_progress_kmeans(X, centroids, previous, idx, K, i) plots the data 12 | # points with colors assigned to each centroid. With the previous 13 | # centroids, it also plots a line between the previous locations and 14 | # current locations of the centroids. 15 | """ 16 | 17 | # Plot the examples 18 | plot_data_points(X, idx, K) 19 | 20 | # Plot the centroids as black x's 21 | plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', s=60, lw=3, edgecolor='k') 22 | plt.draw() 23 | 24 | # Plot the history of the centroids with lines 25 | count = centroids.shape[0] 26 | 27 | arrstr = np.char.mod('%d', np.arange(K)) 28 | c = itertools.cycle("".join(arrstr)) 29 | rgb = np.eye(K, 3) 30 | 31 | for j in range(count): 32 | x = np.r_[centroids[j, 0], previous[j, 0]] 33 | y = np.r_[centroids[j, 1], previous[j, 1]] 34 | plt.plot(x, y, c=rgb[int(next(c))]) 35 | plt.draw() 36 | 37 | plt.title('Iteration number {}'.format(i)) 38 | 39 | plt.show(block=False) 40 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/projectData.py: -------------------------------------------------------------------------------- 1 | def projectData(X, U, K): 2 | """PROJECTDATA Computes the reduced data representation when projecting only 3 | on to the top k eigenvectors 4 | Z = projectData(X, U, K) computes the projection of 5 | the normalized inputs X into the reduced dimensional space spanned by 6 | the first K columns of U. It returns the projected examples in Z. 7 | """ 8 | Ureduce = U[:, 0: K] # take the first k directions 9 | Z = X.dot(Ureduce) # compute the projected data points 10 | return Z 11 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/recoverData.py: -------------------------------------------------------------------------------- 1 | def recoverData(Z, U, K): 2 | """RECOVERDATA Recovers an approximation of the original data when using the 3 | projected data 4 | X_rec = RECOVERDATA(Z, U, K) recovers an approximation the 5 | original data that has been reduced to K dimensions. It returns the 6 | approximate reconstruction in X_rec. 7 | """ 8 | Ureduce = U[:, 0:K] # take the first k directions 9 | X_rec = Z.dot(Ureduce.T) # go back to our original number of features 10 | return X_rec 11 | -------------------------------------------------------------------------------- /algorithms_in_python/week_8/ex7/runKMeans.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from computeCentroids import compute_centroids 3 | from findClosestCentroids import find_closest_centroids 4 | from plotProgresskMeans import plot_progress_kmeans 5 | 6 | 7 | def pause(): 8 | input("") 9 | 10 | 11 | def run_kmeans(X, initial_centroids, max_iters, plot_progress=False): 12 | """[centroids, idx] = RUNKMEANS(X, initial_centroids, max_iters, ... 13 | plot_progress) runs the K-Means algorithm on data matrix X, where each 14 | row of X is a single example. 15 | It uses initial_centroids used as the 16 | initial centroids. max_iters specifies the total number of interactions 17 | of K-Means to execute. plot_progress is a true/false flag that 18 | indicates if the function should also plot its progress as the 19 | learning happens. This is set to false by default. runkMeans returns 20 | centroids, a Kxn matrix of the computed centroids and idx, a m x 1 21 | vector of centroid assignments (i.e. each entry in range [1..K]) 22 | """ 23 | 24 | # Initialize values 25 | m, n = X.shape 26 | K = initial_centroids.shape[0] 27 | centroids = initial_centroids 28 | previous_centroids = centroids 29 | idx = np.zeros((m, 1)) 30 | 31 | # Run K-Means 32 | for i in range(max_iters): 33 | 34 | # Output progress 35 | print('K-Means iteration {}/{}...'.format(i + 1, max_iters)) 36 | 37 | # For each example in X, assign it to the closest centroid 38 | idx = find_closest_centroids(X, centroids) 39 | 40 | # Optionally, plot progress here 41 | if plot_progress: 42 | plot_progress_kmeans(X, centroids, previous_centroids, idx, K, i) 43 | previous_centroids = centroids 44 | print('Press enter to continue.\n') 45 | pause() 46 | 47 | # Given the memberships, compute new centroids 48 | centroids = compute_centroids(X, idx, K) 49 | 50 | return centroids, idx 51 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_9/ex8.pdf -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/checkCostFunction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from cofiCostFunc import cofiCostFunc 4 | from computeNumericalGradient import computeNumericalGradient 5 | 6 | 7 | def checkCostFunction(reg_lambda=0): 8 | """ Creates a collaborative filtering problem 9 | to check your cost function and gradients 10 | checkCostFunction(lambda) Creates a collaborative filtering problem 11 | to check your cost function and gradients, it will output the 12 | analytical gradients produced by your code and the numerical gradients 13 | (computed using computeNumericalGradient). These two gradient 14 | computations should result in very similar values.""" 15 | 16 | # Create small problem 17 | X_t = np.random.rand(4, 3) 18 | Theta_t = np.random.rand(5, 3) 19 | 20 | # Zap out most entries 21 | Y = X_t.dot(Theta_t.T) 22 | rand_data = np.random.randn(*Y.shape) 23 | Y[np.where(rand_data > 0.5)] = 0 24 | R = np.zeros(Y.shape) 25 | R[np.where(Y != 0)] = 1 26 | 27 | # Run Gradient Checking 28 | X = np.random.randn(*X_t.shape) 29 | Theta = np.random.randn(*Theta_t.shape) 30 | num_movies, num_users = Y.shape 31 | num_features = Theta_t.shape[1] 32 | 33 | # build params 34 | params = np.r_[X.flatten(), Theta.flatten()].reshape(-1, 1) 35 | 36 | costFunc = lambda t: cofiCostFunc(t, Y, R, num_users, num_movies, num_features, reg_lambda) 37 | 38 | numgrad = computeNumericalGradient(costFunc, params) 39 | 40 | cost, grad = costFunc(params) 41 | 42 | # make sure both grad have the same shape 43 | grad = grad.reshape(numgrad.shape) 44 | print(np.c_[numgrad.ravel(), grad.ravel()]) 45 | print('The above two columns you get should be very similar. ' 46 | '(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n') 47 | 48 | diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad) 49 | print('If your cost function implementation is correct, then \n the relative difference ' 50 | 'will be small (less than 1e-9). ' 51 | '\n \nRelative Difference: \n', diff) 52 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/cofiCostFunc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def cofiCostFunc(params, Y, R, num_users, num_movies, num_features, reg_lambda, returnCostOnly=False, 5 | returnGradOnly=False): 6 | """Collaborative filtering cost function 7 | [J, grad] = cofiCostFunc(params, Y, R, num_users, num_movies, ... 8 | num_features, lambda) returns the cost and gradient for the 9 | collaborative filtering problem. 10 | """ 11 | 12 | # Unfold the U and W matrices from params 13 | X = params[0:num_movies * num_features].reshape((num_movies, num_features)) 14 | Theta = params[num_movies * num_features:].reshape((num_users, num_features)) 15 | 16 | errors = (X.dot(Theta.T) - Y) * R 17 | J = 1 / 2 * np.sum(np.sum(errors ** 2)) 18 | 19 | penalty = (reg_lambda / 2) * (np.sum(np.sum(Theta ** 2)) + np.sum(np.sum(X ** 2))) 20 | J = J + penalty 21 | 22 | X_grad = errors.dot(Theta) + reg_lambda * X 23 | Theta_grad = errors.T.dot(X) + reg_lambda * Theta 24 | 25 | grad = np.r_[X_grad.flatten(), Theta_grad.flatten()] 26 | 27 | if returnGradOnly: 28 | return grad.flatten() 29 | if returnCostOnly: 30 | return J 31 | 32 | return J, grad 33 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/computeNumericalGradient.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def computeNumericalGradient(J, theta): 5 | """Computes the gradient using "finite differences" 6 | and gives us a numerical estimate of the gradient. 7 | numgrad = computeNumericalGradient(J, theta) computes the numerical 8 | gradient of the function J around theta. Calling y = J(theta) should 9 | return the function value at theta.""" 10 | 11 | numgrad = np.zeros(theta.shape) 12 | perturb = np.zeros(theta.shape) 13 | length = theta.shape[0] 14 | 15 | e = 1e-4 16 | for p in range(length): 17 | # Set perturbation vector 18 | perturb[p] = e 19 | loss1, tmp = J(theta - perturb) 20 | loss2, tmp = J(theta + perturb) 21 | # Compute Numerical Gradient 22 | numgrad[p] = (loss2 - loss1) / (2 * e) 23 | perturb[p] = 0 24 | 25 | return numgrad 26 | 27 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/estimateGaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def estimateGaussian(X): 5 | """This function estimates the parameters of a 6 | Gaussian distribution using the data in X 7 | [mu sigma2] = estimateGaussian(X), 8 | The input X is the dataset with each n-dimensional data point in one row 9 | The output is an n-dimensional vector mu, the mean of the data set 10 | and the variances sigma^2, an n x 1 vector 11 | """ 12 | 13 | # Useful variables 14 | m, n = X.shape 15 | 16 | mu = np.mean(X, 0).reshape(1, n) 17 | sigma2 = (1.0 / m) * sum((X - mu) ** 2).reshape(1, n) 18 | 19 | return mu, sigma2 20 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/ex8_movieParams.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_9/ex8/ex8_movieParams.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/ex8_movies.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_9/ex8/ex8_movies.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/ex8data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_9/ex8/ex8data1.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/ex8data2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_9/ex8/ex8data2.mat -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/loadMovieList.py: -------------------------------------------------------------------------------- 1 | def loadMovieList(filePath='movie_ids.txt'): 2 | """Reads the fixed movie list in movie.txt and returns a 3 | cell array of the words 4 | movieList = loadMovieList() reads the fixed movie list in movie.txt 5 | and returns a cell array of the words in movieList.""" 6 | 7 | movieList = {} 8 | 9 | # Read the fixed movieulary list 10 | f = open(filePath, 'r') 11 | 12 | try: 13 | for idx, line in enumerate(f): 14 | tokens = line.split(' ') 15 | movieName = ' '.join(tokens[1:]) 16 | movieList[idx] = movieName.strip() 17 | finally: 18 | f.close() 19 | 20 | return movieList 21 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/movie_ids.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/algorithms_in_python/week_9/ex8/movie_ids.txt -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/multivariateGaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def multivariateGaussian(X, mu, Sigma2): 5 | """Computes the probability density function of the 6 | multivariate gaussian distribution. 7 | p = multivariateGaussian(X, mu, Sigma2) Computes the probability 8 | density function of the examples X under the multivariate gaussian 9 | distribution with parameters mu and Sigma2. If Sigma2 is a matrix, it is 10 | treated as the covariance matrix. If Sigma2 is a vector, it is treated 11 | as the \sigma^2 values of the variances in each dimension (a diagonal 12 | covariance matrix) 13 | """ 14 | 15 | k = mu.shape[1] 16 | 17 | if len(Sigma2.shape) > 0 and ((Sigma2.shape[0] == 1) or (len(Sigma2.shape) > 1 and Sigma2.shape[1] == 1)): 18 | Sigma2 = np.diag(Sigma2.ravel()) 19 | 20 | X = X - mu 21 | 22 | p = (2 * np.pi) ** (-k / 2) 23 | 24 | p = p * np.linalg.det(Sigma2) ** -0.5 25 | 26 | p = p * np.exp(-0.5 * np.sum(X.dot(np.linalg.pinv(Sigma2)) * X, 1)) 27 | 28 | return p.reshape(-1, 1) 29 | 30 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/normalizeRatings.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def normalizeRatings(Y, R): 5 | """Preprocess data by subtracting mean rating for every 6 | movie (every row) 7 | Ynorm, Ymean = NORMALIZERATINGS(Y, R) normalized Y so that each movie 8 | has a rating of 0 on average, and returns the mean rating in Ymean. 9 | """ 10 | m, n = Y.shape 11 | Ymean = np.zeros((m, 1)) 12 | Ynorm = np.zeros((m, n)) 13 | for i in range(m): 14 | idx = np.where(R[i, :] == 1) 15 | Ymean[i] = np.mean(Y[i, idx]) 16 | Ynorm[i, idx] = Y[i, idx] - Ymean[i] 17 | 18 | return Ynorm, Ymean 19 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/originalGaussian.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import numpy as np 4 | 5 | 6 | def originalGaussian(X, mu, Sigma2): 7 | exp_data = ((X - mu) ** 2) / (2 * Sigma2) 8 | lef_data = (1.0 / (np.sqrt(2 * math.pi * Sigma2))) 9 | p = lef_data * np.exp(- exp_data) 10 | p = np.prod(p, 1) # products of each row 11 | return p 12 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/selectThreshold.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def selectThreshold(yval, pval): 5 | """Finds the best threshold (epsilon) to use for selecting 6 | outliers 7 | [bestEpsilon bestF1] = selectThreshold(yval, pval) finds the best 8 | threshold to use for selecting outliers based on the results from a 9 | validation set (pval) and the ground truth (yval). 10 | """ 11 | 12 | bestEpsilon = 0 13 | bestF1 = 0 14 | stepsize = (max(pval) - min(pval)) / 1000 15 | epsilons = np.arange(min(pval), max(pval), stepsize) 16 | 17 | for epsilon in epsilons: 18 | pred = (pval < epsilon) 19 | tp = np.sum(np.logical_and((pred == 1), (yval == 1)).astype(float)) 20 | fp = np.sum(np.logical_and((pred == 1), (yval == 0)).astype(float)) 21 | fn = np.sum(np.logical_and((pred == 0), (yval == 1)).astype(float)) 22 | prec = tp / (tp + fp) 23 | rec = tp / (tp + fn) 24 | F1 = 2 * (prec * rec) / (prec + rec) 25 | if F1 > bestF1: 26 | bestF1 = F1 27 | bestEpsilon = epsilon 28 | 29 | return bestEpsilon, bestF1 30 | -------------------------------------------------------------------------------- /algorithms_in_python/week_9/ex8/visualizeFit.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | from estimateGaussian import estimateGaussian 5 | from multivariateGaussian import multivariateGaussian 6 | 7 | 8 | def visualizeFit(X, mu, sigma2): 9 | """Visualizes the dataset and its estimated distribution. 10 | visualizeFit(X, p, mu, sigma2) This visualization shows you the 11 | probability density function of the Gaussian distribution. Each example 12 | has a location (x1, x2) that depends on its feature values. 13 | """ 14 | 15 | X1, X2 = np.meshgrid(np.arange(0, 35.5, 0.5), np.arange(0, 35.5, 0.5)) 16 | 17 | Z = multivariateGaussian(np.c_[X1.ravel(), X2.ravel()], mu, sigma2) 18 | Z = Z.reshape(X1.shape) 19 | 20 | plt.plot(X[:, 0], X[:, 1], 'bx') 21 | 22 | cont_levels = [10 ** exp for exp in range(-20, 0, 3)] 23 | 24 | plt.contour(X1, X2, Z, cmap=plt.cm.Paired, alpha=0.9, levels=cont_levels) 25 | -------------------------------------------------------------------------------- /figures/1_linear_regression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/1_linear_regression.png -------------------------------------------------------------------------------- /figures/1_linear_regression_3d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/1_linear_regression_3d.png -------------------------------------------------------------------------------- /figures/2_logistic_regression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/2_logistic_regression.png -------------------------------------------------------------------------------- /figures/3_one_vs_all_classification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/3_one_vs_all_classification.png -------------------------------------------------------------------------------- /figures/4_viz_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/4_viz_nn.png -------------------------------------------------------------------------------- /figures/5_learning_curves.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/5_learning_curves.png -------------------------------------------------------------------------------- /figures/6_spam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/6_spam.png -------------------------------------------------------------------------------- /figures/6_svms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/6_svms.png -------------------------------------------------------------------------------- /figures/7_keams_image_compression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/7_keams_image_compression.png -------------------------------------------------------------------------------- /figures/7_kmeans.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/7_kmeans.png -------------------------------------------------------------------------------- /figures/8_pca_datasets_before.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/8_pca_datasets_before.png -------------------------------------------------------------------------------- /figures/8_pca_faces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/8_pca_faces.png -------------------------------------------------------------------------------- /figures/9_anomaly_detection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/9_anomaly_detection.png -------------------------------------------------------------------------------- /figures/9_collaborative_filtering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/figures/9_collaborative_filtering.png -------------------------------------------------------------------------------- /lectures/w10_Learning_with_Large_Datasets/Lecture17.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w10_Learning_with_Large_Datasets/Lecture17.pdf -------------------------------------------------------------------------------- /lectures/w11_photo_ocr_and_ml_pipline_and_getting_more_data_and_ceiling_analysis/Lecture18.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w11_photo_ocr_and_ml_pipline_and_getting_more_data_and_ceiling_analysis/Lecture18.pdf -------------------------------------------------------------------------------- /lectures/w1_intro_linear_regression_with_one_variable/Lecture1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w1_intro_linear_regression_with_one_variable/Lecture1.pdf -------------------------------------------------------------------------------- /lectures/w1_intro_linear_regression_with_one_variable/Lecture2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w1_intro_linear_regression_with_one_variable/Lecture2.pdf -------------------------------------------------------------------------------- /lectures/w1_intro_linear_regression_with_one_variable/Lecture3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w1_intro_linear_regression_with_one_variable/Lecture3.pdf -------------------------------------------------------------------------------- /lectures/w2_linear_regression_with_multiple_variables/Lecture4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w2_linear_regression_with_multiple_variables/Lecture4.pdf -------------------------------------------------------------------------------- /lectures/w2_linear_regression_with_multiple_variables/Lecture5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w2_linear_regression_with_multiple_variables/Lecture5.pdf -------------------------------------------------------------------------------- /lectures/w3_logitic_regression_and_regularization/Lecture6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w3_logitic_regression_and_regularization/Lecture6.pdf -------------------------------------------------------------------------------- /lectures/w3_logitic_regression_and_regularization/Lecture7.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w3_logitic_regression_and_regularization/Lecture7.pdf -------------------------------------------------------------------------------- /lectures/w4_multi_class_classification_and_neural_networks_representation/Lecture8.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w4_multi_class_classification_and_neural_networks_representation/Lecture8.pdf -------------------------------------------------------------------------------- /lectures/w5_neural_networks_learning/Lecture9.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w5_neural_networks_learning/Lecture9.pdf -------------------------------------------------------------------------------- /lectures/w6_evaluating_learning_algortihm_variance_bias_precision_recall/Lecture10.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w6_evaluating_learning_algortihm_variance_bias_precision_recall/Lecture10.pdf -------------------------------------------------------------------------------- /lectures/w6_evaluating_learning_algortihm_variance_bias_precision_recall/Lecture11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w6_evaluating_learning_algortihm_variance_bias_precision_recall/Lecture11.pdf -------------------------------------------------------------------------------- /lectures/w7_support_vector_machines/Lecture12.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w7_support_vector_machines/Lecture12.pdf -------------------------------------------------------------------------------- /lectures/w7_support_vector_machines/svm-notes-long-08.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w7_support_vector_machines/svm-notes-long-08.pdf -------------------------------------------------------------------------------- /lectures/w8_unsupervised_learning_clustering_kmeans_pca/Lecture13.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w8_unsupervised_learning_clustering_kmeans_pca/Lecture13.pdf -------------------------------------------------------------------------------- /lectures/w8_unsupervised_learning_clustering_kmeans_pca/Lecture14.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w8_unsupervised_learning_clustering_kmeans_pca/Lecture14.pdf -------------------------------------------------------------------------------- /lectures/w9_anomaly_detection_and_recommender_systems/Lecture15.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w9_anomaly_detection_and_recommender_systems/Lecture15.pdf -------------------------------------------------------------------------------- /lectures/w9_anomaly_detection_and_recommender_systems/Lecture16.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hzitoun/machine_learning_from_scratch_matlab_python/362409272b83ff63bb946fb7b59acc7220f02f48/lectures/w9_anomaly_detection_and_recommender_systems/Lecture16.pdf --------------------------------------------------------------------------------