Welcome to our Machine Learning repository! This repository contains a collection of machine learning algorithms, techniques, and resources for anyone interested in learning or working with machine learning.
6 |We welcome contributions to this repository. Whether you are a beginner or an expert in the field, you can contribute in various ways, such as:
8 |All contributors are expected to abide by our code of conduct. Please review our code of conduct before contributing.
35 |If you have any questions or need help with your contributions, please open an issue or reach out to one of the repository maintainers.
37 |Thank you for your contributions! We appreciate your help in making this repository a valuable resource for the Machine Learning community.
38 | 39 | 40 | -------------------------------------------------------------------------------- /lib/data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from random import choice, uniform 3 | import math 4 | import csv 5 | 6 | ################ Data Generation ################ 7 | def generateData(data_size, circles, dim = 2, filename="data.csv"): 8 | """ 9 | data_size: number of vectors to generate 10 | circles: a list of dictionaries, each containing the center coordinates, class label, and radius for a circle 11 | filename: name of the file to save the data to 12 | """ 13 | S = [] 14 | for i in range(data_size): 15 | circle = choice(circles) 16 | x = circle['center'][0] + uniform(-circle['radius']*10, circle['radius']*10)/10 17 | y = circle['center'][1] + uniform(-circle['radius']*10, circle['radius']*10)/10 18 | if dim == 3: 19 | z = circle['center'][2] + uniform(-circle['radius']*10, circle['radius']*10)/10 20 | if math.sqrt((x - circle['center'][0])**2 + (y - circle['center'][1])**2 + (z - circle['center'][2])**2) > circle['radius']: 21 | # If the point is outside the circle, try again 22 | continue 23 | S.append(np.array([1, x, y, z, circle['class']])) 24 | elif dim == 2: 25 | if math.sqrt((x - circle['center'][0])**2 + (y - circle['center'][1])**2) > circle['radius']: 26 | # If the point is outside the circle, try again 27 | continue 28 | S.append(np.array([1, x, y, circle['class']])) 29 | else: 30 | print("Error: dim must be 2 or 3") 31 | return None 32 | np.savetxt(filename, S, delimiter=",",fmt="%f") 33 | return np.asarray(S) 34 | 35 | def makeRegressionData(data_size, filename="data.csv", degree = 1): 36 | """ 37 | description: generates data for a regression problem with a polynomial function 38 | args: 39 | data_size: number of vectors to generate 40 | filename: name of the file to save the data to 41 | degree: degree of the polynomial function 42 | return: a numpy array containing the data 43 | """ 44 | S = [] 45 | for i in range(data_size): 46 | x = uniform(-1, 1) 47 | y = 2*(x**degree) + uniform(-0.3, 0.3) 48 | S.append(np.array([1, x, y])) 49 | np.savetxt(filename, S, delimiter=",",fmt="%f") 50 | return np.asarray(S) 51 | 52 | ######################### Data loading ######################### 53 | def loadData(filename, features): 54 | """ 55 | args: 56 | filename: name of the file to load the data from 57 | features: list of features to load 58 | return: a numpy array containing the data 59 | """ 60 | data = [] 61 | 62 | # open and format data 63 | with open(filename, mode='r') as csv_file: 64 | csv_reader = csv.DictReader(csv_file) 65 | for row in csv_reader: 66 | data.append([float(row[features[i]]) for i in range(len(features))]) 67 | 68 | return np.asarray(data) -------------------------------------------------------------------------------- /lib/plots.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | from matplotlib.colors import ListedColormap 3 | from matplotlib import cm 4 | import numpy as np 5 | 6 | from .transformation import psy 7 | from .hypothesis import h 8 | from .activation import sigmoid 9 | 10 | # hyperplan equation in 2D 11 | def line(wop,x): 12 | return -(wop[1]*x + wop[0])/wop[2] 13 | 14 | # hyperplan equation in 3D 15 | def surface(wop,x,y): 16 | return -(wop[1] * x + wop[2] * y + wop[0])/wop[3] 17 | 18 | # regression curve in 2D 19 | def curve(wop,x): 20 | return sum([wop[i] * x for i in range(len(wop))]) 21 | 22 | 23 | def plotLinearDecisionBoundary2D(X, Y, w0): 24 | """ 25 | description: plot linear decision boundary in 2 dimensions with matplotlib 26 | args: 27 | X: a list of vectors x_i 28 | Y: the list of labels y_i associated 29 | w0: weight vector 30 | return: None 31 | """ 32 | for i in range(len(X)): 33 | if Y[i] == 1 : 34 | plt.plot(X[i][1],X[i][2], "or") 35 | else : 36 | plt.plot(X[i][1],X[i][2], "og") 37 | 38 | Z = [i for i in range(round(min(X[:,1])), round(max(X[:,1])) +1)] 39 | 40 | hyper = [line(w0, e) for e in Z] 41 | plt.xlabel("x1") 42 | plt.ylabel("x2") 43 | plt.title("Plot of linear decision boundary") 44 | plt.plot(Z,hyper,"-b") 45 | plt.legend() 46 | plt.show() 47 | 48 | def plotLinearDecisionBoundary3D(X, Y, w0): 49 | # Create the figure 50 | fig = plt.figure() 51 | # Add an axes 52 | ax = fig.add_subplot(111,projection='3d') 53 | for e in range(len(X)): 54 | if Y[e] == 1 :ax.plot(X[e][1], X[e][2], 1, "o", color = 'red') 55 | else : ax.plot(X[e][1], X[e][2], 0, "o", color = 'green') 56 | 57 | x, y = np.arange(-4,2,0.1),np.arange(-4,2,0.1) 58 | x, y = np.meshgrid(x,y) 59 | z = surface(w0, x, y) 60 | surf = ax.plot_surface(x, y, z,cmap=cm.coolwarm, 61 | linewidth=0, antialiased=False, alpha= 0.6) 62 | fig.colorbar(surf, shrink=0.5, aspect=5) 63 | ax.set_xlabel("x1") 64 | ax.set_ylabel("x2") 65 | ax.set_zlabel("x3") 66 | ax.set_title("Plot of Linear Decision Boundary") 67 | 68 | plt.show() 69 | 70 | 71 | # plot non linear decision boundary in 2 dimensions with matplotlib 72 | def plotNonLinearDecisionBoundary2D(X, Y, w, q): 73 | """ 74 | description: plot non linear decision boundary in 2 dimensions with matplotlib 75 | args: 76 | X: a list of vectors x_i 77 | Y: the list of labels y_i associated 78 | w: weight vector 79 | q: polynom's degree 80 | return: None 81 | """ 82 | 83 | # create meshgrid 84 | print(X.shape) 85 | x_min, x_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5 86 | y_min, y_max = X[:, 2].min() - 0.5, X[:, 2].max() + 0.5 87 | z = (x_max - x_min)/100 88 | xx, yy = np.meshgrid(np.arange(x_min, x_max, z), np.arange(y_min, y_max, z)) 89 | 90 | # create color map 91 | cmap_bold = ListedColormap(['#FF0000', '#0000FF']) 92 | 93 | # create figure 94 | fig = plt.figure(figsize=(10, 10)) 95 | ax = fig.add_subplot(111) 96 | 97 | # plot decision surface 98 | Z = np.array([sigmoid(h(psy(np.array([x,y]),q), w)) for x, y in zip(np.ravel(xx), np.ravel(yy))]) 99 | Z = Z.reshape(xx.shape) 100 | ax.contourf(xx, yy, Z, cmap=cm.viridis, alpha=0.8) 101 | ax.contour(xx, yy, Z, [0.5], linewidths=2, colors='k') 102 | 103 | # plot training points 104 | ax.scatter(X[:, 1], X[:, 2], c=Y, cmap=cmap_bold, edgecolor='k', s=20) 105 | 106 | # set labels 107 | ax.set_xlabel('x1') 108 | ax.set_ylabel('x2') 109 | ax.set_title("Non linear decision boundary\ndegree = " + str(q)) 110 | # add colorbar 111 | fig.colorbar(ax.contourf(xx, yy, Z, cmap=cm.viridis, alpha=0.5), ax=ax, label="Probability") 112 | 113 | plt.show() -------------------------------------------------------------------------------- /lib/bounds.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import itertools 3 | 4 | def create_combi(n, arr, i, combi): 5 | """ 6 | Create all possible combinations of 0 and 1 for the last column of the array 7 | args: 8 | n: the number of rows of the array 9 | arr: the array to create combinations 10 | i: the current row to create combinations 11 | combi: the list to store all combinations 12 | """ 13 | if i == n: 14 | combi.append(np.copy(arr)) 15 | return 16 | arr[i][-1] = 0 17 | create_combi(n, arr, i + 1, combi) 18 | arr[i][-1] = 1 19 | create_combi(n, arr, i + 1, combi) 20 | 21 | 22 | def shatter(classifier, data): 23 | """ 24 | Check if the classifier can shatter the data 25 | args: 26 | classifier: the classifier to check 27 | data: the data to check 28 | return: 29 | True if the classifier can shatter the data 30 | """ 31 | all_combi = [] 32 | X = np.copy(data) 33 | create_combi(len(X), X, 0, all_combi) 34 | all_combi = np.asarray(all_combi) 35 | MAX = all_combi.shape[0] 36 | for i in range(MAX): 37 | if all(all_combi[i][:, -1] == 0) or all(all_combi[i][:, -1] == 1) : 38 | continue 39 | classifier.fit(all_combi[i][:, :-1], all_combi[i][:, -1]) 40 | y_ = classifier.predict(all_combi[i][:, :-1]) 41 | if not all(y_ == all_combi[i][:, -1]): 42 | return False 43 | return True 44 | 45 | def VC_dimension(classifier, data): 46 | """ 47 | Calculate the VC dimension of the classifier 48 | args: 49 | classifier: the classifier to calculate 50 | data: the data to calculate 51 | return: 52 | the VC dimension of the classifier 53 | """ 54 | vc = 1 55 | for k in range(2, data.shape[0] + 1): 56 | A = list(itertools.combinations(data, k)) 57 | i = 1 58 | for subset in A: 59 | if shatter(classifier, np.asarray(subset)): 60 | vc += 1 61 | break 62 | if i == len(A): 63 | return vc 64 | i += 1 65 | return vc 66 | 67 | 68 | def Ne(data, epsilon, d = 2): 69 | """ 70 | function that computes the number of balls of radius epsilon that cover the data 71 | args: 72 | data: a list of points 73 | epsilon: the radius of the balls 74 | d: the norm used to compute the distance between points 75 | returns: 76 | ne: the number of balls of radius epsilon that cover the data 77 | """ 78 | points = list(data) 79 | for i in range(len(points)): 80 | points[i] = list(points[i]) 81 | points[i].append(0) 82 | ne = 0 83 | n_cov = 0 84 | while n_cov < len(points): 85 | current_cov = list() 86 | for p in points: 87 | voisinage_p = [other for other in points if (np.linalg.norm(np.asarray(other[:-1]) - np.asarray(p[:-1]), ord = d) <= epsilon) and (other[-1] != 1)] 88 | current_cov.append([p, len(voisinage_p), voisinage_p]) 89 | current_cov.sort(key=lambda x: x[1], reverse=True) 90 | for e in points: 91 | if e in current_cov[0][2]: 92 | e[-1] = 1 93 | n_cov += current_cov[0][1] 94 | ne += 1 95 | return ne 96 | 97 | def uniform_covering_numbers(classifier, radius, m, data, d = 2): 98 | """ 99 | function that computes the uniform covering number of the data 100 | args: 101 | classifier: the classifier used to compute the covering number 102 | radius: the radius of the balls 103 | m: the number of points in the subset 104 | data: the data 105 | d: the norm used to compute the distance between points 106 | returns: 107 | max_cov_nbr: the uniform covering number of the data 108 | """ 109 | A = np.asarray(list(itertools.combinations(data, m))) 110 | max_cov_nbr = 0 111 | for subset in A: 112 | classifier.fit(subset[:, :-1], subset[:, -1]) 113 | H_subset = classifier.predict(subset[:, :-1]) 114 | for i in range(subset.shape[0]): 115 | subset[i][-1] = H_subset[i] 116 | cov_nbr = Ne(subset, radius, d) 117 | print("cov_nbr = ", cov_nbr) 118 | if cov_nbr > max_cov_nbr: 119 | max_cov_nbr = cov_nbr 120 | 121 | return max_cov_nbr -------------------------------------------------------------------------------- /lib/loss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .hypothesis import h 3 | from .activation import sigmoid, sign 4 | 5 | ################ empirical loss functions ################ 6 | # 0-1 loss 7 | def loss01(X,Y,w): 8 | ''' 9 | args: 10 | X: list of vectors x_i 11 | Y: list of labels (scalars, 1 or 0) y_i 12 | w: vector of weights 13 | returns: average empirical loss 14 | ''' 15 | n = len(X) # size of data sample 16 | misclassified = [1 if sign(X[i], Y[i], w) != 1 else 0 for i in range(len(X))] 17 | return sum(misclassified)/n 18 | 19 | # Arithmetical Mean (AM) 20 | def lossAM(X,Y,w, reg=None, lamda=0.5, alpha = 0.5): 21 | """ 22 | description: AM loss function with regularization variants 23 | args: 24 | X: a list of vectors x_i 25 | Y: the list of labels y_i associated 26 | w: weight vector 27 | reg: regularization variant (Ridge, Lasso, Elastic, None) 28 | lamda: regularization parameter 29 | alpha: ElasticNet mixing parameter 30 | return: loss function with regularization variants 31 | """ 32 | n = len(X) # size of data sample 33 | error = [(Y[i] - w.T @ X[i])**2 for i in range(len(X))] # squared error vector 34 | if reg == "Ridge" : 35 | return np.sum(error)/n + lamda * np.sum(np.power(w,2)) 36 | elif reg == "Lasso" : 37 | return np.sum(error)/n + lamda * np.sum(np.array([abs(e) for e in w])) 38 | elif reg == "Elastic" : 39 | return np.sum(error)/(2*n) + lamda * (1 - alpha)/2 * np.sum(np.power(w,2)) + lamda * alpha * np.sum(np.array([abs(e) for e in w])) 40 | 41 | return np.sum(error)/n 42 | 43 | 44 | # cross-entropy 45 | def cross_entropy(X,Y,w, reg=None, lamda=0.5, alpha = 0.5): 46 | """ 47 | description: loss function with regularization variants (Ridge, Lasso, Elastic, None) 48 | args: 49 | X: a list of vectors x_i 50 | Y: the list of labels y_i associated 51 | w: weight vector 52 | reg: regularization variant (Ridge, Lasso, Elastic, None) 53 | lamda: regularization parameter 54 | alpha: ElasticNet mixing parameter 55 | return: loss function with regularization variants 56 | """ 57 | n = len(X) # size of data sample 58 | error = [- Y[i] * np.log(sigmoid(h(X[i],w))) for i in range(len(X))] 59 | if reg == "Elastic": 60 | return np.sum(error)/n + (lamda * alpha) * np.sum(np.array([abs(e) for e in w]))+ (lamda * (1 - alpha) /(2 *n)) * np.sum(np.power(w,2)) 61 | elif reg == "Ridge" : 62 | return np.sum(error)/n + (lamda/(2 *n)) * np.sum(np.array([abs(e) for e in w])) 63 | elif reg == "Sparse" : 64 | return np.sum(error)/n + (lamda/(2 *n)) * np.sum(np.power(w,2)) 65 | 66 | return np.sum(error)/n 67 | 68 | ################ gradients ################ 69 | # AM loss function 70 | def DlossAM(X,Y,w, reg=None, lamda = 0.5, alpha = 0.5): 71 | """ 72 | description: gradient of loss function 73 | args: 74 | X: a list of vectors x_i 75 | Y: the list of labels y_i associated 76 | w: weight vector 77 | reg: regularization variant (Ridge, Lasso, Elastic, None) 78 | lamda: regularization parameter 79 | alpha: ElasticNet mixing parameter 80 | return: loss function with regularization variants 81 | """ 82 | n = len(X) # size of data sample 83 | error = [(Y[i] - w.T @ X[i]) for i in range(len(X))] # error vector 84 | if reg == "Ridge" : 85 | return (2/n) * np.dot(X.T,error) + lamda * 2 * w # using gradient 86 | elif reg == "Lasso" : 87 | return (2/n) * np.dot(X.T,error) + lamda * np.sign(w) # using sub-gradient 88 | elif reg == "Elastic" : 89 | return (1/n) * np.dot(X.T,error) + lamda * (1 - alpha) * w + lamda * alpha * np.sign(w) 90 | 91 | return (2/n) * np.dot(X.T,error) 92 | 93 | # cross-entropy loss function 94 | def Dcross_entropy(X,Y,w, reg=None, lamda = 0.5, alpha = 0.5): 95 | """ 96 | description: gradient of loss function with regularization variants (Ridge, Sparse, Elastic, None) 97 | args: 98 | X: a list of vectors x_i 99 | Y: the list of labels y_i associated 100 | w: weight vector 101 | reg: regularization variant (Ridge, Sparse, Elastic, None) 102 | lamda: regularization parameter 103 | alpha: ElasticNet mixing parameter 104 | return: gradient of loss function with regularization variants 105 | """ 106 | n = len(X) # size of data sample 107 | error = np.array([sigmoid(h(X[i],w)) - Y[i] for i in range(len(X))]) 108 | 109 | if reg == "Ridge" : 110 | return (2/n) * np.dot(X.T,error) + lamda * 2 * w 111 | elif reg == "Sparse" : 112 | return (2/n) * np.dot(X.T,error) + lamda * np.sign(w) # using sub-gradient 113 | elif reg == "Elastic" : 114 | return (1/n) * np.dot(X.T,error) + lamda * (1 - alpha) * w + lamda * alpha * np.sign(w) 115 | 116 | return (1/n) * np.dot(X.T,error) -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributor Covenant Code of Conduct 3 | 4 | ## Our Pledge 5 | 6 | We as members, contributors, and leaders pledge to make participation in our 7 | community a harassment-free experience for everyone, regardless of age, body 8 | size, visible or invisible disability, ethnicity, sex characteristics, gender 9 | identity and expression, level of experience, education, socio-economic status, 10 | nationality, personal appearance, race, caste, color, religion, or sexual 11 | identity and orientation. 12 | 13 | We pledge to act and interact in ways that contribute to an open, welcoming, 14 | diverse, inclusive, and healthy community. 15 | 16 | ## Our Standards 17 | 18 | Examples of behavior that contributes to a positive environment for our 19 | community include: 20 | 21 | * Demonstrating empathy and kindness toward other people 22 | * Being respectful of differing opinions, viewpoints, and experiences 23 | * Giving and gracefully accepting constructive feedback 24 | * Accepting responsibility and apologizing to those affected by our mistakes, 25 | and learning from the experience 26 | * Focusing on what is best not just for us as individuals, but for the overall 27 | community 28 | 29 | Examples of unacceptable behavior include: 30 | 31 | * The use of sexualized language or imagery, and sexual attention or advances of 32 | any kind 33 | * Trolling, insulting or derogatory comments, and personal or political attacks 34 | * Public or private harassment 35 | * Publishing others' private information, such as a physical or email address, 36 | without their explicit permission 37 | * Other conduct which could reasonably be considered inappropriate in a 38 | professional setting 39 | 40 | ## Enforcement Responsibilities 41 | 42 | Community leaders are responsible for clarifying and enforcing our standards of 43 | acceptable behavior and will take appropriate and fair corrective action in 44 | response to any behavior that they deem inappropriate, threatening, offensive, 45 | or harmful. 46 | 47 | Community leaders have the right and responsibility to remove, edit, or reject 48 | comments, commits, code, wiki edits, issues, and other contributions that are 49 | not aligned to this Code of Conduct, and will communicate reasons for moderation 50 | decisions when appropriate. 51 | 52 | ## Scope 53 | 54 | This Code of Conduct applies within all community spaces, and also applies when 55 | an individual is officially representing the community in public spaces. 56 | Examples of representing our community include using an official e-mail address, 57 | posting via an official social media account, or acting as an appointed 58 | representative at an online or offline event. 59 | 60 | ## Enforcement 61 | 62 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 63 | reported to the community leaders responsible for enforcement (by e-mails or Linkein). 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series of 86 | actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or permanent 93 | ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within the 113 | community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at 126 | [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq 132 | [translations]: https://www.contributor-covenant.org/translations 133 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [](https://github.com/HamzaBamohammed/ML-from-scratch-with-Python/blob/main/LICENSE) 2 |  3 | [](https://www.python.org/downloads/release/python-380/) 4 | 5 | 6 |  7 | ## Overview 8 | In this package, we've implemented a few basic machine learning algorithms from scratch. The algorithms are implemented in Python 3.8. The algorithms are as follows: 9 | 10 | - Single Layer Perceptron Algorithm 11 | - Pocket Perceptron 12 | - Adaline Algorithm with delta rule 13 | - Linear Regression 14 | - Logistic Regression 15 | - Polynomial Regression 16 | - One-vs-All and One-vs-One Classifiers 17 | 18 | Furthermore, we've implemented a few basic algorithms useful in the cristalization of learning theory : 19 | 20 | - Non-Linear Transformation 21 | - Cross Validation with K-Fold 22 | - Gradient Descent for Linear and Logistic Regression 23 | - Regularization for Linear and Logistic Regression 24 | - Bias and Variance (not implemented yet) 25 | - Vapnik Chervonenkis Dimension (VC-Dimension) 26 | - Covering and Uniform Covering Number 27 | 28 | Finally, we also added a few basic functions to help you create dummy data and plot results of the algorithms : 29 | 30 | - Generate Dummy Data (in 2D and 3D) 31 | - Plot Linear Decision Boundary (in 2D and 2D) 32 | - Plot Non-Linear Decision Boundary (in 2D) 33 | - Plot Linear Regression (in 2D) 34 | 35 | All the algorithms and tools are implemented in the `lib` folder. 36 | 37 | These codes were made in the context of the Learning Theory course Fall 2023 at ENSIAS - University Mohammed V - Rabat, Morocco. You can find the LABS and the corresponding PDFs in the `ensias_labs` folder. Do not use them since they are a beta version with a lot of bugs and errors. 38 | 39 | 40 | ## Authors and Contributors 41 | 42 | - [Hamza Bamohammed](https://www.github.com/HamzaBamohammed), Applied Mathematics & AI engineering student at ENSIAS 43 | - [Hicham Filali](https://www.github.com/FILALIHicham), Applied Mathematics & AI engineering student at ENSIAS 44 | - [Bouchra Sahri](https://www.github.com/bouchrasa), Applied Mathematics & AI engineering student at ENSIAS 45 | - [Mohammed Nechba](https://www.github.com/NechbaMohammed), Applied Mathematics & AI engineering student at ENSIAS 46 | - [Hanaa El Afia](https://github.com/hanaa-elafia), Applied Mathematics & AI engineering student at ENSIAS 47 | - [Mohamed Mouhajir](https://github.com/mohamedmohamed2021), Applied Mathematics & AI engineering student at ENSIAS 48 | 49 | ## Installation 50 | 51 | To install the package, you can use the following command : 52 | 53 | ```bash 54 | git clone https://github.com/HamBa-m/scinis-learn.git 55 | ``` 56 | 57 | # Usage/Examples 58 | 59 | ## Data Generation (in 2D) 60 | 61 | ```python 62 | from lib.data import generateData 63 | 64 | # make the list of circles as dictionnaries with keys 'center' and 'radius' and "class" 65 | circles = [ 66 | {'center': [0, 0], 'radius': 1, 'class': 1}, 67 | {'center': [2, 2], 'radius': 1, 'class': -1}, 68 | ] 69 | 70 | # Generate 500 points in 2D 71 | data = generateData(500, circles, filename='data2D.csv') 72 | ``` 73 | PS: Don't forget to manually add the features in the first line of the resulting CSV file before proceeding in this tutorial! 74 | 75 | 76 | ## Perceptron Algorithm 77 | ```python 78 | import numpy as np 79 | from lib.classification import PLA 80 | from lib.data import loadData 81 | 82 | # Load data 83 | data = loadData('data2D.csv', ['x0','x1','x2','y']) 84 | X = data[:, :-1] 85 | y = data[:, -1] 86 | 87 | # Initialize the weights vector 88 | w = np.zeros(X.shape[1]) 89 | 90 | # Initialize the Perceptron Algorithm 91 | w0, t = PLA(X, y, w) 92 | 93 | # Print the weights vector and the number of iterations 94 | print(w0, t) 95 | ``` 96 | 97 | ### Plotting Decision Boundary (in 2D) 98 | ```python 99 | from lib.plots import plotLinearDecisionBoundary2D 100 | 101 | # Plot the decision boundary 102 | plotLinearDecisionBoundary2D(X, y, w0) 103 | ``` 104 | 105 | ### Results 106 | ```Bash 107 | iter= 1 | loss= 0.0 108 | [ 3. -1.513614 -2.503794] 1 109 | ``` 110 |  111 | 112 | ## Logistic Regression 113 | ```python 114 | from lib.classification import LogisticRegression 115 | from lib.data import loadData 116 | import numpy as np 117 | 118 | # Load data 119 | data = loadData('data2Dlogistic.csv', ['x0','x1', 'x2', 'y']) 120 | X = data[:, :-1] 121 | y = data[:, -1] 122 | 123 | # Initialize the logistic regression algorithm 124 | w0, loss = LogisticRegression(X, y) 125 | 126 | # Print the weights vector and the number of iterations 127 | print(w0, loss) 128 | 129 | # Plot the decision boundary 130 | from lib.plots import plotNonLinearDecisionBoundary2D 131 | plotNonLinearDecisionBoundary2D(X, y, w0, 1) 132 | ``` 133 | ### Results 134 | ```Bash 135 | 100% ==================================================>[ 4.94597336 -2.8523291 -2.58496018] 0.013994617998407104 136 | 137 | ``` 138 |  139 | 140 | ## Linear Regression 141 | ```python 142 | # make linear regression data 143 | from lib.data import makeRegressionData 144 | data = makeRegressionData(100, degree=1) 145 | 146 | X = data[:, :-1] 147 | y = data[:, -1] 148 | 149 | # initialize linear regression 150 | from lib.regression import LinearRegression 151 | w0, loss = LinearRegression(X, y) 152 | 153 | # print the weights vector and the loss 154 | print(w0, loss) 155 | 156 | # plot the decision boundary 157 | from lib.plots import plotRegressionLine2D 158 | plotRegressionLine2D(X, y, w0) 159 | ``` 160 | ### Results 161 | ```Bash 162 | [-0.0305375 2.02728691] 0.030354517088163824 163 | ``` 164 |  165 | 166 | ## Polynomial Regression 167 | ```python 168 | # make polynomial regression data 169 | from lib.data import makeRegressionData 170 | data = makeRegressionData(100, degree=3) 171 | 172 | X = data[:, :-1] 173 | y = data[:, -1] 174 | 175 | # transform the data to a higher dimension 176 | from lib.transformation import polyMap 177 | X3 = polyMap(X, 3) 178 | 179 | # initialize polynomial regression 180 | from lib.regression import LinearRegression 181 | w0, loss = LinearRegression(X3, y) 182 | 183 | # print the weights vector and the loss 184 | print(w0, loss) 185 | ``` 186 | 187 | ### Results 188 | ```Bash 189 | [-0.00370442 -0.02731372 -0.00370442 -0.02731372 -0.02731372 -0.01275648 190 | -0.00370442 -0.02731372 -0.01275648 2.12742621] 0.029404600523786663 191 | ``` 192 | -------------------------------------------------------------------------------- /lib/classification.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | import colorama # for colored output in terminal 4 | from .activation import sign 5 | from .loss import loss01, lossAM, cross_entropy, DlossAM, Dcross_entropy 6 | from .transformation import polyMap 7 | 8 | ################ binary classification ERM algorithms ################ 9 | # Single Layer Perceptron 10 | def PLA(X,Y,w): 11 | ''' 12 | args: 13 | X: list of vectors x_i 14 | Y: list of labels (scalars, 1 or 0) y_i 15 | w: vector of weights 16 | returns: 17 | w: vector of weights after training 18 | t: number of iterations 19 | ''' 20 | n, t = len(X), 0 21 | while loss01(X,Y,w) != 0: 22 | for i in range(n): 23 | if sign(X[i], Y[i], w) < 0 : w += X[i]*Y[i] 24 | t += 1 25 | print("iter=",t," | loss=",loss01(X,Y,w)) 26 | return w, t 27 | 28 | # Single Layer Perceptron with Pocket 29 | def Pocket(X,Y,w): 30 | ''' 31 | args: 32 | X: list of vectors x_i 33 | Y: list of labels (scalars, 1 or 0) y_i 34 | w: vector of weights 35 | returns: 36 | w: vector of weights after training 37 | t: number of iterations 38 | loss: average empirical loss 39 | ''' 40 | n, t = len(X), 0 41 | Tmax = 500 42 | w0 = np.array(w) 43 | while t < Tmax: 44 | for i in range(n): 45 | if sign(X[i],Y[i], w0) < 0 : w0 += X[i]*Y[i] 46 | t += 1 47 | print("iter=",t," | loss= ",loss01(X,Y,w)) 48 | if loss01(X,Y,w0) < loss01(X,Y,w) : w = w0 49 | return w, t, loss01(X,Y,w) 50 | 51 | # Single Layer Perceptron with Adaline and delta rule 52 | def Adaline(X,Y,w, delta = 0.2): 53 | ''' 54 | description: Adaline is a Perceptron with a linear activation function 55 | args: 56 | X: list of vectors x_i 57 | Y: list of labels (scalars, 1 or -1) y_i 58 | w: weights vector 59 | eps: precision factor 60 | returns: 61 | w: vector of weights after training 62 | t: number of iterations 63 | loss: loss function value 64 | ''' 65 | n, t, Tmax = len(X), 0, 100 66 | lr = 0.0001 # learning rate for the gradient descent 67 | while abs(DlossAM(X,Y,w)) > delta and t < Tmax : 68 | print(DlossAM(X,Y,w)) 69 | for i in range(n): 70 | if (Y[i] - w.T @ X[i]) != 0 : w += 2 * X[i] * (Y[i] - w.T @ X[i]) * lr 71 | print("iter=",t," | loss=",lossAM(X,Y,w)) 72 | t += 1 73 | return w, t, lossAM(X,Y,w) 74 | 75 | # logistic regression 76 | def LogisticRegression(X, Y, reg = None, lamda = 0.5, alpha = 0.5, lr = 0.1): 77 | ''' 78 | description: logistic regression using gradient descent and regularization 79 | args: 80 | X: matrix of vectors x_i 81 | Y: vector of labels (scalars, 1 or 0) y_i 82 | reg: type of regularization, either Ridge, Sparse, Elastic Net, or None 83 | lamda: parameter of regularization 84 | alpha: parameter of Elastic Net regularization 85 | lr: learning rate of the gradient descent 86 | return: weight vector w and loss function 87 | ''' 88 | w = np.zeros(X.shape[1]) # initialize weights vector 89 | 90 | # Lasso and Elastic Net cases needs an iterative search using gradient descent 91 | Tmax = 2000 # upper bound on number of iterations 92 | for i in range(Tmax): 93 | dw = Dcross_entropy(X, Y, w, reg=reg, lamda=lamda, alpha=alpha) # compute gradient 94 | w -= lr * dw # update weights 95 | sys.stdout.write(colorama.Fore.GREEN + "\r{1}%\t{0}>".format("="*int(50 * ((i+1)/Tmax))+"-" *int(50 * (1 - (i+1)/Tmax)),int(100*(i+1)/Tmax))) 96 | sys.stdout.flush() # progress bar 97 | sys.stdout.write(colorama.Fore.WHITE) # reset color of terminal 98 | 99 | return w, cross_entropy(X,Y,w, reg=reg, lamda=lamda,alpha=alpha) 100 | 101 | ################ multiclass classification ERM algorithms ################ 102 | # One-vs-All 103 | def OVA(data, classes, algo, hyperpara=[]): 104 | w_list = list() 105 | for e in classes: 106 | data_ = np.copy(data) 107 | row = data_.shape[0] 108 | for j in range(row): 109 | if int(data_[j][-1]) != e : data_[j][-1] = -1 110 | else : data_[j][-1] = 1 111 | if algo =="SLP": 112 | # hyperpara = [] 113 | w = np.zeros(data.shape[1] - 1) 114 | col = data_.shape[1] 115 | w, t = PLA(data_[:,:col-1],data_[:,-1],w) 116 | w_list.append(w) 117 | elif algo == "Pocket": 118 | # hyperpara = [] 119 | w = np.zeros(data.shape[1] - 1) 120 | col = data_.shape[1] 121 | w, t, ls = Pocket(data_[:,:col-1],data_[:,-1],w) 122 | w_list.append(w) 123 | elif algo == "Adaline": 124 | # hyperpara = [eps] 125 | w = np.zeros(data.shape[1] - 1) 126 | col = data_.shape[1] 127 | eps = np.copy(hyperpara[0]) 128 | w, t, ls = Adaline(data_[:,:col-1],data_[:,-1],w,eps) 129 | w_list.append(w) 130 | elif algo == "Logistic": 131 | # hyperpara = [lr, Tmax, epsilon] 132 | col = data_.shape[1] 133 | lr = np.copy(hyperpara[0]) 134 | Tmax = np.copy(hyperpara[1]) 135 | eps = np.copy(hyperpara[2]) 136 | w, t, ls = LogisticRegression(data_[:,:col-1],data_[:,-1],lr,Tmax,eps) 137 | w_list.append(w) 138 | elif algo == "Pocket Trans": 139 | # hyperpara = [q] 140 | q = np.copy(hyperpara[0]) 141 | data_ = np.asarray([polyMap(x,q) for x in data_]) 142 | w = np.zeros(data_.shape[1] - 1) 143 | col = data_.shape[1] 144 | w, t, ls = Pocket(data_[:,:col-1],data_[:,-1],w) 145 | w_list.append(w) 146 | elif algo == "Adaline Trans": 147 | # hyperpara = [q] 148 | q = np.copy(hyperpara[0]) 149 | data_ = np.asarray([polyMap(x,q) for x in data_]) 150 | w = np.zeros(data_.shape[1] - 1) 151 | col = data_.shape[1] 152 | eps = np.copy(hyperpara[0]) 153 | w, t, ls = Adaline(data_[:,:col-1],data_[:,-1],w,eps) 154 | w_list.append(w) 155 | return w_list 156 | 157 | # One-vs-One 158 | def OVO(data, classes, algo, hyperpara=[]): 159 | w_list = list() 160 | col = data.shape[1] 161 | for e in classes: 162 | print(classes,e) 163 | classes.remove(e) 164 | for f in classes: 165 | data_ = [] 166 | for k in range(data.shape[0]): 167 | if data[k][-1] == e : 168 | l = list(data[k,:col - 1]) 169 | l.append(1) 170 | data_.append(l) 171 | elif data[k][-1] == f : 172 | l = list(data[k,:col - 1]) 173 | l.append(-1) 174 | data_.append(l) 175 | data_ = np.asarray(data_) 176 | if algo =="SLP": 177 | # hyperpara = [] 178 | w = np.zeros(data.shape[1] - 1) 179 | col = data_.shape[1] 180 | w, t = PLA(data_[:,:col-1],data_[:,-1],w) 181 | w_list.append(w) 182 | elif algo == "Pocket": 183 | # hyperpara = [] 184 | w = np.zeros(data.shape[1] - 1) 185 | col = data_.shape[1] 186 | w, t, ls = Pocket(data_[:,:col-1],data_[:,-1],w) 187 | w_list.append(w) 188 | elif algo == "Adaline": 189 | # hyperpara = [eps] 190 | w = np.zeros(data.shape[1] - 1) 191 | col = data_.shape[1] 192 | eps = np.copy(hyperpara[0]) 193 | w, t, ls = Adaline(data_[:,:col-1],data_[:,-1],w,eps) 194 | w_list.append(w) 195 | elif algo == "Logistic": 196 | # hyperpara = [lr, Tmax, epsilon] 197 | col = data_.shape[1] 198 | lr = np.copy(hyperpara[0]) 199 | Tmax = np.copy(hyperpara[1]) 200 | eps = np.copy(hyperpara[2]) 201 | w, t, ls = LogisticRegression(data_[:,:col-1],data_[:,-1],lr,Tmax,eps) 202 | w_list.append(w) 203 | elif algo == "Pocket Trans": 204 | # hyperpara = [q] 205 | q = np.copy(hyperpara[0]) 206 | data_ = np.asarray([polyMap(x,q) for x in data_]) 207 | w = np.zeros(data_.shape[1] - 1) 208 | col = data_.shape[1] 209 | w, t, ls = Pocket(data_[:,:col-1],data_[:,-1],w) 210 | w_list.append(w) 211 | elif algo == "Adaline Trans": 212 | # hyperpara = [q] 213 | q = np.copy(hyperpara[0]) 214 | data_ = np.asarray([polyMap(x,q) for x in data_]) 215 | w = np.zeros(data_.shape[1] - 1) 216 | col = data_.shape[1] 217 | eps = np.copy(hyperpara[0]) 218 | w, t, ls = Adaline(data_[:,:col-1],data_[:,-1],w,eps) 219 | w_list.append(w) 220 | return w_list --------------------------------------------------------------------------------