├── .gitignore ├── README.md ├── __init__.py └── src ├── __init__.py ├── clustering ├── __init__.py └── k_means.py ├── dimreduction ├── __init__.py └── pca.py ├── kernel ├── __init__.py ├── kernel.py └── support_vector_classifier.py ├── linear ├── __init__.py ├── bayesian_regression.py ├── classifier.py ├── least_squares_classifier.py ├── linear_regression.py ├── logistic_regression.py ├── regression.py ├── ridge_regression.py └── softmax_regression.py ├── preprocess ├── __init__.py ├── label_transformer.py └── polynomial.py └── rv ├── __init__.py ├── bernoulli.py ├── beta.py └── rv.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | __pycache__/ 3 | 4 | # Jupyter Notebook 5 | notebooks/ 6 | .ipynb_checkpoints 7 | *.ipynb 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ml-py 2 | ## machine learning algorithms 3 | All those algorithms coding in python, based on math and numpy packges. 4 | 5 | Thanks for your advice. 6 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TyrionBian/ml-py/a36b5c74d2ac3d108194ed17804b8ad298e3ec6c/__init__.py -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from src import ( 3 | linear 4 | ) 5 | 6 | 7 | __all__ = [ 8 | "linear" 9 | ] 10 | -------------------------------------------------------------------------------- /src/clustering/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TyrionBian/ml-py/a36b5c74d2ac3d108194ed17804b8ad298e3ec6c/src/clustering/__init__.py -------------------------------------------------------------------------------- /src/clustering/k_means.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.spatial.distance import cdist 3 | 4 | class KMeans(object): 5 | 6 | def __init__(self, n_clusters): 7 | self.n_clusters = n_clusters 8 | 9 | def fit(self, X, iter_max=100): 10 | """ 11 | perform k-means algorithm 12 | Parameters 13 | ---------- 14 | X : (sample_size, n_features) ndarray 15 | input data 16 | iter_max : int 17 | maximum number of iterations 18 | Returns 19 | ------- 20 | centers : (n_clusters, n_features) ndarray 21 | center of each cluster 22 | """ 23 | I = np.eye(self.n_clusters) 24 | centers = X[np.random.choice(len(X), self.n_clusters, replace=False)] 25 | for _ in range(iter_max): 26 | prev_centers = np.copy(centers) 27 | D = cdist(X, centers) 28 | cluster_index = np.argmin(D, axis=1) 29 | cluster_index = I[cluster_index] 30 | centers = np.sum(X[:, None, :] * cluster_index[:, :, None], axis=0) / np.sum(cluster_index, axis=0)[:, None] 31 | 32 | if np.allclose(prev_centers, centers): 33 | break 34 | 35 | self.centers = centers 36 | 37 | def predict(self, X): 38 | """ 39 | calculate closest cluster center index 40 | Parameters 41 | ---------- 42 | X : (sample_size, n_features) ndarray 43 | input data 44 | Returns 45 | ------- 46 | index : (sample_size,) ndarray 47 | indicates which cluster they belong 48 | """ 49 | D = cdist(X, self.centers) 50 | return np.argmin(D, axis=1) 51 | -------------------------------------------------------------------------------- /src/dimreduction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TyrionBian/ml-py/a36b5c74d2ac3d108194ed17804b8ad298e3ec6c/src/dimreduction/__init__.py -------------------------------------------------------------------------------- /src/dimreduction/pca.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class PCA(object): 5 | 6 | def __init__(self, n_components): 7 | """ 8 | construct principal component analysis 9 | Parameters 10 | ---------- 11 | n_components : int 12 | number of components 13 | """ 14 | assert isinstance(n_components, int) 15 | self.n_components = n_components 16 | 17 | def fit(self, X, method="eigen", iter_max=100): 18 | """ 19 | maximum likelihood estimate of pca parameters 20 | x ~ \int_z N(x|Wz+mu,sigma^2)N(z|0,I)dz 21 | Parameters 22 | ---------- 23 | X : (sample_size, n_features) ndarray 24 | input data 25 | method : str 26 | method to estimate the parameters 27 | ["eigen", "em"] 28 | iter_max : int 29 | maximum number of iterations for em algorithm 30 | Attributes 31 | ---------- 32 | mean : (n_features,) ndarray 33 | sample mean of the data 34 | W : (n_features, n_components) ndarray 35 | projection matrix 36 | var : float 37 | variance of observation noise 38 | C : (n_features, n_features) ndarray 39 | variance of the marginal dist N(x|mean,C) 40 | Cinv : (n_features, n_features) ndarray 41 | precision of the marginal dist N(x|mean, C) 42 | """ 43 | method_list = ["eigen", "em"] 44 | if method not in method_list: 45 | print("availabel methods are {}".format(method_list)) 46 | self.mean = np.mean(X, axis=0) 47 | getattr(self, method)(X - self.mean, iter_max) 48 | 49 | def eigen(self, X, *arg): 50 | sample_size, n_features = X.shape 51 | if sample_size >= n_features: 52 | cov = np.cov(X, rowvar=False) 53 | values, vectors = np.linalg.eigh(cov) 54 | index = n_features - self.n_components 55 | else: 56 | cov = np.cov(X) 57 | values, vectors = np.linalg.eigh(cov) 58 | vectors = (X.T @ vectors) / np.sqrt(sample_size * values) 59 | index = sample_size - self.n_components 60 | self.I = np.eye(self.n_components) 61 | if index == 0: 62 | self.var = 0 63 | else: 64 | self.var = np.mean(values[:index]) 65 | 66 | self.W = vectors[:, index:].dot(np.sqrt(np.diag(values[index:]) - self.var * self.I)) 67 | self.__M = self.W.T @ self.W + self.var * self.I 68 | self.C = self.W @ self.W.T + self.var * np.eye(n_features) 69 | if index == 0: 70 | self.Cinv = np.linalg.inv(self.C) 71 | else: 72 | self.Cinv = np.eye(n_features) / np.sqrt(self.var) - self.W @ np.linalg.inv(self.__M) @ self.W.T / self.var 73 | 74 | def em(self, X, iter_max): 75 | self.I = np.eye(self.n_components) 76 | self.W = np.eye(np.size(X, 1), self.n_components) 77 | self.var = 1. 78 | for i in range(iter_max): 79 | W = np.copy(self.W) 80 | stats = self._expectation(X) 81 | self._maximization(X, *stats) 82 | if np.allclose(W, self.W): 83 | break 84 | self.C = self.W @ self.W.T + self.var * np.eye(np.size(X, 1)) 85 | self.Cinv = np.linalg.inv(self.C) 86 | 87 | def _expectation(self, X): 88 | self.__M = self.W.T @ self.W + self.var * self.I 89 | Minv = np.linalg.inv(self.__M) 90 | Ez = X @ self.W @ Minv 91 | Ezz = self.var * Minv + Ez[:, :, None] * Ez[:, None, :] 92 | return Ez, Ezz 93 | 94 | def _maximization(self, X, Ez, Ezz): 95 | self.W = X.T @ Ez @ np.linalg.inv(np.sum(Ezz, axis=0)) 96 | self.var = np.mean( 97 | np.mean(X ** 2, axis=1) 98 | - 2 * np.mean(Ez @ self.W.T * X, axis=1) 99 | + np.trace((Ezz @ self.W.T @ self.W).T) / np.size(X, 1)) 100 | 101 | def transform(self, X): 102 | """ 103 | project input data into latent space 104 | p(Z|X) = N(Z|(X-mu)WMinv, sigma^-2M) 105 | Parameters 106 | ---------- 107 | X : (sample_size, n_features) ndarray 108 | input data 109 | Returns 110 | ------- 111 | Z : (sample_size, n_components) ndarray 112 | projected input data 113 | """ 114 | return np.linalg.solve(self.__M, ((X - self.mean) @ self.W).T).T 115 | 116 | def fit_transform(self, X, method="eigen"): 117 | """ 118 | perform pca and whiten the input data 119 | Parameters 120 | ---------- 121 | X : (sample_size, n_features) ndarray 122 | input data 123 | Returns 124 | ------- 125 | Z : (sample_size, n_components) ndarray 126 | projected input data 127 | """ 128 | self.fit(X, method) 129 | return self.transform(X) 130 | 131 | def proba(self, X): 132 | """ 133 | the marginal distribution of the observed variable 134 | Parameters 135 | ---------- 136 | X : (sample_size, n_features) ndarray 137 | input data 138 | Returns 139 | ------- 140 | p : (sample_size,) ndarray 141 | value of the marginal distribution 142 | """ 143 | d = X - self.mean 144 | return ( 145 | np.exp(-0.5 * np.sum(d @ self.Cinv * d, axis=-1)) 146 | / np.sqrt(np.linalg.det(self.C)) 147 | / np.power(2 * np.pi, 0.5 * np.size(X, 1))) -------------------------------------------------------------------------------- /src/kernel/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TyrionBian/ml-py/a36b5c74d2ac3d108194ed17804b8ad298e3ec6c/src/kernel/__init__.py -------------------------------------------------------------------------------- /src/kernel/kernel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class Kernel(object): 5 | """ 6 | Base class for kernel function 7 | """ 8 | 9 | def _pairwise(self, x, y): 10 | """ 11 | all pairs of x and y 12 | Parameters 13 | ---------- 14 | x : (sample_size, n_features) 15 | input 16 | y : (sample_size, n_features) 17 | another input 18 | Returns 19 | ------- 20 | output : tuple 21 | two array with shape (sample_size, sample_size, n_features) 22 | """ 23 | return ( 24 | np.tile(x, (len(y), 1, 1)).transpose(1, 0, 2), 25 | np.tile(y, (len(x), 1, 1)) 26 | ) -------------------------------------------------------------------------------- /src/kernel/support_vector_classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class SupportVectorClassifier(object): 4 | def __init__(self, kernel, C=np.Inf): 5 | """ 6 | construct support vector classifier 7 | Parameters 8 | ---------- 9 | kernel : Kernel 10 | kernel function to compute inner products 11 | C : float 12 | penalty of misclassification 13 | """ 14 | self.kernel = kernel 15 | self.C = C 16 | 17 | def fit(self, X:np.ndarray, t:np.ndarray, tol:float=1e-8): 18 | """ 19 | estimate support vectors and their parameters 20 | Parameters 21 | ---------- 22 | X : (N, D) np.ndarray 23 | training independent variable 24 | t : (N,) np.ndarray 25 | training dependent variable 26 | binary -1 or 1 27 | tol : float, optional 28 | numerical tolerance (the default is 1e-8) 29 | """ 30 | N = len(t) 31 | coef = np.zeros(N) 32 | grad = np.ones(N) 33 | Gram = self.kernel(X, X) 34 | 35 | while True: 36 | tg = t*grad 37 | mask_up = (t==1) & (coef<(self.C-tol)) 38 | mask_up |= (t==-1) & (coef>tol) 39 | 40 | mask_down = (t == -1) & (coef < self.C - tol) 41 | mask_down |= (t == 1) & (coef > tol) 42 | 43 | i = np.where(mask_up)[0][np.argmax(tg[mask_up])] 44 | j = np.where(mask_down)[0][np.argmin(tg[mask_down])] 45 | if tg[i] < tg[j] + tol: 46 | self.b = 0.5 * (tg[i] + tg[j]) 47 | break 48 | else: 49 | A = self.C - coef[i] if t[i] == 1 else coef[i] 50 | B = coef[j] if t[j] == 1 else self.C - coef[j] 51 | direction = (tg[i]-tg[j]) / (Gram(i,i) - 2 * Gram(i,j) + Gram(j,j)) 52 | direction = min(A, B, direction) 53 | coef[i] += direction * t[i] 54 | coef[j] -= direction * t[j] 55 | grad -= direction * t * (Gram[i] - Gram[j]) 56 | support_mask = coef > tol 57 | self.a = coef[support_mask] 58 | self.X = X[support_mask] 59 | self.t = t[support_mask] 60 | 61 | def lagrangian_function(self): 62 | return( 63 | np.sum(self.a) 64 | - self.a 65 | @ (self.t * self.t[:, None] * self.kernel(self.X, self.X)) 66 | @ self.a 67 | ) 68 | 69 | def predict(self, x): 70 | """ 71 | predict labels of the input 72 | Parameters 73 | ---------- 74 | x : (sample_size, n_features) ndarray 75 | input 76 | Returns 77 | ------- 78 | label : (sample_size,) ndarray 79 | predicted labels 80 | """ 81 | y = self.distance(x) 82 | label = np.sign(y) 83 | return label 84 | 85 | def distance(self, x): 86 | """ 87 | calculate distance from the decision boundary 88 | Parameters 89 | ---------- 90 | x : (sample_size, n_features) ndarray 91 | input 92 | Returns 93 | ------- 94 | distance : (sample_size,) ndarray 95 | distance from the boundary 96 | """ 97 | distance = np.sum( 98 | self.a * self.t 99 | * self.kernel(x, self.X), 100 | axis=-1) + self.b 101 | return distance 102 | 103 | 104 | -------------------------------------------------------------------------------- /src/linear/__init__.py: -------------------------------------------------------------------------------- 1 | from src.linear.linear_regression import LinearRegression 2 | from src.linear.ridge_regression import RidgeRegression 3 | from src.linear.bayesian_regression import BayesianRegression 4 | 5 | __all__ = [ 6 | "LinearRegression", 7 | "RidgeRegression", 8 | "BayesianRegression" 9 | ] -------------------------------------------------------------------------------- /src/linear/bayesian_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.linear.regression import Regression 3 | 4 | class BayesianRegression(Regression): 5 | """ 6 | Bayesian regression model 7 | w ~ N(w|0, alpha^(-1)I) 8 | y = X @ w 9 | t ~ N(t|X @ w, beta^(-1)) 10 | """ 11 | 12 | def __init__(self, alpha:float=1., beta:float=1.): 13 | self.alpha = alpha 14 | self.beta = beta 15 | self.w_mean = None 16 | self.w_precision = None 17 | 18 | def _is_prior_defined(self) -> bool: 19 | return self.w_mean is not None and self.w_precision is not None 20 | 21 | def _get_prior(self, ndim:int) -> tuple: 22 | if self._is_prior_defined(): 23 | return self.w_mean, self.w_precision 24 | else: 25 | return np.zeros(ndim), self.alpha * np.eye(ndim) 26 | 27 | def fit(self, X:np.ndarray, t:np.ndarray): 28 | """ 29 | bayesian update of parameters given training dataset 30 | Parameters 31 | ---------- 32 | X : (N, n_features) np.ndarray 33 | training data independent variable 34 | t : (N,) np.ndarray 35 | training data dependent variable 36 | """ 37 | mean_prev, precision_prev = self._get_prior(np.size(X, axis=1)) 38 | w_precision = precision_prev + self.beta * X.T @ X 39 | w_mean = np.linalg.solve( 40 | w_precision, 41 | precision_prev @ mean_prev + self.beta * X.T @ t 42 | ) 43 | self.w_mean = w_mean 44 | self.w_precision = w_precision 45 | self.w_cov = np.linalg.inv(self.w_precision) 46 | 47 | def predict(self, X:np.ndarray, return_std:bool=False, sample_size:int=None): 48 | """ 49 | return mean (and standard deviation) of predictive distribution 50 | Parameters 51 | ---------- 52 | X : (N, n_features) np.ndarray 53 | independent variable 54 | return_std : bool, optional 55 | flag to return standard deviation (the default is False) 56 | sample_size : int, optional 57 | number of samples to draw from the predictive distribution 58 | (the default is None, no sampling from the distribution) 59 | Returns 60 | ------- 61 | y : (N,) np.ndarray 62 | mean of the predictive distribution 63 | y_std : (N,) np.ndarray 64 | standard deviation of the predictive distribution 65 | y_sample : (N, sample_size) np.ndarray 66 | samples from the predictive distribution 67 | """ 68 | 69 | if sample_size is not None: 70 | w_sample = np.random.multivariate_normal( 71 | self.w_mean, self.w_cov, size=sample_size 72 | ) 73 | y_sample = X @ w_sample.T 74 | return y_sample 75 | y = X @ self.w_mean 76 | if return_std: 77 | y_var = 1 / self.beta + np.sum(X @ self.w_cov * X, axis=1) 78 | y_std = np.sqrt(y_var) 79 | return y, y_std 80 | return y -------------------------------------------------------------------------------- /src/linear/classifier.py: -------------------------------------------------------------------------------- 1 | class Classifier(object): 2 | pass -------------------------------------------------------------------------------- /src/linear/least_squares_classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.linear.classifier import Classifier 3 | from src.preprocess.label_transformer import LabelTransformer 4 | 5 | class LeastSquaresClassifier(Classifier): 6 | """ 7 | Least squares classifier model 8 | X : (N, D) 9 | W : (D, K) 10 | y = argmax_k X @ W 11 | """ 12 | def __init__(self, W:np.ndarray=None): 13 | self.W = W 14 | def fit(self, X:np.ndarray, t:np.ndarray): 15 | """ 16 | least squares fitting for classification 17 | Parameters 18 | ---------- 19 | X : (N, D) np.ndarray 20 | training independent variable 21 | t : (N,) or (N, K) np.ndarray 22 | training dependent variable 23 | in class index (N,) or one-of-k coding (N,K) 24 | """ 25 | if t.ndim == 1: 26 | t = LabelTransformer().encode(t) 27 | 28 | self.W = np.linalg.pinv(X, t) 29 | 30 | def classify(self, X:np.ndarray): 31 | """ 32 | classify input data 33 | Parameters 34 | ---------- 35 | X : (N, D) np.ndarray 36 | independent variable to be classified 37 | Returns 38 | ------- 39 | (N,) np.ndarray 40 | class index for each input 41 | """ 42 | return np.argmax(X @ self.W, axis=-1) -------------------------------------------------------------------------------- /src/linear/linear_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.linear.regression import Regression 3 | 4 | 5 | class LinearRegression(Regression): 6 | """ 7 | Linear regression model 8 | y = X @ w 9 | t ~ N(t|X @ w, var) 10 | """ 11 | def fit(self, X:np.ndarray, t:np.ndarray): 12 | """ 13 | perform least squares fitting 14 | Parameters 15 | ---------- 16 | X : (N, D) np.ndarray 17 | training independent variable 18 | t : (N,) np.ndarray 19 | training dependent variable 20 | """ 21 | self.w = np.linalg.pinv(X) @ t # @ operator calls the array's __matmul__ 22 | self.var = np.mean(X @ self.w - t) 23 | 24 | def predict(self, X:np.ndarray, return_std:bool=False): 25 | """ 26 | make prediction given input 27 | Parameters 28 | ---------- 29 | X : (N, D) np.ndarray 30 | samples to predict their output 31 | return_std : bool, optional 32 | returns standard deviation of each predition if True 33 | Returns 34 | ------- 35 | y : (N,) np.ndarray 36 | prediction of each sample 37 | y_std : (N,) np.ndarray 38 | standard deviation of each predition 39 | """ 40 | y = X @ self.w 41 | if return_std: 42 | y_std = np.sqrt(self.var) + np.zeros_like(y) 43 | return y, y_std 44 | return y 45 | 46 | -------------------------------------------------------------------------------- /src/linear/logistic_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.linear.classifier import Classifier 3 | 4 | class LogisticRegression(Classifier): 5 | """ 6 | Logistic regression model 7 | y = sigmoid(X @ w) 8 | t ~ Bernoulli(t|y) 9 | """ 10 | 11 | @staticmethod 12 | def _sigmoid(x): 13 | # (tanh+1)/2 14 | return(1/np.exp(-x)) 15 | 16 | def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100): 17 | """ 18 | maximum likelihood estimation of logistic regression model 19 | Parameters 20 | ---------- 21 | X : (N, D) np.ndarray 22 | training data independent variable 23 | t : (N,) np.ndarray 24 | training data dependent variable 25 | binary 0 or 1 26 | max_iter : int, optional 27 | maximum number of paramter update iteration (the default is 100) 28 | """ 29 | w = np.zeros(np.size(X, axis=1)) 30 | for _ in range(max_iter): 31 | w_prev = np.copy(w) 32 | y = self._sigmoid(X @ w) 33 | grad = X.T @ (y - t) 34 | hession = X.T @ (y * (1 - y)) @ X 35 | try: 36 | w -= np.linalg.solve(hession, grad) 37 | except np.linalg.LinAlgError: 38 | break 39 | if np.allclose(w, w_prev): 40 | break 41 | self.w = w 42 | 43 | def prob(self, X:np.ndarray): 44 | return self._sigmoid(X @ self.w) 45 | 46 | def classify(self, X:np.ndarray, threshold:float=0.5): 47 | return(self.prob(X) > threshold).astype(int) 48 | 49 | -------------------------------------------------------------------------------- /src/linear/regression.py: -------------------------------------------------------------------------------- 1 | class Regression(object): 2 | """ 3 | Base class for regressors 4 | """ 5 | pass -------------------------------------------------------------------------------- /src/linear/ridge_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.linear.regression import Regression 3 | 4 | class RidgeRegression(Regression): 5 | """ 6 | Ridge regression model 7 | w* = argmin |t - X @ w| + alpha * |w|_2^2 8 | """ 9 | def __init__(self, alpha:float=1.): 10 | self.alpha = alpha 11 | 12 | def fit(self, X:np.ndarray, t:np.ndarray): 13 | """ 14 | maximum a posteriori estimation of parameter 15 | Parameters 16 | ---------- 17 | X : (N, D) np.ndarray 18 | training data independent variable 19 | t : (N,) np.ndarray 20 | training data dependent variable 21 | """ 22 | eye = np.eye(np.size(X, axis=1)) 23 | self.w = np.linalg.solve(X.T @ X + self.alpha * eye, X.T @ t) 24 | 25 | def predict(self, X:np.ndarray): 26 | """ 27 | make prediction given input 28 | Parameters 29 | ---------- 30 | X : (N, D) np.ndarray 31 | samples to predict their output 32 | Returns 33 | ------- 34 | (N,) np.ndarray 35 | prediction of each input 36 | """ 37 | y = X @ self.w 38 | return y -------------------------------------------------------------------------------- /src/linear/softmax_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.linear.classifier import Classifier 3 | from src.preprocess.label_transformer import LabelTransformer 4 | class SoftmaxRegression(Classifier): 5 | """ 6 | Softmax regression model 7 | aka 8 | multinomial logistic regression, 9 | multiclass logistic regression, 10 | maximum entropy classifier. 11 | y = softmax(X @ W) 12 | t ~ Categorical(t|y) 13 | """ 14 | 15 | @staticmethod 16 | def _softmax(a): 17 | return np.exp(a)/ np.sum(np.exp(a), axis=-1, keepdims=True) 18 | 19 | def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100, learning_rate:float=0.1): 20 | """ 21 | maximum likelihood estimation of the parameter 22 | Parameters 23 | ---------- 24 | X : (N, D) np.ndarray 25 | training independent variable 26 | t : (N,) or (N, K) np.ndarray 27 | training dependent variable 28 | in class index or one-of-k encoding 29 | max_iter : int, optional 30 | maximum number of iteration (the default is 100) 31 | learning_rate : float, optional 32 | learning rate of gradient descent (the default is 0.1) 33 | """ 34 | if t.ndim==1: 35 | t = LabelTransformer().encode(t) 36 | self.n_classes = np.size(t, axis=1) 37 | W = np.zeros(shape=(np.size(X, axis=1), self.n_classes)) 38 | for _ in range(max_iter): 39 | W_prev = np.copy(W) 40 | y = self._softmax(X @ W) 41 | grad = X.T @ (y - t) 42 | W -= learning_rate*grad 43 | if np.allclose(W_prev, W): 44 | break 45 | self.W = W 46 | 47 | def proba(self, X:np.ndarray): 48 | """ 49 | compute probability of input belonging each class 50 | Parameters 51 | ---------- 52 | X : (N, D) np.ndarray 53 | independent variable 54 | Returns 55 | ------- 56 | (N, K) np.ndarray 57 | probability of each class 58 | """ 59 | return self._softmax(X @ self.W) 60 | 61 | def classify(self, X:np.ndarray): 62 | """ 63 | classify input data 64 | Parameters 65 | ---------- 66 | X : (N, D) np.ndarray 67 | independent variable to be classified 68 | Returns 69 | ------- 70 | (N,) np.ndarray 71 | class index for each input 72 | """ 73 | return np.argmax(self.proba(X), axis=-1) 74 | 75 | 76 | -------------------------------------------------------------------------------- /src/preprocess/__init__.py: -------------------------------------------------------------------------------- 1 | from src.preprocess.polynomial import PolynomialFeature 2 | 3 | 4 | __all__ = [ 5 | "PolynomialFeature" 6 | ] -------------------------------------------------------------------------------- /src/preprocess/label_transformer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class LabelTransformer(object): 4 | """ 5 | Label encoder decoder 6 | Attributes 7 | ---------- 8 | n_classes : int 9 | number of classes, K 10 | """ 11 | def __init__(self, n_classes:int=None): 12 | self.n_classes = n_classes 13 | 14 | @property 15 | def n_classes(self): 16 | return self.__n_classes 17 | 18 | @n_classes.setter 19 | def n_classes(self, K): 20 | self.__n_classes = K 21 | self.__encoder = None if K is None else np.eye(K) 22 | 23 | @property 24 | def encoder(self): 25 | return self.__encoder 26 | 27 | def encode(self, class_indices:np.ndarray): 28 | """ 29 | encode class index into one-of-k code 30 | Parameters 31 | ---------- 32 | class_indices : (N,) np.ndarray 33 | non-negative class index 34 | elements must be integer in [0, n_classes) 35 | Returns 36 | ------- 37 | (N, K) np.ndarray 38 | one-of-k encoding of input 39 | """ 40 | if self.n_classes is None: 41 | self.n_classes = np.max(class_indices) + 1 42 | 43 | return self.encoder[class_indices] 44 | 45 | def decode(self, onehot:np.ndarray): 46 | """ 47 | decode one-of-k code into class index 48 | Parameters 49 | ---------- 50 | onehot : (N, K) np.ndarray 51 | one-of-k code 52 | Returns 53 | ------- 54 | (N,) np.ndarray 55 | class index 56 | """ 57 | return np.argmax(onehot, axis=1) 58 | 59 | -------------------------------------------------------------------------------- /src/preprocess/polynomial.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import functools 3 | import numpy as np 4 | 5 | class PolynomialFeature(object): 6 | """ 7 | polynomial features 8 | transforms input array with polynomial features 9 | Example 10 | ======= 11 | x = 12 | [[a, b], 13 | [c, d]] 14 | y = PolynomialFeatures(degree=2).transform(x) 15 | y = 16 | [[1, a, b, a^2, a * b, b^2], 17 | [1, c, d, c^2, c * d, d^2]] 18 | """ 19 | def __init__(self, degree=2): 20 | """ 21 | construct polynomial features 22 | Parameters 23 | ---------- 24 | degree : int 25 | degree of polynomial 26 | """ 27 | assert isinstance(degree, int) 28 | self.degree = degree 29 | 30 | def transform(self, x): 31 | """ 32 | transforms input array with polynomial features 33 | Parameters 34 | ---------- 35 | x : (sample_size, n) ndarray 36 | input array 37 | Returns 38 | ------- 39 | output : (sample_size, 1 + nC1 + ... + nCd) ndarray 40 | polynomial features 41 | """ 42 | if x.ndim == 1: 43 | x = x[:, None] 44 | x_t = x.transpose() 45 | features = [np.ones(len(x))] 46 | for degree in range(1, self.degree + 1): 47 | for items in itertools.combinations_with_replacement(x_t, degree): 48 | features.append(functools.reduce(lambda x, y: x * y, items)) 49 | return np.asarray(features).transpose() 50 | -------------------------------------------------------------------------------- /src/rv/__init__.py: -------------------------------------------------------------------------------- 1 | from src.rv.bernoulli import Bernoulli 2 | from src.rv.bernoulli_mixture import BernoulliMixture 3 | from src.rv.beta import Beta 4 | from src.rv.categorical import Categorical 5 | from src.rv.dirichlet import Dirichlet 6 | from src.rv.gamma import Gamma 7 | from src.rv.gaussian import Gaussian 8 | from src.rv.multivariate_gaussian import MultivariateGaussian 9 | from src.rv.multivariate_gaussian_mixture import MultivariateGaussianMixture 10 | from src.rv.students_t import StudentsT 11 | from src.rv.uniform import Uniform 12 | from src.rv.variational_gaussian_mixture import VariationalGaussianMixture 13 | 14 | 15 | __all__ = [ 16 | "Bernoulli", 17 | "BernoulliMixture", 18 | "Beta", 19 | "Categorical", 20 | "Dirichlet", 21 | "Gamma", 22 | "Gaussian", 23 | "MultivariateGaussian", 24 | "MultivariateGaussianMixture", 25 | "StudentsT", 26 | "Uniform", 27 | "VariationalGaussianMixture" 28 | ] -------------------------------------------------------------------------------- /src/rv/bernoulli.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from src.rv.rv import RandomVariable 3 | from src.rv.beta import Beta -------------------------------------------------------------------------------- /src/rv/beta.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import gamma 3 | from prml.rv.rv import RandomVariable 4 | 5 | 6 | np.seterr(all="ignore") 7 | 8 | class Beta(RandomVariable): 9 | """ 10 | Beta distribution 11 | p(mu|n_ones, n_zeros) 12 | = gamma(n_ones + n_zeros) 13 | * mu^(n_ones - 1) * (1 - mu)^(n_zeros - 1) 14 | / gamma(n_ones) / gamma(n_zeros) 15 | """ 16 | -------------------------------------------------------------------------------- /src/rv/rv.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class RandomVariable(object): 5 | """ 6 | base class for random variables 7 | """ 8 | 9 | def __init__(self): 10 | self.parameter = {} 11 | 12 | def __repr__(self): 13 | string = f"{self.__class__.__name__}(\n" 14 | for key, value in self.parameter.items(): 15 | string += (" " * 4) 16 | if isinstance(value, RandomVariable): 17 | string += f"{key}={value:8}" 18 | else: 19 | string += f"{key}={value}" 20 | string += "\n" 21 | string += ")" 22 | return string 23 | 24 | def __format__(self, indent="4"): 25 | indent = int(indent) 26 | string = f"{self.__class__.__name__}(\n" 27 | for key, value in self.parameter.items(): 28 | string += (" " * indent) 29 | if isinstance(value, RandomVariable): 30 | string += f"{key}=" + value.__format__(str(indent + 4)) 31 | else: 32 | string += f"{key}={value}" 33 | string += "\n" 34 | string += (" " * (indent - 4)) + ")" 35 | return string 36 | 37 | def fit(self, X, **kwargs): 38 | """ 39 | estimate parameter(s) of the distribution 40 | Parameters 41 | ---------- 42 | X : np.ndarray 43 | observed data 44 | """ 45 | self._check_input(X) 46 | if hasattr(self, "_fit"): 47 | self._fit(X, **kwargs) 48 | else: 49 | raise NotImplementedError 50 | 51 | def pdf(self, X): 52 | """ 53 | compute probability density function 54 | p(X|parameter) 55 | Parameters 56 | ---------- 57 | X : (sample_size, ndim) np.ndarray 58 | input of the function 59 | Returns 60 | ------- 61 | p : (sample_size,) np.ndarray 62 | value of probability density function for each input 63 | """ 64 | self._check_input(X) 65 | if hasattr(self, "_pdf"): 66 | return self._pdf(X) 67 | else: 68 | raise NotImplementedError 69 | 70 | def draw(self, sample_size=1): 71 | """ 72 | draw samples from the distribution 73 | Parameters 74 | ---------- 75 | sample_size : int 76 | sample size 77 | Returns 78 | ------- 79 | sample : (sample_size, ndim) np.ndarray 80 | generated samples from the distribution 81 | """ 82 | assert isinstance(sample_size, int) 83 | if hasattr(self, "_draw"): 84 | return self._draw(sample_size) 85 | else: 86 | raise NotImplementedError 87 | 88 | def _check_input(self, X): 89 | assert isinstance(X, np.ndarray) --------------------------------------------------------------------------------