├── requirements.txt ├── kfda ├── __init__.py └── kfda.py ├── img └── mnist.png ├── setup.py ├── LICENSE ├── examples ├── mnist.py ├── faces.py └── mnist_oneshot.py └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | scikit-learn 2 | -------------------------------------------------------------------------------- /kfda/__init__.py: -------------------------------------------------------------------------------- 1 | from .kfda import Kfda 2 | -------------------------------------------------------------------------------- /img/mnist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/concavegit/kfda/HEAD/img/mnist.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="kfda", 8 | version="0.1.1", 9 | author="Kawin Nikomborirak", 10 | author_email="concavemail@gmail.com", 11 | description="Kernel FDA implementation described in https://arxiv.org/abs/1906.09436", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/concavegit/kfda", 15 | packages=setuptools.find_packages(), 16 | classifiers=[ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ], 21 | python_requires='>=3.6', 22 | ) 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Kawin Nikomborirak 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /examples/mnist.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import fetch_openml 2 | from sklearn.model_selection import train_test_split 3 | import numpy as np 4 | 5 | from kfda import Kfda 6 | 7 | 8 | X, y = fetch_openml('mnist_784', version=1, return_X_y=True) 9 | 10 | X = (X - 127.5) / 127.5 11 | 12 | # train_size=8000 takes up around 12 GB of memory. 13 | # If you don't have that available, lower this number. 14 | X_train, X_test, y_train, y_test = train_test_split( 15 | X, y, train_size=10000, stratify=y) 16 | 17 | cls = Kfda(kernel='rbf', n_components=9) 18 | print('Fitting...') 19 | train_embeddings = cls.fit_transform(X_train, y_train) 20 | print('Scores:') 21 | test_score = cls.score(X_test, y_test) 22 | print(f'Test Score: {test_score}') 23 | train_score = cls.score(X_train, y_train) 24 | print(f'Train Score: {train_score}') 25 | 26 | print('Generating embeddings...') 27 | 28 | test_embeddings = cls.transform(X_test) 29 | 30 | np.savetxt('mnist_test_embeddings.tsv', test_embeddings, delimiter='\t') 31 | np.savetxt('mnist_test_labels.tsv', y_test, delimiter='\t', fmt="%s") 32 | np.savetxt('mnist_train_labels.tsv', y_train, delimiter='\t', fmt="%s") 33 | print('Embeddings saved to *.tsv! Plug them into https://projector.tensorflow.org/ for embedding visualizations.') 34 | -------------------------------------------------------------------------------- /examples/faces.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.datasets import fetch_openml 4 | 5 | from kfda import Kfda 6 | 7 | X, y = fetch_openml('Olivetti_Faces', version=1, return_X_y=True) 8 | y = y.astype('u8') 9 | 10 | X_train, X_test, y_train, y_test = train_test_split( 11 | X, y, train_size=.7, stratify=y) 12 | 13 | cls = Kfda(kernel='rbf', n_components=50) 14 | train_embeddings = cls.fit_transform(X_train, y_train) 15 | print('fitted, scoring...') 16 | 17 | test_score = cls.score(X_test, y_test) 18 | print(f'Test Score: {test_score}') 19 | train_score = cls.score(X_train, y_train) 20 | print(f'Train Score: {train_score}') 21 | 22 | print('Generating embeddings...') 23 | test_embeddings = cls.transform(X_test) 24 | 25 | # Plug these into https://www.google.com/search?q=tensorflow+projector 26 | # to visualize embeddings. 27 | np.savetxt('face_test_embeddings.tsv', test_embeddings, delimiter='\t') 28 | np.savetxt('face_train_embeddings.tsv', train_embeddings, delimiter='\t') 29 | np.savetxt('face_test_labels.tsv', y_test, delimiter='\t', fmt="%s") 30 | np.savetxt('face_train_labels.tsv', y_train, delimiter='\t', fmt="%s") 31 | print('Embeddings saved to *.tsv! Plug them into https://projector.tensorflow.org/ for embedding visualizations.') 32 | -------------------------------------------------------------------------------- /examples/mnist_oneshot.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import fetch_openml 2 | from sklearn.model_selection import train_test_split 3 | import numpy as np 4 | 5 | from kfda import Kfda 6 | 7 | 8 | X, y = fetch_openml('mnist_784', version=1, return_X_y=True) 9 | X = (X - 127.5) / 127.5 10 | 11 | # If you don't have that much memory available, lower this number. 12 | X_train, X_test, y_train, y_test = train_test_split( 13 | X, y, train_size=10000, stratify=y) 14 | 15 | 16 | # Remove nines from the training set 17 | nines_mask = y_train == '9' 18 | X_train_nines = X_train[nines_mask] 19 | X_train = X_train[~nines_mask] 20 | y_train_nines = y_train[nines_mask] 21 | y_train = y_train[~nines_mask] 22 | 23 | # Train 24 | cls = Kfda(kernel='rbf', n_components=8) 25 | print('Fitting...') 26 | train_embeddings = cls.fit_transform(X_train, y_train) 27 | 28 | # Show the algorithm a single nine. 29 | # The same weights are used and there is no lengthy retraining. 30 | print('Adding the 9 class') 31 | cls.fit_additional(X_train_nines[:1], y_train_nines[:1]) 32 | print('Scores:') 33 | 34 | # Show the results 35 | test_score = cls.score(X_test, y_test) 36 | print(f'Test Score: {test_score}') 37 | train_score = cls.score(X_train, y_train) 38 | print(f'Train Score: {train_score}') 39 | 40 | print('Generating embeddings...') 41 | test_embeddings = cls.transform(X_test) 42 | 43 | np.savetxt('mnist_oneshot_test_embeddings.tsv', 44 | test_embeddings, delimiter='\t') 45 | np.savetxt('mnist_oneshot_test_labels.tsv', y_test, delimiter='\t', fmt="%s") 46 | np.savetxt('mnist_oneshot_train_labels.tsv', y_train, delimiter='\t', fmt="%s") 47 | print('Embeddings saved to *.tsv! Plug them into https://projector.tensorflow.org/ for embedding visualizations.') 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kernel FDA 2 | 3 | [![PyPI version](https://badge.fury.io/py/kfda.svg)](https://badge.fury.io/py/kfda) 4 | 5 | This repository implements Kernel Fisher Discriminant Analysis (Kernel FDA) as described in [https://arxiv.org/abs/1906.09436](https://arxiv.org/abs/1906.09436). 6 | FDA, equivalent to Linear Discriminant Analysis (LDA), is a classification method that projects vectors onto a smaller subspace. 7 | This subspace is optimized to maximize between-class scatter and minimize within class scatter, making it an effective classification method. 8 | Kernel FDA improves on regular FDA by enabling nonlinear subspaces using the [kernel trick](https://en.wikipedia.org/wiki/Kernel_method). 9 | The 10 | [`MNIST example`](https://github.com/concavegit/kfda/tree/master/examples/mnist.py) 11 | and the 12 | [Colab Notebook](https://colab.research.google.com/drive/1nnVphyZ_0QKYZbmdJaIBjm-zYO4xwF0b) 13 | demonstrate 97% accuracy by only training on one-seventh of the MNIST dataset. 14 | 15 | FDA and Kernel FDA classify vectors by comparing their projection in the fisher subspace to class centroids, adding a new class is just a matter of adding a new centroid. 16 | Thus, this model is implemented here with the hope of using Kernel FDA as a oneshot learning algorithm. 17 | 18 | ## Installation 19 | `kfda` is available on [PyPI](https://badge.fury.io/py/kfda): 20 | 21 | ``` 22 | pip install kfda 23 | ``` 24 | 25 | ## Usage 26 | `Kfda` uses `scikit-learn`'s interface. 27 | 28 | - Initializing: `cls = Kfda(n_components=2, kernel='linear')` for a classifier that a linear kernel with 2 components. 29 | Use `Kfda(n_components=2, kernel='poly', degree=2)` for a polynomial kernel of degree 2. 30 | See https://scikit-learn.org/stable/modules/metrics.html#polynomial-kernel for a list of kernels and their parameters, or the [source code docstrings](https://github.com/concavegit/kfda/blob/master/kfda/kfda.py) for a complete description of the parameters. 31 | 32 | - Fitting: `cls.fit(X, y)` 33 | 34 | - Prediction: `cls.predict(X)` 35 | 36 | - Scoring: `cls.score(X, y)` 37 | 38 | - Introducing new classes without retraining (fewshot learning): `cls.fit_additional(X, y)` 39 | 40 | ## Oneshot Learning 41 | Oneshot learning means that an algorithm can learn a new class with as little as one sample. 42 | This is possible for Kernel FDA because it finds a subspace that purposefully spreads out distinct classes. 43 | Introducing a new label involves simply adding another centroid in this subspace for use in prediction. 44 | See the 45 | [Colab Notebook](https://colab.research.google.com/drive/1nnVphyZ_0QKYZbmdJaIBjm-zYO4xwF0b) 46 | or the 47 | [example](https://github.com/concavegit/kfda/blob/master/examples/mnist_oneshot.py) for examples. 48 | 49 | ## Examples 50 | See [`examples`](https://github.com/concavegit/kfda/tree/master/examples) for examples on MNIST, faces, and oneshot learning. 51 | 52 | After running them, you can plug corresponding pairs of generated 53 | `*embeddings.tsv` and `*labels.tsv` into Tensorflow's 54 | [Embedding Projector](https://projector.tensorflow.org/) 55 | to visualize the embeddings. 56 | For example, running `mnist.py` and then loading 57 | `mnist_test_embeddings.tsv` and `mnist_test_labels.tsv` shows the 58 | following using the UMAP visualizer where each color is a different digit: 59 | 60 | ![MNIST Kernel FDA embeddings](https://github.com/concavegit/kfda/blob/master/img/mnist.png?raw=true) 61 | 62 | The effectiveness of the classifier is shown by the clear class separation shown. 63 | 64 | ## Notebook 65 | Another place to see example usage is the 66 | [Colab Notebook](https://colab.research.google.com/drive/1nnVphyZ_0QKYZbmdJaIBjm-zYO4xwF0b). 67 | The notebook walks through training, evaluation, and oneshot usage. 68 | 69 | ## Caveats 70 | Similar to SVM, the most glaring constraint of KFDA is the memory limit in training. 71 | Training a Kernel FDA classifier requires creating matrices that are `n_samples` by `n_samples` large, meaning the memory requirement grows with respect to `O(n_samples^2)`. 72 | 73 | The accuracy, while high (0.97 on MNIST), seems to be limited by the training set size. 74 | With a training size of 10000 and a testing size of 60000, performance on MNIST averages around 0.97 accuracy using 9 fisher directions and the RBF kernel: 75 | 76 | ```python 77 | cls = Kfda(kernel='rbf', n_components=9) 78 | ``` 79 | 80 | Accuracy can be improved without increasing training size by implementing invariant kernels that would implicitly handle scale and rotation without requiring an extended dataset. 81 | -------------------------------------------------------------------------------- /kfda/kfda.py: -------------------------------------------------------------------------------- 1 | """Module kfda""" 2 | import warnings 3 | from scipy.sparse.linalg import eigsh 4 | from scipy.sparse import eye 5 | from sklearn.base import BaseEstimator, ClassifierMixin, TransformerMixin 6 | from sklearn.metrics import pairwise_kernels 7 | from sklearn.neighbors import NearestCentroid 8 | from sklearn.utils.multiclass import unique_labels 9 | from sklearn.utils.validation import check_X_y, check_array, check_is_fitted 10 | from sklearn.preprocessing import OneHotEncoder 11 | import numpy as np 12 | 13 | 14 | class Kfda(BaseEstimator, ClassifierMixin, TransformerMixin): 15 | """Kernel Fisher Discriminant Analysis classifier. 16 | 17 | Each class is represented by a centroid using projections in a 18 | Hilbert space. 19 | 20 | See https://arxiv.org/abs/1906.09436 for mathematical details. 21 | 22 | Parameters 23 | ---------- 24 | n_components : int the amount of Fisher directions to use. 25 | This is limited by the amount of classes minus one. 26 | See the paper for further discussion of this limit. 27 | 28 | kernel : str, ['linear', 'poly', 'sigmoid', 'rbf','laplacian', 'chi2'] 29 | The kernel to use. 30 | Use **kwds to pass arguments to these functions. 31 | See 32 | https://scikit-learn.org/stable/modules/metrics.html#polynomial-kernel 33 | for more details. 34 | 35 | robustness_offset : float 36 | The small value to add along the diagonal of N to gurantee 37 | valid fisher directions. 38 | Set this to 0 to disable the feature. Default: 1e-8. 39 | 40 | **kwds : parameters to pass to the kernel function. 41 | 42 | Attributes 43 | ---------- 44 | centroids_ : array_like of shape (n_classes, n_samples) that 45 | represent the class centroids. 46 | 47 | classes_ : array of shape (n_classes,) 48 | The unique class labels 49 | 50 | weights_ : array of shape (n_components, n_samples) that 51 | represent the fisher components. 52 | 53 | clf_ : The internal NearestCentroid classifier used in prediction. 54 | """ 55 | 56 | def __init__(self, n_components=2, kernel='linear', robustness_offset=1e-8, 57 | **kwds): 58 | self.kernel = kernel 59 | self.n_components = n_components 60 | self.kwds = kwds 61 | self.robustness_offset = robustness_offset 62 | 63 | if kernel is None: 64 | self.kernel = 'linear' 65 | 66 | def fit(self, X, y): 67 | """ 68 | Fit the NearestCentroid model according to the given training data. 69 | 70 | Parameters 71 | ---------- 72 | X : {array-like, sparse matrix} of shape (n_samples, n_features) 73 | Training vector, where n_samples is the number of samples and 74 | n_features is the number of features. 75 | 76 | y : array, shape = [n_samples] 77 | Target values (integers) 78 | """ 79 | X, y = check_X_y(X, y) 80 | self.classes_ = unique_labels(y) 81 | if self.n_components > self.classes_.size - 1: 82 | warnings.warn( 83 | "n_components > classes_.size - 1." 84 | "Only the first classes_.size - 1 components will be valid." 85 | ) 86 | self.X_ = X 87 | self.y_ = y 88 | 89 | y_onehot = OneHotEncoder().fit_transform( 90 | self.y_[:, np.newaxis]) 91 | 92 | K = pairwise_kernels( 93 | X, X, metric=self.kernel, **self.kwds) 94 | 95 | m_classes = y_onehot.T @ K / y_onehot.T.sum(1) 96 | indices = (y_onehot @ np.arange(self.classes_.size)).astype('i') 97 | N = K @ (K - m_classes[indices]) 98 | 99 | # Add value to diagonal for rank robustness 100 | N += eye(self.y_.size) * self.robustness_offset 101 | 102 | m_classes_centered = m_classes - K.mean(1) 103 | M = m_classes_centered.T @ m_classes_centered 104 | 105 | # Find weights 106 | w, self.weights_ = eigsh(M, self.n_components, N, which='LM') 107 | 108 | # Compute centers 109 | centroids_ = m_classes @ self.weights_ 110 | 111 | # Train nearest centroid classifier 112 | self.clf_ = NearestCentroid().fit(centroids_, self.classes_) 113 | 114 | return self 115 | 116 | def transform(self, X): 117 | """Project the points in X onto the fisher directions. 118 | 119 | Parameters 120 | ---------- 121 | X : {array-like} of shape (n_samples, n_features) to be 122 | projected onto the fisher directions. 123 | """ 124 | check_is_fitted(self) 125 | return pairwise_kernels( 126 | X, self.X_, metric=self.kernel, **self.kwds 127 | ) @ self.weights_ 128 | 129 | def predict(self, X): 130 | """Perform classification on an array of test vectors X. 131 | 132 | The predicted class C for each sample in X is returned. 133 | 134 | Parameters 135 | ---------- 136 | X : array-like of shape (n_samples, n_features) 137 | 138 | Returns 139 | ------- 140 | C : ndarray of shape (n_samples,) 141 | """ 142 | check_is_fitted(self) 143 | 144 | X = check_array(X) 145 | 146 | projected_points = self.transform(X) 147 | predictions = self.clf_.predict(projected_points) 148 | 149 | return predictions 150 | 151 | def fit_additional(self, X, y): 152 | """Fit new classes without recomputing weights. 153 | 154 | Parameters 155 | ---------- 156 | X : array-like of shape (n_new_samples, n_nfeatures) 157 | y : array, shape = [n_samples] 158 | Target values (integers) 159 | """ 160 | check_is_fitted(self) 161 | X, y = check_X_y(X, y) 162 | 163 | new_classes = np.unique(y) 164 | 165 | projections = self.transform(X) 166 | y_onehot = OneHotEncoder().fit_transform( 167 | y[:, np.newaxis]) 168 | new_centroids = y_onehot.T @ projections / y_onehot.T.sum(1) 169 | 170 | concatenated_classes = np.concatenate([self.classes_, new_classes]) 171 | concatenated_centroids = np.concatenate( 172 | [self.clf_.centroids_, new_centroids]) 173 | 174 | self.clf_.fit(concatenated_centroids, concatenated_classes) 175 | 176 | return self 177 | --------------------------------------------------------------------------------