├── examples ├── data │ ├── example1_circle │ ├── example1_square │ ├── example1_star │ ├── example2_baboon │ ├── example2_lena │ ├── example2_mona │ ├── example1_pentagon │ ├── example2_texture │ └── example2_cameraman └── Robust least squares with constraints.ipynb ├── src └── cocktail │ ├── __init__.py │ ├── utils.py │ └── cocktail.py ├── README.md ├── setup.py ├── LICENSE ├── .gitignore └── tests └── test_lstsq.py /examples/data/example1_circle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example1_circle -------------------------------------------------------------------------------- /examples/data/example1_square: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example1_square -------------------------------------------------------------------------------- /examples/data/example1_star: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example1_star -------------------------------------------------------------------------------- /examples/data/example2_baboon: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example2_baboon -------------------------------------------------------------------------------- /examples/data/example2_lena: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example2_lena -------------------------------------------------------------------------------- /examples/data/example2_mona: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example2_mona -------------------------------------------------------------------------------- /examples/data/example1_pentagon: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example1_pentagon -------------------------------------------------------------------------------- /examples/data/example2_texture: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example2_texture -------------------------------------------------------------------------------- /src/cocktail/__init__.py: -------------------------------------------------------------------------------- 1 | from .cocktail import whiten, nica, nica_nmf 2 | from .utils import lstsq, lstsq_ransac 3 | -------------------------------------------------------------------------------- /examples/data/example2_cameraman: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcromani/cocktail/HEAD/examples/data/example2_cameraman -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cocktail 2 | A blind source separation package using non-negative matrix factorization and non-negative ICA. 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import find_packages, setup 4 | 5 | setup( 6 | name='cocktail', 7 | version='1.0', 8 | description=( 9 | 'A blind source separation package using non-negative matrix factorization ' 10 | 'and non-negative ICA.' 11 | ), 12 | author='Marc Romaní', 13 | author_email='marcromani.ub@gmail.com', 14 | package_dir={'': 'src'}, 15 | packages=find_packages(where='src'), 16 | install_requires=['scikit-learn'] 17 | ) 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Marc Romaní 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Editors 2 | .vscode/ 3 | .idea/ 4 | 5 | # Vagrant 6 | .vagrant/ 7 | 8 | # Mac/OSX 9 | .DS_Store 10 | 11 | # Windows 12 | Thumbs.db 13 | 14 | # Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | build/ 26 | develop-eggs/ 27 | dist/ 28 | downloads/ 29 | eggs/ 30 | .eggs/ 31 | lib/ 32 | lib64/ 33 | parts/ 34 | sdist/ 35 | var/ 36 | wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | local_settings.py 72 | db.sqlite3 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # Jupyter Notebook 88 | .ipynb_checkpoints 89 | 90 | # IPython 91 | profile_default/ 92 | ipython_config.py 93 | 94 | # pyenv 95 | .python-version 96 | 97 | # celery beat schedule file 98 | celerybeat-schedule 99 | 100 | # SageMath parsed files 101 | *.sage.py 102 | 103 | # Environments 104 | .env 105 | .venv 106 | env/ 107 | venv/ 108 | ENV/ 109 | env.bak/ 110 | venv.bak/ 111 | 112 | # Spyder project settings 113 | .spyderproject 114 | .spyproject 115 | 116 | # Rope project settings 117 | .ropeproject 118 | 119 | # mkdocs documentation 120 | /site 121 | 122 | # mypy 123 | .mypy_cache/ 124 | .dmypy.json 125 | dmypy.json 126 | -------------------------------------------------------------------------------- /examples/Robust least squares with constraints.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "from cocktail import lstsq, lstsq_ransac" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "k = 50\n", 20 | "size = 10\n", 21 | "\n", 22 | "# k random points in the square [-size/2, size/2) x [-size/2, size/2)\n", 23 | "# Each column is a 2D point\n", 24 | "X = size * np.random.random(size=(2, k)) - size / 2\n", 25 | "\n", 26 | "# X plus a last row of ones\n", 27 | "X = np.concatenate((X, np.ones((1, X.shape[1]))))" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 3, 33 | "metadata": {}, 34 | "outputs": [ 35 | { 36 | "name": "stdout", 37 | "output_type": "stream", 38 | "text": [ 39 | "[[ 2 0 -4]\n", 40 | " [ 0 3 1]]\n" 41 | ] 42 | } 43 | ], 44 | "source": [ 45 | "sx = 2\n", 46 | "sy = 3\n", 47 | "tx = -4\n", 48 | "ty = 1\n", 49 | "\n", 50 | "# Unobservable 2D affine transformation\n", 51 | "A = np.array([\n", 52 | " [sx, 0, tx],\n", 53 | " [0, sy, ty]\n", 54 | "])\n", 55 | "\n", 56 | "print(A)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 4, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "# Transformed points (plus some noise)\n", 66 | "Y = np.matmul(A, X)\n", 67 | "# Y += 0.01 * np.random.normal(size=Y.shape)\n", 68 | "\n", 69 | "p = 0.5 # Ratio of inliers\n", 70 | "r = int(p * k) # Number of inliers\n", 71 | "\n", 72 | "# Last k - r points are not from y = A*x model but random\n", 73 | "Y[:, r:] = size * np.random.random(size=(Y.shape[0], k - r)) - size / 2" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 5, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "data": { 83 | "text/plain": [ 84 | "array([[ 1.33823781e+00, -3.88578059e-16, -2.56842278e+00],\n", 85 | " [ 0.00000000e+00, 1.45412911e+00, 1.21909974e+00]])" 86 | ] 87 | }, 88 | "execution_count": 5, 89 | "metadata": {}, 90 | "output_type": "execute_result" 91 | } 92 | ], 93 | "source": [ 94 | "constraints = np.empty((2, 3))\n", 95 | "constraints[:] = np.nan\n", 96 | "constraints[0, 1] = constraints[1, 0] = 0\n", 97 | "\n", 98 | "# This yields a bad model!\n", 99 | "lstsq(X.T, Y.T, constraints).T" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 6, 105 | "metadata": {}, 106 | "outputs": [ 107 | { 108 | "data": { 109 | "text/plain": [ 110 | "array([[ 2.00000000e+00, 2.22044605e-16, -4.00000000e+00],\n", 111 | " [ 2.55945800e-16, 3.00000000e+00, 1.00000000e+00]])" 112 | ] 113 | }, 114 | "execution_count": 6, 115 | "metadata": {}, 116 | "output_type": "execute_result" 117 | } 118 | ], 119 | "source": [ 120 | "num_iter = 100\n", 121 | "sample_size = 2\n", 122 | "min_num_inliers = 8\n", 123 | "tol = 0.05\n", 124 | "\n", 125 | "constraints = np.empty((2, 3))\n", 126 | "constraints[:] = np.nan\n", 127 | "constraints[0, 1] = constraints[1, 0] = 0\n", 128 | "\n", 129 | "# This yields a good model\n", 130 | "lstsq_ransac(X.T, Y.T, num_iter, sample_size, min_num_inliers, tol, constraints).T" 131 | ] 132 | } 133 | ], 134 | "metadata": { 135 | "kernelspec": { 136 | "display_name": "Python 3", 137 | "language": "python", 138 | "name": "python3" 139 | }, 140 | "language_info": { 141 | "codemirror_mode": { 142 | "name": "ipython", 143 | "version": 3 144 | }, 145 | "file_extension": ".py", 146 | "mimetype": "text/x-python", 147 | "name": "python", 148 | "nbconvert_exporter": "python", 149 | "pygments_lexer": "ipython3", 150 | "version": "3.8.0" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 4 155 | } 156 | -------------------------------------------------------------------------------- /src/cocktail/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy as np 4 | 5 | 6 | def _lstsq_vector(A, b, constraints=None): 7 | """Minimize || A*x - b || subject to equality constraints x_i = c_i. 8 | 9 | Let A be a matrix of shape (m, n) and b a vector of length m. This function 10 | solves the minimization problem || A*x - b || for x, subject to 0 <= r <= n 11 | equality constraints x_i = c_i. The n entries of the vector of constraints 12 | must be either NaN (if there is no constraint for that entry) or a scalar. 13 | 14 | Args: 15 | A: Coefficient matrix. 16 | b: Dependent variable. 17 | constraints: Constraint vector. 18 | 19 | Returns: 20 | x: The minimizer of the problem. 21 | 22 | """ 23 | if constraints is None: 24 | return np.linalg.lstsq(A, b, rcond=None)[0] 25 | 26 | # Indices of the constraints on x 27 | indices = np.nonzero(np.isfinite(constraints))[0] 28 | # Values of the constraints on x 29 | c = constraints[np.isfinite(constraints)] 30 | # Number of constraints 31 | r = c.size 32 | 33 | n = A.shape[1] 34 | 35 | # Matrix P.T projects (x_1, ..., x_n) to (x_i1, ..., x_ir) 36 | # where ij are the indices of the constraints on x 37 | P = np.zeros((n, r)) 38 | for i, x in enumerate(indices): 39 | P[x][i] = 1 40 | 41 | A00 = np.matmul(A.T, A) 42 | A01 = P 43 | A10 = P.T 44 | A11 = np.zeros(2*(P.shape[1],)) 45 | 46 | # Augmented A 47 | A_ = np.block([[A00, A01], [A10, A11]]) 48 | 49 | b0 = np.matmul(A.T, b) 50 | b1 = c 51 | 52 | # Augmented b 53 | b_ = np.block([b0, b1]) 54 | 55 | # Solve the augmented system 56 | x = np.linalg.lstsq(A_, b_, rcond=None)[0] 57 | 58 | return x[:n] 59 | 60 | 61 | def _lstsq_matrix(X, Y, constraints=None): 62 | """Minimize || A*X - Y || for A, subject to equality constraints a_ij = c_ij. 63 | 64 | Let X, Y be matrices of shapes (n, k), (m, k) respectively, so that A is a 65 | matrix of shape (m, n). This function solves the minimization problem 66 | || A*X - Y || for A, subject to 0 <= r <= m*n equality constraints a_ij = c_ij. 67 | The entries of the (m, n) matrix of constraints must be either NaN (if there 68 | is no constraint for that entry) or a scalar. 69 | 70 | Args: 71 | X: Input matrix. 72 | Y: Output matrix. 73 | constraints: Constraint matrix. 74 | 75 | Returns: 76 | A: The minimizer of the problem. 77 | 78 | """ 79 | if constraints is None: 80 | return np.linalg.lstsq(X.T, Y.T, rcond=None)[0].T 81 | 82 | A = np.empty((Y.shape[0], X.shape[0])) 83 | 84 | for i in range(Y.shape[0]): 85 | A[i] = _lstsq_vector(X.T, Y[i], constraints[i]) 86 | 87 | return A 88 | 89 | 90 | def lstsq(a, b, constraints=None): 91 | """Minimize || a*x - b || for x, subject to equality constraints on the elements of x. 92 | 93 | Find the minimum of the function L(x) = || a*x - b ||, where a is a matrix of shape 94 | (m, n) and b is either a vector of length m or a matrix of shape (m, k). In the first 95 | case, the solution x is a vector of length n, in the latter, it is a matrix of shape 96 | (n, k). If x is a vector (resp. a matrix), 0 <= r <= n (resp. 0 <= r <= n*k) constraints 97 | of the form x_i = c_i (resp. x_ij = c_ij) can be provided. In particular, the entries 98 | of the n-vector (resp. (n, k)-matrix) of constraints must be either NaN (if there is no 99 | constraint for that entry) or a scalar. 100 | 101 | Args: 102 | a: Matrix of shape (m, n). 103 | b: Vector of shape (m,) or matrix of shape (m, k). 104 | constraints: Vector or matrix of constraints. 105 | 106 | Returns: 107 | x: The least-squares solution. 108 | 109 | """ 110 | if b.ndim == 1: 111 | return _lstsq_vector(a, b, constraints) 112 | 113 | return _lstsq_matrix(a.T, b.T, constraints).T 114 | 115 | 116 | def lstsq_ransac(a, b, num_iter, sample_size, min_num_inliers, tol, constraints=None): 117 | best_x = None 118 | best_err = float('inf') 119 | 120 | for i in range(num_iter): 121 | # Randomly select some distinct pairs of rows (a_i, b_i) 122 | sample = random.sample(range(a.shape[0]), min(sample_size, a.shape[0])) 123 | 124 | a_ = a[sample] 125 | b_ = b[sample] 126 | 127 | # Estimate a model with these pairs 128 | x = lstsq(a_, b_, constraints) 129 | 130 | also_inliers = [] 131 | 132 | # For every other pair 133 | for k in set(range(a.shape[0])).difference(sample): 134 | ak = a[k] 135 | bk = b[k] 136 | 137 | # If it agrees with the model keep it 138 | if np.linalg.norm(bk - np.matmul(ak, x)) < tol: 139 | also_inliers.append(k) 140 | 141 | # If there is enough data that agrees with this model 142 | if len(also_inliers) >= min_num_inliers: 143 | a_ = a[sample + also_inliers] 144 | b_ = b[sample + also_inliers] 145 | 146 | # Estimate a final model with all the data 147 | x = lstsq(a_, b_, constraints) 148 | 149 | err = np.linalg.norm(b_ - np.matmul(a_, x)) 150 | 151 | # If this estimate is better than the last one 152 | if err < best_err: 153 | # Update the best model 154 | best_x = x 155 | best_err = err 156 | 157 | return best_x 158 | -------------------------------------------------------------------------------- /tests/test_lstsq.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import numpy as np 4 | 5 | from cocktail import lstsq, lstsq_ransac 6 | 7 | 8 | def test_lstsq_none_constraints_is_solution_for_vectors(): 9 | for m, n in itertools.permutations([10, 20]): 10 | # Solution 11 | x = np.random.randint(-100, 101, size=n) 12 | 13 | A = np.random.randint(-100, 101, size=(m, n)) 14 | b = np.matmul(A, x) 15 | 16 | # Result 17 | r = lstsq(A, b) 18 | 19 | # Test result is a solution 20 | assert np.allclose(np.matmul(A, r), b) 21 | 22 | 23 | def test_lstsq_nan_constraints_is_solution_for_vectors(): 24 | for m, n in itertools.permutations([10, 20]): 25 | # Solution 26 | x = np.random.randint(-100, 101, size=n) 27 | 28 | A = np.random.randint(-100, 101, size=(m, n)) 29 | b = np.matmul(A, x) 30 | 31 | # Constraints 32 | constraints = np.empty(n) 33 | constraints[:] = np.nan 34 | 35 | # Result 36 | r = lstsq(A, b, constraints) 37 | 38 | # Test result is a solution 39 | assert np.allclose(np.matmul(A, r), b) 40 | 41 | 42 | def test_lstsq_with_constraints_is_solution_for_vectors(): 43 | for m, n in itertools.permutations([10, 20]): 44 | # Solution 45 | x = np.random.randint(-100, 101, size=n) 46 | 47 | A = np.random.randint(-100, 101, size=(m, n)) 48 | b = np.matmul(A, x) 49 | 50 | # Constraints 51 | constraints = np.empty(n) 52 | constraints[:] = np.nan 53 | 54 | size = 5 55 | i = np.random.randint(n - size + 1) 56 | 57 | constraints[i:i+size] = x[i:i+size] 58 | 59 | # Result 60 | r = lstsq(A, b, constraints) 61 | 62 | # Test result is a solution 63 | assert np.allclose(np.matmul(A, r), b) 64 | # Test result has constrained elements 65 | assert np.allclose(r[i:i+size], constraints[i:i+size]) 66 | 67 | 68 | def test_lstsq_none_constraints_is_solution_for_matrices(): 69 | for m, n, k in itertools.permutations([10, 20, 30]): 70 | # Solution 71 | A = np.random.randint(-100, 101, size=(m, n)) 72 | 73 | X = np.random.randint(-100, 101, size=(n, k)) 74 | Y = np.matmul(A, X) 75 | 76 | # Result 77 | r = lstsq(X.T, Y.T).T 78 | 79 | # Test result is a solution 80 | assert np.allclose(np.matmul(r, X), Y) 81 | 82 | 83 | def test_lstsq_nan_constraints_is_solution_for_matrices(): 84 | for m, n, k in itertools.permutations([10, 20, 30]): 85 | # Solution 86 | A = np.random.randint(-100, 101, size=(m, n)) 87 | 88 | X = np.random.randint(-100, 101, size=(n, k)) 89 | Y = np.matmul(A, X) 90 | 91 | # Constraints 92 | constraints = np.empty((m, n)) 93 | constraints[:] = np.nan 94 | 95 | # Result 96 | r = lstsq(X.T, Y.T, constraints).T 97 | 98 | # Test result is a solution 99 | assert np.allclose(np.matmul(r, X), Y) 100 | 101 | 102 | def test_lstsq_with_constraints_is_solution_for_matrices(): 103 | for m, n, k in itertools.permutations([10, 20, 30]): 104 | # Solution 105 | A = np.random.randint(-100, 101, size=(m, n)) 106 | 107 | X = np.random.randint(-100, 101, size=(n, k)) 108 | Y = np.matmul(A, X) 109 | 110 | # Constraints 111 | constraints = np.empty((m, n)) 112 | constraints[:] = np.nan 113 | 114 | size = 5 115 | i = np.random.randint(m - size + 1) 116 | j = np.random.randint(n - size + 1) 117 | 118 | constraints[i:i+size, j:j+size] = A[i:i+size, j:j+size] 119 | 120 | # Result 121 | r = lstsq(X.T, Y.T, constraints).T 122 | 123 | # Test result is a solution 124 | assert np.allclose(np.matmul(r, X), Y) 125 | # Test result has constrained elements 126 | assert np.allclose(r[i:i+size, j:j+size], constraints[i:i+size, j:j+size]) 127 | 128 | 129 | def test_lstsq_none_and_nan_constraints_coincide_for_vectors(): 130 | for m, n in itertools.permutations([10, 20]): 131 | # Solution 132 | x = np.random.randint(-100, 101, size=n) 133 | 134 | A = np.random.randint(-100, 101, size=(m, n)) 135 | b = np.matmul(A, x) 136 | 137 | # Constraints 138 | constraints = np.empty(n) 139 | constraints[:] = np.nan 140 | 141 | # Results 142 | results = [ 143 | lstsq(A, b), 144 | lstsq(A, b, constraints) 145 | ] 146 | 147 | # Test they are all close to each other 148 | assert np.alltrue([np.allclose(ri, rj) for ri, rj in itertools.combinations(results, r=2)]) 149 | 150 | 151 | def test_lstsq_none_and_nan_constraints_coincide_for_matrices(): 152 | for m, n, k in itertools.permutations([10, 20, 30]): 153 | # Solution 154 | A = np.random.randint(-100, 101, size=(m, n)) 155 | 156 | X = np.random.randint(-100, 101, size=(n, k)) 157 | Y = np.matmul(A, X) 158 | 159 | # Constraints 160 | constraints = np.empty((m, n)) 161 | constraints[:] = np.nan 162 | 163 | # Results 164 | results = [ 165 | lstsq(X.T, Y.T).T, 166 | lstsq(X.T, Y.T, constraints).T 167 | ] 168 | 169 | # Test they are all close to each other 170 | assert np.alltrue([np.allclose(ri, rj) for ri, rj in itertools.combinations(results, r=2)]) 171 | 172 | 173 | def test_lstsq_ransac_is_robust_solution(): 174 | k = 50 175 | size = 10 176 | 177 | # k random points in the square [-size/2, size/2) x [-size/2, size/2) 178 | # Each column is a 2D point 179 | X = size * np.random.random(size=(2, k)) - size / 2 180 | 181 | # X plus a last row of ones 182 | X = np.concatenate((X, np.ones((1, X.shape[1])))) 183 | 184 | sx = 2 185 | sy = 3 186 | tx = -4 187 | ty = 1 188 | 189 | # Unobservable 2D affine transformation 190 | A = np.array([ 191 | [sx, 0, tx], 192 | [0, sy, ty] 193 | ]) 194 | 195 | # Transformed points 196 | Y = np.matmul(A, X) 197 | 198 | p = 0.5 # Ratio of inliers 199 | r = int(p * k) # Number of inliers 200 | 201 | # Last k - r points are not from y = A*x model but random 202 | Y[:, r:] = size * np.random.random(size=(Y.shape[0], k - r)) - size / 2 203 | 204 | constraints = np.empty((2, 3)) 205 | constraints[:] = np.nan 206 | constraints[0, 1] = constraints[1, 0] = 0 207 | 208 | # This yields a bad model! 209 | x1 = lstsq(X.T, Y.T, constraints).T 210 | assert not np.allclose(x1, A) 211 | 212 | num_iter = 100 213 | sample_size = 2 214 | min_num_inliers = 8 215 | tol = 0.05 216 | 217 | constraints = np.empty((2, 3)) 218 | constraints[:] = np.nan 219 | constraints[0, 1] = constraints[1, 0] = 0 220 | 221 | # This yields a good model 222 | x2 = lstsq_ransac(X.T, Y.T, num_iter, sample_size, min_num_inliers, tol, constraints).T 223 | assert np.allclose(x2, A) 224 | -------------------------------------------------------------------------------- /src/cocktail/cocktail.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import numpy as np 4 | from sklearn.decomposition import NMF 5 | 6 | 7 | def whiten(X, num_components=None, center=True, rowvar=True): 8 | """Whiten the data in matrix X using PCA decomposition. 9 | 10 | The data corresponds to n samples of a p-dimensional random vector. The shape 11 | of the matrix can be either (n, p) if each row is considered to be a sample or 12 | (p, n) if each column is considered to be a sample. How to read the matrix entries 13 | is specified by the rowvar parameter. Before whitening, a dimensionality reduction 14 | step can be applied to the data to reduce the p dimensions of each sample to 15 | num_components dimensions. If num_components is None, the number of dimensions kept 16 | is the maximum possible (nº of non-zero eigenvalues). For example, if X is full rank 17 | (rank(X) = min(n, p)), then num_components = p if p < n, and num_components = n-1 18 | if p >= n. 19 | 20 | Args: 21 | X: Data matrix. 22 | num_components: Number of PCA dimensions of the whitened samples. 23 | center: Whether to center the samples or not (zero-mean whitened samples). 24 | rowvar: Whether each row of X corresponds to one of the p variables or not. 25 | 26 | Returns: 27 | (Z, V): The whitened data matrix and the whitening matrix. 28 | 29 | """ 30 | r = num_components 31 | 32 | if rowvar: 33 | X = X.transpose() 34 | 35 | # Data matrix contains n observations of a p-dimensional vector 36 | # Each observation is a row of X 37 | n, p = X.shape 38 | 39 | # Arbitrary (but sensible) choice. In any case, we remove the eigenvectors of 0 eigenvalue later 40 | if r is None: 41 | r = min(n, p) 42 | 43 | # Compute the mean of the observations (p-dimensional vector) 44 | mu = np.mean(X, axis=0) 45 | 46 | # If p > n compute the eigenvectors efficiently 47 | if p > n: 48 | # n x n matrix 49 | M = np.matmul((X-mu), (X-mu).transpose()) 50 | 51 | # Eigenvector decomposition 52 | vals, vecs = np.linalg.eig(M) 53 | vals, vecs = vals.real, vecs.real 54 | 55 | # Sort the eigenvectors by "importance" and get the first r 56 | pairs = sorted([(vals[i], vecs[:, i]) for i in range(len(vals))], key=lambda x: x[0], reverse=True) 57 | pairs = [p for p in pairs if abs(p[0]) > 1e-10] # Remove the eigenvectors of 0 eigenvalue 58 | pairs = pairs[:r] 59 | 60 | # nxr matrix of eigenvectors (each column is an n-dimensional eigenvector) 61 | E = np.array([p[1] for p in pairs]).transpose() 62 | 63 | # pxr matrix of the first r eigenvectors of the covariance of X 64 | # Note that we normalize! 65 | E = np.matmul((X-mu).transpose(), E) 66 | E /= np.linalg.norm(E, axis=0) 67 | 68 | # Eigenvalues of cov(X) to the -1/2 69 | # Note that we rescale the eigenvalues of M to get the eigenvalues of cov(X)! 70 | diag = np.array([1/np.sqrt(p[0]/(n-1)) for p in pairs]) 71 | 72 | else: 73 | # p x p matrix 74 | C = np.cov(X, rowvar=False) 75 | 76 | # Eigenvector decomposition 77 | vals, vecs = np.linalg.eig(C) 78 | vals, vecs = vals.real, vecs.real 79 | 80 | # Sort the eigenvectors by "importance" and get the first r 81 | pairs = sorted([(vals[i], vecs[:, i]) for i in range(len(vals))], key=lambda x: x[0], reverse=True) 82 | pairs = [p for p in pairs if abs(p[0]) > 1e-10] # Remove the eigenvectors of 0 eigenvalue 83 | pairs = pairs[:r] 84 | 85 | # pxr matrix of the first r eigenvectors of the covariance of X 86 | E = np.array([p[1] for p in pairs]).transpose() 87 | 88 | # Eigenvalues of cov(X) to the -1/2 89 | diag = np.array([1/np.sqrt(p[0]) for p in pairs]) 90 | 91 | # Warn that the specified number of components is larger 92 | # than the number of non-zero eigenvalues. 93 | if num_components is not None: 94 | if num_components > len(pairs): 95 | warnings.warn( 96 | 'The desired number of components (%d) is larger than the actual dimension' 97 | ' of the PCA subespace (%d)' % (num_components, len(pairs)) 98 | ) 99 | 100 | # Center and whiten the data 101 | if center: 102 | X = X - mu 103 | 104 | # Whitening matrix 105 | V = E * diag 106 | 107 | # White data 108 | Z = np.matmul(X, V) 109 | 110 | if rowvar: 111 | Z = Z.transpose() 112 | 113 | # Since X is assumed to be (n, p) through the computations, the current 114 | # whitening matrix V is in fact the transpose of the actual whitening matrix. 115 | # Observation: If z = V * x for random column vectors x, z, then Z = X * V 116 | # for the (n, p) and (n, r) matrices X, Z of observations of x, z. 117 | V = V.transpose() 118 | 119 | return Z, V 120 | 121 | 122 | def nica(X, num_sources, lr=0.03, max_iter=5000, tol=1e-8, rowvar=True): 123 | """Compute the non-negative independent components of the linear generative model x = A * s. 124 | 125 | Here, x is a p-dimensional observable random vector and s is the latent random vector 126 | of length num_sources, whose components are statistically independent and non-negative. 127 | The matrix X is assumed to hold n samples of x, stacked in rows (shape(X) = (n, p)) or 128 | columns (shape(X) = (p, n)), which can be specified by the rowvar parameter. In practice, 129 | if shape(X) = (p, n) (resp. shape(X) = (n, p)) this function solves X = A * S 130 | (resp. X = S.T * A.T) both for S and A, where A is the so-called mixing matrix, with shape 131 | (p, num_sources), and S is a (num_sources, n) matrix which contains n samples of the latent 132 | source vector, stacked in columns. 133 | 134 | This function implements the method presented in: 135 | `Blind Separation of Positive Sources by Globally Convergent Gradient Search´ 136 | (https://core.ac.uk/download/pdf/76988305.pdf) 137 | 138 | Args: 139 | X: Data matrix. 140 | num_sources: Dimension of s. Number of latent random variables. 141 | lr: Learning rate of gradient descent. 142 | max_iter: Maximum number of iterations of gradient descent. 143 | tol: Tolerance on update at each iteration. 144 | rowvar: Whether each row of X corresponds to one of the p variables or not. 145 | 146 | Returns: 147 | (S, A) if rowvar == True. 148 | (S.T, A) if rowvar == False. 149 | 150 | """ 151 | # Whiten the data 152 | Z, V = whiten(X, num_sources, center=False, rowvar=rowvar) 153 | 154 | if num_sources > V.shape[0]: 155 | warnings.warn( 156 | 'The desired number of sources (%d) is larger than the actual dimension' 157 | ' of the whitened observable random vector (%d). The number of sources' 158 | ' will be set to %d' % (num_sources, V.shape[0], V.shape[0]) 159 | ) 160 | num_sources = V.shape[0] 161 | 162 | # We assume rowvar is True throughout the algorithm 163 | if not rowvar: 164 | Z = Z.transpose() 165 | 166 | # Initialize W 167 | W = np.eye(num_sources) 168 | 169 | for i in range(max_iter): 170 | W0 = W 171 | 172 | # Compute gradient 173 | Y = np.matmul(W, Z) 174 | f = np.minimum(0, Y) 175 | f_Y = np.matmul(f, Y.transpose()) 176 | E = (f_Y - f_Y.transpose()) / Y.shape[1] 177 | 178 | # Gradient descent 179 | W -= lr * np.matmul(E, W) 180 | 181 | # Symmetric orthogonalization 182 | M = np.matmul(W, W.transpose()) 183 | vals, vecs = np.linalg.eig(M) 184 | vals, vecs = vals.real, vecs.real 185 | 186 | W_sqrt = vecs / np.sqrt(vals) 187 | W_sqrt = np.matmul(W_sqrt, vecs.transpose()) 188 | W = np.matmul(W_sqrt, W) 189 | 190 | if np.linalg.norm(W - W0) < tol: 191 | break 192 | 193 | # Independent sources (up to an unknown permutation y = Q * s) 194 | Y = np.matmul(W, Z) 195 | 196 | # Compute the mixing matrix A' = A * Q.T 197 | # (which is A up to a permutation of its columns) 198 | # from the identity y = Q * s = W * V * A * s. 199 | # It then holds x = A * s = A * Q.T * y = A' * y. 200 | # Note: A' is computed as the right Moore-Penrose 201 | # inverse of W * V, but A' may not be unique since 202 | # in general p != num_sources and any right inverse 203 | # could be taken as A'. 204 | WV = np.matmul(W, V) 205 | WV_ = np.matmul(WV, WV.transpose()) 206 | WV_ = np.linalg.inv(WV_) 207 | WV_ = np.matmul(WV.transpose(), WV_) 208 | 209 | if not rowvar: 210 | Y = Y.transpose() 211 | 212 | return Y, WV_ 213 | 214 | 215 | def nica_nmf(X, num_components, lr=0.03, max_iter=5000, tol=1e-8, rowvar=True): 216 | """Non-negative matrix factorization with non-negative ICA (NICA) initialization. 217 | 218 | Under the linear generative model x = A * s, where x is a p-dimensional observable 219 | random vector, s is the latent non-negative random vector of length num_components and 220 | A is a fixed (but unknown) non-negative matrix, this function tries to determine both s 221 | and A. The data matrix X is assumed to hold n samples of x, stacked in rows (shape(X) = (n, p)) 222 | or columns (shape(X) = (p, n)), which can be specified by the rowvar parameter. In practice, 223 | if shape(X) = (p, n) (resp. shape(X) = (n, p)) this function solves X = A * S 224 | (resp. X = S.T * A.T) both for S and A, where A is the so-called mixing matrix, with shape 225 | (p, num_sources), and S is a (num_sources, n) matrix which contains n samples of the latent 226 | source vector, stacked in columns. 227 | 228 | The non-uniqueness (non-convexity) property of NMF implies that the solution depends on the 229 | initial factor matrices. This function implements the idea presented in: 230 | `Efficient initialization for nonnegative matrix factorization based on nonnegative independent component analysis´ 231 | (https://ieeexplore.ieee.org/document/7602947) 232 | which suggests that a good initialization is based on the factorization given by non-negative ICA. 233 | 234 | Args: 235 | X: Data matrix. 236 | num_components: Dimension of s. Number of latent random variables. 237 | lr: Learning rate of gradient descent for NICA. 238 | max_iter: Maximum number of iterations of gradient descent for NICA. 239 | tol: Tolerance on update at each iteration for NICA. 240 | rowvar: Whether each row of X corresponds to one of the p variables or not. 241 | 242 | Returns: 243 | (S, A) if rowvar == True. 244 | (S.T, A) if rowvar == False. 245 | 246 | """ 247 | S, A = nica(X, num_components, lr, max_iter, tol, rowvar) 248 | 249 | # We assume rowvar is True throughout the algorithm 250 | if not rowvar: 251 | X = X.transpose() 252 | S = S.transpose() 253 | 254 | # Initial NMF factorization: X = F0 * G0 255 | F0 = np.abs(A) 256 | G0 = np.abs(S) 257 | 258 | W0 = G0.transpose().copy() # Make array C-contiguous 259 | H0 = F0.transpose() 260 | 261 | nmf = NMF(n_components=num_components, init='custom') 262 | 263 | W = nmf.fit_transform(X.transpose(), W=W0, H=H0) 264 | H = nmf.components_ 265 | 266 | A = H.transpose() 267 | S = W.transpose() 268 | 269 | if not rowvar: 270 | S = S.transpose() 271 | 272 | return S, A 273 | --------------------------------------------------------------------------------