├── docs ├── .nojekyll ├── objects.inv ├── _static │ ├── up.png │ ├── down.png │ ├── file.png │ ├── plus.png │ ├── comment.png │ ├── minus.png │ ├── ajax-loader.gif │ ├── down-pressed.png │ ├── up-pressed.png │ ├── comment-bright.png │ ├── comment-close.png │ ├── fonts │ │ ├── Inconsolata.ttf │ │ ├── Lato-Bold.ttf │ │ ├── Lato-Regular.ttf │ │ ├── Inconsolata-Bold.ttf │ │ ├── Lato │ │ │ ├── lato-bold.eot │ │ │ ├── lato-bold.ttf │ │ │ ├── lato-bold.woff │ │ │ ├── lato-bold.woff2 │ │ │ ├── lato-italic.eot │ │ │ ├── lato-italic.ttf │ │ │ ├── lato-italic.woff │ │ │ ├── lato-italic.woff2 │ │ │ ├── lato-regular.eot │ │ │ ├── lato-regular.ttf │ │ │ ├── lato-regular.woff │ │ │ ├── lato-bolditalic.eot │ │ │ ├── lato-bolditalic.ttf │ │ │ ├── lato-bolditalic.woff │ │ │ ├── lato-regular.woff2 │ │ │ └── lato-bolditalic.woff2 │ │ ├── RobotoSlab-Bold.ttf │ │ ├── RobotoSlab-Regular.ttf │ │ ├── Inconsolata-Regular.ttf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ ├── fontawesome-webfont.woff2 │ │ └── RobotoSlab │ │ │ ├── roboto-slab-v7-bold.eot │ │ │ ├── roboto-slab-v7-bold.ttf │ │ │ ├── roboto-slab-v7-bold.woff │ │ │ ├── roboto-slab-v7-bold.woff2 │ │ │ ├── roboto-slab-v7-regular.eot │ │ │ ├── roboto-slab-v7-regular.ttf │ │ │ ├── roboto-slab-v7-regular.woff │ │ │ └── roboto-slab-v7-regular.woff2 │ └── documentation_options.js ├── src │ ├── prml.nn.io.rst │ ├── prml.clustering.rst │ ├── prml.rst │ ├── index.rst │ ├── prml.nn.normalization.rst │ ├── Makefile │ ├── prml.nn.loss.rst │ ├── prml.bayesnet.rst │ ├── prml.dimreduction.rst │ ├── make.bat │ ├── prml.nn.random.rst │ ├── prml.nn.image.rst │ ├── prml.sampling.rst │ ├── prml.preprocess.rst │ ├── prml.nn.distribution.rst │ ├── prml.nn.array.rst │ ├── prml.nn.rst │ ├── prml.markov.rst │ ├── prml.nn.nonlinear.rst │ ├── prml.nn.optimizer.rst │ ├── prml.kernel.rst │ ├── prml.rv.rst │ └── prml.nn.math.rst └── _sources │ ├── prml.nn.io.rst.txt │ ├── prml.clustering.rst.txt │ ├── prml.rst.txt │ ├── index.rst.txt │ ├── prml.nn.normalization.rst.txt │ ├── prml.nn.loss.rst.txt │ ├── prml.bayesnet.rst.txt │ ├── prml.dimreduction.rst.txt │ ├── prml.nn.random.rst.txt │ ├── prml.nn.image.rst.txt │ ├── prml.sampling.rst.txt │ ├── prml.preprocess.rst.txt │ ├── prml.nn.distribution.rst.txt │ ├── prml.nn.array.rst.txt │ ├── prml.nn.rst.txt │ ├── prml.markov.rst.txt │ ├── prml.nn.nonlinear.rst.txt │ ├── prml.nn.optimizer.rst.txt │ ├── prml.kernel.rst.txt │ ├── prml.rv.rst.txt │ └── prml.nn.math.rst.txt ├── test ├── __init__.py ├── test_nn │ ├── __init__.py │ ├── test_image │ │ ├── __init__.py │ │ ├── test_max_pooling2d.py │ │ ├── test_deconvolve2d.py │ │ └── test_convolve2d.py │ ├── test_loss │ │ ├── __init__.py │ │ └── test_sigmoid_cross_entropy.py │ ├── test_math │ │ ├── __init__.py │ │ ├── test_negative.py │ │ ├── test_log.py │ │ ├── test_matmul.py │ │ ├── test_add.py │ │ └── test_multiply.py │ ├── test_random │ │ └── __init__.py │ ├── test_distribution │ │ ├── __init__.py │ │ ├── test_gaussian.py │ │ └── test_bernoulli.py │ ├── test_nonlinear │ │ ├── __init__.py │ │ ├── test_tanh.py │ │ ├── test_sigmoid.py │ │ ├── test_log_softmax.py │ │ └── test_softmax.py │ └── test_backward.py └── test_bayesnet │ ├── __init__.py │ └── test_discrete.py ├── prml ├── nn │ ├── distribution │ │ ├── __init__.py │ │ ├── distribution.py │ │ ├── bernoulli.py │ │ ├── gaussian.py │ │ └── categorical.py │ ├── nonlinear │ │ ├── __init__.py │ │ ├── relu.py │ │ ├── logit.py │ │ ├── tanh.py │ │ ├── sigmoid.py │ │ ├── softplus.py │ │ ├── softmax.py │ │ └── log_softmax.py │ ├── io │ │ ├── __init__.py │ │ └── io.py │ ├── normalization │ │ ├── __init__.py │ │ └── batch_normalization.py │ ├── array │ │ ├── broadcast.py │ │ ├── ones.py │ │ ├── zeros.py │ │ ├── __init__.py │ │ └── reshape.py │ ├── random │ │ ├── __init__.py │ │ ├── uniform.py │ │ ├── normal.py │ │ ├── dropout.py │ │ └── random.py │ ├── loss │ │ ├── __init__.py │ │ ├── sigmoid_cross_entropy.py │ │ ├── softmax_cross_entropy.py │ │ └── kl.py │ ├── image │ │ ├── __init__.py │ │ └── util.py │ ├── optimizer │ │ ├── __init__.py │ │ ├── gradient.py │ │ ├── momentum.py │ │ ├── rmsprop.py │ │ ├── optimizer.py │ │ ├── ada_grad.py │ │ ├── ada_delta.py │ │ └── adam.py │ ├── math │ │ ├── negative.py │ │ ├── log.py │ │ ├── exp.py │ │ ├── square.py │ │ ├── sqrt.py │ │ ├── multiply.py │ │ ├── matmul.py │ │ ├── subtract.py │ │ ├── divide.py │ │ ├── mean.py │ │ ├── power.py │ │ ├── __init__.py │ │ ├── product.py │ │ ├── add.py │ │ └── sum.py │ ├── queue.py │ ├── config.py │ ├── network.py │ └── __init__.py ├── bayesnet │ ├── probability_function.py │ ├── random_variable.py │ └── __init__.py ├── clustering │ ├── __init__.py │ └── k_means.py ├── linear │ ├── classifier.py │ ├── regression.py │ ├── ridge_regression.py │ ├── __init__.py │ ├── least_squares_classifier.py │ ├── perceptron.py │ ├── linear_regression.py │ ├── bayesian_logistic_regression.py │ ├── logistic_regression.py │ ├── fishers_linear_discriminant.py │ ├── softmax_regression.py │ ├── emprical_bayes_regression.py │ ├── bayesian_regression.py │ └── variational_logistic_regression.py ├── markov │ ├── state_space_model.py │ ├── __init__.py │ ├── categorical_hmm.py │ └── gaussian_hmm.py ├── dimreduction │ ├── __init__.py │ ├── autoencoder.py │ └── bayesian_pca.py ├── sampling │ ├── __init__.py │ ├── sir.py │ ├── metropolis.py │ ├── metropolis_hastings.py │ └── rejection_sampling.py ├── __init__.py ├── preprocess │ ├── __init__.py │ ├── polynomial.py │ ├── gaussian.py │ ├── label_transformer.py │ └── sigmoidal.py ├── kernel │ ├── kernel.py │ ├── __init__.py │ ├── polynomial.py │ ├── gaussian_process_classifier.py │ └── rbf.py └── rv │ ├── __init__.py │ ├── dirichlet.py │ ├── uniform.py │ ├── beta.py │ ├── gamma.py │ └── multivariate_gaussian.py ├── setup.py ├── LICENSE ├── .gitignore └── README.md /docs/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prml/nn/distribution/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_bayesnet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/test_image/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/test_loss/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/test_math/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/test_random/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/test_distribution/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_nn/test_nonlinear/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/objects.inv -------------------------------------------------------------------------------- /docs/_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/up.png -------------------------------------------------------------------------------- /docs/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/down.png -------------------------------------------------------------------------------- /docs/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/file.png -------------------------------------------------------------------------------- /docs/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/plus.png -------------------------------------------------------------------------------- /prml/bayesnet/probability_function.py: -------------------------------------------------------------------------------- 1 | class ProbabilityFunction(object): 2 | pass 3 | -------------------------------------------------------------------------------- /docs/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/comment.png -------------------------------------------------------------------------------- /docs/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/minus.png -------------------------------------------------------------------------------- /docs/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/up-pressed.png -------------------------------------------------------------------------------- /prml/clustering/__init__.py: -------------------------------------------------------------------------------- 1 | from .k_means import KMeans 2 | 3 | 4 | __all__ = [ 5 | "KMeans" 6 | ] 7 | -------------------------------------------------------------------------------- /docs/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/comment-close.png -------------------------------------------------------------------------------- /prml/nn/io/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.io.io import save_parameter, load_parameter, save_object, load_object 2 | -------------------------------------------------------------------------------- /prml/nn/normalization/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.normalization.batch_normalization import BatchNormalization 2 | -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Inconsolata.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /prml/nn/array/broadcast.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function, broadcast, broadcast_to 3 | -------------------------------------------------------------------------------- /docs/_static/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Inconsolata-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab-Bold.ttf -------------------------------------------------------------------------------- /prml/linear/classifier.py: -------------------------------------------------------------------------------- 1 | class Classifier(object): 2 | """ 3 | Base class for classifiers 4 | """ 5 | pass 6 | -------------------------------------------------------------------------------- /prml/linear/regression.py: -------------------------------------------------------------------------------- 1 | class Regression(object): 2 | """ 3 | Base class for regressors 4 | """ 5 | pass 6 | -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab-Regular.ttf -------------------------------------------------------------------------------- /prml/bayesnet/random_variable.py: -------------------------------------------------------------------------------- 1 | class RandomVariable(object): 2 | """ 3 | Base class for random variable 4 | """ 5 | -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /prml/markov/state_space_model.py: -------------------------------------------------------------------------------- 1 | class StateSpaceModel(object): 2 | """ 3 | Base class for state-space models 4 | """ 5 | pass 6 | -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cranmer/PRML/master/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /prml/bayesnet/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.bayesnet.discrete import discrete, DiscreteVariable 2 | 3 | 4 | __all__ = [ 5 | "DiscreteVariable", 6 | "discrete" 7 | ] 8 | -------------------------------------------------------------------------------- /prml/nn/random/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.random.dropout import dropout 2 | from prml.nn.random.normal import normal, truncnormal 3 | from prml.nn.random.uniform import uniform 4 | -------------------------------------------------------------------------------- /prml/nn/random/uniform.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.array.array import asarray 3 | 4 | 5 | def uniform(min, max, size): 6 | return asarray(np.random.uniform(min, max, size)) 7 | -------------------------------------------------------------------------------- /prml/nn/array/ones.py: -------------------------------------------------------------------------------- 1 | from prml.nn.array.array import Array 2 | from prml.nn.config import config 3 | import numpy as np 4 | 5 | 6 | def ones(size): 7 | return Array(np.ones(size, dtype=config.dtype)) 8 | -------------------------------------------------------------------------------- /prml/nn/array/zeros.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.array.array import Array 3 | from prml.nn.config import config 4 | 5 | 6 | def zeros(size): 7 | return Array(np.zeros(size, dtype=config.dtype)) 8 | -------------------------------------------------------------------------------- /prml/nn/loss/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.loss.kl import kl_divergence 2 | from prml.nn.loss.sigmoid_cross_entropy import sigmoid_cross_entropy 3 | from prml.nn.loss.softmax_cross_entropy import softmax_cross_entropy 4 | -------------------------------------------------------------------------------- /prml/nn/array/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.array.array import Array, array, asarray 2 | from prml.nn.array.reshape import reshape_method 3 | from prml.nn.function import broadcast, broadcast_to 4 | 5 | 6 | Array.reshape = reshape_method 7 | -------------------------------------------------------------------------------- /prml/nn/image/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.image.convolve2d import convolve2d, Convolve2d 2 | from prml.nn.image.deconvolve2d import deconvolve2d, Deconvolve2d 3 | from prml.nn.image.max_pooling2d import max_pooling2d 4 | from prml.nn.image.util import img2patch, patch2img 5 | -------------------------------------------------------------------------------- /prml/dimreduction/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.dimreduction.autoencoder import Autoencoder 2 | from prml.dimreduction.bayesian_pca import BayesianPCA 3 | from prml.dimreduction.pca import PCA 4 | 5 | 6 | __all__ = [ 7 | "Autoencoder", 8 | "BayesianPCA", 9 | "PCA", 10 | ] 11 | -------------------------------------------------------------------------------- /prml/nn/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.optimizer.ada_delta import AdaDelta 2 | from prml.nn.optimizer.ada_grad import AdaGrad 3 | from prml.nn.optimizer.adam import Adam 4 | from prml.nn.optimizer.gradient import Gradient 5 | from prml.nn.optimizer.momentum import Momentum 6 | from prml.nn.optimizer.rmsprop import RMSProp 7 | -------------------------------------------------------------------------------- /prml/nn/math/negative.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class Negative(Function): 5 | 6 | @staticmethod 7 | def _forward(x): 8 | return -x 9 | 10 | @staticmethod 11 | def _backward(delta, x): 12 | return -delta 13 | 14 | 15 | def negative(x): 16 | return Negative().forward(x) 17 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/relu.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class ReLU(Function): 5 | 6 | @staticmethod 7 | def _forward(x): 8 | return x.clip(min=0) 9 | 10 | @staticmethod 11 | def _backward(delta, x): 12 | return delta * (x > 0) 13 | 14 | 15 | def relu(x): 16 | return ReLU().forward(x) 17 | -------------------------------------------------------------------------------- /prml/nn/math/log.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Log(Function): 6 | 7 | @staticmethod 8 | def _forward(x): 9 | return np.log(x) 10 | 11 | @staticmethod 12 | def _backward(delta, x): 13 | return delta / x 14 | 15 | 16 | def log(x): 17 | return Log().forward(x) 18 | -------------------------------------------------------------------------------- /prml/nn/random/normal.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.stats import truncnorm 3 | from prml.nn.array.array import asarray 4 | 5 | 6 | def normal(mean, std, size): 7 | return asarray(np.random.normal(mean, std, size)) 8 | 9 | 10 | def truncnormal(min, max, scale, size): 11 | return asarray(truncnorm(a=min, b=max, scale=scale).rvs(size)) 12 | -------------------------------------------------------------------------------- /prml/nn/math/exp.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Exp(Function): 6 | 7 | def _forward(self, x): 8 | self.output = np.exp(x) 9 | return self.output 10 | 11 | def _backward(self, delta, x): 12 | return delta * self.output 13 | 14 | 15 | def exp(x): 16 | return Exp().forward(x) 17 | -------------------------------------------------------------------------------- /docs/_static/documentation_options.js: -------------------------------------------------------------------------------- 1 | var DOCUMENTATION_OPTIONS = { 2 | URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), 3 | VERSION: '', 4 | LANGUAGE: 'en', 5 | COLLAPSE_INDEX: false, 6 | FILE_SUFFIX: '.html', 7 | HAS_SOURCE: true, 8 | SOURCELINK_SUFFIX: '.txt', 9 | NAVIGATION_WITH_KEYS: false, 10 | }; -------------------------------------------------------------------------------- /prml/nn/math/square.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Square(Function): 6 | 7 | @staticmethod 8 | def _forward(x): 9 | return np.square(x) 10 | 11 | @staticmethod 12 | def _backward(delta, x): 13 | return 2 * delta * x 14 | 15 | 16 | def square(x): 17 | return Square().forward(x) 18 | -------------------------------------------------------------------------------- /prml/nn/math/sqrt.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Sqrt(Function): 6 | 7 | def _forward(self, x): 8 | self.output = np.sqrt(x) 9 | return self.output 10 | 11 | def _backward(self, delta, x): 12 | return 0.5 * delta / self.output 13 | 14 | 15 | def sqrt(x): 16 | return Sqrt().forward(x) 17 | -------------------------------------------------------------------------------- /prml/sampling/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.sampling.metropolis import metropolis 2 | from prml.sampling.metropolis_hastings import metropolis_hastings 3 | from prml.sampling.rejection_sampling import rejection_sampling 4 | from prml.sampling.sir import sir 5 | 6 | 7 | __all__ = [ 8 | "metropolis", 9 | "metropolis_hastings", 10 | "rejection_sampling", 11 | "sir" 12 | ] 13 | -------------------------------------------------------------------------------- /prml/markov/__init__.py: -------------------------------------------------------------------------------- 1 | from .categorical_hmm import CategoricalHMM 2 | from .gaussian_hmm import GaussianHMM 3 | from prml.markov.kalman import Kalman, kalman_filter, kalman_smoother 4 | from .particle import Particle 5 | 6 | 7 | __all__ = [ 8 | "GaussianHMM", 9 | "CategoricalHMM", 10 | "Kalman", 11 | "kalman_filter", 12 | "kalman_smoother", 13 | "Particle" 14 | ] 15 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/logit.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Logit(Function): 6 | 7 | @staticmethod 8 | def _forward(x): 9 | return np.arctanh(2 * x - 1) * 2 10 | 11 | @staticmethod 12 | def _backward(delta, x): 13 | return delta / x / (1 - x) 14 | 15 | 16 | def logit(x): 17 | return Logit().forward(x) 18 | -------------------------------------------------------------------------------- /prml/nn/optimizer/gradient.py: -------------------------------------------------------------------------------- 1 | from prml.nn.optimizer.optimizer import Optimizer 2 | 3 | 4 | class Gradient(Optimizer): 5 | 6 | def __init__(self, parameter, learning_rate=1e-3): 7 | super().__init__(parameter, learning_rate) 8 | 9 | def update(self): 10 | for param in self.parameter.values(): 11 | param.value += self.learning_rate * param.grad 12 | 13 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/tanh.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Tanh(Function): 6 | 7 | def _forward(self, x): 8 | self.out = np.tanh(x) 9 | return self.out 10 | 11 | def _backward(self, delta, x): 12 | dx = delta * (1 - self.out ** 2) 13 | return dx 14 | 15 | 16 | def tanh(x): 17 | return Tanh().forward(x) 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | setup( 5 | name="prml", 6 | version="0.0.1", 7 | description="Collection of PRML algorithms", 8 | author="ctgk", 9 | python_requires=">=3.6", 10 | install_requires=["numpy", "scipy", "matplotlib", "scikit-learn"], 11 | packages=find_packages(exclude=["test", "test.*"]), 12 | test_suite="test" 13 | ) 14 | -------------------------------------------------------------------------------- /prml/__init__.py: -------------------------------------------------------------------------------- 1 | from prml import ( 2 | bayesnet, 3 | clustering, 4 | dimreduction, 5 | kernel, 6 | linear, 7 | markov, 8 | nn, 9 | rv, 10 | sampling 11 | ) 12 | 13 | 14 | __all__ = [ 15 | "bayesnet", 16 | "clustering", 17 | "dimreduction", 18 | "kernel", 19 | "linear", 20 | "markov", 21 | "nn", 22 | "rv", 23 | "sampling" 24 | ] 25 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/sigmoid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Sigmoid(Function): 6 | 7 | def _forward(self, x): 8 | self.out = np.tanh(x * 0.5) * 0.5 + 0.5 9 | return self.out 10 | 11 | def _backward(self, delta, x): 12 | return delta * self.out * (1 - self.out) 13 | 14 | 15 | def sigmoid(x): 16 | return Sigmoid().forward(x) 17 | -------------------------------------------------------------------------------- /prml/preprocess/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.preprocess.gaussian import GaussianFeature 2 | from prml.preprocess.label_transformer import LabelTransformer 3 | from prml.preprocess.polynomial import PolynomialFeature 4 | from prml.preprocess.sigmoidal import SigmoidalFeature 5 | 6 | 7 | __all__ = [ 8 | "GaussianFeature", 9 | "LabelTransformer", 10 | "PolynomialFeature", 11 | "SigmoidalFeature" 12 | ] 13 | -------------------------------------------------------------------------------- /docs/src/prml.nn.io.rst: -------------------------------------------------------------------------------- 1 | prml.nn.io package 2 | ================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.io.io module 8 | -------------------- 9 | 10 | .. automodule:: prml.nn.io.io 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: prml.nn.io 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.io.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.io package 2 | ================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.io.io module 8 | -------------------- 9 | 10 | .. automodule:: prml.nn.io.io 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: prml.nn.io 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /prml/nn/math/multiply.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Multiply(Function): 6 | enable_auto_broadcast = True 7 | 8 | @staticmethod 9 | def _forward(x, y): 10 | return x * y 11 | 12 | @staticmethod 13 | def _backward(delta, x, y): 14 | dx = delta * y 15 | dy = delta * x 16 | return dx, dy 17 | 18 | 19 | def multiply(x, y): 20 | return Multiply().forward(x, y) 21 | -------------------------------------------------------------------------------- /prml/nn/math/matmul.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class Matmul(Function): 5 | 6 | @staticmethod 7 | def _forward(x, y): 8 | return x @ y 9 | 10 | @staticmethod 11 | def _backward(delta, x, y): 12 | dx = delta @ y.T 13 | dy = x.T @ delta 14 | return dx, dy 15 | 16 | 17 | def matmul(x, y): 18 | return Matmul().forward(x, y) 19 | 20 | 21 | def rmatmul(x, y): 22 | return Matmul().forward(y, x) 23 | -------------------------------------------------------------------------------- /docs/src/prml.clustering.rst: -------------------------------------------------------------------------------- 1 | prml.clustering package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.clustering.k\_means module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.clustering.k_means 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: prml.clustering 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/src/prml.rst: -------------------------------------------------------------------------------- 1 | prml package 2 | ============ 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | prml.bayesnet 10 | prml.clustering 11 | prml.dimreduction 12 | prml.kernel 13 | prml.linear 14 | prml.markov 15 | prml.nn 16 | prml.preprocess 17 | prml.rv 18 | prml.sampling 19 | 20 | Module contents 21 | --------------- 22 | 23 | .. automodule:: prml 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | -------------------------------------------------------------------------------- /docs/_sources/prml.clustering.rst.txt: -------------------------------------------------------------------------------- 1 | prml.clustering package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.clustering.k\_means module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.clustering.k_means 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: prml.clustering 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/_sources/prml.rst.txt: -------------------------------------------------------------------------------- 1 | prml package 2 | ============ 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | prml.bayesnet 10 | prml.clustering 11 | prml.dimreduction 12 | prml.kernel 13 | prml.linear 14 | prml.markov 15 | prml.nn 16 | prml.preprocess 17 | prml.rv 18 | prml.sampling 19 | 20 | Module contents 21 | --------------- 22 | 23 | .. automodule:: prml 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | -------------------------------------------------------------------------------- /prml/nn/array/reshape.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class Reshape(Function): 5 | 6 | @staticmethod 7 | def _forward(x, shape): 8 | return x.reshape(*shape) 9 | 10 | @staticmethod 11 | def _backward(delta, x, shape): 12 | return delta.reshape(*x.shape) 13 | 14 | 15 | def reshape(x, shape): 16 | return Reshape().forward(x, shape=shape) 17 | 18 | 19 | def reshape_method(x, *shape): 20 | return Reshape().forward(x, shape=shape) 21 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/softplus.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Softplus(Function): 6 | 7 | @staticmethod 8 | def _forward(x): 9 | return np.maximum(x, 0) + np.log1p(np.exp(-np.abs(x))) 10 | 11 | @staticmethod 12 | def _backward(delta, x): 13 | return (np.tanh(0.5 * x) * 0.5 + 0.5) * delta 14 | 15 | 16 | def softplus(x): 17 | """ 18 | y = log(1 + exp(x)) 19 | """ 20 | return Softplus().forward(x) 21 | -------------------------------------------------------------------------------- /prml/nn/math/subtract.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Subtract(Function): 6 | enable_auto_broadcast = True 7 | 8 | @staticmethod 9 | def _forward(x, y): 10 | return x - y 11 | 12 | @staticmethod 13 | def _backward(delta, x, y): 14 | return delta, -delta 15 | 16 | 17 | def subtract(x, y): 18 | return Subtract().forward(x, y) 19 | 20 | 21 | def rsubtract(x, y): 22 | return Subtract().forward(y, x) 23 | -------------------------------------------------------------------------------- /prml/nn/math/divide.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class Divide(Function): 5 | enable_auto_broadcast = True 6 | 7 | @staticmethod 8 | def _forward(x, y): 9 | return x / y 10 | 11 | @staticmethod 12 | def _backward(delta, x, y): 13 | dx = delta / y 14 | dy = -delta * x / (y ** 2) 15 | return dx, dy 16 | 17 | 18 | def divide(x, y): 19 | return Divide().forward(x, y) 20 | 21 | 22 | def rdivide(x, y): 23 | return Divide().forward(y, x) 24 | -------------------------------------------------------------------------------- /docs/src/index.rst: -------------------------------------------------------------------------------- 1 | .. prml documentation master file, created by 2 | sphinx-quickstart on Tue May 28 10:43:30 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to prml's documentation! 7 | ================================ 8 | 9 | .. toctree:: 10 | :maxdepth: 4 11 | :caption: Contents: 12 | 13 | prml 14 | 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | -------------------------------------------------------------------------------- /test/test_nn/test_math/test_negative.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestNegative(unittest.TestCase): 7 | 8 | def test_negative(self): 9 | npx = np.random.randn(8, 9) 10 | x = nn.asarray(npx) 11 | y = -x 12 | self.assertTrue(np.allclose(y.value, -npx)) 13 | 14 | npg = np.random.randn(8, 9) 15 | y.backward(npg) 16 | self.assertTrue(np.allclose(x.grad, -npg)) 17 | 18 | 19 | if __name__ == "__main__": 20 | unittest.main() 21 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/softmax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import logsumexp 3 | from prml.nn.function import Function 4 | 5 | 6 | class Softmax(Function): 7 | 8 | def _forward(self, x): 9 | self.output = np.exp(x - logsumexp(x, axis=-1, keepdims=True)) 10 | return self.output 11 | 12 | def _backward(self, delta, x): 13 | dx = self.output * delta 14 | dx -= self.output * dx.sum(axis=-1, keepdims=True) 15 | return dx 16 | 17 | 18 | def softmax(x): 19 | return Softmax().forward(x) 20 | -------------------------------------------------------------------------------- /prml/nn/random/dropout.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class DropoutFunction(Function): 6 | 7 | def _forward(self, x, drop_ratio=.5): 8 | self.coef = 1 / (1 - drop_ratio) 9 | self.mask = (np.random.rand(*x.shape) > drop_ratio) * self.coef 10 | return x * self.mask 11 | 12 | def _backward(self, delta, x, drop_ratio): 13 | return delta * self.mask 14 | 15 | 16 | def dropout(x, drop_ratio=.5): 17 | return DropoutFunction().forward(x, drop_ratio=drop_ratio) 18 | -------------------------------------------------------------------------------- /docs/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. prml documentation master file, created by 2 | sphinx-quickstart on Tue May 28 10:43:30 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to prml's documentation! 7 | ================================ 8 | 9 | .. toctree:: 10 | :maxdepth: 4 11 | :caption: Contents: 12 | 13 | prml 14 | 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | -------------------------------------------------------------------------------- /docs/src/prml.nn.normalization.rst: -------------------------------------------------------------------------------- 1 | prml.nn.normalization package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.normalization.batch\_normalization module 8 | ------------------------------------------------- 9 | 10 | .. automodule:: prml.nn.normalization.batch_normalization 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: prml.nn.normalization 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /test/test_nn/test_math/test_log.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestLog(unittest.TestCase): 7 | 8 | def test_log(self): 9 | npx = np.random.uniform(0, 10, (4, 5)) 10 | x = nn.asarray(npx) 11 | y = nn.log(x) 12 | self.assertTrue(np.allclose(y.value, np.log(npx))) 13 | 14 | npg = np.random.randn(4, 5) 15 | y.backward(npg) 16 | self.assertTrue(np.allclose(x.grad, npg / npx)) 17 | 18 | 19 | if __name__ == "__main__": 20 | unittest.main() 21 | -------------------------------------------------------------------------------- /prml/nn/nonlinear/log_softmax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import logsumexp 3 | from prml.nn.function import Function 4 | 5 | 6 | class LogSoftmax(Function): 7 | 8 | def _forward(self, x): 9 | self.output = x - logsumexp(x, axis=-1, keepdims=True) 10 | return self.output 11 | 12 | def _backward(self, delta, x): 13 | softmax = np.exp(self.output) 14 | dx = delta - softmax * delta.sum(axis=-1, keepdims=True) 15 | return dx 16 | 17 | 18 | def log_softmax(x): 19 | return LogSoftmax().forward(x) 20 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.normalization.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.normalization package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.normalization.batch\_normalization module 8 | ------------------------------------------------- 9 | 10 | .. automodule:: prml.nn.normalization.batch_normalization 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: prml.nn.normalization 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /test/test_nn/test_nonlinear/test_tanh.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestTanh(unittest.TestCase): 7 | 8 | def test_tanh(self): 9 | npx = np.random.randn(4, 7) 10 | x = nn.asarray(npx) 11 | y = nn.tanh(x) 12 | self.assertTrue(np.allclose(y.value, np.tanh(npx))) 13 | 14 | npg = np.random.randn(4, 7) 15 | y.backward(npg) 16 | self.assertTrue(np.allclose(x.grad, npg * (1 - y.value ** 2))) 17 | 18 | 19 | if __name__ == "__main__": 20 | unittest.main() 21 | -------------------------------------------------------------------------------- /test/test_nn/test_nonlinear/test_sigmoid.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestSigmoid(unittest.TestCase): 7 | 8 | def test_sigmoid(self): 9 | npx = np.random.randn(3, 5) 10 | x = nn.asarray(npx) 11 | y = nn.sigmoid(x) 12 | self.assertTrue(np.allclose(y.value, np.tanh(npx * 0.5) * 0.5 + 0.5)) 13 | 14 | npg = np.random.randn(3, 5) 15 | y.backward(npg) 16 | self.assertTrue(np.allclose(x.grad, npg * y.value * (1 - y.value))) 17 | 18 | 19 | if __name__ == "__main__": 20 | unittest.main() 21 | -------------------------------------------------------------------------------- /prml/nn/loss/sigmoid_cross_entropy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class SigmoidCrossEntropy(Function): 6 | enable_auto_broadcast = True 7 | 8 | @staticmethod 9 | def _forward(x, t): 10 | return np.maximum(x, 0) - t * x + np.log1p(np.exp(-np.abs(x))) 11 | 12 | @staticmethod 13 | def _backward(delta, x, t): 14 | y = np.tanh(x * 0.5) * 0.5 + 0.5 15 | dx = delta * (y - t) 16 | dt = -delta * x 17 | return dx, dt 18 | 19 | 20 | def sigmoid_cross_entropy(x, t): 21 | return SigmoidCrossEntropy().forward(x, t) 22 | -------------------------------------------------------------------------------- /prml/nn/loss/softmax_cross_entropy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import logsumexp 3 | from prml.nn.function import Function 4 | 5 | 6 | class SoftmaxCrossEntropy(Function): 7 | 8 | def _forward(self, x, t): 9 | self.log_softmax = x - logsumexp(x, axis=-1, keepdims=True) 10 | return -t * self.log_softmax 11 | 12 | def _backward(self, delta, x, t): 13 | dx = delta * (np.exp(self.log_softmax) - t) 14 | dt = -delta * self.log_softmax 15 | return dx, dt 16 | 17 | 18 | def softmax_cross_entropy(x, t): 19 | return SoftmaxCrossEntropy().forward(x, t) 20 | -------------------------------------------------------------------------------- /docs/src/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = ../_build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /prml/nn/math/mean.py: -------------------------------------------------------------------------------- 1 | from prml.nn.math.sum import sum 2 | 3 | 4 | def mean(x, axis=None, keepdims=False): 5 | """ 6 | returns arithmetic mean of the elements along given axis 7 | """ 8 | if axis is None: 9 | return sum(x, axis=None, keepdims=keepdims) / x.size 10 | elif isinstance(axis, int): 11 | N = x.shape[axis] 12 | return sum(x, axis=axis, keepdims=keepdims) / N 13 | elif isinstance(axis, tuple): 14 | N = 1 15 | for ax in axis: 16 | N *= x.shape[ax] 17 | return sum(x, axis=axis, keepdims=keepdims) / N 18 | else: 19 | raise TypeError(f"Unsupported type for axis: {type(axis)}") 20 | -------------------------------------------------------------------------------- /test/test_nn/test_math/test_matmul.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestMatmul(unittest.TestCase): 7 | 8 | def test_matmul(self): 9 | npa = np.random.randn(4, 6) 10 | npb = np.random.randn(6, 3) 11 | a = nn.asarray(npa) 12 | b = nn.asarray(npb) 13 | c = a @ b 14 | self.assertTrue(np.allclose(c.value, npa @ npb)) 15 | 16 | npg = np.random.randn(4, 3) 17 | c.backward(npg) 18 | self.assertTrue(np.allclose(a.grad, npg @ npb.T)) 19 | self.assertTrue(np.allclose(b.grad, npa.T @ npg)) 20 | 21 | 22 | if __name__ == "__main__": 23 | unittest.main() 24 | -------------------------------------------------------------------------------- /prml/nn/io/io.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | 4 | 5 | def save_parameter(filename: str, parameter: dict): 6 | dict_ = {key: param.value for key, param in parameter.items()} 7 | np.savez_compressed(filename, **dict_) 8 | 9 | 10 | def load_parameter(filename: str, parameter: dict): 11 | loaded = np.load(filename) 12 | for key in parameter: 13 | np.copyto(parameter[key].value, loaded[key]) 14 | 15 | 16 | def save_object(filename: str, obj): 17 | with open(filename, "wb") as file: 18 | pickle.dump(obj, file) 19 | 20 | 21 | def load_object(filename: str, obj): 22 | with open(filename, "rb") as file: 23 | return pickle.load(filename) 24 | -------------------------------------------------------------------------------- /prml/kernel/kernel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class Kernel(object): 5 | """ 6 | Base class for kernel function 7 | """ 8 | 9 | def _pairwise(self, x, y): 10 | """ 11 | all pairs of x and y 12 | 13 | Parameters 14 | ---------- 15 | x : (sample_size, n_features) 16 | input 17 | y : (sample_size, n_features) 18 | another input 19 | 20 | Returns 21 | ------- 22 | output : tuple 23 | two array with shape (sample_size, sample_size, n_features) 24 | """ 25 | return ( 26 | np.tile(x, (len(y), 1, 1)).transpose(1, 0, 2), 27 | np.tile(y, (len(x), 1, 1)) 28 | ) -------------------------------------------------------------------------------- /prml/kernel/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.kernel.polynomial import PolynomialKernel 2 | from prml.kernel.rbf import RBF 3 | 4 | from prml.kernel.gaussian_process_classifier import GaussianProcessClassifier 5 | from prml.kernel.gaussian_process_regressor import GaussianProcessRegressor 6 | from prml.kernel.relevance_vector_classifier import RelevanceVectorClassifier 7 | from prml.kernel.relevance_vector_regressor import RelevanceVectorRegressor 8 | from prml.kernel.support_vector_classifier import SupportVectorClassifier 9 | 10 | 11 | __all__ = [ 12 | "PolynomialKernel", 13 | "RBF", 14 | "GaussianProcessClassifier", 15 | "GaussianProcessRegressor", 16 | "RelevanceVectorClassifier", 17 | "RelevanceVectorRegressor", 18 | "SupportVectorClassifier" 19 | ] 20 | -------------------------------------------------------------------------------- /prml/nn/math/power.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Power(Function): 6 | """ 7 | First array elements raised to powers from second array 8 | """ 9 | 10 | def _forward(self, x, y): 11 | self.output = np.power(x, y) 12 | return self.output 13 | 14 | def _backward(self, delta, x, y): 15 | dx = y * np.power(x, y - 1) * delta 16 | if (x > 0).all(): 17 | dy = self.output * np.log(x) * delta 18 | return dx, dy 19 | return dx 20 | 21 | 22 | def power(x, y): 23 | """ 24 | First array elements raised to powers from second array 25 | """ 26 | return Power().forward(x, y) 27 | 28 | 29 | def rpower(x, y): 30 | return Power().forward(y, x) 31 | -------------------------------------------------------------------------------- /prml/nn/optimizer/momentum.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.optimizer.optimizer import Optimizer 3 | 4 | 5 | class Momentum(Optimizer): 6 | 7 | def __init__(self, parameter: dict, learning_rate=1e-3, momentum=0.9): 8 | super().__init__(parameter, learning_rate) 9 | self.momentum = momentum 10 | self.inertia = {key: np.zeros(value.shape) for key, value in parameter.items()} 11 | 12 | def update(self): 13 | for key in self.parameter: 14 | param, inertia = self.parameter[key], self.inertia[key] 15 | if param.grad is None: 16 | continue 17 | inertia *= self.momentum 18 | inertia += self.learning_rate * (1 - self.momentum) * param.grad 19 | param.value += inertia 20 | -------------------------------------------------------------------------------- /prml/nn/queue.py: -------------------------------------------------------------------------------- 1 | class BackPropQueue(object): 2 | 3 | def __init__(self): 4 | self.queue = [] 5 | 6 | def __len__(self): 7 | return len(self.queue) 8 | 9 | def enqueue(self, array): 10 | array.is_in_queue = True 11 | self.queue.append(array) 12 | 13 | def dequeue(self, depth_to_dequeue): 14 | queue = self.queue[0] 15 | for candidate in self.queue: 16 | if candidate.depth == depth_to_dequeue: 17 | queue = candidate 18 | break 19 | elif candidate.depth > queue.depth: 20 | queue = candidate 21 | self.queue.remove(queue) 22 | queue.is_in_queue = False 23 | return queue 24 | 25 | 26 | backprop_queue = BackPropQueue() 27 | 28 | -------------------------------------------------------------------------------- /test/test_nn/test_nonlinear/test_log_softmax.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestLogSoftmax(unittest.TestCase): 7 | 8 | def test_forward(self): 9 | npx = np.random.randn(5, 3) 10 | npy = np.log(np.exp(npx) / np.exp(npx).sum(axis=-1, keepdims=True)) 11 | self.assertTrue(np.allclose(npy, nn.log_softmax(npx).value)) 12 | 13 | def test_backward(self): 14 | npx = np.random.randn(1, 5) 15 | x = nn.asarray(npx) 16 | nn.softmax(x).backward() 17 | grad1 = np.copy(x.grad) 18 | x.cleargrad() 19 | nn.exp(nn.log_softmax(x)).backward() 20 | grad2 = np.copy(x.grad) 21 | self.assertTrue(np.allclose(grad1, grad2)) 22 | 23 | 24 | if __name__ == "__main__": 25 | unittest.main() 26 | -------------------------------------------------------------------------------- /prml/nn/optimizer/rmsprop.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.optimizer.optimizer import Optimizer 3 | 4 | 5 | class RMSProp(Optimizer): 6 | 7 | def __init__(self, parameter: dict, learning_rate=1e-3, rho=0.9, epsilon=1e-8): 8 | super().__init__(parameter, learning_rate) 9 | self.rho = rho 10 | self.epsilon = epsilon 11 | self.mean_squared_grad = {key: np.zeros(value.shape) for key, value in parameter.items()} 12 | 13 | def update(self): 14 | for key in self.parameter: 15 | param, msg = self.parameter[key], self.mean_squared_grad[key] 16 | if param.grad is None: 17 | continue 18 | msg *= self.rho 19 | msg += (1 - self.rho) * (param.grad ** 2) 20 | param.value += self.learning_rate * param.grad / (np.sqrt(msg) + self.epsilon) 21 | -------------------------------------------------------------------------------- /test/test_nn/test_loss/test_sigmoid_cross_entropy.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestSigmoidCrossEntropy(unittest.TestCase): 7 | 8 | def test_sigmoid_cross_entropy(self): 9 | npx = np.random.randn(10, 3) 10 | npy = np.tanh(npx * 0.5) * 0.5 + 0.5 11 | npt = np.random.uniform(0, 1, (10, 3)) 12 | x = nn.asarray(npx) 13 | t = nn.asarray(npt) 14 | loss = nn.loss.sigmoid_cross_entropy(x, t) 15 | self.assertTrue(np.allclose(loss.value, -npt * np.log(npy) - (1 - npt) * np.log(1 - npy))) 16 | 17 | npg = np.random.randn(10, 3) 18 | loss.backward(npg) 19 | self.assertTrue(np.allclose(x.grad, npg * (npy - npt))) 20 | self.assertTrue(np.allclose(t.grad, -npg * npx)) 21 | 22 | 23 | if __name__ == "__main__": 24 | unittest.main() 25 | -------------------------------------------------------------------------------- /docs/src/prml.nn.loss.rst: -------------------------------------------------------------------------------- 1 | prml.nn.loss package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.loss.kl module 8 | ---------------------- 9 | 10 | .. automodule:: prml.nn.loss.kl 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.loss.sigmoid\_cross\_entropy module 16 | ------------------------------------------- 17 | 18 | .. automodule:: prml.nn.loss.sigmoid_cross_entropy 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.loss.softmax\_cross\_entropy module 24 | ------------------------------------------- 25 | 26 | .. automodule:: prml.nn.loss.softmax_cross_entropy 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: prml.nn.loss 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /prml/nn/math/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.math.negative import negative 2 | from prml.nn.math.add import add 3 | from prml.nn.math.subtract import subtract, rsubtract 4 | from prml.nn.math.divide import divide, rdivide 5 | from prml.nn.math.mean import mean 6 | from prml.nn.math.multiply import multiply 7 | from prml.nn.math.matmul import matmul, rmatmul 8 | from prml.nn.math.power import power, rpower 9 | from prml.nn.math.sum import sum 10 | from prml.nn.array import Array 11 | 12 | 13 | Array.__neg__ = negative 14 | Array.__add__ = add 15 | Array.__radd__ = add 16 | Array.__sub__ = subtract 17 | Array.__rsub__ = rsubtract 18 | Array.__truediv__ = divide 19 | Array.__rtruediv__ = rdivide 20 | Array.__mul__ = multiply 21 | Array.__rmul__ = multiply 22 | Array.__matmul__ = matmul 23 | Array.__rmatmul__ = rmatmul 24 | Array.__pow__ = power 25 | Array.__rpow__ = rpower 26 | Array.sum = sum 27 | Array.mean = mean 28 | -------------------------------------------------------------------------------- /test/test_nn/test_nonlinear/test_softmax.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestSoftmax(unittest.TestCase): 7 | 8 | def test_forward(self): 9 | npx = np.random.randn(5, 3) 10 | npy = np.exp(npx) / np.exp(npx).sum(axis=-1, keepdims=True) 11 | self.assertTrue(np.allclose(npy, nn.softmax(npx).value)) 12 | 13 | def test_backward(self): 14 | npx = np.random.randn(1, 4) 15 | x = nn.asarray(npx) 16 | y = nn.square(nn.softmax(x)).sum() 17 | y.backward() 18 | grad = x.grad 19 | 20 | eps = np.zeros(4) 21 | eps[0] = 1e-3 22 | numerical_grad = (nn.square(nn.softmax(npx + eps)).sum() - nn.square(nn.softmax(npx - eps)).sum()) / 2e-3 23 | self.assertAlmostEqual(grad[0][0], numerical_grad.value[0], 3) 24 | 25 | 26 | if __name__ == "__main__": 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /docs/src/prml.bayesnet.rst: -------------------------------------------------------------------------------- 1 | prml.bayesnet package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.bayesnet.discrete module 8 | ----------------------------- 9 | 10 | .. automodule:: prml.bayesnet.discrete 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.bayesnet.probability\_function module 16 | ------------------------------------------ 17 | 18 | .. automodule:: prml.bayesnet.probability_function 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.bayesnet.random\_variable module 24 | ------------------------------------- 25 | 26 | .. automodule:: prml.bayesnet.random_variable 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: prml.bayesnet 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/src/prml.dimreduction.rst: -------------------------------------------------------------------------------- 1 | prml.dimreduction package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.dimreduction.autoencoder module 8 | ------------------------------------ 9 | 10 | .. automodule:: prml.dimreduction.autoencoder 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.dimreduction.bayesian\_pca module 16 | -------------------------------------- 17 | 18 | .. automodule:: prml.dimreduction.bayesian_pca 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.dimreduction.pca module 24 | ---------------------------- 25 | 26 | .. automodule:: prml.dimreduction.pca 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: prml.dimreduction 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.loss.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.loss package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.loss.kl module 8 | ---------------------- 9 | 10 | .. automodule:: prml.nn.loss.kl 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.loss.sigmoid\_cross\_entropy module 16 | ------------------------------------------- 17 | 18 | .. automodule:: prml.nn.loss.sigmoid_cross_entropy 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.loss.softmax\_cross\_entropy module 24 | ------------------------------------------- 25 | 26 | .. automodule:: prml.nn.loss.softmax_cross_entropy 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: prml.nn.loss 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /prml/sampling/sir.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def sir(func, rv, n): 5 | """ 6 | sampling-importance-resampling 7 | 8 | Parameters 9 | ---------- 10 | func : callable 11 | (un)normalized distribution to be sampled from 12 | rv : RandomVariable 13 | distribution to generate sample 14 | n : int 15 | number of samples to draw 16 | 17 | Returns 18 | ------- 19 | sample : (n, ndim) ndarray 20 | generated sample 21 | """ 22 | assert hasattr(rv, "draw"), "the distribution has no method to draw random samples" 23 | sample_candidate = rv.draw(n * 10) 24 | weight = np.squeeze(func(sample_candidate) / rv.pdf(sample_candidate)) 25 | assert weight.shape == (n * 10,), weight.shape 26 | weight /= np.sum(weight) 27 | index = np.random.choice(n * 10, n, p=weight) 28 | sample = sample_candidate[index] 29 | return sample 30 | -------------------------------------------------------------------------------- /docs/_sources/prml.bayesnet.rst.txt: -------------------------------------------------------------------------------- 1 | prml.bayesnet package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.bayesnet.discrete module 8 | ----------------------------- 9 | 10 | .. automodule:: prml.bayesnet.discrete 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.bayesnet.probability\_function module 16 | ------------------------------------------ 17 | 18 | .. automodule:: prml.bayesnet.probability_function 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.bayesnet.random\_variable module 24 | ------------------------------------- 25 | 26 | .. automodule:: prml.bayesnet.random_variable 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: prml.bayesnet 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/_sources/prml.dimreduction.rst.txt: -------------------------------------------------------------------------------- 1 | prml.dimreduction package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.dimreduction.autoencoder module 8 | ------------------------------------ 9 | 10 | .. automodule:: prml.dimreduction.autoencoder 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.dimreduction.bayesian\_pca module 16 | -------------------------------------- 17 | 18 | .. automodule:: prml.dimreduction.bayesian_pca 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.dimreduction.pca module 24 | ---------------------------- 25 | 26 | .. automodule:: prml.dimreduction.pca 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: prml.dimreduction 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/src/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /prml/rv/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.rv.bernoulli import Bernoulli 2 | from prml.rv.bernoulli_mixture import BernoulliMixture 3 | from prml.rv.beta import Beta 4 | from prml.rv.categorical import Categorical 5 | from prml.rv.dirichlet import Dirichlet 6 | from prml.rv.gamma import Gamma 7 | from prml.rv.gaussian import Gaussian 8 | from prml.rv.multivariate_gaussian import MultivariateGaussian 9 | from prml.rv.multivariate_gaussian_mixture import MultivariateGaussianMixture 10 | from prml.rv.students_t import StudentsT 11 | from prml.rv.uniform import Uniform 12 | from prml.rv.variational_gaussian_mixture import VariationalGaussianMixture 13 | 14 | 15 | __all__ = [ 16 | "Bernoulli", 17 | "BernoulliMixture", 18 | "Beta", 19 | "Categorical", 20 | "Dirichlet", 21 | "Gamma", 22 | "Gaussian", 23 | "MultivariateGaussian", 24 | "MultivariateGaussianMixture", 25 | "StudentsT", 26 | "Uniform", 27 | "VariationalGaussianMixture" 28 | ] 29 | -------------------------------------------------------------------------------- /prml/nn/distribution/distribution.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class Distribution(Function): 5 | is_categorical = False 6 | 7 | def __init__(self, data=None): 8 | self.data = data 9 | 10 | def draw(self): 11 | self.data = self.forward() 12 | return self.data 13 | 14 | def pdf(self, x=None): 15 | if x is not None: 16 | return self._pdf(x) 17 | elif self.data is not None: 18 | return self._pdf(self.data) 19 | else: 20 | raise ValueError 21 | 22 | def _pdf(self, x): 23 | raise NotImplementedError 24 | 25 | def log_pdf(self, x=None): 26 | if x is not None: 27 | return self._log_pdf(x) 28 | elif self.data is not None: 29 | return self._log_pdf(self.data) 30 | else: 31 | raise ValueError 32 | 33 | def _log_pdf(self, *args, **kwargs): 34 | raise NotImplementedError 35 | -------------------------------------------------------------------------------- /prml/nn/optimizer/optimizer.py: -------------------------------------------------------------------------------- 1 | class Optimizer(object): 2 | 3 | def __init__(self, parameter: dict, learning_rate: float): 4 | if isinstance(parameter, list): 5 | self.parameter = {f"parameter{i}" : param for i, param in enumerate(parameter)} 6 | elif isinstance(parameter, dict): 7 | self.parameter = parameter 8 | self.learning_rate = learning_rate 9 | self.iter_count = 0 10 | 11 | def increment_iter_count(self): 12 | self.iter_count += 1 13 | 14 | def minimize(self, loss): 15 | if self.learning_rate > 0: 16 | self.learning_rate *= -1 17 | self.optimize(loss) 18 | 19 | def maximize(self, score): 20 | if self.learning_rate < 0: 21 | self.learning_rate *= -1 22 | self.optimize(score) 23 | 24 | def optimize(self, array): 25 | self.increment_iter_count() 26 | array.backward() 27 | self.update() 28 | 29 | def update(self): 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /docs/src/prml.nn.random.rst: -------------------------------------------------------------------------------- 1 | prml.nn.random package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.random.dropout module 8 | ----------------------------- 9 | 10 | .. automodule:: prml.nn.random.dropout 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.random.normal module 16 | ---------------------------- 17 | 18 | .. automodule:: prml.nn.random.normal 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.random.random module 24 | ---------------------------- 25 | 26 | .. automodule:: prml.nn.random.random 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.random.uniform module 32 | ----------------------------- 33 | 34 | .. automodule:: prml.nn.random.uniform 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.nn.random 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.random.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.random package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.random.dropout module 8 | ----------------------------- 9 | 10 | .. automodule:: prml.nn.random.dropout 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.random.normal module 16 | ---------------------------- 17 | 18 | .. automodule:: prml.nn.random.normal 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.random.random module 24 | ---------------------------- 25 | 26 | .. automodule:: prml.nn.random.random 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.random.uniform module 32 | ----------------------------- 33 | 34 | .. automodule:: prml.nn.random.uniform 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.nn.random 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /prml/sampling/metropolis.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | 4 | 5 | def metropolis(func, rv, n, downsample=1): 6 | """ 7 | Metropolis algorithm 8 | 9 | Parameters 10 | ---------- 11 | func : callable 12 | (un)normalized distribution to be sampled from 13 | rv : RandomVariable 14 | proposal distribution which is symmetric at the origin 15 | n : int 16 | number of samples to draw 17 | downsample : int 18 | downsampling factor 19 | 20 | Returns 21 | ------- 22 | sample : (n, ndim) ndarray 23 | generated sample 24 | """ 25 | x = np.zeros((1, rv.ndim)) 26 | sample = [] 27 | for i in range(n * downsample): 28 | x_new = x + rv.draw() 29 | accept_proba = func(x_new) / func(x) 30 | if random.random() < accept_proba: 31 | x = x_new 32 | if i % downsample == 0: 33 | sample.append(x[0]) 34 | sample = np.asarray(sample) 35 | assert sample.shape == (n, rv.ndim), sample.shape 36 | return sample 37 | -------------------------------------------------------------------------------- /test/test_nn/test_image/test_max_pooling2d.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestMaxPooling2d(unittest.TestCase): 7 | 8 | def test_max_pooling2d(self): 9 | img = np.array([ 10 | [2, 3, 4, 1], 11 | [2, 5, 1, 2], 12 | [3, 5, 1, 3], 13 | [3, 7, 8, 2] 14 | ]).astype(np.float) 15 | img = img[None, :, :, None] 16 | expected = np.array([[5, 4], [7, 8]]) 17 | actual = nn.max_pooling2d(img, 2, 2).value.squeeze() 18 | self.assertTrue((expected == actual).all(), actual) 19 | 20 | expected = np.array([ 21 | [0, 0, 1, 0], 22 | [0, 1, 0, 0], 23 | [0, 0, 0, 0], 24 | [0, 1, 1, 0] 25 | ]) 26 | img = nn.asarray(img) 27 | nn.max_pooling2d(img, 2, 2).backward(np.ones((1, 2, 2, 1))) 28 | actual = img.grad.squeeze() 29 | self.assertTrue((expected == actual).all()) 30 | 31 | 32 | if __name__ == "__main__": 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /docs/src/prml.nn.image.rst: -------------------------------------------------------------------------------- 1 | prml.nn.image package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.image.convolve2d module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.nn.image.convolve2d 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.image.deconvolve2d module 16 | --------------------------------- 17 | 18 | .. automodule:: prml.nn.image.deconvolve2d 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.image.max\_pooling2d module 24 | ----------------------------------- 25 | 26 | .. automodule:: prml.nn.image.max_pooling2d 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.image.util module 32 | ------------------------- 33 | 34 | .. automodule:: prml.nn.image.util 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.nn.image 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /prml/nn/config.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class Config(object): 5 | __dtype = np.float32 6 | __is_updating_bn = False 7 | __available_dtypes = (np.float16, np.float32, np.float64) 8 | __enable_backprop = True 9 | 10 | @property 11 | def dtype(self): 12 | return self.__dtype 13 | 14 | @dtype.setter 15 | def dtype(self, dtype): 16 | if dtype in self.__available_dtypes: 17 | self.__dtype = dtype 18 | else: 19 | raise ValueError 20 | 21 | @property 22 | def is_updating_bn(self): 23 | return self.__is_updating_bn 24 | 25 | @is_updating_bn.setter 26 | def is_updating_bn(self, flag): 27 | assert(isinstance(flag, bool)) 28 | self.__is_updating_bn = flag 29 | 30 | @property 31 | def enable_backprop(self): 32 | return self.__enable_backprop 33 | 34 | @enable_backprop.setter 35 | def enable_backprop(self, flag): 36 | assert(isinstance(flag, bool)) 37 | self.__enable_backprop = flag 38 | 39 | 40 | config = Config() 41 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.image.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.image package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.image.convolve2d module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.nn.image.convolve2d 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.image.deconvolve2d module 16 | --------------------------------- 17 | 18 | .. automodule:: prml.nn.image.deconvolve2d 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.image.max\_pooling2d module 24 | ----------------------------------- 25 | 26 | .. automodule:: prml.nn.image.max_pooling2d 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.image.util module 32 | ------------------------- 33 | 34 | .. automodule:: prml.nn.image.util 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.nn.image 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /prml/sampling/metropolis_hastings.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | 4 | 5 | def metropolis_hastings(func, rv, n, downsample=1): 6 | """ 7 | Metropolis Hastings algorith 8 | 9 | Parameters 10 | ---------- 11 | func : callable 12 | (un)normalized distribution to be sampled from 13 | rv : RandomVariable 14 | proposal distribution 15 | n : int 16 | number of samples to draw 17 | downsample : int 18 | downsampling factor 19 | 20 | Returns 21 | ------- 22 | sample : (n, ndim) ndarray 23 | generated sample 24 | """ 25 | x = np.zeros((1, rv.ndim)) 26 | sample = [] 27 | for i in range(n * downsample): 28 | x_new = x + rv.draw() 29 | accept_proba = func(x_new) * rv.pdf(x - x_new) / (func(x) * rv.pdf(x_new - x)) 30 | if random.random() < accept_proba: 31 | x = x_new 32 | if i % downsample == 0: 33 | sample.append(x[0]) 34 | sample = np.asarray(sample) 35 | assert sample.shape == (n, rv.ndim), sample.shape 36 | return sample 37 | -------------------------------------------------------------------------------- /docs/src/prml.sampling.rst: -------------------------------------------------------------------------------- 1 | prml.sampling package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.sampling.metropolis module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.sampling.metropolis 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.sampling.metropolis\_hastings module 16 | ----------------------------------------- 17 | 18 | .. automodule:: prml.sampling.metropolis_hastings 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.sampling.rejection\_sampling module 24 | ---------------------------------------- 25 | 26 | .. automodule:: prml.sampling.rejection_sampling 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.sampling.sir module 32 | ------------------------ 33 | 34 | .. automodule:: prml.sampling.sir 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.sampling 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/_sources/prml.sampling.rst.txt: -------------------------------------------------------------------------------- 1 | prml.sampling package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.sampling.metropolis module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.sampling.metropolis 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.sampling.metropolis\_hastings module 16 | ----------------------------------------- 17 | 18 | .. automodule:: prml.sampling.metropolis_hastings 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.sampling.rejection\_sampling module 24 | ---------------------------------------- 25 | 26 | .. automodule:: prml.sampling.rejection_sampling 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.sampling.sir module 32 | ------------------------ 33 | 34 | .. automodule:: prml.sampling.sir 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.sampling 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/src/prml.preprocess.rst: -------------------------------------------------------------------------------- 1 | prml.preprocess package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.preprocess.gaussian module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.preprocess.gaussian 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.preprocess.label\_transformer module 16 | ----------------------------------------- 17 | 18 | .. automodule:: prml.preprocess.label_transformer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.preprocess.polynomial module 24 | --------------------------------- 25 | 26 | .. automodule:: prml.preprocess.polynomial 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.preprocess.sigmoidal module 32 | -------------------------------- 33 | 34 | .. automodule:: prml.preprocess.sigmoidal 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.preprocess 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /prml/sampling/rejection_sampling.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | 4 | 5 | def rejection_sampling(func, rv, k, n): 6 | """ 7 | perform rejection sampling n times 8 | 9 | Parameters 10 | ---------- 11 | func : callable 12 | (un)normalized distribution to be sampled from 13 | rv : RandomVariable 14 | distribution to generate sample 15 | k : float 16 | constant to be multiplied with the distribution 17 | n : int 18 | number of samples to draw 19 | 20 | Returns 21 | ------- 22 | sample : (n, ndim) ndarray 23 | generated sample 24 | """ 25 | assert hasattr(rv, "draw"), "the distribution has no method to draw random samples" 26 | sample = [] 27 | while len(sample) < n: 28 | sample_candidate = rv.draw() 29 | accept_proba = func(sample_candidate) / (k * rv.pdf(sample_candidate)) 30 | if random.random() < accept_proba: 31 | sample.append(sample_candidate[0]) 32 | sample = np.asarray(sample) 33 | assert sample.shape == (n, rv.ndim), sample.shape 34 | return sample 35 | -------------------------------------------------------------------------------- /prml/nn/optimizer/ada_grad.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.config import config 3 | from prml.nn.optimizer.optimizer import Optimizer 4 | 5 | 6 | class AdaGrad(Optimizer): 7 | """ 8 | AdaGrad optimizer 9 | initialization 10 | G = 0 11 | update rule 12 | G += gradient ** 2 13 | param -= learning_rate * gradient / sqrt(G + eps) 14 | """ 15 | 16 | def __init__(self, parameter: dict, learning_rate=0.001, epsilon=1e-8): 17 | super().__init__(parameter, learning_rate) 18 | self.epsilon = epsilon 19 | self.G = [] 20 | for key, param in self.parameter.items(): 21 | self.G[key] = np.zeros(param.shape, dtype=config.dtype) 22 | 23 | def update(self): 24 | """ 25 | update parameters 26 | """ 27 | for key in self.parameter: 28 | param, G = self.parameter[key], self.G[key] 29 | if param.grad is None: 30 | continue 31 | grad = param.grad 32 | G += grad ** 2 33 | param.value += self.learning_rate * grad / (np.sqrt(G) + self.epsilon) 34 | -------------------------------------------------------------------------------- /docs/_sources/prml.preprocess.rst.txt: -------------------------------------------------------------------------------- 1 | prml.preprocess package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.preprocess.gaussian module 8 | ------------------------------- 9 | 10 | .. automodule:: prml.preprocess.gaussian 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.preprocess.label\_transformer module 16 | ----------------------------------------- 17 | 18 | .. automodule:: prml.preprocess.label_transformer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.preprocess.polynomial module 24 | --------------------------------- 25 | 26 | .. automodule:: prml.preprocess.polynomial 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.preprocess.sigmoidal module 32 | -------------------------------- 33 | 34 | .. automodule:: prml.preprocess.sigmoidal 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.preprocess 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 ctgk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /prml/kernel/polynomial.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.kernel.kernel import Kernel 3 | 4 | 5 | class PolynomialKernel(Kernel): 6 | """ 7 | Polynomial kernel 8 | k(x,y) = (x @ y + c)^M 9 | """ 10 | 11 | def __init__(self, degree=2, const=0.): 12 | """ 13 | construct Polynomial kernel 14 | 15 | Parameters 16 | ---------- 17 | const : float 18 | a constant to be added 19 | degree : int 20 | degree of polynomial order 21 | """ 22 | self.const = const 23 | self.degree = degree 24 | 25 | def __call__(self, x, y, pairwise=True): 26 | """ 27 | calculate pairwise polynomial kernel 28 | 29 | Parameters 30 | ---------- 31 | x : (..., ndim) ndarray 32 | input 33 | y : (..., ndim) ndarray 34 | another input with the same shape 35 | 36 | Returns 37 | ------- 38 | output : ndarray 39 | polynomial kernel 40 | """ 41 | if pairwise: 42 | x, y = self._pairwise(x, y) 43 | return (np.sum(x * y, axis=-1) + self.const) ** self.degree 44 | -------------------------------------------------------------------------------- /docs/src/prml.nn.distribution.rst: -------------------------------------------------------------------------------- 1 | prml.nn.distribution package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.distribution.bernoulli module 8 | ------------------------------------- 9 | 10 | .. automodule:: prml.nn.distribution.bernoulli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.distribution.categorical module 16 | --------------------------------------- 17 | 18 | .. automodule:: prml.nn.distribution.categorical 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.distribution.distribution module 24 | ---------------------------------------- 25 | 26 | .. automodule:: prml.nn.distribution.distribution 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.distribution.gaussian module 32 | ------------------------------------ 33 | 34 | .. automodule:: prml.nn.distribution.gaussian 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.nn.distribution 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.distribution.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.distribution package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.distribution.bernoulli module 8 | ------------------------------------- 9 | 10 | .. automodule:: prml.nn.distribution.bernoulli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.distribution.categorical module 16 | --------------------------------------- 17 | 18 | .. automodule:: prml.nn.distribution.categorical 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.distribution.distribution module 24 | ---------------------------------------- 25 | 26 | .. automodule:: prml.nn.distribution.distribution 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.distribution.gaussian module 32 | ------------------------------------ 33 | 34 | .. automodule:: prml.nn.distribution.gaussian 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: prml.nn.distribution 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /prml/kernel/gaussian_process_classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class GaussianProcessClassifier(object): 5 | 6 | def __init__(self, kernel, noise_level=1e-4): 7 | """ 8 | construct gaussian process classifier 9 | 10 | Parameters 11 | ---------- 12 | kernel 13 | kernel function to be used to compute Gram matrix 14 | noise_level : float 15 | parameter to ensure the matrix to be positive 16 | """ 17 | self.kernel = kernel 18 | self.noise_level = noise_level 19 | 20 | def _sigmoid(self, a): 21 | return np.tanh(a * 0.5) * 0.5 + 0.5 22 | 23 | def fit(self, X, t): 24 | if X.ndim == 1: 25 | X = X[:, None] 26 | self.X = X 27 | self.t = t 28 | Gram = self.kernel(X, X) 29 | self.covariance = Gram + np.eye(len(Gram)) * self.noise_level 30 | self.precision = np.linalg.inv(self.covariance) 31 | 32 | def predict(self, X): 33 | if X.ndim == 1: 34 | X = X[:, None] 35 | K = self.kernel(X, self.X) 36 | a_mean = K @ self.precision @ self.t 37 | return self._sigmoid(a_mean) 38 | -------------------------------------------------------------------------------- /test/test_nn/test_image/test_deconvolve2d.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | from scipy.ndimage.filters import correlate 4 | import prml.nn as nn 5 | 6 | 7 | class TestDeconvolve2d(unittest.TestCase): 8 | 9 | def test_deconvolve2d_forward(self): 10 | img = np.random.randn(1, 3, 3, 1).astype(np.float32) 11 | kernel = np.random.randn(3, 3, 1, 1).astype(np.float32) 12 | output = nn.deconvolve2d(img, kernel, (1, 1), (0, 0)) 13 | self.assertTrue(np.allclose(output.value[0,1:-1,1:-1,0], correlate(img[0,:,:,0], kernel[::-1,::-1,0,0], mode="constant"))) 14 | 15 | def test_deconvolve2d_backward(self): 16 | x = nn.random.normal(0, 1, (1, 3, 3, 1)) 17 | w = nn.random.normal(0, 1, (3, 3, 1, 1)) 18 | for _ in range(1000): 19 | x.cleargrad() 20 | w.cleargrad() 21 | output = nn.deconvolve2d(x, w, (2, 2), (1, 1)) 22 | output.backward(2 * (output.value - 1)) 23 | x.value -= x.grad * 0.01 24 | w.value -= w.grad * 0.01 25 | self.assertTrue(np.allclose(output.value, 1), output.value) 26 | 27 | 28 | if __name__ == "__main__": 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /docs/src/prml.nn.array.rst: -------------------------------------------------------------------------------- 1 | prml.nn.array package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.array.array module 8 | -------------------------- 9 | 10 | .. automodule:: prml.nn.array.array 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.array.broadcast module 16 | ------------------------------ 17 | 18 | .. automodule:: prml.nn.array.broadcast 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.array.ones module 24 | ------------------------- 25 | 26 | .. automodule:: prml.nn.array.ones 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.array.reshape module 32 | ---------------------------- 33 | 34 | .. automodule:: prml.nn.array.reshape 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.array.zeros module 40 | -------------------------- 41 | 42 | .. automodule:: prml.nn.array.zeros 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: prml.nn.array 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.array.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.array package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.array.array module 8 | -------------------------- 9 | 10 | .. automodule:: prml.nn.array.array 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.array.broadcast module 16 | ------------------------------ 17 | 18 | .. automodule:: prml.nn.array.broadcast 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.array.ones module 24 | ------------------------- 25 | 26 | .. automodule:: prml.nn.array.ones 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.array.reshape module 32 | ---------------------------- 33 | 34 | .. automodule:: prml.nn.array.reshape 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.array.zeros module 40 | -------------------------- 41 | 42 | .. automodule:: prml.nn.array.zeros 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: prml.nn.array 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /prml/linear/ridge_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.regression import Regression 3 | 4 | 5 | class RidgeRegression(Regression): 6 | """ 7 | Ridge regression model 8 | 9 | w* = argmin |t - X @ w| + alpha * |w|_2^2 10 | """ 11 | 12 | def __init__(self, alpha:float=1.): 13 | self.alpha = alpha 14 | 15 | def fit(self, X:np.ndarray, t:np.ndarray): 16 | """ 17 | maximum a posteriori estimation of parameter 18 | 19 | Parameters 20 | ---------- 21 | X : (N, D) np.ndarray 22 | training data independent variable 23 | t : (N,) np.ndarray 24 | training data dependent variable 25 | """ 26 | 27 | eye = np.eye(np.size(X, 1)) 28 | self.w = np.linalg.solve(self.alpha * eye + X.T @ X, X.T @ t) 29 | 30 | def predict(self, X:np.ndarray): 31 | """ 32 | make prediction given input 33 | 34 | Parameters 35 | ---------- 36 | X : (N, D) np.ndarray 37 | samples to predict their output 38 | 39 | Returns 40 | ------- 41 | (N,) np.ndarray 42 | prediction of each input 43 | """ 44 | return X @ self.w 45 | -------------------------------------------------------------------------------- /docs/src/prml.nn.rst: -------------------------------------------------------------------------------- 1 | prml.nn package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | prml.nn.array 10 | prml.nn.distribution 11 | prml.nn.image 12 | prml.nn.io 13 | prml.nn.loss 14 | prml.nn.math 15 | prml.nn.nonlinear 16 | prml.nn.normalization 17 | prml.nn.optimizer 18 | prml.nn.random 19 | 20 | Submodules 21 | ---------- 22 | 23 | prml.nn.config module 24 | --------------------- 25 | 26 | .. automodule:: prml.nn.config 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.function module 32 | ----------------------- 33 | 34 | .. automodule:: prml.nn.function 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.network module 40 | ---------------------- 41 | 42 | .. automodule:: prml.nn.network 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.queue module 48 | -------------------- 49 | 50 | .. automodule:: prml.nn.queue 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: prml.nn 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | prml.nn.array 10 | prml.nn.distribution 11 | prml.nn.image 12 | prml.nn.io 13 | prml.nn.loss 14 | prml.nn.math 15 | prml.nn.nonlinear 16 | prml.nn.normalization 17 | prml.nn.optimizer 18 | prml.nn.random 19 | 20 | Submodules 21 | ---------- 22 | 23 | prml.nn.config module 24 | --------------------- 25 | 26 | .. automodule:: prml.nn.config 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.function module 32 | ----------------------- 33 | 34 | .. automodule:: prml.nn.function 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.network module 40 | ---------------------- 41 | 42 | .. automodule:: prml.nn.network 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.queue module 48 | -------------------- 49 | 50 | .. automodule:: prml.nn.queue 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: prml.nn 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /prml/nn/network.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from prml.nn.array.array import Array 3 | 4 | 5 | class Network(object): 6 | 7 | def __init__(self): 8 | self._setting_parameter = False 9 | self.parameter = {} 10 | 11 | @property 12 | def setting_parameter(self): 13 | return getattr(self, "_setting_parameter", False) 14 | 15 | @contextmanager 16 | def set_parameter(self): 17 | prev_scope = self._setting_parameter 18 | object.__setattr__(self, "_setting_parameter", True) 19 | try: 20 | yield 21 | finally: 22 | object.__setattr__(self, "_setting_parameter", prev_scope) 23 | 24 | def __setattr__(self, key, value): 25 | if self.setting_parameter: 26 | if isinstance(value, Array): 27 | self.parameter[self.__class__.__name__ + "." + key] = value 28 | elif isinstance(value, Network): 29 | for name, param in value.parameter.items(): 30 | self.parameter[self.__class__.__name__ + "." + key + "." + name] = param 31 | 32 | object.__setattr__(self, key, value) 33 | 34 | def clear(self): 35 | for param in self.parameter.values(): 36 | param.cleargrad() 37 | -------------------------------------------------------------------------------- /prml/linear/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.linear.bayesian_logistic_regression import BayesianLogisticRegression 2 | from prml.linear.bayesian_regression import BayesianRegression 3 | from prml.linear.emprical_bayes_regression import EmpiricalBayesRegression 4 | from prml.linear.least_squares_classifier import LeastSquaresClassifier 5 | from prml.linear.linear_regression import LinearRegression 6 | from prml.linear.fishers_linear_discriminant import FishersLinearDiscriminant 7 | from prml.linear.logistic_regression import LogisticRegression 8 | from prml.linear.perceptron import Perceptron 9 | from prml.linear.ridge_regression import RidgeRegression 10 | from prml.linear.softmax_regression import SoftmaxRegression 11 | from prml.linear.variational_linear_regression import VariationalLinearRegression 12 | from prml.linear.variational_logistic_regression import VariationalLogisticRegression 13 | 14 | 15 | __all__ = [ 16 | "BayesianLogisticRegression", 17 | "BayesianRegression", 18 | "EmpiricalBayesRegression", 19 | "LeastSquaresClassifier", 20 | "LinearRegression", 21 | "FishersLinearDiscriminant", 22 | "LogisticRegression", 23 | "Perceptron", 24 | "RidgeRegression", 25 | "SoftmaxRegression", 26 | "VariationalLinearRegression", 27 | "VariationalLogisticRegression" 28 | ] 29 | -------------------------------------------------------------------------------- /prml/nn/optimizer/ada_delta.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.config import config 3 | from prml.nn.optimizer.optimizer import Optimizer 4 | 5 | 6 | class AdaDelta(Optimizer): 7 | """ 8 | AdaDelta optimizer 9 | """ 10 | 11 | def __init__(self, parameter: dict, rho=0.95, epsilon=1e-8): 12 | super().__init__(parameter, None) 13 | self.rho = rho 14 | self.epsilon = epsilon 15 | self.mean_squared_deriv = {} 16 | self.mean_squared_update = {} 17 | for key, param in self.parameter.items(): 18 | self.mean_squared_deriv[key] = np.zeros(param.shape, dtype=config.dtype) 19 | self.mean_squared_update[key] = np.zeros(param.shape, dtype=config.dtype) 20 | 21 | def update(self): 22 | for key in self.parameter: 23 | param = self.parameter[key] 24 | if param.grad is None: 25 | continue 26 | msd = self.mean_squared_deriv[key] 27 | msu = self.mean_squared_update[key] 28 | grad = param.grad 29 | msd *= self.rho 30 | msd += (1 - self.rho) * grad ** 2 31 | delta = np.sqrt((msu + self.epsilon) / (msd + self.epsilon)) * grad 32 | msu *= self.rho 33 | msu *= (1 - self.rho) * delta ** 2 34 | param.value += delta 35 | -------------------------------------------------------------------------------- /prml/nn/__init__.py: -------------------------------------------------------------------------------- 1 | from prml.nn.config import config 2 | from prml.nn.network import Network 3 | from prml.nn import array 4 | from prml.nn import io 5 | from prml.nn import loss 6 | from prml.nn import optimizer 7 | from prml.nn import random 8 | from prml.nn.array.array import array, asarray 9 | from prml.nn.array.ones import ones 10 | from prml.nn.array.reshape import reshape 11 | from prml.nn.array.zeros import zeros 12 | from prml.nn.distribution.bernoulli import Bernoulli 13 | from prml.nn.distribution.categorical import Categorical 14 | from prml.nn.distribution.gaussian import Gaussian, GaussianRadial 15 | from prml.nn.image.convolve2d import convolve2d 16 | from prml.nn.image.deconvolve2d import deconvolve2d 17 | from prml.nn.image.max_pooling2d import max_pooling2d 18 | from prml.nn.math.exp import exp 19 | from prml.nn.math.log import log 20 | from prml.nn.math.sqrt import sqrt 21 | from prml.nn.math.square import square 22 | from prml.nn.math.sum import sum 23 | from prml.nn.nonlinear.log_softmax import log_softmax 24 | from prml.nn.nonlinear.relu import relu 25 | from prml.nn.nonlinear.sigmoid import sigmoid 26 | from prml.nn.nonlinear.softmax import softmax 27 | from prml.nn.nonlinear.softplus import softplus 28 | from prml.nn.nonlinear.tanh import tanh 29 | from prml.nn.normalization.batch_normalization import BatchNormalization 30 | -------------------------------------------------------------------------------- /test/test_nn/test_distribution/test_gaussian.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestGaussian(unittest.TestCase): 7 | 8 | def test_gaussian_draw_forward(self): 9 | mu = nn.array(0) 10 | sigma = nn.softplus(nn.array(-1)) 11 | gaussian = nn.Gaussian(mu, sigma) 12 | sample = [] 13 | for _ in range(1000): 14 | sample.append(gaussian.draw().value) 15 | self.assertTrue(np.allclose(np.mean(sample), 0, rtol=0.1, atol=0.1), np.mean(sample)) 16 | self.assertTrue(np.allclose(np.std(sample), gaussian.std.value, 0.1, 0.1)) 17 | 18 | def test_gaussian_draw_backward(self): 19 | mu = nn.array(0) 20 | s = nn.array(2) 21 | optimizer = nn.optimizer.Gradient({0: mu, 1: s}, 0.01) 22 | prior = nn.Gaussian(1, 1) 23 | for _ in range(1000): 24 | mu.cleargrad() 25 | s.cleargrad() 26 | gaussian = nn.Gaussian(mu, nn.softplus(s)) 27 | gaussian.draw() 28 | loss = nn.loss.kl_divergence(gaussian, prior).sum() 29 | optimizer.minimize(loss) 30 | self.assertTrue(np.allclose(gaussian.mean.value, 1, 0.1, 0.1)) 31 | self.assertTrue(np.allclose(gaussian.std.value, 1, 0.1, 0.1)) 32 | 33 | 34 | if __name__ == "__main__": 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /prml/nn/random/random.py: -------------------------------------------------------------------------------- 1 | from prml.nn.function import Function 2 | 3 | 4 | class RandomVariable(Function): 5 | 6 | def __init__(self, data=None, p=None): 7 | if data is not None and p is not None: 8 | raise ValueError 9 | if data is not None: 10 | data = self._convert2array(data) 11 | self.data = data 12 | self.observed = (data is not None) 13 | self.p = p 14 | 15 | def draw(self): 16 | if self.observed: 17 | raise ValueError 18 | self.data = self.forward() 19 | return self.data 20 | 21 | def pdf(self, x=None): 22 | if x is not None: 23 | return self._pdf(x) 24 | if self.data is not None: 25 | return self._pdf(self.data) 26 | raise ValueError 27 | 28 | def _pdf(self, *args): 29 | raise NotImplementedError 30 | 31 | def log_pdf(self, x=None): 32 | if x is not None: 33 | return self._log_pdf(x) 34 | if self.data is not None: 35 | return self._log_pdf(self.data) 36 | raise ValueError 37 | 38 | def _log_pdf(self, *args): 39 | raise ValueError 40 | 41 | def KLqp(self): 42 | if self.p is None or self.data is None: 43 | raise ValueError 44 | return self._log_pdf(self.data) - self.p._log_pdf(self.data) 45 | -------------------------------------------------------------------------------- /prml/linear/least_squares_classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.classifier import Classifier 3 | from prml.preprocess.label_transformer import LabelTransformer 4 | 5 | 6 | class LeastSquaresClassifier(Classifier): 7 | """ 8 | Least squares classifier model 9 | 10 | X : (N, D) 11 | W : (D, K) 12 | y = argmax_k X @ W 13 | """ 14 | 15 | def __init__(self, W:np.ndarray=None): 16 | self.W = W 17 | 18 | def fit(self, X:np.ndarray, t:np.ndarray): 19 | """ 20 | least squares fitting for classification 21 | 22 | Parameters 23 | ---------- 24 | X : (N, D) np.ndarray 25 | training independent variable 26 | t : (N,) or (N, K) np.ndarray 27 | training dependent variable 28 | in class index (N,) or one-of-k coding (N,K) 29 | """ 30 | if t.ndim == 1: 31 | t = LabelTransformer().encode(t) 32 | self.W = np.linalg.pinv(X) @ t 33 | 34 | def classify(self, X:np.ndarray): 35 | """ 36 | classify input data 37 | 38 | Parameters 39 | ---------- 40 | X : (N, D) np.ndarray 41 | independent variable to be classified 42 | 43 | Returns 44 | ------- 45 | (N,) np.ndarray 46 | class index for each input 47 | """ 48 | return np.argmax(X @ self.W, axis=-1) 49 | -------------------------------------------------------------------------------- /prml/linear/perceptron.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.classifier import Classifier 3 | 4 | 5 | class Perceptron(Classifier): 6 | """ 7 | Perceptron model 8 | """ 9 | 10 | def fit(self, X, t, max_epoch=100): 11 | """ 12 | fit perceptron model on given input pair 13 | 14 | Parameters 15 | ---------- 16 | X : (N, D) np.ndarray 17 | training independent variable 18 | t : (N,) 19 | training dependent variable 20 | binary -1 or 1 21 | max_epoch : int, optional 22 | maximum number of epoch (the default is 100) 23 | """ 24 | self.w = np.zeros(np.size(X, 1)) 25 | for _ in range(max_epoch): 26 | X_error=X[np.sign(X@self.w)!=t] 27 | t_error=t[np.sign(X@self.w)!=t] 28 | idx=np.random.choice(len(X_error)) 29 | self.w+=X_error[idx]*t_error[idx] 30 | if (X@self.w*t>0).all(): 31 | break 32 | 33 | def classify(self, X): 34 | """ 35 | classify input data 36 | 37 | Parameters 38 | ---------- 39 | X : (N, D) np.ndarray 40 | independent variable to be classified 41 | 42 | Returns 43 | ------- 44 | (N,) np.ndarray 45 | binary class (-1 or 1) for each input 46 | """ 47 | return np.sign(X @ self.w).astype(np.int) 48 | -------------------------------------------------------------------------------- /docs/src/prml.markov.rst: -------------------------------------------------------------------------------- 1 | prml.markov package 2 | =================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.markov.categorical\_hmm module 8 | ----------------------------------- 9 | 10 | .. automodule:: prml.markov.categorical_hmm 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.markov.gaussian\_hmm module 16 | -------------------------------- 17 | 18 | .. automodule:: prml.markov.gaussian_hmm 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.markov.hmm module 24 | ---------------------- 25 | 26 | .. automodule:: prml.markov.hmm 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.markov.kalman module 32 | ------------------------- 33 | 34 | .. automodule:: prml.markov.kalman 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.markov.particle module 40 | --------------------------- 41 | 42 | .. automodule:: prml.markov.particle 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.markov.state\_space\_model module 48 | -------------------------------------- 49 | 50 | .. automodule:: prml.markov.state_space_model 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: prml.markov 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /prml/nn/math/product.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Product(Function): 6 | 7 | def __init__(self, axis=None, keepdims=False): 8 | if isinstance(axis, int): 9 | axis = (axis,) 10 | elif isinstance(axis, tuple): 11 | axis = tuple(sorted(axis)) 12 | self.axis = axis 13 | self.keepdims = keepdims 14 | 15 | def _forward(self, x): 16 | self.output = np.prod(x, axis=self.axis, keepdims=True) 17 | if not self.keepdims: 18 | return np.squeeze(self.output) 19 | else: 20 | return self.output 21 | 22 | def backward(self, delta, x): 23 | if not self.keepdims and self.axis is not None: 24 | for ax in self.axis: 25 | delta = np.expand_dims(delta, ax) 26 | dx = delta * self.output / x 27 | return dx 28 | 29 | 30 | def prod(x, axis=None, keepdims=False): 31 | """ 32 | product of all element in the array 33 | Parameters 34 | ---------- 35 | x : tensor_like 36 | input array 37 | axis : int, tuple of ints 38 | axis or axes along which a product is performed 39 | keepdims : bool 40 | keep dimensionality or not 41 | Returns 42 | ------- 43 | product : tensor_like 44 | product of all element 45 | """ 46 | return Product(axis=axis, keepdims=keepdims).forward(x) 47 | -------------------------------------------------------------------------------- /docs/_sources/prml.markov.rst.txt: -------------------------------------------------------------------------------- 1 | prml.markov package 2 | =================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.markov.categorical\_hmm module 8 | ----------------------------------- 9 | 10 | .. automodule:: prml.markov.categorical_hmm 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.markov.gaussian\_hmm module 16 | -------------------------------- 17 | 18 | .. automodule:: prml.markov.gaussian_hmm 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.markov.hmm module 24 | ---------------------- 25 | 26 | .. automodule:: prml.markov.hmm 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.markov.kalman module 32 | ------------------------- 33 | 34 | .. automodule:: prml.markov.kalman 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.markov.particle module 40 | --------------------------- 41 | 42 | .. automodule:: prml.markov.particle 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.markov.state\_space\_model module 48 | -------------------------------------- 49 | 50 | .. automodule:: prml.markov.state_space_model 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: prml.markov 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /prml/linear/linear_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.regression import Regression 3 | 4 | 5 | class LinearRegression(Regression): 6 | """ 7 | Linear regression model 8 | y = X @ w 9 | t ~ N(t|X @ w, var) 10 | """ 11 | 12 | def fit(self, X:np.ndarray, t:np.ndarray): 13 | """ 14 | perform least squares fitting 15 | 16 | Parameters 17 | ---------- 18 | X : (N, D) np.ndarray 19 | training independent variable 20 | t : (N,) np.ndarray 21 | training dependent variable 22 | """ 23 | self.w = np.linalg.pinv(X) @ t 24 | self.var = np.mean(np.square(X @ self.w - t)) 25 | 26 | def predict(self, X:np.ndarray, return_std:bool=False): 27 | """ 28 | make prediction given input 29 | 30 | Parameters 31 | ---------- 32 | X : (N, D) np.ndarray 33 | samples to predict their output 34 | return_std : bool, optional 35 | returns standard deviation of each predition if True 36 | 37 | Returns 38 | ------- 39 | y : (N,) np.ndarray 40 | prediction of each sample 41 | y_std : (N,) np.ndarray 42 | standard deviation of each predition 43 | """ 44 | y = X @ self.w 45 | if return_std: 46 | y_std = np.sqrt(self.var) + np.zeros_like(y) 47 | return y, y_std 48 | return y 49 | -------------------------------------------------------------------------------- /prml/nn/distribution/bernoulli.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.array.array import asarray 3 | from prml.nn.math.log import log 4 | from prml.nn.nonlinear.sigmoid import sigmoid 5 | from prml.nn.nonlinear.logit import logit as logit_func 6 | from prml.nn.distribution.distribution import Distribution 7 | from prml.nn.loss.sigmoid_cross_entropy import sigmoid_cross_entropy 8 | 9 | 10 | class Bernoulli(Distribution): 11 | is_categorical = True 12 | 13 | def __init__(self, mean=None, logit=None): 14 | super().__init__() 15 | if mean is not None: 16 | self.mean = asarray(mean) 17 | assert((self.mean.value >= 0).all() and (self.mean.value <= 1).all()) 18 | self.logit = logit_func(mean) 19 | self._log_pdf = self._log_pdf_mu 20 | elif logit is not None: 21 | self.mean = sigmoid(logit) 22 | self.logit = asarray(logit) 23 | self._log_pdf = self._log_pdf_logit 24 | else: 25 | raise ValueError 26 | 27 | def forward(self): 28 | binary = (np.random.uniform(size=self.mean.shape) < self.mean.value) 29 | return asarray(binary) 30 | 31 | def _pdf(self, x): 32 | return (self.mean ** x) * (1 - self.mean) ** (1 - x) 33 | 34 | def _log_pdf_mu(self, x): 35 | return x * log(self.mean) + (1 - x) * log(1 - self.mean) 36 | 37 | def _log_pdf_logit(self, x): 38 | return -sigmoid_cross_entropy(self.logit, x) 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | 92 | # VSCode project settings 93 | .vscode/ 94 | -------------------------------------------------------------------------------- /prml/nn/math/add.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class Add(Function): 6 | enable_auto_broadcast = True 7 | 8 | @staticmethod 9 | def _forward(x, y): 10 | return x + y 11 | 12 | @staticmethod 13 | def _backward(delta, x, y): 14 | return delta, delta 15 | 16 | 17 | class AddBias(Function): 18 | 19 | @staticmethod 20 | def _forward(x, y): 21 | return x + y 22 | 23 | @staticmethod 24 | def _backward(delta, x, y): 25 | dx = delta 26 | dy = np.sum(delta, axis=tuple(i for i in range(x.ndim - 1))) 27 | return dx, dy 28 | 29 | 30 | class AddScalar(Function): 31 | 32 | @staticmethod 33 | def _forward(x, y): 34 | return x + y 35 | 36 | @staticmethod 37 | def _backward(delta, x, y): 38 | dx = delta 39 | dy = np.atleast_1d(np.sum(delta)) 40 | return dx, dy 41 | 42 | 43 | def add(x, y): 44 | return Add().forward(x, y) 45 | # x = Function._convert2array(x) 46 | # y = Function._convert2array(y) 47 | # if x.shape == y.shape: 48 | # return Add().forward(x, y) 49 | # elif x.size == 1: 50 | # return AddScalar().forward(y, x) 51 | # elif y.size == 1: 52 | # return AddScalar().forward(x, y) 53 | # elif x.shape[-1] == y.shape[-1]: 54 | # if x.ndim == 1: 55 | # return AddBias().forward(y, x) 56 | # elif y.ndim == 1: 57 | # return AddBias().forward(x, y) 58 | # else: 59 | # raise ValueError 60 | -------------------------------------------------------------------------------- /test/test_nn/test_math/test_add.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestAdd(unittest.TestCase): 7 | 8 | def test_add(self): 9 | npa = np.random.randn(4, 5) 10 | npb = np.random.randn(4, 5) 11 | a = nn.asarray(npa) 12 | b = nn.asarray(npb) 13 | c = a + b 14 | self.assertTrue(np.allclose(c.value, npa + npb)) 15 | 16 | npg = np.random.randn(4, 5) 17 | c.backward(npg) 18 | self.assertTrue(np.allclose(a.grad, npg)) 19 | self.assertTrue(np.allclose(b.grad, npg)) 20 | 21 | def test_add_bias(self): 22 | npa = np.random.randn(4, 3) 23 | npb = np.random.randn(3) 24 | a = nn.asarray(npa) 25 | b = nn.asarray(npb) 26 | c = a + b 27 | self.assertTrue(np.allclose(c.value, npa + npb)) 28 | 29 | npg = np.random.randn(4, 3) 30 | c.backward(npg) 31 | self.assertTrue(np.allclose(a.grad, npg)) 32 | self.assertTrue(np.allclose(b.grad, npg.sum(axis=0))) 33 | 34 | def test_add_scalar(self): 35 | npa = np.random.randn(5, 6) 36 | npb = 2 37 | a = nn.asarray(npa) 38 | b = nn.asarray(npb) 39 | c = a + b 40 | self.assertTrue(np.allclose(c.value, npa + npb)) 41 | 42 | npg = np.random.randn(5, 6) 43 | c.backward(npg) 44 | self.assertTrue(np.allclose(a.grad, npg)) 45 | self.assertTrue(np.allclose(b.grad, np.sum(npg))) 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /prml/preprocess/polynomial.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import functools 3 | import numpy as np 4 | 5 | 6 | class PolynomialFeature(object): 7 | """ 8 | polynomial features 9 | 10 | transforms input array with polynomial features 11 | 12 | Example 13 | ======= 14 | x = 15 | [[a, b], 16 | [c, d]] 17 | 18 | y = PolynomialFeatures(degree=2).transform(x) 19 | y = 20 | [[1, a, b, a^2, a * b, b^2], 21 | [1, c, d, c^2, c * d, d^2]] 22 | """ 23 | 24 | def __init__(self, degree=2): 25 | """ 26 | construct polynomial features 27 | 28 | Parameters 29 | ---------- 30 | degree : int 31 | degree of polynomial 32 | """ 33 | assert isinstance(degree, int) 34 | self.degree = degree 35 | 36 | def transform(self, x): 37 | """ 38 | transforms input array with polynomial features 39 | 40 | Parameters 41 | ---------- 42 | x : (sample_size, n) ndarray 43 | input array 44 | 45 | Returns 46 | ------- 47 | output : (sample_size, 1 + nC1 + ... + nCd) ndarray 48 | polynomial features 49 | """ 50 | if x.ndim == 1: 51 | x = x[:, None] 52 | x_t = x.transpose() 53 | features = [np.ones(len(x))] 54 | for degree in range(1, self.degree + 1): 55 | for items in itertools.combinations_with_replacement(x_t, degree): 56 | features.append(functools.reduce(lambda x, y: x * y, items)) 57 | return np.asarray(features).transpose() 58 | -------------------------------------------------------------------------------- /prml/rv/dirichlet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import gamma 3 | from prml.rv.rv import RandomVariable 4 | 5 | 6 | class Dirichlet(RandomVariable): 7 | """ 8 | Dirichlet distribution 9 | p(mu|alpha) 10 | = gamma(sum(alpha)) 11 | * prod_k mu_k ^ (alpha_k - 1) 12 | / gamma(alpha_1) / ... / gamma(alpha_K) 13 | """ 14 | 15 | def __init__(self, alpha): 16 | """ 17 | construct dirichlet distribution 18 | 19 | Parameters 20 | ---------- 21 | alpha : (size,) np.ndarray 22 | pseudo count of each outcome, aka concentration parameter 23 | """ 24 | super().__init__() 25 | self.alpha = alpha 26 | 27 | @property 28 | def alpha(self): 29 | return self.parameter["alpha"] 30 | 31 | @alpha.setter 32 | def alpha(self, alpha): 33 | assert isinstance(alpha, np.ndarray) 34 | assert alpha.ndim == 1 35 | assert (alpha >= 0).all() 36 | self.parameter["alpha"] = alpha 37 | 38 | @property 39 | def ndim(self): 40 | return self.alpha.ndim 41 | 42 | @property 43 | def size(self): 44 | return self.alpha.size 45 | 46 | @property 47 | def shape(self): 48 | return self.alpha.shape 49 | 50 | def _pdf(self, mu): 51 | return ( 52 | gamma(self.alpha.sum()) 53 | * np.prod(mu ** (self.alpha - 1), axis=-1) 54 | / np.prod(gamma(self.alpha), axis=-1) 55 | ) 56 | 57 | def _draw(self, sample_size=1): 58 | return np.random.dirichlet(self.alpha, sample_size) 59 | -------------------------------------------------------------------------------- /test/test_nn/test_math/test_multiply.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestMultiply(unittest.TestCase): 7 | 8 | def test_multiply(self): 9 | npx = np.random.randn(5, 6) 10 | npy = np.random.randn(5, 6) 11 | x = nn.asarray(npx) 12 | y = nn.asarray(npy) 13 | z = x * y 14 | self.assertTrue(np.allclose(z.value, npx * npy)) 15 | 16 | npg = np.random.randn(5, 6) 17 | z.backward(npg) 18 | self.assertTrue(np.allclose(x.grad, npg * npy)) 19 | self.assertTrue(np.allclose(y.grad, npg * npx)) 20 | 21 | def test_multiply_vector(self): 22 | npx = np.random.randn(5, 6) 23 | npy = np.random.randn(6) 24 | x = nn.asarray(npx) 25 | y = nn.asarray(npy) 26 | z = x * y 27 | self.assertTrue(np.allclose(z.value, npx * npy)) 28 | 29 | npg = np.random.randn(5, 6) 30 | z.backward(npg) 31 | self.assertTrue(np.allclose(x.grad, npg * npy)) 32 | self.assertTrue(np.allclose(y.grad, np.sum(npg * npx, axis=0))) 33 | 34 | def test_multiply_scalar(self): 35 | npx = np.random.randn(5, 6) 36 | npy = 3.5 37 | x = nn.asarray(npx) 38 | y = nn.asarray(npy) 39 | z = x * y 40 | self.assertTrue(np.allclose(z.value, npx * npy)) 41 | 42 | npg = np.random.randn(5, 6) 43 | z.backward(npg) 44 | self.assertTrue(np.allclose(x.grad, npg * npy)) 45 | self.assertTrue(np.allclose(y.grad, np.sum(npg * npx))) 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | 51 | -------------------------------------------------------------------------------- /prml/preprocess/gaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class GaussianFeature(object): 5 | """ 6 | Gaussian feature 7 | 8 | gaussian function = exp(-0.5 * (x - m) / v) 9 | """ 10 | 11 | def __init__(self, mean, var): 12 | """ 13 | construct gaussian features 14 | 15 | Parameters 16 | ---------- 17 | mean : (n_features, ndim) or (n_features,) ndarray 18 | places to locate gaussian function at 19 | var : float 20 | variance of the gaussian function 21 | """ 22 | if mean.ndim == 1: 23 | mean = mean[:, None] 24 | else: 25 | assert mean.ndim == 2 26 | assert isinstance(var, float) or isinstance(var, int) 27 | self.mean = mean 28 | self.var = var 29 | 30 | def _gauss(self, x, mean): 31 | return np.exp(-0.5 * np.sum(np.square(x - mean), axis=-1) / self.var) 32 | 33 | def transform(self, x): 34 | """ 35 | transform input array with gaussian features 36 | 37 | Parameters 38 | ---------- 39 | x : (sample_size, ndim) or (sample_size,) 40 | input array 41 | 42 | Returns 43 | ------- 44 | output : (sample_size, n_features) 45 | gaussian features 46 | """ 47 | if x.ndim == 1: 48 | x = x[:, None] 49 | else: 50 | assert x.ndim == 2 51 | assert np.size(x, 1) == np.size(self.mean, 1) 52 | basis = [np.ones(len(x))] 53 | for m in self.mean: 54 | basis.append(self._gauss(x, m)) 55 | return np.asarray(basis).transpose() 56 | -------------------------------------------------------------------------------- /test/test_nn/test_distribution/test_bernoulli.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestBernoulli(unittest.TestCase): 7 | 8 | def test_bernoulli_kl(self): 9 | logit = nn.random.normal(0, 1, (5, 3)) 10 | mu = nn.random.uniform(0, 1, (5, 3)) 11 | q = nn.Bernoulli(mu) 12 | for _ in range(1000): 13 | logit.cleargrad() 14 | p = nn.Bernoulli(logit=logit) 15 | nn.loss.kl_divergence(p, q).backward() 16 | logit.value -= 0.1 * logit.grad 17 | self.assertTrue(np.allclose(p.mean.value, q.mean.value, 1e-2, 1e-2)) 18 | self.assertTrue(logit.value.dtype, nn.config.dtype) 19 | self.assertEqual(p.mean.value.dtype, nn.config.dtype) 20 | self.assertEqual(q.mean.value.dtype, nn.config.dtype) 21 | self.assertEqual(nn.config.dtype, np.float32) 22 | 23 | def test_bernoulli_mu(self): 24 | npmu = np.random.uniform(0, 1, (5, 3)) 25 | npt = np.random.uniform(0, 1, (5, 3)) 26 | p = nn.Bernoulli(npmu) 27 | log_likelihood = p.log_pdf(npt) 28 | self.assertTrue(np.allclose(log_likelihood.value, npt * np.log(npmu) + (1 - npt) * np.log(1 - npmu))) 29 | 30 | def test_bernoulli_logit(self): 31 | nplogit = np.random.randn(5, 3) 32 | npmu = np.tanh(nplogit * 0.5) * 0.5 + 0.5 33 | npt = np.random.uniform(0, 1, (5, 3)) 34 | p = nn.Bernoulli(logit=nplogit) 35 | log_likelihood = p.log_pdf(npt) 36 | self.assertTrue(np.allclose(log_likelihood.value, npt * np.log(npmu) + (1 - npt) * np.log(1 - npmu))) 37 | 38 | 39 | if __name__ == "__main__": 40 | unittest.main() 41 | -------------------------------------------------------------------------------- /docs/src/prml.nn.nonlinear.rst: -------------------------------------------------------------------------------- 1 | prml.nn.nonlinear package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.nonlinear.log\_softmax module 8 | ------------------------------------- 9 | 10 | .. automodule:: prml.nn.nonlinear.log_softmax 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.nonlinear.logit module 16 | ------------------------------ 17 | 18 | .. automodule:: prml.nn.nonlinear.logit 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.nonlinear.relu module 24 | ----------------------------- 25 | 26 | .. automodule:: prml.nn.nonlinear.relu 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.nonlinear.sigmoid module 32 | -------------------------------- 33 | 34 | .. automodule:: prml.nn.nonlinear.sigmoid 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.nonlinear.softmax module 40 | -------------------------------- 41 | 42 | .. automodule:: prml.nn.nonlinear.softmax 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.nonlinear.softplus module 48 | --------------------------------- 49 | 50 | .. automodule:: prml.nn.nonlinear.softplus 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.nn.nonlinear.tanh module 56 | ----------------------------- 57 | 58 | .. automodule:: prml.nn.nonlinear.tanh 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: prml.nn.nonlinear 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /prml/dimreduction/autoencoder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml import nn 3 | 4 | 5 | class Autoencoder(nn.Network): 6 | 7 | def __init__(self, *args): 8 | self.n_unit = len(args) 9 | super().__init__() 10 | for i in range(self.n_unit - 1): 11 | self.parameter[f"w_encode{i}"] = nn.Parameter(np.random.randn(args[i], args[i + 1])) 12 | self.parameter[f"b_encode{i}"] = nn.Parameter(np.zeros(args[i + 1])) 13 | self.parameter[f"w_decode{i}"] = nn.Parameter(np.random.randn(args[i + 1], args[i])) 14 | self.parameter[f"b_decode{i}"] = nn.Parameter(np.zeros(args[i])) 15 | 16 | def transform(self, x): 17 | h = x 18 | for i in range(self.n_unit - 1): 19 | h = nn.tanh(h @ self.parameter[f"w_encode{i}"] + self.parameter[f"b_encode{i}"]) 20 | return h.value 21 | 22 | def forward(self, x): 23 | h = x 24 | for i in range(self.n_unit - 1): 25 | h = nn.tanh(h @ self.parameter[f"w_encode{i}"] + self.parameter[f"b_encode{i}"]) 26 | for i in range(self.n_unit - 2, 0, -1): 27 | h = nn.tanh(h @ self.parameter[f"w_decode{i}"] + self.parameter[f"b_decode{i}"]) 28 | x_ = h @ self.parameter["w_decode0"] + self.parameter["b_decode0"] 29 | self.px = nn.random.Gaussian(x_, 1., data=x) 30 | 31 | def fit(self, x, n_iter=100, learning_rate=1e-3): 32 | optimizer = nn.optimizer.Adam(self.parameter, learning_rate) 33 | for _ in range(n_iter): 34 | self.clear() 35 | self.forward(x) 36 | log_likelihood = self.log_pdf() 37 | log_likelihood.backward() 38 | optimizer.update() 39 | -------------------------------------------------------------------------------- /prml/nn/loss/kl.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | from prml.nn.distribution.bernoulli import Bernoulli 4 | from prml.nn.distribution.categorical import Categorical 5 | from prml.nn.distribution.gaussian import Gaussian 6 | from prml.nn.math.log import log 7 | from prml.nn.math.square import square 8 | from prml.nn.nonlinear.log_softmax import log_softmax 9 | from prml.nn.nonlinear.softplus import softplus 10 | 11 | 12 | def kl_divergence(q, p, data=None): 13 | """ 14 | compute sample approximation of kl divergence from p to q 15 | KL(q||p) 16 | 17 | Parameters 18 | ---------- 19 | q : Distribution 20 | one generated a sample 21 | p : Distribution 22 | data : Array 23 | """ 24 | if isinstance(q, Bernoulli) and isinstance(p, Bernoulli): 25 | return kl_bernoulli(q, p) 26 | elif isinstance(q, Categorical) and isinstance(p, Categorical): 27 | return kl_categorical(q, p) 28 | elif isinstance(q, Gaussian) and isinstance(p, Gaussian): 29 | return kl_gaussian(q, p) 30 | elif q.data.depth > 0: 31 | return q.log_pdf() - p.log_pdf(q.data) 32 | else: 33 | return q.pdf() * (q.log_pdf() - p.log_pdf(q.data)) 34 | 35 | 36 | def kl_bernoulli(q, p): 37 | return (q.mean - 1) * (q.logit - p.logit) \ 38 | - softplus(-q.logit) + softplus(p.logit) 39 | 40 | 41 | def kl_categorical(q, p): 42 | return (q.mean * (log_softmax(q.logit) - log_softmax(p.logit))).sum(axis=-1) 43 | 44 | 45 | def kl_gaussian(q, p): 46 | qvar = square(q.std) 47 | pvar = square(p.std) 48 | return log(p.std) - log(q.std) + 0.5 * (qvar + square(p.mean - q.mean)) / pvar - 0.5 49 | 50 | 51 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.nonlinear.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.nonlinear package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.nonlinear.log\_softmax module 8 | ------------------------------------- 9 | 10 | .. automodule:: prml.nn.nonlinear.log_softmax 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.nonlinear.logit module 16 | ------------------------------ 17 | 18 | .. automodule:: prml.nn.nonlinear.logit 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.nonlinear.relu module 24 | ----------------------------- 25 | 26 | .. automodule:: prml.nn.nonlinear.relu 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.nonlinear.sigmoid module 32 | -------------------------------- 33 | 34 | .. automodule:: prml.nn.nonlinear.sigmoid 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.nonlinear.softmax module 40 | -------------------------------- 41 | 42 | .. automodule:: prml.nn.nonlinear.softmax 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.nonlinear.softplus module 48 | --------------------------------- 49 | 50 | .. automodule:: prml.nn.nonlinear.softplus 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.nn.nonlinear.tanh module 56 | ----------------------------- 57 | 58 | .. automodule:: prml.nn.nonlinear.tanh 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: prml.nn.nonlinear 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /prml/preprocess/label_transformer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class LabelTransformer(object): 5 | """ 6 | Label encoder decoder 7 | 8 | Attributes 9 | ---------- 10 | n_classes : int 11 | number of classes, K 12 | """ 13 | 14 | def __init__(self, n_classes:int=None): 15 | self.n_classes = n_classes 16 | 17 | @property 18 | def n_classes(self): 19 | return self.__n_classes 20 | 21 | @n_classes.setter 22 | def n_classes(self, K): 23 | self.__n_classes = K 24 | self.__encoder = None if K is None else np.eye(K) 25 | 26 | @property 27 | def encoder(self): 28 | return self.__encoder 29 | 30 | def encode(self, class_indices:np.ndarray): 31 | """ 32 | encode class index into one-of-k code 33 | 34 | Parameters 35 | ---------- 36 | class_indices : (N,) np.ndarray 37 | non-negative class index 38 | elements must be integer in [0, n_classes) 39 | 40 | Returns 41 | ------- 42 | (N, K) np.ndarray 43 | one-of-k encoding of input 44 | """ 45 | if self.n_classes is None: 46 | self.n_classes = np.max(class_indices) + 1 47 | 48 | return self.encoder[class_indices] 49 | 50 | def decode(self, onehot:np.ndarray): 51 | """ 52 | decode one-of-k code into class index 53 | 54 | Parameters 55 | ---------- 56 | onehot : (N, K) np.ndarray 57 | one-of-k code 58 | 59 | Returns 60 | ------- 61 | (N,) np.ndarray 62 | class index 63 | """ 64 | 65 | return np.argmax(onehot, axis=1) 66 | -------------------------------------------------------------------------------- /docs/src/prml.nn.optimizer.rst: -------------------------------------------------------------------------------- 1 | prml.nn.optimizer package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.optimizer.ada\_delta module 8 | ----------------------------------- 9 | 10 | .. automodule:: prml.nn.optimizer.ada_delta 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.optimizer.ada\_grad module 16 | ---------------------------------- 17 | 18 | .. automodule:: prml.nn.optimizer.ada_grad 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.optimizer.adam module 24 | ----------------------------- 25 | 26 | .. automodule:: prml.nn.optimizer.adam 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.optimizer.gradient module 32 | --------------------------------- 33 | 34 | .. automodule:: prml.nn.optimizer.gradient 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.optimizer.momentum module 40 | --------------------------------- 41 | 42 | .. automodule:: prml.nn.optimizer.momentum 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.optimizer.optimizer module 48 | ---------------------------------- 49 | 50 | .. automodule:: prml.nn.optimizer.optimizer 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.nn.optimizer.rmsprop module 56 | -------------------------------- 57 | 58 | .. automodule:: prml.nn.optimizer.rmsprop 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: prml.nn.optimizer 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.optimizer.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.optimizer package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.optimizer.ada\_delta module 8 | ----------------------------------- 9 | 10 | .. automodule:: prml.nn.optimizer.ada_delta 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.optimizer.ada\_grad module 16 | ---------------------------------- 17 | 18 | .. automodule:: prml.nn.optimizer.ada_grad 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.optimizer.adam module 24 | ----------------------------- 25 | 26 | .. automodule:: prml.nn.optimizer.adam 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.optimizer.gradient module 32 | --------------------------------- 33 | 34 | .. automodule:: prml.nn.optimizer.gradient 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.optimizer.momentum module 40 | --------------------------------- 41 | 42 | .. automodule:: prml.nn.optimizer.momentum 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.optimizer.optimizer module 48 | ---------------------------------- 49 | 50 | .. automodule:: prml.nn.optimizer.optimizer 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.nn.optimizer.rmsprop module 56 | -------------------------------- 57 | 58 | .. automodule:: prml.nn.optimizer.rmsprop 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: prml.nn.optimizer 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /prml/clustering/k_means.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.spatial.distance import cdist 3 | 4 | 5 | class KMeans(object): 6 | 7 | def __init__(self, n_clusters): 8 | self.n_clusters = n_clusters 9 | 10 | def fit(self, X, iter_max=100): 11 | """ 12 | perform k-means algorithm 13 | 14 | Parameters 15 | ---------- 16 | X : (sample_size, n_features) ndarray 17 | input data 18 | iter_max : int 19 | maximum number of iterations 20 | 21 | Returns 22 | ------- 23 | centers : (n_clusters, n_features) ndarray 24 | center of each cluster 25 | """ 26 | I = np.eye(self.n_clusters) 27 | centers = X[np.random.choice(len(X), self.n_clusters, replace=False)] 28 | for _ in range(iter_max): 29 | prev_centers = np.copy(centers) 30 | D = cdist(X, centers) 31 | cluster_index = np.argmin(D, axis=1) 32 | cluster_index = I[cluster_index] 33 | centers = np.sum(X[:, None, :] * cluster_index[:, :, None], axis=0) / np.sum(cluster_index, axis=0)[:, None] 34 | if np.allclose(prev_centers, centers): 35 | break 36 | self.centers = centers 37 | 38 | def predict(self, X): 39 | """ 40 | calculate closest cluster center index 41 | 42 | Parameters 43 | ---------- 44 | X : (sample_size, n_features) ndarray 45 | input data 46 | 47 | Returns 48 | ------- 49 | index : (sample_size,) ndarray 50 | indicates which cluster they belong 51 | """ 52 | D = cdist(X, self.centers) 53 | return np.argmin(D, axis=1) 54 | -------------------------------------------------------------------------------- /prml/nn/math/sum.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.function import Function 3 | 4 | 5 | class SumAxisOrKeepdims(Function): 6 | """ 7 | summation along given axis 8 | y = sum_i=1^N x_i 9 | """ 10 | 11 | def __init__(self, axis=None, keepdims=False): 12 | if isinstance(axis, int): 13 | axis = (axis,) 14 | self.axis = axis 15 | self.keepdims = keepdims 16 | 17 | def _forward(self, x): 18 | return x.sum(axis=self.axis, keepdims=self.keepdims) 19 | 20 | def _backward(self, delta, x): 21 | if isinstance(delta, np.ndarray) and (not self.keepdims) and (self.axis is not None): 22 | axis_positive = [] 23 | for axis in self.axis: 24 | if axis < 0: 25 | axis_positive.append(x.ndim + axis) 26 | else: 27 | axis_positive.append(axis) 28 | for axis in sorted(axis_positive): 29 | delta = np.expand_dims(delta, axis) 30 | dx = np.broadcast_to(delta, x.shape) 31 | return dx 32 | 33 | class SumSimple(Function): 34 | 35 | @staticmethod 36 | def _forward(x): 37 | return x.sum() 38 | 39 | @staticmethod 40 | def _backward(delta, x): 41 | return np.broadcast_to(delta, x.shape) 42 | 43 | 44 | def sum(x, axis=None, keepdims=False): 45 | """ 46 | returns summation of the elements along given axis 47 | y = sum_i=1^N x_i 48 | """ 49 | x = Function._convert2array(x) 50 | if x.ndim == 1: 51 | return SumAxisOrKeepdims(axis=axis, keepdims=True).forward(x) 52 | elif axis is None and keepdims == False: 53 | return SumSimple().forward(x) 54 | return SumAxisOrKeepdims(axis=axis, keepdims=keepdims).forward(x) 55 | 56 | -------------------------------------------------------------------------------- /prml/kernel/rbf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.kernel.kernel import Kernel 3 | 4 | 5 | class RBF(Kernel): 6 | 7 | def __init__(self, params): 8 | """ 9 | construct Radial basis kernel function 10 | 11 | Parameters 12 | ---------- 13 | params : (ndim + 1,) ndarray 14 | parameters of radial basis function 15 | 16 | Attributes 17 | ---------- 18 | ndim : int 19 | dimension of expected input data 20 | """ 21 | assert params.ndim == 1 22 | self.params = params 23 | self.ndim = len(params) - 1 24 | 25 | def __call__(self, x, y, pairwise=True): 26 | """ 27 | calculate radial basis function 28 | k(x, y) = c0 * exp(-0.5 * c1 * (x1 - y1) ** 2 ...) 29 | 30 | Parameters 31 | ---------- 32 | x : ndarray [..., ndim] 33 | input of this kernel function 34 | y : ndarray [..., ndim] 35 | another input 36 | 37 | Returns 38 | ------- 39 | output : ndarray 40 | output of this radial basis function 41 | """ 42 | assert x.shape[-1] == self.ndim 43 | assert y.shape[-1] == self.ndim 44 | if pairwise: 45 | x, y = self._pairwise(x, y) 46 | d = self.params[1:] * (x - y) ** 2 47 | return self.params[0] * np.exp(-0.5 * np.sum(d, axis=-1)) 48 | 49 | def derivatives(self, x, y, pairwise=True): 50 | if pairwise: 51 | x, y = self._pairwise(x, y) 52 | d = self.params[1:] * (x - y) ** 2 53 | delta = np.exp(-0.5 * np.sum(d, axis=-1)) 54 | deltas = -0.5 * (x - y) ** 2 * (delta * self.params[0])[:, :, None] 55 | return np.concatenate((np.expand_dims(delta, 0), deltas.T)) 56 | 57 | def update_parameters(self, updates): 58 | self.params += updates 59 | -------------------------------------------------------------------------------- /test/test_nn/test_backward.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import prml.nn as nn 4 | 5 | 6 | class TestBackward(unittest.TestCase): 7 | 8 | def test_backward(self): 9 | def sigmoid(x): 10 | return np.tanh(x * 0.5) * 0.5 + 0.5 11 | 12 | x_train = nn.asarray(np.array([[0, 0], [0, 1], [1, 0], [1, 1]])) 13 | y_train = nn.asarray(np.array([[0], [1], [1], [0]])) 14 | w1 = nn.asarray(np.random.normal(0, 1, (2, 3))) 15 | b1 = nn.asarray(np.zeros(3)) 16 | w2 = nn.asarray(np.random.normal(0, 1, (3, 1))) 17 | b2 = nn.asarray(np.zeros(1)) 18 | parameter = [w1, b1, w2, b2] 19 | for _ in range(10): 20 | for param in parameter: 21 | param.cleargrad() 22 | x_train.cleargrad() 23 | h = x_train @ w1 + b1 24 | a = nn.tanh(h) 25 | logit = a @ w2 + b2 26 | loss = nn.loss.sigmoid_cross_entropy(logit, y_train) 27 | loss.backward() 28 | dlogit = sigmoid(logit.value) - y_train.value 29 | db2 = np.sum(dlogit, axis=0) 30 | dw2 = a.value.T @ dlogit 31 | da = dlogit @ w2.value.T 32 | dh = da * (1 - a.value ** 2) 33 | db1 = np.sum(dh, axis=0) 34 | dw1 = x_train.value.T @ dh 35 | dx = dh @ w1.value.T 36 | self.assertTrue(np.allclose(logit.grad, dlogit)) 37 | self.assertTrue(np.allclose(b2.grad, db2)) 38 | self.assertTrue(np.allclose(w2.grad, dw2)) 39 | self.assertTrue(np.allclose(a.grad, da)) 40 | self.assertTrue(np.allclose(h.grad, dh)) 41 | self.assertTrue(np.allclose(b1.grad, db1)) 42 | self.assertTrue(np.allclose(w1.grad, dw1)) 43 | self.assertTrue(np.allclose(x_train.grad, dx)) 44 | 45 | 46 | if __name__ == "__main__": 47 | unittest.main() 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PRML 2 | Python codes implementing algorithms described in Bishop's book "Pattern Recognition and Machine Learning" 3 | 4 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/cranmer/PRML/master) 5 | 6 | ## Required Packages 7 | - python 3 8 | - numpy 9 | - scipy 10 | - jupyter (optional: to run jupyter notebooks) 11 | - matplotlib (optional: to plot results in the notebooks) 12 | - sklearn (optional: to fetch data) 13 | 14 | ## Notebooks 15 | - [ch1. Introduction](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch01_Introduction.ipynb) 16 | - [ch2. Probability Distributions](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch02_Probability_Distributions.ipynb) 17 | - [ch3. Linear Models for Regression](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch03_Linear_Models_for_Regression.ipynb) 18 | - [ch4. Linear Models for Classification](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch04_Linear_Models_for_Classfication.ipynb) 19 | - [ch5. Neural Networks](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch05_Neural_Networks.ipynb) 20 | - [ch6. Kernel Methods](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch06_Kernel_Methods.ipynb) 21 | - [ch7. Sparse Kernel Machines](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch07_Sparse_Kernel_Machines.ipynb) 22 | - [ch9. Mixture Models and EM](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch09_Mixture_Models_and_EM.ipynb) 23 | - [ch10. Approximate Inference](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch10_Approximate_Inference.ipynb) 24 | - [ch11. Sampling Methods](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch11_Sampling_Methods.ipynb) 25 | - [ch12. Continuous Latent Variables](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch12_Continuous_Latent_Variables.ipynb) 26 | -------------------------------------------------------------------------------- /prml/preprocess/sigmoidal.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class SigmoidalFeature(object): 5 | """ 6 | Sigmoidal features 7 | 8 | 1 / (1 + exp((m - x) @ c) 9 | """ 10 | 11 | def __init__(self, mean, coef=1): 12 | """ 13 | construct sigmoidal features 14 | 15 | Parameters 16 | ---------- 17 | mean : (n_features, ndim) or (n_features,) ndarray 18 | center of sigmoid function 19 | coef : (ndim,) ndarray or int or float 20 | coefficient to be multplied with the distance 21 | """ 22 | if mean.ndim == 1: 23 | mean = mean[:, None] 24 | else: 25 | assert mean.ndim == 2 26 | if isinstance(coef, int) or isinstance(coef, float): 27 | if np.size(mean, 1) == 1: 28 | coef = np.array([coef]) 29 | else: 30 | raise ValueError("mismatch of dimension") 31 | else: 32 | assert coef.ndim == 1 33 | assert np.size(mean, 1) == len(coef) 34 | self.mean = mean 35 | self.coef = coef 36 | 37 | def _sigmoid(self, x, mean): 38 | return np.tanh((x - mean) @ self.coef * 0.5) * 0.5 + 0.5 39 | 40 | def transform(self, x): 41 | """ 42 | transform input array with sigmoidal features 43 | 44 | Parameters 45 | ---------- 46 | x : (sample_size, ndim) or (sample_size,) ndarray 47 | input array 48 | 49 | Returns 50 | ------- 51 | output : (sample_size, n_features) ndarray 52 | sigmoidal features 53 | """ 54 | if x.ndim == 1: 55 | x = x[:, None] 56 | else: 57 | assert x.ndim == 2 58 | assert np.size(x, 1) == np.size(self.mean, 1) 59 | basis = [np.ones(len(x))] 60 | for m in self.mean: 61 | basis.append(self._sigmoid(x, m)) 62 | return np.asarray(basis).transpose() 63 | -------------------------------------------------------------------------------- /prml/rv/uniform.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.rv.rv import RandomVariable 3 | 4 | 5 | class Uniform(RandomVariable): 6 | """ 7 | Uniform distribution 8 | p(x|a, b) 9 | = 1 / ((b_0 - a_0) * (b_1 - a_1)) if a <= x <= b else 0 10 | """ 11 | 12 | def __init__(self, low, high): 13 | """ 14 | construct uniform distribution 15 | 16 | Parameters 17 | ---------- 18 | low : int, float, or np.ndarray 19 | lower boundary 20 | high : int, float, or np.ndarray 21 | higher boundary 22 | """ 23 | super().__init__() 24 | low = np.asarray(low) 25 | high = np.asarray(high) 26 | assert low.shape == high.shape 27 | assert (low <= high).all() 28 | self.low = low 29 | self.high = high 30 | self.value = 1 / np.prod(high - low) 31 | 32 | @property 33 | def low(self): 34 | return self.parameter["low"] 35 | 36 | @low.setter 37 | def low(self, low): 38 | self.parameter["low"] = low 39 | 40 | @property 41 | def high(self): 42 | return self.parameter["high"] 43 | 44 | @high.setter 45 | def high(self, high): 46 | self.parameter["high"] = high 47 | 48 | @property 49 | def ndim(self): 50 | return self.low.ndim 51 | 52 | @property 53 | def size(self): 54 | return self.low.size 55 | 56 | @property 57 | def shape(self): 58 | return self.low.shape 59 | 60 | @property 61 | def mean(self): 62 | return 0.5 * (self.low + self.high) 63 | 64 | def _pdf(self, X): 65 | higher = np.logical_and.reduce(X >= self.low, 1) 66 | lower = np.logical_and.reduce(X <= self.high, 1) 67 | return self.value * np.logical_and(higher, lower) 68 | 69 | def _draw(self, sample_size=1): 70 | u01 = np.random.uniform(size=(sample_size,) + self.shape) 71 | return u01 * (self.high - self.low) + self.low 72 | -------------------------------------------------------------------------------- /docs/src/prml.kernel.rst: -------------------------------------------------------------------------------- 1 | prml.kernel package 2 | =================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.kernel.gaussian\_process\_classifier module 8 | ------------------------------------------------ 9 | 10 | .. automodule:: prml.kernel.gaussian_process_classifier 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.kernel.gaussian\_process\_regressor module 16 | ----------------------------------------------- 17 | 18 | .. automodule:: prml.kernel.gaussian_process_regressor 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.kernel.kernel module 24 | ------------------------- 25 | 26 | .. automodule:: prml.kernel.kernel 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.kernel.polynomial module 32 | ----------------------------- 33 | 34 | .. automodule:: prml.kernel.polynomial 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.kernel.rbf module 40 | ---------------------- 41 | 42 | .. automodule:: prml.kernel.rbf 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.kernel.relevance\_vector\_classifier module 48 | ------------------------------------------------ 49 | 50 | .. automodule:: prml.kernel.relevance_vector_classifier 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.kernel.relevance\_vector\_regressor module 56 | ----------------------------------------------- 57 | 58 | .. automodule:: prml.kernel.relevance_vector_regressor 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | prml.kernel.support\_vector\_classifier module 64 | ---------------------------------------------- 65 | 66 | .. automodule:: prml.kernel.support_vector_classifier 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | 72 | Module contents 73 | --------------- 74 | 75 | .. automodule:: prml.kernel 76 | :members: 77 | :undoc-members: 78 | :show-inheritance: 79 | -------------------------------------------------------------------------------- /docs/_sources/prml.kernel.rst.txt: -------------------------------------------------------------------------------- 1 | prml.kernel package 2 | =================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.kernel.gaussian\_process\_classifier module 8 | ------------------------------------------------ 9 | 10 | .. automodule:: prml.kernel.gaussian_process_classifier 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.kernel.gaussian\_process\_regressor module 16 | ----------------------------------------------- 17 | 18 | .. automodule:: prml.kernel.gaussian_process_regressor 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.kernel.kernel module 24 | ------------------------- 25 | 26 | .. automodule:: prml.kernel.kernel 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.kernel.polynomial module 32 | ----------------------------- 33 | 34 | .. automodule:: prml.kernel.polynomial 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.kernel.rbf module 40 | ---------------------- 41 | 42 | .. automodule:: prml.kernel.rbf 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.kernel.relevance\_vector\_classifier module 48 | ------------------------------------------------ 49 | 50 | .. automodule:: prml.kernel.relevance_vector_classifier 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.kernel.relevance\_vector\_regressor module 56 | ----------------------------------------------- 57 | 58 | .. automodule:: prml.kernel.relevance_vector_regressor 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | prml.kernel.support\_vector\_classifier module 64 | ---------------------------------------------- 65 | 66 | .. automodule:: prml.kernel.support_vector_classifier 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | 72 | Module contents 73 | --------------- 74 | 75 | .. automodule:: prml.kernel 76 | :members: 77 | :undoc-members: 78 | :show-inheritance: 79 | -------------------------------------------------------------------------------- /test/test_nn/test_image/test_convolve2d.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | from scipy.ndimage.filters import correlate 4 | import prml.nn as nn 5 | 6 | 7 | class TestConvolve2d(unittest.TestCase): 8 | 9 | def test_convolve2d_forward(self): 10 | img = np.random.randn(1, 5, 5, 1) 11 | kernel = np.random.randn(3, 3, 1, 1) 12 | output = nn.convolve2d(img, kernel) 13 | self.assertTrue( 14 | np.allclose( 15 | output.value[0, ..., 0], 16 | correlate(img[0, ..., 0], kernel[..., 0, 0])[1:-1, 1:-1] 17 | ) 18 | ) 19 | self.assertEqual(nn.config.dtype, np.float32) 20 | self.assertEqual(output.value.dtype, nn.config.dtype) 21 | 22 | def test_convolve2d_backward(self): 23 | x = nn.random.normal(0, 1, (1, 5, 5, 1)) 24 | w = nn.random.normal(0, 1, (3, 3, 1, 1)) 25 | for _ in range(1000): 26 | x.cleargrad() 27 | w.cleargrad() 28 | output = nn.convolve2d(x, w, (2, 2), (1, 1)) 29 | output.backward(2 * (output.value - 1)) 30 | x.value -= x.grad * 0.01 31 | w.value -= w.grad * 0.01 32 | self.assertTrue(np.allclose(output.value, 1)) 33 | self.assertEqual(nn.config.dtype, np.float32) 34 | self.assertEqual(x.dtype, nn.config.dtype) 35 | self.assertEqual(w.dtype, nn.config.dtype) 36 | self.assertEqual(output.dtype, nn.config.dtype) 37 | 38 | def test_convolve2d_network(self): 39 | x = nn.random.normal(0, 1, (1, 5, 5, 1)) 40 | kernel = nn.random.normal(0, 1, (3, 3, 1, 1)) 41 | conv = nn.image.Convolve2d(kernel, (1, 1), (0, 0)) 42 | for _ in range(1000): 43 | x.cleargrad() 44 | conv.clear() 45 | output = conv(x) 46 | output.backward(2 * (output.value - 1)) 47 | x.value -= x.grad * 0.01 48 | for param in conv.parameter.values(): 49 | param.value -= param.grad * 0.01 50 | self.assertTrue(np.allclose(output.value, 1)) 51 | 52 | 53 | if __name__ == "__main__": 54 | unittest.main() 55 | -------------------------------------------------------------------------------- /prml/linear/bayesian_logistic_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.logistic_regression import LogisticRegression 3 | 4 | 5 | class BayesianLogisticRegression(LogisticRegression): 6 | """ 7 | Logistic regression model 8 | 9 | w ~ Gaussian(0, alpha^(-1)I) 10 | y = sigmoid(X @ w) 11 | t ~ Bernoulli(t|y) 12 | """ 13 | 14 | def __init__(self, alpha:float=1.): 15 | self.alpha = alpha 16 | 17 | def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100): 18 | """ 19 | bayesian estimation of logistic regression model 20 | using Laplace approximation 21 | 22 | Parameters 23 | ---------- 24 | X : (N, D) np.ndarray 25 | training data independent variable 26 | t : (N,) np.ndarray 27 | training data dependent variable 28 | binary 0 or 1 29 | max_iter : int, optional 30 | maximum number of paramter update iteration (the default is 100) 31 | """ 32 | w = np.zeros(np.size(X, 1)) 33 | eye = np.eye(np.size(X, 1)) 34 | self.w_mean = np.copy(w) 35 | self.w_precision = self.alpha * eye 36 | for _ in range(max_iter): 37 | w_prev = np.copy(w) 38 | y = self._sigmoid(X @ w) 39 | grad = X.T @ (y - t) + self.w_precision @ (w - self.w_mean) 40 | hessian = (X.T * y * (1 - y)) @ X + self.w_precision 41 | try: 42 | w -= np.linalg.solve(hessian, grad) 43 | except np.linalg.LinAlgError: 44 | break 45 | if np.allclose(w, w_prev): 46 | break 47 | self.w_mean = w 48 | self.w_precision = hessian 49 | 50 | def proba(self, X:np.ndarray): 51 | """ 52 | compute probability of input belonging class 1 53 | 54 | Parameters 55 | ---------- 56 | X : (N, D) np.ndarray 57 | training data independent variable 58 | 59 | Returns 60 | ------- 61 | (N,) np.ndarray 62 | probability of positive 63 | """ 64 | mu_a = X @ self.w_mean 65 | var_a = np.sum(np.linalg.solve(self.w_precision, X.T).T * X, axis=1) 66 | return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8)) 67 | -------------------------------------------------------------------------------- /prml/nn/distribution/gaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.special as sp 3 | from prml.nn.array.array import asarray 4 | from prml.nn.distribution.distribution import Distribution 5 | from prml.nn.function import Function 6 | from prml.nn.math.log import log 7 | from prml.nn.math.sqrt import sqrt 8 | from prml.nn.math.square import square 9 | from prml.nn.random.normal import normal 10 | 11 | 12 | class Gaussian(Distribution): 13 | 14 | def __init__(self, mean, std): 15 | super().__init__() 16 | self.mean = asarray(mean) 17 | self.std = asarray(std) 18 | assert((self.std.value > 0).all()) 19 | 20 | def forward(self): 21 | if np.prod(self.mean.shape) > np.prod(self.std.shape): 22 | eps = normal(0, 1, self.mean.shape) 23 | else: 24 | eps = normal(0, 1, self.std.shape) 25 | return self.mean + self.std * eps 26 | 27 | def _log_pdf(self, x): 28 | return GaussianLogPDF().forward(x, self.mean, self.std) 29 | 30 | 31 | class GaussianLogPDF(Function): 32 | enable_auto_broadcast = True 33 | log2pi = np.log(2 * np.pi) 34 | 35 | def _forward(self, x, mean, std): 36 | self.mahalanobis_distance = np.square((x - mean) / std) 37 | log_pdf = -0.5 * ( 38 | self.mahalanobis_distance 39 | + 2 * np.log(std) 40 | + self.log2pi 41 | ) 42 | return log_pdf 43 | 44 | def _backward(self, delta, x, mean, std): 45 | dx = -0.5 * delta * (x - mean) / np.square(std) 46 | dmean = -dx 47 | dstd = (self.mahalanobis_distance - 1) / std 48 | return dx, dmean, dstd 49 | 50 | 51 | class GaussianRadial(Distribution): 52 | 53 | def __init__(self, std, ndim: int): 54 | super().__init__() 55 | self.std = asarray(std) 56 | self.ndim = ndim 57 | assert((self.std.value >= 0).all()) 58 | 59 | def forward(self): 60 | eps = normal(0, 1, (self.ndim,) + self.std.shape) 61 | return sqrt(square(self.std * eps).sum(axis=0)) 62 | 63 | def _log_pdf(self, x): 64 | return ( 65 | (self.ndim - 1) * log(x) 66 | - 0.5 * square(x / self.std) 67 | - self.ndim * log(self.std) 68 | - np.log(sp.gamma(0.5 * self.ndim)) 69 | ) 70 | -------------------------------------------------------------------------------- /prml/rv/beta.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import gamma 3 | from prml.rv.rv import RandomVariable 4 | 5 | 6 | np.seterr(all="ignore") 7 | 8 | 9 | class Beta(RandomVariable): 10 | """ 11 | Beta distribution 12 | p(mu|n_ones, n_zeros) 13 | = gamma(n_ones + n_zeros) 14 | * mu^(n_ones - 1) * (1 - mu)^(n_zeros - 1) 15 | / gamma(n_ones) / gamma(n_zeros) 16 | """ 17 | 18 | def __init__(self, n_zeros, n_ones): 19 | """ 20 | construct beta distribution 21 | 22 | Parameters 23 | ---------- 24 | n_zeros : int, float, or np.ndarray 25 | pseudo count of zeros 26 | n_ones : int, float, or np.ndarray 27 | pseudo count of ones 28 | """ 29 | super().__init__() 30 | if not isinstance(n_ones, (int, float, np.number, np.ndarray)): 31 | raise ValueError( 32 | "{} is not supported for n_ones" 33 | .format(type(n_ones)) 34 | ) 35 | if not isinstance(n_zeros, (int, float, np.number, np.ndarray)): 36 | raise ValueError( 37 | "{} is not supported for n_zeros" 38 | .format(type(n_zeros)) 39 | ) 40 | n_ones = np.asarray(n_ones) 41 | n_zeros = np.asarray(n_zeros) 42 | if n_ones.shape != n_zeros.shape: 43 | raise ValueError( 44 | "the sizes of the arrays don't match: {}, {}" 45 | .format(n_ones.shape, n_zeros.shape) 46 | ) 47 | self.n_ones = n_ones 48 | self.n_zeros = n_zeros 49 | 50 | @property 51 | def ndim(self): 52 | return self.n_ones.ndim 53 | 54 | @property 55 | def size(self): 56 | return self.n_ones.size 57 | 58 | @property 59 | def shape(self): 60 | return self.n_ones.shape 61 | 62 | def _pdf(self, mu): 63 | return ( 64 | gamma(self.n_ones + self.n_zeros) 65 | * np.power(mu, self.n_ones - 1) 66 | * np.power(1 - mu, self.n_zeros - 1) 67 | / gamma(self.n_ones) 68 | / gamma(self.n_zeros) 69 | ) 70 | 71 | def _draw(self, sample_size=1): 72 | return np.random.beta( 73 | self.n_ones, self.n_zeros, size=(sample_size,) + self.shape 74 | ) 75 | -------------------------------------------------------------------------------- /prml/nn/normalization/batch_normalization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.array.ones import ones 3 | from prml.nn.array.zeros import zeros 4 | from prml.nn.config import config 5 | from prml.nn.function import Function 6 | from prml.nn.network import Network 7 | 8 | 9 | class BatchNormalizationFunction(Function): 10 | 11 | def _forward(self, x): 12 | self.mean = x.mean(axis=0) 13 | self.xc = x - self.mean 14 | self.var = np.mean(self.xc ** 2, axis=0) 15 | self.std = np.sqrt(self.var + 1e-7) 16 | return self.xc / self.std 17 | 18 | def _backward(self, delta, x): 19 | # dstd = -np.mean((delta * self.xc) / (self.std ** 2), axis=0) 20 | dxc = delta / self.std - self.xc * np.mean((delta * self.xc) / (self.std ** 3), axis=0) 21 | return dxc - np.mean(dxc, axis=0) 22 | 23 | # dstd = -np.mean((delta * self.xc) / (self.std ** 2), axis=0) 24 | # dxc = delta / self.std + self.xc * dstd / self.std 25 | # return dxc - np.mean(dxc, axis=0) 26 | 27 | # dxn = delta 28 | # dxc = dxn / self.std 29 | # dstd = -np.sum((dxn * self.xc) / (self.std ** 2), axis=0) 30 | # dvar = 0.5 * dstd / self.std 31 | # dxc += 2.0 * self.xc * dvar / delta.shape[0] 32 | # dmu = np.sum(dxc, axis=0) 33 | # dx = dxc - dmu / delta.shape[0] 34 | # return dx 35 | 36 | 37 | class BatchNormalization(Network): 38 | 39 | def __init__(self, ndim, scale=None, bias=None, momentum=0.9): 40 | super().__init__() 41 | self.momentum = momentum 42 | with self.set_parameter(): 43 | self.mean = zeros(ndim) 44 | self.var = ones(ndim) 45 | 46 | def __call__(self, x): 47 | shape = x.shape 48 | x = x.reshape(-1, x.shape[-1]) 49 | if config.is_updating_bn: 50 | func = BatchNormalizationFunction() 51 | out = func.forward(x) 52 | self.mean.value = self.momentum * self.mean.value + (1 - self.momentum) * func.mean 53 | self.var.value = self.momentum * self.var.value + (1 - self.momentum) * func.var 54 | del func.mean 55 | del func.var 56 | else: 57 | xc = x - self.mean 58 | out = xc / np.sqrt(self.var.value + 1e-7) 59 | return out.reshape(*shape) 60 | -------------------------------------------------------------------------------- /prml/linear/logistic_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.classifier import Classifier 3 | 4 | 5 | class LogisticRegression(Classifier): 6 | """ 7 | Logistic regression model 8 | 9 | y = sigmoid(X @ w) 10 | t ~ Bernoulli(t|y) 11 | """ 12 | 13 | @staticmethod 14 | def _sigmoid(a): 15 | return np.tanh(a * 0.5) * 0.5 + 0.5 16 | 17 | def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100): 18 | """ 19 | maximum likelihood estimation of logistic regression model 20 | 21 | Parameters 22 | ---------- 23 | X : (N, D) np.ndarray 24 | training data independent variable 25 | t : (N,) np.ndarray 26 | training data dependent variable 27 | binary 0 or 1 28 | max_iter : int, optional 29 | maximum number of paramter update iteration (the default is 100) 30 | """ 31 | w = np.zeros(np.size(X, 1)) 32 | for _ in range(max_iter): 33 | w_prev = np.copy(w) 34 | y = self._sigmoid(X @ w) 35 | grad = X.T @ (y - t) 36 | hessian = (X.T * y * (1 - y)) @ X 37 | try: 38 | w -= np.linalg.solve(hessian, grad) 39 | except np.linalg.LinAlgError: 40 | break 41 | if np.allclose(w, w_prev): 42 | break 43 | self.w = w 44 | 45 | def proba(self, X:np.ndarray): 46 | """ 47 | compute probability of input belonging class 1 48 | 49 | Parameters 50 | ---------- 51 | X : (N, D) np.ndarray 52 | training data independent variable 53 | 54 | Returns 55 | ------- 56 | (N,) np.ndarray 57 | probability of positive 58 | """ 59 | return self._sigmoid(X @ self.w) 60 | 61 | def classify(self, X:np.ndarray, threshold:float=0.5): 62 | """ 63 | classify input data 64 | 65 | Parameters 66 | ---------- 67 | X : (N, D) np.ndarray 68 | independent variable to be classified 69 | threshold : float, optional 70 | threshold of binary classification (default is 0.5) 71 | 72 | Returns 73 | ------- 74 | (N,) np.ndarray 75 | binary class for each input 76 | """ 77 | return (self.proba(X) > threshold).astype(np.int) 78 | -------------------------------------------------------------------------------- /prml/dimreduction/bayesian_pca.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.dimreduction.pca import PCA 3 | 4 | 5 | class BayesianPCA(PCA): 6 | 7 | def fit(self, X, iter_max=100, initial="random"): 8 | """ 9 | empirical bayes estimation of pca parameters 10 | 11 | Parameters 12 | ---------- 13 | X : (sample_size, n_features) ndarray 14 | input data 15 | iter_max : int 16 | maximum number of em steps 17 | 18 | Returns 19 | ------- 20 | mean : (n_features,) ndarray 21 | sample mean fo the input data 22 | W : (n_features, n_components) ndarray 23 | projection matrix 24 | var : float 25 | variance of observation noise 26 | """ 27 | initial_list = ["random", "eigen"] 28 | self.mean = np.mean(X, axis=0) 29 | self.I = np.eye(self.n_components) 30 | if initial not in initial_list: 31 | print("availabel initializations are {}".format(initial_list)) 32 | if initial == "random": 33 | self.W = np.eye(np.size(X, 1), self.n_components) 34 | self.var = 1. 35 | elif initial == "eigen": 36 | self.eigen(X) 37 | self.alpha = len(self.mean) / np.sum(self.W ** 2, axis=0).clip(min=1e-10) 38 | for i in range(iter_max): 39 | W = np.copy(self.W) 40 | stats = self._expectation(X - self.mean) 41 | self._maximization(X - self.mean, *stats) 42 | self.alpha = len(self.mean) / np.sum(self.W ** 2, axis=0).clip(min=1e-10) 43 | if np.allclose(W, self.W): 44 | break 45 | self.n_iter = i + 1 46 | 47 | def _maximization(self, X, Ez, Ezz): 48 | self.W = X.T @ Ez @ np.linalg.inv(np.sum(Ezz, axis=0) + self.var * np.diag(self.alpha)) 49 | self.var = np.mean( 50 | np.mean(X ** 2, axis=-1) 51 | - 2 * np.mean(Ez @ self.W.T * X, axis=-1) 52 | + np.trace((Ezz @ self.W.T @ self.W).T) / len(self.mean)) 53 | 54 | def maximize(self, D, Ez, Ezz): 55 | self.W = D.T.dot(Ez).dot(np.linalg.inv(np.sum(Ezz, axis=0) + self.var * np.diag(self.alpha))) 56 | self.var = np.mean( 57 | np.mean(D ** 2, axis=-1) 58 | - 2 * np.mean(Ez.dot(self.W.T) * D, axis=-1) 59 | + np.trace(Ezz.dot(self.W.T).dot(self.W).T) / self.ndim) 60 | -------------------------------------------------------------------------------- /prml/linear/fishers_linear_discriminant.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.classifier import Classifier 3 | from prml.rv.gaussian import Gaussian 4 | 5 | 6 | class FishersLinearDiscriminant(Classifier): 7 | """ 8 | Fisher's Linear discriminant model 9 | """ 10 | 11 | def __init__(self, w:np.ndarray=None, threshold:float=None): 12 | self.w = w 13 | self.threshold = threshold 14 | 15 | def fit(self, X:np.ndarray, t:np.ndarray): 16 | """ 17 | estimate parameter given training dataset 18 | 19 | Parameters 20 | ---------- 21 | X : (N, D) np.ndarray 22 | training dataset independent variable 23 | t : (N,) np.ndarray 24 | training dataset dependent variable 25 | binary 0 or 1 26 | """ 27 | X0 = X[t == 0] 28 | X1 = X[t == 1] 29 | m0 = np.mean(X0, axis=0) 30 | m1 = np.mean(X1, axis=0) 31 | cov_inclass = np.cov(X0, rowvar=False) + np.cov(X1, rowvar=False) 32 | self.w = np.linalg.solve(cov_inclass, m1 - m0) 33 | self.w /= np.linalg.norm(self.w).clip(min=1e-10) 34 | 35 | g0 = Gaussian() 36 | g0.fit((X0 @ self.w)) 37 | g1 = Gaussian() 38 | g1.fit((X1 @ self.w)) 39 | root = np.roots([ 40 | g1.var - g0.var, 41 | 2 * (g0.var * g1.mu - g1.var * g0.mu), 42 | g1.var * g0.mu ** 2 - g0.var * g1.mu ** 2 43 | - g1.var * g0.var * np.log(g1.var / g0.var) 44 | ]) 45 | if g0.mu < root[0] < g1.mu or g1.mu < root[0] < g0.mu: 46 | self.threshold = root[0] 47 | else: 48 | self.threshold = root[1] 49 | 50 | def transform(self, X:np.ndarray): 51 | """ 52 | project data 53 | 54 | Parameters 55 | ---------- 56 | X : (N, D) np.ndarray 57 | independent variable 58 | 59 | Returns 60 | ------- 61 | y : (N,) np.ndarray 62 | projected data 63 | """ 64 | return X @ self.w 65 | 66 | def classify(self, X:np.ndarray): 67 | """ 68 | classify input data 69 | 70 | Parameters 71 | ---------- 72 | X : (N, D) np.ndarray 73 | independent variable to be classified 74 | 75 | Returns 76 | ------- 77 | (N,) np.ndarray 78 | binary class for each input 79 | """ 80 | return (X @ self.w > self.threshold).astype(np.int) 81 | -------------------------------------------------------------------------------- /prml/markov/categorical_hmm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .hmm import HiddenMarkovModel 3 | 4 | 5 | class CategoricalHMM(HiddenMarkovModel): 6 | """ 7 | Hidden Markov Model with categorical emission model 8 | """ 9 | 10 | def __init__(self, initial_proba, transition_proba, means): 11 | """ 12 | construct hidden markov model with categorical emission model 13 | 14 | Parameters 15 | ---------- 16 | initial_proba : (n_hidden,) np.ndarray 17 | probability of initial latent state 18 | transition_proba : (n_hidden, n_hidden) np.ndarray 19 | transition probability matrix 20 | (i, j) component denotes the transition probability from i-th to j-th hidden state 21 | means : (n_hidden, ndim) np.ndarray 22 | mean parameters of categorical distribution 23 | 24 | Returns 25 | ------- 26 | ndim : int 27 | number of observation categories 28 | n_hidden : int 29 | number of hidden states 30 | """ 31 | assert initial_proba.size == transition_proba.shape[0] == transition_proba.shape[1] == means.shape[0] 32 | assert np.allclose(means.sum(axis=1), 1) 33 | super().__init__(initial_proba, transition_proba) 34 | self.ndim = means.shape[1] 35 | self.means = means 36 | 37 | def draw(self, n=100): 38 | """ 39 | draw random sequence from this model 40 | 41 | Parameters 42 | ---------- 43 | n : int 44 | length of the random sequence 45 | 46 | Returns 47 | ------- 48 | seq : (n,) np.ndarray 49 | generated random sequence 50 | """ 51 | hidden_state = np.random.choice(self.n_hidden, p=self.initial_proba) 52 | seq = [] 53 | while len(seq) < n: 54 | seq.append(np.random.choice(self.ndim, p=self.means[hidden_state])) 55 | hidden_state = np.random.choice(self.n_hidden, p=self.transition_proba[hidden_state]) 56 | return np.asarray(seq) 57 | 58 | def likelihood(self, X): 59 | return self.means[X] 60 | 61 | def maximize(self, seq, p_hidden, p_transition): 62 | self.initial_proba = p_hidden[0] / np.sum(p_hidden[0]) 63 | self.transition_proba = np.sum(p_transition, axis=0) / np.sum(p_transition, axis=(0, 2)) 64 | x = p_hidden[:, None, :] * (np.eye(self.ndim)[seq])[:, :, None] 65 | self.means = np.sum(x, axis=0) / np.sum(p_hidden, axis=0) 66 | -------------------------------------------------------------------------------- /prml/nn/optimizer/adam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.config import config 3 | from prml.nn.optimizer.optimizer import Optimizer 4 | 5 | 6 | class Adam(Optimizer): 7 | """ 8 | Adam optimizer 9 | initialization 10 | m1 = 0 (Initial 1st moment of gradient) 11 | m2 = 0 (Initial 2nd moment of gradient) 12 | n_iter = 0 13 | update rule 14 | n_iter += 1 15 | learning_rate *= sqrt(1 - beta2^n) / (1 - beta1^n) 16 | m1 = beta1 * m1 + (1 - beta1) * gradient 17 | m2 = beta2 * m2 + (1 - beta2) * gradient^2 18 | param += learning_rate * m1 / (sqrt(m2) + epsilon) 19 | """ 20 | 21 | def __init__(self, parameter, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8): 22 | """ 23 | construct Adam optimizer 24 | Parameters 25 | ---------- 26 | parameters : dict 27 | dict of parameters to be optimized 28 | learning_rate : float 29 | beta1 : float 30 | exponential decay rate for the 1st moment 31 | beta2 : float 32 | exponential decay rate for the 2nd moment 33 | epsilon : float 34 | small constant to be added to denominator for numerical stability 35 | Attributes 36 | ---------- 37 | n_iter : int 38 | number of iterations performed 39 | moment1 : dict 40 | 1st moment of each learnable parameter 41 | moment2 : dict 42 | 2nd moment of each learnable parameter 43 | """ 44 | super().__init__(parameter, learning_rate) 45 | self.beta1 = beta1 46 | self.beta2 = beta2 47 | self.epsilon = epsilon 48 | self.moment1 = {} 49 | self.moment2 = {} 50 | for key, param in self.parameter.items(): 51 | self.moment1[key] = np.zeros(param.shape, dtype=config.dtype) 52 | self.moment2[key] = np.zeros(param.shape, dtype=config.dtype) 53 | 54 | def update(self): 55 | """ 56 | update parameter of the neural network 57 | """ 58 | lr = ( 59 | self.learning_rate 60 | * (1 - self.beta2 ** self.iter_count) ** 0.5 61 | / (1 - self.beta1 ** self.iter_count)) 62 | for kp in self.parameter: 63 | p, m1, m2 = self.parameter[kp], self.moment1[kp], self.moment2[kp] 64 | if p.grad is None: 65 | continue 66 | m1 += (1 - self.beta1) * (p.grad - m1) 67 | m2 += (1 - self.beta2) * (p.grad ** 2 - m2) 68 | p.value += lr * m1 / (np.sqrt(m2) + self.epsilon) 69 | -------------------------------------------------------------------------------- /prml/nn/distribution/categorical.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.nn.array.array import asarray 3 | from prml.nn.function import Function 4 | from prml.nn.math.log import log 5 | from prml.nn.nonlinear.softmax import softmax 6 | from prml.nn.distribution.distribution import Distribution 7 | from prml.nn.loss.softmax_cross_entropy import softmax_cross_entropy 8 | 9 | 10 | class Categorical(Distribution): 11 | is_categorical = True 12 | 13 | def __init__(self, mean=None, logit=None, use_gumbel_softmax=True, tau=0.1): 14 | super().__init__() 15 | if mean is not None: 16 | self.mean = asarray(mean) 17 | assert((self.mean.value >= 0).all() and np.allclose(self.mean.value.sum(axis=-1), 1)) 18 | self.logit = log(self.mean) 19 | self._log_pdf = self._log_pdf_mean 20 | elif logit is not None: 21 | self.mean = softmax(logit) 22 | self.logit = asarray(logit) 23 | self._log_pdf = self._log_pdf_logit 24 | else: 25 | raise ValueError 26 | self.n_category = self.mean.shape[-1] 27 | if use_gumbel_softmax: 28 | self.forward = self._forward_gumbel_softmax 29 | else: 30 | self.forward = self._forward 31 | self.tau = tau 32 | self.eye = np.eye(self.n_category) 33 | 34 | def _forward_gumbel_softmax(self): 35 | g = np.random.gumbel(size=self.mean.shape) 36 | return softmax((self.logit + g) / self.tau) 37 | 38 | def _forward(self): 39 | if self.mean.ndim == 1: 40 | index = np.random.choice(self.n_category, p=self.mean.value) 41 | return asarray(self.eye[index]) 42 | else: 43 | mean = self.mean.value.reshape(-1, self.n_category) 44 | indices = [np.random.choice(self.n_category, p=p) for p in mean] 45 | onehot = self.eye[np.array(indices)] 46 | return asarray(onehot) 47 | 48 | def _pdf(self, x): 49 | return CategoricalPDF().forward(self.mean, x) 50 | 51 | def _log_pdf_mean(self, x): 52 | return x * log(self.mean) 53 | 54 | def _log_pdf_logit(self, x): 55 | return -softmax_cross_entropy(self.logit, x) 56 | 57 | 58 | class CategoricalPDF(Function): 59 | 60 | def _forward(self, mean, x): 61 | proba = np.ones_like(mean) 62 | self.indices = np.where(x == 1) 63 | proba[self.indices] = mean[self.indices] 64 | return proba 65 | 66 | def _backward(self, delta, mean, x): 67 | dmean = np.zeros_like(mean) 68 | dmean[self.indices] = delta[self.indices] 69 | return dmean 70 | -------------------------------------------------------------------------------- /prml/linear/softmax_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.classifier import Classifier 3 | from prml.preprocess.label_transformer import LabelTransformer 4 | 5 | 6 | class SoftmaxRegression(Classifier): 7 | """ 8 | Softmax regression model 9 | aka 10 | multinomial logistic regression, 11 | multiclass logistic regression, 12 | maximum entropy classifier. 13 | 14 | y = softmax(X @ W) 15 | t ~ Categorical(t|y) 16 | """ 17 | 18 | @staticmethod 19 | def _softmax(a): 20 | a_max = np.max(a, axis=-1, keepdims=True) 21 | exp_a = np.exp(a - a_max) 22 | return exp_a / np.sum(exp_a, axis=-1, keepdims=True) 23 | 24 | def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100, learning_rate:float=0.1): 25 | """ 26 | maximum likelihood estimation of the parameter 27 | 28 | Parameters 29 | ---------- 30 | X : (N, D) np.ndarray 31 | training independent variable 32 | t : (N,) or (N, K) np.ndarray 33 | training dependent variable 34 | in class index or one-of-k encoding 35 | max_iter : int, optional 36 | maximum number of iteration (the default is 100) 37 | learning_rate : float, optional 38 | learning rate of gradient descent (the default is 0.1) 39 | """ 40 | if t.ndim == 1: 41 | t = LabelTransformer().encode(t) 42 | self.n_classes = np.size(t, 1) 43 | W = np.zeros((np.size(X, 1), self.n_classes)) 44 | for _ in range(max_iter): 45 | W_prev = np.copy(W) 46 | y = self._softmax(X @ W) 47 | grad = X.T @ (y - t) 48 | W -= learning_rate * grad 49 | if np.allclose(W, W_prev): 50 | break 51 | self.W = W 52 | 53 | def proba(self, X:np.ndarray): 54 | """ 55 | compute probability of input belonging each class 56 | 57 | Parameters 58 | ---------- 59 | X : (N, D) np.ndarray 60 | independent variable 61 | 62 | Returns 63 | ------- 64 | (N, K) np.ndarray 65 | probability of each class 66 | """ 67 | return self._softmax(X @ self.W) 68 | 69 | def classify(self, X:np.ndarray): 70 | """ 71 | classify input data 72 | 73 | Parameters 74 | ---------- 75 | X : (N, D) np.ndarray 76 | independent variable to be classified 77 | 78 | Returns 79 | ------- 80 | (N,) np.ndarray 81 | class index for each input 82 | """ 83 | return np.argmax(self.proba(X), axis=-1) 84 | -------------------------------------------------------------------------------- /prml/nn/image/util.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import numpy as np 3 | from numpy.lib.stride_tricks import as_strided 4 | 5 | 6 | def img2patch(img, size, step=1): 7 | """ 8 | convert batch of image array into patches 9 | Parameters 10 | ---------- 11 | img : (n_batch, xlen_in, ylen_in, in_chaprml.nnels) ndarray 12 | batch of images 13 | size : tuple or int 14 | patch size 15 | step : tuple or int 16 | stride of patches 17 | Returns 18 | ------- 19 | patch : (n_batch, xlen_out, ylen_out, size, size, in_chaprml.nnels) ndarray 20 | batch of patches at all points 21 | len_out = (len_in - size) // step + 1 22 | """ 23 | ndim = img.ndim 24 | if isinstance(size, int): 25 | size = (size,) * (ndim - 2) 26 | if isinstance(step, int): 27 | step = (step,) * (ndim - 2) 28 | 29 | slices = tuple(slice(None, None, s) for s in step) 30 | window_strides = img.strides[1:] 31 | index_strides = img[(slice(None),) + slices].strides[:-1] 32 | 33 | out_shape = tuple( 34 | np.subtract(img.shape[1: -1], size) // np.array(step) + 1) 35 | out_shape = (len(img),) + out_shape + size + (np.size(img, -1),) 36 | strides = index_strides + window_strides 37 | patch = as_strided(img, shape=out_shape, strides=strides) 38 | return patch 39 | 40 | 41 | def _patch2img(x, stride, shape): 42 | """ 43 | sum up patches and form an image 44 | Parameters 45 | ---------- 46 | x : (n_batch, xlen_in, ylen_in, kx, ky, in_chaprml.nnels) ndarray 47 | batch of patches at all points 48 | stride : tuple 49 | applied stride to take patches 50 | shape : (n_batch, xlen_out, ylen_out, in_chaprml.nnels) tuple 51 | output image shape 52 | Returns 53 | ------- 54 | img : (n_batch, xlen_out, ylen_out, in_chaprml.nnels) ndarray 55 | image 56 | """ 57 | img = np.zeros(shape, dtype=x.dtype) 58 | kx, ky = x.shape[3: 5] 59 | for i, j in itertools.product(range(kx), range(ky)): 60 | slices = tuple(slice(b, b + s * len_, s) for b, s, len_ in zip([i, j], stride, x.shape[1: 3])) 61 | img[(slice(None),) + slices] += x[..., i, j, :] 62 | return img 63 | 64 | 65 | def patch2img(x, stride, shape): 66 | img = np.zeros(shape, dtype=x.dtype) 67 | kx, ky = x.shape[3:5] 68 | patch = img2patch(img, (kx, ky), stride) 69 | for i, j in itertools.product(range(kx), range(ky)): 70 | patch[..., i, j, :] += x[..., i, j, :] 71 | return img 72 | 73 | 74 | def patch2img_no_overlap(x, stride, shape): 75 | img = np.zeros(shape, dtype=x.dtype) 76 | patch = img2patch(img, x.shape[3:5], stride) 77 | patch += x 78 | return img 79 | -------------------------------------------------------------------------------- /prml/rv/gamma.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import gamma 3 | from prml.rv.rv import RandomVariable 4 | 5 | 6 | np.seterr(all="ignore") 7 | 8 | 9 | class Gamma(RandomVariable): 10 | """ 11 | Gamma distribution 12 | p(x|a, b) 13 | = b^a x^(a-1) exp(-bx) / gamma(a) 14 | """ 15 | 16 | def __init__(self, a, b): 17 | """ 18 | construct Gamma distribution 19 | 20 | Parameters 21 | ---------- 22 | a : int, float, or np.ndarray 23 | shape parameter 24 | b : int, float, or np.ndarray 25 | rate parameter 26 | """ 27 | super().__init__() 28 | a = np.asarray(a) 29 | b = np.asarray(b) 30 | assert a.shape == b.shape 31 | self.a = a 32 | self.b = b 33 | 34 | @property 35 | def a(self): 36 | return self.parameter["a"] 37 | 38 | @a.setter 39 | def a(self, a): 40 | if isinstance(a, (int, float, np.number)): 41 | if a <= 0: 42 | raise ValueError("a must be positive") 43 | self.parameter["a"] = np.asarray(a) 44 | elif isinstance(a, np.ndarray): 45 | if (a <= 0).any(): 46 | raise ValueError("a must be positive") 47 | self.parameter["a"] = a 48 | else: 49 | if a is not None: 50 | raise TypeError(f"{type(a)} is not supported for a") 51 | self.parameter["a"] = None 52 | 53 | @property 54 | def b(self): 55 | return self.parameter["b"] 56 | 57 | @b.setter 58 | def b(self, b): 59 | if isinstance(b, (int, float, np.number)): 60 | if b <= 0: 61 | raise ValueError("b must be positive") 62 | self.parameter["b"] = np.asarray(b) 63 | elif isinstance(b, np.ndarray): 64 | if (b <= 0).any(): 65 | raise ValueError("b must be positive") 66 | self.parameter["b"] = b 67 | else: 68 | if b is not None: 69 | raise TypeError(f"{type(b)} is not supported for b") 70 | self.parameter["b"] = None 71 | 72 | @property 73 | def ndim(self): 74 | return self.a.ndim 75 | 76 | @property 77 | def shape(self): 78 | return self.a.shape 79 | 80 | @property 81 | def size(self): 82 | return self.a.size 83 | 84 | def _pdf(self, X): 85 | return ( 86 | self.b ** self.a 87 | * X ** (self.a - 1) 88 | * np.exp(-self.b * X) 89 | / gamma(self.a)) 90 | 91 | def _draw(self, sample_size=1): 92 | return np.random.gamma( 93 | shape=self.a, 94 | scale=1 / self.b, 95 | size=(sample_size,) + self.shape 96 | ) 97 | -------------------------------------------------------------------------------- /docs/src/prml.rv.rst: -------------------------------------------------------------------------------- 1 | prml.rv package 2 | =============== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.rv.bernoulli module 8 | ------------------------ 9 | 10 | .. automodule:: prml.rv.bernoulli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.rv.bernoulli\_mixture module 16 | --------------------------------- 17 | 18 | .. automodule:: prml.rv.bernoulli_mixture 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.rv.beta module 24 | ------------------- 25 | 26 | .. automodule:: prml.rv.beta 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.rv.categorical module 32 | -------------------------- 33 | 34 | .. automodule:: prml.rv.categorical 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.rv.dirichlet module 40 | ------------------------ 41 | 42 | .. automodule:: prml.rv.dirichlet 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.rv.gamma module 48 | -------------------- 49 | 50 | .. automodule:: prml.rv.gamma 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.rv.gaussian module 56 | ----------------------- 57 | 58 | .. automodule:: prml.rv.gaussian 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | prml.rv.multivariate\_gaussian module 64 | ------------------------------------- 65 | 66 | .. automodule:: prml.rv.multivariate_gaussian 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | prml.rv.multivariate\_gaussian\_mixture module 72 | ---------------------------------------------- 73 | 74 | .. automodule:: prml.rv.multivariate_gaussian_mixture 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | prml.rv.rv module 80 | ----------------- 81 | 82 | .. automodule:: prml.rv.rv 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | prml.rv.students\_t module 88 | -------------------------- 89 | 90 | .. automodule:: prml.rv.students_t 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | prml.rv.uniform module 96 | ---------------------- 97 | 98 | .. automodule:: prml.rv.uniform 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | prml.rv.variational\_gaussian\_mixture module 104 | --------------------------------------------- 105 | 106 | .. automodule:: prml.rv.variational_gaussian_mixture 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: prml.rv 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /docs/_sources/prml.rv.rst.txt: -------------------------------------------------------------------------------- 1 | prml.rv package 2 | =============== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.rv.bernoulli module 8 | ------------------------ 9 | 10 | .. automodule:: prml.rv.bernoulli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.rv.bernoulli\_mixture module 16 | --------------------------------- 17 | 18 | .. automodule:: prml.rv.bernoulli_mixture 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.rv.beta module 24 | ------------------- 25 | 26 | .. automodule:: prml.rv.beta 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.rv.categorical module 32 | -------------------------- 33 | 34 | .. automodule:: prml.rv.categorical 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.rv.dirichlet module 40 | ------------------------ 41 | 42 | .. automodule:: prml.rv.dirichlet 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.rv.gamma module 48 | -------------------- 49 | 50 | .. automodule:: prml.rv.gamma 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.rv.gaussian module 56 | ----------------------- 57 | 58 | .. automodule:: prml.rv.gaussian 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | prml.rv.multivariate\_gaussian module 64 | ------------------------------------- 65 | 66 | .. automodule:: prml.rv.multivariate_gaussian 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | prml.rv.multivariate\_gaussian\_mixture module 72 | ---------------------------------------------- 73 | 74 | .. automodule:: prml.rv.multivariate_gaussian_mixture 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | prml.rv.rv module 80 | ----------------- 81 | 82 | .. automodule:: prml.rv.rv 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | prml.rv.students\_t module 88 | -------------------------- 89 | 90 | .. automodule:: prml.rv.students_t 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | prml.rv.uniform module 96 | ---------------------- 97 | 98 | .. automodule:: prml.rv.uniform 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | prml.rv.variational\_gaussian\_mixture module 104 | --------------------------------------------- 105 | 106 | .. automodule:: prml.rv.variational_gaussian_mixture 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: prml.rv 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /docs/src/prml.nn.math.rst: -------------------------------------------------------------------------------- 1 | prml.nn.math package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.math.add module 8 | ----------------------- 9 | 10 | .. automodule:: prml.nn.math.add 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.math.divide module 16 | -------------------------- 17 | 18 | .. automodule:: prml.nn.math.divide 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.math.exp module 24 | ----------------------- 25 | 26 | .. automodule:: prml.nn.math.exp 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.math.log module 32 | ----------------------- 33 | 34 | .. automodule:: prml.nn.math.log 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.math.matmul module 40 | -------------------------- 41 | 42 | .. automodule:: prml.nn.math.matmul 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.math.mean module 48 | ------------------------ 49 | 50 | .. automodule:: prml.nn.math.mean 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.nn.math.multiply module 56 | ---------------------------- 57 | 58 | .. automodule:: prml.nn.math.multiply 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | prml.nn.math.negative module 64 | ---------------------------- 65 | 66 | .. automodule:: prml.nn.math.negative 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | prml.nn.math.power module 72 | ------------------------- 73 | 74 | .. automodule:: prml.nn.math.power 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | prml.nn.math.product module 80 | --------------------------- 81 | 82 | .. automodule:: prml.nn.math.product 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | prml.nn.math.sqrt module 88 | ------------------------ 89 | 90 | .. automodule:: prml.nn.math.sqrt 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | prml.nn.math.square module 96 | -------------------------- 97 | 98 | .. automodule:: prml.nn.math.square 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | prml.nn.math.subtract module 104 | ---------------------------- 105 | 106 | .. automodule:: prml.nn.math.subtract 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | prml.nn.math.sum module 112 | ----------------------- 113 | 114 | .. automodule:: prml.nn.math.sum 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: prml.nn.math 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /docs/_sources/prml.nn.math.rst.txt: -------------------------------------------------------------------------------- 1 | prml.nn.math package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | prml.nn.math.add module 8 | ----------------------- 9 | 10 | .. automodule:: prml.nn.math.add 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | prml.nn.math.divide module 16 | -------------------------- 17 | 18 | .. automodule:: prml.nn.math.divide 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | prml.nn.math.exp module 24 | ----------------------- 25 | 26 | .. automodule:: prml.nn.math.exp 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | prml.nn.math.log module 32 | ----------------------- 33 | 34 | .. automodule:: prml.nn.math.log 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | prml.nn.math.matmul module 40 | -------------------------- 41 | 42 | .. automodule:: prml.nn.math.matmul 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | prml.nn.math.mean module 48 | ------------------------ 49 | 50 | .. automodule:: prml.nn.math.mean 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | prml.nn.math.multiply module 56 | ---------------------------- 57 | 58 | .. automodule:: prml.nn.math.multiply 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | prml.nn.math.negative module 64 | ---------------------------- 65 | 66 | .. automodule:: prml.nn.math.negative 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | prml.nn.math.power module 72 | ------------------------- 73 | 74 | .. automodule:: prml.nn.math.power 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | prml.nn.math.product module 80 | --------------------------- 81 | 82 | .. automodule:: prml.nn.math.product 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | prml.nn.math.sqrt module 88 | ------------------------ 89 | 90 | .. automodule:: prml.nn.math.sqrt 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | prml.nn.math.square module 96 | -------------------------- 97 | 98 | .. automodule:: prml.nn.math.square 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | prml.nn.math.subtract module 104 | ---------------------------- 105 | 106 | .. automodule:: prml.nn.math.subtract 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | prml.nn.math.sum module 112 | ----------------------- 113 | 114 | .. automodule:: prml.nn.math.sum 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: prml.nn.math 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /prml/rv/multivariate_gaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.rv.rv import RandomVariable 3 | 4 | 5 | class MultivariateGaussian(RandomVariable): 6 | """ 7 | The multivariate Gaussian distribution 8 | p(x|mu, cov) 9 | = exp{-0.5 * (x - mu)^T @ cov^-1 @ (x - mu)} 10 | / (2pi)^(D/2) / |cov|^0.5 11 | """ 12 | 13 | def __init__(self, mu=None, cov=None, tau=None): 14 | super().__init__() 15 | self.mu = mu 16 | if cov is not None: 17 | self.cov = cov 18 | elif tau is not None: 19 | self.tau = tau 20 | else: 21 | self.cov = None 22 | self.tau = None 23 | 24 | @property 25 | def mu(self): 26 | return self.parameter["mu"] 27 | 28 | @mu.setter 29 | def mu(self, mu): 30 | if isinstance(mu, np.ndarray): 31 | assert mu.ndim == 1 32 | self.parameter["mu"] = mu 33 | else: 34 | assert mu is None 35 | self.parameter["mu"] = None 36 | 37 | @property 38 | def cov(self): 39 | return self.parameter["cov"] 40 | 41 | @cov.setter 42 | def cov(self, cov): 43 | if isinstance(cov, np.ndarray): 44 | assert cov.ndim == 2 45 | self.tau_ = np.linalg.inv(cov) 46 | self.parameter["cov"] = cov 47 | else: 48 | assert cov is None 49 | self.tau_ = None 50 | self.parameter["cov"] = None 51 | 52 | @property 53 | def tau(self): 54 | return self.tau_ 55 | 56 | @tau.setter 57 | def tau(self, tau): 58 | if isinstance(tau, np.ndarray): 59 | assert tau.ndim == 2 60 | self.parameter["cov"] = np.linalg.inv(tau) 61 | self.tau_ = tau 62 | else: 63 | assert tau is None 64 | self.tau_ = None 65 | self.parameter["cov"] = None 66 | 67 | @property 68 | def ndim(self): 69 | if hasattr(self.mu, "ndim"): 70 | return self.mu.ndim 71 | else: 72 | return None 73 | 74 | @property 75 | def size(self): 76 | if hasattr(self.mu, "size"): 77 | return self.mu.size 78 | else: 79 | return None 80 | 81 | @property 82 | def shape(self): 83 | if hasattr(self.mu, "shape"): 84 | return self.mu.shape 85 | else: 86 | return None 87 | 88 | def _fit(self, X): 89 | self.mu = np.mean(X, axis=0) 90 | self.cov = np.atleast_2d(np.cov(X.T, bias=True)) 91 | 92 | def _pdf(self, X): 93 | d = X - self.mu 94 | return ( 95 | np.exp(-0.5 * np.sum(d @ self.tau * d, axis=-1)) 96 | * np.sqrt(np.linalg.det(self.tau)) 97 | / np.power(2 * np.pi, 0.5 * self.size)) 98 | 99 | def _draw(self, sample_size=1): 100 | return np.random.multivariate_normal(self.mu, self.cov, sample_size) 101 | -------------------------------------------------------------------------------- /prml/linear/emprical_bayes_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.bayesian_regression import BayesianRegression 3 | 4 | 5 | class EmpiricalBayesRegression(BayesianRegression): 6 | """ 7 | Empirical Bayes Regression model 8 | a.k.a. 9 | type 2 maximum likelihood, 10 | generalized maximum likelihood, 11 | evidence approximation 12 | 13 | w ~ N(w|0, alpha^(-1)I) 14 | y = X @ w 15 | t ~ N(t|X @ w, beta^(-1)) 16 | evidence function p(t|X,alpha,beta) = S p(t|w;X,beta)p(w|0;alpha) dw 17 | """ 18 | 19 | def __init__(self, alpha:float=1., beta:float=1.): 20 | super().__init__(alpha, beta) 21 | 22 | def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100): 23 | """ 24 | maximization of evidence function with respect to 25 | the hyperparameters alpha and beta given training dataset 26 | 27 | Parameters 28 | ---------- 29 | X : (N, D) np.ndarray 30 | training independent variable 31 | t : (N,) np.ndarray 32 | training dependent variable 33 | max_iter : int 34 | maximum number of iteration 35 | """ 36 | M = X.T @ X 37 | eigenvalues = np.linalg.eigvalsh(M) 38 | eye = np.eye(np.size(X, 1)) 39 | N = len(t) 40 | for _ in range(max_iter): 41 | params = [self.alpha, self.beta] 42 | 43 | w_precision = self.alpha * eye + self.beta * X.T @ X 44 | w_mean = self.beta * np.linalg.solve(w_precision, X.T @ t) 45 | 46 | gamma = np.sum(eigenvalues / (self.alpha + eigenvalues)) 47 | self.alpha = float(gamma / np.sum(w_mean ** 2).clip(min=1e-10)) 48 | self.beta = float( 49 | (N - gamma) / np.sum(np.square(t - X @ w_mean)) 50 | ) 51 | if np.allclose(params, [self.alpha, self.beta]): 52 | break 53 | self.w_mean = w_mean 54 | self.w_precision = w_precision 55 | self.w_cov = np.linalg.inv(w_precision) 56 | 57 | def _log_prior(self, w): 58 | return -0.5 * self.alpha * np.sum(w ** 2) 59 | 60 | def _log_likelihood(self, X, t, w): 61 | return -0.5 * self.beta * np.square(t - X @ w).sum() 62 | 63 | def _log_posterior(self, X, t, w): 64 | return self._log_likelihood(X, t, w) + self._log_prior(w) 65 | 66 | def log_evidence(self, X:np.ndarray, t:np.ndarray): 67 | """ 68 | logarithm or the evidence function 69 | 70 | Parameters 71 | ---------- 72 | X : (N, D) np.ndarray 73 | indenpendent variable 74 | t : (N,) np.ndarray 75 | dependent variable 76 | Returns 77 | ------- 78 | float 79 | log evidence 80 | """ 81 | N = len(t) 82 | D = np.size(X, 1) 83 | return 0.5 * ( 84 | D * np.log(self.alpha) + N * np.log(self.beta) 85 | - np.linalg.slogdet(self.w_precision)[1] - D * np.log(2 * np.pi) 86 | ) + self._log_posterior(X, t, self.w_mean) 87 | -------------------------------------------------------------------------------- /prml/linear/bayesian_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.regression import Regression 3 | 4 | 5 | class BayesianRegression(Regression): 6 | """ 7 | Bayesian regression model 8 | 9 | w ~ N(w|0, alpha^(-1)I) 10 | y = X @ w 11 | t ~ N(t|X @ w, beta^(-1)) 12 | """ 13 | 14 | def __init__(self, alpha:float=1., beta:float=1.): 15 | self.alpha = alpha 16 | self.beta = beta 17 | self.w_mean = None 18 | self.w_precision = None 19 | 20 | def _is_prior_defined(self) -> bool: 21 | return self.w_mean is not None and self.w_precision is not None 22 | 23 | def _get_prior(self, ndim:int) -> tuple: 24 | if self._is_prior_defined(): 25 | return self.w_mean, self.w_precision 26 | else: 27 | return np.zeros(ndim), self.alpha * np.eye(ndim) 28 | 29 | def fit(self, X:np.ndarray, t:np.ndarray): 30 | """ 31 | bayesian update of parameters given training dataset 32 | 33 | Parameters 34 | ---------- 35 | X : (N, n_features) np.ndarray 36 | training data independent variable 37 | t : (N,) np.ndarray 38 | training data dependent variable 39 | """ 40 | 41 | mean_prev, precision_prev = self._get_prior(np.size(X, 1)) 42 | 43 | w_precision = precision_prev + self.beta * X.T @ X 44 | w_mean = np.linalg.solve( 45 | w_precision, 46 | precision_prev @ mean_prev + self.beta * X.T @ t 47 | ) 48 | self.w_mean = w_mean 49 | self.w_precision = w_precision 50 | self.w_cov = np.linalg.inv(self.w_precision) 51 | 52 | def predict(self, X:np.ndarray, return_std:bool=False, sample_size:int=None): 53 | """ 54 | return mean (and standard deviation) of predictive distribution 55 | 56 | Parameters 57 | ---------- 58 | X : (N, n_features) np.ndarray 59 | independent variable 60 | return_std : bool, optional 61 | flag to return standard deviation (the default is False) 62 | sample_size : int, optional 63 | number of samples to draw from the predictive distribution 64 | (the default is None, no sampling from the distribution) 65 | 66 | Returns 67 | ------- 68 | y : (N,) np.ndarray 69 | mean of the predictive distribution 70 | y_std : (N,) np.ndarray 71 | standard deviation of the predictive distribution 72 | y_sample : (N, sample_size) np.ndarray 73 | samples from the predictive distribution 74 | """ 75 | 76 | if sample_size is not None: 77 | w_sample = np.random.multivariate_normal( 78 | self.w_mean, self.w_cov, size=sample_size 79 | ) 80 | y_sample = X @ w_sample.T 81 | return y_sample 82 | y = X @ self.w_mean 83 | if return_std: 84 | y_var = 1 / self.beta + np.sum(X @ self.w_cov * X, axis=1) 85 | y_std = np.sqrt(y_var) 86 | return y, y_std 87 | return y 88 | -------------------------------------------------------------------------------- /test/test_bayesnet/test_discrete.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | from prml import bayesnet as bn 4 | 5 | 6 | class TestDiscrete(unittest.TestCase): 7 | 8 | def test_discrete(self): 9 | a = bn.discrete([0.2, 0.8]) 10 | b = bn.discrete([[0.1, 0.2], [0.9, 0.8]], a) 11 | self.assertTrue(np.allclose(b.proba, [0.18, 0.82])) 12 | a.observe(0) 13 | self.assertTrue(np.allclose(b.proba, [0.1, 0.9])) 14 | 15 | a = bn.discrete([0.1, 0.9]) 16 | b = bn.discrete([[0.7, 0.2], [0.3, 0.8]], a) 17 | c = bn.discrete([[0.8, 0.4], [0.2, 0.6]], b) 18 | self.assertTrue(np.allclose(a.proba, [0.1, 0.9])) 19 | self.assertTrue(np.allclose(b.proba, [0.25, 0.75])) 20 | self.assertTrue(np.allclose(c.proba, [0.5, 0.5])) 21 | c.observe(0) 22 | self.assertTrue(np.allclose(a.proba, [0.136, 0.864])) 23 | self.assertTrue(np.allclose(b.proba, [0.4, 0.6])) 24 | self.assertTrue(np.allclose(c.proba, [1, 0])) 25 | 26 | a = bn.discrete([0.1, 0.9], name="p(a)") 27 | b = bn.discrete([0.1, 0.9], name="p(b)") 28 | c = bn.discrete( 29 | [[[0.9, 0.8], 30 | [0.8, 0.2]], 31 | [[0.1, 0.2], 32 | [0.2, 0.8]]], 33 | a, b, name="p(c|a,b)" 34 | ) 35 | c.observe(0) 36 | self.assertTrue(np.allclose(a.proba, [0.25714286, 0.74285714])) 37 | b.observe(0) 38 | self.assertTrue(np.allclose(a.proba, [0.11111111, 0.88888888])) 39 | 40 | a = bn.discrete([0.1, 0.9], name="p(a)") 41 | b = bn.discrete([0.1, 0.9], name="p(b)") 42 | c = bn.discrete( 43 | [[[0.9, 0.8], 44 | [0.8, 0.2]], 45 | [[0.1, 0.2], 46 | [0.2, 0.8]]], 47 | a, b, name="p(c|a,b)" 48 | ) 49 | c.observe(0, proprange=0) 50 | self.assertTrue(np.allclose(a.proba, [0.1, 0.9])) 51 | b.observe(0, proprange=1) 52 | self.assertTrue(np.allclose(a.proba, [0.1, 0.9])) 53 | b.send_message(proprange=2) 54 | self.assertTrue(np.allclose(a.proba, [0.11111111, 0.88888888])) 55 | a.send_message() 56 | c.send_message() 57 | self.assertTrue(np.allclose(a.proba, [0.11111111, 0.88888888]), a.message_from) 58 | 59 | a = bn.discrete([0.1, 0.9], name="p(a)") 60 | b = bn.discrete([0.1, 0.9], name="p(b)") 61 | c = bn.discrete( 62 | [[[0.9, 0.8], 63 | [0.8, 0.2]], 64 | [[0.1, 0.2], 65 | [0.2, 0.8]]], 66 | a, b, name="p(c|a,b)" 67 | ) 68 | b.observe(0) 69 | self.assertTrue(np.allclose(a.proba, [0.1, 0.9])) 70 | c.send_message() 71 | self.assertTrue(np.allclose(a.proba, [0.1, 0.9]), a.message_from) 72 | 73 | def test_joint_discrete(self): 74 | a = bn.DiscreteVariable(2) 75 | b = bn.DiscreteVariable(2) 76 | bn.discrete([[0.1, 0.2], [0.3, 0.4]], out=[a, b]) 77 | b.observe(1) 78 | self.assertTrue(np.allclose(a.proba, [1/3, 2/3])) 79 | 80 | 81 | if __name__ == "__main__": 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /prml/linear/variational_logistic_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.linear.logistic_regression import LogisticRegression 3 | 4 | 5 | class VariationalLogisticRegression(LogisticRegression): 6 | 7 | def __init__(self, alpha:float=None, a0:float=1., b0:float=1.): 8 | """ 9 | construct variational logistic regressor 10 | 11 | Parameters 12 | ---------- 13 | alpha : float 14 | precision parameter of the prior 15 | if None, this is also the subject to estimate 16 | a0 : float 17 | a parameter of hyper prior Gamma dist. 18 | Gamma(alpha|a0,b0) 19 | if alpha is not None, this argument will be ignored 20 | b0 : float 21 | another parameter of hyper prior Gamma dist. 22 | Gamma(alpha|a0,b0) 23 | if alpha is not None, this argument will be ignored 24 | """ 25 | if alpha is not None: 26 | self.__alpha = alpha 27 | else: 28 | self.a0 = a0 29 | self.b0 = b0 30 | 31 | def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=1000): 32 | """ 33 | variational bayesian estimation of the parameter 34 | 35 | Parameters 36 | ---------- 37 | X : (N, D) np.ndarray 38 | training independent variable 39 | t : (N,) np.ndarray 40 | training dependent variable 41 | iter_max : int, optional 42 | maximum number of iteration (the default is 1000) 43 | """ 44 | N, D = X.shape 45 | if hasattr(self, "a0"): 46 | self.a = self.a0 + 0.5 * D 47 | xi = np.random.uniform(-1, 1, size=N) 48 | I = np.eye(D) 49 | param = np.copy(xi) 50 | for _ in range(iter_max): 51 | lambda_ = np.tanh(xi) * 0.25 / xi 52 | self.w_var = np.linalg.inv(I / self.alpha + 2 * (lambda_ * X.T) @ X) 53 | self.w_mean = self.w_var @ np.sum(X.T * (t - 0.5), axis=1) 54 | xi = np.sqrt(np.sum(X @ (self.w_var + self.w_mean * self.w_mean[:, None]) * X, axis=-1)) 55 | if np.allclose(xi, param): 56 | break 57 | else: 58 | param = np.copy(xi) 59 | 60 | @property 61 | def alpha(self): 62 | if hasattr(self, "__alpha"): 63 | return self.__alpha 64 | else: 65 | try: 66 | self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var)) 67 | except AttributeError: 68 | self.b = self.b0 69 | return self.a / self.b 70 | 71 | def proba(self, X:np.ndarray): 72 | """ 73 | compute probability of input belonging class 1 74 | 75 | Parameters 76 | ---------- 77 | X : (N, D) np.ndarray 78 | training data independent variable 79 | 80 | Returns 81 | ------- 82 | (N,) np.ndarray 83 | probability of positive 84 | """ 85 | mu_a = X @ self.w_mean 86 | var_a = np.sum(X @ self.w_var * X, axis=1) 87 | y = self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8)) 88 | return y 89 | -------------------------------------------------------------------------------- /prml/markov/gaussian_hmm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from prml.rv import MultivariateGaussian 3 | from .hmm import HiddenMarkovModel 4 | 5 | 6 | class GaussianHMM(HiddenMarkovModel): 7 | """ 8 | Hidden Markov Model with Gaussian emission model 9 | """ 10 | 11 | def __init__(self, initial_proba, transition_proba, means, covs): 12 | """ 13 | construct hidden markov model with Gaussian emission model 14 | 15 | Parameters 16 | ---------- 17 | initial_proba : (n_hidden,) np.ndarray or None 18 | probability of initial states 19 | transition_proba : (n_hidden, n_hidden) np.ndarray or None 20 | transition probability matrix 21 | (i, j) component denotes the transition probability from i-th to j-th hidden state 22 | means : (n_hidden, ndim) np.ndarray 23 | mean of each gaussian component 24 | covs : (n_hidden, ndim, ndim) np.ndarray 25 | covariance matrix of each gaussian component 26 | 27 | Attributes 28 | ---------- 29 | ndim : int 30 | dimensionality of observation space 31 | n_hidden : int 32 | number of hidden states 33 | """ 34 | assert initial_proba.size == transition_proba.shape[0] == transition_proba.shape[1] == means.shape[0] == covs.shape[0] 35 | assert means.shape[1] == covs.shape[1] == covs.shape[2] 36 | super().__init__(initial_proba, transition_proba) 37 | self.ndim = means.shape[1] 38 | self.means = means 39 | self.covs = covs 40 | self.precisions = np.linalg.inv(self.covs) 41 | self.gaussians = [MultivariateGaussian(m, cov) for m, cov in zip(means, covs)] 42 | 43 | def draw(self, n=100): 44 | """ 45 | draw random sequence from this model 46 | 47 | Parameters 48 | ---------- 49 | n : int 50 | length of the random sequence 51 | 52 | Returns 53 | ------- 54 | seq : (n, ndim) np.ndarray 55 | generated random sequence 56 | """ 57 | hidden_state = np.random.choice(self.n_hidden, p=self.initial_proba) 58 | seq = [] 59 | while len(seq) < n: 60 | seq.extend(self.gaussians[hidden_state].draw()) 61 | hidden_state = np.random.choice(self.n_hidden, p=self.transition_proba[hidden_state]) 62 | return np.asarray(seq) 63 | 64 | def likelihood(self, X): 65 | diff = X[:, None, :] - self.means 66 | exponents = np.sum( 67 | np.einsum('nki,kij->nkj', diff, self.precisions) * diff, axis=-1) 68 | return np.exp(-0.5 * exponents) / np.sqrt(np.linalg.det(self.covs) * (2 * np.pi) ** self.ndim) 69 | 70 | def maximize(self, seq, p_hidden, p_transition): 71 | self.initial_proba = p_hidden[0] / np.sum(p_hidden[0]) 72 | self.transition_proba = np.sum(p_transition, axis=0) / np.sum(p_transition, axis=(0, 2)) 73 | Nk = np.sum(p_hidden, axis=0) 74 | self.means = (seq.T @ p_hidden / Nk).T 75 | diffs = seq[:, None, :] - self.means 76 | self.covs = np.einsum('nki,nkj->kij', diffs, diffs * p_hidden[:, :, None]) / Nk[:, None, None] 77 | --------------------------------------------------------------------------------