├── .gitignore ├── LICENSE ├── README.md ├── data ├── .DS_Store └── fertility │ ├── x_step1_mean.np │ ├── x_step2_mean.np │ ├── y_step1_mean.np │ └── y_step2_mean.np ├── error_func.py ├── hetmogp ├── __init__.py ├── het_likelihood.py ├── svmogp.py ├── svmogp_inf.py └── util.py ├── likelihoods ├── __init__.py ├── bernoulli.py ├── beta.py ├── categorical.py ├── dirichlet.py ├── exponential.py ├── gamma.py ├── gaussian.py ├── hetgaussian.py ├── ordinal.py ├── poisson.py └── student.py ├── notebooks ├── fertility_2x2.ipynb ├── synthetic_data │ ├── X1_train_syn.npy │ ├── X2_train_syn.npy │ ├── X_test_syn.npy │ ├── Y1_train_syn.npy │ ├── Y2_train_syn.npy │ ├── Y_test_syn.npy │ └── Z_syn.npy ├── toy_data_multioutput.ipynb └── toy_data_singleoutput.ipynb ├── poster └── multitask_aggregated_poster.pdf └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # celery beat schedule file 95 | celerybeat-schedule 96 | 97 | # SageMath parsed files 98 | *.sage.py 99 | 100 | # Environments 101 | .env 102 | .venv 103 | env/ 104 | venv/ 105 | ENV/ 106 | env.bak/ 107 | venv.bak/ 108 | 109 | # Spyder project settings 110 | .spyderproject 111 | .spyproject 112 | 113 | # Rope project settings 114 | .ropeproject 115 | 116 | # mkdocs documentation 117 | /site 118 | 119 | # mypy 120 | .mypy_cache/ 121 | .dmypy.json 122 | dmypy.json 123 | 124 | # Pyre type checker 125 | .pyre/ 126 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019, Fariba Yousefi 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aggregated Multi-task GP 2 | This repository contains the source code for https://arxiv.org/abs/1906.09412 3 | It is based on the [GPy](https://github.com/SheffieldML/GPy) and [HetMOGP](https://github.com/pmorenoz/HetMOGP). Here we use a modified version of the GPy which can be access [here](https://github.com/frb-yousefi/GPy-multitask/tree/multitask-gp). 4 | 5 | For any questions please contact f dot yousefi at sheffield dot ac dot uk. 6 | 7 | If you use this code, please cite the following paper: 8 | ``` 9 | @inproceedings{YousefiSmithAlvarez19, 10 | title = {Multi-task Learning for Aggregated Data using {G}aussian Processes}, 11 | author = {Yousefi, Fariba and Smith, Michael T and \'{A}lvarez, Mauricio}, 12 | booktitle = {Advances in Neural Information Processing Systems (NeurIPS) 32}, 13 | year = {2019}, 14 | publisher = {Curran Associates, Inc.}, 15 | url = {http://papers.nips.cc/paper/9644-multi-task-learning-for-aggregated-data-using-gaussian-processes.pdf} 16 | } 17 | ``` 18 | 19 | ## Setup Instructions 20 | 21 | ### Prerequisites 22 | For reproducibility, all instructions below are tested with this Docker image: [continuumio/anaconda3:2019.07](https://hub.docker.com/layers/continuumio/anaconda3/2019.07/images/sha256-9fad434f3f775ed245f0f888cda954bc93f81ffdc31d4e3e37d69283260c3f41) - which is based on Debian 10 and Anaconda3 and Python 3.7.3 (Ubuntu works as well). The authors may release a Docker image as well in the future. 23 | 24 | If you already have a Python 3.7 environment, you may want to skip to the `GPy-multitask installation` part below (and ignore all Docker commands) 25 | 26 | ```bash 27 | # pull the image 28 | docker pull continuumio/anaconda3:2019.07 29 | 30 | # run the image 31 | docker run -i -t --rm -p 8888:8888 continuumio/anaconda3:2019.07 /bin/bash 32 | 33 | # the rest of the commands here are assumed to be executed from the Docker image 34 | 35 | # install the required packages (needed for Cython extensions of the GPy) 36 | apt update && apt install -y build-essential 37 | 38 | # GPy-multitask installation 39 | # clone a modified version of the GPy toolkit: 40 | git clone https://github.com/frb-yousefi/GPy-multitask.git 41 | cd GPy-multitask 42 | 43 | # switch to the multitask-gp branch 44 | git checkout multitask-gp 45 | 46 | # Now install GPy based on the instructions provided in the README file (https://github.com/frb-yousefi/GPy-multitask/blob/multitask-gp/README.md) 47 | python setup.py build_ext --inplace 48 | 49 | # set PYTHONPATH environment variable to include the GPy-multitask path 50 | echo "export PYTHONPATH=$PWD" >> ~/.bashrc 51 | source ~/.bashrc 52 | cd .. 53 | ``` 54 | 55 | ### Multi-task GP Setup 56 | ```bash 57 | # clone this repository 58 | git clone https://github.com/frb-yousefi/aggregated-multitask-gp 59 | cd aggregated-multitask-gp 60 | 61 | # install the required Python packages 62 | pip install -r requirements.txt 63 | 64 | ``` 65 | 66 | ### Sample Notebooks 67 | A sample notebook is provided with this repository. Assuming you're running the Docker image, you can execute this command to start a Jupyter Notebook server and check the sample notebook: 68 | 69 | ```bash 70 | # starting from the root folder of this cloned repo 71 | cd notebooks 72 | jupyter notebook --port 8888 --ip '*' --allow-root 73 | ``` 74 | 75 | ### Contributors 76 | Fariba Yousefi, Michael T. Smith, and Mauricio A Álvarez. 77 | 78 | For further information, please contact: [f......@gmail.com](https://mailhide.io/e/GduGS) 79 | -------------------------------------------------------------------------------- /data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/data/.DS_Store -------------------------------------------------------------------------------- /data/fertility/x_step1_mean.np: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/data/fertility/x_step1_mean.np -------------------------------------------------------------------------------- /data/fertility/x_step2_mean.np: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/data/fertility/x_step2_mean.np -------------------------------------------------------------------------------- /data/fertility/y_step1_mean.np: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/data/fertility/y_step1_mean.np -------------------------------------------------------------------------------- /data/fertility/y_step2_mean.np: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/data/fertility/y_step2_mean.np -------------------------------------------------------------------------------- /error_func.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | 6 | # Standardised Mean Squared Error 7 | def smse(mu_star_list, Y_test_list): 8 | error_k = [] 9 | for k in range(len(Y_test_list)): 10 | res = mu_star_list[k] - Y_test_list[k] 11 | error = (res**2).mean() 12 | error = error / Y_test_list[k].var() 13 | error_k.append(error) 14 | return np.array(error_k) 15 | 16 | # snlp 17 | def snlp(var_star_list, Y_train_list, Y_test_list, mu_star_list): 18 | error_k = [] 19 | for k in range(len(var_star_list)): 20 | res = mu_star_list[k] - Y_test_list[k] 21 | nlp = 0.5 * (np.log(2 * np.pi * var_star_list[k]) + res**2 / var_star_list[k]).mean() 22 | muY = Y_train_list[k].mean() 23 | varY = Y_train_list[k].var() 24 | error = nlp - 0.5 * (np.log(2 * np.pi * varY) + ((Y_test_list - muY) ** 2) / varY).mean() 25 | error_k.append(error) 26 | return np.array(error_k) 27 | 28 | -------------------------------------------------------------------------------- /hetmogp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/hetmogp/__init__.py -------------------------------------------------------------------------------- /hetmogp/het_likelihood.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import numpy as np 5 | from GPy.likelihoods import link_functions 6 | from GPy.likelihoods import Likelihood 7 | from itertools import compress 8 | 9 | 10 | class HetLikelihood(Likelihood): 11 | """ 12 | Heterogeneous Likelihood where 13 | 14 | """ 15 | 16 | def __init__(self, likelihoods_list, gp_link=None ,name='heterogeneous_likelihood'): 17 | if gp_link is None: 18 | gp_link = link_functions.Identity() 19 | 20 | super(HetLikelihood, self).__init__(gp_link=gp_link, name=name) 21 | 22 | self.likelihoods_list = likelihoods_list 23 | 24 | def generate_metadata(self): 25 | """ 26 | Generates Metadata: Given an Heterogeneous likelihood, it calculates the number functions f required in the 27 | model, the assignments of each f to its likelihood function, dimensionality of y_d and functions needed for 28 | predictions. 29 | """ 30 | t_index = np.arange(len(self.likelihoods_list)) 31 | y_index = np.empty((1,1)) 32 | f_index = np.empty((1,1)) 33 | d_index = np.empty((1,1)) 34 | p_index = np.empty((1,1)) 35 | for t, lik in enumerate(self.likelihoods_list): 36 | dim_y, dim_f, dim_pred = lik.get_metadata() 37 | y_index = np.hstack(( y_index, t*np.ones((1,dim_y)) )) 38 | f_index = np.hstack(( f_index, t*np.ones((1,dim_f)) )) 39 | d_index = np.hstack(( d_index, np.arange(0,dim_f)[None,:] )) 40 | p_index = np.hstack((p_index, t * np.ones((1, dim_pred)))) 41 | 42 | metadata = {'task_index': t_index, 'y_index': np.int_(y_index[0,1:]), 'function_index': np.int_(f_index[0,1:]), 43 | 'd_index': np.int_(d_index[0,1:]),'pred_index': np.int_(p_index[0,1:])} 44 | return metadata 45 | 46 | def pdf(self, f, Y, Y_metadata): 47 | """ 48 | Returns a list of PDFs from all likelihoods. 49 | """ 50 | t_ind = Y_metadata['task_index'].flatten() 51 | y_ind = Y_metadata['y_index'].flatten() 52 | f_ind = Y_metadata['function_index'].flatten() 53 | tasks = np.unique(t_ind) 54 | pdf = np.zeros((Y.shape[0], t_ind.shape[0])) 55 | for t in tasks: 56 | pdf[:, t_ind == t] = self.likelihoods_list[t].pdf(f[:, f_ind == t], Y[:, y_ind == t], Y_metadata=None) 57 | return pdf 58 | 59 | def logpdf(self, f, Y, Y_metadata): 60 | """ 61 | Returns a list of log-PDFs from all likelihoods. 62 | """ 63 | t_ind = Y_metadata['task_index'].flatten() 64 | y_ind = Y_metadata['y_index'].flatten() 65 | f_ind = Y_metadata['function_index'].flatten() 66 | tasks = np.unique(t_ind) 67 | logpdf = np.zeros((Y.shape[0], t_ind.shape[0])) 68 | for t in tasks: 69 | logpdf[:, t_ind == t] = self.likelihoods_list[t].logpdf(f[:, f_ind == t], Y[:, y_ind == t], Y_metadata=None) 70 | return logpdf 71 | 72 | def samples(self, F, Y_metadata): 73 | """ 74 | Returns a list of samples from all likelihoods. 75 | """ 76 | t_ind = Y_metadata['task_index'].flatten() 77 | y_ind = Y_metadata['y_index'].flatten() 78 | f_ind = Y_metadata['function_index'].flatten() 79 | tasks = np.unique(t_ind) 80 | samples = [] 81 | for t in tasks: 82 | samples.append(self.likelihoods_list[t].samples(F[t], num_samples=1, Y_metadata=None)) 83 | return samples 84 | 85 | def num_output_functions(self, Y_metadata): 86 | """ 87 | Returns the number of functions f that are required in the model for a given heterogeneous likelihood. 88 | """ 89 | f_ind = Y_metadata['function_index'].flatten() 90 | return f_ind.shape[0] 91 | 92 | def num_latent_functions(self): 93 | pass 94 | 95 | def ismulti(self, task): 96 | """ 97 | For a given task d (heterogeneous output) returns if y_d is or is not multivariate. 98 | """ 99 | return self.likelihoods_list[task].ismulti() 100 | 101 | def var_exp(self, Y, mu_F, v_F, Y_metadata): 102 | """ 103 | Returns a list of variational expectations from all likelihoods wrt to parameter functions (PFs) f. 104 | """ 105 | t_ind = Y_metadata['task_index'].flatten() 106 | y_ind = Y_metadata['y_index'].flatten() 107 | f_ind = Y_metadata['function_index'].flatten() 108 | d_ind = Y_metadata['d_index'].flatten() 109 | tasks = np.unique(t_ind) 110 | var_exp = [] 111 | for t in tasks: 112 | 113 | ve_task = self.likelihoods_list[t].var_exp(Y[t], mu_F[t], v_F[t], Y_metadata=None) 114 | var_exp.append(ve_task) 115 | return var_exp 116 | 117 | def var_exp_derivatives(self, Y, mu_F, v_F, Y_metadata): 118 | """ 119 | Returns a list of variational expectations from all likelihood derivatives wrt to parameter functions (PFs) f. 120 | """ 121 | t_ind = Y_metadata['task_index'].flatten() 122 | y_ind = Y_metadata['y_index'].flatten() 123 | f_ind = Y_metadata['function_index'].flatten() 124 | tasks = np.unique(t_ind) 125 | var_exp_dm = [] 126 | var_exp_dv = [] 127 | for t in tasks: 128 | ve_task_dm, ve_task_dv = self.likelihoods_list[t].var_exp_derivatives(Y[t], mu_F[t], v_F[t], Y_metadata=None) 129 | var_exp_dm.append(ve_task_dm) 130 | var_exp_dv.append(ve_task_dv) 131 | return var_exp_dm, var_exp_dv 132 | 133 | def predictive(self, mu_F_pred, v_F_pred, Y_metadata): 134 | """ 135 | Returns a list of predictive mean and variance from all likelihoods. 136 | """ 137 | t_ind = Y_metadata['task_index'].flatten() 138 | y_ind = Y_metadata['y_index'].flatten() 139 | f_ind = Y_metadata['function_index'].flatten() 140 | p_ind = Y_metadata['pred_index'].flatten() 141 | tasks = np.unique(t_ind) 142 | m_pred = [] 143 | v_pred = [] 144 | for t in tasks: 145 | m_pred_task, v_pred_task = self.likelihoods_list[t].predictive(mu_F_pred[t], v_F_pred[t], Y_metadata=None) 146 | m_pred.append(m_pred_task) 147 | v_pred.append(v_pred_task) 148 | return m_pred, v_pred 149 | 150 | def log_predictive(self, Ytest, mu_F_star, v_F_star, Y_metadata, num_samples): 151 | """ 152 | Returns the log-predictive density (LPD) of the model over each test data Ytest. 153 | the variable logpred is a list of length equal to the number of outputs of the model 154 | and each list position contains the evaluation of LPD of each Ntest data per output 155 | """ 156 | t_ind = Y_metadata['task_index'].flatten() 157 | y_ind = Y_metadata['y_index'].flatten() 158 | f_ind = Y_metadata['function_index'].flatten() 159 | p_ind = Y_metadata['pred_index'].flatten() 160 | tasks = np.unique(t_ind) 161 | logpred = [] #this to allow the return of the nlogpred of each task 162 | for t in tasks: 163 | logpred.append(self.likelihoods_list[t].log_predictive(Ytest[t], mu_F_star[t], v_F_star[t], num_samples)) # it is negative due to "Negative" LPD 164 | 165 | return logpred 166 | -------------------------------------------------------------------------------- /hetmogp/svmogp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import numpy as np 5 | import GPy 6 | from hetmogp.util import draw_mini_slices 7 | from hetmogp.svmogp_inf import SVMOGPInf 8 | from GPy.core.parameterization.param import Param 9 | from GPy.plotting.matplot_dep.util import fixed_inputs 10 | import matplotlib.pyplot as plt 11 | from GPy.util import choleskies 12 | from GPy.util.misc import kmm_init 13 | from hetmogp import util 14 | import random 15 | 16 | class SVMOGP(GPy.core.SparseGP): 17 | def __init__(self, X, Y, Z, kern_list, likelihood, Y_metadata, name='SVMOGP', batch_size=None): 18 | 19 | self.batch_size = batch_size 20 | self.kern_list = kern_list 21 | self.likelihood = likelihood 22 | self.Y_metadata = Y_metadata 23 | 24 | self.num_inducing = Z.shape[0] # M 25 | self.num_latent_funcs = len(kern_list) # Q 26 | self.num_output_funcs = likelihood.num_output_functions(self.Y_metadata) 27 | self.W_list, self.kappa_list = util.random_W_kappas(self.num_latent_funcs, self.num_output_funcs, rank=1) 28 | 29 | self.Xmulti = X 30 | self.Ymulti = Y 31 | 32 | # Batch the data 33 | self.Xmulti_all, self.Ymulti_all = X, Y 34 | if batch_size is None: 35 | #self.stochastic = False 36 | Xmulti_batch, Ymulti_batch = X, Y 37 | else: 38 | # Makes a climin slicer to make drawing minibatches much quicker 39 | #self.stochastic = False #"This was True as Pablo had it" 40 | self.slicer_list = [] 41 | [self.slicer_list.append(draw_mini_slices(Xmulti_task.shape[0], self.batch_size)) for Xmulti_task in self.Xmulti] 42 | Xmulti_batch, Ymulti_batch = self.new_batch() 43 | self.Xmulti, self.Ymulti = Xmulti_batch, Ymulti_batch 44 | 45 | # Initialize inducing points Z 46 | #Z = kmm_init(self.X_all, self.num_inducing) 47 | self.Xdim = Z.shape[1] 48 | Z = np.tile(Z,(1,self.num_latent_funcs)) 49 | 50 | inference_method = SVMOGPInf() 51 | 52 | super(SVMOGP, self).__init__(X=Xmulti_batch[0][1:10], Y=Ymulti_batch[0][1:10], Z=Z, kernel=kern_list[0], likelihood=likelihood, 53 | mean_function=None, X_variance=None, inference_method=inference_method, 54 | Y_metadata=Y_metadata, name=name, normalizer=False) 55 | 56 | self.unlink_parameter(self.kern) # Unlink SparseGP default param kernel 57 | 58 | 59 | _, self.B_list = util.LCM(input_dim=self.Xdim, output_dim=self.num_output_funcs, rank=1, kernels_list=self.kern_list, 60 | W_list=self.W_list, kappa_list=self.kappa_list) 61 | 62 | # Set-up optimization parameters: [Z, m_u, L_u] 63 | self.q_u_means = Param('m_u', 0*np.random.randn(self.num_inducing, self.num_latent_funcs) + 64 | 0*np.tile(np.random.randn(1,self.num_latent_funcs),(self.num_inducing,1))) 65 | chols = choleskies.triang_to_flat(np.tile(np.eye(self.num_inducing)[None,:,:], (self.num_latent_funcs,1,1))) 66 | self.q_u_chols = Param('L_u', chols) 67 | 68 | self.link_parameter(self.Z, index=0) 69 | self.link_parameter(self.q_u_means) 70 | self.link_parameters(self.q_u_chols) 71 | [self.link_parameter(kern_q) for kern_q in kern_list] # link all kernels 72 | [self.link_parameter(B_q) for B_q in self.B_list] 73 | 74 | self.vem_step = True # [True=VE-step, False=VM-step] 75 | self.ve_count = 0 76 | self.elbo = np.zeros((1,1)) 77 | self.index_VEM = 0 #this is a variable to index correctly the self.elbo when using VEM 78 | 79 | 80 | def log_likelihood(self): 81 | return self._log_marginal_likelihood 82 | 83 | def test_log_likelihood(self,Xtest,Ytest): 84 | 85 | if not Xtest.__len__()==Ytest.__len__(): 86 | print("The length of the input list has to coincide with the output list") 87 | return 0 88 | 89 | test_log_likelihood, _, _, _ = self.inference_method.inference(q_u_means=self.q_u_means,q_u_chols=self.q_u_chols, X=Xtest, Y=Ytest, Z=self.Z,kern_list=self.kern_list, likelihood=self.likelihood,B_list=self.B_list, Y_metadata=self.Y_metadata, batch_scale=None) 90 | return test_log_likelihood 91 | 92 | def parameters_changed(self): 93 | f_index = self.Y_metadata['function_index'].flatten() 94 | d_index = self.Y_metadata['d_index'].flatten() 95 | T = len(self.likelihood.likelihoods_list) 96 | self.batch_scale = [] 97 | [self.batch_scale.append(float(self.Xmulti_all[t].shape[0])/float(self.Xmulti[t].shape[0])) for t in range(T)] 98 | self._log_marginal_likelihood, self.gradients, self.posteriors, _ = self.inference_method.inference(q_u_means=self.q_u_means, 99 | q_u_chols=self.q_u_chols, X=self.Xmulti, Y=self.Ymulti, Z=self.Z, 100 | kern_list=self.kern_list, likelihood=self.likelihood, 101 | B_list=self.B_list, Y_metadata=self.Y_metadata, batch_scale=self.batch_scale) 102 | D = self.likelihood.num_output_functions(self.Y_metadata) 103 | N = self.X.shape[0] 104 | M = self.num_inducing 105 | # _, B_list = util.LCM(input_dim=self.Xdim, output_dim=D, rank=1, kernels_list=self.kern_list, W_list=self.W_list, 106 | # kappa_list=self.kappa_list) 107 | Z_grad = np.zeros_like(self.Z.values) 108 | for q, kern_q in enumerate(self.kern_list): 109 | # Update the variational parameter gradients: 110 | # SVI + VEM 111 | # if self.stochastic: 112 | # if self.vem_step: 113 | # self.q_u_means[:, q:q + 1].gradient = self.gradients['dL_dmu_u'][q] 114 | # self.q_u_chols[:, q:q + 1].gradient = self.gradients['dL_dL_u'][q] 115 | # else: 116 | # self.q_u_means[:, q:q+1].gradient = np.zeros(self.gradients['dL_dmu_u'][q].shape) 117 | # self.q_u_chols[:,q:q+1].gradient = np.zeros(self.gradients['dL_dL_u'][q].shape) 118 | # else: 119 | # self.q_u_means[:, q:q + 1].gradient = self.gradients['dL_dmu_u'][q] 120 | # self.q_u_chols[:, q:q + 1].gradient = self.gradients['dL_dL_u'][q] 121 | 122 | self.q_u_means[:, q:q + 1].gradient = self.gradients['dL_dmu_u'][q] 123 | self.q_u_chols[:, q:q + 1].gradient = self.gradients['dL_dL_u'][q] 124 | 125 | # Update kernel hyperparameters: lengthscale and variance 126 | kern_q.update_gradients_full(self.gradients['dL_dKmm'][q], self.Z[:,q*self.Xdim:q*self.Xdim+self.Xdim]) 127 | grad = kern_q.gradient.copy() 128 | 129 | # Update kernel hyperparameters: W + kappa 130 | Kffdiag = [] 131 | KuqF = [] 132 | for d in range(D): 133 | Kffdiag.append(kern_q.Kdiag(self.Xmulti[f_index[d]]) * self.gradients['dL_dKdiag'][q][d]) 134 | #Kffdiag.append(self.gradients['dL_dKdiag'][q][d]) #old line 135 | #KuqF.append(self.gradients['dL_dKmn'][q][d] * kern_q.K(self.Z[:,q*self.Xdim:q*self.Xdim+self.Xdim], self.Xmulti[f_index[d]])) #old line 136 | KuqF.append(kern_q.K(self.Z[:,q*self.Xdim:q*self.Xdim+self.Xdim], self.Xmulti[f_index[d]]) * self.gradients['dL_dKmn'][q][d]) 137 | 138 | util.update_gradients_diag(self.B_list[q], Kffdiag) 139 | Bgrad = self.B_list[q].gradient.copy() 140 | util.update_gradients_Kmn(self.B_list[q], KuqF, D) 141 | Bgrad += self.B_list[q].gradient.copy() 142 | # SVI + VEM 143 | # if self.stochastic: 144 | # if self.vem_step: 145 | # self.B_list[q].gradient = np.zeros(Bgrad.shape) 146 | # else: 147 | # self.B_list[q].gradient = Bgrad 148 | # else: 149 | # self.B_list[q].gradient = Bgrad 150 | 151 | self.B_list[q].gradient = Bgrad 152 | 153 | for d in range(self.likelihood.num_output_functions(self.Y_metadata)): 154 | #kern_q.update_gradients_full(self.gradients['dL_dKmn'][q][d], self.Z[:,q*self.Xdim:q*self.Xdim+self.Xdim], self.Xmulti[f_index[d]]) 155 | kern_q.update_gradients_full(self.B_list[q].W[d] * self.gradients['dL_dKmn'][q][d],self.Z[:, q * self.Xdim:q * self.Xdim + self.Xdim],self.Xmulti[f_index[d]]) 156 | 157 | #grad += B_list[q].W[d]*kern_q.gradient.copy() #old line 158 | #grad += self.B_list[q].W[d] * kern_q.gradient.copy() #Juan wrote this 159 | grad += kern_q.gradient.copy() # Juan wrote this 160 | 161 | #kern_q.update_gradients_diag(self.gradients['dL_dKdiag'][q][d], self.Xmulti[f_index[d]]) 162 | kern_q.update_gradients_diag(self.B_list[q].B[d,d] *self.gradients['dL_dKdiag'][q][d], self.Xmulti[f_index[d]]) 163 | #grad += B_list[q].B[d,d] * kern_q.gradient.copy() #old line 164 | #grad += self.B_list[q].B[d, d] * kern_q.gradient.copy() #Juan wrote this line 165 | grad += kern_q.gradient.copy() # Juan wrote this line 166 | # SVI + VEM 167 | # if self.stochastic: 168 | # if self.vem_step: 169 | # kern_q.gradient = np.zeros(grad.shape) 170 | # else: 171 | # kern_q.gradient = grad 172 | # else: 173 | # kern_q.gradient = grad 174 | 175 | kern_q.gradient = grad 176 | 177 | if not self.Z.is_fixed: 178 | Z_grad[:,q*self.Xdim:q*self.Xdim+self.Xdim] += kern_q.gradients_X(self.gradients['dL_dKmm'][q], self.Z[:,q*self.Xdim:q*self.Xdim+self.Xdim]).copy() 179 | for d in range(self.likelihood.num_output_functions(self.Y_metadata)): 180 | Z_grad[:,q*self.Xdim:q*self.Xdim+self.Xdim]+= self.B_list[q].W[d]*kern_q.gradients_X(self.gradients['dL_dKmn'][q][d], self.Z[:, q * self.Xdim:q * self.Xdim + self.Xdim],self.Xmulti[f_index[d]]).copy() 181 | #Z_grad[:,q*self.Xdim:q*self.Xdim+self.Xdim] += kern_q.gradients_X(self.B_list[q].W[d]*self.gradients['dL_dKmn'][q][d], self.Z[:,q*self.Xdim:q*self.Xdim+self.Xdim], self.Xmulti[f_index[d]]) 182 | 183 | #self.Z.gradient[:] = Z_grad 184 | self.Z.gradient[:] = Z_grad 185 | # if not self.Z.is_fixed: 186 | # #SVI + VEM 187 | # if self.stochastic: 188 | # if self.vem_step: 189 | # self.Z.gradient[:] = np.zeros(Z_grad.shape) 190 | # else: 191 | # self.Z.gradient[:] = Z_grad 192 | # else: 193 | # self.Z.gradient[:] = Z_grad 194 | # 195 | # self.Z.gradient[:] = Z_grad 196 | 197 | def set_data(self, X, Y): 198 | """ 199 | Set the data without calling parameters_changed to avoid wasted computation 200 | If this is called by the stochastic_grad function this will immediately update the gradients 201 | """ 202 | self.Xmulti, self.Ymulti = X, Y 203 | 204 | def new_batch(self): 205 | """ 206 | Return a new batch of X and Y by taking a chunk of data from the complete X and Y 207 | """ 208 | T = len(self.likelihood.likelihoods_list) 209 | Xmulti_batch = [] 210 | Ymulti_batch = [] 211 | for t in range(T): 212 | i_task = next(self.slicer_list[t]) 213 | Xmulti_batch.append(self.Xmulti_all[t][i_task]) 214 | Ymulti_batch.append(self.Ymulti_all[t][i_task]) 215 | return Xmulti_batch, Ymulti_batch 216 | 217 | def stochastic_grad(self, parameters): 218 | self.set_data(*self.new_batch()) 219 | stochastic_gradients = self._grads(parameters) 220 | # if self.vem_step: 221 | # if self.ve_count > 2: 222 | # self.ve_count = 0 223 | # self.vem_step = False 224 | # else: 225 | # self.ve_count += 1 226 | # else: 227 | # self.vem_step = True 228 | return stochastic_gradients 229 | 230 | def callback(self, i, max_iter, verbose=True, verbose_plot=False): 231 | ll = self.log_likelihood() 232 | self.elbo[self.index_VEM+i['n_iter'],0] = self.log_likelihood()[0] 233 | if verbose: 234 | if i['n_iter']%50 ==0: 235 | print('svi - iteration '+str(i['n_iter'])+'/'+str(int(max_iter))) 236 | 237 | if verbose_plot: 238 | plt.ion() 239 | plt.show() 240 | plt.plot(i['n_iter'],ll,'k+') 241 | plt.draw() 242 | plt.pause(1e-5) 243 | 244 | if i['n_iter'] > max_iter: 245 | return True 246 | return False 247 | 248 | 249 | def posteriors_F(self, Xnew, kern_list=None): 250 | # This function returns all the q(f*) associated to each output (It is the ) 251 | # We assume that Xnew can be a list of length equal to the number of likelihoods defined for the HetMOGP 252 | # or Xnew can be a numpy array so that we can replicate it per each outout 253 | if kern_list is None: 254 | kern_list = self.kern_list 255 | 256 | if isinstance(Xnew, list): 257 | Xmulti_all_new = Xnew 258 | else: 259 | Xmulti_all_new = [] 260 | for i in range(self.num_output_funcs): 261 | Xmulti_all_new.append(Xnew.copy()) 262 | 263 | posteriors_F = self.inference_method.inference(q_u_means=self.q_u_means, 264 | q_u_chols=self.q_u_chols, X=Xmulti_all_new, Y=self.Ymulti_all, 265 | Z=self.Z, 266 | kern_list=self.kern_list, likelihood=self.likelihood, 267 | B_list=self.B_list, Y_metadata=self.Y_metadata, predictive=True) 268 | return posteriors_F 269 | 270 | def predictive_f_star(self, Xnew, latent_function_ind=None, kern_list=None): 271 | #This function is just to select one of the predictive Posteriors_F_star associated to each output 272 | if latent_function_ind is None: 273 | latent_function_ind = 0 274 | 275 | posterior = self.posteriors_F(Xnew=Xnew)[latent_function_ind] 276 | return posterior.mean.copy(), np.diag(posterior.covariance).copy()[:,None] 277 | 278 | def predict(self, Xnew): 279 | """ 280 | Description: Make a prediction of p(y*|y,Xnew) 281 | The function retuns two lists m_pred and v_pred with length equal to the number of outputs 282 | m_pred contains the prediction of each output 283 | v_pred contains the variance of each prediction 284 | """ 285 | D = self.num_output_funcs #This D is the number of total latent functions that parameterize all Likelihooods; it is literally J= \sum Jd (in the paper) 286 | f_index = self.Y_metadata['function_index'].flatten() 287 | d_index = self.Y_metadata['d_index'].flatten() 288 | m_F_pred = [] 289 | v_F_pred = [] 290 | 291 | if isinstance(Xnew, list): 292 | Xpred = Xnew 293 | else: 294 | Xpred = [] 295 | for i in range(self.num_output_funcs): 296 | Xpred.append(Xnew.copy()) 297 | 298 | posteriors_F = self.posteriors_F(Xnew=Xpred) 299 | for t in range(len(self.likelihood.likelihoods_list)): 300 | _,num_f_task,_ = self.likelihood.likelihoods_list[t].get_metadata() 301 | m_task_pred = np.empty((Xpred[t].shape[0], num_f_task)) 302 | v_task_pred = np.empty((Xpred[t].shape[0], num_f_task)) 303 | for d in range(D): 304 | if f_index[d] == t: 305 | #m_task_pred[:,d_index[d],None], v_task_pred[:,d_index[d],None] = self._raw_predict_f(Xpred[f_index[d]], output_function_ind=d) 306 | m_task_pred[:, d_index[d], None], v_task_pred[:, d_index[d], None] = posteriors_F[d].mean.copy(), np.diag(posteriors_F[d].covariance.copy()) [:,None] 307 | 308 | m_F_pred.append(m_task_pred) 309 | v_F_pred.append(v_task_pred) 310 | 311 | m_pred, v_pred = self.likelihood.predictive(m_F_pred, v_F_pred, self.Y_metadata) 312 | return m_pred, v_pred 313 | 314 | def negative_log_predictive(self, Xtest, Ytest, num_samples=1000): 315 | # The variables Xtest and Ytest have to be lists with the same length, and 316 | # each position Xtest[i] and Ytest[i] have to be N_i X D and N_i x 1 respectively, 317 | # where N_i is the number of data to test per output. 318 | # For instance N_1 (output 1) can be different to N_2 (output 2) 319 | 320 | f_index = self.Y_metadata['function_index'].flatten() 321 | T = len(self.Ymulti) 322 | mu_F_star = [] 323 | v_F_star = [] 324 | NLPD = np.zeros(T) 325 | 326 | posteriors_F = self.posteriors_F(Xnew=Xtest) 327 | for t in range(T): 328 | mu_F_star_task = np.empty((Ytest[t].shape[0],1)) 329 | v_F_star_task = np.empty((Ytest[t].shape[0], 1)) 330 | for d in range(self.num_output_funcs): 331 | if f_index[d] == t: 332 | #m_fd_star, v_fd_star = self._raw_predict_f(Xtest[t], output_function_ind=d) 333 | m_fd_star, v_fd_star = posteriors_F[d].mean.copy(), np.diag(posteriors_F[d].covariance.copy())[:, None] 334 | mu_F_star_task = np.hstack((mu_F_star_task, m_fd_star)) 335 | v_F_star_task = np.hstack((v_F_star_task, v_fd_star)) 336 | 337 | mu_F_star.append(mu_F_star_task[:,1:]) 338 | v_F_star.append(v_F_star_task[:,1:]) 339 | 340 | Log_Pred_Density = self.likelihood.log_predictive(Ytest, mu_F_star, v_F_star, Y_metadata=self.Y_metadata, num_samples=num_samples) 341 | 342 | "NLPD loss is the negative mean of all the log predictive: frac{-1}{N}\sum^N_{n=1} \log p(y_n=Ytest_n|Xtest_n)" 343 | 344 | for i,LPD in enumerate(Log_Pred_Density): 345 | NLPD[i] = -LPD.mean() #Negative Log Predictive Density Loss 346 | 347 | return NLPD 348 | -------------------------------------------------------------------------------- /hetmogp/svmogp_inf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | import GPy 7 | from GPy.inference.latent_function_inference import LatentFunctionInference 8 | from GPy.inference.latent_function_inference.posterior import Posterior 9 | from GPy.util import choleskies 10 | from GPy.util import linalg 11 | from hetmogp import util 12 | from collections import namedtuple 13 | from scipy.linalg.blas import dtrmm 14 | import matplotlib.pyplot as plt 15 | import random 16 | 17 | qfd = namedtuple("q_fd", "m_fd v_fd Kfdu Afdu S_fd") 18 | qu = namedtuple("q_U", "mu_u chols_u") 19 | pu = namedtuple("p_U", "Kuu Luu Kuui") 20 | 21 | class SVMOGPInf(LatentFunctionInference): 22 | 23 | def inference(self, q_u_means, q_u_chols, X, Y, Z, kern_list, likelihood, B_list, Y_metadata, KL_scale=1.0, 24 | batch_scale=None, predictive=False): 25 | M = Z.shape[0] 26 | T = len(Y) 27 | if batch_scale is None: 28 | batch_scale = [1.0]*T 29 | Ntask = [] 30 | [Ntask.append(Y[t].shape[0]) for t in range(T)] 31 | Q = len(kern_list) 32 | D = likelihood.num_output_functions(Y_metadata) 33 | Kuu, Luu, Kuui = util.latent_funs_cov(Z, kern_list) 34 | p_U = pu(Kuu=Kuu, Luu=Luu, Kuui=Kuui) 35 | q_U = qu(mu_u=q_u_means, chols_u=q_u_chols) 36 | 37 | # for every latent function f_d calculate q(f_d) and keep it as q(F): 38 | q_F = [] 39 | posteriors_F = [] 40 | f_index = Y_metadata['function_index'].flatten() 41 | d_index = Y_metadata['d_index'].flatten() 42 | 43 | for d in range(D): 44 | Xtask = X[f_index[d]] 45 | q_fd = self.calculate_q_f(X=Xtask, Z=Z, q_U=q_U, p_U=p_U, kern_list=kern_list, B=B_list, 46 | M=M, N=Xtask.shape[0], Q=Q, D=D, d=d) 47 | # Posterior objects for output functions (used in prediction) 48 | posterior_fd = Posterior(mean=q_fd.m_fd.copy(), cov=q_fd.S_fd.copy(), 49 | K=util.function_covariance(X=Xtask, B=B_list, kernel_list=kern_list, d=d), 50 | prior_mean=np.zeros(q_fd.m_fd.shape)) 51 | posteriors_F.append(posterior_fd) 52 | q_F.append(q_fd) 53 | 54 | mu_F = [] 55 | v_F = [] 56 | for t in range(T): 57 | mu_F_task = np.empty((X[t].shape[0],1)) 58 | v_F_task = np.empty((X[t].shape[0], 1)) 59 | for d, q_fd in enumerate(q_F): 60 | if f_index[d] == t: 61 | mu_F_task = np.hstack((mu_F_task, q_fd.m_fd)) 62 | v_F_task = np.hstack((v_F_task, q_fd.v_fd)) 63 | 64 | mu_F.append(mu_F_task[:,1:]) 65 | v_F.append(v_F_task[:,1:]) 66 | 67 | # posterior_Fnew for predictive 68 | if predictive: 69 | return posteriors_F 70 | # inference for rest of cases 71 | else: 72 | # Variational Expectations 73 | VE = likelihood.var_exp(Y, mu_F, v_F, Y_metadata) 74 | VE_dm, VE_dv = likelihood.var_exp_derivatives(Y, mu_F, v_F, Y_metadata) 75 | for t in range(T): 76 | VE[t] = VE[t]*batch_scale[t] 77 | VE_dm[t] = VE_dm[t]*batch_scale[t] 78 | VE_dv[t] = VE_dv[t]*batch_scale[t] 79 | 80 | # KL Divergence 81 | KL = self.calculate_KL(q_U=q_U, p_U=p_U,M=M, Q=Q) 82 | 83 | # Log Marginal log(p(Y)) 84 | F = 0 85 | for t in range(T): 86 | F += VE[t].sum() 87 | 88 | log_marginal = F - KL 89 | 90 | # Gradients and Posteriors 91 | dL_dS_u = [] 92 | dL_dmu_u = [] 93 | dL_dL_u = [] 94 | dL_dKmm = [] 95 | dL_dKmn = [] 96 | dL_dKdiag = [] 97 | posteriors = [] 98 | for q in range(Q): 99 | (dL_dmu_q, dL_dL_q, dL_dS_q, posterior_q, dL_dKqq, dL_dKdq, dL_dKdiag_q) = self.calculate_gradients(q_U=q_U, p_U=p_U, 100 | q_F=q_F,VE_dm=VE_dm, VE_dv=VE_dv, Ntask=Ntask, M=M, Q=Q, D=D, f_index=f_index, d_index=d_index, q=q) 101 | dL_dmu_u.append(dL_dmu_q) 102 | dL_dL_u.append(dL_dL_q) 103 | dL_dS_u.append(dL_dS_q) 104 | dL_dKmm.append(dL_dKqq) 105 | dL_dKmn.append(dL_dKdq) 106 | dL_dKdiag.append(dL_dKdiag_q) 107 | posteriors.append(posterior_q) 108 | 109 | gradients = {'dL_dmu_u':dL_dmu_u, 'dL_dL_u':dL_dL_u,'dL_dS_u':dL_dS_u, 'dL_dKmm':dL_dKmm, 'dL_dKmn':dL_dKmn, 'dL_dKdiag': dL_dKdiag} 110 | 111 | return log_marginal, gradients, posteriors, posteriors_F 112 | 113 | def calculate_gradients(self, q_U, p_U, q_F, VE_dm, VE_dv, Ntask, M, Q, D, f_index, d_index,q): 114 | """ 115 | Calculates gradients of the Log-marginal distribution p(Y) wrt variational 116 | parameters mu_q, S_q 117 | """ 118 | # Algebra for q(u) and p(u): 119 | m_u = q_U.mu_u.copy() 120 | L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) 121 | S_u = np.empty((Q, M, M)) 122 | [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] 123 | Kuu = p_U.Kuu.copy() 124 | Luu = p_U.Luu.copy() 125 | Kuui = p_U.Kuui.copy() 126 | S_qi, _ = linalg.dpotri(np.asfortranarray(L_u[q, :, :])) 127 | 128 | if np.any(np.isinf(S_qi)): 129 | raise ValueError("Sqi: Cholesky representation unstable") 130 | 131 | # KL Terms 132 | dKL_dmu_q = np.dot(Kuui[q,:,:],m_u[:, q, None]) 133 | dKL_dS_q = 0.5 * (Kuui[q,:,:] - S_qi) 134 | dKL_dKqq = 0.5 * Kuui[q,:,:] - 0.5 * Kuui[q,:,:].dot(S_u[q, :, :]).dot(Kuui[q,:,:]) \ 135 | - 0.5 * np.dot(Kuui[q,:,:],np.dot(m_u[:, q, None],m_u[:, q, None].T)).dot(Kuui[q,:,:].T) 136 | 137 | # VE Terms 138 | dVE_dmu_q = np.zeros((M, 1)) 139 | dVE_dS_q = np.zeros((M, M)) 140 | dVE_dKqq = np.zeros((M, M)) 141 | dVE_dKqd = [] 142 | dVE_dKdiag = [] 143 | 144 | for d, q_fd in enumerate(q_F): 145 | Nt = Ntask[f_index[d]] 146 | dVE_dmu_q += np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:,d_index[d]])[:, None] 147 | Adv = q_fd.Afdu[q,:,:].T * VE_dv[f_index[d]][:,d_index[d],None].T 148 | Adv = np.ascontiguousarray(Adv) 149 | AdvA = np.dot(Adv.reshape(-1, Nt), q_fd.Afdu[q, :, :]).reshape(M, M) 150 | dVE_dS_q += AdvA 151 | 152 | # Derivatives dKuquq 153 | tmp_dv = np.dot(AdvA, S_u[q, :, :]).dot(Kuui[q,:,:]) 154 | dVE_dKqq += AdvA - tmp_dv - tmp_dv.T 155 | Adm = np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:,d_index[d],None]) 156 | dVE_dKqq += - np.dot(Adm, np.dot(Kuui[q,:,:], m_u[:, q, None]).T) 157 | 158 | # Derivatives dKuqfd 159 | tmp = np.dot(S_u[q, :, :], Kuui[q,:,:]) 160 | tmp = 2. * (tmp - np.eye(M)) 161 | dve_kqd = np.dot(np.dot(Kuui[q,:,:], m_u[:, q, None]), VE_dm[f_index[d]][:,d_index[d],None].T) 162 | dve_kqd += np.dot(tmp.T, Adv) 163 | dVE_dKqd.append(dve_kqd) 164 | 165 | # Derivatives dKdiag 166 | dVE_dKdiag.append(VE_dv[f_index[d]][:,d_index[d]]) 167 | 168 | dVE_dKqq = 0.5 * (dVE_dKqq + dVE_dKqq.T) 169 | # Sum of VE and KL terms 170 | dL_dmu_q = dVE_dmu_q - dKL_dmu_q 171 | dL_dS_q = dVE_dS_q - dKL_dS_q 172 | dL_dKqq = dVE_dKqq - dKL_dKqq 173 | dL_dKdq = dVE_dKqd 174 | dL_dKdiag = dVE_dKdiag 175 | 176 | # Pass S_q gradients to its low-triangular representation L_q 177 | chol_u = q_U.chols_u.copy() 178 | L_q = choleskies.flat_to_triang(chol_u[:,q:q+1]) 179 | dL_dL_q = 2. * np.array([np.dot(a, b) for a, b in zip(dL_dS_q[None,:,:], L_q)]) 180 | dL_dL_q = choleskies.triang_to_flat(dL_dL_q) 181 | 182 | # Posterior 183 | posterior_q = Posterior(mean=m_u[:, q, None], cov=S_u[q, :, :], K=Kuu[q,:,:], prior_mean=np.zeros(m_u[:, q, None].shape)) 184 | 185 | return dL_dmu_q, dL_dL_q, dL_dS_q, posterior_q, dL_dKqq, dL_dKdq, dL_dKdiag 186 | 187 | 188 | def calculate_q_f(self, X, Z, q_U, p_U, kern_list, B, M, N, Q, D, d): 189 | """ 190 | Calculates the mean and variance of q(f_d) as 191 | Equation: E_q(U)\{p(f_d|U)\} 192 | """ 193 | # Algebra for q(u): 194 | m_u = q_U.mu_u.copy() 195 | L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) 196 | S_u = np.empty((Q, M, M)) 197 | [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] 198 | 199 | # Algebra for p(f_d|u): 200 | Kfdu = util.cross_covariance(X, Z, B, kern_list, d) 201 | Kuu = p_U.Kuu.copy() 202 | Luu = p_U.Luu.copy() 203 | Kuui = p_U.Kuui.copy() 204 | Kff = util.function_covariance(X, B, kern_list, d) 205 | Kff_diag = np.diag(Kff) 206 | 207 | # Algebra for q(f_d) = E_{q(u)}[p(f_d|u)] 208 | Afdu = np.empty((Q, N, M)) #Afdu = K_{fduq}Ki_{uquq} 209 | m_fd = np.zeros((N, 1)) 210 | v_fd = np.zeros((N, 1)) 211 | S_fd = np.zeros((N, N)) 212 | v_fd += Kff_diag[:,None] 213 | S_fd += Kff 214 | for q in range(Q): 215 | # Expectation part 216 | R, _ = linalg.dpotrs(np.asfortranarray(Luu[q, :, :]), Kfdu[:, q * M:(q * M) + M].T) 217 | Afdu[q, :, :] = R.T 218 | m_fd += np.dot(Afdu[q, :, :], m_u[:, q, None]) #exp 219 | tmp = dtrmm(alpha=1.0, a=L_u[q, :, :].T, b=R, lower=0, trans_a=0) 220 | v_fd += np.sum(np.square(tmp), 0)[:,None] - np.sum(R * Kfdu[:, q * M:(q * M) + M].T,0)[:,None] #exp 221 | S_fd += np.dot(np.dot(R.T,S_u[q, :, :]),R) - np.dot(Kfdu[:, q * M:(q * M) + M],R) 222 | 223 | if (v_fd<0).any(): 224 | print('v negative!') 225 | 226 | q_fd = qfd(m_fd=m_fd, v_fd=v_fd, Kfdu=Kfdu, Afdu=Afdu, S_fd=S_fd) 227 | return q_fd 228 | 229 | def calculate_KL(self, q_U, p_U, M, Q): 230 | """ 231 | Calculates the KL divergence (see KL-div for multivariate normals) 232 | Equation: \sum_Q KL{q(uq)|p(uq)} 233 | """ 234 | # Algebra for q(u): 235 | m_u = q_U.mu_u.copy() 236 | L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) 237 | S_u = np.empty((Q, M, M)) 238 | [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] 239 | 240 | # Algebra for p(u): 241 | Kuu = p_U.Kuu.copy() 242 | Luu = p_U.Luu.copy() 243 | Kuui = p_U.Kuui.copy() 244 | 245 | KL = 0 246 | for q in range(Q): 247 | KL += 0.5 * np.sum(Kuui[q, :, :] * S_u[q, :, :]) \ 248 | + 0.5 * np.dot(m_u[:, q, None].T,np.dot(Kuui[q,:,:],m_u[:, q, None])) \ 249 | - 0.5 * M \ 250 | + 0.5 * 2. * np.sum(np.log(np.abs(np.diag(Luu[q, :, :])))) \ 251 | - 0.5 * 2. * np.sum(np.log(np.abs(np.diag(L_u[q, :, :])))) 252 | return KL 253 | -------------------------------------------------------------------------------- /hetmogp/util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | 5 | from GPy import kern 6 | from GPy.util import linalg 7 | import random 8 | import warnings 9 | import numpy as np 10 | import climin 11 | from functools import partial 12 | import matplotlib.pyplot as plt 13 | import sys 14 | # import VariationalOptimization as vo 15 | 16 | 17 | def get_batch_scales(X_all, X): 18 | batch_scales = [] 19 | for t, X_all_task in enumerate(X_all): 20 | batch_scales.append(float(X_all_task.shape[0]) / float(X[t].shape[0])) 21 | return batch_scales 22 | 23 | def true_u_functions(X_list, Q): 24 | u_functions = [] 25 | amplitude = (1.5-0.5)*np.random.rand(Q,3) + 0.5 26 | freq = (3-1)*np.random.rand(Q,3) + 1 27 | shift = 2*np.random.rand(Q,3) 28 | for X in X_list: 29 | u_task = np.empty((X.shape[0],Q)) 30 | for q in range(Q): 31 | u_task[:,q,None] = 3*amplitude[q,0]*np.cos(freq[q,0]*np.pi*X + shift[q,0]*np.pi) - \ 32 | 2*amplitude[q,1]*np.sin(2*freq[q,1]*np.pi*X + shift[q,1]*np.pi) + \ 33 | amplitude[q,2] * np.cos(4*freq[q, 2] * np.pi * X + shift[q, 2] * np.pi) 34 | 35 | u_functions.append(u_task) 36 | return u_functions 37 | 38 | def true_f_functions(true_u, W_list, D, likelihood_list, Y_metadata): 39 | true_f = [] 40 | f_index = Y_metadata['function_index'].flatten() 41 | d_index = Y_metadata['d_index'].flatten() 42 | for t, u_task in enumerate(true_u): 43 | Ntask = u_task.shape[0] 44 | _, num_f_task, _ = likelihood_list[t].get_metadata() 45 | F = np.zeros((Ntask, num_f_task)) 46 | for q, W in enumerate(W_list): 47 | for d in range(D): 48 | if f_index[d] == t: 49 | F[:,d_index[d],None] += np.tile(W[d].T, (Ntask, 1)) * u_task[:, q, None] 50 | 51 | true_f.append(F) 52 | return true_f 53 | 54 | def mini_slices(n_samples, batch_size): 55 | """Yield slices of size `batch_size` that work with a container of length 56 | `n_samples`.""" 57 | n_batches, rest = divmod(n_samples, batch_size) 58 | if rest != 0: 59 | n_batches += 1 60 | 61 | return [slice(i * batch_size, (i + 1) * batch_size) for i in range(n_batches)] 62 | 63 | 64 | def draw_mini_slices(n_samples, batch_size, with_replacement=False): 65 | slices = mini_slices(n_samples, batch_size) 66 | idxs = list(range(len(slices))) # change this line 67 | 68 | if with_replacement: 69 | yield random.choice(slices) 70 | else: 71 | while True: 72 | random.shuffle(list(idxs)) 73 | for i in idxs: 74 | yield slices[i] 75 | 76 | 77 | def latent_functions_prior(Q, lenghtscale=None, variance=None, input_dim=None): 78 | if lenghtscale is None: 79 | lenghtscale = np.random.rand(Q) 80 | else: 81 | lenghtscale = lenghtscale 82 | 83 | if variance is None: 84 | variance = np.random.rand(Q) 85 | else: 86 | variance = variance 87 | kern_list = [] 88 | for q in range(Q): 89 | # For the one dimensional case 90 | kern_q = kern.Mix_Integral_extend(input_dim=input_dim, lengthscale=[lenghtscale[q]], variance=variance[q], name='Mix_Integral_extend') #+ kern.White(input_dim) 91 | # For the two dimensional case 92 | # kern_q = kern.Mix_Integral_extend(input_dim=input_dim, lengthscale=[lenghtscale[q],lenghtscale[q+1]], variance=variance[q], name='Mix_Integral_extend') #+ kern.White(input_dim) 93 | kern_q.name = 'kern_q'+str(q) 94 | kern_list.append(kern_q) 95 | return kern_list 96 | 97 | def random_W_kappas(Q,D,rank, experiment=False): 98 | W_list = [] 99 | kappa_list = [] 100 | for q in range(Q): 101 | p = np.random.binomial(n=1, p=0.5*np.ones((D,1))) 102 | Ws = p*np.random.normal(loc=0.5, scale=0.5, size=(D,1)) - (p-1)*np.random.normal(loc=-0.5, scale=0.5, size=(D,1)) 103 | W_list.append(Ws / np.sqrt(rank)) # deberían ser tanto positivos como negativos 104 | if experiment: 105 | kappa_list.append(np.zeros(D)) 106 | else: 107 | kappa_list.append(np.zeros(D)) 108 | return W_list, kappa_list 109 | 110 | 111 | def ICM(input_dim, output_dim, kernel, rank, W=None, kappa=None, name='ICM'): 112 | """ 113 | Builds a kernel for an Intrinsic Coregionalization Model 114 | :input_dim: Input dimensionality (does not include dimension of indices) 115 | :num_outputs: Number of outputs 116 | :param kernel: kernel that will be multiplied by the coregionalize kernel (matrix B). 117 | :type kernel: a GPy kernel 118 | :param W_rank: number tuples of the corregionalization parameters 'W' 119 | :type W_rank: integer 120 | """ 121 | kern_q = kernel.copy() 122 | if kernel.input_dim != input_dim: 123 | kernel.input_dim = input_dim 124 | warnings.warn("kernel's input dimension overwritten to fit input_dim parameter.") 125 | B = kern.Coregionalize(input_dim=input_dim, output_dim=output_dim, rank=rank, W=W, kappa=kappa) 126 | B.name = name 127 | K = kern_q.prod(B, name=name) 128 | return K, B 129 | 130 | 131 | def LCM(input_dim, output_dim, kernels_list, W_list, kappa_list, rank, name='B_q'): 132 | """ 133 | Builds a kernel for an Linear Coregionalization Model 134 | :input_dim: Input dimensionality (does not include dimension of indices) 135 | :num_outputs: Number of outputs 136 | :param kernel: kernel that will be multiplied by the coregionalize kernel (matrix B). 137 | :type kernel: a GPy kernel 138 | :param W_rank: number tuples of the corregionalization parameters 'W' 139 | :type W_rank: integer 140 | """ 141 | B_q = [] 142 | K, B = ICM(input_dim, output_dim, kernels_list[0], W=W_list[0], kappa=kappa_list[0], rank=rank, name='%s%s' %(name,0)) 143 | B_q.append(B) 144 | for q, kernel in enumerate(kernels_list[1:]): 145 | Kq, Bq = ICM(input_dim, output_dim, kernel, W=W_list[q+1], kappa=kappa_list[q+1], rank=rank, name='%s%s' %(name,q+1)) 146 | B_q.append(Bq) 147 | K += Kq 148 | return K, B_q 149 | 150 | def cross_covariance(X, Z, B, kernel_list, d): 151 | """ 152 | Builds the cross-covariance cov[f_d(x),u(z)] of a Multi-output GP 153 | :param X: Input data 154 | :param Z: Inducing Points 155 | :param B: Coregionalization matric 156 | :param kernel_list: Kernels of u_q functions 157 | :param d: output function f_d index 158 | :return: Kfdu 159 | """ 160 | N,_ = X.shape 161 | M,Dz = Z.shape 162 | Q = len(B) 163 | Xdim = int(Dz/Q) 164 | Kfdu = np.empty([N,M*Q]) 165 | for q, B_q in enumerate(B): 166 | Kfdu[:, q * M:(q * M) + M] = B_q.W[d] * kernel_list[q].K(X, Z[:, q*Xdim:q*Xdim+Xdim]) 167 | #Kfdu[:, q * M:(q * M) + M] = B_q.B[d,d] * kernel_list[q].K(X, Z[:,q,None]) 168 | return Kfdu 169 | 170 | def function_covariance(X, B, kernel_list, d): 171 | """ 172 | Builds the cross-covariance Kfdfd = cov[f_d(x),f_d(x)] of a Multi-output GP 173 | :param X: Input data 174 | :param B: Coregionalization matrix 175 | :param kernel_list: Kernels of u_q functions 176 | :param d: output function f_d index 177 | :return: Kfdfd 178 | """ 179 | N,_ = X.shape 180 | Kfdfd = np.zeros((N, N)) 181 | for q, B_q in enumerate(B): 182 | Kfdfd += B_q.B[d,d]*kernel_list[q].K(X,X) 183 | return Kfdfd 184 | 185 | def latent_funs_cov(Z, kernel_list): 186 | """ 187 | Builds the full-covariance cov[u(z),u(z)] of a Multi-output GP 188 | for a Sparse approximation 189 | :param Z: Inducing Points 190 | :param kernel_list: Kernels of u_q functions priors 191 | :return: Kuu 192 | """ 193 | Q = len(kernel_list) 194 | M,Dz = Z.shape 195 | Xdim = int(Dz/Q) 196 | #Kuu = np.zeros([Q*M,Q*M]) 197 | Kuu = np.empty((Q, M, M)) 198 | Luu = np.empty((Q, M, M)) 199 | Kuui = np.empty((Q, M, M)) 200 | for q, kern in enumerate(kernel_list): 201 | Kuu[q, :, :] = kern.K(Z[:,q*Xdim:q*Xdim+Xdim], Z[:,q*Xdim:q*Xdim+Xdim]) 202 | Luu[q, :, :] = linalg.jitchol(Kuu[q, :, :],maxtries=10) 203 | Kuui[q, :, :], _ = linalg.dpotri(np.asfortranarray(Luu[q, :, :])) 204 | return Kuu, Luu, Kuui 205 | 206 | def generate_toy_U(X,Q): 207 | arg = np.tile(X, (1,Q)) 208 | rnd = np.tile(np.random.rand(1,Q), (X.shape)) 209 | U = 2*rnd*np.sin(10*rnd*arg + np.random.randn(1)) + 2*rnd*np.cos(20*rnd*arg + np.random.randn(1)) 210 | return U 211 | 212 | def _gradient_reduce_numpy(coreg, dL_dK, index, index2): 213 | index, index2 = index[:,0], index2[:,0] 214 | dL_dK_small = np.zeros_like(coreg.B) 215 | for i in range(coreg.output_dim): 216 | tmp1 = dL_dK[index==i] 217 | for j in range(coreg.output_dim): 218 | dL_dK_small[j,i] = tmp1[:,index2==j].sum() 219 | return dL_dK_small 220 | 221 | def _gradient_B(coreg, dL_dK, index, index2): 222 | index, index2 = index[:,0], index2[:,0] 223 | B = coreg.B 224 | isqrtB = 1 / np.sqrt(B) 225 | dL_dK_small = np.zeros_like(B) 226 | for i in range(coreg.output_dim): 227 | tmp1 = dL_dK[index==i] 228 | for j in range(coreg.output_dim): 229 | dL_dK_small[j,i] = (0.5 * isqrtB[i,j] * tmp1[:,index2==j]).sum() 230 | return dL_dK_small 231 | 232 | def update_gradients_diag(coreg, dL_dKdiag): 233 | dL_dKdiag_small = np.array([dL_dKdiag_task.sum() for dL_dKdiag_task in dL_dKdiag]) 234 | coreg.W.gradient = 2.*coreg.W*dL_dKdiag_small[:, None] # should it be 2*..? R/Yes Pablo, it should be :) 235 | coreg.kappa.gradient = dL_dKdiag_small 236 | 237 | def update_gradients_full(coreg, dL_dK, X, X2=None): 238 | index = np.asarray(X, dtype=np.int) 239 | if X2 is None: 240 | index2 = index 241 | else: 242 | index2 = np.asarray(X2, dtype=np.int) 243 | 244 | dL_dK_small = _gradient_reduce_numpy(coreg, dL_dK, index, index2) 245 | dkappa = np.diag(dL_dK_small).copy() 246 | dL_dK_small += dL_dK_small.T 247 | dW = (coreg.W[:, None, :]*dL_dK_small[:, :, None]).sum(0) 248 | 249 | coreg.W.gradient = dW 250 | coreg.kappa.gradient = dkappa 251 | 252 | def update_gradients_Kmn(coreg, dL_dK, D): 253 | dW = np.zeros((D,1)) 254 | dkappa = np.zeros((D)) # not used 255 | for d in range(D): 256 | dW[d,:] = dL_dK[d].sum() 257 | 258 | coreg.W.gradient = dW 259 | coreg.kappa.gradient = dkappa 260 | 261 | def gradients_coreg(coreg, dL_dK, X, X2=None): 262 | index = np.asarray(X, dtype=np.int) 263 | if X2 is None: 264 | index2 = index 265 | else: 266 | index2 = np.asarray(X2, dtype=np.int) 267 | 268 | dK_dB = _gradient_B(coreg, dL_dK, index, index2) 269 | dkappa = np.diag(dK_dB).copy() 270 | dK_dB += dK_dB.T 271 | dW = (coreg.W[:, None, :]*dK_dB[:, :, None]).sum(0) 272 | coreg.W.gradient = dW 273 | coreg.kappa.gradient = dkappa 274 | 275 | def gradients_coreg_diag(coreg, dL_dKdiag, kern_q, X, X2=None): 276 | # dL_dKdiag is (NxD) 277 | if X2 is None: 278 | X2 = X 279 | N,D = dL_dKdiag.shape 280 | matrix_sum = np.zeros((D,1)) 281 | for d in range(D): 282 | matrix_sum[d,0] = np.sum(np.diag(kern_q.K(X, X2)) * dL_dKdiag[:,d,None]) 283 | 284 | dW = 2 * coreg.W * matrix_sum 285 | dkappa = matrix_sum 286 | return dW, dkappa 287 | 288 | 289 | def vem_algorithm(model, vem_iters=None, maxIter_perVEM = None, step_rate=None ,verbose=False, optZ=True, verbose_plot=False, non_chained=True): 290 | if vem_iters is None: 291 | vem_iters = 5 292 | if maxIter_perVEM is None: 293 | maxIter_perVEM = 100 294 | 295 | model['.*.kappa'].fix() # must be always fixed 296 | #model.elbo = np.empty((vem_iters,1)) 297 | 298 | if model.batch_size is None: 299 | 300 | for i in range(vem_iters): 301 | # VARIATIONAL E-STEP 302 | model['.*.lengthscale'].fix() 303 | model['.*.variance'].fix() 304 | model.Z.fix() 305 | model['.*.W'].fix() 306 | 307 | model.q_u_means.unfix() 308 | model.q_u_chols.unfix() 309 | model.optimize(messages=verbose, max_iters=maxIter_perVEM) 310 | print('iteration ('+str(i+1)+') VE step, log_likelihood='+str(model.log_likelihood().flatten())) 311 | sys.stdout.flush() 312 | sys.stderr.flush() 313 | 314 | # VARIATIONAL M-STEP 315 | model['.*.lengthscale'].unfix() 316 | model['.*.variance'].unfix() 317 | if optZ: 318 | model.Z.unfix() 319 | if non_chained: 320 | model['.*.W'].unfix() 321 | 322 | model.q_u_means.fix() 323 | model.q_u_chols.fix() 324 | model.optimize(messages=verbose, max_iters=maxIter_perVEM) 325 | print('iteration (' + str(i+1) + ') VM step, log_likelihood=' + str(model.log_likelihood().flatten())) 326 | sys.stdout.flush() 327 | sys.stderr.flush() 328 | else: 329 | 330 | if step_rate is None: 331 | step_rate = 0.01 332 | 333 | # Here the E step has maxIter_perVEM (100 by default) and 334 | # the M step has also maxIter_perVEM (100 by default) 335 | model.elbo = np.empty((2*maxIter_perVEM*vem_iters+2, 1)) 336 | model.elbo[0,0]=model.log_likelihood() 337 | c_full = partial(model.callback, max_iter=maxIter_perVEM, verbose=verbose, verbose_plot=verbose_plot) 338 | 339 | for i in range(vem_iters): 340 | # VARIATIONAL E-STEP 341 | model['.*.lengthscale'].fix() 342 | model['.*.variance'].fix() 343 | model.Z.fix() 344 | model['.*.W'].fix() 345 | 346 | model.q_u_means.unfix() 347 | model.q_u_chols.unfix() 348 | optimizer = climin.Adam(model.optimizer_array, model.stochastic_grad, step_rate=step_rate, 349 | decay_mom1=1 - 0.9, decay_mom2=1 - 0.999) 350 | # model.index_VEM = 2*(i) * maxIter_perVEM 351 | optimizer.minimize_until(c_full) 352 | # vo.variational_opt_HetMOGP(model=model, max_iters=maxIter_perVEM, step_size=step_rate, momentum=0.0,prior_lambda=1.0e-1,MC=1) 353 | 354 | print('iteration (' + str(i + 1) + ') VE step, mini-batch log_likelihood=' + str( 355 | model.log_likelihood().flatten())) 356 | # 357 | # # VARIATIONAL M-STEP 358 | model['.*.lengthscale'].unfix() 359 | model['.*.variance'].unfix() 360 | if optZ: 361 | model.Z.unfix() 362 | if non_chained: 363 | model['.*.W'].unfix() 364 | 365 | model.q_u_means.fix() 366 | model.q_u_chols.fix() 367 | optimizer = climin.Adam(model.optimizer_array, model.stochastic_grad, step_rate=step_rate,decay_mom1=1 - 0.9, decay_mom2=1 - 0.999) 368 | # model.index_VEM = 2*(i) * maxIter_perVEM +maxIter_perVEM 369 | optimizer.minimize_until(c_full) 370 | # vo.variational_opt_HetMOGP(model=model, max_iters=maxIter_perVEM, step_size=step_rate, momentum=0.0,prior_lambda=1.0e-1,MC=1) 371 | print('iteration (' + str(i + 1) + ') VM step, mini-batch log_likelihood=' + str( 372 | model.log_likelihood().flatten())) 373 | 374 | return model 375 | -------------------------------------------------------------------------------- /likelihoods/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/likelihoods/__init__.py -------------------------------------------------------------------------------- /likelihoods/bernoulli.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | from GPy.util.univariate_Gaussian import std_norm_pdf, std_norm_cdf 10 | try: 11 | from scipy.misc import logsumexp 12 | except: 13 | from scipy.special import logsumexp 14 | 15 | 16 | class Bernoulli(Likelihood): 17 | """ 18 | Bernoulli likelihood with a latent function over its parameter 19 | 20 | """ 21 | 22 | def __init__(self, gp_link=None): 23 | if gp_link is None: 24 | gp_link = link_functions.Identity() 25 | 26 | super(Bernoulli, self).__init__(gp_link, name='Bernoulli') 27 | 28 | def pdf(self, f, y, Y_metadata=None): 29 | ef = safe_exp(f) 30 | p = ef / (1 + ef) 31 | pdf = p ** (y) * (1 - p) ** (1 - y) 32 | return pdf 33 | 34 | def logpdf(self, f, y, Y_metadata=None): 35 | ef = safe_exp(f) 36 | p = ef / (1 + ef) 37 | p = np.clip(p, 1e-9, 1. - 1e-9) #numerical stability 38 | logpdf = (y * np.log(p)) + ((1 - y) * np.log(1 - p)) 39 | return logpdf 40 | 41 | def mean(self, f, Y_metadata=None): 42 | ef = safe_exp(f) 43 | p = ef / (1 + ef) 44 | p = np.clip(p, 1e-9, 1. - 1e-9) #numerical stability 45 | mean = p 46 | return mean 47 | 48 | def mean_sq(self, f, Y_metadata=None): 49 | ef = safe_exp(f) 50 | p = ef / (1 + ef) 51 | p = np.clip(p, 1e-9, 1. - 1e-9) #numerical stability 52 | mean_sq = np.square(p) 53 | return mean_sq 54 | 55 | def variance(self, f, Y_metadata=None): 56 | ef = safe_exp(f) 57 | p = ef / (1 + ef) 58 | p = np.clip(p, 1e-9, 1. - 1e-9) #numerical stability 59 | var = p*(1 - p) 60 | return var 61 | 62 | def samples(self, f ,num_samples, Y_metadata=None): 63 | ef = safe_exp(f) 64 | p = ef / (1 + ef) 65 | p = np.clip(p, 1e-9, 1. - 1e-9) # numerical stability 66 | samples = np.random.binomial(n=1, p=p) 67 | return samples 68 | 69 | def dlogp_df(self, f, y, Y_metadata=None): 70 | ef = safe_exp(f) 71 | p = ef / (1 + ef) 72 | p = np.clip(p, 1e-9, 1. - 1e-9) # numerical stability 73 | dlogp = ((y - p) / (1 - p)) * (1 / (1 + ef)) 74 | #dlogp = ( y - (1 - y)*(p / (1 - p)))*(1 / (1 + ef)) 75 | return dlogp 76 | 77 | def d2logp_df2(self, f, y, Y_metadata=None): 78 | ef = safe_exp(f) 79 | p = ef / (1 + ef) 80 | p = np.clip(p, 1e-9, 1. - 1e-9) # numerical stability 81 | d2logp = - p / (1 + ef) 82 | #d2logp = (1 / (1 + ef))*((p*(y - 1)/((1 + ef)*(1 - p)**2)) - p*y + (1 - y)*(p**2 / (1 - p))) 83 | return d2logp 84 | 85 | def var_exp(self, Y, m, v, gh_points=None, Y_metadata=None): 86 | # Variational Expectation 87 | # gh: Gaussian-Hermite quadrature 88 | if gh_points is None: 89 | gh_f, gh_w = self._gh_points() 90 | else: 91 | gh_f, gh_w = gh_points 92 | 93 | gh_w = gh_w / np.sqrt(np.pi) 94 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 95 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 96 | logp = self.logpdf(f, np.tile(Y[:, None], (1, f.shape[1]))) 97 | var_exp = logp.dot(gh_w[:,None]) 98 | return var_exp 99 | 100 | def var_exp_derivatives(self, Y, m, v, gh_points=None, Y_metadata=None): 101 | # Variational Expectations of derivatives 102 | # gh: Gaussian-Hermite quadrature 103 | if gh_points is None: 104 | gh_f, gh_w = self._gh_points() 105 | else: 106 | gh_f, gh_w = gh_points 107 | gh_w = gh_w / np.sqrt(np.pi) 108 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 109 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 110 | dlogp_df = self.dlogp_df(f, np.tile(Y[:, None], (1, f.shape[1]))) 111 | var_exp_dm = dlogp_df.dot(gh_w[:,None]) 112 | GN = False #If True means we approximate the grads with Gauss-Newton so checkgrad(verbose) can change a bit 113 | if GN is False: 114 | d2logp_df2 = self.d2logp_df2(f, np.tile(Y[:, None], (1, f.shape[1]))) 115 | var_exp_dv = 0.5*d2logp_df2.dot(gh_w[:, None]) 116 | else: 117 | var_exp_dv = -0.5 * dlogp_df.__pow__(2).dot(gh_w[:, None]) 118 | return var_exp_dm, var_exp_dv 119 | 120 | # def predictive(self, m, v, gh_points=None, Y_metadata=None): 121 | # # Variational Expectation 122 | # # gh: Gaussian-Hermite quadrature 123 | # if gh_points is None: 124 | # gh_f, gh_w = self._gh_points() 125 | # else: 126 | # gh_f, gh_w = gh_points 127 | # 128 | # gh_w = gh_w / np.sqrt(np.pi) 129 | # m, v= m.flatten(), v.flatten() 130 | # f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 131 | # mean = self.mean(f) 132 | # var = self.variance(f).dot(gh_w[:,None]) + self.mean_sq(f).dot(gh_w[:,None]) - np.square(mean.dot(gh_w[:,None])) 133 | # mean_pred = mean.dot(gh_w[:,None]) 134 | # var_pred = var 135 | # return mean_pred, var_pred 136 | 137 | def predictive(self, m, v,Y_metadata=None): 138 | # This can be checked in eq 4.152 Pattern recognition Bishop 139 | # with lambda = 1 140 | mean_pred = std_norm_cdf(m / np.sqrt(1 + v)) #Here the mean prediction is already influenced by the variance v 141 | var_pred = np.ones_like(mean_pred)*np.nan #Since the prediction is already a probability informing the uncertainty 142 | return mean_pred, var_pred #of the prediciton, so with don't need any confidence variance 143 | 144 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 145 | Ntest, D = mu_F_star.shape 146 | F_samples = np.empty((Ntest, num_samples, D)) 147 | # function samples: 148 | for d in range(D): 149 | mu_fd_star = mu_F_star[:, d][:, None] 150 | var_fd_star = v_F_star[:, d][:, None] 151 | F_samples[:, :, d] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 152 | 153 | # monte-carlo: 154 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf(F_samples[:,:,0], Ytest), axis=-1) 155 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 156 | "I just changed this to have the log_predictive of each data point and not a mean values" 157 | #log_predictive = (1/num_samples)*log_pred.sum() 158 | 159 | return log_pred 160 | 161 | def get_metadata(self): 162 | dim_y = 1 163 | dim_f = 1 164 | dim_p = 1 165 | return dim_y, dim_f, dim_p 166 | 167 | def ismulti(self): 168 | # Returns if the distribution is multivariate 169 | return False 170 | -------------------------------------------------------------------------------- /likelihoods/beta.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | 5 | import sys 6 | import numpy as np 7 | from GPy.likelihoods import link_functions 8 | from GPy.likelihoods import Likelihood 9 | from GPy.util.misc import safe_exp, safe_square 10 | from scipy.special import beta, betaln, psi, zeta, gammaln, gamma 11 | from functools import reduce 12 | try: 13 | from scipy.misc import logsumexp 14 | except: 15 | from scipy.special import logsumexp 16 | 17 | class Beta(Likelihood): 18 | """ 19 | Beta likelihood with a latent function over its parameter 20 | """ 21 | 22 | def __init__(self, gp_link=None): 23 | if gp_link is None: 24 | gp_link = link_functions.Identity() 25 | 26 | super(Beta, self).__init__(gp_link, name='Beta') 27 | 28 | def pdf(self, F, y, Y_metadata=None): 29 | eF = safe_exp(F) 30 | pdf = (y**(eF[:,0] - 1)) * ((1-y)**(eF[:,1] - 1)) / beta(eF[:,0], eF[:,1]) 31 | return pdf 32 | 33 | def logpdf(self, F, y, Y_metadata=None): 34 | eF = safe_exp(F) 35 | a = eF[:,0,None] 36 | b = eF[:,1,None] 37 | a = np.clip(a, 1e-9, 1e9) # numerical stability 38 | b = np.clip(b, 1e-9, 1e9) # numerical stability 39 | logpdf = ((a - 1)*np.log(y)) + ((b - 1)*np.log(1-y)) - betaln(a, b) 40 | return logpdf 41 | 42 | def logpdf_sampling(self, F, y, Y_metadata=None): 43 | eF = safe_exp(F) 44 | a = eF[:, 0, :] 45 | b = eF[:, 1, :] 46 | a = np.clip(a, 1e-9, 1e9) # numerical stability 47 | b = np.clip(b, 1e-9, 1e9) # numerical stability 48 | ym =np.tile(y, (1,F.shape[2])) 49 | logpdf = ((a - 1) * np.log(ym)) + ((b - 1) * np.log(1 - ym)) - betaln(a, b) 50 | return logpdf 51 | 52 | 53 | def samples(self, F ,num_samples, Y_metadata=None): 54 | eF = safe_exp(F) 55 | a = eF[:,0,None] 56 | b = eF[:,1,None] 57 | a = np.clip(a, 1e-9, 1e9) # numerical stability 58 | b = np.clip(b, 1e-9, 1e9) # numerical stability 59 | samples = np.random.beta(a=a, b=b) 60 | return samples 61 | 62 | def mean(self, F, Y_metadata=None): 63 | eF = safe_exp(F) 64 | a = eF[:,0, None] 65 | b = eF[:,1, None] 66 | a = np.clip(a, 1e-9, 1e9) # numerical stability 67 | b = np.clip(b, 1e-9, 1e9) # numerical stability 68 | mean = a / (a+b) 69 | return mean 70 | 71 | def mean_sq(self, F, Y_metadata=None): 72 | eF = safe_exp(F) 73 | a = eF[:,0, None] 74 | b = eF[:,1, None] 75 | a = np.clip(a, 1e-9, 1e9) # numerical stability 76 | b = np.clip(b, 1e-9, 1e9) # numerical stability 77 | mean = a / (a+b) 78 | mean_sq = np.square(mean) 79 | return mean_sq 80 | 81 | def variance(self, F, Y_metadata=None): 82 | eF = safe_exp(F) 83 | a = eF[:,0, None] 84 | b = eF[:,1, None] 85 | a = np.clip(a, 1e-9, 1e9) # numerical stability 86 | b = np.clip(b, 1e-9, 1e9) # numerical stability 87 | var = a*b / ((a+b)**2 * (a+b+1)) 88 | return var 89 | 90 | 91 | def dlogp_df(self, F, y, Y_metadata=None): 92 | eF = safe_exp(F) 93 | a = eF[:,0, None] 94 | b = eF[:,1, None] 95 | a = np.clip(a, 1e-9, 1e9) # numerical stability 96 | b = np.clip(b, 1e-9, 1e9) # numerical stability 97 | psi_ab = psi(a+b) 98 | psi_a = psi(a) 99 | psi_b = psi(b) 100 | dlogp_dfa = (psi_ab - psi_a + np.log(y)) * a 101 | dlogp_dfb = (psi_ab - psi_b + np.log(1 - y)) * b 102 | return dlogp_dfa, dlogp_dfb 103 | 104 | 105 | def d2logp_df2(self, F, y, Y_metadata=None): 106 | eF = safe_exp(F) 107 | a = eF[:,0, None] 108 | b = eF[:,1, None] 109 | a = np.clip(a, 1e-9, 1e9) # numerical stability 110 | b = np.clip(b, 1e-9, 1e9) # numerical stability 111 | psi_ab = psi(a+b) 112 | psi_a = psi(a) 113 | psi_b = psi(b) 114 | zeta_ab = zeta(2,a+b) 115 | zeta_a = zeta(2,a) 116 | zeta_b = zeta(2,b) 117 | d2logp_dfa2 = (psi_ab + (a*zeta_ab) - psi_a - (a*zeta_a) + np.log(y)) * a 118 | d2logp_dfb2 = (psi_ab + (b*zeta_ab) - psi_b - (b*zeta_b) + np.log(1-y)) * b 119 | return d2logp_dfa2, d2logp_dfb2 120 | 121 | def var_exp(self, y, M, V, gh_points=None, Y_metadata=None): 122 | # Variational Expectation 123 | # gh: Gaussian-Hermite quadrature 124 | if gh_points is None: 125 | gh_f, gh_w = self._gh_points(T=16) 126 | else: 127 | gh_f, gh_w = gh_points 128 | gh_w = gh_w / np.sqrt(np.pi) 129 | D = M.shape[1] 130 | expanded_F_tuples = [] 131 | grid_tuple = [M.shape[0]] 132 | for d in range(D): 133 | grid_tuple.append(gh_f.shape[0]) 134 | expanded_fd_tuple = [1] * (D + 1) 135 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 136 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 137 | 138 | # mean-variance tuple 139 | mv_tuple = [1] * (D + 1) 140 | mv_tuple[0] = M.shape[0] 141 | mv_tuple = tuple(mv_tuple) 142 | 143 | # building, normalizing and reshaping the grids 144 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 145 | for d in range(D): 146 | fd = np.zeros(tuple(grid_tuple)) 147 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 148 | + np.reshape(M[:, d], mv_tuple) 149 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 150 | 151 | # function evaluation 152 | Y_full = np.repeat(y, gh_f.shape[0] ** D, axis=0) 153 | logp = self.logpdf(F, Y_full) 154 | logp = logp.reshape(tuple(grid_tuple)) 155 | 156 | # calculating quadrature 157 | var_exp = logp.dot(gh_w)# / np.sqrt(np.pi) 158 | for d in range(D - 1): 159 | var_exp = var_exp.dot(gh_w)# / np.sqrt(np.pi) 160 | 161 | return var_exp[:, None] 162 | 163 | def var_exp_derivatives(self, y, M, V, gh_points=None, Y_metadata=None): 164 | # Variational Expectation 165 | # gh: Gaussian-Hermite quadrature 166 | if gh_points is None: 167 | gh_f, gh_w = self._gh_points(T=16) 168 | else: 169 | gh_f, gh_w = gh_points 170 | gh_w = gh_w / np.sqrt(np.pi) 171 | D = M.shape[1] 172 | expanded_F_tuples = [] 173 | grid_tuple = [M.shape[0]] 174 | for d in range(D): 175 | grid_tuple.append(gh_f.shape[0]) 176 | expanded_fd_tuple = [1] * (D + 1) 177 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 178 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 179 | 180 | # mean-variance tuple 181 | mv_tuple = [1] * (D + 1) 182 | mv_tuple[0] = M.shape[0] 183 | mv_tuple = tuple(mv_tuple) 184 | 185 | # building, normalizing and reshaping the grids 186 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 187 | for d in range(D): 188 | fd = np.zeros(tuple(grid_tuple)) 189 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 190 | + np.reshape(M[:, d], mv_tuple) 191 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 192 | 193 | # function evaluation 194 | Y_full = np.repeat(y, gh_f.shape[0] ** D, axis=0) 195 | 196 | dlogp_a, dlogp_b = self.dlogp_df(F, Y_full) 197 | d2logp_a, d2logp_b = self.d2logp_df2(F, Y_full) 198 | 199 | dlogp_a = dlogp_a.reshape(tuple(grid_tuple)) 200 | dlogp_b = dlogp_b.reshape(tuple(grid_tuple)) 201 | d2logp_a = d2logp_a.reshape(tuple(grid_tuple)) 202 | d2logp_b = d2logp_b.reshape(tuple(grid_tuple)) 203 | 204 | ve_dm_fa = dlogp_a.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 205 | ve_dm_fb = dlogp_b.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 206 | ve_dv_fa = d2logp_a.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 207 | ve_dv_fb = d2logp_b.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 208 | 209 | var_exp_dm = np.hstack((ve_dm_fa[:,None], ve_dm_fb[:,None])) 210 | var_exp_dv = 0.5*np.hstack((ve_dv_fa[:,None], ve_dv_fb[:,None])) 211 | 212 | return var_exp_dm, var_exp_dv 213 | 214 | def predictive(self, M, V, gh_points=None, Y_metadata=None): 215 | # Variational Expectation 216 | # gh: Gaussian-Hermite quadrature 217 | if gh_points is None: 218 | gh_f, gh_w = self._gh_points(T=20) 219 | else: 220 | gh_f, gh_w = gh_points 221 | gh_w = gh_w / np.sqrt(np.pi) 222 | D = M.shape[1] 223 | expanded_F_tuples = [] 224 | grid_tuple = [M.shape[0]] 225 | for d in range(D): 226 | grid_tuple.append(gh_f.shape[0]) 227 | expanded_fd_tuple = [1] * (D + 1) 228 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 229 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 230 | 231 | # mean-variance tuple 232 | mv_tuple = [1] * (D + 1) 233 | mv_tuple[0] = M.shape[0] 234 | mv_tuple = tuple(mv_tuple) 235 | 236 | # building, normalizing and reshaping the grids 237 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 238 | for d in range(D): 239 | fd = np.zeros(tuple(grid_tuple)) 240 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 241 | + np.reshape(M[:, d], mv_tuple) 242 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 243 | 244 | mean = self.mean(F) 245 | mean = mean.reshape(tuple(grid_tuple)) 246 | mean_pred = mean.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 247 | 248 | var = self.variance(F) 249 | var = var.reshape(tuple(grid_tuple)) 250 | var_int = var.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 251 | mean_sq = self.mean_sq(F) 252 | mean_sq = mean_sq.reshape(tuple(grid_tuple)) 253 | mean_sq_int = mean_sq.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 254 | 255 | var_pred = var_int + mean_sq_int - safe_square(mean_pred) 256 | return mean_pred[:,None] , var_pred[:,None] 257 | 258 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 259 | Ntest, D = mu_F_star.shape 260 | F_samples = np.empty((Ntest, D, num_samples)) 261 | # function samples: 262 | for d in range(D): 263 | mu_fd_star = mu_F_star[:, d][:, None] 264 | var_fd_star = v_F_star[:, d][:, None] 265 | F_samples[:, d, :] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 266 | 267 | # monte-carlo: 268 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf_sampling(F_samples, Ytest), axis=-1) 269 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 270 | "I just changed this to have the log_predictive of each data point and not a mean values" 271 | #log_predictive = (1/num_samples)*log_pred.sum() 272 | 273 | return log_pred 274 | 275 | def get_metadata(self): 276 | dim_y = 1 277 | dim_f = 2 278 | dim_p = 1 279 | return dim_y, dim_f, dim_p 280 | 281 | def ismulti(self): 282 | # Returns if the distribution is multivariate 283 | return False 284 | -------------------------------------------------------------------------------- /likelihoods/categorical.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import numpy as np 5 | from GPy.likelihoods import link_functions 6 | from GPy.likelihoods import Likelihood 7 | from scipy.stats import multinomial 8 | from functools import reduce 9 | from GPy.util.misc import safe_exp, safe_square 10 | try: 11 | from scipy.misc import logsumexp 12 | except: 13 | from scipy.special import logsumexp 14 | 15 | class Categorical(Likelihood): 16 | """ 17 | Categorical likelihood with K dimensional 18 | Needs (K-1) latent functions (see Link Functions) 19 | 20 | """ 21 | def __init__(self, K, gp_link=None): 22 | if gp_link is None: 23 | gp_link = link_functions.Identity() 24 | 25 | super(Categorical, self).__init__(gp_link, name='Categorical') 26 | self.K = K 27 | 28 | def pdf(self, F, y, Y_metadata=None): 29 | Y_oneK = self.onehot(y) 30 | eF = safe_exp(F) 31 | den = 1 + eF.sum(1)[:, None] 32 | p = eF / np.tile(den, eF.shape[1]) 33 | p = np.hstack((p, 1 / den)) 34 | p = np.clip(p, 1e-9, 1 - 1e-9) 35 | p = p / np.tile(p.sum(1)[:,None], (1, p.shape[1])) 36 | pdf = multinomial.pmf(x=Y_oneK, n=1, p=p) 37 | return pdf 38 | 39 | def logpdf(self, F, y, Y_metadata=None): 40 | Y_oneK = self.onehot(y) 41 | eF = safe_exp(F) 42 | den = 1 + eF.sum(1)[:, None] 43 | p = eF / np.tile(den, eF.shape[1]) 44 | p = np.hstack((p, 1 / den)) 45 | p = np.clip(p, 1e-9, 1- 1e-9) 46 | p = p / np.tile(p.sum(1)[:,None], (1, p.shape[1])) 47 | logpdf = multinomial.logpmf(x=Y_oneK, n=1, p=p) 48 | return logpdf 49 | 50 | def logpdf_sampling(self, F, y, Y_metadata=None): 51 | Y_oneK = self.onehot(y) 52 | eF = safe_exp(F) 53 | den = 1 + eF.sum(1)[:, None, :] 54 | p = eF / np.tile(den, (1, eF.shape[1] ,1)) 55 | p = np.hstack((p, 1 / den)) 56 | p = np.clip(p, 1e-9, 1 - 1e-9) 57 | p = p / np.tile(p.sum(1)[:,None,:], (1, p.shape[1],1)) 58 | Y_oneK_rep = np.tile(Y_oneK, (eF.shape[2],1)) 59 | p_rep = np.empty((p.shape[0]*p.shape[2],p.shape[1])) 60 | for s in range(p.shape[2]): 61 | p_rep[s * p.shape[0]:(s * p.shape[0]) + p.shape[0], :] = p[:, :, s] 62 | 63 | logpdf = multinomial.logpmf(x=Y_oneK_rep, n=1, p=p_rep) 64 | logpdf = logpdf.reshape(p.shape[0], p.shape[2]) 65 | return logpdf 66 | 67 | def samples(self, F, num_samples,Y_metadata=None): 68 | eF = safe_exp(F) 69 | den = 1 + eF.sum(1)[:, None] 70 | p = eF / np.tile(den, eF.shape[1]) 71 | p = np.hstack((p, 1 / den)) 72 | p = np.clip(p, 1e-9, 1 - 1e-9) 73 | p = p / np.tile(p.sum(1)[:,None], (1, p.shape[1])) 74 | samples = np.empty((F.shape[0], self.K)) 75 | for i in range(F.shape[0]): 76 | samples[i,:] = multinomial.rvs(n=1, p=p[i,:], size=1) 77 | return self.invonehot(Y=samples) 78 | 79 | def onehot(self, y): 80 | # One-Hot Encoding of Categorical Data 81 | Y_onehot = np.zeros((y.shape[0], self.K)) 82 | for k in range(self.K): 83 | Y_onehot[:,k,None] = (y==k+1).astype(np.int) 84 | return Y_onehot 85 | 86 | def invonehot(self, Y): 87 | # One-Hot Encoding of Categorical Data 88 | ycat = np.where( Y == 1)[1] + 1 89 | return ycat[:,None] 90 | 91 | def rho_k(self, F, k): 92 | # Probability of class k: P(y=k) 93 | Kminus1 = F.shape[1] 94 | eF = safe_exp(F) 95 | rho = eF / (1 + np.tile(eF.sum(1)[:,None], (1, F.shape[1]))) 96 | rho = np.clip(rho, 1e-9, 1. - 1e-9) # numerical stability 97 | # Gym has changed this parts: 98 | # rho = rho / np.tile(rho.sum(1)[:,None], (1, rho.shape[1])) 99 | # if k>Kminus1: 100 | # rho_k = 1 - rho.sum(1) 101 | # else: 102 | # rho_k = rho[:,k] 103 | if k < Kminus1: 104 | rho_k = rho[:, k] 105 | else: 106 | rho_k = 1 - rho.sum(1) 107 | return rho_k 108 | 109 | def dlogp_df(self, df, F, y, Y_metadata=None): 110 | # df: indicated the derivated function f from F 111 | Y_oneK = self.onehot(y) 112 | eF = safe_exp(F) 113 | den = 1 + eF.sum(1)[:, None] 114 | p = eF[:, df, None] / den 115 | p = np.clip(p, 1e-9, 1. - 1e-9) # numerical stability 116 | p = p / np.tile(p.sum(1)[:,None], (1, p.shape[1])) 117 | yp = Y_oneK*np.tile(p, (1, Y_oneK.shape[1])) #old, new is simpler 118 | dlogp = Y_oneK[:,df,None] - yp.sum(1)[:,None] #old, new is simpler 119 | #dlogp = Y_oneK[:,df,None] - p 120 | return dlogp 121 | 122 | def d2logp_df2(self, df, F, y, Y_metadata=None): 123 | # df: indicated the derivated function f from F 124 | Y_oneK = self.onehot(y) 125 | eF = safe_exp(F) 126 | den = 1 + eF.sum(1)[:, None] 127 | num = F + np.tile(F[:,df,None],(1,F.shape[1])) 128 | enum = safe_exp(num) 129 | enum[:,df] = safe_exp(F[:,df]) 130 | num = enum.sum(1)[:,None] 131 | p = num / safe_square(den) #añadir clip 132 | #p = p / np.tile(p.sum(1), (1, p.shape[1])) 133 | yp = Y_oneK*np.tile(p, (1, Y_oneK.shape[1])) #old, new is simpler 134 | d2logp = - yp.sum(1)[:,None] #old, new is simpler 135 | return d2logp 136 | 137 | def var_exp(self, y, M, V, gh_points=None, Y_metadata=None): 138 | # Variational Expectation 139 | # gh: Gaussian-Hermite quadrature 140 | if gh_points is None: 141 | gh_f, gh_w = self._gh_points(T=10) 142 | else: 143 | gh_f, gh_w = gh_points 144 | D = M.shape[1] 145 | # grid-size and fd tuples 146 | expanded_F_tuples = [] 147 | grid_tuple = [M.shape[0]] 148 | for d in range(D): 149 | grid_tuple.append(gh_f.shape[0]) 150 | expanded_fd_tuple = [1]*(D+1) 151 | expanded_fd_tuple[d+1] = gh_f.shape[0] 152 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 153 | 154 | # mean-variance tuple 155 | mv_tuple = [1]*(D+1) 156 | mv_tuple[0] = M.shape[0] 157 | mv_tuple = tuple(mv_tuple) 158 | 159 | # building, normalizing and reshaping the grids 160 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple),D)) 161 | for d in range(D): 162 | fd = np.zeros(tuple(grid_tuple)) 163 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d])*np.sqrt(2*np.reshape(V[:,d],mv_tuple)) \ 164 | + np.reshape(M[:,d],mv_tuple) 165 | F[:,d,None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 166 | 167 | # function evaluation 168 | Y_full = np.repeat(y, gh_f.shape[0]**D, axis=0) 169 | logp = self.logpdf(F, Y_full) 170 | logp = logp.reshape(tuple(grid_tuple)) 171 | 172 | # calculating quadrature 173 | var_exp = logp.dot(gh_w) / np.sqrt(np.pi) 174 | for d in range(D-1): 175 | var_exp = var_exp.dot(gh_w) / np.sqrt(np.pi) 176 | 177 | return var_exp[:,None] 178 | 179 | def var_exp_derivatives(self, y, M, V, gh_points=None, Y_metadata=None): 180 | # Variational Expectation 181 | # gh: Gaussian-Hermite quadrature 182 | if gh_points is None: 183 | gh_f, gh_w = self._gh_points(T=10) 184 | else: 185 | gh_f, gh_w = gh_points 186 | N = M.shape[0] 187 | D = M.shape[1] 188 | # grid-size and fd tuples 189 | expanded_F_tuples = [] 190 | grid_tuple = [M.shape[0]] 191 | for d in range(D): 192 | grid_tuple.append(gh_f.shape[0]) 193 | expanded_fd_tuple = [1] * (D + 1) 194 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 195 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 196 | 197 | # mean-variance tuple 198 | mv_tuple = [1] * (D + 1) 199 | mv_tuple[0] = M.shape[0] 200 | mv_tuple = tuple(mv_tuple) 201 | 202 | # building, normalizing and reshaping the grids 203 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 204 | for d in range(D): 205 | fd = np.zeros(tuple(grid_tuple)) 206 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 207 | + np.reshape(M[:, d], mv_tuple) 208 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 209 | 210 | # function evaluation 211 | Y_full = np.repeat(y, gh_f.shape[0] ** D, axis=0) 212 | var_exp_dm = np.empty((N,D)) 213 | var_exp_dv = np.empty((N,D)) 214 | for d in range(D): 215 | # wrt to the mean 216 | dlogp = self.dlogp_df(d, F, Y_full) 217 | dlogp = dlogp.reshape(tuple(grid_tuple)) 218 | ve_dm = dlogp.dot(gh_w) / np.sqrt(np.pi) 219 | # wrt to the variance 220 | d2logp = self.d2logp_df2(d, F, Y_full) 221 | d2logp = d2logp.reshape(tuple(grid_tuple)) 222 | ve_dv = d2logp.dot(gh_w) / np.sqrt(np.pi) 223 | for fd in range(D - 1): 224 | ve_dm = ve_dm.dot(gh_w) / np.sqrt(np.pi) 225 | ve_dv = ve_dv.dot(gh_w) / np.sqrt(np.pi) 226 | 227 | var_exp_dm[:,d] = ve_dm 228 | var_exp_dv[:,d] = 0.5 * ve_dv 229 | return var_exp_dm, var_exp_dv 230 | 231 | def predictive(self, M, V, gh_points=None, Y_metadata=None): 232 | # Variational Expectation 233 | # gh: Gaussian-Hermite quadrature 234 | if gh_points is None: 235 | gh_f, gh_w = self._gh_points(T=10) 236 | else: 237 | gh_f, gh_w = gh_points 238 | N = M.shape[0] 239 | D = M.shape[1] 240 | # grid-size and fd tuples 241 | expanded_F_tuples = [] 242 | grid_tuple = [M.shape[0]] 243 | for d in range(D): 244 | grid_tuple.append(gh_f.shape[0]) 245 | expanded_fd_tuple = [1] * (D + 1) 246 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 247 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 248 | 249 | # mean-variance tuple 250 | mv_tuple = [1] * (D + 1) 251 | mv_tuple[0] = M.shape[0] 252 | mv_tuple = tuple(mv_tuple) 253 | 254 | # building, normalizing and reshaping the grids 255 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 256 | for d in range(D): 257 | fd = np.zeros(tuple(grid_tuple)) 258 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 259 | + np.reshape(M[:, d], mv_tuple) 260 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 261 | 262 | # function evaluation 263 | mean_pred = np.empty((N, D)) 264 | var_pred = np.zeros((N, D)) 265 | for d in range(D): 266 | # wrt to the mean 267 | mean_k = self.rho_k(F, d) 268 | mean_k = mean_k.reshape(tuple(grid_tuple)) 269 | mean_pred_k = mean_k.dot(gh_w) / np.sqrt(np.pi) 270 | # wrt to the variance 271 | # NOT IMPLEMENTED 272 | for fd in range(D - 1): 273 | mean_pred_k = mean_pred_k.dot(gh_w) / np.sqrt(np.pi) 274 | 275 | mean_pred[:, d] = mean_pred_k 276 | return mean_pred, var_pred 277 | 278 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 279 | Ntest, D = mu_F_star.shape 280 | F_samples = np.empty((Ntest, D, num_samples)) 281 | # function samples: 282 | for d in range(D): 283 | mu_fd_star = mu_F_star[:, d, None] 284 | var_fd_star = v_F_star[:, d, None] 285 | F_samples[:, d, :] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 286 | 287 | # monte-carlo: 288 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf_sampling(F_samples, Ytest), axis=-1) 289 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 290 | #log_predictive = (1/num_samples)*log_pred.sum() 291 | 292 | return log_pred 293 | 294 | def get_metadata(self): 295 | dim_y = 1 296 | dim_f = self.K - 1 297 | dim_p = self.K - 1 298 | return dim_y, dim_f, dim_p 299 | 300 | def ismulti(self): 301 | # Returns if the distribution is multivariate 302 | return True 303 | -------------------------------------------------------------------------------- /likelihoods/dirichlet.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | 10 | 11 | class Dirichlet(Likelihood): 12 | """ 13 | Dirichlet likelihood with a latent function over its parameter 14 | 15 | """ 16 | 17 | def __init__(self, gp_link=None): 18 | if gp_link is None: 19 | gp_link = link_functions.Identity() 20 | 21 | super(Dirichlet, self).__init__(gp_link, name='Dirichlet') -------------------------------------------------------------------------------- /likelihoods/exponential.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | try: 10 | from scipy.misc import logsumexp 11 | except: 12 | from scipy.special import logsumexp 13 | 14 | class Exponential(Likelihood): 15 | """ 16 | Exponential likelihood with a latent function over its parameter 17 | 18 | """ 19 | 20 | def __init__(self, gp_link=None): 21 | if gp_link is None: 22 | gp_link = link_functions.Identity() 23 | 24 | super(Exponential, self).__init__(gp_link, name='Exponential') 25 | 26 | def pdf(self, f, y, Y_metadata=None): 27 | b = safe_exp(-f) 28 | pdf = safe_exp(-y/b) / b 29 | return pdf 30 | 31 | def logpdf(self, f, y, Y_metadata=None): 32 | b = safe_exp(-f) # a veces se va a zero -> np.clip 33 | b = np.clip(b, 1e-9, 1e9) # numerical stability 34 | logpdf = - np.log(b) - (y/b) 35 | return logpdf 36 | 37 | def mean(self, f, Y_metadata=None): 38 | b = safe_exp(-f) 39 | b = np.clip(b, 1e-9, 1e9) # numerical stability 40 | mean = b 41 | return mean 42 | 43 | def mean_sq(self, f, Y_metadata=None): 44 | b = safe_exp(-f) 45 | b = np.clip(b, 1e-9, 1e9) # numerical stability 46 | mean_sq = safe_square(b) 47 | return mean_sq 48 | 49 | def variance(self, f, Y_metadata=None): 50 | b = safe_exp(-f) 51 | b = np.clip(b, 1e-9, 1e9) # numerical stability 52 | var = safe_square(b) 53 | return var 54 | 55 | def samples(self, f, num_samples, Y_metadata=None): 56 | b = safe_exp(-f) 57 | b = np.clip(b, 1e-9, 1e9) # numerical stability 58 | samples = np.random.exponential(scale=b) 59 | return samples 60 | 61 | def dlogp_df(self, f, y, Y_metadata=None): 62 | b = safe_exp(-f) 63 | b = np.clip(b, 1e-9, 1e9) # numerical stability 64 | dlogp = 1 - (y/b) 65 | return dlogp 66 | 67 | def d2logp_df2(self, f, y, Y_metadata=None): 68 | b = safe_exp(-f) 69 | b = np.clip(b, 1e-9, 1e9) # numerical stability 70 | d2logp = - y/b 71 | return d2logp 72 | 73 | def var_exp(self, Y, m, v, gh_points=None, Y_metadata=None): 74 | # Variational Expectation 75 | # gh: Gaussian-Hermite quadrature 76 | if gh_points is None: 77 | gh_f, gh_w = self._gh_points() 78 | else: 79 | gh_f, gh_w = gh_points 80 | 81 | gh_w = gh_w / np.sqrt(np.pi) 82 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 83 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 84 | logp = self.logpdf(f, np.tile(Y[:, None], (1, f.shape[1]))) 85 | var_exp = logp.dot(gh_w[:, None]) 86 | return var_exp 87 | 88 | def var_exp_derivatives(self, Y, m, v, gh_points=None, Y_metadata=None): 89 | # Variational Expectations of derivatives 90 | # gh: Gaussian-Hermite quadrature 91 | if gh_points is None: 92 | gh_f, gh_w = self._gh_points() 93 | else: 94 | gh_f, gh_w = gh_points 95 | gh_w = gh_w / np.sqrt(np.pi) 96 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 97 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 98 | dlogp_df = self.dlogp_df(f, np.tile(Y[:, None], (1, f.shape[1]))) 99 | d2logp_df2 = self.d2logp_df2(f, np.tile(Y[:, None], (1, f.shape[1]))) 100 | var_exp_dm = dlogp_df.dot(gh_w[:, None]) 101 | var_exp_dv = 0.5 * d2logp_df2.dot(gh_w[:, None]) 102 | return var_exp_dm, var_exp_dv 103 | 104 | def predictive(self, m, v, gh_points=None, Y_metadata=None): 105 | # Variational Expectation 106 | # gh: Gaussian-Hermite quadrature 107 | if gh_points is None: 108 | gh_f, gh_w = self._gh_points() 109 | else: 110 | gh_f, gh_w = gh_points 111 | 112 | gh_w = gh_w / np.sqrt(np.pi) 113 | m, v = m.flatten(), v.flatten() 114 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 115 | mean = self.mean(f) 116 | var = self.variance(f).dot(gh_w[:, None]) + self.mean_sq(f).dot(gh_w[:, None]) - np.square( 117 | mean.dot(gh_w[:, None])) 118 | mean_pred = mean.dot(gh_w[:, None]) 119 | var_pred = var 120 | return mean_pred, var_pred 121 | 122 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 123 | Ntest, D = mu_F_star.shape 124 | F_samples = np.empty((Ntest, num_samples, D)) 125 | # function samples: 126 | for d in range(D): 127 | mu_fd_star = mu_F_star[:, d][:, None] 128 | var_fd_star = v_F_star[:, d][:, None] 129 | F_samples[:,:,d] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 130 | 131 | # monte-carlo: 132 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf(F_samples[:,:,0], Ytest), axis=-1) 133 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 134 | log_predictive = (1/num_samples)*log_pred.sum() 135 | 136 | return log_predictive 137 | 138 | def get_metadata(self): 139 | dim_y = 1 140 | dim_f = 1 141 | dim_p = 1 142 | return dim_y, dim_f, dim_p 143 | 144 | def ismulti(self): 145 | # Returns if the distribution is multivariate 146 | return False 147 | -------------------------------------------------------------------------------- /likelihoods/gamma.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | from scipy.special import beta, betaln, psi, zeta, gamma, gammaln 10 | from functools import reduce 11 | 12 | 13 | class Gamma(Likelihood): 14 | """ 15 | Gamma likelihood with a latent function over its parameter 16 | 17 | """ 18 | 19 | def __init__(self, gp_link=None): 20 | if gp_link is None: 21 | gp_link = link_functions.Identity() 22 | 23 | super(Gamma, self).__init__(gp_link, name='Gamma') 24 | 25 | def pdf(self, F, y, Y_metadata=None): 26 | eF = safe_exp(F) 27 | a = eF[:,0,None] 28 | b = eF[:,1,None] 29 | a = np.clip(a, 1e-9, 1e9) # numerical stability 30 | b = np.clip(b, 1e-9, 1e9) # numerical stability 31 | pdf = (b**a) * (y**(a-1)) * safe_exp(-b*y) / gamma(a) 32 | return pdf 33 | 34 | def logpdf(self, F, y, Y_metadata=None): 35 | eF = safe_exp(F) 36 | a = eF[:,0,None] 37 | b = eF[:,1,None] 38 | a = np.clip(a, 1e-9, 1e9) # numerical stability 39 | b = np.clip(b, 1e-9, 1e9) # numerical stability 40 | logpdf = - gammaln(a) + (a*np.log(b)) + ((a-1)*np.log(y)) - (b*y) 41 | return logpdf 42 | 43 | def logpdf_sampling(self, F, y, Y_metadata=None): 44 | eF = safe_exp(F) 45 | a = eF[:,0,:] 46 | b = eF[:,1,:] 47 | a = np.clip(a, 1e-9, 1e9) # numerical stability 48 | b = np.clip(b, 1e-9, 1e9) # numerical stability 49 | ym = np.tile(y, (1, F.shape[2])) 50 | logpdf = - gammaln(a) + (a*np.log(b)) + ((a-1)*np.log(ym)) - (b*ym) 51 | return logpdf 52 | 53 | def samples(self, F ,num_samples, Y_metadata=None): 54 | eF = safe_exp(F) 55 | a = eF[:,0,None] 56 | b = eF[:,1,None] 57 | a = np.clip(a, 1e-9, 1e9) # numerical stability 58 | b = np.clip(b, 1e-9, 1e9) # numerical stability 59 | samples = np.random.gamma(shape=a, scale=1/b) 60 | return samples 61 | 62 | def mean(self, F, Y_metadata=None): 63 | eF = safe_exp(F) 64 | a = eF[:,0, None] 65 | b = eF[:,1, None] 66 | a = np.clip(a, 1e-9, 1e9) # numerical stability 67 | b = np.clip(b, 1e-9, 1e9) # numerical stability 68 | mean = a / b 69 | return mean 70 | 71 | def mean_sq(self, F, Y_metadata=None): 72 | eF = safe_exp(F) 73 | a = eF[:,0, None] 74 | b = eF[:,1, None] 75 | a = np.clip(a, 1e-9, 1e9) # numerical stability 76 | b = np.clip(b, 1e-9, 1e9) # numerical stability 77 | mean = a / b 78 | mean_sq = np.square(mean) 79 | return mean_sq 80 | 81 | def variance(self, F, Y_metadata=None): 82 | eF = safe_exp(F) 83 | a = eF[:,0, None] 84 | b = eF[:,1, None] 85 | a = np.clip(a, 1e-9, 1e9) # numerical stability 86 | b = np.clip(b, 1e-9, 1e9) # numerical stability 87 | var = a / (b**2) 88 | return var 89 | 90 | def dlogp_df(self, F, y, Y_metadata=None): 91 | eF = safe_exp(F) 92 | a = eF[:,0, None] 93 | b = eF[:,1, None] 94 | a = np.clip(a, 1e-9, 1e9) # numerical stability 95 | b = np.clip(b, 1e-9, 1e9) # numerical stability 96 | psi_a = psi(a) 97 | dlogp_dfa = (-psi_a + np.log(b) + np.log(y)) * a 98 | dlogp_dfb = a - (b*y) 99 | return dlogp_dfa, dlogp_dfb 100 | 101 | def d2logp_df2(self, F, y, Y_metadata=None): 102 | eF = safe_exp(F) 103 | a = eF[:,0, None] 104 | b = eF[:,1, None] 105 | a = np.clip(a, 1e-9, 1e9) # numerical stability 106 | b = np.clip(b, 1e-9, 1e9) # numerical stability 107 | psi_a = psi(a) 108 | zeta_a = zeta(2,a) 109 | d2logp_dfa2 = (- psi_a - (a*zeta_a) + np.log(b) + np.log(y)) * a 110 | d2logp_dfb2 = - y*b 111 | return d2logp_dfa2, d2logp_dfb2 112 | 113 | def var_exp(self, y, M, V, gh_points=None, Y_metadata=None): 114 | # Variational Expectation 115 | # gh: Gaussian-Hermite quadrature 116 | if gh_points is None: 117 | gh_f, gh_w = self._gh_points(T=16) 118 | else: 119 | gh_f, gh_w = gh_points 120 | gh_w = gh_w / np.sqrt(np.pi) 121 | D = M.shape[1] 122 | expanded_F_tuples = [] 123 | grid_tuple = [M.shape[0]] 124 | for d in range(D): 125 | grid_tuple.append(gh_f.shape[0]) 126 | expanded_fd_tuple = [1] * (D + 1) 127 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 128 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 129 | 130 | # mean-variance tuple 131 | mv_tuple = [1] * (D + 1) 132 | mv_tuple[0] = M.shape[0] 133 | mv_tuple = tuple(mv_tuple) 134 | 135 | # building, normalizing and reshaping the grids 136 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 137 | for d in range(D): 138 | fd = np.zeros(tuple(grid_tuple)) 139 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 140 | + np.reshape(M[:, d], mv_tuple) 141 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 142 | 143 | # function evaluation 144 | Y_full = np.repeat(y, gh_f.shape[0] ** D, axis=0) 145 | logp = self.logpdf(F, Y_full) 146 | logp = logp.reshape(tuple(grid_tuple)) 147 | 148 | # calculating quadrature 149 | var_exp = logp.dot(gh_w)# / np.sqrt(np.pi) 150 | for d in range(D - 1): 151 | var_exp = var_exp.dot(gh_w)# / np.sqrt(np.pi) 152 | 153 | return var_exp[:, None] 154 | 155 | def var_exp_derivatives(self, y, M, V, gh_points=None, Y_metadata=None): 156 | # Variational Expectation 157 | # gh: Gaussian-Hermite quadrature 158 | if gh_points is None: 159 | gh_f, gh_w = self._gh_points(T=16) 160 | else: 161 | gh_f, gh_w = gh_points 162 | gh_w = gh_w / np.sqrt(np.pi) 163 | D = M.shape[1] 164 | expanded_F_tuples = [] 165 | grid_tuple = [M.shape[0]] 166 | for d in range(D): 167 | grid_tuple.append(gh_f.shape[0]) 168 | expanded_fd_tuple = [1] * (D + 1) 169 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 170 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 171 | 172 | # mean-variance tuple 173 | mv_tuple = [1] * (D + 1) 174 | mv_tuple[0] = M.shape[0] 175 | mv_tuple = tuple(mv_tuple) 176 | 177 | # building, normalizing and reshaping the grids 178 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 179 | for d in range(D): 180 | fd = np.zeros(tuple(grid_tuple)) 181 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 182 | + np.reshape(M[:, d], mv_tuple) 183 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 184 | 185 | # function evaluation 186 | Y_full = np.repeat(y, gh_f.shape[0] ** D, axis=0) 187 | 188 | dlogp_a, dlogp_b = self.dlogp_df(F, Y_full) 189 | d2logp_a, d2logp_b = self.d2logp_df2(F, Y_full) 190 | 191 | dlogp_a = dlogp_a.reshape(tuple(grid_tuple)) 192 | dlogp_b = dlogp_b.reshape(tuple(grid_tuple)) 193 | d2logp_a = d2logp_a.reshape(tuple(grid_tuple)) 194 | d2logp_b = d2logp_b.reshape(tuple(grid_tuple)) 195 | 196 | ve_dm_fa = dlogp_a.dot(gh_w).dot(gh_w)# / np.square(np.sqrt(np.pi)) 197 | ve_dm_fb = dlogp_b.dot(gh_w).dot(gh_w) #/ np.square(np.sqrt(np.pi)) 198 | ve_dv_fa = d2logp_a.dot(gh_w).dot(gh_w) #/ np.square(np.sqrt(np.pi)) 199 | ve_dv_fb = d2logp_b.dot(gh_w).dot(gh_w) #/ np.square(np.sqrt(np.pi)) 200 | 201 | var_exp_dm = np.hstack((ve_dm_fa[:,None], ve_dm_fb[:,None])) 202 | var_exp_dv = 0.5*np.hstack((ve_dv_fa[:,None], ve_dv_fb[:,None])) 203 | 204 | return var_exp_dm, var_exp_dv 205 | 206 | def predictive(self, M, V, gh_points=None, Y_metadata=None): 207 | # Variational Expectation 208 | # gh: Gaussian-Hermite quadrature 209 | if gh_points is None: 210 | gh_f, gh_w = self._gh_points(T=20) 211 | else: 212 | gh_f, gh_w = gh_points 213 | gh_w = gh_w / np.sqrt(np.pi) 214 | D = M.shape[1] 215 | expanded_F_tuples = [] 216 | grid_tuple = [M.shape[0]] 217 | for d in range(D): 218 | grid_tuple.append(gh_f.shape[0]) 219 | expanded_fd_tuple = [1] * (D + 1) 220 | expanded_fd_tuple[d + 1] = gh_f.shape[0] 221 | expanded_F_tuples.append(tuple(expanded_fd_tuple)) 222 | 223 | # mean-variance tuple 224 | mv_tuple = [1] * (D + 1) 225 | mv_tuple[0] = M.shape[0] 226 | mv_tuple = tuple(mv_tuple) 227 | 228 | # building, normalizing and reshaping the grids 229 | F = np.zeros((reduce(lambda x, y: x * y, grid_tuple), D)) 230 | for d in range(D): 231 | fd = np.zeros(tuple(grid_tuple)) 232 | fd[:] = np.reshape(gh_f, expanded_F_tuples[d]) * np.sqrt(2 * np.reshape(V[:, d], mv_tuple)) \ 233 | + np.reshape(M[:, d], mv_tuple) 234 | F[:, d, None] = fd.reshape(reduce(lambda x, y: x * y, grid_tuple), -1, order='C') 235 | 236 | mean = self.mean(F) 237 | mean = mean.reshape(tuple(grid_tuple)) 238 | mean_pred = mean.dot(gh_w).dot(gh_w) #/ np.square(np.sqrt(np.pi)) 239 | 240 | var = self.variance(F) 241 | var = var.reshape(tuple(grid_tuple)) 242 | var_int = var.dot(gh_w).dot(gh_w) #/ np.square(np.sqrt(np.pi)) 243 | mean_sq = self.mean_sq(F) 244 | mean_sq = mean_sq.reshape(tuple(grid_tuple)) 245 | mean_sq_int = mean_sq.dot(gh_w).dot(gh_w) #/ np.square(np.sqrt(np.pi)) 246 | 247 | var_pred = var_int + mean_sq_int - safe_square(mean_pred) 248 | return mean_pred[:,None] , var_pred[:,None] 249 | 250 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 251 | Ntest, D = mu_F_star.shape 252 | F_samples = np.empty((Ntest, D, num_samples)) 253 | # function samples: 254 | for d in range(D): 255 | mu_fd_star = mu_F_star[:, d][:, None] 256 | var_fd_star = v_F_star[:, d][:, None] 257 | F_samples[:, d, :] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 258 | 259 | # monte-carlo: 260 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf_sampling(F_samples, Ytest), axis=-1) 261 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 262 | "I just changed this to have the log_predictive of each data point and not a mean values" 263 | #log_predictive = (1/num_samples)*log_pred.sum() 264 | 265 | return log_pred 266 | 267 | def get_metadata(self): 268 | dim_y = 1 269 | dim_f = 2 270 | dim_p = 1 271 | return dim_y, dim_f, dim_p 272 | 273 | def ismulti(self): 274 | # Returns if the distribution is multivariate 275 | return False 276 | -------------------------------------------------------------------------------- /likelihoods/gaussian.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import numpy as np 5 | from GPy.likelihoods import link_functions 6 | from GPy.likelihoods import Likelihood 7 | from scipy.stats import norm 8 | try: 9 | from scipy.misc import logsumexp 10 | except: 11 | from scipy.special import logsumexp 12 | 13 | class Gaussian(Likelihood): 14 | """ 15 | Gaussian likelihood with a latent function over its mean parameter 16 | 17 | """ 18 | 19 | def __init__(self, sigma=None, gp_link=None): 20 | if gp_link is None: 21 | gp_link = link_functions.Identity() 22 | 23 | if sigma is None: 24 | self.sigma = 0.5 25 | else: 26 | self.sigma = sigma 27 | 28 | super(Gaussian, self).__init__(gp_link, name='Gaussian') 29 | 30 | def pdf(self, f, y, Y_metadata=None): 31 | pdf = norm.pdf(y, loc=f) 32 | return pdf 33 | 34 | def logpdf(self, f, y, Y_metadata=None): 35 | logpdf = norm.logpdf(y, loc=f) 36 | return logpdf 37 | 38 | def samples(self, f , num_samples, Y_metadata=None): 39 | samples = np.random.normal(loc=f, scale=self.sigma) 40 | #samples = np.random.normal(loc=f, size=(num_samples, f.shape[1])) 41 | return samples 42 | 43 | def var_exp(self, Y, m, v, gh_points=None, Y_metadata=None): 44 | # Variational Expectation (Analytical) 45 | # E_q(fid)[log(p(yi|fid))] 46 | lik_v = np.square(self.sigma) 47 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 48 | m = m[:,None] 49 | v = v[:,None] 50 | Y = Y[:,None] 51 | var_exp = -0.5 * np.log(2 * np.pi) - 0.5 * np.log(lik_v) \ 52 | - 0.5 * (np.square(Y) + np.square(m) + v - (2 * m * Y)) / lik_v 53 | return var_exp 54 | 55 | def var_exp_derivatives(self, Y, m, v, gh_points=None, Y_metadata=None): 56 | # Variational Expectations of derivatives 57 | lik_v = np.square(self.sigma) 58 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 59 | m = m[:,None] 60 | v = v[:,None] 61 | Y = Y[:,None] 62 | var_exp_dm = - (m - Y) / lik_v 63 | GN = False 64 | if GN is False: 65 | var_exp_dv = - 0.5 * (1 / np.tile(lik_v, (m.shape[0],1))) 66 | else: 67 | var_exp_dv = - 0.5*(Y**2 - 2*Y*m + v + m**2)/ np.square(lik_v) 68 | return var_exp_dm, var_exp_dv 69 | 70 | def predictive(self, m, v, Y_metadata): 71 | mean_pred = m 72 | var_pred = np.square(self.sigma) + v 73 | return mean_pred, var_pred 74 | 75 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 76 | Ntest, D = mu_F_star.shape 77 | F_samples = np.empty((Ntest, num_samples, D)) 78 | # function samples: 79 | for d in range(D): 80 | mu_fd_star = mu_F_star[:, d][:, None] 81 | var_fd_star = v_F_star[:, d][:, None] 82 | F_samples[:, :, d] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 83 | 84 | # monte-carlo: 85 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf(F_samples[:,:,0], Ytest), axis=-1) 86 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 87 | "I just changed this to have the log_predictive of each data point and not a mean values" 88 | #log_predictive = (1/num_samples)*log_pred.sum() 89 | return log_pred 90 | 91 | def get_metadata(self): 92 | dim_y = 1 93 | dim_f = 1 94 | dim_p = 1 95 | return dim_y, dim_f, dim_p 96 | 97 | def ismulti(self): 98 | # Returns if the distribution is multivariate 99 | return False 100 | -------------------------------------------------------------------------------- /likelihoods/hetgaussian.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | try: 10 | from scipy.misc import logsumexp 11 | except: 12 | from scipy.special import logsumexp 13 | 14 | class HetGaussian(Likelihood): 15 | """ 16 | Heterocedastic Gaussian likelihood with a latent function over its parameter 17 | 18 | """ 19 | 20 | def __init__(self, gp_link=None): 21 | if gp_link is None: 22 | gp_link = link_functions.Identity() 23 | 24 | super(HetGaussian, self).__init__(gp_link, name='HetGaussian') 25 | 26 | def pdf(self, F, y, Y_metadata): 27 | e_var = safe_exp(F[:,1]) 28 | ym = (y - F[:,0]) 29 | pdf = safe_exp(-0.5 * np.log(2 * np.pi * e_var) - 0.5 * ( (ym**2) / e_var)) 30 | return pdf 31 | 32 | def logpdf(self, F, y, Y_metadata=None): 33 | e_var = safe_exp(F[:,1]) 34 | ym =(y - F[:,0]) 35 | logpdf = -0.5*np.log(2*np.pi) - (0.5*F[:,1]) - 0.5*((safe_square(ym)) / e_var) 36 | return logpdf 37 | 38 | def logpdf_sampling(self, F, y, Y_metadata=None): 39 | e_var = safe_exp(F[:,1,:]) 40 | ym =(np.tile(y, (1,F.shape[2])) - F[:,0,:]) 41 | logpdf = -0.5*np.log(2*np.pi) - (0.5*F[:,1]) - 0.5*((safe_square(ym)) / e_var) 42 | return logpdf 43 | 44 | def samples(self, F , num_samples, Y_metadata=None): 45 | e_var = safe_exp(F[:,1]) 46 | samples = np.random.normal(loc=F[:,0], scale=np.sqrt(e_var)) 47 | return samples[:,None] 48 | 49 | def var_exp(self, Y, M, V, gh_points=None, Y_metadata=None): 50 | # Variational Expectation (Analytical) 51 | # E_q(fid)[log(p(yi|fid))] 52 | m_fmean, m_fvar = M[:,0,None], M[:,1,None] 53 | v_fmean, v_fvar = V[:, 0,None], V[:, 1,None] 54 | precision = safe_exp(- m_fvar + (0.5*v_fvar)) 55 | precision = np.clip(precision, -1e9, 1e9) # numerical stability 56 | squares = (safe_square(Y) + safe_square(m_fmean) + v_fmean - (2*m_fmean*Y)) 57 | squares = np.clip(squares, -1e9, 1e9) # numerical stability 58 | var_exp = - (0.5*np.log(2 * np.pi)) - (0.5*m_fvar) - (0.5*precision*squares) 59 | return var_exp 60 | 61 | def var_exp_derivatives(self, Y, M, V, gh_points=None, Y_metadata=None): 62 | # Variational Expectations of derivatives 63 | var_exp_dm = np.empty((M.shape[0],2)) 64 | var_exp_dv = np.empty((M.shape[0],2)) 65 | 66 | m_fmean, m_fvar = M[:,0, None], M[:,1, None] 67 | v_fmean, v_fvar = V[:, 0, None], V[:, 1, None] 68 | precision = safe_exp(- m_fvar + (0.5*v_fvar)) 69 | precision = np.clip(precision, -1e9, 1e9) # numerical stability 70 | squares = (np.square(Y) + np.square(m_fmean) + v_fmean - (2*m_fmean*Y)) 71 | squares = np.clip(squares, -1e9, 1e9) # numerical stability 72 | var_exp_dm[:, 0, None] = precision*(Y - m_fmean) 73 | var_exp_dm[:, 1, None] = 0.5*((precision * squares) - 1.) 74 | var_exp_dv[:, 0, None] = -0.5*precision 75 | var_exp_dv[:, 1, None] = -0.25*precision*squares 76 | return var_exp_dm, var_exp_dv 77 | 78 | def predictive(self, M, V, gh_points=None, Y_metadata=None): 79 | # Variational Expectation 80 | # gh: Gaussian-Hermite quadrature 81 | if gh_points is None: 82 | gh_f, gh_w = self._gh_points() 83 | else: 84 | gh_f, gh_w = gh_points 85 | 86 | gh_w = gh_w / np.sqrt(np.pi) 87 | f1 = gh_f[None, :] * np.sqrt(2. * V[:,0,None]) + M[:,0,None] 88 | f2 = gh_f[None, :] * np.sqrt(2. * V[:,1,None]) + M[:,1,None] 89 | mean_pred = M[:,0,None] 90 | var_pred = safe_exp(f2).dot(gh_w[:,None]) + safe_square(f1).dot(gh_w[:,None]) - np.square(M[:,0])[:,None] 91 | return mean_pred, var_pred 92 | 93 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 94 | Ntest, D = mu_F_star.shape 95 | F_samples = np.empty((Ntest, D, num_samples)) 96 | # function samples: 97 | for d in range(D): 98 | mu_fd_star = mu_F_star[:, d][:, None] 99 | var_fd_star = v_F_star[:, d][:, None] 100 | F_samples[:, d, :] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 101 | 102 | # monte-carlo: 103 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf_sampling(F_samples, Ytest), axis=-1) 104 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 105 | "I just changed this to have the log_predictive of each data point and not a mean values" 106 | #log_predictive = (1/num_samples)*log_pred.sum() 107 | 108 | return log_pred 109 | 110 | def get_metadata(self): 111 | dim_y = 1 112 | dim_f = 2 113 | dim_p = 1 114 | return dim_y, dim_f, dim_p 115 | 116 | def ismulti(self): 117 | # Returns if the distribution is multivariate 118 | return False 119 | -------------------------------------------------------------------------------- /likelihoods/ordinal.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | 10 | 11 | class Ordinal(Likelihood): 12 | """ 13 | Ordinal likelihood with a latent function over its parameter 14 | 15 | """ 16 | 17 | def __init__(self, gp_link=None): 18 | if gp_link is None: 19 | gp_link = link_functions.Identity() 20 | 21 | super(Ordinal, self).__init__(gp_link, name='Ordinal') -------------------------------------------------------------------------------- /likelihoods/poisson.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from scipy import special 7 | from GPy.likelihoods import link_functions 8 | from GPy.likelihoods import Likelihood 9 | from GPy.util.misc import safe_exp, safe_square 10 | from scipy.stats import poisson 11 | try: 12 | from scipy.misc import logsumexp 13 | except: 14 | from scipy.special import logsumexp 15 | 16 | 17 | 18 | class Poisson(Likelihood): 19 | """ 20 | Poisson likelihood with a latent function over its parameter 21 | 22 | """ 23 | def __init__(self, gp_link=None): 24 | if gp_link is None: 25 | gp_link = link_functions.Identity() 26 | 27 | super(Poisson, self).__init__(gp_link, name='Poisson') 28 | 29 | 30 | def pdf(self, f, y, Y_metadata=None): 31 | pdf = safe_exp(self.logpdf(f, y, Y_metadata)) 32 | return pdf 33 | 34 | def logpdf(self, f, y, Y_metadata=None): 35 | ef = safe_exp(f) 36 | logpdf = -ef + (y*f) - special.gammaln(y+1) 37 | return logpdf 38 | 39 | def mean(self, f, Y_metadata=None): 40 | ef = safe_exp(f) 41 | mean = ef 42 | return mean 43 | 44 | def mean_sq(self, f, Y_metadata=None): 45 | ef = safe_exp(f) 46 | mean = np.square(ef) 47 | return mean 48 | 49 | def variance(self, f, Y_metadata=None): 50 | ef = safe_exp(f) 51 | var = ef 52 | return var 53 | 54 | def samples(self, f ,num_samples, Y_metadata=None): 55 | ef = safe_exp(f) 56 | samples = np.random.poisson(lam=ef) 57 | return samples 58 | 59 | def dlogp_df(self, f, y, Y_metadata=None): 60 | ef = safe_exp(f) 61 | dlogp = - ef + y 62 | return dlogp 63 | 64 | def d2logp_df2(self, f, y, Y_metadata=None): 65 | ef = safe_exp(f) 66 | d2logp = - ef 67 | return d2logp 68 | 69 | def var_exp(self, Y, m, v, gh_points=None, Y_metadata=None): 70 | # Variational Expectation 71 | # gh: Gaussian-Hermite quadrature 72 | if gh_points is None: 73 | gh_f, gh_w = self._gh_points() 74 | else: 75 | gh_f, gh_w = gh_points 76 | 77 | gh_w = gh_w / np.sqrt(np.pi) 78 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 79 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 80 | logp = self.logpdf(f, np.tile(Y[:, None], (1, f.shape[1]))) 81 | var_exp = logp.dot(gh_w[:,None]) 82 | return var_exp 83 | 84 | def var_exp_derivatives(self, Y, m, v, gh_points=None, Y_metadata=None): 85 | # Variational Expectations of derivatives 86 | # gh: Gaussian-Hermite quadrature 87 | if gh_points is None: 88 | gh_f, gh_w = self._gh_points() 89 | else: 90 | gh_f, gh_w = gh_points 91 | gh_w = gh_w / np.sqrt(np.pi) 92 | m, v, Y = m.flatten(), v.flatten(), Y.flatten() 93 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 94 | dlogp_df = self.dlogp_df(f, np.tile(Y[:, None], (1, f.shape[1]))) 95 | d2logp_df2 = self.d2logp_df2(f, np.tile(Y[:, None], (1, f.shape[1]))) 96 | var_exp_dm = dlogp_df.dot(gh_w[:,None]) 97 | var_exp_dv = 0.5*d2logp_df2.dot(gh_w[:, None]) 98 | return var_exp_dm, var_exp_dv 99 | 100 | def predictive(self, m, v, gh_points=None, Y_metadata=None): 101 | # Variational Expectation 102 | # gh: Gaussian-Hermite quadrature 103 | if gh_points is None: 104 | gh_f, gh_w = self._gh_points() 105 | else: 106 | gh_f, gh_w = gh_points 107 | 108 | gh_w = gh_w / np.sqrt(np.pi) 109 | m, v= m.flatten(), v.flatten() 110 | f = gh_f[None, :] * np.sqrt(2. * v[:, None]) + m[:, None] 111 | mean = self.mean(f) 112 | var = self.variance(f).dot(gh_w[:,None]) + self.mean_sq(f).dot(gh_w[:,None]) - np.square(mean.dot(gh_w[:,None])) 113 | mean_pred = mean.dot(gh_w[:,None]) 114 | var_pred = var 115 | return mean_pred, var_pred 116 | 117 | def log_predictive(self, Ytest, mu_F_star, v_F_star, num_samples): 118 | Ntest, D = mu_F_star.shape 119 | F_samples = np.empty((Ntest, num_samples, D)) 120 | # function samples: 121 | for d in range(D): 122 | mu_fd_star = mu_F_star[:, d][:, None] 123 | var_fd_star = v_F_star[:, d][:, None] 124 | F_samples[:, :, d] = np.random.normal(mu_fd_star, np.sqrt(var_fd_star), size=(Ntest, num_samples)) 125 | 126 | # monte-carlo: 127 | log_pred = -np.log(num_samples) + logsumexp(self.logpdf(F_samples[:,:,0], Ytest), axis=-1) 128 | log_pred = np.array(log_pred).reshape(*Ytest.shape) 129 | log_predictive = (1/num_samples)*log_pred.sum() 130 | 131 | return log_predictive 132 | 133 | def get_metadata(self): 134 | dim_y = 1 135 | dim_f = 1 136 | dim_p = 1 137 | return dim_y, dim_f, dim_p 138 | 139 | def ismulti(self): 140 | # Returns if the distribution is multivariate 141 | return False 142 | -------------------------------------------------------------------------------- /likelihoods/student.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Pablo Moreno-Munoz 2 | # Universidad Carlos III de Madrid and University of Sheffield 3 | 4 | import sys 5 | import numpy as np 6 | from GPy.likelihoods import link_functions 7 | from GPy.likelihoods import Likelihood 8 | from GPy.util.misc import safe_exp, safe_square 9 | from scipy.stats import poisson 10 | 11 | 12 | class Student(Likelihood): 13 | """ 14 | Student likelihood with a latent function over its parameter 15 | 16 | """ 17 | 18 | def __init__(self, gp_link=None): 19 | if gp_link is None: 20 | gp_link = link_functions.Identity() 21 | 22 | super(Student, self).__init__(gp_link, name='Student') -------------------------------------------------------------------------------- /notebooks/fertility_2x2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys\n", 10 | "import climin\n", 11 | "from functools import partial\n", 12 | "import warnings\n", 13 | "import os\n", 14 | "sys.path.append('..')\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "from scipy.stats import multinomial\n", 18 | "from scipy.linalg.blas import dtrmm\n", 19 | "\n", 20 | "import GPy\n", 21 | "from GPy.util import choleskies\n", 22 | "from GPy.core.parameterization.param import Param\n", 23 | "from GPy.kern import Coregionalize\n", 24 | "from GPy.likelihoods import Likelihood\n", 25 | "from GPy.util import linalg\n", 26 | "\n", 27 | "from likelihoods.bernoulli import Bernoulli\n", 28 | "from likelihoods.gaussian import Gaussian\n", 29 | "from likelihoods.categorical import Categorical\n", 30 | "from likelihoods.hetgaussian import HetGaussian\n", 31 | "from likelihoods.beta import Beta\n", 32 | "from likelihoods.gamma import Gamma\n", 33 | "from likelihoods.exponential import Exponential\n", 34 | "\n", 35 | "from hetmogp.util import draw_mini_slices\n", 36 | "from hetmogp.het_likelihood import HetLikelihood\n", 37 | "from hetmogp.svmogp import SVMOGP\n", 38 | "from hetmogp import util\n", 39 | "from hetmogp.util import vem_algorithm as VEM\n", 40 | "\n", 41 | "import matplotlib.pyplot as plt\n", 42 | "from matplotlib.pyplot import gca\n", 43 | "from matplotlib import rc, font_manager\n", 44 | "from matplotlib import rcParams\n", 45 | "\n", 46 | "warnings.filterwarnings(\"ignore\")\n", 47 | "\n", 48 | "from sklearn.model_selection import train_test_split\n", 49 | "from sklearn.cluster import KMeans\n", 50 | "\n", 51 | "import pickle\n", 52 | "import random\n", 53 | "import time\n", 54 | "\n", 55 | "import error_func\n", 56 | "import click\n", 57 | "import logging" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 2, 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "Multi output mode, y_dim: 1\n", 70 | "\n", 71 | "Name : SVMOGP\n", 72 | "Objective : 1651.0470505077521\n", 73 | "Number of Parameters : 5657\n", 74 | "Number of Optimization Parameters : 5657\n", 75 | "Updates : True\n", 76 | "Parameters:\n", 77 | " \u001b[1mSVMOGP. \u001b[0;0m | value | constraints | priors\n", 78 | " \u001b[1minducing_inputs \u001b[0;0m | (100, 5) | | \n", 79 | " \u001b[1mm_u \u001b[0;0m | (100, 1) | | \n", 80 | " \u001b[1mL_u \u001b[0;0m | (5050, 1) | | \n", 81 | " \u001b[1mkern_q0.variance \u001b[0;0m | 1.0 | +ve | \n", 82 | " \u001b[1mkern_q0.lengthscale\u001b[0;0m | (2,) | +ve | \n", 83 | " \u001b[1mB_q0.W \u001b[0;0m | (2, 1) | | \n", 84 | " \u001b[1mB_q0.kappa \u001b[0;0m | (2,) | +ve | \n", 85 | "model.kern_q0.lengthscale \u001b[1mindex\u001b[0;0m | SVMOGP.kern_q0.lengthscale | constraints | priors\n", 86 | " \u001b[1m[0] \u001b[0;0m | 2.00000000 | +ve | \n", 87 | " \u001b[1m[1] \u001b[0;0m | 2.00000000 | +ve | \n", 88 | "Optimiser max iter: 1000\n" 89 | ] 90 | } 91 | ], 92 | "source": [ 93 | "Q = 1 # number of latent functions\n", 94 | "mode = 'multi_output'\n", 95 | "y_dim = 1\n", 96 | "M = 100\n", 97 | "\n", 98 | "x_high = np.load('../data/fertility/x_step1_mean.np')\n", 99 | "x_low = np.load('../data/fertility/x_step2_mean.np')\n", 100 | "\n", 101 | "y_high = np.load('../data/fertility/y_step1_mean.np')\n", 102 | "y_low = np.load('../data/fertility/y_step2_mean.np')\n", 103 | "\n", 104 | "\n", 105 | "x_high_new = np.zeros((x_high.shape[0], x_high.shape[1]+1))\n", 106 | "x_high_new[:,0] = x_high[:,1].copy()\n", 107 | "x_high_new[:,1] = x_high[:,0].copy()\n", 108 | "x_high_new[:,2] = x_high[:,3].copy() - 15\n", 109 | "x_high_new[:,3] = x_high[:,2].copy() - 15\n", 110 | "\n", 111 | "x_low_new = np.zeros((x_low.shape[0], x_low.shape[1]+1))\n", 112 | "x_low_new[:,0] = x_low[:,1].copy()\n", 113 | "x_low_new[:,1] = x_low[:,0].copy()\n", 114 | "x_low_new[:,2] = x_low[:,3].copy() - 15\n", 115 | "x_low_new[:,3] = x_low[:,2].copy() - 15\n", 116 | "\n", 117 | "x1_high, x2_high, x3_high, x4_high = x_high_new.copy(), x_high_new.copy(), x_high_new.copy(), x_high_new.copy()\n", 118 | "Y1_high, Y2_high, Y3_high, Y4_high = y_high[:,0].reshape(-1, 1), y_high[:,1].reshape(-1, 1), y_high[:,2].reshape(-1, 1), y_high[:,3].reshape(-1, 1)\n", 119 | "\n", 120 | "x1_low, x2_low, x3_low, x4_low = x_low_new.copy(), x_low_new.copy(), x_low_new.copy(), x_low_new.copy()\n", 121 | "Y1_low, Y2_low, Y3_low, Y4_low = y_low[:,0].reshape(-1, 1), y_low[:,1].reshape(-1, 1), y_low[:,2].reshape(-1, 1), y_low[:,3].reshape(-1, 1)\n", 122 | "\n", 123 | "\n", 124 | "# rnd_idx\n", 125 | "random.seed(8)\n", 126 | "size_random_idx = 1000\n", 127 | "rnd_idx1 = random.sample(range(x1_high.shape[0]), size_random_idx)\n", 128 | "rnd_idx2 = random.sample(range(x2_high.shape[0]), size_random_idx)\n", 129 | "\n", 130 | "# sorted_rnd_idx\n", 131 | "sorted_rnd_idx1 = np.sort(rnd_idx1)\n", 132 | "sorted_rnd_idx2 = np.sort(rnd_idx2)\n", 133 | "\n", 134 | "X_test1 = x1_high[sorted_rnd_idx1]\n", 135 | "X_test2 = x2_high[sorted_rnd_idx2]\n", 136 | "\n", 137 | "if y_dim == 1:\n", 138 | " Y_test = [Y1_high[sorted_rnd_idx1]]\n", 139 | "elif y_dim == 2:\n", 140 | " Y_test = [Y1_high[sorted_rnd_idx1], Y2_high[sorted_rnd_idx2]]\n", 141 | "\n", 142 | "x1_high_new = np.delete(x1_high, sorted_rnd_idx1, axis=0)\n", 143 | "Y1_high_new = np.delete(Y1_high, sorted_rnd_idx1, axis=0)\n", 144 | "\n", 145 | "x2_high_new = np.delete(x2_high, sorted_rnd_idx2, axis=0)\n", 146 | "Y2_high_new = np.delete(Y2_high, sorted_rnd_idx2, axis=0)\n", 147 | "\n", 148 | "# Normalising outputs\n", 149 | "Y1_low_mean = Y1_low.mean().copy()\n", 150 | "Y1_low_std = Y1_low.std().copy()\n", 151 | "\n", 152 | "Y2_low_mean = Y2_low.mean().copy()\n", 153 | "Y2_low_std = Y2_low.std().copy()\n", 154 | "\n", 155 | "Y1_high_mean = Y1_high.mean().copy()\n", 156 | "Y1_high_std = Y1_high.std().copy()\n", 157 | "\n", 158 | "Y2_high_mean = Y2_high.mean().copy()\n", 159 | "Y2_high_std = Y2_high.std().copy()\n", 160 | "\n", 161 | "Y1_low_norm = (Y1_low - Y1_low_mean) / Y1_low_std\n", 162 | "Y2_low_norm = (Y2_low - Y2_low_mean) / Y2_low_std\n", 163 | "\n", 164 | "Y1_high_norm = (Y1_high_new - Y1_high_mean) / Y1_high_std\n", 165 | "Y2_high_norm = (Y2_high_new - Y2_high_mean) / Y2_high_std\n", 166 | "\n", 167 | "random.seed(8+4)\n", 168 | "high_res_data_random_size = 80\n", 169 | "rnd_idx_high_res = random.sample(range(x1_high_new.shape[0]), high_res_data_random_size)\n", 170 | "\n", 171 | "# sorted_rnd_idx\n", 172 | "sorted_rnd_idx_high_res = np.sort(rnd_idx_high_res)\n", 173 | "\n", 174 | "if mode == 'multi_output':\n", 175 | " print('Multi output mode, y_dim:', y_dim)\n", 176 | " if y_dim == 1:\n", 177 | " X = [x1_high_new[sorted_rnd_idx_high_res], x1_low]\n", 178 | " Y = [Y1_high_norm[sorted_rnd_idx_high_res], Y1_low_norm]\n", 179 | "\n", 180 | " # Creating inducing inputs using cluster centers of the original images\n", 181 | " X_ = np.vstack((X[0], X[1]))\n", 182 | " \n", 183 | " # Heterogeneous Likelihood Definition\n", 184 | " likelihoods_list = [Gaussian(sigma=1.), Gaussian(sigma=1.)]\n", 185 | " elif y_dim == 2:\n", 186 | " X = [x1_high_new[sorted_rnd_idx_high_res], x1_low,\n", 187 | " x2_high_new[sorted_rnd_idx_high_res], x2_low]\n", 188 | " Y = [Y1_high_norm[sorted_rnd_idx_high_res], Y1_low_norm,\n", 189 | " Y2_high_norm[sorted_rnd_idx_high_res], Y2_low_norm]\n", 190 | " \n", 191 | " # Creating inducing inputs using cluster centers of the original images\n", 192 | " X_ = np.vstack((X[0], X[1], X[2], X[3]))\n", 193 | " \n", 194 | " # Heterogeneous Likelihood Definition\n", 195 | " likelihoods_list = [Gaussian(sigma=1.), Gaussian(sigma=1.), Gaussian(sigma=1.), Gaussian(sigma=1.)]\n", 196 | "\n", 197 | " ls_q = np.array(([2., 2.] * Q))\n", 198 | " var_q = np.array(([1.0]*Q))\n", 199 | "else:\n", 200 | " print('Single output mode')\n", 201 | " if y_dim == 1:\n", 202 | " X = [x1_high_new[sorted_rnd_idx_high_res]]\n", 203 | " Y = [Y1_high_norm[sorted_rnd_idx_high_res]]\n", 204 | " elif y_dim == 2:\n", 205 | " X = [x2_high_new[sorted_rnd_idx_high_res]]\n", 206 | " Y = [Y2_high_norm[sorted_rnd_idx_high_res]]\n", 207 | " \n", 208 | " # Creating inducing inputs using cluster centers of the original images\n", 209 | " X_ = np.vstack((X[0], X[0]))\n", 210 | " \n", 211 | " # Heterogeneous Likelihood Definition\n", 212 | " likelihoods_list = [Gaussian(sigma=1.)]\n", 213 | " ls_q = np.array(([2., 2.]*Q))\n", 214 | " var_q = np.array(([1.0]*Q))\n", 215 | "\n", 216 | "if M > X_.shape[0]:\n", 217 | " logging.warning(\"More inducing points than X - setting Z to %d\", X_.shape[0])\n", 218 | " M = X_.shape[0]\n", 219 | " \n", 220 | "kmeans_X = KMeans(n_clusters=M, random_state=0).fit(X_)\n", 221 | "kmeans_X.cluster_centers_.shape\n", 222 | "Z = kmeans_X.cluster_centers_\n", 223 | "Z[:,-1] = 1\n", 224 | "\n", 225 | "likelihood = HetLikelihood(likelihoods_list)\n", 226 | "Y_metadata = likelihood.generate_metadata()\n", 227 | "\n", 228 | "D = likelihood.num_output_functions(Y_metadata)\n", 229 | "\n", 230 | "W_list, _ = util.random_W_kappas(Q, D, rank=1, experiment=True)\n", 231 | "\n", 232 | "# KERNELS\n", 233 | "input_dim = 5\n", 234 | "\n", 235 | "kern_list = util.latent_functions_prior(Q, lenghtscale=ls_q, variance=var_q, input_dim=input_dim)\n", 236 | "\n", 237 | "model = SVMOGP(X=X, Y=Y, Z=Z, kern_list=kern_list, likelihood=likelihood, Y_metadata=Y_metadata, batch_size=50)\n", 238 | "\n", 239 | "print(model)\n", 240 | "\n", 241 | "print('model.kern_q0.lengthscale', model.kern_q0.lengthscale)\n", 242 | "\n", 243 | "# Z should be fixed, is not implemented\n", 244 | "model.Z.fix()\n", 245 | "\n", 246 | "def transform_y(data_test):\n", 247 | " mpred, vpred = model.predict(data_test)\n", 248 | " mpred_transformed = []\n", 249 | " vpred_transformed = []\n", 250 | " if mode == 'multi_output':\n", 251 | " for i in range(len(mpred)):\n", 252 | " if i % 2 == 1:\n", 253 | " continue\n", 254 | " if i == 0:\n", 255 | " m_pred_star = mpred[i] * Y1_high_std + Y1_high_mean\n", 256 | " v_pred_star = vpred[i] * Y1_high_std * Y1_high_std\n", 257 | " elif i == 2:\n", 258 | " m_pred_star = mpred[i] * Y2_high_std + Y2_high_mean\n", 259 | " v_pred_star = vpred[i] * Y2_high_std * Y2_high_std\n", 260 | " else:\n", 261 | " raise Exception(\"Not implemented!\")\n", 262 | " mpred_transformed.append(m_pred_star)\n", 263 | " vpred_transformed.append(v_pred_star)\n", 264 | " else:\n", 265 | " if y_dim == 1:\n", 266 | " m_pred_star = mpred[0] * Y1_high_std + Y1_high_mean\n", 267 | " v_pred_star = vpred[0] * Y1_high_std * Y1_high_std\n", 268 | " elif y_dim == 2:\n", 269 | " m_pred_star = mpred[0] * Y2_high_std + Y2_high_mean\n", 270 | " v_pred_star = vpred[0] * Y2_high_std * Y2_high_std\n", 271 | "\n", 272 | " mpred_transformed.append(m_pred_star)\n", 273 | " vpred_transformed.append(v_pred_star)\n", 274 | "\n", 275 | " return mpred_transformed, vpred_transformed\n", 276 | "\n", 277 | "def transform_y_test_set(data_set):\n", 278 | " mpred_transformed = []\n", 279 | " vpred_transformed = []\n", 280 | " if model == 'multi_output':\n", 281 | " iters_count = y_dim\n", 282 | " else:\n", 283 | " iters_count = 1\n", 284 | "\n", 285 | " for i in range(iters_count):\n", 286 | " mpred, vpred = model.predict(data_set)\n", 287 | " if i == 0:\n", 288 | " if mode == 'multi_output':\n", 289 | " m_pred_star = mpred[i*2] * Y1_high_std + Y1_high_mean\n", 290 | " v_pred_star = vpred[i*2] * Y1_high_std * Y1_high_std\n", 291 | " else:\n", 292 | " if y_dim == 1:\n", 293 | " m_pred_star = mpred[0] * Y1_high_std + Y1_high_mean\n", 294 | " v_pred_star = vpred[0] * Y1_high_std * Y1_high_std\n", 295 | " elif y_dim == 2:\n", 296 | " m_pred_star = mpred[0] * Y2_high_std + Y2_high_mean\n", 297 | " v_pred_star = vpred[0] * Y2_high_std * Y2_high_std\n", 298 | " elif i == 1:\n", 299 | " assert mode == 'multi_output'\n", 300 | " m_pred_star = mpred[i*2] * Y2_high_std + Y2_high_mean\n", 301 | " v_pred_star = vpred[i*2] * Y2_high_std * Y2_high_std\n", 302 | " else:\n", 303 | " raise Exception(\"Not implemented!\")\n", 304 | "\n", 305 | " mpred_transformed.append(m_pred_star)\n", 306 | " vpred_transformed.append(v_pred_star)\n", 307 | "\n", 308 | " return mpred_transformed, vpred_transformed\n", 309 | "\n", 310 | "\n", 311 | "def error_calc(data_test, true_labels, calculate_snlp=False, test_mode=False, Y_train_snlp=None, \n", 312 | " single_dim_only=False):\n", 313 | " if test_mode == False:\n", 314 | " m_pred_star, v_pred_star = transform_y(data_test)\n", 315 | " else:\n", 316 | "# if test_mode is set, means that we want inference for test set data, x is shared, y is a list of y1, y2, ...\n", 317 | " m_pred_star, v_pred_star = transform_y_test_set(data_test)\n", 318 | "\n", 319 | " smse_error = error_func.smse(mu_star_list=m_pred_star, Y_test_list=true_labels)\n", 320 | " if calculate_snlp:\n", 321 | " if single_dim_only:\n", 322 | " snlp_error = error_func.snlp(mu_star_list=[m_pred_star[0]], var_star_list=[v_pred_star[0]], \n", 323 | " Y_test_list=true_labels, Y_train_list=Y_train_snlp)\n", 324 | " else:\n", 325 | " snlp_error = error_func.snlp(mu_star_list=m_pred_star, var_star_list=v_pred_star, \n", 326 | " Y_test_list=true_labels, Y_train_list=Y_train_snlp)\n", 327 | "\n", 328 | " else:\n", 329 | " snlp_error = None\n", 330 | " msll_error = None\n", 331 | " return smse_error, snlp_error\n", 332 | "\n", 333 | "max_iter = 1000\n", 334 | "print(\"Optimiser max iter: \", max_iter)\n" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 3, 340 | "metadata": { 341 | "scrolled": false 342 | }, 343 | "outputs": [ 344 | { 345 | "name": "stdout", 346 | "output_type": "stream", 347 | "text": [ 348 | "svi - iteration 100/1000:[[-1457.20677999]]\n", 349 | "svi - iteration 200/1000:[[-1218.87779029]]\n", 350 | "svi - iteration 300/1000:[[-983.52022031]]\n", 351 | "svi - iteration 400/1000:[[-871.14472836]]\n", 352 | "svi - iteration 500/1000:[[-808.53632294]]\n", 353 | "svi - iteration 600/1000:[[-803.52985688]]\n", 354 | "svi - iteration 700/1000:[[-853.80240453]]\n", 355 | "svi - iteration 800/1000:[[-831.94530008]]\n", 356 | "svi - iteration 900/1000:[[-810.09994718]]\n", 357 | "svi - iteration 1000/1000:[[-775.16090062]]\n", 358 | "--- 2523.0613961219788 seconds ---\n", 359 | "Test SMSE: [0.04392279]\n", 360 | "Test SNLP: [-3.68148376]\n", 361 | "***********************************************************************\n", 362 | "\n", 363 | "Name : SVMOGP\n", 364 | "Objective : 772.1801584682671\n", 365 | "Number of Parameters : 5657\n", 366 | "Number of Optimization Parameters : 5157\n", 367 | "Updates : True\n", 368 | "Parameters:\n", 369 | " \u001b[1mSVMOGP. \u001b[0;0m | value | constraints | priors\n", 370 | " \u001b[1minducing_inputs \u001b[0;0m | (100, 5) | fixed | \n", 371 | " \u001b[1mm_u \u001b[0;0m | (100, 1) | | \n", 372 | " \u001b[1mL_u \u001b[0;0m | (5050, 1) | | \n", 373 | " \u001b[1mkern_q0.variance \u001b[0;0m | 0.4221733527473745 | +ve | \n", 374 | " \u001b[1mkern_q0.lengthscale\u001b[0;0m | (2,) | +ve | \n", 375 | " \u001b[1mB_q0.W \u001b[0;0m | (2, 1) | | \n", 376 | " \u001b[1mB_q0.kappa \u001b[0;0m | (2,) | +ve | \n" 377 | ] 378 | } 379 | ], 380 | "source": [ 381 | "def callback(i):\n", 382 | " if i['n_iter'] % 100 == 0:\n", 383 | " print('svi - iteration ' + str(i['n_iter']) + '/' + str(max_iter) + \":\" + (str(model.log_likelihood())))\n", 384 | "\n", 385 | " # Note that X_test is numpy array, and Y_test is a list\n", 386 | " if y_dim == 1:\n", 387 | " test_smse_error, test_snlp_error = error_calc(X_test1, Y_test, calculate_snlp=True, \n", 388 | " test_mode=True, \n", 389 | " Y_train_snlp=Y)\n", 390 | "\n", 391 | " elif y_dim == 2: \n", 392 | " if mode == 'multi_output':\n", 393 | " test_smse_error_y1, test_snlp_error_y1 = error_calc(X_test1, [Y_test[0]], \n", 394 | " calculate_snlp=True, \n", 395 | " test_mode=True,\n", 396 | " Y_train_snlp=[Y[0]],\n", 397 | " single_dim_only=True)\n", 398 | " test_smse_error_y2, test_snlp_error_y2 = error_calc(X_test2, [Y_test[1]], \n", 399 | " calculate_snlp=True, \n", 400 | " test_mode=True,\n", 401 | " Y_train_snlp=[Y[2]],\n", 402 | " single_dim_only=True)\n", 403 | " test_smse_error = (test_smse_error_y1 + test_smse_error_y2) / 2.0\n", 404 | " test_snlp_error = (test_snlp_error_y1 + test_snlp_error_y2) / 2.0\n", 405 | "\n", 406 | "\n", 407 | " else:\n", 408 | " if y_dim == 1:\n", 409 | " test_smse_error, test_snlp_error = error_calc(X_test1, Y_test, \n", 410 | " calculate_snlp=True, \n", 411 | " test_mode=True, Y_train_snlp=Y)\n", 412 | " elif y_dim == 2:\n", 413 | " test_smse_error, test_snlp_error = error_calc(X_test2, [Y_test[1]], \n", 414 | " calculate_snlp=True, \n", 415 | " test_mode=True, Y_train_snlp=Y)\n", 416 | "\n", 417 | " if i['n_iter'] > max_iter:\n", 418 | " return True\n", 419 | "\n", 420 | " return False\n", 421 | "\n", 422 | "start_time = time.time()\n", 423 | "opt = climin.Adam(model.optimizer_array, model.stochastic_grad, step_rate=0.01, decay_mom1=1 - 0.9, \n", 424 | " decay_mom2=1 - 0.999)\n", 425 | "\n", 426 | "opt.minimize_until(callback)\n", 427 | "print(\"--- %s seconds ---\" % (time.time() - start_time))\n", 428 | "test_smse_error, test_snlp_error = error_calc(X_test1, Y_test, calculate_snlp=True, \n", 429 | " test_mode=True, \n", 430 | " Y_train_snlp=Y)\n", 431 | "\n", 432 | "print(\"Test SMSE:\", test_smse_error)\n", 433 | "print(\"Test SNLP:\", test_snlp_error)\n", 434 | "print(\"***********************************************************************\")\n", 435 | "\n", 436 | "print(model)\n" 437 | ] 438 | } 439 | ], 440 | "metadata": { 441 | "kernelspec": { 442 | "display_name": "Python 3", 443 | "language": "python", 444 | "name": "python3" 445 | }, 446 | "language_info": { 447 | "codemirror_mode": { 448 | "name": "ipython", 449 | "version": 3 450 | }, 451 | "file_extension": ".py", 452 | "mimetype": "text/x-python", 453 | "name": "python", 454 | "nbconvert_exporter": "python", 455 | "pygments_lexer": "ipython3", 456 | "version": "3.7.3" 457 | } 458 | }, 459 | "nbformat": 4, 460 | "nbformat_minor": 2 461 | } 462 | -------------------------------------------------------------------------------- /notebooks/synthetic_data/X1_train_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/X1_train_syn.npy -------------------------------------------------------------------------------- /notebooks/synthetic_data/X2_train_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/X2_train_syn.npy -------------------------------------------------------------------------------- /notebooks/synthetic_data/X_test_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/X_test_syn.npy -------------------------------------------------------------------------------- /notebooks/synthetic_data/Y1_train_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/Y1_train_syn.npy -------------------------------------------------------------------------------- /notebooks/synthetic_data/Y2_train_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/Y2_train_syn.npy -------------------------------------------------------------------------------- /notebooks/synthetic_data/Y_test_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/Y_test_syn.npy -------------------------------------------------------------------------------- /notebooks/synthetic_data/Z_syn.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/notebooks/synthetic_data/Z_syn.npy -------------------------------------------------------------------------------- /notebooks/toy_data_singleoutput.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys\n", 10 | "import climin\n", 11 | "from functools import partial\n", 12 | "import warnings\n", 13 | "import os\n", 14 | "sys.path.append('..')\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "from scipy.stats import multinomial\n", 18 | "from scipy.linalg.blas import dtrmm\n", 19 | "\n", 20 | "import GPy\n", 21 | "from GPy.util import choleskies\n", 22 | "from GPy.core.parameterization.param import Param\n", 23 | "from GPy.kern import Coregionalize\n", 24 | "from GPy.likelihoods import Likelihood\n", 25 | "from GPy.util import linalg\n", 26 | "\n", 27 | "from likelihoods.bernoulli import Bernoulli\n", 28 | "from likelihoods.gaussian import Gaussian\n", 29 | "from likelihoods.poisson import Poisson\n", 30 | "\n", 31 | "from likelihoods.hetgaussian import HetGaussian\n", 32 | "\n", 33 | "from hetmogp.util import draw_mini_slices\n", 34 | "from hetmogp.het_likelihood import HetLikelihood\n", 35 | "from hetmogp.svmogp import SVMOGP\n", 36 | "from hetmogp import util\n", 37 | "from hetmogp.util import vem_algorithm as VEM\n", 38 | "\n", 39 | "import matplotlib.pyplot as plt\n", 40 | "from matplotlib.pyplot import gca\n", 41 | "from matplotlib import rc, font_manager\n", 42 | "from matplotlib import rcParams\n", 43 | "\n", 44 | "warnings.filterwarnings(\"ignore\")\n", 45 | "\n", 46 | "from GPy.util.misc import safe_exp\n", 47 | "from GPy import kern\n", 48 | "\n", 49 | "from sklearn.cluster import KMeans\n", 50 | "\n", 51 | "import random" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 2, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "X1_train = np.load('./synthetic_data/X1_train_syn.npy')\n", 61 | "X2_train = np.load('./synthetic_data/X2_train_syn.npy')\n", 62 | "Y1_train = np.load('./synthetic_data/Y1_train_syn.npy')\n", 63 | "Y2_train = np.load('./synthetic_data/Y2_train_syn.npy')\n", 64 | "X1test = np.load('./synthetic_data/X_test_syn.npy')\n", 65 | "Y1test = np.load('./synthetic_data/Y_test_syn.npy')\n", 66 | "Z = np.load('./synthetic_data/Z_syn.npy')" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 3, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "M = 100 # number of inducing points\n", 76 | "Q = 1 # number of latent functions\n", 77 | "l = 4.0\n", 78 | "v_q = 1\n", 79 | "\n", 80 | "# Heterogeneous Likelihood Definition\n", 81 | "likelihoods_list = [Poisson()] \n", 82 | "likelihood = HetLikelihood(likelihoods_list)\n", 83 | "Y_metadata = likelihood.generate_metadata()\n", 84 | "D = likelihood.num_output_functions(Y_metadata)\n", 85 | "\n", 86 | "W_list, _ = util.random_W_kappas(Q, D, rank=1, experiment=True)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 4, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "# KERNELS\n", 96 | "input_dim = 3\n", 97 | "\n", 98 | "ls_q = np.array(([l]*Q))\n", 99 | "var_q = np.array(([v_q]*Q))\n", 100 | "kern_list = util.latent_functions_prior(Q, lenghtscale=ls_q, variance=var_q, input_dim=input_dim)" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 5, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "X = [X1_train]\n", 110 | "Y = [Y1_train]" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 6, 116 | "metadata": {}, 117 | "outputs": [ 118 | { 119 | "name": "stdout", 120 | "output_type": "stream", 121 | "text": [ 122 | "which mode:\n", 123 | " adam\n", 124 | "svi - iteration 50/1000[[-530.79598275]]\n", 125 | "svi - iteration 100/1000[[-406.90272871]]\n", 126 | "svi - iteration 150/1000[[-363.45208413]]\n", 127 | "svi - iteration 200/1000[[-339.98805796]]\n", 128 | "svi - iteration 250/1000[[-328.07267344]]\n", 129 | "svi - iteration 300/1000[[-322.95477695]]\n", 130 | "svi - iteration 350/1000[[-320.82677013]]\n", 131 | "svi - iteration 400/1000[[-319.92917758]]\n", 132 | "svi - iteration 450/1000[[-319.54872332]]\n", 133 | "svi - iteration 500/1000[[-319.39691053]]\n", 134 | "svi - iteration 550/1000[[-319.34496013]]\n", 135 | "svi - iteration 600/1000[[-319.33275921]]\n", 136 | "svi - iteration 650/1000[[-319.33300944]]\n", 137 | "svi - iteration 700/1000[[-319.3342831]]\n", 138 | "svi - iteration 750/1000[[-319.33270149]]\n", 139 | "svi - iteration 800/1000[[-319.32775139]]\n", 140 | "svi - iteration 850/1000[[-319.32019336]]\n", 141 | "svi - iteration 900/1000[[-319.31107781]]\n", 142 | "svi - iteration 950/1000[[-319.30134013]]\n", 143 | "svi - iteration 1000/1000[[-319.29167879]]\n" 144 | ] 145 | } 146 | ], 147 | "source": [ 148 | "step_rate = 0.01\n", 149 | "mode = \"adam\"\n", 150 | "\n", 151 | "if mode == \"adam\":\n", 152 | " model = SVMOGP(X=X, Y=Y, Z=Z, kern_list=kern_list, likelihood=likelihood, Y_metadata=Y_metadata, batch_size=150)\n", 153 | " model.Z.fix()\n", 154 | " print(\"which mode:\\n\", mode)\n", 155 | " max_iter = 1000\n", 156 | " def callback(i):\n", 157 | " # global max_iter\n", 158 | " if i['n_iter'] % 50 == 0:\n", 159 | " print('svi - iteration '+ str(i['n_iter'])+'/' + str(int(max_iter)) + str(model.log_likelihood()))\n", 160 | "\n", 161 | " if i['n_iter'] > max_iter:\n", 162 | " return True\n", 163 | " return False\n", 164 | "\n", 165 | " opt = climin.Adam(model.optimizer_array, model.stochastic_grad, step_rate=step_rate, \n", 166 | " decay_mom1=1 - 0.9, decay_mom2=1 - 0.999)\n", 167 | " opt.minimize_until(callback)\n", 168 | "\n", 169 | "elif mode == \"vem\": \n", 170 | " print(\"which mode:\\n\", mode)\n", 171 | " model = SVMOGP(X=X, Y=Y, Z=Z, kern_list=kern_list, likelihood=likelihood, Y_metadata=Y_metadata)\n", 172 | " model = VEM(model, vem_iters=10, maxIter_perVEM=100, optZ=False, verbose=False,verbose_plot=False,non_chained=True)\n", 173 | " model.kern_q0.variance.fix()\n", 174 | "# model = SVMOGP(X=X, Y=Y, Z=Z, kern_list=kern_list, likelihood=likelihood,Y_metadata=Y_metadata, batch_size=30)\n", 175 | "# model = VEM(model, vem_iters=30, maxIter_perVEM=100, step_rate=step_rate, optZ=False, verbose=True, verbose_plot=False)" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 7, 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "name": "stdout", 185 | "output_type": "stream", 186 | "text": [ 187 | "model:\n", 188 | " \n", 189 | "Name : SVMOGP\n", 190 | "Objective : 273.0394174184195\n", 191 | "Number of Parameters : 5454\n", 192 | "Number of Optimization Parameters : 5154\n", 193 | "Updates : True\n", 194 | "Parameters:\n", 195 | " \u001b[1mSVMOGP. \u001b[0;0m | value | constraints | priors\n", 196 | " \u001b[1minducing_inputs \u001b[0;0m | (100, 3) | fixed | \n", 197 | " \u001b[1mm_u \u001b[0;0m | (100, 1) | | \n", 198 | " \u001b[1mL_u \u001b[0;0m | (5050, 1) | | \n", 199 | " \u001b[1mkern_q0.variance \u001b[0;0m | 1.2499084186527034 | +ve | \n", 200 | " \u001b[1mkern_q0.lengthscale\u001b[0;0m | 3.626884787694605 | +ve | \n", 201 | " \u001b[1mB_q0.W \u001b[0;0m | [-0.80724708] | | \n", 202 | " \u001b[1mB_q0.kappa \u001b[0;0m | 5.562684646268137e-309 | +ve | \n" 203 | ] 204 | }, 205 | { 206 | "data": { 207 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFCFJREFUeJzt3X+MHGd9x/HPN3YiYn4oxT5QQ7g7IiGatCKm3ouCgio20MoJUagqsIJMhSqkkwx/BJUKJaClwlb+8EmhoLZYsgIFyW7okjYF0QoacYMKqpTsXn40SR1aSB0ThcYHhQINok3y7R+z2/uR/TGz98zOPLPvl7Sam+eenXmemWc/N/vc7J25uwAA8big7AYAAPIhuAEgMgQ3AESG4AaAyBDcABAZghsAIkNwA0BkCG4AiAzBDQCR2V3ERvft2+eLi4tFbBoAamltbe2H7j6XpW4hwb24uKhut1vEpgGglszsyax1mSoBgMgQ3AAQGYIbACJDcANAZAhuAIjM2OA2szeY2UObHj81sw9No3EIYGVFSpKtZUmSloeoD2Dqxga3u3/H3fe7+35JByQ9K+mewluGMJaWpEOHNsI4SdL1paUw9QFMXd77uN8m6Xvunvl+Q5Ss2ZTa7TR8jxyRTpxI15vNMPUBTF3eOe6bJd016BtmtmxmXTPrrq+v77xlCKfZTEP42LF0OS6E89YHMFWZg9vMLpJ0k6QvDfq+u59094a7N+bmMn1qE9OSJOmVc6uVLrfPYe+0PoCpynPFfb2kB9z9maIagwL056jbbeno0Y1pkGFhnLc+gKnLE9zv0ZBpElRYp7N1jro/h93phKkPYOrM3cdXMtsj6fuSLnf3/xpXv9FoOH9kCgCyM7M1d29kqZvprhJ3f1bS3h21CgAQBJ+cBIDIENwAEBmCGwAiQ3ADQGQIbgCIDMENAJEhuAEgMgQ3AESG4AaAyBDcABAZghsAIkNwA0BkCG4AiAzBDQCRIbgBIDIENwBEhuAGgMgQ3AAQGYIbACKTKbjN7BIzu9vMHjezM2b25qIbhpxWVqQk2VqWJGk5gOwieC1lveL+tKSvufuvSbpK0pnimoSJLC1Jhw5tDLgkSdeXlsptFxCbCF5L5u6jK5i9QtLDki73cZV7Go2Gd7vdAM1DLv0BduSIdOKE1G5LzWbZrQLiU8JryczW3L2RpW6WK+7LJa1L+gsze9DM7jSzlw7Y6bKZdc2su76+nrPJCKLZTAfasWPpktAGJlPx11KW4N4t6TclnXD3N0n6b0m3bq/k7ifdveHujbm5ucDNRCZJkl4dtFrpcvs8HYBsKv5ayhLcT0l6yt3v663frTTIUSX9t3bttnT0aLrcPE8HIJsIXktjg9vd/0PS983sDb2it0n6l0Jbhfw6na3zcM1mut7plNsuIDYRvJbG/nJSksxsv6Q7JV0k6QlJf+DuPx5Wn19OAkA+eX45uTtLJXd/SFKmDQIAisUnJwEgMgQ3AESG4AaAyBDcABAZghsAIkNwA0BkCG4AiAzBDQCRIbgBIDIENwBEhuAGgMgQ3AAQGYIbACJDcANAZAhuAIgMwQ0AkSG4ASAyBDcARIbgBoDIZPqfk2Z2VtLPJD0v6bms/9ASABBenivuprvvJ7Rn1MqKlCRby5IkLcfOcGzjUKHzxFQJsllakg4d2hi4SZKuLy2V26464NjGoUrnyd3HPiT9u6QHJK1JWh5X/8CBA44aWl1137fPvdVKl6urZbeoPji2cSjwPEnqeoY8dvfMwX1pb/kqSQ9L+q0BdZYldSV15+fng3UGFdNqpcOm1Sq7JfXDsY1DQecpT3Bnmipx96d7y/OS7pF09YA6J9294e6Nubm5Hb0LQEUliXTihNRqpcvt832YHMc2DlU5T+OSXdJLJb1809f/JOngqOcwVVJD/beI/beG29cxOY5tHAo+Twp8xf1qSd82s4cl3S/p79z9a8X8GEFldTpSuy01m+l6s5mudzrltqsOOLZxqNB5sjTow2o0Gt7tdoNvFwDqyszWPOPt1twOCACRIbgBIDIENwBEhuAGgMgQ3AAQGYIbACJDcANAZAhuAIgMwQ0AkSG4ASAyBDcARIbgBoDIENwAEBmCGwAiQ3ADQGQIbgCIDMENAJEhuAEgMgQ3AEQmc3Cb2S4ze9DMvlpkgwAAo+W54r5F0pmiGoKIraxISbK1LEnScqCqIh63mYLbzC6T9A5JdxbbHERpaUk6dGjjRZAk6frSUrntAkaJeNxmveL+lKSPSHqhwLYgVs2m1G6ng/7jH0+X7XZaDlRVxON2bHCb2Y2Szrv72ph6y2bWNbPu+vp6sAYiEs2mdOSIdOxYuoxg8AOxjtssV9zXSrrJzM5K+qKk68zs1PZK7n7S3Rvu3pibmwvcTFRekkgnTkitVrrcPncIVFGk43ZscLv7be5+mbsvSrpZ0qq7v7fwliEe/bnBdls6enTj7WckLwLMqIjHLfdxY+c6na1zg/25w06n3HYBo0Q8bs3dg2+00Wh4t9sNvl0AqCszW3P3Rpa6XHEDQGQIbgCIDMENAJEhuAEgMgQ3AESG4AaAyBDcABAZghsAIkNwA0BkCG4AiAzBDQCRIbgBIDIENwBEhuAGgMgQ3AAQGYIbACJDcANAZAhuAIgMwQ0AkSG4ASAyY4PbzF5iZveb2cNm9piZfWIaDQMwwsqKlCRby5IkLceGmh6nLFfcv5R0nbtfJWm/pINmdk2xzQIw0tKSdOjQRiglSbq+tFRuu6qmpsdp97gK7u6Sft5bvbD38CIbBWCMZlNqt9MQOnJEOnEiXW82y25ZtdT0OGWa4zazXWb2kKTzku519/sG1Fk2s66ZddfX10O3E8B2zWYaRseOpcvIw6gwNTxOmYLb3Z939/2SLpN0tZn9xoA6J9294e6Nubm50O0EsF2SpFeQrVa63D6Xi1QNj1Ouu0rc/SeSvinpYCGtAZBNf6623ZaOHt2YDqhBKAVV0+OU5a6SOTO7pPf1xZLeLunxohsGYIROZ+tcbX8ut9Mpt11VU9PjZOnvHkdUMHujpC9I2qU06NvufnTUcxqNhne73WCNBIC6M7M1d29kqZvlrpJ/lvSmHbcKABAEn5wEgMgQ3AAQGYIbACJDcANAZAhuAIgMwQ0AkSG4ASAyBDcARIbgBoDIENwAEBmCGwAiQ3ADQGQIbgCIDMENAJEhuAEgMgQ3AESG4AaAyBDcABAZghsAIpPlv7y/1swSMztjZo+Z2S3TaBgGWFmRkmRrWZKk5WVuq8x9oJ5iGjsltDXLFfdzkj7s7ldIukbSB83sysJahOGWlqRDhzYGSZKk60tL5W6rzH2gnmIaO2W01d1zPSR9WdJvj6pz4MABR0FWV9337XNvtdLl6mo1tlXmPlBPMY2dAG2V1PWsOZy1YrpdLUo6J+kVA763LKkrqTs/Pz/5AcB4rVZ66lqtam2rzH2gnmIaOztsayHBLellktYk/d64ulxxF4grbsyKmMZOFa+4JV0o6euS/jBLfYK7IP3B0R8U29fL2laZ+0A9xTR2ArU1T3BnuavEJH1W0hl3/2S42XXk1ulI7bbUbKbrzWa63umUu60y94F6imnslNBWS4N+RAWzt0j6lqRHJL3QK/6ou//9sOc0Gg3vdrvBGgkAdWdma+7eyFJ397gK7v5tSbbjVgEAguCTkwAQGYIbACJDcANAZAhuAIgMwQ2g1k6flhYXpQsuSJenT4//3qjnVAHBDaCS8obqoPLTp6XlZenJJyX3dLm8PPp7H/jA6OdUItCzflInz6Mun5w8dcp9YcHdLF2eOjVZechtlbmPWd133ftXxX2fOuW+Z0/62e7+Y88e9yNH8pXv3bu1rP9YWEgfg763a9fg8r17B+9j8/HaCRX1R6ayPmIL7kGDJ9TACbmtMvcxq/uue/+quu+8oTqsfNjDLH3kec6wx8JCmBzKE9xjPzk5iap+cvL0aeljH5POnZPm56Xbb0/Ll5elZ5/dqLdnj3TxxdKPfvTibezaJT3/fPbyhYV0+eSTO99WmfuY1X3XvX9V3fe5c2ksFmWS/g1jJr3wwvh647eT/ZOTmdI976OKV9zDrh6GvZUK9Qj5k73Mfczqvuvev6ruO9QV96jpjbzvKEZNu4yabspKszxVMuwADhsIeR95B84kc2lV3Mes7rvu/avqvkNPVYaYw8/bprzhPbPBPezAnjqV/+ph2E/qKs4Vlj0fWed9171/Vd13//Uc6pesITNm+z6G/YDLO/c9E8Gd9wAO+964t1Kx/HZ+GvuY1X3XvX9V3Xcshl0UmuXbTpTBHeIty6CD1z+Ao67G6zB4AJRjZq+4Q/2SYNScWX8/BDSAkEZdFOaRJ7grcTvg4mKY23Kk9Fa+7bf2nTwpHT6cbzsAkNWgW43zZk6e2wEr8ZH3c+cGl+cN7YWFNKQXFtJ7K/vrhDaAIh0+LJ09m97PffZs8Zkz9j/gTMP8fL4r7r17pV/84sVX1v2fcgQ1gDqrxBX37benwbvZnj3pJxoHlX/601xZA5hdWf7L++fM7LyZPVpUIw4fHhzEn/nM8ICe9lsTAKiKLFfcn5d0sNBWrKzo8KXJ1iC+NEnL8wT0yoqUJFvLknQ7wZ4zyT6w1bBjeMMN4Y7tNM5T3n6E7F/eNlVxfIZsa0z9DiHLrSeSFiU9mvVWldz3ca+uuu/bly4HrRe5nbzPCdXWWTbsGN5xR7hjO43zlLcfIfuXt01VHJ8h2xpTv4dQ6Pu4Cw9u940D3Wrt7IBPsp28zwnV1lk27BiGPLbTOE95+1Fmm6ootvNdoFKCW9KypK6k7vz8/GQtb7XSJrVakz1/J9vJ+5xQbZ1lw45hyGM7jfOUtx9ltqmKYjvfBeGKmyvu6uOKmytu9/jOd4HiC27muGcLc9zTb1MVxydz3FsEDW5Jd0n6gaT/lfSUpPePe07u4D5+/MUHeHU1LS96O3mfE6qts2zYMbz++nDHdhrnKW8/QvYvb5uqOD5DtjWmfg+RJ7gr8bdKAGDWRfe3SgAA2RHcABAZghsAIkNwA0BkCG4AiAzBDQCRIbgBIDIENwBEhuAGgMgQ3AAQGYIbACJDcANAZAhuAIgMwQ0AkSG4ASAyBDcARIbgBoDIENwAEBmCGwAikym4zeygmX3HzL5rZrcW3SgAwHBjg9vMdkn6c0nXS7pS0nvM7MqiGyZJWlmRkmRrWZJIN9wwuHxlJf+2hj0nZP28/Zikf3U2ybHNe55GHfO8+wgp1L5D9jvvuA05nss8F1Uy7t/AS3qzpK9vWr9N0m2jnnPgwIEw/69+ddV93750uXn9jjsGl/fX82xr2HNC1s/bj0n6V2eTHNu852nUMc+7j5BC7Ttkv/OO25DjucxzUTBJXR+Tx/1HluB+l6Q7N63/vqQ/G/WcYMHtvnFiWq3BJ2x7+STbmkb9vP2YpH91NsmxzbutkPsIKdS+Q/a7zPFc09dG6OB+94Dg/tMB9ZYldSV15+fnw/ao1Uqb2mplK59kW9Oon7cfk/SvziY5tnm3FXIfIYXad8h+lzmea/jaCB3c5U2VuHPFXbOriolxxc0V97g2RS50cO+W9ISk10m6SNLDkn591HOY4962HmqusCYDNLdJjm3e88Qc94vrhRq3zHFnEjS40+3pBkn/Kul7kj42rn6w4D5+/MUnZHXV/frrB5cfP55/W8OeE7J+3n5M0r86m+TY5j1Po4553n2EFGrfIfudd9yGHM9lnouC5QluS+uH1Wg0vNvtBt8uANSVma25eyNLXT45CQCRIbgBIDIENwBEhuAGgMgQ3AAQmULuKjGzdUlPTvj0fZJ+GLA5saDfs4V+z5Ys/V5w97ksGyskuHfCzLpZb4mpE/o9W+j3bAndb6ZKACAyBDcARKaKwX2y7AaUhH7PFvo9W4L2u3Jz3ACA0ap4xQ0AGKEywT1L/5DYzD5nZufN7NFNZa80s3vN7N96y18ps42hmdlrzSwxszNm9piZ3dIrr3W/JcnMXmJm95vZw72+f6JX/jozu6/X978ys4vKbmtoZrbLzB40s6/21mvfZ0kys7Nm9oiZPWRm3V5ZsLFeieAu9R8Sl+Pzkg5uK7tV0jfc/fWSvtFbr5PnJH3Y3a+QdI2kD/bOcd37LUm/lHSdu18lab+kg2Z2jaTjkv6k1/cfS3p/iW0syi2Szmxan4U+9zXdff+m2wCDjfVKBLekqyV9192fcPf/kfRFSe8suU2Fcfd/lPSf24rfKekLva+/IOl3p9qogrn7D9z9gd7XP1P6Yn6Nat5vSer9ueWf91Yv7D1c0nWS7u6V167vZnaZpHdIurO3bqp5n8cINtarEtyvkfT9TetP9cpmyavd/QdSGnKSXlVyewpjZouS3iTpPs1Iv3tTBg9JOi/pXqX/lOQn7v5cr0odx/ynJH1E0gu99b2qf5/7XNI/mNmamS33yoKN9d0BGhiCDSjjdpcaMrOXSfprSR9y95+mF2H15+7PS9pvZpdIukfSFYOqTbdVxTGzGyWdd/c1M3trv3hA1dr0eZtr3f1pM3uVpHvN7PGQG6/KFfdTkl67af0ySU+X1JayPGNmvypJveX5ktsTnJldqDS0T7v73/SKa9/vzdz9J5K+qXSe/xIz61881W3MXyvpJjM7q3Tq8zqlV+B17vP/c/ene8vzSn9QX62AY70qwd2R9Preb5wvknSzpK+U3KZp+4qk9/W+fp+kL5fYluB685uflXTG3T+56Vu17rckmdlc70pbZnaxpLcrneNPJL2rV61WfXf329z9MndfVPp6XnX3w6pxn/vM7KVm9vL+15J+R9KjCjjWK/MBHDO7QelP5F2SPufut5fcpMKY2V2S3qr0L4Y9I+mPJf2tpLakeUnnJL3b3bf/AjNaZvYWSd+S9Ig25jw/qnSeu7b9liQze6PSX0btUnqx1Hb3o2Z2udKr0VdKelDSe939l+W1tBi9qZI/cvcbZ6HPvT7e01vdLekv3f12M9urQGO9MsENAMimKlMlAICMCG4AiAzBDQCRIbgBIDIENwBEhuAGgMgQ3AAQGYIbACLzf/ePDePA8N15AAAAAElFTkSuQmCC\n", 208 | "text/plain": [ 209 | "
" 210 | ] 211 | }, 212 | "metadata": { 213 | "needs_background": "light" 214 | }, 215 | "output_type": "display_data" 216 | }, 217 | { 218 | "data": { 219 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXuQHVd957+/GT2vhGUxslgWrDtAZSF4ywEsE1PE7GqBbGxYYPPAwmPHKbw1RnnBOo81mjLJklKCBKKWPAQISJbQs7G17MqxVbULa2Zq88+GHRkMyDEsOJYGxwbLBgTyENszc/aP063bt6dP9+m+/Tjn3u+n6tad29OP3/md098+/evT5ydKKRBCCPGHsbYNIIQQUgwKNyGEeAaFmxBCPIPCTQghnkHhJoQQz6BwE0KIZ1C4CSHEMyjchBDiGRRuQgjxjHV17HTHjh1qcnKyjl0TQshQct999z2hlLrEZt1ahHtychInT56sY9eEEDKUiMgZ23UZKiGEEM+gcBNCiGdQuAkhxDMo3IQQ4hkUbkII8QwKNyFVcugQMD/fv2x+Xi8npCIo3IRUyZVXAm9/e0+85+f17yuvbNcuMlTUMo6bkJFlzx7g2DEt1vv2AR/9qP69Z0/blpEhgj1uQqpmzx4t2n/wB/qbok0qhsJNSNXMz+ue9u236+9kzJuQAaFwE1IlUUz72DHg/e/vhU0o3qRCKNyEVMnCQn9MO4p5Lyy0axcZKkQpVflOd+/erTjJFCGE2CMi9ymldtusyx43IYR4BoWbEEI8g8JNCCGeQeEmhBDPoHATQohnULgJIcQzKNyEEOIZFG5CCPEMCjchhHgGhZsQQjyDwk0IIZ5B4SaEEM+gcBNCiGdQuAkhxDOshFtE/r2IPCAip0Tkr0RkU92GEUdhFnNSBt/bjWP25wq3iLwAwG8C2K2U+ucAxgHsrdsw4ijMYk7K4Hu7cc1+pVTmB8ALAHwbwHOhs8KfAPCzWdtcccUVigwxc3NK7dih1O236++5ubYtIj7ge7up2X4AJ1WOHkef3B63UuofAHwIwCKAxwCcU0p9PrmeiEyLyEkROXn27NnqrizEPZjFnJTB93bjkP02oZLtAN4K4EUA/imALSJyQ3I9pdRRpdRupdTuSy65pHpLiTswizkpg+/txiX787rkAH4JwKdiv38ZwJGsbRgqGWKi28XoNjH5m5A0fG83DdiPKkMl0CGSq0SkIyIC4PUAHqzpOkJch1nMSRl8bzeO2W+V5V1E/iOA6wAsA/gygH+nlHratD6zvBNCSDGKZHlfZ7OSUur3APzeQFYRQgipBL45SQghnkHhJoQQz6BwE0KIZ1C4CSHEMyjchBDiGRRuQgjxDAo3IYR4BoWbEEI8g8JNCCGeQeEmhBDPoHATQohnULgJIcQzKNyEEOIZFO5RxbGs1d4T92f0d9yfPvmWbcN5KNyjimtZq30n5s/Zc2/C5OtfgrF/9S8wefg3MDvzgF++ZdtwH9tUOUU+TF3mCb5n3XaNuTkVbJ1WnfVPK0Bd+HTwlAr2n2rbumKwbTQOKk5dRoYVh7JWDwV79mBm/CCWnt3Qt3gJHczMXtaSUSVh23AaCvco41LW6mFgfh6L5y5K/dfiYn6KQKdg23AaCveoEsUtjx0D3v9+/R2Pa5JihP7ctTM9FesuecQf37JtOA+Fe1RxLGu194T+PPDhzeisf7bvX50OcOC2H/rjW7YN57HK8l4UZnkno8zsLDAzAywuArt2AQcOAFNTbVtFXKfyLO+EEHumpijUpF4YKiGEEM+gcBNCiGdQuAkhxDMo3IQQ4hkUbkIqYnYWmJwExsb09+xs2xaRYYWjSgipgNlZYHoaWFrSv8+c0b8BjjAh1cMeNyEVMDPTE+2IpSW9nJCqoXATUgGLi8WWEzIIFG5CKmDXrmLLCRkECjchFXDggJ6TJE6no5cTUjUUbkIqYGoKOHoU6HYBEf199CgfTJJ64KgSQiqCc5SQprDqcYvIxSLyWRH5uog8KCKvqdswQggh6diGSj4C4H8qpV4G4KcAPFifSQmYcZoQQvrIFW4RuQjA6wB8CgCUUs8opX5Qt2EXYMZpQkiLuPhGrE2P+8UAzgL4CxH5soh8UkS21GxXjyj7xtvfDrzvfb2USkxeSgipmeiN2DNnAKV6b8S2Ld42wr0OwKsAfFQp9UoATwG4LbmSiEyLyEkROXn27NlqrWTGaUJIC7j6RqyNcD8C4BGl1BfD35+FFvI+lFJHlVK7lVK7L7nkkiptZMZpQkgruPpGbK5wK6W+A+DbIvLScNHrAfxdrVbFYcZpQkhLuPpGrO2okt8AMCsiXwXwCgB/WJ9JCZhxmhDSEq6+Ecss74QQksHsrI5pLy7qnvaBA/W8aMUs74QQUhEuvhHLuUoIqQEXx/6S4YE9bkIqhtlwSN2wx01Ixbg69pcMDxRuQirG1bG/ZHigcBNSMa6O/SXDA4WbkIpxdewvGR4o3IRUDLPhkLqhcBNSA1NTwOnTwOqq/qZo+4fLQzo5HJAQQhK4PqSTPW5CCEng+pBOCjchhCRwfUgnhZsQQhK4PqSTwk0IIQnShnSuXw+cP+/Gw0q3hZsZ3okP+NROm7LVJ58kOXQIU39zC46+54ELQzonnvMMZOVZPPmkG7kn3RZuZngnPuBTO23KVp98kuTKK4E77sDUn70Wp/9iHqtfmMfW89/BM6vr+1Zr9WGlUqryzxVXXKEqY25OqR07lLr9dv09N1fdvgmpCp/aaVO2+uSTJHNzSl10kVKdjlKbNyvBitJ97f6PSHWHBHBSWWqs+8KtlK54QH8T4io+tdOmbPXJJ0ki2wHV3fb9VOHudqs73HAJt89XbTI6+NRO2ePOJ9HjDjbfrDobl/tEu9NRKgiqO+TwCHdU8VGFJ38T4gI+tdOmbE3sN9h/SnXHFpXIqup2qxW8yolEe9s2/Xf4O9h8s+ruXFIiqpYyFBFutx9OMsM78QGf2mlTtobHmX10D3bsAG74w8twZvVSKCWtj8jIZWEB2LsXOH5c+2fPHuCuuzB14zhO/9afODH/DLO8E0JqITnfR5JuVwsg0RTJ8u52j5sQ4i1p833EceX1cR+hcBNCaiFPmF15fdxHKNyEVIDLcze3RZYwMyPQYFC4CRmQKJZ75owbr0O7Qtp8HwAwMcGMQINC4SZkQFyfu7kt0lK4BQHwxBMU7UHhqBJCBmRsTPe0k4jo1GWE2MBRJYQ0iOtzN5Phg8JNyICkxXL58I3UCYWbkAFJi+Xy4RupE2Z5J6QCpqYo1KQ52OMmhBDPoHATQohnULgJIcQzrIVbRMZF5MsicqJOgwghhGRTpMf9bgAP1mUIIZVQVXZxn7OUt8mw+c3R8lgJt4i8EMCbAHyyXnMIGZCqsov7nKW8TYbNb66WxyZNDoDPArgCwL8EcCJv/cqTBRNShKpyHfqcM7FNhs1vDZUHVeacBPBmAEfCv43CDWAawEkAJ3ft2lVLwQixpqrs4j5nKW+TYfNbA+WpWrj/CMAjAE4D+A6AJQBB1jbscZNWYY+7XYbNbz72uFW/iDNUQtymqizmPmVud4mYn4JA6azoWFHdnUtuZ3Y30WA7KCLcHMdNhouqspj7lLndJWLZ3aengTOPb4bCGM48vtnP5BKOtgPOx00IqZzJSZ0JKAkzu5vhfNyEkFYxJQpmZvdq8EK4mYiV+Mqotl0ml6gX54WbiVjdY1TFqCij3HZ9Ty7hfBu3fYpZ5FPlqJJuV499SX663coOQQoQBEp1Ov110ekoP0cM1Myot90g0GUV0d/79vX/drXNtNXGUWBUifMPJ02JWAGdMZqT1zcLHzrZwyTCPaK7j6Wl3rJOx81MQW218aF6OJkVExuV2866KHo7ODub3qABPnRKg3HeHjMz/aIN6N8zM+3Yk4UPD1adF+60WFmEqxXvA0Xjr9H6JkZRjPLwPc5bJT6IYYQPF1znhTtKxGrCxYr3gaI9oLT1I0ZVjPJgEuEePohhhBcXXNtgeJFPHa+8j/qDnqoRSfenSLH1AXcfMhF38O2hdvLBahN2YhhfeffiKugRRXtApuXd7mj2IEkxfLv7mJrSDyJXV/W3a3Z6I9y+VbzrFL0Q8sJJBsV1MbTBmfHdtl3zIh/ODugHRW8H27h9JMQV6g73YJjGcRNCiAvUPb57qMZxE0KIC7g0pNF54b4QUxKFyef9uD+mdMst+hOnrgzMjmZ7LkRUhkOHMDvzQBirU5jcfg6zMw/0lyWrvC76Im5T9Hfcpjrsy/OD6z4sY0ORbWLtrS/Zbny5R+zadq7Q8lqxjakU+VQV406NKW1c1jGluTmlLrpIqW3bmslSMgwZUUKbg+tPqA6e6h8GiBUFrPZi11nlddEXSfu2bdPtI83eOo5Z9LcLPixjQ5Ftov8dPpz+7dO5o5QK9p9ac9508JQK9p+qZP+oK3WZ7acq4TaO3d72/bUnQBP57YYhl97cnOqOLRrHZPc9cMkqr4u+iNsUXdTrti/PD677sIwNRbaJ1r3xRv1U+8Yb3WkvJQj2n1LdsUUlWFXdscXKRFupIRJu40siWOnPttxkRukhyF4tWM0U7r4XmwzlDQJ9ARWsqO6277szwiRub1N1lXecrP+70J7K2FBkm2jdq69uv6xVUFOdDY1wm3rc47LcS0C6/xR73EWw6HFfeIPSUN4g0CGr1BBWm7TQ487tgbHHPVQ97jrrbGiEOy3GjURvsS/GxBh3Nhkx7jU97p1LxvJ2dy6Zt2m5bE3GuHNjnoxxD1WMu+46KyLcTo8q6XtbEgrjY6sApG+dJXQwM3uZ/lFnBmZHsz0XIizD1E89gKP7H0a3CwAKgv6x/J0OcGDPvcbyLj6+KXX3puWNEK+fhQXg+HHgrrv03zXV1cyRF2IJ/a+TLqGDmSMvXGsT0G+HC+2pjA1FtonWXV7W37fe2v/bp3MHcKPOQrx6AYcT09fD7Kye/W9xUc9JcuDA2teR4+uMjQErK2v3M2rJFNgeSZUM7Qs4Pk0N6RN5c0gk5+5OE+1RnLeE7TEdZ+bzGGK8Em5OdNQOprm4x8dHe8Ivtse1jHKC5CbxKlQC2N3Wk2phSMAM22M/zElaniKhEu+EmzQPT0ZiCy/y5RnaGDdpB4YERodB49OM+zcDhZvk4lsSCz4cK0cV8Wle5JuBoRIyVETiE3+Y2um4faFxhapCYoz7l4MxbjKyMB5fHlN8GgCCgOJbN4xxk5HFpcnufSMrDs0hfW5B4SZDhUl8nvvcZu2I40vM/dpr9TOMNJaWdPiDuAGFmwDwR1yymJ0Fzp9P/9+PftROmXx5IWV2Fvj0p82hEoB3LS7hrXAPg9C4gi/ikkVUhiefTP//M8+002NMe+vUxd6r6e3YOBzS5w5ePpzkyIFqGYYHeqYyxGnjJRBfXkjJejAJ8PxqgkofTorIpSIyLyIPisgDIvLuwU3MJ6tH7UsvxheG4YGeja1t9Bh9eCFldlafZyZcH7c/itiESpYB/JZS6icBXAXg10Tk5XUaZbx133sPMD+PxTPpXQPT8j6ayK7dVAbvQ4d0lvv4sebn9TLbYx06hF1b0uMLuzZ+V2eD334OY6Iw+bwf97LBz88DP/3T+lhxklnOI/vimdcj+9IynpfJ0H7oEHZd8uPMYnbWP4sDL/7kwL4qaueBq+5BZ2P/dIqdjSs4cNU99r669lrgwx/u33eWTwq0v+hcS53xcf2zCK67pzdjZFYG+0OHitnoIrZ+c6HsthkXog+Avwbwxqx1Bs2AY0wSHGZl6W4/Vz4DSxOZR5rKbjI315/pPvk7hSDQ/hXR38H+UyrY9M7UTC77XvOl9Awv15/Qx9mypf9YaeWO7Dl8WH93Or3fadlgymSvmZtTwdbpNenUotyaUTmL+irV30XtDG3rbj+n0+1tP6eCrdNrs8Bk+WrfPl1hhw+n+znLzpz1JybSz7XxsRUV4HrzMZO/Dx8uZqOLRHW1c0mfHzuXdF2ZEkBXXHbUlboMwCSARQAXZa03qHAbkwSHeRCDrdOqM/6P/YJSJOdhE7n+msonGJ3wnY5SmzfninYyFVyn0xPvrpzR4iJndC7FruECKmf6BTAvy3lk34YN5ryDg+aLjE66KIHxzqW17aGAr7KOU9hO27yLWb6KhN7WJzn1EgRm0Y4ScgfXnyiWT7OojY5RKJdqDWWvRbgBbAVwH4CfN/x/GsBJACd37dpV2Og4RsHohivcfrsK8A7V3fhY+SzjTWTXbjrLeM6xMv2asg/jBRQrF9axyvYe33dWpu9BM7TbbGPpK+vj2Nppm+k8y1dFfWJYPz2Xa/9nYvP59It8sn6Tx7Cwcc1dX9tJpkNydSdJibJnUblwA1gP4HMAbrVZf9Aet7FnGKhqskaPaI/bfCezmrqPvB53sP9Ufg+lwR63KSP9BZHYuaSCTe+srMfdu0sx9EiT2zjS487qaevPqpqQJ/JFrESvM/PcbpnMO/0krve4obPz/iWA/2S70yqyvKdelavIGj3CMW7r0Ee4j7Rs8PEYd1fOpO8vetbQYIw7zd/B/lNrRSLKwj5gjDs3w3tym7w221CMOwjyRFsLe/R8IPUin3aMw4dVgOt1LD8jPly4V9sg3Z1L2e05IhkL334u+3mAJVUL988AUAC+CuD+8HNt1jZVCHcqBw9qZ0TfSvX/PnjQfh9xbLctamedx4iOM504Oebm9LKUY6X2dtY/o4I9n0jfxzXX6Fj3xT/QvcqdS1qYwvIJVtJPbqyutS/ySdy+aHncX/H1ojLk+c7g7+7FP8gWiQxf5R3HuO+Lf5C+TV6bzfLVNddoUYjbmeWTjPZnEs5k7ze3fIljBNfdrTrrn1l793Xd3X1mFOrVNkxw3d3pd5CJMqSut/6Z/vVKnO+1PZy0/dQm3GRg8uKLReKPLvWe0uyuUyRcFqAsTHZHPe2ovouGNGzbgkttJg2b9l9XGSjcpBRFT1ZX4pUmO0yxXNsTLOskdl2A0ggCpcbHzaKdtr7tRdz2QuZKmxmEui7aIyXcrj6h9pGs22iTb13wv8nuiYnyImEaeRH1Sn0ToKyRJFXYXeRC5kKbGQT2uAfEt5PHdbJuo132bVYPqKxIZF3EIj9ULUDJsdXx0MWgmMozPl7NMUbpXKyrrCMj3D7errpM3oMrV31bdTuwGXlRZt9ZQh8ESq1fv/Y4GzZUI36D3N7bXqB870kXoY6yjoxw+/qAyFVM4uG6b6vsAdm8nFLGD3k2Zl00q+gVl724jVJPum1GRrjZ466evJczXPVtXm920JEyyU/aw7wsstqqTQ9/ULEsK8A8x5rDe+EucmvG3kC1ZMW5B+nFtnULXbSN5MX5TcKbR55fbY9V1geRAEejSga128U7L9/xWrjLDEkblbhaE1T9EKvti2vRHqNtj7uqnqtpeJ7pU5RB/c8ed3N4LdxsKO1StdDWPZohj6I9xujt8qLiXSZWnPc8Ic3moj4b9HxKs1tE+ym5HjtQg+G1cPPWrH2qPAkH7akOStHxxWkitXWrnajmkRzuV+ZT9IJXxflkupj5OqbdVbwW7jI9BF7t28Xk/6zXzqu6k7J5hd9WVIq8yFOmHLYjVvLWaTrUkRU+yrqw8S65GF4Lt6+vXY8qWf63iRcPcidlW/e2D+fyXuTJKodNe7ONn9v4bpBQR9Hzo0zoaNC6HUW8Fm6lqhm+VcfVvmzPvs434opQx51Jlv9tTvhB6mnQMEhSwPL2l9Ujt6GIP/J657aiWEXbK/PAtq5zcJjxXriL0FRMPC/Ol0ZWTLOqN+LaHjqZ5f+8Ez7tIVdVx05iI/J5PqprhEb8kxx/bhp1UjY0U6bObUM8Vbct16m6I+SfcCfnY56e7p9/OGPe5Mx5g+fm9FzGprmx43MiR3Mhx3+/+c1KHTyoguvuNk4sf6GRxuapjrZJztmb/IzLst4uPj913N5o2fS0/kS2X3ONCq4/oSY2P2V9wlj1TgvO75233/QTvt+PqfMdB1G9robCtaq6W5/QCRwi/8zNqe7WnCwtsXZlTsO2qtT09IV5x4FVNT62oo958Q/0XOUve9kF/weB0hPoR/8vcLLmCeDE5qf662JuznqO6NS6MZ0bGx7ttaf43N8x316o7+lppV79ahXsP1XowarRLzZz1We1w6zz2bT/+PmTdcwCc+jX0RHyT7hjGUv0SbGiunhYn6g5mUqMmUiuP2HONJLMvHL4cC9r+UUX6d+djlIiKrj+RHgiZzfU7thi37FM2TTWCC3Oq2DzzemZUaJsMImsMcHrP6U6OG8WgJRbd6veacGMOkoZGjCeUvveshiK+qoax7MxQUzxXSzDSJa4CVbUvpfPX/BJsPnmtZP3x0+eWF0bLzBhOrO1bei8bkORMVu2WPskiyBQauI5T6+9gCWz58TaZrB1Wk10ngq30WnFjGnSYogYsthgpb+Oo4xDUeadePah6LwI19/3lkVjAo3UzkCSjOw8feuY2mHW+Wz6nawvwzGtMryHZN09le19+yfcyiTA51Ww/qbckyTYf6qXjRoPq+C1R9IrMy0fXPS/G2/UB9248UI+wrTUXebPat++izzQ6W45a7Y3anSxfIXdscXCPR7reHDU0AvkZUzeMu57y+LaurSM1+b16gQrKhjv+aLvZOumnDBRp+Btx9bYFIVqjCmropRu0YV9kFyVSZ/F26wpX2Vk+2uPrLlQD/S6+s6l9DZmyveZaBPB5pvDi8/afVuFALPOx/g6Jp/nbZ/2/5xtgkDZZ3hX9cyk6aVwGye9x8N2WZPzMr+nZGDuxaB1T2YMywpRb/9txwo9lBGsqOBtx3LLk/5ZWVvGtEziYdbvZG/NRpAL3dqVzISeNxoiL15rM2fHhTZx9dW6vrd9Pz/GGJZn3xX/Z80Jl3lBiddLFdnhDXblZaXv4mGrOo5jer7SWf90z0+mbPVpdqWUf6AHn5ZlN/o8b/usMqRsU8cbtkUfznon3PqEtbitMxHe5nTG/7G/kUZXzMTV1iZep28H8wWyr6LGFrUtgemtOFMZV/tvyzJ63MFrj+TeqgLpD+isHqaU6HFH+7Yao7wm7r+qJiZ0z9f+9e9VtQ9/ura+0y5EMV+a71TS66XOHrd1r3PHjozEvUXqYlVN4KwOyyV7oAV73AOXPwpLbPu+7mTtXEqvtwZ73EUHOdi096IDJLwT7qx4cHfnUnZMMayQ7vZz6dtvP9dXUWkhmSyBKLp838ZPGGO5W3DOKLoX7MyIcQfXnwjjxfm2lxqKVSLGfaEOuxY2hXFD0212sY9BbOPlTsQyTTHf1JMOK7XEuNPsMsZ5w/ZguuAU7Q1eOJdiMeuiMe6Byx91srLCEi3EuK0zvMfIGvVT5hz0TrjNIzZWez1m08iGCxnHM3rssQoa9JVjwWr4oKiYoETbZol+sP+UcVTJvp/8QqG4eamx4iVGlVyowxzbBKt6JETGg8IqPn29nMRIgWLHXe35IzaqpIhPMv2c1ktMG+Vw8KDubKT1oA11nNl7nJtT6tWvLjyqpEybMJXdKJLdWNkbHlVSdvROVrsf+hj3oC822O7DNoaa9ykzrjVqmFniYXrrz+Zik9aAXJiFL/6JKHIBKjoRU17ct+4pVOvCGLNOqWPXJ2pzdT6iMuOyq9CuCO+Eu4oxkaa4cvSUO++2pqgwFL0IxPMUZolHfBIh25nq8jKaNzGXS1654qJh2/ONfGG7vk2bsWkHrr48YivIae3GpTK5fmEpQpXjub0T7sgBg4qLSbxsJgkqIwxFBShe1rxtNm4stu8iE/XXdRLb9gptL3rxNxbzLmBFZs0zPzxubzoCG2x6qmlCMugbqlVT11u8bVFVx8hL4a6CspPhRLc1WWGJtBO6qADFqSLOG5+f2bS/sbH05XX2bmwacl74J3mrmXX3UfY1bhfmjymCzW25L71ZmzbSxJ2iS4yscJcRw6zZ5GwaTFEBih9jkLuAZC8qqxeZ10trg6zym4Q4HjYpmoJrGLAJB7pa30UxtY86LrCuXCBGVriLimEVWVjKCFB82zJxd1PjLTJixoUe2KgLcRlMdSyiBdzl+i5CVluuMqziUthmZIVbqWIPDaus/LICZBO/NfWyk9jux+d44qhTJhzocn2n9XZtzuGsURvx8zHur7QOj0uhpZEWbqXsQiZlhuvUhc3oEZtbxCIT9RM/KRMOdLW+03q769ebn8vYnBNFzyWXhiaOvHDnxXtd7IGYHowWGRFgEyry7ZaZ9FM0HOhyfVf5IlbRO5HovMoaRts0Iy/cSq0Vwugq7noMddAHJVlx8zJZwol7FHk24nJ9lx0FVvenqiQnRSki3KLXr5bdu3erkydPVr5fYsfsLDA9DSwt9ZaJAO96F3DkSHt2keqYnQVuvFFLjYmJCeCJJ5qzqSiTk8CZM21bsZa2/CYi9ymldtusO1a3MaR5pqaAo0eBblcLdrcLfOYzFO1hYmpKX4hF0v/f6QAf+UizNhXlwAGz/XFEgA0b6rcn4nvfa+5YZWGPmxCPmZ0FZmZ0z3V8HFhZ0RfqAwe0uLuOjXAHgf5+97uBJ5+s1x5A++/06fqPk4Q9bkJGhKkpLTJKAcvL+vv0aT9EG9AimcXEhC7L1JQOX+zbZyf20bZBoH1iu12noy96rmMl3CLycyLyDRH5lojcVrdRhJDR4MABLZZppIV7jhzRYb9I8MfH9XdclCPBfuKJ3gUs2m5iwmzL+LgOMfpw0csNlYjIOID/B+CNAB4BsADgHUqpvzNtUypUcugQcOWVwJ49vWXz88DCAvC7v1tsHwsLwEMPAXv36uUf/CDwhjcAd94JvOIVwMc/Dtxyi/7f3r29Y8zP63V/53e0HdH+gN460XYf/3jvmOvWAffeC1x6ae+YCwt62zvuAF7yEnMZDh3q2RptAwC33aZtjZYDveNE9kX+ufJKbXd0/Kj8L31pr8wA8OijwNVX93wUfdv6N8/nUf2ZfBv9XbTMy8s9G6P9AL19RG1mfn6tv9Pa1S23APffD3zgA9nbZpW3TDs12QL02lPW/pJ+i9cxAFx3Xa++9+7tlSdqH1G7KWq3yf54209rk6Z9J87T2R2/iZk//ic4c347xseAlVVB9+If4sCvPoKpe9+pt/mqRqPLAAAIe0lEQVTAB9LbCtCryzvu0L/37tX/j3yyvKz/XrcOuPNO/OrDv42Pnf1FKPTUvrPuaRz9hc9j6patxdpBvN3blD2DIqGS/PGCwGsAfC72+70A3pu1TanhgDZZQWz3kczaHo3K37SpPxVV9P94aqO0DDTJjCDxbBqHD/cGhSYzxdtkCklm94gy3mzZ0m9f/DhxO+Pf8f1s2aLHNm3apP/euDF9+6pScCXtSPNtWlYSmzIfPtx/rLS6MGVmMR27bFaXQdppyewsfdub6njTpv76DrMmrWknVZxfaW0/rU2a9p12nnY6vTJs2dL/v6iu0tpK2t/R96ZNvfYT2Rr6J9jwK6orZ3TqNDysgg2/Yn/OJstQpOwZoMpx3AB+EcAnY79vBPCnWduUHsedl0uuyD6SWdujyo9lS0/Nr5e0w5SDL25nVFkpmeKtT+gov96GDWb74seJ1kkTgmg/kS3RPt/4xvTtBiXu8zzfli1z2n7i+8jyd1q7st02q7xl2qnJFtv9ZdVxsr7T2kdV55dtm8zbT/yciZchXid5bSW5j/j/IyFPakDeMYv4omjZDVQt3L+UItx/krLeNICTAE7u2rWrlOFKKbvsz7b7CLOiX9hfcrkpo3XSDpuM0aZjFrU5z760MuTtJ/63abtBKeLbsmVO249t9nXLzOWFy1vGjwUzkBu3N9WxTfuo4vyybZN5+zGVwVRXaW3FVPZkHRc5ZpkyDODXqoW7mVCJUuxxs8fNHjd73OxxVyTc6wD8PYAXAdgA4CsALsvahjFuxrgZ47bcljHuVmLcmcf0IMadOxxQKbUM4NcBfA7AgwCOKaUesHryWYSFBeDYsd4T6z179O9oBEGRfSwv6yfL99wD3HWXHtj6oQ8Bl1+unzLfeqv+/9SU/v/CQu94997bs2NhATh+vH+dvXv1PiI7l5f1vk+f7j/m8rL+vu667DIsLOjtjh/X2xw/Dpw4AVx2Wb998eMcO6bLEJU1sju+n6kp4PBhXeapKeCmm4A3vUm/Zxzfroh/83we2WXybXSsomWORgXE9xPfx549+pPm77R2tXevPlbetlnlLdNOTbbE21PW/pJ+i9fx5Zf31/eJE70RFcl2XdRuk/15bdK077Tz9IYbemWYmuqdP5dd1qurtLZyww3672gfUdkvu0zvL2o/ka2XX96/ftoxi7SDeLuv8rzKgW9OEkKIA/DNSUIIGWIo3IQQ4hkUbkII8QwKNyGEeAaFmxBCPKOWUSUichZA2dwWOwC4mLeDdhXHVdtoVzFoV3HK2NZVSl1is2Itwj0IInLSdkhMk9Cu4rhqG+0qBu0qTt22MVRCCCGeQeEmhBDPcFG4j7ZtgAHaVRxXbaNdxaBdxanVNudi3IQQQrJxscdNCCEkA2eE25WExCJyqYjMi8iDIvKAiLw7XP77IvIPInJ/+Lm2JftOi8jXQhtOhsueKyL/S0S+GX5vb9iml8b8cr+I/FBE3tOGz0Tkz0XkcRE5FVuW6h/R/HHY5r4qIq9qwbYPisjXw+MfF5GLw+WTIvLjmO8+1rBdxroTkfeGPvuGiPzrhu26M2bTaRG5P1zepL9MGtFcO7Od/7XOD4BxAA8BeDF6c36/vCVbng/gVeHfz4FOlPxyAL8P4Lcd8NVpADsSyw4BuC38+zYAB1uuy+8A6LbhMwCvA/AqAKfy/APgWgD/A4AAuArAF1uw7WcBrAv/PhizbTK+Xgt2pdZdeC58BcBG6Dn6HwIw3pRdif8fBvC+Fvxl0ojG2pkrPe5XA/iWUurvlVLPALgDwFvbMEQp9ZhS6kvh3z+CnoP8BW3YUoC3Avh0+PenAbytRVteD+AhpVTZF7AGQin1NwC+l1hs8s9bAfyl0vwtgItF5PlN2qaU+rzSc94DwN8CeGFdxy9iVwZvBXCHUupppdTDAL4Fff42apeICIC3A/irOo6dRYZGNNbOXBHuFwD4duz3I3BALEVkEsArAXwxXPTr4a3OnzcdjoihAHxeRO4Tkelw2fOUUo8BulEB2NmSbQCwF/0nkws+M/nHtXb3TuieWcSLROTLIvK/ReTqFuxJqztXfHY1gO8qpb4ZW9a4vxIa0Vg7c0W4JWVZq8NdRGQrgP8G4D1KqR8C+CiAlwB4BYDHoG/T2uC1SqlXAbgGwK+JyOtasmMNIrIBwFsA/NdwkSs+M+FMuxORGQDLAGbDRY8B2KWUeiWAWwH8FxG5qEGTTHXnis/egf4OQuP+StEI46opywbymSvC/QiAS2O/Xwjg0ZZsgYish66QWaXUfwcApdR3lVIrSqlVAJ9ATbeHeSilHg2/HwdwPLTju9GtV/j9eBu2QV9MvqSU+m5ooxM+g9k/TrQ7EbkJwJsBTKkwKBqGIp4M/74POpb8z5qyKaPuWveZiKwD8PMA7oyWNe2vNI1Ag+3MFeFeAPATIvKisNe2F8DdbRgSxs4+BeBBpdSHY8vjMal/C+BUctsGbNsiIs+J/oZ+sHUK2lc3havdBOCvm7YtpK8X5ILPQkz+uRvAL4dP/a8CcC661W0KEfk5AP8BwFuUUkux5ZeIyHj494sB/AR00u6m7DLV3d0A9orIRhF5UWjX/23KrpA3APi6UuqRaEGT/jJpBJpsZ008hbV8Unst9NPZhwDMtGjHz0DfxnwVwP3h51oAnwHwtXD53QCe34JtL4Z+ov8VAA9EfgIwAeALAL4Zfj+3Bds6AJ4EsC22rHGfQV84HgPwLHRP52aTf6BvYf8sbHNfA7C7Bdu+BR3/jNrax8J1fyGs468A+BKAf9OwXca6AzAT+uwbAK5p0q5w+X8G8K7Euk36y6QRjbUzvjlJCCGe4UqohBBCiCUUbkII8QwKNyGEeAaFmxBCPIPCTQghnkHhJoQQz6BwE0KIZ1C4CSHEM/4/ajF0tBvmsAEAAAAASUVORK5CYII=\n", 220 | "text/plain": [ 221 | "
" 222 | ] 223 | }, 224 | "metadata": { 225 | "needs_background": "light" 226 | }, 227 | "output_type": "display_data" 228 | } 229 | ], 230 | "source": [ 231 | "mu_pred, var_pred = model.predict(X1test)\n", 232 | "posteriors = model.posteriors_F(X1test)\n", 233 | "mu_fstar, var_fstar = posteriors[0].mean.copy(), np.diag(posteriors[0].covariance.copy())[:, None]\n", 234 | "\n", 235 | "plt.figure()\n", 236 | "\n", 237 | "# plt.title(\"Test\")\n", 238 | "plt.plot(Y1test, 'rx')\n", 239 | "\n", 240 | "# plt.title(\"Train\")\n", 241 | "plt.plot(mu_pred[0], 'bo')\n", 242 | "\n", 243 | "mu_pred_train, var_pred_train = model.predict(X[0])\n", 244 | "posteriors_ = model.posteriors_F(X[0])\n", 245 | "mu_fstar_train, var_fstar_train = posteriors_[0].mean.copy(), np.diag(posteriors_[0].covariance.copy())[:, None]\n", 246 | "\n", 247 | "plt.figure()\n", 248 | "plt.plot(Y[0], 'rx')\n", 249 | "plt.plot(mu_pred_train[0], 'bo')\n", 250 | "\n", 251 | "# np.save('toy_data_gp_params.npy', model.param_array)\n", 252 | "print(\"model:\\n\", model)" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 8, 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "X1_train_slice1 = X[0][:131].copy()\n", 262 | "Y1_train_slice1 = Y[0][:131].copy()\n", 263 | "X1_train_slice2 = X[0][131:].copy()\n", 264 | "Y1_train_slice2 = Y[0][131:].copy()" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 9, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "values = mu_pred[0].copy()\n", 274 | "values = np.floor(values)" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 10, 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [ 283 | "values_train = mu_pred_train[0].copy()\n", 284 | "values_train = np.floor(values_train)" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 11, 290 | "metadata": {}, 291 | "outputs": [ 292 | { 293 | "data": { 294 | "text/plain": [ 295 | "Text(0, 0.5, 'Count')" 296 | ] 297 | }, 298 | "execution_count": 11, 299 | "metadata": {}, 300 | "output_type": "execute_result" 301 | }, 302 | { 303 | "data": { 304 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmAAAADYCAYAAABMbV/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEqhJREFUeJzt3X+QJGV9x/H3Fw4UBAv1NiZBzzXGWDFSAWoTMVRdVPyBeCeaIhaWGPBHdq1ggglVCVqVQOWXpoyCWkZ3jYhWFIpCTDhiDCogsUqRu5OSw4vRgCB6cviboMEcfPNH997tzc3sTN/uPD07835Vbc129zPd35lnn9lPdfd0R2YiSZKkcg5puwBJkqRJYwCTJEkqzAAmSZJUmAFMkiSpMAOYJElSYQYwSZKkwgxgkiRJhRnAJEmSCjOASZIkFbau7QL6Wb9+fU5PT7ddhiRJUl/btm37bmZO9Ws38gFsenqarVu3tl2GJElSXxFx1yDtPAQpSZJUmAFMkiSpMAOYJElSYQYwSZKkwgxgkiRJhRnAJK05EdWPJK1VBjBJkqTCDGCSJEmFGcAkSZIKM4BJkiQVZgCTJEkqzAAmSZJUmAFMkiSpMAOYJElSYQYwSZKkwgxgkiRJhRnAJEmSCjOASZIkFWYAkyRJKswAJkmSVFjxABYRfxwRt0fEjoi4PCIeWboGSZKkNhUNYBFxLPBHwExmPgM4FDizZA2SJElta+MQ5DrgiIhYBxwJfLuFGiRJklpTNIBl5reAvwfuBnYBP8rM6zrbRcRsRGyNiK333XdfyRKlgUVUP5LUlkn+HFrrr730IcjHAKcDTwZ+EXhURJzV2S4zFzJzJjNnpqamSpYoSZI0dKUPQT4PuDMz78vM/wOuBn6rcA2SJEmtKh3A7gZOiogjIyKAU4CdhWuQJElqVelzwG4GrgK2A7fV218oWYMkSVLb1pXeYGZeCFxYeruSJEmjwivhS5IkFWYAkyRJKswAJkmSVJgBTJIkqTADmCRJUmEGMEmSpMIMYJIkSYUZwCRJkgozgEmSJBVmAJMkSSrMACZJklRY8XtBSm2LqB4z261DzS32Xef0OPZl52sb59c6TuwnDco9YJIkSYUZwCRJkgozgEmSJBVmAJMkSSrMACZJklSYAUySJKkwA5gkSVJhBjBJkqTCDGCSJEmFGcAkSZIKM4BJkiQVZgCTJEkqzAAmSZJUWPEAFhHHRMRVEfGfEbEzIp5VugZJkqQ2rWthm+8EPpmZZ0TE4cCRLdQgSZLUmqIBLCIeDWwEzgHIzJ8BPytZgyRJUttK7wH7JeA+4IMR8evANuC8zHxgaaOImAVmATZs2FC4xN4iqsfMduuQJtamuf2nr51vp44hWvyc0T5+9mocDXwOWERsjIijeiw7KiI2DrCadcCJwHsz8wTgAeCCzkaZuZCZM5k5MzU1NWiJkiRJa0KTk/BvAJ7eY9nT6uX93APck5k319NXUQUySZKkidEkgC23Y/wRwEP9VpCZ3wG+GRFPq2edAnylQQ2SJElr3rLngEXENNV5W4tmuhyGPAJ4DXD3gNv8Q+Aj9Tcg7wBePeDzJEmSxkK/k/DPBi4Esv55N/vvCct6eg9w7iAbzMxbgZnGlUqSJI2JfgHsMuBGqpB1PVXI6jxk+CDwX5n5/dUuTpIkaRwtG8Ay8y7gLoCIeA6wPTPvL1GYJEnSuBr4OmCZ+dlhFiJJkjQpmlwH7PCIuLC+h+NPIuKhjp89wyxUkiRpXDS5Ev7bqM4B+zfgaqpzvyRJktRQkwB2BnBhZv7NsIqRJEmaBE0uxHoU8PlhFSJJkjQpmgSwLcAg93uUJEnSMpocgnw38OGIeBj4BHDAdb8y847VKkySJGlcNQlgi4cfL6K6On43h66oGkmSpAnQJIC9hurWQ5IkSVqBJhdivWyIdUiSJE2MJnvAJlZE/zaShmfvGNzUsWDTXP3LfMFq2rX4XmTB4xFtbHNU6pjk1z4q9o3/arzPzi5OL7ZYm+N/4AAWEZf2aZKZ+doV1iNJkjT2muwBey4HngP2WOBo4If1jyRJkvpocg7YdLf5EbEReB/wylWqSZIkaaw1uRBrV5l5E3Ax1XXCJEmS1MeKA1jtDuCEVVqXJEnSWFtxAIuIdcA5wD0rrkaSJGkCNPkW5PVdZh8O/ArwOOD1q1WUJEnSOGvyLchDOPBbkPcDVwNXZOaNq1WUJEnSOGvyLchnD7EOSZKkibFaJ+FLkiRpQI0CWEQcFxFXRcR9EbEnInZHxJURcdywCpQkSRo3TU7C/w3gs8BPgWuA7wA/D2wGXhwRGzNz21CqlCRJGiNNTsJ/C7ADOCUz71+cGRFHA5+ul79gdcuTJEkaP00OQZ4EvGVp+AKop/8OeNagK4qIQyPiSxFxbYPtS5IkjYUmAazzEhRNly91HrCzQXtJkqSx0SSA3Qy8uT7kuFdEPAr4M+ALg6wkIp4AvBj4xwbbliRJGhtNzgF7M3AjcFd96HAX1Un4LwaOBH57wPVcAvwpcHSvBhExC8wCbNiwoUGJklYionrMJvuzS6x/09yq17LWDbuvRsHia5xkne/B3JbOsTBfrJZRsbDQdgWrY+A9YJn5RarzwK4HXgj8CXBqPf3MzLyl3zoiYhOwu9+3JTNzITNnMnNmampq0BIlSZLWhGX3gEXEIVR7uO7MzB2Z+WXgjI42xwHTwG0DbO9k4CURcRrwSODREfFPmXnWwRQvSZK0FvXbA3YWcDnwwDJt7gcuj4hX9NtYZr4pM5+QmdPAmcD1hi9JkjRpBglgH8zMO3s1yMxvAB8Azl7FuiRJksZWvwB2InDdAOv5NDDTZMOZeWNmbmryHEmSpHHQL4AdDfxggPX8gGW+1ShJkqR9+gWw7wJPGmA9G+q2kiRJ6qNfAPscg53bdU7dVpIkSX30C2CXAKdExMURcXjnwog4LCLeCTwXuHgYBUqSJI2bZa8Dlpmfj4jzgbcDr4yI64C76sVPAp4PPA44PzMHuhWRJEnSpOt7K6LMvCQitgMXAC8DjqgX/ZTq1kRvzcz/GFqFkiRJY2age0Fm5k3ATfWV8dfXs7+XmQ8NrTJJkqQx1eRm3GTmw8DuIdUiSZI0EQa+GbckSZJWhwFMkiSpMAOYJElSYY3OAZt4m+YAiM3VZG6Z329xRD0/Sxa1vFGsqbTF92Cvuh9hvrNp1+d1vndzW+rnb1psN9+13VrS+R6Nyt/N3rr63DU2Nld90jkme62z7de11g377+OAMTtCVuu1N15P/bm1sNB9fr/PszVt72vsrsn4HyXuAZMkSSrMACZJklSYAUySJKkwA5gkSVJhBjBJkqTCDGCSJEmFGcAkSZIKM4BJkiQVZgCTJEkqzAAmSZJUmAFMkiSpMAOYJElSYQYwSZKkwooGsIh4YkTcEBE7I+L2iDiv5PYlSZJGwbrC29sDnJ+Z2yPiaGBbRHwqM79SuA5JkqTWFN0Dlpm7MnN7/fv9wE7g2JI1SJIkta30HrC9ImIaOAG4ucuyWWAWYMOGDUXrWmpuy1z1y6buy2PzXMec+VXZbkT1mLkqq1uzNSytY9GK6tm0f3919l9u6d5/B/Zz9/VGVM/vV2Pna9q7/QGft7RdxMrek721bOr1Gvd/T1a1P7qsd3F9/etaZl2b58gt8wO/NwfbH6utVx295ncub3usHoyDrX01/w4Pdgwt/n9YWKimZ2frBZsW19vs86Cz3UrGwLjY9x40a9/rvRy1MdLKSfgRcRTwMeCNmfnjzuWZuZCZM5k5MzU1Vb5ASZKkISoewCLiMKrw9ZHMvLr09iVJktpW+luQAXwA2JmZ7yi5bUmSpFFReg/YycCrgOdGxK31z2mFa5AkSWpV0ZPwM/NzQJ9TSiVJksabV8KXJEkqzAAmSZJUmAFMkiSpMAOYJElSYQYwSZKkwgxgkiRJhRnAJEmSCjOASZIkFWYAkyRJKswAJkmSVJgBTJIkqTADmCRJUmGRmW3XsKyZmZncunXrULcR9e3BF9+K2Dy3Oiu+dn6/9fbb7nLzI6rpzsfO5XunO17D7Gz1uPCS+f3m96uNTT3ei2vnu84e9LVGxy3ZD3gPOvugz3vZzdyWOeY3z69efw5o8b2e31zV3PTvK7d0f28H+bvoNb9z+dyWqoaFhWavaW/7Hv3R9++m6XYayi1Vfy99XPp30Dm/ybbed231ODe7//z5hf3n95vuZf4gX3MvczkPc3Mwv+9xPua6zl+6fNl1drymzvmD6vX8pu/B62n+udBp7xhZ4d/HoHr9jc92vIcrGQOD6vX50KvdIOuDwf6nDbKelX6O9PoMHraI2JaZM/3auQdMkiSpMAOYJElSYQYwSZKkwgxgkiRJhRnAJEmSCjOASZIkFWYAkyRJKswAJkmSVJgBTJIkqTADmCRJUmEGMEmSpMIMYJIkSYUZwCRJkgorHsAi4tSI+GpEfD0iLii9fUmSpLYVDWARcSjwHuBFwNOBV0TE00vWIEmS1LbSe8B+E/h6Zt6RmT8DrgBOL1yDJElSqyIzy20s4gzg1Mx8XT39KuCZmfmGjnazwGw9+TTgqwXKWw98t8B2NDj7ZDTZL6PHPhlN9svoKdEnT8rMqX6N1g25iE7RZd4BCTAzF4CF4ZezT0RszcyZktvU8uyT0WS/jB77ZDTZL6NnlPqk9CHIe4AnLpl+AvDtwjVIkiS1qnQAuwV4akQ8OSIOB84ErilcgyRJUquKHoLMzD0R8Qbg34FDgUsz8/aSNSyj6CFPDcQ+GU32y+ixT0aT/TJ6RqZPip6EL0mSJK+EL0mSVJwBTJIkqbCJD2DeGml0RMQ3IuK2iLg1IrbW8x4bEZ+KiK/Vj49pu85xFxGXRsTuiNixZF7XfojKu+rx8+WIOLG9ysdXjz65KCK+VY+XWyPitCXL3lT3yVcj4oXtVD3eIuKJEXFDROyMiNsj4rx6vmOlRcv0y8iNl4kOYN4aaSQ9JzOPX3KdlguAz2TmU4HP1NMarsuAUzvm9eqHFwFPrX9mgfcWqnHSXMaBfQJwcT1ejs/MTwDUn2FnAr9WP+cf6s86ra49wPmZ+avAScC59XvvWGlXr36BERsvEx3A8NZIa8HpwIfq3z8EvLTFWiZCZt4EfL9jdq9+OB34cFa+ABwTEb9QptLJ0aNPejkduCIzH8zMO4GvU33WaRVl5q7M3F7/fj+wEzgWx0qrlumXXlobL5MewI4Fvrlk+h6W7ygNVwLXRcS2+nZUAI/PzF1QDSzg51qrbrL16gfHULveUB/OunTJ4Xn7pLCImAZOAG7GsTIyOvoFRmy8THoAG+jWSCrm5Mw8kWpX/bkRsbHtgtSXY6g97wWeAhwP7ALeXs+3TwqKiKOAjwFvzMwfL9e0yzz7ZUi69MvIjZdJD2DeGmmEZOa368fdwMepdgPfu7ibvn7c3V6FE61XPziGWpKZ92bmQ5n5MPB+9h02sU8KiYjDqP7JfyQzr65nO1Za1q1fRnG8THoA89ZIIyIiHhURRy/+DrwA2EHVH2fXzc4G/qWdCider364Bvi9+hteJwE/Wjz8ouHqOH/oZVTjBao+OTMiHhERT6Y66fuLpesbdxERwAeAnZn5jiWLHCst6tUvozheit6KaNSM+K2RJs3jgY9XY4d1wEcz85MRcQtwZUS8Frgb+N0Wa5wIEXE58GxgfUTcA1wIvJXu/fAJ4DSqE1d/Ary6eMEToEefPDsijqc6XPINYA4gM2+PiCuBr1B9I+zczHyojbrH3MnAq4DbIuLWet6bcay0rVe/vGLUxou3IpIkSSps0g9BSpIkFWcAkyRJKswAJkmSVJgBTJIkqTADmCRJUmEGMElrQkScExEZEb/cch3HRMRFEXFim3VIWtsMYJLUzDFU1+EygEk6aAYwSZKkwgxgktakiLgxIj4XEc+LiO0R8ZOI2BERL+1od1F96PK4iLihbrcrIv4yIg5Z0m7xEOd0t+fXv08Dd9aL3l+3z4g4Z4gvVdIYMoBJWsueArwTeAfwO8Au4Koe54n9M/Bp4KXAR4E/B/6i4fZ21dsBeAvwrPrnXxtXLmmiTfS9ICWteeuBjZn5NYCI2E4Vkl4O/G1H2/dn5lvr36+LiEcD50fEJZn5w0E2lpkPRsSX6sk7MvMLK38JkiaRe8AkrWVfWwxfAJm5G9gNbOjS9sqO6SuAo4BnDK88SerOACZpLft+l3kPAo/sMv/eHtPHrmpFkjQAA5ikSfH4HtPfqh//t348vKPd44ZWkaSJZQCTNCle3jF9JvA/wI56+q76ce8hyYhYB7yg43kP1o9HrHaBkiaHJ+FLmhS/X1924hbghcDrgIuWnIB/C/DfwNvqdg8CfwA8omM99wLfA86MiC8DDwB3Zub3CrwGSWPCPWCSJsXpwPOBa4CzgL8G/mpxYWbuqdt8E7gMeA/wqfp3lrR7mCq8PYbqsha3AJuHXbyk8RKZ2XYNkjQ0EXER1a2DDqtDliS1zj1gkiRJhRnAJEmSCvMQpCRJUmHuAZMkSSrMACZJklSYAUySJKkwA5gkSVJhBjBJkqTC/h9/rXEWGCv4ZQAAAABJRU5ErkJggg==\n", 305 | "text/plain": [ 306 | "
" 307 | ] 308 | }, 309 | "metadata": { 310 | "needs_background": "light" 311 | }, 312 | "output_type": "display_data" 313 | } 314 | ], 315 | "source": [ 316 | "plt.figure(figsize=(10,3))\n", 317 | "plt.bar([i for i in range(0, 131)], Y1_train_slice1.reshape((-1,)), color='b', width=0.9)\n", 318 | "plt.bar([i for i in range(131, 181)], Y1test.reshape((-1,)), color='b', width=0.9)\n", 319 | "plt.bar([i for i in range(181, 250)], Y1_train_slice2.reshape((-1,)), color='b', width=0.9)\n", 320 | "\n", 321 | "plt.bar([i for i in range(0, 131)], values_train[:131].reshape((-1,)), color='g', width=0.9, alpha=0.6)\n", 322 | "plt.bar([i for i in range(131, 181)], values.reshape((-1,)), color='r', width=0.9, alpha=0.6)\n", 323 | "plt.bar([i for i in range(181, 250)], values_train[131:].reshape((-1,)), color='g', width=0.9, alpha=0.6)\n", 324 | "\n", 325 | "plt.xlabel(\"Input\", fontsize=16)\n", 326 | "plt.ylabel(\"Count\", fontsize=16)\n" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [] 335 | } 336 | ], 337 | "metadata": { 338 | "kernelspec": { 339 | "display_name": "Python 3", 340 | "language": "python", 341 | "name": "python3" 342 | }, 343 | "language_info": { 344 | "codemirror_mode": { 345 | "name": "ipython", 346 | "version": 3 347 | }, 348 | "file_extension": ".py", 349 | "mimetype": "text/x-python", 350 | "name": "python", 351 | "nbconvert_exporter": "python", 352 | "pygments_lexer": "ipython3", 353 | "version": "3.7.3" 354 | } 355 | }, 356 | "nbformat": 4, 357 | "nbformat_minor": 2 358 | } 359 | -------------------------------------------------------------------------------- /poster/multitask_aggregated_poster.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frb-yousefi/aggregated-multitask-gp/4b2b7617cc859be95858e7b7450c72cd88cb341b/poster/multitask_aggregated_poster.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | climin==0.1a1 2 | paramz==0.9.5 3 | coloredlogs==10.0 4 | click==7.0 5 | 6 | --------------------------------------------------------------------------------