├── .coveragerc ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bayesian_benchmarks ├── __init__.py ├── bayesian_benchmarksrc ├── data.py ├── database_utils.py ├── models │ ├── README.md │ ├── deep_gp_doubly_stochastic │ │ ├── README.md │ │ └── models.py │ ├── get_model.py │ ├── non_bayesian_models.py │ ├── template.py │ ├── variationally_sparse_gp │ │ ├── README.md │ │ └── models.py │ └── variationally_sparse_gp_minibatch │ │ └── models.py ├── paths.py ├── results │ ├── calibration.ipynb │ └── view_results.ipynb ├── scripts │ ├── make_experiments.py │ └── run_all_pytest.py └── tasks │ ├── active_learning_continuous.py │ ├── active_learning_discrete.py │ ├── adversarial.py │ ├── classification.py │ ├── conditional_density_estimation.py │ ├── learned_synthetic.py │ ├── mmd.py │ ├── regression.py │ └── utils.py ├── setup.py └── tests ├── test_data.py ├── test_database.py ├── test_models.py └── test_tasks.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = *tests*, setup.py 3 | exclude_lines = 4 | pragma: no cover 5 | def __repr__ 6 | def __str__ 7 | def _repr_html_ 8 | def _html_table_rows 9 | if self.debug: 10 | if settings.DEBUG 11 | raise AssertionError 12 | raise NotImplementedError 13 | if __name__ == .__main__.: 14 | print -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Intellij files 107 | .idea/ 108 | 109 | # data 110 | bayesian_benchmarks/data 111 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: python 4 | python: 5 | - 3.6 6 | 7 | before_install: 8 | - sudo apt-get update -qq 9 | 10 | install: 11 | - pip install numpy scipy sklearn pandas pytest matplotlib pytest-xdist pytest-cov codecov ddt 12 | - python setup.py install 13 | - git clone https://github.com/GPflow/GPflow.git 14 | - cd GPflow 15 | - git reset --hard 3a169917326bdacef9bb7003a8122766d7cf6472 16 | - pip install . 17 | - cd ../ 18 | - git clone https://github.com/ICL-SML/Doubly-Stochastic-DGP.git 19 | - cd Doubly-Stochastic-DGP 20 | - git reset --hard 23343e1143390168fdb7abe19c5878933dea9e06 21 | - pip install . 22 | - cd ../ 23 | 24 | script: 25 | - pytest -W ignore::UserWarning --durations=50 --cov=./bayesian_benchmarks -d --tx 3*popen//python=python3.6 --pyargs ./tests 26 | - codecov --token=fe99b84e-0656-4739-96b9-79f021477eac 27 | 28 | cache: 29 | apt: true 30 | pip: true 31 | directories: 32 | - $HOME/.cache/pip 33 | - $HOME/download 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bayesian Benchmarks 2 | 3 | [![Build Status](https://travis-ci.org/hughsalimbeni/bayesian_benchmarks.svg?branch=master)](https://travis-ci.org/hughsalimbeni/bayesian_benchmarks) 4 | [![codecov](https://codecov.io/gh/hughsalimbeni/bayesian_benchmarks/branch/master/graph/badge.svg)](https://codecov.io/gh/hughsalimbeni/bayesian_benchmarks) 5 | 6 | This is a set of tools for evaluating Bayesian models, together with benchmark implementations and results. 7 | 8 | Motivations: 9 | * There is a lack of standardized tasks that meaningfully assess the quality of uncertainty quantification for Bayesian black-box models. 10 | * Variations between tasks in the literature make a direct comparison between methods difficult. 11 | * Implementing competing methods takes considerable effort, and there little incentive to do a good job. 12 | * Published papers may not always provide complete details of implementations due to space considerations. 13 | 14 | Aims: 15 | * Curate a set of benchmarks that meaningfully compare the efficacy of Bayesian models in real-world tasks. 16 | * Maintain a fair assessment of benchmark methods, with full implementations and results. 17 | 18 | Tasks: 19 | * Classification and regression 20 | * Density estimation (real world and synthetic) (TODO) 21 | * Active learning 22 | * Adversarial robustness (TODO) 23 | 24 | Current implementations: 25 | * Sparse variational GP, for [Gaussian](http://proceedings.mlr.press/v5/titsias09a/titsias09a.pdf) and [non-Gaussian](http://proceedings.mlr.press/v38/hensman15.pdf) likelihoods 26 | * Sparse variational GP, with [minibatches](https://arxiv.org/pdf/1309.6835.pdf) 27 | * 2 layer Deep Gaussian process, with [doubly-stochastic variational inference](http://papers.nips.cc/paper/7045-doubly-stochastic-variational-inference-for-deep-gaussian-processes.pdf) 28 | * A variety of sklearn models 29 | 30 | See the models folder for instruction for adding new models. 31 | 32 | Coming soon: 33 | * [Structured Variational Learning of Bayesian Neural Networks with Horseshoe Priors](https://arxiv.org/pdf/1806.05975.pdf) 34 | * [Differentiable Compositional Kernel Learning for Gaussian Processes](https://arxiv.org/abs/1806.04326) 35 | * [Deep Gaussian Processes using Stochastic Gradient Hamiltonian Monte Carlo 36 | ](https://arxiv.org/pdf/1806.05490.pdf) 37 | 38 | -------------------------------------------------------------------------------- /bayesian_benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/secondmind-labs/bayesian_benchmarks/0fd113b5bb28739d7f29e0c850344b552d18bd35/bayesian_benchmarks/__init__.py -------------------------------------------------------------------------------- /bayesian_benchmarks/bayesian_benchmarksrc: -------------------------------------------------------------------------------- 1 | [paths] 2 | data_path = ./data 3 | results_path = ./results/results.db 4 | 5 | [seeds] 6 | seed = 0 7 | -------------------------------------------------------------------------------- /bayesian_benchmarks/data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Hugh Salimbeni 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import numpy as np 16 | import os 17 | import pandas 18 | import logging 19 | from datetime import datetime 20 | from scipy.io import loadmat 21 | import pickle 22 | import shutil 23 | 24 | from urllib.request import urlopen 25 | logging.getLogger().setLevel(logging.INFO) 26 | import zipfile 27 | 28 | from bayesian_benchmarks.paths import DATA_PATH, BASE_SEED 29 | 30 | _ALL_REGRESSION_DATATSETS = {} 31 | _ALL_CLASSIFICATION_DATATSETS = {} 32 | 33 | def add_regression(C): 34 | _ALL_REGRESSION_DATATSETS.update({C.name:C}) 35 | return C 36 | 37 | def add_classficiation(C): 38 | _ALL_CLASSIFICATION_DATATSETS.update({C.name:C}) 39 | return C 40 | 41 | def normalize(X): 42 | X_mean = np.average(X, 0)[None, :] 43 | X_std = 1e-6 + np.std(X, 0)[None, :] 44 | return (X - X_mean) / X_std, X_mean, X_std 45 | 46 | 47 | class Dataset(object): 48 | def __init__(self, split=0, prop=0.9): 49 | 50 | if self.needs_download: 51 | self.download() 52 | 53 | X_raw, Y_raw = self.read_data() 54 | X, Y = self.preprocess_data(X_raw, Y_raw) 55 | 56 | ind = np.arange(self.N) 57 | 58 | np.random.seed(BASE_SEED + split) 59 | np.random.shuffle(ind) 60 | 61 | n = int(self.N * prop) 62 | 63 | self.X_train = X[ind[:n]] 64 | self.Y_train = Y[ind[:n]] 65 | 66 | self.X_test = X[ind[n:]] 67 | self.Y_test = Y[ind[n:]] 68 | 69 | @property 70 | def datadir(self): 71 | dir = os.path.join(DATA_PATH, self.name) 72 | if not os.path.isdir(dir): 73 | os.mkdir(dir) 74 | return dir 75 | 76 | @property 77 | def datapath(self): 78 | filename = self.url.split('/')[-1] # this is for the simple case with no zipped files 79 | return os.path.join(self.datadir, filename) 80 | 81 | @property 82 | def needs_download(self): 83 | return not os.path.isfile(self.datapath) 84 | 85 | def download(self): 86 | logging.info('downloading {} data'.format(self.name)) 87 | 88 | is_zipped = np.any([z in self.url for z in ['.gz', '.zip', '.tar']]) 89 | 90 | if is_zipped: 91 | filename = os.path.join(self.datadir, self.url.split('/')[-1]) 92 | else: 93 | filename = self.datapath 94 | 95 | with urlopen(self.url) as response, open(filename, 'wb') as out_file: 96 | data = response.read() 97 | out_file.write(data) 98 | 99 | if is_zipped: 100 | zip_ref = zipfile.ZipFile(filename, 'r') 101 | zip_ref.extractall(self.datadir) 102 | zip_ref.close() 103 | 104 | # os.remove(filename) 105 | 106 | logging.info('finished downloading {} data'.format(self.name)) 107 | 108 | def read_data(self): 109 | raise NotImplementedError 110 | 111 | def preprocess_data(self, X, Y): 112 | X, self.X_mean, self.X_std = normalize(X) 113 | Y, self.Y_mean, self.Y_std = normalize(Y) 114 | return X, Y 115 | 116 | 117 | uci_base_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/' 118 | 119 | 120 | @add_regression 121 | class Boston(Dataset): 122 | N, D, name = 506, 13, 'boston' 123 | url = uci_base_url + 'housing/housing.data' 124 | 125 | def read_data(self): 126 | data = pandas.read_fwf(self.datapath, header=None).values 127 | return data[:, :-1], data[:, -1].reshape(-1, 1) 128 | 129 | 130 | @add_regression 131 | class Concrete(Dataset): 132 | N, D, name = 1030, 8, 'concrete' 133 | url = uci_base_url + 'concrete/compressive/Concrete_Data.xls' 134 | 135 | def read_data(self): 136 | data = pandas.read_excel(self.datapath).values 137 | return data[:, :-1], data[:, -1].reshape(-1, 1) 138 | 139 | 140 | @add_regression 141 | class Energy(Dataset): 142 | N, D, name = 768, 8, 'energy' 143 | url = uci_base_url + '00242/ENB2012_data.xlsx' 144 | def read_data(self): 145 | # NB this is the first output (aka Energy1, as opposed to Energy2) 146 | data = pandas.read_excel(self.datapath).values[:, :-1] 147 | return data[:, :-1], data[:, -1].reshape(-1, 1) 148 | 149 | 150 | @add_regression 151 | class Kin8nm(Dataset): 152 | N, D, name = 8192, 8, 'kin8nm' 153 | url = 'https://www.dcc.fc.up.pt/~ltorgo/Regression/kinematics.tar.gz' 154 | 155 | @property 156 | def datapath(self): 157 | return os.path.join(self.datadir, 'Kinematics/kin8nm.data') 158 | 159 | def download(self): 160 | logging.info('downloading {} data'.format(self.name)) 161 | 162 | filename = os.path.join(self.datadir, self.url.split('/')[-1]) 163 | with urlopen(self.url) as response, open(filename, 'wb') as outfile: 164 | data = response.read() 165 | outfile.write(data) 166 | 167 | import tarfile 168 | with tarfile.open(filename, 'r:gz') as t: 169 | t.extractall(path=self.datadir) 170 | 171 | logging.info('finished downloading {} data'.format(self.name)) 172 | 173 | def read_data(self): 174 | data = pandas.read_csv(self.datapath, header=None).values 175 | X_raw, Y_raw = data[..., :-1], data[..., -1].reshape(-1, 1) 176 | assert X_raw.shape == (self.N, self.D) 177 | assert Y_raw.shape == (self.N, 1) 178 | return X_raw.astype(np.float), Y_raw.astype(np.float) 179 | 180 | 181 | @add_regression 182 | class Naval(Dataset): 183 | N, D, name = 11934, 14, 'naval' 184 | url = uci_base_url + '00316/UCI%20CBM%20Dataset.zip' 185 | 186 | @property 187 | def datapath(self): 188 | return os.path.join(self.datadir, 'UCI CBM Dataset/data.txt') 189 | 190 | def read_data(self): 191 | data = pandas.read_fwf(self.datapath, header=None).values 192 | # NB this is the first output 193 | X = data[:, :-2] 194 | Y = data[:, -2].reshape(-1, 1) 195 | 196 | # dims 8 and 11 have std=0: 197 | X = np.delete(X, [8, 11], axis=1) 198 | return X, Y 199 | 200 | 201 | @add_regression 202 | class Power(Dataset): 203 | N, D, name = 9568, 4, 'power' 204 | url = uci_base_url + '00294/CCPP.zip' 205 | 206 | @property 207 | def datapath(self): 208 | return os.path.join(self.datadir, 'CCPP/Folds5x2_pp.xlsx') 209 | 210 | def read_data(self): 211 | data = pandas.read_excel(self.datapath).values 212 | return data[:, :-1], data[:, -1].reshape(-1, 1) 213 | 214 | 215 | @add_regression 216 | class Protein(Dataset): 217 | N, D, name = 45730, 9, 'protein' 218 | url = uci_base_url + '00265/CASP.csv' 219 | 220 | def read_data(self): 221 | data = pandas.read_csv(self.datapath).values 222 | return data[:, 1:], data[:, 0].reshape(-1, 1) 223 | 224 | 225 | @add_regression 226 | class WineRed(Dataset): 227 | N, D, name = 1599, 11, 'winered' 228 | url = uci_base_url + 'wine-quality/winequality-red.csv' 229 | 230 | def read_data(self): 231 | data = pandas.read_csv(self.datapath, delimiter=';').values 232 | return data[:, :-1], data[:, -1].reshape(-1, 1) 233 | 234 | 235 | @add_regression 236 | class WineWhite(WineRed): 237 | N, D, name = 4898, 11, 'winewhite' 238 | url = uci_base_url + 'wine-quality/winequality-white.csv' 239 | 240 | 241 | @add_regression 242 | class Yacht(Dataset): 243 | N, D, name = 308, 6, 'yacht' 244 | url = uci_base_url + '/00243/yacht_hydrodynamics.data' 245 | 246 | def read_data(self): 247 | data = pandas.read_fwf(self.datapath, header=None).values 248 | return data[:, :-1], data[:, -1].reshape(-1, 1) 249 | 250 | 251 | class Classification(Dataset): 252 | def preprocess_data(self, X, Y): 253 | X, self.X_mean, self.X_std = normalize(X) 254 | return X, Y 255 | 256 | @property 257 | def needs_download(self): 258 | if os.path.isfile(os.path.join(DATA_PATH, 'classification_data', 'iris', 'iris_R.dat')): 259 | return False 260 | else: 261 | return True 262 | 263 | def download(self): 264 | logging.info('donwloading classification data. WARNING: downloading 195MB file'.format(self.name)) 265 | 266 | filename = os.path.join(DATA_PATH, 'classification_data.tar.gz') 267 | 268 | url = 'http://persoal.citius.usc.es/manuel.fernandez.delgado/papers/jmlr/data.tar.gz' 269 | with urlopen(url) as response, open(filename, 'wb') as out_file: 270 | data = response.read() 271 | out_file.write(data) 272 | 273 | import tarfile 274 | tar = tarfile.open(filename) 275 | tar.extractall(path=os.path.join(DATA_PATH, 'classification_data')) 276 | tar.close() 277 | 278 | logging.info('finished donwloading {} data'.format(self.name)) 279 | 280 | 281 | def read_data(self): 282 | datapath = os.path.join(DATA_PATH, 'classification_data', self.name, self.name + '_R.dat') 283 | if os.path.isfile(datapath): 284 | data = np.array(pandas.read_csv(datapath, header=0, delimiter='\t').values).astype(float) 285 | else: 286 | data_path1 = os.path.join(DATA_PATH, 'classification_data', self.name, self.name + '_train_R.dat') 287 | data1 = np.array(pandas.read_csv(data_path1, header=0, delimiter='\t').values).astype(float) 288 | 289 | data_path2 = os.path.join(DATA_PATH, 'classification_data', self.name, self.name + '_test_R.dat') 290 | data2 = np.array(pandas.read_csv(data_path2, header=0, delimiter='\t').values).astype(float) 291 | 292 | data = np.concatenate([data1, data2], 0) 293 | 294 | return data[:, :-1], data[:, -1].reshape(-1, 1) 295 | 296 | 297 | rescale = lambda x, a, b: b[0] + (b[1] - b[0]) * x / (a[1] - a[0]) 298 | 299 | 300 | def convert_to_day_minute(d): 301 | day_of_week = rescale(float(d.weekday()), [0, 6], [0, 2 * np.pi]) 302 | time_of_day = rescale(d.time().hour * 60 + d.time().minute, [0, 24 * 60], [0, 2 * np.pi]) 303 | return day_of_week, time_of_day 304 | 305 | 306 | def process_time(pickup_datetime, dropoff_datetime): 307 | d_pickup = datetime.strptime(pickup_datetime, "%Y-%m-%d %H:%M:%S") 308 | d_dropoff = datetime.strptime(dropoff_datetime, "%Y-%m-%d %H:%M:%S") 309 | duration = (d_dropoff - d_pickup).total_seconds() 310 | 311 | pickup_day_of_week, pickup_time_of_day = convert_to_day_minute(d_pickup) 312 | dropoff_day_of_week, dropoff_time_of_day = convert_to_day_minute(d_dropoff) 313 | 314 | return [pickup_day_of_week, pickup_time_of_day, 315 | dropoff_day_of_week, dropoff_time_of_day, 316 | duration] 317 | 318 | 319 | class NYTaxiBase(Dataset): 320 | x_bounds = [-74.04, -73.75] 321 | y_bounds = [40.62, 40.86] 322 | too_close_radius = 0.00001 323 | min_duration = 30 324 | max_duration = 3 * 3600 325 | name = 'nytaxi' 326 | 327 | def _read_data(self): 328 | data = pandas.read_csv(self.datapath)#, nrows=10000) 329 | data = data.values 330 | 331 | # print(data.dtypes.index) 332 | # 'id', 0 333 | # 'vendor_id', 1 334 | # 'pickup_datetime', 2 335 | # 'dropoff_datetime',3 336 | # 'passenger_count', 4 337 | # 'pickup_longitude', 5 338 | # 'pickup_latitude',6 339 | # 'dropoff_longitude', 7 340 | # 'dropoff_latitude', 8 341 | # 'store_and_fwd_flag',9 342 | # 'trip_duration'10 343 | 344 | pickup_loc = np.array((data[:, 5], data[:, 6])).T 345 | dropoff_loc = np.array((data[:, 7], data[:, 8])).T 346 | 347 | ind = np.ones(len(data)).astype(bool) 348 | ind[data[:, 5] < self.x_bounds[0]] = False 349 | ind[data[:, 5] > self.x_bounds[1]] = False 350 | ind[data[:, 6] < self.y_bounds[0]] = False 351 | ind[data[:, 6] > self.y_bounds[1]] = False 352 | 353 | ind[data[:, 7] < self.x_bounds[0]] = False 354 | ind[data[:, 7] > self.x_bounds[1]] = False 355 | ind[data[:, 8] < self.y_bounds[0]] = False 356 | ind[data[:, 8] > self.y_bounds[1]] = False 357 | 358 | print('discarding {} out of bounds {} {}'.format(np.sum(np.invert(ind).astype(int)), self.x_bounds, 359 | self.y_bounds)) 360 | 361 | early_stop = ((data[:, 5] - data[:, 7]) ** 2 + (data[:, 6] - data[:, 8]) ** 2 < self.too_close_radius) 362 | ind[early_stop] = False 363 | print('discarding {} trip less than {} gp dist'.format(np.sum(early_stop.astype(int)), 364 | self.too_close_radius ** 0.5)) 365 | 366 | times = np.array([process_time(d_pickup, d_dropoff) for (d_pickup, d_dropoff) in data[:, 2:4]]) 367 | pickup_time = times[:, :2] 368 | dropoff_time = times[:, 2:4] 369 | duration = times[:, 4] 370 | 371 | short_journeys = (duration < self.min_duration) 372 | ind[short_journeys] = False 373 | print('discarding {} less than {}s journeys'.format(np.sum(short_journeys.astype(int)), self.min_duration)) 374 | 375 | long_journeys = (duration > self.max_duration) 376 | ind[long_journeys] = False 377 | print( 378 | 'discarding {} more than {}h journeys'.format(np.sum(long_journeys.astype(int)), self.max_duration / 3600.)) 379 | 380 | pickup_loc = pickup_loc[ind, :] 381 | dropoff_loc = dropoff_loc[ind, :] 382 | pickup_time = pickup_time[ind, :] 383 | dropoff_time = dropoff_time[ind, :] 384 | duration = duration[ind] 385 | 386 | print('{} total rejected journeys'.format(np.sum(np.invert(ind).astype(int)))) 387 | return pickup_loc, dropoff_loc, pickup_time, dropoff_time, duration 388 | 389 | @property 390 | def datapath(self): 391 | filename = 'train.csv' 392 | return os.path.join(self.datadir, filename) 393 | 394 | def download(self): 395 | raise NotImplementedError 396 | 397 | 398 | @add_regression 399 | class NYTaxiTimePrediction(NYTaxiBase): 400 | N, D = 1420068, 8 401 | # N, D = 9741, 6 402 | 403 | def read_data(self): 404 | path = os.path.join(DATA_PATH, 'taxitime_preprocessed.npz') 405 | if os.path.isfile(path): 406 | with open(path, 'rb') as file: 407 | f = np.load(file) 408 | X, Y = f['X'], f['Y'] 409 | 410 | else: 411 | pickup_loc, dropoff_loc, pickup_datetime, dropoff_datetime, duration = self._read_data() 412 | 413 | pickup_sc = np.array([np.sin(pickup_datetime[:, 0]), 414 | np.cos(pickup_datetime[:, 0]), 415 | np.sin(pickup_datetime[:, 1]), 416 | np.cos(pickup_datetime[:, 1])]).T 417 | 418 | X = np.concatenate([pickup_loc, dropoff_loc, pickup_sc], 1) 419 | Y = duration.reshape(-1, 1) 420 | X, Y = np.array(X).astype(float), np.array(Y).astype(float) 421 | with open(path, 'wb') as file: 422 | np.savez(file, X=X, Y=Y) 423 | 424 | return X, Y 425 | 426 | 427 | class NYTaxiLocationPrediction(NYTaxiBase): 428 | N, D = 1420068, 6 429 | def read_data(self): 430 | path = os.path.join(DATA_PATH, 'taxiloc_preprocessed.npz') 431 | if os.path.isfile(path): 432 | with open(path, 'rb') as file: 433 | f = np.load(file) 434 | X, Y = f['X'], f['Y'] 435 | 436 | else: 437 | 438 | pickup_loc, dropoff_loc, pickup_datetime, dropoff_datetime, duration = self._read_data() 439 | 440 | pickup_sc = np.array([np.sin(pickup_datetime[:, 0]), 441 | np.cos(pickup_datetime[:, 0]), 442 | np.sin(pickup_datetime[:, 1]), 443 | np.cos(pickup_datetime[:, 1])]).T 444 | # X = np.concatenate([pickup_loc, pickup_sc, duration.reshape(-1, 1)], 1) 445 | X = np.concatenate([pickup_loc, pickup_sc], 1) 446 | Y = dropoff_loc 447 | X, Y = np.array(X).astype(float), np.array(Y).astype(float) 448 | 449 | with open(path, 'wb') as file: 450 | np.savez(file, X=X, Y=Y) 451 | 452 | return X, Y 453 | 454 | def preprocess_data(self, X, Y): 455 | return X, Y 456 | 457 | # Andrew Wilson's datasets 458 | #https://drive.google.com/open?id=0BxWe_IuTnMFcYXhxdUNwRHBKTlU 459 | class WilsonDataset(Dataset): 460 | @property 461 | def datapath(self): 462 | n = self.name[len('wilson_'):] 463 | return '{}/uci/{}/{}.mat'.format(DATA_PATH, n, n) 464 | 465 | def read_data(self): 466 | data = loadmat(self.datapath)['data'] 467 | return data[:, :-1], data[:, -1, None] 468 | 469 | 470 | @add_regression 471 | class Wilson_3droad(WilsonDataset): 472 | name, N, D = 'wilson_3droad', 434874, 3 473 | 474 | 475 | @add_regression 476 | class Wilson_challenger(WilsonDataset): 477 | name, N, D = 'wilson_challenger', 23, 4 478 | 479 | 480 | @add_regression 481 | class Wilson_gas(WilsonDataset): 482 | name, N, D = 'wilson_gas', 2565, 128 483 | 484 | 485 | @add_regression 486 | class Wilson_servo(WilsonDataset): 487 | name, N, D = 'wilson_servo', 167, 4 488 | 489 | 490 | @add_regression 491 | class Wilson_tamielectric(WilsonDataset): 492 | name, N, D = 'wilson_tamielectric', 45781, 3 493 | 494 | 495 | @add_regression 496 | class Wilson_airfoil(WilsonDataset): 497 | name, N, D = 'wilson_airfoil', 1503, 5 498 | 499 | 500 | @add_regression 501 | class Wilson_concrete(WilsonDataset): 502 | name, N, D = 'wilson_concrete', 1030, 8 503 | 504 | 505 | @add_regression 506 | class Wilson_machine(WilsonDataset): 507 | name, N, D = 'wilson_machine', 209, 7 508 | 509 | 510 | @add_regression 511 | class Wilson_skillcraft(WilsonDataset): 512 | name, N, D = 'wilson_skillcraft', 3338, 19 513 | 514 | 515 | @add_regression 516 | class Wilson_wine(WilsonDataset): 517 | name, N, D = 'wilson_wine', 1599, 11 518 | 519 | 520 | @add_regression 521 | class Wilson_autompg(WilsonDataset): 522 | name, N, D = 'wilson_autompg', 392, 7 523 | 524 | 525 | @add_regression 526 | class Wilson_concreteslump(WilsonDataset): 527 | name, N, D = 'wilson_concreteslump', 103, 7 528 | 529 | 530 | @add_regression 531 | class Wilson_houseelectric(WilsonDataset): 532 | name, N, D = 'wilson_houseelectric', 2049280, 11 533 | 534 | 535 | @add_regression 536 | class Wilson_parkinsons(WilsonDataset): 537 | name, N, D = 'wilson_parkinsons', 5875, 20 538 | 539 | 540 | @add_regression 541 | class Wilson_slice(WilsonDataset): 542 | name, N, D = 'wilson_slice', 53500, 385 543 | 544 | 545 | @add_regression 546 | class Wilson_yacht(WilsonDataset): 547 | name, N, D = 'wilson_yacht', 308, 6 548 | 549 | 550 | @add_regression 551 | class Wilson_autos(WilsonDataset): 552 | name, N, D = 'wilson_autos', 159, 25 553 | 554 | 555 | @add_regression 556 | class Wilson_elevators(WilsonDataset): 557 | name, N, D = 'wilson_elevators', 16599, 18 558 | 559 | 560 | @add_regression 561 | class Wilson_housing(WilsonDataset): 562 | name, N, D = 'wilson_housing', 506, 13 563 | 564 | 565 | @add_regression 566 | class Wilson_pendulum(WilsonDataset): 567 | name, N, D = 'wilson_pendulum', 630, 9 568 | 569 | 570 | @add_regression 571 | class Wilson_sml(WilsonDataset): 572 | name, N, D = 'wilson_sml', 4137, 26 573 | 574 | 575 | @add_regression 576 | class Wilson_bike(WilsonDataset): 577 | name, N, D = 'wilson_bike', 17379, 17 578 | 579 | 580 | @add_regression 581 | class Wilson_energy(WilsonDataset): 582 | name, N, D = 'wilson_energy', 768, 8 583 | 584 | 585 | @add_regression 586 | class Wilson_keggdirected(WilsonDataset): 587 | name, N, D = 'wilson_keggdirected', 48827, 20 588 | 589 | 590 | @add_regression 591 | class Wilson_pol(WilsonDataset): 592 | name, N, D = 'wilson_pol', 15000, 26 593 | 594 | 595 | @add_regression 596 | class Wilson_solar(WilsonDataset): 597 | name, N, D = 'wilson_solar', 1066, 10 598 | 599 | 600 | @add_regression 601 | class Wilson_breastcancer(WilsonDataset): 602 | name, N, D = 'wilson_breastcancer', 194, 33 603 | 604 | 605 | @add_regression 606 | class Wilson_fertility(WilsonDataset): 607 | name, N, D = 'wilson_fertility', 100, 9 608 | 609 | 610 | @add_regression 611 | class Wilson_keggundirected(WilsonDataset): 612 | name, N, D = 'wilson_keggundirected', 63608, 27 613 | 614 | 615 | @add_regression 616 | class Wilson_protein(WilsonDataset): 617 | name, N, D = 'wilson_protein', 45730, 9 618 | 619 | 620 | @add_regression 621 | class Wilson_song(WilsonDataset): 622 | name, N, D = 'wilson_song', 515345, 90 623 | 624 | 625 | @add_regression 626 | class Wilson_buzz(WilsonDataset): 627 | name, N, D = 'wilson_buzz', 583250, 77 628 | 629 | 630 | @add_regression 631 | class Wilson_forest(WilsonDataset): 632 | name, N, D = 'wilson_forest', 517, 12 633 | 634 | 635 | @add_regression 636 | class Wilson_kin40k(WilsonDataset): 637 | name, N, D = 'wilson_kin40k', 40000, 8 638 | 639 | 640 | @add_regression 641 | class Wilson_pumadyn32nm(WilsonDataset): 642 | name, N, D = 'wilson_pumadyn32nm', 8192, 32 643 | 644 | 645 | @add_regression 646 | class Wilson_stock(WilsonDataset): 647 | name, N, D = 'wilson_stock', 536, 11 648 | 649 | 650 | classification_datasets = [ 651 | ['heart-va', 200, 13, 5], 652 | ['connect-4', 67557, 43, 2], 653 | ['wine', 178, 14, 3], 654 | ['tic-tac-toe', 958, 10, 2], 655 | ['fertility', 100, 10, 2], 656 | ['statlog-german-credit', 1000, 25, 2], 657 | ['car', 1728, 7, 4], 658 | ['libras', 360, 91, 15], 659 | ['spambase', 4601, 58, 2], 660 | ['pittsburg-bridges-MATERIAL', 106, 8, 3], 661 | ['hepatitis', 155, 20, 2], 662 | ['acute-inflammation', 120, 7, 2], 663 | ['pittsburg-bridges-TYPE', 105, 8, 6], 664 | ['arrhythmia', 452, 263, 13], 665 | ['musk-2', 6598, 167, 2], 666 | ['twonorm', 7400, 21, 2], 667 | ['nursery', 12960, 9, 5], 668 | ['breast-cancer-wisc-prog', 198, 34, 2], 669 | ['seeds', 210, 8, 3], 670 | ['lung-cancer', 32, 57, 3], 671 | ['waveform', 5000, 22, 3], 672 | ['audiology-std', 196, 60, 18], 673 | ['trains', 10, 30, 2], 674 | ['horse-colic', 368, 26, 2], 675 | ['miniboone', 130064, 51, 2], 676 | ['pittsburg-bridges-SPAN', 92, 8, 3], 677 | ['breast-cancer-wisc-diag', 569, 31, 2], 678 | ['statlog-heart', 270, 14, 2], 679 | ['blood', 748, 5, 2], 680 | ['primary-tumor', 330, 18, 15], 681 | ['cylinder-bands', 512, 36, 2], 682 | ['glass', 214, 10, 6], 683 | ['contrac', 1473, 10, 3], 684 | ['statlog-shuttle', 58000, 10, 7], 685 | ['zoo', 101, 17, 7], 686 | ['musk-1', 476, 167, 2], 687 | ['hill-valley', 1212, 101, 2], 688 | ['hayes-roth', 160, 4, 3], 689 | ['optical', 5620, 63, 10], 690 | ['credit-approval', 690, 16, 2], 691 | ['pendigits', 10992, 17, 10], 692 | ['pittsburg-bridges-REL-L', 103, 8, 3], 693 | ['dermatology', 366, 35, 6], 694 | ['soybean', 683, 36, 18], 695 | ['ionosphere', 351, 34, 2], 696 | ['planning', 182, 13, 2], 697 | ['energy-y1', 768, 9, 3], 698 | ['acute-nephritis', 120, 7, 2], 699 | ['pittsburg-bridges-T-OR-D', 102, 8, 2], 700 | ['letter', 20000, 17, 26], 701 | ['titanic', 2201, 4, 2], 702 | ['adult', 48842, 15, 2], 703 | ['lymphography', 148, 19, 4], 704 | ['statlog-australian-credit', 690, 15, 2], 705 | ['chess-krvk', 28056, 7, 18], 706 | ['bank', 4521, 17, 2], 707 | ['statlog-landsat', 6435, 37, 6], 708 | ['heart-hungarian', 294, 13, 2], 709 | ['flags', 194, 29, 8], 710 | ['mushroom', 8124, 22, 2], 711 | ['conn-bench-sonar-mines-rocks', 208, 61, 2], 712 | ['image-segmentation', 2310, 19, 7], 713 | ['congressional-voting', 435, 17, 2], 714 | ['annealing', 898, 32, 5], 715 | ['semeion', 1593, 257, 10], 716 | ['echocardiogram', 131, 11, 2], 717 | ['statlog-image', 2310, 19, 7], 718 | ['wine-quality-white', 4898, 12, 7], 719 | ['lenses', 24, 5, 3], 720 | ['plant-margin', 1600, 65, 100], 721 | ['post-operative', 90, 9, 3], 722 | ['thyroid', 7200, 22, 3], 723 | ['monks-2', 601, 7, 2], 724 | ['molec-biol-promoter', 106, 58, 2], 725 | ['chess-krvkp', 3196, 37, 2], 726 | ['balloons', 16, 5, 2], 727 | ['low-res-spect', 531, 101, 9], 728 | ['plant-texture', 1599, 65, 100], 729 | ['haberman-survival', 306, 4, 2], 730 | ['spect', 265, 23, 2], 731 | ['plant-shape', 1600, 65, 100], 732 | ['parkinsons', 195, 23, 2], 733 | ['oocytes_merluccius_nucleus_4d', 1022, 42, 2], 734 | ['conn-bench-vowel-deterding', 990, 12, 11], 735 | ['ilpd-indian-liver', 583, 10, 2], 736 | ['heart-cleveland', 303, 14, 5], 737 | ['synthetic-control', 600, 61, 6], 738 | ['vertebral-column-2clases', 310, 7, 2], 739 | ['teaching', 151, 6, 3], 740 | ['cardiotocography-10clases', 2126, 22, 10], 741 | ['heart-switzerland', 123, 13, 5], 742 | ['led-display', 1000, 8, 10], 743 | ['molec-biol-splice', 3190, 61, 3], 744 | ['wall-following', 5456, 25, 4], 745 | ['statlog-vehicle', 846, 19, 4], 746 | ['ringnorm', 7400, 21, 2], 747 | ['energy-y2', 768, 9, 3], 748 | ['oocytes_trisopterus_nucleus_2f', 912, 26, 2], 749 | ['yeast', 1484, 9, 10], 750 | ['oocytes_merluccius_states_2f', 1022, 26, 3], 751 | ['oocytes_trisopterus_states_5b', 912, 33, 3], 752 | ['breast-cancer-wisc', 699, 10, 2], 753 | ['steel-plates', 1941, 28, 7], 754 | ['mammographic', 961, 6, 2], 755 | ['monks-3', 554, 7, 2], 756 | ['balance-scale', 625, 5, 3], 757 | ['ecoli', 336, 8, 8], 758 | ['spectf', 267, 45, 2], 759 | ['monks-1', 556, 7, 2], 760 | ['page-blocks', 5473, 11, 5], 761 | ['magic', 19020, 11, 2], 762 | ['pima', 768, 9, 2], 763 | ['breast-tissue', 106, 10, 6], 764 | ['ozone', 2536, 73, 2], 765 | ['iris', 150, 5, 3], 766 | ['waveform-noise', 5000, 41, 3], 767 | ['cardiotocography-3clases', 2126, 22, 3], 768 | ['wine-quality-red', 1599, 12, 6], 769 | ['vertebral-column-3clases', 310, 7, 3], 770 | ['breast-cancer', 286, 10, 2], 771 | ['abalone', 4177, 9, 3], 772 | ] 773 | 774 | 775 | for name, N, D, K in classification_datasets: 776 | @add_classficiation 777 | class C(Classification): 778 | name, N, D, K = name, N, D, K 779 | 780 | 781 | # URLs of Mujoco data sets 782 | urls_mujoco_dataset = {'Ant-v2': 'https://drive.google.com/uc?export=download&id=1NdoF79nOg53_fjTlth4iN1RHpQRYgYw0', 783 | 'HalfCheetah-v2': 'https://drive.google.com/uc?export=download&id=1lcxCBRKFG-9JzcLroLSd6e-GgagcEnY6', 784 | 'Hopper-v2': 'https://drive.google.com/uc?export=download&id=10BZ_4n7a14K1wV_2Z6Ne-BT6SLJnm4U9', 785 | 'Humanoid-v2': 'https://drive.google.com/uc?export=download&id=1K_juBeAuCeMGNAqnWDFUoAYvQDYXyaTB', 786 | 'Humanoid-v2-1': 'https://drive.google.com/uc?export=download&id=1Sjai5ksAgud8ipVCADtLuIlcOMkB01WP', 787 | 'Humanoid-v2-2': 'https://drive.google.com/uc?export=download&id=1EfvIZ7oPy0hNgXOVxXIv8gu9ZkK4wHlw', 788 | 'Humanoid-v2-3': 'https://drive.google.com/uc?export=download&id=1YZMqBBNt-7fky3SDeJ-NvdzcCgzD7XGk', 789 | 'Humanoid-v2-4': 'https://drive.google.com/uc?export=download&id=1-FVKSOJQIA7PiKpVoXXfHl7Q19U9e0_o', 790 | 'Humanoid-v2-5': 'https://drive.google.com/uc?export=download&id=1wWtZaYI_Pm_t_PCSIQ5yZIU4MGIvgoyy', 791 | 'InvertedDoublePendulum-v2': 'https://drive.google.com/uc?export=download&id=1qqasWqUtBKq49ylAtiCj6hnL365APwj2', 792 | 'InvertedPendulum-v2': 'https://drive.google.com/uc?export=download&id=1cTsIsxUcIQpZ-INHnDBaQtHOuyWMLPJp', 793 | 'Pendulum-v0': 'https://drive.google.com/uc?export=download&id=11-T5IBWniyw3EF0z4TgvUvRgCQ8_QpTm', 794 | 'Reacher-v2': 'https://drive.google.com/uc?export=download&id=1uVj4MOb4xYk5aXmdsf6cIDVCHVVsafYQ', 795 | 'Swimmer-v2': 'https://drive.google.com/uc?export=download&id=1narm3mDgVSFzE26kHzy2Mkif6XR_DSuB', 796 | 'Walker2d-v2': 'https://drive.google.com/uc?export=download&id=1i3sG35FSPHdZ0S-VqyPF0a9gCoh9NdWz'} 797 | 798 | policy_checkpoints = [str(i * 100000) for i in range(11)] 799 | evaluation_suffix = 'sac_policy.eval' 800 | evaluations = 10 801 | 802 | 803 | class MujocoSoftActorCriticDataset(Dataset): 804 | """ 805 | A single Mujoco data set for one specific environment was created as follows. A soft actor-critic agent [Haarnoja et al., 806 | Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor, ICML, 2018] was trained 807 | for 1 million time steps and 11 policy checkpoints created every 100,000 steps (including step 0) using the implementation 808 | of [Leibfried et al., A Unified Bellman Optimality Principle Combining Reward Maximization and Empowerment, arXiv, 2019]. 809 | After that, each policy checkpoint was evaluated offline 10 times yielding 10 trajectories with a maximum length of 1,000. 810 | The Bayesian benchmark data sets create a single training and test set from ALL the policy transitions WITHOUT 811 | distinguishing different policy checkpoints. The inputs `X' are concatenated observation-action pairs, the outputs `Y' are 812 | the next-observation-observation difference vectors concatenated with the scalar reward signal. The dimensions of the 813 | observation space and the action space are given by the attributes `observation_dimension' and `action_dimension' 814 | respectively. 815 | """ 816 | 817 | @property 818 | def datadir(self): 819 | dir = os.path.join(DATA_PATH, self.name) 820 | return dir 821 | 822 | @property 823 | def needs_download(self): 824 | if not os.path.isdir(self.datadir): 825 | return True 826 | return False 827 | 828 | def download(self): 829 | filename = self.datadir + '.zip' 830 | with urlopen(self.url_mujoco_dataset) as response, open(filename, 'wb') as out_file: 831 | data = response.read() 832 | out_file.write(data) 833 | zip_ref = zipfile.ZipFile(filename, 'r') 834 | zip_ref.extractall(self.datadir) 835 | zip_ref.close() 836 | os.remove(filename) 837 | 838 | def read_data(self): 839 | """ 840 | `X_raw' stores concatenated observation-action vectors 841 | `Y_raw' stores the difference vectors between the next and the current observation vectors, 842 | concatenated with the scalar reward signal 843 | :return: X_raw [transitions x (observation_dimension + action_dimension)] 844 | Y_raw [transitions x (observation_dimension + 1)] 845 | """ 846 | X_raw, Y_raw = None, None 847 | outer_evaluation_dir = os.path.join(self.datadir, 'env=' + self.name) 848 | if not os.path.isdir(outer_evaluation_dir): 849 | raise Exception('There is no evaluation direcory for environment ' + self.name) 850 | for policy_checkpoint in policy_checkpoints: 851 | 852 | evaluation_dir = os.path.join(outer_evaluation_dir, 'environment_step=' + policy_checkpoint) 853 | if not os.path.isdir(evaluation_dir): 854 | raise Exception('There is no evaluation direcory for policy checkpoint ' + policy_checkpoint) 855 | 856 | evaluation_file = os.path.join(evaluation_dir, evaluation_suffix) 857 | if not os.path.isfile(evaluation_file): 858 | raise Exception('There is no evaluation file for policy checkpoint ' + policy_checkpoint) 859 | 860 | X_raw_checkpoint, Y_raw_checkpoint = self._read_checkpoint_data(evaluation_file) 861 | if X_raw is None and Y_raw is None: 862 | X_raw = X_raw_checkpoint 863 | Y_raw = Y_raw_checkpoint 864 | else: 865 | X_raw = np.concatenate((X_raw, X_raw_checkpoint)) 866 | Y_raw = np.concatenate((Y_raw, Y_raw_checkpoint)) 867 | 868 | assert len(X_raw) == len(Y_raw) 869 | assert X_raw.shape[1] == self.observation_dimension + self.action_dimension 870 | assert Y_raw.shape[1] == self.observation_dimension + 1 871 | self.N = len(X_raw) 872 | return X_raw, Y_raw 873 | 874 | def _read_checkpoint_data(self, evaluation_file): 875 | 876 | with open(evaluation_file, 'rb') as handle: 877 | evaluation_result = pickle.load(handle) 878 | assert len(evaluation_result) == evaluations 879 | 880 | X_raw_checkpoint, Y_raw_checkpoint = None, None 881 | for evaluation in range(evaluations): 882 | 883 | evaluation_result_trajec = evaluation_result[evaluation] 884 | observation_trajec = evaluation_result_trajec['observations'] 885 | action_trajec = evaluation_result_trajec['actions'] 886 | reward_trajec = evaluation_result_trajec['rewards'] 887 | assert len(observation_trajec) == len(action_trajec) + 1 == len(reward_trajec) + 1 888 | assert observation_trajec.shape[1] == self.observation_dimension 889 | assert action_trajec.shape[1] == self.action_dimension 890 | assert len(reward_trajec.shape) == 1 891 | 892 | X_raw_trajec = np.concatenate((observation_trajec[:-1, :], action_trajec), axis=1) 893 | delta_observation_trajec = observation_trajec[1:, :] - observation_trajec[:-1, :] 894 | Y_raw_trajec = np.concatenate((delta_observation_trajec, reward_trajec[:, None]), axis=1) 895 | if X_raw_checkpoint is None and Y_raw_checkpoint is None: 896 | X_raw_checkpoint = X_raw_trajec 897 | Y_raw_checkpoint = Y_raw_trajec 898 | else: 899 | X_raw_checkpoint = np.concatenate((X_raw_checkpoint, X_raw_trajec)) 900 | Y_raw_checkpoint = np.concatenate((Y_raw_checkpoint, Y_raw_trajec)) 901 | 902 | assert len(X_raw_checkpoint) == len(Y_raw_checkpoint) 903 | assert X_raw_checkpoint.shape[1] == self.observation_dimension + self.action_dimension 904 | assert Y_raw_checkpoint.shape[1] == self.observation_dimension + 1 905 | return X_raw_checkpoint, Y_raw_checkpoint 906 | 907 | 908 | @add_regression 909 | class Ant(MujocoSoftActorCriticDataset): 910 | name = 'Ant-v2' 911 | url_mujoco_dataset = urls_mujoco_dataset[name] 912 | observation_dimension = 111 913 | action_dimension = 8 914 | 915 | 916 | @add_regression 917 | class HalfCheetah(MujocoSoftActorCriticDataset): 918 | name = 'HalfCheetah-v2' 919 | url_mujoco_dataset = urls_mujoco_dataset[name] 920 | observation_dimension = 17 921 | action_dimension = 6 922 | 923 | 924 | @add_regression 925 | class Hopper(MujocoSoftActorCriticDataset): 926 | name = 'Hopper-v2' 927 | url_mujoco_dataset = urls_mujoco_dataset[name] 928 | observation_dimension = 11 929 | action_dimension = 3 930 | 931 | 932 | @add_regression 933 | class Humanoid(MujocoSoftActorCriticDataset): 934 | name = 'Humanoid-v2' 935 | url_mujoco_sub_datasets = {'1': urls_mujoco_dataset[name + '-1'], 936 | '2': urls_mujoco_dataset[name + '-2'], 937 | '3': urls_mujoco_dataset[name + '-3'], 938 | '4': urls_mujoco_dataset[name + '-4'], 939 | '5': urls_mujoco_dataset[name + '-5']} 940 | observation_dimension = 376 941 | action_dimension = 17 942 | 943 | def download(self): 944 | for i in range(1, 6): 945 | filename = self.datadir + '-' + str(i) + '.zip' 946 | with urlopen(self.url_mujoco_sub_datasets[str(i)]) as response, open(filename, 'wb') as out_file: 947 | data = response.read() 948 | out_file.write(data) 949 | zip_ref = zipfile.ZipFile(filename, 'r') 950 | zip_ref.extractall(self.datadir + '-' + str(i)) 951 | zip_ref.close() 952 | os.remove(filename) 953 | os.mkdir(self.datadir) 954 | outer_evaluation_dir = os.path.join(self.datadir, 'env=' + self.name) 955 | os.mkdir(outer_evaluation_dir) 956 | readme_counter = 0 957 | for i in range(1, 6): 958 | dirname = os.path.join(self.datadir + '-' + str(i), 'env=' + self.name + '-' + str(i)) 959 | elements = os.listdir(dirname) 960 | for element in elements: 961 | if element == 'readme.txt' and readme_counter == 1: 962 | continue 963 | if element == 'readme.txt': 964 | readme_counter += 1 965 | shutil.move(os.path.join(dirname, element), outer_evaluation_dir) 966 | shutil.rmtree(self.datadir + '-' + str(i)) 967 | 968 | 969 | @add_regression 970 | class InvertedDoublePendulum(MujocoSoftActorCriticDataset): 971 | name = 'InvertedDoublePendulum-v2' 972 | url_mujoco_dataset = urls_mujoco_dataset[name] 973 | observation_dimension = 11 974 | action_dimension = 1 975 | 976 | 977 | @add_regression 978 | class InvertedPendulum(MujocoSoftActorCriticDataset): 979 | name = 'InvertedPendulum-v2' 980 | url_mujoco_dataset = urls_mujoco_dataset[name] 981 | observation_dimension = 4 982 | action_dimension = 1 983 | 984 | 985 | @add_regression 986 | class Pendulum(MujocoSoftActorCriticDataset): 987 | name = 'Pendulum-v0' 988 | url_mujoco_dataset = urls_mujoco_dataset[name] 989 | observation_dimension = 3 990 | action_dimension = 1 991 | 992 | 993 | @add_regression 994 | class Reacher(MujocoSoftActorCriticDataset): 995 | name = 'Reacher-v2' 996 | url_mujoco_dataset = urls_mujoco_dataset[name] 997 | observation_dimension = 11 998 | action_dimension = 2 999 | 1000 | 1001 | @add_regression 1002 | class Swimmer(MujocoSoftActorCriticDataset): 1003 | name = 'Swimmer-v2' 1004 | url_mujoco_dataset = urls_mujoco_dataset[name] 1005 | observation_dimension = 8 1006 | action_dimension = 2 1007 | 1008 | 1009 | @add_regression 1010 | class Walker2d(MujocoSoftActorCriticDataset): 1011 | name = 'Walker2d-v2' 1012 | url_mujoco_dataset = urls_mujoco_dataset[name] 1013 | observation_dimension = 17 1014 | action_dimension = 6 1015 | 1016 | 1017 | ########################## 1018 | 1019 | regression_datasets = list(_ALL_REGRESSION_DATATSETS.keys()) 1020 | regression_datasets.sort() 1021 | 1022 | classification_datasets = list(_ALL_CLASSIFICATION_DATATSETS.keys()) 1023 | classification_datasets.sort() 1024 | 1025 | def get_regression_data(name, *args, **kwargs): 1026 | return _ALL_REGRESSION_DATATSETS[name](*args, **kwargs) 1027 | 1028 | def get_classification_data(name, *args, **kwargs): 1029 | return _ALL_CLASSIFICATION_DATATSETS[name](*args, **kwargs) 1030 | -------------------------------------------------------------------------------- /bayesian_benchmarks/database_utils.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import numpy as np 3 | import io 4 | 5 | from bayesian_benchmarks.paths import RESULTS_DB_PATH 6 | 7 | ## from https://stackoverflow.com/questions/18621513/python-insert-numpy-array-into-sqlite3-database 8 | 9 | def adapt_array(arr): 10 | """ 11 | http://stackoverflow.com/a/31312102/190597 (SoulNibbler) 12 | """ 13 | out = io.BytesIO() 14 | np.save(out, arr) 15 | out.seek(0) 16 | return sqlite3.Binary(out.read()) 17 | 18 | def convert_array(text): 19 | out = io.BytesIO(text) 20 | out.seek(0) 21 | return np.load(out) 22 | 23 | # Converts np.array to TEXT when inserting 24 | sqlite3.register_adapter(np.ndarray, adapt_array) 25 | 26 | # Converts TEXT to np.array when selecting 27 | sqlite3.register_converter("array", convert_array) 28 | 29 | 30 | class Database: 31 | def __init__(self, name=None): 32 | self.conn = None 33 | self.cursor = None 34 | name = name or RESULTS_DB_PATH 35 | 36 | self.open(name) 37 | 38 | def open(self, name): 39 | try: 40 | self.conn = sqlite3.connect(name, detect_types=sqlite3.PARSE_DECLTYPES) 41 | self.cursor = self.conn.cursor() 42 | 43 | except sqlite3.Error as e: # pragma: no cover 44 | print("Error connecting to database!") 45 | 46 | def close(self): 47 | if self.conn: 48 | self.conn.commit() 49 | self.cursor.close() 50 | self.conn.close() 51 | 52 | def delete(self, table, delete_dict): 53 | keys, vals = dict_to_lists(delete_dict) 54 | t = ' AND '.join(['{}=?'.format(k) for k in keys]) 55 | s = "DELETE FROM {} WHERE ({})".format(table, t) 56 | # doesn't appear to be working 57 | self.cursor.execute(s, vals) 58 | 59 | def read(self, table_name : str, fields_to_return : list, search_dict : dict, limit=None): 60 | """ 61 | Read the database 62 | :param table_name: table to search 63 | :param fields_to_return: list of fields to return 64 | :param search_dict: dict of fields to match and the values 65 | :param limit: limit of number of values to return 66 | :return: 67 | """ 68 | keys, vals = dict_to_lists(search_dict) 69 | t = ' AND '.join(['{}=?'.format(k) for k in keys]) 70 | s = "SELECT {} FROM {} WHERE {}".format(', '.join(fields_to_return), table_name, t) 71 | self.cursor.execute(s, vals) 72 | 73 | rows = self.cursor.fetchall() 74 | 75 | return rows[len(rows) - limit if limit else 0:] 76 | 77 | def check_table_has_columns(self, table_name : str): 78 | """ 79 | True if the table has any columns, False otherwise 80 | :param table_name: name of table to check 81 | :return: bool 82 | """ 83 | self.cursor.execute('PRAGMA table_info({})'.format(table_name)) 84 | return len(self.cursor.fetchall()) > 0 85 | 86 | def write(self, table_name, results_dict): 87 | """ 88 | Writes a row in the table, creating the columns if necessary inferring the types. It is assumed that 89 | the values are either strings, floats or numpy arrays 90 | 91 | :param table_name: name of table to update 92 | :param results_dict: a dictionary of results 93 | :return: 94 | """ 95 | keys, values = dict_to_lists(results_dict) 96 | if not self.check_table_has_columns(table_name): 97 | types = [infer_type(v) for v in values] 98 | t = 'CREATE TABLE {} ({})'.format(table_name, ' ,'.join(['{} {}'.format(k, t) for k, t in zip(keys, types)])) 99 | self.cursor.execute(t) 100 | self.conn.commit() 101 | 102 | query = "INSERT INTO {} ({}) VALUES ({});".format(table_name, ', '.join(keys), (' ?,'*len(keys))[1:-1]) 103 | self.cursor.execute(query, values) 104 | 105 | def __exit__(self, exc_type, exc_value, traceback): 106 | self.close() 107 | 108 | def __enter__(self): 109 | return self 110 | 111 | def dict_to_lists(d : dict): 112 | """ 113 | Flattens a dict into keys and values, in the same order 114 | 115 | :param d: dict to flatten 116 | :return: list of keys, list of values 117 | """ 118 | keys = d.keys() 119 | return keys, [d[k] for k in keys] 120 | 121 | def infer_type(val): 122 | """ 123 | The sqlite type of val 124 | 125 | :param val: either a string, float, int, or np.ndarray 126 | :return: 'text', 'real', 'int' or 'array' 127 | """ 128 | if isinstance(val, str): 129 | return 'text' 130 | elif isinstance(val, int): 131 | return 'int' 132 | elif isinstance(val, float) : 133 | return 'real' 134 | elif isinstance(val, np.ndarray): 135 | return 'array' 136 | else: 137 | raise NotImplementedError('unrecognised type for value {}'.format(val)) 138 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/README.md: -------------------------------------------------------------------------------- 1 | To add a model create a new folder and add a file models.py that implements the model. 2 | 3 | See template.py for the model API. 4 | 5 | Add the name of your model to `get_models.py` under `all_regression_models` or `all_classification_models` as appropriate. 6 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/deep_gp_doubly_stochastic/README.md: -------------------------------------------------------------------------------- 1 | Run with 2 | gpflow 147ce61e840ba80d3b387018c24026f32231e200 3 | doubly_stochastic_dgp d2c2e1c7ebda45067e5c41996587e0e06e7cdce5 4 | 5 | This is the model from 6 | 7 | @inproceedings{salimbeni2017doubly, 8 | title={Doubly stochastic variational inference for deep gaussian processes}, 9 | author={Salimbeni, Hugh and Deisenroth, Marc}, 10 | booktitle={Advances in Neural Information Processing Systems}, 11 | year={2017} 12 | } 13 | 14 | using natural gradients from 15 | 16 | @article{salimbeni2018natural, 17 | title={Natural Gradients in Practice: Non-Conjugate Variational Inference in Gaussian Process Models}, 18 | author={Salimbeni, Hugh and Eleftheriadis, Stefanos and Hensman, James}, 19 | journal={Artificial Intelligence and Statistics}, 20 | year={2018} 21 | } 22 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/deep_gp_doubly_stochastic/models.py: -------------------------------------------------------------------------------- 1 | import gpflow 2 | from doubly_stochastic_dgp.dgp import DGP 3 | 4 | import numpy as np 5 | 6 | from scipy.cluster.vq import kmeans2 7 | from scipy.stats import norm 8 | from scipy.special import logsumexp 9 | 10 | 11 | 12 | 13 | class RegressionModel(object): 14 | def __init__(self, is_test=False, seed=0): 15 | if is_test: 16 | class ARGS: 17 | num_inducing = 2 18 | iterations = 2 19 | small_iterations = 1 20 | adam_lr = 0.01 21 | gamma = 0.1 22 | minibatch_size = 100 23 | num_posterior_samples = 2 24 | initial_likelihood_var = 0.01 25 | else: # pragma: no cover 26 | class ARGS: 27 | num_inducing = 100 28 | iterations = 10000 29 | small_iterations = 1000 30 | adam_lr = 0.01 31 | gamma = 0.1 32 | minibatch_size = 1000 33 | num_posterior_samples = 1000 34 | initial_likelihood_var = 0.01 35 | self.ARGS = ARGS 36 | self.model = None 37 | 38 | def fit(self, X, Y): 39 | initial_likelihood_var = self.ARGS.initial_likelihood_var 40 | class Lik(gpflow.likelihoods.Gaussian): 41 | def __init__(self): 42 | gpflow.likelihoods.Gaussian.__init__(self) 43 | self.variance = initial_likelihood_var 44 | return self._fit(X, Y, Lik) 45 | 46 | def _fit(self, X, Y, Lik, **kwargs): 47 | if X.shape[0] > self.ARGS.num_inducing: 48 | Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] 49 | else: 50 | # pad with random values 51 | Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) 52 | 53 | if not self.model: 54 | kerns = [] 55 | for _ in range(2): 56 | kerns.append(gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5)) 57 | 58 | mb_size = self.ARGS.minibatch_size if X.shape[0] > self.ARGS.minibatch_size else None 59 | 60 | self.model = DGP(X, Y, Z, kerns, Lik(), 61 | minibatch_size=mb_size, 62 | **kwargs) 63 | 64 | self.model.layers[0].q_sqrt = self.model.layers[0].q_sqrt.read_value() * 1e-5 65 | 66 | if isinstance(self.model.likelihood.likelihood, gpflow.likelihoods.Gaussian): 67 | var_list = [[self.model.layers[-1].q_mu, self.model.layers[-1].q_sqrt]] 68 | self.model.layers[-1].q_mu.set_trainable(False) 69 | self.model.layers[-1].q_sqrt.set_trainable(False) 70 | self.ng = gpflow.train.NatGradOptimizer(gamma=self.ARGS.gamma).make_optimize_tensor(self.model, var_list=var_list) 71 | else: 72 | self.ng = None 73 | 74 | self.adam = gpflow.train.AdamOptimizer(self.ARGS.adam_lr).make_optimize_tensor(self.model) 75 | 76 | iters = self.ARGS.iterations 77 | self.sess = self.model.enquire_session() 78 | else: 79 | iters = self.ARGS.small_iterations # after first time use fewer iterations 80 | 81 | # we might have new data 82 | self.model.X.assign(X, session=self.sess) 83 | self.model.Y.assign(Y, session=self.sess) 84 | 85 | self.model.layers[0].feature.Z.assign(Z, session=self.sess) 86 | self.model.layers[0].q_mu.assign(np.zeros((self.ARGS.num_inducing, X.shape[1])), session=self.sess) 87 | self.model.layers[0].q_sqrt.assign(1e-5*np.tile(np.eye(self.ARGS.num_inducing)[None], [X.shape[1], 1, 1]), session=self.sess) 88 | 89 | self.model.layers[1].feature.Z.assign(Z, session=self.sess) 90 | num_outputs = self.model.layers[1].q_sqrt.shape[0] 91 | self.model.layers[1].q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) 92 | self.model.layers[1].q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) 93 | 94 | try: 95 | for _ in range(iters): 96 | 97 | if _ % 100 == 0: 98 | print('{} {}'.format(_, self.sess.run(self.model.likelihood_tensor))) 99 | if self.ng: 100 | self.sess.run(self.ng) 101 | self.sess.run(self.adam) 102 | 103 | except KeyboardInterrupt: # pragma: no cover 104 | pass 105 | 106 | self.model.anchor(session=self.sess) 107 | 108 | def _predict(self, Xs, S): 109 | ms, vs = [], [] 110 | n = max(len(Xs) / 100, 1) # predict in small batches 111 | for xs in np.array_split(Xs, n): 112 | m, v = self.model.predict_y(xs, S, session=self.sess) 113 | ms.append(m) 114 | vs.append(v) 115 | 116 | return np.concatenate(ms, 1), np.concatenate(vs, 1) # num_posterior_samples, N_test, D_y 117 | 118 | def predict(self, Xs): 119 | ms, vs = self._predict(Xs, self.ARGS.num_posterior_samples) 120 | 121 | # the first two moments 122 | m = np.average(ms, 0) 123 | v = np.average(vs + ms**2, 0) - m**2 124 | return m, v 125 | 126 | def sample(self, Xs, S): 127 | ms, vs = self._predict(Xs, S) 128 | return ms + vs**0.5 * np.random.randn(*ms.shape) 129 | 130 | 131 | class ClassificationModel(RegressionModel): 132 | def __init__(self, K, is_test=False, seed=0): 133 | self.K = K 134 | RegressionModel.__init__(self, is_test=is_test, seed=seed) 135 | 136 | def fit(self, X, Y): 137 | if self.K == 2: 138 | Lik = gpflow.likelihoods.Bernoulli 139 | num_latent = 1 140 | else: 141 | K = self.K 142 | class Lik(gpflow.likelihoods.MultiClass): 143 | def __init__(self): 144 | gpflow.likelihoods.MultiClass.__init__(self, K) 145 | num_latent = K 146 | 147 | return self._fit(X, Y, Lik, num_outputs=num_latent) 148 | 149 | 150 | def predict(self, Xs): 151 | m, v = self.model.predict_y(Xs, self.ARGS.num_posterior_samples) # num_samples, N_test, K 152 | m = np.average(m, 0) # N_test, K 153 | if self.K == 2: 154 | # Convert Bernoulli to onehot 155 | return np.concatenate([1 - m, m], -1) 156 | else: 157 | return m 158 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/get_model.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | import os 3 | 4 | from bayesian_benchmarks.models.non_bayesian_models import non_bayesian_model 5 | 6 | abs_path = os.path.abspath(__file__)[:-len('/get_model.py')] 7 | 8 | def get_regression_model(name): 9 | assert name in all_regression_models 10 | return non_bayesian_model(name, 'regression') or \ 11 | import_module('bayesian_benchmarks.models.{}.models'.format(name)).RegressionModel 12 | 13 | def get_classification_model(name): 14 | assert name in all_classification_models 15 | return non_bayesian_model(name, 'classification') or \ 16 | import_module('bayesian_benchmarks.models.{}.models'.format(name)).ClassificationModel 17 | 18 | # add new regression models here 19 | all_regression_models = [ 20 | 'linear', 21 | 'variationally_sparse_gp', 22 | 'variationally_sparse_gp_minibatch', 23 | 'deep_gp_doubly_stochastic', 24 | 'svm', 25 | 'knn', 26 | 'decision_tree', 27 | 'random_forest', 28 | 'gradient_boosting_machine', 29 | 'adaboost', 30 | 'mlp', 31 | ] 32 | 33 | # add new classification models here 34 | all_classification_models = [ 35 | 'linear', 36 | 'variationally_sparse_gp', 37 | 'variationally_sparse_gp_minibatch', 38 | 'deep_gp_doubly_stochastic', 39 | 'svm', 40 | 'naive_bayes', 41 | 'knn', 42 | 'decision_tree', 43 | 'random_forest', 44 | 'gradient_boosting_machine', 45 | 'adaboost', 46 | 'mlp', 47 | ] 48 | 49 | all_models = list(set(all_regression_models).union(set(all_classification_models))) 50 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/non_bayesian_models.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sklearn import linear_model 4 | from sklearn import neighbors 5 | from sklearn import svm 6 | from sklearn import naive_bayes 7 | from sklearn import tree 8 | from sklearn import ensemble 9 | from sklearn import neural_network 10 | 11 | def regression_model(model): 12 | class SKLWrapperRegression(object): 13 | def __init__(self, is_test=False, seed=0): 14 | self.model = model 15 | 16 | def fit(self, X, Y): 17 | self.model.fit(X, Y.flatten()) 18 | self.std = np.std(self.model.predict(X) - Y.flatten()) 19 | 20 | def predict(self, Xs): 21 | pred_mean = self.model.predict(Xs)[:, None] 22 | return pred_mean, np.ones_like(pred_mean) * (self.std + 1e-6) ** 2 23 | 24 | def sample(self, Xs, num_samples): 25 | m, v = self.predict(Xs) 26 | N, D = np.shape(m) 27 | m, v = np.expand_dims(m, 0), np.expand_dims(v, 0) 28 | return m + np.random.randn(num_samples, N, D) * (v ** 0.5) 29 | 30 | return SKLWrapperRegression 31 | 32 | 33 | def classification_model(model): 34 | class SKLWrapperClassification(object): 35 | def __init__(self, K, is_test=False, seed=0): 36 | self.model = model 37 | self.K = K 38 | 39 | def fit(self, X, Y): 40 | self.Ks_seen = list(set(Y.flatten().astype(int))) 41 | self.Ks_seen.sort() 42 | self.model.fit(X, Y.ravel()) 43 | 44 | def predict(self, Xs): 45 | ps = self.model.predict_proba(Xs) 46 | if len(self.Ks_seen) == self.K: 47 | return ps 48 | else: # not all classes have been seen, and sklearn doesn't allow setting of n_classes 49 | ret = np.zeros((len(ps), self.K)) 50 | for k, p in zip(self.Ks_seen, ps.T): 51 | ret[:, k] = p 52 | return ret 53 | 54 | return SKLWrapperClassification 55 | 56 | 57 | def non_bayesian_model(name, task): 58 | if name == 'linear' and task == 'regression': 59 | return regression_model(linear_model.LinearRegression()) 60 | 61 | elif name == 'linear' and task == 'classification': 62 | return classification_model(linear_model.LogisticRegression()) 63 | 64 | if name == 'svm' and task == 'regression': 65 | return regression_model(svm.SVR()) 66 | 67 | elif name == 'svm' and task == 'classification': 68 | return classification_model(svm.SVC(probability=True)) 69 | 70 | if name == 'knn' and task == 'regression': 71 | return regression_model(neighbors.KNeighborsRegressor()) # default is K=5 72 | 73 | elif name == 'knn' and task == 'classification': 74 | return classification_model(neighbors.KNeighborsClassifier()) # default is K=5 75 | 76 | elif name == 'naive_bayes' and task == 'classification': 77 | return classification_model(naive_bayes.GaussianNB()) 78 | 79 | if name == 'decision_tree' and task == 'regression': 80 | return regression_model(tree.DecisionTreeRegressor()) 81 | 82 | elif name == 'decision_tree' and task == 'classification': 83 | return classification_model(tree.DecisionTreeClassifier()) 84 | 85 | if name == 'random_forest' and task == 'regression': 86 | return regression_model(ensemble.RandomForestRegressor()) 87 | 88 | elif name == 'random_forest' and task == 'classification': 89 | return classification_model(ensemble.RandomForestClassifier()) # default is 10 estimators 90 | 91 | if name == 'gradient_boosting_machine' and task == 'regression': 92 | return regression_model(ensemble.GradientBoostingRegressor()) 93 | 94 | elif name == 'gradient_boosting_machine' and task == 'classification': 95 | return classification_model(ensemble.GradientBoostingClassifier()) # default is 100 estimators 96 | 97 | if name == 'adaboost' and task == 'regression': 98 | return regression_model(ensemble.AdaBoostRegressor()) 99 | 100 | elif name == 'adaboost' and task == 'classification': 101 | return classification_model(ensemble.AdaBoostClassifier()) # default is 100 estimators 102 | 103 | if name == 'mlp' and task == 'regression': 104 | return regression_model(neural_network.MLPRegressor()) 105 | 106 | elif name == 'mlp' and task == 'classification': 107 | return classification_model(neural_network.MLPClassifier()) 108 | 109 | else: # pragma: no cover 110 | return None 111 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/template.py: -------------------------------------------------------------------------------- 1 | """ 2 | To add a model, create a new directory in bayesian_benchmarks.models (here) and make file models.py containing at 3 | least one of the classes below. 4 | 5 | Model usage is similar to sklearn. For regression: 6 | 7 | model = RegressionModel(is_test=False) 8 | model.fit(X, Y) 9 | mean, var = model.predict(X_test) # first two moments of the posterior 10 | samples = model.sample(X_test, S) # samples from the posterior 11 | 12 | For classification: 13 | model = ClassificationModel(K, is_test=False) # K is the number of classes 14 | model.fit(X, Y) 15 | p = model.predict(X_test) # predictive probabilities for each class (i.e. onehot) 16 | 17 | It should be feasible to call fit and predict many times (e.g. avoid rebuilding a tensorflow graph on each call). 18 | 19 | """ 20 | 21 | import numpy as np 22 | 23 | 24 | class RegressionModel: 25 | def __init__(self, is_test=False, seed=0): 26 | """ 27 | If is_test is True your model should train and predict in a few seconds (i.e. suitable for travis) 28 | """ 29 | pass 30 | 31 | def fit(self, X : np.ndarray, Y : np.ndarray): 32 | """ 33 | Train the model (and probably create the model, too, since there is no shape information on the __init__) 34 | 35 | :param X: numpy array, of shape N, Dx 36 | :param Y: numpy array, of shape N, Dy 37 | :return: 38 | """ 39 | pass 40 | 41 | def predict(self, Xs : np.ndarray): 42 | """ 43 | The predictive mean and variance 44 | 45 | :param Xs: numpy array, of shape N, Dx 46 | :return: mean, var, both of shape N, Dy 47 | """ 48 | raise NotImplementedError 49 | 50 | def sample(self, Xs : np.ndarray, S : int): 51 | """ 52 | Samples from the posterior 53 | :param Xs: numpy array, of shape N, Dx 54 | :param S: number of samples 55 | :return: numpy array, of shape (S, N, Dy) 56 | """ 57 | raise NotImplementedError 58 | 59 | 60 | class ClassificationModel: 61 | def __init__(self, K, is_test=False, seed=0): 62 | """ 63 | :param K: number of classes 64 | :param is_test: whether to run quickly for testing purposes 65 | """ 66 | 67 | def fit(self, X : np.ndarray, Y : np.ndarray): 68 | """ 69 | Train the model (and probably create the model, too, since there is no shape information on the __init__) 70 | 71 | Note Y is not onehot, but is an int array of labels in {0, 1, ..., K-1} 72 | 73 | :param X: numpy array, of shape N, Dx 74 | :param Y: numpy array, of shape N, 1 75 | :return: 76 | """ 77 | pass 78 | 79 | def predict(self, Xs : np.ndarray): 80 | """ 81 | The predictive probabilities 82 | 83 | :param Xs: numpy array, of shape N, Dx 84 | :return: p, of shape (N, K) 85 | """ 86 | raise NotImplementedError 87 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/variationally_sparse_gp/README.md: -------------------------------------------------------------------------------- 1 | Regression models: 2 | * Less than 5000 points: SGPR [Titsias 2009] with lbfgs. 3 | * More than 5000 points: SVGP [Hensman 2013] with natural gradients [Salimbeni 2018] for the variational parameters and Adam optimizer the remaining parameters. 4 | 5 | For classification models the model is SVGP [Hensman 2015] with a Bernoulli likelihood, and Robust Max for multiclass classification, with natural gradients [Salimbeni 2018] for the variational parameters and Adam optimizer the remaining parameters. 6 | 7 | 8 | @article{titsias2009variational, 9 | title={Variational learning of inducing variables in sparse Gaussian processes}, 10 | author={Titsias, Michalis}, 11 | journal={Artificial Intelligence and Statistics}, 12 | year={2009} 13 | } 14 | 15 | @article{hensman2013gaussian, 16 | title={Gaussian processes for big data}, 17 | author={Hensman, James and Fusi, Nicolo and Lawrence, Neil D}, 18 | journal={Uncertainty in Artificial Intelligence}, 19 | year={2013} 20 | } 21 | 22 | @article{hensman2015scalable, 23 | title={Scalable variational Gaussian process classification}, 24 | author={Hensman, James and Matthews, Alexander G de G and Ghahramani, Zoubin}, 25 | journal={Artificial Intelligence and Statistics}, 26 | year={2015} 27 | } 28 | 29 | @article{salimbeni2018natural, 30 | title={Natural Gradients in Practice: Non-Conjugate Variational Inference in Gaussian Process Models}, 31 | author={Salimbeni, Hugh and Eleftheriadis, Stefanos and Hensman, James}, 32 | journal={Artificial Intelligence and Statistics}, 33 | year={2018} 34 | } 35 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/variationally_sparse_gp/models.py: -------------------------------------------------------------------------------- 1 | import gpflow 2 | import numpy as np 3 | from scipy.cluster.vq import kmeans2 4 | from scipy.stats import norm 5 | 6 | class RegressionModel(object): 7 | def __init__(self, is_test=False, seed=0): 8 | if is_test: 9 | class ARGS: 10 | num_inducing = 2 11 | iterations = 1 12 | small_iterations = 1 13 | initial_likelihood_var = 0.01 14 | else: # pragma: no cover 15 | class ARGS: 16 | num_inducing = 100 17 | iterations = 10000 18 | small_iterations = 1000 19 | initial_likelihood_var = 0.01 20 | self.ARGS = ARGS 21 | self.model = None 22 | 23 | def fit(self, X, Y): 24 | if X.shape[0] > self.ARGS.num_inducing: 25 | Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] 26 | else: 27 | # pad with random values 28 | Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) 29 | 30 | # make model if necessary 31 | if not self.model: 32 | kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5) 33 | lik = gpflow.likelihoods.Gaussian() 34 | lik.variance = self.ARGS.initial_likelihood_var 35 | 36 | self.model = gpflow.models.SGPR(X, Y, kern, feat=Z) 37 | self.model.likelihood.variance = lik.variance.read_value() 38 | self.sess = self.model.enquire_session() 39 | self.opt = gpflow.train.ScipyOptimizer() 40 | 41 | # we might have new data 42 | self.model.X.assign(X, session=self.sess) 43 | self.model.Y.assign(Y, session=self.sess) 44 | self.model.feature.Z.assign(Z, session=self.sess) 45 | 46 | self.opt.minimize(self.model, session=self.sess, maxiter=self.ARGS.iterations) 47 | 48 | def predict(self, Xs): 49 | return self.model.predict_y(Xs, session=self.sess) 50 | 51 | def sample(self, Xs, num_samples): 52 | m, v = self.predict(Xs) 53 | N, D = np.shape(m) 54 | m, v = np.expand_dims(m, 0), np.expand_dims(v, 0) 55 | return m + np.random.randn(num_samples, N, D) * (v ** 0.5) 56 | 57 | 58 | class ClassificationModel(object): 59 | def __init__(self, K, is_test=False, seed=0): 60 | if is_test: 61 | class ARGS: 62 | num_inducing = 2 63 | iterations = 1 64 | small_iterations = 1 65 | initial_likelihood_var = 0.01 66 | else: # pragma: no cover 67 | class ARGS: 68 | num_inducing = 100 69 | iterations = 5000 70 | small_iterations = 1000 71 | initial_likelihood_var = 0.01 72 | 73 | self.ARGS = ARGS 74 | self.K = K 75 | self.model = None 76 | 77 | def fit(self, X, Y): 78 | Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if X.shape[0] > self.ARGS.num_inducing else X.copy() 79 | 80 | if not self.model: 81 | if self.K == 2: 82 | lik = gpflow.likelihoods.Bernoulli() 83 | num_latent = 1 84 | else: 85 | lik = gpflow.likelihoods.MultiClass(self.K) 86 | num_latent = self.K 87 | 88 | kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1]) ** 0.5) 89 | self.model = gpflow.models.SVGP(X, Y, kern, lik, 90 | feat=Z, 91 | whiten=False, 92 | num_latent=num_latent, 93 | minibatch_size=None) 94 | 95 | self.sess = self.model.enquire_session() 96 | self.opt = gpflow.train.ScipyOptimizer() 97 | 98 | iters = self.ARGS.iterations 99 | 100 | else: 101 | iters = self.ARGS.small_iterations 102 | 103 | # we might have new data 104 | self.model.X.assign(X, session=self.sess) 105 | self.model.Y.assign(Y, session=self.sess) 106 | self.model.feature.Z.assign(Z, session=self.sess) 107 | 108 | num_outputs = self.model.q_sqrt.shape[0] 109 | self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) 110 | self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) 111 | 112 | self.opt.minimize(self.model, maxiter=iters, session=self.sess) 113 | 114 | def predict(self, Xs): 115 | m, v = self.model.predict_y(Xs, session=self.sess) 116 | if self.K == 2: 117 | # convert Bernoulli to onehot 118 | return np.concatenate([1 - m, m], 1) 119 | else: 120 | return m 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /bayesian_benchmarks/models/variationally_sparse_gp_minibatch/models.py: -------------------------------------------------------------------------------- 1 | import gpflow 2 | import numpy as np 3 | from scipy.cluster.vq import kmeans2 4 | from scipy.stats import norm 5 | 6 | class RegressionModel(object): 7 | def __init__(self, is_test=False, seed=0): 8 | if is_test: 9 | class ARGS: 10 | num_inducing = 2 11 | iterations = 1 12 | small_iterations = 1 13 | adam_lr = 0.01 14 | gamma = 0.1 15 | minibatch_size = 100 16 | initial_likelihood_var = 0.01 17 | else: # pragma: no cover 18 | class ARGS: 19 | num_inducing = 100 20 | iterations = 10000 21 | small_iterations = 1000 22 | adam_lr = 0.01 23 | gamma = 0.1 24 | minibatch_size = 1000 25 | initial_likelihood_var = 0.01 26 | self.ARGS = ARGS 27 | self.model = None 28 | 29 | def fit(self, X, Y): 30 | if X.shape[0] > self.ARGS.num_inducing: 31 | Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] 32 | else: 33 | # pad with random values 34 | Z = np.concatenate([X, np.random.randn(self.ARGS.num_inducing - X.shape[0], X.shape[1])], 0) 35 | 36 | # make model if necessary 37 | if not self.model: 38 | kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1])**0.5) 39 | lik = gpflow.likelihoods.Gaussian() 40 | lik.variance = self.ARGS.initial_likelihood_var 41 | mb_size = self.ARGS.minibatch_size if X.shape[0] > self.ARGS.minibatch_size else None 42 | self.model = gpflow.models.SVGP(X, Y, kern, lik, feat=Z, minibatch_size=mb_size) 43 | 44 | var_list = [[self.model.q_mu, self.model.q_sqrt]] 45 | self.model.q_mu.set_trainable(False) 46 | self.model.q_sqrt.set_trainable(False) 47 | self.ng = gpflow.train.NatGradOptimizer(gamma=self.ARGS.gamma).make_optimize_tensor(self.model, var_list=var_list) 48 | self.adam = gpflow.train.AdamOptimizer(self.ARGS.adam_lr).make_optimize_tensor(self.model) 49 | 50 | self.sess = self.model.enquire_session() 51 | 52 | iters = self.ARGS.iterations 53 | 54 | else: 55 | iters = self.ARGS.small_iterations 56 | 57 | # we might have new data 58 | self.model.X.assign(X, session=self.sess) 59 | self.model.Y.assign(Y, session=self.sess) 60 | self.model.feature.Z.assign(Z, session=self.sess) 61 | 62 | self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, Y.shape[1])), session=self.sess) 63 | self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [Y.shape[1], 1, 1]), session=self.sess) 64 | 65 | 66 | for _ in range(iters): 67 | self.sess.run(self.ng) 68 | self.sess.run(self.adam) 69 | self.model.anchor(session=self.sess) 70 | 71 | def predict(self, Xs): 72 | return self.model.predict_y(Xs, session=self.sess) 73 | 74 | def sample(self, Xs, num_samples): 75 | m, v = self.predict(Xs) 76 | N, D = np.shape(m) 77 | m, v = np.expand_dims(m, 0), np.expand_dims(v, 0) 78 | return m + np.random.randn(num_samples, N, D) * (v ** 0.5) 79 | 80 | 81 | class ClassificationModel(object): 82 | def __init__(self, K, is_test=False, seed=0): 83 | if is_test: 84 | class ARGS: 85 | num_inducing = 2 86 | iterations = 1 87 | small_iterations = 1 88 | adam_lr = 0.01 89 | minibatch_size = 100 90 | else: # pragma: no cover 91 | class ARGS: 92 | num_inducing = 100 93 | iterations = 10000 94 | small_iterations = 1000 95 | adam_lr = 0.01 96 | minibatch_size = 1000 97 | self.ARGS = ARGS 98 | 99 | self.K = K 100 | self.model = None 101 | 102 | def fit(self, X, Y): 103 | Z = kmeans2(X, self.ARGS.num_inducing, minit='points')[0] if X.shape[0] > self.ARGS.num_inducing else X.copy() 104 | 105 | if not self.model: 106 | # NB mb_size does not change once the model is created 107 | mb_size = self.ARGS.minibatch_size if X.shape[0] >= self.ARGS.minibatch_size else None 108 | 109 | if self.K == 2: 110 | lik = gpflow.likelihoods.Bernoulli() 111 | num_latent = 1 112 | else: 113 | lik = gpflow.likelihoods.MultiClass(self.K) 114 | num_latent = self.K 115 | 116 | kern = gpflow.kernels.RBF(X.shape[1], lengthscales=float(X.shape[1]) ** 0.5) 117 | self.model = gpflow.models.SVGP(X, Y, kern, lik, 118 | feat=Z, 119 | whiten=False, 120 | num_latent=num_latent, 121 | minibatch_size=mb_size) 122 | 123 | self.opt = gpflow.train.AdamOptimizer(self.ARGS.adam_lr) 124 | 125 | self.sess = self.model.enquire_session() 126 | iters = self.ARGS.iterations 127 | 128 | else: 129 | iters = self.ARGS.small_iterations 130 | 131 | # we might have new data 132 | self.model.X.assign(X, session=self.sess) 133 | self.model.Y.assign(Y, session=self.sess) 134 | self.model.feature.Z.assign(Z, session=self.sess) 135 | 136 | num_outputs = self.model.q_sqrt.shape[0] 137 | self.model.q_mu.assign(np.zeros((self.ARGS.num_inducing, num_outputs)), session=self.sess) 138 | self.model.q_sqrt.assign(np.tile(np.eye(self.ARGS.num_inducing)[None], [num_outputs, 1, 1]), session=self.sess) 139 | 140 | self.opt.minimize(self.model, maxiter=iters, session=self.sess) 141 | 142 | def predict(self, Xs): 143 | m, v = self.model.predict_y(Xs, session=self.sess) 144 | if self.K == 2: 145 | # convert Bernoulli to onehot 146 | return np.concatenate([1 - m, m], 1) 147 | else: 148 | return m 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /bayesian_benchmarks/paths.py: -------------------------------------------------------------------------------- 1 | from six.moves import configparser 2 | import os 3 | 4 | cfg = configparser.ConfigParser() 5 | dirs = [os.curdir, os.path.dirname(os.path.realpath(__file__)), 6 | os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')] 7 | locations = map(os.path.abspath, dirs) 8 | 9 | for loc in locations: 10 | if cfg.read(os.path.join(loc, 'bayesian_benchmarksrc')): 11 | break 12 | 13 | def expand_to_absolute(path): 14 | if './' == path[:2]: 15 | return os.path.join(os.path.dirname(os.path.realpath(__file__)), path[2:]) 16 | else: 17 | return path 18 | 19 | DATA_PATH = expand_to_absolute(cfg['paths']['data_path']) 20 | BASE_SEED = int(cfg['seeds']['seed']) 21 | RESULTS_DB_PATH = expand_to_absolute(cfg['paths']['results_path']) 22 | 23 | if not os.path.isdir(DATA_PATH): 24 | os.mkdir(DATA_PATH) 25 | -------------------------------------------------------------------------------- /bayesian_benchmarks/scripts/make_experiments.py: -------------------------------------------------------------------------------- 1 | from bayesian_benchmarks.data import regression_datasets, classification_datasets 2 | from bayesian_benchmarks.database_utils import Database 3 | 4 | import itertools 5 | import os 6 | from subprocess import call 7 | 8 | 9 | def make_experiment_combinations(combinations: list): 10 | """ 11 | The product of all combinations of arguments. 12 | :param combinations: A list of dictionaries, each with a list of args 13 | :return: A list of dictionaries for all combinations 14 | """ 15 | fields = [] 16 | vals = [] 17 | for p in combinations: 18 | for k in p: 19 | fields.append(k) 20 | vals.append(p[k]) 21 | 22 | ret = [] 23 | for a in itertools.product(*vals): 24 | d = {} 25 | 26 | for f, arg in zip(fields, a): 27 | d[f] = arg 28 | 29 | ret.append(d) 30 | 31 | return ret 32 | 33 | 34 | def make_local_jobs(script: str, experiments: list, overwrite=False): 35 | """ 36 | Writes a file of commands to be run in in series on a single machine, e.g. 37 | #!/usr/bin/env bash 38 | python run_regression --split=0 39 | python run_regression --split=1 40 | etc. 41 | 42 | If overwrite=True then a new file is written with a shebang, otherwise lines are appended. 43 | 44 | :param script: name of python script to run 45 | :param experiments: list of dictionaries of args 46 | :return: None 47 | """ 48 | if overwrite: 49 | with open('local_run', 'w') as f: 50 | f.write('#!/usr/bin/env bash\n\n') 51 | 52 | with open('local_run', 'a') as f: 53 | for e in experiments: 54 | s = 'python {}.py '.format(script) 55 | for k in e: 56 | s += '--{}={} '.format(k, e[k]) 57 | s += '\n' 58 | f.write(s) 59 | 60 | 61 | def make_condor_jobs(script: str, experiments: list, overwrite=False): 62 | """ 63 | Writes a condor submission file, and also creates the executable if necessary. Preamble for the 64 | exectable (e.g. for setting up the python environment) should go in 'preamble.txt.txt'. Preamble 65 | for the condor submission should go in condor_preamble.txt.txt.txt. 66 | 67 | If overwrite=True then a new file is written with the condor preamble from condor_preamble.txt.txt, 68 | otherwise lines are appended. 69 | 70 | :param script: name of python script to run 71 | :param experiments: list of dictionaries of args 72 | :return: None 73 | """ 74 | condor_run_file = 'condor_run' 75 | if not os.path.isfile(condor_run_file): 76 | with open(condor_run_file, 'w') as f: 77 | f.write("#!/usr/bin/env bash\n") 78 | 79 | preamble = 'preamble.txt' 80 | if os.path.isfile(preamble): 81 | for l in open(preamble, 'r'): 82 | f.writelines(l) 83 | 84 | t = "python " 85 | for i in range(1, 10): 86 | t += '$' + str(i) + ' ' 87 | for i in range(10, 20): 88 | t += '${' + str(i) + '} ' 89 | 90 | f.write(t + '\n') 91 | 92 | call(["chmod", '777', condor_run_file]) 93 | 94 | if overwrite: 95 | with open('condor_jobs', 'w') as f: 96 | with open('condor_preamble.txt.txt', 'r') as ff: 97 | f.writelines(ff) 98 | 99 | with open('condor_jobs', 'a') as f: 100 | for e in experiments: 101 | t = 'Arguments = {}.py '.format(script) 102 | for k in e: 103 | t += '--{}={} '.format(k, e[k]) 104 | t += '\nQueue 1\n' 105 | f.write(t) 106 | 107 | def remove_already_run_experiments(table, experiments): 108 | res = [] 109 | 110 | with Database() as db: 111 | for e in experiments: 112 | if len(db.read(table, ['test_loglik'], e)) == 0: 113 | res.append(e) 114 | 115 | s = 'originally {} experiments, but {} have already been run, so running {} experiments' 116 | print(s.format(len(experiments), len(experiments) - len(res), len(res))) 117 | return res 118 | 119 | ################################################# 120 | models = [ 121 | 'linear', 122 | 'variationally_sparse_gp', 123 | 'variationally_sparse_gp_minibatch', 124 | 'deep_gp_doubly_stochastic', 125 | 'svm', 126 | 'knn', 127 | 'naive_bayes', 128 | 'decision_tree', 129 | 'random_forest', 130 | 'gradient_boosting_machine', 131 | 'adaboost', 132 | 'mlp', 133 | ] 134 | 135 | 136 | ############# Regression 137 | combinations = [] 138 | combinations.append({'dataset' : regression_datasets}) 139 | combinations.append({'split' : range(10)}) 140 | combinations.append({'model' : models}) 141 | experiments = make_experiment_combinations(combinations) 142 | experiments = remove_already_run_experiments('regression', experiments) 143 | 144 | make_local_jobs('../tasks/regression', experiments, overwrite=True) 145 | make_condor_jobs('../tasks/regression', experiments, overwrite=True) 146 | 147 | # make_local_jobs('../tasks/active_learning_continuous', experiments) 148 | # make_condor_jobs('../tasks/active_learning_continuous', experiments) 149 | 150 | # make_local_jobs('../tasks/conditional_density_estimation', experiments) 151 | # make_condor_jobs('../tasks/conditional_density_estimation', experiments) 152 | 153 | ############# Classification 154 | combinations = [] 155 | combinations.append({'dataset' : classification_datasets}) 156 | combinations.append({'split' : range(10)}) 157 | combinations.append({'model' : models}) 158 | 159 | experiments = make_experiment_combinations(combinations) 160 | experiments = remove_already_run_experiments('classification', experiments) 161 | 162 | make_local_jobs('../tasks/classification', experiments) 163 | make_condor_jobs('../tasks/classification', experiments) 164 | # 165 | # # make_local_jobs('../tasks/active_learning_discrete', experiments) 166 | # # make_condor_jobs('../tasks/active_learning_discrete', experiments) 167 | -------------------------------------------------------------------------------- /bayesian_benchmarks/scripts/run_all_pytest.py: -------------------------------------------------------------------------------- 1 | """ 2 | If pytest-xdist is installed, pytest can multiple independent jobs in parallel on a single machine. 3 | 4 | Here we use the facility to run experiments. 5 | 6 | To run 32 experiments in parallel install xdist (pip install pytest-xdist), then following command can be used 7 | 8 | python -m pytest bayesian_benchmarks/scripts/run_all_pytest.py -n 32 9 | 10 | """ 11 | import pytest 12 | 13 | from bayesian_benchmarks.tasks.regression import run as run_regression 14 | from bayesian_benchmarks.tasks.classification import run as run_classification 15 | from bayesian_benchmarks.tasks.active_learning_continuous import run as run_AL_cont 16 | from bayesian_benchmarks.tasks.active_learning_discrete import run as run_AL_disc 17 | from bayesian_benchmarks.tasks.mmd import run as run_mmd 18 | 19 | from bayesian_benchmarks.data import regression_datasets, classification_datasets 20 | from bayesian_benchmarks.database_utils import Database 21 | 22 | 23 | all_regression_models = [ 24 | 'linear', 25 | # 'variationally_sparse_gp', 26 | # 'variationally_sparse_gp_minibatch', 27 | # 'deep_gp_doubly_stochastic', 28 | 'svm', 29 | # 'knn', 30 | # 'decision_tree', 31 | # 'random_forest', 32 | # 'gradient_boosting_machine', 33 | # 'adaboost', 34 | # 'mlp', 35 | ] 36 | 37 | all_classification_models = [ 38 | 'linear', 39 | # 'variationally_sparse_gp', 40 | # 'variationally_sparse_gp_minibatch', 41 | # 'deep_gp_doubly_stochastic', 42 | 'svm', 43 | # 'naive_bayes', 44 | # 'knn', 45 | # 'decision_tree', 46 | # 'random_forest', 47 | # 'gradient_boosting_machine', 48 | # 'adaboost', 49 | # 'mlp', 50 | ] 51 | 52 | class ConvertToNamespace(object): 53 | def __init__(self, adict): 54 | adict.update({'seed':0, 55 | 'database_path':''}) 56 | self.__dict__.update(adict) 57 | 58 | def check_needs_run(table, d): 59 | with Database() as db: 60 | try: 61 | return (len(db.read(table, ['test_loglik'], d.__dict__)) == 0) 62 | except: 63 | return True 64 | 65 | 66 | @pytest.mark.parametrize('model', all_regression_models) 67 | @pytest.mark.parametrize('dataset', regression_datasets) 68 | @pytest.mark.parametrize('split', range(10)) 69 | def test_run_all_regression(model, dataset, split): 70 | d = ConvertToNamespace({'dataset':dataset, 71 | 'model' : model, 72 | 'split' : split}) 73 | 74 | if check_needs_run('regression', d): 75 | run_regression(d, is_test=False) 76 | 77 | 78 | 79 | @pytest.mark.parametrize('dataset', classification_datasets) 80 | @pytest.mark.parametrize('model', all_classification_models) 81 | @pytest.mark.parametrize('split', range(10)) 82 | def test_classification(model, dataset, split): 83 | d = {'dataset':dataset, 84 | 'model' : model, 85 | 'split' : split} 86 | if check_needs_run('classification', d): 87 | run_classification(ConvertToNamespace(d), is_test=False) 88 | 89 | 90 | 91 | # @pytest.mark.parametrize('dataset', ['iris', 'planning']) # binary and multiclass 92 | # @pytest.mark.parametrize('model', all_regression_models) 93 | # def test_active_learning_discrete(model, dataset): 94 | # d = {'dataset':dataset, 95 | # 'model' : model, 96 | # 'iterations': 2, 97 | # 'num_initial_points': 10} 98 | # 99 | # run_AL_disc(ConvertToNamespace(d), is_test=True) 100 | 101 | 102 | # @pytest.mark.parametrize('model', all_regression_models) 103 | # @pytest.mark.parametrize('dataset', regression_datasets) 104 | # @pytest.mark.parametrize('split', range(10)) 105 | # def test_active_learning_continuous(model): 106 | # d = {'dataset':'boston', 107 | # 'model' : model, 108 | # 'iterations': 2, 109 | # 'num_initial_points': 10} 110 | # 111 | # run_AL_cont(ConvertToNamespace(d), is_test=True) 112 | 113 | # 114 | # @pytest.mark.parametrize('model', all_regression_models) 115 | # @pytest.mark.parametrize('pca_dim', [0, 2]) 116 | # def test_mmd(model, pca_dim): 117 | # d = {'dataset':'boston', 118 | # 'model' : model, 119 | # 'num_samples' : 2, 120 | # 'pca_dim' : pca_dim} 121 | # 122 | # run_mmd(ConvertToNamespace(d), is_test=True) 123 | -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/active_learning_continuous.py: -------------------------------------------------------------------------------- 1 | """ 2 | Active learning for continuous data, using the max variance criterion to select new points 3 | 4 | """ 5 | 6 | import sys 7 | sys.path.append('../') 8 | 9 | import argparse 10 | import numpy as np 11 | from scipy.stats import norm 12 | from importlib import import_module 13 | 14 | from bayesian_benchmarks.data import get_regression_data 15 | from bayesian_benchmarks.database_utils import Database 16 | from bayesian_benchmarks.models.non_bayesian_models import non_bayesian_model 17 | 18 | def parse_args(): # pragma: no cover 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("--model", default='variationally_sparse_gp', nargs='?', type=str) 21 | parser.add_argument("--dataset", default='boston', nargs='?', type=str) 22 | parser.add_argument("--split", default=0, nargs='?', type=int) 23 | parser.add_argument("--seed", default=0, nargs='?', type=int) 24 | parser.add_argument("--iterations", default=10, nargs='?', type=int) 25 | parser.add_argument("--num_initial_points", default=3, nargs='?', type=int) 26 | return parser.parse_args() 27 | 28 | def run(ARGS, is_test): 29 | data = get_regression_data(ARGS.dataset, split=ARGS.split, prop=1.) 30 | 31 | ind = np.zeros(data.X_train.shape[0]).astype(bool) 32 | ind[:ARGS.num_initial_points] = True 33 | 34 | X, Y = data.X_train, data.Y_train 35 | 36 | Model = non_bayesian_model(ARGS.model, 'regression') or\ 37 | import_module('bayesian_benchmarks.models.{}.models'.format(ARGS.model)).RegressionModel 38 | model = Model(is_test=is_test, seed=ARGS.seed) 39 | 40 | test_ll = [] 41 | train_ll = [] 42 | all_ll = [] 43 | test_rmse = [] 44 | train_rmse = [] 45 | all_rmse = [] 46 | 47 | for _ in range(min(ARGS.iterations, X.shape[0] - ARGS.num_initial_points)): 48 | model.fit(X[ind], Y[ind]) 49 | 50 | m, v = model.predict(X) # ND 51 | 52 | vars = v.copy() 53 | 54 | # set the seen ones to -inf so we don't choose them 55 | vars[ind] = -np.inf 56 | 57 | # choose the highest variance point 58 | i = np.argmax(vars) 59 | ind[i] = True 60 | 61 | logp = norm.logpdf(Y, loc=m, scale=v**0.5) # N 62 | d2 = (Y - m)**2 63 | 64 | test_ll.append(np.average(logp[np.invert(ind)])) 65 | train_ll.append(np.average(logp[ind])) 66 | all_ll.append(np.average(logp)) 67 | test_rmse.append(np.average(d2[np.invert(ind)])**0.5) 68 | train_rmse.append(np.average(d2[ind])**0.5) 69 | all_rmse.append(np.average(d2)**0.5) 70 | 71 | 72 | # save 73 | res = {'test_loglik':np.array(test_ll), 74 | 'train_loglik':np.array(train_ll), 75 | 'total_loglik':np.array(all_ll), 76 | 'test_rmse':np.array(test_rmse), 77 | 'train_rmse':np.array(train_rmse), 78 | 'total_rmse':np.array(all_rmse), 79 | } 80 | res.update(ARGS.__dict__) 81 | 82 | if not is_test: # pragma: no cover 83 | with Database() as db: 84 | db.write('active_learning_continuous', res) 85 | 86 | if __name__ == '__main__': 87 | run(parse_args()) 88 | -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/active_learning_discrete.py: -------------------------------------------------------------------------------- 1 | """ 2 | Active learning with labels, using the max entropy criterion to select new points 3 | 4 | """ 5 | 6 | import sys 7 | sys.path.append('../') 8 | 9 | import argparse 10 | import numpy as np 11 | from scipy.stats import multinomial 12 | from importlib import import_module 13 | 14 | from bayesian_benchmarks.data import get_classification_data 15 | from bayesian_benchmarks.database_utils import Database 16 | from bayesian_benchmarks.models.get_model import get_classification_model 17 | 18 | def parse_args(): # pragma: no cover 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("--model", default='variationally_sparse_gp', nargs='?', type=str) 21 | parser.add_argument("--dataset", default='statlog-landsat', nargs='?', type=str) 22 | parser.add_argument("--split", default=0, nargs='?', type=int) 23 | parser.add_argument("--seed", default=0, nargs='?', type=int) 24 | parser.add_argument("--iterations", default=10, nargs='?', type=int) 25 | parser.add_argument("--num_initial_points", default=3, nargs='?', type=int) 26 | parser.add_argument("--database_path", default='', nargs='?', type=str) 27 | return parser.parse_args() 28 | 29 | def run(ARGS, is_test=False): 30 | data = get_classification_data(ARGS.dataset, split=ARGS.split, prop=1.) 31 | 32 | ind = np.zeros(data.X_train.shape[0]).astype(bool) 33 | ind[:ARGS.num_initial_points] = True 34 | 35 | X, Y = data.X_train, data.Y_train 36 | 37 | def onehot(Y, K): 38 | return np.eye(K)[Y.flatten().astype(int)].reshape(Y.shape[:-1]+(K,)) 39 | 40 | Y_oh = onehot(Y, data.K) 41 | 42 | Model = get_classification_model(ARGS.model) 43 | model = Model(data.K, is_test=is_test, seed=ARGS.seed) 44 | 45 | test_ll = [] 46 | train_ll = [] 47 | all_ll = [] 48 | test_acc = [] 49 | train_acc = [] 50 | all_acc = [] 51 | 52 | for _ in range(min(ARGS.iterations, X.shape[0] - ARGS.num_initial_points)): 53 | model.fit(X[ind], Y[ind]) 54 | 55 | p = model.predict(X) # NK 56 | # clip very large and small probs 57 | eps = 1e-12 58 | p = np.clip(p, eps, 1 - eps) 59 | p = p / np.expand_dims(np.sum(p, -1), -1) 60 | 61 | # entropy of predictions at all points 62 | ent = multinomial.entropy(n=1, p=p) 63 | 64 | # set the seen ones to -inf so we don't choose them 65 | ent[ind] = -np.inf 66 | 67 | # choose the highest entropy point to see next 68 | i = np.argmax(ent) 69 | ind[i] = True 70 | 71 | logp = multinomial.logpmf(Y_oh, n=1, p=p) # N 72 | is_correct = (np.argmax(p, 1) == Y.flatten()) # N 73 | 74 | test_ll.append(np.average(logp[np.invert(ind)])) 75 | train_ll.append(np.average(logp[ind])) 76 | all_ll.append(np.average(logp)) 77 | test_acc.append(np.average(is_correct[np.invert(ind)])) 78 | train_acc.append(np.average(is_correct[ind])) 79 | all_acc.append(np.average(is_correct)) 80 | 81 | res = {'test_loglik':np.array(test_ll), 82 | 'train_loglik':np.array(train_ll), 83 | 'total_loglik':np.array(all_ll), 84 | 'test_acc':np.array(test_acc), 85 | 'train_acc':np.array(train_acc), 86 | 'total_acc':np.array(all_acc), 87 | } 88 | res.update(ARGS.__dict__) 89 | 90 | if not is_test: # pragma: no cover 91 | with Database(ARGS.database_path) as db: 92 | db.write('active_learning_discrete', res) 93 | 94 | if __name__ == '__main__': 95 | run(parse_args()) -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/adversarial.py: -------------------------------------------------------------------------------- 1 | #TODO -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/classification.py: -------------------------------------------------------------------------------- 1 | """ 2 | A classification task, which can be either binary or multiclass. 3 | 4 | Metrics reported are test loglikelihood, classification accuracy. Also the predictions are stored for 5 | analysis of calibration etc. 6 | 7 | """ 8 | 9 | import sys 10 | sys.path.append('../') 11 | 12 | import argparse 13 | import numpy as np 14 | 15 | from scipy.stats import multinomial 16 | 17 | from bayesian_benchmarks.data import get_classification_data 18 | from bayesian_benchmarks.models.get_model import get_classification_model 19 | from bayesian_benchmarks.database_utils import Database 20 | from bayesian_benchmarks.tasks.utils import meanlogsumexp 21 | 22 | def parse_args(): # pragma: no cover 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument("--model", default='variationally_sparse_gp', nargs='?', type=str) 25 | parser.add_argument("--dataset", default='statlog-german-credit', nargs='?', type=str) 26 | parser.add_argument("--split", default=0, nargs='?', type=int) 27 | parser.add_argument("--seed", default=0, nargs='?', type=int) 28 | parser.add_argument("--database_path", default='', nargs='?', type=str) 29 | return parser.parse_args() 30 | 31 | def run(ARGS, data=None, model=None, is_test=False): 32 | 33 | data = data or get_classification_data(ARGS.dataset, split=ARGS.split) 34 | model = model or get_classification_model(ARGS.model)(data.K, is_test=is_test, seed=ARGS.seed) 35 | 36 | def onehot(Y, K): 37 | return np.eye(K)[Y.flatten().astype(int)].reshape(Y.shape[:-1] + (K,)) 38 | 39 | Y_oh = onehot(data.Y_test, data.K)[None, :, :] # [1 x N_test x K] 40 | 41 | model.fit(data.X_train, data.Y_train) 42 | p = model.predict(data.X_test) # [N_test x K] or [samples x N_test x K] 43 | 44 | assert p.ndim in {2, 3} # 3-dim in case of approximate predictions (multiple samples per each X) 45 | 46 | # clip very large and small probs 47 | eps = 1e-12 48 | p = np.clip(p, eps, 1 - eps) 49 | p = p / np.expand_dims(np.sum(p, -1), -1) 50 | 51 | assert np.all(p >= 0.0) and np.all(p <= 1.0) 52 | 53 | # evaluation metrics 54 | res = {} 55 | 56 | if p.ndim == 2: # keep analysis as in the original code in case 2-dim predictions 57 | 58 | logp = multinomial.logpmf(Y_oh, n=1, p=p) # [N_test] 59 | 60 | res['test_loglik'] = np.average(logp) 61 | 62 | pred = np.argmax(p, axis=-1) 63 | 64 | else: # compute metrics in case of 3-dim predictions 65 | 66 | res['test_loglik'] = [] 67 | 68 | for n in range(p.shape[0]): # iterate through samples 69 | logp = multinomial.logpmf(Y_oh, n=1, p=p[n]) # [N_test] 70 | res['test_loglik'].append(logp) 71 | 72 | # Mixture test likelihood (mean over per data point evaluations) 73 | res['test_loglik'] = meanlogsumexp(res['test_loglik']) 74 | 75 | p = np.mean(p, axis=0) 76 | pred = np.argmax(p, axis=-1) 77 | 78 | res['test_acc'] = np.average(np.array(pred == data.Y_test.flatten()).astype(float)) 79 | 80 | if not is_test: 81 | res.update(ARGS.__dict__) 82 | 83 | if not is_test: # pragma: no cover 84 | with Database(ARGS.database_path) as db: 85 | db.write('classification', res) 86 | 87 | return res 88 | 89 | 90 | if __name__ == '__main__': 91 | run(parse_args()) -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/conditional_density_estimation.py: -------------------------------------------------------------------------------- 1 | # import sys 2 | # sys.path.append('../') 3 | # 4 | # import argparse 5 | # import numpy as np 6 | # from importlib import import_module 7 | # 8 | # parser = argparse.ArgumentParser() 9 | # 10 | # parser.add_argument("--model", default='linear', nargs='?', type=str) 11 | # parser.add_argument("--dataset", default='boston', nargs='?', type=str) 12 | # parser.add_argument("--split", default=0, nargs='?', type=int) 13 | # parser.add_argument("--levels", default=1000, nargs='?', type=int) 14 | # 15 | # ARGS = parser.parse_args() 16 | # 17 | # from bayesian_benchmarks.data import ALL_REGRESSION_DATATSETS 18 | # data = ALL_REGRESSION_DATATSETS[ARGS.dataset]() 19 | # 20 | # run_path = '../models/{}/models.py'.format(ARGS.model) 21 | # 22 | # Y_test = data.Y_test.flatten() # N_test, 23 | # levels = np.linspace(np.min(Y_test), np.max(Y_test), ARGS.levels) 24 | # 25 | # lower_ind = np.argmin(Y_test[None, :] > levels[:, None], 0) - 1 26 | # interp = [] 27 | # for y, l in zip(Y_test, lower_ind): 28 | # interp.append((y - levels[l])/(levels[l+1] - levels[l])) 29 | # interp = np.array(interp) 30 | # 31 | # models = import_module('models.{}.models'.format(ARGS.model)) 32 | # 33 | # model = models.DensityEstimationModel() 34 | # model.fit(data.X_train, data.Y_train) 35 | # 36 | # logps = model.predict(data.X_test, levels) 37 | # 38 | # 39 | # logp = interp[:, None] * logps[lower_ind, :, 0] + (1-interp)[:, None] * logps[lower_ind + 1, :, :] 40 | # 41 | # print(logp.shape) 42 | # # test_lik = np.average(l) 43 | # # 44 | # # 45 | # # res = {} 46 | # # res['test_likelihood'] = test_lik 47 | # # 48 | -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/learned_synthetic.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import tensorflow as tf 4 | import time 5 | tfd = tf.contrib.distributions 6 | tfb = tfd.bijectors 7 | layers = tf.contrib.layers 8 | 9 | from bayesian_benchmarks.data import NYTaxi 10 | data = NYTaxi() 11 | 12 | DTYPE = tf.float32 13 | NP_DTYPE = np.float32 14 | 15 | # quite easy to interpret - multiplying by alpha causes a contraction in volume. 16 | class LeakyReLU(tfb.Bijector): 17 | def __init__(self, alpha=0.5, validate_args=False, name="leaky_relu"): 18 | super(LeakyReLU, self).__init__( 19 | event_ndims=1, validate_args=validate_args, name=name) 20 | self.alpha = alpha 21 | 22 | def _forward(self, x): 23 | return tf.where(tf.greater_equal(x, 0), x, self.alpha * x) 24 | 25 | def _inverse(self, y): 26 | return tf.where(tf.greater_equal(y, 0), y, 1. / self.alpha * y) 27 | 28 | def _inverse_log_det_jacobian(self, y): 29 | event_dims = self._event_dims_tensor(y) 30 | I = tf.ones_like(y) 31 | J_inv = tf.where(tf.greater_equal(y, 0), I, 1.0 / self.alpha * I) 32 | # abs is actually redundant here, since this det Jacobian is > 0 33 | log_abs_det_J_inv = tf.log(tf.abs(J_inv)) 34 | return tf.reduce_sum(log_abs_det_J_inv, axis=event_dims) 35 | 36 | 37 | def net(x, out_size): 38 | return layers.stack(x, layers.fully_connected, [512, 512, out_size]) 39 | 40 | 41 | class NVPCoupling(tfb.Bijector): 42 | """ 43 | NVP affine coupling layer for 2D units. 44 | """ 45 | def __init__(self, D, d, layer_id=0, validate_args=False, name="NVPCoupling"): 46 | """ 47 | Args: 48 | d: First d units are pass-thru units. 49 | """ 50 | # first d numbers decide scaling/shift factor for remaining D-d numbers. 51 | super(NVPCoupling, self).__init__( 52 | event_ndims=1, validate_args=validate_args, name=name) 53 | self.D, self.d = D, d 54 | self.id = layer_id 55 | # create variables here 56 | tmp = tf.placeholder(dtype=DTYPE, shape=[1, self.d]) 57 | self.s(tmp) 58 | self.t(tmp) 59 | 60 | def s(self, xd): 61 | with tf.variable_scope('s%d' % self.id, reuse=tf.AUTO_REUSE): 62 | return net(xd, self.D - self.d) 63 | 64 | def t(self, xd): 65 | with tf.variable_scope('t%d' % self.id, reuse=tf.AUTO_REUSE): 66 | return net(xd, self.D - self.d) 67 | 68 | def _forward(self, x): 69 | xd, xD = x[:, :self.d], x[:, self.d:] 70 | yD = xD * tf.exp(self.s(xd)) + self.t(xd) # [batch, D-d] 71 | return tf.concat([xd, yD], axis=1) 72 | 73 | def _inverse(self, y): 74 | yd, yD = y[:, :self.d], y[:, self.d:] 75 | xD = (yD - self.t(yd)) * tf.exp(-self.s(yd)) 76 | return tf.concat([yd, xD], axis=1) 77 | 78 | def _forward_log_det_jacobian(self, x): 79 | event_dims = self._event_dims_tensor(x) 80 | xd = x[:, :self.d] 81 | return tf.reduce_sum(self.s(xd), axis=event_dims) 82 | 83 | 84 | class BatchNorm(tfb.Bijector): 85 | def __init__(self, eps=1e-5, decay=0.95, validate_args=False, name="batch_norm"): 86 | super(BatchNorm, self).__init__( 87 | event_ndims=1, validate_args=validate_args, name=name) 88 | self._vars_created = False 89 | self.eps = eps 90 | self.decay = decay 91 | 92 | def _create_vars(self, x): 93 | n = x.get_shape().as_list()[1] 94 | with tf.variable_scope(self.name): 95 | self.beta = tf.get_variable('beta', [1, n], dtype=DTYPE) 96 | self.gamma = tf.get_variable('gamma', [1, n], dtype=DTYPE) 97 | self.train_m = tf.get_variable( 98 | 'mean', [1, n], dtype=DTYPE, trainable=False) 99 | self.train_v = tf.get_variable( 100 | 'var', [1, n], dtype=DTYPE, initializer=tf.ones_initializer, trainable=False) 101 | self._vars_created = True 102 | 103 | def _forward(self, u): 104 | if not self._vars_created: 105 | self._create_vars(u) 106 | return (u - self.beta) * tf.exp(-self.gamma) * tf.sqrt(self.train_v + self.eps) + self.train_m 107 | 108 | def _inverse(self, x): 109 | # Eq 22. Called during training of a normalizing flow. 110 | if not self._vars_created: 111 | self._create_vars(x) 112 | # statistics of current minibatch 113 | m, v = tf.nn.moments(x, axes=[0], keep_dims=True) 114 | # update train statistics via exponential moving average 115 | update_train_m = tf.assign_sub( 116 | self.train_m, self.decay * (self.train_m - m)) 117 | update_train_v = tf.assign_sub( 118 | self.train_v, self.decay * (self.train_v - v)) 119 | # normalize using current minibatch statistics, followed by BN scale and shift 120 | with tf.control_dependencies([update_train_m, update_train_v]): 121 | return (x - m) * 1. / tf.sqrt(v + self.eps) * tf.exp(self.gamma) + self.beta 122 | 123 | def _inverse_log_det_jacobian(self, x): 124 | # at training time, the log_det_jacobian is computed from statistics of the 125 | # current minibatch. 126 | if not self._vars_created: 127 | self._create_vars(x) 128 | _, v = tf.nn.moments(x, axes=[0], keep_dims=True) 129 | abs_log_det_J_inv = tf.reduce_sum( 130 | self.gamma - .5 * tf.log(v + self.eps)) 131 | return abs_log_det_J_inv 132 | 133 | class Model: 134 | def __init__(self, X, batchsize=1000): 135 | data = tf.data.Dataset.from_tensor_slices(tf.identity(X)) 136 | data = data.repeat() 137 | data_batch = data.batch(batch_size=batchsize) 138 | 139 | self._iterator_tensor = data_batch.make_initializable_iterator() 140 | self.X_train = self._iterator_tensor.get_next() 141 | self.X_test = tf.placeholder(DTYPE, [None, 2]) 142 | 143 | flow = self.make_flow() 144 | base_dist = tfd.MultivariateNormalDiag(loc=tf.zeros([2], DTYPE)) 145 | self.dist = tfd.TransformedDistribution(distribution=base_dist, bijector=flow) 146 | 147 | x = base_dist.sample(512) 148 | samples = [x] 149 | names = [base_dist.name] 150 | for bijector in reversed(self.dist.bijector.bijectors): 151 | x = bijector.forward(x) 152 | samples.append(x) 153 | names.append(bijector.name) 154 | self.names = names 155 | self.samples = samples 156 | 157 | self.logp_train = self.dist.log_prob(self.X_train) 158 | self.logp_test = self.dist.log_prob(self.X_test) 159 | self.loss = -tf.reduce_mean(self.logp_train) 160 | 161 | self.train_op = tf.train.AdamOptimizer(1e-3).minimize(self.loss) 162 | 163 | def make_flow(self): 164 | d, r = 2, 2 165 | bijectors = [] 166 | num_layers = 6 167 | for i in range(num_layers): 168 | with tf.variable_scope('bijector_%d' % i): 169 | V = tf.get_variable('V', [d, r], dtype=DTYPE) # factor loading 170 | shift = tf.get_variable('shift', [d], dtype=DTYPE) # affine shift 171 | L = tf.get_variable('L', [d * (d + 1) / 2], 172 | dtype=DTYPE) # lower triangular 173 | bijectors.append(tfb.Affine( 174 | scale_tril=tfd.fill_triangular(L), 175 | scale_perturb_factor=V, 176 | shift=shift, 177 | )) 178 | alpha = tf.abs(tf.get_variable('alpha', [], dtype=DTYPE)) + .01 179 | bijectors.append(LeakyReLU(alpha=alpha)) 180 | # Last layer is affine. Note that tfb.Chain takes a list of bijectors in the *reverse* order 181 | # that they are applied.. 182 | mlp_bijector = tfb.Chain( 183 | list(reversed(bijectors[:-1])), name='2d_mlp_bijector') 184 | return mlp_bijector 185 | 186 | def plot_samples(self, sess): 187 | samples = sess.run(model.samples) 188 | f, arr = plt.subplots(1, len(samples), figsize=(4 * (len(samples)), 4)) 189 | X0 = samples[0] 190 | 191 | 192 | arr[-1].scatter(data.X_train[::100, 0], data.X_train[::100, 1], 193 | marker='.', color='yellow', alpha=0.3) 194 | 195 | for i in range(len(samples)): 196 | X1 = samples[i] 197 | idx = np.logical_and(X0[:, 0] < 0, X0[:, 1] < 0) 198 | arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='red') 199 | idx = np.logical_and(X0[:, 0] > 0, X0[:, 1] < 0) 200 | arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='green') 201 | idx = np.logical_and(X0[:, 0] < 0, X0[:, 1] > 0) 202 | arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='blue') 203 | idx = np.logical_and(X0[:, 0] > 0, X0[:, 1] > 0) 204 | arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='black') 205 | # arr[i].set_xlim([-2, 2]) 206 | # arr[i].set_ylim([-2, 2]) 207 | arr[i].set_title(self.names[i]) 208 | 209 | plt.show() 210 | 211 | 212 | USE_BATCHNORM = True 213 | num_bijectors = 8 214 | 215 | class FancyModel(Model): 216 | def __init__(self, model_type, *args, **kwargs): 217 | self.model_type = model_type 218 | Model.__init__(self, *args, **kwargs) 219 | 220 | def make_flow(self): 221 | bijectors = [] 222 | 223 | for i in range(num_bijectors): 224 | if self.model_type == 'NVP': 225 | bijectors.append(NVPCoupling(D=2, d=1, layer_id=i)) 226 | 227 | elif self.model_type == 'MAF': 228 | bijectors.append(tfb.MaskedAutoregressiveFlow( 229 | shift_and_log_scale_fn=tfb.masked_autoregressive_default_template( 230 | hidden_layers=[512, 512]))) 231 | 232 | elif self.model_type == 'IAF': 233 | bijectors.append(tfb.Invert(tfb.MaskedAutoregressiveFlow( 234 | shift_and_log_scale_fn=tfb.masked_autoregressive_default_template( 235 | hidden_layers=[512, 512])))) 236 | 237 | if USE_BATCHNORM and i % 2 == 0: 238 | # BatchNorm helps to stabilize deep normalizing flows, esp. Real-NVP 239 | bijectors.append(BatchNorm(name='batch_norm%d' % i)) 240 | 241 | bijectors.append(tfb.Permute(permutation=[1, 0])) 242 | 243 | # Discard the last Permute layer. 244 | return tfb.Chain(list(reversed(bijectors[:-1]))) 245 | 246 | 247 | 248 | 249 | 250 | sess = tf.Session() 251 | model = FancyModel('IAF', data.X_train[:, :2].astype(np.float32), 252 | batchsize=5000) 253 | sess.run(tf.global_variables_initializer()) 254 | sess.run(model._iterator_tensor.initializer) 255 | 256 | L = [] 257 | plot_freq = 100 258 | 259 | 260 | 261 | t = time.time() 262 | 263 | try: 264 | for it in range(int(1e4)): 265 | _, loss = sess.run([model.train_op, model.loss])#, {model.X_train : X}) 266 | L.append(loss) 267 | if it % plot_freq == 0: 268 | print('{} {}'.format(it, loss)) 269 | except KeyboardInterrupt: 270 | pass 271 | 272 | print('train time {:.4f}s'.format(time.time() - t)) 273 | 274 | # plt.plot(L) 275 | # plt.show() 276 | 277 | model.plot_samples(sess) 278 | 279 | 280 | # plt.axes().set_xlim(-74.1, -73.7) 281 | # plt.axes().set_ylim(40.6, 40.9) 282 | 283 | 284 | 285 | # x2_dist = tfd.Normal(loc=0., scale=4.) 286 | # x2_samples = x2_dist.sample(batch_size) 287 | # x1 = tfd.Normal(loc=.25 * tf.square(x2_samples), 288 | # scale=tf.ones(batch_size, dtype=tf.float32)) 289 | # x1_samples = x1.sample() 290 | # x_samples = tf.stack([x1_samples, x2_samples], axis=1) 291 | # 292 | # 293 | # 294 | # x_placeholder = tf.placeholder(tf.float32, [None, 2]) 295 | # logp = dist.log_prob(x_placeholder) 296 | # 297 | # 298 | 299 | # 300 | # 301 | # # results = sess.run(samples) 302 | 303 | # # 304 | # 305 | # 306 | # NUM_STEPS = int(3e5) 307 | # global_step = [] 308 | # np_losses = [] 309 | # for i in range(NUM_STEPS): 310 | # _, np_loss = sess.run([train_op, loss]) 311 | # if i % 1000 == 0: 312 | # global_step.append(i) 313 | # np_losses.append(np_loss) 314 | # if i % int(1e4) == 0: 315 | # print(i, np_loss) 316 | # start = 10 317 | # plt.plot(np_losses[start:]) 318 | # plt.show() 319 | # 320 | # 321 | # # results = sess.run(samples) 322 | # # f, arr = plt.subplots(1, len(results), figsize=(4 * (len(results)), 4)) 323 | # # X0 = results[0] 324 | # # for i in range(len(results)): 325 | # # X1 = results[i] 326 | # # idx = np.logical_and(X0[:, 0] < 0, X0[:, 1] < 0) 327 | # # arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='red') 328 | # # idx = np.logical_and(X0[:, 0] > 0, X0[:, 1] < 0) 329 | # # arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='green') 330 | # # idx = np.logical_and(X0[:, 0] < 0, X0[:, 1] > 0) 331 | # # arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='blue') 332 | # # idx = np.logical_and(X0[:, 0] > 0, X0[:, 1] > 0) 333 | # # arr[i].scatter(X1[idx, 0], X1[idx, 1], s=10, color='black') 334 | # # # arr[i].set_xlim([-2, 2]) 335 | # # # arr[i].set_ylim([-2, 2]) 336 | # # arr[i].set_title(names[i]) 337 | # # plt.show() 338 | # 339 | # 340 | # X1 = sess.run(dist.sample(4000)) 341 | # plt.scatter(X1[:, 0], X1[:, 1], color='red', s=2) 342 | # # arr[i].set_ylim([-.5, .5]) 343 | # plt.show() 344 | # 345 | # 346 | # # x_samples_np = sess.run([x1_samples, x2_samples]) 347 | # # plt.scatter(*x_samples_np) 348 | # # plt.show() 349 | # 350 | # N = 100 351 | # l = np.linspace(-5, 5, N) 352 | # X_grid = np.array([x.flatten() for x in np.meshgrid(l, l)]).T 353 | # 354 | # 355 | # lp = sess.run(logp, {x_placeholder:X_grid}) 356 | # 357 | # plt.pcolor(np.exp(lp).reshape(N, N)) 358 | # plt.colorbar() 359 | # plt.show() 360 | # 361 | # print(np.sum(np.exp(lp))) 362 | # 363 | -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/mmd.py: -------------------------------------------------------------------------------- 1 | """ 2 | A conditional Gaussian estimation task: model p(y_n|x_n) = N(a(x_n), b(x_n)) 3 | 4 | Metrics reported are test log likelihood, mean squared error, and absolute error, all for normalized and unnormalized y. 5 | 6 | """ 7 | 8 | import argparse 9 | import numpy as np 10 | from sklearn.decomposition.pca import PCA 11 | 12 | from bayesian_benchmarks.database_utils import Database 13 | from bayesian_benchmarks.models.get_model import get_regression_model 14 | from bayesian_benchmarks.data import get_regression_data 15 | 16 | import gpflow 17 | 18 | def parse_args(): #pragma: no cover 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("--model", default='variationally_sparse_gp', nargs='?', type=str) 21 | parser.add_argument("--dataset", default='yacht', nargs='?', type=str) 22 | parser.add_argument("--split", default=0, nargs='?', type=int) 23 | parser.add_argument("--seed", default=0, nargs='?', type=int) 24 | parser.add_argument("--num_samples", default=100, nargs='?', type=int) 25 | parser.add_argument("--pca_dim", default=2, type=int) 26 | parser.add_argument("--database_path", default='', nargs='?', type=str) 27 | return parser.parse_args() 28 | 29 | 30 | def mmd(A, B, kernel): 31 | KAA = kernel.K(A, A) 32 | KAB = kernel.K(A, B) 33 | KBB = kernel.K(B, B) 34 | 35 | n = float(A.shape[0]) 36 | m = float(B.shape[0]) 37 | 38 | return np.sum(KAA)/m/m - 2*np.sum(KAB)/m/n + np.sum(KBB)/n/n 39 | 40 | def run(ARGS, data=None, model=None, is_test=False): 41 | data = data or get_regression_data(ARGS.dataset, split=ARGS.split) 42 | model = model or get_regression_model(ARGS.model)(is_test=is_test, seed=ARGS.seed) 43 | 44 | model.fit(data.X_train, data.Y_train) 45 | 46 | res = {} 47 | 48 | samples = model.sample(data.X_test, ARGS.num_samples) 49 | data_tiled = np.tile(data.X_test[None, :, :], [ARGS.num_samples, 1, 1]) 50 | shape = [ARGS.num_samples * data.X_test.shape[0], data.X_test.shape[1] + data.Y_test.shape[1]] 51 | A = np.reshape(np.concatenate([data_tiled, samples], -1), shape) 52 | B = np.concatenate([data.X_test, data.Y_test], -1) 53 | 54 | 55 | if ARGS.pca_dim > 0: 56 | AB = np.concatenate([A, B], 0) 57 | pca = PCA(n_components=ARGS.pca_dim).fit(AB) 58 | A = pca.transform(A) 59 | B = pca.transform(B) 60 | 61 | # import matplotlib.pyplot as plt 62 | # plt.scatter(A[:, 0], A[:, 1], color='b') 63 | # plt.scatter(B[:, 0], B[:, 1], color='r') 64 | # plt.show() 65 | 66 | kernel = gpflow.kernels.RBF(A.shape[-1]) 67 | res['mmd'] = mmd(A, B, kernel) 68 | 69 | print(res) 70 | 71 | res.update(ARGS.__dict__) 72 | if not is_test: # prgama: no cover 73 | with Database(ARGS.database_path) as db: 74 | db.write('mmd', res) 75 | 76 | 77 | if __name__ == '__main__': 78 | run(parse_args()) -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/regression.py: -------------------------------------------------------------------------------- 1 | """ 2 | A conditional Gaussian estimation task: model p(y_n|x_n) = N(a(x_n), b(x_n)) 3 | 4 | Metrics reported are test log likelihood, mean squared error, and absolute error, all for normalized and unnormalized y. 5 | 6 | """ 7 | 8 | import argparse 9 | import numpy as np 10 | from scipy.stats import norm 11 | 12 | from bayesian_benchmarks.data import get_regression_data 13 | from bayesian_benchmarks.database_utils import Database 14 | from bayesian_benchmarks.models.get_model import get_regression_model 15 | from bayesian_benchmarks.tasks.utils import meanlogsumexp 16 | 17 | def parse_args(): # pragma: no cover 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument("--model", default='linear', nargs='?', type=str) 20 | parser.add_argument("--dataset", default='energy', nargs='?', type=str) 21 | parser.add_argument("--split", default=0, nargs='?', type=int) 22 | parser.add_argument("--seed", default=0, nargs='?', type=int) 23 | parser.add_argument("--database_path", default='', nargs='?', type=str) 24 | return parser.parse_args() 25 | 26 | def run(ARGS, data=None, model=None, is_test=False): 27 | 28 | data = data or get_regression_data(ARGS.dataset, split=ARGS.split) 29 | model = model or get_regression_model(ARGS.model)(is_test=is_test, seed=ARGS.seed) 30 | 31 | model.fit(data.X_train, data.Y_train) 32 | m, v = model.predict(data.X_test) # both [data points x output dim] or [samples x data points x output dim] 33 | 34 | assert m.ndim == v.ndim 35 | assert m.ndim in {2, 3} # 3-dim in case of approximate predictions (multiple samples per each X) 36 | assert np.all(v >= 0.0) 37 | 38 | res = {} 39 | log_eps = np.log(1e-12) # log probability threshold 40 | log_1_minus_eps = np.log(1.0 - 1e-12) 41 | 42 | if m.ndim == 2: # keep analysis as in the original code in case of 2-dim predictions 43 | 44 | l = norm.logpdf(data.Y_test, loc=m, scale=v ** 0.5) # [] 45 | l = np.clip(l, log_eps, log_1_minus_eps) # clip 46 | res['test_loglik'] = np.average(l) 47 | 48 | lu = norm.logpdf(data.Y_test * data.Y_std, loc=m * data.Y_std, scale=(v ** 0.5) * data.Y_std) 49 | lu = np.clip(lu, log_eps, log_1_minus_eps) # clip 50 | res['test_loglik_unnormalized'] = np.average(lu) 51 | 52 | d = data.Y_test - m 53 | du = d * data.Y_std 54 | 55 | res['test_mae'] = np.average(np.abs(d)) 56 | res['test_mae_unnormalized'] = np.average(np.abs(du)) 57 | 58 | res['test_rmse'] = np.average(d ** 2) ** 0.5 59 | res['test_rmse_unnormalized'] = np.average(du ** 2) ** 0.5 60 | 61 | else: # compute metrics in case of 3-dim predictions 62 | 63 | res['test_loglik'] = [] 64 | res['test_loglik_unnormalized'] = [] 65 | 66 | for n in range(m.shape[0]): # iterate through samples 67 | l = norm.logpdf(data.Y_test, loc=m[n], scale=v[n] ** 0.5) 68 | l = np.clip(l, log_eps, log_1_minus_eps) # clip 69 | res['test_loglik'].append(l) 70 | 71 | lu = norm.logpdf(data.Y_test * data.Y_std, loc=m[n] * data.Y_std, scale=(v[n] ** 0.5) * data.Y_std) 72 | lu = np.clip(lu, log_eps, log_1_minus_eps) # clip 73 | res['test_loglik_unnormalized'].append(lu) 74 | 75 | # Mixture test likelihood (mean over per data point evaluations) 76 | res['test_loglik'] = meanlogsumexp(res['test_loglik']) 77 | 78 | # Mixture test likelihood (mean over per data point evaluations) 79 | res['test_loglik_unnormalized'] = meanlogsumexp(res['test_loglik_unnormalized']) 80 | 81 | d = data.Y_test - np.mean(m, axis=0) 82 | du = d * data.Y_std 83 | 84 | res['test_mae'] = np.average(np.abs(d)) 85 | res['test_mae_unnormalized'] = np.average(np.abs(du)) 86 | 87 | res['test_rmse'] = np.average(d ** 2) ** 0.5 88 | res['test_rmse_unnormalized'] = np.average(du ** 2) ** 0.5 89 | 90 | if not is_test: 91 | res.update(ARGS.__dict__) 92 | 93 | if not is_test: # pragma: no cover 94 | with Database(ARGS.database_path) as db: 95 | db.write('regression', res) 96 | 97 | return res 98 | 99 | 100 | if __name__ == '__main__': 101 | run(parse_args()) -------------------------------------------------------------------------------- /bayesian_benchmarks/tasks/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Helper functions... 3 | 4 | """ 5 | 6 | from typing import List, Union 7 | 8 | import numpy as np 9 | from scipy.special import logsumexp 10 | 11 | def meanlogsumexp(logps: Union[List[np.ndarray], np.ndarray], axis: int=0) -> np.ndarray: 12 | """ 13 | Mean log sum exp of a log p list. 14 | :param logps: list of log probs [samples x data points] or [samples x data points x output dim] 15 | :param axis: determines reduction 16 | :return: avg probability value (empty shape) 17 | """ 18 | logps = np.asarray(logps) 19 | assert axis < logps.ndim 20 | return np.mean(logsumexp(logps, axis=axis) - np.log(logps.shape[axis])) 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | requirements = [ 4 | 'numpy>=1.10.0', 5 | 'scipy>=0.18.0', 6 | 'xlrd>=1.1.0', 7 | 'pandas>=0.23', 8 | 'pytest>=4.4.0', 9 | ] 10 | 11 | setup(name='bayesian_benchmarks', 12 | version='alpha', 13 | author="Hugh Salimbeni", 14 | author_email="hrs13@ic.ac.uk", 15 | description=("Bayesian benchmarking"), 16 | license="Apache License 2.0", 17 | keywords="machine-learning bayesian-methods", 18 | url="https://github.com/hughsalimbeni/bayesian_benchmarks", 19 | python_requires=">=3.6", 20 | packages=find_packages(include=["bayesian_benchmarks", 21 | "bayesian_benchmarks.*"]), 22 | install_requires=requirements, 23 | package_data={"":["bayesian_benchmarksrc"]}, 24 | include_package_data=True, 25 | classifiers=[ 26 | 'License :: OSI Approved :: Apache Software License', 27 | 'Natural Language :: English', 28 | 'Operating System :: MacOS :: MacOS X', 29 | 'Operating System :: Microsoft :: Windows', 30 | 'Operating System :: POSIX :: Linux', 31 | 'Programming Language :: Python :: 3.6', 32 | 'Topic :: Scientific/Engineering :: Artificial Intelligence' 33 | ]) 34 | -------------------------------------------------------------------------------- /tests/test_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Hugh Salimbeni 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from numpy.testing import assert_almost_equal 18 | import numpy as np 19 | 20 | from bayesian_benchmarks.data import regression_datasets, get_regression_data, get_classification_data 21 | 22 | @pytest.mark.parametrize('d', ['boston']) 23 | def test_regression(d): 24 | data = get_regression_data(d) 25 | 26 | assert_almost_equal(np.average(np.concatenate([data.X_train, data.X_test], 0), 0), 27 | np.zeros(data.X_train.shape[1])) 28 | 29 | assert_almost_equal(np.std(np.concatenate([data.X_train, data.X_test], 0), 0), 30 | np.ones(data.X_train.shape[1]), 31 | decimal=3) 32 | 33 | assert_almost_equal(np.average(np.concatenate([data.Y_train, data.Y_test], 0), 0), 34 | np.zeros(data.Y_train.shape[1])) 35 | 36 | assert_almost_equal(np.std(np.concatenate([data.Y_train, data.Y_test], 0), 0), 37 | np.ones(data.Y_train.shape[1]), 38 | decimal=3) 39 | 40 | assert data.X_train.shape[0] == data.Y_train.shape[0] 41 | assert data.X_test.shape[0] == data.Y_test.shape[0] 42 | assert data.X_train.shape[1] == data.X_test.shape[1] 43 | assert data.Y_train.shape[1] == data.Y_test.shape[1] 44 | 45 | @pytest.mark.parametrize('d', ['iris', 'thyroid']) 46 | def test_classification(d): 47 | data = get_classification_data(d) 48 | 49 | assert_almost_equal(np.average(np.concatenate([data.X_train, data.X_test], 0), 0), 50 | np.zeros(data.X_train.shape[1])) 51 | 52 | assert_almost_equal(np.std(np.concatenate([data.X_train, data.X_test], 0), 0), 53 | np.ones(data.X_train.shape[1]), 54 | decimal=3) 55 | 56 | K = len(list(set(np.concatenate([data.Y_train, data.Y_test], 0).astype(int).flatten()))) 57 | 58 | assert K == data.K 59 | 60 | assert data.X_train.shape[0] == data.Y_train.shape[0] 61 | assert data.X_test.shape[0] == data.Y_test.shape[0] 62 | assert data.X_train.shape[1] == data.X_test.shape[1] 63 | assert data.Y_train.shape[1] == data.Y_test.shape[1] 64 | -------------------------------------------------------------------------------- /tests/test_database.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Hugh Salimbeni 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | 17 | import numpy as np 18 | import os 19 | 20 | from bayesian_benchmarks.database_utils import Database 21 | 22 | 23 | class TestDatabase(unittest.TestCase): 24 | def setUp(self): 25 | self.data1 = {'a': '1', 'b': 1, 'c': 1., 'd': np.ones(1), 'e':'data1'} 26 | self.data2 = {'a': '2', 'b': 2, 'c': 2., 'd': 2*np.ones(1), 'e':'data2'} 27 | 28 | self.tmp_path = 'test.db' 29 | with Database(self.tmp_path) as db: 30 | db.write('test', self.data2) 31 | 32 | with Database(self.tmp_path) as db: 33 | db.write('test', self.data1) 34 | 35 | def tearDown(self): 36 | os.remove(self.tmp_path) 37 | 38 | def test_read(self): 39 | fields = ['b', 'c', 'd', 'e'] 40 | with Database(self.tmp_path) as db: 41 | results1 = db.read('test', fields, {'a':'1'}) 42 | results2 = db.read('test', fields, {'a':'2'}) 43 | 44 | for k, r1, r2 in zip(fields, results1[0], results2[0]): 45 | assert r1 == self.data1[k] 46 | assert r2 == self.data2[k] 47 | 48 | def test_delete(self): 49 | d = {'a': '3', 'b': 3, 'c': 3., 'd': 3 * np.ones(1), 'e': 'data3'} 50 | 51 | with Database(self.tmp_path) as db: 52 | db.write('test', d) 53 | 54 | with Database(self.tmp_path) as db: 55 | assert len(db.read('test', ['b'], {'a':3})) == 1 56 | 57 | with Database(self.tmp_path) as db: 58 | db.delete('test', {'a':'3'}) 59 | 60 | with Database(self.tmp_path) as db: 61 | assert len(db.read('test', ['b'], {'a':3})) == 0 62 | 63 | 64 | if __name__ == '__main__': 65 | unittest.main() 66 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Hugh Salimbeni 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | import numpy as np 18 | from numpy.testing import assert_allclose 19 | 20 | from bayesian_benchmarks.models.template import RegressionModel as RegressionTemplate 21 | from bayesian_benchmarks.models.template import ClassificationModel as ClassificationTemplate 22 | 23 | from bayesian_benchmarks.models.get_model import all_regression_models, all_classification_models 24 | from bayesian_benchmarks.models.get_model import get_regression_model, get_classification_model 25 | 26 | @pytest.mark.parametrize('name', all_regression_models) 27 | @pytest.mark.parametrize('N', [100, 6000]) 28 | def test_models_regression(name, N): 29 | S, Ns, D = 5, 3, 2 30 | 31 | model = get_regression_model(name)(is_test=True) 32 | model.fit(np.random.randn(N, D), np.random.randn(N, 1)) 33 | model.fit(np.random.randn(N, D), np.random.randn(N, 1)) 34 | m, v = model.predict(np.random.randn(Ns, D)) 35 | assert m.shape == (Ns, 1) 36 | assert v.shape == (Ns, 1) 37 | 38 | samples = model.sample(np.random.randn(Ns, D), S) 39 | assert samples.shape == (S, Ns, 1) 40 | 41 | 42 | @pytest.mark.parametrize('name', all_classification_models) 43 | @pytest.mark.parametrize('K', [2, 3]) 44 | @pytest.mark.parametrize('N', [100, 6000]) 45 | def test_models_classification(name, K, N): 46 | S, Ns, D = 4, 3, 2 47 | 48 | model = get_classification_model(name)(K, is_test=True) 49 | # make sure the multiclass can cope with a missing class in the train data 50 | model.fit(np.random.randn(N, D), np.random.choice(range(2), size=(N, 1)).astype(float)) 51 | model.fit(np.random.randn(N, D), np.random.choice(range(K), size=(N, 1)).astype(float)) 52 | p = model.predict(np.random.randn(Ns, D)) 53 | assert p.shape == (Ns, K) 54 | assert_allclose(np.sum(p, 1), np.ones(Ns), atol=0.001) 55 | 56 | 57 | def test_templates(): 58 | regression_model = RegressionTemplate() 59 | classification_model = ClassificationTemplate(2) 60 | 61 | regression_model.fit(np.ones((1, 1)), np.ones((1, 1))) 62 | classification_model.fit(np.ones((1, 1)), np.ones((1, 1))) 63 | 64 | with pytest.raises(NotImplementedError): 65 | regression_model.predict(np.ones((1, 1))) 66 | 67 | with pytest.raises(NotImplementedError): 68 | classification_model.predict(np.ones((1, 1))) 69 | 70 | with pytest.raises(NotImplementedError): 71 | regression_model.sample(np.ones((1, 1)), 1) 72 | -------------------------------------------------------------------------------- /tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Hugh Salimbeni 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import numpy as np 17 | 18 | from bayesian_benchmarks.tasks.regression import run as run_regression 19 | from bayesian_benchmarks.tasks.classification import run as run_classification 20 | from bayesian_benchmarks.tasks.active_learning_continuous import run as run_AL_cont 21 | from bayesian_benchmarks.tasks.active_learning_discrete import run as run_AL_disc 22 | from bayesian_benchmarks.tasks.mmd import run as run_mmd 23 | 24 | # only test these 25 | all_regression_models = ['linear'] 26 | all_classification_models = ['linear'] 27 | 28 | 29 | class ConvertToNamespace(object): 30 | def __init__(self, adict): 31 | adict.update({'seed':0, 'split':0}) 32 | self.__dict__.update(adict) 33 | 34 | @pytest.mark.parametrize('model', all_regression_models) 35 | def test_regression(model): 36 | d = {'dataset':'boston', 37 | 'model' : model} 38 | 39 | run_regression(ConvertToNamespace(d), is_test=True) 40 | 41 | 42 | @pytest.mark.parametrize('model', all_regression_models) 43 | def test_active_learning_continuous(model): 44 | d = {'dataset':'boston', 45 | 'model' : model, 46 | 'iterations': 2, 47 | 'num_initial_points': 10} 48 | 49 | run_AL_cont(ConvertToNamespace(d), is_test=True) 50 | 51 | 52 | @pytest.mark.parametrize('model', all_regression_models) 53 | @pytest.mark.parametrize('pca_dim', [0, 2]) 54 | def test_mmd(model, pca_dim): 55 | d = {'dataset':'boston', 56 | 'model' : model, 57 | 'num_samples' : 2, 58 | 'pca_dim' : pca_dim} 59 | 60 | run_mmd(ConvertToNamespace(d), is_test=True) 61 | 62 | 63 | @pytest.mark.parametrize('dataset', ['iris', 'planning']) # binary and multiclass 64 | @pytest.mark.parametrize('model', all_classification_models) 65 | def test_classification(model, dataset): 66 | d = {'dataset':dataset, 67 | 'model' : model} 68 | 69 | run_classification(ConvertToNamespace(d), is_test=True) 70 | 71 | 72 | @pytest.mark.parametrize('dataset', ['iris', 'planning']) # binary and multiclass 73 | @pytest.mark.parametrize('model', all_regression_models) 74 | def test_active_learning_discrete(model, dataset): 75 | d = {'dataset':dataset, 76 | 'model' : model, 77 | 'iterations': 2, 78 | 'num_initial_points': 10} 79 | 80 | run_AL_disc(ConvertToNamespace(d), is_test=True) 81 | 82 | 83 | # Testing modified regression and classification runs with mocks 84 | 85 | class RegressionMock(object): 86 | """ 87 | Regression mock. 88 | """ 89 | def fit(self, X: np.ndarray, Y:np.ndarray) -> None: 90 | pass 91 | def predict(self, X: np.ndarray) -> (np.ndarray, np.ndarray): 92 | mu = np.array([[1., 2., 3.], [4., 5., 6.]]) # [data points x output dim] 93 | var = np.array([[.1, .2, .3], [.4, .5, .6]]) # [data points x output dim] 94 | return mu, var 95 | 96 | class ApproximateRegressionMock(RegressionMock): 97 | """ 98 | Approximate regression mock. 99 | """ 100 | def predict(self, X: np.ndarray) -> (np.ndarray, np.ndarray): 101 | mu = np.array([[[1., 2., 3.], [4., 5., 6.]], 102 | [[1.5, 2.5, 3.5], [4.5, 5.5, 6.5]]]) # [samples x data points x output dim] 103 | var = np.array([[[.1, .2, .3], [.4, .5, .6]], 104 | [[.1, .2, .3], [.4, .5, .6]]]) # [samples x data points x output dim] 105 | return mu, var 106 | 107 | class ClassificationMock(object): 108 | """ 109 | Classification mock. 110 | """ 111 | def fit(self, X: np.ndarray, Y:np.ndarray) -> None: 112 | pass 113 | def predict(self, X: np.ndarray) -> np.ndarray: 114 | p = np.array([[.1, .2, .7], [.6, .3, .1]]) # [data points x output dim] 115 | return p 116 | 117 | class ApproximateClassificationMock(ClassificationMock): 118 | """ 119 | Approximate classification mock. 120 | """ 121 | def predict(self, X: np.ndarray) -> np.ndarray: 122 | p = np.array([[[.1, .2, .7], [.6, .3, .1]], 123 | [[.01, .02, .97], [.64, .05, .31]]]) # [samples x data points x output dim] 124 | return p 125 | 126 | class RegressionDataMock(object): 127 | """ 128 | Regression data mock. 129 | """ 130 | X_train, Y_train = np.empty(shape=()), np.empty(shape=()) 131 | X_test, Y_test = np.empty(shape=()), np.array([[1., 1., 1.], [1., 1., 1.]]) # [data points x output dim] 132 | Y_std = 2. # Y_test must be compatible with regression mocks... 133 | 134 | class ClassificationDataMock(object): 135 | """ 136 | Classification data mock. 137 | """ 138 | X_train, Y_train = np.empty(shape=()), np.empty(shape=()) 139 | X_test, Y_test = np.empty(shape=()), np.array([[0], [1]]) # [data points x output dim] 140 | K = 3 # Y_test and K must be compatible with classification mocks... 141 | 142 | # Below correct results computed by hand... 143 | 144 | regression_results = {} 145 | regression_results['test_loglik'] = -9.8963 146 | regression_results['test_loglik_unnormalized'] = -10.5507 147 | regression_results['test_mae'] = 2.5 148 | regression_results['test_mae_unnormalized'] = 5.0 149 | regression_results['test_rmse'] = 3.0277 150 | regression_results['test_rmse_unnormalized'] = 6.0553 151 | 152 | approximate_regression_results = {} 153 | approximate_regression_results['test_loglik'] = -10.5197 154 | approximate_regression_results['test_loglik_unnormalized'] = -11.1836 155 | approximate_regression_results['test_mae'] = 2.75 156 | approximate_regression_results['test_mae_unnormalized'] = 5.5 157 | approximate_regression_results['test_rmse'] = 3.2372 158 | approximate_regression_results['test_rmse_unnormalized'] = 6.4743 159 | 160 | classification_results = {} 161 | classification_results['test_loglik'] = -1.7533 162 | classification_results['test_acc'] = 0.0 163 | 164 | approximate_classification_results = {} 165 | approximate_classification_results['test_loglik'] = -2.3217 166 | approximate_classification_results['test_acc'] = 0.0 167 | 168 | # Below two tests, one for regression and one for classification (2-dim and 3-dim case respectively) 169 | 170 | regression_tuple = (RegressionDataMock(), RegressionMock(), regression_results) 171 | approx_regression_tuple = (RegressionDataMock(), ApproximateRegressionMock(), approximate_regression_results) 172 | 173 | @pytest.mark.parametrize('tuple', [regression_tuple, approx_regression_tuple]) 174 | def test_regression(tuple): 175 | data, model, correct_result = tuple 176 | result = run_regression(None, data=data, model=model, is_test=True) 177 | 178 | evaluation_metrics = {'test_loglik', 'test_loglik_unnormalized', 'test_mae', 'test_mae_unnormalized', 179 | 'test_rmse', 'test_rmse_unnormalized'} 180 | for evaluation_metric in evaluation_metrics: 181 | np.testing.assert_almost_equal(correct_result[evaluation_metric], result[evaluation_metric], decimal=3) 182 | 183 | classification_tuple = (ClassificationDataMock(), ClassificationMock(), classification_results) 184 | approx_classification_tuple = (ClassificationDataMock(), ApproximateClassificationMock(), approximate_classification_results) 185 | 186 | @pytest.mark.parametrize('tuple', [classification_tuple, approx_classification_tuple]) 187 | def test_classification(tuple): 188 | data, model, correct_result = tuple 189 | result = run_classification(None, data=data, model=model, is_test=True) 190 | 191 | evaluation_metrics = {'test_loglik', 'test_acc'} 192 | for evaluation_metric in evaluation_metrics: 193 | np.testing.assert_almost_equal(correct_result[evaluation_metric], result[evaluation_metric], decimal=3) 194 | --------------------------------------------------------------------------------