├── tests ├── __init__.py └── integration │ ├── __init__.py │ ├── test_demo.py │ └── test_notebook.py ├── CODEOWNERS ├── .gitignore ├── data ├── README.rst └── mldata │ └── mnist-original.mat ├── .gitattributes ├── images └── boosting.jpg ├── requirements.txt ├── qboost ├── __init__.py └── qboost.py ├── README.rst ├── demo.py ├── LICENSE └── demo.ipynb /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @yanboxue -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | env/* 2 | .idea/* 3 | *.pyc -------------------------------------------------------------------------------- /data/README.rst: -------------------------------------------------------------------------------- 1 | save your data to this folder -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | data/mldata/mnist-original.mat filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /images/boosting.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dwavesystems/qboost/HEAD/images/boosting.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==2.1.2 2 | numpy==1.14.2 3 | pandas==0.19.2 4 | scipy==1.0.0 5 | scikit-learn==0.19.1 6 | dwave-system==0.3.2 7 | jupyter==1.0.0 8 | -------------------------------------------------------------------------------- /data/mldata/mnist-original.mat: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:59b5e6181102902314137ac772a1ec7e9870d582dc9cf8da15e9f77a2ecd6ee2 3 | size 55440440 4 | -------------------------------------------------------------------------------- /qboost/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 D-Wave Systems Inc. 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License") 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http: // www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from __future__ import absolute_import 15 | 16 | 17 | from qboost.qboost import * 18 | -------------------------------------------------------------------------------- /tests/integration/test_demo.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import sys 4 | 5 | from dwave.system.samplers import DWaveSampler 6 | 7 | try: 8 | DWaveSampler() 9 | _config_found = True 10 | except ValueError: 11 | _config_found = False 12 | 13 | # /tests/integration/test_demo.py 14 | PY2 = sys.version_info.major == 2 15 | project_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 16 | 17 | 18 | @unittest.skipUnless(_config_found, "no configuration found for accessing the D-Wave System") 19 | class TestDemo(unittest.TestCase): 20 | def test_smoke(self): 21 | """run demo.py and check that nothing crashes""" 22 | 23 | demo_file = os.path.join(project_dir, 'demo.py') 24 | 25 | # assume qboost is on the path 26 | exit_status = os.system('python {} --mnist --wisc'.format(demo_file)) 27 | 28 | assert isinstance(exit_status, int) 29 | self.assertFalse(exit_status, "running demo.py returned an exit status != 0") 30 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | **WARNING**: This repository is obsolete. The Qboost demo is now maintained in 2 | `dwave-examples/qboost `_. 3 | 4 | Demo of Qboost 5 | ============== 6 | 7 | The D-Wave quantum computer has been widely studied as a discrete optimization engine that accepts 8 | any problem formulated as quadratic unconstrained binary optimization (QUBO). In 2008, Google and 9 | D-Wave published a paper, `Training a Binary Classifier with the Quantum Adiabatic Algorithm`_\ , 10 | which describes how the `Qboost` ensemble method makes binary classification amenable to quantum 11 | computing: the problem is formulated as a thresholded linear superposition of a set of weak classifiers 12 | and the D-Wave quantum computer is used to optimize the weights in a learning process that 13 | strives to minimize the training error and number of weak classifiers 14 | 15 | This code demonstrates the use of the D-Wave system to solve a binary classification problem using the 16 | Qboost algorithm. 17 | 18 | Disclaimer 19 | ---------- 20 | 21 | This demo and its code are intended for demonstrative purposes only and are not 22 | designed for performance. 23 | 24 | For state-of-the-art machine learning, please contact Quadrant, the 25 | machine learning business unit of D-Wave Systems. 26 | 27 | Setting Up the Demo 28 | ---------------------- 29 | 30 | It's recommended that you work in a `virtual environment`_ on your local machine; depending on 31 | your machine, you may need to first install virtualenv and then create a virtual environment, 32 | for example: 33 | 34 | .. code-block:: bash 35 | 36 | virtualenv env 37 | cd env 38 | source ./bin/activate 39 | 40 | Copy (clone) this Qboost repository to your local machine's newly created virtual environment. 41 | 42 | To set up the required dependencies, in the root directory of a copy (clone) of this repository, run the following: 43 | 44 | .. code-block:: bash 45 | 46 | pip install -r requirements.txt 47 | 48 | Configuring the Demo 49 | -------------------- 50 | 51 | Access to a D-Wave system must be configured, as described in the `dwave-cloud-client`_ documentation. 52 | A default solver is required. 53 | 54 | Running the Demo 55 | ---------------- 56 | 57 | A minimal working example using the main interface function can be seen by running: 58 | 59 | .. code-block:: bash 60 | 61 | python demo.py --wisc --mnist 62 | 63 | License 64 | ------- 65 | 66 | Released under the Apache License 2.0. See LICENSE file. 67 | 68 | .. _`dwave-cloud-client`: http://dwave-cloud-client.readthedocs.io/en/latest/#module-dwave.cloud.config 69 | .. _`Training a Binary Classifier with the Quantum Adiabatic Algorithm`: https://arxiv.org/pdf/0811.0416.pdf 70 | .. _`virtual environment`: https://packaging.python.org/guides/installing-using-pip-and-virtualenv 71 | -------------------------------------------------------------------------------- /tests/integration/test_notebook.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import nbformat 4 | from nbconvert.preprocessors import ExecutePreprocessor 5 | 6 | from dwave.system.samplers import DWaveSampler 7 | 8 | try: 9 | DWaveSampler() 10 | _config_found = True 11 | except ValueError: 12 | _config_found = False 13 | 14 | 15 | def run_notebook(path): 16 | with open(path) as notebook_file: 17 | print('Running notebook {}'.format(path)) 18 | nb = nbformat.read(notebook_file, nbformat.current_nbformat) 19 | ep = ExecutePreprocessor(timeout=600, kernel_name='python2', allow_errors=True) 20 | ep.preprocess(nb, {'metadata': {'path': os.path.dirname(path)}}) 21 | 22 | return nb 23 | 24 | 25 | def _modify_grid_search_cell(nb): 26 | grid_search_line = 'from helpers import grid_search' 27 | for cell in nb.cells: 28 | source = cell.source.split('\n') 29 | if source[0] == grid_search_line: 30 | new_source = [] 31 | en_scales_var = 'energy_scales' 32 | offset_mag_var = 'offset_magnitudes' 33 | modified_en = False 34 | modified_offset = False 35 | for line in source: 36 | if len(line) >= len(en_scales_var) and line[:len(en_scales_var)] == en_scales_var: 37 | new_source.append('{} = [1.]'.format(en_scales_var)) 38 | modified_en = True 39 | elif len(line) >= len(offset_mag_var) and line[:len(offset_mag_var)] == offset_mag_var: 40 | new_source.append('{} = [.1]'.format(offset_mag_var)) 41 | modified_offset = True 42 | else: 43 | new_source.append(line) 44 | 45 | cell.source = '\n'.join(new_source) 46 | assert modified_en and modified_offset, "Failed to modify grid_search cell. Can't continue test" 47 | 48 | return nb 49 | 50 | 51 | def extract_errors(notebook): 52 | errors = [] 53 | for i, cell in enumerate(notebook.cells): 54 | for output in cell.get('outputs', []): 55 | if output.output_type == "error": 56 | errors.append((i, output)) 57 | 58 | return errors 59 | 60 | 61 | class TestNotebook(unittest.TestCase): 62 | 63 | @classmethod 64 | def setUpClass(cls): 65 | print('Running modified notebook...') 66 | notebook_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'demo.ipynb') 67 | with open(notebook_path) as notebook_file: 68 | cls.notebook = nbformat.read(notebook_file, nbformat.current_nbformat) 69 | cls.notebook = _modify_grid_search_cell(cls.notebook) 70 | ep = ExecutePreprocessor(timeout=600, kernel_name='python2', allow_errors=True) 71 | ep.preprocess(cls.notebook, {'metadata': {'path': os.path.dirname(notebook_path)}}) 72 | 73 | def test_no_errors(self): 74 | self.assertListEqual(extract_errors(self.notebook), []) 75 | 76 | 77 | if __name__ == '__main__': 78 | unittest.main() 79 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 D-Wave Systems Inc. 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License") 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http: // www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from __future__ import print_function, division 17 | 18 | import os 19 | import sys 20 | 21 | import numpy as np 22 | import matplotlib.pyplot as plt 23 | 24 | from sklearn import preprocessing, metrics 25 | from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier 26 | from sklearn.datasets.mldata import fetch_mldata 27 | from sklearn.datasets import load_breast_cancer 28 | from dwave.system.samplers import DWaveSampler 29 | from dwave.system.composites import EmbeddingComposite 30 | 31 | from qboost import WeakClassifiers, QBoostClassifier, QboostPlus 32 | 33 | 34 | def metric(y, y_pred): 35 | 36 | return metrics.accuracy_score(y, y_pred) 37 | 38 | 39 | def train_model(X_train, y_train, X_test, y_test, lmd): 40 | """ 41 | Train qboost model 42 | 43 | :param X_train: train input 44 | :param y_train: train label 45 | :param X_test: test input 46 | :param y_test: test label 47 | :param lmd: lmbda to control regularization term 48 | :return: 49 | """ 50 | NUM_READS = 3000 51 | NUM_WEAK_CLASSIFIERS = 35 52 | # lmd = 0.5 53 | TREE_DEPTH = 3 54 | 55 | # define sampler 56 | dwave_sampler = DWaveSampler() 57 | # sa_sampler = micro.dimod.SimulatedAnnealingSampler() 58 | emb_sampler = EmbeddingComposite(dwave_sampler) 59 | 60 | N_train = len(X_train) 61 | N_test = len(X_test) 62 | 63 | print("\n======================================") 64 | print("Train#: %d, Test: %d" %(N_train, N_test)) 65 | print('Num weak classifiers:', NUM_WEAK_CLASSIFIERS) 66 | print('Tree depth:', TREE_DEPTH) 67 | 68 | 69 | # input: dataset X and labels y (in {+1, -1} 70 | 71 | # Preprocessing data 72 | imputer = preprocessing.Imputer() 73 | # scaler = preprocessing.MinMaxScaler() 74 | scaler = preprocessing.StandardScaler() 75 | normalizer = preprocessing.Normalizer() 76 | centerer = preprocessing.KernelCenterer() 77 | 78 | 79 | # X = imputer.fit_transform(X) 80 | X_train = scaler.fit_transform(X_train) 81 | X_train = normalizer.fit_transform(X_train) 82 | X_train = centerer.fit_transform(X_train) 83 | 84 | # X_test = imputer.fit_transform(X_test) 85 | X_test = scaler.fit_transform(X_test) 86 | X_test = normalizer.fit_transform(X_test) 87 | X_test = centerer.fit_transform(X_test) 88 | 89 | 90 | ## Adaboost 91 | print('\nAdaboost') 92 | 93 | clf = AdaBoostClassifier(n_estimators=NUM_WEAK_CLASSIFIERS) 94 | 95 | # scores = cross_val_score(clf, X, y, cv=5, scoring='accuracy') 96 | print('fitting...') 97 | clf.fit(X_train, y_train) 98 | 99 | hypotheses_ada = clf.estimators_ 100 | # clf.estimator_weights_ = np.random.uniform(0,1,size=NUM_WEAK_CLASSIFIERS) 101 | print('testing...') 102 | y_train_pred = clf.predict(X_train) 103 | y_test_pred = clf.predict(X_test) 104 | 105 | print('accu (train): %5.2f'%(metric(y_train, y_train_pred))) 106 | print('accu (test): %5.2f'%(metric(y_test, y_test_pred))) 107 | 108 | # Ensembles of Decision Tree 109 | print('\nDecision tree') 110 | 111 | clf2 = WeakClassifiers(n_estimators=NUM_WEAK_CLASSIFIERS, max_depth=TREE_DEPTH) 112 | clf2.fit(X_train, y_train) 113 | 114 | y_train_pred2 = clf2.predict(X_train) 115 | y_test_pred2 = clf2.predict(X_test) 116 | print(clf2.estimator_weights) 117 | 118 | print('accu (train): %5.2f' % (metric(y_train, y_train_pred2))) 119 | print('accu (test): %5.2f' % (metric(y_test, y_test_pred2))) 120 | 121 | # Ensembles of Decision Tree 122 | print('\nQBoost') 123 | 124 | DW_PARAMS = {'num_reads': NUM_READS, 125 | 'auto_scale': True, 126 | # "answer_mode": "histogram", 127 | 'num_spin_reversal_transforms': 10, 128 | # 'annealing_time': 10, 129 | 'postprocess': 'optimization', 130 | } 131 | 132 | clf3 = QBoostClassifier(n_estimators=NUM_WEAK_CLASSIFIERS, max_depth=TREE_DEPTH) 133 | clf3.fit(X_train, y_train, emb_sampler, lmd=lmd, **DW_PARAMS) 134 | 135 | y_train_dw = clf3.predict(X_train) 136 | y_test_dw = clf3.predict(X_test) 137 | 138 | print(clf3.estimator_weights) 139 | 140 | print('accu (train): %5.2f' % (metric(y_train, y_train_dw))) 141 | print('accu (test): %5.2f' % (metric(y_test, y_test_dw))) 142 | 143 | 144 | # Ensembles of Decision Tree 145 | print('\nQBoostPlus') 146 | clf4 = QboostPlus([clf, clf2, clf3]) 147 | clf4.fit(X_train, y_train, emb_sampler, lmd=lmd, **DW_PARAMS) 148 | y_train4 = clf4.predict(X_train) 149 | y_test4 = clf4.predict(X_test) 150 | print(clf4.estimator_weights) 151 | 152 | print('accu (train): %5.2f' % (metric(y_train, y_train4))) 153 | print('accu (test): %5.2f' % (metric(y_test, y_test4))) 154 | 155 | 156 | print("=============================================") 157 | print("Method \t Adaboost \t DecisionTree \t Qboost \t QboostIt") 158 | print("Train\t %5.2f \t\t %5.2f \t\t\t %5.2f \t\t %5.2f"% (metric(y_train, y_train_pred), 159 | metric(y_train, y_train_pred2), 160 | metric(y_train, y_train_dw), 161 | metric(y_train, y_train4))) 162 | print("Test\t %5.2f \t\t %5.2f \t\t\t %5.2f \t\t %5.2f"% (metric(y_test, y_test_pred), 163 | metric(y_test,y_test_pred2), 164 | metric(y_test, y_test_dw), 165 | metric(y_test, y_test4))) 166 | print("=============================================") 167 | 168 | # plt.subplot(211) 169 | # plt.bar(range(len(y_test)), y_test) 170 | # plt.subplot(212) 171 | # plt.bar(range(len(y_test)), y_test_dw) 172 | # plt.show() 173 | 174 | return 175 | 176 | 177 | if __name__ == '__main__': 178 | 179 | if '--mnist' in sys.argv: 180 | 181 | mnist = fetch_mldata('MNIST original', data_home='data') 182 | 183 | idx_01 = np.where(mnist.target <= 10)[0] 184 | 185 | np.random.shuffle(idx_01) 186 | idx_01 = idx_01[:5000] 187 | idx_train = idx_01[:2*len(idx_01)//3] 188 | idx_test = idx_01[2*len(idx_01)//3:] 189 | 190 | X_train = mnist.data[idx_train] 191 | X_test = mnist.data[idx_test] 192 | 193 | y_train = 2*(mnist.target[idx_train] <= 4) - 1 194 | y_test = 2*(mnist.target[idx_test] <= 4) - 1 195 | 196 | clfs = train_model(X_train, y_train, X_test, y_test, 1.0) 197 | 198 | if '--wisc' in sys.argv: 199 | 200 | wisc = load_breast_cancer() 201 | 202 | idx = np.arange(len(wisc.target)) 203 | np.random.shuffle(idx) 204 | 205 | # train on a random 2/3 and test on the remaining 1/3 206 | idx_train = idx[:2*len(idx)//3] 207 | idx_test = idx[2*len(idx)//3:] 208 | 209 | X_train = wisc.data[idx_train] 210 | X_test = wisc.data[idx_test] 211 | 212 | y_train = 2 * wisc.target[idx_train] - 1 # binary -> spin 213 | y_test = 2 * wisc.target[idx_test] - 1 214 | 215 | clfs = train_model(X_train, y_train, X_test, y_test, 1.0) 216 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /qboost/qboost.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 D-Wave Systems Inc. 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License") 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http: // www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor 17 | from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier, AdaBoostRegressor 18 | import numpy as np 19 | from copy import deepcopy 20 | 21 | def weight_penalty(prediction, y, percent = 0.1): 22 | """ 23 | For Regression we have to introduce a metric to penalize differences of the prediction from the label y. 24 | Percent gives the maximum deviation of the prediction from the label that is not penalized. 25 | """ 26 | diff = np.abs(prediction-y) 27 | min_ = diff.min() 28 | max_ = diff.max() 29 | norm = (diff-min_)/(max_-min_) 30 | norm = 1.0*(norm < percent) 31 | return norm 32 | 33 | class WeakClassifiers(object): 34 | """ 35 | Weak Classifiers based on DecisionTree 36 | """ 37 | 38 | def __init__(self, n_estimators=50, max_depth=3): 39 | self.n_estimators = n_estimators 40 | self.estimators_ = [] 41 | self.max_depth = max_depth 42 | self.__construct_wc() 43 | 44 | def __construct_wc(self): 45 | 46 | self.estimators_ = [DecisionTreeClassifier(max_depth=self.max_depth, 47 | random_state=np.random.randint(1000000,10000000)) 48 | for _ in range(self.n_estimators)] 49 | 50 | def fit(self, X, y): 51 | """ 52 | fit estimators 53 | :param X: 54 | :param y: 55 | :return: 56 | """ 57 | 58 | self.estimator_weights = np.zeros(self.n_estimators) 59 | 60 | d = np.ones(len(X)) / len(X) 61 | for i, h in enumerate(self.estimators_): 62 | h.fit(X, y, sample_weight=d) 63 | pred = h.predict(X) 64 | eps = d.dot(pred != y) 65 | if eps == 0: # to prevent divided by zero error 66 | eps = 1e-20 67 | w = (np.log(1 - eps) - np.log(eps)) / 2 68 | d = d * np.exp(- w * y * pred) 69 | d = d / d.sum() 70 | self.estimator_weights[i] = w 71 | 72 | def predict(self, X): 73 | """ 74 | predict label of X 75 | :param X: 76 | :return: 77 | """ 78 | 79 | if not hasattr(self, 'estimator_weights'): 80 | raise Exception('Not Fitted Error!') 81 | 82 | y = np.zeros(len(X)) 83 | 84 | for (h, w) in zip(self.estimators_, self.estimator_weights): 85 | y += w * h.predict(X) 86 | 87 | y = np.sign(y) 88 | 89 | return y 90 | 91 | def copy(self): 92 | 93 | classifier = WeakClassifiers(n_estimators=self.n_estimators, max_depth=self.max_depth) 94 | classifier.estimators_ = deepcopy(self.estimators_) 95 | if hasattr(self, 'estimator_weights'): 96 | classifier.estimator_weights = np.array(self.estimator_weights) 97 | 98 | return classifier 99 | 100 | 101 | class QBoostClassifier(WeakClassifiers): 102 | """ 103 | Qboost Classifier 104 | """ 105 | def __init__(self, n_estimators=50, max_depth=3): 106 | super(QBoostClassifier, self).__init__(n_estimators=n_estimators, 107 | max_depth=max_depth) 108 | 109 | def fit(self, X, y, sampler, lmd=0.2, **kwargs): 110 | 111 | n_data = len(X) 112 | 113 | # step 1: fit weak classifiers 114 | super(QBoostClassifier, self).fit(X, y) 115 | 116 | # step 2: create QUBO 117 | hij = [] 118 | for h in self.estimators_: 119 | hij.append(h.predict(X)) 120 | 121 | hij = np.array(hij) 122 | # scale hij to [-1/N, 1/N] 123 | hij = 1. * hij / self.n_estimators 124 | 125 | ## Create QUBO 126 | qii = n_data * 1. / (self.n_estimators ** 2) + lmd - 2 * np.dot(hij, y) 127 | qij = np.dot(hij, hij.T) 128 | Q = dict() 129 | Q.update(dict(((k, k), v) for (k, v) in enumerate(qii))) 130 | for i in range(self.n_estimators): 131 | for j in range(i + 1, self.n_estimators): 132 | Q[(i, j)] = qij[i, j] 133 | 134 | # step 3: optimize QUBO 135 | res = sampler.sample_qubo(Q, **kwargs) 136 | samples = np.array([[samp[k] for k in range(self.n_estimators)] for samp in res]) 137 | 138 | # take the optimal solution as estimator weights 139 | self.estimator_weights = samples[0] 140 | 141 | def predict(self, X): 142 | n_data = len(X) 143 | pred_all = np.array([h.predict(X) for h in self.estimators_]) 144 | temp1 = np.dot(self.estimator_weights, pred_all) 145 | T1 = np.sum(temp1, axis=0) / (n_data * self.n_estimators * 1.) 146 | y = np.sign(temp1 - T1) #binary classes are either 1 or -1 147 | 148 | return y 149 | 150 | 151 | class WeakRegressor(object): 152 | """ 153 | Weak Regressor based on DecisionTreeRegressor 154 | """ 155 | 156 | def __init__(self, n_estimators=50, max_depth=3, DT = True, Ada = False, ): 157 | self.n_estimators = n_estimators 158 | self.estimators_ = [] 159 | self.max_depth = max_depth 160 | self.__construct_wc() 161 | 162 | def __construct_wc(self): 163 | 164 | self.estimators_ = [DecisionTreeRegressor(max_depth=self.max_depth, 165 | random_state=np.random.randint(1000000,10000000)) 166 | for _ in range(self.n_estimators)] 167 | # self.estimators_ = [AdaBoostRegressor(random_state=np.random.randint(1000000,10000000)) 168 | # for _ in range(self.n_estimators)] 169 | 170 | def fit(self, X, y): 171 | """ 172 | fit estimators 173 | :param X: 174 | :param y: 175 | :return: 176 | """ 177 | 178 | self.estimator_weights = np.zeros(self.n_estimators) #initialize all estimator weights to zero 179 | 180 | d = np.ones(len(X)) / len(X) 181 | for i, h in enumerate(self.estimators_): #fit all estimators 182 | h.fit(X, y, sample_weight=d) 183 | pred = h.predict(X) 184 | # For classification one simply compares (pred != y) 185 | # For regression we have to define another metric 186 | norm = weight_penalty(pred, y) 187 | eps = d.dot(norm) 188 | if eps == 0: # to prevent divided by zero error 189 | eps = 1e-20 190 | w = (np.log(1 - eps) - np.log(eps)) / 2 191 | d = d * np.exp(- w * y * pred) 192 | d = d / d.sum() 193 | self.estimator_weights[i] = w 194 | 195 | def predict(self, X): 196 | """ 197 | predict label of X 198 | :param X: 199 | :return: 200 | """ 201 | 202 | if not hasattr(self, 'estimator_weights'): 203 | raise Exception('Not Fitted Error!') 204 | 205 | y = np.zeros(len(X)) 206 | 207 | for (h, w) in zip(self.estimators_, self.estimator_weights): 208 | y += w * h.predict(X) 209 | 210 | y = np.sign(y) 211 | 212 | return y 213 | 214 | def copy(self): 215 | 216 | classifier = WeakRegressor(n_estimators=self.n_estimators, max_depth=self.max_depth) 217 | classifier.estimators_ = deepcopy(self.estimators_) 218 | if hasattr(self, 'estimator_weights'): 219 | classifier.estimator_weights = np.array(self.estimator_weights) 220 | 221 | return classifier 222 | 223 | 224 | class QBoostRegressor(WeakRegressor): 225 | """ 226 | Qboost Regressor 227 | """ 228 | def __init__(self, n_estimators=50, max_depth=3): 229 | super(QBoostRegressor, self).__init__(n_estimators=n_estimators, 230 | max_depth=max_depth) 231 | self.Qu = 0.0 232 | self.hij = 0.0 233 | self.var1 = 0.0 234 | self.qij = 0.0 235 | 236 | def fit(self, X, y, sampler, lmd=0.2, **kwargs): 237 | 238 | n_data = len(X) 239 | 240 | # step 1: fit weak classifiers 241 | super(QBoostRegressor, self).fit(X, y) 242 | 243 | # step 2: create QUBO 244 | hij = [] 245 | for h in self.estimators_: 246 | hij.append(h.predict(X)) 247 | 248 | hij = np.array(hij) 249 | # scale hij to [-1/N, 1/N] 250 | hij = 1. * hij / self.n_estimators 251 | self.hij = hij 252 | ## Create QUBO 253 | qii = n_data * 1. / (self.n_estimators ** 2) + lmd - 2 * np.dot(hij, y) 254 | self.var1 = qii 255 | qij = np.dot(hij, hij.T) 256 | self.qij = qij 257 | Q = dict() 258 | Q.update(dict(((k, k), v) for (k, v) in enumerate(qii))) 259 | for i in range(self.n_estimators): 260 | for j in range(i + 1, self.n_estimators): 261 | Q[(i, j)] = qij[i, j] 262 | 263 | self.Qu = Q 264 | # step 3: optimize QUBO 265 | res = sampler.sample_qubo(Q, **kwargs) 266 | samples = np.array([[samp[k] for k in range(self.n_estimators)] for samp in res]) 267 | 268 | # take the optimal solution as estimator weights 269 | # self.estimator_weights = np.mean(samples, axis=0) 270 | self.estimator_weights = samples[0] 271 | 272 | def predict(self, X): 273 | n_data = len(X) 274 | pred_all = np.array([h.predict(X) for h in self.estimators_]) 275 | temp1 = np.dot(self.estimator_weights, pred_all) 276 | norm = np.sum(self.estimator_weights) 277 | if norm > 0: 278 | y = temp1 / norm 279 | else: 280 | y = temp1 281 | return y 282 | 283 | 284 | class QboostPlus(object): 285 | """ 286 | Only for Classifiers 287 | Quantum boost existing (weak) classifiers 288 | """ 289 | 290 | def __init__(self, weak_classifier_list): 291 | self.estimators_ = weak_classifier_list 292 | self.n_estimators = len(self.estimators_) 293 | self.estimator_weights = np.ones(self.n_estimators) #estimator weights will be binary (Dwave output) 294 | 295 | def fit(self, X, y, sampler, lmd=0.2, **kwargs): 296 | 297 | n_data = len(X) 298 | # step 1: create QUBO 299 | hij = [] 300 | for h in self.estimators_: 301 | hij.append(h.predict(X)) 302 | 303 | hij = np.array(hij) 304 | # scale hij to [-1/N, 1/N] 305 | hij = 1. * hij / self.n_estimators 306 | 307 | ## Create QUBO 308 | qii = n_data * 1. / (self.n_estimators ** 2) + lmd - 2 * np.dot(hij, y) 309 | qij = np.dot(hij, hij.T) 310 | Q = dict() 311 | Q.update(dict(((k, k), v) for (k, v) in enumerate(qii))) 312 | for i in range(self.n_estimators): 313 | for j in range(i + 1, self.n_estimators): 314 | Q[(i, j)] = qij[i, j] 315 | 316 | # step 3: optimize QUBO 317 | res = sampler.sample_qubo(Q, **kwargs) 318 | samples = np.array([[samp[k] for k in range(self.n_estimators)] for samp in res]) 319 | 320 | # take the optimal solution as estimator weights 321 | self.estimator_weights = samples[0] 322 | 323 | def predict(self, X): 324 | 325 | n_data = len(X) 326 | T = 0 327 | y = np.zeros(n_data) 328 | for i, h in enumerate(self.estimators_): 329 | y0 = self.estimator_weights[i] * h.predict(X) # prediction of weak classifier 330 | y += y0 331 | T += np.sum(y0) 332 | 333 | y = np.sign(y - T / (n_data*self.n_estimators)) 334 | 335 | return y 336 | 337 | class QboostPlusRegression(object): 338 | """ 339 | Quantum boost existing (weak) regressors 340 | """ 341 | 342 | def __init__(self, weak_Regressor_list): 343 | self.estimators_ = weak_Regressor_list 344 | self.n_estimators = len(self.estimators_) 345 | self.estimator_weights = np.ones(self.n_estimators) 346 | 347 | def fit(self, X, y, sampler, lmd=0.2, **kwargs): 348 | 349 | n_data = len(X) 350 | # step 1: create QUBO 351 | hij = [] 352 | for h in self.estimators_: 353 | hij.append(h.predict(X)) 354 | 355 | hij = np.array(hij) 356 | # scale hij to [-1/N, 1/N] 357 | hij = 1. * hij / self.n_estimators 358 | 359 | ## Create QUBO 360 | qii = n_data * 1. / (self.n_estimators ** 2) + lmd - 2 * np.dot(hij, y) 361 | qij = np.dot(hij, hij.T) 362 | Q = dict() 363 | Q.update(dict(((k, k), v) for (k, v) in enumerate(qii))) 364 | for i in range(self.n_estimators): 365 | for j in range(i + 1, self.n_estimators): 366 | Q[(i, j)] = qij[i, j] 367 | 368 | # step 3: optimize QUBO 369 | res = sampler.sample_qubo(Q, **kwargs) 370 | samples = np.array([[samp[k] for k in range(self.n_estimators)] for samp in res]) 371 | 372 | # take the optimal solution as estimator weights 373 | self.estimator_weights = samples[0] 374 | 375 | def predict(self, X): 376 | 377 | n_data = len(X) 378 | T = 0 379 | y = np.zeros(n_data) 380 | for i, h in enumerate(self.estimators_): 381 | y0 = self.estimator_weights[i] * h.predict(X) # prediction of weak classifier 382 | y += y0 383 | T += np.sum(y0) 384 | 385 | norm = np.sum(self.estimator_weights) 386 | if norm > 0: 387 | y = y / norm 388 | else: 389 | y = y 390 | 391 | return y 392 | 393 | -------------------------------------------------------------------------------- /demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Qboost: Binary Classification with Quantum Computer\n", 8 | "\n", 9 | "The D-Wave quantum computer has been widely studied as a discrete optimization engine that accepts any problem formulated as quadratic unconstrained binary optimization (QUBO). In 2008, Google and D-Wave published a paper, [Training a Binary Classifier with the Quantum Adiabatic Algorithm](https://arxiv.org/pdf/0811.0416.pdf), which describes how the `Qboost` ensemble method makes binary classification amenable to quantum computing: the problem is formulated as a thresholded linear superposition of a set of weak classifiers and the D-Wave quantum computer is used to optimize the weights in a learning process that strives to minimize the training error and number of weak classifiers.\n", 10 | "\n", 11 | "This notebook demonstrates and explains how the Qboost algorithm can be used to solve a binary classification problem. " 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## A Few Words on Ensemble Methods\n", 19 | "\n", 20 | "Ensemble methods build a strong classifier (an improved model) by combining weak classifiers with the goal of:\n", 21 | "\n", 22 | "* decreasing variance (bagging)\n", 23 | "* decreasing bias (boosting)\n", 24 | "* improving prediction (voting)\n", 25 | "\n", 26 | "![Boosting Algorithm](images/boosting.jpg)\n", 27 | "\n", 28 | "### Bagging, Boosting, and Voting\n", 29 | "\n", 30 | "The ensemble method produces new training data sets by random sampling with replacement from the original set. In _bagging_, any element has the same probability to appear in a new dataset; in _boosting_, data elements are weighted before they are collected in the new dataset. Another distinction is that bagging is parallelizable but boosting has to be executed sequentially. You can learn more about the differences between these methods here: https://quantdare.com/what-is-the-difference-between-bagging-and-boosting/.\n", 31 | "\n", 32 | "Voting operates on labels only. Unlike boosting, the aggeragated classification performance is not used to further polish each weak classifier. Voting has two typical requirements of its collection of weak classifiers: that there be __many__ and that they be __diverse__. " 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "## Weak and Strong Classifiers\n", 40 | "For this reference example we chose the following four classifiers:\n", 41 | " 1. Adaboost\n", 42 | " 2. Decision Trees\n", 43 | " 3. Random Forest\n", 44 | " 4. Qboost\n", 45 | "Note that you can replace any of these with any commonly used classification model. Also, an ensemble method can use a strong classifier instead of a weak one, and in this example we embed the Qboost classifier itself, __QboostPlus__ in the following code, with the first three. \n", 46 | "\n", 47 | "### Adaboost\n", 48 | "Adaboost combines a number of $N$ weak classifiers into a strong one as\n", 49 | "$$C(x) = sign\\left(\\sum_i^N w_i c_i(x)\\right),$$\n", 50 | "with $c_i(x) \\in [-1, +1]$ being the $i$-th weak classifier:\n", 51 | "\n", 52 | "$$c_i(x) = sign(w'*x + b)$$\n", 53 | "\n", 54 | "The loss function of Adaboost is defined as\n", 55 | "$$\n", 56 | "L = \\sum_{n=1}^N \\exp\\left\\{ - y_n \\sum_{s=1}^S w_sc_k(x_n)\\right\\}.\n", 57 | "$$\n", 58 | "\n", 59 | "The strong classifier $C(\\cdot)$ is constructed in an iterative fashion. In each iteration, one weak classifier\n", 60 | "is selected and re-learned to minimize the weighted error function. Its weight is adjusted and renormalized to make sure the sum of all weights equals 1. \n", 61 | "\n", 62 | "The final classification model will be decided by a weighted “vote” of all the weak classifiers. \n", 63 | "\n", 64 | "### Decision Trees\n", 65 | "A decision tree builds on a tree structure with non-leaf nodes encoding decision rules and leaf nodes encoding labels. You construct a decision tree by optimizing either entropic or information-theoretic metrics. Controlling the depth of a decision tree indirectly decides the sub-dimension of the dataset. \n", 66 | "\n", 67 | "Decision trees are often chosen as the weak classifiers in Adaboost because they are both simple to construct and fast to do inference. The `scikit-learn` package implements its Adaboost method with decision trees of depth 1, also known as _tree stumps_. This reference examples demonstrates an alternative implementation of boosting with a number of deeper decision trees.\n", 68 | "### Random Forest \n", 69 | "Random forest is an ensemble method that typically implements bagging on a set of decision trees. By introducing randomness in the selection of an optimized feature in the training of the underlying decision trees, the ensemble diversifies the weightings of its collection of weak classifiers, generally resulting in an improved model. \n", 70 | "\n", 71 | "### Qboost\n", 72 | "To make use of the optimization power of D-Wave quantum annealer, we needs to formulate our objective function as a quadratic unconstrained binary optimization (QUBO) problem. Therefore, we replace the exponential loss as in Adaboost with the following quadratic loss\n", 73 | "$$\n", 74 | "w* = \\arg\\min_w\\left(\\sum_s \\left(\\frac{1}{N}\\sum_n^N w_nc_n(x_s) - y_s\\right)^2\\right) + \\lambda ||w||_0,\n", 75 | "$$\n", 76 | "where the regularization term is added to enable controlling of weight sparsity.\n", 77 | "\n", 78 | "Note in Qboost, the weight vector is binary.\n", 79 | "\n" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "## Training and Comparing Performance\n", 87 | "Now we define functions used in the following experiemnts to train our selected classifiers and provide metrics for comparing performance.\n", 88 | "First, let us import the required packages." 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "# import necessary packages\n", 98 | "from sklearn import preprocessing, metrics\n", 99 | "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n", 100 | "from sklearn.datasets.mldata import fetch_mldata\n", 101 | "from sklearn.datasets import load_breast_cancer\n", 102 | "from dwave.system.samplers import DWaveSampler\n", 103 | "from dwave.system.composites import EmbeddingComposite\n", 104 | "\n", 105 | "from qboost import WeakClassifiers, QBoostClassifier, QboostPlus" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "Next, let us define the `metric` and `train_model` functions:" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "# Define the functions required in this example\n", 122 | "def metric(y, y_pred):\n", 123 | " \"\"\"\n", 124 | " :param y: true label\n", 125 | " :param y_pred: predicted label\n", 126 | " :return: metric score\n", 127 | " \"\"\"\n", 128 | "\n", 129 | " return metrics.accuracy_score(y, y_pred)\n", 130 | "\n", 131 | "\n", 132 | "def train_model(X_train, y_train, X_test, y_test, lmd):\n", 133 | " \"\"\"\n", 134 | " :param X_train: training data\n", 135 | " :param y_train: training label\n", 136 | " :param X_test: testing data\n", 137 | " :param y_test: testing label\n", 138 | " :param lmd: lambda used in regularization\n", 139 | " :return:\n", 140 | " \"\"\"\n", 141 | "\n", 142 | " # define parameters used in this function\n", 143 | " NUM_READS = 1000\n", 144 | " NUM_WEAK_CLASSIFIERS = 30\n", 145 | " TREE_DEPTH = 2\n", 146 | " DW_PARAMS = {'num_reads': NUM_READS,\n", 147 | " 'auto_scale': True,\n", 148 | " 'num_spin_reversal_transforms': 10,\n", 149 | " 'postprocess': 'optimization',\n", 150 | " }\n", 151 | "\n", 152 | " # define sampler\n", 153 | " dwave_sampler = DWaveSampler()\n", 154 | " emb_sampler = EmbeddingComposite(dwave_sampler)\n", 155 | "\n", 156 | " N_train = len(X_train)\n", 157 | " N_test = len(X_test)\n", 158 | " print(\"\\n======================================\")\n", 159 | " print(\"Train size: %d, Test size: %d\" %(N_train, N_test))\n", 160 | " print('Num weak classifiers:', NUM_WEAK_CLASSIFIERS)\n", 161 | "\n", 162 | " # Preprocessing data\n", 163 | " imputer = preprocessing.Imputer()\n", 164 | " scaler = preprocessing.StandardScaler()\n", 165 | " normalizer = preprocessing.Normalizer()\n", 166 | "\n", 167 | " X_train = scaler.fit_transform(X_train)\n", 168 | " X_train = normalizer.fit_transform(X_train)\n", 169 | "\n", 170 | " X_test = scaler.fit_transform(X_test)\n", 171 | " X_test = normalizer.fit_transform(X_test)\n", 172 | "\n", 173 | " ## Adaboost\n", 174 | " print('\\nAdaboost')\n", 175 | " clf1 = AdaBoostClassifier(n_estimators=NUM_WEAK_CLASSIFIERS)\n", 176 | " clf1.fit(X_train, y_train)\n", 177 | " y_train1 = clf1.predict(X_train)\n", 178 | " y_test1 = clf1.predict(X_test)\n", 179 | "# print(clf1.estimator_weights_)\n", 180 | " print('accu (train): %5.2f'%(metric(y_train, y_train1)))\n", 181 | " print('accu (test): %5.2f'%(metric(y_test, y_test1)))\n", 182 | "\n", 183 | " # Ensembles of Decision Tree\n", 184 | " print('\\nDecision tree')\n", 185 | " clf2 = WeakClassifiers(n_estimators=NUM_WEAK_CLASSIFIERS, max_depth=TREE_DEPTH)\n", 186 | " clf2.fit(X_train, y_train)\n", 187 | " y_train2 = clf2.predict(X_train)\n", 188 | " y_test2 = clf2.predict(X_test)\n", 189 | "# print(clf2.estimator_weights)\n", 190 | " print('accu (train): %5.2f' % (metric(y_train, y_train2)))\n", 191 | " print('accu (test): %5.2f' % (metric(y_test, y_test2)))\n", 192 | " \n", 193 | " # Random forest\n", 194 | " print('\\nRandom Forest')\n", 195 | " clf3 = RandomForestClassifier(max_depth=TREE_DEPTH, n_estimators=NUM_WEAK_CLASSIFIERS)\n", 196 | " clf3.fit(X_train, y_train)\n", 197 | " y_train3 = clf3.predict(X_train)\n", 198 | " y_test3 = clf3.predict(X_test)\n", 199 | " print('accu (train): %5.2f' % (metric(y_train, y_train3)))\n", 200 | " print('accu (test): %5.2f' % (metric(y_test, y_test3)))\n", 201 | "\n", 202 | " # Qboost\n", 203 | " print('\\nQBoost')\n", 204 | " clf4 = QBoostClassifier(n_estimators=NUM_WEAK_CLASSIFIERS, max_depth=TREE_DEPTH)\n", 205 | " clf4.fit(X_train, y_train, emb_sampler, lmd=lmd, **DW_PARAMS)\n", 206 | " y_train4 = clf4.predict(X_train)\n", 207 | " y_test4 = clf4.predict(X_test)\n", 208 | " print(clf4.estimator_weights)\n", 209 | " print('accu (train): %5.2f' % (metric(y_train, y_train4)))\n", 210 | " print('accu (test): %5.2f' % (metric(y_test, y_test4)))\n", 211 | "\n", 212 | " # QboostPlus\n", 213 | " print('\\nQBoostPlus')\n", 214 | " clf5 = QboostPlus([clf1, clf2, clf3, clf4])\n", 215 | " clf5.fit(X_train, y_train, emb_sampler, lmd=lmd, **DW_PARAMS)\n", 216 | " y_train5 = clf5.predict(X_train)\n", 217 | " y_test5 = clf5.predict(X_test)\n", 218 | " print(clf5.estimator_weights)\n", 219 | " print('accu (train): %5.2f' % (metric(y_train, y_train5)))\n", 220 | " print('accu (test): %5.2f' % (metric(y_test, y_test5)))\n", 221 | "\n", 222 | " print(\"===========================================================================\")\n", 223 | " print(\"Method \\t Adaboost \\t DecisionTree \\t RandomForest \\t Qboost \\t Qboost+\")\n", 224 | " print(\"Train\\t %5.2f \\t\\t %5.2f \\t\\t %5.2f \\t\\t %5.2f \\t\\t %5.2f\"% (metric(y_train, y_train1),\n", 225 | " metric(y_train, y_train2),\n", 226 | " metric(y_train, y_train3),\n", 227 | " metric(y_train, y_train4),\n", 228 | " metric(y_train, y_train5),\n", 229 | " ))\n", 230 | " print(\"Test\\t %5.2f \\t\\t %5.2f \\t\\t %5.2f \\t\\t %5.2f \\t\\t %5.2f\"% (metric(y_test, y_test1),\n", 231 | " metric(y_test, y_test2),\n", 232 | " metric(y_test, y_test3),\n", 233 | " metric(y_test, y_test4),\n", 234 | " metric(y_test, y_test5)))\n", 235 | " print(\"===========================================================================\")\n", 236 | " \n", 237 | " return [clf1, clf2, clf3, clf4, clf5]" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "## Experiments\n", 245 | "Now we're ready to run some experiments.\n", 246 | "First, import the required packages. " 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "metadata": {}, 253 | "outputs": [], 254 | "source": [ 255 | "import numpy as np\n", 256 | "#import os\n", 257 | "import matplotlib.pyplot as plt" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "metadata": {}, 263 | "source": [ 264 | "\n", 265 | "### Experiment 1: Binary Classfication on the MNIST Dataset \n", 266 | "This example transforms the MNIST dataset (handwritten digits) into a binary classification problem. We assume all digits that are smaller than 5 are labelled as -1 and the rest digits are labelled as +1.\n", 267 | "\n", 268 | "First, let us load the MINIST dataset:" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": {}, 275 | "outputs": [], 276 | "source": [ 277 | "mnist = fetch_mldata('MNIST original', data_home='data')\n", 278 | "\n", 279 | "idx_01 = np.where(mnist.target <= 9)[0]\n", 280 | "np.random.shuffle(idx_01)\n", 281 | "idx_01 = idx_01[:15000]\n", 282 | "idx_train = idx_01[:2*len(idx_01)//3]\n", 283 | "idx_test = idx_01[2*len(idx_01)//3:]\n", 284 | "\n", 285 | "X_train = mnist.data[idx_train]\n", 286 | "X_test = mnist.data[idx_test]\n", 287 | "\n", 288 | "y_train = 2*(mnist.target[idx_train] >4) - 1\n", 289 | "y_test = 2*(mnist.target[idx_test] >4) - 1\n", 290 | "\n", 291 | "print(\"Training data size: (%d, %d)\" %(X_train.shape))\n", 292 | "print(\"Testing data size: (%d, %d)\" %(X_test.shape))" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "Let us visualize the digits: digits with class $+1$ are shown as images with a black background while digits with class $-1$ as images with a white background." 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": null, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "for i in range(16):\n", 309 | " if y_train[i] == 1:\n", 310 | " COLORMAP = 'gray'\n", 311 | " else:\n", 312 | " COLORMAP = 'gray_r'\n", 313 | " plt.subplot(4,4, i+1)\n", 314 | " plt.imshow(X_train[i].reshape(28,28), cmap=COLORMAP)\n", 315 | " plt.axis('off')" 316 | ] 317 | }, 318 | { 319 | "cell_type": "markdown", 320 | "metadata": {}, 321 | "source": [ 322 | "Now train the model and compare the results of the selected classifiers." 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": null, 328 | "metadata": {}, 329 | "outputs": [], 330 | "source": [ 331 | "# start training the model\n", 332 | "clfs = train_model(X_train, y_train, X_test, y_test, 1.0)" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "metadata": {}, 339 | "outputs": [], 340 | "source": [ 341 | "# TODO: for this cell, Graphviz executables must be on systems PATH \n", 342 | "# You can optionally visualize the decision trees by uncommenting the following code\n", 343 | "# import graphviz\n", 344 | "# from sklearn import tree\n", 345 | "# clf = clfs[0]\n", 346 | "# graph = graphviz.Source(tree.export_graphviz(clf.estimators_[0], out_file=None))\n", 347 | "# graph.render(None, view=True)" 348 | ] 349 | }, 350 | { 351 | "cell_type": "markdown", 352 | "metadata": { 353 | "collapsed": true 354 | }, 355 | "source": [ 356 | "### Experiment 2: Wisconsin Breast Cancer\n", 357 | "\n", 358 | "This example classifies tumors in scikit-learn's Wisconsis breast cancer dataset as either malignant or benign (binary classification).\n", 359 | "\n", 360 | "First, let us load the dataset." 361 | ] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": null, 366 | "metadata": {}, 367 | "outputs": [], 368 | "source": [ 369 | "wisc = load_breast_cancer()\n", 370 | "\n", 371 | "idx = np.arange(len(wisc.target))\n", 372 | "np.random.shuffle(idx)\n", 373 | "\n", 374 | "# train on a random 2/3 and test on the remaining 1/3\n", 375 | "idx_train = idx[:2*len(idx)//3]\n", 376 | "idx_test = idx[2*len(idx)//3:]\n", 377 | "\n", 378 | "X_train = wisc.data[idx_train]\n", 379 | "X_test = wisc.data[idx_test]\n", 380 | "\n", 381 | "y_train = 2 * wisc.target[idx_train] - 1 # binary -> spin\n", 382 | "y_test = 2 * wisc.target[idx_test] - 1" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "Now train the model and compare the results of the selected classifiers." 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": null, 395 | "metadata": {}, 396 | "outputs": [], 397 | "source": [ 398 | "# train the model\n", 399 | "clfs = train_model(X_train, y_train, X_test, y_test, 1.0)" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": null, 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [] 408 | } 409 | ], 410 | "metadata": { 411 | "kernelspec": { 412 | "display_name": "Python 2", 413 | "language": "python", 414 | "name": "python2" 415 | }, 416 | "language_info": { 417 | "codemirror_mode": { 418 | "name": "ipython", 419 | "version": 2 420 | }, 421 | "file_extension": ".py", 422 | "mimetype": "text/x-python", 423 | "name": "python", 424 | "nbconvert_exporter": "python", 425 | "pygments_lexer": "ipython2", 426 | "version": "2.7.12" 427 | } 428 | }, 429 | "nbformat": 4, 430 | "nbformat_minor": 1 431 | } 432 | --------------------------------------------------------------------------------