├── LICENSE ├── README.md ├── nas-bench-graph ├── architecture.py ├── light │ ├── arxiv.bench │ ├── citeseer.bench │ ├── computers.bench │ ├── cora.bench │ ├── cs.bench │ ├── photo.bench │ ├── physics.bench │ ├── proteins.bench │ └── pubmed.bench ├── readbench.py └── runnni.py └── setup.py /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NAS-Bench-Graph 2 | 3 | This repository provides the official codes and all evaluated architectures for NAS-Bench-Graph, a tailored benchmark for graph neural architecture search. 4 | 5 | Yijian Qin, Ziwei Zhang, Xin Wang, Zeyang Zhang, Wenwu Zhu, [NAS-Bench-Graph: Benchmarking Graph Neural Architecture Search](https://openreview.net/pdf?id=bBff294gqLp) (NeurIPS 2022) 6 | 7 | ## Install from PyPI 8 | You can directly install our benchmark by `pip install nas_bench_graph` 9 | 10 | ## Usage 11 | First, read the benchmark of a certain dataset by specifying the name. The nine supported datasets are: cora, citeseer, pubmed, cs, physics, photo, computers, arxiv, and proteins. For example, for the Cora dataset: 12 | ``` 13 | from nas_bench_graph import lightread 14 | bench = lightread('cora') 15 | ``` 16 | The data is stored as a `dict` in Python. 17 | 18 | Then, an architecture needs to be specified by its macro space and operations. 19 | We consider the macro space as a directed acyclic graph (DAG) and constrain the DAG to have only one input node for each intermediate node. Therefore, the macro space can be specificed by a list of integers, indicating the input node index for each computing node (0 for the raw input, 1 for the first computing node, etc.). Then, the operations can be specified by a list of strings with the same length. For example, we provide the code to specify the architecture in the following figure: 20 | ![arch](https://user-images.githubusercontent.com/17705534/173767528-eda1bc64-f4d8-4da1-a0e9-8470f55ccc6a.png) 21 | 22 | ``` 23 | from nas_bench_graph import Arch 24 | arch = Arch([0, 1, 2, 1], ['gcn', 'gin', 'fc', 'cheb']) 25 | # 0 means the inital computing node is connected to the input node 26 | # 1 means the next computing node is connected to the first computing node 27 | # 2 means the next computing node is connected to the second computing node 28 | # 1 means there is another computing node connected to the first computing node 29 | ``` 30 | 31 | Notice that we assume all leaf nodes (i.e., nodes without descendants) are connected to the output, so there is no need to specific the output node. 32 | 33 | Besides, the list can be specified in any order, e.g., the following code can specific the same architecture: 34 | ``` 35 | arch = Arch([0, 1, 1, 2], ['gcn', 'cheb', 'gin', 'fc']) 36 | ``` 37 | 38 | The benchmark data can be obtained by a look-up table. In this repository, we only provide the validation and test performance, the latency, and the number of parameters as follows: 39 | 40 | ``` 41 | info = bench[arch.valid_hash()] 42 | info['valid_perf'] # validation performance 43 | info['perf'] # test performance 44 | info['latency'] # latency 45 | info['para'] # number of parameters 46 | ``` 47 | 48 | For the complete benchmark, please downloadfrom https://figshare.com/articles/dataset/NAS-bench-Graph/20070371, which contains the training/validation/testing performance at each epoch. Since we run each dataset with three random seeds, each dataset has 3 files, e.g., 49 | 50 | ``` 51 | from nas_bench_graph import read 52 | bench = read('cora0.bench') # cora1.bench and cora2.bench 53 | ``` 54 | 55 | The full metric for any epoch can be obtained as follows. 56 | ``` 57 | info = bench[arch.valid_hash()] 58 | epoch = 50 59 | info['dur'][epoch][0] # training performance 60 | info['dur'][epoch][1] # validation performance 61 | info['dur'][epoch][2] # testing performance 62 | info['dur'][epoch][3] # training loss 63 | info['dur'][epoch][4] # validation loss 64 | info['dur'][epoch][5] # testing loss 65 | info['dur'][epoch][6] # best performance 66 | ``` 67 | 68 | ## Example usage of NNI and AutoGL 69 | NAS-Bench-Graph can be used together with other libraries such AutoGL and NNI. 70 | 71 | For the usage of [AutoGL](https://github.com/THUMNLab/AutoGL), please refer to the [agnn branch](https://github.com/THUMNLab/AutoGL/tree/agnn). 72 | 73 | You can also refer to `runnni.py` to use the benchmark together with [NNI](https://github.com/microsoft/nni/). 74 | 75 | 76 | ## Citation 77 | If you find that NAS-Bench-Graph helps your research, please consider citing it: 78 | 79 | ``` 80 | @inproceedings{qin2022nas, 81 | title = {NAS-Bench-Graph: Benchmarking Graph Neural Architecture Search}, 82 | author = {Qin, Yijian and Zhang, Ziwei and Wang, Xin and Zhang, Zeyang and Zhu, Wenwu}, 83 | booktitle = {Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track} 84 | year = {2022} 85 | } 86 | ``` 87 | 88 | -------------------------------------------------------------------------------- /nas-bench-graph/architecture.py: -------------------------------------------------------------------------------- 1 | import random 2 | import json 3 | 4 | link_list = [ 5 | [0,0,0,0], 6 | [0,0,0,1], 7 | [0,0,1,1], 8 | [0,0,1,2], 9 | [0,0,1,3], 10 | [0,1,1,1], 11 | [0,1,1,2], 12 | [0,1,2,2], 13 | [0,1,2,3] 14 | ] 15 | 16 | gnn_list = [ 17 | "gat", # GAT with 2 heads 18 | "gcn", # GCN 19 | "gin", # GIN 20 | "cheb", # chebnet 21 | "sage", # sage 22 | "arma", 23 | "graph", # k-GNN 24 | "fc", # fully-connected 25 | "skip" # skip connection 26 | ] 27 | 28 | gnn_list_proteins = [ 29 | "gcn", # GCN 30 | "sage", # sage 31 | "arma", 32 | "fc", # fully-connected 33 | "skip" # skip connection 34 | ] 35 | 36 | class Arch: 37 | def __init__(self, lk=None, op=None): 38 | self.link = lk 39 | self.ops = op 40 | 41 | def random_arch(self): 42 | self.ops = [] 43 | self.link = random.choice(link_list) 44 | for i in self.link: 45 | self.ops.append(random.choice(gnn_list)) 46 | 47 | def hash_arch(self, use_proteins = False): 48 | lk = self.link 49 | op = self.ops 50 | if use_proteins: 51 | gnn_g = {name: i for i, name in enumerate(gnn_list_proteins)} 52 | b = len(gnn_list_proteins) + 1 53 | else: 54 | gnn_g = {name: i for i, name in enumerate(gnn_list)} 55 | b = len(gnn_list) + 1 56 | if lk == [0,0,0,0]: 57 | lk_hash = 0 58 | elif lk == [0,0,0,1]: 59 | lk_hash = 1 60 | elif lk == [0,0,1,1]: 61 | lk_hash = 2 62 | elif lk == [0,0,1,2]: 63 | lk_hash = 3 64 | elif lk == [0,0,1,3]: 65 | lk_hash = 4 66 | elif lk == [0,1,1,1]: 67 | lk_hash = 5 68 | elif lk == [0,1,1,2]: 69 | lk_hash = 6 70 | elif lk == [0,1,2,2]: 71 | lk_hash = 7 72 | elif lk == [0,1,2,3]: 73 | lk_hash = 8 74 | 75 | for i in op: 76 | lk_hash = lk_hash * b + gnn_g[i] 77 | return lk_hash 78 | 79 | def regularize(self): 80 | lk = self.link[:] 81 | ops = self.ops[:] 82 | if lk == [0,0,0,2]: 83 | lk = [0,0,0,1] 84 | ops = [ops[1], ops[0], ops[2], ops[3]] 85 | elif lk == [0,0,0,3]: 86 | lk = [0,0,0,1] 87 | ops = [ops[2], ops[0], ops[1], ops[3]] 88 | elif lk == [0,0,1,0]: 89 | lk = [0,0,0,1] 90 | ops = [ops[0], ops[1], ops[3], ops[2]] 91 | elif lk == [0,0,2,0]: 92 | lk = [0,0,0,1] 93 | ops = [ops[1], ops[0], ops[3], ops[2]] 94 | elif lk == [0,0,2,1]: 95 | lk = [0,0,1,2] 96 | ops = [ops[1], ops[0], ops[2], ops[3]] 97 | elif lk == [0,0,2,2]: 98 | lk = [0,0,1,1] 99 | ops = [ops[1], ops[0], ops[2], ops[3]] 100 | elif lk == [0,0,2,3]: 101 | lk = [0,0,1,3] 102 | ops = [ops[1], ops[0], ops[2], ops[3]] 103 | elif lk == [0,1,0,0]: 104 | lk = [0,0,0,1] 105 | ops = [ops[0], ops[2], ops[3], ops[1]] 106 | elif lk == [0,1,0,1]: 107 | lk = [0,0,1,1] 108 | ops = [ops[0], ops[2], ops[1], ops[3]] 109 | elif lk == [0,1,0,2]: 110 | lk = [0,0,1,3] 111 | ops = [ops[0], ops[2], ops[1], ops[3]] 112 | elif lk == [0,1,0,3]: 113 | lk = [0,0,1,2] 114 | ops = [ops[0], ops[2], ops[1], ops[3]] 115 | elif lk == [0,1,1,0]: 116 | lk = [0,0,1,1] 117 | ops = [ops[0], ops[3], ops[1], ops[2]] 118 | elif lk == [0,1,1,3]: 119 | lk = [0,1,1,2] 120 | ops = [ops[0], ops[2], ops[1], ops[3]] 121 | elif lk == [0,1,2,0]: 122 | lk = [0,0,1,3] 123 | ops = [ops[0], ops[3], ops[1], ops[2]] 124 | elif lk == [0,1,2,1]: 125 | lk = [0,1,1,2] 126 | ops = [ops[0], ops[1], ops[3], ops[2]] 127 | return Arch(lk, ops) 128 | 129 | def equalpart_sort(self): 130 | lk = self.link 131 | op = self.ops 132 | ops = op[:] 133 | def part_sort(ids, ops): 134 | gnn_g = {name: i for i, name in enumerate(gnn_list)} 135 | opli = [gnn_g[ops[i]] for i in ids] 136 | opli.sort() 137 | for posid, opid in zip(ids, opli): 138 | ops[posid] = gnn_list[opid] 139 | return ops 140 | 141 | def sort0012(ops): 142 | gnn_g = {name: i for i, name in enumerate(gnn_list)} 143 | if gnn_g[op[0]] > gnn_g[op[1]] or op[0] == op[1] and gnn_g[op[2]] > gnn_g[op[3]]: 144 | ops = [ops[1], ops[0], ops[3], ops[2]] 145 | return ops 146 | 147 | if lk == [0,0,0,0]: 148 | ids = [0,1,2,3] 149 | elif lk == [0,0,0,1]: 150 | ids = [1,2] 151 | elif lk == [0,0,1,1]: 152 | ids = [2,3] 153 | elif lk == [0,0,1,2]: 154 | ids = None 155 | ops = sort0012(ops) 156 | elif lk == [0,1,1,1]: 157 | ids = [1,2,3] 158 | elif lk == [0,1,2,2]: 159 | ids = [2,3] 160 | else: 161 | ids = None 162 | 163 | if ids: 164 | ops = part_sort(ids, ops) 165 | 166 | self.ops = ops 167 | 168 | def move_skip_op(self): 169 | link = self.link[:] 170 | ops = self.ops[:] 171 | def move_one(k, link, ops): 172 | ops = [ops[k]] + ops[:k] + ops[k + 1:] 173 | for i, father in enumerate(link): 174 | if father == k + 1: 175 | link[i] = link[k] 176 | if father <= k: 177 | link[i] = link[i] + 1 178 | link = [0] + link[:k] + link[k + 1:] 179 | return link, ops 180 | 181 | def check_dim(k, link, ops): 182 | # check if a dimension is original dimension 183 | while k > -1: 184 | if ops[k] != 'skip': 185 | return False 186 | k = link[k] - 1 187 | return True 188 | 189 | for i in range(len(link)): 190 | if ops[i] != 'skip': 191 | continue 192 | son = False 193 | brother = False 194 | for j, fa in enumerate(link): 195 | if fa == i + 1: 196 | son = True 197 | elif j != i and fa == link[i]: 198 | brother = True 199 | if son or not brother or check_dim(i, link, ops) and not son: 200 | link, ops = move_one(i, link, ops) 201 | 202 | if link == [0,1,2,1]: 203 | link = [0,1,1,2] 204 | ops = ops[:2] + [ops[3], ops[2]] 205 | elif link == [0,1,1,3]: 206 | link = [0,1,1,2] 207 | ops = [ops[0], ops[2], ops[1], ops[3]] 208 | 209 | #if link not in link_list: 210 | # print(lk, link) 211 | 212 | self.link = link 213 | self.ops = ops 214 | 215 | def valid_hash(self): 216 | b = self.regularize() 217 | b.move_skip_op() 218 | b.equalpart_sort() 219 | return b.hash_arch() 220 | 221 | def check_isomorph(self): 222 | link, ops = self.link, self.ops 223 | linkm = link[:] 224 | opsm = ops[:] 225 | self.move_skip_op() 226 | self.equalpart_sort() 227 | #print(self.link, self.ops) 228 | return linkm == self.link and opsm == self.ops 229 | 230 | def check_isom(): 231 | lk = [0, 0, 0, 0] 232 | op = ['skip', 'gcn', 'gcn', 'skip'] 233 | arch = Arch(lk, op) 234 | print(arch.check_isomorph()) 235 | 236 | def all_archs(): 237 | # all combination of link_list and ops 238 | ng = len(gnn_list) 239 | archs = [] 240 | for i in gnn_list: 241 | for j in gnn_list: 242 | for k in gnn_list: 243 | for l in gnn_list: 244 | if i == 'skip' and j == 'skip' and k == 'skip' and l == 'skip': 245 | continue 246 | for lk in link_list: 247 | arch = Arch(lk, [i, j, k, l]) 248 | if arch.check_isomorph(): 249 | archs.append(arch) 250 | 251 | print(len(archs)) 252 | return archs 253 | 254 | class HP: 255 | def __repr__(self): 256 | return json.dumps(self, default=lambda obj: obj.__dict__) 257 | 258 | def random_hp(): 259 | randfloat = lambda x, y: x + (y - x) * random.random() 260 | hp = HP() 261 | hp.dropout = random.choice([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]) 262 | hp.dim = random.choice((64, 128, 256)) 263 | hp.num_cells = 1# random.randint(1,2) 264 | hp.num_pre = 0#random.randint(0, 1) 265 | hp.num_pro = random.randint(0, 1) 266 | 267 | #hp.lr = random.choice((0.05, 0.02, 0.01, 0.005, 0.002, 0.001)) 268 | hp.lr = random.choice((0.5, 0.2, 0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001)) 269 | hp.wd = random.choice((0, 5e-4)) 270 | #hp.optimizer = 'Adam' 271 | hp.optimizer = random.choice(('Adam', 'SGD')) 272 | hp.num_epochs = random.choice((200, 300, 400, 500)) 273 | return hp 274 | 275 | anchors = [ 276 | Arch([0, 1, 2, 3], ['skip', 'skip', 'gcn', 'gcn']), 277 | Arch([0, 1, 2, 3], ['skip', 'skip', 'gat', 'gat']), 278 | Arch([0, 1, 2, 3], ['skip', 'skip', 'gin', 'gin']), 279 | Arch([0, 1, 2, 3], ['skip', 'skip', 'graph', 'graph']), 280 | Arch([0, 1, 2, 3], ['skip', 'skip', 'sage', 'sage']), 281 | Arch([0, 1, 2, 3], ['skip', 'gcn', 'gcn', 'gcn']), 282 | Arch([0, 1, 2, 3], ['skip', 'gat', 'gat', 'gat']), 283 | Arch([0, 1, 2, 3], ['skip', 'gin', 'gin', 'gin']), 284 | Arch([0, 1, 2, 3], ['skip', 'graph', 'graph', 'graph']), 285 | Arch([0, 1, 2, 3], ['skip', 'sage', 'sage', 'sage']) 286 | ] 287 | 288 | if __name__ == '__main__': 289 | #all_archs() 290 | check_isom() 291 | -------------------------------------------------------------------------------- /nas-bench-graph/light/arxiv.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/arxiv.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/citeseer.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/citeseer.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/computers.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/computers.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/cora.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/cora.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/cs.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/cs.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/photo.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/photo.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/physics.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/physics.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/proteins.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/proteins.bench -------------------------------------------------------------------------------- /nas-bench-graph/light/pubmed.bench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUMNLab/NAS-Bench-Graph/be42534b56f6aaab2c3d6145b10eda9cbde10193/nas-bench-graph/light/pubmed.bench -------------------------------------------------------------------------------- /nas-bench-graph/readbench.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from architecture import all_archs 3 | 4 | def light_read(dname): 5 | f = open("light/{}.bench".format(dname), "rb") 6 | bench = pickle.load(f) 7 | f.close() 8 | return bench 9 | 10 | def read(name): 11 | f = open(name, "rb") 12 | bench = pickle.load(f) 13 | f.close() 14 | return bench 15 | 16 | if __name__ == "__main__": 17 | read() 18 | -------------------------------------------------------------------------------- /nas-bench-graph/runnni.py: -------------------------------------------------------------------------------- 1 | from modulefinder import Module 2 | import torch 3 | import nni 4 | import sys 5 | import torch.nn.functional as F 6 | import nni.retiarii.nn.pytorch as nn 7 | from nni.retiarii import model_wrapper 8 | from nni.retiarii.evaluator import FunctionalEvaluator 9 | import nni.retiarii.strategy as strategy 10 | 11 | from architecture import gnn_list, gnn_list_proteins 12 | from architecture import all_archs, HP, Arch 13 | from readbench import light_read 14 | from nni.retiarii.experiment.pytorch import RetiariiExperiment, RetiariiExeConfig 15 | dnames = ['cora', 'citeseer', 'pubmed', 'cs', 'physics', 'photo', 'computers', 'arxiv', 'proteins'] 16 | op_proteins = ['gcn', 'arma', 'cheb', 'fc', 'skip'] 17 | #op_proteins = gnn_list 18 | 19 | # model 20 | class StrModule(nn.Module): 21 | def __init__(self, lambd): 22 | super().__init__() 23 | self.name = lambd 24 | 25 | def forward(self, *args, **kwargs): 26 | return self.str 27 | 28 | def __repr__(self): 29 | return "{}({})".format(self.__class__.__name__, self.str) 30 | 31 | 32 | def map_nn(l): 33 | return nn.LayerChoice([StrModule(x) for x in l]) 34 | 35 | def map_value(l, label): 36 | return nn.ValueChoice(l, label = label) 37 | 38 | @model_wrapper 39 | class Space_rl(nn.Module): 40 | def __init__(self): 41 | super().__init__() 42 | self.lk0 = 0 43 | self.op0 = map_value(op_proteins, "op0") 44 | self.lk1 = map_value([0, 1], "lk1") 45 | self.op1 = map_value(op_proteins, "op1") 46 | self.lk2 = map_value([0, 1, 2], "lk2") 47 | self.op2 = map_value(op_proteins, "op2") 48 | self.lk3 = map_value([0, 1, 2, 3], "lk3") 49 | self.op3 = map_value(op_proteins, "op3") 50 | 51 | def forward(self, bench): 52 | lks = [getattr(self, "lk" + str(i)) for i in range(4)] 53 | ops = [getattr(self, "op" + str(i)) for i in range(4)] 54 | arch = Arch(lks, ops) 55 | h = arch.valid_hash() 56 | if h == "88888": 57 | return 0 58 | return bench[h]['perf'] 59 | 60 | @model_wrapper 61 | class Space(nn.Module): 62 | def __init__(self): 63 | super().__init__() 64 | self.links = nn.ModuleList() 65 | self.ops = nn.ModuleList() 66 | 67 | for i in range(4): 68 | self.links.append(map_nn(range(i + 1))) 69 | self.ops.append(map_nn(op_proteins)) 70 | 71 | def forward(self, bench): 72 | lks = [i.name for i in self.links] 73 | ops = [i.name for i in self.ops] 74 | arch = Arch(lks, ops) 75 | h = arch.valid_hash() 76 | if h == "88888": 77 | return 0 78 | return bench[h]['perf'] 79 | 80 | def seeinsed(self): 81 | for i in self.ops: 82 | print(i.name) 83 | 84 | def evaluate_model(model_cls, dname): 85 | bench = light_read(dname) 86 | model = model_cls() 87 | ans = model(bench) 88 | nni.report_final_result(ans) 89 | return ans 90 | 91 | def atest(nas, dname, port): 92 | space = Space_rl() 93 | evaluator = FunctionalEvaluator(evaluate_model, dname=dname) 94 | if nas == "random": 95 | exploration_strategy = strategy.Random(dedup=True) 96 | elif nas == "ea": 97 | exploration_strategy = strategy.RegularizedEvolution() 98 | elif nas == "rl": 99 | exploration_strategy = strategy.PolicyBasedRL() 100 | 101 | exp = RetiariiExperiment(space, evaluator, [], exploration_strategy) 102 | exp_config = RetiariiExeConfig('local') 103 | exp_config.experiment_name = 'mnist_search' 104 | exp_config.max_trial_number = 40 # spawn 4 trials at most 105 | exp_config.trial_concurrency = 10 # will run two trials concurrently 106 | exp_config.trial_gpu_number = 1 107 | exp_config.training_service.use_active_gpu = True 108 | exp.run(exp_config, port) 109 | for model_dict in exp.export_top_models(formatter='dict'): 110 | return model_dict 111 | 112 | def main(): 113 | fin = {} 114 | alg = sys.argv[1] 115 | dname = sys.argv[2] 116 | port = int(sys.argv[3]) 117 | ans = atest(alg, dname, port) 118 | fin[alg + dname] = ans 119 | fo = open("pickle/{}{}.txt".format(alg, dname), "w") 120 | for key in fin: 121 | fo.write(key + "\n") 122 | fo.write(str(fin[key]) + "\n") 123 | fo.close() 124 | 125 | main() 126 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("README.md", 'r') as fh: 4 | long_description = fh.read() 5 | 6 | ''' https://packaging.python.org/guides/distributing-packages-using-setuptools/ ''' 7 | ''' https://setuptools.readthedocs.io/en/latest/ ''' 8 | setup( 9 | name='nas-bench-graph', 10 | version='1.0', 11 | author='THUMNLab/aglteam', 12 | maintainer='THUMNLab/aglteam', 13 | author_email='autogl@tsinghua.edu.cn', 14 | description='NAS benchmark for graph data', 15 | long_description=long_description, 16 | long_description_content_type='text/markdown', 17 | include_package_data=True, 18 | packages=find_packages(), 19 | # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires 20 | python_requires='~=3.6', 21 | # https://pypi.org/classifiers/ 22 | classifiers=[ 23 | "Development Status :: 4 - Beta", 24 | "License :: OSI Approved :: Apache Software License", 25 | "Programming Language :: Python :: 3 :: Only", 26 | "Programming Language :: Python :: 3.6" 27 | ], 28 | # https://setuptools.readthedocs.io/en/latest/userguide/dependency_management.html 29 | # note that setup_requires and tests_require are deprecated 30 | install_requires=[ 31 | 'pickle', 32 | 'torch', 33 | 'modulefinder', 34 | 'nni', 35 | 'sys' 36 | ] 37 | ) 38 | --------------------------------------------------------------------------------