├── .gitignore ├── experiments ├── predictor │ ├── result │ │ └── README.md │ ├── result_parser.py │ └── predictor_compare.py ├── weight_factor_analysis │ ├── result │ │ └── README.md │ ├── result_parser.py │ └── prediction_weight_factor_compare.py ├── utils.py ├── algorithms │ ├── NPNAS.py │ ├── FRNAS.py │ └── NPENAS.py └── bias_analysis │ ├── sample_size_irg_change_compare.py │ └── instance_relation_compare.py ├── nas_lib ├── config.py └── model_predictor │ ├── search_space │ ├── search_space.py │ ├── nasbench101_search_space.py │ ├── nasbench201_search_space.py │ └── darts_search_space.py │ ├── trainer │ ├── utils.py │ ├── gin_trainer.py │ └── fr_gin_trainer.py │ ├── agent │ ├── npenas_gin_predictor.py │ ├── npnas_gcn_predictor.py │ ├── multi_transform_gin_predictor.py │ └── fr_gin_predictor.py │ └── dataset │ ├── nasbench201_dataset.py │ ├── nasbench101_dataset.py │ └── darts_dataset.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *\result 3 | *.zip -------------------------------------------------------------------------------- /experiments/predictor/result/README.md: -------------------------------------------------------------------------------- 1 | Each folder here contains experimental data. -------------------------------------------------------------------------------- /experiments/weight_factor_analysis/result/README.md: -------------------------------------------------------------------------------- 1 | Each folder here contains experimental data. -------------------------------------------------------------------------------- /nas_lib/config.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | EVOXBENCH_DATABASE_PATH = "#Path To#/evoxbench/database20220713/database" 4 | EVOXBENCH_DATA_PATH = "#Path To#/evoxbench/data20221028/data" 5 | 6 | 7 | if __name__ == '__main__': 8 | # setup before first run 9 | from evoxbench.database.init import config 10 | config(EVOXBENCH_DATABASE_PATH, EVOXBENCH_DATA_PATH) -------------------------------------------------------------------------------- /nas_lib/model_predictor/search_space/search_space.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | class DatabaseSearchSpace: 5 | """ 6 | Interface for search space with database 7 | """ 8 | def __init__(self): 9 | pass 10 | 11 | def query(self, archs): 12 | raise NotImplementedError 13 | 14 | 15 | def sample_data(self, sample_size): 16 | raise NotImplementedError -------------------------------------------------------------------------------- /nas_lib/model_predictor/trainer/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import math 3 | import numpy as np 4 | 5 | from torch.optim.lr_scheduler import CosineAnnealingLR, ConstantLR 6 | 7 | 8 | def get_lr_scheduler(optimizer, epochs, train_date_size, batch_size, lr="CosineAnnealingLR"): 9 | total_steps = epochs*int(train_date_size / batch_size) 10 | if lr=="CosineAnnealingLR": 11 | scheduler = CosineAnnealingLR(optimizer=optimizer, T_max=total_steps, eta_min=0) 12 | elif lr=="Constant": 13 | scheduler = ConstantLR(optimizer=optimizer) 14 | else: 15 | raise NotImplementedError() 16 | return scheduler 17 | 18 | 19 | 20 | def make_agent_optimizer(model, base_lr, weight_deacy=1e-4, bias_multiply=True): 21 | params = [] 22 | for key, value in model.named_parameters(): 23 | if not value.requires_grad: 24 | continue 25 | lr = base_lr 26 | weight_decay = weight_deacy 27 | if "bias" in key: 28 | if bias_multiply: 29 | lr = base_lr*2.0 30 | weight_decay = 0.0 31 | else: 32 | lr = base_lr 33 | weight_decay = weight_deacy 34 | 35 | params += [{"params": [value], "lr": lr, "weight_decay": weight_decay}] 36 | optimizer = torch.optim.Adam(params, base_lr, (0.0, 0.9)) 37 | return optimizer 38 | 39 | 40 | 41 | 42 | 43 | 44 | if __name__ == '__main__': 45 | pass 46 | -------------------------------------------------------------------------------- /experiments/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy 4 | import numpy as np 5 | import torch 6 | 7 | from scipy.stats import kendalltau 8 | from scipy.stats import spearmanr 9 | 10 | def set_random_seed(seed): 11 | random.seed(seed) 12 | np.random.seed(seed) 13 | torch.manual_seed(seed) 14 | torch.cuda.manual_seed_all(seed) 15 | 16 | 17 | def evaluate_stats(x: np.ndarray, y: np.ndarray, metrics=None, prefix=""): 18 | if metrics is None: 19 | metrics = ["MAE", "MSE", "kendalltau", "spearman"] 20 | 21 | result = {} 22 | for metric in metrics: 23 | if metric == "MAE": 24 | result[prefix + metric] = calculate_mean_error(x, y) 25 | if metric == "MSE": 26 | result[prefix + metric] = calculate_mean_square_error(x, y) 27 | if metric == "kendalltau": 28 | result[prefix + metric] = calclulate_kendalltau_correlation(x, y) 29 | if metric == "spearman": 30 | result[prefix + metric] = calclulate_spearman_rank_correlation(x, y) 31 | 32 | return result 33 | 34 | 35 | def calculate_mean_error(x: np.ndarray, y: np.ndarray): 36 | """ 37 | input two numpy list 38 | retrun mean error 39 | """ 40 | result = np.mean(np.abs(x - y)).tolist() 41 | return result 42 | 43 | 44 | def calculate_mean_square_error(x: np.ndarray, y: np.ndarray): 45 | """ 46 | input two numpy list 47 | retrun mean square error 48 | """ 49 | result = np.mean((x - y) ** 2).tolist() 50 | return result 51 | 52 | 53 | def calclulate_kendalltau_correlation(x: np.ndarray, y: np.ndarray): 54 | """ 55 | input two numpy list 56 | retrun kendall tau correlation, pvalue 57 | """ 58 | correlation, pvalue = kendalltau(x, y) 59 | return correlation, pvalue 60 | 61 | def calclulate_spearman_rank_correlation(x: np.ndarray, y: np.ndarray): 62 | """ 63 | input two numpy list 64 | retrun kendall tau correlation, pvalue 65 | """ 66 | correlation, pvalue = spearmanr(x, y) 67 | return correlation, pvalue 68 | 69 | 70 | def test(): 71 | x = numpy.array([1,2,3,4,5]) 72 | y = numpy.array([6,2,3,2,5]) 73 | res = evaluate_stats(x,y) 74 | print(res) 75 | 76 | if __name__ == '__main__': 77 | test() -------------------------------------------------------------------------------- /experiments/algorithms/NPNAS.py: -------------------------------------------------------------------------------- 1 | 2 | # utils 3 | from experiments.utils import evaluate_stats 4 | 5 | # trainer 6 | from nas_lib.model_predictor.trainer.gin_trainer import GinPredictorTrainer 7 | 8 | # algorithms 9 | from nas_lib.model_predictor.agent.npnas_gcn_predictor import NPNASGcnPredictorAgent 10 | 11 | from nas_lib.model_predictor.agent.multi_transform_gin_predictor import multi_transform, undirected_transform, \ 12 | no_transform, reverse_transform 13 | 14 | 15 | from nas_lib.model_predictor.search_space.nasbench101_search_space import NasBench101SearchSpace 16 | from nas_lib.model_predictor.dataset.nasbench101_dataset import NasBench101Dataset 17 | from nas_lib.model_predictor.dataset.nasbench201_dataset import NasBench201Dataset 18 | from nas_lib.model_predictor.dataset.darts_dataset import DartsDataset 19 | 20 | AvailableDataset = { 21 | "nasbench-101": NasBench101Dataset, 22 | "nasbench-201": NasBench201Dataset, 23 | "darts": DartsDataset, 24 | } 25 | 26 | 27 | 28 | 29 | def NPNAS(search_space, train_archs, test_archs, learning_rate, batch_size, epochs, scaling_factor, weight_decay, bias_multiply, device): 30 | 31 | Dataset = AvailableDataset[search_space] 32 | 33 | train_dataset = Dataset(arch_data_list=train_archs) 34 | test_dataset = Dataset(arch_data_list=test_archs) 35 | 36 | agent = NPNASGcnPredictorAgent(input_dim=Dataset.input_dim, directed_gcn=True) 37 | trainer = GinPredictorTrainer(agent, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, device=device, 38 | bias_multiply=bias_multiply, weight_decay = weight_decay 39 | ) 40 | trainer.fit(train_dataset) 41 | 42 | # train dataset evaluation 43 | pred_result, label_result = trainer.test(train_dataset) 44 | pred_result = pred_result.cpu().numpy() 45 | label_result = label_result.cpu().numpy() 46 | 47 | stats = evaluate_stats(pred_result, label_result) 48 | print(stats) 49 | 50 | # test dataset evaluation 51 | pred_result, label_result = trainer.test(test_dataset) 52 | 53 | pred_result = pred_result.cpu().numpy() 54 | label_result = label_result.cpu().numpy() 55 | # print(pred_result) 56 | # print(label_result) 57 | stats = evaluate_stats(pred_result, label_result) 58 | print(stats) 59 | return stats 60 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/agent/npenas_gin_predictor.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import Sequential, Linear, ReLU 5 | from torch_geometric.nn import GINConv 6 | from torch_geometric.nn import global_mean_pool 7 | 8 | 9 | 10 | class NPENASGinPredictorAgent(nn.Module): 11 | ''' 12 | Graph Isomorphism Network Predictor 13 | 14 | Wei, Chen, Chuang Niu, Yiping Tang, Yue Wang, Haihong Hu, and Jimin Liang. 15 | “NPENAS: Neural Predictor Guided Evolution for Neural Architecture Search.” 16 | IEEE Transactions on Neural Networks and Learning Systems, 2022, 1–15. 17 | https://doi.org/10.1109/TNNLS.2022.3151160. 18 | ''' 19 | 20 | def __init__(self, input_dim=6): 21 | super(NPENASGinPredictorAgent, self).__init__() 22 | layers = [] 23 | dim = 32 24 | dim2 = 16 25 | nn1 = Sequential(Linear(input_dim, dim, bias=True), ReLU(), Linear(dim, dim, bias=True)) 26 | self.conv1 = GINConv(nn1) 27 | self.bn1 = torch.nn.BatchNorm1d(dim) 28 | 29 | nn2 = Sequential(Linear(dim, dim, bias=True), ReLU(), Linear(dim, dim, bias=True)) 30 | self.conv2 = GINConv(nn2) 31 | self.bn2 = torch.nn.BatchNorm1d(dim) 32 | # 33 | nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim)) 34 | self.conv3 = GINConv(nn3) 35 | self.bn3 = torch.nn.BatchNorm1d(dim) 36 | 37 | self.linear_before = torch.nn.Linear(dim, dim2, bias=True) 38 | 39 | self.linear_mean = Linear(dim2, 1) 40 | layers.append(self.linear_mean) 41 | layers.append(self.linear_before) 42 | self.out_layer = torch.nn.Sigmoid() 43 | 44 | # cause worse performance on NB101 and NB201, better performance on NB301 45 | # for layer in layers: 46 | # if isinstance(layer, nn.Linear): 47 | # nn.init.kaiming_uniform_(layer.weight, a=1) 48 | # nn.init.constant_(layer.bias, 0) 49 | 50 | def forward(self, graph_data): 51 | x = graph_data.x 52 | edge_index = graph_data.edge_index 53 | batch = graph_data.batch 54 | return self.forward_batch(x, edge_index, batch) 55 | 56 | def forward_batch(self, x, edge_index, batch): 57 | x1 = F.relu(self.conv1(x, edge_index)) 58 | x1 = self.bn1(x1) 59 | 60 | x2 = F.relu(self.conv2(x1, edge_index)) 61 | x2 = self.bn2(x2) 62 | 63 | x3 = F.relu(self.conv3(x2, edge_index)) 64 | x3 = self.bn3(x3) 65 | 66 | x_embedding = global_mean_pool(x3, batch) 67 | x_embedding_mean = F.relu(self.linear_before(x_embedding)) 68 | x_embedding_drop = F.dropout(x_embedding_mean, p=0.1, training=self.training) 69 | mean = self.linear_mean(x_embedding_drop) 70 | mean = self.out_layer(mean) 71 | return mean 72 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/agent/npnas_gcn_predictor.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch_geometric.nn.conv import GCNConv 5 | from torch_geometric.nn import global_mean_pool 6 | 7 | class DirectedGCNConv(nn.Module): 8 | """ 9 | "To make information flow both ways, we always use the average of two GCN layers: 10 | one where we use A to propagate information in the forward directions 11 | and another where we use A^T to reverse the direction" 12 | 13 | """ 14 | 15 | def __init__(self, in_channels, out_channels, directed_gcn=True): 16 | super(DirectedGCNConv, self).__init__() 17 | self.forward_layer = GCNConv(in_channels=in_channels, out_channels=out_channels) 18 | self.backward_layer = GCNConv(in_channels=in_channels, out_channels=out_channels) 19 | self.directed_gcn = directed_gcn 20 | 21 | def forward(self, x, edge_index): 22 | output1 = F.relu(self.forward_layer(x, edge_index)) 23 | if self.directed_gcn: 24 | reverse_idx = edge_index[[1, 0]] # swap the order 25 | else: 26 | reverse_idx = edge_index[[0, 1]] 27 | output2 = F.relu(self.backward_layer(x, reverse_idx)) 28 | output = (output1 + output2) / 2 29 | 30 | return output 31 | 32 | 33 | 34 | 35 | class NPNASGcnPredictorAgent(nn.Module): 36 | """ 37 | Graph Convolutional Network Predictor 38 | 39 | Implemented based on paper: 40 | 41 | Wen Wei, Hanxiao Liu, Hai Li, Yiran Chen, Gabriel Bender, and Pieter-Jan Kindermans. 42 | “Neural Predictor for Neural Architecture Search.” arXiv, December 2, 2019. http://arxiv.org/abs/1912.00848. 43 | refer to https://github.com/ultmaster/neuralpredictor.pytorch 44 | """ 45 | 46 | def __init__(self, input_dim=5, gcn_hidden=144, gcn_layers=3, linear_hidden=128, directed_gcn=True): 47 | super(NPNASGcnPredictorAgent, self).__init__() 48 | 49 | 50 | self.gcn = [DirectedGCNConv(in_channels=input_dim if i == 0 else gcn_hidden, out_channels=gcn_hidden, directed_gcn=directed_gcn) 51 | for i in range(gcn_layers)] 52 | 53 | 54 | self.gcn = nn.ModuleList(self.gcn) 55 | 56 | self.dropout = nn.Dropout(0.1) 57 | 58 | self.fc1 = nn.Linear(gcn_hidden, linear_hidden) 59 | self.fc2 = nn.Linear(linear_hidden, 1) 60 | 61 | # not mentioned on original papers 62 | # nn.init.kaiming_uniform_(self.fc1.weight, a=1) 63 | # nn.init.constant_(self.fc1.bias, 0) 64 | # nn.init.kaiming_uniform_(self.fc2.weight, a=1) 65 | # nn.init.constant_(self.fc2.bias, 0) 66 | 67 | def forward(self, graph_data): 68 | x = graph_data.x 69 | edge_index = graph_data.edge_index 70 | batch = graph_data.batch 71 | 72 | for layer in self.gcn: 73 | x = layer(x, edge_index) 74 | out = global_mean_pool(x, batch) 75 | 76 | # print(out) 77 | out = self.fc1(out) 78 | out = self.dropout(out) 79 | out = self.fc2(out).view(-1) 80 | 81 | return out 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ## FR-NAS: Forward-and-Reverse Graph Predictor for Efficient Neural Architecture Search [arXiv](https://arxiv.org/abs/2404.15622) 3 | 4 | Neural Architecture Search (NAS) has emerged as a key tool in identifying optimal configurations of deep neural networks tailored to specific tasks. 5 | However, training and assessing numerous architectures introduces considerable computational overhead. 6 | We propose a GIN predictor and new training procedure for performance evaluation in NAS. 7 | 8 | 9 | ## Train and Evaluation 10 | run `experiments/predictor/predictor_compare.py` for prediction compare. 11 | 12 | The results are stored in `experiments/predictor/result)`. 13 | 14 | Process the result using code in `experiments/predictor/result_parser.py`. 15 | 16 | ## Source File Structure 17 | - `experiments`: Experiments conducted for the paper. 18 | - `nas_lib/model_predictor`: 19 | - `agent`: Neural predictors. 20 | - `search_space`: Search spaces, including database interactions and data preprocessing. 21 | - `trainer`: Training procedures for models. 22 | 23 | 24 | ## Environment Setup 25 | 26 | - python=3.8 27 | - scipy=1.4.1 28 | - torch=1.12.1+cu116 29 | - torch-geometric 30 | 31 | 32 | ### Installing EvoXBench 33 | 34 | To utilize the [EvoXBench](https://github.com/EMI-Group/evoxbench) database, configure it after installation: 35 | 36 | ```python 37 | from evoxbench.database.init import config 38 | 39 | config("Path to database", "Path to data") 40 | # For instance: 41 | # With this structure: 42 | # /home/Downloads/ 43 | # └─ database/ 44 | # | | __init__.py 45 | # | | db.sqlite3 46 | # | | ... 47 | # | 48 | # └─ data/ 49 | # └─ darts/ 50 | # └─ mnv3/ 51 | # └─ ... 52 | # Then, execute: 53 | # config("/home/Downloads/database", "/home/Downloads/data") 54 | ``` 55 | 56 | ### Installation Commands 57 | 58 | #### For Windows (CPU version) 59 | ```bash 60 | # pytorch 1.12.1 61 | pip install torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 62 | # torch-geometric 63 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcpu/torch_cluster-1.6.0-cp38-cp38-win_amd64.whl 64 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcpu/torch_scatter-2.0.9-cp38-cp38-win_amd64.whl 65 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcpu/torch_sparse-0.6.14-cp38-cp38-win_amd64.whl 66 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcpu/torch_spline_conv-1.2.1-cp38-cp38-win_amd64.whl 67 | pip install torch-geometric 68 | ``` 69 | 70 | #### For Windows (CUDA 11.3 version) 71 | ```bash 72 | # pytorch 1.12.1 73 | pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113 74 | # torch-geometric 75 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcu113/torch_cluster-1.6.0-cp38-cp38-win_amd64.whl 76 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcu113/torch_scatter-2.0.9-cp38-cp38-win_amd64.whl 77 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcu113/torch_sparse-0.6.14-cp38-cp38-win_amd64.whl 78 | pip install https://data.pyg.org/whl/torch-1.12.0%2Bcu113/torch_spline_conv-1.2.1-cp38-cp38-win_amd64.whl 79 | pip install torch-geometric 80 | ``` 81 | 82 | ## Citing FR-NAS 83 | 84 | If you use FR-NAS in your research and want to cite it in your work, please use: 85 | 86 | ``` 87 | @misc{zhang2024frnas, 88 | title={FR-NAS: Forward-and-Reverse Graph Predictor for Efficient Neural Architecture Search}, 89 | author={Haoming Zhang and Ran Cheng}, 90 | booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)}, 91 | year={2024} 92 | } 93 | ``` 94 | 95 | 96 | ## Acknowledge 97 | This code is implemented based on the framework provided by [NPENASv1](https://github.com/auroua/NPENASv1?tab=readme-ov-file#acknowledge) 98 | -------------------------------------------------------------------------------- /experiments/weight_factor_analysis/result_parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | import argparse 4 | import scipy.stats as stats 5 | 6 | 7 | def merge_data(total_trial, trial): 8 | # data structures of every trials shall be same 9 | for key in trial.keys(): 10 | stats = trial[key] 11 | if key in total_trial: 12 | total_trial[key].append(stats) 13 | else: 14 | total_trial[key] = [stats] 15 | 16 | 17 | def merge_stats(total_stats, stats): 18 | # data structures of every stats shall be same 19 | for metric in stats.keys(): 20 | s = stats[metric] 21 | if metric == "kendalltau": 22 | s = s[0] 23 | total_stats[metric].append(s) 24 | 25 | 26 | def merge_trials(total_trials, trial): 27 | # data structures of every trial shall be same 28 | for algoname in trial.keys(): 29 | stats = trial[algoname] 30 | if algoname in total_trials: 31 | merge_stats(total_trials[algoname], stats) 32 | else: 33 | new_stats = {} 34 | for metric in stats.keys(): 35 | new_stats[metric] = [] 36 | total_trials[algoname] = new_stats 37 | merge_stats(total_trials[algoname], stats) 38 | 39 | 40 | def load_from_file(file_path): 41 | with open(file_path+"/results.json", 'r') as f: 42 | data = json.load(f) 43 | 44 | with open(file_path + "/config.json", 'r') as f: 45 | config = json.load(f) 46 | 47 | total_trials = {} 48 | num_trial = 0 49 | 50 | for k in data.keys(): 51 | if not k.isdigit(): 52 | raise NotImplementedError("such file is not supported") 53 | trial = data[k] 54 | merge_trials(total_trials, trial) 55 | num_trial += 1 56 | 57 | # assert num_trial == config["trials"] 58 | 59 | # print(json.dumps(total_trials, indent=2)) 60 | return total_trials, num_trial, config 61 | 62 | 63 | def load_and_evaluate(file_path, metrics=None): 64 | if metrics is None: 65 | metrics = [ 66 | "MAE", 67 | "kendalltau" 68 | ] 69 | 70 | # load_old_file(SAVE_PATH) 71 | data, num_trial, config = load_from_file(file_path) 72 | 73 | result = {} 74 | for algoname in data.keys(): 75 | algo_stats = data[algoname] 76 | result[algoname] = {} 77 | for metric in algo_stats.keys(): 78 | if not metric in metrics: 79 | continue 80 | s = np.array(algo_stats[metric]) 81 | result[algoname][metric] = (np.nanmean(s, axis=0).tolist(), np.nanstd(s, axis=0).tolist()) 82 | 83 | return result, num_trial, config 84 | 85 | 86 | def table(result_dict, algo_names): 87 | metric = "kendalltau" 88 | 89 | for algo in algo_names: 90 | result = [] 91 | for sample_size in result_dict.keys(): 92 | mean = result_dict[sample_size][algo][metric][0] 93 | std = result_dict[sample_size][algo][metric][1] 94 | mean = round(mean, 4) 95 | std = round(std, 4) 96 | result.append((mean,std)) 97 | # result = result + " & " + str(mean) + " $\\pm$ " + str(std) 98 | print(algo, result) 99 | 100 | 101 | 102 | def main(): 103 | experiment_path_list = [ 104 | "./result/weight_factor_new_#Replace With Date#_darts", 105 | ] 106 | 107 | algo_name = [] 108 | 109 | result_dict = {} # sample_size --> trail_result 110 | for file_path in experiment_path_list: 111 | result, num_trial, config = load_and_evaluate(file_path) 112 | for algo in result.keys(): 113 | algo_name.append(algo) 114 | print(num_trial) 115 | if config["sample_size"] in result_dict: 116 | result_dict[config["sample_size"]].update(result) 117 | else: 118 | result_dict[config["sample_size"]] = result 119 | 120 | print(json.dumps(result_dict, indent=2)) 121 | 122 | table(result_dict, algo_name) 123 | 124 | 125 | 126 | if __name__ == '__main__': 127 | main() 128 | 129 | # print(data) 130 | # print(stats.ranksums(data["neural_predictor"], data["neural_predictor_reverse_only"])) 131 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/dataset/nasbench201_dataset.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy 4 | import torch 5 | from torch_geometric.data import Dataset, Data 6 | 7 | from nas_lib.model_predictor.search_space.nasbench201_search_space import OPS 8 | 9 | 10 | def operation_to_one_hot(ops): 11 | op_dims = len(OPS) 12 | output_ops = [] 13 | 14 | for op in ops: 15 | if op in OPS: 16 | index = OPS[op] 17 | feature = [0 for _ in range(op_dims)] 18 | feature[index] = 1 19 | output_ops.append(feature) 20 | else: 21 | raise Exception() 22 | 23 | return output_ops 24 | 25 | 26 | def matrix_to_edge_index(matrix): 27 | matrix = numpy.array(matrix) 28 | shape = matrix.shape 29 | assert len(shape) == 2 and shape[0] == shape[1] # must be square 30 | total_nodes = matrix.shape[0] 31 | 32 | node_out = [] 33 | node_in = [] 34 | 35 | for i in range(total_nodes): 36 | for j in range(total_nodes): 37 | if matrix[i][j] != 0: 38 | node_out.append(i) 39 | node_in.append(j) 40 | 41 | return [node_out, node_in] 42 | 43 | 44 | class NasBench201Dataset(Dataset): 45 | 46 | input_dim = 7 47 | 48 | 49 | def __init__(self, arch_data_list, use_valid_error=True, key_matrix = "matrix", key_ops = "ops"): 50 | """ 51 | :param arch_data_list: 52 | :param use_valid_data: 53 | """ 54 | super().__init__(root=None, transform=None, pre_transform=None, pre_filter=None) 55 | 56 | self.data_list = [] 57 | 58 | for data in arch_data_list: 59 | ops = operation_to_one_hot(data[key_ops]) 60 | edge_index = matrix_to_edge_index(data[key_matrix]) 61 | if use_valid_error: 62 | label = data["valid_error"] 63 | else: 64 | label = data["test_error"] 65 | 66 | ops = torch.from_numpy(numpy.array(ops)).type(torch.float) 67 | edge_index = torch.from_numpy(numpy.array(edge_index)).type(torch.long) 68 | label = torch.from_numpy(numpy.array(label)).type(torch.float) 69 | 70 | self.data_list.append(Data(x=ops, edge_index=edge_index, y=label)) 71 | 72 | def to_device(self, device): 73 | for data in self.data_list: 74 | data.to(device) 75 | 76 | def len(self): 77 | return len(self.data_list) 78 | 79 | def get(self, idx): 80 | return self.data_list[idx] 81 | 82 | 83 | class NasBench201MultiTransformDataset(Dataset): 84 | 85 | 86 | input_dim = 7 87 | 88 | def __init__(self, arch_data_list, use_valid_error=True, device=None): 89 | """ 90 | :param arch_data_list: 91 | :param use_valid_data: 92 | """ 93 | super().__init__(root=None, transform=None, pre_transform=None, pre_filter=None) 94 | 95 | self.data_list = [] 96 | 97 | for data in arch_data_list: 98 | 99 | matrix_list = data["transform_matrix_list"] 100 | graph_data_list = [] 101 | for matrix in matrix_list: 102 | ops = operation_to_one_hot(data["ops"]) 103 | edge_index = matrix_to_edge_index(matrix) 104 | 105 | if use_valid_error: 106 | label = data["valid_error"] 107 | else: 108 | label = data["test_error"] 109 | 110 | ops = torch.from_numpy(numpy.array(ops)).type(torch.float) 111 | edge_index = torch.from_numpy(numpy.array(edge_index)).type(torch.long) 112 | label = torch.from_numpy(numpy.array(label)).type(torch.float) 113 | 114 | graph_data_list.append(Data(x=ops, edge_index=edge_index, y=label)) 115 | self.data_list.append(graph_data_list) 116 | 117 | if device: 118 | self.to_device(device) 119 | 120 | def to_device(self, device): 121 | for data in self.data_list: 122 | for graph_data in data: 123 | graph_data.to(device) 124 | 125 | def len(self): 126 | return len(self.data_list) 127 | 128 | def get(self, idx): 129 | return self.data_list[idx] 130 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/search_space/nasbench101_search_space.py: -------------------------------------------------------------------------------- 1 | import random 2 | from nas_lib.model_predictor.search_space.search_space import DatabaseSearchSpace 3 | from evoxbench.benchmarks.nb101 import NASBench101Result 4 | from evoxbench.benchmarks.nb101 import NASBench101Graph 5 | 6 | OPS = { 7 | 'input': 0, 8 | 'conv3x3-bn-relu': 1, 9 | 'conv1x1-bn-relu': 2, 10 | 'maxpool3x3': 3, 11 | 'output': 4, 12 | 'isolate': 5 13 | } 14 | 15 | NODE_NUM = 7 16 | 17 | 18 | class NasBench101SearchSpace(DatabaseSearchSpace): 19 | def __init__(self, determistic=True): 20 | super(NasBench101SearchSpace, self).__init__() 21 | self.determistic = determistic 22 | 23 | def _extract_data(self, data_list): 24 | """ 25 | given data list from database 26 | """ 27 | output_data = [] 28 | for data in data_list: 29 | index = data["index"] 30 | matrix = data["phenotype"]["module_adjacency"] 31 | ops = data["phenotype"]["module_operations"] 32 | valid_acc_list = data["final_validation_accuracy"]["epoch108"] 33 | test_acc_list = data["final_test_accuracy"]["epoch108"] 34 | 35 | if self.determistic: 36 | valid_error = (1 - sum(valid_acc_list) / len(valid_acc_list)) * 100 37 | test_error = (1 - sum(test_acc_list) / len(test_acc_list)) * 100 38 | else: 39 | valid_error = (1 - random.choice(valid_acc_list)) * 100 40 | test_error = (1 - random.choice(test_acc_list)) * 100 41 | 42 | output_data.append( 43 | { 44 | "index": index, 45 | "matrix": matrix, 46 | "ops": ops, 47 | "valid_error": valid_error, 48 | "test_error": test_error 49 | } 50 | ) 51 | return output_data 52 | 53 | def query(self, archs): 54 | """ 55 | query archs 56 | :param archs: lists of arch: {"matrix":[square adjacent matrix], "ops": [original operations]} 57 | """ 58 | query_result = [] 59 | for i, arch in enumerate(archs): 60 | model_spec = NASBench101Graph(matrix=arch['matrix'], ops=arch['ops']) 61 | index = model_spec.hash_spec(['conv3x3-bn-relu', 'conv1x1-bn-relu', 'maxpool3x3']) # hash value of one arch 62 | result = NASBench101Result.objects.values("index", "phenotype", "final_test_accuracy", 63 | "final_validation_accuracy").get(index=index) 64 | query_result.append(result) 65 | query_result = self._extract_data(data_list=query_result) 66 | return query_result 67 | 68 | def sample_data(self, sample_size): 69 | return self.sample_data_uniform(sample_size=sample_size) 70 | 71 | def sample_data_uniform(self, sample_size): 72 | """ 73 | Uniformly sample from nasbench-101 74 | """ 75 | print("sampling data in nasbench-101: {}".format(sample_size)) 76 | query_set = NASBench101Result.objects.order_by('?') \ 77 | .values("index", "phenotype", "final_test_accuracy", "final_validation_accuracy")[ 78 | :sample_size] 79 | # NOTE: this will return a list QuerySet, every time we get the value from same QuerySet will get 80 | # different result because of random shuffle , must assign value to get exact result. 81 | query_result = [] 82 | for query in query_set: 83 | query_result.append(query) 84 | 85 | 86 | result = self._extract_data(data_list=query_result) 87 | return result 88 | 89 | 90 | if __name__ == '__main__': 91 | nb101data = NasBench101SearchSpace() 92 | sample = nb101data.sample_data_uniform(10) 93 | query_archs = [] 94 | for s in sample: 95 | print(s) 96 | query_archs.append({"matrix": s["matrix"], "ops": s["ops"]}) 97 | print("-" * 20) 98 | 99 | result = nb101data.query(query_archs) 100 | for s in result: 101 | print(s) 102 | query_archs.append({"matrix": s["matrix"], "ops": s["ops"]}) 103 | print("-" * 20) 104 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/dataset/nasbench101_dataset.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy 4 | import torch 5 | from torch_geometric.data import Dataset, Data 6 | 7 | from nas_lib.model_predictor.search_space.nasbench101_search_space import OPS 8 | 9 | 10 | def operation_to_one_hot(ops): 11 | op_dims = len(OPS) 12 | output_ops = [] 13 | 14 | for op in ops: 15 | if op in OPS: 16 | index = OPS[op] 17 | feature = [0 for _ in range(op_dims)] 18 | feature[index] = 1 19 | output_ops.append(feature) 20 | else: 21 | raise Exception() 22 | 23 | return output_ops 24 | 25 | 26 | def matrix_to_edge_index(matrix): 27 | matrix = numpy.array(matrix) 28 | shape = matrix.shape 29 | assert len(shape) == 2 and shape[0] == shape[1] # must be square 30 | total_nodes = matrix.shape[0] 31 | 32 | node_out = [] 33 | node_in = [] 34 | 35 | for i in range(total_nodes): 36 | for j in range(total_nodes): 37 | if matrix[i][j] != 0: 38 | node_out.append(i) 39 | node_in.append(j) 40 | 41 | return [node_out, node_in] 42 | 43 | 44 | class NasBench101Dataset(Dataset): 45 | 46 | input_dim = 6 47 | 48 | def __init__(self, arch_data_list, use_valid_error=True, key_matrix = "matrix", key_ops = "ops", device=None): 49 | """ 50 | :param arch_data_list: 51 | :param use_valid_data: 52 | """ 53 | 54 | super().__init__(root=None, transform=None, pre_transform=None, pre_filter=None) 55 | 56 | self.data_list = [] 57 | 58 | for data in arch_data_list: 59 | ops = operation_to_one_hot(data[key_ops]) 60 | edge_index = matrix_to_edge_index(data[key_matrix]) 61 | if use_valid_error: 62 | label = data["valid_error"] 63 | else: 64 | label = data["test_error"] 65 | 66 | ops = torch.from_numpy(numpy.array(ops)).type(torch.float) 67 | edge_index = torch.from_numpy(numpy.array(edge_index)).type(torch.long) 68 | label = torch.from_numpy(numpy.array(label)).type(torch.float) 69 | 70 | self.data_list.append(Data(x=ops, edge_index=edge_index, y=label)) 71 | 72 | if device: 73 | self.to_device(device) 74 | 75 | 76 | def to_device(self, device): 77 | for data in self.data_list: 78 | data.to(device) 79 | 80 | def len(self): 81 | return len(self.data_list) 82 | 83 | def get(self, idx): 84 | return self.data_list[idx] 85 | 86 | 87 | class NasBench101MultiTransformDataset(Dataset): 88 | 89 | input_dim = 6 90 | 91 | def __init__(self, arch_data_list, use_valid_error=True, device=None): 92 | """ 93 | :param arch_data_list: 94 | :param use_valid_data: 95 | """ 96 | super().__init__(root=None, transform=None, pre_transform=None, pre_filter=None ) 97 | 98 | self.data_list = [] 99 | 100 | for data in arch_data_list: 101 | 102 | matrix_list = data["transform_matrix_list"] 103 | graph_data_list = [] 104 | for matrix in matrix_list: 105 | ops = operation_to_one_hot(data["ops"]) 106 | edge_index = matrix_to_edge_index(matrix) 107 | 108 | if use_valid_error: 109 | label = data["valid_error"] 110 | else: 111 | label = data["test_error"] 112 | 113 | ops = torch.from_numpy(numpy.array(ops)).type(torch.float) 114 | edge_index = torch.from_numpy(numpy.array(edge_index)).type(torch.long) 115 | label = torch.from_numpy(numpy.array(label)).type(torch.float) 116 | 117 | graph_data_list.append(Data(x=ops, edge_index=edge_index, y=label)) 118 | self.data_list.append(graph_data_list) 119 | 120 | if device: 121 | self.to_device(device) 122 | 123 | def to_device(self, device): 124 | for data in self.data_list: 125 | for graph_data in data: 126 | graph_data.to(device) 127 | 128 | def len(self): 129 | return len(self.data_list) 130 | 131 | def get(self, idx): 132 | return self.data_list[idx] 133 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/dataset/darts_dataset.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy 4 | import torch 5 | from torch_geometric.data import Dataset, Data 6 | 7 | from nas_lib.model_predictor.search_space.darts_search_space import OPS 8 | 9 | 10 | def operation_to_one_hot(ops): 11 | op_dims = len(OPS) 12 | output_ops = [] 13 | 14 | for op in ops: 15 | if op in OPS: 16 | index = OPS[op] 17 | feature = [0 for _ in range(op_dims)] 18 | feature[index] = 1 19 | output_ops.append(feature) 20 | else: 21 | print(op, "is not a valid operation!") 22 | raise Exception() 23 | 24 | return output_ops 25 | 26 | 27 | def matrix_to_edge_index(matrix): 28 | matrix = numpy.array(matrix) 29 | shape = matrix.shape 30 | assert len(shape) == 2 and shape[0] == shape[1] # must be square 31 | total_nodes = matrix.shape[0] 32 | 33 | node_out = [] 34 | node_in = [] 35 | 36 | for i in range(total_nodes): 37 | for j in range(total_nodes): 38 | if matrix[i][j] != 0: 39 | node_out.append(i) 40 | node_in.append(j) 41 | 42 | return [node_out, node_in] 43 | 44 | 45 | class DartsDataset(Dataset): 46 | input_dim = 13 47 | 48 | def __init__(self, arch_data_list, use_valid_error=True, key_matrix="matrix", key_ops="ops", device=None): 49 | """ 50 | :param arch_data_list: 51 | :param use_valid_data: 52 | """ 53 | super().__init__(root=None, transform=None, pre_transform=None, pre_filter=None) 54 | 55 | self.data_list = [] 56 | 57 | for data in arch_data_list: 58 | ops = operation_to_one_hot(data[key_ops]) 59 | edge_index = matrix_to_edge_index(data[key_matrix]) 60 | if use_valid_error: 61 | label = data["valid_error"] 62 | else: 63 | label = data["test_error"] 64 | 65 | ops = torch.from_numpy(numpy.array(ops)).type(torch.float) 66 | edge_index = torch.from_numpy(numpy.array(edge_index)).type(torch.long) 67 | label = torch.from_numpy(numpy.array(label)).type(torch.float) 68 | 69 | self.data_list.append(Data(x=ops, edge_index=edge_index, y=label)) 70 | 71 | if device: 72 | self.to_device(device) 73 | 74 | def to_device(self, device): 75 | for data in self.data_list: 76 | data.to(device) 77 | 78 | def len(self): 79 | return len(self.data_list) 80 | 81 | def get(self, idx): 82 | return self.data_list[idx] 83 | 84 | 85 | class DartsMultiTransformDataset(Dataset): 86 | input_dim = 13 87 | 88 | def __init__(self, arch_data_list, use_valid_error=True, device=None): 89 | """ 90 | :param arch_data_list: 91 | :param use_valid_data: 92 | """ 93 | super().__init__(root=None, transform=None, pre_transform=None, pre_filter=None) 94 | 95 | self.data_list = [] 96 | 97 | for data in arch_data_list: 98 | 99 | matrix_list = data["transform_matrix_list"] 100 | graph_data_list = [] 101 | for matrix in matrix_list: 102 | ops = operation_to_one_hot(data["ops"]) 103 | edge_index = matrix_to_edge_index(matrix) 104 | 105 | if use_valid_error: 106 | label = data["valid_error"] 107 | else: 108 | label = data["test_error"] 109 | 110 | ops = torch.from_numpy(numpy.array(ops)).type(torch.float) 111 | edge_index = torch.from_numpy(numpy.array(edge_index)).type(torch.long) 112 | label = torch.from_numpy(numpy.array(label)).type(torch.float) 113 | 114 | graph_data_list.append(Data(x=ops, edge_index=edge_index, y=label)) 115 | self.data_list.append(graph_data_list) 116 | 117 | if device: 118 | self.to_device(device) 119 | 120 | 121 | def to_device(self, device): 122 | for data in self.data_list: 123 | for graph_data in data: 124 | graph_data.to(device) 125 | 126 | def len(self): 127 | return len(self.data_list) 128 | 129 | def get(self, idx): 130 | return self.data_list[idx] 131 | -------------------------------------------------------------------------------- /experiments/predictor/result_parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | import argparse 4 | import scipy.stats as stats 5 | 6 | 7 | def merge_data(total_trial, trial): 8 | # data structures of every trials shall be same 9 | for key in trial.keys(): 10 | stats = trial[key] 11 | if key in total_trial: 12 | total_trial[key].append(stats) 13 | else: 14 | total_trial[key] = [stats] 15 | 16 | 17 | def merge_stats(total_stats, stats): 18 | # data structures of every stats shall be same 19 | for metric in stats.keys(): 20 | s = stats[metric] 21 | if metric == "kendalltau": 22 | s = s[0] 23 | total_stats[metric].append(s) 24 | 25 | 26 | def merge_trials(total_trials, trial): 27 | # data structures of every trial shall be same 28 | for algoname in trial.keys(): 29 | stats = trial[algoname] 30 | if algoname in total_trials: 31 | merge_stats(total_trials[algoname], stats) 32 | else: 33 | new_stats = {} 34 | for metric in stats.keys(): 35 | new_stats[metric] = [] 36 | total_trials[algoname] = new_stats 37 | merge_stats(total_trials[algoname], stats) 38 | 39 | 40 | def load_from_file(file_path): 41 | with open(file_path+"/results.json", 'r') as f: 42 | data = json.load(f) 43 | 44 | with open(file_path + "/config.json", 'r') as f: 45 | config = json.load(f) 46 | 47 | total_trials = {} 48 | num_trial = 0 49 | 50 | for k in data.keys(): 51 | if not k.isdigit(): 52 | raise NotImplementedError("such file is not supported") 53 | trial = data[k] 54 | merge_trials(total_trials, trial) 55 | num_trial += 1 56 | 57 | # assert num_trial == config["trials"] 58 | 59 | # print(json.dumps(total_trials, indent=2)) 60 | return total_trials, num_trial, config 61 | 62 | 63 | def load_and_evaluate(file_path, metrics=None): 64 | if metrics is None: 65 | metrics = [ 66 | "MAE", 67 | "kendalltau" 68 | ] 69 | 70 | # load_old_file(SAVE_PATH) 71 | data, num_trial, config = load_from_file(file_path) 72 | 73 | result = {} 74 | for algoname in data.keys(): 75 | algo_stats = data[algoname] 76 | result[algoname] = {} 77 | for metric in algo_stats.keys(): 78 | if not metric in metrics: 79 | continue 80 | s = np.array(algo_stats[metric]) 81 | result[algoname][metric] = (np.nanmean(s, axis=0).tolist(), np.nanstd(s, axis=0).tolist()) 82 | 83 | return result, num_trial, config 84 | 85 | 86 | def table(result_dict, algo_names): 87 | metric = "MAE" 88 | 89 | for algo in algo_names: 90 | result = [] 91 | for sample_size in result_dict.keys(): 92 | mean = result_dict[sample_size][algo][metric][0] 93 | std = result_dict[sample_size][algo][metric][1] 94 | mean = round(mean, 4) 95 | std = round(std, 4) 96 | result.append((sample_size, mean,std)) 97 | # result = result + " & " + str(mean) + " $\\pm$ " + str(std) 98 | print(algo, result) 99 | 100 | 101 | 102 | def main(): 103 | experiment_path_list = [ 104 | "predictor_compare_#Replace With Date#_darts_400", 105 | "predictor_compare_#Replace With Date#_darts_300", 106 | "predictor_compare_#Replace With Date#_darts_200", 107 | "predictor_compare_#Replace With Date#_darts_150", 108 | "predictor_compare_#Replace With Date#_darts_100", 109 | "predictor_compare_#Replace With Date#_darts_50", 110 | ] 111 | 112 | algo_name = [] 113 | 114 | result_dict = {} # sample_size --> trail_result 115 | for file_path in experiment_path_list: 116 | file_path = "./result/" + file_path 117 | result, num_trial, config = load_and_evaluate(file_path) 118 | for algo in result.keys(): 119 | if algo not in algo_name: 120 | algo_name.append(algo) 121 | print(num_trial) 122 | if config["sample_size"] in result_dict: 123 | result_dict[config["sample_size"]].update(result) 124 | else: 125 | result_dict[config["sample_size"]] = result 126 | 127 | print(json.dumps(result_dict, indent=2)) 128 | 129 | table(result_dict, algo_name) 130 | 131 | 132 | 133 | if __name__ == '__main__': 134 | main() 135 | 136 | # print(data) 137 | # print(stats.ranksums(data["neural_predictor"], data["neural_predictor_reverse_only"])) 138 | -------------------------------------------------------------------------------- /experiments/weight_factor_analysis/prediction_weight_factor_compare.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | import numpy as np 4 | import os 5 | 6 | from datetime import datetime 7 | 8 | # utils 9 | from experiments.utils import set_random_seed 10 | 11 | from experiments.utils import evaluate_stats 12 | 13 | # search space 14 | from nas_lib.model_predictor.search_space.nasbench101_search_space import NasBench101SearchSpace 15 | from nas_lib.model_predictor.search_space.nasbench201_search_space import NasBench201SearchSpace 16 | from nas_lib.model_predictor.search_space.darts_search_space import DartsSearchSpace 17 | 18 | # algorithms 19 | # from experiments.algorithms.NPENAS_BKD import NPENAS_BKD 20 | from experiments.algorithms.FRNAS import FRNAS 21 | 22 | WeightFactorList = [ 23 | 0, 24 | 0.1, 25 | 0.2, 26 | 0.3, 27 | 0.4, 28 | 0.5, 29 | 0.6, 30 | 0.7, 31 | 0.8, 32 | 0.9, 33 | ] 34 | 35 | AvailableSearchSpace = { 36 | "nasbench-101": NasBench101SearchSpace, 37 | "nasbench-201": NasBench201SearchSpace, 38 | "darts": DartsSearchSpace, 39 | } 40 | 41 | DEVICE = "cpu" # the bottleneck of this type of predictor is cpu and memory 42 | 43 | 44 | def main(args): 45 | sample_size, test_size = args.sample_size, args.test_size 46 | data_dict = {} 47 | 48 | # search space 49 | 50 | search_space = AvailableSearchSpace[args.search_space]() 51 | sampled_archs = search_space.sample_data_uniform(sample_size + test_size) 52 | train_archs = sampled_archs[:sample_size] 53 | test_archs = sampled_archs[sample_size:] 54 | 55 | for weight_factor in WeightFactorList: 56 | for loss in ["MSE", "IRG"]: 57 | algorithm = "FRNAS_{}_wf{}".format(loss, weight_factor) 58 | print("Running algorithm: {}".format(algorithm)) 59 | if loss == "IRG": 60 | algo_result = FRNAS(search_space=args.search_space, 61 | train_archs=train_archs, 62 | test_archs=test_archs, 63 | learning_rate=0.005, batch_size=16, epochs=200, scaling_factor=20, 64 | weight_decay=1e-4, 65 | weight_factor=weight_factor, 66 | device=DEVICE, 67 | loss=loss) 68 | elif loss == "MSE": 69 | algo_result = FRNAS(search_space=args.search_space, 70 | train_archs=train_archs, 71 | test_archs=test_archs, 72 | learning_rate=0.005, batch_size=16, epochs=200, scaling_factor=20, 73 | weight_decay=1e-4, 74 | weight_factor=weight_factor, 75 | device=DEVICE, 76 | loss=loss) 77 | 78 | data_dict[algorithm] = algo_result 79 | 80 | return data_dict 81 | 82 | 83 | if __name__ == '__main__': 84 | parser = argparse.ArgumentParser(description='Args for algorithms compare.') 85 | parser.add_argument('--search_space', type=str, default="darts", choices=["nasbench-101", "nasbench-201", "darts"], 86 | help='search space') 87 | parser.add_argument('--trials', type=int, default=100, help='Number of trials') 88 | parser.add_argument('--seed', type=int, default=15234, help='random seed') 89 | parser.add_argument('--save_path', type=str, default='./result/', help='path to save') 90 | parser.add_argument('--save_prefix', type=str, default='weight_factor_new', help='prefix of the file') 91 | 92 | parser.add_argument('--sample_size', type=int, default=200, help='Number of sampled architectures') 93 | 94 | parser.add_argument('--test_size', type=int, default=5000, help='Number of test architectures') 95 | parser.add_argument('--save_interval', type=int, default=30, help='intervals to save') 96 | args = parser.parse_args() 97 | # print(args) 98 | set_random_seed(args.seed) 99 | 100 | init_time = datetime.now() 101 | init_time = init_time.strftime("%y%m%d") + "-" + init_time.strftime("%H%M%S") 102 | 103 | 104 | save_path = args.save_path + args.save_prefix + "_" + init_time + "_" + args.search_space 105 | os.mkdir(save_path) 106 | 107 | config = { 108 | "seed": args.seed, 109 | "sample_size": args.sample_size, 110 | "trials": args.trials, 111 | "test_size": args.test_size, 112 | "search_space": args.search_space, 113 | "epochs": 200, 114 | } 115 | 116 | with open(save_path + "/config.json", "w") as f: 117 | json.dump(config, f, indent=2) 118 | 119 | trials_result_dict = {} 120 | for trial in range(args.trials): 121 | print("trial {}".format(trial)) 122 | result_dict = main(args) 123 | trials_result_dict[trial] = result_dict 124 | if (trial + 1) % args.save_interval == 0: 125 | with open(save_path + "/ckpt-{}.json".format(trial+1), "w") as f: 126 | json.dump(trials_result_dict, f, indent=2) 127 | 128 | 129 | with open(save_path + "/results.json", "w") as f: 130 | json.dump(trials_result_dict, f, indent=2) 131 | -------------------------------------------------------------------------------- /experiments/algorithms/FRNAS.py: -------------------------------------------------------------------------------- 1 | 2 | # utils 3 | from experiments.utils import evaluate_stats 4 | 5 | # datas 6 | from nas_lib.model_predictor.dataset.nasbench101_dataset import NasBench101MultiTransformDataset, NasBench101Dataset 7 | from nas_lib.model_predictor.dataset.nasbench201_dataset import NasBench201MultiTransformDataset, NasBench201Dataset 8 | from nas_lib.model_predictor.dataset.darts_dataset import DartsMultiTransformDataset, DartsDataset 9 | 10 | # trainer 11 | from nas_lib.model_predictor.trainer.fr_gin_trainer import FRGinPredictorTrainer 12 | from nas_lib.model_predictor.trainer.gin_trainer import GinPredictorTrainer 13 | 14 | # algorithms 15 | 16 | from nas_lib.model_predictor.agent.multi_transform_gin_predictor import multi_transform 17 | 18 | from nas_lib.model_predictor.agent.fr_gin_predictor import FRGinPredictorAgent 19 | from nas_lib.model_predictor.agent.fr_gin_predictor import FRGinPredictorAgentAfter 20 | 21 | AvailableMultiTransformDataset = { 22 | "nasbench-101": NasBench101MultiTransformDataset, 23 | "nasbench-201": NasBench201MultiTransformDataset, 24 | "darts": DartsMultiTransformDataset, 25 | } 26 | 27 | AvailableDataset = { 28 | "nasbench-101": NasBench101Dataset, 29 | "nasbench-201": NasBench201Dataset, 30 | "darts": DartsDataset, 31 | } 32 | 33 | 34 | 35 | def FRNAS(search_space, train_archs, test_archs, learning_rate, batch_size, epochs, scaling_factor, device, weight_factor, weight_decay, loss="IRG"): 36 | for arch in train_archs: 37 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 38 | 39 | for arch in test_archs: 40 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 41 | 42 | Dataset = AvailableMultiTransformDataset[search_space] 43 | 44 | train_dataset = Dataset(arch_data_list=train_archs, device=device) 45 | test_dataset = Dataset(arch_data_list=test_archs, device=device) 46 | 47 | agent_1 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 48 | agent_2 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 49 | 50 | trainer = FRGinPredictorTrainer(agent_1, agent_2, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, 51 | weight_factor=weight_factor, weight_decay=weight_decay, 52 | device=device, loss=loss) 53 | 54 | trainer.fit(train_dataset) 55 | 56 | # train dataset evaluation 57 | pred_result, label_result = trainer.test(train_dataset) 58 | pred_result = pred_result.cpu().numpy() 59 | label_result = label_result.cpu().numpy() 60 | stats = evaluate_stats(pred_result, label_result) 61 | print(stats) 62 | 63 | # test dataset evaluation 64 | pred_result, label_result = trainer.test(test_dataset) 65 | 66 | pred_result = pred_result.cpu().numpy() 67 | label_result = label_result.cpu().numpy() 68 | # print(pred_result) 69 | # print(label_result) 70 | stats = evaluate_stats(pred_result, label_result) 71 | print(stats) 72 | return stats 73 | 74 | 75 | def FRNAS_finetune(search_space, train_archs, test_archs, learning_rate, batch_size, epochs, scaling_factor, device, weight_factor, weight_decay, loss="IRG"): 76 | for arch in train_archs: 77 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 78 | 79 | for arch in test_archs: 80 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 81 | 82 | Dataset = AvailableMultiTransformDataset[search_space] 83 | 84 | train_dataset = Dataset(arch_data_list=train_archs, device=device) 85 | test_dataset = Dataset(arch_data_list=test_archs, device=device) 86 | 87 | agent_1 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 88 | agent_2 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 89 | 90 | trainer = FRGinPredictorTrainer(agent_1, agent_2, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, 91 | weight_factor=weight_factor, weight_decay=weight_decay, 92 | device=device, loss=loss) 93 | 94 | trainer.fit(train_dataset) 95 | 96 | # train dataset evaluation 97 | pred_result, label_result = trainer.test(train_dataset) 98 | pred_result = pred_result.cpu().numpy() 99 | label_result = label_result.cpu().numpy() 100 | stats = evaluate_stats(pred_result, label_result) 101 | print(stats) 102 | 103 | # finetune after training 104 | 105 | Dataset = AvailableDataset[search_space] 106 | 107 | train_dataset = Dataset(arch_data_list=train_archs) 108 | test_dataset = Dataset(arch_data_list=test_archs) 109 | 110 | new_agent = FRGinPredictorAgentAfter(agent_1.gin_encoder, agent_2.gin_encoder) 111 | trainer = GinPredictorTrainer(new_agent, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, weight_decay=weight_decay, device=device) 112 | trainer.fit(train_dataset) 113 | 114 | # train dataset evaluation 115 | pred_result, label_result = trainer.test(train_dataset) 116 | pred_result = pred_result.cpu().numpy() 117 | label_result = label_result.cpu().numpy() 118 | stats = evaluate_stats(pred_result, label_result) 119 | print(stats) 120 | 121 | # test dataset evaluation 122 | pred_result, label_result = trainer.test(test_dataset) 123 | 124 | pred_result = pred_result.cpu().numpy() 125 | label_result = label_result.cpu().numpy() 126 | # print(pred_result) 127 | # print(label_result) 128 | stats = evaluate_stats(pred_result, label_result) 129 | print(stats) 130 | return stats -------------------------------------------------------------------------------- /nas_lib/model_predictor/trainer/gin_trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) XiDian University and Xi'an University of Posts&Telecommunication. All Rights Reserved 2 | 3 | import torch 4 | import random 5 | 6 | from torch_geometric.data import Data, Batch 7 | from torch_geometric.loader import DataLoader 8 | from nas_lib.model_predictor.trainer.utils import make_agent_optimizer, get_lr_scheduler 9 | 10 | 11 | class RankingLoss(torch.nn.Module): 12 | def __init__(self, margin=0.1): 13 | super(RankingLoss, self).__init__() 14 | self.margin = margin 15 | self.mse_loss = torch.nn.MSELoss() 16 | 17 | def forward(self, pred, target): 18 | loss = torch.tensor(0) 19 | for i in range(len(target)): 20 | Tj = target < target[i] 21 | si = pred[i] 22 | sj = pred[Tj] 23 | if len(sj) == 0: 24 | continue 25 | loss = loss + torch.sum(torch.maximum(torch.tensor(0), self.margin - (si - sj))) / len(sj) 26 | loss = loss / len(pred) 27 | return loss 28 | 29 | 30 | class GinPredictorTrainer: 31 | def __init__(self, 32 | predictor_agent, 33 | lr=0.01, 34 | device=None, 35 | epochs=10, 36 | batch_size=10, 37 | scaling_factor=10, 38 | weight_decay=1e-4, 39 | bias_multiply=True, 40 | lr_curve="CosineAnnealingLR", 41 | loss="MSE" 42 | ): 43 | 44 | self.predictor_agent = predictor_agent 45 | 46 | if loss == "MSE": 47 | self.criterion = torch.nn.MSELoss() 48 | elif loss == "Ranking": 49 | self.criterion = RankingLoss() 50 | else: 51 | raise NotImplementedError() 52 | 53 | self.device = device 54 | self.predictor_agent.to(self.device) 55 | self.lr = lr 56 | 57 | self.batch_size = batch_size 58 | self.epoch = epochs 59 | self.scaling_factor = scaling_factor 60 | 61 | self.weight_decay = weight_decay 62 | 63 | self.bias_multiply = bias_multiply 64 | 65 | self.lr_curve = lr_curve 66 | 67 | def fit(self, dataset): 68 | data_size = len(dataset) 69 | data_loader = DataLoader(dataset=dataset, batch_size=self.batch_size, shuffle=True) 70 | 71 | self.optimizer = make_agent_optimizer(self.predictor_agent, base_lr=self.lr, weight_deacy=self.weight_decay, 72 | bias_multiply=self.bias_multiply) 73 | 74 | self.scheduler = get_lr_scheduler(self.optimizer, self.epoch, data_size, self.batch_size, lr=self.lr_curve) 75 | 76 | self.predictor_agent.train() 77 | 78 | for epoch in range(self.epoch): 79 | num_batch = 0 80 | for batch in data_loader: 81 | num_batch += 1 82 | if isinstance(batch, list): 83 | for data in batch: 84 | data.to(self.device) 85 | val_tensor = batch[0].y 86 | else: 87 | batch = batch.to(self.device) 88 | val_tensor = batch.y 89 | 90 | pred = self.predictor_agent(batch) * self.scaling_factor 91 | pred = pred.squeeze() 92 | loss = self.criterion(pred, val_tensor) 93 | # print(loss) 94 | self.optimizer.zero_grad() 95 | loss.backward() 96 | self.optimizer.step() 97 | 98 | self.scheduler.step() # NOTE: the original code uses: self.scheduler.step(epoch + int(i/30)) 99 | # print(num_batch) 100 | 101 | def pred(self, dataset, batch_size = 64): 102 | data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=False) 103 | pred_list = [] 104 | 105 | self.predictor_agent.eval() 106 | with torch.no_grad(): 107 | for batch in data_loader: 108 | 109 | if isinstance(batch, list): 110 | for data in batch: 111 | data.to(self.device) 112 | else: 113 | batch = batch.to(self.device) 114 | pred = self.predictor_agent(batch) * self.scaling_factor 115 | pred = pred.squeeze() 116 | 117 | if len(pred.size()) == 0: 118 | pred.unsqueeze(0) 119 | pred_list.append(pred) 120 | return torch.cat(pred_list, dim=0) 121 | 122 | def test(self, dataset, batch_size = 64): 123 | data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=False) 124 | pred_list = [] 125 | valid_list = [] 126 | self.predictor_agent.eval() 127 | with torch.no_grad(): 128 | for batch in data_loader: 129 | if isinstance(batch, list): 130 | for data in batch: 131 | data.to(self.device) 132 | val_tensor = batch[0].y 133 | else: 134 | batch = batch.to(self.device) 135 | val_tensor = batch.y 136 | pred = self.predictor_agent(batch) * self.scaling_factor 137 | pred = pred.squeeze() 138 | 139 | if len(pred.size()) == 0: 140 | pred.unsqueeze(0) 141 | pred_list.append(pred) 142 | valid_list.append(val_tensor) 143 | 144 | # print(pred_list) 145 | # print(valid_list) 146 | return torch.cat(pred_list, dim=0), torch.cat(valid_list, dim=0) 147 | 148 | 149 | def test_ranking_loss(): 150 | y = torch.tensor([2,1,5,4,3]) 151 | x = torch.tensor([1,2,3,4,5]) 152 | loss = RankingLoss() 153 | print(loss(y,x)) 154 | 155 | if __name__ == '__main__': 156 | test_ranking_loss() -------------------------------------------------------------------------------- /experiments/algorithms/NPENAS.py: -------------------------------------------------------------------------------- 1 | 2 | # utils 3 | from experiments.utils import evaluate_stats 4 | 5 | # trainer 6 | from nas_lib.model_predictor.trainer.gin_trainer import GinPredictorTrainer 7 | 8 | # algorithms 9 | from nas_lib.model_predictor.agent.npenas_gin_predictor import NPENASGinPredictorAgent 10 | from nas_lib.model_predictor.agent.fr_gin_predictor import DirectedGinPredictorAgent 11 | 12 | from nas_lib.model_predictor.agent.multi_transform_gin_predictor import MultiTransformGinPredictorAgent 13 | from nas_lib.model_predictor.agent.multi_transform_gin_predictor import multi_transform, undirected_transform, \ 14 | no_transform, reverse_transform 15 | 16 | 17 | from nas_lib.model_predictor.search_space.nasbench101_search_space import NasBench101SearchSpace 18 | from nas_lib.model_predictor.dataset.nasbench101_dataset import NasBench101Dataset 19 | from nas_lib.model_predictor.dataset.nasbench201_dataset import NasBench201Dataset 20 | from nas_lib.model_predictor.dataset.darts_dataset import DartsDataset 21 | 22 | AvailableDataset = { 23 | "nasbench-101": NasBench101Dataset, 24 | "nasbench-201": NasBench201Dataset, 25 | "darts": DartsDataset, 26 | } 27 | 28 | 29 | def NPENAS(search_space, train_archs, test_archs, learning_rate, batch_size, epochs, scaling_factor, weight_decay, device): 30 | 31 | Dataset = AvailableDataset[search_space] 32 | 33 | train_dataset = Dataset(arch_data_list=train_archs) 34 | test_dataset = Dataset(arch_data_list=test_archs) 35 | 36 | agent = NPENASGinPredictorAgent(input_dim=Dataset.input_dim) 37 | trainer = GinPredictorTrainer(agent, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, weight_decay=weight_decay, device=device) 38 | trainer.fit(train_dataset) 39 | 40 | # train dataset evaluation 41 | pred_result, label_result = trainer.test(train_dataset) 42 | pred_result = pred_result.cpu().numpy() 43 | label_result = label_result.cpu().numpy() 44 | stats = evaluate_stats(pred_result, label_result) 45 | print(stats) 46 | 47 | # test dataset evaluation 48 | pred_result, label_result = trainer.test(test_dataset) 49 | 50 | pred_result = pred_result.cpu().numpy() 51 | label_result = label_result.cpu().numpy() 52 | stats = evaluate_stats(pred_result, label_result) 53 | print(stats) 54 | return stats 55 | 56 | 57 | 58 | def NPENAS_ensemble_transform(search_space, train_archs, test_archs, learning_rate, batch_size, epochs, scaling_factor, weight_decay, device, reverse=True): 59 | for arch in train_archs: 60 | arch["matrix_forward"] = no_transform(arch["matrix"]) 61 | if reverse: 62 | arch["matrix_backward"] = reverse_transform(arch["matrix"]) 63 | else: 64 | arch["matrix_backward"] = no_transform(arch["matrix"]) 65 | 66 | for arch in test_archs: 67 | arch["matrix_forward"] = no_transform(arch["matrix"]) 68 | if reverse: 69 | arch["matrix_backward"] = reverse_transform(arch["matrix"]) 70 | else: 71 | arch["matrix_backward"] = no_transform(arch["matrix"]) 72 | 73 | Dataset = AvailableDataset[search_space] 74 | 75 | train_dataset_forward = Dataset(arch_data_list=train_archs, key_matrix="matrix_forward") 76 | test_dataset_forward = Dataset(arch_data_list=test_archs, key_matrix="matrix_forward") 77 | 78 | train_dataset_backward = Dataset(arch_data_list=train_archs, key_matrix="matrix_backward") 79 | test_dataset_backward = Dataset(arch_data_list=test_archs, key_matrix="matrix_backward") 80 | 81 | agent_forward = NPENASGinPredictorAgent(input_dim=Dataset.input_dim) 82 | trainer_forward = GinPredictorTrainer(agent_forward, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, weight_decay=weight_decay, device=device) 83 | trainer_forward.fit(train_dataset_forward) 84 | 85 | agent_backward = NPENASGinPredictorAgent(input_dim=Dataset.input_dim) 86 | trainer_backward = GinPredictorTrainer(agent_backward, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, weight_decay=weight_decay, device=device) 87 | trainer_backward.fit(train_dataset_backward) 88 | 89 | pred_result_forward, label_result_forward = trainer_forward.test(test_dataset_forward) 90 | pred_result_backward, label_result_backward = trainer_backward.test(test_dataset_backward) 91 | 92 | if not label_result_forward.equal(label_result_backward): 93 | raise Exception("disorder!") 94 | 95 | pred_result_forward = pred_result_forward.cpu().numpy() 96 | pred_result_backward = pred_result_backward.cpu().numpy() 97 | pred_result = (pred_result_forward + pred_result_backward) / 2 98 | 99 | label_result = label_result_forward.cpu().numpy() 100 | 101 | stats = evaluate_stats(pred_result, label_result) 102 | print(stats) 103 | return stats 104 | 105 | 106 | def NPENAS_Directed(search_space, train_archs, test_archs, learning_rate, batch_size, epochs, scaling_factor, weight_decay, device): 107 | 108 | Dataset = AvailableDataset[search_space] 109 | 110 | train_dataset = Dataset(arch_data_list=train_archs) 111 | test_dataset = Dataset(arch_data_list=test_archs) 112 | 113 | agent = DirectedGinPredictorAgent(input_dim=Dataset.input_dim) 114 | trainer = GinPredictorTrainer(agent, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, weight_decay=weight_decay, device=device) 115 | trainer.fit(train_dataset) 116 | 117 | # train dataset evaluation 118 | pred_result, label_result = trainer.test(train_dataset) 119 | pred_result = pred_result.cpu().numpy() 120 | label_result = label_result.cpu().numpy() 121 | stats = evaluate_stats(pred_result, label_result) 122 | print(stats) 123 | 124 | # test dataset evaluation 125 | pred_result, label_result = trainer.test(test_dataset) 126 | 127 | pred_result = pred_result.cpu().numpy() 128 | label_result = label_result.cpu().numpy() 129 | stats = evaluate_stats(pred_result, label_result) 130 | print(stats) 131 | return stats -------------------------------------------------------------------------------- /nas_lib/model_predictor/search_space/nasbench201_search_space.py: -------------------------------------------------------------------------------- 1 | import random 2 | from nas_lib.model_predictor.search_space.search_space import DatabaseSearchSpace 3 | 4 | from evoxbench.benchmarks.nb201 import NASBench201SearchSpace 5 | from evoxbench.benchmarks.nb201 import NASBench201Evaluator # the evaluator uses database 6 | 7 | import numpy as np 8 | 9 | OPS = { 10 | 'input': 0, 11 | 'skip_connect': 1, 12 | 'nor_conv_1x1': 2, 13 | 'nor_conv_3x3': 3, 14 | 'avg_pool_3x3': 4, 15 | 'output': 5, 16 | 'isolate': 6, 17 | } 18 | 19 | OP_LIST = ['none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] 20 | 21 | 22 | NODE_NUM = 8 23 | 24 | 25 | class NasBench201SearchSpace(DatabaseSearchSpace): 26 | def __init__(self, 27 | determistic=True, 28 | dataset='cifar10', # ['cifar10', 'cifar100', 'ImageNet16-120'] 29 | ): 30 | super(NasBench201SearchSpace, self).__init__() 31 | 32 | self.determistic = determistic 33 | self.search_space = NASBench201SearchSpace() 34 | self.evaluator = NASBench201Evaluator(objs="err", dataset=dataset) 35 | 36 | def query(self, archs): 37 | """ 38 | query archs 39 | :param archs: lists of arch dict with index {'index': [0, 3, 1, 2, 4, 4], ...} 40 | """ 41 | # query_result = [] 42 | # for i, arch in enumerate(archs): 43 | # # TODO: implement this 44 | # 45 | # query_result.append(result) 46 | # query_result = self._extract_data(data_list=query_result) 47 | # return query_result 48 | raise NotImplementedError() 49 | 50 | def sample_data(self, sample_size): 51 | """ 52 | sample from nasbench-201 53 | """ 54 | return self.sample_data_uniform(sample_size=sample_size) 55 | 56 | def sample_data_uniform(self, sample_size): 57 | """ 58 | Uniformly sample from nasbench-201, i.e. choose operations randomly 59 | Using interface from evoxbench 60 | """ 61 | print("sampling data in nasbench-201: {}".format(sample_size)) 62 | sample = self.search_space.sample(n_samples=sample_size) 63 | # sample : ['|none~0|+|skip_connect~0|nor_conv_3x3~1|+|skip_connect~0|skip_connect~1|none~2|', ] 64 | 65 | performance_valid = self.evaluator.evaluate(archs=sample, true_valid=True) 66 | performance_test = self.evaluator.evaluate(archs=sample, true_eval=True) 67 | # print(sample) 68 | # print(performance_valid) 69 | # print(performance_test) 70 | data_list = [] 71 | for i in range(len(sample)): 72 | data_list.append({ 73 | "code": sample[i], 74 | "index": self.search_space.encode([sample[i]])[0].tolist(), 75 | "valid_error": performance_valid[i]["err"], 76 | "test_error": performance_test[i]["err"], 77 | }) 78 | 79 | result = self._extract_data(data_list=data_list) 80 | return result 81 | 82 | def _extract_data(self, data_list): 83 | """ 84 | data_list: sample ['|none~0|+|skip_connect~0|nor_conv_3x3~1|+|skip_connect~0|skip_connect~1|none~2|', ] 85 | """ 86 | output_data = [] 87 | for data in data_list: 88 | 89 | genotype = self._str_to_genotypes(data["code"]) 90 | 91 | matrix, ops = self._genotype_to_nodes_edges(genotype) 92 | 93 | output_data.append( 94 | { 95 | "index": data["index"], 96 | "matrix": matrix, 97 | "ops": ops, 98 | "valid_error": data["valid_error"], # No test error? 99 | "test_error": data["test_error"] 100 | } 101 | ) 102 | return output_data 103 | 104 | def _genotype_to_nodes_edges(self, genotype_data): 105 | """ 106 | codes from NPENAS 107 | """ 108 | ops = ['input'] 109 | data_list = [] 110 | for k in genotype_data: 111 | data_list.append(k) 112 | ops.append(data_list[0][0][0]) # 0--->1 113 | ops.append(data_list[1][0][0]) # 0--->2 114 | ops.append(data_list[2][0][0]) # 0--->3 115 | ops.append(data_list[1][1][0]) # 1--->4 116 | ops.append(data_list[2][1][0]) # 1--->5 117 | ops.append(data_list[2][2][0]) # 2--->6 118 | ops.append('output') 119 | 120 | adjacency_matrix = np.zeros((8, 8)) 121 | adjacency_matrix[0, 1] = 1 122 | adjacency_matrix[0, 2] = 1 123 | adjacency_matrix[0, 3] = 1 124 | adjacency_matrix[1, 4] = 1 125 | adjacency_matrix[1, 5] = 1 126 | adjacency_matrix[2, 6] = 1 127 | adjacency_matrix[4, 6] = 1 128 | adjacency_matrix[3, 7] = 1 129 | adjacency_matrix[5, 7] = 1 130 | adjacency_matrix[6, 7] = 1 131 | 132 | del_idxs = [id for id, op in enumerate(ops) if op == 'none'] 133 | ops = [op for op in ops if op != 'none'] 134 | 135 | counter = 0 136 | for id in del_idxs: 137 | temp_id = id - counter 138 | adjacency_matrix = np.delete(adjacency_matrix, temp_id, axis=0) 139 | adjacency_matrix = np.delete(adjacency_matrix, temp_id, axis=1) 140 | counter += 1 141 | return adjacency_matrix, ops 142 | 143 | def _str_to_genotypes(self, xstr): 144 | assert isinstance(xstr, str), 'must take string (not {:}) as input'.format(type(xstr)) 145 | nodestrs = xstr.split('+') 146 | genotypes = [] 147 | for i, node_str in enumerate(nodestrs): 148 | inputs = list(filter(lambda x: x != '', node_str.split('|'))) 149 | for xinput in inputs: assert len(xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput) 150 | inputs = (xi.split('~') for xi in inputs) 151 | input_infos = tuple((op, int(IDX)) for (op, IDX) in inputs) 152 | genotypes.append(input_infos) 153 | return genotypes 154 | 155 | 156 | if __name__ == '__main__': 157 | # query_result = NASBench201Result.objects.order_by('?')[:2] 158 | serach_space = NasBench201SearchSpace() 159 | query_result = serach_space.sample_data(5) 160 | print(query_result) 161 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/agent/multi_transform_gin_predictor.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import Sequential, Linear, ReLU 5 | from torch_geometric.nn import GINConv 6 | from torch_geometric.nn import global_mean_pool 7 | import numpy 8 | 9 | 10 | class MultiTransformGinPredictorAgent(nn.Module): 11 | def __init__(self, input_dim=6, num_transforms=2): 12 | super(MultiTransformGinPredictorAgent, self).__init__() 13 | 14 | self.num_transforms = num_transforms 15 | self.input_dim = input_dim 16 | self.dim = 36 17 | self.linear_dim = 54 18 | self.out_dim = 18 19 | 20 | self.build_gnn_block() 21 | 22 | self.mlp_layers = nn.ModuleList() 23 | for i in range(num_transforms): 24 | self.mlp_layers.append(Linear(self.dim, self.linear_dim, bias=True)) 25 | 26 | self.mlp_layers.append(Linear(self.dim * num_transforms, self.linear_dim, bias=True)) 27 | self.mlp_layers.append(Linear(self.linear_dim, self.out_dim)) 28 | self.mlp_layers.append(Linear(self.out_dim, 1)) 29 | self.out_layer = nn.Sigmoid() 30 | self.init_mlp() 31 | 32 | def init_mlp(self): 33 | for layer in self.mlp_layers: 34 | if isinstance(layer, nn.Linear): 35 | nn.init.kaiming_uniform_(layer.weight, a=1) 36 | nn.init.constant_(layer.bias, 0) 37 | 38 | def build_gnn_block(self): 39 | input_dim = self.input_dim 40 | dim = self.dim 41 | num_transforms = self.num_transforms 42 | 43 | self.gnn_list = nn.ModuleList() 44 | 45 | self.gnn_weight_mask = [True for _ in range(num_transforms)] 46 | self.gnn_forward_switch = -1 47 | 48 | for _ in range(num_transforms): 49 | nn1 = Sequential(Linear(input_dim, dim, bias=True), ReLU(), Linear(dim, dim, bias=True)) 50 | conv1 = GINConv(nn1) 51 | bn1 = torch.nn.BatchNorm1d(dim) 52 | 53 | nn2 = Sequential(Linear(dim, dim, bias=True), ReLU(), Linear(dim, dim, bias=True)) 54 | conv2 = GINConv(nn2) 55 | bn2 = torch.nn.BatchNorm1d(dim) 56 | 57 | nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim)) 58 | conv3 = GINConv(nn3) 59 | bn3 = torch.nn.BatchNorm1d(dim) 60 | 61 | self.gnn_list.append(nn.ModuleList([conv1, bn1, conv2, bn2, conv3, bn3])) 62 | 63 | def forward(self, graph_data_list): 64 | x_embedding_list = [] 65 | # print("-"*30) 66 | gnn_index = 0 67 | for graph_data in graph_data_list: 68 | if self.gnn_forward_switch < 0 or self.gnn_forward_switch == gnn_index: 69 | x = graph_data.x 70 | edge_index = graph_data.edge_index 71 | batch = graph_data.batch 72 | # print(x) 73 | # print(edge_index) 74 | # print(batch) 75 | x_embedding = self.forward_batch(x, edge_index, batch, gnn_index) 76 | x_embedding_list.append(x_embedding) 77 | gnn_index += 1 78 | 79 | # print(x_embedding_list) 80 | x_embedding = torch.cat(x_embedding_list, dim=1) # note the dim 0 are for batches 81 | if self.gnn_forward_switch < 0: 82 | x_embedding_all = F.relu(self.mlp_layers[-3](x_embedding)) 83 | else: 84 | x_embedding_all = F.relu(self.mlp_layers[self.gnn_forward_switch](x_embedding)) 85 | 86 | x_embedding = F.relu(self.mlp_layers[-2](x_embedding_all)) 87 | x_embedding_drop = F.dropout(x_embedding, p=0.1, training=self.training) 88 | mean = self.mlp_layers[-1](x_embedding_drop) 89 | mean = self.out_layer(mean) 90 | return mean 91 | 92 | def forward_batch(self, x, edge_index, batch, gnn_index): 93 | x1 = x 94 | index = 0 95 | for layer in self.gnn_list[gnn_index]: 96 | if index % 2 == 0: 97 | x1 = F.relu(layer(x1, edge_index)) 98 | else: 99 | x1 = layer(x1) 100 | index += 1 101 | x_embedding = global_mean_pool(x1, batch) 102 | return x_embedding 103 | 104 | def set_gnn_weight_masking(self, gnn_weight_mask): 105 | """ 106 | mask: one dim list, [True,False] indicating first gnn is activated 107 | weight controls gradient calculation 108 | forward controls calculation 109 | """ 110 | self.gnn_weight_mask = gnn_weight_mask 111 | 112 | # freezing weight 113 | index = 0 114 | for gnn in self.gnn_list: 115 | if not gnn_weight_mask[index]: 116 | for params in gnn.parameters(): 117 | params.requires_grad = False 118 | else: 119 | for params in gnn.parameters(): 120 | params.requires_grad = True 121 | index += 1 122 | 123 | def switch_gnn_forward(self, gnn_forward_switch): 124 | """ 125 | 126 | :param gnn_forward_switch: 127 | :return: 128 | """ 129 | self.gnn_forward_switch = gnn_forward_switch 130 | # freezing weight 131 | gnn_weight_mask = [False for _ in range(self.num_transforms)] 132 | if gnn_forward_switch == -1: 133 | gnn_weight_mask = [True for _ in range(self.num_transforms)] 134 | else: 135 | gnn_weight_mask[gnn_forward_switch] = True 136 | self.set_gnn_weight_masking(gnn_weight_mask=gnn_weight_mask) 137 | 138 | 139 | def multi_transform(matrix): 140 | transform_matrix_list = [] 141 | transform_matrix_list.append(no_transform(matrix)) 142 | transform_matrix_list.append(reverse_transform(matrix)) 143 | return transform_matrix_list 144 | 145 | 146 | def no_transform(matrix): 147 | matrix = numpy.array(matrix) 148 | assert len(matrix.shape) == 2 and matrix.shape[0] == matrix.shape[1] 149 | new_matrix = [] 150 | 151 | for i in range(matrix.shape[0]): 152 | row = [] 153 | for j in range(matrix.shape[1]): 154 | row.append(matrix[i][j]) 155 | new_matrix.append(row) 156 | 157 | return matrix 158 | 159 | 160 | def reverse_transform(matrix): 161 | matrix = numpy.array(matrix) 162 | assert len(matrix.shape) == 2 and matrix.shape[0] == matrix.shape[1] 163 | matrix = matrix.transpose() 164 | new_matrix = [] 165 | for i in range(matrix.shape[0]): 166 | row = [] 167 | for j in range(matrix.shape[1]): 168 | row.append(matrix[i][j]) 169 | new_matrix.append(row) 170 | 171 | return matrix 172 | 173 | 174 | def undirected_transform(matrix): 175 | """ 176 | :param matrix: square matrix containing only 0 and 1 177 | """ 178 | matrix = numpy.array(matrix) 179 | assert len(matrix.shape) == 2 and matrix.shape[0] == matrix.shape[1] 180 | transpose_matrix = matrix.transpose() 181 | new_matrix = [] 182 | for i in range(matrix.shape[0]): 183 | row = [] 184 | for j in range(matrix.shape[1]): 185 | row.append(min(matrix[i][j] + transpose_matrix[i][j], 1)) 186 | new_matrix.append(row) 187 | return new_matrix 188 | -------------------------------------------------------------------------------- /experiments/bias_analysis/sample_size_irg_change_compare.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | 4 | import numpy 5 | import numpy as np 6 | import torch 7 | 8 | # utils 9 | from experiments.utils import set_random_seed 10 | from experiments.utils import evaluate_stats 11 | 12 | # datas 13 | from nas_lib.model_predictor.search_space.nasbench101_search_space import NasBench101SearchSpace 14 | from nas_lib.model_predictor.search_space.nasbench201_search_space import NasBench201SearchSpace 15 | from nas_lib.model_predictor.search_space.darts_search_space import DartsSearchSpace 16 | from nas_lib.model_predictor.dataset.nasbench101_dataset import NasBench101MultiTransformDataset, NasBench101Dataset 17 | from nas_lib.model_predictor.dataset.nasbench201_dataset import NasBench201MultiTransformDataset, NasBench201Dataset 18 | from nas_lib.model_predictor.dataset.darts_dataset import DartsMultiTransformDataset, DartsDataset 19 | 20 | # trainer 21 | from nas_lib.model_predictor.trainer.fr_gin_trainer import FRGinPredictorTrainer 22 | from nas_lib.model_predictor.trainer.gin_trainer import GinPredictorTrainer 23 | 24 | # algorithms 25 | 26 | from nas_lib.model_predictor.agent.multi_transform_gin_predictor import multi_transform 27 | 28 | from nas_lib.model_predictor.agent.fr_gin_predictor import FRGinPredictorAgent 29 | from nas_lib.model_predictor.agent.fr_gin_predictor import FRGinPredictorAgentAfter 30 | 31 | AvailableMultiTransformDataset = { 32 | "nasbench-101": NasBench101MultiTransformDataset, 33 | "nasbench-201": NasBench201MultiTransformDataset, 34 | "darts": DartsMultiTransformDataset, 35 | } 36 | 37 | AvailableDataset = { 38 | "nasbench-101": NasBench101Dataset, 39 | "nasbench-201": NasBench201Dataset, 40 | "darts": DartsDataset, 41 | } 42 | 43 | AvailableSearchSpace = { 44 | 45 | "nasbench-101": NasBench101SearchSpace, 46 | "nasbench-201": NasBench201SearchSpace, 47 | "darts": DartsSearchSpace, 48 | } 49 | DEVICE = "cpu" 50 | 51 | 52 | def get_IRG( features): 53 | dist_matrix_list = [] 54 | for feature in features: 55 | dist_matrix = torch.norm(feature[:, None] - feature, dim=2, p=2) # broadcast, calculate A 56 | dist_matrix_list.append(dist_matrix) 57 | return dist_matrix_list 58 | 59 | def FRNAS(search_space, train_archs, test_archs, draw_archs, learning_rate, batch_size, epochs, scaling_factor, device, weight_factor, weight_decay, loss="IRG"): 60 | for arch in train_archs: 61 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 62 | 63 | for arch in test_archs: 64 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 65 | 66 | Dataset = AvailableMultiTransformDataset[search_space] 67 | 68 | train_dataset = Dataset(arch_data_list=train_archs, device=device) 69 | test_dataset = Dataset(arch_data_list=test_archs, device=device) 70 | draw_dataset = Dataset(arch_data_list=draw_archs, device=device) 71 | 72 | agent_1 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 73 | agent_2 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 74 | 75 | trainer = FRGinPredictorTrainer(agent_1, agent_2, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, 76 | weight_factor=weight_factor, weight_decay=weight_decay, 77 | device=device, loss=loss) 78 | 79 | trainer.fit(train_dataset) 80 | 81 | 82 | 83 | # test dataset evaluation 84 | 85 | pred_result, label_result = trainer.test(test_dataset) 86 | pred_result = pred_result.cpu().numpy() 87 | label_result = label_result.cpu().numpy() 88 | # print(pred_result) 89 | # print(label_result) 90 | stats = evaluate_stats(pred_result, label_result) 91 | 92 | pred_result, label_result = trainer.test(test_dataset) 93 | forward_feats = agent_1.features 94 | reverse_feats = agent_2.features 95 | forward_feats = get_IRG(forward_feats) 96 | reverse_feats = get_IRG(reverse_feats) 97 | forward_feats = forward_feats[0].cpu().numpy() 98 | reverse_feats = reverse_feats[0].cpu().numpy() 99 | 100 | # print(reverse_feats - forward_feats) 101 | 102 | return forward_feats, reverse_feats, stats 103 | 104 | def run(train_archs, test_archs, draw_archs, weight_factor, name): 105 | stats = FRNAS(search_space=args.search_space, 106 | train_archs=train_archs, 107 | test_archs=test_archs, 108 | draw_archs=draw_archs, 109 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 110 | weight_decay=1e-4, 111 | weight_factor=weight_factor, 112 | device=DEVICE, 113 | loss="IRG") 114 | 115 | result = numpy.sum(numpy.abs(stats[0]-stats[1])) 116 | performance = stats[2]["kendalltau"][0] 117 | print(result) 118 | return result, performance 119 | 120 | 121 | def main(learning_rate, sampled_archs, sample_size, sample_size_2, sample_size_3, test_size): 122 | data_dict = {} 123 | 124 | # the dataset is fixed, we only do retrains 125 | train_archs = sampled_archs[:sample_size] 126 | train_archs_2 = sampled_archs[:sample_size_2] 127 | train_archs_3 = sampled_archs[:sample_size_3] 128 | test_archs = sampled_archs[sample_size_3:] 129 | draw_archs = sampled_archs[:32] 130 | 131 | print("Running") 132 | 133 | result_50 = run(train_archs, test_archs, draw_archs, 0, "IRG_feat_50") 134 | # run(train_archs_2, test_archs, draw_archs, 0, "IRG_feat_200_0") 135 | result_400 = run(train_archs_3, test_archs, draw_archs, 0, "IRG_feat_400") 136 | 137 | 138 | return {"IRG_feat_50":result_50, "IRG_feat_400":result_400} 139 | 140 | 141 | if __name__ == '__main__': 142 | parser = argparse.ArgumentParser(description='Args for algorithms compare.') 143 | parser.add_argument('--search_space', type=str, default="nasbench-201", choices=["nasbench-101", "nasbench-201", "darts"], help='search space') 144 | parser.add_argument('--trials', type=int, default=50, help='Number of trials') 145 | parser.add_argument('--seed', type=int, default=24657, help='random seed') 146 | parser.add_argument('--lr', type=float, default=0.005, help='Loss used to train architecture.') 147 | parser.add_argument('--sample_size', type=int, default=50, help='Number of sampled architectures') 148 | parser.add_argument('--sample_size_2', type=int, default=200, help='Number of sampled architectures') 149 | parser.add_argument('--sample_size_3', type=int, default=400, help='Number of sampled architectures') 150 | parser.add_argument('--test_size', type=int, default=5000, help='Number of test architectures') 151 | args = parser.parse_args() 152 | # print(args) 153 | set_random_seed(args.seed) 154 | 155 | trials_result_dict = {} 156 | 157 | search_space = AvailableSearchSpace[args.search_space]() 158 | for trial in range(args.trials): 159 | print("trial {}".format(trial)) 160 | sampled_archs = search_space.sample_data_uniform(args.sample_size_3 + args.test_size) 161 | result_dict = main(args.lr, sampled_archs, args.sample_size, args.sample_size_2, args.sample_size_3, args.test_size) 162 | trials_result_dict[trial] = result_dict 163 | 164 | 165 | with open("result_sample_size_irg_change_compare.json", "w") as f: 166 | json.dump(trials_result_dict, f) 167 | 168 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/agent/fr_gin_predictor.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import Sequential, Linear, ReLU 5 | from torch_geometric.nn import GINConv 6 | from torch_geometric.nn import global_mean_pool 7 | 8 | 9 | 10 | class DirectedGINConv(nn.Module): 11 | """ 12 | "To make information flow both ways, we always use the average of two GCN layers: 13 | one where we use A to propagate information in the forward directions 14 | and another where we use A^T to reverse the direction" 15 | Reimplement in GIN way, using NPNAS idea using directed convolution 16 | """ 17 | 18 | def __init__(self, in_dim, out_dim): 19 | super(DirectedGINConv, self).__init__() 20 | nn1 = Sequential(Linear(in_dim, out_dim, bias=True), ReLU(), Linear(out_dim, out_dim, bias=True)) 21 | nn2 = Sequential(Linear(in_dim, out_dim, bias=True), ReLU(), Linear(out_dim, out_dim, bias=True)) 22 | self.forward_layer = GINConv(nn1) 23 | self.backward_layer = GINConv(nn2) 24 | self.batch_norm = nn.BatchNorm1d(out_dim) 25 | 26 | def forward(self, x, edge_index): 27 | output1 = F.relu(self.forward_layer(x, edge_index)) 28 | reverse_idx = edge_index[[1, 0]] # swap the order 29 | output2 = F.relu(self.backward_layer(x, reverse_idx)) 30 | output = (output1 + output2) / 2 31 | output = self.batch_norm(output) 32 | return output 33 | 34 | class DirectedGinPredictorAgent(nn.Module): 35 | ''' 36 | Reimplement the method 37 | ''' 38 | 39 | def __init__(self, input_dim=6, encoder_dim=32, decoder_dim=16): 40 | super(DirectedGinPredictorAgent, self).__init__() 41 | 42 | self.directed_gin_1 = DirectedGINConv(in_dim=input_dim, out_dim=encoder_dim) 43 | self.directed_gin_2 = DirectedGINConv(in_dim=encoder_dim, out_dim=encoder_dim) 44 | self.directed_gin_3 = DirectedGINConv(in_dim=encoder_dim, out_dim=encoder_dim) 45 | 46 | self.linear_before = torch.nn.Linear(encoder_dim, decoder_dim, bias=True) 47 | 48 | self.linear_mean = Linear(decoder_dim, 1) 49 | self.out_layer = torch.nn.Sigmoid() 50 | 51 | self.features = [] 52 | 53 | 54 | def forward(self, graph_data): 55 | x = graph_data.x 56 | edge_index = graph_data.edge_index 57 | batch = graph_data.batch 58 | x = self.forward_batch(x, edge_index, batch) 59 | return x 60 | 61 | def forward_batch(self, x, edge_index, batch): 62 | x = self.directed_gin_1(x, edge_index) 63 | x = self.directed_gin_2(x, edge_index) 64 | x = self.directed_gin_3(x, edge_index) 65 | 66 | x_embedding = global_mean_pool(x, batch) 67 | 68 | x_embedding_mean = F.relu(self.linear_before(x_embedding)) 69 | x_embedding_drop = F.dropout(x_embedding_mean, p=0.1, training=self.training) 70 | mean = self.linear_mean(x_embedding_drop) 71 | mean = self.out_layer(mean) 72 | return mean 73 | 74 | # using normal GIN predictor 75 | 76 | 77 | class GIN_Encoder(nn.Module): 78 | def __init__(self, input_dim=6, hidden_dim=32): 79 | super(GIN_Encoder, self).__init__() 80 | 81 | dim = hidden_dim 82 | 83 | self.layers = nn.ModuleList() 84 | 85 | nn1 = Sequential(Linear(input_dim, dim, bias=True), ReLU(), Linear(dim, dim, bias=True)) 86 | self.layers.append(GINConv(nn1)) 87 | self.layers.append(torch.nn.BatchNorm1d(dim)) 88 | 89 | nn2 = Sequential(Linear(dim, dim, bias=True), ReLU(), Linear(dim, dim, bias=True)) 90 | self.layers.append(GINConv(nn2)) 91 | self.layers.append(torch.nn.BatchNorm1d(dim)) 92 | # 93 | nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim)) 94 | self.layers.append(GINConv(nn3)) 95 | self.layers.append(torch.nn.BatchNorm1d(dim)) 96 | 97 | self.num_layers = len(self.layers)//2 98 | 99 | def forward(self, x, edge_index, batch): 100 | for i in range(self.num_layers): 101 | x = self.layers[i*2](x, edge_index) 102 | x = self.layers[i*2+1](x) 103 | 104 | return x 105 | 106 | 107 | class FRGinPredictorAgentAfter(nn.Module): 108 | def __init__(self, encoder_forward, encoder_reverse, encoder_dim=32, decoder_dim=16): 109 | super(FRGinPredictorAgentAfter, self).__init__() 110 | 111 | self.gin_encoder_forward = encoder_forward 112 | self.gin_encoder_reverse = encoder_reverse 113 | 114 | for name, param in self.named_parameters(): 115 | param.requires_grad = False 116 | 117 | self.linear_before = torch.nn.Linear(encoder_dim * 2, decoder_dim * 2, bias=True) 118 | self.linear_middle = torch.nn.Linear(decoder_dim * 2, decoder_dim, bias=True) 119 | self.linear_mean = Linear(decoder_dim, 1) 120 | self.out_layer = torch.nn.Sigmoid() 121 | 122 | def forward(self, graph_data): 123 | 124 | x = graph_data.x 125 | edge_index = graph_data.edge_index 126 | batch = graph_data.batch 127 | x = self.forward_batch(x, edge_index, batch) 128 | return x 129 | 130 | def forward_batch(self, x, edge_index, batch): 131 | reverse_edge_idx = edge_index[[1, 0]] # swap the order 132 | x_forward = self.gin_encoder_forward(x, edge_index, batch) 133 | x_embedding_forward = global_mean_pool(x_forward, batch) 134 | 135 | x_reverse = self.gin_encoder_reverse(x, reverse_edge_idx, batch) 136 | x_embedding_reverse = global_mean_pool(x_reverse, batch) 137 | 138 | x_embedding = torch.cat((x_embedding_forward, x_embedding_reverse), 1) 139 | 140 | x_embedding_mean = F.relu(self.linear_before(x_embedding)) 141 | x_embedding_mean = F.relu(self.linear_middle(x_embedding_mean)) 142 | x_embedding_drop = F.dropout(x_embedding_mean, p=0.1, training=self.training) 143 | mean = self.linear_mean(x_embedding_drop) 144 | mean = self.out_layer(mean) 145 | return mean 146 | 147 | class FRGinPredictorAgent(nn.Module): 148 | ''' 149 | Reimplement the method 150 | ''' 151 | 152 | def __init__(self, input_dim=6, encoder_dim=32, decoder_dim=16): 153 | super(FRGinPredictorAgent, self).__init__() 154 | 155 | self.gin_encoder = GIN_Encoder(input_dim=input_dim, hidden_dim=encoder_dim) 156 | 157 | self.linear_before = torch.nn.Linear(encoder_dim, decoder_dim, bias=True) 158 | 159 | self.linear_mean = Linear(decoder_dim, 1) 160 | self.out_layer = torch.nn.Sigmoid() 161 | 162 | self.features = [] 163 | 164 | 165 | def forward(self, graph_data): 166 | self.features = [] 167 | 168 | x = graph_data.x 169 | edge_index = graph_data.edge_index 170 | batch = graph_data.batch 171 | x = self.forward_batch(x, edge_index, batch) 172 | return x 173 | 174 | def forward_batch(self, x, edge_index, batch): 175 | x = self.gin_encoder(x, edge_index, batch) 176 | 177 | x_embedding = global_mean_pool(x, batch) 178 | 179 | self.features.append(x_embedding) 180 | 181 | x_embedding_mean = F.relu(self.linear_before(x_embedding)) 182 | x_embedding_drop = F.dropout(x_embedding_mean, p=0.1, training=self.training) 183 | mean = self.linear_mean(x_embedding_drop) 184 | mean = self.out_layer(mean) 185 | return mean 186 | 187 | 188 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/search_space/darts_search_space.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy as np 4 | 5 | from nas_lib.model_predictor.search_space.search_space import DatabaseSearchSpace 6 | from evoxbench.benchmarks.darts import NASBench301Result 7 | 8 | from collections import namedtuple 9 | import json 10 | 11 | OPS = { 12 | 'skip_connect': 0, 13 | 'avg_pool_3x3': 1, 14 | 'max_pool_3x3': 2, 15 | 'sep_conv_3x3': 3, 16 | 'sep_conv_5x5': 4, 17 | 'dil_conv_3x3': 5, 18 | 'dil_conv_5x5': 6, 19 | 'normal_input_0': 7, 20 | 'normal_input_1': 8, 21 | 'reduce_input_0': 9, 22 | 'reduce_input_1': 10, 23 | 'add': 11, 24 | 'concat': 12, 25 | } 26 | 27 | Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat') 28 | Genotype_cell = namedtuple('CellGenotype', 'cell concat') 29 | 30 | 31 | class DartsSearchSpace(DatabaseSearchSpace): 32 | """ 33 | DARTS search space, using prediction result from EvoxBench and NasBench301 database 34 | Datasets from nasbench301 training data 35 | """ 36 | 37 | def __init__(self, determistic=True, graph_encoding="operation_on_node"): 38 | super(DartsSearchSpace, self).__init__() 39 | self.determistic = determistic 40 | self.graph_encoding = graph_encoding 41 | 42 | def query(self, archs): 43 | """ 44 | query archs 45 | :param archs: lists of arch: {"matrix":[square adjacent matrix], "ops": [original operations]} 46 | """ 47 | raise NotImplementedError() 48 | 49 | def sample_data(self, sample_size, from_database=True): 50 | if from_database: 51 | return self.sample_data_nb301_database(sample_size=sample_size) 52 | else: 53 | raise NotImplementedError() 54 | 55 | def sample_data_nb301_database(self, sample_size): 56 | """ 57 | Uniformly sample from nasbench-301 database, 58 | it samples and evaluates darts model using different sampling methods 59 | """ 60 | print("sampling data in nasbench-301: {}".format(sample_size)) 61 | query_set = NASBench301Result.objects.order_by('?')[:sample_size] 62 | # NOTE: this will return a list QuerySet, every time we get the value from same QuerySet will get 63 | # different result because of random shuffle , must assign value to get exact result. 64 | query_result = [] 65 | for query in query_set: 66 | query_result.append(query) 67 | 68 | result = self._extract_data(data_list=query_result) 69 | 70 | return result 71 | 72 | def sample_data_uniform(self, sample_size): 73 | return self.sample_data_nb301_database(sample_size) 74 | 75 | def _extract_data(self, data_list): 76 | output_data = [] 77 | for data in data_list: 78 | index = data.id 79 | genotype = data.genotype # the value is a str type, 80 | genotype = genotype.replace('\'', '\"') # note eval() method is not allowed 81 | genotype = json.loads(genotype) 82 | normal = [tuple(x) for x in genotype[0]] 83 | normal_concat = genotype[1] 84 | reduce = [tuple(x) for x in genotype[2]] 85 | reduce_concat = genotype[3] 86 | 87 | # print(genotype) 88 | # print(normal) 89 | # print(normal_concat) 90 | # print(reduce) 91 | # print(reduce_concat) 92 | 93 | genotype = Genotype(normal=normal, normal_concat=normal_concat, reduce=reduce, reduce_concat=reduce_concat) 94 | 95 | matrix, ops = self._genotype_to_nodes_edges(genotype) 96 | 97 | valid_acc = data.result["val_accuracy"] 98 | test_acc = data.result["test_accuracy"] 99 | 100 | valid_error = 100 - valid_acc 101 | test_error = 100 - test_acc 102 | 103 | output_data.append( 104 | { 105 | "index": index, 106 | "matrix": matrix, 107 | "ops": ops, 108 | "valid_error": valid_error, 109 | "test_error": test_error 110 | } 111 | ) 112 | return output_data 113 | 114 | def _cell_to_graph(self, cell, cell_concat, cell_type="normal"): 115 | """ 116 | given cell and cell concat, return the nodes and edge index 117 | """ 118 | 119 | assert cell_type in ["normal", "reduce"] 120 | state_index_list = [0, 1] 121 | ops_index_list = [] 122 | ops = ['{}_input_0'.format(cell_type), '{}_input_1'.format(cell_type)] 123 | edges = [] # list of edges (u,v) : u -> v 124 | node_num = 2 125 | op_num = 0 126 | state_num = 2 127 | for op, idx in cell: 128 | ops_index_list.append(node_num) 129 | ops.append(op) 130 | edges.append((state_index_list[idx], node_num)) 131 | node_num += 1 132 | op_num += 1 133 | 134 | # every output of two operator forms a new state 135 | if op_num % 2 == 0: 136 | state_index_list.append(node_num) 137 | ops.append('add') 138 | edges.append((ops_index_list[-1], node_num)) 139 | edges.append((ops_index_list[-2], node_num)) 140 | node_num += 1 141 | state_num += 1 142 | 143 | ops.append('concat') 144 | for idx in cell_concat: 145 | edges.append((state_index_list[idx], node_num)) 146 | node_num += 1 147 | 148 | return ops, edges 149 | 150 | def _genotype_to_nodes_edges(self, genotype_data): 151 | # normal cell 152 | normal_ops, normal_edges = self._cell_to_graph(cell=genotype_data.normal, 153 | cell_concat=genotype_data.normal_concat, 154 | cell_type="normal") 155 | # reduce cell 156 | reduce_ops, reduce_edges = self._cell_to_graph(cell=genotype_data.reduce, 157 | cell_concat=genotype_data.reduce_concat, 158 | cell_type="reduce") 159 | # merge two cells together 160 | num_nodes = len(normal_ops) 161 | new_reduce_edges = [] 162 | for u, v in reduce_edges: 163 | new_reduce_edges.append((u + num_nodes, v + num_nodes)) 164 | ops = normal_ops + reduce_ops 165 | edges = normal_edges + new_reduce_edges 166 | num_nodes = len(ops) 167 | matrix = np.zeros((num_nodes, num_nodes), dtype=np.int8) 168 | for u,v in edges: 169 | matrix[u,v] = 1 170 | 171 | matrix = matrix.tolist() 172 | 173 | return matrix, ops 174 | 175 | 176 | def _test_database(): 177 | print("total archs: ", len(NASBench301Result.objects.order_by('?')[:])) 178 | 179 | query_set = NASBench301Result.objects.order_by('?')[:2] 180 | query_result = [] 181 | for query in query_set: 182 | query_result.append(query) 183 | 184 | for query in query_result: 185 | print("------") 186 | print(query) 187 | print(query.id) 188 | print(query.phenotype) 189 | print(query.genotype) 190 | print(query.normal) 191 | print(query.normal_concat) 192 | print(query.reduce) 193 | print(query.reduce_concat) 194 | print(query.result) 195 | print(query.result["val_accuracy"]) 196 | print(query.result["test_accuracy"]) 197 | for query in query_result: 198 | print("------") 199 | print(query) 200 | print(query.id) 201 | print(query.phenotype) 202 | print(query.genotype) 203 | print(query.normal) 204 | print(query.normal_concat) 205 | print(query.reduce) 206 | print(query.reduce_concat) 207 | print(query.result) 208 | print(query.result["val_accuracy"]) 209 | print(query.result["test_accuracy"]) 210 | 211 | 212 | def _test(): 213 | darts_search_space = DartsSearchSpace() 214 | sample = darts_search_space.sample_data(1) 215 | print(sample) 216 | 217 | 218 | if __name__ == '__main__': 219 | _test_database() 220 | _test() 221 | -------------------------------------------------------------------------------- /experiments/bias_analysis/instance_relation_compare.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | 4 | import numpy 5 | import numpy as np 6 | import torch 7 | 8 | # utils 9 | from experiments.utils import set_random_seed 10 | from experiments.utils import evaluate_stats 11 | 12 | # datas 13 | from nas_lib.model_predictor.search_space.nasbench101_search_space import NasBench101SearchSpace 14 | from nas_lib.model_predictor.search_space.nasbench201_search_space import NasBench201SearchSpace 15 | from nas_lib.model_predictor.search_space.darts_search_space import DartsSearchSpace 16 | from nas_lib.model_predictor.dataset.nasbench101_dataset import NasBench101MultiTransformDataset, NasBench101Dataset 17 | from nas_lib.model_predictor.dataset.nasbench201_dataset import NasBench201MultiTransformDataset, NasBench201Dataset 18 | from nas_lib.model_predictor.dataset.darts_dataset import DartsMultiTransformDataset, DartsDataset 19 | 20 | # trainer 21 | from nas_lib.model_predictor.trainer.fr_gin_trainer import FRGinPredictorTrainer 22 | from nas_lib.model_predictor.trainer.gin_trainer import GinPredictorTrainer 23 | 24 | # algorithms 25 | 26 | from nas_lib.model_predictor.agent.multi_transform_gin_predictor import multi_transform 27 | 28 | from nas_lib.model_predictor.agent.fr_gin_predictor import FRGinPredictorAgent 29 | from nas_lib.model_predictor.agent.fr_gin_predictor import FRGinPredictorAgentAfter 30 | 31 | AvailableMultiTransformDataset = { 32 | "nasbench-101": NasBench101MultiTransformDataset, 33 | "nasbench-201": NasBench201MultiTransformDataset, 34 | "darts": DartsMultiTransformDataset, 35 | } 36 | 37 | AvailableDataset = { 38 | "nasbench-101": NasBench101Dataset, 39 | "nasbench-201": NasBench201Dataset, 40 | "darts": DartsDataset, 41 | } 42 | 43 | AvailableSearchSpace = { 44 | 45 | "nasbench-101": NasBench101SearchSpace, 46 | "nasbench-201": NasBench201SearchSpace, 47 | "darts": DartsSearchSpace, 48 | } 49 | DEVICE = "cpu" 50 | 51 | 52 | def get_IRG( features): 53 | dist_matrix_list = [] 54 | for feature in features: 55 | dist_matrix = torch.norm(feature[:, None] - feature, dim=2, p=2) # broadcast, calculate A 56 | dist_matrix_list.append(dist_matrix) 57 | return dist_matrix_list 58 | 59 | 60 | def plot(matrix, file_name, vmin=0, vmax=4, cmap="rocket_r", center=None): 61 | from matplotlib import pyplot as plt 62 | import seaborn as sns 63 | from matplotlib.backends.backend_pdf import PdfPages 64 | 65 | plt.rcParams["font.family"] = "serif" 66 | 67 | fig, axs = plt.subplots(1, 1) 68 | fig.set_figheight(5) 69 | fig.set_figwidth(6) 70 | 71 | sns.heatmap(data=matrix, vmin=vmin, vmax=vmax, cmap=cmap, center=center) 72 | 73 | # axs.grid(linestyle='--', which='both', linewidth='0.5', color=(0.9, 0.9, 0.9, 0.5)) 74 | axs.set_xlabel('Instance', fontsize=14) 75 | axs.set_ylabel('Instance', fontsize=14) 76 | 77 | # lg3 = plt.legend(loc='lower right', fontsize=14, numpoints=1) 78 | # lg3.get_frame().set_linewidth(0.75) 79 | # lg3.get_frame().set_edgecolor("k") 80 | # plt.gca().add_artist(lg3) 81 | 82 | with PdfPages('./{}.pdf'.format(file_name)) as pdf: 83 | # plt.plot(....) 84 | pdf.savefig(bbox_inches = 'tight') 85 | 86 | 87 | 88 | 89 | 90 | def FRNAS(search_space, train_archs, test_archs, draw_archs, learning_rate, batch_size, epochs, scaling_factor, device, weight_factor, weight_decay, loss="IRG"): 91 | for arch in train_archs: 92 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 93 | 94 | for arch in test_archs: 95 | arch["transform_matrix_list"] = multi_transform(arch["matrix"]) 96 | 97 | Dataset = AvailableMultiTransformDataset[search_space] 98 | 99 | train_dataset = Dataset(arch_data_list=train_archs, device=device) 100 | test_dataset = Dataset(arch_data_list=test_archs, device=device) 101 | draw_dataset = Dataset(arch_data_list=draw_archs, device=device) 102 | 103 | agent_1 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 104 | agent_2 = FRGinPredictorAgent(input_dim=Dataset.input_dim) 105 | 106 | trainer = FRGinPredictorTrainer(agent_1, agent_2, lr=learning_rate, epochs=epochs, batch_size=batch_size, scaling_factor=scaling_factor, 107 | weight_factor=weight_factor, weight_decay=weight_decay, 108 | device=device, loss=loss) 109 | 110 | trainer.fit(train_dataset) 111 | 112 | 113 | 114 | # test dataset evaluation 115 | 116 | pred_result, label_result = trainer.test(test_dataset) 117 | pred_result = pred_result.cpu().numpy() 118 | label_result = label_result.cpu().numpy() 119 | # print(pred_result) 120 | # print(label_result) 121 | stats = evaluate_stats(pred_result, label_result) 122 | 123 | # draw dataset 124 | pred_result, label_result = trainer.test(draw_dataset) 125 | pred_result = pred_result.cpu().numpy() 126 | label_result = label_result.cpu().numpy() 127 | error_distribution = numpy.abs(pred_result-label_result) 128 | stats["error"] = error_distribution 129 | stats["target"] = label_result 130 | stats["pred"] = pred_result 131 | pred_result, label_result = trainer.test(draw_dataset) 132 | forward_feats = agent_1.features 133 | reverse_feats = agent_2.features 134 | forward_feats = get_IRG(forward_feats) 135 | reverse_feats = get_IRG(reverse_feats) 136 | forward_feats = forward_feats[0].cpu().numpy() 137 | reverse_feats = reverse_feats[0].cpu().numpy() 138 | 139 | # print(reverse_feats - forward_feats) 140 | 141 | # forward_reverse 142 | pred_result_1, pred_result_2, label_result = trainer.test_feature_difference(test_dataset) 143 | pred_result_1 = pred_result_1.cpu().numpy() 144 | pred_result_2 = pred_result_2.cpu().numpy() 145 | label_result = label_result.cpu().numpy() 146 | stats["forward"] = evaluate_stats(pred_result_1, label_result) 147 | stats["reverse"] = evaluate_stats(pred_result_2, label_result) 148 | return forward_feats, reverse_feats, stats 149 | 150 | def run(train_archs, test_archs, draw_archs, weight_factor, name): 151 | stats = FRNAS(search_space=args.search_space, 152 | train_archs=train_archs, 153 | test_archs=test_archs, 154 | draw_archs=draw_archs, 155 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 156 | weight_decay=1e-4, 157 | weight_factor=weight_factor, 158 | device=DEVICE, 159 | loss="IRG") 160 | plot(stats[0], name+"_f") 161 | plot(stats[1], name+"_r") 162 | plot(numpy.abs(stats[0]-stats[1]), name+"_diff") 163 | print(name, stats[2]) 164 | 165 | 166 | def main(learning_rate, sampled_archs, sample_size, sample_size_2, sample_size_3, test_size): 167 | data_dict = {} 168 | 169 | # the dataset is fixed, we only do retrains 170 | train_archs = sampled_archs[:sample_size] 171 | train_archs_2 = sampled_archs[:sample_size_2] 172 | train_archs_3 = sampled_archs[:sample_size_3] 173 | test_archs = sampled_archs[sample_size_3:] 174 | draw_archs = sampled_archs[:32] 175 | 176 | print("Running") 177 | 178 | # run(train_archs_3, test_archs, draw_archs, 1, "test") 179 | 180 | run(train_archs, test_archs, draw_archs, 0.8, "IRG_feat_50_0.8") 181 | run(train_archs_2, test_archs, draw_archs, 0.8, "IRG_feat_200_0.8") 182 | run(train_archs_3, test_archs, draw_archs, 0.8, "IRG_feat_400_0.8") 183 | run(train_archs, test_archs, draw_archs, 0, "IRG_feat_50_0") 184 | run(train_archs_2, test_archs, draw_archs, 0, "IRG_feat_200_0") 185 | run(train_archs_3, test_archs, draw_archs, 0, "IRG_feat_400_0") 186 | 187 | 188 | return None 189 | 190 | 191 | if __name__ == '__main__': 192 | parser = argparse.ArgumentParser(description='Args for algorithms compare.') 193 | parser.add_argument('--search_space', type=str, default="nasbench-201", choices=["nasbench-101", "nasbench-201", "darts"], help='search space') 194 | parser.add_argument('--trials', type=int, default=1, help='Number of trials') 195 | parser.add_argument('--seed', type=int, default=24657, help='random seed') 196 | parser.add_argument('--lr', type=float, default=0.005, help='Loss used to train architecture.') 197 | parser.add_argument('--sample_size', type=int, default=50, help='Number of sampled architectures') 198 | parser.add_argument('--sample_size_2', type=int, default=200, help='Number of sampled architectures') 199 | parser.add_argument('--sample_size_3', type=int, default=400, help='Number of sampled architectures') 200 | parser.add_argument('--test_size', type=int, default=5000, help='Number of test architectures') 201 | args = parser.parse_args() 202 | # print(args) 203 | set_random_seed(args.seed) 204 | 205 | trials_result_dict = {} 206 | 207 | search_space = AvailableSearchSpace[args.search_space]() 208 | sampled_archs = search_space.sample_data_uniform(args.sample_size_3 + args.test_size) 209 | for trial in range(args.trials): 210 | print("trial {}".format(trial)) 211 | result_dict = main(args.lr, sampled_archs, args.sample_size, args.sample_size_2, args.sample_size_3, args.test_size) 212 | trials_result_dict[trial] = result_dict 213 | 214 | -------------------------------------------------------------------------------- /experiments/predictor/predictor_compare.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | import numpy as np 4 | import os 5 | 6 | from datetime import datetime 7 | 8 | # utils 9 | from experiments.utils import set_random_seed 10 | 11 | from experiments.utils import evaluate_stats 12 | 13 | # search space 14 | from nas_lib.model_predictor.search_space.nasbench101_search_space import NasBench101SearchSpace 15 | from nas_lib.model_predictor.search_space.nasbench201_search_space import NasBench201SearchSpace 16 | from nas_lib.model_predictor.search_space.darts_search_space import DartsSearchSpace 17 | 18 | # algorithms 19 | from experiments.algorithms.NPNAS import NPNAS 20 | from experiments.algorithms.NPENAS import NPENAS, NPENAS_ensemble_transform, NPENAS_Directed 21 | from experiments.algorithms.FRNAS import FRNAS, FRNAS_finetune 22 | 23 | ALGORITHMS = [ 24 | "NPNAS", 25 | "NPNAS_FIX", 26 | "NPENAS", 27 | "NPENAS_Directed", 28 | "NPENAS_ensemble_transform", 29 | "NPENAS_ensemble_no_transform", 30 | "FRNAS_IRG_ep200_wf08", 31 | "FRNAS_IRG_ep300_wf08", 32 | "FRNAS_IRG_ep300_wf06", 33 | "FRNAS_MSE_finetune", 34 | "FRNAS_IRG_finetune", 35 | ] 36 | 37 | AvailableSearchSpace = { 38 | "nasbench-101": NasBench101SearchSpace, 39 | "nasbench-201": NasBench201SearchSpace, 40 | "darts": DartsSearchSpace, 41 | } 42 | 43 | DEVICE = "cpu" # the bottleneck of this type of predictor is cpu and memory 44 | 45 | 46 | def main(args): 47 | sample_size, test_size = args.sample_size, args.test_size 48 | data_dict = {} 49 | 50 | # search space 51 | 52 | search_space = AvailableSearchSpace[args.search_space]() 53 | sampled_archs = search_space.sample_data_uniform(sample_size + test_size) 54 | train_archs = sampled_archs[:sample_size] 55 | test_archs = sampled_archs[sample_size:] 56 | 57 | for algorithm in ALGORITHMS: 58 | print("Running algorithm: {}".format(algorithm)) 59 | algo_result = None 60 | if algorithm == "NPENAS": 61 | algo_result = NPENAS(search_space=args.search_space, train_archs=train_archs, test_archs=test_archs, 62 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, weight_decay=1e-4, 63 | device=DEVICE) 64 | if algorithm == "NPENAS_Directed": 65 | algo_result = NPENAS_Directed(search_space=args.search_space, 66 | train_archs=train_archs, 67 | test_archs=test_archs, 68 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 69 | weight_decay=1e-4, 70 | device=DEVICE) 71 | 72 | if algorithm == "NPENAS_ensemble_no_transform": 73 | algo_result = NPENAS_ensemble_transform(search_space=args.search_space, 74 | train_archs=train_archs, 75 | test_archs=test_archs, 76 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 77 | weight_decay=1e-4, 78 | device=DEVICE, 79 | reverse=False) 80 | 81 | if algorithm == "NPENAS_ensemble_transform": 82 | algo_result = NPENAS_ensemble_transform(search_space=args.search_space, 83 | train_archs=train_archs, 84 | test_archs=test_archs, 85 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 86 | weight_decay=1e-4, 87 | device=DEVICE, 88 | reverse=True) 89 | if algorithm == "FRNAS_IRG_ep300_wf06": 90 | algo_result = FRNAS(search_space=args.search_space, 91 | train_archs=train_archs, 92 | test_archs=test_archs, 93 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 94 | weight_decay=1e-4, 95 | weight_factor=0.8, 96 | device=DEVICE, 97 | loss="IRG") 98 | 99 | if algorithm == "FRNAS_IRG_ep300_wf08": 100 | algo_result = FRNAS(search_space=args.search_space, 101 | train_archs=train_archs, 102 | test_archs=test_archs, 103 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 104 | weight_decay=1e-4, 105 | weight_factor=0.8, 106 | device=DEVICE, 107 | loss="IRG") 108 | 109 | if algorithm == "FRNAS_IRG_ep200_wf08": 110 | algo_result = FRNAS(search_space=args.search_space, 111 | train_archs=train_archs, 112 | test_archs=test_archs, 113 | learning_rate=0.005, batch_size=16, epochs=200, scaling_factor=20, 114 | weight_decay=1e-4, 115 | weight_factor=0.8, 116 | device=DEVICE, 117 | loss="IRG") 118 | 119 | # if algorithm == "FRNAS_MSE_ep200_wf08": 120 | # algo_result = FRNAS(search_space=args.search_space, 121 | # train_archs=train_archs, 122 | # test_archs=test_archs, 123 | # learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 124 | # weight_decay=1e-4, 125 | # weight_factor=0.8, 126 | # device=DEVICE, 127 | # loss="MSE") 128 | 129 | if algorithm == "FRNAS_IRG_finetune": 130 | algo_result = FRNAS_finetune(search_space=args.search_space, 131 | train_archs=train_archs, 132 | test_archs=test_archs, 133 | learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 134 | weight_decay=1e-4, 135 | weight_factor=0.6, 136 | device=DEVICE, 137 | loss="IRG") 138 | # if algorithm == "FRNAS_MSE_finetune": 139 | # algo_result = FRNAS_finetune(search_space=args.search_space, 140 | # train_archs=train_archs, 141 | # test_archs=test_archs, 142 | # learning_rate=0.005, batch_size=16, epochs=300, scaling_factor=20, 143 | # weight_decay=1e-4, 144 | # weight_factor=0.6, 145 | # device=DEVICE, 146 | # loss="MSE") 147 | 148 | if algorithm == "NPNAS": # follows the original paper 149 | algo_result = NPNAS(search_space=args.search_space, 150 | train_archs=train_archs, 151 | test_archs=test_archs, 152 | learning_rate=0.0001, batch_size=10, epochs=300, scaling_factor=100, 153 | weight_decay=1e-3, 154 | bias_multiply=False, 155 | device=DEVICE) 156 | 157 | 158 | if algorithm == "NPNAS_FIX": # larger LR, LR from original paper are too low for 101 and 201 159 | algo_result = NPNAS(search_space=args.search_space, 160 | train_archs=train_archs, 161 | test_archs=test_archs, 162 | learning_rate=0.001, batch_size=10, epochs=300, scaling_factor=100, 163 | weight_decay=1e-3, 164 | bias_multiply=False, 165 | device=DEVICE) 166 | 167 | if algo_result is None: 168 | print("no matching algorithm") 169 | else: 170 | data_dict[algorithm] = algo_result 171 | 172 | return data_dict 173 | 174 | 175 | if __name__ == '__main__': 176 | parser = argparse.ArgumentParser(description='Args for algorithms compare.') 177 | parser.add_argument('--search_space', type=str, default="darts", choices=["nasbench-101", "nasbench-201", "darts"], 178 | help='search space') 179 | parser.add_argument('--trials', type=int, default=200, help='Number of trials') 180 | parser.add_argument('--seed', type=int, default=5386, help='random seed') 181 | parser.add_argument('--save_path', type=str, default='./result/', help='path to save') 182 | parser.add_argument('--save_prefix', type=str, default='predictor_compare', help='prefix of the file') 183 | 184 | parser.add_argument('--sample_size', type=int, default=50, help='Number of sampled architectures') 185 | # 50 100 150 200 300 400 186 | parser.add_argument('--test_size', type=int, default=5000, help='Number of test architectures') 187 | parser.add_argument('--save_interval', type=int, default=30, help='intervals to save') 188 | args = parser.parse_args() 189 | # print(args) 190 | set_random_seed(args.seed) 191 | 192 | init_time = datetime.now() 193 | init_time = init_time.strftime("%y%m%d") + "-" + init_time.strftime("%H%M%S") 194 | 195 | 196 | save_path = args.save_path + args.save_prefix + "_" + init_time + "_" + args.search_space + "_" + str(args.sample_size) 197 | os.mkdir(save_path) 198 | 199 | config = { 200 | "seed": args.seed, 201 | "sample_size": args.sample_size, 202 | "trials": args.trials, 203 | "test_size": args.test_size, 204 | "search_space": args.search_space, 205 | } 206 | 207 | with open(save_path + "/config.json", "w") as f: 208 | json.dump(config, f, indent=2) 209 | 210 | trials_result_dict = {} 211 | for trial in range(args.trials): 212 | print("trial {}".format(trial)) 213 | result_dict = main(args) 214 | trials_result_dict[trial] = result_dict 215 | if (trial + 1) % args.save_interval == 0: 216 | with open(save_path + "/ckpt-{}.json".format(trial+1), "w") as f: 217 | json.dump(trials_result_dict, f, indent=2) 218 | with open(save_path + "/results.json", "w") as f: 219 | json.dump(trials_result_dict, f, indent=2) 220 | 221 | with open(save_path + "/results.json", "w") as f: 222 | json.dump(trials_result_dict, f, indent=2) 223 | -------------------------------------------------------------------------------- /nas_lib/model_predictor/trainer/fr_gin_trainer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import random 3 | 4 | from torch_geometric.data import Data, Batch 5 | from torch_geometric.loader import DataLoader 6 | from nas_lib.model_predictor.trainer.utils import make_agent_optimizer, get_lr_scheduler 7 | 8 | 9 | 10 | 11 | class IRGLoss(torch.nn.Module): 12 | """ 13 | A simpler version of IRG loss 14 | """ 15 | def __init__(self, weight_factor=0.8): 16 | super(IRGLoss, self).__init__() 17 | 18 | self.logit_loss = torch.nn.MSELoss() 19 | self.feature_loss = torch.nn.MSELoss() 20 | self.weight_factor = weight_factor 21 | 22 | def forward(self, pred, target, pred_features, target_features): 23 | """ 24 | given a batch of prediction, with a batch of features 25 | """ 26 | # print(pred_features) 27 | # print(target_features) 28 | 29 | label_loss = self.logit_loss(pred, target) 30 | 31 | num_features = len(pred_features) 32 | 33 | pred_irg = self.get_IRG(pred_features) 34 | target_irg = self.get_IRG(target_features) 35 | 36 | feature_loss = 0 37 | for i in range(0, len(pred_irg)): 38 | # feature_loss = feature_loss + torch.pow(torch.norm(target_irg[i] - pred_irg[i], p=2), 2) # euclidean distance should be calculated, not l2 norm. 39 | feature_loss = feature_loss + self.feature_loss(pred_irg[i], target_irg[i]) 40 | 41 | # print(label_loss, feature_loss) 42 | 43 | return (1-self.weight_factor) * label_loss + self.weight_factor * feature_loss / num_features 44 | 45 | def get_IRG(self, features): 46 | dist_matrix_list = [] 47 | for feature in features: 48 | dist_matrix = torch.norm(feature[:, None] - feature, dim=2, p=2) # broadcast, calculate A 49 | dist_matrix_list.append(dist_matrix) 50 | return dist_matrix_list 51 | 52 | 53 | class CELoss(torch.nn.Module): 54 | def __init__(self, weight_factor=0.8): 55 | super(CELoss, self).__init__() 56 | 57 | self.logit_loss = torch.nn.MSELoss() 58 | self.feature_loss = torch.nn.CrossEntropyLoss() 59 | self.weight_factor = weight_factor 60 | 61 | def forward(self, pred, target, pred_features, target_features): 62 | """ 63 | given a batch of prediction, with a batch of features 64 | """ 65 | num_features = len(pred_features) 66 | 67 | label_loss = self.logit_loss(pred, target) 68 | 69 | feature_loss = 0 70 | for i in range(0, len(pred_features)): 71 | feature_loss = feature_loss + self.feature_loss(pred_features[i], target_features[i]) 72 | 73 | return (1-self.weight_factor) * label_loss + self.weight_factor * feature_loss / num_features 74 | 75 | class MSELoss(torch.nn.Module): 76 | def __init__(self, weight_factor=0.8): 77 | super(MSELoss, self).__init__() 78 | 79 | self.logit_loss = torch.nn.MSELoss() 80 | self.feature_loss = torch.nn.MSELoss() 81 | self.weight_factor = weight_factor 82 | 83 | def forward(self, pred, target, pred_features, target_features): 84 | """ 85 | given a batch of prediction, with a batch of features 86 | """ 87 | # print(pred_features) 88 | # print(target_features) 89 | 90 | label_loss = self.logit_loss(pred, target) 91 | 92 | num_features = len(pred_features) 93 | 94 | feature_loss = 0 95 | for i in range(0, len(pred_features)): 96 | feature_loss = feature_loss + self.feature_loss(pred_features[i], target_features[i]) 97 | 98 | # print(label_loss, feature_loss) 99 | 100 | return (1-self.weight_factor) * label_loss + self.weight_factor * feature_loss / num_features 101 | 102 | 103 | 104 | 105 | class FRGinPredictorTrainer: 106 | def __init__(self, 107 | predictor_agent_1, 108 | predictor_agent_2, 109 | lr=0.01, 110 | device=None, 111 | epochs=10, 112 | batch_size=10, 113 | scaling_factor=10, 114 | weight_decay=1e-4, 115 | bias_multiply=True, 116 | loss = "IRG", 117 | weight_factor = 0.8 118 | ): 119 | 120 | self.predictor_agent_1 = predictor_agent_1 121 | self.predictor_agent_2 = predictor_agent_2 122 | 123 | 124 | if loss=="IRG": 125 | self.criterion = IRGLoss(weight_factor=weight_factor) 126 | elif loss=="CE": 127 | self.criterion = CELoss(weight_factor=weight_factor) 128 | elif loss=="MSE": 129 | self.criterion = MSELoss(weight_factor=weight_factor) 130 | 131 | 132 | self.device = device 133 | self.predictor_agent_1.to(self.device) 134 | self.predictor_agent_2.to(self.device) 135 | self.lr = lr 136 | 137 | self.batch_size = batch_size 138 | self.epoch = epochs 139 | self.scaling_factor = scaling_factor 140 | 141 | self.weight_decay = weight_decay 142 | 143 | self.bias_multiply = bias_multiply 144 | 145 | def fit(self, dataset): 146 | data_size = len(dataset) 147 | data_loader = DataLoader(dataset=dataset, batch_size=self.batch_size, shuffle=True) 148 | 149 | self.optimizer_1 = make_agent_optimizer(self.predictor_agent_1, base_lr=self.lr, weight_deacy=self.weight_decay, 150 | bias_multiply=self.bias_multiply) 151 | 152 | self.optimizer_2 = make_agent_optimizer(self.predictor_agent_2, base_lr=self.lr, weight_deacy=self.weight_decay, 153 | bias_multiply=self.bias_multiply) 154 | 155 | self.scheduler_1 = get_lr_scheduler(self.optimizer_1, self.epoch, data_size, self.batch_size) 156 | self.scheduler_2 = get_lr_scheduler(self.optimizer_2, self.epoch, data_size, self.batch_size) 157 | 158 | self.predictor_agent_1.train() 159 | self.predictor_agent_2.train() 160 | 161 | for epoch in range(self.epoch): 162 | num_batch = 0 163 | for batch in data_loader: 164 | num_batch += 1 165 | # for data in batch: 166 | # data.to(self.device) 167 | val_tensor = batch[0].y 168 | 169 | pred_1 = self.predictor_agent_1(batch[0]) * self.scaling_factor 170 | _ = self.predictor_agent_2(batch[1]) * self.scaling_factor 171 | pred_1 = pred_1.squeeze() 172 | loss_1 = self.criterion(pred_1, val_tensor, self.predictor_agent_1.features, 173 | self.predictor_agent_2.features) 174 | 175 | self.optimizer_1.zero_grad() 176 | loss_1.backward() 177 | self.optimizer_1.step() 178 | 179 | _ = self.predictor_agent_1(batch[0]) * self.scaling_factor 180 | pred_2 = self.predictor_agent_2(batch[1]) * self.scaling_factor 181 | pred_2 = pred_2.squeeze() 182 | loss_2 = self.criterion(pred_2, val_tensor, self.predictor_agent_2.features, 183 | self.predictor_agent_1.features) 184 | 185 | self.optimizer_2.zero_grad() 186 | loss_2.backward() 187 | self.optimizer_2.step() 188 | 189 | self.scheduler_1.step() # NOTE: the original code uses: self.scheduler.step(epoch + int(i/30)) 190 | self.scheduler_2.step() 191 | # print(num_batch) 192 | 193 | def pred(self, dataset, batch_size = 64): 194 | data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=False) 195 | pred_list = [] 196 | 197 | self.predictor_agent_1.eval() 198 | self.predictor_agent_2.eval() 199 | with torch.no_grad(): 200 | for batch in data_loader: 201 | 202 | # for data in batch: 203 | # data.to(self.device) 204 | 205 | pred_1 = self.predictor_agent_1(batch[0]) * self.scaling_factor 206 | pred_2 = self.predictor_agent_2(batch[1]) * self.scaling_factor 207 | pred_1 = pred_1.squeeze() 208 | pred_2 = pred_2.squeeze() 209 | 210 | if len(pred_1.size()) == 0: 211 | pred_1.unsqueeze(0) 212 | if len(pred_2.size()) == 0: 213 | pred_2.unsqueeze(0) 214 | # print(pred_1) 215 | # print(pred_2) 216 | pred = (pred_1 + pred_2) / 2 217 | # print(pred) 218 | pred_list.append(pred) 219 | return torch.cat(pred_list, dim=0) 220 | 221 | def test(self, dataset, batch_size=64): 222 | data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=False) 223 | pred_list = [] 224 | valid_list = [] 225 | 226 | self.predictor_agent_1.eval() 227 | self.predictor_agent_2.eval() 228 | with torch.no_grad(): 229 | for batch in data_loader: 230 | # for data in batch: 231 | # data.to(self.device) 232 | val_tensor = batch[0].y 233 | 234 | pred_1 = self.predictor_agent_1(batch[0]) * self.scaling_factor 235 | pred_2 = self.predictor_agent_2(batch[1]) * self.scaling_factor 236 | pred_1 = pred_1.squeeze() 237 | pred_2 = pred_2.squeeze() 238 | 239 | if len(pred_1.size()) == 0: 240 | pred_1.unsqueeze(0) 241 | if len(pred_2.size()) == 0: 242 | pred_2.unsqueeze(0) 243 | # print(pred_1) 244 | # print(pred_2) 245 | pred = (pred_1 + pred_2) / 2 246 | # print(pred) 247 | pred_list.append(pred) 248 | valid_list.append(val_tensor) 249 | 250 | # print(pred_list) 251 | # print(valid_list) 252 | return torch.cat(pred_list, dim=0), torch.cat(valid_list, dim=0) 253 | 254 | 255 | def test_feature_difference(self, dataset, batch_size=64): 256 | 257 | data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=False) 258 | pred_1_list = [] 259 | pred_2_list = [] 260 | valid_list = [] 261 | 262 | self.predictor_agent_1.eval() 263 | self.predictor_agent_2.eval() 264 | with torch.no_grad(): 265 | for batch in data_loader: 266 | # for data in batch: 267 | # data.to(self.device) 268 | val_tensor = batch[0].y 269 | 270 | pred_1 = self.predictor_agent_1(batch[0]) * self.scaling_factor 271 | pred_2 = self.predictor_agent_2(batch[1]) * self.scaling_factor 272 | pred_1 = pred_1.squeeze() 273 | pred_2 = pred_2.squeeze() 274 | 275 | if len(pred_1.size()) == 0: 276 | pred_1.unsqueeze(0) 277 | if len(pred_2.size()) == 0: 278 | pred_2.unsqueeze(0) 279 | pred_1_list.append(pred_1) 280 | pred_2_list.append(pred_2) 281 | valid_list.append(val_tensor) 282 | 283 | # print(pred_list) 284 | # print(valid_list) 285 | return torch.cat(pred_1_list, dim=0), torch.cat(pred_2_list, dim=0), torch.cat(valid_list, dim=0) 286 | 287 | --------------------------------------------------------------------------------