├── Calculate.py ├── FedNets.py ├── Fed_NN.py ├── Fed_Svm.py ├── Noise_add.py ├── Privacy.py ├── README.md ├── Update.py ├── __pycache__ ├── Calculate.cpython-37.pyc ├── FedNets.cpython-37.pyc ├── Noise_add.cpython-37.pyc ├── Privacy.cpython-37.pyc ├── Update.cpython-37.pyc ├── averaging.cpython-37.pyc ├── options.cpython-37.pyc └── sampling.cpython-37.pyc ├── averaging.py ├── dataset ├── ADULT.dat └── mnist │ └── MNIST │ ├── processed │ ├── test.pt │ └── training.pt │ └── raw │ ├── t10k-images-idx3-ubyte │ ├── t10k-images-idx3-ubyte.gz │ ├── t10k-labels-idx1-ubyte │ ├── t10k-labels-idx1-ubyte.gz │ ├── train-images-idx3-ubyte │ ├── train-images-idx3-ubyte.gz │ ├── train-labels-idx1-ubyte │ └── train-labels-idx1-ubyte.gz ├── local └── events.out.tfevents.1612356994.DESKTOP-U8MK2VQ ├── options.py └── sampling.py /Calculate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Sep 26 17:19:31 2019 4 | 5 | @author: WEIKANG 6 | """ 7 | import torch 8 | import numpy as np 9 | import copy 10 | import math 11 | 12 | def inner_product(params_a, params_b): 13 | sum = 0 14 | for i in params_a.keys(): 15 | sum += np.sum(np.multiply(params_a[i].cpu().numpy(),\ 16 | params_b[i].cpu().numpy())) 17 | return sum 18 | 19 | 20 | def avg_grads(g): 21 | grad_avg = copy.deepcopy(g[0]) 22 | for k in grad_avg.keys(): 23 | for i in range(1, len(g)): 24 | grad_avg[k] += g[i][k] 25 | grad_avg[k] = torch.div(grad_avg[k], len(g)) 26 | return grad_avg 27 | 28 | def calculate_grads(args, w_before, w_new): 29 | grads = copy.deepcopy(w_before) 30 | for k in grads.keys(): 31 | grads[k] =(w_before[k]-w_new[k]) * 1.0 / args.lr 32 | return grads 33 | 34 | 35 | def f_zero(args, f, num_iter): 36 | x0 = 0 37 | x1 = args.max_epochs 38 | if f(x0)*f(x1)>=0: 39 | if abs(f(x0))>abs(f(x1)): 40 | x0 = copy.deepcopy(x1) 41 | else: 42 | y = copy.deepcopy(args.max_epochs) 43 | for i in range(100): 44 | if f(x0)*f(x1)<0: 45 | y = copy.deepcopy(x1) 46 | x1 = copy.deepcopy((x0+x1)/2) 47 | else: 48 | x1 = copy.deepcopy(y) 49 | x0 = copy.deepcopy((x0+x1)/2) 50 | if abs(x0-x1)<0.01: 51 | break 52 | if (x0+num_iter) > args.max_epochs: 53 | x0 = copy.deepcopy(args.max_epochs) 54 | return x0 55 | 56 | def get_l2_norm(args, params_a): 57 | sum = 0 58 | if args.gpu != -1: 59 | tmp_a = np.array([v.detach().cpu().numpy() for v in params_a]) 60 | else: 61 | tmp_a = np.array([v.detach().numpy() for v in params_a]) 62 | a = [] 63 | for i in tmp_a: 64 | x = i.flatten() 65 | for k in x: 66 | a.append(k) 67 | for i in range(len(a)): 68 | sum += (a[i] - 0) ** 2 69 | norm = np.sqrt(sum) 70 | return norm 71 | 72 | def get_1_norm(params_a): 73 | sum = 0 74 | if isinstance(params_a,np.ndarray) == True: 75 | sum += pow(np.linalg.norm(params_a, ord=2),2) 76 | else: 77 | for i in params_a.keys(): 78 | if len(params_a[i]) == 1: 79 | sum += pow(np.linalg.norm(params_a[i].cpu().numpy(), ord=2),2) 80 | else: 81 | a = copy.deepcopy(params_a[i].cpu().numpy()) 82 | for j in a: 83 | x = copy.deepcopy(j.flatten()) 84 | sum += pow(np.linalg.norm(x, ord=2),2) 85 | norm = np.sqrt(sum) 86 | return norm 87 | 88 | def get_2_norm(params_a, params_b): 89 | sum = 0 90 | for i in params_a.keys(): 91 | if len(params_a[i]) == 1: 92 | sum += pow(np.linalg.norm(params_a[i].cpu().numpy()-\ 93 | params_b[i].cpu().numpy(), ord=2),2) 94 | else: 95 | a = copy.deepcopy(params_a[i].cpu().numpy()) 96 | b = copy.deepcopy(params_b[i].cpu().numpy()) 97 | x = [] 98 | y = [] 99 | for j in a: 100 | x.append(copy.deepcopy(j.flatten())) 101 | for k in b: 102 | y.append(copy.deepcopy(k.flatten())) 103 | for m in range(len(x)): 104 | sum += pow(np.linalg.norm(x[m]-y[m], ord=2),2) 105 | norm = np.sqrt(sum) 106 | return norm 107 | 108 | def para_estimate(args, list_loss, loss_locals, w_glob_before, w_locals_before,\ 109 | w_locals, w_glob): 110 | Lipz_c = [] 111 | Lipz_s = [] 112 | beta = [] 113 | delta = [] 114 | norm_grads_locals = [] 115 | Grads_locals = copy.deepcopy(w_locals) 116 | for idx in range(args.num_Chosenusers): 117 | ### Calculate▽F_i(w(t))=[w(t)-w_i(t)]/lr ### 118 | Grads_locals[idx] = copy.deepcopy(calculate_grads(args, w_glob, w_locals[idx])) 119 | ### Calculate▽F(w(t)) ### 120 | Grads_glob = copy.deepcopy(avg_grads(Grads_locals)) 121 | for idx in range(args.num_Chosenusers): 122 | ### Calculate ||w(t-1)-w(t)|| ### 123 | diff_weights_glob = copy.deepcopy(get_2_norm(w_glob_before, w_glob)) 124 | ### Calculate ||▽F_i(w(t-1))-▽F_i(w(t))|| ### 125 | diff_grads = copy.deepcopy(get_2_norm(calculate_grads(args, w_glob_before, \ 126 | w_locals_before[idx]), calculate_grads(args, w_glob, w_locals[idx]))) 127 | ### Calculate ||w(t)-w_i(t)|| ### 128 | diff_weights_locals = copy.deepcopy(get_2_norm(w_glob, w_locals[idx])) 129 | ### Calculate ||▽F(w(t))-▽F_i(w(t))|| ### 130 | Grads_variance = copy.deepcopy(get_2_norm(Grads_glob, Grads_locals[idx])) 131 | ### Calculate ||▽F(w(t))|| ### 132 | norm_grads_glob = copy.deepcopy(get_1_norm(Grads_glob)) 133 | ### Calculate ||▽F_i(w(t))|| ### 134 | norm_grads_locals.append(copy.deepcopy(get_1_norm(Grads_locals[idx]))) 135 | ### Calculate Lipz_s=||▽F_i(w(t-1))-▽F_i(w(t))||/||w(t-1)-w(t)|| ### 136 | Lipz_s.append(copy.deepcopy(diff_grads/diff_weights_glob)) 137 | ### Calculate Lipz_c=||F_i(w(t))-F_i(w_i(t))||/||w(t)-w_i(t)|| ### 138 | Lipz_c.append(copy.deepcopy(abs(list_loss[idx]-loss_locals[idx])/diff_weights_locals)) 139 | ### Calculate delta= ||▽F(w(t))-▽F_i(w(t))||### 140 | delta.append(copy.deepcopy(Grads_variance)) 141 | beta = copy.deepcopy(np.sqrt(sum(c*c for c in norm_grads_locals)/args.num_Chosenusers)/norm_grads_glob) 142 | return Lipz_s, Lipz_c, delta, beta, Grads_glob, Grads_locals, norm_grads_glob, norm_grads_locals 143 | -------------------------------------------------------------------------------- /FedNets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Python version: 3.6 4 | 5 | import torch 6 | from torch import nn 7 | import torch.nn.functional as F 8 | 9 | torch.manual_seed(1) 10 | torch.cuda.manual_seed_all(1) 11 | torch.cuda.manual_seed(1) 12 | 13 | class MLP(nn.Module): 14 | def __init__(self, dim_in, dim_hidden, dim_out): 15 | super(MLP, self).__init__() 16 | print("NN: MLP is created") 17 | self.layer_input = nn.Linear(dim_in, dim_hidden) 18 | self.relu = nn.ReLU() 19 | self.dropout = nn.Dropout() 20 | self.layer_hidden = nn.Linear(dim_hidden, dim_out) 21 | self.softmax = nn.Softmax(dim=1) 22 | 23 | def forward(self, x): 24 | x = x.view(-1, x.shape[1]*x.shape[-2]*x.shape[-1]) 25 | x = self.layer_input(x) 26 | x = self.dropout(x) 27 | x = self.relu(x) 28 | x = self.layer_hidden(x) 29 | return self.softmax(x) 30 | 31 | class MLP1(nn.Module): 32 | def __init__(self, dim_in, dim_hidden, dim_out): 33 | super(MLP1, self).__init__() 34 | print("NN: MLP is created") 35 | self.layer_input = nn.Linear(dim_in, dim_hidden) 36 | self.layer_hidden = nn.Linear(dim_hidden, dim_out) 37 | # self.softmax = nn.Softmax(dim=1) 38 | 39 | # weights_init = 0.001 40 | # bias_init = 0.001 41 | # 42 | # nn.init.constant_(self.layer_input.weight,weights_init) 43 | # nn.init.constant_(self.layer_input.bias, bias_init) 44 | # nn.init.constant_(self.layer_hidden.weight, weights_init) 45 | # nn.init.constant_(self.layer_hidden.bias, bias_init) 46 | 47 | def forward(self, x): 48 | x = x.view(-1, x.shape[1]*x.shape[-2]*x.shape[-1]) 49 | # x = x.view(-1, 1) 50 | x = self.layer_input(x) 51 | x = self.layer_hidden(x) 52 | return F.log_softmax(x, dim=1) 53 | 54 | class MLP_regression(nn.Module): 55 | def __init__(self, dim_in, dim_hidden, dim_out): 56 | super(MLP_regression, self).__init__() 57 | print("NN: MLP is created") 58 | self.layer_input = nn.Linear(dim_in, dim_hidden) 59 | self.layer_hidden = nn.Linear(dim_hidden, dim_out) 60 | 61 | def forward(self, x): 62 | x = x.view(-1, x.shape[1]*x.shape[-2]*x.shape[-1]) 63 | x = self.layer_input(x) 64 | x = self.layer_hidden(x) 65 | return F.log_softmax(x, dim=1) 66 | 67 | class CNN_test(nn.Module): 68 | def __init__(self, args): 69 | super(CNN_test, self).__init__() 70 | self.conv1 = nn.Sequential( # input shape (1, 28, 28) 71 | nn.Conv2d( 72 | in_channels=1, # input height 73 | out_channels=16, # n_filters 74 | kernel_size=5, # filter size 75 | stride=1, # filter movement/step 76 | padding=2, # if want same width and length of this image after Conv2d, padding=(kernel_size-1)/2 if stride=1 77 | ), # output shape (16, 28, 28) 78 | nn.ReLU(), # activation 79 | nn.MaxPool2d(kernel_size=2), # choose max value in 2x2 area, output shape (16, 14, 14) 80 | ) 81 | self.conv2 = nn.Sequential( # input shape (16, 14, 14) 82 | nn.Conv2d(16, 32, 5, 1, 2), # output shape (32, 14, 14) 83 | nn.ReLU(), # activation 84 | nn.MaxPool2d(2), # output shape (32, 7, 7) 85 | ) 86 | self.out = nn.Linear(32* 7 * 7, args.num_classes) # fully connected layer, output 10 classes 87 | 88 | def forward(self, x): 89 | x = self.conv1(x) 90 | x = self.conv2(x) 91 | x = x.view(x.size(0), -1) # flatten the output of conv2 to (batch_size, 32 * 7 * 7) 92 | output = self.out(x) 93 | return F.log_softmax(output, dim=1) # return x for visualization 94 | 95 | 96 | class CNNMnist(nn.Module): 97 | def __init__(self, args): 98 | super(CNNMnist, self).__init__() 99 | print("NN: CNNMnist is created") 100 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 101 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 102 | self.conv2_drop = nn.Dropout2d() 103 | self.fc1 = nn.Linear(320, 50) 104 | self.fc2 = nn.Linear(50, args.num_classes) 105 | 106 | def forward(self, x): 107 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 108 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 109 | x = x.view(-1, x.shape[1]*x.shape[2]*x.shape[3]) 110 | x = F.relu(self.fc1(x)) 111 | x = F.dropout(x, training=self.training) 112 | x = self.fc2(x) 113 | return F.log_softmax(x, dim=1) 114 | 115 | 116 | class CNNCifar(nn.Module): 117 | def __init__(self, args): 118 | super(CNNCifar, self).__init__() 119 | self.conv1 = nn.Conv2d(3, 6, 5) 120 | self.pool = nn.MaxPool2d(2, 2) 121 | self.conv2 = nn.Conv2d(6, 16, 5) 122 | self.fc1 = nn.Linear(16 * 5 * 5, 120) 123 | self.fc2 = nn.Linear(120, 84) 124 | self.fc3 = nn.Linear(84, args.num_classes) 125 | 126 | def forward(self, x): 127 | x = self.pool(F.relu(self.conv1(x))) 128 | x = self.pool(F.relu(self.conv2(x))) 129 | x = x.view(-1, 16 * 5 * 5) 130 | x = F.relu(self.fc1(x)) 131 | x = F.relu(self.fc2(x)) 132 | x = self.fc3(x) 133 | return F.log_softmax(x, dim=1) 134 | -------------------------------------------------------------------------------- /Fed_NN.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Python version: 3.6 4 | import time 5 | import matplotlib 6 | matplotlib.use('Agg') 7 | import os 8 | import copy 9 | import pandas as pd 10 | import random 11 | from torchvision import datasets, transforms 12 | import torch 13 | from tensorboardX import SummaryWriter 14 | 15 | from sampling import mnist_iid, mnist_noniid, cifar_iid, cifar_noniid 16 | from options import args_parser 17 | from Update import LocalUpdate 18 | from FedNets import MLP1, CNNMnist, CNN_test 19 | from averaging import average_weights 20 | from Privacy import Privacy_account, Adjust_T, Noise_TB_decay 21 | from Noise_add import noise_add, users_sampling, clipping 22 | from Calculate import para_estimate 23 | 24 | 25 | def main(args): 26 | #####-Choose Variable-##### 27 | set_variable = args.set_num_Chosenusers 28 | set_variable0 = copy.deepcopy(args.set_epochs) 29 | set_variable1 = copy.deepcopy(args.set_privacy_budget) 30 | 31 | if not os.path.exists('./exper_result'): 32 | os.mkdir('./exper_result') 33 | 34 | # load dataset and split users 35 | dataset_train,dataset_test = [],[] 36 | dataset_train = datasets.MNIST('./dataset/mnist/', train=True, download=True, 37 | transform=transforms.Compose([ 38 | transforms.ToTensor(), 39 | transforms.Normalize((0.1307,), (0.3081,)) 40 | ])) 41 | dataset_test = datasets.MNIST('./dataset/mnist/', train=False, download=True, 42 | transform=transforms.Compose([ 43 | transforms.ToTensor(), 44 | transforms.Normalize((0.1307,), (0.3081,)) 45 | ])) 46 | 47 | # sample users 48 | if args.iid: 49 | dict_users = mnist_iid(dataset_train, args.num_users, args.num_items_train) 50 | # dict_users_test = mnist_iid(dataset_test, args.num_users, args.num_items_test) 51 | dict_sever = mnist_iid(dataset_test, args.num_users, args.num_items_test) 52 | else: 53 | dict_users = mnist_noniid(dataset_train, args.num_users) 54 | dict_sever = mnist_noniid(dataset_test, args.num_users) 55 | 56 | img_size = dataset_train[0][0].shape 57 | 58 | for v in range(len(set_variable)): 59 | final_train_loss = [[0 for i in range(len(set_variable1))] for j in range(len(set_variable0))] 60 | final_train_accuracy = [[0 for i in range(len(set_variable1))] for j in range(len(set_variable0))] 61 | final_test_loss = [[0 for i in range(len(set_variable1))] for j in range(len(set_variable0))] 62 | final_test_accuracy = [[0 for i in range(len(set_variable1))] for j in range(len(set_variable0))] 63 | final_com_cons = [[0 for i in range(len(set_variable1))] for j in range(len(set_variable0))] 64 | args.num_Chosenusers = copy.deepcopy(set_variable[v]) 65 | for s in range(len(set_variable0)): 66 | for j in range(len(set_variable1)): 67 | args.epochs = copy.deepcopy(set_variable0[s]) 68 | args.privacy_budget = copy.deepcopy(set_variable1[j]) 69 | print("dataset:", args.dataset, " num_users:", args.num_users, " num_chosen_users:", args.num_Chosenusers, " Privacy budget:", args.privacy_budget,\ 70 | " epochs:", args.epochs, "local_ep:", args.local_ep, "local train size", args.num_items_train, "batch size:", args.local_bs) 71 | loss_test, loss_train = [], [] 72 | acc_test, acc_train = [], [] 73 | for m in range(args.num_experiments): 74 | # build model 75 | net_glob = None 76 | if args.model == 'cnn' and args.dataset == 'mnist': 77 | if args.gpu != -1: 78 | torch.cuda.set_device(args.gpu) 79 | # net_glob = CNNMnist(args=args).cuda() 80 | net_glob = CNN_test(args=args).cuda() 81 | else: 82 | net_glob = CNNMnist(args=args) 83 | elif args.model == 'mlp': 84 | len_in = 1 85 | for x in img_size: 86 | len_in *= x 87 | if args.gpu != -1: 88 | torch.cuda.set_device(args.gpu) 89 | net_glob = MLP1(dim_in=len_in, dim_hidden=256,\ 90 | dim_out=args.num_classes).cuda() 91 | else: 92 | net_glob = MLP1(dim_in=len_in, dim_hidden=256,\ 93 | dim_out=args.num_classes) 94 | else: 95 | exit('Error: unrecognized model') 96 | print("Nerual Net:",net_glob) 97 | 98 | net_glob.train() #Train() does not change the weight values 99 | # copy weights 100 | w_glob = net_glob.state_dict() 101 | w_size = 0 102 | w_size_all = 0 103 | for k in w_glob.keys(): 104 | size = w_glob[k].size() 105 | if(len(size)==1): 106 | nelements = size[0] 107 | else: 108 | nelements = size[0] * size[1] 109 | w_size += nelements*4 110 | w_size_all += nelements 111 | # print("Size ", k, ": ",nelements*4) 112 | print("Weight Size:", w_size, " bytes") 113 | print("Weight & Grad Size:", w_size*2, " bytes") 114 | print("Each user Training size:", 784* 8/8* args.local_bs, " bytes") 115 | print("Total Training size:", 784 * 8 / 8 * 60000, " bytes") 116 | # training 117 | threshold_epochs = copy.deepcopy(args.epochs) 118 | threshold_epochs_list, noise_list = [], [] 119 | loss_avg_list, acc_avg_list, list_loss, loss_avg = [], [], [], [] 120 | eps_tot_list, eps_tot = [], 0 121 | com_cons = [] 122 | ### FedAvg Aglorithm ### 123 | ### Compute noise scale ### 124 | noise_scale = copy.deepcopy(Privacy_account(args,\ 125 | threshold_epochs, noise_list, 0)) 126 | for iter in range(args.epochs): 127 | print('\n','*' * 20,f'Epoch: {iter}','*' * 20) 128 | start_time = time.time() 129 | if args.num_Chosenusers < args.num_users: 130 | chosenUsers = random.sample(range(1,args.num_users)\ 131 | ,args.num_Chosenusers) 132 | chosenUsers.sort() 133 | else: 134 | chosenUsers = range(args.num_users) 135 | print("\nChosen users:", chosenUsers) 136 | 137 | if iter >= 1 and args.para_est == True: 138 | w_locals_before = copy.deepcopy(w_locals_org) 139 | w_glob_before = copy.deepcopy(w_glob) 140 | w_locals, w_locals_1ep, loss_locals, acc_locals = [], [], [], [] 141 | for idx in range(len(chosenUsers)): 142 | local = LocalUpdate(args=args, dataset=dataset_train,\ 143 | idxs=dict_users[chosenUsers[idx]], tb=summary) 144 | w_1st_ep, w, loss, acc = local.update_weights(\ 145 | net=copy.deepcopy(net_glob)) 146 | w_locals.append(copy.deepcopy(w)) 147 | ### get 1st ep local weights ### 148 | w_locals_1ep.append(copy.deepcopy(w_1st_ep)) 149 | loss_locals.append(copy.deepcopy(loss)) 150 | # print("User ", chosenUsers[idx], " Acc:", acc, " Loss:", loss) 151 | acc_locals.append(copy.deepcopy(acc)) 152 | 153 | w_locals_org = copy.deepcopy(w_locals) 154 | 155 | ### Clipping ### 156 | for idx in range(len(chosenUsers)): 157 | w_locals[idx] = copy.deepcopy(clipping(args, w_locals[idx])) 158 | # print(get_2_norm(w_locals[idx], w_glob)) 159 | 160 | ### perturb 'w_local' ### 161 | w_locals = noise_add(args, noise_scale, w_locals) 162 | 163 | ### update global weights ### 164 | # w_locals = users_sampling(args, w_locals, chosenUsers) 165 | w_glob = average_weights(w_locals) 166 | 167 | # copy weight to net_glob 168 | net_glob.load_state_dict(w_glob) 169 | # global test 170 | list_acc, list_loss = [], [] 171 | net_glob.eval() 172 | for c in range(args.num_users): 173 | net_local = LocalUpdate(args=args,dataset=dataset_test,\ 174 | idxs=dict_sever[idx], tb=summary) 175 | acc, loss = net_local.test(net=net_glob) 176 | # acc, loss = net_local.test_gen(net=net_glob,\ 177 | # idxs=dict_users[c], dataset=dataset_test) 178 | list_acc.append(acc) 179 | list_loss.append(loss) 180 | # print("\nEpoch:{},Global test loss:{}, Global test acc:{:.2f}%".\ 181 | # format(iter, sum(list_loss) / len(list_loss),\ 182 | # 100. * sum(list_acc) / len(list_acc))) 183 | 184 | # print loss 185 | loss_avg = sum(loss_locals) / len(loss_locals) 186 | acc_avg = sum(acc_locals) / len(acc_locals) 187 | loss_avg_list.append(loss_avg) 188 | acc_avg_list.append(acc_avg) 189 | print("\nTrain loss: {}, Train acc: {}".\ 190 | format(loss_avg_list[-1], acc_avg_list[-1])) 191 | print("\nTest loss: {}, Test acc: {}".\ 192 | format(sum(list_loss) / len(list_loss),\ 193 | sum(list_acc) / len(list_acc))) 194 | 195 | noise_list.append(noise_scale) 196 | threshold_epochs_list.append(threshold_epochs) 197 | print('\nNoise Scale:', noise_list) 198 | print('\nThreshold epochs:', threshold_epochs_list) 199 | if args.dp_mechanism == 'CRD' and iter >= 1: 200 | threshold_epochs = Adjust_T(args, loss_avg_list,\ 201 | threshold_epochs_list, iter) 202 | noise_scale = copy.deepcopy(Privacy_account(args,\ 203 | threshold_epochs, noise_list, iter)) 204 | 205 | # print run time of each experiment 206 | end_time = time.time() 207 | print('Run time: %f second' % (end_time - start_time)) 208 | # estimate some paramters of the loss function 209 | if iter >= 1 and args.para_est == True: 210 | Lipz_s,Lipz_c,delta,_,_,_,_,_=para_estimate(args, list_loss, loss_locals, w_glob_before, w_locals_before,\ 211 | w_locals_org, w_glob) 212 | print('Lipschitz smooth, lipschitz continuous, gradient divergence:',\ 213 | Lipz_s,Lipz_c,delta) 214 | 215 | if iter >= threshold_epochs: 216 | break 217 | loss_train.append(loss_avg) 218 | acc_train.append(acc_avg) 219 | loss_test.append(sum(list_loss) / len(list_loss)) 220 | acc_test.append(sum(list_acc) / len(list_acc)) 221 | com_cons.append(iter+1) 222 | 223 | # record results 224 | final_train_loss[s][j]=copy.deepcopy(sum(loss_train)/len(loss_train)) 225 | final_train_accuracy[s][j]=copy.deepcopy(sum(acc_train)/len(acc_train)) 226 | final_test_loss[s][j]=copy.deepcopy(sum(loss_test)/len(loss_test)) 227 | final_test_accuracy[s][j]=copy.deepcopy(sum(acc_test)/len(acc_test)) 228 | final_com_cons[s][j]=copy.deepcopy(sum(com_cons)/len(com_cons)) 229 | 230 | print('\nFinal train loss:', final_train_loss) 231 | print('\nFinal train acc:', final_train_accuracy) 232 | print('\nFinal test loss:', final_test_loss) 233 | print('\nFinal test acc:', final_test_accuracy) 234 | 235 | timeslot = int(time.time()) 236 | data_test_loss = pd.DataFrame(index = set_variable0, columns =\ 237 | set_variable1, data = final_train_loss) 238 | data_test_loss.to_csv('./exper_result/'+'train_loss_{}_{}_{}.csv'.\ 239 | format(set_variable[v],args.dp_mechanism,timeslot)) 240 | data_test_loss = pd.DataFrame(index = set_variable0, columns =\ 241 | set_variable1, data = final_test_loss) 242 | data_test_loss.to_csv('./exper_result/'+'test_loss_{}_{}_{}.csv'.\ 243 | format(set_variable[v],args.dp_mechanism,timeslot)) 244 | data_test_acc = pd.DataFrame(index = set_variable0, columns =\ 245 | set_variable1, data = final_train_accuracy) 246 | data_test_acc.to_csv('./exper_result/'+'train_acc_{}_{}_{}.csv'.\ 247 | format(set_variable[v],args.dp_mechanism,timeslot)) 248 | data_test_acc = pd.DataFrame(index = set_variable0, columns =\ 249 | set_variable1, data = final_test_accuracy) 250 | data_test_acc.to_csv('./exper_result/'+'test_acc_{}_{}_{}.csv'.\ 251 | format(set_variable[v],args.dp_mechanism,timeslot)) 252 | data_test_acc = pd.DataFrame(index = set_variable0, columns =\ 253 | set_variable1, data = final_com_cons) 254 | data_test_acc.to_csv('./exper_result/'+'aggregation_consuming_{}_{}_{}.csv'.\ 255 | format(set_variable[v],args.dp_mechanism,timeslot)) 256 | 257 | if __name__ == '__main__': 258 | # return the available GPU 259 | av_GPU = torch.cuda.is_available() 260 | if av_GPU == False: 261 | exit('No available GPU') 262 | # parse args 263 | args = args_parser() 264 | # define paths 265 | path_project = os.path.abspath('..') 266 | 267 | summary = SummaryWriter('local') 268 | ### differential privacy ### 269 | args.dp_mechanism = 'Origi' ### CRD or Origi### 270 | args.dec_cons = 0.8 ## discount constant 271 | # args.privacy_budget = 100 272 | args.delta = 0.01 273 | 274 | args.gpu = -1 # -1 (CPU only) or GPU = 0 275 | args.lr = 0.002 # Learning rate 276 | args.model = 'mlp' # 'mlp' or 'cnn' 277 | args.dataset = 'mnist' # 'mnist' 278 | args.num_users = 10 ### numb of users ### 279 | args.num_Chosenusers = 10 280 | args.epochs = 100 # numb of global iters 281 | args.local_ep = 5 # numb of local iters 282 | args.num_items_train = 800 # numb of local data size # 283 | args.num_items_test = 512 284 | args.local_bs = 128 ### Local Batch size (1200 = full dataset ### 285 | ### size of a user for mnist, 2000 for cifar) ### 286 | 287 | args.set_privacy_budget = range(80000,240000,40000) 288 | args.set_epochs = range(100,305,50) 289 | args.set_num_Chosenusers = [10] 290 | args.set_dec_cons = [0.7,0.75,0.80,0.85,0.9,0.95] 291 | 292 | args.num_experiments = 100 293 | args.clipthr = 20 294 | args.para_est = True 295 | args.iid = True 296 | 297 | main(args) 298 | -------------------------------------------------------------------------------- /Fed_Svm.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import time 3 | import copy 4 | import numpy as np 5 | import random 6 | from torch import nn 7 | from sklearn.preprocessing import MinMaxScaler 8 | from sklearn.preprocessing import Normalizer 9 | from sklearn.model_selection import RepeatedKFold 10 | 11 | def load_dat(filepath, minmax=None, normalize=False, bias_term=True): 12 | """ load a dat file 13 | 14 | args: 15 | minmax: tuple(min, max), dersired range of transformed data 16 | normalize: boolean, normalize samples individually to unit norm if True 17 | bias_term: boolean, add a dummy column of 1s 18 | """ 19 | lines = np.loadtxt(filepath) 20 | labels = lines[:, -1] 21 | features = lines[:, :-1] 22 | 23 | N, dim = features.shape 24 | 25 | if minmax is not None: 26 | minmax = MinMaxScaler(feature_range=minmax, copy=False) 27 | minmax.fit_transform(features) 28 | 29 | if normalize: 30 | # make sure each entry's L2 norm is 1 31 | normalizer = Normalizer(copy=False) 32 | normalizer.fit_transform(features) 33 | 34 | if bias_term: 35 | X = np.hstack([np.ones(shape=(N, 1)), features]) 36 | else: 37 | X = features 38 | 39 | return X, labels 40 | 41 | def svm_grad(w, X, y, clip=-1): 42 | y2d = np.atleast_2d(y) 43 | 44 | ywx = y * np.dot(X, w) 45 | loc = ywx < 1 46 | per_grad = -1.0 * y2d[:, loc].T * X[loc] 47 | 48 | if clip > 0: 49 | norm = np.linalg.norm(per_grad, axis=1) 50 | to_clip = norm > clip 51 | per_grad[to_clip, :] = ((clip * per_grad[to_clip]) 52 | / np.atleast_2d(norm[to_clip]).T) 53 | grad = np.sum(per_grad, axis=0) 54 | else: 55 | grad = np.sum(per_grad, axis=0) 56 | 57 | return grad 58 | 59 | def svm_loss(w, X, y, clip=-1): 60 | is_1d = w.ndim == 1 61 | 62 | w = np.atleast_2d(w) 63 | y = np.atleast_2d(y) 64 | 65 | wx = np.dot(X, w.T) 66 | obj = 1.0 - (y.T * wx) 67 | obj[obj < 0] = 0 68 | 69 | # clipping 70 | if clip > 0: 71 | obj[obj > clip] = clip 72 | 73 | # reg = lmbda * np.sum(np.square(w[:, 1:]), axis=1) 74 | loss = np.sum(obj, axis=0) 75 | 76 | # loss = hinge + reg 77 | 78 | if is_1d: 79 | loss = np.asscalar(loss) 80 | 81 | return loss 82 | 83 | def svm_test(w, X, y): 84 | is_1d = w.ndim == 1 85 | 86 | N = X.shape[0] 87 | w2d = np.atleast_2d(w) 88 | y2d = np.atleast_2d(y) 89 | 90 | wx = np.dot(X, w2d.T) 91 | sign = y2d.T * wx 92 | 93 | cnt = np.count_nonzero(sign > 0, axis=0) 94 | 95 | if is_1d: 96 | cnt = np.squeeze(cnt) 97 | 98 | return cnt / float(N) 99 | 100 | def get_1_norm(params_a): 101 | sum = 0 102 | if isinstance(params_a,np.ndarray) == True: 103 | sum += pow(np.linalg.norm(params_a, ord=2),2) 104 | else: 105 | for i in params_a.keys(): 106 | if len(params_a[i]) == 1: 107 | sum += pow(np.linalg.norm(params_a[i].cpu().numpy(), ord=2),2) 108 | else: 109 | a = copy.deepcopy(params_a[i].cpu().numpy()) 110 | for j in a: 111 | x = copy.deepcopy(j.flatten()) 112 | sum += pow(np.linalg.norm(x, ord=2),2) 113 | norm = np.sqrt(sum) 114 | return norm 115 | 116 | def clipping(w, clipthr): 117 | if get_1_norm(w) > clipthr: 118 | w_local = copy.deepcopy(w) 119 | for i in w.keys(): 120 | w_local[i]=copy.deepcopy(w[i]*clipthr/get_1_norm(w)) 121 | else: 122 | w_local = copy.deepcopy(w) 123 | return w_local 124 | 125 | def noise_add(w, noise_scale,dim): 126 | w_noise = copy.deepcopy(w) 127 | noise = np.random.normal(0, noise_scale ,dim) 128 | noise = np.clip(noise,-3*noise_scale,3*noise_scale) 129 | w_noise = w_noise + noise 130 | return w_noise 131 | 132 | 133 | def main(args): 134 | fpath = "./dataset/{0}.dat".format(args.dname) 135 | X, y = load_dat(fpath, minmax=(0, 1), normalize=False, bias_term=True) 136 | N, dim = X.shape 137 | y[y < 1] = -1 138 | 139 | grad_clip = args.grad_clip 140 | set_num_epochs = args.num_epochs 141 | reg_coeff = args.reg_coeff 142 | step_size = args.step_size 143 | 144 | delta = args.delta 145 | set_privacy_budget = args.set_privacy_budget 146 | clipthr = args.clipthr 147 | 148 | num_experiments = args.num_experiments 149 | 150 | num_users = args.num_users 151 | num_Chosenusers = args.num_Chosenusers 152 | num_train = args.num_train 153 | 154 | for eps in range(len(set_privacy_budget)): 155 | privacy_budget = copy.deepcopy(set_privacy_budget[eps]) 156 | print('Privacy budget:{}, Clients number:{},Chosen clients:{}, Dataset size:{}, Experiment times:{}\n'.format(privacy_budget,\ 157 | num_users, num_Chosenusers, num_train, num_experiments)) 158 | epo_acc, epo_obj = [], [0] 159 | for j in range(len(set_num_epochs)): 160 | num_epochs = copy.deepcopy(set_num_epochs[j]) 161 | avg_acc, avg_obj =[], [] 162 | q_s = num_Chosenusers/num_users 163 | if privacy_budget>10000: 164 | noise_scale = 0 165 | else: 166 | noise_scale = 2*clipthr*np.sqrt(2*q_s*num_epochs*np.log(1/delta))/(privacy_budget*num_train) 167 | # noise_scale = 0 168 | for num_exper in range(num_experiments): 169 | acc_test = np.zeros(num_users) 170 | obj_test = np.zeros(num_users) 171 | avg_acc_test, avg_obj_test = [], [] 172 | acc_train = np.zeros(num_users) 173 | obj_train = np.zeros(num_users) 174 | avg_acc_train, avg_obj_train = [], [] 175 | sol_glob = copy.deepcopy(np.zeros(dim)) 176 | for i in range(num_epochs): 177 | sol_locals = [] 178 | if num_Chosenusers < num_users: 179 | chosenUsers = random.sample(range(1,num_users),num_Chosenusers) 180 | chosenUsers.sort() 181 | else: 182 | chosenUsers = range(num_users) 183 | # print("\nChosen users:", chosenUsers) 184 | for k in chosenUsers: 185 | train_X, train_y = X[k*num_train:(k+1)*num_train,:], y[k*num_train:(k+1)*num_train] 186 | test_X, test_y = X[num_users*num_train:,:], y[num_users*num_train:] 187 | 188 | n_train = train_X.shape[0] 189 | n_test = test_X.shape[0] 190 | N, dim = train_X.shape 191 | sol = copy.deepcopy(sol_glob) 192 | if args.batch_size > 0: 193 | # build a mini-batch 194 | idx_len = int(np.ceil(N/args.batch_size)) 195 | for batch_idx in range(idx_len): 196 | mini_X = X[batch_idx*args.batch_size:(batch_idx+1)*args.batch_size, :] 197 | mini_y = y[batch_idx*args.batch_size:(batch_idx+1)*args.batch_size] 198 | grad = svm_grad(sol, mini_X, mini_y, grad_clip) 199 | if reg_coeff > 0: 200 | grad += reg_coeff * sol 201 | sol += - step_size * grad 202 | # rand_idx = np.random.choice(N, size=args.batch_size, replace=False) 203 | else: 204 | mini_X = X 205 | mini_y = y 206 | grad = svm_grad(sol, mini_X, mini_y, grad_clip) 207 | if reg_coeff > 0: 208 | grad += reg_coeff * sol 209 | sol += - step_size * grad 210 | sol_locals.append(sol) 211 | obj_train[k] = svm_loss(sol, train_X, train_y) / n_train 212 | acc_train[k] = svm_test(sol, train_X, train_y) * 100.0 213 | 214 | sol_glob = np.zeros(dim) 215 | for k in range(len(chosenUsers)): 216 | ### Clipping ### 217 | sol_locals[k] = copy.deepcopy(clipping(sol_locals[k], clipthr)) 218 | # print('\nLocal parameters' ,w_locals[i]) 219 | ### Add noise ### 220 | sol_locals[k] = copy.deepcopy(noise_add(sol_locals[k], noise_scale, dim)) 221 | sol_glob += sol_locals[k] 222 | sol_glob = sol_glob/len(chosenUsers) 223 | 224 | obj_test = svm_loss(sol_glob, test_X, test_y) / n_test 225 | acc_test = svm_test(sol_glob, test_X, test_y) * 100.0 226 | 227 | avg_acc_train.append(sum(acc_train)/len(acc_train)) 228 | avg_obj_train.append(sum(obj_train)/len(obj_train)) 229 | avg_acc_test.append(acc_test) 230 | avg_obj_test.append(obj_test) 231 | 232 | avg_acc.append(avg_acc_test[-1]) 233 | avg_obj.append(avg_obj_test[-1]) 234 | epo_obj.append(sum(avg_obj)/len(avg_obj)) 235 | epo_acc.append(sum(avg_acc)/len(avg_acc)) 236 | # print('*' * 20,f'Epoch[{i+1}/{num_epochs}]','*' * 20) 237 | print(f'loss: {sum(avg_obj)/len(avg_obj):.6f}, acc: {sum(avg_acc)/len(avg_acc):.6f}, STD: {noise_scale}') 238 | if (epo_obj[-1]+epo_obj[-2])/2>epo_obj[1]: 239 | break 240 | print('Total loss:{}\n'.format(epo_obj)) 241 | 242 | if __name__ == "__main__": 243 | parser = argparse.ArgumentParser(description='adaptive sgd') 244 | # parser.add_argument('--dname', help='IPUMS-US') 245 | parser.add_argument('--dname', default='ADULT') 246 | parser.add_argument('--rep', type=int, default=1) 247 | parser.add_argument('--step_size', type=float, default=0.001) 248 | parser.add_argument('--grad_clip', type=float, default=3.0) 249 | parser.add_argument('--num_epochs', type=list, default=range(10,205,10)) 250 | parser.add_argument('--reg_coeff', type=float, default=0.001) 251 | parser.add_argument('--batch_size', type=int, default=10) 252 | parser.add_argument('--num_users', type=int, default=50) 253 | parser.add_argument('--num_Chosenusers', type=int, default=50) 254 | parser.add_argument('--num_train', type=int, default=128) 255 | parser.add_argument('--num_experiments', type=int, default=2) 256 | parser.add_argument('--delta', type=float, default=0.01) 257 | parser.add_argument('--set_privacy_budget', type=int, default=[100000000]) 258 | parser.add_argument('--clipthr', type=int, default=10) 259 | 260 | args = parser.parse_args() 261 | 262 | # fpath = "./dataset/{0}.dat".format(args.dname) 263 | # X, y = load_dat(fpath, minmax=(0, 1), normalize=False, bias_term=True) 264 | 265 | print("Running the program ... [{0}]".format( 266 | time.strftime("%m/%d/%Y %H:%M:%S"))) 267 | print("Parameters") 268 | print("----------") 269 | 270 | for arg in vars(args): 271 | print(" - {0:22s}: {1}".format(arg, getattr(args, arg))) 272 | 273 | #start_time = time.clock() 274 | 275 | main(args) 276 | 277 | #elapsed = time.clock() - start_time 278 | #mins, sec = divmod(elapsed, 60) 279 | #hrs, mins = divmod(mins, 60) 280 | 281 | print("The program finished. [{0}]".format( 282 | time.strftime("%m/%d/%Y %H:%M:%S"))) 283 | #print("Elasepd time: %d:%02d:%02d" % (hrs, mins, sec)) 284 | -------------------------------------------------------------------------------- /Noise_add.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sat Jul 6 10:59:16 2019 4 | 5 | @author: WEIKANG 6 | """ 7 | 8 | import numpy as np 9 | import copy 10 | import torch 11 | import random 12 | from Calculate import get_1_norm, get_2_norm, inner_product, avg_grads 13 | 14 | def noise_add(args, noise_scale, w): 15 | w_noise = copy.deepcopy(w) 16 | if isinstance(w[0],np.ndarray) == True: 17 | noise = np.random.normal(0,noise_scale,w.size()) 18 | w_noise = w_noise + noise 19 | else: 20 | for k in range(len(w)): 21 | for i in w[k].keys(): 22 | noise = np.random.normal(0,noise_scale,w[k][i].size()) 23 | if args.gpu != -1: 24 | noise = torch.from_numpy(noise).float().cuda() 25 | else: 26 | noise = torch.from_numpy(noise).float() 27 | w_noise[k][i] = w_noise[k][i] + noise 28 | return w_noise 29 | 30 | def users_sampling(args, w, chosenUsers): 31 | if args.num_chosenUsers < args.num_users: 32 | w_locals = [] 33 | for i in range(len(chosenUsers)): 34 | w_locals.append(w[chosenUsers[i]]) 35 | else: 36 | w_locals = copy.deepcopy(w) 37 | return w_locals 38 | 39 | def clipping(args, w): 40 | if get_1_norm(w) > args.clipthr: 41 | w_local = copy.deepcopy(w) 42 | for i in w.keys(): 43 | w_local[i]=copy.deepcopy(w[i]*args.clipthr/get_1_norm(w)) 44 | else: 45 | w_local = copy.deepcopy(w) 46 | return w_local -------------------------------------------------------------------------------- /Privacy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Jul 5 20:37:06 2019 4 | 5 | @author: WEIKANG 6 | """ 7 | 8 | import numpy as np 9 | import copy 10 | import torch 11 | import math 12 | from Calculate import f_zero 13 | 14 | 15 | def Privacy_account(args, threshold_epochs, noise_list, iter): 16 | q_s = args.num_Chosenusers/args.num_users 17 | delta_s = 2*args.clipthr/args.num_items_train 18 | if args.dp_mechanism != 'CRD': 19 | noise_scale = delta_s*np.sqrt(2*q_s*threshold_epochs*np.log(1/args.delta))/args.privacy_budget 20 | elif args.dp_mechanism == 'CRD': 21 | noise_sum = 0 22 | for i in range(len(noise_list)): 23 | noise_sum += pow(1/noise_list[i],2) 24 | if pow(args.privacy_budget/delta_s,2)/(2*q_s*np.log(1/args.delta))>noise_sum: 25 | noise_scale = np.sqrt((threshold_epochs-iter)/(pow(args.privacy_budget/delta_s,2)/(2*q_s*np.log(1/args.delta))-noise_sum)) 26 | else: 27 | noise_scale = noise_list[-1] 28 | return noise_scale 29 | 30 | 31 | def Adjust_T(args, loss_avg_list, threshold_epochs_list, iter): 32 | if loss_avg_list[iter-1]-loss_avg_list[iter-2]>=0: 33 | threshold_epochs = copy.deepcopy(math.floor( math.ceil(args.dec_cons*threshold_epochs_list[-1]))) 34 | # print('\nThreshold epochs:', threshold_epochs_list) 35 | else: 36 | threshold_epochs = threshold_epochs_list[-1] 37 | return threshold_epochs 38 | 39 | def Noise_TB_decay(args, noise_list, loss_avg_list, dec_cons, iter, method_selected): 40 | if loss_avg_list[-1]-loss_avg_list[-2]>=0: 41 | if method_selected == 'UD': 42 | noise_scale = copy.deepcopy(noise_list[-1]*dec_cons) 43 | elif method_selected == 'TBD': 44 | noise_scale = copy.deepcopy(noise_list[0]/(1+dec_cons*iter)) 45 | elif method_selected == 'ED': 46 | noise_scale = copy.deepcopy(noise_list[0]*np.exp(-dec_cons*iter)) 47 | else: 48 | noise_scale = copy.deepcopy(noise_list[-1]) 49 | q_s = args.num_Chosenusers/args.num_users 50 | eps_tot = 0 51 | for i in range(len(noise_list)): 52 | eps_tot = copy.deepcopy(eps_tot + 1/pow(noise_list[i], 2)) 53 | eps_tot = copy.deepcopy(np.sqrt(eps_tot*2*q_s*np.log(1/args.delta)/pow(args.num_users,2))) 54 | return noise_scale, eps_tot 55 | 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Federated Learning with Differential Privacy 2 | ====== 3 | Citation 4 | ----- 5 | If you find "federated learning with DP" useful in your research, please consider citing: 6 | 7 | @ARTICLE{Wei2020Fed, 8 | author={Kang Wei and Jun Li and Ming Ding and Chuan Ma 9 | and Howard H. Yang and Farhad Farokhi and Shi Jin 10 | and Tony Q. S. Quek and H. Vincent Poor}, 11 | journal={{IEEE} Transactions on Information Forensics and Security}, 12 | title={Federated Learning with Differential Privacy: {Algorithms} and Performance Analysis}, 13 | year={2020}, 14 | volume={15}, 15 | number={}, 16 | pages={3454-3469},} 17 | 18 | @ARTICLE{Ma202On, 19 | author={C. {Ma} and J. {Li} and M. {Ding} and H. H. {Yang} and F. {Shu} and T. Q. S. {Quek} and H. V. {Poor}}, 20 | title={On Safeguarding Privacy and Security in the Framework of Federated Learning}, 21 | journal = {{IEEE} Network}, 22 | volume = {34}, 23 | number = {4}, 24 | pages = {242-248}, 25 | year = {2020},} 26 | 27 | Prerequisites 28 | ----- 29 | Python 3.6 30 | Torch 1.5.1 31 | Models&Data 32 | ----- 33 | 34 | Training 35 | ----- 36 | 37 | Description 38 | ----- 39 | -------------------------------------------------------------------------------- /Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Python version: 3.6 4 | 5 | import torch 6 | from torch import nn, autograd 7 | from torch.utils.data import DataLoader, Dataset 8 | import numpy as np 9 | from sklearn import metrics 10 | import copy 11 | #matplotlib.use('Agg') 12 | 13 | class DatasetSplit(Dataset): 14 | def __init__(self, dataset, idxs): 15 | self.dataset = dataset 16 | self.idxs = list(idxs) 17 | 18 | def __len__(self): 19 | return len(self.idxs) 20 | 21 | def __getitem__(self, item): 22 | 23 | image, label = self.dataset[int(self.idxs[item])] 24 | return image, label 25 | 26 | 27 | class LocalUpdate(object): 28 | def __init__(self, args, dataset, idxs, tb): 29 | self.args = args 30 | self.tb = tb 31 | self.loss_func = nn.CrossEntropyLoss() 32 | # self.loss_func = nn.NLLLoss() 33 | self.ldr_train, self.ldr_test = self.train_val_test(dataset, list(idxs)) 34 | 35 | def train_val_test(self, dataset, idxs): 36 | # split train, and test 37 | idxs_train = idxs 38 | if (self.args.dataset == 'mnist') or (self.args.dataset == 'cifar'): 39 | idxs_test = idxs 40 | train = DataLoader(DatasetSplit(dataset, idxs_train), batch_size=self.args.local_bs, shuffle=True) 41 | #val = DataLoader(DatasetSplit(dataset, idxs_val), batch_size=int(len(idxs_val)/10), shuffle=True) 42 | test = DataLoader(DatasetSplit(dataset, idxs_test), batch_size=int(len(idxs_test)), shuffle=False) 43 | else: 44 | train = self.args.dataset_train[idxs] 45 | test = self.args.dataset_test[idxs] 46 | return train, test 47 | 48 | def update_weights(self, net): 49 | net.train() 50 | # train and update 51 | optimizer = torch.optim.SGD(net.parameters(), lr=self.args.lr, momentum=0) 52 | 53 | epoch_loss = [] 54 | epoch_acc = [] 55 | for iter in range(self.args.local_ep): 56 | batch_loss = [] 57 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 58 | if self.args.gpu != -1: 59 | images, labels = images.cuda(), labels.cuda() 60 | images, labels = autograd.Variable(images),\ 61 | autograd.Variable(labels) 62 | net.zero_grad() 63 | log_probs = net(images) 64 | loss = self.loss_func(log_probs, labels) 65 | loss.backward() 66 | optimizer.step() 67 | if self.args.gpu == -1: 68 | loss = loss.cpu() 69 | self.tb.add_scalar('loss', loss.data.item()) 70 | batch_loss.append(loss.data.item()) 71 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 72 | acc, _, = self.test(net) 73 | epoch_acc.append(acc) 74 | if iter == 0: 75 | w_1st_ep = copy.deepcopy(net.state_dict()) 76 | avg_loss = sum(epoch_loss)/len(epoch_loss) 77 | avg_acc = sum(epoch_acc)/len(epoch_acc) 78 | w = net.state_dict() 79 | return w_1st_ep, w, avg_loss ,avg_acc 80 | 81 | def test(self, net): 82 | loss = 0 83 | log_probs = [] 84 | labels = [] 85 | for batch_idx, (images, labels) in enumerate(self.ldr_test): 86 | if self.args.gpu != -1: 87 | images, labels = images.cuda(), labels.cuda() 88 | images, labels = autograd.Variable(images), autograd.Variable(labels) 89 | net = net.float() 90 | log_probs = net(images) 91 | loss = self.loss_func(log_probs, labels) 92 | if self.args.gpu == -1: 93 | loss = loss.cpu() 94 | log_probs = log_probs.cpu() 95 | labels = labels.cpu() 96 | y_pred = np.argmax(log_probs.data, axis=1) 97 | acc = metrics.accuracy_score(y_true=labels.data, y_pred=y_pred) 98 | loss = loss.data.item() 99 | return acc, loss -------------------------------------------------------------------------------- /__pycache__/Calculate.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/Calculate.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/FedNets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/FedNets.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/Noise_add.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/Noise_add.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/Privacy.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/Privacy.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/Update.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/Update.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/averaging.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/averaging.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/options.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/options.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/sampling.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/__pycache__/sampling.cpython-37.pyc -------------------------------------------------------------------------------- /averaging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Python version: 3.6 4 | 5 | import copy 6 | import torch 7 | import numpy as np 8 | from torch import nn 9 | 10 | 11 | def average_weights(w): 12 | w_avg = copy.deepcopy(w[0]) 13 | if isinstance(w[0],np.ndarray) == True: 14 | for i in range(1, len(w)): 15 | w_avg += w[i] 16 | w_avg = w_avg/len(w) 17 | else: 18 | for k in w_avg.keys(): 19 | for i in range(1, len(w)): 20 | w_avg[k] += w[i][k] 21 | w_avg[k] = torch.div(w_avg[k], len(w)) 22 | return w_avg 23 | 24 | def average_FSVRG_weights(w, ag_scalar, net, gpu=-1): 25 | """ 26 | This method is for using FSVRG algo to update global parameters 27 | :param w: list of client's state_dict 28 | :param ag_scalar: simpilicity for A Matrix 29 | :param net: global net model 30 | :return: global state_dict 31 | """ 32 | w_t = copy.deepcopy(net.state_dict()) 33 | #print("=======================before==============================") 34 | #print(w_t) 35 | sg = {} 36 | total_size = np.array(np.sum([u[0] for u in w])) 37 | for key in w_t.keys(): 38 | sg[key] = np.zeros(w_t[key].shape) 39 | for l in range(len(w)): 40 | for k in sg.keys(): 41 | # += ag_scalar * w[l][0] * (w[l][1][k] - w_t[k]) / total_size 42 | if(gpu!= -1): 43 | tmp_w = (w[l][1][k] - w_t[k]).cpu() 44 | sg[k] = np.add(sg[k], w[l][0] * tmp_w) 45 | else: 46 | sg[k] = np.add(sg[k], w[l][0] * (w[l][1][k] - w_t[k]))#np.add(sg[k].long(), torch.div(ag_scalar * w[l][0] * (torch.add(w[l][1][k], -w_t[k])).long(), total_size.long()).long()) 47 | for key in w_t.keys(): 48 | if (gpu != -1): 49 | w_t[key] = np.add(w_t[key].cpu(), np.divide(ag_scalar * sg[key], total_size)) 50 | else: 51 | w_t[key] = np.add(w_t[key], np.divide(ag_scalar * sg[key], total_size)) 52 | #print('===========================after===================================') 53 | #print(w_t) 54 | return w_t 55 | -------------------------------------------------------------------------------- /dataset/mnist/MNIST/processed/test.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/processed/test.pt -------------------------------------------------------------------------------- /dataset/mnist/MNIST/processed/training.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/processed/training.pt -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/t10k-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/t10k-images-idx3-ubyte -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/t10k-labels-idx1-ubyte: -------------------------------------------------------------------------------- 1 | '                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/train-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/train-images-idx3-ubyte -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/train-labels-idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/train-labels-idx1-ubyte -------------------------------------------------------------------------------- /dataset/mnist/MNIST/raw/train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/dataset/mnist/MNIST/raw/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /local/events.out.tfevents.1612356994.DESKTOP-U8MK2VQ: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheWitcher05/Federated_learning_with_differential_privacy/77c622399519520959030e6365ca4cb038d32294/local/events.out.tfevents.1612356994.DESKTOP-U8MK2VQ -------------------------------------------------------------------------------- /options.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Python version: 3.6 4 | 5 | import argparse 6 | 7 | 8 | def args_parser(): 9 | parser = argparse.ArgumentParser() 10 | # federated arguments 11 | parser.add_argument('--epochs', type=int, default=10, help="rounds of training") 12 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 13 | parser.add_argument('--frac', type=float, default=0.1, help='the fraction of clients: C') 14 | parser.add_argument('--local_ep', type=int, default=5, help="the number of local epochs: E") 15 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 16 | parser.add_argument('--lr', type=float, default=0.001, help='learning rate') 17 | parser.add_argument('--momentum', type=float, default=0.5, help='SGD momentum (default: 0.5)') 18 | 19 | # model arguments 20 | parser.add_argument('--model', type=str, default='mlp', help='model name') 21 | parser.add_argument('--kernel_num', type=int, default=9, help='number of each kind of kernel') 22 | parser.add_argument('--kernel_sizes', type=str, default='3,4,5', 23 | help='comma-separated kernel size to use for convolution') 24 | parser.add_argument('--norm', type=str, default='batch_norm', help="batch_norm, layer_norm, or None") 25 | parser.add_argument('--num_filters', type=int, default=32, 26 | help="number of filters for conv nets -- 32 for miniimagenet, 64 for omiglot.") 27 | parser.add_argument('--max_pool', type=str, default='True', 28 | help="Whether use max pooling rather than strided convolutions") 29 | 30 | # other arguments 31 | parser.add_argument('--dataset', type=str, default='mnist', help="name of dataset") 32 | parser.add_argument('--iid', type=int, default=0, help='whether i.i.d or not, 1 for iid, 0 for non-iid') 33 | parser.add_argument('--num_classes', type=int, default=10, help="number of classes") 34 | parser.add_argument('--num_channels', type=int, default=3, help="number of channels of imges") 35 | parser.add_argument('--gpu', type=int, default=-1, help="GPU ID") 36 | parser.add_argument('--stopping_rounds', type=int, default=10, help='rounds of early stopping') 37 | parser.add_argument('--verbose', type=int, default=1, help='verbose print, 1 for True, 0 for False') 38 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 39 | parser.add_argument('--ag_scalar', type=float, default=1.0, help="global aggregation updating scalar, simplicity for A Matrix") 40 | parser.add_argument('--lg_scalar', type=float, default=1.0, help="client local updating scalar, simplicity for S Matrix") 41 | parser.add_argument('--algorithm', type=str, default='fedavg', help='algorithm for optimization') 42 | args = parser.parse_args() 43 | return args 44 | 45 | -------------------------------------------------------------------------------- /sampling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Python version: 3.6 4 | 5 | 6 | import numpy as np 7 | import copy 8 | from torchvision import datasets, transforms 9 | 10 | # np.random.seed(1) 11 | def unique_index(L,f): 12 | return [i for (i,value) in enumerate(L) if value==f] 13 | 14 | def mnist_iid(dataset, num_users, num_items): 15 | """ 16 | Sample I.I.D. client data from MNIST dataset 17 | :param dataset: 18 | :param num_users: 19 | :return: dict of image index 20 | """ 21 | dict_users, all_idxs = {}, [i for i in range(len(dataset))] 22 | if len(dataset) == 60000: 23 | num_digits = int(num_items/10) 24 | labels = dataset.train_labels.numpy() 25 | classes = np.unique(labels) 26 | classes_index = [] 27 | for i in range(len(classes)): 28 | classes_index.append(unique_index(labels, classes[i])) 29 | for i in range(num_users): 30 | c = [] 31 | for j in range(10): 32 | b = (np.random.choice(classes_index[j], num_digits, replace=False)) 33 | for m in range(num_digits): 34 | c.append(b[m]) 35 | # print(c) 36 | dict_users[i] = set(c) 37 | else: 38 | # num_digits = int(num_items/10) 39 | # labels = dataset.test_labels.numpy() 40 | # classes = np.unique(labels) 41 | # classes_index = [] 42 | # for i in range(len(classes)): 43 | # classes_index.append(unique_index(labels, classes[i])) 44 | # c = [] 45 | # for j in range(10): 46 | # b = (np.random.choice(classes_index[j], num_digits, replace=False)) 47 | # for m in range(num_digits): 48 | # c.append(b[m]) 49 | c = set(np.random.choice(all_idxs, num_items, replace=False)) 50 | for i in range(num_users): 51 | dict_users[i] = copy.deepcopy(c) 52 | # print("\nDivide", len(all_idxs)) 53 | return dict_users 54 | 55 | def mnist_noniid(dataset, num_users): 56 | """ 57 | Sample non-I.I.D client data from MNIST dataset 58 | :param dataset: 59 | :param num_users: 60 | :return: 61 | """ 62 | num_shards, num_imgs = 200, 300 63 | idx_shard = [i for i in range(num_shards)] 64 | dict_users = {i: np.array([]) for i in range(num_users)} 65 | idxs = np.arange(num_shards*num_imgs) 66 | labels = dataset.train_labels.numpy() 67 | # print("total_data:",len(labels)) 68 | 69 | # sort labels 70 | idxs_labels = np.vstack((idxs, labels)) 71 | idxs_labels = idxs_labels[:,idxs_labels[1,:].argsort()] 72 | idxs = idxs_labels[0,:] 73 | print(idxs) 74 | 75 | # divide and assign 76 | for i in range(num_users): 77 | rand_set = set(np.random.choice(idx_shard, 4, replace=False)) 78 | #idx_shard = list(set(idx_shard) - rand_set) 79 | for rand in rand_set: 80 | dict_users[i] = np.concatenate((dict_users[i], idxs[rand*num_imgs:(rand+1)*num_imgs]), axis=0) 81 | 82 | return dict_users 83 | 84 | 85 | def cifar_iid(dataset, num_users): 86 | """ 87 | Sample I.I.D. client data from CIFAR10 dataset 88 | :param dataset: 89 | :param num_users: 90 | :return: dict of image index 91 | """ 92 | num_items = int(len(dataset)/num_users) 93 | dict_users, all_idxs = {}, [i for i in range(len(dataset))] 94 | for i in range(num_users): 95 | dict_users[i] = set(np.random.choice(all_idxs, num_items, replace=False)) 96 | all_idxs = list(set(all_idxs) - dict_users[i]) 97 | return dict_users 98 | 99 | 100 | # Divide into 100 portions of total data. Allocate 2 random portions for each user 101 | def cifar_noniid(dataset, num_users): 102 | num_shards, num_imgs = 100, 500 103 | idx_shard = [i for i in range(num_shards)] 104 | dict_users = {i: np.array([]) for i in range(num_users)} 105 | idxs = np.arange(num_shards * num_imgs) 106 | labels = np.array(dataset.train_labels)#.numpy() 107 | print(len(idxs)) 108 | print(len(labels)) 109 | # sort labels 110 | idxs_labels = np.vstack((idxs, labels)) 111 | idxs_labels = idxs_labels[:, idxs_labels[1, :].argsort()] 112 | idxs = idxs_labels[0, :] 113 | 114 | # divide and assign 115 | for i in range(num_users): 116 | rand_set = set(np.random.choice(idx_shard, 4, replace=False)) 117 | #idx_shard = list(set(idx_shard) - rand_set) 118 | for rand in rand_set: 119 | dict_users[i] = np.concatenate((dict_users[i], idxs[rand * num_imgs:(rand + 1) * num_imgs]), axis=0) 120 | #np.random.shuffle(dict_users[i]) 121 | return dict_users 122 | 123 | 124 | if __name__ == '__main__': 125 | dataset_train = datasets.MNIST('./data/mnist/', train=True, download=True, 126 | transform=transforms.Compose([ 127 | transforms.ToTensor(), 128 | transforms.Normalize((0.1307,), (0.3081,)) 129 | ])) 130 | num = 100 131 | d = mnist_noniid(dataset_train, num) --------------------------------------------------------------------------------