├── Federated Learning.pdf ├── Federated Implementation.pdf ├── Federated Learning └── Federated-Learning-and-Split-Learning-with-raspberry-pi-master │ ├── ensemble_learning │ ├── README.md │ └── ecg │ │ ├── README.md │ │ ├── ecg_es_client_rasp.py │ │ ├── ecg_es_client.ipynb │ │ └── ecg_es_client_rasp.ipynb │ ├── federated_learning │ ├── README.md │ ├── ecg │ │ ├── README.md │ │ ├── ecg_fd_client_rasp.py │ │ ├── ecg_fd_client_rasp.ipynb │ │ └── ecg_fd_server.ipynb │ ├── cifar10_MobileNet │ │ ├── README.md │ │ ├── cifar10_mobile_fd_client_rasp.py │ │ ├── cifar10_mobile_fd_client.ipynb │ │ └── cifar10_mobile_fd_client_rasp.ipynb │ └── cifar10_Resnet20 │ │ ├── README.md │ │ ├── cifar10_resnet20_fd_client_rasp.py │ │ ├── cifar10_resnet20_fd_client.ipynb │ │ └── cifar10_resnet20_fd_client_rasp.ipynb │ └── LICENSE └── README.md /Federated Learning.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SyedUmaidAhmed/Federated-and-Split-Learning-on-Edge/HEAD/Federated Learning.pdf -------------------------------------------------------------------------------- /Federated Implementation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SyedUmaidAhmed/Federated-and-Split-Learning-on-Edge/HEAD/Federated Implementation.pdf -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/ensemble_learning/README.md: -------------------------------------------------------------------------------- 1 | # Ensemble-Learning 2 | This directory is for ensemble learning source code 3 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/README.md: -------------------------------------------------------------------------------- 1 | # Federated-Learning 2 | This directory is for federated learning source code 3 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/ecg/README.md: -------------------------------------------------------------------------------- 1 | ## ECG federated learning 2 | - Use 4conv layer 1D CNN model for training 3 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_MobileNet/README.md: -------------------------------------------------------------------------------- 1 | ## CIFAR 10 federated learning 2 | - Use mobilenet for training 3 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_Resnet20/README.md: -------------------------------------------------------------------------------- 1 | ## CIFAR 10 federated learning 2 | - Use resnet20 for training 3 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/ensemble_learning/ecg/README.md: -------------------------------------------------------------------------------- 1 | ## ECG ensemble learning 2 | - Use three models (4conv, 5conv, 6conv layers of 1D CNN models) for training. 3 | - In this code, client has only 2 conv layers. 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Federated-and-Split-Learning-on-Edge 2 | This is the official implementation of Federated and Split Learning on multiple Raspberry Pi's. It is the demonstration of training the data on edge devices without having threat to security issues. 3 | 4 | The complete explanation and implementation video can be seen on my YouTube Channel: 5 | 6 | https://youtu.be/zYb4NuuMpiU 7 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Minki-Kim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/ensemble_learning/ecg/ecg_es_client_rasp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # # ECG Ensemble 1D-CNN Client Side with Raspberry pie 5 | # This code is the server part of ECG ensemble 1D-CNN model for **multi** client and a server. 6 | 7 | # In[1]: 8 | 9 | 10 | users = 3 # number of clients 11 | 12 | 13 | # ## Import required packages 14 | 15 | # In[2]: 16 | 17 | 18 | import os 19 | import struct 20 | import socket 21 | import pickle 22 | import time 23 | 24 | import h5py 25 | from tqdm import tqdm 26 | 27 | import torch 28 | import torch.nn as nn 29 | from torch.utils.data import Dataset, DataLoader 30 | from torch.optim import Adam 31 | import copy 32 | 33 | 34 | # In[3]: 35 | 36 | 37 | root_path = '../../models/' 38 | 39 | 40 | # In[ ]: 41 | 42 | 43 | def getFreeDescription(): 44 | free = os.popen("free -h") 45 | i = 0 46 | while True: 47 | i = i + 1 48 | line = free.readline() 49 | if i == 1: 50 | return (line.split()[0:7]) 51 | 52 | 53 | def getFree(): 54 | free = os.popen("free -h") 55 | i = 0 56 | while True: 57 | i = i + 1 58 | line = free.readline() 59 | if i == 2: 60 | return (line.split()[0:7]) 61 | 62 | from gpiozero import CPUTemperature 63 | 64 | 65 | def printPerformance(): 66 | cpu = CPUTemperature() 67 | 68 | print("temperature: " + str(cpu.temperature)) 69 | 70 | description = getFreeDescription() 71 | mem = getFree() 72 | 73 | print(description[0] + " : " + mem[1]) 74 | print(description[1] + " : " + mem[2]) 75 | print(description[2] + " : " + mem[3]) 76 | print(description[3] + " : " + mem[4]) 77 | print(description[4] + " : " + mem[5]) 78 | print(description[5] + " : " + mem[6]) 79 | 80 | 81 | # In[ ]: 82 | 83 | 84 | printPerformance() 85 | 86 | 87 | # ## SET CUDA 88 | 89 | # In[4]: 90 | 91 | 92 | # device = "cuda:0" if torch.cuda.is_available() else "cpu" 93 | device = "cpu" 94 | torch.manual_seed(777) 95 | if device =="cuda:0": 96 | torch.cuda.manual_seed_all(777) 97 | 98 | 99 | # In[5]: 100 | 101 | 102 | client_order = int(input("client_order(start from 0): ")) 103 | 104 | 105 | # In[6]: 106 | 107 | 108 | num_traindata = 13244 // users 109 | 110 | 111 | # ## Define ECG dataset class 112 | 113 | # In[7]: 114 | 115 | 116 | class ECG(Dataset): 117 | def __init__(self, train=True): 118 | if train: 119 | # total: 13244 120 | with h5py.File(os.path.join(root_path, 'ecg_data', 'train_ecg.hdf5'), 'r') as hdf: 121 | self.x = hdf['x_train'][num_traindata * client_order : num_traindata * (client_order + 1)] 122 | self.y = hdf['y_train'][num_traindata * client_order : num_traindata * (client_order + 1)] 123 | 124 | else: 125 | with h5py.File(os.path.join(root_path, 'ecg_data', 'test_ecg.hdf5'), 'r') as hdf: 126 | self.x = hdf['x_test'][:] 127 | self.y = hdf['y_test'][:] 128 | 129 | def __len__(self): 130 | return len(self.x) 131 | 132 | def __getitem__(self, idx): 133 | return torch.tensor(self.x[idx], dtype=torch.float), torch.tensor(self.y[idx]) 134 | 135 | 136 | # ### Set batch size 137 | 138 | # In[8]: 139 | 140 | 141 | batch_size = 32 142 | 143 | 144 | # ## Make train and test dataset batch generator 145 | 146 | # In[9]: 147 | 148 | 149 | train_dataset = ECG(train=True) 150 | test_dataset = ECG(train=False) 151 | train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) 152 | test_loader = DataLoader(test_dataset, batch_size=batch_size) 153 | 154 | 155 | # ### Size check 156 | 157 | # In[10]: 158 | 159 | 160 | x_train, y_train = next(iter(train_loader)) 161 | print(x_train.size()) 162 | print(y_train.size()) 163 | 164 | 165 | # ### Total number of batches 166 | 167 | # In[11]: 168 | 169 | 170 | total_batch = len(train_loader) 171 | print(total_batch) 172 | 173 | 174 | # ## Define ECG client model 175 | # Client side has only **2 convolutional layers**. 176 | 177 | # In[12]: 178 | 179 | 180 | class Ecgclient(nn.Module): 181 | def __init__(self): 182 | super(Ecgclient, self).__init__() 183 | self.conv1 = nn.Conv1d(1, 16, 7, padding=3) # 128 x 16 184 | self.relu1 = nn.LeakyReLU() 185 | self.pool1 = nn.MaxPool1d(2) # 64 x 16 186 | self.conv2 = nn.Conv1d(16, 16, 5, padding=2) # 64 x 16 187 | self.relu2 = nn.LeakyReLU() 188 | 189 | def forward(self, x): 190 | x = self.conv1(x) 191 | x = self.relu1(x) 192 | x = self.pool1(x) 193 | x = self.conv2(x) 194 | x = self.relu2(x) 195 | return x 196 | 197 | 198 | # In[13]: 199 | 200 | 201 | ecg_client = Ecgclient().to(device) 202 | print(ecg_client) 203 | 204 | 205 | # In[14]: 206 | 207 | 208 | # from torchsummary import summary 209 | 210 | # print('ECG 1D CNN clients') 211 | # summary(ecg_client, (1, 130)) 212 | 213 | 214 | # ### Set other hyperparameters in the model 215 | # Hyperparameters here should be same with the server side. 216 | 217 | # In[15]: 218 | 219 | 220 | epoch = 20 # default 221 | criterion = nn.CrossEntropyLoss() 222 | lr = 0.001 223 | optimizer = Adam(ecg_client.parameters(), lr=lr) 224 | 225 | 226 | # ## Socket initialization 227 | 228 | # ### Required socket functions 229 | 230 | # In[16]: 231 | 232 | 233 | def send_msg(sock, msg): 234 | # prefix each message with a 4-byte length in network byte order 235 | msg = pickle.dumps(msg) 236 | msg = struct.pack('>I', len(msg)) + msg 237 | sock.sendall(msg) 238 | 239 | def recv_msg(sock): 240 | # read message length and unpack it into an integer 241 | raw_msglen = recvall(sock, 4) 242 | if not raw_msglen: 243 | return None 244 | msglen = struct.unpack('>I', raw_msglen)[0] 245 | # read the message data 246 | msg = recvall(sock, msglen) 247 | msg = pickle.loads(msg) 248 | return msg 249 | 250 | def recvall(sock, n): 251 | # helper function to receive n bytes or return None if EOF is hit 252 | data = b'' 253 | while len(data) < n: 254 | packet = sock.recv(n - len(data)) 255 | if not packet: 256 | return None 257 | data += packet 258 | return data 259 | 260 | 261 | # In[ ]: 262 | 263 | 264 | printPerformance() 265 | 266 | 267 | # ### Set host address and port number 268 | 269 | # In[17]: 270 | 271 | 272 | host = input("IP address: ") 273 | port = 10080 274 | 275 | 276 | # ## SET TIMER 277 | 278 | # In[18]: 279 | 280 | 281 | start_time = time.time() # store start time 282 | print("timmer start!") 283 | 284 | 285 | # ### Open the client socket 286 | 287 | # In[19]: 288 | 289 | 290 | s = socket.socket() 291 | s.connect((host, port)) 292 | 293 | 294 | # In[20]: 295 | 296 | 297 | msg = recv_msg(s) # get epoch 298 | epochs = msg['epochs'] 299 | users = msg['users'] 300 | msg = total_batch 301 | send_msg(s, msg) # send total_batch of train dataset 302 | 303 | 304 | # ## Real training process 305 | 306 | # In[21]: 307 | 308 | 309 | for e in range(epochs): 310 | for u in range(users): 311 | client_weights = recv_msg(s) 312 | ecg_client.load_state_dict(client_weights) 313 | ecg_client.eval() 314 | for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Epoch '+str(e+1)+ '_' +str(u))): 315 | x, label = data 316 | x = x.to(device) 317 | label = label.to(device) 318 | 319 | optimizer.zero_grad() 320 | output = ecg_client(x) 321 | client_output = output.clone().detach().requires_grad_(True) 322 | msg = { 323 | 'client_output': client_output, 324 | 'label': label 325 | } 326 | send_msg(s, msg) 327 | client_grad = recv_msg(s) 328 | output.backward(client_grad) 329 | optimizer.step() 330 | msg = copy.deepcopy(ecg_client.state_dict()) 331 | send_msg(s, msg) 332 | 333 | 334 | # In[ ]: 335 | 336 | 337 | printPerformance() 338 | 339 | 340 | # In[22]: 341 | 342 | 343 | end_time = time.time() #store end time 344 | print("WorkingTime of ",device ,": {} sec".format(end_time - start_time)) 345 | 346 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/ecg/ecg_fd_client_rasp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # # ECG Federated 1D-CNN Client Side 5 | # This code is the server part of ECG federated 1D-CNN model for **multi** client and a server. 6 | 7 | # In[ ]: 8 | 9 | 10 | users = 2 # number of clients 11 | 12 | 13 | # In[1]: 14 | 15 | 16 | import os 17 | import h5py 18 | 19 | import socket 20 | import struct 21 | import pickle 22 | 23 | import torch 24 | import torch.nn as nn 25 | import torch.nn.functional as F 26 | import torch.optim as optim 27 | 28 | from torch.utils.data import Dataset, DataLoader 29 | from torch.optim import Adam 30 | 31 | # for image 32 | # import matplotlib.pyplot as plt 33 | # import numpy as np 34 | 35 | import time 36 | 37 | from tqdm import tqdm 38 | 39 | 40 | # In[ ]: 41 | 42 | 43 | def getFreeDescription(): 44 | free = os.popen("free -h") 45 | i = 0 46 | while True: 47 | i = i + 1 48 | line = free.readline() 49 | if i == 1: 50 | return (line.split()[0:7]) 51 | 52 | 53 | def getFree(): 54 | free = os.popen("free -h") 55 | i = 0 56 | while True: 57 | i = i + 1 58 | line = free.readline() 59 | if i == 2: 60 | return (line.split()[0:7]) 61 | 62 | from gpiozero import CPUTemperature 63 | 64 | 65 | def printPerformance(): 66 | cpu = CPUTemperature() 67 | 68 | print("temperature: " + str(cpu.temperature)) 69 | 70 | description = getFreeDescription() 71 | mem = getFree() 72 | 73 | print(description[0] + " : " + mem[1]) 74 | print(description[1] + " : " + mem[2]) 75 | print(description[2] + " : " + mem[3]) 76 | print(description[3] + " : " + mem[4]) 77 | print(description[4] + " : " + mem[5]) 78 | print(description[5] + " : " + mem[6]) 79 | 80 | 81 | # In[ ]: 82 | 83 | 84 | printPerformance() 85 | 86 | 87 | # In[2]: 88 | 89 | 90 | root_path = '../../models/' 91 | 92 | 93 | # ## Cuda 94 | 95 | # In[3]: 96 | 97 | 98 | # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 99 | device = "cpu" 100 | print(device) 101 | 102 | 103 | # In[ ]: 104 | 105 | 106 | client_order = int(input("client_order(start from 0): ")) 107 | 108 | 109 | # In[ ]: 110 | 111 | 112 | num_traindata = 13244 // users 113 | 114 | 115 | # ## Data load 116 | 117 | # In[4]: 118 | 119 | 120 | class ECG(Dataset): 121 | def __init__(self, train=True): 122 | if train: 123 | # total: 13244 124 | with h5py.File(os.path.join(root_path, 'ecg_data', 'train_ecg.hdf5'), 'r') as hdf: 125 | self.x = hdf['x_train'][num_traindata * client_order : num_traindata * (client_order + 1)] 126 | self.y = hdf['y_train'][num_traindata * client_order : num_traindata * (client_order + 1)] 127 | 128 | else: 129 | with h5py.File(os.path.join(root_path, 'ecg_data', 'test_ecg.hdf5'), 'r') as hdf: 130 | self.x = hdf['x_test'][:] 131 | self.y = hdf['y_test'][:] 132 | 133 | def __len__(self): 134 | return len(self.x) 135 | 136 | def __getitem__(self, idx): 137 | return torch.tensor(self.x[idx], dtype=torch.float), torch.tensor(self.y[idx]) 138 | 139 | 140 | # ## Making Batch Generator 141 | 142 | # In[5]: 143 | 144 | 145 | batch_size = 32 146 | 147 | 148 | # ### `DataLoader` for batch generating 149 | # `torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)` 150 | 151 | # In[6]: 152 | 153 | 154 | train_dataset = ECG(train=True) 155 | test_dataset = ECG(train=False) 156 | trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) 157 | testloader = DataLoader(test_dataset, batch_size=batch_size) 158 | 159 | 160 | # ### Number of total batches 161 | 162 | # In[7]: 163 | 164 | 165 | train_total_batch = len(trainloader) 166 | print(train_total_batch) 167 | test_batch = len(testloader) 168 | print(test_batch) 169 | 170 | 171 | # ## Pytorch layer modules for *Conv1D* Network 172 | # 173 | # 174 | # 175 | # ### `Conv1d` layer 176 | # - `torch.nn.Conv1d(in_channels, out_channels, kernel_size)` 177 | # 178 | # ### `MaxPool1d` layer 179 | # - `torch.nn.MaxPool1d(kernel_size, stride=None)` 180 | # - Parameter `stride` follows `kernel_size`. 181 | # 182 | # ### `ReLU` layer 183 | # - `torch.nn.ReLU()` 184 | # 185 | # ### `Linear` layer 186 | # - `torch.nn.Linear(in_features, out_features, bias=True)` 187 | # 188 | # ### `Softmax` layer 189 | # - `torch.nn.Softmax(dim=None)` 190 | # - Parameter `dim` is usually set to `1`. 191 | # 192 | # ## Construct 1D-CNN ECG classification model 193 | 194 | # In[8]: 195 | 196 | 197 | class EcgConv1d(nn.Module): 198 | def __init__(self): 199 | super(EcgConv1d, self).__init__() 200 | self.conv1 = nn.Conv1d(1, 16, 7) # 124 x 16 201 | self.relu1 = nn.LeakyReLU() 202 | self.pool1 = nn.MaxPool1d(2) # 62 x 16 203 | self.conv2 = nn.Conv1d(16, 16, 5) # 58 x 16 204 | self.relu2 = nn.LeakyReLU() 205 | self.conv3 = nn.Conv1d(16, 16, 5) # 54 x 16 206 | self.relu3 = nn.LeakyReLU() 207 | self.conv4 = nn.Conv1d(16, 16, 5) # 50 x 16 208 | self.relu4 = nn.LeakyReLU() 209 | self.pool4 = nn.MaxPool1d(2) # 25 x 16 210 | self.linear5 = nn.Linear(25 * 16, 128) 211 | self.relu5 = nn.LeakyReLU() 212 | self.linear6 = nn.Linear(128, 5) 213 | self.softmax6 = nn.Softmax(dim=1) 214 | 215 | def forward(self, x): 216 | x = self.conv1(x) 217 | x = self.relu1(x) 218 | x = self.pool1(x) 219 | x = self.conv2(x) 220 | x = self.relu2(x) 221 | x = self.conv3(x) 222 | x = self.relu3(x) 223 | x = self.conv4(x) 224 | x = self.relu4(x) 225 | x = self.pool4(x) 226 | x = x.view(-1, 25 * 16) 227 | x = self.linear5(x) 228 | x = self.relu5(x) 229 | x = self.linear6(x) 230 | x = self.softmax6(x) 231 | return x 232 | 233 | 234 | # In[9]: 235 | 236 | 237 | ecg_net = EcgConv1d() 238 | ecg_net.to(device) 239 | 240 | 241 | # In[10]: 242 | 243 | 244 | criterion = nn.CrossEntropyLoss() 245 | rounds = 400 # default 246 | local_epochs = 1 # default 247 | lr = 0.001 248 | optimizer = Adam(ecg_net.parameters(), lr=lr) 249 | 250 | 251 | # ## Socket initialization 252 | # ### Required socket functions 253 | 254 | # In[11]: 255 | 256 | 257 | def send_msg(sock, msg): 258 | # prefix each message with a 4-byte length in network byte order 259 | msg = pickle.dumps(msg) 260 | msg = struct.pack('>I', len(msg)) + msg 261 | sock.sendall(msg) 262 | 263 | def recv_msg(sock): 264 | # read message length and unpack it into an integer 265 | raw_msglen = recvall(sock, 4) 266 | if not raw_msglen: 267 | return None 268 | msglen = struct.unpack('>I', raw_msglen)[0] 269 | # read the message data 270 | msg = recvall(sock, msglen) 271 | msg = pickle.loads(msg) 272 | return msg 273 | 274 | def recvall(sock, n): 275 | # helper function to receive n bytes or return None if EOF is hit 276 | data = b'' 277 | while len(data) < n: 278 | packet = sock.recv(n - len(data)) 279 | if not packet: 280 | return None 281 | data += packet 282 | return data 283 | 284 | 285 | # In[ ]: 286 | 287 | 288 | printPerformance() 289 | 290 | 291 | # ### Set host address and port number 292 | 293 | # In[12]: 294 | 295 | 296 | host = input("IP address: ") 297 | port = 10080 298 | max_recv = 100000 299 | 300 | 301 | # ### Open the client socket 302 | 303 | # In[13]: 304 | 305 | 306 | s = socket.socket() 307 | s.connect((host, port)) 308 | 309 | 310 | # ## SET TIMER 311 | 312 | # In[14]: 313 | 314 | 315 | start_time = time.time() # store start time 316 | print("timmer start!") 317 | 318 | 319 | # In[15]: 320 | 321 | 322 | msg = recv_msg(s) 323 | rounds = msg['rounds'] 324 | client_id = msg['client_id'] 325 | local_epochs = msg['local_epoch'] 326 | send_msg(s, len(train_dataset)) 327 | 328 | 329 | # In[16]: 330 | 331 | 332 | # update weights from server 333 | # train 334 | for r in range(rounds): # loop over the dataset multiple times 335 | weights = recv_msg(s) 336 | ecg_net.load_state_dict(weights) 337 | ecg_net.eval() 338 | for local_epoch in range(local_epochs): 339 | 340 | for i, data in enumerate(tqdm(trainloader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))): 341 | 342 | # get the inputs; data is a list of [inputs, labels] 343 | inputs, labels = data 344 | inputs = inputs.to(device) 345 | labels = labels.clone().detach().long().to(device) 346 | 347 | # zero the parameter gradients 348 | optimizer.zero_grad() 349 | 350 | # forward + backward + optimize 351 | outputs = ecg_net(inputs) 352 | loss = criterion(outputs, labels) 353 | loss.backward() 354 | optimizer.step() 355 | 356 | msg = ecg_net.state_dict() 357 | send_msg(s, msg) 358 | 359 | print('Finished Training') 360 | 361 | 362 | # In[ ]: 363 | 364 | 365 | printPerformance() 366 | 367 | 368 | # In[ ]: 369 | 370 | 371 | end_time = time.time() #store end time 372 | print("Training Time: {} sec".format(end_time - start_time)) 373 | 374 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_MobileNet/cifar10_mobile_fd_client_rasp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # # CIFAR10 Federated Mobilenet Client Side 5 | # This code is the server part of CIFAR10 federated mobilenet for **multi** client and a server. 6 | 7 | # In[3]: 8 | 9 | 10 | users = 1 # number of clients 11 | 12 | 13 | # In[4]: 14 | 15 | 16 | import os 17 | import h5py 18 | 19 | import socket 20 | import struct 21 | import pickle 22 | 23 | import torch 24 | import torch.nn as nn 25 | import torch.nn.functional as F 26 | import torchvision 27 | import torchvision.transforms as transforms 28 | import torch.optim as optim 29 | 30 | from torch.utils.data import Dataset, DataLoader 31 | 32 | import time 33 | 34 | from tqdm import tqdm 35 | 36 | 37 | # In[2]: 38 | 39 | 40 | def getFreeDescription(): 41 | free = os.popen("free -h") 42 | i = 0 43 | while True: 44 | i = i + 1 45 | line = free.readline() 46 | if i == 1: 47 | return (line.split()[0:7]) 48 | 49 | 50 | def getFree(): 51 | free = os.popen("free -h") 52 | i = 0 53 | while True: 54 | i = i + 1 55 | line = free.readline() 56 | if i == 2: 57 | return (line.split()[0:7]) 58 | 59 | from gpiozero import CPUTemperature 60 | 61 | 62 | def printPerformance(): 63 | cpu = CPUTemperature() 64 | 65 | print("temperature: " + str(cpu.temperature)) 66 | 67 | description = getFreeDescription() 68 | mem = getFree() 69 | 70 | print(description[0] + " : " + mem[1]) 71 | print(description[1] + " : " + mem[2]) 72 | print(description[2] + " : " + mem[3]) 73 | print(description[3] + " : " + mem[4]) 74 | print(description[4] + " : " + mem[5]) 75 | print(description[5] + " : " + mem[6]) 76 | 77 | 78 | # In[3]: 79 | 80 | 81 | printPerformance() 82 | 83 | 84 | # In[5]: 85 | 86 | 87 | root_path = '../../models/cifar10_data' 88 | 89 | 90 | # ## Cuda 91 | 92 | # In[6]: 93 | 94 | 95 | # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 96 | device = "cpu" 97 | print(device) 98 | 99 | 100 | # In[7]: 101 | 102 | 103 | client_order = int(input("client_order(start from 0): ")) 104 | 105 | 106 | # In[8]: 107 | 108 | 109 | num_traindata = 50000 // users 110 | 111 | 112 | # ## Data load 113 | 114 | # In[9]: 115 | 116 | 117 | transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))]) 118 | 119 | from torch.utils.data import Subset 120 | 121 | 122 | indices = list(range(50000)) 123 | 124 | part_tr = indices[num_traindata * client_order : num_traindata * (client_order + 1)] 125 | 126 | 127 | # In[10]: 128 | 129 | 130 | trainset = torchvision.datasets.CIFAR10 (root=root_path, train=True, download=True, transform=transform) 131 | 132 | trainset_sub = Subset(trainset, part_tr) 133 | 134 | train_loader = torch.utils.data.DataLoader(trainset_sub, batch_size=4, shuffle=True, num_workers=2) 135 | 136 | testset = torchvision.datasets.CIFAR10 (root=root_path, train=False, download=True, transform=transform) 137 | test_loader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2) 138 | 139 | 140 | # In[11]: 141 | 142 | 143 | classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') 144 | 145 | 146 | # ### Number of total batches 147 | 148 | # In[13]: 149 | 150 | 151 | train_total_batch = len(train_loader) 152 | print(train_total_batch) 153 | test_batch = len(test_loader) 154 | print(test_batch) 155 | 156 | 157 | # ## Pytorch layer modules for *Conv1D* Network 158 | # 159 | # 160 | # 161 | # ### `Conv1d` layer 162 | # - `torch.nn.Conv1d(in_channels, out_channels, kernel_size)` 163 | # 164 | # ### `MaxPool1d` layer 165 | # - `torch.nn.MaxPool1d(kernel_size, stride=None)` 166 | # - Parameter `stride` follows `kernel_size`. 167 | # 168 | # ### `ReLU` layer 169 | # - `torch.nn.ReLU()` 170 | # 171 | # ### `Linear` layer 172 | # - `torch.nn.Linear(in_features, out_features, bias=True)` 173 | # 174 | # ### `Softmax` layer 175 | # - `torch.nn.Softmax(dim=None)` 176 | # - Parameter `dim` is usually set to `1`. 177 | 178 | # In[14]: 179 | 180 | 181 | # -*- coding: utf-8 -*- 182 | """ 183 | Created on Thu Nov 1 14:23:31 2018 184 | @author: tshzzz 185 | """ 186 | 187 | import torch 188 | import torch.nn as nn 189 | 190 | 191 | def conv_dw(inplane,outplane,stride=1): 192 | return nn.Sequential( 193 | nn.Conv2d(inplane,inplane,kernel_size = 3,groups = inplane,stride=stride,padding=1), 194 | nn.BatchNorm2d(inplane), 195 | nn.ReLU(), 196 | nn.Conv2d(inplane,outplane,kernel_size = 1,groups = 1,stride=1), 197 | nn.BatchNorm2d(outplane), 198 | nn.ReLU() 199 | ) 200 | 201 | def conv_bw(inplane,outplane,kernel_size = 3,stride=1): 202 | return nn.Sequential( 203 | nn.Conv2d(inplane,outplane,kernel_size = kernel_size,groups = 1,stride=stride,padding=1), 204 | nn.BatchNorm2d(outplane), 205 | nn.ReLU() 206 | ) 207 | 208 | 209 | class MobileNet(nn.Module): 210 | 211 | def __init__(self,num_class=10): 212 | super(MobileNet,self).__init__() 213 | 214 | layers = [] 215 | layers.append(conv_bw(3,32,3,1)) 216 | layers.append(conv_dw(32,64,1)) 217 | layers.append(conv_dw(64,128,2)) 218 | layers.append(conv_dw(128,128,1)) 219 | layers.append(conv_dw(128,256,2)) 220 | layers.append(conv_dw(256,256,1)) 221 | layers.append(conv_dw(256,512,2)) 222 | 223 | for i in range(5): 224 | layers.append(conv_dw(512,512,1)) 225 | layers.append(conv_dw(512,1024,2)) 226 | layers.append(conv_dw(1024,1024,1)) 227 | 228 | self.classifer = nn.Sequential( 229 | nn.Dropout(0.5), 230 | nn.Linear(1024,num_class) 231 | ) 232 | self.feature = nn.Sequential(*layers) 233 | 234 | 235 | 236 | def forward(self,x): 237 | out = self.feature(x) 238 | out = out.mean(3).mean(2) 239 | out = out.view(-1,1024) 240 | out = self.classifer(out) 241 | return out 242 | 243 | 244 | # In[15]: 245 | 246 | 247 | mobile_net = MobileNet() 248 | mobile_net.to(device) 249 | 250 | 251 | # In[16]: 252 | 253 | 254 | lr = 0.001 255 | criterion = nn.CrossEntropyLoss() 256 | optimizer = optim.SGD(mobile_net.parameters(), lr=lr, momentum=0.9) 257 | 258 | rounds = 400 # default 259 | local_epochs = 1 # default 260 | 261 | 262 | # ## Socket initialization 263 | # ### Required socket functions 264 | 265 | # In[17]: 266 | 267 | 268 | def send_msg(sock, msg): 269 | # prefix each message with a 4-byte length in network byte order 270 | msg = pickle.dumps(msg) 271 | msg = struct.pack('>I', len(msg)) + msg 272 | sock.sendall(msg) 273 | 274 | def recv_msg(sock): 275 | # read message length and unpack it into an integer 276 | raw_msglen = recvall(sock, 4) 277 | if not raw_msglen: 278 | return None 279 | msglen = struct.unpack('>I', raw_msglen)[0] 280 | # read the message data 281 | msg = recvall(sock, msglen) 282 | msg = pickle.loads(msg) 283 | return msg 284 | 285 | def recvall(sock, n): 286 | # helper function to receive n bytes or return None if EOF is hit 287 | data = b'' 288 | while len(data) < n: 289 | packet = sock.recv(n - len(data)) 290 | if not packet: 291 | return None 292 | data += packet 293 | return data 294 | 295 | 296 | # In[15]: 297 | 298 | 299 | printPerformance() 300 | 301 | 302 | # ### Set host address and port number 303 | 304 | # In[18]: 305 | 306 | 307 | host = input("IP address: ") 308 | port = 10080 309 | max_recv = 100000 310 | 311 | 312 | # ### Open the client socket 313 | 314 | # In[19]: 315 | 316 | 317 | s = socket.socket() 318 | s.connect((host, port)) 319 | 320 | 321 | # ## SET TIMER 322 | 323 | # In[20]: 324 | 325 | 326 | start_time = time.time() # store start time 327 | print("timmer start!") 328 | 329 | 330 | # In[21]: 331 | 332 | 333 | msg = recv_msg(s) 334 | rounds = msg['rounds'] 335 | client_id = msg['client_id'] 336 | local_epochs = msg['local_epoch'] 337 | send_msg(s, len(trainset_sub)) 338 | 339 | 340 | # In[22]: 341 | 342 | 343 | # update weights from server 344 | # train 345 | for r in range(rounds): # loop over the dataset multiple times 346 | 347 | 348 | 349 | weights = recv_msg(s) 350 | mobile_net.load_state_dict(weights) 351 | mobile_net.eval() 352 | for local_epoch in range(local_epochs): 353 | 354 | for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))): 355 | 356 | # get the inputs; data is a list of [inputs, labels] 357 | inputs, labels = data 358 | inputs = inputs.to(device) 359 | labels = labels.clone().detach().long().to(device) 360 | 361 | # zero the parameter gradients 362 | optimizer.zero_grad() 363 | 364 | # forward + backward + optimize 365 | outputs = mobile_net(inputs) 366 | loss = criterion(outputs, labels) 367 | loss.backward() 368 | optimizer.step() 369 | 370 | msg = mobile_net.state_dict() 371 | send_msg(s, msg) 372 | 373 | print('Finished Training') 374 | 375 | 376 | # In[ ]: 377 | 378 | 379 | printPerformance() 380 | 381 | 382 | # In[23]: 383 | 384 | 385 | end_time = time.time() #store end time 386 | print("Training Time: {} sec".format(end_time - start_time)) 387 | 388 | 389 | # In[ ]: 390 | 391 | 392 | 393 | 394 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_Resnet20/cifar10_resnet20_fd_client_rasp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # # CIFAR10 Federated resnet20 Client Side 5 | # This code is the server part of CIFAR10 federated resnet20 for **multi** client and a server. 6 | 7 | # In[3]: 8 | 9 | 10 | users = 1 # number of clients 11 | 12 | 13 | # In[4]: 14 | 15 | 16 | import os 17 | import h5py 18 | 19 | import socket 20 | import struct 21 | import pickle 22 | 23 | import torch 24 | import torch.nn as nn 25 | import torch.nn.functional as F 26 | import torchvision 27 | import torchvision.transforms as transforms 28 | import torch.optim as optim 29 | 30 | from torch.utils.data import Dataset, DataLoader 31 | 32 | import time 33 | 34 | from tqdm import tqdm 35 | 36 | 37 | # In[2]: 38 | 39 | 40 | def getFreeDescription(): 41 | free = os.popen("free -h") 42 | i = 0 43 | while True: 44 | i = i + 1 45 | line = free.readline() 46 | if i == 1: 47 | return (line.split()[0:7]) 48 | 49 | 50 | def getFree(): 51 | free = os.popen("free -h") 52 | i = 0 53 | while True: 54 | i = i + 1 55 | line = free.readline() 56 | if i == 2: 57 | return (line.split()[0:7]) 58 | 59 | from gpiozero import CPUTemperature 60 | 61 | 62 | def printPerformance(): 63 | cpu = CPUTemperature() 64 | 65 | print("temperature: " + str(cpu.temperature)) 66 | 67 | description = getFreeDescription() 68 | mem = getFree() 69 | 70 | print(description[0] + " : " + mem[1]) 71 | print(description[1] + " : " + mem[2]) 72 | print(description[2] + " : " + mem[3]) 73 | print(description[3] + " : " + mem[4]) 74 | print(description[4] + " : " + mem[5]) 75 | print(description[5] + " : " + mem[6]) 76 | 77 | 78 | # In[3]: 79 | 80 | 81 | printPerformance() 82 | 83 | 84 | # In[5]: 85 | 86 | 87 | root_path = '../../models/cifar10_data' 88 | 89 | 90 | # ## Cuda 91 | 92 | # In[6]: 93 | 94 | 95 | # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 96 | device = "cpu" 97 | print(device) 98 | 99 | 100 | # In[7]: 101 | 102 | 103 | client_order = int(input("client_order(start from 0): ")) 104 | 105 | 106 | # In[8]: 107 | 108 | 109 | num_traindata = 50000 // users 110 | 111 | 112 | # ## Data load 113 | 114 | # In[9]: 115 | 116 | 117 | transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))]) 118 | 119 | from torch.utils.data import Subset 120 | 121 | 122 | indices = list(range(50000)) 123 | 124 | part_tr = indices[num_traindata * client_order : num_traindata * (client_order + 1)] 125 | 126 | 127 | # In[10]: 128 | 129 | 130 | trainset = torchvision.datasets.CIFAR10 (root=root_path, train=True, download=True, transform=transform) 131 | 132 | trainset_sub = Subset(trainset, part_tr) 133 | 134 | train_loader = torch.utils.data.DataLoader(trainset_sub, batch_size=4, shuffle=True, num_workers=2) 135 | 136 | testset = torchvision.datasets.CIFAR10 (root=root_path, train=False, download=True, transform=transform) 137 | test_loader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2) 138 | 139 | 140 | # In[11]: 141 | 142 | 143 | classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') 144 | 145 | 146 | # ### Number of total batches 147 | 148 | # In[13]: 149 | 150 | 151 | train_total_batch = len(train_loader) 152 | print(train_total_batch) 153 | test_batch = len(test_loader) 154 | print(test_batch) 155 | 156 | 157 | from torch.autograd import Variable 158 | import torch.nn.init as init 159 | 160 | def _weights_init(m): 161 | classname = m.__class__.__name__ 162 | #print(classname) 163 | if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d): 164 | init.kaiming_normal_(m.weight) 165 | 166 | class LambdaLayer(nn.Module): 167 | def __init__(self, lambd): 168 | super(LambdaLayer, self).__init__() 169 | self.lambd = lambd 170 | 171 | def forward(self, x): 172 | return self.lambd(x) 173 | 174 | 175 | class BasicBlock(nn.Module): 176 | expansion = 1 177 | 178 | def __init__(self, in_planes, planes, stride=1, option='A'): 179 | super(BasicBlock, self).__init__() 180 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 181 | self.bn1 = nn.BatchNorm2d(planes) 182 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 183 | self.bn2 = nn.BatchNorm2d(planes) 184 | 185 | self.shortcut = nn.Sequential() 186 | if stride != 1 or in_planes != planes: 187 | if option == 'A': 188 | """ 189 | For CIFAR10 ResNet paper uses option A. 190 | """ 191 | self.shortcut = LambdaLayer(lambda x: 192 | F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), "constant", 0)) 193 | elif option == 'B': 194 | self.shortcut = nn.Sequential( 195 | nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), 196 | nn.BatchNorm2d(self.expansion * planes) 197 | ) 198 | 199 | def forward(self, x): 200 | out = F.relu(self.bn1(self.conv1(x))) 201 | out = self.bn2(self.conv2(out)) 202 | out += self.shortcut(x) 203 | out = F.relu(out) 204 | return out 205 | 206 | 207 | class ResNet(nn.Module): 208 | def __init__(self, block, num_blocks, num_classes=10): 209 | super(ResNet, self).__init__() 210 | self.in_planes = 16 211 | 212 | self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False) 213 | self.bn1 = nn.BatchNorm2d(16) 214 | self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1) 215 | self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2) 216 | self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) 217 | self.linear = nn.Linear(64, num_classes) 218 | 219 | self.apply(_weights_init) 220 | 221 | def _make_layer(self, block, planes, num_blocks, stride): 222 | strides = [stride] + [1]*(num_blocks-1) 223 | layers = [] 224 | for stride in strides: 225 | layers.append(block(self.in_planes, planes, stride)) 226 | self.in_planes = planes * block.expansion 227 | 228 | return nn.Sequential(*layers) 229 | 230 | def forward(self, x): 231 | out = F.relu(self.bn1(self.conv1(x))) 232 | out = self.layer1(out) 233 | out = self.layer2(out) 234 | out = self.layer3(out) 235 | out = F.avg_pool2d(out, out.size()[3]) 236 | out = out.view(out.size(0), -1) 237 | out = self.linear(out) 238 | return out 239 | 240 | 241 | def resnet20(): 242 | return ResNet(BasicBlock, [3, 3, 3]) 243 | 244 | # In[15]: 245 | 246 | 247 | res_net = resnet20() 248 | res_net.to(device) 249 | 250 | 251 | # In[16]: 252 | 253 | 254 | lr = 0.001 255 | criterion = nn.CrossEntropyLoss() 256 | optimizer = optim.SGD(res_net.parameters(), lr=lr, momentum=0.9) 257 | 258 | rounds = 400 # default 259 | local_epochs = 1 # default 260 | 261 | 262 | # ## Socket initialization 263 | # ### Required socket functions 264 | 265 | # In[17]: 266 | 267 | 268 | def send_msg(sock, msg): 269 | # prefix each message with a 4-byte length in network byte order 270 | msg = pickle.dumps(msg) 271 | msg = struct.pack('>I', len(msg)) + msg 272 | sock.sendall(msg) 273 | 274 | def recv_msg(sock): 275 | # read message length and unpack it into an integer 276 | raw_msglen = recvall(sock, 4) 277 | if not raw_msglen: 278 | return None 279 | msglen = struct.unpack('>I', raw_msglen)[0] 280 | # read the message data 281 | msg = recvall(sock, msglen) 282 | msg = pickle.loads(msg) 283 | return msg 284 | 285 | def recvall(sock, n): 286 | # helper function to receive n bytes or return None if EOF is hit 287 | data = b'' 288 | while len(data) < n: 289 | packet = sock.recv(n - len(data)) 290 | if not packet: 291 | return None 292 | data += packet 293 | return data 294 | 295 | 296 | # In[15]: 297 | 298 | 299 | printPerformance() 300 | 301 | 302 | # ### Set host address and port number 303 | 304 | # In[18]: 305 | 306 | 307 | host = input("IP address: ") 308 | port = 10080 309 | max_recv = 100000 310 | 311 | 312 | # ### Open the client socket 313 | 314 | # In[19]: 315 | 316 | 317 | s = socket.socket() 318 | s.connect((host, port)) 319 | 320 | 321 | # ## SET TIMER 322 | 323 | # In[20]: 324 | 325 | 326 | start_time = time.time() # store start time 327 | print("timmer start!") 328 | 329 | 330 | # In[21]: 331 | 332 | 333 | msg = recv_msg(s) 334 | rounds = msg['rounds'] 335 | client_id = msg['client_id'] 336 | local_epochs = msg['local_epoch'] 337 | send_msg(s, len(trainset_sub)) 338 | 339 | 340 | # In[22]: 341 | 342 | 343 | # update weights from server 344 | # train 345 | for r in range(rounds): # loop over the dataset multiple times 346 | 347 | 348 | 349 | weights = recv_msg(s) 350 | res_net.load_state_dict(weights) 351 | res_net.eval() 352 | for local_epoch in range(local_epochs): 353 | 354 | for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))): 355 | 356 | # get the inputs; data is a list of [inputs, labels] 357 | inputs, labels = data 358 | inputs = inputs.to(device) 359 | labels = labels.clone().detach().long().to(device) 360 | 361 | # zero the parameter gradients 362 | optimizer.zero_grad() 363 | 364 | # forward + backward + optimize 365 | outputs = res_net(inputs) 366 | loss = criterion(outputs, labels) 367 | loss.backward() 368 | optimizer.step() 369 | 370 | msg = res_net.state_dict() 371 | send_msg(s, msg) 372 | 373 | print('Finished Training') 374 | 375 | 376 | # In[ ]: 377 | 378 | 379 | printPerformance() 380 | 381 | 382 | # In[23]: 383 | 384 | 385 | end_time = time.time() #store end time 386 | print("Training Time: {} sec".format(end_time - start_time)) 387 | 388 | 389 | # In[ ]: 390 | 391 | 392 | 393 | 394 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/ensemble_learning/ecg/ecg_es_client.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# ECG Ensemble 1D-CNN Client Side\n", 8 | "This code is the server part of ECG ensemble 1D-CNN model for **multi** client and a server." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "users = 3 # number of clients" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "## Import required packages" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "import os\n", 34 | "import struct\n", 35 | "import socket\n", 36 | "import pickle\n", 37 | "import time\n", 38 | "\n", 39 | "import h5py\n", 40 | "from tqdm import tqdm\n", 41 | "\n", 42 | "import torch\n", 43 | "import torch.nn as nn\n", 44 | "from torch.utils.data import Dataset, DataLoader\n", 45 | "from torch.optim import Adam\n", 46 | "import copy" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 3, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "root_path = '../../models/'" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "## SET CUDA" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n", 72 | "device = \"cpu\"\n", 73 | "torch.manual_seed(777)\n", 74 | "if device ==\"cuda:0\":\n", 75 | " torch.cuda.manual_seed_all(777)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 5, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "client_order(start from 0): 0\n" 88 | ] 89 | } 90 | ], 91 | "source": [ 92 | "client_order = int(input(\"client_order(start from 0): \"))" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 6, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "num_traindata = 13244 // users" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "## Define ECG dataset class" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 7, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "class ECG(Dataset):\n", 118 | " def __init__(self, train=True):\n", 119 | " if train:\n", 120 | " # total: 13244\n", 121 | " with h5py.File(os.path.join(root_path, 'ecg_data', 'train_ecg.hdf5'), 'r') as hdf:\n", 122 | " self.x = hdf['x_train'][num_traindata * client_order : num_traindata * (client_order + 1)]\n", 123 | " self.y = hdf['y_train'][num_traindata * client_order : num_traindata * (client_order + 1)]\n", 124 | "\n", 125 | " else:\n", 126 | " with h5py.File(os.path.join(root_path, 'ecg_data', 'test_ecg.hdf5'), 'r') as hdf:\n", 127 | " self.x = hdf['x_test'][:]\n", 128 | " self.y = hdf['y_test'][:]\n", 129 | " \n", 130 | " def __len__(self):\n", 131 | " return len(self.x)\n", 132 | " \n", 133 | " def __getitem__(self, idx):\n", 134 | " return torch.tensor(self.x[idx], dtype=torch.float), torch.tensor(self.y[idx])" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "### Set batch size" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 8, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "batch_size = 32" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "## Make train and test dataset batch generator" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 9, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "train_dataset = ECG(train=True)\n", 167 | "test_dataset = ECG(train=False)\n", 168 | "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", 169 | "test_loader = DataLoader(test_dataset, batch_size=batch_size)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "### Size check" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 10, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "name": "stdout", 186 | "output_type": "stream", 187 | "text": [ 188 | "torch.Size([32, 1, 130])\n", 189 | "torch.Size([32])\n" 190 | ] 191 | } 192 | ], 193 | "source": [ 194 | "x_train, y_train = next(iter(train_loader))\n", 195 | "print(x_train.size())\n", 196 | "print(y_train.size())" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "### Total number of batches" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 11, 209 | "metadata": {}, 210 | "outputs": [ 211 | { 212 | "name": "stdout", 213 | "output_type": "stream", 214 | "text": [ 215 | "138\n" 216 | ] 217 | } 218 | ], 219 | "source": [ 220 | "total_batch = len(train_loader)\n", 221 | "print(total_batch)" 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "metadata": {}, 227 | "source": [ 228 | "## Define ECG client model\n", 229 | "Client side has only **2 convolutional layers**." 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 12, 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [ 238 | "class Ecgclient(nn.Module):\n", 239 | " def __init__(self):\n", 240 | " super(Ecgclient, self).__init__()\n", 241 | " self.conv1 = nn.Conv1d(1, 16, 7, padding=3) # 128 x 16\n", 242 | " self.relu1 = nn.LeakyReLU()\n", 243 | " self.pool1 = nn.MaxPool1d(2) # 64 x 16\n", 244 | " self.conv2 = nn.Conv1d(16, 16, 5, padding=2) # 64 x 16\n", 245 | " self.relu2 = nn.LeakyReLU()\n", 246 | " \n", 247 | " def forward(self, x):\n", 248 | " x = self.conv1(x)\n", 249 | " x = self.relu1(x)\n", 250 | " x = self.pool1(x)\n", 251 | " x = self.conv2(x)\n", 252 | " x = self.relu2(x)\n", 253 | " return x " 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 13, 259 | "metadata": {}, 260 | "outputs": [ 261 | { 262 | "name": "stdout", 263 | "output_type": "stream", 264 | "text": [ 265 | "Ecgclient(\n", 266 | " (conv1): Conv1d(1, 16, kernel_size=(7,), stride=(1,), padding=(3,))\n", 267 | " (relu1): LeakyReLU(negative_slope=0.01)\n", 268 | " (pool1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", 269 | " (conv2): Conv1d(16, 16, kernel_size=(5,), stride=(1,), padding=(2,))\n", 270 | " (relu2): LeakyReLU(negative_slope=0.01)\n", 271 | ")\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "ecg_client = Ecgclient().to(device)\n", 277 | "print(ecg_client)" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": 14, 283 | "metadata": {}, 284 | "outputs": [], 285 | "source": [ 286 | "# from torchsummary import summary\n", 287 | "\n", 288 | "# print('ECG 1D CNN clients')\n", 289 | "# summary(ecg_client, (1, 130))" 290 | ] 291 | }, 292 | { 293 | "cell_type": "markdown", 294 | "metadata": {}, 295 | "source": [ 296 | "### Set other hyperparameters in the model\n", 297 | "Hyperparameters here should be same with the server side." 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 15, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "epoch = 20 # default\n", 307 | "criterion = nn.CrossEntropyLoss()\n", 308 | "lr = 0.001\n", 309 | "optimizer = Adam(ecg_client.parameters(), lr=lr)" 310 | ] 311 | }, 312 | { 313 | "cell_type": "markdown", 314 | "metadata": {}, 315 | "source": [ 316 | "## Socket initialization" 317 | ] 318 | }, 319 | { 320 | "cell_type": "markdown", 321 | "metadata": {}, 322 | "source": [ 323 | "### Required socket functions" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 16, 329 | "metadata": {}, 330 | "outputs": [], 331 | "source": [ 332 | "def send_msg(sock, msg):\n", 333 | " # prefix each message with a 4-byte length in network byte order\n", 334 | " msg = pickle.dumps(msg)\n", 335 | " msg = struct.pack('>I', len(msg)) + msg\n", 336 | " sock.sendall(msg)\n", 337 | "\n", 338 | "def recv_msg(sock):\n", 339 | " # read message length and unpack it into an integer\n", 340 | " raw_msglen = recvall(sock, 4)\n", 341 | " if not raw_msglen:\n", 342 | " return None\n", 343 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 344 | " # read the message data\n", 345 | " msg = recvall(sock, msglen)\n", 346 | " msg = pickle.loads(msg)\n", 347 | " return msg\n", 348 | "\n", 349 | "def recvall(sock, n):\n", 350 | " # helper function to receive n bytes or return None if EOF is hit\n", 351 | " data = b''\n", 352 | " while len(data) < n:\n", 353 | " packet = sock.recv(n - len(data))\n", 354 | " if not packet:\n", 355 | " return None\n", 356 | " data += packet\n", 357 | " return data" 358 | ] 359 | }, 360 | { 361 | "cell_type": "markdown", 362 | "metadata": {}, 363 | "source": [ 364 | "### Set host address and port number" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": 17, 370 | "metadata": {}, 371 | "outputs": [ 372 | { 373 | "name": "stdout", 374 | "output_type": "stream", 375 | "text": [ 376 | "IP address: 192.168.83.1\n" 377 | ] 378 | } 379 | ], 380 | "source": [ 381 | "host = input(\"IP address: \")\n", 382 | "port = 10080" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "## SET TIMER" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": 18, 395 | "metadata": {}, 396 | "outputs": [ 397 | { 398 | "name": "stdout", 399 | "output_type": "stream", 400 | "text": [ 401 | "timmer start!\n" 402 | ] 403 | } 404 | ], 405 | "source": [ 406 | "start_time = time.time() # store start time\n", 407 | "print(\"timmer start!\")" 408 | ] 409 | }, 410 | { 411 | "cell_type": "markdown", 412 | "metadata": {}, 413 | "source": [ 414 | "### Open the client socket" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": 19, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "s = socket.socket()\n", 424 | "s.connect((host, port))" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": 20, 430 | "metadata": {}, 431 | "outputs": [], 432 | "source": [ 433 | "msg = recv_msg(s) # get epoch\n", 434 | "epochs = msg['epochs']\n", 435 | "users = msg['users']\n", 436 | "msg = total_batch\n", 437 | "send_msg(s, msg) # send total_batch of train dataset" 438 | ] 439 | }, 440 | { 441 | "cell_type": "markdown", 442 | "metadata": {}, 443 | "source": [ 444 | "## Real training process" 445 | ] 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": 21, 450 | "metadata": {}, 451 | "outputs": [ 452 | { 453 | "name": "stderr", 454 | "output_type": "stream", 455 | "text": [ 456 | "Epoch 1_0: 0%| | 0/138 [00:00I', len(msg)) + msg\n", 389 | " sock.sendall(msg)\n", 390 | "\n", 391 | "def recv_msg(sock):\n", 392 | " # read message length and unpack it into an integer\n", 393 | " raw_msglen = recvall(sock, 4)\n", 394 | " if not raw_msglen:\n", 395 | " return None\n", 396 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 397 | " # read the message data\n", 398 | " msg = recvall(sock, msglen)\n", 399 | " msg = pickle.loads(msg)\n", 400 | " return msg\n", 401 | "\n", 402 | "def recvall(sock, n):\n", 403 | " # helper function to receive n bytes or return None if EOF is hit\n", 404 | " data = b''\n", 405 | " while len(data) < n:\n", 406 | " packet = sock.recv(n - len(data))\n", 407 | " if not packet:\n", 408 | " return None\n", 409 | " data += packet\n", 410 | " return data" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "metadata": {}, 417 | "outputs": [], 418 | "source": [ 419 | "printPerformance()" 420 | ] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": {}, 425 | "source": [ 426 | "### Set host address and port number" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": 17, 432 | "metadata": {}, 433 | "outputs": [ 434 | { 435 | "name": "stdout", 436 | "output_type": "stream", 437 | "text": [ 438 | "IP address: 192.168.83.1\n" 439 | ] 440 | } 441 | ], 442 | "source": [ 443 | "host = input(\"IP address: \")\n", 444 | "port = 10080" 445 | ] 446 | }, 447 | { 448 | "cell_type": "markdown", 449 | "metadata": {}, 450 | "source": [ 451 | "## SET TIMER" 452 | ] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "execution_count": 18, 457 | "metadata": {}, 458 | "outputs": [ 459 | { 460 | "name": "stdout", 461 | "output_type": "stream", 462 | "text": [ 463 | "timmer start!\n" 464 | ] 465 | } 466 | ], 467 | "source": [ 468 | "start_time = time.time() # store start time\n", 469 | "print(\"timmer start!\")" 470 | ] 471 | }, 472 | { 473 | "cell_type": "markdown", 474 | "metadata": {}, 475 | "source": [ 476 | "### Open the client socket" 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": 19, 482 | "metadata": {}, 483 | "outputs": [], 484 | "source": [ 485 | "s = socket.socket()\n", 486 | "s.connect((host, port))" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 20, 492 | "metadata": {}, 493 | "outputs": [], 494 | "source": [ 495 | "msg = recv_msg(s) # get epoch\n", 496 | "epochs = msg['epochs']\n", 497 | "users = msg['users']\n", 498 | "msg = total_batch\n", 499 | "send_msg(s, msg) # send total_batch of train dataset" 500 | ] 501 | }, 502 | { 503 | "cell_type": "markdown", 504 | "metadata": {}, 505 | "source": [ 506 | "## Real training process" 507 | ] 508 | }, 509 | { 510 | "cell_type": "code", 511 | "execution_count": 21, 512 | "metadata": {}, 513 | "outputs": [ 514 | { 515 | "name": "stderr", 516 | "output_type": "stream", 517 | "text": [ 518 | "Epoch 1_0: 0%| | 0/138 [00:00I', len(msg)) + msg\n", 391 | " sock.sendall(msg)\n", 392 | "\n", 393 | "def recv_msg(sock):\n", 394 | " # read message length and unpack it into an integer\n", 395 | " raw_msglen = recvall(sock, 4)\n", 396 | " if not raw_msglen:\n", 397 | " return None\n", 398 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 399 | " # read the message data\n", 400 | " msg = recvall(sock, msglen)\n", 401 | " msg = pickle.loads(msg)\n", 402 | " return msg\n", 403 | "\n", 404 | "def recvall(sock, n):\n", 405 | " # helper function to receive n bytes or return None if EOF is hit\n", 406 | " data = b''\n", 407 | " while len(data) < n:\n", 408 | " packet = sock.recv(n - len(data))\n", 409 | " if not packet:\n", 410 | " return None\n", 411 | " data += packet\n", 412 | " return data" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": null, 418 | "metadata": {}, 419 | "outputs": [], 420 | "source": [ 421 | "printPerformance()" 422 | ] 423 | }, 424 | { 425 | "cell_type": "markdown", 426 | "metadata": {}, 427 | "source": [ 428 | "### Set host address and port number" 429 | ] 430 | }, 431 | { 432 | "cell_type": "code", 433 | "execution_count": 12, 434 | "metadata": {}, 435 | "outputs": [ 436 | { 437 | "name": "stdout", 438 | "output_type": "stream", 439 | "text": [ 440 | "IP address: localhost\n" 441 | ] 442 | } 443 | ], 444 | "source": [ 445 | "host = input(\"IP address: \")\n", 446 | "port = 10080\n", 447 | "max_recv = 100000" 448 | ] 449 | }, 450 | { 451 | "cell_type": "markdown", 452 | "metadata": {}, 453 | "source": [ 454 | "### Open the client socket" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": 13, 460 | "metadata": {}, 461 | "outputs": [], 462 | "source": [ 463 | "s = socket.socket()\n", 464 | "s.connect((host, port))" 465 | ] 466 | }, 467 | { 468 | "cell_type": "markdown", 469 | "metadata": {}, 470 | "source": [ 471 | "## SET TIMER" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 14, 477 | "metadata": {}, 478 | "outputs": [ 479 | { 480 | "name": "stdout", 481 | "output_type": "stream", 482 | "text": [ 483 | "timmer start!\n" 484 | ] 485 | } 486 | ], 487 | "source": [ 488 | "start_time = time.time() # store start time\n", 489 | "print(\"timmer start!\")" 490 | ] 491 | }, 492 | { 493 | "cell_type": "code", 494 | "execution_count": 15, 495 | "metadata": {}, 496 | "outputs": [], 497 | "source": [ 498 | "msg = recv_msg(s)\n", 499 | "rounds = msg['rounds'] \n", 500 | "client_id = msg['client_id']\n", 501 | "local_epochs = msg['local_epoch']\n", 502 | "send_msg(s, len(train_dataset))" 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": 16, 508 | "metadata": {}, 509 | "outputs": [ 510 | { 511 | "name": "stderr", 512 | "output_type": "stream", 513 | "text": [ 514 | "Round 1_1: 100%|██████████████████████████████████████████████████| 207/207 [00:04<00:00, 43.61it/s]\n", 515 | "Round 2_1: 100%|██████████████████████████████████████████████████| 207/207 [00:03<00:00, 55.96it/s]\n" 516 | ] 517 | }, 518 | { 519 | "name": "stdout", 520 | "output_type": "stream", 521 | "text": [ 522 | "Finished Training\n", 523 | "Training Time: 12.980786323547363 sec\n" 524 | ] 525 | } 526 | ], 527 | "source": [ 528 | "# update weights from server\n", 529 | "# train\n", 530 | "for r in range(rounds): # loop over the dataset multiple times\n", 531 | " weights = recv_msg(s)\n", 532 | " ecg_net.load_state_dict(weights)\n", 533 | " ecg_net.eval()\n", 534 | " for local_epoch in range(local_epochs):\n", 535 | " \n", 536 | " for i, data in enumerate(tqdm(trainloader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))):\n", 537 | " \n", 538 | " # get the inputs; data is a list of [inputs, labels]\n", 539 | " inputs, labels = data\n", 540 | " inputs = inputs.to(device)\n", 541 | " labels = labels.clone().detach().long().to(device)\n", 542 | "\n", 543 | " # zero the parameter gradients\n", 544 | " optimizer.zero_grad()\n", 545 | "\n", 546 | " # forward + backward + optimize\n", 547 | " outputs = ecg_net(inputs)\n", 548 | " loss = criterion(outputs, labels)\n", 549 | " loss.backward()\n", 550 | " optimizer.step()\n", 551 | "\n", 552 | " msg = ecg_net.state_dict()\n", 553 | " send_msg(s, msg)\n", 554 | "\n", 555 | "print('Finished Training')\n", 556 | "\n" 557 | ] 558 | }, 559 | { 560 | "cell_type": "code", 561 | "execution_count": null, 562 | "metadata": {}, 563 | "outputs": [], 564 | "source": [ 565 | "printPerformance()" 566 | ] 567 | }, 568 | { 569 | "cell_type": "code", 570 | "execution_count": null, 571 | "metadata": {}, 572 | "outputs": [], 573 | "source": [ 574 | "end_time = time.time() #store end time\n", 575 | "print(\"Training Time: {} sec\".format(end_time - start_time))" 576 | ] 577 | } 578 | ], 579 | "metadata": { 580 | "kernelspec": { 581 | "display_name": "Python 3", 582 | "language": "python", 583 | "name": "python3" 584 | }, 585 | "language_info": { 586 | "codemirror_mode": { 587 | "name": "ipython", 588 | "version": 3 589 | }, 590 | "file_extension": ".py", 591 | "mimetype": "text/x-python", 592 | "name": "python", 593 | "nbconvert_exporter": "python", 594 | "pygments_lexer": "ipython3", 595 | "version": "3.6.10" 596 | } 597 | }, 598 | "nbformat": 4, 599 | "nbformat_minor": 2 600 | } 601 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_MobileNet/cifar10_mobile_fd_client.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# CIFAR10 Federated Mobilenet Client Side\n", 8 | "This code is the server part of CIFAR10 federated mobilenet for **multi** client and a server." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 3, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "users = 1 # number of clients" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 4, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "import os\n", 27 | "import h5py\n", 28 | "\n", 29 | "import socket\n", 30 | "import struct\n", 31 | "import pickle\n", 32 | "\n", 33 | "import torch\n", 34 | "import torch.nn as nn\n", 35 | "import torch.nn.functional as F\n", 36 | "import torchvision\n", 37 | "import torchvision.transforms as transforms\n", 38 | "import torch.optim as optim\n", 39 | "\n", 40 | "from torch.utils.data import Dataset, DataLoader\n", 41 | "\n", 42 | "import time\n", 43 | "\n", 44 | "from tqdm import tqdm" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 5, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "root_path = '../../models/cifar10_data'" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "## Cuda" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 6, 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "name": "stdout", 70 | "output_type": "stream", 71 | "text": [ 72 | "cpu\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 78 | "device = \"cpu\"\n", 79 | "print(device)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 7, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "name": "stdout", 89 | "output_type": "stream", 90 | "text": [ 91 | "client_order(start from 0): 0\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "client_order = int(input(\"client_order(start from 0): \"))" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 8, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "num_traindata = 50000 // users" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "## Data load" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 9, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n", 122 | "\n", 123 | "from torch.utils.data import Subset\n", 124 | "\n", 125 | "\n", 126 | "indices = list(range(50000))\n", 127 | "\n", 128 | "part_tr = indices[num_traindata * client_order : num_traindata * (client_order + 1)]\n" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 10, 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../models/cifar10_data\\cifar-10-python.tar.gz\n" 141 | ] 142 | }, 143 | { 144 | "data": { 145 | "application/vnd.jupyter.widget-view+json": { 146 | "model_id": "62295362754042e980b2371d8d0c3390", 147 | "version_major": 2, 148 | "version_minor": 0 149 | }, 150 | "text/plain": [ 151 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" 152 | ] 153 | }, 154 | "metadata": {}, 155 | "output_type": "display_data" 156 | }, 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "Extracting ../../models/cifar10_data\\cifar-10-python.tar.gz to ../../models/cifar10_data\n", 162 | "Files already downloaded and verified\n" 163 | ] 164 | } 165 | ], 166 | "source": [ 167 | "trainset = torchvision.datasets.CIFAR10 (root=root_path, train=True, download=True, transform=transform)\n", 168 | "\n", 169 | "trainset_sub = Subset(trainset, part_tr)\n", 170 | "\n", 171 | "train_loader = torch.utils.data.DataLoader(trainset_sub, batch_size=4, shuffle=True, num_workers=2)\n", 172 | "\n", 173 | "testset = torchvision.datasets.CIFAR10 (root=root_path, train=False, download=True, transform=transform)\n", 174 | "test_loader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 11, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "### Number of total batches" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 13, 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "name": "stdout", 200 | "output_type": "stream", 201 | "text": [ 202 | "25\n", 203 | "2500\n" 204 | ] 205 | } 206 | ], 207 | "source": [ 208 | "train_total_batch = len(train_loader)\n", 209 | "print(train_total_batch)\n", 210 | "test_batch = len(test_loader)\n", 211 | "print(test_batch)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 14, 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "# -*- coding: utf-8 -*-\n", 221 | "\"\"\"\n", 222 | "Created on Thu Nov 1 14:23:31 2018\n", 223 | "@author: tshzzz\n", 224 | "\"\"\"\n", 225 | "\n", 226 | "import torch\n", 227 | "import torch.nn as nn\n", 228 | "\n", 229 | "\n", 230 | "def conv_dw(inplane,outplane,stride=1):\n", 231 | " return nn.Sequential(\n", 232 | " nn.Conv2d(inplane,inplane,kernel_size = 3,groups = inplane,stride=stride,padding=1),\n", 233 | " nn.BatchNorm2d(inplane),\n", 234 | " nn.ReLU(),\n", 235 | " nn.Conv2d(inplane,outplane,kernel_size = 1,groups = 1,stride=1),\n", 236 | " nn.BatchNorm2d(outplane),\n", 237 | " nn.ReLU() \n", 238 | " )\n", 239 | "\n", 240 | "def conv_bw(inplane,outplane,kernel_size = 3,stride=1):\n", 241 | " return nn.Sequential(\n", 242 | " nn.Conv2d(inplane,outplane,kernel_size = kernel_size,groups = 1,stride=stride,padding=1),\n", 243 | " nn.BatchNorm2d(outplane),\n", 244 | " nn.ReLU() \n", 245 | " )\n", 246 | "\n", 247 | "\n", 248 | "class MobileNet(nn.Module):\n", 249 | " \n", 250 | " def __init__(self,num_class=10):\n", 251 | " super(MobileNet,self).__init__()\n", 252 | " \n", 253 | " layers = []\n", 254 | " layers.append(conv_bw(3,32,3,1))\n", 255 | " layers.append(conv_dw(32,64,1))\n", 256 | " layers.append(conv_dw(64,128,2))\n", 257 | " layers.append(conv_dw(128,128,1))\n", 258 | " layers.append(conv_dw(128,256,2))\n", 259 | " layers.append(conv_dw(256,256,1))\n", 260 | " layers.append(conv_dw(256,512,2))\n", 261 | "\n", 262 | " for i in range(5):\n", 263 | " layers.append(conv_dw(512,512,1))\n", 264 | " layers.append(conv_dw(512,1024,2))\n", 265 | " layers.append(conv_dw(1024,1024,1))\n", 266 | "\n", 267 | " self.classifer = nn.Sequential(\n", 268 | " nn.Dropout(0.5),\n", 269 | " nn.Linear(1024,num_class)\n", 270 | " )\n", 271 | " self.feature = nn.Sequential(*layers)\n", 272 | " \n", 273 | " \n", 274 | "\n", 275 | " def forward(self,x):\n", 276 | " out = self.feature(x)\n", 277 | " out = out.mean(3).mean(2)\n", 278 | " out = out.view(-1,1024)\n", 279 | " out = self.classifer(out)\n", 280 | " return out\n", 281 | "\n" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": 15, 287 | "metadata": {}, 288 | "outputs": [ 289 | { 290 | "data": { 291 | "text/plain": [ 292 | "MobileNet(\n", 293 | " (classifer): Sequential(\n", 294 | " (0): Dropout(p=0.5, inplace=False)\n", 295 | " (1): Linear(in_features=1024, out_features=10, bias=True)\n", 296 | " )\n", 297 | " (feature): Sequential(\n", 298 | " (0): Sequential(\n", 299 | " (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 300 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 301 | " (2): ReLU()\n", 302 | " )\n", 303 | " (1): Sequential(\n", 304 | " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n", 305 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 306 | " (2): ReLU()\n", 307 | " (3): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n", 308 | " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 309 | " (5): ReLU()\n", 310 | " )\n", 311 | " (2): Sequential(\n", 312 | " (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=64)\n", 313 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 314 | " (2): ReLU()\n", 315 | " (3): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n", 316 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 317 | " (5): ReLU()\n", 318 | " )\n", 319 | " (3): Sequential(\n", 320 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n", 321 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 322 | " (2): ReLU()\n", 323 | " (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n", 324 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 325 | " (5): ReLU()\n", 326 | " )\n", 327 | " (4): Sequential(\n", 328 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=128)\n", 329 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 330 | " (2): ReLU()\n", 331 | " (3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1))\n", 332 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 333 | " (5): ReLU()\n", 334 | " )\n", 335 | " (5): Sequential(\n", 336 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n", 337 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 338 | " (2): ReLU()\n", 339 | " (3): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", 340 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 341 | " (5): ReLU()\n", 342 | " )\n", 343 | " (6): Sequential(\n", 344 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=256)\n", 345 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 346 | " (2): ReLU()\n", 347 | " (3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1))\n", 348 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 349 | " (5): ReLU()\n", 350 | " )\n", 351 | " (7): Sequential(\n", 352 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 353 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 354 | " (2): ReLU()\n", 355 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 356 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 357 | " (5): ReLU()\n", 358 | " )\n", 359 | " (8): Sequential(\n", 360 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 361 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 362 | " (2): ReLU()\n", 363 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 364 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 365 | " (5): ReLU()\n", 366 | " )\n", 367 | " (9): Sequential(\n", 368 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 369 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 370 | " (2): ReLU()\n", 371 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 372 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 373 | " (5): ReLU()\n", 374 | " )\n", 375 | " (10): Sequential(\n", 376 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 377 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 378 | " (2): ReLU()\n", 379 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 380 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 381 | " (5): ReLU()\n", 382 | " )\n", 383 | " (11): Sequential(\n", 384 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 385 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 386 | " (2): ReLU()\n", 387 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 388 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 389 | " (5): ReLU()\n", 390 | " )\n", 391 | " (12): Sequential(\n", 392 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=512)\n", 393 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 394 | " (2): ReLU()\n", 395 | " (3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 396 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 397 | " (5): ReLU()\n", 398 | " )\n", 399 | " (13): Sequential(\n", 400 | " (0): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1024)\n", 401 | " (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 402 | " (2): ReLU()\n", 403 | " (3): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 404 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 405 | " (5): ReLU()\n", 406 | " )\n", 407 | " )\n", 408 | ")" 409 | ] 410 | }, 411 | "execution_count": 15, 412 | "metadata": {}, 413 | "output_type": "execute_result" 414 | } 415 | ], 416 | "source": [ 417 | "mobile_net = MobileNet()\n", 418 | "mobile_net.to(device)" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": 16, 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "lr = 0.001\n", 428 | "criterion = nn.CrossEntropyLoss()\n", 429 | "optimizer = optim.SGD(mobile_net.parameters(), lr=lr, momentum=0.9)\n", 430 | "\n", 431 | "rounds = 400 # default\n", 432 | "local_epochs = 1 # default" 433 | ] 434 | }, 435 | { 436 | "cell_type": "markdown", 437 | "metadata": {}, 438 | "source": [ 439 | "## Socket initialization\n", 440 | "### Required socket functions" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": 17, 446 | "metadata": {}, 447 | "outputs": [], 448 | "source": [ 449 | "def send_msg(sock, msg):\n", 450 | " # prefix each message with a 4-byte length in network byte order\n", 451 | " msg = pickle.dumps(msg)\n", 452 | " msg = struct.pack('>I', len(msg)) + msg\n", 453 | " sock.sendall(msg)\n", 454 | "\n", 455 | "def recv_msg(sock):\n", 456 | " # read message length and unpack it into an integer\n", 457 | " raw_msglen = recvall(sock, 4)\n", 458 | " if not raw_msglen:\n", 459 | " return None\n", 460 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 461 | " # read the message data\n", 462 | " msg = recvall(sock, msglen)\n", 463 | " msg = pickle.loads(msg)\n", 464 | " return msg\n", 465 | "\n", 466 | "def recvall(sock, n):\n", 467 | " # helper function to receive n bytes or return None if EOF is hit\n", 468 | " data = b''\n", 469 | " while len(data) < n:\n", 470 | " packet = sock.recv(n - len(data))\n", 471 | " if not packet:\n", 472 | " return None\n", 473 | " data += packet\n", 474 | " return data" 475 | ] 476 | }, 477 | { 478 | "cell_type": "markdown", 479 | "metadata": {}, 480 | "source": [ 481 | "### Set host address and port number" 482 | ] 483 | }, 484 | { 485 | "cell_type": "code", 486 | "execution_count": 18, 487 | "metadata": {}, 488 | "outputs": [ 489 | { 490 | "name": "stdout", 491 | "output_type": "stream", 492 | "text": [ 493 | "IP address: 192.168.83.1\n" 494 | ] 495 | } 496 | ], 497 | "source": [ 498 | "host = input(\"IP address: \")\n", 499 | "port = 10080\n", 500 | "max_recv = 100000" 501 | ] 502 | }, 503 | { 504 | "cell_type": "markdown", 505 | "metadata": {}, 506 | "source": [ 507 | "### Open the client socket" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 19, 513 | "metadata": {}, 514 | "outputs": [], 515 | "source": [ 516 | "s = socket.socket()\n", 517 | "s.connect((host, port))" 518 | ] 519 | }, 520 | { 521 | "cell_type": "markdown", 522 | "metadata": {}, 523 | "source": [ 524 | "## SET TIMER" 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "execution_count": 20, 530 | "metadata": {}, 531 | "outputs": [ 532 | { 533 | "name": "stdout", 534 | "output_type": "stream", 535 | "text": [ 536 | "timmer start!\n" 537 | ] 538 | } 539 | ], 540 | "source": [ 541 | "start_time = time.time() # store start time\n", 542 | "print(\"timmer start!\")" 543 | ] 544 | }, 545 | { 546 | "cell_type": "code", 547 | "execution_count": 21, 548 | "metadata": {}, 549 | "outputs": [], 550 | "source": [ 551 | "msg = recv_msg(s)\n", 552 | "rounds = msg['rounds'] \n", 553 | "client_id = msg['client_id']\n", 554 | "local_epochs = msg['local_epoch']\n", 555 | "send_msg(s, len(trainset_sub))" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 22, 561 | "metadata": {}, 562 | "outputs": [ 563 | { 564 | "name": "stderr", 565 | "output_type": "stream", 566 | "text": [ 567 | "Round 1_1: 100%|████████████████████████████████████████████████████| 25/25 [00:11<00:00, 2.13it/s]" 568 | ] 569 | }, 570 | { 571 | "name": "stdout", 572 | "output_type": "stream", 573 | "text": [ 574 | "Finished Training\n" 575 | ] 576 | }, 577 | { 578 | "name": "stderr", 579 | "output_type": "stream", 580 | "text": [ 581 | "\n", 582 | "C:\\Users\\rlaal\\anaconda3\\envs\\py36\\lib\\site-packages\\torch\\storage.py:34: FutureWarning: pickle support for Storage will be removed in 1.5. Use `torch.save` instead\n", 583 | " warnings.warn(\"pickle support for Storage will be removed in 1.5. Use `torch.save` instead\", FutureWarning)\n" 584 | ] 585 | } 586 | ], 587 | "source": [ 588 | "# update weights from server\n", 589 | "# train\n", 590 | "for r in range(rounds): # loop over the dataset multiple times\n", 591 | "\n", 592 | " \n", 593 | " \n", 594 | " weights = recv_msg(s)\n", 595 | " mobile_net.load_state_dict(weights)\n", 596 | " mobile_net.eval()\n", 597 | " for local_epoch in range(local_epochs):\n", 598 | " \n", 599 | " for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))):\n", 600 | " \n", 601 | " # get the inputs; data is a list of [inputs, labels]\n", 602 | " inputs, labels = data\n", 603 | " inputs = inputs.to(device)\n", 604 | " labels = labels.clone().detach().long().to(device)\n", 605 | "\n", 606 | " # zero the parameter gradients\n", 607 | " optimizer.zero_grad()\n", 608 | "\n", 609 | " # forward + backward + optimize\n", 610 | " outputs = mobile_net(inputs)\n", 611 | " loss = criterion(outputs, labels)\n", 612 | " loss.backward()\n", 613 | " optimizer.step()\n", 614 | "\n", 615 | " msg = mobile_net.state_dict()\n", 616 | " send_msg(s, msg)\n", 617 | "\n", 618 | "print('Finished Training')\n", 619 | "\n" 620 | ] 621 | }, 622 | { 623 | "cell_type": "code", 624 | "execution_count": 23, 625 | "metadata": {}, 626 | "outputs": [ 627 | { 628 | "name": "stdout", 629 | "output_type": "stream", 630 | "text": [ 631 | "Training Time: 12.0054190158844 sec\n" 632 | ] 633 | } 634 | ], 635 | "source": [ 636 | "end_time = time.time() #store end time\n", 637 | "print(\"Training Time: {} sec\".format(end_time - start_time))" 638 | ] 639 | }, 640 | { 641 | "cell_type": "code", 642 | "execution_count": null, 643 | "metadata": {}, 644 | "outputs": [], 645 | "source": [] 646 | } 647 | ], 648 | "metadata": { 649 | "kernelspec": { 650 | "display_name": "Python 3", 651 | "language": "python", 652 | "name": "python3" 653 | }, 654 | "language_info": { 655 | "codemirror_mode": { 656 | "name": "ipython", 657 | "version": 3 658 | }, 659 | "file_extension": ".py", 660 | "mimetype": "text/x-python", 661 | "name": "python", 662 | "nbconvert_exporter": "python", 663 | "pygments_lexer": "ipython3", 664 | "version": "3.6.10" 665 | } 666 | }, 667 | "nbformat": 4, 668 | "nbformat_minor": 2 669 | } 670 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_Resnet20/cifar10_resnet20_fd_client.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# CIFAR10 Federated Resnet20 Client Side\n", 8 | "This code is the server part of CIFAR10 federated resnet20 for **multi** client and a server." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 3, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "users = 1 # number of clients" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 4, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "import os\n", 27 | "import h5py\n", 28 | "\n", 29 | "import socket\n", 30 | "import struct\n", 31 | "import pickle\n", 32 | "\n", 33 | "import torch\n", 34 | "import torch.nn as nn\n", 35 | "import torch.nn.functional as F\n", 36 | "import torchvision\n", 37 | "import torchvision.transforms as transforms\n", 38 | "import torch.optim as optim\n", 39 | "\n", 40 | "from torch.utils.data import Dataset, DataLoader\n", 41 | "\n", 42 | "import time\n", 43 | "\n", 44 | "from tqdm import tqdm" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 5, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "root_path = '../../models/cifar10_data'" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "## Cuda" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 6, 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "name": "stdout", 70 | "output_type": "stream", 71 | "text": [ 72 | "cpu\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 78 | "device = \"cpu\"\n", 79 | "print(device)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 7, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "name": "stdout", 89 | "output_type": "stream", 90 | "text": [ 91 | "client_order(start from 0): 0\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "client_order = int(input(\"client_order(start from 0): \"))" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 8, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "num_traindata = 50000 // users" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "## Data load" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 9, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n", 122 | "\n", 123 | "from torch.utils.data import Subset\n", 124 | "\n", 125 | "\n", 126 | "indices = list(range(50000))\n", 127 | "\n", 128 | "part_tr = indices[num_traindata * client_order : num_traindata * (client_order + 1)]\n" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 10, 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../models/cifar10_data\\cifar-10-python.tar.gz\n" 141 | ] 142 | }, 143 | { 144 | "data": { 145 | "application/vnd.jupyter.widget-view+json": { 146 | "model_id": "62295362754042e980b2371d8d0c3390", 147 | "version_major": 2, 148 | "version_minor": 0 149 | }, 150 | "text/plain": [ 151 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" 152 | ] 153 | }, 154 | "metadata": {}, 155 | "output_type": "display_data" 156 | }, 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "Extracting ../../models/cifar10_data\\cifar-10-python.tar.gz to ../../models/cifar10_data\n", 162 | "Files already downloaded and verified\n" 163 | ] 164 | } 165 | ], 166 | "source": [ 167 | "trainset = torchvision.datasets.CIFAR10 (root=root_path, train=True, download=True, transform=transform)\n", 168 | "\n", 169 | "trainset_sub = Subset(trainset, part_tr)\n", 170 | "\n", 171 | "train_loader = torch.utils.data.DataLoader(trainset_sub, batch_size=4, shuffle=True, num_workers=2)\n", 172 | "\n", 173 | "testset = torchvision.datasets.CIFAR10 (root=root_path, train=False, download=True, transform=transform)\n", 174 | "test_loader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 11, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "### Number of total batches" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 13, 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "name": "stdout", 200 | "output_type": "stream", 201 | "text": [ 202 | "25\n", 203 | "2500\n" 204 | ] 205 | } 206 | ], 207 | "source": [ 208 | "train_total_batch = len(train_loader)\n", 209 | "print(train_total_batch)\n", 210 | "test_batch = len(test_loader)\n", 211 | "print(test_batch)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 14, 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "from torch.autograd import Variable\n", 221 | "import torch.nn.init as init\n", 222 | "\n", 223 | "def _weights_init(m):\n", 224 | " classname = m.__class__.__name__\n", 225 | " #print(classname)\n", 226 | " if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):\n", 227 | " init.kaiming_normal_(m.weight)\n", 228 | "\n", 229 | "class LambdaLayer(nn.Module):\n", 230 | " def __init__(self, lambd):\n", 231 | " super(LambdaLayer, self).__init__()\n", 232 | " self.lambd = lambd\n", 233 | "\n", 234 | " def forward(self, x):\n", 235 | " return self.lambd(x)\n", 236 | "\n", 237 | "\n", 238 | "class BasicBlock(nn.Module):\n", 239 | " expansion = 1\n", 240 | "\n", 241 | " def __init__(self, in_planes, planes, stride=1, option='A'):\n", 242 | " super(BasicBlock, self).__init__()\n", 243 | " self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n", 244 | " self.bn1 = nn.BatchNorm2d(planes)\n", 245 | " self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n", 246 | " self.bn2 = nn.BatchNorm2d(planes)\n", 247 | "\n", 248 | " self.shortcut = nn.Sequential()\n", 249 | " if stride != 1 or in_planes != planes:\n", 250 | " if option == 'A':\n", 251 | " \"\"\"\n", 252 | " For CIFAR10 ResNet paper uses option A.\n", 253 | " \"\"\"\n", 254 | " self.shortcut = LambdaLayer(lambda x:\n", 255 | " F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), \"constant\", 0))\n", 256 | " elif option == 'B':\n", 257 | " self.shortcut = nn.Sequential(\n", 258 | " nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),\n", 259 | " nn.BatchNorm2d(self.expansion * planes)\n", 260 | " )\n", 261 | "\n", 262 | " def forward(self, x):\n", 263 | " out = F.relu(self.bn1(self.conv1(x)))\n", 264 | " out = self.bn2(self.conv2(out))\n", 265 | " out += self.shortcut(x)\n", 266 | " out = F.relu(out)\n", 267 | " return out\n", 268 | "\n", 269 | "\n", 270 | "class ResNet(nn.Module):\n", 271 | " def __init__(self, block, num_blocks, num_classes=10):\n", 272 | " super(ResNet, self).__init__()\n", 273 | " self.in_planes = 16\n", 274 | "\n", 275 | " self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n", 276 | " self.bn1 = nn.BatchNorm2d(16)\n", 277 | " self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)\n", 278 | " self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)\n", 279 | " self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)\n", 280 | " self.linear = nn.Linear(64, num_classes)\n", 281 | "\n", 282 | " self.apply(_weights_init)\n", 283 | "\n", 284 | " def _make_layer(self, block, planes, num_blocks, stride):\n", 285 | " strides = [stride] + [1]*(num_blocks-1)\n", 286 | " layers = []\n", 287 | " for stride in strides:\n", 288 | " layers.append(block(self.in_planes, planes, stride))\n", 289 | " self.in_planes = planes * block.expansion\n", 290 | "\n", 291 | " return nn.Sequential(*layers)\n", 292 | "\n", 293 | " def forward(self, x):\n", 294 | " out = F.relu(self.bn1(self.conv1(x)))\n", 295 | " out = self.layer1(out)\n", 296 | " out = self.layer2(out)\n", 297 | " out = self.layer3(out)\n", 298 | " out = F.avg_pool2d(out, out.size()[3])\n", 299 | " out = out.view(out.size(0), -1)\n", 300 | " out = self.linear(out)\n", 301 | " return out\n", 302 | "\n", 303 | "\n", 304 | "def resnet20():\n", 305 | " return ResNet(BasicBlock, [3, 3, 3])\n" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 15, 311 | "metadata": {}, 312 | "outputs": [ 313 | { 314 | "data": { 315 | "text/plain": [ 316 | "MobileNet(\n", 317 | " (classifer): Sequential(\n", 318 | " (0): Dropout(p=0.5, inplace=False)\n", 319 | " (1): Linear(in_features=1024, out_features=10, bias=True)\n", 320 | " )\n", 321 | " (feature): Sequential(\n", 322 | " (0): Sequential(\n", 323 | " (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 324 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 325 | " (2): ReLU()\n", 326 | " )\n", 327 | " (1): Sequential(\n", 328 | " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n", 329 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 330 | " (2): ReLU()\n", 331 | " (3): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n", 332 | " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 333 | " (5): ReLU()\n", 334 | " )\n", 335 | " (2): Sequential(\n", 336 | " (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=64)\n", 337 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 338 | " (2): ReLU()\n", 339 | " (3): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n", 340 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 341 | " (5): ReLU()\n", 342 | " )\n", 343 | " (3): Sequential(\n", 344 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n", 345 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 346 | " (2): ReLU()\n", 347 | " (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n", 348 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 349 | " (5): ReLU()\n", 350 | " )\n", 351 | " (4): Sequential(\n", 352 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=128)\n", 353 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 354 | " (2): ReLU()\n", 355 | " (3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1))\n", 356 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 357 | " (5): ReLU()\n", 358 | " )\n", 359 | " (5): Sequential(\n", 360 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n", 361 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 362 | " (2): ReLU()\n", 363 | " (3): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", 364 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 365 | " (5): ReLU()\n", 366 | " )\n", 367 | " (6): Sequential(\n", 368 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=256)\n", 369 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 370 | " (2): ReLU()\n", 371 | " (3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1))\n", 372 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 373 | " (5): ReLU()\n", 374 | " )\n", 375 | " (7): Sequential(\n", 376 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 377 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 378 | " (2): ReLU()\n", 379 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 380 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 381 | " (5): ReLU()\n", 382 | " )\n", 383 | " (8): Sequential(\n", 384 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 385 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 386 | " (2): ReLU()\n", 387 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 388 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 389 | " (5): ReLU()\n", 390 | " )\n", 391 | " (9): Sequential(\n", 392 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 393 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 394 | " (2): ReLU()\n", 395 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 396 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 397 | " (5): ReLU()\n", 398 | " )\n", 399 | " (10): Sequential(\n", 400 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 401 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 402 | " (2): ReLU()\n", 403 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 404 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 405 | " (5): ReLU()\n", 406 | " )\n", 407 | " (11): Sequential(\n", 408 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 409 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 410 | " (2): ReLU()\n", 411 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 412 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 413 | " (5): ReLU()\n", 414 | " )\n", 415 | " (12): Sequential(\n", 416 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=512)\n", 417 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 418 | " (2): ReLU()\n", 419 | " (3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 420 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 421 | " (5): ReLU()\n", 422 | " )\n", 423 | " (13): Sequential(\n", 424 | " (0): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1024)\n", 425 | " (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 426 | " (2): ReLU()\n", 427 | " (3): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 428 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 429 | " (5): ReLU()\n", 430 | " )\n", 431 | " )\n", 432 | ")" 433 | ] 434 | }, 435 | "execution_count": 15, 436 | "metadata": {}, 437 | "output_type": "execute_result" 438 | } 439 | ], 440 | "source": [ 441 | "res_net = resnet20()\n", 442 | "res_net.to(device)" 443 | ] 444 | }, 445 | { 446 | "cell_type": "code", 447 | "execution_count": 16, 448 | "metadata": {}, 449 | "outputs": [], 450 | "source": [ 451 | "lr = 0.001\n", 452 | "criterion = nn.CrossEntropyLoss()\n", 453 | "optimizer = optim.SGD(res_net.parameters(), lr=lr, momentum=0.9)\n", 454 | "\n", 455 | "rounds = 400 # default\n", 456 | "local_epochs = 1 # default" 457 | ] 458 | }, 459 | { 460 | "cell_type": "markdown", 461 | "metadata": {}, 462 | "source": [ 463 | "## Socket initialization\n", 464 | "### Required socket functions" 465 | ] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": 17, 470 | "metadata": {}, 471 | "outputs": [], 472 | "source": [ 473 | "def send_msg(sock, msg):\n", 474 | " # prefix each message with a 4-byte length in network byte order\n", 475 | " msg = pickle.dumps(msg)\n", 476 | " msg = struct.pack('>I', len(msg)) + msg\n", 477 | " sock.sendall(msg)\n", 478 | "\n", 479 | "def recv_msg(sock):\n", 480 | " # read message length and unpack it into an integer\n", 481 | " raw_msglen = recvall(sock, 4)\n", 482 | " if not raw_msglen:\n", 483 | " return None\n", 484 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 485 | " # read the message data\n", 486 | " msg = recvall(sock, msglen)\n", 487 | " msg = pickle.loads(msg)\n", 488 | " return msg\n", 489 | "\n", 490 | "def recvall(sock, n):\n", 491 | " # helper function to receive n bytes or return None if EOF is hit\n", 492 | " data = b''\n", 493 | " while len(data) < n:\n", 494 | " packet = sock.recv(n - len(data))\n", 495 | " if not packet:\n", 496 | " return None\n", 497 | " data += packet\n", 498 | " return data" 499 | ] 500 | }, 501 | { 502 | "cell_type": "markdown", 503 | "metadata": {}, 504 | "source": [ 505 | "### Set host address and port number" 506 | ] 507 | }, 508 | { 509 | "cell_type": "code", 510 | "execution_count": 18, 511 | "metadata": {}, 512 | "outputs": [ 513 | { 514 | "name": "stdout", 515 | "output_type": "stream", 516 | "text": [ 517 | "IP address: 192.168.83.1\n" 518 | ] 519 | } 520 | ], 521 | "source": [ 522 | "host = input(\"IP address: \")\n", 523 | "port = 10080\n", 524 | "max_recv = 100000" 525 | ] 526 | }, 527 | { 528 | "cell_type": "markdown", 529 | "metadata": {}, 530 | "source": [ 531 | "### Open the client socket" 532 | ] 533 | }, 534 | { 535 | "cell_type": "code", 536 | "execution_count": 19, 537 | "metadata": {}, 538 | "outputs": [], 539 | "source": [ 540 | "s = socket.socket()\n", 541 | "s.connect((host, port))" 542 | ] 543 | }, 544 | { 545 | "cell_type": "markdown", 546 | "metadata": {}, 547 | "source": [ 548 | "## SET TIMER" 549 | ] 550 | }, 551 | { 552 | "cell_type": "code", 553 | "execution_count": 20, 554 | "metadata": {}, 555 | "outputs": [ 556 | { 557 | "name": "stdout", 558 | "output_type": "stream", 559 | "text": [ 560 | "timmer start!\n" 561 | ] 562 | } 563 | ], 564 | "source": [ 565 | "start_time = time.time() # store start time\n", 566 | "print(\"timmer start!\")" 567 | ] 568 | }, 569 | { 570 | "cell_type": "code", 571 | "execution_count": 21, 572 | "metadata": {}, 573 | "outputs": [], 574 | "source": [ 575 | "msg = recv_msg(s)\n", 576 | "rounds = msg['rounds'] \n", 577 | "client_id = msg['client_id']\n", 578 | "local_epochs = msg['local_epoch']\n", 579 | "send_msg(s, len(trainset_sub))" 580 | ] 581 | }, 582 | { 583 | "cell_type": "code", 584 | "execution_count": 22, 585 | "metadata": {}, 586 | "outputs": [ 587 | { 588 | "name": "stderr", 589 | "output_type": "stream", 590 | "text": [ 591 | "Round 1_1: 100%|████████████████████████████████████████████████████| 25/25 [00:11<00:00, 2.13it/s]" 592 | ] 593 | }, 594 | { 595 | "name": "stdout", 596 | "output_type": "stream", 597 | "text": [ 598 | "Finished Training\n" 599 | ] 600 | }, 601 | { 602 | "name": "stderr", 603 | "output_type": "stream", 604 | "text": [ 605 | "\n", 606 | "C:\\Users\\rlaal\\anaconda3\\envs\\py36\\lib\\site-packages\\torch\\storage.py:34: FutureWarning: pickle support for Storage will be removed in 1.5. Use `torch.save` instead\n", 607 | " warnings.warn(\"pickle support for Storage will be removed in 1.5. Use `torch.save` instead\", FutureWarning)\n" 608 | ] 609 | } 610 | ], 611 | "source": [ 612 | "# update weights from server\n", 613 | "# train\n", 614 | "for r in range(rounds): # loop over the dataset multiple times\n", 615 | "\n", 616 | " \n", 617 | " \n", 618 | " weights = recv_msg(s)\n", 619 | " res_net.load_state_dict(weights)\n", 620 | " res_net.eval()\n", 621 | " for local_epoch in range(local_epochs):\n", 622 | " \n", 623 | " for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))):\n", 624 | " \n", 625 | " # get the inputs; data is a list of [inputs, labels]\n", 626 | " inputs, labels = data\n", 627 | " inputs = inputs.to(device)\n", 628 | " labels = labels.clone().detach().long().to(device)\n", 629 | "\n", 630 | " # zero the parameter gradients\n", 631 | " optimizer.zero_grad()\n", 632 | "\n", 633 | " # forward + backward + optimize\n", 634 | " outputs = res_net(inputs)\n", 635 | " loss = criterion(outputs, labels)\n", 636 | " loss.backward()\n", 637 | " optimizer.step()\n", 638 | "\n", 639 | " msg = res_net.state_dict()\n", 640 | " send_msg(s, msg)\n", 641 | "\n", 642 | "print('Finished Training')\n", 643 | "\n" 644 | ] 645 | }, 646 | { 647 | "cell_type": "code", 648 | "execution_count": 23, 649 | "metadata": {}, 650 | "outputs": [ 651 | { 652 | "name": "stdout", 653 | "output_type": "stream", 654 | "text": [ 655 | "Training Time: 12.0054190158844 sec\n" 656 | ] 657 | } 658 | ], 659 | "source": [ 660 | "end_time = time.time() #store end time\n", 661 | "print(\"Training Time: {} sec\".format(end_time - start_time))" 662 | ] 663 | }, 664 | { 665 | "cell_type": "code", 666 | "execution_count": null, 667 | "metadata": {}, 668 | "outputs": [], 669 | "source": [] 670 | } 671 | ], 672 | "metadata": { 673 | "kernelspec": { 674 | "display_name": "Python 3", 675 | "language": "python", 676 | "name": "python3" 677 | }, 678 | "language_info": { 679 | "codemirror_mode": { 680 | "name": "ipython", 681 | "version": 3 682 | }, 683 | "file_extension": ".py", 684 | "mimetype": "text/x-python", 685 | "name": "python", 686 | "nbconvert_exporter": "python", 687 | "pygments_lexer": "ipython3", 688 | "version": "3.6.10" 689 | } 690 | }, 691 | "nbformat": 4, 692 | "nbformat_minor": 2 693 | } 694 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_MobileNet/cifar10_mobile_fd_client_rasp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# CIFAR10 Federated Mobilenet Client Side\n", 8 | "This code is the server part of CIFAR10 federated mobilenet for **multi** client and a server." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 3, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "users = 1 # number of clients" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 4, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "import os\n", 27 | "import h5py\n", 28 | "\n", 29 | "import socket\n", 30 | "import struct\n", 31 | "import pickle\n", 32 | "\n", 33 | "import torch\n", 34 | "import torch.nn as nn\n", 35 | "import torch.nn.functional as F\n", 36 | "import torchvision\n", 37 | "import torchvision.transforms as transforms\n", 38 | "import torch.optim as optim\n", 39 | "\n", 40 | "from torch.utils.data import Dataset, DataLoader\n", 41 | "\n", 42 | "import time\n", 43 | "\n", 44 | "from tqdm import tqdm" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "def getFreeDescription():\n", 54 | " free = os.popen(\"free -h\")\n", 55 | " i = 0\n", 56 | " while True:\n", 57 | " i = i + 1\n", 58 | " line = free.readline()\n", 59 | " if i == 1:\n", 60 | " return (line.split()[0:7])\n", 61 | "\n", 62 | "\n", 63 | "def getFree():\n", 64 | " free = os.popen(\"free -h\")\n", 65 | " i = 0\n", 66 | " while True:\n", 67 | " i = i + 1\n", 68 | " line = free.readline()\n", 69 | " if i == 2:\n", 70 | " return (line.split()[0:7])\n", 71 | "\n", 72 | "from gpiozero import CPUTemperature\n", 73 | "\n", 74 | "\n", 75 | "def printPerformance():\n", 76 | " cpu = CPUTemperature()\n", 77 | "\n", 78 | " print(\"temperature: \" + str(cpu.temperature))\n", 79 | "\n", 80 | " description = getFreeDescription()\n", 81 | " mem = getFree()\n", 82 | "\n", 83 | " print(description[0] + \" : \" + mem[1])\n", 84 | " print(description[1] + \" : \" + mem[2])\n", 85 | " print(description[2] + \" : \" + mem[3])\n", 86 | " print(description[3] + \" : \" + mem[4])\n", 87 | " print(description[4] + \" : \" + mem[5])\n", 88 | " print(description[5] + \" : \" + mem[6])" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "printPerformance()" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 5, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "root_path = '../../models/cifar10_data'" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Cuda" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 6, 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "name": "stdout", 123 | "output_type": "stream", 124 | "text": [ 125 | "cpu\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 131 | "device = \"cpu\"\n", 132 | "print(device)" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 7, 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "client_order(start from 0): 0\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "client_order = int(input(\"client_order(start from 0): \"))" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 8, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "num_traindata = 50000 // users" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "## Data load" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 9, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n", 175 | "\n", 176 | "from torch.utils.data import Subset\n", 177 | "\n", 178 | "\n", 179 | "indices = list(range(50000))\n", 180 | "\n", 181 | "part_tr = indices[num_traindata * client_order : num_traindata * (client_order + 1)]\n" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 10, 187 | "metadata": {}, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../models/cifar10_data\\cifar-10-python.tar.gz\n" 194 | ] 195 | }, 196 | { 197 | "data": { 198 | "application/vnd.jupyter.widget-view+json": { 199 | "model_id": "62295362754042e980b2371d8d0c3390", 200 | "version_major": 2, 201 | "version_minor": 0 202 | }, 203 | "text/plain": [ 204 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" 205 | ] 206 | }, 207 | "metadata": {}, 208 | "output_type": "display_data" 209 | }, 210 | { 211 | "name": "stdout", 212 | "output_type": "stream", 213 | "text": [ 214 | "Extracting ../../models/cifar10_data\\cifar-10-python.tar.gz to ../../models/cifar10_data\n", 215 | "Files already downloaded and verified\n" 216 | ] 217 | } 218 | ], 219 | "source": [ 220 | "trainset = torchvision.datasets.CIFAR10 (root=root_path, train=True, download=True, transform=transform)\n", 221 | "\n", 222 | "trainset_sub = Subset(trainset, part_tr)\n", 223 | "\n", 224 | "train_loader = torch.utils.data.DataLoader(trainset_sub, batch_size=4, shuffle=True, num_workers=2)\n", 225 | "\n", 226 | "testset = torchvision.datasets.CIFAR10 (root=root_path, train=False, download=True, transform=transform)\n", 227 | "test_loader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 11, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [ 236 | "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')" 237 | ] 238 | }, 239 | { 240 | "cell_type": "markdown", 241 | "metadata": {}, 242 | "source": [ 243 | "### Number of total batches" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 13, 249 | "metadata": {}, 250 | "outputs": [ 251 | { 252 | "name": "stdout", 253 | "output_type": "stream", 254 | "text": [ 255 | "25\n", 256 | "2500\n" 257 | ] 258 | } 259 | ], 260 | "source": [ 261 | "train_total_batch = len(train_loader)\n", 262 | "print(train_total_batch)\n", 263 | "test_batch = len(test_loader)\n", 264 | "print(test_batch)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 14, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "# -*- coding: utf-8 -*-\n", 274 | "\"\"\"\n", 275 | "Created on Thu Nov 1 14:23:31 2018\n", 276 | "@author: tshzzz\n", 277 | "\"\"\"\n", 278 | "\n", 279 | "import torch\n", 280 | "import torch.nn as nn\n", 281 | "\n", 282 | "\n", 283 | "def conv_dw(inplane,outplane,stride=1):\n", 284 | " return nn.Sequential(\n", 285 | " nn.Conv2d(inplane,inplane,kernel_size = 3,groups = inplane,stride=stride,padding=1),\n", 286 | " nn.BatchNorm2d(inplane),\n", 287 | " nn.ReLU(),\n", 288 | " nn.Conv2d(inplane,outplane,kernel_size = 1,groups = 1,stride=1),\n", 289 | " nn.BatchNorm2d(outplane),\n", 290 | " nn.ReLU() \n", 291 | " )\n", 292 | "\n", 293 | "def conv_bw(inplane,outplane,kernel_size = 3,stride=1):\n", 294 | " return nn.Sequential(\n", 295 | " nn.Conv2d(inplane,outplane,kernel_size = kernel_size,groups = 1,stride=stride,padding=1),\n", 296 | " nn.BatchNorm2d(outplane),\n", 297 | " nn.ReLU() \n", 298 | " )\n", 299 | "\n", 300 | "\n", 301 | "class MobileNet(nn.Module):\n", 302 | " \n", 303 | " def __init__(self,num_class=10):\n", 304 | " super(MobileNet,self).__init__()\n", 305 | " \n", 306 | " layers = []\n", 307 | " layers.append(conv_bw(3,32,3,1))\n", 308 | " layers.append(conv_dw(32,64,1))\n", 309 | " layers.append(conv_dw(64,128,2))\n", 310 | " layers.append(conv_dw(128,128,1))\n", 311 | " layers.append(conv_dw(128,256,2))\n", 312 | " layers.append(conv_dw(256,256,1))\n", 313 | " layers.append(conv_dw(256,512,2))\n", 314 | "\n", 315 | " for i in range(5):\n", 316 | " layers.append(conv_dw(512,512,1))\n", 317 | " layers.append(conv_dw(512,1024,2))\n", 318 | " layers.append(conv_dw(1024,1024,1))\n", 319 | "\n", 320 | " self.classifer = nn.Sequential(\n", 321 | " nn.Dropout(0.5),\n", 322 | " nn.Linear(1024,num_class)\n", 323 | " )\n", 324 | " self.feature = nn.Sequential(*layers)\n", 325 | " \n", 326 | " \n", 327 | "\n", 328 | " def forward(self,x):\n", 329 | " out = self.feature(x)\n", 330 | " out = out.mean(3).mean(2)\n", 331 | " out = out.view(-1,1024)\n", 332 | " out = self.classifer(out)\n", 333 | " return out\n", 334 | "\n" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 15, 340 | "metadata": {}, 341 | "outputs": [ 342 | { 343 | "data": { 344 | "text/plain": [ 345 | "MobileNet(\n", 346 | " (classifer): Sequential(\n", 347 | " (0): Dropout(p=0.5, inplace=False)\n", 348 | " (1): Linear(in_features=1024, out_features=10, bias=True)\n", 349 | " )\n", 350 | " (feature): Sequential(\n", 351 | " (0): Sequential(\n", 352 | " (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 353 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 354 | " (2): ReLU()\n", 355 | " )\n", 356 | " (1): Sequential(\n", 357 | " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n", 358 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 359 | " (2): ReLU()\n", 360 | " (3): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n", 361 | " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 362 | " (5): ReLU()\n", 363 | " )\n", 364 | " (2): Sequential(\n", 365 | " (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=64)\n", 366 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 367 | " (2): ReLU()\n", 368 | " (3): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n", 369 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 370 | " (5): ReLU()\n", 371 | " )\n", 372 | " (3): Sequential(\n", 373 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n", 374 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 375 | " (2): ReLU()\n", 376 | " (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n", 377 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 378 | " (5): ReLU()\n", 379 | " )\n", 380 | " (4): Sequential(\n", 381 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=128)\n", 382 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 383 | " (2): ReLU()\n", 384 | " (3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1))\n", 385 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 386 | " (5): ReLU()\n", 387 | " )\n", 388 | " (5): Sequential(\n", 389 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n", 390 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 391 | " (2): ReLU()\n", 392 | " (3): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", 393 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 394 | " (5): ReLU()\n", 395 | " )\n", 396 | " (6): Sequential(\n", 397 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=256)\n", 398 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 399 | " (2): ReLU()\n", 400 | " (3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1))\n", 401 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 402 | " (5): ReLU()\n", 403 | " )\n", 404 | " (7): Sequential(\n", 405 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 406 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 407 | " (2): ReLU()\n", 408 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 409 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 410 | " (5): ReLU()\n", 411 | " )\n", 412 | " (8): Sequential(\n", 413 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 414 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 415 | " (2): ReLU()\n", 416 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 417 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 418 | " (5): ReLU()\n", 419 | " )\n", 420 | " (9): Sequential(\n", 421 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 422 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 423 | " (2): ReLU()\n", 424 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 425 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 426 | " (5): ReLU()\n", 427 | " )\n", 428 | " (10): Sequential(\n", 429 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 430 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 431 | " (2): ReLU()\n", 432 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 433 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 434 | " (5): ReLU()\n", 435 | " )\n", 436 | " (11): Sequential(\n", 437 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 438 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 439 | " (2): ReLU()\n", 440 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 441 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 442 | " (5): ReLU()\n", 443 | " )\n", 444 | " (12): Sequential(\n", 445 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=512)\n", 446 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 447 | " (2): ReLU()\n", 448 | " (3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 449 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 450 | " (5): ReLU()\n", 451 | " )\n", 452 | " (13): Sequential(\n", 453 | " (0): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1024)\n", 454 | " (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 455 | " (2): ReLU()\n", 456 | " (3): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 457 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 458 | " (5): ReLU()\n", 459 | " )\n", 460 | " )\n", 461 | ")" 462 | ] 463 | }, 464 | "execution_count": 15, 465 | "metadata": {}, 466 | "output_type": "execute_result" 467 | } 468 | ], 469 | "source": [ 470 | "mobile_net = MobileNet()\n", 471 | "mobile_net.to(device)" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 16, 477 | "metadata": {}, 478 | "outputs": [], 479 | "source": [ 480 | "lr = 0.001\n", 481 | "criterion = nn.CrossEntropyLoss()\n", 482 | "optimizer = optim.SGD(mobile_net.parameters(), lr=lr, momentum=0.9)\n", 483 | "\n", 484 | "rounds = 400 # default\n", 485 | "local_epochs = 1 # default" 486 | ] 487 | }, 488 | { 489 | "cell_type": "markdown", 490 | "metadata": {}, 491 | "source": [ 492 | "## Socket initialization\n", 493 | "### Required socket functions" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": 17, 499 | "metadata": {}, 500 | "outputs": [], 501 | "source": [ 502 | "def send_msg(sock, msg):\n", 503 | " # prefix each message with a 4-byte length in network byte order\n", 504 | " msg = pickle.dumps(msg)\n", 505 | " msg = struct.pack('>I', len(msg)) + msg\n", 506 | " sock.sendall(msg)\n", 507 | "\n", 508 | "def recv_msg(sock):\n", 509 | " # read message length and unpack it into an integer\n", 510 | " raw_msglen = recvall(sock, 4)\n", 511 | " if not raw_msglen:\n", 512 | " return None\n", 513 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 514 | " # read the message data\n", 515 | " msg = recvall(sock, msglen)\n", 516 | " msg = pickle.loads(msg)\n", 517 | " return msg\n", 518 | "\n", 519 | "def recvall(sock, n):\n", 520 | " # helper function to receive n bytes or return None if EOF is hit\n", 521 | " data = b''\n", 522 | " while len(data) < n:\n", 523 | " packet = sock.recv(n - len(data))\n", 524 | " if not packet:\n", 525 | " return None\n", 526 | " data += packet\n", 527 | " return data" 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": 15, 533 | "metadata": {}, 534 | "outputs": [], 535 | "source": [ 536 | "printPerformance()" 537 | ] 538 | }, 539 | { 540 | "cell_type": "markdown", 541 | "metadata": {}, 542 | "source": [ 543 | "### Set host address and port number" 544 | ] 545 | }, 546 | { 547 | "cell_type": "code", 548 | "execution_count": 18, 549 | "metadata": {}, 550 | "outputs": [ 551 | { 552 | "name": "stdout", 553 | "output_type": "stream", 554 | "text": [ 555 | "IP address: 192.168.83.1\n" 556 | ] 557 | } 558 | ], 559 | "source": [ 560 | "host = input(\"IP address: \")\n", 561 | "port = 10080\n", 562 | "max_recv = 100000" 563 | ] 564 | }, 565 | { 566 | "cell_type": "markdown", 567 | "metadata": {}, 568 | "source": [ 569 | "### Open the client socket" 570 | ] 571 | }, 572 | { 573 | "cell_type": "code", 574 | "execution_count": 19, 575 | "metadata": {}, 576 | "outputs": [], 577 | "source": [ 578 | "s = socket.socket()\n", 579 | "s.connect((host, port))" 580 | ] 581 | }, 582 | { 583 | "cell_type": "markdown", 584 | "metadata": {}, 585 | "source": [ 586 | "## SET TIMER" 587 | ] 588 | }, 589 | { 590 | "cell_type": "code", 591 | "execution_count": 20, 592 | "metadata": {}, 593 | "outputs": [ 594 | { 595 | "name": "stdout", 596 | "output_type": "stream", 597 | "text": [ 598 | "timmer start!\n" 599 | ] 600 | } 601 | ], 602 | "source": [ 603 | "start_time = time.time() # store start time\n", 604 | "print(\"timmer start!\")" 605 | ] 606 | }, 607 | { 608 | "cell_type": "code", 609 | "execution_count": 21, 610 | "metadata": {}, 611 | "outputs": [], 612 | "source": [ 613 | "msg = recv_msg(s)\n", 614 | "rounds = msg['rounds'] \n", 615 | "client_id = msg['client_id']\n", 616 | "local_epochs = msg['local_epoch']\n", 617 | "send_msg(s, len(trainset_sub))" 618 | ] 619 | }, 620 | { 621 | "cell_type": "code", 622 | "execution_count": 22, 623 | "metadata": {}, 624 | "outputs": [ 625 | { 626 | "name": "stderr", 627 | "output_type": "stream", 628 | "text": [ 629 | "Round 1_1: 100%|████████████████████████████████████████████████████| 25/25 [00:11<00:00, 2.13it/s]" 630 | ] 631 | }, 632 | { 633 | "name": "stdout", 634 | "output_type": "stream", 635 | "text": [ 636 | "Finished Training\n" 637 | ] 638 | }, 639 | { 640 | "name": "stderr", 641 | "output_type": "stream", 642 | "text": [ 643 | "\n", 644 | "C:\\Users\\rlaal\\anaconda3\\envs\\py36\\lib\\site-packages\\torch\\storage.py:34: FutureWarning: pickle support for Storage will be removed in 1.5. Use `torch.save` instead\n", 645 | " warnings.warn(\"pickle support for Storage will be removed in 1.5. Use `torch.save` instead\", FutureWarning)\n" 646 | ] 647 | } 648 | ], 649 | "source": [ 650 | "# update weights from server\n", 651 | "# train\n", 652 | "for r in range(rounds): # loop over the dataset multiple times\n", 653 | "\n", 654 | " \n", 655 | " \n", 656 | " weights = recv_msg(s)\n", 657 | " mobile_net.load_state_dict(weights)\n", 658 | " mobile_net.eval()\n", 659 | " for local_epoch in range(local_epochs):\n", 660 | " \n", 661 | " for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))):\n", 662 | " \n", 663 | " # get the inputs; data is a list of [inputs, labels]\n", 664 | " inputs, labels = data\n", 665 | " inputs = inputs.to(device)\n", 666 | " labels = labels.clone().detach().long().to(device)\n", 667 | "\n", 668 | " # zero the parameter gradients\n", 669 | " optimizer.zero_grad()\n", 670 | "\n", 671 | " # forward + backward + optimize\n", 672 | " outputs = mobile_net(inputs)\n", 673 | " loss = criterion(outputs, labels)\n", 674 | " loss.backward()\n", 675 | " optimizer.step()\n", 676 | "\n", 677 | " msg = mobile_net.state_dict()\n", 678 | " send_msg(s, msg)\n", 679 | "\n", 680 | "print('Finished Training')\n", 681 | "\n" 682 | ] 683 | }, 684 | { 685 | "cell_type": "code", 686 | "execution_count": null, 687 | "metadata": {}, 688 | "outputs": [], 689 | "source": [ 690 | "printPerformance()" 691 | ] 692 | }, 693 | { 694 | "cell_type": "code", 695 | "execution_count": 23, 696 | "metadata": {}, 697 | "outputs": [ 698 | { 699 | "name": "stdout", 700 | "output_type": "stream", 701 | "text": [ 702 | "Training Time: 12.0054190158844 sec\n" 703 | ] 704 | } 705 | ], 706 | "source": [ 707 | "end_time = time.time() #store end time\n", 708 | "print(\"Training Time: {} sec\".format(end_time - start_time))" 709 | ] 710 | }, 711 | { 712 | "cell_type": "code", 713 | "execution_count": null, 714 | "metadata": {}, 715 | "outputs": [], 716 | "source": [] 717 | } 718 | ], 719 | "metadata": { 720 | "kernelspec": { 721 | "display_name": "Python 3", 722 | "language": "python", 723 | "name": "python3" 724 | }, 725 | "language_info": { 726 | "codemirror_mode": { 727 | "name": "ipython", 728 | "version": 3 729 | }, 730 | "file_extension": ".py", 731 | "mimetype": "text/x-python", 732 | "name": "python", 733 | "nbconvert_exporter": "python", 734 | "pygments_lexer": "ipython3", 735 | "version": "3.6.10" 736 | } 737 | }, 738 | "nbformat": 4, 739 | "nbformat_minor": 2 740 | } 741 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/ecg/ecg_fd_server.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# ECG Federated 1D-CNN Server Side\n", 8 | "This code is the server part of ECG federated 1D-CNN model for **multi** client and a server." 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "## Setting variables" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "rounds = 400\n", 25 | "local_epoch = 1\n", 26 | "users = 2 # number of clients" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 2, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "import os\n", 36 | "import h5py\n", 37 | "\n", 38 | "import socket\n", 39 | "import struct\n", 40 | "import pickle\n", 41 | "import sys\n", 42 | "\n", 43 | "import torch\n", 44 | "import torch.nn as nn\n", 45 | "import torch.nn.functional as F\n", 46 | "import torch.optim as optim\n", 47 | "from threading import Thread\n", 48 | "from threading import Lock\n", 49 | "\n", 50 | "\n", 51 | "import time\n", 52 | "\n", 53 | "from tqdm import tqdm\n", 54 | "\n", 55 | "import copy" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "## Cuda" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "outputs": [ 70 | { 71 | "name": "stdout", 72 | "output_type": "stream", 73 | "text": [ 74 | "cpu\n" 75 | ] 76 | } 77 | ], 78 | "source": [ 79 | "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 80 | "print(device)" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "## Pytorch layer modules for *Conv1D* Network\n", 88 | "\n", 89 | "\n", 90 | "\n", 91 | "### `Conv1d` layer\n", 92 | "- `torch.nn.Conv1d(in_channels, out_channels, kernel_size)`\n", 93 | "\n", 94 | "### `MaxPool1d` layer\n", 95 | "- `torch.nn.MaxPool1d(kernel_size, stride=None)`\n", 96 | "- Parameter `stride` follows `kernel_size`.\n", 97 | "\n", 98 | "### `ReLU` layer\n", 99 | "- `torch.nn.ReLU()`\n", 100 | "\n", 101 | "### `Linear` layer\n", 102 | "- `torch.nn.Linear(in_features, out_features, bias=True)`\n", 103 | "\n", 104 | "### `Softmax` layer\n", 105 | "- `torch.nn.Softmax(dim=None)`\n", 106 | "- Parameter `dim` is usually set to `1`." 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Construct 1D-CNN ECG classification model" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 4, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "class EcgConv1d(nn.Module):\n", 123 | " def __init__(self):\n", 124 | " super(EcgConv1d, self).__init__() \n", 125 | " self.conv1 = nn.Conv1d(1, 16, 7) # 124 x 16 \n", 126 | " self.relu1 = nn.LeakyReLU()\n", 127 | " self.pool1 = nn.MaxPool1d(2) # 62 x 16\n", 128 | " self.conv2 = nn.Conv1d(16, 16, 5) # 58 x 16\n", 129 | " self.relu2 = nn.LeakyReLU() \n", 130 | " self.conv3 = nn.Conv1d(16, 16, 5) # 54 x 16\n", 131 | " self.relu3 = nn.LeakyReLU() \n", 132 | " self.conv4 = nn.Conv1d(16, 16, 5) # 50 x 16\n", 133 | " self.relu4 = nn.LeakyReLU()\n", 134 | " self.pool4 = nn.MaxPool1d(2) # 25 x 16\n", 135 | " self.linear5 = nn.Linear(25 * 16, 128)\n", 136 | " self.relu5 = nn.LeakyReLU() \n", 137 | " self.linear6 = nn.Linear(128, 5)\n", 138 | " self.softmax6 = nn.Softmax(dim=1)\n", 139 | " \n", 140 | " def forward(self, x):\n", 141 | " x = self.conv1(x)\n", 142 | " x = self.relu1(x)\n", 143 | " x = self.pool1(x) \n", 144 | " x = self.conv2(x)\n", 145 | " x = self.relu2(x) \n", 146 | " x = self.conv3(x)\n", 147 | " x = self.relu3(x) \n", 148 | " x = self.conv4(x)\n", 149 | " x = self.relu4(x)\n", 150 | " x = self.pool4(x)\n", 151 | " x = x.view(-1, 25 * 16)\n", 152 | " x = self.linear5(x)\n", 153 | " x = self.relu5(x) \n", 154 | " x = self.linear6(x)\n", 155 | " x = self.softmax6(x)\n", 156 | " return x " 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 5, 162 | "metadata": {}, 163 | "outputs": [ 164 | { 165 | "data": { 166 | "text/plain": [ 167 | "EcgConv1d(\n", 168 | " (conv1): Conv1d(1, 16, kernel_size=(7,), stride=(1,))\n", 169 | " (relu1): LeakyReLU(negative_slope=0.01)\n", 170 | " (pool1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", 171 | " (conv2): Conv1d(16, 16, kernel_size=(5,), stride=(1,))\n", 172 | " (relu2): LeakyReLU(negative_slope=0.01)\n", 173 | " (conv3): Conv1d(16, 16, kernel_size=(5,), stride=(1,))\n", 174 | " (relu3): LeakyReLU(negative_slope=0.01)\n", 175 | " (conv4): Conv1d(16, 16, kernel_size=(5,), stride=(1,))\n", 176 | " (relu4): LeakyReLU(negative_slope=0.01)\n", 177 | " (pool4): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n", 178 | " (linear5): Linear(in_features=400, out_features=128, bias=True)\n", 179 | " (relu5): LeakyReLU(negative_slope=0.01)\n", 180 | " (linear6): Linear(in_features=128, out_features=5, bias=True)\n", 181 | " (softmax6): Softmax(dim=1)\n", 182 | ")" 183 | ] 184 | }, 185 | "execution_count": 5, 186 | "metadata": {}, 187 | "output_type": "execute_result" 188 | } 189 | ], 190 | "source": [ 191 | "ecg_net = EcgConv1d()\n", 192 | "ecg_net.to('cpu')" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "metadata": {}, 198 | "source": [ 199 | "## variables" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": 6, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "clientsoclist = [0]*users\n", 209 | "\n", 210 | "start_time = 0\n", 211 | "weight_count = 0\n", 212 | "\n", 213 | "global_weights = copy.deepcopy(ecg_net.state_dict())\n", 214 | "\n", 215 | "datasetsize = [0]*users\n", 216 | "weights_list = [0]*users\n", 217 | "\n", 218 | "lock = Lock()" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": {}, 224 | "source": [ 225 | "## Comunication overhead" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 7, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "total_sendsize_list = []\n", 235 | "total_receivesize_list = []\n", 236 | "\n", 237 | "client_sendsize_list = [[] for i in range(users)]\n", 238 | "client_receivesize_list = [[] for i in range(users)]\n", 239 | "\n", 240 | "train_sendsize_list = [] \n", 241 | "train_receivesize_list = []" 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "metadata": {}, 247 | "source": [ 248 | "## Socket initialization\n", 249 | "### Set host address and port number" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "### Required socket functions" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 8, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "def send_msg(sock, msg):\n", 266 | " # prefix each message with a 4-byte length in network byte order\n", 267 | " msg = pickle.dumps(msg)\n", 268 | " l_send = len(msg)\n", 269 | " msg = struct.pack('>I', l_send) + msg\n", 270 | " sock.sendall(msg)\n", 271 | " return l_send\n", 272 | "\n", 273 | "def recv_msg(sock):\n", 274 | " # read message length and unpack it into an integer\n", 275 | " raw_msglen = recvall(sock, 4)\n", 276 | " if not raw_msglen:\n", 277 | " return None\n", 278 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 279 | " # read the message data\n", 280 | " msg = recvall(sock, msglen)\n", 281 | " msg = pickle.loads(msg)\n", 282 | " return msg, msglen\n", 283 | "\n", 284 | "def recvall(sock, n):\n", 285 | " # helper function to receive n bytes or return None if EOF is hit\n", 286 | " data = b''\n", 287 | " while len(data) < n:\n", 288 | " packet = sock.recv(n - len(data))\n", 289 | " if not packet:\n", 290 | " return None\n", 291 | " data += packet\n", 292 | " return data" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": 9, 298 | "metadata": {}, 299 | "outputs": [], 300 | "source": [ 301 | "import copy\n", 302 | "\n", 303 | "def average_weights(w, datasize):\n", 304 | " \"\"\"\n", 305 | " Returns the average of the weights.\n", 306 | " \"\"\"\n", 307 | " \n", 308 | " for i, data in enumerate(datasize):\n", 309 | " for key in w[i].keys():\n", 310 | " w[i][key] *= float(data)\n", 311 | " \n", 312 | " w_avg = copy.deepcopy(w[0])\n", 313 | " \n", 314 | " \n", 315 | "\n", 316 | "# when client use only one kinds of device\n", 317 | "\n", 318 | " for key in w_avg.keys():\n", 319 | " for i in range(1, len(w)):\n", 320 | " w_avg[key] += w[i][key]\n", 321 | " w_avg[key] = torch.div(w_avg[key], float(sum(datasize)))\n", 322 | "\n", 323 | "# when client use various devices (cpu, gpu) you need to use it instead\n", 324 | "#\n", 325 | "# for key, val in w_avg.items():\n", 326 | "# common_device = val.device\n", 327 | "# break\n", 328 | "# for key in w_avg.keys():\n", 329 | "# for i in range(1, len(w)):\n", 330 | "# if common_device == 'cpu':\n", 331 | "# w_avg[key] += w[i][key].cpu()\n", 332 | "# else:\n", 333 | "# w_avg[key] += w[i][key].cuda()\n", 334 | "# w_avg[key] = torch.div(w_avg[key], float(sum(datasize)))\n", 335 | "\n", 336 | " return w_avg" 337 | ] 338 | }, 339 | { 340 | "cell_type": "markdown", 341 | "metadata": {}, 342 | "source": [ 343 | "## Thread define" 344 | ] 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "metadata": {}, 349 | "source": [ 350 | "## Receive users before training" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 10, 356 | "metadata": {}, 357 | "outputs": [], 358 | "source": [ 359 | "def run_thread(func, num_user):\n", 360 | " global clientsoclist\n", 361 | " global start_time\n", 362 | " \n", 363 | " thrs = []\n", 364 | " for i in range(num_user):\n", 365 | " conn, addr = s.accept()\n", 366 | " print('Conntected with', addr)\n", 367 | " # append client socket on list\n", 368 | " clientsoclist[i] = conn\n", 369 | " args = (i, num_user, conn)\n", 370 | " thread = Thread(target=func, args=args)\n", 371 | " thrs.append(thread)\n", 372 | " thread.start()\n", 373 | " print(\"timmer start!\")\n", 374 | " start_time = time.time() # store start time\n", 375 | " for thread in thrs:\n", 376 | " thread.join()\n", 377 | " end_time = time.time() # store end time\n", 378 | " print(\"TrainingTime: {} sec\".format(end_time - start_time))" 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": 11, 384 | "metadata": {}, 385 | "outputs": [], 386 | "source": [ 387 | "def receive(userid, num_users, conn): #thread for receive clients\n", 388 | " global weight_count\n", 389 | " \n", 390 | " global datasetsize\n", 391 | "\n", 392 | "\n", 393 | " msg = {\n", 394 | " 'rounds': rounds,\n", 395 | " 'client_id': userid,\n", 396 | " 'local_epoch': local_epoch\n", 397 | " }\n", 398 | "\n", 399 | " datasize = send_msg(conn, msg) #send epoch\n", 400 | " total_sendsize_list.append(datasize)\n", 401 | " client_sendsize_list[userid].append(datasize)\n", 402 | "\n", 403 | " train_dataset_size, datasize = recv_msg(conn) # get total_batch of train dataset\n", 404 | " total_receivesize_list.append(datasize)\n", 405 | " client_receivesize_list[userid].append(datasize)\n", 406 | " \n", 407 | " \n", 408 | " with lock:\n", 409 | " datasetsize[userid] = train_dataset_size\n", 410 | " weight_count += 1\n", 411 | " \n", 412 | " train(userid, train_dataset_size, num_users, conn)" 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "metadata": {}, 418 | "source": [ 419 | "## Train" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": 12, 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "def train(userid, train_dataset_size, num_users, client_conn):\n", 429 | " global weights_list\n", 430 | " global global_weights\n", 431 | " global weight_count\n", 432 | " global ecg_net\n", 433 | " global val_acc\n", 434 | " \n", 435 | " for r in range(rounds):\n", 436 | " with lock:\n", 437 | " if weight_count == num_users:\n", 438 | " for i, conn in enumerate(clientsoclist):\n", 439 | " datasize = send_msg(conn, global_weights)\n", 440 | " total_sendsize_list.append(datasize)\n", 441 | " client_sendsize_list[i].append(datasize)\n", 442 | " train_sendsize_list.append(datasize)\n", 443 | " weight_count = 0\n", 444 | "\n", 445 | " client_weights, datasize = recv_msg(client_conn)\n", 446 | " total_receivesize_list.append(datasize)\n", 447 | " client_receivesize_list[userid].append(datasize)\n", 448 | " train_receivesize_list.append(datasize)\n", 449 | "\n", 450 | " weights_list[userid] = client_weights\n", 451 | " print(\"User\" + str(userid) + \"'s Round \" + str(r + 1) + \" is done\")\n", 452 | " with lock:\n", 453 | " weight_count += 1\n", 454 | " if weight_count == num_users:\n", 455 | " #average\n", 456 | " global_weights = average_weights(weights_list, datasetsize)\n", 457 | " \n", 458 | " \n", 459 | " " 460 | ] 461 | }, 462 | { 463 | "cell_type": "code", 464 | "execution_count": 13, 465 | "metadata": {}, 466 | "outputs": [ 467 | { 468 | "name": "stdout", 469 | "output_type": "stream", 470 | "text": [ 471 | "192.168.0.102\n" 472 | ] 473 | } 474 | ], 475 | "source": [ 476 | "host = socket.gethostbyname(socket.gethostname())\n", 477 | "port = 10080\n", 478 | "print(host)" 479 | ] 480 | }, 481 | { 482 | "cell_type": "code", 483 | "execution_count": 14, 484 | "metadata": {}, 485 | "outputs": [], 486 | "source": [ 487 | "s = socket.socket()\n", 488 | "s.bind((host, port))\n", 489 | "s.listen(5)" 490 | ] 491 | }, 492 | { 493 | "cell_type": "markdown", 494 | "metadata": {}, 495 | "source": [ 496 | "### Open the server socket" 497 | ] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": null, 502 | "metadata": {}, 503 | "outputs": [ 504 | { 505 | "name": "stdout", 506 | "output_type": "stream", 507 | "text": [ 508 | "Conntected with ('192.168.0.108', 58144)\n", 509 | "Conntected with ('192.168.0.107', 59952)\n", 510 | "timmer start!\n", 511 | "User0's Round 1 is done\n", 512 | "User1's Round 1 is done\n", 513 | "User1's Round 2 is done\n", 514 | "User0's Round 2 is done\n" 515 | ] 516 | } 517 | ], 518 | "source": [ 519 | "run_thread(receive, users)" 520 | ] 521 | }, 522 | { 523 | "cell_type": "code", 524 | "execution_count": null, 525 | "metadata": {}, 526 | "outputs": [], 527 | "source": [ 528 | "end_time = time.time() # store end time\n", 529 | "print(\"TrainingTime: {} sec\".format(end_time - start_time))" 530 | ] 531 | }, 532 | { 533 | "cell_type": "markdown", 534 | "metadata": {}, 535 | "source": [ 536 | "## Print all of communication overhead" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": null, 542 | "metadata": {}, 543 | "outputs": [], 544 | "source": [ 545 | "# print('val_acc list')\n", 546 | "# for acc in val_acc:\n", 547 | "# print(acc)\n", 548 | "\n", 549 | "print('\\n')\n", 550 | "print('---total_sendsize_list---')\n", 551 | "total_size = 0\n", 552 | "for size in total_sendsize_list:\n", 553 | "# print(size)\n", 554 | " total_size += size\n", 555 | "print(\"total_sendsize size: {} bytes\".format(total_size))\n", 556 | "print('\\n')\n", 557 | "\n", 558 | "print('---total_receivesize_list---')\n", 559 | "total_size = 0\n", 560 | "for size in total_receivesize_list:\n", 561 | "# print(size)\n", 562 | " total_size += size\n", 563 | "print(\"total receive sizes: {} bytes\".format(total_size) )\n", 564 | "print('\\n')\n", 565 | "\n", 566 | "for i in range(users):\n", 567 | " print('---client_sendsize_list(user{})---'.format(i))\n", 568 | " total_size = 0\n", 569 | " for size in client_sendsize_list[i]:\n", 570 | "# print(size)\n", 571 | " total_size += size\n", 572 | " print(\"total client_sendsizes(user{}): {} bytes\".format(i, total_size))\n", 573 | " print('\\n')\n", 574 | "\n", 575 | " print('---client_receivesize_list(user{})---'.format(i))\n", 576 | " total_size = 0\n", 577 | " for size in client_receivesize_list[i]:\n", 578 | "# print(size)\n", 579 | " total_size += size\n", 580 | " print(\"total client_receive sizes(user{}): {} bytes\".format(i, total_size))\n", 581 | " print('\\n')\n", 582 | "\n", 583 | "print('---train_sendsize_list---')\n", 584 | "total_size = 0\n", 585 | "for size in train_sendsize_list:\n", 586 | "# print(size)\n", 587 | " total_size += size\n", 588 | "print(\"total train_sendsizes: {} bytes\".format(total_size))\n", 589 | "print('\\n')\n", 590 | "\n", 591 | "print('---train_receivesize_list---')\n", 592 | "total_size = 0\n", 593 | "for size in train_receivesize_list:\n", 594 | "# print(size)\n", 595 | " total_size += size\n", 596 | "print(\"total train_receivesizes: {} bytes\".format(total_size))\n", 597 | "print('\\n')\n" 598 | ] 599 | }, 600 | { 601 | "cell_type": "code", 602 | "execution_count": null, 603 | "metadata": {}, 604 | "outputs": [], 605 | "source": [ 606 | "root_path = '../../models/'" 607 | ] 608 | }, 609 | { 610 | "cell_type": "markdown", 611 | "metadata": {}, 612 | "source": [ 613 | "## Defining `ECG` Dataset Class\n" 614 | ] 615 | }, 616 | { 617 | "cell_type": "code", 618 | "execution_count": null, 619 | "metadata": {}, 620 | "outputs": [], 621 | "source": [ 622 | "from torch.utils.data import Dataset, DataLoader\n", 623 | "from torch.optim import Adam" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": null, 629 | "metadata": {}, 630 | "outputs": [], 631 | "source": [ 632 | "class ECG(Dataset):\n", 633 | " def __init__(self, train=True):\n", 634 | " if train:\n", 635 | " with h5py.File(os.path.join(root_path, 'ecg_data', 'train_ecg.hdf5'), 'r') as hdf:\n", 636 | " self.x = hdf['x_train'][:]\n", 637 | " self.y = hdf['y_train'][:]\n", 638 | " else:\n", 639 | " with h5py.File(os.path.join(root_path, 'ecg_data', 'test_ecg.hdf5'), 'r') as hdf:\n", 640 | " self.x = hdf['x_test'][:]\n", 641 | " self.y = hdf['y_test'][:]\n", 642 | " \n", 643 | " def __len__(self):\n", 644 | " return len(self.x)\n", 645 | " \n", 646 | " def __getitem__(self, idx):\n", 647 | " return torch.tensor(self.x[idx], dtype=torch.float), torch.tensor(self.y[idx])" 648 | ] 649 | }, 650 | { 651 | "cell_type": "markdown", 652 | "metadata": {}, 653 | "source": [ 654 | "## Making Batch Generator" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": null, 660 | "metadata": {}, 661 | "outputs": [], 662 | "source": [ 663 | "batch_size = 32" 664 | ] 665 | }, 666 | { 667 | "cell_type": "markdown", 668 | "metadata": {}, 669 | "source": [ 670 | "### `DataLoader` for batch generating\n", 671 | "`torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)`" 672 | ] 673 | }, 674 | { 675 | "cell_type": "code", 676 | "execution_count": null, 677 | "metadata": {}, 678 | "outputs": [], 679 | "source": [ 680 | "train_dataset = ECG(train=True)\n", 681 | "test_dataset = ECG(train=False)\n", 682 | "trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", 683 | "testloader = DataLoader(test_dataset, batch_size=batch_size)" 684 | ] 685 | }, 686 | { 687 | "cell_type": "markdown", 688 | "metadata": {}, 689 | "source": [ 690 | "### Number of total batches" 691 | ] 692 | }, 693 | { 694 | "cell_type": "code", 695 | "execution_count": null, 696 | "metadata": {}, 697 | "outputs": [], 698 | "source": [ 699 | "train_total_batch = len(trainloader)\n", 700 | "print(train_total_batch)\n", 701 | "test_batch = len(testloader)\n", 702 | "print(test_batch)" 703 | ] 704 | }, 705 | { 706 | "cell_type": "code", 707 | "execution_count": null, 708 | "metadata": {}, 709 | "outputs": [], 710 | "source": [ 711 | "lr = 0.001\n", 712 | "optimizer = Adam(ecg_net.parameters(), lr=lr)\n", 713 | "criterion = nn.CrossEntropyLoss()" 714 | ] 715 | }, 716 | { 717 | "cell_type": "markdown", 718 | "metadata": {}, 719 | "source": [ 720 | "## Accuracy of train and each of classes" 721 | ] 722 | }, 723 | { 724 | "cell_type": "code", 725 | "execution_count": null, 726 | "metadata": {}, 727 | "outputs": [], 728 | "source": [ 729 | "ecg_net.load_state_dict(global_weights)\n", 730 | "ecg_net.eval()\n", 731 | "ecg_net = ecg_net.to(device)\n", 732 | "\n", 733 | "# train acc\n", 734 | "with torch.no_grad():\n", 735 | " corr_num = 0\n", 736 | " total_num = 0\n", 737 | " train_loss = 0.0\n", 738 | " for j, trn in enumerate(trainloader):\n", 739 | " trn_x, trn_label = trn\n", 740 | " trn_x = trn_x.to(device)\n", 741 | " trn_label = trn_label.clone().detach().long().to(device)\n", 742 | "\n", 743 | " trn_output = ecg_net(trn_x)\n", 744 | " loss = criterion(trn_output, trn_label)\n", 745 | " train_loss += loss.item()\n", 746 | " model_label = trn_output.argmax(dim=1)\n", 747 | " corr = trn_label[trn_label == model_label].size(0)\n", 748 | " corr_num += corr\n", 749 | " total_num += trn_label.size(0)\n", 750 | " print(\"train_acc: {:.2f}%, train_loss: {:.4f}\".format(corr_num / total_num * 100, train_loss / len(trainloader)))\n", 751 | "\n", 752 | "\n", 753 | "# test acc\n", 754 | "with torch.no_grad():\n", 755 | " corr_num = 0\n", 756 | " total_num = 0\n", 757 | " val_loss = 0.0\n", 758 | " for j, val in enumerate(testloader):\n", 759 | " val_x, val_label = val\n", 760 | " val_x = val_x.to(device)\n", 761 | " val_label = val_label.clone().detach().long().to(device)\n", 762 | "\n", 763 | " val_output = ecg_net(val_x)\n", 764 | " loss = criterion(val_output, val_label)\n", 765 | " val_loss += loss.item()\n", 766 | " model_label = val_output.argmax(dim=1)\n", 767 | " corr = val_label[val_label == model_label].size(0)\n", 768 | " corr_num += corr\n", 769 | " total_num += val_label.size(0)\n", 770 | " accuracy = corr_num / total_num * 100\n", 771 | " test_loss = val_loss / len(testloader)\n", 772 | " print(\"test_acc: {:.2f}%, test_loss: {:.4f}\".format( accuracy, test_loss))\n", 773 | "\n", 774 | "# acc of each acc \n", 775 | "class_correct = list(0. for i in range(5))\n", 776 | "class_total = list(0. for i in range(5))\n", 777 | "classes = ['N', 'L', 'R', 'A', 'V']\n", 778 | "\n", 779 | "with torch.no_grad():\n", 780 | " for data in testloader:\n", 781 | " x, labels = data\n", 782 | " x = x.to(device)\n", 783 | " labels = labels.to(device)\n", 784 | "\n", 785 | " outputs = ecg_net(x)\n", 786 | " labels = labels.long()\n", 787 | " _, predicted = torch.max(outputs, 1)\n", 788 | " c = (predicted == labels).squeeze()\n", 789 | " for i in range(len(labels)):\n", 790 | " label = labels[i]\n", 791 | " class_correct[label] += c[i].item()\n", 792 | " class_total[label] += 1\n", 793 | "\n", 794 | "\n", 795 | "for i in range(5):\n", 796 | " print('Accuracy of %5s : %2d %%' % (\n", 797 | " classes[i], 100 * class_correct[i] / class_total[i]))\n", 798 | "\n", 799 | "# Let's quickly save our trained model:\n", 800 | "PATH = './ecg_fd.pth'\n", 801 | "torch.save(ecg_net.state_dict(), PATH)\n", 802 | "\n", 803 | "end_time = time.time() # store end time\n", 804 | "print(\"WorkingTime: {} sec\".format(end_time - start_time))\n", 805 | "# sys.exit(0)" 806 | ] 807 | }, 808 | { 809 | "cell_type": "code", 810 | "execution_count": null, 811 | "metadata": {}, 812 | "outputs": [], 813 | "source": [] 814 | } 815 | ], 816 | "metadata": { 817 | "kernelspec": { 818 | "display_name": "Python 3 (ipykernel)", 819 | "language": "python", 820 | "name": "python3" 821 | }, 822 | "language_info": { 823 | "codemirror_mode": { 824 | "name": "ipython", 825 | "version": 3 826 | }, 827 | "file_extension": ".py", 828 | "mimetype": "text/x-python", 829 | "name": "python", 830 | "nbconvert_exporter": "python", 831 | "pygments_lexer": "ipython3", 832 | "version": "3.8.6" 833 | } 834 | }, 835 | "nbformat": 4, 836 | "nbformat_minor": 2 837 | } 838 | -------------------------------------------------------------------------------- /Federated Learning/Federated-Learning-and-Split-Learning-with-raspberry-pi-master/federated_learning/cifar10_Resnet20/cifar10_resnet20_fd_client_rasp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# CIFAR10 Federated Resnet20 Client Side\n", 8 | "This code is the server part of CIFAR10 federated resnet20 for **multi** client and a server." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 3, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "users = 1 # number of clients" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 4, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "import os\n", 27 | "import h5py\n", 28 | "\n", 29 | "import socket\n", 30 | "import struct\n", 31 | "import pickle\n", 32 | "\n", 33 | "import torch\n", 34 | "import torch.nn as nn\n", 35 | "import torch.nn.functional as F\n", 36 | "import torchvision\n", 37 | "import torchvision.transforms as transforms\n", 38 | "import torch.optim as optim\n", 39 | "\n", 40 | "from torch.utils.data import Dataset, DataLoader\n", 41 | "\n", 42 | "import time\n", 43 | "\n", 44 | "from tqdm import tqdm" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "def getFreeDescription():\n", 54 | " free = os.popen(\"free -h\")\n", 55 | " i = 0\n", 56 | " while True:\n", 57 | " i = i + 1\n", 58 | " line = free.readline()\n", 59 | " if i == 1:\n", 60 | " return (line.split()[0:7])\n", 61 | "\n", 62 | "\n", 63 | "def getFree():\n", 64 | " free = os.popen(\"free -h\")\n", 65 | " i = 0\n", 66 | " while True:\n", 67 | " i = i + 1\n", 68 | " line = free.readline()\n", 69 | " if i == 2:\n", 70 | " return (line.split()[0:7])\n", 71 | "\n", 72 | "from gpiozero import CPUTemperature\n", 73 | "\n", 74 | "\n", 75 | "def printPerformance():\n", 76 | " cpu = CPUTemperature()\n", 77 | "\n", 78 | " print(\"temperature: \" + str(cpu.temperature))\n", 79 | "\n", 80 | " description = getFreeDescription()\n", 81 | " mem = getFree()\n", 82 | "\n", 83 | " print(description[0] + \" : \" + mem[1])\n", 84 | " print(description[1] + \" : \" + mem[2])\n", 85 | " print(description[2] + \" : \" + mem[3])\n", 86 | " print(description[3] + \" : \" + mem[4])\n", 87 | " print(description[4] + \" : \" + mem[5])\n", 88 | " print(description[5] + \" : \" + mem[6])" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "printPerformance()" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 5, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "root_path = '../../models/cifar10_data'" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Cuda" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 6, 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "name": "stdout", 123 | "output_type": "stream", 124 | "text": [ 125 | "cpu\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 131 | "device = \"cpu\"\n", 132 | "print(device)" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 7, 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "client_order(start from 0): 0\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "client_order = int(input(\"client_order(start from 0): \"))" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 8, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "num_traindata = 50000 // users" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "## Data load" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 9, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n", 175 | "\n", 176 | "from torch.utils.data import Subset\n", 177 | "\n", 178 | "\n", 179 | "indices = list(range(50000))\n", 180 | "\n", 181 | "part_tr = indices[num_traindata * client_order : num_traindata * (client_order + 1)]\n" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 10, 187 | "metadata": {}, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../models/cifar10_data\\cifar-10-python.tar.gz\n" 194 | ] 195 | }, 196 | { 197 | "data": { 198 | "application/vnd.jupyter.widget-view+json": { 199 | "model_id": "62295362754042e980b2371d8d0c3390", 200 | "version_major": 2, 201 | "version_minor": 0 202 | }, 203 | "text/plain": [ 204 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" 205 | ] 206 | }, 207 | "metadata": {}, 208 | "output_type": "display_data" 209 | }, 210 | { 211 | "name": "stdout", 212 | "output_type": "stream", 213 | "text": [ 214 | "Extracting ../../models/cifar10_data\\cifar-10-python.tar.gz to ../../models/cifar10_data\n", 215 | "Files already downloaded and verified\n" 216 | ] 217 | } 218 | ], 219 | "source": [ 220 | "trainset = torchvision.datasets.CIFAR10 (root=root_path, train=True, download=True, transform=transform)\n", 221 | "\n", 222 | "trainset_sub = Subset(trainset, part_tr)\n", 223 | "\n", 224 | "train_loader = torch.utils.data.DataLoader(trainset_sub, batch_size=4, shuffle=True, num_workers=2)\n", 225 | "\n", 226 | "testset = torchvision.datasets.CIFAR10 (root=root_path, train=False, download=True, transform=transform)\n", 227 | "test_loader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 11, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [ 236 | "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')" 237 | ] 238 | }, 239 | { 240 | "cell_type": "markdown", 241 | "metadata": {}, 242 | "source": [ 243 | "### Number of total batches" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 13, 249 | "metadata": {}, 250 | "outputs": [ 251 | { 252 | "name": "stdout", 253 | "output_type": "stream", 254 | "text": [ 255 | "25\n", 256 | "2500\n" 257 | ] 258 | } 259 | ], 260 | "source": [ 261 | "train_total_batch = len(train_loader)\n", 262 | "print(train_total_batch)\n", 263 | "test_batch = len(test_loader)\n", 264 | "print(test_batch)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 14, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "from torch.autograd import Variable\n", 274 | "import torch.nn.init as init\n", 275 | "\n", 276 | "def _weights_init(m):\n", 277 | " classname = m.__class__.__name__\n", 278 | " #print(classname)\n", 279 | " if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):\n", 280 | " init.kaiming_normal_(m.weight)\n", 281 | "\n", 282 | "class LambdaLayer(nn.Module):\n", 283 | " def __init__(self, lambd):\n", 284 | " super(LambdaLayer, self).__init__()\n", 285 | " self.lambd = lambd\n", 286 | "\n", 287 | " def forward(self, x):\n", 288 | " return self.lambd(x)\n", 289 | "\n", 290 | "\n", 291 | "class BasicBlock(nn.Module):\n", 292 | " expansion = 1\n", 293 | "\n", 294 | " def __init__(self, in_planes, planes, stride=1, option='A'):\n", 295 | " super(BasicBlock, self).__init__()\n", 296 | " self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n", 297 | " self.bn1 = nn.BatchNorm2d(planes)\n", 298 | " self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n", 299 | " self.bn2 = nn.BatchNorm2d(planes)\n", 300 | "\n", 301 | " self.shortcut = nn.Sequential()\n", 302 | " if stride != 1 or in_planes != planes:\n", 303 | " if option == 'A':\n", 304 | " \"\"\"\n", 305 | " For CIFAR10 ResNet paper uses option A.\n", 306 | " \"\"\"\n", 307 | " self.shortcut = LambdaLayer(lambda x:\n", 308 | " F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), \"constant\", 0))\n", 309 | " elif option == 'B':\n", 310 | " self.shortcut = nn.Sequential(\n", 311 | " nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),\n", 312 | " nn.BatchNorm2d(self.expansion * planes)\n", 313 | " )\n", 314 | "\n", 315 | " def forward(self, x):\n", 316 | " out = F.relu(self.bn1(self.conv1(x)))\n", 317 | " out = self.bn2(self.conv2(out))\n", 318 | " out += self.shortcut(x)\n", 319 | " out = F.relu(out)\n", 320 | " return out\n", 321 | "\n", 322 | "\n", 323 | "class ResNet(nn.Module):\n", 324 | " def __init__(self, block, num_blocks, num_classes=10):\n", 325 | " super(ResNet, self).__init__()\n", 326 | " self.in_planes = 16\n", 327 | "\n", 328 | " self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n", 329 | " self.bn1 = nn.BatchNorm2d(16)\n", 330 | " self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)\n", 331 | " self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)\n", 332 | " self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)\n", 333 | " self.linear = nn.Linear(64, num_classes)\n", 334 | "\n", 335 | " self.apply(_weights_init)\n", 336 | "\n", 337 | " def _make_layer(self, block, planes, num_blocks, stride):\n", 338 | " strides = [stride] + [1]*(num_blocks-1)\n", 339 | " layers = []\n", 340 | " for stride in strides:\n", 341 | " layers.append(block(self.in_planes, planes, stride))\n", 342 | " self.in_planes = planes * block.expansion\n", 343 | "\n", 344 | " return nn.Sequential(*layers)\n", 345 | "\n", 346 | " def forward(self, x):\n", 347 | " out = F.relu(self.bn1(self.conv1(x)))\n", 348 | " out = self.layer1(out)\n", 349 | " out = self.layer2(out)\n", 350 | " out = self.layer3(out)\n", 351 | " out = F.avg_pool2d(out, out.size()[3])\n", 352 | " out = out.view(out.size(0), -1)\n", 353 | " out = self.linear(out)\n", 354 | " return out\n", 355 | "\n", 356 | "\n", 357 | "def resnet20():\n", 358 | " return ResNet(BasicBlock, [3, 3, 3])\n" 359 | ] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": 15, 364 | "metadata": {}, 365 | "outputs": [ 366 | { 367 | "data": { 368 | "text/plain": [ 369 | "MobileNet(\n", 370 | " (classifer): Sequential(\n", 371 | " (0): Dropout(p=0.5, inplace=False)\n", 372 | " (1): Linear(in_features=1024, out_features=10, bias=True)\n", 373 | " )\n", 374 | " (feature): Sequential(\n", 375 | " (0): Sequential(\n", 376 | " (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 377 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 378 | " (2): ReLU()\n", 379 | " )\n", 380 | " (1): Sequential(\n", 381 | " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n", 382 | " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 383 | " (2): ReLU()\n", 384 | " (3): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n", 385 | " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 386 | " (5): ReLU()\n", 387 | " )\n", 388 | " (2): Sequential(\n", 389 | " (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=64)\n", 390 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 391 | " (2): ReLU()\n", 392 | " (3): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n", 393 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 394 | " (5): ReLU()\n", 395 | " )\n", 396 | " (3): Sequential(\n", 397 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n", 398 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 399 | " (2): ReLU()\n", 400 | " (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n", 401 | " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 402 | " (5): ReLU()\n", 403 | " )\n", 404 | " (4): Sequential(\n", 405 | " (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=128)\n", 406 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 407 | " (2): ReLU()\n", 408 | " (3): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1))\n", 409 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 410 | " (5): ReLU()\n", 411 | " )\n", 412 | " (5): Sequential(\n", 413 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n", 414 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 415 | " (2): ReLU()\n", 416 | " (3): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", 417 | " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 418 | " (5): ReLU()\n", 419 | " )\n", 420 | " (6): Sequential(\n", 421 | " (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=256)\n", 422 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 423 | " (2): ReLU()\n", 424 | " (3): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1))\n", 425 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 426 | " (5): ReLU()\n", 427 | " )\n", 428 | " (7): Sequential(\n", 429 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 430 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 431 | " (2): ReLU()\n", 432 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 433 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 434 | " (5): ReLU()\n", 435 | " )\n", 436 | " (8): Sequential(\n", 437 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 438 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 439 | " (2): ReLU()\n", 440 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 441 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 442 | " (5): ReLU()\n", 443 | " )\n", 444 | " (9): Sequential(\n", 445 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 446 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 447 | " (2): ReLU()\n", 448 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 449 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 450 | " (5): ReLU()\n", 451 | " )\n", 452 | " (10): Sequential(\n", 453 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 454 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 455 | " (2): ReLU()\n", 456 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 457 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 458 | " (5): ReLU()\n", 459 | " )\n", 460 | " (11): Sequential(\n", 461 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n", 462 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 463 | " (2): ReLU()\n", 464 | " (3): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n", 465 | " (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 466 | " (5): ReLU()\n", 467 | " )\n", 468 | " (12): Sequential(\n", 469 | " (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=512)\n", 470 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 471 | " (2): ReLU()\n", 472 | " (3): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 473 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 474 | " (5): ReLU()\n", 475 | " )\n", 476 | " (13): Sequential(\n", 477 | " (0): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1024)\n", 478 | " (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 479 | " (2): ReLU()\n", 480 | " (3): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1))\n", 481 | " (4): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 482 | " (5): ReLU()\n", 483 | " )\n", 484 | " )\n", 485 | ")" 486 | ] 487 | }, 488 | "execution_count": 15, 489 | "metadata": {}, 490 | "output_type": "execute_result" 491 | } 492 | ], 493 | "source": [ 494 | "res_net = resnet20()\n", 495 | "res_net.to(device)" 496 | ] 497 | }, 498 | { 499 | "cell_type": "code", 500 | "execution_count": 16, 501 | "metadata": {}, 502 | "outputs": [], 503 | "source": [ 504 | "lr = 0.001\n", 505 | "criterion = nn.CrossEntropyLoss()\n", 506 | "optimizer = optim.SGD(res_net.parameters(), lr=lr, momentum=0.9)\n", 507 | "\n", 508 | "rounds = 400 # default\n", 509 | "local_epochs = 1 # default" 510 | ] 511 | }, 512 | { 513 | "cell_type": "markdown", 514 | "metadata": {}, 515 | "source": [ 516 | "## Socket initialization\n", 517 | "### Required socket functions" 518 | ] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": 17, 523 | "metadata": {}, 524 | "outputs": [], 525 | "source": [ 526 | "def send_msg(sock, msg):\n", 527 | " # prefix each message with a 4-byte length in network byte order\n", 528 | " msg = pickle.dumps(msg)\n", 529 | " msg = struct.pack('>I', len(msg)) + msg\n", 530 | " sock.sendall(msg)\n", 531 | "\n", 532 | "def recv_msg(sock):\n", 533 | " # read message length and unpack it into an integer\n", 534 | " raw_msglen = recvall(sock, 4)\n", 535 | " if not raw_msglen:\n", 536 | " return None\n", 537 | " msglen = struct.unpack('>I', raw_msglen)[0]\n", 538 | " # read the message data\n", 539 | " msg = recvall(sock, msglen)\n", 540 | " msg = pickle.loads(msg)\n", 541 | " return msg\n", 542 | "\n", 543 | "def recvall(sock, n):\n", 544 | " # helper function to receive n bytes or return None if EOF is hit\n", 545 | " data = b''\n", 546 | " while len(data) < n:\n", 547 | " packet = sock.recv(n - len(data))\n", 548 | " if not packet:\n", 549 | " return None\n", 550 | " data += packet\n", 551 | " return data" 552 | ] 553 | }, 554 | { 555 | "cell_type": "code", 556 | "execution_count": 15, 557 | "metadata": {}, 558 | "outputs": [], 559 | "source": [ 560 | "printPerformance()" 561 | ] 562 | }, 563 | { 564 | "cell_type": "markdown", 565 | "metadata": {}, 566 | "source": [ 567 | "### Set host address and port number" 568 | ] 569 | }, 570 | { 571 | "cell_type": "code", 572 | "execution_count": 18, 573 | "metadata": {}, 574 | "outputs": [ 575 | { 576 | "name": "stdout", 577 | "output_type": "stream", 578 | "text": [ 579 | "IP address: 192.168.83.1\n" 580 | ] 581 | } 582 | ], 583 | "source": [ 584 | "host = input(\"IP address: \")\n", 585 | "port = 10080\n", 586 | "max_recv = 100000" 587 | ] 588 | }, 589 | { 590 | "cell_type": "markdown", 591 | "metadata": {}, 592 | "source": [ 593 | "### Open the client socket" 594 | ] 595 | }, 596 | { 597 | "cell_type": "code", 598 | "execution_count": 19, 599 | "metadata": {}, 600 | "outputs": [], 601 | "source": [ 602 | "s = socket.socket()\n", 603 | "s.connect((host, port))" 604 | ] 605 | }, 606 | { 607 | "cell_type": "markdown", 608 | "metadata": {}, 609 | "source": [ 610 | "## SET TIMER" 611 | ] 612 | }, 613 | { 614 | "cell_type": "code", 615 | "execution_count": 20, 616 | "metadata": {}, 617 | "outputs": [ 618 | { 619 | "name": "stdout", 620 | "output_type": "stream", 621 | "text": [ 622 | "timmer start!\n" 623 | ] 624 | } 625 | ], 626 | "source": [ 627 | "start_time = time.time() # store start time\n", 628 | "print(\"timmer start!\")" 629 | ] 630 | }, 631 | { 632 | "cell_type": "code", 633 | "execution_count": 21, 634 | "metadata": {}, 635 | "outputs": [], 636 | "source": [ 637 | "msg = recv_msg(s)\n", 638 | "rounds = msg['rounds'] \n", 639 | "client_id = msg['client_id']\n", 640 | "local_epochs = msg['local_epoch']\n", 641 | "send_msg(s, len(trainset_sub))" 642 | ] 643 | }, 644 | { 645 | "cell_type": "code", 646 | "execution_count": 22, 647 | "metadata": {}, 648 | "outputs": [ 649 | { 650 | "name": "stderr", 651 | "output_type": "stream", 652 | "text": [ 653 | "Round 1_1: 100%|████████████████████████████████████████████████████| 25/25 [00:11<00:00, 2.13it/s]" 654 | ] 655 | }, 656 | { 657 | "name": "stdout", 658 | "output_type": "stream", 659 | "text": [ 660 | "Finished Training\n" 661 | ] 662 | }, 663 | { 664 | "name": "stderr", 665 | "output_type": "stream", 666 | "text": [ 667 | "\n", 668 | "C:\\Users\\rlaal\\anaconda3\\envs\\py36\\lib\\site-packages\\torch\\storage.py:34: FutureWarning: pickle support for Storage will be removed in 1.5. Use `torch.save` instead\n", 669 | " warnings.warn(\"pickle support for Storage will be removed in 1.5. Use `torch.save` instead\", FutureWarning)\n" 670 | ] 671 | } 672 | ], 673 | "source": [ 674 | "# update weights from server\n", 675 | "# train\n", 676 | "for r in range(rounds): # loop over the dataset multiple times\n", 677 | "\n", 678 | " \n", 679 | " \n", 680 | " weights = recv_msg(s)\n", 681 | " res_net.load_state_dict(weights)\n", 682 | " res_net.eval()\n", 683 | " for local_epoch in range(local_epochs):\n", 684 | " \n", 685 | " for i, data in enumerate(tqdm(train_loader, ncols=100, desc='Round '+str(r+1)+'_'+str(local_epoch+1))):\n", 686 | " \n", 687 | " # get the inputs; data is a list of [inputs, labels]\n", 688 | " inputs, labels = data\n", 689 | " inputs = inputs.to(device)\n", 690 | " labels = labels.clone().detach().long().to(device)\n", 691 | "\n", 692 | " # zero the parameter gradients\n", 693 | " optimizer.zero_grad()\n", 694 | "\n", 695 | " # forward + backward + optimize\n", 696 | " outputs = res_net(inputs)\n", 697 | " loss = criterion(outputs, labels)\n", 698 | " loss.backward()\n", 699 | " optimizer.step()\n", 700 | "\n", 701 | " msg = res_net.state_dict()\n", 702 | " send_msg(s, msg)\n", 703 | "\n", 704 | "print('Finished Training')\n", 705 | "\n" 706 | ] 707 | }, 708 | { 709 | "cell_type": "code", 710 | "execution_count": null, 711 | "metadata": {}, 712 | "outputs": [], 713 | "source": [ 714 | "printPerformance()" 715 | ] 716 | }, 717 | { 718 | "cell_type": "code", 719 | "execution_count": 23, 720 | "metadata": {}, 721 | "outputs": [ 722 | { 723 | "name": "stdout", 724 | "output_type": "stream", 725 | "text": [ 726 | "Training Time: 12.0054190158844 sec\n" 727 | ] 728 | } 729 | ], 730 | "source": [ 731 | "end_time = time.time() #store end time\n", 732 | "print(\"Training Time: {} sec\".format(end_time - start_time))" 733 | ] 734 | }, 735 | { 736 | "cell_type": "code", 737 | "execution_count": null, 738 | "metadata": {}, 739 | "outputs": [], 740 | "source": [] 741 | } 742 | ], 743 | "metadata": { 744 | "kernelspec": { 745 | "display_name": "Python 3", 746 | "language": "python", 747 | "name": "python3" 748 | }, 749 | "language_info": { 750 | "codemirror_mode": { 751 | "name": "ipython", 752 | "version": 3 753 | }, 754 | "file_extension": ".py", 755 | "mimetype": "text/x-python", 756 | "name": "python", 757 | "nbconvert_exporter": "python", 758 | "pygments_lexer": "ipython3", 759 | "version": "3.6.10" 760 | } 761 | }, 762 | "nbformat": 4, 763 | "nbformat_minor": 2 764 | } 765 | --------------------------------------------------------------------------------