├── CIFAR ├── Keras │ ├── 1_cifar_conv.py │ └── 2_cifar_DA.py └── Pytorch │ ├── README.MD │ ├── cifar_cnn_1.py │ ├── cifar_cnn_2_da_lra.py │ └── cifar_cnn_3_resnet_block.py ├── MNIST ├── Keras │ ├── 1_mlp_basic.py │ ├── 2_mlp_batchnorm.py │ ├── 3_mlp_BN_GN.py │ ├── 4_mlp_BN_GN_LRA.py │ └── 5_mlp_BN_GN_LRA_DA.py └── Pytorch │ ├── mnist_mlp_1.py │ ├── mnist_mlp_1_no_api.py │ ├── mnist_mlp_1_sequential.py │ ├── mnist_mlp_2_bn.py │ ├── mnist_mlp_3_lra.py │ └── mnist_mlp_4_da.py └── README.md /CIFAR/Keras/1_cifar_conv.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import cifar10 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Dropout, Activation, Flatten 7 | from keras.layers import Conv2D, MaxPooling2D 8 | from keras.layers import BatchNormalization as BN 9 | from keras.layers import GaussianNoise as GN 10 | from tensorflow.keras.optimizers import SGD 11 | 12 | batch_size = 100 13 | num_classes = 10 14 | epochs = 75 15 | 16 | 17 | #### LOAD AND TRANSFORM 18 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 19 | 20 | x_train = x_train.astype('float32') 21 | x_test = x_test.astype('float32') 22 | 23 | x_train /= 255 24 | x_test /= 255 25 | 26 | print(x_train.shape) 27 | print(x_test.shape) 28 | 29 | y_train = keras.utils.to_categorical(y_train, num_classes) 30 | y_test = keras.utils.to_categorical(y_test, num_classes) 31 | 32 | 33 | 34 | ## DEF A BLOCK CONV + BN + GN + MAXPOOL 35 | def CBGN(model,filters,ishape=0): 36 | if (ishape!=0): 37 | model.add(Conv2D(filters, (3, 3), padding='same', 38 | input_shape=ishape)) 39 | else: 40 | model.add(Conv2D(filters, (3, 3), padding='same')) 41 | 42 | model.add(BN()) 43 | model.add(GN(0.3)) 44 | model.add(Activation('relu')) 45 | model.add(MaxPooling2D(pool_size=(2, 2))) 46 | 47 | 48 | return model 49 | 50 | 51 | ## DEF NN TOPOLOGY 52 | model = Sequential() 53 | 54 | model=CBGN(model,32,x_train.shape[1:]) 55 | model=CBGN(model,64) 56 | model=CBGN(model,128) 57 | model=CBGN(model,256) 58 | model=CBGN(model,512) 59 | 60 | model.add(Flatten()) 61 | model.add(Dense(512)) 62 | model.add(Activation('relu')) 63 | 64 | model.add(Dense(num_classes)) 65 | model.add(Activation('softmax')) 66 | 67 | 68 | model.summary() 69 | 70 | 71 | ## OPTIM AND COMPILE 72 | opt = SGD(learning_rate=0.1) 73 | 74 | model.compile(loss='categorical_crossentropy', 75 | optimizer=opt, 76 | metrics=['accuracy']) 77 | 78 | 79 | ## TRAINING 80 | model.fit(x_train, y_train, 81 | batch_size=batch_size, 82 | epochs=epochs, 83 | validation_data=(x_test, y_test), 84 | shuffle=True) 85 | 86 | ## TEST 87 | scores = model.evaluate(x_test, y_test, verbose=1) 88 | print('Test loss:', scores[0]) 89 | print('Test accuracy:', scores[1]) 90 | 91 | -------------------------------------------------------------------------------- /CIFAR/Keras/2_cifar_DA.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import cifar10 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Dropout, Activation, Flatten 7 | from keras.layers import Conv2D, MaxPooling2D 8 | from keras.layers import BatchNormalization as BN 9 | from keras.layers import GaussianNoise as GN 10 | from tensorflow.keras.optimizers import SGD 11 | 12 | from keras.callbacks import LearningRateScheduler as LRS 13 | from keras.preprocessing.image import ImageDataGenerator 14 | 15 | 16 | batch_size = 100 17 | num_classes = 10 18 | epochs = 75 19 | 20 | 21 | #### LOAD AND TRANSFORM 22 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 23 | 24 | x_train = x_train.astype('float32') 25 | x_test = x_test.astype('float32') 26 | 27 | x_train /= 255 28 | x_test /= 255 29 | 30 | print(x_train.shape) 31 | print(x_test.shape) 32 | 33 | y_train = keras.utils.to_categorical(y_train, num_classes) 34 | y_test = keras.utils.to_categorical(y_test, num_classes) 35 | 36 | 37 | 38 | ## DEFINE A DATA AUGMENTATION GENERATOR 39 | 40 | datagen = ImageDataGenerator( 41 | width_shift_range=0.2, 42 | height_shift_range=0.2, 43 | horizontal_flip=True) 44 | 45 | 46 | ## DEF A BLOCK CONV + BN + GN + MAXPOOL 47 | def CBGN(model,filters,ishape=0): 48 | if (ishape!=0): 49 | model.add(Conv2D(filters, (3, 3), padding='same', 50 | input_shape=ishape)) 51 | else: 52 | model.add(Conv2D(filters, (3, 3), padding='same')) 53 | 54 | model.add(BN()) 55 | model.add(GN(0.3)) 56 | model.add(Activation('relu')) 57 | model.add(MaxPooling2D(pool_size=(2, 2))) 58 | 59 | 60 | return model 61 | 62 | 63 | ## DEF NN TOPOLOGY 64 | model = Sequential() 65 | 66 | model=CBGN(model,32,x_train.shape[1:]) 67 | model=CBGN(model,64) 68 | model=CBGN(model,128) 69 | model=CBGN(model,256) 70 | model=CBGN(model,512) 71 | 72 | model.add(Flatten()) 73 | model.add(Dense(512)) 74 | model.add(Activation('relu')) 75 | 76 | model.add(Dense(num_classes)) 77 | model.add(Activation('softmax')) 78 | 79 | 80 | model.summary() 81 | 82 | 83 | ## OPTIM AND COMPILE 84 | opt = SGD(learning_rate=0.1) 85 | 86 | model.compile(loss='categorical_crossentropy', 87 | optimizer=opt, 88 | metrics=['accuracy']) 89 | 90 | # DEFINE A LEARNING RATE SCHEDULER 91 | def scheduler(epoch): 92 | if epoch < 25: 93 | return .1 94 | elif epoch < 50: 95 | return 0.01 96 | else: 97 | return 0.001 98 | 99 | set_lr = LRS(scheduler) 100 | 101 | 102 | ## TRAINING with DA and LRA 103 | history=model.fit(datagen.flow(x_train, y_train,batch_size=batch_size), 104 | steps_per_epoch=len(x_train) / batch_size, 105 | epochs=epochs, 106 | validation_data=(x_test, y_test), 107 | callbacks=[set_lr], 108 | verbose=1) 109 | 110 | 111 | ## TEST 112 | scores = model.evaluate(x_test, y_test, verbose=1) 113 | print('Test loss:', scores[0]) 114 | print('Test accuracy:', scores[1]) 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /CIFAR/Pytorch/README.MD: -------------------------------------------------------------------------------- 1 | Pytorch version for CIFAR -------------------------------------------------------------------------------- /CIFAR/Pytorch/cifar_cnn_1.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torch.nn as nn 4 | from tqdm import tqdm 5 | import multiprocessing 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torch.utils.data import Dataset 9 | from torch.utils.data import DataLoader 10 | 11 | print("Torch version: ", torch. __version__) 12 | 13 | #################################################################### 14 | # Set Device 15 | #################################################################### 16 | 17 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 18 | print("Device: ", device) 19 | 20 | 21 | #################################################################### 22 | # Dataset Class 23 | #################################################################### 24 | 25 | class CIFAR10_dataset(Dataset): 26 | 27 | def __init__(self, partition = "train"): 28 | 29 | print("\nLoading CIFAR10 ", partition, " Dataset...") 30 | self.partition = partition 31 | if self.partition == "train": 32 | self.data = torchvision.datasets.CIFAR10('.data/', 33 | train=True, 34 | download=True) 35 | else: 36 | self.data = torchvision.datasets.CIFAR10('.data/', 37 | train=False, 38 | download=True) 39 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 40 | 41 | def from_pil_to_tensor(self, image): 42 | return torchvision.transforms.ToTensor()(image) 43 | 44 | def __len__(self): 45 | return len(self.data) 46 | 47 | def __getitem__(self, idx): 48 | 49 | # Image 50 | image = self.data[idx][0] 51 | # PIL Image to torch tensor 52 | image_tensor = self.from_pil_to_tensor(image) 53 | 54 | # Label 55 | label = torch.tensor(self.data[idx][1]) 56 | label = F.one_hot(label, num_classes=10).float() 57 | 58 | return {"img": image_tensor, "label": label} 59 | 60 | train_dataset = CIFAR10_dataset(partition="train") 61 | test_dataset = CIFAR10_dataset(partition="test") 62 | 63 | #################################################################### 64 | # DataLoader Class 65 | #################################################################### 66 | 67 | batch_size = 100 68 | num_workers = multiprocessing.cpu_count()-1 69 | print("Num workers", num_workers) 70 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 71 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 72 | 73 | #################################################################### 74 | # Neural Network Class 75 | #################################################################### 76 | 77 | # Define the CNN model 78 | class SimpleCNN(nn.Module): 79 | def __init__(self, num_classes=10): 80 | super(SimpleCNN, self).__init__() 81 | 82 | self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1, padding=1) 83 | self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1) 84 | self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1) 85 | self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1) 86 | self.conv5 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1) 87 | 88 | self.bn1 = nn.BatchNorm2d(32) 89 | self.bn2 = nn.BatchNorm2d(64) 90 | self.bn3 = nn.BatchNorm2d(128) 91 | self.bn4 = nn.BatchNorm2d(256) 92 | self.bn5 = nn.BatchNorm2d(512) 93 | 94 | self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2) 95 | 96 | self.fc1 = nn.Linear(512, 512) 97 | self.fc2 = nn.Linear(512, num_classes) 98 | self.relu = nn.ReLU() 99 | 100 | def forward(self, x): 101 | 102 | x = self.relu(self.bn1(self.conv1(x))) 103 | x = self.maxpool(x) 104 | x = self.relu(self.bn2(self.conv2(x))) 105 | x = self.maxpool(x) 106 | x = self.relu(self.bn3(self.conv3(x))) 107 | x = self.maxpool(x) 108 | x = self.relu(self.bn4(self.conv4(x))) 109 | x = self.maxpool(x) 110 | x = self.relu(self.bn5(self.conv5(x))) 111 | x = self.maxpool(x) 112 | 113 | x = torch.flatten(x, start_dim=1) 114 | x = self.relu(self.fc1(x)) 115 | x = self.fc2(x) 116 | return x 117 | 118 | 119 | # Instantiating the network and printing its architecture 120 | num_classes = 10 121 | net = SimpleCNN(num_classes) 122 | print(net) 123 | 124 | def count_parameters(model): 125 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 126 | print("Params: ", count_parameters(net)) 127 | 128 | #################################################################### 129 | # Training settings 130 | #################################################################### 131 | 132 | # Training hyperparameters 133 | criterion = nn.CrossEntropyLoss() 134 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 135 | epochs = 25 136 | 137 | 138 | #################################################################### 139 | # Training 140 | #################################################################### 141 | 142 | # Load model in GPU 143 | net.to(device) 144 | 145 | print("\n---- Start Training ----") 146 | best_accuracy = -1 147 | best_epoch = 0 148 | for epoch in range(epochs): 149 | 150 | 151 | # TRAIN NETWORK 152 | train_loss, train_correct = 0, 0 153 | net.train() 154 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 155 | for batch in tepoch: 156 | 157 | # Returned values of Dataset Class 158 | images = batch["img"].to(device) 159 | labels = batch["label"].to(device) 160 | 161 | # zero the parameter gradients 162 | optimizer.zero_grad() 163 | 164 | # Forward 165 | outputs = net(images) 166 | loss = criterion(outputs, labels) 167 | 168 | # Calculate gradients 169 | loss.backward() 170 | 171 | # Update gradients 172 | optimizer.step() 173 | 174 | # one hot -> labels 175 | labels = torch.argmax(labels, dim=1) 176 | pred = torch.argmax(outputs, dim=1) 177 | train_correct += pred.eq(labels).sum().item() 178 | 179 | # print statistics 180 | train_loss += loss.item() 181 | 182 | train_loss /= (len(train_dataloader.dataset) / batch_size) 183 | 184 | # TEST NETWORK 185 | test_loss, test_correct = 0, 0 186 | net.eval() 187 | with torch.no_grad(): 188 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 189 | for batch in tepoch: 190 | 191 | images = batch["img"].to(device) 192 | labels = batch["label"].to(device) 193 | 194 | # Forward 195 | outputs = net(images) 196 | test_loss += criterion(outputs, labels) 197 | 198 | # one hot -> labels 199 | labels = torch.argmax(labels, dim=1) 200 | pred = torch.argmax(outputs, dim=1) 201 | 202 | test_correct += pred.eq(labels).sum().item() 203 | 204 | test_loss /= (len(test_dataloader.dataset) / batch_size) 205 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 206 | 207 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 208 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 209 | )) 210 | 211 | if test_accuracy > best_accuracy: 212 | best_accuracy = test_accuracy 213 | best_epoch = epoch 214 | 215 | # Save best weights 216 | torch.save(net.state_dict(), "best_model.pt") 217 | 218 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 219 | -------------------------------------------------------------------------------- /CIFAR/Pytorch/cifar_cnn_2_da_lra.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torch.nn as nn 4 | from tqdm import tqdm 5 | import multiprocessing 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torchvision import transforms 9 | from torch.utils.data import Dataset 10 | from torch.utils.data import DataLoader 11 | 12 | print("Torch version: ", torch. __version__) 13 | 14 | #################################################################### 15 | # Set Device 16 | #################################################################### 17 | 18 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 19 | print("Device: ", device) 20 | 21 | #################################################################### 22 | # Dataset Class 23 | #################################################################### 24 | 25 | da_train = transforms.Compose([ 26 | transforms.RandomHorizontalFlip(), 27 | transforms.RandomAffine(degrees=5, translate=(0.2, 0.2)), 28 | transforms.ToTensor(), 29 | ]) 30 | 31 | da_test = transforms.Compose([ 32 | transforms.ToTensor(), 33 | ]) 34 | 35 | class CIFAR10_dataset(Dataset): 36 | 37 | def __init__(self, transform, partition = "train"): 38 | 39 | print("\nLoading CIFAR10 ", partition, " Dataset...") 40 | self.partition = partition 41 | self.transform = transform 42 | if self.partition == "train": 43 | self.data = torchvision.datasets.CIFAR10('.data/', 44 | train=True, 45 | download=True) 46 | else: 47 | self.data = torchvision.datasets.CIFAR10('.data/', 48 | train=False, 49 | download=True) 50 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 51 | 52 | def __len__(self): 53 | return len(self.data) 54 | 55 | def __getitem__(self, idx): 56 | 57 | # Image 58 | image = self.data[idx][0] 59 | image_tensor = self.transform(image) 60 | 61 | # Label 62 | label = torch.tensor(self.data[idx][1]) 63 | label = F.one_hot(label, num_classes=10).float() 64 | 65 | return {"img": image_tensor, "label": label} 66 | 67 | train_dataset = CIFAR10_dataset(da_train, partition="train") 68 | test_dataset = CIFAR10_dataset(da_test, partition="test") 69 | 70 | #################################################################### 71 | # DataLoader Class 72 | #################################################################### 73 | 74 | batch_size = 100 75 | num_workers = multiprocessing.cpu_count()-1 76 | print("Num workers", num_workers) 77 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 78 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 79 | 80 | #################################################################### 81 | # Neural Network Class 82 | #################################################################### 83 | 84 | # Define ConvBlocks to create a CNN 85 | class ConvBlock(nn.Module): 86 | def __init__(self, in_channels, out_channels): 87 | super(ConvBlock, self).__init__() 88 | 89 | self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) 90 | self.bn = nn.BatchNorm2d(out_channels) 91 | self.relu = nn.ReLU() 92 | self.pool = nn.MaxPool2d(kernel_size=2, stride=2) 93 | 94 | def forward(self, x): 95 | 96 | x = self.conv(x) 97 | x = self.bn(x) 98 | x = self.relu(x) 99 | x = self.pool(x) 100 | 101 | return x 102 | 103 | # Define the CNN model 104 | class SimpleCNN(nn.Module): 105 | def __init__(self, num_classes=10): 106 | super(SimpleCNN, self).__init__() 107 | 108 | self.conv_layers = nn.Sequential( 109 | ConvBlock(3, 32), 110 | ConvBlock(32, 64), 111 | ConvBlock(64, 128), 112 | ConvBlock(128, 256), 113 | ConvBlock(256, 512) 114 | ) 115 | 116 | self.fc1 = nn.Linear(512, 512) 117 | self.fc2 = nn.Linear(512, num_classes) 118 | 119 | self.relu = nn.ReLU() 120 | 121 | def forward(self, x): 122 | for layer in self.conv_layers: 123 | x = layer(x) 124 | 125 | x = torch.flatten(x, start_dim=1) 126 | x = F.relu(self.fc1(x)) 127 | x = self.fc2(x) 128 | 129 | return x 130 | 131 | # Instantiating the network and printing its architecture 132 | num_classes = 10 133 | net = SimpleCNN( 134 | num_classes 135 | ) 136 | print(net) 137 | 138 | def count_parameters(model): 139 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 140 | print("Params: ", count_parameters(net)) 141 | 142 | #################################################################### 143 | # Training settings 144 | #################################################################### 145 | 146 | # Training hyperparameters 147 | criterion = nn.CrossEntropyLoss() 148 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 149 | lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=10, min_lr=0.00001) 150 | epochs = 100 151 | 152 | 153 | #################################################################### 154 | # Training 155 | #################################################################### 156 | 157 | # Load model in GPU 158 | net.to(device) 159 | 160 | print("\n---- Start Training ----") 161 | best_accuracy = -1 162 | best_epoch = 0 163 | for epoch in range(epochs): 164 | 165 | 166 | # TRAIN NETWORK 167 | train_loss, train_correct = 0, 0 168 | net.train() 169 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 170 | for batch in tepoch: 171 | 172 | # Returned values of Dataset Class 173 | images = batch["img"].to(device) 174 | labels = batch["label"].to(device) 175 | 176 | # zero the parameter gradients 177 | optimizer.zero_grad() 178 | 179 | # Forward 180 | outputs = net(images) 181 | loss = criterion(outputs, labels) 182 | 183 | # Calculate gradients 184 | loss.backward() 185 | 186 | # Update gradients 187 | optimizer.step() 188 | 189 | # one hot -> labels 190 | labels = torch.argmax(labels, dim=1) 191 | pred = torch.argmax(outputs, dim=1) 192 | train_correct += pred.eq(labels).sum().item() 193 | 194 | # print statistics 195 | train_loss += loss.item() 196 | 197 | train_loss /= (len(train_dataloader.dataset) / batch_size) 198 | 199 | # TEST NETWORK 200 | test_loss, test_correct = 0, 0 201 | net.eval() 202 | with torch.no_grad(): 203 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 204 | for batch in tepoch: 205 | 206 | images = batch["img"].to(device) 207 | labels = batch["label"].to(device) 208 | 209 | # Forward 210 | outputs = net(images) 211 | test_loss += criterion(outputs, labels) 212 | 213 | # one hot -> labels 214 | labels = torch.argmax(labels, dim=1) 215 | pred = torch.argmax(outputs, dim=1) 216 | 217 | test_correct += pred.eq(labels).sum().item() 218 | 219 | lr_scheduler.step(test_loss) 220 | 221 | test_loss /= (len(test_dataloader.dataset) / batch_size) 222 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 223 | 224 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 225 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 226 | )) 227 | 228 | if test_accuracy > best_accuracy: 229 | best_accuracy = test_accuracy 230 | best_epoch = epoch 231 | 232 | # Save best weights 233 | torch.save(net.state_dict(), "best_model.pt") 234 | 235 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 236 | -------------------------------------------------------------------------------- /CIFAR/Pytorch/cifar_cnn_3_resnet_block.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | #################################################################### 4 | # ResNet Block 5 | #################################################################### 6 | 7 | ''' 8 | Deep Residual Learning for Image Recognition 9 | https://arxiv.org/pdf/1512.03385.pdf 10 | ''' 11 | class ResNetBlock(nn.Module): 12 | def __init__(self, in_channels, out_channels): 13 | super(ResNetBlock, self).__init__() 14 | 15 | self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding='same') 16 | self.bn1 = nn.BatchNorm2d(out_channels) 17 | 18 | self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding='same') 19 | self.bn2 = nn.BatchNorm2d(out_channels) 20 | 21 | self.conv_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding='same') 22 | 23 | self.relu = nn.ReLU() 24 | 25 | def forward(self, x): 26 | previous_x = x 27 | 28 | out = self.conv1(x) 29 | out = self.bn1(out) 30 | out = self.relu(out) 31 | 32 | out = self.conv2(out) 33 | out = self.bn2(out) 34 | 35 | previous_x = self.conv_shortcut(previous_x) 36 | 37 | out += previous_x 38 | out = self.relu(out) 39 | 40 | return out -------------------------------------------------------------------------------- /MNIST/Keras/1_mlp_basic.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import mnist 5 | from keras.models import Sequential 6 | from keras.layers import Dense 7 | from tensorflow.keras.optimizers import SGD 8 | from keras.utils import np_utils 9 | 10 | batch_size = 100 11 | epochs = 25 12 | num_classes=10 13 | 14 | # the data, shuffled and split between train and test sets 15 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 16 | 17 | print('training set', x_train.shape) 18 | print('test set', x_test.shape) 19 | 20 | x_train = x_train.reshape(60000, 784) 21 | x_test = x_test.reshape(10000, 784) 22 | x_train = x_train.astype('float32') 23 | x_test = x_test.astype('float32') 24 | 25 | # Normalize [0..255]-->[0..1] 26 | x_train /= 255 27 | x_test /= 255 28 | 29 | # convert class vectors to binary class matrices 30 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 31 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 32 | 33 | # A three hidden layer of 1024 34 | model = Sequential() 35 | 36 | model.add(Dense(1024, activation='relu', input_shape=(784,))) 37 | model.add(Dense(1024, activation='relu')) 38 | model.add(Dense(1024, activation='relu')) 39 | 40 | model.add(Dense(num_classes, activation='softmax')) 41 | ##### 42 | 43 | model.summary() 44 | 45 | # Optimizer 46 | sgd=SGD(learning_rate=0.01, decay=1e-6, momentum=0.9) 47 | 48 | # Compile Model 49 | model.compile(loss='categorical_crossentropy', 50 | optimizer=sgd, 51 | metrics=['accuracy']) 52 | 53 | # Training 54 | history = model.fit(x_train, y_train, 55 | batch_size=batch_size, 56 | epochs=epochs, 57 | verbose=1, 58 | validation_data=(x_test, y_test)) 59 | 60 | # Evaluate over test 61 | score = model.evaluate(x_test, y_test, verbose=0) 62 | 63 | 64 | print('Test loss:', score[0]) 65 | print('Test accuracy:', score[1]) 66 | -------------------------------------------------------------------------------- /MNIST/Keras/2_mlp_batchnorm.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import mnist 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Activation 7 | from keras.layers import BatchNormalization as BN 8 | from tensorflow.keras.optimizers import SGD 9 | from keras.utils import np_utils 10 | 11 | batch_size = 100 12 | epochs = 25 13 | num_classes=10 14 | 15 | # the data, shuffled and split between train and test sets 16 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 17 | 18 | print('training set', x_train.shape) 19 | print('test set', x_test.shape) 20 | 21 | x_train = x_train.reshape(60000, 784) 22 | x_test = x_test.reshape(10000, 784) 23 | x_train = x_train.astype('float32') 24 | x_test = x_test.astype('float32') 25 | 26 | # Normalize [0..255]-->[0..1] 27 | x_train /= 255 28 | x_test /= 255 29 | 30 | # convert class vectors to binary class matrices 31 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 32 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 33 | 34 | 35 | ## NN with BN 36 | model = Sequential() 37 | model.add(Dense(1024, input_shape=(784,))) 38 | model.add(BN()) 39 | model.add(Activation('relu')) 40 | 41 | model.add(Dense(1024)) 42 | model.add(BN()) 43 | model.add(Activation('relu')) 44 | 45 | model.add(Dense(1024)) 46 | model.add(BN()) 47 | model.add(Activation('relu')) 48 | 49 | model.add(Dense(num_classes, activation='softmax')) 50 | ## 51 | 52 | model.summary() 53 | 54 | 55 | ################################ 56 | # Note the higher learning rate 57 | ################################ 58 | sgd=SGD(learning_rate=0.1, decay=1e-6, momentum=0.9) 59 | 60 | model.compile(loss='categorical_crossentropy', 61 | optimizer=sgd, 62 | metrics=['accuracy']) 63 | 64 | history = model.fit(x_train, y_train, 65 | batch_size=batch_size, 66 | epochs=epochs, 67 | verbose=1, 68 | validation_data=(x_test, y_test)) 69 | 70 | score = model.evaluate(x_test, y_test, verbose=0) 71 | 72 | print('Test loss:', score[0]) 73 | print('Test accuracy:', score[1]) 74 | -------------------------------------------------------------------------------- /MNIST/Keras/3_mlp_BN_GN.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import mnist 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Activation, Reshape 7 | from keras.layers import BatchNormalization as BN 8 | from tensorflow.keras.optimizers import SGD 9 | from keras.utils import np_utils 10 | from keras.layers import GaussianNoise as GN 11 | 12 | batch_size = 100 13 | epochs = 25 14 | num_classes=10 15 | 16 | # the data, shuffled and split between train and test sets 17 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 18 | 19 | print('training set', x_train.shape) 20 | print('test set', x_test.shape) 21 | 22 | x_train = x_train.reshape(60000, 784) 23 | x_test = x_test.reshape(10000, 784) 24 | x_train = x_train.astype('float32') 25 | x_test = x_test.astype('float32') 26 | 27 | # Normalize [0..255]-->[0..1] 28 | x_train /= 255 29 | x_test /= 255 30 | 31 | # convert class vectors to binary class matrices 32 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 33 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 34 | 35 | # NN with BN + Gaussian Noise 36 | model = Sequential() 37 | ## noise in input 38 | model.add(Reshape(target_shape=(784,), input_shape=(784,))) 39 | model.add(GN(0.3)) 40 | 41 | model.add(Dense(1024, input_shape=(784,))) 42 | model.add(BN()) 43 | model.add(GN(0.3)) 44 | model.add(Activation('relu')) 45 | 46 | model.add(Dense(1024)) 47 | model.add(BN()) 48 | model.add(GN(0.3)) 49 | model.add(Activation('relu')) 50 | 51 | model.add(Dense(1024)) 52 | model.add(BN()) 53 | model.add(GN(0.3)) 54 | model.add(Activation('relu')) 55 | model.add(Dense(num_classes, activation='softmax')) 56 | ## 57 | 58 | model.summary() 59 | 60 | sgd=SGD(learning_rate=0.1, decay=1e-6, momentum=0.9) 61 | 62 | model.compile(loss='categorical_crossentropy', 63 | optimizer=sgd, 64 | metrics=['accuracy']) 65 | 66 | history = model.fit(x_train, y_train, 67 | batch_size=batch_size, 68 | epochs=epochs, 69 | verbose=1, 70 | validation_data=(x_test, y_test)) 71 | 72 | score = model.evaluate(x_test, y_test, verbose=0) 73 | 74 | print('Test loss:', score[0]) 75 | print('Test accuracy:', score[1]) -------------------------------------------------------------------------------- /MNIST/Keras/4_mlp_BN_GN_LRA.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import keras 4 | from keras.datasets import mnist 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Activation, Reshape 7 | from keras.layers import BatchNormalization as BN 8 | from tensorflow.keras.optimizers import SGD 9 | from keras.utils import np_utils 10 | from keras.layers import GaussianNoise as GN 11 | 12 | from keras.callbacks import LearningRateScheduler as LRS 13 | 14 | 15 | batch_size = 100 16 | epochs = 75 17 | num_classes=10 18 | 19 | # the data, shuffled and split between train and test sets 20 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 21 | 22 | print('training set', x_train.shape) 23 | print('test set', x_test.shape) 24 | 25 | x_train = x_train.reshape(60000, 784) 26 | x_test = x_test.reshape(10000, 784) 27 | x_train = x_train.astype('float32') 28 | x_test = x_test.astype('float32') 29 | 30 | # Normalize [0..255]-->[0..1] 31 | x_train /= 255 32 | x_test /= 255 33 | 34 | # convert class vectors to binary class matrices 35 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 36 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 37 | 38 | 39 | 40 | model = Sequential() 41 | model.add(Reshape(target_shape=(784,), input_shape=(784,))) 42 | model.add(GN(0.3)) 43 | 44 | model.add(Dense(1024)) 45 | model.add(BN()) 46 | model.add(GN(0.3)) 47 | model.add(Activation('relu')) 48 | 49 | model.add(Dense(1024)) 50 | model.add(BN()) 51 | model.add(GN(0.3)) 52 | model.add(Activation('relu')) 53 | 54 | model.add(Dense(1024)) 55 | model.add(BN()) 56 | model.add(GN(0.3)) 57 | model.add(Activation('relu')) 58 | 59 | model.add(Dense(num_classes, activation='softmax')) 60 | 61 | model.summary() 62 | 63 | 64 | sgd=SGD(learning_rate=0.1, decay=0.0, momentum=0.0) 65 | 66 | ## define a learning rate scheduler 67 | def scheduler(epoch): 68 | if epoch < 25: 69 | return .1 70 | elif epoch < 50: 71 | return 0.01 72 | else: 73 | return 0.001 74 | 75 | set_lr = LRS(scheduler) 76 | ### 77 | 78 | model.compile(loss='categorical_crossentropy', 79 | optimizer=sgd, 80 | metrics=['accuracy']) 81 | 82 | 83 | ## Training, note the callbacks list 84 | history = model.fit(x_train, y_train, 85 | batch_size=batch_size, 86 | epochs=epochs, 87 | verbose=1, 88 | validation_data=(x_test, y_test), 89 | callbacks=[set_lr]) 90 | 91 | ## Test 92 | score = model.evaluate(x_test, y_test, verbose=0) 93 | 94 | print('Test loss:', score[0]) 95 | print('Test accuracy:', score[1]) 96 | 97 | 98 | import matplotlib.pyplot as plt 99 | 100 | plt.plot(history.history['accuracy']) 101 | plt.plot(history.history['val_accuracy']) 102 | plt.title('model accuracy') 103 | plt.ylabel('accuracy') 104 | plt.xlabel('epoch') 105 | plt.legend(['train', 'test'], loc='upper left') 106 | plt.show() 107 | 108 | -------------------------------------------------------------------------------- /MNIST/Keras/5_mlp_BN_GN_LRA_DA.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | 4 | import keras 5 | from keras.datasets import mnist 6 | from keras.models import Sequential 7 | from keras.layers import Dense, Activation, Reshape 8 | from keras.layers import BatchNormalization as BN 9 | from tensorflow.keras.optimizers import SGD 10 | from keras.utils import np_utils 11 | from keras.layers import GaussianNoise as GN 12 | 13 | from keras.callbacks import LearningRateScheduler as LRS 14 | 15 | from keras.preprocessing.image import ImageDataGenerator 16 | 17 | 18 | batch_size = 100 19 | epochs = 75 20 | num_classes=10 21 | 22 | # the data, shuffled and split between train and test sets 23 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 24 | 25 | print('training set', x_train.shape) 26 | print('test set', x_test.shape) 27 | 28 | x_train = x_train.reshape(60000, 28,28,1) 29 | x_test = x_test.reshape(10000, 28,28,1) 30 | 31 | x_train = x_train.astype('float32') 32 | x_test = x_test.astype('float32') 33 | 34 | # Normalize [0..255]-->[0..1] 35 | x_train /= 255 36 | x_test /= 255 37 | 38 | # convert class vectors to binary class matrices 39 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 40 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 41 | 42 | 43 | ## Data Augmentation with an ImageGenerator 44 | datagen = ImageDataGenerator( 45 | width_shift_range=0.1, 46 | height_shift_range=0.1, 47 | horizontal_flip=False) 48 | 49 | 50 | ## Model, note the reshape 51 | model = Sequential() 52 | model.add(Reshape(target_shape=(784,), input_shape=(28,28,1))) 53 | model.add(GN(0.3)) 54 | 55 | model.add(Dense(1024)) 56 | model.add(BN()) 57 | model.add(GN(0.3)) 58 | model.add(Activation('relu')) 59 | 60 | model.add(Dense(1024)) 61 | model.add(BN()) 62 | model.add(GN(0.3)) 63 | model.add(Activation('relu')) 64 | 65 | model.add(Dense(1024)) 66 | model.add(BN()) 67 | model.add(GN(0.3)) 68 | model.add(Activation('relu')) 69 | 70 | model.add(Dense(num_classes, activation='softmax')) 71 | 72 | model.summary() 73 | ## 74 | 75 | sgd=SGD(learning_rate=0.1, decay=0.0, momentum=0.0) 76 | 77 | def scheduler(epoch): 78 | if epoch < 25: 79 | return .1 80 | elif epoch < 50: 81 | return 0.01 82 | else: 83 | return 0.001 84 | 85 | set_lr = LRS(scheduler) 86 | 87 | 88 | model.compile(loss='categorical_crossentropy', 89 | optimizer=sgd, 90 | metrics=['accuracy']) 91 | 92 | 93 | 94 | history=model.fit(datagen.flow(x_train, y_train,batch_size=batch_size), 95 | steps_per_epoch=len(x_train) / batch_size, 96 | epochs=epochs, 97 | validation_data=(x_test, y_test), 98 | callbacks=[set_lr], 99 | verbose=1) 100 | 101 | 102 | score = model.evaluate(x_test, y_test, verbose=0) 103 | 104 | print('Test loss:', score[0]) 105 | print('Test accuracy:', score[1]) 106 | 107 | import matplotlib.pyplot as plt 108 | 109 | plt.plot(history.history['accuracy']) 110 | plt.plot(history.history['val_accuracy']) 111 | plt.title('model accuracy') 112 | plt.ylabel('accuracy') 113 | plt.xlabel('epoch') 114 | plt.legend(['train', 'test'], loc='upper left') 115 | plt.show() 116 | -------------------------------------------------------------------------------- /MNIST/Pytorch/mnist_mlp_1.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import pandas as pd 4 | import torch.nn as nn 5 | from tqdm import tqdm 6 | import multiprocessing 7 | import torch.optim as optim 8 | import torch.nn.functional as F 9 | from torch.utils.data import Dataset 10 | from torch.utils.data import DataLoader 11 | 12 | print("Torch version: ", torch. __version__) 13 | 14 | #################################################################### 15 | # Set Device 16 | #################################################################### 17 | 18 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 19 | print("Device: ", device) 20 | 21 | 22 | #################################################################### 23 | # Prepare Data 24 | #################################################################### 25 | 26 | train_set = torchvision.datasets.MNIST('.data/', train=True, download=True) 27 | #train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True) 28 | 29 | test_set = torchvision.datasets.MNIST('.data/', train=False, download=True) 30 | #test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=True) 31 | 32 | print("Train images: ", train_set) 33 | print("Image: ", train_set[0][0]) 34 | print("Label: ", train_set[0][1]) 35 | print("Label one hot: ", F.one_hot(torch.tensor(train_set[0][1]), num_classes=10)) 36 | 37 | 38 | #################################################################### 39 | # Dataset Class 40 | #################################################################### 41 | 42 | class MNIST_dataset(Dataset): 43 | 44 | def __init__(self, data, partition = "train"): 45 | 46 | print("\nLoading MNIST ", partition, " Dataset...") 47 | self.data = data 48 | self.partition = partition 49 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 50 | 51 | def __len__(self): 52 | return len(self.data) 53 | 54 | def from_pil_to_tensor(self, image): 55 | return torchvision.transforms.ToTensor()(image) 56 | 57 | def __getitem__(self, idx): 58 | 59 | # Image 60 | image = self.data[idx][0] 61 | # PIL Image to torch tensor 62 | image_tensor = self.from_pil_to_tensor(image) 63 | # care! net expect a 784 size vector and our dataset 64 | # provide 1x28x28 (channels, height, width) -> Reshape! 65 | image_tensor = image_tensor.view(-1) 66 | 67 | # Label 68 | label = torch.tensor(self.data[idx][1]) 69 | label = F.one_hot(label, num_classes=10).float() 70 | 71 | return {"img": image_tensor, "label": label} 72 | 73 | train_dataset = MNIST_dataset(train_set, partition="train") 74 | test_dataset = MNIST_dataset(test_set, partition="test") 75 | 76 | 77 | #################################################################### 78 | # DataLoader Class 79 | #################################################################### 80 | 81 | batch_size = 100 82 | num_workers = multiprocessing.cpu_count()-1 83 | print("Num workers", num_workers) 84 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 85 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 86 | 87 | 88 | #################################################################### 89 | # Neural Network Class 90 | #################################################################### 91 | 92 | # Creating our Neural Network - Fully Connected 93 | class Net(nn.Module): 94 | def __init__(self, num_classes): 95 | super(Net, self).__init__() 96 | self.linear1 = nn.Linear(784, 1024) 97 | self.relu1 = nn.ReLU() 98 | self.linear2 = nn.Linear(1024, 1024) 99 | self.relu2 = nn.ReLU() 100 | self.linear3 = nn.Linear(1024, 1024) 101 | self.relu3 = nn.ReLU() 102 | self.classifier = nn.Linear(1024, num_classes) 103 | 104 | def forward(self, x): 105 | out = self.relu1(self.linear1(x)) 106 | out = self.relu2(self.linear2(out)) 107 | out = self.relu3(self.linear3(out)) 108 | out = self.classifier(out) 109 | return out 110 | 111 | 112 | # Instantiating the network and printing its architecture 113 | num_classes = 10 114 | net = Net(num_classes) 115 | print(net) 116 | 117 | def count_parameters(model): 118 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 119 | print("Params: ", count_parameters(net)) 120 | 121 | #################################################################### 122 | # Training settings 123 | #################################################################### 124 | 125 | # Training hyperparameters 126 | criterion = nn.CrossEntropyLoss() 127 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 128 | epochs = 25 129 | 130 | 131 | #################################################################### 132 | # Training 133 | #################################################################### 134 | 135 | # Load model in GPU 136 | net.to(device) 137 | 138 | print("\n---- Start Training ----") 139 | best_accuracy = -1 140 | best_epoch = 0 141 | for epoch in range(epochs): 142 | 143 | 144 | # TRAIN NETWORK 145 | train_loss, train_correct = 0, 0 146 | net.train() 147 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 148 | for batch in tepoch: 149 | 150 | # Returned values of Dataset Class 151 | images = batch["img"].to(device) 152 | labels = batch["label"].to(device) 153 | 154 | # zero the parameter gradients 155 | optimizer.zero_grad() 156 | 157 | # Forward 158 | outputs = net(images) 159 | loss = criterion(outputs, labels) 160 | 161 | # Calculate gradients 162 | loss.backward() 163 | 164 | # Update gradients 165 | optimizer.step() 166 | 167 | # one hot -> labels 168 | labels = torch.argmax(labels, dim=1) 169 | pred = torch.argmax(outputs, dim=1) 170 | train_correct += pred.eq(labels).sum().item() 171 | 172 | # print statistics 173 | train_loss += loss.item() 174 | 175 | train_loss /= len(train_dataloader.dataset) 176 | 177 | # TEST NETWORK 178 | test_loss, test_correct = 0, 0 179 | net.eval() 180 | with torch.no_grad(): 181 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 182 | for batch in tepoch: 183 | 184 | images = batch["img"].to(device) 185 | labels = batch["label"].to(device) 186 | 187 | # Forward 188 | outputs = net(images) 189 | test_loss += criterion(outputs, labels) 190 | 191 | # one hot -> labels 192 | labels = torch.argmax(labels, dim=1) 193 | pred = torch.argmax(outputs, dim=1) 194 | 195 | test_correct += pred.eq(labels).sum().item() 196 | 197 | test_loss /= len(test_dataloader.dataset) 198 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 199 | 200 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 201 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 202 | )) 203 | 204 | if test_accuracy > best_accuracy: 205 | best_accuracy = test_accuracy 206 | best_epoch = epoch 207 | 208 | # Save best weights 209 | torch.save(net.state_dict(), "best_model.pt") 210 | 211 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 212 | 213 | 214 | 215 | #################################################################### 216 | # Load best weights 217 | #################################################################### 218 | 219 | # Load best weights 220 | net.load_state_dict(torch.load("best_model.pt")) 221 | 222 | test_loss, test_correct = 0, 0 223 | net.eval() 224 | with torch.no_grad(): 225 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 226 | for batch in tepoch: 227 | 228 | images = batch["img"].to(device) 229 | labels = batch["label"].to(device) 230 | 231 | # Forward 232 | outputs = net(images) 233 | test_loss += criterion(outputs, labels) 234 | 235 | # one hot -> labels 236 | labels = torch.argmax(labels, dim=1) 237 | pred = torch.argmax(outputs, dim=1) 238 | 239 | test_correct += pred.eq(labels).sum().item() 240 | 241 | test_loss /= len(test_dataloader.dataset) 242 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 243 | print("Final best acc: ", test_accuracy) 244 | -------------------------------------------------------------------------------- /MNIST/Pytorch/mnist_mlp_1_no_api.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import torch 3 | import struct 4 | import numpy as np 5 | import torchvision 6 | import pandas as pd 7 | import torch.nn as nn 8 | from tqdm import tqdm 9 | import multiprocessing 10 | import torch.optim as optim 11 | import torch.nn.functional as F 12 | from torch.utils.data import Dataset 13 | from torch.utils.data import DataLoader 14 | 15 | print("Torch version: ", torch. __version__) 16 | 17 | #################################################################### 18 | # Set Device 19 | #################################################################### 20 | 21 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 22 | print("Device: ", device) 23 | 24 | 25 | #################################################################### 26 | # Prepare Data 27 | #################################################################### 28 | 29 | 30 | # Dowload Data 31 | #!wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz 32 | #!wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz 33 | #!wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz 34 | #!wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz 35 | 36 | # load compressed MNIST gz files and return numpy arrays 37 | def load_data(filename, label=False): 38 | with gzip.open(filename) as gz: 39 | gz.read(4) 40 | n_items = struct.unpack('>I', gz.read(4)) 41 | if not label: 42 | n_rows = struct.unpack('>I', gz.read(4))[0] 43 | n_cols = struct.unpack('>I', gz.read(4))[0] 44 | res = np.frombuffer(gz.read(n_items[0] * n_rows * n_cols), dtype=np.uint8) 45 | res = res.reshape(n_items[0], n_rows, n_cols) 46 | else: 47 | res = np.frombuffer(gz.read(n_items[0]), dtype=np.uint8).tolist() 48 | return res 49 | 50 | train_images = torch.tensor(load_data('./content/train-images-idx3-ubyte.gz', label=False)) 51 | test_images = torch.tensor(load_data('./content/t10k-images-idx3-ubyte.gz', label=False)) 52 | 53 | train_labels = torch.tensor(load_data('./content/train-labels-idx1-ubyte.gz', label=True)) 54 | test_labels = torch.tensor(load_data('./content/t10k-labels-idx1-ubyte.gz', label=True)) 55 | 56 | print("Train images shape: ", train_images.shape) 57 | print("Test images shape: ", test_images.shape) 58 | print("Train labels shape: ", train_labels.shape) 59 | print("Train labels shape: ", test_labels.shape) 60 | 61 | 62 | train_labels = F.one_hot(train_labels, num_classes=10) 63 | test_labels = F.one_hot(test_labels, num_classes=10) 64 | 65 | print("Train labels shape: ", train_labels.shape) 66 | print("Train labels shape: ", test_labels.shape) 67 | 68 | 69 | #################################################################### 70 | # Dataset Class 71 | #################################################################### 72 | 73 | class MNIST_dataset(Dataset): 74 | 75 | def __init__(self, images, labels, partition = "train"): 76 | 77 | print("\nLoading MNIST ", partition, " Dataset...") 78 | self.image_name_list = images 79 | self.label_list = labels 80 | self.partition = partition 81 | print("\tTotal Len.: ", len(self.label_list), "\n", 50*"-") 82 | 83 | def __len__(self): 84 | return len(self.label_list) 85 | 86 | def from_pil_to_tensor(self, image): 87 | return torchvision.transforms.ToTensor()(image) 88 | 89 | def __getitem__(self, idx): 90 | 91 | # Image 92 | image_tensor = self.image_name_list[idx].float() / 255 93 | # care! net expect a 784 size vector and our dataset 94 | # provide 1x28x28 (channels, height, width) -> Reshape! 95 | image_tensor = image_tensor.view(-1) 96 | 97 | # Label 98 | label = self.label_list[idx].float() 99 | 100 | return {"img": image_tensor, "label": label} 101 | 102 | train_dataset = MNIST_dataset(train_images, train_labels, partition="train") 103 | test_dataset = MNIST_dataset(test_images, test_labels, partition="test") 104 | 105 | 106 | #################################################################### 107 | # DataLoader Class 108 | #################################################################### 109 | 110 | batch_size = 100 111 | num_workers = multiprocessing.cpu_count()-1 112 | print("Num workers", num_workers) 113 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 114 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 115 | 116 | 117 | #################################################################### 118 | # Neural Network Class 119 | #################################################################### 120 | 121 | # Creating our Neural Network - Fully Connected 122 | class Net(nn.Module): 123 | def __init__(self, num_classes): 124 | super(Net, self).__init__() 125 | self.linear1 = nn.Linear(784, 1024) 126 | self.relu1 = nn.ReLU() 127 | self.linear2 = nn.Linear(1024, 1024) 128 | self.relu2 = nn.ReLU() 129 | self.linear3 = nn.Linear(1024, 1024) 130 | self.relu3 = nn.ReLU() 131 | self.classifier = nn.Linear(1024, num_classes) 132 | 133 | def forward(self, x): 134 | out = self.relu1(self.linear1(x)) 135 | out = self.relu2(self.linear2(out)) 136 | out = self.relu3(self.linear3(out)) 137 | out = self.classifier(out) 138 | return out 139 | 140 | 141 | # Instantiating the network and printing its architecture 142 | num_classes = 10 143 | net = Net(num_classes) 144 | print(net) 145 | 146 | def count_parameters(model): 147 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 148 | print("Params: ", count_parameters(net)) 149 | 150 | #################################################################### 151 | # Training settings 152 | #################################################################### 153 | 154 | # Training hyperparameters 155 | criterion = nn.CrossEntropyLoss() 156 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 157 | epochs = 25 158 | 159 | 160 | #################################################################### 161 | # Training 162 | #################################################################### 163 | 164 | # Load model in GPU 165 | net.to(device) 166 | 167 | print("\n---- Start Training ----") 168 | best_accuracy = -1 169 | best_epoch = 0 170 | for epoch in range(epochs): 171 | 172 | 173 | # TRAIN NETWORK 174 | train_loss, train_correct = 0, 0 175 | net.train() 176 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 177 | for batch in tepoch: 178 | 179 | # Returned values of Dataset Class 180 | images = batch["img"].to(device) 181 | labels = batch["label"].to(device) 182 | 183 | # zero the parameter gradients 184 | optimizer.zero_grad() 185 | 186 | # Forward 187 | outputs = net(images) 188 | loss = criterion(outputs, labels) 189 | 190 | # Calculate gradients 191 | loss.backward() 192 | 193 | # Update gradients 194 | optimizer.step() 195 | 196 | # one hot -> labels 197 | labels = torch.argmax(labels, dim=1) 198 | pred = torch.argmax(outputs, dim=1) 199 | train_correct += pred.eq(labels).sum().item() 200 | 201 | # print statistics 202 | train_loss += loss.item() 203 | 204 | train_loss /= len(train_dataloader.dataset) 205 | 206 | # TEST NETWORK 207 | test_loss, test_correct = 0, 0 208 | net.eval() 209 | with torch.no_grad(): 210 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 211 | for batch in tepoch: 212 | 213 | images = batch["img"].to(device) 214 | labels = batch["label"].to(device) 215 | 216 | # Forward 217 | outputs = net(images) 218 | test_loss += criterion(outputs, labels) 219 | 220 | # one hot -> labels 221 | labels = torch.argmax(labels, dim=1) 222 | pred = torch.argmax(outputs, dim=1) 223 | 224 | test_correct += pred.eq(labels).sum().item() 225 | 226 | test_loss /= len(test_dataloader.dataset) 227 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 228 | 229 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 230 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 231 | )) 232 | 233 | if test_accuracy > best_accuracy: 234 | best_accuracy = test_accuracy 235 | best_epoch = epoch 236 | 237 | # Save best weights 238 | torch.save(net.state_dict(), "best_model.pt") 239 | 240 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 241 | 242 | 243 | 244 | #################################################################### 245 | # Load best weights 246 | #################################################################### 247 | 248 | # Load best weights 249 | net.load_state_dict(torch.load("best_model.pt")) 250 | 251 | test_loss, test_correct = 0, 0 252 | net.eval() 253 | with torch.no_grad(): 254 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 255 | for batch in tepoch: 256 | 257 | images = batch["img"].to(device) 258 | labels = batch["label"].to(device) 259 | 260 | # Forward 261 | outputs = net(images) 262 | test_loss += criterion(outputs, labels) 263 | 264 | # one hot -> labels 265 | labels = torch.argmax(labels, dim=1) 266 | pred = torch.argmax(outputs, dim=1) 267 | 268 | test_correct += pred.eq(labels).sum().item() 269 | 270 | test_loss /= len(test_dataloader.dataset) 271 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 272 | print("Final best acc: ", test_accuracy) 273 | -------------------------------------------------------------------------------- /MNIST/Pytorch/mnist_mlp_1_sequential.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import pandas as pd 4 | import torch.nn as nn 5 | from tqdm import tqdm 6 | import multiprocessing 7 | import torch.optim as optim 8 | import torch.nn.functional as F 9 | from torch.utils.data import Dataset 10 | from torch.utils.data import DataLoader 11 | 12 | print("Torch version: ", torch. __version__) 13 | 14 | #################################################################### 15 | # Set Device 16 | #################################################################### 17 | 18 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 19 | print("Device: ", device) 20 | 21 | 22 | #################################################################### 23 | # Prepare Data 24 | #################################################################### 25 | 26 | train_set = torchvision.datasets.MNIST('.data/', train=True, download=True) 27 | #train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True) 28 | 29 | test_set = torchvision.datasets.MNIST('.data/', train=False, download=True) 30 | #test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=True) 31 | 32 | print("Train images: ", train_set) 33 | print("Image: ", train_set[0][0]) 34 | print("Label: ", train_set[0][1]) 35 | print("Label one hot: ", F.one_hot(torch.tensor(train_set[0][1]), num_classes=10)) 36 | 37 | 38 | #################################################################### 39 | # Dataset Class 40 | #################################################################### 41 | 42 | class MNIST_dataset(Dataset): 43 | 44 | def __init__(self, data, partition = "train"): 45 | 46 | print("\nLoading MNIST ", partition, " Dataset...") 47 | self.data = data 48 | self.partition = partition 49 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 50 | 51 | def __len__(self): 52 | return len(self.data) 53 | 54 | def from_pil_to_tensor(self, image): 55 | return torchvision.transforms.ToTensor()(image) 56 | 57 | def __getitem__(self, idx): 58 | 59 | # Image 60 | image = self.data[idx][0] 61 | # PIL Image to torch tensor 62 | image_tensor = self.from_pil_to_tensor(image) 63 | # care! net expect a 784 size vector and our dataset 64 | # provide 1x28x28 (channels, height, width) -> Reshape! 65 | image_tensor = image_tensor.view(-1) 66 | 67 | # Label 68 | label = torch.tensor(self.data[idx][1]) 69 | label = F.one_hot(label, num_classes=10).float() 70 | 71 | return {"img": image_tensor, "label": label} 72 | 73 | train_dataset = MNIST_dataset(train_set, partition="train") 74 | test_dataset = MNIST_dataset(test_set, partition="test") 75 | 76 | 77 | #################################################################### 78 | # DataLoader Class 79 | #################################################################### 80 | 81 | batch_size = 100 82 | num_workers = multiprocessing.cpu_count()-1 83 | print("Num workers", num_workers) 84 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 85 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 86 | 87 | 88 | #################################################################### 89 | # Neural Network Class 90 | #################################################################### 91 | 92 | # Instantiating the network and printing its architecture 93 | num_classes = 10 94 | net = nn.Sequential( 95 | nn.Linear(784, 1024), 96 | nn.ReLU(), 97 | nn.Linear(1024, 1024), 98 | nn.ReLU(), 99 | nn.Linear(1024, 1024), 100 | nn.ReLU(), 101 | nn.Linear(1024, num_classes) 102 | ) 103 | print(net) 104 | 105 | def count_parameters(model): 106 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 107 | print("Params: ", count_parameters(net)) 108 | 109 | #################################################################### 110 | # Training settings 111 | #################################################################### 112 | 113 | # Training hyperparameters 114 | criterion = nn.CrossEntropyLoss() 115 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 116 | epochs = 25 117 | 118 | 119 | #################################################################### 120 | # Training 121 | #################################################################### 122 | 123 | # Load model in GPU 124 | net.to(device) 125 | 126 | print("\n---- Start Training ----") 127 | best_accuracy = -1 128 | best_epoch = 0 129 | for epoch in range(epochs): 130 | 131 | 132 | # TRAIN NETWORK 133 | train_loss, train_correct = 0, 0 134 | net.train() 135 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 136 | for batch in tepoch: 137 | 138 | # Returned values of Dataset Class 139 | images = batch["img"].to(device) 140 | labels = batch["label"].to(device) 141 | 142 | # zero the parameter gradients 143 | optimizer.zero_grad() 144 | 145 | # Forward 146 | outputs = net(images) 147 | loss = criterion(outputs, labels) 148 | 149 | # Calculate gradients 150 | loss.backward() 151 | 152 | # Update gradients 153 | optimizer.step() 154 | 155 | # one hot -> labels 156 | labels = torch.argmax(labels, dim=1) 157 | pred = torch.argmax(outputs, dim=1) 158 | train_correct += pred.eq(labels).sum().item() 159 | 160 | # print statistics 161 | train_loss += loss.item() 162 | 163 | train_loss /= len(train_dataloader.dataset) 164 | 165 | # TEST NETWORK 166 | test_loss, test_correct = 0, 0 167 | net.eval() 168 | with torch.no_grad(): 169 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 170 | for batch in tepoch: 171 | 172 | images = batch["img"].to(device) 173 | labels = batch["label"].to(device) 174 | 175 | # Forward 176 | outputs = net(images) 177 | test_loss += criterion(outputs, labels) 178 | 179 | # one hot -> labels 180 | labels = torch.argmax(labels, dim=1) 181 | pred = torch.argmax(outputs, dim=1) 182 | 183 | test_correct += pred.eq(labels).sum().item() 184 | 185 | test_loss /= len(test_dataloader.dataset) 186 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 187 | 188 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 189 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 190 | )) 191 | 192 | if test_accuracy > best_accuracy: 193 | best_accuracy = test_accuracy 194 | best_epoch = epoch 195 | 196 | # Save best weights 197 | torch.save(net.state_dict(), "best_model.pt") 198 | 199 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 200 | 201 | 202 | 203 | #################################################################### 204 | # Load best weights 205 | #################################################################### 206 | 207 | # Load best weights 208 | net.load_state_dict(torch.load("best_model.pt")) 209 | 210 | test_loss, test_correct = 0, 0 211 | net.eval() 212 | with torch.no_grad(): 213 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 214 | for batch in tepoch: 215 | 216 | images = batch["img"].to(device) 217 | labels = batch["label"].to(device) 218 | 219 | # Forward 220 | outputs = net(images) 221 | test_loss += criterion(outputs, labels) 222 | 223 | # one hot -> labels 224 | labels = torch.argmax(labels, dim=1) 225 | pred = torch.argmax(outputs, dim=1) 226 | 227 | test_correct += pred.eq(labels).sum().item() 228 | 229 | test_loss /= len(test_dataloader.dataset) 230 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 231 | print("Final best acc: ", test_accuracy) 232 | -------------------------------------------------------------------------------- /MNIST/Pytorch/mnist_mlp_2_bn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torch.nn as nn 4 | from tqdm import tqdm 5 | import multiprocessing 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torch.utils.data import Dataset 9 | from torch.utils.data import DataLoader 10 | 11 | print("Torch version: ", torch. __version__) 12 | 13 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 14 | print("Device: ", device) 15 | 16 | class MNIST_dataset(Dataset): 17 | 18 | def __init__(self, partition = "train"): 19 | 20 | print("\nLoading MNIST ", partition, " Dataset...") 21 | self.partition = partition 22 | if self.partition == "train": 23 | self.data = torchvision.datasets.MNIST('.data/', train=True, download=True) 24 | else: 25 | self.data = torchvision.datasets.MNIST('.data/', train=False, download=True) 26 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 27 | 28 | def __len__(self): 29 | return len(self.data) 30 | 31 | def from_pil_to_tensor(self, image): 32 | return torchvision.transforms.ToTensor()(image) 33 | 34 | def __getitem__(self, idx): 35 | 36 | # Image 37 | image = self.data[idx][0] 38 | # PIL Image to torch tensor 39 | image = self.from_pil_to_tensor(image) 40 | # care! net expect a 784 size vector and our dataset 41 | # provide 1x28x28 (channels, height, width) -> Reshape! 42 | image = image.view(-1) 43 | 44 | # Label 45 | label = torch.tensor(self.data[idx][1]) 46 | label = F.one_hot(label, num_classes=10).float() 47 | 48 | return {"idx": idx, "img": image, "label": label} 49 | 50 | train_dataset = MNIST_dataset(partition="train") 51 | test_dataset = MNIST_dataset(partition="test") 52 | 53 | batch_size = 100 54 | num_workers = multiprocessing.cpu_count()-1 55 | print("Num workers", num_workers) 56 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 57 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 58 | 59 | #################################################################### 60 | # Adding Batch Normalization from torch.nn 61 | #################################################################### 62 | 63 | class Net(nn.Module): 64 | def __init__(self, sizes=[[784, 1024], [1024, 1024], [1024, 1024], [1024, 10]], criterion=None): 65 | super(Net, self).__init__() 66 | 67 | self.layers = nn.ModuleList() 68 | 69 | for i in range(len(sizes)-1): 70 | dims = sizes[i] 71 | self.layers.append(nn.Linear(dims[0], dims[1])) 72 | self.layers.append(nn.BatchNorm1d(dims[1])) 73 | self.layers.append(nn.ReLU()) 74 | 75 | dims = sizes[-1] 76 | self.classifier = nn.Linear(dims[0], dims[1]) 77 | 78 | self.criterion = criterion 79 | 80 | def forward(self, x, y=None): 81 | for layer in self.layers: 82 | x = layer(x) 83 | x = self.classifier(x) 84 | 85 | if y != None: 86 | loss = self.criterion(x, y) 87 | return loss, x 88 | return x 89 | 90 | 91 | # Training Settings 92 | criterion = nn.CrossEntropyLoss() 93 | 94 | # Instantiating the network and printing its architecture 95 | num_classes = 10 96 | net = Net(sizes=[ 97 | [784, 1024], 98 | [1024, 1024], 99 | [1024, 1024], 100 | [1024, num_classes] 101 | ], 102 | criterion=criterion) 103 | print(net) 104 | 105 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 106 | net = net.to(device) 107 | 108 | # Start training 109 | epochs = 25 110 | 111 | print("\n---- Start Training ----") 112 | best_accuracy = -1 113 | best_epoch = 0 114 | for epoch in range(epochs): 115 | 116 | 117 | # TRAIN NETWORK 118 | train_loss, train_correct = 0, 0 119 | net.train() 120 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 121 | for batch in tepoch: 122 | 123 | images = batch["img"].to(device) 124 | labels = batch["label"].to(device) 125 | ids = batch["idx"].to('cpu').numpy() 126 | 127 | # zero the parameter gradients 128 | optimizer.zero_grad() 129 | 130 | # Forward 131 | loss, outputs = net(images, labels) 132 | 133 | loss.backward() 134 | 135 | optimizer.step() 136 | 137 | # one hot -> labels 138 | labels = torch.argmax(labels, dim=1) 139 | pred = torch.argmax(outputs, dim=1) 140 | train_correct += pred.eq(labels).sum().item() 141 | 142 | # print statistics 143 | train_loss += loss.item() 144 | 145 | train_loss /= len(train_dataloader.dataset) 146 | 147 | # TEST NETWORK 148 | test_loss, test_correct = 0, 0 149 | net.eval() 150 | with torch.no_grad(): 151 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 152 | for batch in tepoch: 153 | 154 | images = batch["img"].to(device) 155 | labels = batch["label"].to(device) 156 | ids = batch["idx"].to('cpu').numpy() 157 | 158 | # Forward 159 | outputs = net(images) 160 | test_loss += criterion(outputs, labels) 161 | 162 | # one hot -> labels 163 | labels = torch.argmax(labels, dim=1) 164 | pred = torch.argmax(outputs, dim=1) 165 | 166 | test_correct += pred.eq(labels).sum().item() 167 | 168 | test_loss /= len(test_dataloader.dataset) 169 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 170 | 171 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 172 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 173 | )) 174 | 175 | if test_accuracy > best_accuracy: 176 | best_accuracy = test_accuracy 177 | best_epoch = epoch 178 | 179 | # Save best weights 180 | torch.save(net.state_dict(), "best_model.pt") 181 | 182 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 183 | 184 | 185 | # Load best weights 186 | net.load_state_dict(torch.load("best_model.pt")) 187 | 188 | test_loss, test_correct = 0, 0 189 | net.eval() 190 | with torch.no_grad(): 191 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 192 | for batch in tepoch: 193 | 194 | images = batch["img"].to(device) 195 | labels = batch["label"].to(device) 196 | ids = batch["idx"].to('cpu').numpy() 197 | 198 | # Forward 199 | outputs = net(images) 200 | test_loss += criterion(outputs, labels) 201 | 202 | # one hot -> labels 203 | labels = torch.argmax(labels, dim=1) 204 | pred = torch.argmax(outputs, dim=1) 205 | 206 | test_correct += pred.eq(labels).sum().item() 207 | 208 | test_loss /= len(test_dataloader.dataset) 209 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 210 | print("Final best acc: ", test_accuracy) 211 | -------------------------------------------------------------------------------- /MNIST/Pytorch/mnist_mlp_3_lra.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torch.nn as nn 4 | from tqdm import tqdm 5 | import multiprocessing 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torch.utils.data import Dataset 9 | from torch.utils.data import DataLoader 10 | 11 | print("Torch version: ", torch. __version__) 12 | 13 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 14 | print("Device: ", device) 15 | 16 | 17 | class MNIST_dataset(Dataset): 18 | 19 | def __init__(self, partition = "train"): 20 | 21 | print("\nLoading MNIST ", partition, " Dataset...") 22 | self.partition = partition 23 | if self.partition == "train": 24 | self.data = torchvision.datasets.MNIST('.data/', train=True, download=True) 25 | else: 26 | self.data = torchvision.datasets.MNIST('.data/', train=False, download=True) 27 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 28 | 29 | def __len__(self): 30 | return len(self.data) 31 | 32 | def from_pil_to_tensor(self, image): 33 | return torchvision.transforms.ToTensor()(image) 34 | 35 | def __getitem__(self, idx): 36 | 37 | # Image 38 | image = self.data[idx][0] 39 | # PIL Image to torch tensor 40 | image = self.from_pil_to_tensor(image) 41 | # care! net expect a 784 size vector and our dataset 42 | # provide 1x28x28 (channels, height, width) -> Reshape! 43 | image = image.view(-1) 44 | 45 | # Label 46 | label = torch.tensor(self.data[idx][1]) 47 | label = F.one_hot(label, num_classes=10).float() 48 | 49 | return {"idx": idx, "img": image, "label": label} 50 | 51 | train_dataset = MNIST_dataset(partition="train") 52 | test_dataset = MNIST_dataset(partition="test") 53 | 54 | batch_size = 100 55 | num_workers = multiprocessing.cpu_count()-1 56 | print("Num workers", num_workers) 57 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 58 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 59 | 60 | #################################################################### 61 | # Adding Batch Normalization from torch.nn 62 | #################################################################### 63 | 64 | class Net(nn.Module): 65 | def __init__(self, sizes=[[784, 1024], [1024, 1024], [1024, 1024], [1024, 10]], criterion=None): 66 | super(Net, self).__init__() 67 | 68 | self.layers = nn.ModuleList() 69 | 70 | for i in range(len(sizes)-1): 71 | dims = sizes[i] 72 | self.layers.append(nn.Linear(dims[0], dims[1])) 73 | self.layers.append(nn.BatchNorm1d(dims[1])) 74 | self.layers.append(nn.ReLU()) 75 | 76 | dims = sizes[-1] 77 | self.classifier = nn.Linear(dims[0], dims[1]) 78 | 79 | self.criterion = criterion 80 | 81 | def forward(self, x, y=None): 82 | for layer in self.layers: 83 | x = layer(x) 84 | x = self.classifier(x) 85 | 86 | if y != None: 87 | loss = self.criterion(x, y) 88 | return loss, x 89 | return x 90 | 91 | 92 | # Training Settings 93 | criterion = nn.CrossEntropyLoss() 94 | 95 | # Instantiating the network and printing its architecture 96 | num_classes = 10 97 | net = Net(sizes=[ 98 | [784, 1024], 99 | [1024, 1024], 100 | [1024, 1024], 101 | [1024, num_classes] 102 | ], 103 | criterion=criterion) 104 | print(net) 105 | 106 | #################################################################### 107 | # Adding LRA 108 | #################################################################### 109 | 110 | # Training hyperparameters 111 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 112 | 113 | # Learning Rate Annealing (LRA) scheduling 114 | # lr = 0.1 if epoch < 25 115 | # lr = 0.01 if 25 <= epoch < 50 116 | # lr = 0.001 if epoch >= 50 117 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[25, 50], gamma=0.1) 118 | 119 | net = net.to(device) 120 | 121 | # Start training 122 | epochs = 75 123 | 124 | print("\n---- Start Training ----") 125 | best_accuracy = -1 126 | best_epoch = 0 127 | for epoch in range(epochs): 128 | 129 | 130 | # TRAIN NETWORK 131 | train_loss, train_correct = 0, 0 132 | net.train() 133 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 134 | for batch in tepoch: 135 | 136 | images = batch["img"].to(device) 137 | labels = batch["label"].to(device) 138 | ids = batch["idx"].to('cpu').numpy() 139 | 140 | # zero the parameter gradients 141 | optimizer.zero_grad() 142 | 143 | # Forward 144 | loss, outputs = net(images, labels) 145 | 146 | loss.backward() 147 | 148 | optimizer.step() 149 | 150 | # one hot -> labels 151 | labels = torch.argmax(labels, dim=1) 152 | pred = torch.argmax(outputs, dim=1) 153 | train_correct += pred.eq(labels).sum().item() 154 | 155 | # print statistics 156 | train_loss += loss.item() 157 | 158 | scheduler.step() 159 | print("\tLR: ", optimizer.param_groups[0]['lr']) 160 | 161 | 162 | train_loss /= len(train_dataloader.dataset) 163 | 164 | # TEST NETWORK 165 | test_loss, test_correct = 0, 0 166 | net.eval() 167 | with torch.no_grad(): 168 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 169 | for batch in tepoch: 170 | 171 | images = batch["img"].to(device) 172 | labels = batch["label"].to(device) 173 | ids = batch["idx"].to('cpu').numpy() 174 | 175 | # Forward 176 | outputs = net(images) 177 | test_loss += criterion(outputs, labels) 178 | 179 | # one hot -> labels 180 | labels = torch.argmax(labels, dim=1) 181 | pred = torch.argmax(outputs, dim=1) 182 | 183 | test_correct += pred.eq(labels).sum().item() 184 | 185 | test_loss /= len(test_dataloader.dataset) 186 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 187 | 188 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 189 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 190 | )) 191 | 192 | if test_accuracy > best_accuracy: 193 | best_accuracy = test_accuracy 194 | best_epoch = epoch 195 | 196 | # Save best weights 197 | torch.save(net.state_dict(), "best_model.pt") 198 | 199 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 200 | 201 | 202 | # Load best weights 203 | net.load_state_dict(torch.load("best_model.pt")) 204 | 205 | test_loss, test_correct = 0, 0 206 | net.eval() 207 | with torch.no_grad(): 208 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 209 | for batch in tepoch: 210 | 211 | images = batch["img"].to(device) 212 | labels = batch["label"].to(device) 213 | ids = batch["idx"].to('cpu').numpy() 214 | 215 | # Forward 216 | outputs = net(images) 217 | test_loss += criterion(outputs, labels) 218 | 219 | # one hot -> labels 220 | labels = torch.argmax(labels, dim=1) 221 | pred = torch.argmax(outputs, dim=1) 222 | 223 | test_correct += pred.eq(labels).sum().item() 224 | 225 | test_loss /= len(test_dataloader.dataset) 226 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 227 | print("Final best acc: ", test_accuracy) 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /MNIST/Pytorch/mnist_mlp_4_da.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torch.nn as nn 4 | from tqdm import tqdm 5 | import multiprocessing 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torchvision import transforms 9 | from torch.utils.data import Dataset 10 | from torch.utils.data import DataLoader 11 | 12 | print("Torch version: ", torch. __version__) 13 | 14 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 15 | print("Device: ", device) 16 | 17 | #################################################################### 18 | # Defining a Compose my a D.A. 19 | #################################################################### 20 | 21 | train_transform = transforms.Compose( 22 | [ 23 | transforms.RandomRotation(3), 24 | transforms.RandomAffine(degrees=2, translate=(0.002,0.001), scale=(0.001, 1.64)), 25 | transforms.ToTensor(), 26 | ]) 27 | 28 | test_transform = transforms.Compose( 29 | [ 30 | transforms.ToTensor(), 31 | ]) 32 | 33 | class MNIST_dataset(Dataset): 34 | 35 | def __init__(self, partition = "train", transform=None): 36 | 37 | print("\nLoading MNIST ", partition, " Dataset...") 38 | self.partition = partition 39 | self.transform = transform 40 | if self.partition == "train": 41 | self.data = torchvision.datasets.MNIST('.data/', train=True, download=True) 42 | else: 43 | self.data = torchvision.datasets.MNIST('.data/', train=False, download=True) 44 | print("\tTotal Len.: ", len(self.data), "\n", 50*"-") 45 | 46 | def __len__(self): 47 | return len(self.data) 48 | 49 | def __getitem__(self, idx): 50 | 51 | # Image 52 | image = self.data[idx][0] 53 | image = self.transform(image) 54 | # care! net expect a 784 size vector and our dataset 55 | # provide 1x28x28 (channels, height, width) -> Reshape! 56 | image = image.view(-1) 57 | 58 | # Label 59 | label = torch.tensor(self.data[idx][1]) 60 | label = F.one_hot(label, num_classes=10).float() 61 | 62 | return {"idx": idx, "img": image, "label": label} 63 | 64 | train_dataset = MNIST_dataset(partition="train", transform=train_transform) 65 | test_dataset = MNIST_dataset(partition="test", transform=test_transform) 66 | 67 | batch_size = 100 68 | num_workers = multiprocessing.cpu_count()-1 69 | print("Num workers", num_workers) 70 | train_dataloader = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers) 71 | test_dataloader = DataLoader(test_dataset, batch_size, shuffle=False, num_workers=num_workers) 72 | 73 | class Net(nn.Module): 74 | def __init__(self, sizes=[[784, 1024], [1024, 1024], [1024, 1024], [1024, 10]], criterion=None): 75 | super(Net, self).__init__() 76 | 77 | self.layers = nn.ModuleList() 78 | 79 | for i in range(len(sizes)-1): 80 | dims = sizes[i] 81 | self.layers.append(nn.Linear(dims[0], dims[1])) 82 | self.layers.append(nn.BatchNorm1d(dims[1])) 83 | self.layers.append(nn.ReLU()) 84 | 85 | dims = sizes[-1] 86 | self.classifier = nn.Linear(dims[0], dims[1]) 87 | 88 | self.criterion = criterion 89 | 90 | def forward(self, x, y=None): 91 | for layer in self.layers: 92 | x = layer(x) 93 | x = self.classifier(x) 94 | 95 | if y != None: 96 | loss = self.criterion(x, y) 97 | return loss, x 98 | return x 99 | 100 | 101 | # Training Settings 102 | criterion = nn.CrossEntropyLoss() 103 | 104 | # Instantiating the network and printing its architecture 105 | num_classes = 10 106 | net = Net(sizes=[ 107 | [784, 1024], 108 | [1024, 1024], 109 | [1024, 1024], 110 | [1024, num_classes] 111 | ], 112 | criterion=criterion) 113 | print(net) 114 | optimizer = optim.SGD(net.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9) 115 | 116 | # Learning Rate Annealing (LRA) scheduling 117 | # lr = 0.1 if epoch < 25 118 | # lr = 0.01 if 25 <= epoch < 50 119 | # lr = 0.001 if epoch >= 50 120 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[25, 50], gamma=0.1) 121 | 122 | net = net.to(device) 123 | 124 | # Start training 125 | epochs = 75 126 | 127 | print("\n---- Start Training ----") 128 | best_accuracy = -1 129 | best_epoch = 0 130 | for epoch in range(epochs): 131 | 132 | 133 | # TRAIN NETWORK 134 | train_loss, train_correct = 0, 0 135 | net.train() 136 | with tqdm(iter(train_dataloader), desc="Epoch " + str(epoch), unit="batch") as tepoch: 137 | for batch in tepoch: 138 | 139 | images = batch["img"].to(device) 140 | labels = batch["label"].to(device) 141 | ids = batch["idx"].to('cpu').numpy() 142 | 143 | # zero the parameter gradients 144 | optimizer.zero_grad() 145 | 146 | # Forward 147 | loss, outputs = net(images, labels) 148 | 149 | loss.backward() 150 | 151 | optimizer.step() 152 | 153 | # one hot -> labels 154 | labels = torch.argmax(labels, dim=1) 155 | pred = torch.argmax(outputs, dim=1) 156 | 157 | train_correct += pred.eq(labels).sum().item() 158 | 159 | # print statistics 160 | train_loss += loss.item() 161 | 162 | scheduler.step() 163 | print("\tLR: ", optimizer.param_groups[0]['lr']) 164 | 165 | train_loss /= len(train_dataloader.dataset) 166 | 167 | # TEST NETWORK 168 | test_loss, test_correct = 0, 0 169 | net.eval() 170 | with torch.no_grad(): 171 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 172 | for batch in tepoch: 173 | 174 | images = batch["img"].to(device) 175 | labels = batch["label"].to(device) 176 | ids = batch["idx"].to('cpu').numpy() 177 | 178 | # Forward 179 | outputs = net(images) 180 | test_loss += criterion(outputs, labels) 181 | 182 | # one hot -> labels 183 | labels = torch.argmax(labels, dim=1) 184 | pred = torch.argmax(outputs, dim=1) 185 | 186 | test_correct += pred.eq(labels).sum().item() 187 | 188 | test_loss /= len(test_dataloader.dataset) 189 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 190 | 191 | print("[Epoch {}] Train Loss: {:.6f} - Test Loss: {:.6f} - Train Accuracy: {:.2f}% - Test Accuracy: {:.2f}%".format( 192 | epoch + 1, train_loss, test_loss, 100. * train_correct / len(train_dataloader.dataset), test_accuracy 193 | )) 194 | 195 | if test_accuracy > best_accuracy: 196 | best_accuracy = test_accuracy 197 | best_epoch = epoch 198 | 199 | # Save best weights 200 | torch.save(net.state_dict(), "best_model.pt") 201 | 202 | print("\nBEST TEST ACCURACY: ", best_accuracy, " in epoch ", best_epoch) 203 | 204 | 205 | # Load best weights 206 | net.load_state_dict(torch.load("best_model.pt")) 207 | 208 | test_loss, test_correct = 0, 0 209 | net.eval() 210 | with torch.no_grad(): 211 | with tqdm(iter(test_dataloader), desc="Test " + str(epoch), unit="batch") as tepoch: 212 | for batch in tepoch: 213 | 214 | images = batch["img"].to(device) 215 | labels = batch["label"].to(device) 216 | ids = batch["idx"].to('cpu').numpy() 217 | 218 | # Forward 219 | outputs = net(images) 220 | test_loss += criterion(outputs, labels) 221 | 222 | # one hot -> labels 223 | labels = torch.argmax(labels, dim=1) 224 | pred = torch.argmax(outputs, dim=1) 225 | 226 | test_correct += pred.eq(labels).sum().item() 227 | 228 | test_loss /= len(test_dataloader.dataset) 229 | test_accuracy = 100. * test_correct / len(test_dataloader.dataset) 230 | print("Final best acc: ", test_accuracy) 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning Lab 2 | Some of the lab examples and exercises of my deep learning lectures. 3 | 4 | ## Examples 5 | 6 | + Two different tasks: 7 | 8 | - [MNIST](http://yann.lecun.com/exdb/mnist/) and 9 | - [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) 10 | 11 | + Two different toolkits: 12 | - [Keras](https://keras.io) 13 | - [Pythorch](https://pytorch.org) 14 | 15 | 16 | ## Recommendations 17 | 18 | Use [Googe Colab](https://colab.research.google.com) to run the experiments in virtual machines equiped with GPUs just in case you don't have a GPU computer to run them 19 | --------------------------------------------------------------------------------