├── README.md
├── Lenet5_.py
├── VGG.py
├── ResNet_.py
├── Inception_.py
└── EfficientNet_.py


/README.md:
--------------------------------------------------------------------------------
1 | # Deep_Learning_models_implementation_from-scratch_using_pytorch_
2 | Deep learning model implementation from scratch using pytorch</br>
3 | 
4 | LeNet5 paper: http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf </br>
5 | VGG paper: https://arxiv.org/pdf/1409.1556 </br>
6 | ResNet paper: https://arxiv.org/abs/1512.03385 </br>
7 | Inception paper: https://arxiv.org/abs/1409.4842 </br>
8 | EfficientNet paper: https://arxiv.org/abs/1905.11946 </br>
9 | 


--------------------------------------------------------------------------------
/Lenet5_.py:
--------------------------------------------------------------------------------
 1 | import torch
 2 | import torch.nn as nn
 3 | 
 4 | 
 5 | class LeNet(nn.Module):
 6 |     def __init__(self):
 7 |         super(LeNet, self).__init__()
 8 |         self.relu = nn.ReLU()
 9 |         self.pool = nn.AvgPool2d(kernel_size=(2, 2), stride=(2, 2))
10 |         self.conv1 = nn.Conv2d(
11 |             in_channels=1,
12 |             out_channels=6,
13 |             kernel_size=(5, 5),
14 |             stride=(1, 1),
15 |             padding=(0, 0),
16 |         )
17 |         self.conv2 = nn.Conv2d(
18 |             in_channels=6,
19 |             out_channels=16,
20 |             kernel_size=(5, 5),
21 |             stride=(1, 1),
22 |             padding=(0, 0),
23 |         )
24 |         self.conv3 = nn.Conv2d(
25 |             in_channels=16,
26 |             out_channels=120,
27 |             kernel_size=(5, 5),
28 |             stride=(1, 1),
29 |             padding=(0, 0),
30 |         )
31 |         self.linear1 = nn.Linear(120, 84)
32 |         self.linear2 = nn.Linear(84, 10)
33 | 
34 |     def forward(self, x):
35 |         x = self.relu(self.conv1(x))
36 |         x = self.pool(x)
37 |         x = self.relu(self.conv2(x))
38 |         x = self.pool(x)
39 |         x = self.relu(
40 |             self.conv3(x)
41 |         )  # num_examples x 120 x 1 x 1 --> num_examples x 120
42 |         x = x.reshape(x.shape[0], -1)
43 |         x = self.relu(self.linear1(x))
44 |         x = self.linear2(x)
45 |         return x
46 | 
47 | 
48 | def test_lenet():
49 |     x = torch.randn(64, 1, 32, 32)
50 |     model = LeNet()
51 |     return model(x)
52 | 
53 | 
54 | if __name__ == "__main__":
55 |     out = test_lenet()
56 |     print(out.shape)
57 | 
58 | # ref https://github.com/aladdinpersson/Machine-Learning-Collection


--------------------------------------------------------------------------------
/VGG.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | 
  4 | 
  5 | VGG_types = {
  6 |     "VGG11": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
  7 |     "VGG13": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
  8 |     "VGG16": [
  9 |         64,
 10 |         64,
 11 |         "M",
 12 |         128,
 13 |         128,
 14 |         "M",
 15 |         256,
 16 |         256,
 17 |         256,
 18 |         "M",
 19 |         512,
 20 |         512,
 21 |         512,
 22 |         "M",
 23 |         512,
 24 |         512,
 25 |         512,
 26 |         "M",
 27 |     ],
 28 |     "VGG19": [
 29 |         64,
 30 |         64,
 31 |         "M",
 32 |         128,
 33 |         128,
 34 |         "M",
 35 |         256,
 36 |         256,
 37 |         256,
 38 |         256,
 39 |         "M",
 40 |         512,
 41 |         512,
 42 |         512,
 43 |         512,
 44 |         "M",
 45 |         512,
 46 |         512,
 47 |         512,
 48 |         512,
 49 |         "M",
 50 |     ],
 51 | }
 52 | 
 53 | 
 54 | class VGGnet(nn.Module):
 55 |     def __init__(self, in_channels=3, num_classes=1000):
 56 |         super(VGGnet, self).__init__()
 57 |         self.in_channels = in_channels
 58 |         self.conv_layers = self.create_conv_layers(VGG_types["VGG16"])
 59 | 
 60 |         self.fcs = nn.Sequential(
 61 |             nn.Linear(512 * 7 * 7, 4096),
 62 |             nn.ReLU(),
 63 |             nn.Dropout(p=0.5),
 64 |             nn.Linear(4096, 4096),
 65 |             nn.ReLU(),
 66 |             nn.Dropout(p=0.5),
 67 |             nn.Linear(4096, num_classes),
 68 |         )
 69 | 
 70 |     def forward(self, x):
 71 |         x = self.conv_layers(x)
 72 |         x = x.reshape(x.shape[0], -1)
 73 |         x = self.fcs(x)
 74 |         return x
 75 | 
 76 |     def create_conv_layers(self, architecture):
 77 |         layers = []
 78 |         in_channels = self.in_channels
 79 | 
 80 |         for x in architecture:
 81 |             if type(x) == int:
 82 |                 out_channels = x
 83 | 
 84 |                 layers += [
 85 |                     nn.Conv2d(
 86 |                         in_channels=in_channels,
 87 |                         out_channels=out_channels,
 88 |                         kernel_size=(3, 3),
 89 |                         stride=(1, 1),
 90 |                         padding=(1, 1),
 91 |                     ),
 92 |                     nn.BatchNorm2d(x),
 93 |                     nn.ReLU(),
 94 |                 ]
 95 |                 in_channels = x
 96 |             elif x == "M":
 97 |                 layers += [nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))]
 98 | 
 99 |         return nn.Sequential(*layers)
100 | 
101 | 
102 | if __name__ == "__main__":
103 |     device = "cuda" if torch.cuda.is_available() else "cpu"
104 |     model = VGGnet(in_channels=3, num_classes=1000).to(device)
105 |     print(model)
106 |     ## N = 3 (Mini batch size)
107 |     # x = torch.randn(3, 3, 224, 224).to(device)
108 |     # print(model(x).shape)
109 | 
110 | # ref https://github.com/aladdinpersson/Machine-Learning-Collection


--------------------------------------------------------------------------------
/ResNet_.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | 
  4 | 
  5 | class block(nn.Module):
  6 |     def __init__(
  7 |         self, in_channels, intermediate_channels, identity_downsample=None, stride=1
  8 |     ):
  9 |         super(block, self).__init__()
 10 |         self.expansion = 4
 11 |         self.conv1 = nn.Conv2d(
 12 |             in_channels, intermediate_channels, kernel_size=1, stride=1, padding=0, bias=False
 13 |         )
 14 |         self.bn1 = nn.BatchNorm2d(intermediate_channels)
 15 |         self.conv2 = nn.Conv2d(
 16 |             intermediate_channels,
 17 |             intermediate_channels,
 18 |             kernel_size=3,
 19 |             stride=stride,
 20 |             padding=1,
 21 |             bias=False
 22 |         )
 23 |         self.bn2 = nn.BatchNorm2d(intermediate_channels)
 24 |         self.conv3 = nn.Conv2d(
 25 |             intermediate_channels,
 26 |             intermediate_channels * self.expansion,
 27 |             kernel_size=1,
 28 |             stride=1,
 29 |             padding=0,
 30 |             bias=False
 31 |         )
 32 |         self.bn3 = nn.BatchNorm2d(intermediate_channels * self.expansion)
 33 |         self.relu = nn.ReLU()
 34 |         self.identity_downsample = identity_downsample
 35 |         self.stride = stride
 36 | 
 37 |     def forward(self, x):
 38 |         identity = x.clone()
 39 | 
 40 |         x = self.conv1(x)
 41 |         x = self.bn1(x)
 42 |         x = self.relu(x)
 43 |         x = self.conv2(x)
 44 |         x = self.bn2(x)
 45 |         x = self.relu(x)
 46 |         x = self.conv3(x)
 47 |         x = self.bn3(x)
 48 | 
 49 |         if self.identity_downsample is not None:
 50 |             identity = self.identity_downsample(identity)
 51 | 
 52 |         x += identity
 53 |         x = self.relu(x)
 54 |         return x
 55 | 
 56 | 
 57 | class ResNet(nn.Module):
 58 |     def __init__(self, block, layers, image_channels, num_classes):
 59 |         super(ResNet, self).__init__()
 60 |         self.in_channels = 64
 61 |         self.conv1 = nn.Conv2d(image_channels, 64, kernel_size=7, stride=2, padding=3, bias=False)
 62 |         self.bn1 = nn.BatchNorm2d(64)
 63 |         self.relu = nn.ReLU()
 64 |         self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 65 | 
 66 |         # Essentially the entire ResNet architecture are in these 4 lines below
 67 |         self.layer1 = self._make_layer(
 68 |             block, layers[0], intermediate_channels=64, stride=1
 69 |         )
 70 |         self.layer2 = self._make_layer(
 71 |             block, layers[1], intermediate_channels=128, stride=2
 72 |         )
 73 |         self.layer3 = self._make_layer(
 74 |             block, layers[2], intermediate_channels=256, stride=2
 75 |         )
 76 |         self.layer4 = self._make_layer(
 77 |             block, layers[3], intermediate_channels=512, stride=2
 78 |         )
 79 | 
 80 |         self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
 81 |         self.fc = nn.Linear(512 * 4, num_classes)
 82 | 
 83 |     def forward(self, x):
 84 |         x = self.conv1(x)
 85 |         x = self.bn1(x)
 86 |         x = self.relu(x)
 87 |         x = self.maxpool(x)
 88 |         x = self.layer1(x)
 89 |         x = self.layer2(x)
 90 |         x = self.layer3(x)
 91 |         x = self.layer4(x)
 92 | 
 93 |         x = self.avgpool(x)
 94 |         x = x.reshape(x.shape[0], -1)
 95 |         x = self.fc(x)
 96 | 
 97 |         return x
 98 | 
 99 |     def _make_layer(self, block, num_residual_blocks, intermediate_channels, stride):
100 |         identity_downsample = None
101 |         layers = []
102 | 
103 |         # Either if we half the input space for ex, 56x56 -> 28x28 (stride=2), or channels changes
104 |         # we need to adapt the Identity (skip connection) so it will be able to be added
105 |         # to the layer that's ahead
106 |         if stride != 1 or self.in_channels != intermediate_channels * 4:
107 |             identity_downsample = nn.Sequential(
108 |                 nn.Conv2d(
109 |                     self.in_channels,
110 |                     intermediate_channels * 4,
111 |                     kernel_size=1,
112 |                     stride=stride,
113 |                     bias=False
114 |                 ),
115 |                 nn.BatchNorm2d(intermediate_channels * 4),
116 |             )
117 | 
118 |         layers.append(
119 |             block(self.in_channels, intermediate_channels, identity_downsample, stride)
120 |         )
121 | 
122 |         # The expansion size is always 4 for ResNet 50,101,152
123 |         self.in_channels = intermediate_channels * 4
124 | 
125 |         # For example for first resnet layer: 256 will be mapped to 64 as intermediate layer,
126 |         # then finally back to 256. Hence no identity downsample is needed, since stride = 1,
127 |         # and also same amount of channels.
128 |         for i in range(num_residual_blocks - 1):
129 |             layers.append(block(self.in_channels, intermediate_channels))
130 | 
131 |         return nn.Sequential(*layers)
132 | 
133 | 
134 | def ResNet50(img_channel=3, num_classes=1000):
135 |     return ResNet(block, [3, 4, 6, 3], img_channel, num_classes)
136 | 
137 | 
138 | def ResNet101(img_channel=3, num_classes=1000):
139 |     return ResNet(block, [3, 4, 23, 3], img_channel, num_classes)
140 | 
141 | 
142 | def ResNet152(img_channel=3, num_classes=1000):
143 |     return ResNet(block, [3, 8, 36, 3], img_channel, num_classes)
144 | 
145 | 
146 | def test():
147 |     net = ResNet101(img_channel=3, num_classes=1000)
148 |     device = "cuda" if torch.cuda.is_available() else "cpu"
149 |     y = net(torch.randn(4, 3, 224, 224)).to(device)
150 |     print(y.size())
151 | 
152 | 
153 | test()
154 | 
155 | # ref https://github.com/aladdinpersson/Machine-Learning-Collection


--------------------------------------------------------------------------------
/Inception_.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from torch import nn
  3 | 
  4 | 
  5 | class GoogLeNet(nn.Module):
  6 |     def __init__(self, aux_logits=True, num_classes=1000):
  7 |         super(GoogLeNet, self).__init__()
  8 |         assert aux_logits == True or aux_logits == False
  9 |         self.aux_logits = aux_logits
 10 | 
 11 |         # Write in_channels, etc, all explicit in self.conv1, rest will write to
 12 |         # make everything as compact as possible, kernel_size=3 instead of (3,3)
 13 |         self.conv1 = conv_block(
 14 |             in_channels=3,
 15 |             out_channels=64,
 16 |             kernel_size=(7, 7),
 17 |             stride=(2, 2),
 18 |             padding=(3, 3),
 19 |         )
 20 | 
 21 |         self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 22 |         self.conv2 = conv_block(64, 192, kernel_size=3, stride=1, padding=1)
 23 |         self.maxpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 24 | 
 25 |         # In this order: in_channels, out_1x1, red_3x3, out_3x3, red_5x5, out_5x5, out_1x1pool
 26 |         self.inception3a = Inception_block(192, 64, 96, 128, 16, 32, 32)
 27 |         self.inception3b = Inception_block(256, 128, 128, 192, 32, 96, 64)
 28 |         self.maxpool3 = nn.MaxPool2d(kernel_size=(3, 3), stride=2, padding=1)
 29 | 
 30 |         self.inception4a = Inception_block(480, 192, 96, 208, 16, 48, 64)
 31 |         self.inception4b = Inception_block(512, 160, 112, 224, 24, 64, 64)
 32 |         self.inception4c = Inception_block(512, 128, 128, 256, 24, 64, 64)
 33 |         self.inception4d = Inception_block(512, 112, 144, 288, 32, 64, 64)
 34 |         self.inception4e = Inception_block(528, 256, 160, 320, 32, 128, 128)
 35 |         self.maxpool4 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 36 | 
 37 |         self.inception5a = Inception_block(832, 256, 160, 320, 32, 128, 128)
 38 |         self.inception5b = Inception_block(832, 384, 192, 384, 48, 128, 128)
 39 | 
 40 |         self.avgpool = nn.AvgPool2d(kernel_size=7, stride=1)
 41 |         self.dropout = nn.Dropout(p=0.4)
 42 |         self.fc1 = nn.Linear(1024, num_classes)
 43 | 
 44 |         if self.aux_logits:
 45 |             self.aux1 = InceptionAux(512, num_classes)
 46 |             self.aux2 = InceptionAux(528, num_classes)
 47 |         else:
 48 |             self.aux1 = self.aux2 = None
 49 | 
 50 |     def forward(self, x):
 51 |         x = self.conv1(x)
 52 |         x = self.maxpool1(x)
 53 |         x = self.conv2(x)
 54 |         # x = self.conv3(x)
 55 |         x = self.maxpool2(x)
 56 | 
 57 |         x = self.inception3a(x)
 58 |         x = self.inception3b(x)
 59 |         x = self.maxpool3(x)
 60 | 
 61 |         x = self.inception4a(x)
 62 | 
 63 |         # Auxiliary Softmax classifier 1
 64 |         if self.aux_logits and self.training:
 65 |             aux1 = self.aux1(x)
 66 | 
 67 |         x = self.inception4b(x)
 68 |         x = self.inception4c(x)
 69 |         x = self.inception4d(x)
 70 | 
 71 |         # Auxiliary Softmax classifier 2
 72 |         if self.aux_logits and self.training:
 73 |             aux2 = self.aux2(x)
 74 | 
 75 |         x = self.inception4e(x)
 76 |         x = self.maxpool4(x)
 77 |         x = self.inception5a(x)
 78 |         x = self.inception5b(x)
 79 |         x = self.avgpool(x)
 80 |         x = x.reshape(x.shape[0], -1)
 81 |         x = self.dropout(x)
 82 |         x = self.fc1(x)
 83 | 
 84 |         if self.aux_logits and self.training:
 85 |             return aux1, aux2, x
 86 |         else:
 87 |             return x
 88 | 
 89 | 
 90 | class Inception_block(nn.Module):
 91 |     def __init__(
 92 |         self, in_channels, out_1x1, red_3x3, out_3x3, red_5x5, out_5x5, out_1x1pool
 93 |     ):
 94 |         super(Inception_block, self).__init__()
 95 |         self.branch1 = conv_block(in_channels, out_1x1, kernel_size=(1, 1))
 96 | 
 97 |         self.branch2 = nn.Sequential(
 98 |             conv_block(in_channels, red_3x3, kernel_size=(1, 1)),
 99 |             conv_block(red_3x3, out_3x3, kernel_size=(3, 3), padding=(1, 1)),
100 |         )
101 | 
102 |         self.branch3 = nn.Sequential(
103 |             conv_block(in_channels, red_5x5, kernel_size=(1, 1)),
104 |             conv_block(red_5x5, out_5x5, kernel_size=(5, 5), padding=(2, 2)),
105 |         )
106 | 
107 |         self.branch4 = nn.Sequential(
108 |             nn.MaxPool2d(kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)),
109 |             conv_block(in_channels, out_1x1pool, kernel_size=(1, 1)),
110 |         )
111 | 
112 |     def forward(self, x):
113 |         return torch.cat(
114 |             [self.branch1(x), self.branch2(x), self.branch3(x), self.branch4(x)], 1
115 |         )
116 | 
117 | 
118 | class InceptionAux(nn.Module):
119 |     def __init__(self, in_channels, num_classes):
120 |         super(InceptionAux, self).__init__()
121 |         self.relu = nn.ReLU()
122 |         self.dropout = nn.Dropout(p=0.7)
123 |         self.pool = nn.AvgPool2d(kernel_size=5, stride=3)
124 |         self.conv = conv_block(in_channels, 128, kernel_size=1)
125 |         self.fc1 = nn.Linear(2048, 1024)
126 |         self.fc2 = nn.Linear(1024, num_classes)
127 | 
128 |     def forward(self, x):
129 |         x = self.pool(x)
130 |         x = self.conv(x)
131 |         x = x.reshape(x.shape[0], -1)
132 |         x = self.relu(self.fc1(x))
133 |         x = self.dropout(x)
134 |         x = self.fc2(x)
135 | 
136 |         return x
137 | 
138 | 
139 | class conv_block(nn.Module):
140 |     def __init__(self, in_channels, out_channels, **kwargs):
141 |         super(conv_block, self).__init__()
142 |         self.relu = nn.ReLU()
143 |         self.conv = nn.Conv2d(in_channels, out_channels, **kwargs)
144 |         self.batchnorm = nn.BatchNorm2d(out_channels)
145 | 
146 |     def forward(self, x):
147 |         return self.relu(self.batchnorm(self.conv(x)))
148 | 
149 | 
150 | if __name__ == "__main__":
151 |     # N = 3 (Mini batch size)
152 |     x = torch.randn(3, 3, 224, 224)
153 |     model = GoogLeNet(aux_logits=True, num_classes=1000)
154 |     print(model(x)[2].shape)
155 | 
156 | 
157 | # ref https://github.com/aladdinpersson/Machine-Learning-Collection


--------------------------------------------------------------------------------
/EfficientNet_.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | from math import ceil
  4 | 
  5 | base_model = [
  6 |     # using table 1 from the official paper
  7 |     [1, 16, 1, 1, 3],
  8 |     [6, 24, 2, 2, 3],
  9 |     [6, 40, 2, 2, 5],
 10 |     [6, 80, 3, 2, 3],
 11 |     [6, 112, 3, 1, 5],
 12 |     [6, 192, 4, 2, 5],
 13 |     [6, 320, 1, 1, 3],
 14 | ]
 15 | 
 16 | phi_values = {
 17 |     # tuple: (phi_value, resolution, drop_rate)
 18 |     "b0": (0, 224, 0.2), # alpha, beta, gamma, depth = alpha** phi
 19 |     "b1": (0.5, 240, 0.2),
 20 |     "b2": (1, 260, 0.3),
 21 |     "b3": (2, 300, 0.3),
 22 |     "b4": (3, 380, 0.4),
 23 |     "b5": (4, 456, 0.4),
 24 |     "b6": (5, 528, 0.5),
 25 |     "b7": (6, 600, 0.5),
 26 | }
 27 | 
 28 | 
 29 | class CNNBlock(nn.Module):
 30 |     def __init__(self, input_channels, output_channels, kernel, stride , padding, groups= 1):
 31 |         super(CNNBlock, self).__init__()
 32 |         self.conv = nn.Conv2d(
 33 |             input_channels,
 34 |             output_channels,
 35 |             kernel,
 36 |             stride,
 37 |             padding,
 38 |             groups=groups,
 39 |             bias=False,
 40 |         )
 41 |         self.bn = nn.BatchNorm2d(output_channels)
 42 |         self.silu = nn.SiLU() # SiLU same as swish
 43 | 
 44 |     def forward(self, x):
 45 |         return self.silu(self.bn(self.conv(x)))
 46 | 
 47 | 
 48 | class SqueezeExecution(nn.Module): # to compute attenstion score for each of the channel
 49 |     def __init__(self, input_channels, reduced_dim):
 50 |         super(SqueezeExecution, self).__init__()
 51 |         self.att_sc = nn.Sequential(
 52 |             nn.AdaptiveAvgPool2d(1), # C x W x H = C x 1 x 1
 53 |             nn.Conv2d(input_channels, reduced_dim, 1),
 54 |             nn.SiLU(),
 55 |             nn.Conv2d(reduced_dim, input_channels, 1),
 56 |             nn.Sigmoid()
 57 |         )
 58 | 
 59 |     def forward(self, x):
 60 |         return x * self.att_sc(x)
 61 | 
 62 | 
 63 | class InvertedResidualBlock(nn.Module):
 64 |     def __init__(self, input_channels, output_channels, kernel, stride,
 65 |                  padding, expand_ratio, reduction=4, survival_prob=0.8):
 66 |         # reduction for squeezexcitation and survival for stochastic depth
 67 |         super(InvertedResidualBlock, self).__init__()
 68 |         self.survival_prob = survival_prob
 69 |         self.use_residual = input_channels == output_channels and stride == 1
 70 |         hidden_dim = input_channels * expand_ratio
 71 |         self.expand = input_channels != hidden_dim
 72 |         reduced_dim = int(input_channels/reduction)
 73 | 
 74 |         if self.expand:
 75 |             self.expand_conv = CNNBlock(input_channels, hidden_dim, kernel=3, stride=1, padding=1,)
 76 | 
 77 |         self.conv = nn.Sequential(
 78 |                 CNNBlock(hidden_dim, hidden_dim, kernel, stride, padding, groups=hidden_dim),
 79 |                 SqueezeExecution(hidden_dim, reduced_dim),
 80 |                 nn.Conv2d(hidden_dim, output_channels, 1,  bias=False),
 81 |                 nn.BatchNorm2d(output_channels),
 82 |             )
 83 | 
 84 |     def stochastic_depth(self, x):
 85 |         if not self.training:
 86 |             return x
 87 | 
 88 |         binary_tensor = torch.rand(x.shape[0], 1, 1, 1, device=x.device) < self.survival_prob
 89 |         return torch.div(x, self.survival_prob) * binary_tensor
 90 | 
 91 |     def forward(self, inputs):
 92 |         x = self.expand_conv(inputs) if self.expand else inputs
 93 | 
 94 |         if self.use_residual:
 95 |             return self.stochastic_depth(self.conv(x)) + inputs
 96 |         else:
 97 |             return self.conv(x)
 98 | 
 99 | 
100 | class EfficientNet(nn.Module):
101 |     def __init__(self, version, num_classes):
102 |         super(EfficientNet, self).__init__()
103 |         width_factor, depth_factor, dropout_rate = self.calculate_factors(version)
104 |         last_channel = ceil(1280 * width_factor)
105 |         self.pool = nn.AdaptiveAvgPool2d(1)
106 |         self.features = self.create_features(width_factor, depth_factor, last_channel)
107 |         self.classifier = nn.Sequential(
108 |             nn.Dropout(dropout_rate),
109 |             nn.Linear(last_channel, num_classes)
110 |         )
111 | 
112 |     def calculate_factors(self, version, alpha=1.2, beta=1.1):
113 |         phi, res, drop_rate = phi_values[version]
114 |         depth_factor = alpha ** phi
115 |         width_factor = beta ** phi
116 |         return width_factor, depth_factor, drop_rate
117 | 
118 |     def create_features(self, width_factor, depth_factor, last_channel):
119 |         channels = int(32 * width_factor)
120 |         features = [CNNBlock(3, channels, 3, stride=2, padding=1)]
121 |         in_channels = channels
122 | 
123 |         for expand_ratio, channels, repeats, stride, kernel_size in base_model:
124 |             out_channels = 4 * ceil(int(channels * width_factor) / 4)
125 |             layers_repeats = ceil(repeats * depth_factor)
126 | 
127 |             for layer in range(layers_repeats):
128 |                 features.append(
129 |                     InvertedResidualBlock(in_channels, out_channels, expand_ratio=expand_ratio,
130 |                                           stride=stride if layer == 0 else 1,
131 |                                           kernel=kernel_size,
132 |                                           padding=kernel_size//2) # if k=1, p=0 or k=3, p=1
133 |                 )
134 |                 in_channels = out_channels
135 |         features.append(
136 |             CNNBlock(in_channels, last_channel, kernel=1, stride=1, padding=0)
137 |         )
138 | 
139 |         return nn.Sequential(*features)
140 | 
141 |     def forward(self, x):
142 |         x = self.pool(self.features(x))
143 |         return self.classifier(x.view(x.shape[0], -1))
144 | 
145 | 
146 | def testing():
147 |     device = "cuda" if torch.cuda.is_available() else "cpu"
148 |     version = "b1"
149 |     phi, res, drop_rate = phi_values[version]
150 |     num_examples, num_classes = 8, 12
151 |     x = torch.randn((num_examples, 3, res, res)).to(device)
152 |     model = EfficientNet(
153 |         version=version,
154 |         num_classes=num_classes
155 |     ).to(device)
156 | 
157 |     print(model(x).shape)
158 | 
159 | 
160 | testing()
161 | 
162 | # ref https://github.com/aladdinpersson/Machine-Learning-Collections


--------------------------------------------------------------------------------