├── 0.LinearRegression(manual)-Multivariate.py ├── 0.LinearRegression(manual)-Univariate.py ├── 0.LogisticRegression(manual)-Binary.py ├── 0.LogisticRegression(manual)-Softmax.py ├── 1.LinearRegression(auto)-Multivariate.py ├── 1.LinearRegression(auto)-Univariate.py ├── 1.LogisticRegression(auto)-Binary.py ├── 1.LogisticRegression(auto)-Softmax.py ├── 1.Pytorch-Basic.py ├── 2.DNN-Dropout.py ├── 2.DNN-Optimization.py ├── 2.DNN1.py ├── 2.DNN2.py ├── 3.DataLoader-Cifar10.py ├── 3.DataLoader-Cifar100.py ├── 3.DataLoader-ImageFolder.py ├── 3.DataLoader-MNIST.py ├── 4.CNN-Introduce.py ├── 4.CNN-MNIST(Inference).py ├── 4.CNN-MNIST(Train).py ├── 5.RNN-Introduce.py ├── 5.RNN-PredictWord.py ├── 6.LSTM-AutoComplete.py ├── 6.LSTM-Introduce.py ├── README.md ├── data ├── cat │ └── cat.jpg └── dog │ └── dog.jpg ├── mnist_cnn.pt └── practice ├── DNN-MNIST.py └── MNIST-Viewer.py /0.LinearRegression(manual)-Multivariate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pure Python 3 | Code by Tae Hwan Jung(@graykode) 4 | """ 5 | # Answer y = x1*1 + x2*2 + x3*3 6 | x_data = [ 7 | [1.0, 2.0, 3.0], 8 | [4.0, 5.0, 6.0], 9 | [7.0, 8.0, 9.0], 10 | ] 11 | y_data = [14.0, 32.0, 50.0] 12 | 13 | # Traninalbe Parameters 14 | w1 = 2.0 15 | w2 = 2.0 16 | w3 = 2.0 17 | 18 | def forward(x1, x2, x3): 19 | return x1 * w1 + x2 * w2 + x3 * w3 20 | 21 | # Loss function 22 | def loss(x, y): 23 | y_pred = forward(x[0], x[1], x[2]) 24 | return (y_pred - y) * (y_pred - y) 25 | 26 | # compute gradient 27 | def gradient1(x_vals, y): # d_loss/d_w1 28 | return x_vals[0] * (x_vals[0] * w1 + x_vals[1] * w2 + x_vals[2] * w3 - y) 29 | 30 | def gradient2(x_vals, y): # d_loss/d_w2 31 | return x_vals[1] * (x_vals[0] * w1 + x_vals[1] * w2 + x_vals[2] * w3 - y) 32 | 33 | def gradient3(x_vals, y): # d_loss/d_w3 34 | return x_vals[2] * (x_vals[0] * w1 + x_vals[1] * w2 + x_vals[2] * w3 - y) 35 | 36 | # Before training 37 | print("Parameter w1 :%f, w2 : %f, w3 : %f" % (w1, w2, w3)) 38 | 39 | # Training loop 40 | for epoch in range(100): 41 | l = 0 42 | for x_vals, y_val in zip(x_data, y_data): 43 | 44 | grad1 = gradient1(x_vals, y_val) # d_loss/d_w1 45 | grad2 = gradient2(x_vals, y_val) # # d_loss/d_w2 46 | grad3 = gradient3(x_vals, y_val) # # d_loss/d_w3 47 | 48 | w1 = w1 - 0.01 * grad1 49 | w2 = w2 - 0.01 * grad2 50 | w3 = w3 - 0.01 * grad3 51 | 52 | print("\tgrad: ", x_vals, y_val, round(grad1, 2), round(grad2, 2), round(grad3, 2)) 53 | l += loss(x_vals, y_val) 54 | 55 | print("progress:", epoch+1, "w1=%f, w2=%f, w3=%f"% (round(w1, 2), round(w2, 2), round(w3, 2)), "loss=", round(l, 2)) 56 | 57 | # After training 58 | print("Parameter w1 :%f, w2 : %f, w3 : %f" % (w1, w2, w3)) -------------------------------------------------------------------------------- /0.LinearRegression(manual)-Univariate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pure Python 3 | Reference : https://github.com/hunkim/PyTorchZeroToAll/blob/master/02_manual_gradient.py 4 | """ 5 | # Answer y = x * 3 6 | x_data = [1.0, 2.0, 3.0] 7 | y_data = [3.0, 6.0, 9.0] 8 | 9 | # Traninalbe Parameters 10 | w = 1.0 11 | 12 | def forward(x): 13 | return x * w 14 | 15 | # Loss function 16 | def loss(x, y): 17 | y_pred = forward(x) 18 | return (y_pred - y) * (y_pred - y) 19 | 20 | # compute gradient 21 | def gradient(x, y): # d_loss/d_w 22 | return 2 * x * (x * w - y) 23 | 24 | # Before training 25 | print("Parameter w : %f, Predict y : %f" % (w, forward(4))) 26 | 27 | # Training loop 28 | for epoch in range(10): 29 | l = 0 30 | for x_val, y_val in zip(x_data, y_data): 31 | grad = gradient(x_val, y_val) 32 | w = w - 0.01 * grad # 0.01 is learning rate 33 | print("\tgrad: ", x_val, y_val, round(grad, 2)) 34 | l += loss(x_val, y_val) 35 | print("progress:", epoch, "w=", round(w, 2), "loss=", round(l, 2)) 36 | 37 | # After training 38 | print("Parameter w : %f, Predict y : %f" % (w, forward(4))) -------------------------------------------------------------------------------- /0.LogisticRegression(manual)-Binary.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pure Python 3 | Code by Tae Hwan Jung(@graykode) 4 | """ 5 | import numpy as np 6 | 7 | x_data = [-2.0, -1.0, 1.0, 5.0] 8 | y_data = [1.0, 1.0, 2.0, 2.0] 9 | 10 | # Traninalbe Parameters 11 | w = -10.0 12 | 13 | def sigmoid(x): 14 | return 1 / (1 + np.exp(-w * x)) 15 | 16 | # Loss function 17 | def loss(x, y): 18 | return -(y * np.log(sigmoid(x)) + (1-y) * np.log(1- sigmoid(x))) 19 | 20 | # compute gradient 21 | # if you see Proof about Partial derivative of J(θ) see, https://wikidocs.net/4289 22 | def gradient(x, y): # d_loss/d_w 23 | return x * (sigmoid(x) - y) 24 | 25 | # Training loop 26 | for epoch in range(50): 27 | l = 0 28 | for x_val, y_val in zip(x_data, y_data): 29 | grad = gradient(x_val, y_val) 30 | w = w - 0.01 * grad # 0.01 is learning rate 31 | print("\tgrad: ", x_val, y_val, round(grad, 2)) 32 | l += loss(x_val, y_val) 33 | print("progress:", epoch, "w=", round(w, 2), "loss=", round(l, 2)) 34 | 35 | # test 36 | for x in x_data: 37 | y_pred = sigmoid(x) 38 | print(round(y_pred, 5)) -------------------------------------------------------------------------------- /0.LogisticRegression(manual)-Softmax.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pure Python 3 | Code by Tae Hwan Jung(@graykode) 4 | """ 5 | import numpy as np 6 | 7 | x_data = [-2.0, -1.0, 1.0, 5.0] 8 | y_data = [1.0, 1.0, 2.0, 3.0] # 3 classes 9 | 10 | # Traninalbe Parameters 11 | w = 0.0 12 | 13 | def softmax(x): 14 | new_x_data = [w * i for i in x_data] 15 | return np.exp(w * x) / np.sum(np.exp(new_x_data)) 16 | 17 | # Loss function, Cross Entropy is loss function in Logistic Regression 18 | # if you see Proof about Partial derivative of J(θ) see, https://deepnotes.io/softmax-crossentropy 19 | def loss(x, y): # Cross Entropy function 20 | return -(y * softmax(x) / len(x_data)) 21 | 22 | # compute gradient 23 | def gradient(x, y): # d_loss/d_w 24 | return softmax(x) - y / len(x_data) 25 | 26 | # Training loop 27 | for epoch in range(50): 28 | l = 0 29 | for x_val, y_val in zip(x_data, y_data): 30 | grad = gradient(x_val, y_val) 31 | w = w - 0.01 * grad # 0.01 is learning rate 32 | print("\tgrad: ", x_val, y_val, round(grad, 2)) 33 | l += loss(x_val, y_val) 34 | print("progress:", epoch, "w=", round(w, 2), "loss=", round(l, 2)) 35 | 36 | # test 37 | for x in x_data: 38 | y_pred = softmax(x) 39 | print(round(y_pred, 5)) -------------------------------------------------------------------------------- /1.LinearRegression(auto)-Multivariate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | 8 | # Answer y = x1*1 + x2*2 + x3*3 9 | x_data = torch.Tensor([ 10 | [1.0, 2.0, 3.0], 11 | [4.0, 5.0, 6.0], 12 | [7.0, 8.0, 9.0], 13 | ]) 14 | y_data = torch.Tensor([ 15 | [14.0], 16 | [32.0], 17 | [50.0] 18 | ]) 19 | 20 | class Model(nn.Module): 21 | def __init__(self): 22 | super(Model, self).__init__() 23 | # Model Parameters in Here 24 | self.w = nn.Linear(3, 1) 25 | 26 | def forward(self, x): 27 | # feeforwarding of Model in Here 28 | y = self.w(x) 29 | return y 30 | 31 | model = Model() 32 | criterion = nn.MSELoss() 33 | optimizer = optim.SGD(model.parameters(), lr=0.01) 34 | 35 | # Before training 36 | print("Parameter w :", model.w.weight) 37 | 38 | for epoch in range(100): 39 | output = model(x_data) 40 | loss = criterion(output, y_data) 41 | 42 | # Zero gradients, perform a backward pass, and update the weights. 43 | optimizer.zero_grad() 44 | loss.backward() 45 | optimizer.step() 46 | 47 | print("progress:", epoch, "loss=", loss.item()) 48 | 49 | # After training 50 | print("Parameter w :", model.w.weight) -------------------------------------------------------------------------------- /1.LinearRegression(auto)-Univariate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | Reference : https://github.com/hunkim/PyTorchZeroToAll/blob/master/05_linear_regression.py 4 | """ 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | 9 | x_data = torch.Tensor([ 10 | [1.0], 11 | [2.0], 12 | [3.0] 13 | ]) 14 | y_data = torch.Tensor([ 15 | [2.0], 16 | [4.0], 17 | [6.0] 18 | ]) 19 | 20 | class Model(nn.Module): 21 | def __init__(self): 22 | super(Model, self).__init__() 23 | # Model Parameters in Here 24 | self.w = nn.Linear(1, 1) 25 | 26 | def forward(self, x): 27 | # feeforwarding of Model in Here 28 | y = self.w(x) 29 | return y 30 | 31 | model = Model() 32 | criterion = nn.MSELoss() 33 | optimizer = optim.SGD(model.parameters(), lr=0.01) 34 | 35 | # Before training 36 | print("Parameter w :", model.w.weight.item(), "Predict y :" , model(torch.Tensor([4.0])).item() ) 37 | 38 | for epoch in range(10): 39 | output = model(x_data) 40 | loss = criterion(output, y_data) 41 | 42 | # Zero gradients, perform a backward pass, and update the weights. 43 | optimizer.zero_grad() 44 | loss.backward() 45 | optimizer.step() 46 | 47 | print("progress:", epoch, "loss=", loss.item()) 48 | 49 | # After training 50 | print("Parameter w :", model.w.weight.item(), "Predict y :" , model(torch.Tensor([4.0])).item() ) -------------------------------------------------------------------------------- /1.LogisticRegression(auto)-Binary.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | 8 | x_data = torch.Tensor([ 9 | [-2.0], 10 | [-1.0], 11 | [1.0], 12 | [5.0] 13 | ]) 14 | 15 | y_data = torch.Tensor([ 16 | [1.0], 17 | [1.0], 18 | [2.0], 19 | [2.0] 20 | ]) 21 | 22 | class Model(nn.Module): 23 | def __init__(self): 24 | super(Model, self).__init__() 25 | # Model Parameters in Here 26 | self.w = nn.Linear(1, 1) 27 | 28 | def forward(self, x): 29 | # feeforwarding of Model in Here 30 | y = self.w(x) 31 | y = torch.sigmoid(y) 32 | return y 33 | 34 | model = Model() 35 | 36 | criterion = torch.nn.BCELoss() 37 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 38 | 39 | for epoch in range(100): 40 | output = model(x_data) 41 | loss = criterion(output, y_data) 42 | 43 | # Zero gradients, perform a backward pass, and update the weights. 44 | optimizer.zero_grad() 45 | loss.backward() 46 | optimizer.step() 47 | 48 | print("progress:", epoch, "loss=", loss.item()) 49 | 50 | # test 51 | for x in x_data: 52 | y_pred = torch.sigmoid(x) 53 | print(y_pred) 54 | print(1 if y_pred >= 0.5 else 0) -------------------------------------------------------------------------------- /1.LogisticRegression(auto)-Softmax.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | 8 | x_data = torch.Tensor([ 9 | [-2.0], 10 | [-1.0], 11 | [1.0], 12 | [5.0] 13 | ]) 14 | 15 | y_data = torch.LongTensor([1, 1, 2, 3]) # 3 classes 16 | 17 | class Model(nn.Module): 18 | def __init__(self): 19 | super(Model, self).__init__() 20 | # Model Parameters in Here 21 | self.w = nn.Linear(1, 4) 22 | 23 | def forward(self, x): 24 | # feeforwarding of Model in Here 25 | y = self.w(x) 26 | return y 27 | 28 | model = Model() 29 | 30 | criterion = torch.nn.CrossEntropyLoss() 31 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 32 | 33 | for epoch in range(10000): 34 | output = model(x_data) 35 | 36 | # when use CrossEntropyLoss, first shape : [batch, n_class], secnod shape : [batch] 37 | loss = criterion(output, y_data) 38 | 39 | # Zero gradients, perform a backward pass, and update the weights. 40 | optimizer.zero_grad() 41 | loss.backward() 42 | optimizer.step() 43 | 44 | print("progress:", epoch, "loss=", loss.item()) 45 | 46 | # test 47 | for x in x_data: 48 | y_pred = model(x) 49 | print(y_pred) 50 | predict = y_pred.max(dim=0)[1].item() 51 | print(predict) -------------------------------------------------------------------------------- /1.Pytorch-Basic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import torch.nn as nn 5 | 6 | class Model(nn.Module): 7 | def __init__(self , W , b): 8 | super(Model, self).__init__() 9 | # Model Parameters in Here 10 | self.W = W 11 | self.b = b 12 | 13 | def forward(self, x): 14 | # feeforwarding of Model in Here 15 | y = self.W * x + self.b 16 | return y 17 | 18 | model = Model(W=10, b=5) 19 | output = model(1) # 10 * 1 + 5 20 | print(output) 21 | 22 | -------------------------------------------------------------------------------- /2.DNN-Dropout.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | Reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/01%20-%20Classification.py 4 | Dropout example 5 | """ 6 | import torch 7 | import torch.nn as nn 8 | 9 | # [fur, wing] 10 | x_data = torch.Tensor([ 11 | [0, 0], 12 | [1, 0], 13 | [0, 0], 14 | [0, 0], 15 | [0, 1] 16 | ]) 17 | 18 | y_data = torch.LongTensor([ 19 | 0, # etc 20 | 1, # mammal 21 | 0, 22 | 0, 23 | 2 # bird 24 | ]) 25 | 26 | new_x_data = torch.Tensor([ 27 | [1, 1], 28 | ]) # answer is 2(bird) 29 | 30 | criterion = torch.nn.CrossEntropyLoss() 31 | 32 | def train(model): 33 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 34 | 35 | for epoch in range(10000): 36 | output = model(x_data) 37 | 38 | # when use CrossEntropyLoss, first shape : [batch, n_class], secnod shape : [batch] 39 | loss = criterion(output, y_data) 40 | 41 | # Zero gradients, perform a backward pass, and update the weights. 42 | optimizer.zero_grad() 43 | loss.backward() 44 | optimizer.step() 45 | 46 | if (epoch + 1)%1000 == 0: 47 | print("progress:", epoch + 1, "loss=", loss.item()) 48 | 49 | def test(): 50 | y_pred = model(new_x_data) 51 | print(y_pred) 52 | predict = y_pred.max(dim=-1)[1].item() 53 | print(predict) 54 | 55 | class ModelDropout(nn.Module): 56 | def __init__(self): 57 | super(ModelDropout, self).__init__() 58 | self.w1 = nn.Linear(2, 10) 59 | self.bias1 = torch.zeros([10]) 60 | self.dropout = nn.Dropout(0.5) 61 | 62 | self.w2 = nn.Linear(10, 3) 63 | self.bias2 = torch.zeros([3]) 64 | self.relu = nn.ReLU() 65 | self.softmax = nn.Softmax(dim=0) 66 | 67 | def forward(self, x): 68 | y = self.w1(x) + self.bias1 69 | y = self.relu(y) 70 | y = self.dropout(y) # make to be comment this line and check diff. 71 | y = self.w2(y) + self.bias2 72 | return y 73 | 74 | model = ModelDropout() 75 | train(model) 76 | test() -------------------------------------------------------------------------------- /2.DNN-Optimization.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | Reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/01%20-%20Classification.py 4 | 2 Layer DNN 5 | """ 6 | import torch 7 | import torch.nn as nn 8 | 9 | # [fur, wing] 10 | x_data = torch.Tensor([ 11 | [0, 0], 12 | [1, 0], 13 | [1, 1], 14 | [0, 0], 15 | [0, 0], 16 | [0, 1] 17 | ]) 18 | 19 | y_data = torch.LongTensor([ 20 | 0, # etc 21 | 1, # mammal 22 | 2, # birds 23 | 0, 24 | 0, 25 | 2 26 | ]) 27 | 28 | class DNN(nn.Module): 29 | def __init__(self): 30 | super(DNN, self).__init__() 31 | self.w1 = nn.Linear(2, 10) 32 | self.bias1 = torch.zeros([10]) 33 | 34 | self.w2 = nn.Linear(10, 3) 35 | self.bias2 = torch.zeros([3]) 36 | self.relu = nn.ReLU() 37 | self.softmax = nn.Softmax(dim=0) 38 | 39 | def forward(self, x): 40 | y = self.w1(x) + self.bias1 41 | y = self.relu(y) 42 | 43 | y = self.w2(y) + self.bias2 44 | return y 45 | 46 | model = DNN() 47 | 48 | criterion = torch.nn.CrossEntropyLoss() 49 | 50 | # See http://shuuki4.github.io/deep%20learning/2016/05/20/Gradient-Descent-Algorithm-Overview.html 51 | # Pytorch https://pytorch.org/docs/stable/optim.html 52 | optimSGD = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=0) 53 | optimAdam = torch.optim.Adam(model.parameters(), lr=0.01) 54 | optimAdagrad = torch.optim.Adagrad(model.parameters(), lr=0.01) 55 | optimAdadelta = torch.optim.Adadelta(model.parameters(), lr=0.01) 56 | optimRMSprop = torch.optim.RMSprop(model.parameters(), lr=0.01) 57 | 58 | for epoch in range(1000): 59 | output = model(x_data) 60 | 61 | # when use CrossEntropyLoss, first shape : [batch, n_class], secnod shape : [batch] 62 | loss = criterion(output, y_data) 63 | 64 | # Zero gradients, perform a backward pass, and update the weights. 65 | optimSGD.zero_grad() 66 | loss.backward() 67 | optimSGD.step() 68 | 69 | print("progress:", epoch, "loss=", loss.item()) 70 | 71 | # test 72 | for x in x_data: 73 | y_pred = model(x) 74 | print(y_pred) 75 | predict = y_pred.max(dim=0)[1].item() 76 | print(predict) -------------------------------------------------------------------------------- /2.DNN1.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | Reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/01%20-%20Classification.py 4 | 1 Layer DNN 5 | """ 6 | import torch 7 | import torch.nn as nn 8 | from torch.autograd import Variable 9 | 10 | # [fur, wing] 11 | x_data = torch.Tensor([ 12 | [0, 0], 13 | [1, 0], 14 | [1, 1], 15 | [0, 0], 16 | [0, 0], 17 | [0, 1] 18 | ]) 19 | 20 | y_data = torch.LongTensor([ 21 | 0, # etc 22 | 1, # mammal 23 | 2, # birds 24 | 0, 25 | 0, 26 | 2 27 | ]) 28 | 29 | class DNN(nn.Module): 30 | def __init__(self): 31 | super(DNN, self).__init__() 32 | self.w = nn.Linear(2, 3) 33 | self.bias = torch.zeros([3]) 34 | self.relu = nn.ReLU() 35 | 36 | def forward(self, x): 37 | y = self.w(x) + self.bias 38 | y = self.relu(y) 39 | return y 40 | 41 | model = DNN() 42 | 43 | criterion = torch.nn.CrossEntropyLoss() 44 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 45 | 46 | for epoch in range(1000): 47 | output = model(x_data) 48 | 49 | # when use CrossEntropyLoss, first shape : [batch, n_class], secnod shape : [batch] 50 | loss = criterion(output, y_data) 51 | 52 | # Zero gradients, perform a backward pass, and update the weights. 53 | optimizer.zero_grad() 54 | loss.backward() 55 | optimizer.step() 56 | 57 | print("progress:", epoch, "loss=", loss.item()) 58 | 59 | # test 60 | for x in x_data: 61 | y_pred = model(x) 62 | print(y_pred) 63 | predict = y_pred.max(dim=0)[1].item() 64 | print(predict) -------------------------------------------------------------------------------- /2.DNN2.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | Reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/01%20-%20Classification.py 4 | 2 Layer DNN 5 | """ 6 | import torch 7 | import torch.nn as nn 8 | 9 | # [fur, wing] 10 | x_data = torch.Tensor([ 11 | [0, 0], 12 | [1, 0], 13 | [1, 1], 14 | [0, 0], 15 | [0, 0], 16 | [0, 1] 17 | ]) 18 | 19 | y_data = torch.LongTensor([ 20 | 0, # etc 21 | 1, # mammal 22 | 2, # birds 23 | 0, 24 | 0, 25 | 2 26 | ]) 27 | 28 | class DNN(nn.Module): 29 | def __init__(self): 30 | super(DNN, self).__init__() 31 | self.w1 = nn.Linear(2, 10) 32 | self.bias1 = torch.zeros([10]) 33 | 34 | self.w2 = nn.Linear(10, 3) 35 | self.bias2 = torch.zeros([3]) 36 | self.relu = nn.ReLU() 37 | self.softmax = nn.Softmax(dim=0) 38 | 39 | def forward(self, x): 40 | y = self.w1(x) + self.bias1 41 | y = self.relu(y) 42 | 43 | y = self.w2(y) + self.bias2 44 | return y 45 | 46 | model = DNN() 47 | 48 | criterion = torch.nn.CrossEntropyLoss() 49 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 50 | 51 | for epoch in range(1000): 52 | output = model(x_data) 53 | 54 | # when use CrossEntropyLoss, first shape : [batch, n_class], secnod shape : [batch] 55 | loss = criterion(output, y_data) 56 | 57 | # Zero gradients, perform a backward pass, and update the weights. 58 | optimizer.zero_grad() 59 | loss.backward() 60 | optimizer.step() 61 | 62 | print("progress:", epoch, "loss=", loss.item()) 63 | 64 | # test 65 | for x in x_data: 66 | y_pred = model(x) 67 | print(y_pred) 68 | predict = y_pred.max(dim=0)[1].item() 69 | print(predict) -------------------------------------------------------------------------------- /3.DataLoader-Cifar10.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import numpy as np 5 | import torch 6 | import matplotlib.pyplot as plt 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | 10 | classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') 11 | 12 | mnistdata = datasets.CIFAR10('./data/', train=True, download=True, transform=transforms.ToTensor()) 13 | print('number of image : ',len(mnistdata)) 14 | 15 | batch_size = 10 16 | print('Data Loader') 17 | data_loader = torch.utils.data.DataLoader(dataset=mnistdata, batch_size=batch_size, shuffle=True) 18 | 19 | count = 0 20 | for batch_idx, (data, targets) in enumerate(data_loader): 21 | targets = [classes[target] for target in targets] 22 | data = Variable(data) 23 | count += batch_size 24 | print('batch :', batch_idx + 1,' ', count, '/', len(mnistdata), 25 | 'image:', data.shape, 'target : ', targets) -------------------------------------------------------------------------------- /3.DataLoader-Cifar100.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import numpy as np 5 | import torch 6 | import matplotlib.pyplot as plt 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | 10 | classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') 11 | 12 | mnistdata = datasets.CIFAR100('./data/', train=True, download=True, transform=transforms.ToTensor()) 13 | print('number of image : ',len(mnistdata)) 14 | 15 | batch_size = 10 16 | print('Data Loader') 17 | data_loader = torch.utils.data.DataLoader(dataset=mnistdata, batch_size=batch_size, shuffle=True) 18 | 19 | count = 0 20 | for batch_idx, (data, targets) in enumerate(data_loader): 21 | targets = [classes[target] for target in targets] 22 | data = Variable(data) 23 | count += batch_size 24 | print('batch :', batch_idx + 1,' ', count, '/', len(mnistdata), 25 | 'image:', data.shape, 'target : ', targets) -------------------------------------------------------------------------------- /3.DataLoader-ImageFolder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import numpy as np 5 | import torch 6 | import matplotlib.pyplot as plt 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | 10 | # image from https://github.com/ardamavi/Dog-Cat-Classifier/tree/master/Data/Train_Data 11 | # 0 : cat, 1 : dog 12 | 13 | def img_show(image): 14 | image = image / 2 + 0.5 15 | plt.imshow(np.transpose(image.numpy(), (1, 2, 0))) 16 | plt.show(block=False) 17 | 18 | def pick_image(data, index): 19 | image, target = data[index] 20 | img_show(image) 21 | 22 | original_image = datasets.ImageFolder(root='../data/', transform=transforms.ToTensor()) 23 | print(original_image,'\n') 24 | pick_image(original_image, 1) 25 | 26 | # make transformation (resizing image) 27 | resized_transform = transforms.Compose([ 28 | transforms.Resize((227, 227)), 29 | transforms.ToTensor() 30 | ]) 31 | resized_image = datasets.ImageFolder(root='../data/', transform=resized_transform) 32 | print('resized image to 227x227x3') 33 | pick_image(resized_image, 1) 34 | 35 | # make transformation (crop image) 36 | cropped_transform = transforms.Compose([ 37 | transforms.CenterCrop((10, 10)), 38 | transforms.ToTensor() 39 | ]) 40 | cropped_image = datasets.ImageFolder(root='../data/', transform=cropped_transform) 41 | print('cropped image to 10x10x3') 42 | pick_image(cropped_image, 1) 43 | 44 | # make transformation (normalized image) 45 | normalized_transform = transforms.Compose([ 46 | transforms.ToTensor(), 47 | transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) 48 | ]) 49 | 50 | normalized_image = datasets.ImageFolder(root='../data/', transform=normalized_transform) 51 | print('normalized image to mean and std (0.5, 0.5, 0.5)') 52 | pick_image(normalized_image, 1) 53 | 54 | 55 | # How to use Data Loader (Input Pipeline) 56 | # same batch images should have same height, weight, channel 57 | batch_size = 2 58 | print('Data Loader') 59 | dataloader = torch.utils.data.DataLoader(dataset=resized_image, batch_size=batch_size, shuffle=True) 60 | 61 | count = 0 62 | for batch_idx, (data, target) in enumerate(dataloader): 63 | data, target = Variable(data), Variable(target) 64 | count += batch_size 65 | print('batch :', batch_idx + 1,' ', count, '/', len(original_image), 66 | 'image:', data.shape, 'target : ', target) -------------------------------------------------------------------------------- /3.DataLoader-MNIST.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import numpy as np 5 | import torch 6 | import matplotlib.pyplot as plt 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | 10 | 11 | mnistdata = datasets.MNIST('./data/', train=True, download=True, transform=transforms.ToTensor()) 12 | print('number of image : ',len(mnistdata)) 13 | 14 | batch_size = 10 15 | print('Data Loader') 16 | data_loader = torch.utils.data.DataLoader(dataset=mnistdata, batch_size=batch_size, shuffle=True) 17 | 18 | count = 0 19 | for batch_idx, (data, target) in enumerate(data_loader): 20 | data, target = Variable(data), Variable(target) 21 | count += batch_size 22 | print('batch :', batch_idx + 1,' ', count, '/', len(mnistdata), 23 | 'image:', data.shape, 'target : ', target) -------------------------------------------------------------------------------- /4.CNN-Introduce.py: -------------------------------------------------------------------------------- 1 | ''' 2 | code by Tae Hwan Jung @graykode 3 | ''' 4 | import numpy as np 5 | import torch.nn as nn 6 | import matplotlib.pyplot as plt 7 | from torchvision import datasets, transforms 8 | 9 | def img_show(image): 10 | image = image / 2 + 0.5 11 | if image.shape[0] == 3: 12 | plt.imshow(np.transpose(image.numpy(), (1, 2, 0))) 13 | elif image.shape[0] == 1: 14 | plt.imshow(image.squeeze(0)) 15 | plt.show(block=False) 16 | 17 | transform = transforms.Compose([ 18 | transforms.Resize((227, 227)), 19 | transforms.ToTensor(), 20 | ]) 21 | dataset = datasets.ImageFolder(root='../data/', transform=transform) 22 | cat, dog = dataset[0][0], dataset[1][0] 23 | 24 | # What is filter(=kernel) in CNN 25 | print('original image') 26 | img_show(cat) 27 | 28 | print('in_channels=3, out_channels=6, kernel_size=4 Convolution') 29 | outputs = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=4)(cat.unsqueeze(0)).data 30 | for i in range(outputs.shape[1]): 31 | print(i+1,'channel') 32 | img_show(outputs[:,i,:,:]) 33 | 34 | print('in_channels=3, out_channels=6, kernel_size=40 Convolution') 35 | outputs = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=40)(cat.unsqueeze(0)).data 36 | for i in range(outputs.shape[1]): 37 | print(i+1,'channel') 38 | img_show(outputs[:,i,:,:]) 39 | 40 | print('in_channels=3, out_channels=6, kernel_size=3 stride=4 Convolution') 41 | outputs = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=3, stride=4)(cat.unsqueeze(0)).data 42 | for i in range(outputs.shape[1]): 43 | print(i+1,'channel') 44 | img_show(outputs[:,i,:,:]) -------------------------------------------------------------------------------- /4.CNN-MNIST(Inference).py: -------------------------------------------------------------------------------- 1 | ''' 2 | code by Tae Hwan Jung @graykode 3 | reference : https://github.com/pytorch/examples/blob/master/mnist/main.py 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import matplotlib.pyplot as plt 9 | import torch.optim as optim 10 | from torchvision import datasets, transforms 11 | 12 | class Model(nn.Module): 13 | def __init__(self): 14 | super(Model, self).__init__() 15 | self.conv1 = nn.Conv2d(in_channels=1, out_channels=20, kernel_size=5, stride=1) 16 | self.conv2 = nn.Conv2d(in_channels=20, out_channels=50, kernel_size=5, stride=1) 17 | self.fc1 = nn.Linear(4 * 4 * 50, 500) 18 | self.fc2 = nn.Linear(500, 10) 19 | 20 | def forward(self, x): 21 | x = F.relu(self.conv1(x)) 22 | x = F.max_pool2d(x, kernel_size=2, stride=2) 23 | x = F.relu(self.conv2(x)) 24 | x = F.max_pool2d(x, kernel_size=2, stride=2) 25 | 26 | x = x.view(-1, 4 * 4 * 50) # [batch_size, 50, 4, 4] 27 | x = F.relu(self.fc1(x)) 28 | x = self.fc2(x) 29 | return x 30 | 31 | def img_show(image): 32 | plt.imshow(image.numpy().reshape((28, 28)), cmap='gray') 33 | plt.show(block=False) 34 | 35 | if __name__ == '__main__': 36 | use_cuda = torch.cuda.is_available() 37 | device = torch.device("cuda" if use_cuda else "cpu") 38 | 39 | transform = transforms.Compose([ 40 | transforms.ToTensor(), 41 | transforms.Normalize((0.1307,), (0.3081,)) 42 | ]) 43 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 44 | 45 | test_data = datasets.MNIST('./data', train=False, transform=transform) 46 | 47 | model = Model().to(device) 48 | model.load_state_dict(torch.load('mnist_cnn.pt', map_location=lambda storage, loc: storage)) 49 | model.eval() 50 | 51 | criterion = torch.nn.CrossEntropyLoss() 52 | optimizer = optim.SGD(model.parameters(), lr=0.01) 53 | 54 | with torch.no_grad(): 55 | for i in range(10): 56 | data = test_data[i][0] 57 | target = test_data[i][1] 58 | img_show(data) 59 | 60 | data = data.unsqueeze(0) 61 | data = data.to(device) 62 | 63 | output = model(data) 64 | predict = output.max(dim=1)[1] 65 | print('model predict : ', predict, ' answer : ',target) -------------------------------------------------------------------------------- /4.CNN-MNIST(Train).py: -------------------------------------------------------------------------------- 1 | ''' 2 | code by Tae Hwan Jung @graykode 3 | reference : https://github.com/pytorch/examples/blob/master/mnist/main.py 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import torch.optim as optim 9 | from torchvision import datasets, transforms 10 | 11 | class Model(nn.Module): 12 | def __init__(self): 13 | super(Model, self).__init__() 14 | self.conv1 = nn.Conv2d(in_channels=1, out_channels=20, kernel_size=5, stride=1) 15 | self.conv2 = nn.Conv2d(in_channels=20, out_channels=50, kernel_size=5, stride=1) 16 | self.fc1 = nn.Linear(4 * 4 * 50, 500) 17 | self.fc2 = nn.Linear(500, 10) 18 | 19 | def forward(self, x): 20 | x = F.relu(self.conv1(x)) 21 | x = F.max_pool2d(x, kernel_size=2, stride=2) 22 | x = F.relu(self.conv2(x)) 23 | x = F.max_pool2d(x, kernel_size=2, stride=2) 24 | 25 | x = x.view(-1, 4 * 4 * 50) # [batch_size, 50, 4, 4] 26 | x = F.relu(self.fc1(x)) 27 | x = self.fc2(x) 28 | return x 29 | 30 | def train(model, device, train_loader, epoch, optimizer, criterion): 31 | model.train() 32 | for batch_idx, (data, target) in enumerate(train_loader): 33 | data, target = data.to(device), target.to(device) 34 | optimizer.zero_grad() 35 | output = model(data) 36 | loss = criterion(output, target) 37 | loss.backward() 38 | optimizer.step() 39 | 40 | if batch_idx % 20 == 0: 41 | print('Train Epoch: {} [{}/{} ({:.0f}%)] Loss: {:.6f}'.format( 42 | epoch, batch_idx * len(data), len(train_loader.dataset), 43 | 100. * batch_idx / len(train_loader), loss.item())) 44 | 45 | def test(model, device, test_loader, criterion): 46 | model.eval() 47 | test_loss = 0 48 | correct = 0 49 | with torch.no_grad(): 50 | for data, target in test_loader: 51 | data, target = data.to(device), target.to(device) 52 | output = model(data) 53 | test_loss += criterion(output, target).item() # sum up batch loss 54 | pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability 55 | correct += pred.eq(target.view_as(pred)).sum().item() 56 | 57 | test_loss /= len(test_loader.dataset) 58 | 59 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 60 | test_loss, correct, len(test_loader.dataset), 61 | 100. * correct / len(test_loader.dataset))) 62 | 63 | if __name__ == '__main__': 64 | 65 | batch_size = 100 66 | epochs = 50 67 | 68 | use_cuda = torch.cuda.is_available() 69 | device = torch.device("cuda" if use_cuda else "cpu") 70 | 71 | transform = transforms.Compose([ 72 | transforms.ToTensor(), 73 | transforms.Normalize((0.1307,), (0.3081,)) 74 | ]) 75 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 76 | train_data = datasets.MNIST('./data', train=True, download=True, transform=transform) 77 | train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True, **kwargs) 78 | 79 | test_data = datasets.MNIST('./data', train=False, transform=transform) 80 | test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, shuffle=True, **kwargs) 81 | 82 | model = Model().to(device) 83 | criterion = torch.nn.CrossEntropyLoss() 84 | optimizer = optim.SGD(model.parameters(), lr=0.01) 85 | 86 | for epoch in range(1, epochs + 1): 87 | train(model, device, train_loader, epoch, optimizer, criterion) 88 | test(model, device, test_loader, criterion) 89 | 90 | torch.save(model.state_dict(),"mnist_cnn.pt") 91 | 92 | -------------------------------------------------------------------------------- /5.RNN-Introduce.py: -------------------------------------------------------------------------------- 1 | """ 2 | code by Tae Hwan Jung(Jeff Jung) @graykode 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | 7 | batch = 1 8 | seq_len = 5 9 | input_size = 10 10 | hidden_size= 3 11 | 12 | rnn = nn.RNN(input_size=input_size, hidden_size=hidden_size, batch_first=True) 13 | 14 | # input of shape (batch, seq_len, input_size) 15 | input = torch.rand([batch, seq_len, input_size]) 16 | 17 | # h_0 of shape (num_layers * num_directions, batch, hidden_size) 18 | hidden = torch.rand([1 * 1, batch, hidden_size]) 19 | 20 | # output of shape (batch, seq_len, num_directions * hidden_size) 21 | # final_hidden of shape (num_layers * num_directions, batch, hidden_size) 22 | output, final_hidden = rnn(input, hidden) 23 | print('final_hidden shape :', final_hidden.shape) 24 | print(final_hidden) 25 | 26 | # final_hidden state is same with last time seqeuence output 27 | for index, out in enumerate(output[0]): 28 | print(index + 1, 'time seqeuence output :', out) -------------------------------------------------------------------------------- /5.RNN-PredictWord.py: -------------------------------------------------------------------------------- 1 | """ 2 | code by Tae Hwan Jung(Jeff Jung) @graykode 3 | """ 4 | import numpy as np 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | from torch.autograd import Variable 9 | 10 | dtype = torch.FloatTensor 11 | 12 | sentences = [ "i like dog", "i love coffee", "i hate milk"] 13 | 14 | word_list = " ".join(sentences).split() 15 | word_list = list(set(word_list)) 16 | word_dict = {w: i for i, w in enumerate(word_list)} 17 | number_dict = {i: w for i, w in enumerate(word_list)} 18 | n_class = len(word_dict) 19 | 20 | # TextRNN Parameter 21 | batch_size = len(sentences) 22 | n_step = 2 # number of cells(= number of Step) 23 | n_hidden = 5 # number of hidden units in one cell 24 | 25 | def make_batch(sentences): 26 | input_batch = [] 27 | target_batch = [] 28 | 29 | for sen in sentences: 30 | word = sen.split() 31 | input = [word_dict[n] for n in word[:-1]] 32 | target = word_dict[word[-1]] 33 | 34 | input_batch.append(np.eye(n_class)[input]) 35 | target_batch.append(target) 36 | 37 | return input_batch, target_batch 38 | 39 | # to Torch.Tensor 40 | input_batch, target_batch = make_batch(sentences) 41 | input_batch = Variable(torch.Tensor(input_batch)) 42 | target_batch = Variable(torch.LongTensor(target_batch)) 43 | 44 | class TextRNN(nn.Module): 45 | def __init__(self): 46 | super(TextRNN, self).__init__() 47 | 48 | self.rnn = nn.RNN(input_size=n_class, hidden_size=n_hidden) 49 | self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype)) 50 | self.b = nn.Parameter(torch.randn([n_class]).type(dtype)) 51 | 52 | def forward(self, hidden, X): 53 | X = X.transpose(0, 1) # X : [n_step, batch_size, n_class] 54 | outputs, hidden = self.rnn(X, hidden) 55 | # outputs : [n_step, batch_size, num_directions(=1) * n_hidden] 56 | # hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden] 57 | outputs = outputs[-1] # [batch_size, num_directions(=1) * n_hidden] 58 | model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class] 59 | return model 60 | 61 | model = TextRNN() 62 | 63 | criterion = nn.CrossEntropyLoss() 64 | optimizer = optim.Adam(model.parameters(), lr=0.001) 65 | 66 | # Training 67 | for epoch in range(5000): 68 | # hidden : [num_layers * num_directions, batch, hidden_size] 69 | hidden = Variable(torch.zeros(1, batch_size, n_hidden)) 70 | # input_batch : [batch_size, n_step, n_class] 71 | output = model(hidden, input_batch) 72 | 73 | # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot) 74 | loss = criterion(output, target_batch) 75 | if (epoch + 1) % 1000 == 0: 76 | print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) 77 | 78 | optimizer.zero_grad() 79 | loss.backward() 80 | optimizer.step() 81 | 82 | input = [sen.split()[:2] for sen in sentences] 83 | 84 | # Predict 85 | hidden = Variable(torch.zeros(1, batch_size, n_hidden)) 86 | predict = model(hidden, input_batch).data.max(1, keepdim=True)[1] 87 | print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()]) -------------------------------------------------------------------------------- /6.LSTM-AutoComplete.py: -------------------------------------------------------------------------------- 1 | """ 2 | code by Tae Hwan Jung(Jeff Jung) @graykode 3 | """ 4 | import numpy as np 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | from torch.autograd import Variable 9 | 10 | dtype = torch.FloatTensor 11 | 12 | char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz'] 13 | word_dict = {n: i for i, n in enumerate(char_arr)} 14 | number_dict = {i: w for i, w in enumerate(char_arr)} 15 | n_class = len(word_dict) # number of class(=number of vocab) 16 | 17 | seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star'] 18 | 19 | # TextLSTM Parameters 20 | n_step = 3 21 | n_hidden = 128 22 | 23 | def make_batch(seq_data): 24 | input_batch, target_batch = [], [] 25 | 26 | for seq in seq_data: 27 | input = [word_dict[n] for n in seq[:-1]] # 'm', 'a' , 'k' is input 28 | target = word_dict[seq[-1]] # 'e' is target 29 | input_batch.append(np.eye(n_class)[input]) 30 | target_batch.append(target) 31 | 32 | return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch)) 33 | 34 | class TextLSTM(nn.Module): 35 | def __init__(self): 36 | super(TextLSTM, self).__init__() 37 | 38 | self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden) 39 | self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype)) 40 | self.b = nn.Parameter(torch.randn([n_class]).type(dtype)) 41 | 42 | def forward(self, X): 43 | input = X.transpose(0, 1) # X : [n_step, batch_size, n_class] 44 | 45 | hidden_state = Variable(torch.zeros(1, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden] 46 | cell_state = Variable(torch.zeros(1, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden] 47 | 48 | outputs, (_, _) = self.lstm(input, (hidden_state, cell_state)) 49 | outputs = outputs[-1] # [batch_size, n_hidden] 50 | model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class] 51 | return model 52 | 53 | input_batch, target_batch = make_batch(seq_data) 54 | 55 | model = TextLSTM() 56 | 57 | criterion = nn.CrossEntropyLoss() 58 | optimizer = optim.Adam(model.parameters(), lr=0.001) 59 | 60 | output = model(input_batch) 61 | 62 | # Training 63 | for epoch in range(1000): 64 | optimizer.zero_grad() 65 | 66 | output = model(input_batch) 67 | loss = criterion(output, target_batch) 68 | if (epoch + 1) % 100 == 0: 69 | print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) 70 | 71 | loss.backward() 72 | optimizer.step() 73 | 74 | inputs = [sen[:3] for sen in seq_data] 75 | 76 | predict = model(input_batch).data.max(1, keepdim=True)[1] 77 | print(inputs, '->', [number_dict[n.item()] for n in predict.squeeze()]) -------------------------------------------------------------------------------- /6.LSTM-Introduce.py: -------------------------------------------------------------------------------- 1 | """ 2 | code by Tae Hwan Jung(Jeff Jung) @graykode 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | 7 | batch = 1 8 | seq_len = 5 9 | input_size = 10 10 | hidden_size= 3 11 | 12 | lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, batch_first=True) 13 | 14 | # input of shape (batch, seq_len, input_size) 15 | input = torch.rand([batch, seq_len, input_size]) 16 | 17 | # h_0 of shape (num_layers * num_directions, batch, hidden_size) 18 | # c_0 of shape (num_layers * num_directions, batch, hidden_size) 19 | hidden_state = torch.rand([1 * 1, batch, hidden_size]) 20 | cell_state = torch.rand([1 * 1, batch, hidden_size]) 21 | 22 | # output of shape (batch, seq_len, num_directions * hidden_size) 23 | # final_hidden_state of shape (num_layers * num_directions, batch, hidden_size) 24 | # final_cell_state of shape (num_layers * num_directions, batch, hidden_size) 25 | output, (final_hidden_state, final_cell_state) = lstm(input, (hidden_state, cell_state)) 26 | print('final_hidden_state shape :', final_hidden_state.shape) 27 | print(final_hidden_state) 28 | 29 | print('final_cell_state shape :', final_cell_state.shape) 30 | print(final_cell_state) 31 | 32 | # final_hidden state is same with last time seqeuence output 33 | for index, out in enumerate(output[0]): 34 | print(index + 1, 'time seqeuence output :', out) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## DeepLearning Basic Study 2 | 3 |

4 | 5 | 6 | 7 |

8 | 9 | 10 | This is repository for Deep Learning Study in Kyung Hee University Computer Engineering Club `D.COM`. 11 | 12 | #### Recommend this study to those who want to review the Machine Learning concept again and to those who have just learned Python. 13 | 14 | - I've created a course material that will be accessible to the first person to start Python. 15 | - [Tae Hwan Jung(@graykode)](https://github.com/graykode) will lead this Study with **Pytorch** for DeepLearning Framework. But I will implement Tensorflow, Pytorch, Keras for beginner. 16 | - We deal with **basic mathematical theory** and **basic models in Deep Learning** such as ` DNN, CNN, RNN, LSTM ` in 1st Study **All of Code were implemented with less than 30 lines.** 17 | - We will use `Google Colaboratory` GPU for memory resource so you can run easily in Colab link.(Thank for Google!) 18 | - First, I made lecture with page link material in Korean, only wrote Contents in English 19 | 20 | 21 | ### Contribution Guide 22 | If you find English link or helpful link irrespective of language, Please give me contribution in README, Markdown like this. 23 | > Linear Regression([Eng[(your contribution link), Kor) 24 | 25 | 26 | ## Curriculum 27 | Please see down Contents. 28 | - 1 Weeks 29 | - Basic Probability Review 30 | - Supervisor Learning vs. Un-supervisor Learning 31 | - Linear Regression, Logistic Regression `manual` Gradient Descent implementation using `pure python` 32 | - 2 Weeks 33 | - method using Google Colaboratory. 34 | - Linear Regression, Logistic Regression Review, Convert `manual` to `auto` implementation using `Pytorch` 35 | - 3 Weeks 36 | - Classification with DNN(Deep Neural Network) in `Pytorch` 37 | - apply Regularization(DropOut) concept to DNN 38 | - Optimization function in `Pytorch`, mini-batch, SGD, Adagrad, RMSProp, AdaDelta, Adam optimizer 39 | - 4 Weeks 40 | - Basic Convolution Neural Network 41 | - load dataset and use data loader with `torchvision` 42 | - apply Machine Learning Diagnostic(Train Set, Cross Validation Set, Test Set) concept to DNN 43 | - Implementation MNIST Classification using CNN 44 | - 5 Weeks 45 | - Basic RNN(Recurrent Neural Network) and LSTM in Pytorch 46 | - Teacher Forcing vs. No Teacher Forcing 47 | - Practice : Predict Next word using RNN or LSTM 48 | - 6 Weeks - Hackathon 49 | - Topic1 : Classification Cat , Dog Image, [Dataset](https://github.com/ardamavi/Dog-Cat-Classifier/tree/master/Data/Train_Data) 50 | - Topic2 : Classification Positive or Negative Korean Naver Movie, [Dataset](https://github.com/e9t/nsmc) 51 | 52 | 53 | ## Contents 54 | 55 | #### 0. Review Basic mathematical Theory with pure `Python` 56 | - **Supervisor Learning vs. Unsupervisor Learning : In this Study, We will deal with only supervisor concept model.** 57 | 58 | - Basic Probability Review 59 | - Bayers Theorem(Eng, [Kor](https://taeoh-kim.github.io/blog/bayes-theorem%EA%B3%BC-sigmoid%EC%99%80-softmax%EC%82%AC%EC%9D%B4%EC%9D%98-%EA%B4%80%EA%B3%84/)), Bayesian inference in Generative Model(Eng, [Kor](https://statkclee.github.io/statistics/bayesian-ab-testing-in-practice.html)) 60 | - Generative Model vs. Discriminative Model(Eng, [Kor](http://sanghyukchun.github.io/61/)) 61 | - Maximum Likelihood vs. Maximum A Posteriori(Eng, [Kor](https://darkpgmr.tistory.com/62)) 62 | - Maximizing Likelihood is Minimizing Cross-Entropy(Eng, [Kor](https://taeoh-kim.github.io/blog/cross-entropy%EC%9D%98-%EC%A0%95%ED%99%95%ED%95%9C-%ED%99%95%EB%A5%A0%EC%A0%81-%EC%9D%98%EB%AF%B8/)) 63 | - Linear Regression(Eng, [Kor](https://wikidocs.net/4212)) 64 | - Univariate Linear Regression(Eng, [Kor](https://wikidocs.net/4213)) vs. Multivariate Linear Regression(Eng, [Kor](https://wikidocs.net/7639)) 65 | - **loss function and activation function** in Linear Regression 66 | - activation function : identity map(Eng, [Kor](https://ko.wikipedia.org/wiki/%ED%95%AD%EB%93%B1_%ED%95%A8%EC%88%98)) 67 | - loss function : MSE function([Eng, Kor](https://en.wikipedia.org/wiki/Mean_squared_error)) 68 | - Gradient Descent in Linear Regression 69 | - manual : [0.LinearRegression(manual)-Univariate.py](https://github.com/graykode/DeepLearning-Study/blob/master/0.LinearRegression(manual)-Univariate.py) 70 | - manual : [0.LinearRegression(manual)-Multivariate.py](https://github.com/graykode/DeepLearning-Study/blob/master/0.LinearRegression(manual)-Multivariate.py) 71 | - Problem : XOR 72 | 73 | - Logistic Regression 74 | - What is different with Linear Regression?(Eng, [Kor](https://wikidocs.net/4267)) 75 | - **loss function and activation function** in Logistic Regression 76 | - activation function : [sigmoid ](https://en.wikipedia.org/wiki/Sigmoid_function)vs. [tanh](https://en.wikipedia.org/wiki/Hyperbolic_function) vs. [ReLu](https://en.wikipedia.org/wiki/Rectifier_(neural_networks)) vs. [Softmax](https://en.wikipedia.org/wiki/Softmax_function) 77 | - loss function : Maximizing Likelihood is Minimizing Cross-Entropy(Eng, [Kor](https://taeoh-kim.github.io/blog/cross-entropy%EC%9D%98-%EC%A0%95%ED%99%95%ED%95%9C-%ED%99%95%EB%A5%A0%EC%A0%81-%EC%9D%98%EB%AF%B8/)) 78 | - Gradient Descent in Logistic Regression 79 | - manual : [0.LogisticRegression(manual)-Binary.py](https://github.com/graykode/DeepLearning-Study/blob/master/0.LogisticRegression(manual)-Binary.py) 80 | - manual : [0.LogisticRegression(manual)-Softmax.py](https://github.com/graykode/DeepLearning-Study/blob/master/0.LogisticRegression(manual)-Softmax.py) 81 | - different with binary classification and multi classification(sigmoid vs. Softmax)(Eng, [Kor1](https://wikidocs.net/4291), [Kor2](https://taeoh-kim.github.io/blog/bayes-theorem%EA%B3%BC-sigmoid%EC%99%80-softmax%EC%82%AC%EC%9D%B4%EC%9D%98-%EA%B4%80%EA%B3%84/)) 82 | - different with Multi-Classification and Multi-labels Classification([Eng](https://stats.stackexchange.com/questions/11859/what-is-the-difference-between-multiclass-and-multilabel-problem), Kor) 83 | 84 | - Optimizing 85 | - What is batch and mini-batch?(Eng, [Kor](http://shuuki4.github.io/deep%20learning/2016/05/20/Gradient-Descent-Algorithm-Overview.html)) 86 | - role of Momentum(Eng, [Kor](http://shuuki4.github.io/deep%20learning/2016/05/20/Gradient-Descent-Algorithm-Overview.html)) 87 | - SGD, Adagrad, RMSProp, AdaDelta, Adam optimizer([Eng](http://ruder.io/optimizing-gradient-descent/?fbclid=IwAR3-EUWRXxLwNlGIEBaETVeVU9VOnDH8hIlp1PJvMG0StbM72gEKMpWA_VA), [Kor](http://shuuki4.github.io/deep%20learning/2016/05/20/Gradient-Descent-Algorithm-Overview.html)) : [2.DNN-Optimization.py](https://github.com/graykode/DeepLearning-Study/blob/master/2.DNN-Optimization.py) 88 | 89 | - Regularization 90 | - What is Overfitting?(Eng, [Kor](https://wikidocs.net/4269)) 91 | - Regularization : weight decay 92 | - weight decay : Linear Regression(Eng, [Kor](https://wikidocs.net/4330)) 93 | - weight decay : Logistic Regression(Eng, [Kor](https://wikidocs.net/4331)) 94 | - Regularization : dropout(Eng, [Kor](https://pythonkim.tistory.com/42)) 95 | 96 | - Machine Learning Diagnostic 97 | - Train Set, Cross Validation Set, Test Set(Eng, [Kor](https://wikidocs.net/4656)) 98 | - Bias vs. Variance(Eng, [Kor](https://wikidocs.net/4657)) 99 | - Learning Curves(Eng, [Kor](https://wikidocs.net/4658)) 100 | 101 | 102 | #### 1.DeepLearning FrameWork Basic 103 | - Abstract Model using Pytorch Class : [1.Pytorch-Basic.py](https://github.com/graykode/DeepLearning-Study/blob/master/1.Pytorch-Basic.py) 104 | - method using Google Colaboratory 105 | - Convert `manual gradient descent` to `auto graident descent` 106 | - [1.LinearRegression(auto)-Univariate.py](https://github.com/graykode/DeepLearning-Study/blob/master/1.LinearRegression(auto)-Univariate.py) 107 | - [1.LinearRegression(auto)-Multivariate.py](https://github.com/graykode/DeepLearning-Study/blob/master/1.LinearRegression(auto)-Multivariate.py) 108 | - [1.LogisticRegression(auto)-Binary.py](https://github.com/graykode/DeepLearning-Study/blob/master/1.LogisticRegression(auto)-Binary.py) 109 | - [1.LogisticRegression(auto)-Softmax.py](https://github.com/graykode/DeepLearning-Study/blob/master/1.LogisticRegression(auto)-Softmax.py) 110 | 111 | 112 | #### 2.DNN(Deep Neural Network) 113 | - Mathematical Back Propagation in Deep Neural Network(Eng, [Kor1](https://wikidocs.net/4262), [Kor2](https://wikidocs.net/4279)) 114 | - Basic Classification using Deep Neural Network 115 | - ~~Classification : Linear Regression in Deep Neural Network~~ 116 | - Classification : Logistic Regression in Deep Neural Network 117 | - 1 Layer Classification : [2.DNN-LinearRegression1.py](https://github.com/graykode/DeepLearning-Study/blob/master/2.DNN-LinearRegression1.py) 118 | - 2 Layers Classification : [2.DNN-LinearRegression2.py](https://github.com/graykode/DeepLearning-Study/blob/master/2.DNN-LinearRegression2.py) 119 | - Dropout in Deep Neural Network : [2.DNN-Dropout.py](https://github.com/graykode/DeepLearning-Study/blob/master/2.DNN-Dropout.py) 120 | 121 | 122 | #### 3.DataLoader and basic Dataset and Image handler 123 | - MNIST : [3.DataLoader-MNIST.py](https://github.com/graykode/DeepLearning-Study/blob/master/3.DataLoader-MNIST.py) 124 | - Cifar10 : [3.DataLoader-Cifar10.py](https://github.com/graykode/DeepLearning-Study/blob/master/3.DataLoader-Cifar10.py) 125 | - Cifar100 : [3.DataLoader-Cifar100.py](https://github.com/graykode/DeepLearning-Study/blob/master/3.DataLoader-Cifar100.py) 126 | - Image Folder : [3.DataLoader-ImageFolder.py](https://github.com/graykode/DeepLearning-Study/blob/master/3.DataLoader-ImageFolder.py) 127 | 128 | 129 | #### 4.CNN(Convolution Neural Network) 130 | - [awesome lecture](https://stanford.edu/~shervine/teaching/cs-230/cheatsheet-convolutional-neural-networks?fbclid=IwAR21k7YvRmCC1RqAJznzLjDPEf8EaZ2jBGeevX4GkiXruocr1akBAIX9-4U) 131 | - Structure of CNN 132 | - [4.CNN-Introduce.py](https://github.com/graykode/DeepLearning-Study/blob/master/4.CNN-Introduce.py) 133 | - Convolutional Layer 134 | - Role of filter(=kernel) vs. receptive fields 135 | - Role of Padding 136 | - Weight sharing in Convolutional Layer 137 | - Role of Channel, Reason using Multi Channel 138 | - Weight sharing in CNN 139 | - Pooling Layer 140 | - Max Pooling 141 | - Average Pooling 142 | - FeedForward in Convolution Neural Network 143 | - Mathematical Back Propagation in Convolution Neural Network 144 | - Practice : Classification MNIST 145 | 146 | 147 | #### 5.RNN(Recurrent Neural Network) 148 | - [awesome lecture](https://stanford.edu/~shervine/teaching/cs-230/cheatsheet-recurrent-neural-networks?fbclid=IwAR0rE5QoMJ3l005fhvqoer0Jo_6GiXAF8XM86iWCXD78e3Ud_nDtw_NGzzY) 149 | - Structure of RNN 150 | - [5.RNN-Introduce.py](https://github.com/graykode/DeepLearning-Study/blob/master/5.RNN-Introduce.py) 151 | - One-to-one vs. One-to-many vs. Many-to-one vs. Many-to-many 152 | - Hidden State 153 | - Output Layer 154 | - Weight sharing in RNN 155 | - [Teacher Forcing vs. No Teacher Forcing](https://www.quora.com/What-is-the-teacher-forcing-in-RNN?awc=15748_1550222926_22685c53296b51dfb8cb6be25b6ce096&uiv=6&txtv=8&source=awin&medium=ad&campaign=uad_mkt_en_acq_us_awin&set=awin) 156 | - FeedForward in Recurrent Neural Network(Eng, [Kor](https://ratsgo.github.io/natural%20language%20processing/2017/03/09/rnnlstm/)) 157 | - Mathematical Back Propagation in Recurrent Neural Network(Eng, [Kor](https://ratsgo.github.io/natural%20language%20processing/2017/03/09/rnnlstm/)) 158 | - Practice : [Predict Next word using RNN](https://github.com/graykode/DeepLearning-Study/blob/master/5.RNN-PredictWord.py) 159 | 160 | 161 | #### 6.LSTM(Long Short Term Memory) 162 | - Structure of LSTM 163 | - [6.LSTM-Introduce.py](https://github.com/graykode/DeepLearning-Study/blob/master/6.LSTM-Introduce.py) 164 | - Hidden State, Cell State 165 | - Different of RNN with LSTM 166 | - Output Layer 167 | - Weight sharing in RNN 168 | - FeedForward in LSTM(Eng, [Kor](https://ratsgo.github.io/natural%20language%20processing/2017/03/09/rnnlstm/)) 169 | - Mathematical Back Propagation in LSTM(Eng, [Kor](https://ratsgo.github.io/natural%20language%20processing/2017/03/09/rnnlstm/)) 170 | - Bi-directional LSTM(BiLSTM)(Eng, [Kor](https://ratsgo.github.io/natural%20language%20processing/2017/10/22/manning/)) 171 | - Practice : [LSTM-AutoComplete with LSTM](https://github.com/graykode/DeepLearning-Study/blob/master/6.LSTM-AutoComplete.py) 172 | 173 | 174 | #### 7. Application Level 175 | - Vision : Cat or Dog Image Classification. 176 | - Natural Language Processing : Positive or Negative Classification with Naver Movie Review. 177 | 178 | 179 | ## Reference 180 | - Andrew NG - Machine Learning Lecture 181 | - Korean Andrew Ng NoteBook : [WikiBook](https://wikidocs.net/book/587) 182 | 183 | 184 | ## Author 185 | - Tae Hwan Jung(Jeff Jung) @graykode 186 | - Author Email : [nlkey2022@gmail.com](mailto:nlkey2022@gmail.com) 187 | 188 | 189 | ## License 190 |

191 | 193 | CC0 194 | 195 |

-------------------------------------------------------------------------------- /data/cat/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graykode/DeepLearning-Study/0af921431124d87993700973a1bf2e223f1294bf/data/cat/cat.jpg -------------------------------------------------------------------------------- /data/dog/dog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graykode/DeepLearning-Study/0af921431124d87993700973a1bf2e223f1294bf/data/dog/dog.jpg -------------------------------------------------------------------------------- /mnist_cnn.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graykode/DeepLearning-Study/0af921431124d87993700973a1bf2e223f1294bf/mnist_cnn.pt -------------------------------------------------------------------------------- /practice/DNN-MNIST.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | from torchvision import datasets, transforms 8 | 9 | train_data = datasets.MNIST('./data/', train=True, download=True, transform=transforms.ToTensor()) 10 | train_batch_size = 5000 11 | train_dataloader = torch.utils.data.DataLoader(dataset=train_data, batch_size=train_batch_size, shuffle=True) 12 | 13 | class DNN(nn.Module): 14 | def __init__(self): 15 | super(DNN, self).__init__() 16 | 17 | def forward(self, x): 18 | """ 19 | put your code 20 | """ 21 | out = x 22 | return out 23 | 24 | model = DNN() 25 | criterion = torch.nn.CrossEntropyLoss() 26 | optimizer = optim.SGD(model.parameters(), lr=0.001) 27 | 28 | # Training 29 | count = 0 30 | model.train() 31 | for batch_idx, (data, target) in enumerate(train_dataloader): 32 | optimizer.zero_grad() 33 | output = model(data) 34 | loss = criterion(output, target) 35 | loss.backward() 36 | optimizer.step() 37 | 38 | count += train_batch_size 39 | print('batch :', batch_idx + 1,' ', count, '/', len(train_data), 40 | 'image:', data.shape, 'target : ', target) 41 | 42 | # Test 43 | test_data = datasets.MNIST('./data/', train=False, download=True, transform=transforms.ToTensor()) 44 | test_batch_size = 5000 45 | test_dataloader = torch.utils.data.DataLoader(dataset=test_data, batch_size=test_batch_size, shuffle=True) 46 | 47 | model.eval() 48 | test_loss = 0 49 | correct = 0 50 | with torch.no_grad(): 51 | for data, target in test_dataloader: 52 | output = model(data) 53 | test_loss += criterion(output, target).item() # sum up batch loss 54 | pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability 55 | correct += pred.eq(target.view_as(pred)).sum().item() 56 | 57 | test_loss /= len(test_dataloader.dataset) 58 | 59 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 60 | test_loss, correct, len(test_dataloader.dataset), 61 | 100. * correct / len(test_dataloader.dataset))) -------------------------------------------------------------------------------- /practice/MNIST-Viewer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code by Tae Hwan Jung(@graykode) 3 | """ 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | from torchvision import datasets, transforms 7 | 8 | def show(image): 9 | pixels = np.array(image, dtype='float').reshape((28, 28)) 10 | plt.imshow(pixels, cmap='gray') 11 | plt.show() 12 | 13 | mnistdata = datasets.MNIST('./data/', train=True, download=True, transform=transforms.ToTensor()) 14 | # first is index, second is (image, label) 15 | print(mnistdata[0][1]) 16 | show(mnistdata[0][0]) --------------------------------------------------------------------------------