├── Data └── 코드실행하면 이쪽이 채워 집니다..txt ├── .idea ├── vcs.xml ├── misc.xml ├── inspectionProfiles │ └── profiles_settings.xml ├── modules.xml ├── Advanced_Models.iml └── workspace.xml ├── README.md ├── Models ├── Resnet_Model │ ├── Parser_args.py │ ├── BasicBlock.py │ ├── main.py │ ├── Bottle_neck.py │ └── Resnet.py ├── VAE_Model │ ├── Decoder.py │ ├── Encoder.py │ ├── Parser_args.py │ ├── main.py │ └── VAE.py ├── GPT2_Model │ ├── Model │ │ ├── LayerNorm.py │ │ ├── GPT2LMHead.py │ │ ├── GPT2LMHeadModel.py │ │ ├── GPT2Model.py │ │ └── Block.py │ ├── Parser_args.py │ ├── config.py │ ├── main.py │ ├── text_Encoder.py │ └── Utils.py ├── GAN_Model │ ├── Discriminator.py │ ├── Generator.py │ ├── main.py │ ├── Parser_args.py │ └── GAN.py ├── CGAN_Model │ ├── Discriminator.py │ ├── Parser_args.py │ ├── Generator.py │ ├── main.py │ └── CGAN.py ├── DCGAN_Model │ ├── Parser_args.py │ ├── main.py │ ├── Generator.py │ ├── DCGAN.py │ └── Discriminator.py └── SAGAN_Model │ ├── main.py │ ├── Parser_args.py │ ├── Generator.py │ ├── SAGAN.py │ └── Discriminator.py ├── Modules ├── scaled_dot_product_attention_layer.py ├── self_attention_Layer.py └── multi_head_attention_Layer.py └── Utils.py /Data/코드실행하면 이쪽이 채워 집니다..txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/Advanced_Models.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Advanced_Models 2 | 여러가지 유명한 신경망 모델들을 제공합니다. 3 | 4 | 제공 모델들 : DCGAN, CGAN, SA-GAN ,GAN, Resnet, VAE, Multi-Head Attention, GPT-2 5 | 6 | # VAE 7 | 8 | 해당 모델에 대한 개념의 정립이 필요하시면 아래 링크로 가세요~ 9 | 10 | https://blog.naver.com/jk96491/222177759271 11 | 12 | 13 | # GAN 14 | 15 | 해당 모델에 대한 개념의 정립이 필요하시면 아래 링크로 가세요~ 16 | 17 | https://blog.naver.com/jk96491/222190307569 18 | -------------------------------------------------------------------------------- /Models/Resnet_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--n_epochs", type=int, default=100, help="number of epochs of training") 7 | parser.add_argument("--batch_size", type=int, default=64, help="size of the batches") 8 | parser.add_argument("--channels", type=int, default=3, help="number of image channels") 9 | parser.add_argument("--learning_rate", type=float, default=0.001) 10 | parser.add_argument("--layers", type=int, default=[3, 4, 6, 3]) 11 | args = parser.parse_args() 12 | return args -------------------------------------------------------------------------------- /Modules/scaled_dot_product_attention_layer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class scaled_dot_product_attention(nn.Module): 6 | def __init__(self, key_dim): 7 | super(scaled_dot_product_attention, self).__init__() 8 | self.key_dim = key_dim 9 | self.softmax = nn.Softmax(dim=-1) 10 | 11 | def forward(self, query, key, value): 12 | 13 | dot_product = torch.bmm(query, key) 14 | attention_score = self.softmax(dot_product) 15 | output = torch.bmm(value, attention_score.permute(0, 2, 1)) 16 | 17 | return output, attention_score -------------------------------------------------------------------------------- /Models/VAE_Model/Decoder.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class Decoder(nn.Module): 5 | def __init__(self, input_dim, hidden_dim, hidden_dim2, out_dim): 6 | super(Decoder, self).__init__() 7 | 8 | self.layer1 = nn.Sequential(nn.Linear(input_dim, hidden_dim), 9 | nn.ReLU()) 10 | self.layer2 = nn.Sequential(nn.Linear(hidden_dim, hidden_dim2), 11 | nn.ReLU()) 12 | self.layer3 = nn.Sequential(nn.Linear(hidden_dim2, out_dim)) 13 | 14 | def forward(self, x): 15 | x = self.layer1(x) 16 | x = self.layer2(x) 17 | x = self.layer3(x) 18 | 19 | return x 20 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Model/LayerNorm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class LayerNorm(nn.Module): 6 | def __init__(self, hidden_size, eps=1e-12): 7 | """Construct a layernorm module in the TF style (epsilon inside the square root). 8 | """ 9 | super(LayerNorm, self).__init__() 10 | self.weight = nn.Parameter(torch.ones(hidden_size)) 11 | self.bias = nn.Parameter(torch.zeros(hidden_size)) 12 | self.variance_epsilon = eps 13 | 14 | def forward(self, x): 15 | u = x.mean(-1, keepdim=True) 16 | s = (x - u).pow(2).mean(-1, keepdim=True) 17 | x = (x - u) / torch.sqrt(s + self.variance_epsilon) 18 | return self.weight * x + self.bias 19 | -------------------------------------------------------------------------------- /Models/VAE_Model/Encoder.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class Encoder(nn.Module): 5 | def __init__(self, input_dim, hidden_dim, hidden_dim2, out_dim): 6 | super(Encoder, self).__init__() 7 | self.layer1 = nn.Sequential(nn.Linear(input_dim, hidden_dim), 8 | nn.ReLU()) 9 | self.layer2 = nn.Sequential(nn.Linear(hidden_dim, hidden_dim2), 10 | nn.ReLU()) 11 | self.layer3 = nn.Sequential(nn.Linear(hidden_dim2, out_dim), 12 | nn.ReLU()) 13 | 14 | def forward(self, x): 15 | x = self.layer1(x) 16 | x = self.layer2(x) 17 | x = self.layer3(x) 18 | 19 | return x 20 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--text", type=str, default='korean air is airline', required=False) 7 | parser.add_argument("--quiet", type=bool, default=False) 8 | parser.add_argument("--nsamples", type=int, default=1) 9 | parser.add_argument('--unconditional', action='store_true', help='If true, unconditional generation.') 10 | parser.add_argument("--batch_size", type=int, default=-1) 11 | parser.add_argument("--length", type=int, default=50) 12 | parser.add_argument("--temperature", type=float, default=0.7) 13 | parser.add_argument("--top_k", type=int, default=40) 14 | args = parser.parse_args() 15 | 16 | return args 17 | -------------------------------------------------------------------------------- /Models/GAN_Model/Discriminator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import numpy as np 3 | 4 | 5 | class Discriminator(nn.Module): 6 | def __init__(self, image_shape): 7 | super(Discriminator, self).__init__() 8 | self.image_shape = image_shape 9 | 10 | self.model = nn.Sequential(nn.Linear(int(np.prod(self.image_shape)), 512), 11 | nn.LeakyReLU(0.2, inplace=True), 12 | nn.Linear(512, 256), 13 | nn.LeakyReLU(0.2, inplace=True), 14 | nn.Linear(256, 1), 15 | nn.Sigmoid()) 16 | 17 | def forward(self, image): 18 | image_flatten = image.view(image.size(0), -1) 19 | validity = self.model(image_flatten) 20 | 21 | return validity 22 | 23 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Model/GPT2LMHead.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class GPT2LMHead(nn.Module): 5 | def __init__(self, model_embeddings_weights, config): 6 | super(GPT2LMHead, self).__init__() 7 | self.n_embd = config.n_embd 8 | self.set_embeddings_weights(model_embeddings_weights) 9 | 10 | def set_embeddings_weights(self, model_embeddings_weights): 11 | embed_shape = model_embeddings_weights.shape 12 | self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) 13 | self.decoder.weight = model_embeddings_weights # Tied weights 14 | 15 | def forward(self, hidden_state): 16 | # Truncated Language modeling logits (we remove the last token) 17 | # h_trunc = h[:, :-1].contiguous().view(-1, self.n_embd) 18 | lm_logits = self.decoder(hidden_state) 19 | return lm_logits -------------------------------------------------------------------------------- /Models/GPT2_Model/config.py: -------------------------------------------------------------------------------- 1 | ''' 2 | code by TaeHwan Jung(@graykode) 3 | Original Paper and repository here : https://github.com/openai/gpt-2 4 | GPT2 Pytorch Model : https://github.com/huggingface/pytorch-pretrained-BERT 5 | ''' 6 | class GPT2Config(object): 7 | def __init__( 8 | self, 9 | vocab_size_or_config_json_file=50257, 10 | n_positions=1024, 11 | n_ctx=1024, 12 | n_embd=768, 13 | n_layer=12, 14 | n_head=12, 15 | layer_norm_epsilon=1e-5, 16 | initializer_range=0.02, 17 | ): 18 | self.vocab_size = vocab_size_or_config_json_file 19 | self.n_ctx = n_ctx 20 | self.n_positions = n_positions 21 | self.n_embd = n_embd 22 | self.n_layer = n_layer 23 | self.n_head = n_head 24 | self.layer_norm_epsilon = layer_norm_epsilon 25 | self.initializer_range = initializer_range -------------------------------------------------------------------------------- /Models/Resnet_Model/BasicBlock.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Utils import conv_3x3 3 | 4 | 5 | class BasicBlock(nn.Module): 6 | def __init__(self, in_planes, planes, stride=1, down_sample=None): 7 | super(BasicBlock, self).__init__() 8 | self.conv_layer1 = nn.Sequential(conv_3x3(in_planes, planes, stride), 9 | nn.BatchNorm2d(planes), 10 | nn.ReLU()) 11 | self.conv_layer2 = nn.Sequential(conv_3x3(planes, planes, stride), 12 | nn.BatchNorm2d(planes)) 13 | 14 | self.down_sample = down_sample 15 | self.stride = stride 16 | 17 | def forward(self, inputs): 18 | identity = inputs 19 | 20 | out = self.conv_layer1(inputs) 21 | out = self.conv_layer2(out) 22 | 23 | if self.downsample is not None: 24 | identity = self.downsample(inputs) 25 | 26 | out += identity 27 | out = self.relu(out) 28 | 29 | return out 30 | -------------------------------------------------------------------------------- /Models/GAN_Model/Generator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import numpy as np 3 | 4 | 5 | class Generator(nn.Module): 6 | def __init__(self, image_shape, args): 7 | super(Generator, self).__init__() 8 | self.image_shape = image_shape 9 | 10 | self.model = nn.Sequential(self.block(args.latent_dim, 128), 11 | self.block(128, 256), 12 | self.block(256, 512), 13 | self.block(512, 1024), 14 | nn.Linear(1024, int(np.prod(self.image_shape))), 15 | nn.Tanh()) 16 | 17 | def forward(self, latent): 18 | latent = self.model(latent) 19 | latent = latent.view(latent.size(0), *self.image_shape) 20 | return latent 21 | 22 | def block(self, input_feature, output_feature): 23 | layers = nn.Sequential(nn.Linear(input_feature, output_feature), 24 | nn.LeakyReLU(0.2, inplace=True)) 25 | return layers 26 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Model/GPT2LMHeadModel.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Models.GPT2_Model.Model.GPT2Model import GPT2Model 3 | from Models.GPT2_Model.Model.GPT2LMHead import GPT2LMHead 4 | 5 | 6 | class GPT2LMHeadModel(nn.Module): 7 | def __init__(self, config): 8 | super(GPT2LMHeadModel, self).__init__() 9 | self.transformer = GPT2Model(config) 10 | self.lm_head = GPT2LMHead(self.transformer.wte.weight, config) 11 | 12 | def set_tied(self): 13 | self.lm_head.set_embeddings_weights(self.transformer.wte.weight) 14 | 15 | def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None): 16 | hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past) 17 | lm_logits = self.lm_head(hidden_states) 18 | 19 | if lm_labels is not None: 20 | cross_entropy = nn.CrossEntropyLoss(ignore_index=-1) 21 | loss = cross_entropy(lm_logits.view(-1, lm_logits.size(-1)), lm_labels.view(-1)) 22 | return loss 23 | 24 | return lm_logits, presents -------------------------------------------------------------------------------- /Models/CGAN_Model/Discriminator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | 5 | 6 | class Discriminator(nn.Module): 7 | def __init__(self, image_shape): 8 | super(Discriminator, self).__init__() 9 | self.image_shape = image_shape 10 | 11 | self.Layer1_1 = nn.Linear(int(np.prod(self.image_shape)), 256) 12 | self.Layer1_2 = nn.Linear(10, 256) 13 | 14 | self.Layer2 = nn.Sequential(nn.Linear(512, 512), 15 | nn.LeakyReLU(0.2, inplace=True), 16 | nn.Linear(512, 256), 17 | nn.LeakyReLU(0.2, inplace=True), 18 | nn.Linear(256, 1), 19 | nn.Sigmoid()) 20 | 21 | def forward(self, image, labels): 22 | image_flatten = image.view(image.size(0), -1) 23 | x1 = self.Layer1_1(image_flatten) 24 | x2 = self.Layer1_2(labels) 25 | 26 | validity = torch.cat([x1, x2], 1) 27 | 28 | validity = self.Layer2(validity) 29 | 30 | return validity 31 | 32 | -------------------------------------------------------------------------------- /Models/VAE_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--n_epochs", type=int, default=100, help="number of epochs of training") 7 | parser.add_argument("--latent_dim", type=int, default=8) 8 | parser.add_argument("--input_dim", type=int, default=64*64) 9 | parser.add_argument("--hidden_dim", type=int, default=128) 10 | parser.add_argument("--hidden_dim2", type=int, default=64) 11 | parser.add_argument("--output_dim", type=int, default=32) 12 | parser.add_argument("--batch_size", type=int, default=64, help="size of the batches") 13 | parser.add_argument("--image_size", type=int, default=64, help="size of each image dimension") 14 | parser.add_argument("--channels", type=int, default=3, help="number of image channels") 15 | parser.add_argument("--learning_rate", type=float, default=0.001) 16 | parser.add_argument("--nrow", type=int, default=8) 17 | parser.add_argument("--sample_interval", type=int, default=400, help="interval betwen image samples") 18 | args = parser.parse_args() 19 | return args -------------------------------------------------------------------------------- /Models/GAN_Model/main.py: -------------------------------------------------------------------------------- 1 | from Models.GAN_Model.Parser_args import parse_Arg 2 | from Utils import CIFARLoadData 3 | from Models.GAN_Model.GAN import gan 4 | from torchvision.utils import save_image 5 | from Utils import get_device 6 | 7 | args = parse_Arg() 8 | image_shape = (args.channels, args.image_size, args.image_size) 9 | data_loader = CIFARLoadData(args.batch_size, True, True) 10 | 11 | device = get_device() 12 | 13 | model = gan(image_shape, args, device) 14 | 15 | for epoch in range(args.n_epochs): 16 | for i, (images, _) in enumerate(data_loader): 17 | real_images = images.to(device) 18 | 19 | generator_loss, fake_image = model.learn_generator(images) 20 | discriminator_loss = model.learn_discriminator(real_images, fake_image) 21 | 22 | print( 23 | "[Epoch %d/%d] [Batch %d/%d] [Discriminator_loss: %f] [Generator_loss: %f]" 24 | % (epoch + 1, args.n_epochs, i + 1, len(data_loader), discriminator_loss, generator_loss) 25 | ) 26 | 27 | batches_done = epoch * len(data_loader) + i 28 | if batches_done % args.sample_interval == 0: 29 | save_image(fake_image, "images/%d.png" % batches_done, nrow=args.nrow, normalize=True) 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /Models/GAN_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training") 7 | parser.add_argument("--batch_size", type=int, default=64, help="size of the batches") 8 | parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate") 9 | parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient") 10 | parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient") 11 | parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation") 12 | parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space") 13 | parser.add_argument("--image_size", type=int, default=64, help="size of each image dimension") 14 | parser.add_argument("--channels", type=int, default=3, help="number of image channels") 15 | parser.add_argument("--sample_interval", type=int, default=400, help="interval betwen image samples") 16 | parser.add_argument("--nrow", type=int, default=8) 17 | args = parser.parse_args() 18 | return args -------------------------------------------------------------------------------- /Models/CGAN_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training") 7 | parser.add_argument("--batch_size", type=int, default=64, help="size of the batches") 8 | parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate") 9 | parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient") 10 | parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient") 11 | parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation") 12 | parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space") 13 | parser.add_argument("--image_size", type=int, default=64, help="size of each image dimension") 14 | parser.add_argument("--channels", type=int, default=3, help="number of image channels") 15 | parser.add_argument("--sample_interval", type=int, default=400, help="interval betwen image samples") 16 | parser.add_argument("--nrow", type=int, default=8) 17 | args = parser.parse_args() 18 | return args -------------------------------------------------------------------------------- /Models/VAE_Model/main.py: -------------------------------------------------------------------------------- 1 | from Utils import MnistLoadData 2 | from Utils import CIFARLoadData 3 | from Models.VAE_Model.Parser_args import parse_Arg 4 | from Models.VAE_Model.VAE import vae 5 | from torchvision.utils import save_image 6 | from Utils import get_device 7 | 8 | args = parse_Arg() 9 | image_shape = (args.channels, args.image_size, args.image_size) 10 | data_loader = CIFARLoadData(args.batch_size, True, True) 11 | 12 | device = get_device() 13 | 14 | model = vae(args, device) 15 | 16 | for epoch in range(args.n_epochs): 17 | for i, data in enumerate(data_loader, 0): 18 | inputs, _ = data 19 | current_batch_size = inputs.size(0) 20 | inputs = inputs.view(current_batch_size, args.input_dim * args.channels).to(device) 21 | 22 | loss = model.learn(inputs) 23 | 24 | print("[Epoch %d/%d] [Batch %d/%d] [loss: %f]]" % (epoch + 1, args.n_epochs, i + 1, len(data_loader), loss)) 25 | 26 | batches_done = epoch * len(data_loader) + i 27 | if batches_done % args.sample_interval == 0: 28 | output = model(inputs).data.reshape(args.batch_size, args.channels, args.image_size, args.image_size) 29 | save_image(output, "images/%d.png" % batches_done, nrow=args.nrow, normalize=True) 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /Models/Resnet_Model/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from Models.Resnet_Model.Resnet import ResNet 3 | from Utils import MnistLoadData 4 | from Models.Resnet_Model.Parser_args import parse_Arg 5 | from Utils import get_device 6 | from Utils import CIFARLoadData 7 | 8 | args = parse_Arg() 9 | train_loader = CIFARLoadData(args.batch_size, True, False) 10 | test_loader = CIFARLoadData(args.batch_size, False, False) 11 | 12 | device = get_device() 13 | 14 | # 50 Resnet 15 | model = ResNet(args.channels, args.layers).to(device) 16 | 17 | # 학습 진행 18 | for epoch in range(args.n_epochs): 19 | for i, data in enumerate(train_loader): 20 | inputs, labels = data[0].to(device), data[1].to(device) 21 | 22 | loss = model.Learn(inputs, labels) 23 | 24 | print("[Epoch %d/%d] [Batch %d/%d] [loss: %f]]" % (epoch + 1, args.n_epochs, i + 1, len(train_loader), loss)) 25 | 26 | # 평가 27 | with torch.no_grad(): 28 | model.eval() 29 | 30 | X_test = test_loader.test_data.view(len(test_loader), args.channels, 28, 28).float() 31 | Y_test = test_loader.test_labels 32 | 33 | prediction = model(X_test) 34 | correct_prediction = torch.argmax(prediction, 1) == Y_test 35 | accuracy = correct_prediction.float().mean() 36 | print('Accuracy:', accuracy.item()) 37 | 38 | -------------------------------------------------------------------------------- /Models/CGAN_Model/Generator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | 5 | 6 | class Generator(nn.Module): 7 | def __init__(self, image_shape, args): 8 | super(Generator, self).__init__() 9 | self.image_shape = image_shape 10 | 11 | self.Layer1_1 = nn.Linear(args.latent_dim, 64) 12 | self.Layer1_2 = nn.Linear(10, 64) 13 | 14 | self.Layer2 = nn.Sequential(self.block(128, 128), 15 | self.block(128, 256), 16 | self.block(256, 512), 17 | self.block(512, 1024), 18 | nn.Linear(1024, int(np.prod(self.image_shape))), 19 | nn.Tanh()) 20 | 21 | def forward(self, latent, labels): 22 | x1 = self.Layer1_1(latent) 23 | x2 = self.Layer1_2(labels) 24 | latent = torch.cat([x1, x2], 1) 25 | latent = self.Layer2(latent) 26 | latent = latent.view(latent.size(0), *self.image_shape) 27 | return latent 28 | 29 | def block(self, input_feature, output_feature): 30 | layers = nn.Sequential(nn.Linear(input_feature, output_feature), 31 | nn.LeakyReLU(0.2, inplace=True)) 32 | return layers 33 | -------------------------------------------------------------------------------- /Modules/self_attention_Layer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from Modules.scaled_dot_product_attention_layer import scaled_dot_product_attention 4 | 5 | 6 | class self_attention(nn.Module): 7 | def __init__(self, input_dim): 8 | super(self_attention, self).__init__() 9 | 10 | self.query = nn.Conv2d(input_dim, input_dim // 2, kernel_size=1) 11 | self.key = nn.Conv2d(input_dim, input_dim // 2, kernel_size=1) 12 | self.value = nn.Conv2d(input_dim, input_dim, kernel_size=1) 13 | 14 | self.scaled_dot_product_attention_layer = scaled_dot_product_attention(input_dim // 2) 15 | 16 | self.gamma = nn.Parameter(torch.zeros(1)) 17 | self.softmax = nn.Softmax(dim=-1) 18 | 19 | def forward(self, inputs): 20 | batch_size, channels, width, height = inputs.size() 21 | 22 | query = self.query(inputs).view(batch_size, -1, width * height).permute(0, 2, 1) 23 | key = self.key(inputs).view(batch_size, -1, width * height) 24 | value = self.value(inputs).view(batch_size, -1, width * height) 25 | 26 | output, attention_score = self.scaled_dot_product_attention_layer(query, key, value) 27 | output = output.view(batch_size, channels, width, height) 28 | output = self.gamma * output + inputs 29 | 30 | return output, attention_score 31 | -------------------------------------------------------------------------------- /Models/CGAN_Model/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from Models.CGAN_Model.Parser_args import parse_Arg 3 | from Utils import CIFARLoadData 4 | from Models.CGAN_Model.CGAN import cgan 5 | from torchvision.utils import save_image 6 | from Utils import get_device 7 | 8 | args = parse_Arg() 9 | image_shape = (args.channels, args.image_size, args.image_size) 10 | data_loader = CIFARLoadData(args.batch_size, True, True) 11 | 12 | device = get_device() 13 | 14 | model = cgan(image_shape, args, device) 15 | 16 | for epoch in range(args.n_epochs): 17 | for i, (images, label) in enumerate(data_loader): 18 | real_images = images.to(device) 19 | 20 | y_label_ = torch.zeros(images.size(0), 10) 21 | labels = y_label_.scatter_(1, label.view(images.size(0), 1), 1).to(device) 22 | 23 | generator_loss, generator_image = model.learn_generator(images, labels) 24 | discriminator_loss = model.learn_discriminator(real_images, generator_image, labels) 25 | 26 | print( 27 | "[Epoch %d/%d] [Batch %d/%d] [Discriminator_loss: %f] [Generator_loss: %f]" 28 | % (epoch + 1, args.n_epochs, i + 1, len(data_loader), discriminator_loss, generator_loss) 29 | ) 30 | 31 | batches_done = epoch * len(data_loader) + i 32 | if batches_done % args.sample_interval == 0: 33 | save_image(generator_image, "images/%d.png" % batches_done, nrow=args.nrow, normalize=True) 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /Models/Resnet_Model/Bottle_neck.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Utils import conv_3x3 3 | from Utils import conv_1x1 4 | 5 | 6 | class Bottle_neck(nn.Module): 7 | expansion = 4 8 | 9 | def __init__(self, in_planes, planes, stride=1, down_sample=None): 10 | super(Bottle_neck, self).__init__() 11 | 12 | self.bn1 = nn.BatchNorm2d(planes) 13 | self.bn2 = nn.BatchNorm2d(planes) 14 | self.bn3 = nn.BatchNorm2d(planes * self.expansion) 15 | self.relu = nn.ReLU(inplace=True) 16 | 17 | # 1 X 1 18 | self.Layer1 = nn.Sequential(conv_1x1(in_planes, planes), 19 | self.bn1, 20 | self.relu) 21 | # 3 X 3 22 | self.Layer2 = nn.Sequential(conv_3x3(planes, planes, stride), 23 | self.bn2, 24 | self.relu) 25 | # 1 X 1 26 | self.Layer3 = nn.Sequential(conv_1x1(planes, planes * self.expansion), 27 | self.bn3) 28 | 29 | self.down_sample = down_sample 30 | self.stride = stride 31 | 32 | def forward(self, inputs): 33 | identity = inputs 34 | 35 | out = self.Layer1(inputs) 36 | out = self.Layer2(out) 37 | out = self.Layer3(out) 38 | 39 | if self.down_sample is not None: 40 | identity = self.down_sample(inputs) 41 | 42 | out += identity 43 | out = self.relu(out) 44 | 45 | return out -------------------------------------------------------------------------------- /Models/DCGAN_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training") 7 | parser.add_argument("--batch_size", type=int, default=64, help="size of the batches") 8 | parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate") 9 | parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient") 10 | parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient") 11 | parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation") 12 | parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space") 13 | parser.add_argument("--noise_dim", type=int, default=100, help="size of each image dimension") 14 | parser.add_argument("--noise_filter_genenrator", type=int, default=64, help="size of each image dimension") 15 | parser.add_argument("--noise_filter_discriminator", type=int, default=64, help="size of each image dimension") 16 | parser.add_argument("--channels", type=int, default=3, help="number of image channels") 17 | parser.add_argument("--sample_interval", type=int, default=400, help="interval betwen image samples") 18 | parser.add_argument("--image_size", type=int, default=64, help="size of each image dimension") 19 | parser.add_argument("--nrow", type=int, default=8) 20 | args = parser.parse_args() 21 | return args -------------------------------------------------------------------------------- /Models/DCGAN_Model/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from Models.DCGAN_Model.Parser_args import parse_Arg 3 | from Models.DCGAN_Model.DCGAN import dc_gan 4 | from torchvision.utils import save_image 5 | from Utils import CIFARLoadData 6 | from Utils import get_device 7 | 8 | args = parse_Arg() 9 | 10 | train_loader = CIFARLoadData(args.batch_size, True, True) 11 | 12 | device = get_device() 13 | 14 | model = dc_gan(args).to(device) 15 | 16 | for epoch in range(args.n_epochs): 17 | for i, data in enumerate(train_loader): 18 | real_images, _ = data 19 | current_batch_size = real_images.size(0) 20 | 21 | inputs = real_images.clone().to(device) 22 | noise = torch.zeros(current_batch_size, args.noise_dim, 1, 1).normal_(0, 1) 23 | 24 | real_labels = torch.ones(current_batch_size, 1).detach() 25 | fake_labels = torch.zeros(current_batch_size, 1).detach() 26 | 27 | discriminator_loss, generator_image = model.learn_discriminator(inputs, noise, real_labels, fake_labels) 28 | generator_loss = model.learn_generator(noise, real_labels) 29 | 30 | print( 31 | "[Epoch %d/%d] [Batch %d/%d] [Discriminator_loss: %f] [Generator_loss: %f]" 32 | % (epoch + 1, args.n_epochs, i + 1, len(train_loader), discriminator_loss, generator_loss) 33 | ) 34 | 35 | batches_done = epoch * len(train_loader) + i 36 | if batches_done % args.sample_interval == 0: 37 | temp = generator_image 38 | save_image(generator_image, "images/%d.png" % batches_done, nrow=args.nrow, normalize=True) 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /Models/SAGAN_Model/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from Models.SAGAN_Model.Parser_args import parse_Arg 3 | from Models.SAGAN_Model.SAGAN import sa_gan 4 | from torchvision.utils import save_image 5 | from Utils import CIFARLoadData 6 | from Utils import get_device 7 | 8 | args = parse_Arg() 9 | device1 = get_device('cuda:0') 10 | device2 = get_device('cuda:1') 11 | 12 | train_loader = CIFARLoadData(args.batch_size, True, True) 13 | 14 | model = sa_gan(args, [device1, device2]) 15 | 16 | for epoch in range(args.n_epochs): 17 | for i, data in enumerate(train_loader): 18 | real_images, _ = data 19 | current_batch_size = real_images.size(0) 20 | 21 | inputs = real_images.clone().to(device1) 22 | noise = torch.zeros(current_batch_size, args.noise_dim, 1, 1).normal_(0, 1) 23 | 24 | real_labels = torch.ones(current_batch_size, 1).detach() 25 | fake_labels = torch.zeros(current_batch_size, 1).detach() 26 | 27 | discriminator_loss, generator_image = model.learn_discriminator(inputs, noise, real_labels, fake_labels) 28 | generator_loss = model.learn_generator(noise, real_labels) 29 | 30 | print("[Epoch %d/%d] [Batch %d/%d] [Discriminator_loss: %f] [Generator_loss: %f]" 31 | % (epoch + 1, args.n_epochs, i + 1, len(train_loader), discriminator_loss, generator_loss)) 32 | 33 | batches_done = epoch * len(train_loader) + i 34 | if batches_done % args.sample_interval == 0: 35 | temp = generator_image 36 | save_image(generator_image, "images/%d.png" % batches_done, nrow=args.nrow, normalize=True) 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /Models/SAGAN_Model/Parser_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def parse_Arg(): 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training") 7 | parser.add_argument("--batch_size", type=int, default=9, help="size of the batches") 8 | parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate") 9 | parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient") 10 | parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient") 11 | parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation") 12 | parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space") 13 | parser.add_argument("--noise_dim", type=int, default=100, help="size of each image dimension") 14 | parser.add_argument("--noise_filter_genenrator", type=int, default=64, help="size of each image dimension") 15 | parser.add_argument("--noise_filter_discriminator", type=int, default=64, help="size of each image dimension") 16 | parser.add_argument("--channels", type=int, default=3, help="number of image channels") 17 | parser.add_argument("--sample_interval", type=int, default=400, help="interval betwen image samples") 18 | parser.add_argument("--image_size", type=int, default=64, help="size of each image dimension") 19 | parser.add_argument("--nrow", type=int, default=3) 20 | parser.add_argument("--n_head", type=int, default=4) 21 | parser.add_argument("--use_MHA", type=bool, default=True) 22 | args = parser.parse_args() 23 | return args -------------------------------------------------------------------------------- /Models/DCGAN_Model/Generator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class Generator(nn.Module): 5 | def __init__(self, args): 6 | super(Generator, self).__init__() 7 | self.noise_dim = args.noise_dim 8 | self.channels = args.channels 9 | self.noise_filter = args.noise_filter_genenrator 10 | 11 | self.layer1 = nn.Sequential(nn.ConvTranspose2d(self.noise_dim, self.noise_filter * 8, 4, 1, 0, bias=False), 12 | nn.BatchNorm2d(self.noise_filter * 8), 13 | nn.ReLU(True)) 14 | 15 | self.layer2 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter * 8, self.noise_filter * 4, 4, 2, 1, bias=False), 16 | nn.BatchNorm2d(self.noise_filter * 4), 17 | nn.ReLU(True)) 18 | 19 | self.layer3 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter * 4, self.noise_filter * 2, 4, 2, 1, bias=False), 20 | nn.BatchNorm2d(self.noise_filter * 2), 21 | nn.ReLU(True)) 22 | 23 | self.layer4 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter * 2, self.noise_filter, 4, 2, 1, bias=False), 24 | nn.BatchNorm2d(self.noise_filter), 25 | nn.ReLU(True)) 26 | 27 | self.layer5 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter, self.channels, 4, 2, 1, bias=False), 28 | nn.Tanh()) 29 | 30 | def forward(self, noise): 31 | x = self.layer1(noise) 32 | x = self.layer2(x) 33 | x = self.layer3(x) 34 | x = self.layer4(x) 35 | latent = self.layer5(x) 36 | 37 | return latent 38 | 39 | -------------------------------------------------------------------------------- /Models/DCGAN_Model/DCGAN.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Models.DCGAN_Model.Discriminator import Discriminator 3 | from Models.DCGAN_Model.Generator import Generator 4 | from torch import optim 5 | 6 | 7 | class dc_gan(nn.Module): 8 | def __init__(self, args): 9 | super(dc_gan, self).__init__() 10 | self.args = args 11 | 12 | self.Generator = Generator(self.args) 13 | self.Discriminator = Discriminator(self.args) 14 | 15 | self.optimizer_generator = optim.Adam(self.Generator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 16 | self.optimizer_discriminator = optim.Adam(self.Discriminator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 17 | 18 | self.adversarial_loss = nn.BCELoss() 19 | 20 | self.valid = None 21 | self.fake = None 22 | self.real_loss = None 23 | 24 | def learn_discriminator(self, inputs, noise, real_labels, fake_labels): 25 | output = self.Discriminator(inputs) 26 | real_loss = self.adversarial_loss(output, real_labels) 27 | 28 | fake = self.Generator(noise) 29 | discriminator_result = self.Discriminator(fake.detach()) 30 | fake_loss = self.adversarial_loss(discriminator_result, fake_labels) 31 | 32 | loss = real_loss + fake_loss 33 | 34 | self.optimizer_discriminator.zero_grad() 35 | loss.backward() 36 | self.optimizer_discriminator.step() 37 | 38 | return loss.item(), fake 39 | 40 | def learn_generator(self, noise, label): 41 | fake = self.Generator(noise) 42 | discriminator_result = self.Discriminator(fake) 43 | loss = self.adversarial_loss(discriminator_result, label) 44 | 45 | self.optimizer_generator.zero_grad() 46 | loss.backward() 47 | self.optimizer_generator.step() 48 | 49 | return loss.item() 50 | 51 | 52 | -------------------------------------------------------------------------------- /Utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torchvision import datasets 4 | import torchvision.transforms as transforms 5 | import torch.nn as nn 6 | 7 | 8 | def MnistLoadData(image_size, batch_size, train, generate_image): 9 | if generate_image is True: 10 | os.makedirs("images", exist_ok=True) 11 | 12 | if image_size is None: 13 | transform = transforms.ToTensor() 14 | else: 15 | transform = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Normalize([0.5], [0.5])]) 16 | 17 | os.makedirs("../../Data/mnist", exist_ok=True) 18 | dataloader = torch.utils.data.DataLoader( 19 | datasets.MNIST( 20 | "../../Data/mnist", 21 | train=train, 22 | download=True, 23 | transform=transform), 24 | batch_size=batch_size, 25 | shuffle=True, 26 | ) 27 | 28 | return dataloader 29 | 30 | 31 | def CIFARLoadData(batch_size, Train, generate_image): 32 | if generate_image is True: 33 | os.makedirs("images", exist_ok=True) 34 | 35 | transform = transforms.Compose([transforms.Scale(64), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 36 | dataset = datasets.CIFAR10(root='../../Data/CIFAR/', train=Train, download=True, transform=transform) 37 | data_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) 38 | 39 | return data_loader 40 | 41 | 42 | def conv_3x3(in_planes, out_planes, stride=1): 43 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) 44 | 45 | 46 | def conv_1x1(in_planes, out_planes, stride=1): 47 | return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) 48 | 49 | 50 | def get_device(device_name='cuda:0'): 51 | device = device_name if torch.cuda.is_available() else 'cpu' 52 | return device 53 | 54 | 55 | -------------------------------------------------------------------------------- /Models/DCGAN_Model/Discriminator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class Discriminator(nn.Module): 5 | def __init__(self, args): 6 | self.noise_dim = args.noise_dim 7 | self.channels = args.channels 8 | self.noise_filter = args.noise_filter_discriminator 9 | 10 | super(Discriminator, self).__init__() 11 | 12 | self.layer1 = nn.Sequential(nn.Conv2d(self.channels, self.noise_filter, 13 | kernel_size=4, stride=2, padding=1, bias=False), 14 | nn.LeakyReLU(0.2, inplace=True)) 15 | 16 | self.layer2 = nn.Sequential(nn.Conv2d(self.noise_filter, self.noise_filter * 2, 17 | kernel_size=4, stride=2, padding=1, bias=False), 18 | nn.BatchNorm2d(self.noise_filter * 2), 19 | nn.LeakyReLU(0.2, inplace=True)) 20 | 21 | self.layer3 = nn.Sequential(nn.Conv2d(self.noise_filter * 2, self.noise_filter * 4, 22 | kernel_size=4, stride=2, padding=1, bias=False), 23 | nn.BatchNorm2d(self.noise_filter * 4), 24 | nn.LeakyReLU(0.2, inplace=True)) 25 | 26 | self.layer4 = nn.Sequential(nn.Conv2d(self.noise_filter * 4, self.noise_filter * 8, 27 | kernel_size=4, stride=2, padding=1, bias=False), 28 | nn.BatchNorm2d(self.noise_filter * 8), 29 | nn.Sigmoid()) 30 | 31 | self.layer5 = nn.Sequential(nn.Conv2d(self.noise_filter * 8, 1, 32 | kernel_size=4, stride=1, padding=0, bias=False), 33 | nn.Sigmoid()) 34 | 35 | def forward(self, inputs): 36 | x = self.layer1(inputs) 37 | x = self.layer2(x) 38 | x = self.layer3(x) 39 | x = self.layer4(x) 40 | output = self.layer5(x) 41 | return output.view(-1, 1).squeeze(1) 42 | 43 | -------------------------------------------------------------------------------- /Models/GAN_Model/GAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from Models.GAN_Model.Discriminator import Discriminator 4 | from Models.GAN_Model.Generator import Generator 5 | from torch import optim 6 | 7 | 8 | class gan(nn.Module): 9 | def __init__(self, image_shape, args, device): 10 | super(gan, self).__init__() 11 | self.image_shape = image_shape 12 | self.args = args 13 | self.device = device 14 | 15 | self.Generator = Generator(self.image_shape, self.args) 16 | self.Discriminator = Discriminator(self.image_shape) 17 | 18 | self.optimizer_generator = optim.Adam(self.Generator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 19 | self.optimizer_discriminator = optim.Adam(self.Discriminator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 20 | 21 | self.adversarial_loss = nn.BCELoss() 22 | 23 | self.valid = None 24 | self.fake = None 25 | 26 | self.to(device) 27 | 28 | def learn_generator(self, image): 29 | self.valid = torch.ones(image.size(0), 1).detach().to(self.device) 30 | self.fake = torch.zeros(image.size(0), 1).detach().to(self.device) 31 | 32 | z = torch.zeros(image.size(0), self.args.latent_dim).normal_(0, 1).to(self.device) 33 | 34 | fake_image = self.Generator(z) 35 | discriminator_result = self.Discriminator(fake_image) 36 | loss = self.adversarial_loss(discriminator_result, self.valid) 37 | 38 | self.optimizer_generator.zero_grad() 39 | loss.backward() 40 | self.optimizer_generator.step() 41 | 42 | return loss.item(), fake_image 43 | 44 | def learn_discriminator(self, real_images, fake_image): 45 | real_loss = self.adversarial_loss(self.Discriminator(real_images), self.valid) 46 | fake_loss = self.adversarial_loss(self.Discriminator(fake_image.detach()), self.fake) 47 | 48 | loss = (real_loss + fake_loss) / 2 49 | 50 | self.optimizer_discriminator.zero_grad() 51 | loss.backward() 52 | self.optimizer_discriminator.step() 53 | 54 | return loss.item() 55 | 56 | 57 | -------------------------------------------------------------------------------- /Models/SAGAN_Model/Generator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Modules.multi_head_attention_Layer import multi_head_attention 3 | 4 | 5 | class Generator(nn.Module): 6 | def __init__(self, args): 7 | super(Generator, self).__init__() 8 | self.noise_dim = args.noise_dim 9 | self.channels = args.channels 10 | self.noise_filter = args.noise_filter_genenrator 11 | 12 | if args.use_MHA: 13 | self.n_head = args.n_head 14 | else: 15 | self.n_head = 1 16 | 17 | self.layer1 = nn.Sequential(nn.ConvTranspose2d(self.noise_dim, self.noise_filter * 8, 4, 1, 0, bias=False), 18 | nn.BatchNorm2d(self.noise_filter * 8), 19 | nn.ReLU(True)) 20 | 21 | self.layer2 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter * 8, self.noise_filter * 4, 4, 2, 1, bias=False), 22 | nn.BatchNorm2d(self.noise_filter * 4), 23 | nn.ReLU(True)) 24 | 25 | self.layer3 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter * 4, self.noise_filter * 2, 4, 2, 1, bias=False), 26 | nn.BatchNorm2d(self.noise_filter * 2), 27 | nn.ReLU(True)) 28 | 29 | self.layer4 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter * 2, self.noise_filter, 4, 2, 1, bias=False), 30 | nn.BatchNorm2d(self.noise_filter), 31 | nn.ReLU(True)) 32 | 33 | self.layer5 = nn.Sequential(nn.ConvTranspose2d(self.noise_filter, self.channels, 4, 2, 1, bias=False), 34 | nn.Tanh()) 35 | 36 | self.self_attention_layer = multi_head_attention(self.noise_filter, self.n_head) 37 | 38 | def forward(self, noise): 39 | x = self.layer1(noise) 40 | x = self.layer2(x) 41 | x = self.layer3(x) 42 | x = self.layer4(x) 43 | 44 | x, attention_score = self.self_attention_layer(x) 45 | 46 | latent = self.layer5(x) 47 | 48 | return latent 49 | 50 | -------------------------------------------------------------------------------- /Models/CGAN_Model/CGAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from Models.CGAN_Model.Discriminator import Discriminator 4 | from Models.CGAN_Model.Generator import Generator 5 | from torch import optim 6 | 7 | 8 | class cgan(nn.Module): 9 | def __init__(self, image_shape, args, device): 10 | super(cgan, self).__init__() 11 | self.image_shape = image_shape 12 | self.args = args 13 | self.device = device 14 | 15 | self.Generator = Generator(self.image_shape, self.args) 16 | self.Discriminator = Discriminator(self.image_shape) 17 | 18 | self.optimizer_generator = optim.Adam(self.Generator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 19 | self.optimizer_discriminator = optim.Adam(self.Discriminator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 20 | 21 | self.adversarial_loss = nn.BCELoss() 22 | 23 | self.valid = None 24 | self.fake = None 25 | 26 | self.to(device) 27 | 28 | def learn_generator(self, image, labels): 29 | self.valid = torch.ones(image.size(0), 1).detach().to(self.device) 30 | self.fake = torch.zeros(image.size(0), 1).detach().to(self.device) 31 | 32 | z = torch.zeros(image.size(0), self.args.latent_dim).normal_(0, 1).to(self.device) 33 | 34 | generator_images = self.Generator(z, labels) 35 | discriminator_result = self.Discriminator(generator_images, labels) 36 | loss = self.adversarial_loss(discriminator_result, self.valid) 37 | 38 | self.optimizer_generator.zero_grad() 39 | loss.backward() 40 | self.optimizer_generator.step() 41 | 42 | return loss.item(), generator_images 43 | 44 | def learn_discriminator(self, real_images, generator_images, labels): 45 | real_loss = self.adversarial_loss(self.Discriminator(real_images, labels), self.valid) 46 | fake_loss = self.adversarial_loss(self.Discriminator(generator_images.detach(), labels), self.fake) 47 | 48 | loss = (real_loss + fake_loss) / 2 49 | 50 | self.optimizer_discriminator.zero_grad() 51 | loss.backward() 52 | self.optimizer_discriminator.step() 53 | 54 | return loss.item() 55 | 56 | 57 | -------------------------------------------------------------------------------- /Models/SAGAN_Model/SAGAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from Models.SAGAN_Model.Discriminator import Discriminator 4 | from Models.SAGAN_Model.Generator import Generator 5 | from torch import optim 6 | 7 | 8 | class sa_gan(nn.Module): 9 | def __init__(self, args, device): 10 | super(sa_gan, self).__init__() 11 | self.args = args 12 | 13 | self.device_generator = device[0] 14 | self.device_discriminator = device[1] 15 | 16 | self.Generator = Generator(self.args).to(self.device_generator) 17 | self.Discriminator = Discriminator(self.args).to(self.device_discriminator) 18 | 19 | self.optimizer_generator = optim.Adam(self.Generator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 20 | self.optimizer_discriminator = optim.Adam(self.Discriminator.parameters(), lr=args.lr, betas=(args.b1, args.b2)) 21 | 22 | self.adversarial_loss = nn.BCELoss() 23 | 24 | self.valid = None 25 | self.fake = None 26 | self.real_loss = None 27 | 28 | def learn_discriminator(self, inputs, noise, real_labels, fake_labels): 29 | output = self.Discriminator(inputs.to(self.device_discriminator)) 30 | real_loss = self.adversarial_loss(output, real_labels.to(self.device_discriminator)) 31 | 32 | fake = self.Generator(noise.to(self.device_generator)) 33 | discriminator_result = self.Discriminator(fake.detach().to(self.device_discriminator)) 34 | fake_loss = self.adversarial_loss(discriminator_result, fake_labels.to(self.device_discriminator)) 35 | 36 | loss = real_loss + fake_loss 37 | 38 | self.optimizer_discriminator.zero_grad() 39 | loss.backward() 40 | self.optimizer_discriminator.step() 41 | 42 | return loss.item(), fake 43 | 44 | def learn_generator(self, noise, label): 45 | fake = self.Generator(noise.to(self.device_generator)) 46 | discriminator_result = self.Discriminator(fake.to(self.device_discriminator)) 47 | loss = self.adversarial_loss(discriminator_result, label.to(self.device_discriminator)) 48 | 49 | self.optimizer_generator.zero_grad() 50 | loss.backward() 51 | self.optimizer_generator.step() 52 | 53 | return loss.item() 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /Modules/multi_head_attention_Layer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from Modules.scaled_dot_product_attention_layer import scaled_dot_product_attention 4 | 5 | 6 | class multi_head_attention(nn.Module): 7 | def __init__(self, input_dim, n_head): 8 | super(multi_head_attention, self).__init__() 9 | self.n_head = n_head 10 | self.out_dim = input_dim // 2 11 | 12 | self.query_layers = nn.ModuleList() 13 | self.key_layers = nn.ModuleList() 14 | self.value_layers = nn.ModuleList() 15 | 16 | for i in range(self.n_head): 17 | self.query_layers.append(nn.Conv2d(input_dim, input_dim // 2 , kernel_size=1)) 18 | self.key_layers.append(nn.Conv2d(input_dim, input_dim // 2, kernel_size=1)) 19 | self.value_layers.append(nn.Conv2d(input_dim, input_dim, kernel_size=1)) 20 | 21 | self.fc = nn.Linear(self.out_dim * self.out_dim * self.n_head, self.out_dim * self.out_dim) 22 | 23 | self.scaled_dot_product_attention_layer = scaled_dot_product_attention(input_dim // 2) 24 | 25 | self.gamma = nn.Parameter(torch.zeros(1)) 26 | self.softmax = nn.Softmax(dim=-1) 27 | 28 | def forward(self, inputs): 29 | batch_size, channels, width, height = inputs.size() 30 | 31 | attention_output = [] 32 | attention_score_output = [] 33 | 34 | for i in range(self.n_head): 35 | query = self.query_layers[i](inputs).view(batch_size, -1, width * height).permute(0, 2, 1) 36 | key = self.key_layers[i](inputs).view(batch_size, -1, width * height) 37 | value = self.value_layers[i](inputs).view(batch_size, -1, width * height) 38 | 39 | output, attention_score = self.scaled_dot_product_attention_layer(query, key, value) 40 | 41 | attention_output.append(output) 42 | attention_score_output.append(attention_score) 43 | 44 | attention_output = torch.cat(attention_output, dim=-1) 45 | attention_score_output = torch.cat(attention_score_output, dim=-1) 46 | 47 | output = self.fc(attention_output) 48 | output = output.view(batch_size, channels, self.out_dim, self.out_dim) 49 | output = self.gamma * output + inputs 50 | 51 | return output, attention_score_output 52 | -------------------------------------------------------------------------------- /Models/SAGAN_Model/Discriminator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Modules.self_attention_Layer import self_attention 3 | 4 | 5 | class Discriminator(nn.Module): 6 | def __init__(self, args): 7 | self.noise_dim = args.noise_dim 8 | self.channels = args.channels 9 | self.noise_filter = args.noise_filter_discriminator 10 | 11 | super(Discriminator, self).__init__() 12 | 13 | self.layer1 = nn.Sequential(nn.Conv2d(self.channels, self.noise_filter, 14 | kernel_size=4, stride=2, padding=1, bias=False), 15 | nn.LeakyReLU(0.2, inplace=True)) 16 | 17 | self.layer2 = nn.Sequential(nn.Conv2d(self.noise_filter, self.noise_filter * 2, 18 | kernel_size=4, stride=2, padding=1, bias=False), 19 | nn.BatchNorm2d(self.noise_filter * 2), 20 | nn.LeakyReLU(0.2, inplace=True)) 21 | 22 | self.layer3 = nn.Sequential(nn.Conv2d(self.noise_filter * 2, self.noise_filter * 4, 23 | kernel_size=4, stride=2, padding=1, bias=False), 24 | nn.BatchNorm2d(self.noise_filter * 4), 25 | nn.LeakyReLU(0.2, inplace=True)) 26 | 27 | self.layer4 = nn.Sequential(nn.Conv2d(self.noise_filter * 4, self.noise_filter * 8, 28 | kernel_size=4, stride=2, padding=1, bias=False), 29 | nn.BatchNorm2d(self.noise_filter * 8), 30 | nn.Sigmoid()) 31 | 32 | self.layer5 = nn.Sequential(nn.Conv2d(self.noise_filter * 8, 1, 33 | kernel_size=4, stride=1, padding=0, bias=False), 34 | nn.Sigmoid()) 35 | 36 | self.self_attention_layer = self_attention(self.noise_filter * 8) 37 | 38 | def forward(self, inputs): 39 | x = self.layer1(inputs) 40 | x = self.layer2(x) 41 | x = self.layer3(x) 42 | x = self.layer4(x) 43 | 44 | x, attention_score = self.self_attention_layer(x) 45 | 46 | output = self.layer5(x) 47 | 48 | return output.view(-1, 1).squeeze(1) 49 | 50 | -------------------------------------------------------------------------------- /Models/VAE_Model/VAE.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from Models.VAE_Model.Encoder import Encoder 5 | from Models.VAE_Model.Decoder import Decoder 6 | from torch import optim 7 | 8 | 9 | class vae(nn.Module): 10 | def __init__(self, args, device): 11 | super(vae, self).__init__() 12 | self.input_dim = args.input_dim 13 | self.hidden_dim = args.hidden_dim 14 | self.hidden_dim2 = args.hidden_dim2 15 | self.output_dim = args.output_dim 16 | self.latent_dim = args.latent_dim 17 | self.channels = args.channels 18 | 19 | self.encoder = Encoder(self.input_dim * self.channels, self.hidden_dim, self.hidden_dim2, self.output_dim) 20 | self.decoder = Decoder(self.latent_dim, self.hidden_dim2, self.hidden_dim, self.input_dim * self.channels) 21 | 22 | self.enc_mu = nn.Linear(self.output_dim, self.latent_dim) 23 | self.enc_log_sigma = nn.Linear(self.output_dim, self.latent_dim) 24 | 25 | self.optimizer = optim.Adam(self.parameters(), lr=args.learning_rate) 26 | self.mse_loss = nn.MSELoss() 27 | 28 | self.z_mean = None 29 | self.z_sigma = None 30 | 31 | self.device = device 32 | self.to(self.device) 33 | 34 | def sampling_latent(self, encoder_output): 35 | mu = self.enc_mu(encoder_output) 36 | log_sigma = self.enc_log_sigma(encoder_output) 37 | 38 | sigma = torch.exp(log_sigma) 39 | std_z = torch.from_numpy(np.random.normal(0, 1, size=sigma.size())).float() 40 | 41 | self.z_mean = mu 42 | self.z_sigma = sigma 43 | 44 | return mu + sigma * std_z.to(self.device) 45 | 46 | def forward(self, inputs): 47 | encoder_output = self.encoder(inputs) 48 | latent = self.sampling_latent(encoder_output) 49 | return self.decoder(latent) 50 | 51 | def learn(self, inputs): 52 | decoder_output = self.forward(inputs) 53 | latent_loss = self.get_latent_loss(self.z_mean, self.z_sigma) 54 | loss = self.mse_loss(decoder_output, inputs) + latent_loss 55 | 56 | self.optimizer.zero_grad() 57 | loss.backward() 58 | self.optimizer.step() 59 | 60 | return loss.item() 61 | 62 | def get_latent_loss(self, z_mean, z_std): 63 | mean_sq = z_mean * z_mean 64 | stdd_sq = z_std * z_std 65 | return 0.5 * torch.mean(mean_sq + stdd_sq - torch.log(stdd_sq) - 1) 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Model/GPT2Model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from Models.GPT2_Model.Model.Block import Block 4 | from Models.GPT2_Model.Model.LayerNorm import LayerNorm 5 | import copy 6 | 7 | 8 | class GPT2Model(nn.Module): 9 | def __init__(self, config): 10 | super(GPT2Model, self).__init__() 11 | self.n_layer = config.n_layer 12 | self.n_embd = config.n_embd 13 | self.n_vocab = config.vocab_size 14 | 15 | self.wte = nn.Embedding(config.vocab_size, config.n_embd) 16 | self.wpe = nn.Embedding(config.n_positions, config.n_embd) 17 | block = Block(config.n_ctx, config, scale=True) 18 | self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) 19 | self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) 20 | 21 | def set_embeddings_weights(self, model_embeddings_weights): 22 | embed_shape = model_embeddings_weights.shape 23 | self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) 24 | self.decoder.weight = model_embeddings_weights # Tied weights 25 | 26 | def forward(self, input_ids, position_ids=None, token_type_ids=None, past=None): 27 | if past is None: 28 | past_length = 0 29 | past = [None] * len(self.h) 30 | else: 31 | past_length = past[0][0].size(-2) 32 | if position_ids is None: 33 | position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, 34 | device=input_ids.device) 35 | position_ids = position_ids.unsqueeze(0).expand_as(input_ids) 36 | 37 | input_shape = input_ids.size() 38 | input_ids = input_ids.view(-1, input_ids.size(-1)) 39 | position_ids = position_ids.view(-1, position_ids.size(-1)) 40 | 41 | inputs_embeds = self.wte(input_ids) 42 | position_embeds = self.wpe(position_ids) 43 | if token_type_ids is not None: 44 | token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) 45 | token_type_embeds = self.wte(token_type_ids) 46 | else: 47 | token_type_embeds = 0 48 | hidden_states = inputs_embeds + position_embeds + token_type_embeds 49 | presents = [] 50 | for block, layer_past in zip(self.h, past): 51 | hidden_states, present = block(hidden_states, layer_past) 52 | presents.append(present) 53 | hidden_states = self.ln_f(hidden_states) 54 | output_shape = input_shape + (hidden_states.size(-1),) 55 | return hidden_states.view(*output_shape), presents -------------------------------------------------------------------------------- /Models/GPT2_Model/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import torch 4 | import random 5 | import argparse 6 | import numpy as np 7 | from Models.GPT2_Model.Model.GPT2LMHeadModel import (GPT2LMHeadModel) 8 | from Models.GPT2_Model.Utils import load_weight 9 | from Models.GPT2_Model.config import GPT2Config 10 | from Models.GPT2_Model.Utils import sample_sequence 11 | from Models.GPT2_Model.text_Encoder import get_encoder 12 | 13 | 14 | def text_generator(state_dict): 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--text", type=str, default='korean air is airline', required=False) 17 | parser.add_argument("--quiet", type=bool, default=False) 18 | parser.add_argument("--nsamples", type=int, default=1) 19 | parser.add_argument('--unconditional', action='store_true', help='If true, unconditional generation.') 20 | parser.add_argument("--batch_size", type=int, default=-1) 21 | parser.add_argument("--length", type=int, default=50) 22 | parser.add_argument("--temperature", type=float, default=0.7) 23 | parser.add_argument("--top_k", type=int, default=40) 24 | args = parser.parse_args() 25 | 26 | if args.quiet is False: 27 | print(args) 28 | 29 | if args.batch_size == -1: 30 | args.batch_size = 1 31 | assert args.nsamples % args.batch_size == 0 32 | assert args.nsamples % args.batch_size == 0 33 | 34 | seed = random.randint(0, 2147483647) 35 | np.random.seed(seed) 36 | torch.random.manual_seed(seed) 37 | torch.cuda.manual_seed(seed) 38 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 39 | 40 | # Load Model 41 | enc = get_encoder() 42 | config = GPT2Config() 43 | model = GPT2LMHeadModel(config) 44 | model = load_weight(model, state_dict) 45 | model.to(device) 46 | model.eval() 47 | 48 | if args.length == -1: 49 | args.length = config.n_ctx // 2 50 | elif args.length > config.n_ctx: 51 | raise ValueError("Can't get samples longer than window size: %s" % config.n_ctx) 52 | 53 | print(args.text) 54 | context_tokens = enc.encode(args.text) 55 | 56 | generated = 0 57 | for _ in range(args.nsamples // args.batch_size): 58 | out = sample_sequence( 59 | model=model, length=args.length, 60 | context=context_tokens if not args.unconditional else None, 61 | start_token=enc.encoder['<|endoftext|>'] if args.unconditional else None, 62 | batch_size=args.batch_size, 63 | temperature=args.temperature, top_k=args.top_k, device=device 64 | ) 65 | out = out[:, len(context_tokens):].tolist() 66 | for i in range(args.batch_size): 67 | generated += 1 68 | text = enc.decode(out[i]) 69 | if args.quiet is False: 70 | print("=" * 40 + " SAMPLE " + str(generated) + " " + "=" * 40) 71 | print(text) 72 | 73 | 74 | if __name__ == '__main__': 75 | if os.path.exists('gpt2-pytorch_model.bin'): 76 | state_dict = torch.load('gpt2-pytorch_model.bin', map_location='cpu' if not torch.cuda.is_available() else None) 77 | text_generator(state_dict) 78 | else: 79 | print('Please download gpt2-pytorch_model.bin') 80 | sys.exit() 81 | -------------------------------------------------------------------------------- /Models/GPT2_Model/text_Encoder.py: -------------------------------------------------------------------------------- 1 | from Models.GPT2_Model.Utils import convert_byte_to_unicode 2 | from Models.GPT2_Model.Utils import get_pairs 3 | import regex 4 | import json 5 | 6 | 7 | class Encoder: 8 | def __init__(self, encoder, byte_pair_encoding_merges, errors='replace'): 9 | self.encoder = encoder 10 | self.decoder = {v: k for k, v in self.encoder.items()} 11 | self.errors = errors 12 | self.byte_encoder = convert_byte_to_unicode() 13 | self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} 14 | self.byte_pair_encoding_ranks = dict(zip(byte_pair_encoding_merges, range(len(byte_pair_encoding_merges)))) 15 | self.cache = {} 16 | 17 | self.pat = regex.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""") 18 | 19 | def byte_pair_encoding(self, token): 20 | if token in self.cache: 21 | return self.cache[token] 22 | 23 | word = tuple(token) 24 | pairs = get_pairs(word) 25 | 26 | if not pairs: 27 | return token 28 | 29 | while True: 30 | bigram = min(pairs, key=lambda pair: self.byte_pair_encoding_ranks.get(pair, float('inf'))) 31 | if bigram not in self.byte_pair_encoding_ranks: 32 | break 33 | 34 | first, second = bigram 35 | new_word = [] 36 | i = 0 37 | 38 | while i < len(word): 39 | try: 40 | j = word.index(first, i) 41 | new_word.extend(word[i:j]) 42 | i = j 43 | except: 44 | new_word.extend(word[i:]) 45 | break 46 | 47 | if word[i] == first and i < len(word) - 1 and word[i + 1] == second: 48 | new_word.append(first + second) 49 | i += 2 50 | else: 51 | new_word.append(word[i]) 52 | i += 1 53 | new_word = tuple(new_word) 54 | word = new_word 55 | if len(word) == 1: 56 | break 57 | else: 58 | pairs = get_pairs(word) 59 | 60 | word = ' '.join(word) 61 | self.cache[token] = word 62 | return word 63 | 64 | def encode(self, text): 65 | bpe_tokens = [] 66 | for token in regex.findall(self.pat, text): 67 | token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) 68 | bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.byte_pair_encoding(token).split(' ')) 69 | return bpe_tokens 70 | 71 | def decode(self, tokens): 72 | text = ''.join([self.decoder[token] for token in tokens]) 73 | text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors) 74 | return text 75 | 76 | 77 | def get_encoder(): 78 | with open('encoder.json', 'r') as f: 79 | encoder = json.load(f) 80 | with open('vocab.bpe', 'r', encoding="utf-8") as f: 81 | bpe_data = f.read() 82 | bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]] 83 | 84 | return Encoder(encoder=encoder, byte_pair_encoding_merges=bpe_merges) 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /Models/Resnet_Model/Resnet.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.optim as optim 3 | from Models.Resnet_Model.BasicBlock import BasicBlock 4 | from Models.Resnet_Model.Bottle_neck import Bottle_neck 5 | from Utils import conv_1x1 6 | 7 | 8 | class ResNet(nn.Module): 9 | def __init__(self, input_channel, layers, num_classes=10, zero_init_residual=True): 10 | super(ResNet, self).__init__() 11 | self.in_planes = 64 12 | 13 | self.inputLayer = nn.Sequential(nn.Conv2d(input_channel, 64, kernel_size=7, stride=2, padding=3, bias=False), 14 | nn.BatchNorm2d(64), 15 | nn.ReLU(inplace=True), 16 | nn.MaxPool2d(kernel_size=3, stride=2, padding=1)) 17 | 18 | self.inputLayer2 = nn.Sequential(self.make_layer(Bottle_neck, 64, layers[0]), 19 | self.make_layer(Bottle_neck, 128, layers[1], stride=2), 20 | self.make_layer(Bottle_neck, 256, layers[2], stride=2), 21 | self.make_layer(Bottle_neck, 512, layers[3], stride=2)) 22 | 23 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 24 | self.fc = nn.Linear(512 * Bottle_neck.expansion, num_classes) 25 | 26 | for m in self.modules(): 27 | if isinstance(m, nn.Conv2d): 28 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 29 | elif isinstance(m, nn.BatchNorm2d): 30 | nn.init.constant_(m.weight, 1) 31 | nn.init.constant_(m.bias, 0) 32 | 33 | if zero_init_residual: 34 | for m in self.modules(): 35 | if isinstance(m, Bottle_neck): 36 | nn.init.constant_(m.bn3.weight, 0) 37 | elif isinstance(m, BasicBlock): 38 | nn.init.constant_(m.bn2.weight, 0) 39 | 40 | self.optimizer = optim.Adam(self.parameters(), lr=0.001) 41 | self.cross_entropy = nn.CrossEntropyLoss() 42 | 43 | def make_layer(self, block, planes, blocks, stride=1): 44 | down_sample = None 45 | 46 | if stride != 1 or self.in_planes != planes * block.expansion: 47 | down_sample = nn.Sequential( 48 | conv_1x1(self.in_planes, planes * block.expansion, stride), 49 | nn.BatchNorm2d(planes * block.expansion), 50 | ) 51 | 52 | layers = [] 53 | layers.append(block(self.in_planes, planes, stride, down_sample)) 54 | 55 | self.in_planes = planes * block.expansion 56 | 57 | for _ in range(1, blocks): 58 | layers.append(block(self.in_planes, planes)) 59 | 60 | final_layers = nn.Sequential(*layers) 61 | return final_layers 62 | 63 | def forward(self, x): 64 | x = self.inputLayer(x) 65 | x = self.inputLayer2(x) 66 | x = self.avgpool(x) 67 | 68 | x = x.view(x.size(0), -1) 69 | x = self.fc(x) 70 | 71 | return x 72 | 73 | def Learn(self, inputs, labels): 74 | predict = self.forward(inputs) 75 | loss = self.cross_entropy(predict, labels) 76 | 77 | self.optimizer.zero_grad() 78 | loss.backward() 79 | self.optimizer.step() 80 | 81 | return loss.item() 82 | 83 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from tqdm import trange 4 | 5 | 6 | def convert_byte_to_unicode(): 7 | bs = list(range(ord("!"), ord('~') + 1)) + list(range(ord('!'), ord('-') + 1)) + list(range(ord('@'), ord('y') + 1)) 8 | cs = bs[:] 9 | n = 0 10 | for b in range(2**8): 11 | if b not in bs: 12 | bs.append(b) 13 | cs.append(2**8+n) 14 | n += 1 15 | 16 | cs = [chr(n) for n in cs] 17 | return dict(zip(bs, cs)) 18 | 19 | 20 | def get_pairs(word): 21 | pairs = set() 22 | prev_char = word[0] 23 | 24 | for char in word[1:]: 25 | pairs.add((prev_char, char)) 26 | prev_char = char 27 | 28 | return pairs 29 | 30 | 31 | def load_weight(model, state_dict): 32 | old_keys = [] 33 | new_keys = [] 34 | for key in state_dict.keys(): 35 | new_key = None 36 | if key.endswith(".g"): 37 | new_key = key[:-2] + ".weight" 38 | elif key.endswith(".b"): 39 | new_key = key[:-2] + ".bias" 40 | elif key.endswith(".w"): 41 | new_key = key[:-2] + ".weight" 42 | if new_key: 43 | old_keys.append(key) 44 | new_keys.append(new_key) 45 | for old_key, new_key in zip(old_keys, new_keys): 46 | state_dict[new_key] = state_dict.pop(old_key) 47 | 48 | missing_keys = [] 49 | unexpected_keys = [] 50 | error_msgs = [] 51 | 52 | metadata = getattr(state_dict, "_metadata", None) 53 | state_dict = state_dict.copy() 54 | if metadata is not None: 55 | state_dict._metadata = metadata 56 | 57 | def load(module, prefix=""): 58 | local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) 59 | module._load_from_state_dict( 60 | state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs 61 | ) 62 | for name, child in module._modules.items(): 63 | if child is not None: 64 | load(child, prefix + name + ".") 65 | 66 | start_model = model 67 | if hasattr(model, "transformer") and all(not s.startswith('transformer.') for s in state_dict.keys()): 68 | start_model = model.transformer 69 | load(start_model, prefix="") 70 | 71 | model.set_tied() 72 | return model 73 | 74 | 75 | def top_k_logits(logits, k): 76 | if k == 0: 77 | return logits 78 | values, _ = torch.topk(logits, k) 79 | min_values = values[:, -1] 80 | return torch.where(logits < min_values, torch.ones_like(logits, dtype=logits.dtype) * -1e10, logits) 81 | 82 | 83 | def sample_sequence(model, length, start_token=None, batch_size=None, context=None, temperature=1, top_k=0, device='cuda', sample=True): 84 | if start_token is None: 85 | assert context is not None, 'Specify exactly one of start_token and context!' 86 | context = torch.tensor(context, device=device, dtype=torch.long).unsqueeze(0).repeat(batch_size, 1) 87 | else: 88 | assert context is None, 'Specify exactly one of start_token and context!' 89 | context = torch.full((batch_size, 1), start_token, device=device, dtype=torch.long) 90 | prev = context 91 | output = context 92 | past = None 93 | with torch.no_grad(): 94 | for i in trange(length): 95 | logits, past = model(prev, past=past) 96 | logits = logits[:, -1, :] / temperature 97 | logits = top_k_logits(logits, k=top_k) 98 | log_probs = F.softmax(logits, dim=-1) 99 | if sample: 100 | prev = torch.multinomial(log_probs, num_samples=1) 101 | else: 102 | _, prev = torch.topk(log_probs, k=1, dim=-1) 103 | output = torch.cat((output, prev), dim=1) 104 | return output 105 | -------------------------------------------------------------------------------- /Models/GPT2_Model/Model/Block.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from Models.GPT2_Model.Model.LayerNorm import LayerNorm 3 | import torch 4 | from torch.nn.parameter import Parameter 5 | import math 6 | 7 | 8 | def gelu(x): 9 | return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) 10 | 11 | 12 | class Block(nn.Module): 13 | def __init__(self, n_ctx, config, scale=False): 14 | super(Block, self).__init__() 15 | nx = config.n_embd 16 | self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) 17 | self.attn = Attention(nx, n_ctx, config, scale) 18 | self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) 19 | self.mlp = MLP(4 * nx, config) 20 | 21 | def forward(self, x, layer_past=None): 22 | a, present = self.attn(self.ln_1(x), layer_past=layer_past) 23 | x = x + a 24 | m = self.mlp(self.ln_2(x)) 25 | x = x + m 26 | return x, present 27 | 28 | 29 | class MLP(nn.Module): 30 | def __init__(self, n_state, config): # in MLP: n_state=3072 (4 * n_embd) 31 | super(MLP, self).__init__() 32 | nx = config.n_embd 33 | self.c_fc = Conv1D(n_state, nx) 34 | self.c_proj = Conv1D(nx, n_state) 35 | self.act = gelu 36 | 37 | def forward(self, x): 38 | h = self.act(self.c_fc(x)) 39 | h2 = self.c_proj(h) 40 | return h2 41 | 42 | 43 | class Attention(nn.Module): 44 | def __init__(self, nx, n_ctx, config, scale=False): 45 | super(Attention, self).__init__() 46 | n_state = nx # in Attention: n_state=768 (nx=n_embd) 47 | # [switch nx => n_state from Block to Attention to keep identical to TF implem] 48 | assert n_state % config.n_head == 0 49 | self.register_buffer("bias", torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx)) 50 | self.n_head = config.n_head 51 | self.split_size = n_state 52 | self.scale = scale 53 | self.c_attn = Conv1D(n_state * 3, nx) 54 | self.c_proj = Conv1D(n_state, nx) 55 | 56 | def _attn(self, q, k, v): 57 | w = torch.matmul(q, k) 58 | if self.scale: 59 | w = w / math.sqrt(v.size(-1)) 60 | nd, ns = w.size(-2), w.size(-1) 61 | b = self.bias[:, :, ns-nd:ns, :ns] 62 | w = w * b - 1e10 * (1 - b) 63 | w = nn.Softmax(dim=-1)(w) 64 | return torch.matmul(w, v) 65 | 66 | def merge_heads(self, x): 67 | x = x.permute(0, 2, 1, 3).contiguous() 68 | new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),) 69 | return x.view(*new_x_shape) # in Tensorflow implem: fct merge_states 70 | 71 | def split_heads(self, x, k=False): 72 | new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head) 73 | x = x.view(*new_x_shape) # in Tensorflow implem: fct split_states 74 | if k: 75 | return x.permute(0, 2, 3, 1) # (batch, head, head_features, seq_length) 76 | else: 77 | return x.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features) 78 | 79 | def forward(self, x, layer_past=None): 80 | x = self.c_attn(x) 81 | query, key, value = x.split(self.split_size, dim=2) 82 | query = self.split_heads(query) 83 | key = self.split_heads(key, k=True) 84 | value = self.split_heads(value) 85 | if layer_past is not None: 86 | past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1] # transpose back cf below 87 | key = torch.cat((past_key, key), dim=-1) 88 | value = torch.cat((past_value, value), dim=-2) 89 | present = torch.stack((key.transpose(-2, -1), value)) # transpose to have same shapes for stacking 90 | a = self._attn(query, key, value) 91 | a = self.merge_heads(a) 92 | a = self.c_proj(a) 93 | return a, present 94 | 95 | 96 | class Conv1D(nn.Module): 97 | def __init__(self, nf, nx): 98 | super(Conv1D, self).__init__() 99 | self.nf = nf 100 | w = torch.empty(nx, nf) 101 | nn.init.normal_(w, std=0.02) 102 | self.weight = Parameter(w) 103 | self.bias = Parameter(torch.zeros(nf)) 104 | 105 | def forward(self, x): 106 | size_out = x.size()[:-1] + (self.nf,) 107 | x = torch.addmm(self.bias, x.view(-1, x.size(-1)), self.weight) 108 | x = x.view(*size_out) 109 | return x 110 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 16 | 17 | 18 | 19 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 59 | 60 | 61 | 80 | 81 | 82 | 101 | 102 | 103 | 122 | 123 | 124 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | C:\Users\JGY\AppData\Roaming\Subversion 155 | 125 156 | 157 | 158 | 159 | 160 | 1608091939310 161 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | --------------------------------------------------------------------------------