├── .gitignore ├── README.md ├── docs ├── 马天行.深度学习.Exp0 │ ├── 实验项目结构概述.md │ └── 实验项目结构概述.pdf ├── 马天行.深度学习.Exp1 │ ├── 实验一.md │ ├── 实验一.pdf │ └── 实验一.pptx ├── 马天行.深度学习.Exp2 │ ├── 实验二.md │ ├── 实验二.pdf │ └── 实验二.pptx ├── 马天行.深度学习.Exp3 │ ├── 实验三.md │ ├── 实验三.pdf │ └── 实验三.pptx └── 马天行.深度学习.Exp4 │ ├── 实验四.md │ ├── 实验四.pdf │ └── 实验四.pptx ├── logs ├── dogs_vs_cats.resnet18.log ├── mnist.conv.log ├── movie.conv.log └── poem(640).lstm.log ├── task_01.py ├── task_02.py ├── task_03.py ├── task_04.py ├── task_05.py ├── task_plot.py └── utils ├── __init__.py ├── cv ├── __init__.py ├── config.py ├── dataset.py ├── handler.py ├── nnmodels.py ├── recorder.py └── trainer.py └── nlp ├── __init__.py ├── config.py ├── dataset.py ├── handler.py ├── nnmodels.py ├── recorder.py └── trainer.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | datasets 3 | cache 4 | checkpoints 5 | .idea 6 | .vscode -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CourseDL 2 | 3 | 国科大深度学习课程作业（2023） 4 | 5 | ## 00 概述 6 | 7 | 这个文档是下面四个实验报告的预置内容，主要介绍了一下实验项目文件结构和实验流程。 8 | 9 | 整体代码可以到仓库查看：https://github.com/Noitolar/CourseDL.git 10 | 11 | ## 01 文件结构 12 | 13 | * checkpoints：保存的模型权重 14 | * mnist.conv.pt 15 | * dogs_vs_cats.resnet18.pt 16 | * poem(640).lstm.pt 17 | * movie.conv.pt 18 | * datasets：数据集 19 | * mnist 20 | * dogs_vs_cats 21 | * poem(640) 22 | * movie 23 | * logs：实验日志 24 | * mnist.conv.log 25 | * dogs_vs_cats.resnet18.log 26 | * poem(640).lstm.log 27 | * movie.conv.log 28 | * utils：工具类和工具函数 29 | * \_\_init\_\_.py 30 | * cv：前两个实验用到的 31 | * \_\_init\_\_.py 32 | * config.py：实验配置 33 | * dataset.py：数据集处理 34 | * handler.py：模型操作 35 | * nnmodels.py：神经网络模型 36 | * recorder.py：实验记录 37 | * trainer.py：模型训练和评估 38 | * nlp：后两个实验用到的 39 | * \_\_init\_\_.py 40 | * config.py 41 | * dataset.py 42 | * handler.py 43 | * nnmodels.py 44 | * recorder.py 45 | * trainer.py 46 | * task_01.py：实验一 47 | * task_02.py：实验二 48 | * task_03.py：实验三 49 | * task_04.py：实验四 50 | 51 | ## 02 实验流程 52 | 53 | 由于面对对象式的编程风格，四个实验的实验流程基本一致，包括但不限于： 54 | 55 | * 初始化实验配置： 56 | * 神经网络模型类名 57 | * 神经网络模型参数 58 | * 设备 59 | * 损失函数类名 60 | * 损失函数参数 61 | * 日志文件路径 62 | * 数据集名称 63 | * 随机种子 64 | * 训练集预处理（CV） 65 | * 验证集预处理（CV） 66 | * 批大小 67 | * 迭代轮数 68 | * 优化器类名 69 | * 优化器参数 70 | * 学习率调整策略类名 71 | * 学习率调整策略参数 72 | * 权重文件路径 73 | 74 | * 基于配置初始化模型Handler 75 | * 将配置内容记录在日志 76 | * 基于配置初始化数据集和加载器 77 | * 基于配置初始化训练器 78 | * 进行对应轮次的训练和验证/生成，并将结果记录在日志中 79 | 80 | 下面以task_01.py为例： 81 | 82 | ```python 83 | import torch 84 | import torch.nn as nn 85 | import torch.optim as optim 86 | import torch.utils.data as tdata 87 | import torchvision.transforms as trans 88 | import utils.cv as ucv 89 | 90 | if __name__ == "__main__": 91 | # 实验配置 92 | config = ucv.config.ConfigObject() 93 | config.model_class = ucv.nnmodels.SimpleConvClassifier 94 | config.model_params = {"num_classes": 10, "num_channels": 1} 95 | config.device = "cuda:0" 96 | config.criterion_class = nn.CrossEntropyLoss 97 | config.criterion_params = {} 98 | config.log_path = "./logs/mnist.conv.log" 99 | config.dataset = "mnist" 100 | config.seed = 0 101 | config.trn_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 102 | config.val_preprocess = config.trn_preprocess 103 | config.batch_size = 32 104 | config.num_epochs = 8 105 | config.optimizer_class = optim.SGD 106 | config.optimizer_params = {"lr": 0.001, "momentum": 0.9, "nesterov": True} 107 | config.scheduler_class = optim.lr_scheduler.MultiStepLR 108 | config.scheduler_params = {"milestones": [4, 6], "gamma": 0.1} 109 | config.checkpoint_path = "./checkpoints/mnist.conv.pt" 110 | 111 | # 初始化模型Handler 112 | handler = ucv.handler.ModelHandlerCv(config) 113 | 114 | # 将配置录入日志 115 | handler.log_config() 116 | 117 | # 初始化数据集和加载器 118 | trn_set, val_set = ucv.dataset.get_dataset(config) 119 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 120 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 121 | 122 | # 初始化训练器 123 | trainer = ucv.trainer.Trainer(handler) 124 | 125 | # 训练并记录结果 126 | best_val_accuracy = 0.0 127 | for epoch in range(config.num_epochs): 128 | handler.log(" " + "=" * 40) 129 | trn_report = trainer.train(trn_loader) 130 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 131 | val_report = trainer.validate(val_loader) 132 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 133 | if val_report["accuracy"] > best_val_accuracy: 134 | best_val_accuracy = val_report["accuracy"] 135 | if config.checkpoint_path is not None: 136 | torch.save(handler.model.state_dict, config.checkpoint_path) 137 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 138 | ``` 139 | 140 | -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp0/实验项目结构概述.md: -------------------------------------------------------------------------------- 1 | # 实验项目结构概述 2 | 3 | 202218018670003 马天行 4 | 5 | ## 00 概述 6 | 7 | 这个文档是下面四个实验报告的预置内容，主要介绍了一下实验项目文件结构和实验流程。 8 | 9 | 整体代码可以到仓库查看：https://github.com/Noitolar/CourseDL.git 10 | 11 | ## 01 文件结构 12 | 13 | * checkpoints：保存的模型权重 14 | * mnist.conv.pt 15 | * dogs_vs_cats.resnet18.pt 16 | * poem(640).lstm.pt 17 | * movie.conv.pt 18 | * datasets：数据集 19 | * mnist 20 | * dogs_vs_cats 21 | * poem(640) 22 | * movie 23 | * logs：实验日志 24 | * mnist.conv.log 25 | * dogs_vs_cats.resnet18.log 26 | * poem(640).lstm.log 27 | * movie.conv.log 28 | * utils：工具类和工具函数 29 | * \_\_init\_\_.py 30 | * cv：前两个实验用到的 31 | * \_\_init\_\_.py 32 | * config.py：实验配置 33 | * dataset.py：数据集处理 34 | * handler.py：模型操作 35 | * nnmodels.py：神经网络模型 36 | * recorder.py：实验记录 37 | * trainer.py：模型训练和评估 38 | * nlp：后两个实验用到的 39 | * \_\_init\_\_.py 40 | * config.py 41 | * dataset.py 42 | * handler.py 43 | * nnmodels.py 44 | * recorder.py 45 | * trainer.py 46 | * task_01.py：实验一 47 | * task_02.py：实验二 48 | * task_03.py：实验三 49 | * task_04.py：实验四 50 | 51 | ## 02 实验流程 52 | 53 | 由于面对对象式的编程风格，四个实验的实验流程基本一致，包括但不限于： 54 | 55 | * 初始化实验配置： 56 | * 神经网络模型类名 57 | * 神经网络模型参数 58 | * 设备 59 | * 损失函数类名 60 | * 损失函数参数 61 | * 日志文件路径 62 | * 数据集名称 63 | * 随机种子 64 | * 训练集预处理（CV） 65 | * 验证集预处理（CV） 66 | * 批大小 67 | * 迭代轮数 68 | * 优化器类名 69 | * 优化器参数 70 | * 学习率调整策略类名 71 | * 学习率调整策略参数 72 | * 权重文件路径 73 | 74 | * 基于配置初始化模型Handler 75 | * 将配置内容记录在日志 76 | * 基于配置初始化数据集和加载器 77 | * 基于配置初始化训练器 78 | * 进行对应轮次的训练和验证/生成，并将结果记录在日志中 79 | 80 | 下面以task_01.py为例： 81 | 82 | ```python 83 | import torch 84 | import torch.nn as nn 85 | import torch.optim as optim 86 | import torch.utils.data as tdata 87 | import torchvision.transforms as trans 88 | import utils.cv as ucv 89 | 90 | if __name__ == "__main__": 91 | # 实验配置 92 | config = ucv.config.ConfigObject() 93 | config.model_class = ucv.nnmodels.SimpleConvClassifier 94 | config.model_params = {"num_classes": 10, "num_channels": 1} 95 | config.device = "cuda:0" 96 | config.criterion_class = nn.CrossEntropyLoss 97 | config.criterion_params = {} 98 | config.log_path = "./logs/mnist.conv.log" 99 | config.dataset = "mnist" 100 | config.seed = 0 101 | config.trn_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 102 | config.val_preprocess = config.trn_preprocess 103 | config.batch_size = 32 104 | config.num_epochs = 8 105 | config.optimizer_class = optim.SGD 106 | config.optimizer_params = {"lr": 0.001, "momentum": 0.9, "nesterov": True} 107 | config.scheduler_class = optim.lr_scheduler.MultiStepLR 108 | config.scheduler_params = {"milestones": [4, 6], "gamma": 0.1} 109 | config.checkpoint_path = "./checkpoints/mnist.conv.pt" 110 | 111 | # 初始化模型Handler 112 | handler = ucv.handler.ModelHandlerCv(config) 113 | 114 | # 将配置录入日志 115 | handler.log_config() 116 | 117 | # 初始化数据集和加载器 118 | trn_set, val_set = ucv.dataset.get_dataset(config) 119 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 120 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 121 | 122 | # 初始化训练器 123 | trainer = ucv.trainer.Trainer(handler) 124 | 125 | # 训练并记录结果 126 | best_val_accuracy = 0.0 127 | for epoch in range(config.num_epochs): 128 | handler.log(" " + "=" * 40) 129 | trn_report = trainer.train(trn_loader) 130 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 131 | val_report = trainer.validate(val_loader) 132 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 133 | if val_report["accuracy"] > best_val_accuracy: 134 | best_val_accuracy = val_report["accuracy"] 135 | if config.checkpoint_path is not None: 136 | torch.save(handler.model.state_dict, config.checkpoint_path) 137 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 138 | ``` 139 | 140 | -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp0/实验项目结构概述.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp0/实验项目结构概述.pdf -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp1/实验一.md: -------------------------------------------------------------------------------- 1 | # 实验一 2 | 3 | 202218018670003 马天行 4 | 5 | ## 00 概述 6 | 7 | 实验内容是使用手工Conv网络实现Mnist图片分类任务，文件结构等信息可以在前置文档《实验项目结构概述》中看到。 8 | 9 | 完整代码和文档见仓库：https://github.com/Noitolar/CourseDL.git 10 | 11 | ## 01 数据集和预处理 12 | 13 | 默认预处理为向量化与标准化，没有额外的数据增强 14 | 15 | ```python 16 | import torchvision.datasets as vdatasets 17 | import torchvision.transforms as trans 18 | import utils.cv as ucv 19 | 20 | 21 | def get_dataset(config: ucv.config.ConfigObject): 22 | if config.dataset == "mnist": 23 | if config.trn_preprocess is None: 24 | config.trn_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 25 | if config.val_preprocess is None: 26 | config.val_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 27 | trn_set = vdatasets.MNIST(root=f"./datasets/mnist", train=True, transform=config.trn_preprocess, download=True) 28 | val_set = vdatasets.MNIST(root=f"./datasets/mnist", train=False, transform=config.val_preprocess, download=True) 29 | return trn_set, val_set 30 | elif ... 31 | ``` 32 | 33 | ## 02 神经网络模型 34 | 35 | 两层网络，每一层由二维卷积+ReLU激活+BatchNorm+二维最大池化组成，卷积核大小为5\*5、步长为1、填充为2、池化核大小为2\*2，最终输出32个FeatureMap，每个FeatureMap大小为7\*7，最终摊平经过全连接进行分类。 36 | 37 | ```python 38 | import torch 39 | import torch.nn as nn 40 | import torchvision.models as vmodels 41 | 42 | 43 | class SimpleConvClassifier(nn.Module): 44 | def __init__(self, num_classes, num_channels): 45 | super().__init__() 46 | self.layer1 = self.build_layer(num_channels, 16) 47 | self.layer2 = self.build_layer(16, 32) 48 | self.flatten = nn.Flatten() 49 | self.fc = nn.Linear(32 * 7 * 7, num_classes) 50 | 51 | @staticmethod 52 | def build_layer(conv_in_channels, conv_out_channels, conv_kernel_size=5, conv_stride=1, conv_padding=2, pool_kernel_size=2): 53 | layer = nn.Sequential( 54 | nn.Conv2d(conv_in_channels, conv_out_channels, conv_kernel_size, conv_stride, conv_padding), 55 | nn.ReLU(), nn.BatchNorm2d(conv_out_channels), nn.MaxPool2d(pool_kernel_size)) 56 | return layer 57 | 58 | def forward(self, inputs): 59 | outputs = self.layer1(inputs) 60 | outputs = self.layer2(outputs) 61 | outputs = self.flatten(outputs) 62 | outputs = self.fc(outputs) 63 | return outputs 64 | ``` 65 | 66 | ## 03 模型操作 67 | 68 | 这个类用于对模型进行一些高级操作，成员包括： 69 | 70 | * 配置项 71 | * 记录器（用于记录训练过程中的指标以及进行日志管理） 72 | * 设备 73 | * 神经网络模型 74 | * 损失函数 75 | 76 | 类方法包括： 77 | 78 | * 日志记录：调用成员记录器的方法，将指定信息录入日志，并同步显示在终端 79 | * 配置记录：按照固定格式将实验配置记录到日志，并同步显示在终端 80 | * 数据转移：将内存中的张量数据转移到配置的设备的内存/显存中 81 | * 正向调用：根据输入和标签输出预测值和损失值，如果没有输入标签则只输出预测值 82 | 83 | 在初始化成员对象之前会先设置全局随机数种子，保证实验结果可以复现。 84 | 85 | ```python 86 | import torch 87 | import torch.nn as nn 88 | import torchvision.transforms as trans 89 | import time 90 | import transformers as tfm 91 | import utils.cv as ucv 92 | 93 | 94 | class ModelHandlerCv(nn.Module): 95 | def __init__(self, config: ucv.config.ConfigObject): 96 | super().__init__() 97 | self.config = config 98 | tfm.set_seed(config.seed) 99 | self.recorder = ucv.recorder.Recorder(config.log_path) 100 | self.device = config.device 101 | self.model = config.model_class(**config.model_params).to(config.device) 102 | self.criterion = config.criterion_class(**config.criterion_params) 103 | 104 | def log(self, message): 105 | self.recorder.audit(message) 106 | 107 | def log_config(self): 108 | self.log(f"\n\n[+] exp starts from: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}") 109 | for config_key, config_value in self.config.params_dict.items(): 110 | if config_value is None: 111 | continue 112 | elif config_key.endswith("_class"): 113 | self.log(f"[+] {config_key.replace('_class', '')}: {config_value.__name__}") 114 | elif config_key.endswith("_params") and isinstance(config_value, dict): 115 | for param_key, param_value in config_value.items(): 116 | self.log(f" [-] {config_key.replace('_params', '')}.{param_key}: {param_value}") 117 | elif isinstance(config_value, trans.transforms.Compose): 118 | self.log(f"[+] {config_key}:") 119 | for index, value in enumerate(str(config_value).replace(" ", "").split("\n")[1:-1]): 120 | self.log(f" [-] {index:02d}: {value}") 121 | else: 122 | self.log(f"[+] {config_key}: {config_value}") 123 | 124 | def device_transfer(self, data): 125 | if isinstance(data, torch.Tensor): 126 | data = data.to(self.device) 127 | if isinstance(data, dict): 128 | data = {key: value.to(self.device) for key, value in data.items()} 129 | return data 130 | 131 | def forward(self, inputs: torch.Tensor, targets=None): 132 | inputs = self.device_transfer(inputs) 133 | targets = self.device_transfer(targets) 134 | preds = self.model(inputs) 135 | loss = self.criterion(preds, targets) if targets is not None else None 136 | return preds, loss 137 | ``` 138 | 139 | ## 04 记录器 140 | 141 | 这个类用于记录模型训练过程中的一些指标，以及日志管理的相关功能，成员对象包括： 142 | 143 | * 累计准确率 144 | * 累计损失值 145 | * 累计样本数 146 | * 日志记录器 147 | 148 | 成员方法包括： 149 | 150 | * 计算并更新累计准确率、损失值、样本数 151 | * 还原成员变量 152 | * 返回平均准确率和损失值 153 | * 日志录入 154 | 155 | ```python 156 | import numpy as np 157 | import sklearn.metrics as metrics 158 | import logging 159 | import os 160 | 161 | 162 | class Recorder: 163 | def __init__(self, logpath): 164 | self.accumulative_accuracy = 0.0 165 | self.accumulative_loss = 0.0 166 | self.accumulative_num_samples = 0 167 | self.logger = logging.getLogger(__name__) 168 | self.logger.setLevel(logging.DEBUG) 169 | self.logger.addHandler(logging.StreamHandler(stream=None)) 170 | if logpath is not None: 171 | if not os.path.exists(os.path.dirname(logpath)): 172 | os.makedirs(os.path.dirname(logpath)) 173 | logfile = open(logpath, "a", encoding="utf-8") 174 | logfile.close() 175 | self.logger.addHandler(logging.FileHandler(filename=logpath, mode="a")) 176 | 177 | def update(self, preds, targets, loss): 178 | assert len(preds) == len(targets) 179 | num_samples = len(preds) 180 | preds = np.array([pred.argmax() for pred in preds.detach().cpu().numpy()]) 181 | targets = targets.detach().cpu().numpy() 182 | self.accumulative_accuracy += metrics.accuracy_score(y_pred=preds, y_true=targets) * num_samples 183 | self.accumulative_loss += loss * num_samples 184 | self.accumulative_num_samples += num_samples 185 | 186 | def clear(self): 187 | self.accumulative_accuracy = 0.0 188 | self.accumulative_loss = 0.0 189 | self.accumulative_num_samples = 0 190 | 191 | def accuracy(self): 192 | accuracy = self.accumulative_accuracy / self.accumulative_num_samples 193 | loss = self.accumulative_loss / self.accumulative_num_samples 194 | return accuracy, loss 195 | 196 | def audit(self, msg): 197 | self.logger.debug(msg) 198 | ``` 199 | 200 | ## 05 模型训练 201 | 202 | 用于训练和评估模型的类，成员对象包括： 203 | 204 | * 模型操作器 205 | * 优化器 206 | * 学习率调整策略（可选） 207 | 208 | 成员方法包括： 209 | 210 | * 训练：训练模型一轮，会调用模型操作器（ModelHandler）的记录器（Recorder）计算训练时的准确率和损失，并在本轮结束时返回训练报告并重置记录器 211 | * 验证：验证模型，没有反向传播过程，并且不计算梯度以节省显存和算力 212 | 213 | ```python 214 | import torch 215 | import tqdm 216 | 217 | 218 | class Trainer: 219 | def __init__(self, handler): 220 | self.handler = handler 221 | self.optimizer = handler.config.optimizer_class(handler.model.parameters(), **handler.config.optimizer_params) 222 | self.scheduler = handler.config.scheduler_class(self.optimizer, **handler.config.scheduler_params) if handler.config.scheduler_class is not None else None 223 | 224 | def train(self, loader): 225 | self.handler.train() 226 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] training", delay=0.2, leave=False, ascii="->"): 227 | preds, loss = self.handler(inputs, targets) 228 | self.handler.recorder.update(preds, targets, loss) 229 | self.optimizer.zero_grad() 230 | loss.backward() 231 | self.optimizer.step() 232 | accuracy, loss = self.handler.recorder.accuracy() 233 | self.handler.recorder.clear() 234 | if self.scheduler is not None: 235 | self.scheduler.step() 236 | report = {"loss": loss, "accuracy": accuracy} 237 | return report 238 | 239 | @torch.no_grad() 240 | def validate(self, loader): 241 | self.handler.eval() 242 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] validating", delay=0.2, leave=False, ascii="->"): 243 | preds, loss = self.handler(inputs, targets) 244 | self.handler.recorder.update(preds, targets, loss) 245 | accuracy, loss = self.handler.recorder.accuracy() 246 | self.handler.recorder.clear() 247 | report = {"loss": loss, "accuracy": accuracy} 248 | return report 249 | ``` 250 | 251 | ## 06 实验主函数 252 | 253 | 批大小为32，共训练8轮，SGD优化器初始学习率为0.001，在第4轮和第6轮下降至原来的十分之一。 254 | 255 | ```python 256 | import torch 257 | import torch.nn as nn 258 | import torch.optim as optim 259 | import torch.utils.data as tdata 260 | import torchvision.transforms as trans 261 | import utils.cv as ucv 262 | 263 | if __name__ == "__main__": 264 | config = ucv.config.ConfigObject() 265 | 266 | config.model_class = ucv.nnmodels.SimpleConvClassifier 267 | config.model_params = {"num_classes": 10, "num_channels": 1} 268 | config.device = "cuda:0" 269 | config.criterion_class = nn.CrossEntropyLoss 270 | config.criterion_params = {} 271 | config.log_path = "./logs/mnist.conv.log" 272 | 273 | config.dataset = "mnist" 274 | config.seed = 0 275 | config.trn_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 276 | config.val_preprocess = config.trn_preprocess 277 | config.batch_size = 32 278 | config.num_epochs = 8 279 | config.optimizer_class = optim.SGD 280 | config.optimizer_params = {"lr": 0.001, "momentum": 0.9, "nesterov": True} 281 | config.scheduler_class = optim.lr_scheduler.MultiStepLR 282 | config.scheduler_params = {"milestones": [4, 6], "gamma": 0.1} 283 | config.checkpoint_path = "./checkpoints/mnist.conv.pt" 284 | 285 | handler = ucv.handler.ModelHandlerCv(config) 286 | handler.log_config() 287 | 288 | trn_set, val_set = ucv.dataset.get_dataset(config) 289 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 290 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 291 | trainer = ucv.trainer.Trainer(handler) 292 | 293 | best_val_accuracy = 0.0 294 | for epoch in range(config.num_epochs): 295 | handler.log(" " + "=" * 40) 296 | trn_report = trainer.train(trn_loader) 297 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 298 | val_report = trainer.validate(val_loader) 299 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 300 | if val_report["accuracy"] > best_val_accuracy: 301 | best_val_accuracy = val_report["accuracy"] 302 | if config.checkpoint_path is not None: 303 | torch.save(handler.model.state_dict, config.checkpoint_path) 304 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 305 | ``` 306 | 307 | ## 07 实验结果 308 | 309 | 实验日志如下，包含了本次实验的配置以及实验结果，最高验证集准确率为99.18%。 310 | 311 | ``` 312 | [+] exp starts from: 2023-04-08 23:32:41 313 | [+] model: SimpleConvClassifier 314 | [-] model.num_classes: 10 315 | [-] model.num_channels: 1 316 | [+] device: cuda:0 317 | [+] criterion: CrossEntropyLoss 318 | [+] log_path: ./logs/mnist.conv.log 319 | [+] dataset: mnist 320 | [+] seed: 0 321 | [+] trn_preprocess: 322 | [-] 00: ToTensor() 323 | [-] 01: Normalize(mean=[0.485],std=[0.229]) 324 | [+] val_preprocess: 325 | [-] 00: ToTensor() 326 | [-] 01: Normalize(mean=[0.485],std=[0.229]) 327 | [+] batch_size: 32 328 | [+] num_epochs: 8 329 | [+] optimizer: SGD 330 | [-] optimizer.lr: 0.001 331 | [-] optimizer.momentum: 0.9 332 | [-] optimizer.nesterov: True 333 | [+] scheduler: MultiStepLR 334 | [-] scheduler.milestones: [4, 6] 335 | [-] scheduler.gamma: 0.1 336 | [+] checkpoint_path: ./checkpoints/mnist.conv.pt 337 | ======================================== 338 | [001] trn-loss: 0.1085 --- trn-acc: 96.72% 339 | [001] val-loss: 0.0428 --- val-acc: 98.63% 340 | ======================================== 341 | [002] trn-loss: 0.0394 --- trn-acc: 98.84% 342 | [002] val-loss: 0.0339 --- val-acc: 98.94% 343 | ======================================== 344 | [003] trn-loss: 0.0271 --- trn-acc: 99.21% 345 | [003] val-loss: 0.0311 --- val-acc: 98.94% 346 | ======================================== 347 | [004] trn-loss: 0.0196 --- trn-acc: 99.45% 348 | [004] val-loss: 0.0317 --- val-acc: 99.02% 349 | ======================================== 350 | [005] trn-loss: 0.0118 --- trn-acc: 99.77% 351 | [005] val-loss: 0.0266 --- val-acc: 99.12% 352 | ======================================== 353 | [006] trn-loss: 0.0103 --- trn-acc: 99.80% 354 | [006] val-loss: 0.0261 --- val-acc: 99.17% 355 | ======================================== 356 | [007] trn-loss: 0.0095 --- trn-acc: 99.83% 357 | [007] val-loss: 0.0262 --- val-acc: 99.18% 358 | ======================================== 359 | [008] trn-loss: 0.0096 --- trn-acc: 99.84% 360 | [008] val-loss: 0.0260 --- val-acc: 99.17% 361 | [=] best-val-acc: 99.18% 362 | ``` 363 | 364 | -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp1/实验一.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp1/实验一.pdf -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp1/实验一.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp1/实验一.pptx -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp2/实验二.md: -------------------------------------------------------------------------------- 1 | # 实验二 2 | 3 | 202218018670003 马天行 4 | 5 | ## 00 概述 6 | 7 | 实验内容是使用Resnet18网络实现猫狗图片分类任务，文件结构等信息可以在前置文档《实验项目结构概述》中看到。 8 | 9 | 完整代码和文档见仓库：https://github.com/Noitolar/CourseDL.git 10 | 11 | ## 01 数据集和预处理 12 | 13 | 使用Kaggle官方的猫狗分类数据集，在实验之前已经将带有标签的训练集的95%划分到本实验的训练集，另外5%作为验证集。在攻击函数中没有默认的预处理配置项，具体预处理配置详见第06节。 14 | 15 | ```python 16 | import torchvision.datasets as vdatasets 17 | import torchvision.transforms as trans 18 | import utils.cv as ucv 19 | 20 | 21 | def get_dataset(config: ucv.config.ConfigObject): 22 | if config.dataset == "mnist": 23 | ... 24 | else: 25 | assert config.trn_preprocess is not None 26 | assert config.val_preprocess is not None 27 | trn_set = vdatasets.ImageFolder(root=f"./datasets/{config.dataset}/train", transform=config.trn_preprocess) 28 | val_set = vdatasets.ImageFolder(root=f"./datasets/{config.dataset}/validation", transform=config.val_preprocess) 29 | return trn_set, val_set 30 | ``` 31 | 32 | ## 02 神经网络模型 33 | 34 | 基于torchvision官方提供的resnet18网络，可以自定义配置输出的通道数量（例如在进行Mnist分类任务时应该将输入通道数设为1），以及预训练权重，其他不做改动。 35 | 36 | ```python 37 | import torch 38 | import torch.nn as nn 39 | import torchvision.models as vmodels 40 | 41 | 42 | class Resnet18Classifier(nn.Module): 43 | def __init__(self, num_classes, num_channels=3, from_pretrained=None): 44 | super().__init__() 45 | if from_pretrained == "default": 46 | self.resnet = vmodels.resnet18(weights=vmodels.ResNet18_Weights.DEFAULT) 47 | elif from_pretrained == "imagenet": 48 | self.resnet = vmodels.resnet18(weights=vmodels.ResNet18_Weights.IMAGENET1K_V1) 49 | else: 50 | self.resnet = vmodels.resnet18(weights=None) 51 | self.resnet.conv1 = nn.Conv2d(num_channels, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) 52 | self.resnet.fc = nn.Linear(512, num_classes) 53 | 54 | def forward(self, inputs: torch.Tensor): 55 | outputs = self.resnet(inputs) 56 | return outputs 57 | ``` 58 | 59 | ## 03 模型操作 60 | 61 | 第03节到第05节实验一完全一致，如果不想看可以直接跳转到第06节。 62 | 63 | 这个类用于对模型进行一些高级操作，成员包括： 64 | 65 | * 配置项 66 | * 记录器（用于记录训练过程中的指标以及进行日志管理） 67 | * 设备 68 | * 神经网络模型 69 | * 损失函数 70 | 71 | 类方法包括： 72 | 73 | * 日志记录：调用成员记录器的方法，将指定信息录入日志，并同步显示在终端 74 | * 配置记录：按照固定格式将实验配置记录到日志，并同步显示在终端 75 | * 数据转移：将内存中的张量数据转移到配置的设备的内存/显存中 76 | * 正向调用：根据输入和标签输出预测值和损失值，如果没有输入标签则只输出预测值 77 | 78 | 在初始化成员对象之前会先设置全局随机数种子，保证实验结果可以复现。 79 | 80 | ```python 81 | import torch 82 | import torch.nn as nn 83 | import torchvision.transforms as trans 84 | import time 85 | import transformers as tfm 86 | import utils.cv as ucv 87 | 88 | 89 | class ModelHandlerCv(nn.Module): 90 | def __init__(self, config: ucv.config.ConfigObject): 91 | super().__init__() 92 | self.config = config 93 | tfm.set_seed(config.seed) 94 | self.recorder = ucv.recorder.Recorder(config.log_path) 95 | self.device = config.device 96 | self.model = config.model_class(**config.model_params).to(config.device) 97 | self.criterion = config.criterion_class(**config.criterion_params) 98 | 99 | def log(self, message): 100 | self.recorder.audit(message) 101 | 102 | def log_config(self): 103 | self.log(f"\n\n[+] exp starts from: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}") 104 | for config_key, config_value in self.config.params_dict.items(): 105 | if config_value is None: 106 | continue 107 | elif config_key.endswith("_class"): 108 | self.log(f"[+] {config_key.replace('_class', '')}: {config_value.__name__}") 109 | elif config_key.endswith("_params") and isinstance(config_value, dict): 110 | for param_key, param_value in config_value.items(): 111 | self.log(f" [-] {config_key.replace('_params', '')}.{param_key}: {param_value}") 112 | elif isinstance(config_value, trans.transforms.Compose): 113 | self.log(f"[+] {config_key}:") 114 | for index, value in enumerate(str(config_value).replace(" ", "").split("\n")[1:-1]): 115 | self.log(f" [-] {index:02d}: {value}") 116 | else: 117 | self.log(f"[+] {config_key}: {config_value}") 118 | 119 | def device_transfer(self, data): 120 | if isinstance(data, torch.Tensor): 121 | data = data.to(self.device) 122 | if isinstance(data, dict): 123 | data = {key: value.to(self.device) for key, value in data.items()} 124 | return data 125 | 126 | def forward(self, inputs: torch.Tensor, targets=None): 127 | inputs = self.device_transfer(inputs) 128 | targets = self.device_transfer(targets) 129 | preds = self.model(inputs) 130 | loss = self.criterion(preds, targets) if targets is not None else None 131 | return preds, loss 132 | ``` 133 | 134 | ## 04 记录器 135 | 136 | 这个类用于记录模型训练过程中的一些指标，以及日志管理的相关功能，成员对象包括： 137 | 138 | * 累计准确率 139 | * 累计损失值 140 | * 累计样本数 141 | * 日志记录器 142 | 143 | 成员方法包括： 144 | 145 | * 计算并更新累计准确率、损失值、样本数 146 | * 还原成员变量 147 | * 返回平均准确率和损失值 148 | * 日志录入 149 | 150 | ```python 151 | import numpy as np 152 | import sklearn.metrics as metrics 153 | import logging 154 | import os 155 | 156 | 157 | class Recorder: 158 | def __init__(self, logpath): 159 | self.accumulative_accuracy = 0.0 160 | self.accumulative_loss = 0.0 161 | self.accumulative_num_samples = 0 162 | self.logger = logging.getLogger(__name__) 163 | self.logger.setLevel(logging.DEBUG) 164 | self.logger.addHandler(logging.StreamHandler(stream=None)) 165 | if logpath is not None: 166 | if not os.path.exists(os.path.dirname(logpath)): 167 | os.makedirs(os.path.dirname(logpath)) 168 | logfile = open(logpath, "a", encoding="utf-8") 169 | logfile.close() 170 | self.logger.addHandler(logging.FileHandler(filename=logpath, mode="a")) 171 | 172 | def update(self, preds, targets, loss): 173 | assert len(preds) == len(targets) 174 | num_samples = len(preds) 175 | preds = np.array([pred.argmax() for pred in preds.detach().cpu().numpy()]) 176 | targets = targets.detach().cpu().numpy() 177 | self.accumulative_accuracy += metrics.accuracy_score(y_pred=preds, y_true=targets) * num_samples 178 | self.accumulative_loss += loss * num_samples 179 | self.accumulative_num_samples += num_samples 180 | 181 | def clear(self): 182 | self.accumulative_accuracy = 0.0 183 | self.accumulative_loss = 0.0 184 | self.accumulative_num_samples = 0 185 | 186 | def accuracy(self): 187 | accuracy = self.accumulative_accuracy / self.accumulative_num_samples 188 | loss = self.accumulative_loss / self.accumulative_num_samples 189 | return accuracy, loss 190 | 191 | def audit(self, msg): 192 | self.logger.debug(msg) 193 | ``` 194 | 195 | ## 05 模型训练 196 | 197 | 用于训练和评估模型的类，成员对象包括： 198 | 199 | * 模型操作器 200 | * 优化器 201 | * 学习率调整策略（可选） 202 | 203 | 成员方法包括： 204 | 205 | * 训练：训练模型一轮，会调用模型操作器（ModelHandler）的记录器（Recorder）计算训练时的准确率和损失，并在本轮结束时返回训练报告并重置记录器 206 | * 验证：验证模型，没有反向传播过程，并且不计算梯度以节省显存和算力 207 | 208 | ```python 209 | import torch 210 | import tqdm 211 | 212 | 213 | class Trainer: 214 | def __init__(self, handler): 215 | self.handler = handler 216 | self.optimizer = handler.config.optimizer_class(handler.model.parameters(), **handler.config.optimizer_params) 217 | self.scheduler = handler.config.scheduler_class(self.optimizer, **handler.config.scheduler_params) if handler.config.scheduler_class is not None else None 218 | 219 | def train(self, loader): 220 | self.handler.train() 221 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] training", delay=0.2, leave=False, ascii="->"): 222 | preds, loss = self.handler(inputs, targets) 223 | self.handler.recorder.update(preds, targets, loss) 224 | self.optimizer.zero_grad() 225 | loss.backward() 226 | self.optimizer.step() 227 | accuracy, loss = self.handler.recorder.accuracy() 228 | self.handler.recorder.clear() 229 | if self.scheduler is not None: 230 | self.scheduler.step() 231 | report = {"loss": loss, "accuracy": accuracy} 232 | return report 233 | 234 | @torch.no_grad() 235 | def validate(self, loader): 236 | self.handler.eval() 237 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] validating", delay=0.2, leave=False, ascii="->"): 238 | preds, loss = self.handler(inputs, targets) 239 | self.handler.recorder.update(preds, targets, loss) 240 | accuracy, loss = self.handler.recorder.accuracy() 241 | self.handler.recorder.clear() 242 | report = {"loss": loss, "accuracy": accuracy} 243 | return report 244 | ``` 245 | 246 | ## 06 实验主函数 247 | 248 | 配置、对象初始化、训练和评估。 249 | 250 | 数据集的基本与处理是缩放到固定尺寸、张量化以及标准化，训练集在此基础上还有一定的数据增强处理，包括图片裁剪以及随机水平翻转。模型使用自imagenet上训练的预训练权重。批大小为32，共训练8轮，SGD优化器初始学习率为0.001，每4轮下降至原来的十分之一。 251 | 252 | ```python 253 | import torch 254 | import torch.nn as nn 255 | import torch.optim as optim 256 | import torch.utils.data as tdata 257 | import torchvision.transforms as trans 258 | import utils.cv as ucv 259 | 260 | if __name__ == "__main__": 261 | config = ucv.config.ConfigObject() 262 | 263 | config.model_class = ucv.nnmodels.Resnet18Classifier 264 | config.model_params = {"num_classes": 2, "num_channels": 3, "from_pretrained": "imagenet"} 265 | config.device = "cuda:0" 266 | config.criterion_class = nn.CrossEntropyLoss 267 | config.criterion_params = {} 268 | config.log_path = "./logs/dogs_vs_cats.resnet18.log" 269 | 270 | config.dataset = "dogs_vs_cats" 271 | config.seed = 0 272 | config.trn_preprocess = trans.Compose([ 273 | trans.Resize((256, 256)), 274 | trans.RandomCrop((224, 224)), 275 | trans.RandomHorizontalFlip(), 276 | trans.ToTensor(), 277 | trans.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.2225)) 278 | ]) 279 | config.val_preprocess = trans.Compose([ 280 | trans.Resize((256, 256)), 281 | trans.ToTensor(), 282 | trans.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.2225)) 283 | ]) 284 | config.batch_size = 32 285 | config.num_epochs = 8 286 | config.optimizer_class = optim.SGD 287 | config.optimizer_params = {"lr": 0.001, "momentum": 0.9, "nesterov": True} 288 | config.scheduler_class = optim.lr_scheduler.StepLR 289 | config.scheduler_params = {"step_size": 4, "gamma": 0.1} 290 | config.checkpoint_path = "./checkpoints/dogs_vs_cats.resnet18.pt" 291 | 292 | handler = ucv.handler.ModelHandlerCv(config) 293 | handler.log_config() 294 | 295 | trn_set, val_set = ucv.dataset.get_dataset(config) 296 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 297 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 298 | trainer = ucv.trainer.Trainer(handler) 299 | 300 | best_val_accuracy = 0.0 301 | for epoch in range(config.num_epochs): 302 | handler.log(" " + "=" * 40) 303 | trn_report = trainer.train(trn_loader) 304 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 305 | val_report = trainer.validate(val_loader) 306 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 307 | if val_report["accuracy"] > best_val_accuracy: 308 | best_val_accuracy = val_report["accuracy"] 309 | if config.checkpoint_path is not None: 310 | torch.save(handler.model.state_dict, config.checkpoint_path) 311 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 312 | ``` 313 | 314 | ## 07 实验结果 315 | 316 | 实验日志如下，包含了本次实验的配置以及实验结果，最高验证集准确率为98.56%。 317 | 318 | ``` 319 | [+] exp starts from: 2023-04-08 23:34:21 320 | [+] model: Resnet18Classifier 321 | [-] model.num_classes: 2 322 | [-] model.num_channels: 3 323 | [-] model.from_pretrained: imagenet 324 | [+] device: cuda:0 325 | [+] criterion: CrossEntropyLoss 326 | [+] log_path: ./logs/dogs_vs_cats.resnet18.log 327 | [+] dataset: dogs_vs_cats 328 | [+] seed: 0 329 | [+] trn_preprocess: 330 | [-] 00: Resize(size=(256,256),interpolation=bilinear,max_size=None,antialias=None) 331 | [-] 01: RandomCrop(size=(224,224),padding=None) 332 | [-] 02: RandomHorizontalFlip(p=0.5) 333 | [-] 03: ToTensor() 334 | [-] 04: Normalize(mean=(0.485,0.456,0.406),std=(0.229,0.224,0.2225)) 335 | [+] val_preprocess: 336 | [-] 00: Resize(size=(256,256),interpolation=bilinear,max_size=None,antialias=None) 337 | [-] 01: ToTensor() 338 | [-] 02: Normalize(mean=(0.485,0.456,0.406),std=(0.229,0.224,0.2225)) 339 | [+] batch_size: 32 340 | [+] num_epochs: 8 341 | [+] optimizer: SGD 342 | [-] optimizer.lr: 0.001 343 | [-] optimizer.momentum: 0.9 344 | [-] optimizer.nesterov: True 345 | [+] scheduler: StepLR 346 | [-] scheduler.step_size: 4 347 | [-] scheduler.gamma: 0.1 348 | [+] checkpoint_path: ./checkpoints/dogs_vs_cats.resnet18.pt 349 | ======================================== 350 | [001] trn-loss: 0.2173 --- trn-acc: 90.49% 351 | [001] val-loss: 0.1256 --- val-acc: 94.80% 352 | ======================================== 353 | [002] trn-loss: 0.0862 --- trn-acc: 96.55% 354 | [002] val-loss: 0.0705 --- val-acc: 97.36% 355 | ======================================== 356 | [003] trn-loss: 0.0627 --- trn-acc: 97.63% 357 | [003] val-loss: 0.0622 --- val-acc: 97.52% 358 | ======================================== 359 | [004] trn-loss: 0.0517 --- trn-acc: 97.99% 360 | [004] val-loss: 0.0604 --- val-acc: 97.84% 361 | ======================================== 362 | [005] trn-loss: 0.0369 --- trn-acc: 98.67% 363 | [005] val-loss: 0.0514 --- val-acc: 98.08% 364 | ======================================== 365 | [006] trn-loss: 0.0308 --- trn-acc: 98.85% 366 | [006] val-loss: 0.0474 --- val-acc: 98.32% 367 | ======================================== 368 | [007] trn-loss: 0.0301 --- trn-acc: 98.88% 369 | [007] val-loss: 0.0498 --- val-acc: 98.32% 370 | ======================================== 371 | [008] trn-loss: 0.0297 --- trn-acc: 98.92% 372 | [008] val-loss: 0.0419 --- val-acc: 98.56% 373 | [=] best-val-acc: 98.56% 374 | ``` 375 | 376 | -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp2/实验二.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp2/实验二.pdf -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp2/实验二.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp2/实验二.pptx -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp3/实验三.md: -------------------------------------------------------------------------------- 1 | # 实验三 2 | 3 | 202218018670003 马天行 4 | 5 | ## 00 概述 6 | 7 | 实验内容是使用手工LSTM网络实现唐诗生成任务，文件结构等信息可以在前置文档《实验项目结构概述》中看到。 8 | 9 | 完整代码和文档见仓库：https://github.com/Noitolar/CourseDL.git 10 | 11 | ## 01 数据集和预处理 12 | 13 | 课程网站上给出的原始数据是空格填充在前，实际诗句在后，总长度125。个人认为这样的数据不利于训练循环神经网络，所及将空格放在后面，并且设置了最大序列长度（默认为50）保证不会出现文字和空格填充之间没有结束符的情况。同时由于提供的原属数据太大，这里也设置了数据使用量（默认全量使用）来加速训练过程。同时数据集对象还有token的编解码功能。 14 | 15 | 数据集每次取数据（假设序列长度为N）的前N-1个token作为模型输入，后N-1个token作为模型标签。 16 | 17 | ```python 18 | import os 19 | import tqdm 20 | import torch 21 | import torch.utils.data as tdata 22 | import numpy as np 23 | 24 | 25 | class DatasetPoemGenerator(tdata.Dataset): 26 | def __init__(self, sequence_length=50, use_samples=-1): 27 | npz_data = np.load(f"./datasets/poem/tang.npz", allow_pickle=True) 28 | self.vocab = {"encode": npz_data["word2ix"].item(), "decode": npz_data["ix2word"].item()} 29 | if use_samples == -1: 30 | self.sentences = npz_data["data"] 31 | else: 32 | self.sentences = npz_data["data"][:use_samples] 33 | self.sequence_length = sequence_length 34 | self.preprocess() 35 | 36 | def preprocess(self): 37 | new_sentences = [] 38 | for sentence in self.sentences: 39 | new_sentence = [token for token in sentence if token != 8292] 40 | if len(new_sentence) < self.sequence_length: 41 | new_sentence.extend([8292] * (self.sequence_length - len(new_sentence))) 42 | else: 43 | new_sentence = new_sentence[:self.sequence_length] 44 | new_sentences.append(new_sentence) 45 | self.sentences = np.array(new_sentences) 46 | self.sentences = torch.tensor(self.sentences, dtype=torch.long) 47 | 48 | def encode(self, character: str): 49 | return self.vocab["encode"][character] 50 | 51 | def decode(self, token: int): 52 | return self.vocab["decode"][token] 53 | 54 | def __getitem__(self, index): 55 | sentence = self.sentences[index, :-1] 56 | target = self.sentences[index, 1:] 57 | return sentence, target 58 | 59 | def __len__(self): 60 | return len(self.sentences) 61 | ``` 62 | 63 | ## 02 神经网络模型 64 | 65 | 模型为一个多层LSTM结构，具体结构为; 66 | 67 | * 嵌入层：输入维度就是词典长度，输出维度由外部配置 68 | * LSTM层：由外部配置决定层数以及层与层之间的dropout率（在Windows下使用带有dropout的多层LSTM会产生CUDA告警，在StackOverflow上发现这个可能是Torch当前版本存在的问题） 69 | * 分类层：由两层全连接组成，中间加了一个Tahn激活 70 | 71 | ```python 72 | import torch 73 | import torch.nn as nn 74 | import torch.nn.functional as func 75 | 76 | 77 | class LstmGnerator(nn.Module): 78 | def __init__(self, vocab_size, embed_size, lstm_output_size, num_lstm_layers=1, lstm_dropout=0.0): 79 | super().__init__() 80 | self.lstm_output_size = lstm_output_size 81 | self.num_lstm_layers = num_lstm_layers 82 | # 在windows上多层lstm使用dropout或导致driver shutdown告警，应该是torch的问题 83 | self.embedding = nn.Embedding(vocab_size, embed_size) 84 | self.lstm = nn.LSTM(embed_size, lstm_output_size, num_layers=num_lstm_layers, batch_first=True, dropout=lstm_dropout) 85 | self.fc = nn.Sequential( 86 | nn.Linear(lstm_output_size, 2048), 87 | nn.Tanh(), 88 | nn.Linear(2048, vocab_size)) 89 | 90 | def forward(self, inputs, hiddens): 91 | outputs = self.embedding(inputs) 92 | # lstm_outputs.shape: (batch_size, sequence_length, vocab_size) 93 | # lstm_hiddens: (lstm_h0, lstm_c0) 94 | outputs, lstm_hiddens = self.lstm(outputs, hiddens) 95 | outputs = self.fc(outputs) 96 | return outputs, lstm_hiddens 97 | ``` 98 | 99 | ## 03 模型操作 100 | 101 | 这个类用于对模型进行一些高级操作，成员包括： 102 | 103 | * 配置项 104 | * 记录器（用于记录训练过程中的指标以及进行日志管理） 105 | * 设备 106 | * 神经网络模型 107 | * 损失函数 108 | 109 | 类方法包括： 110 | 111 | * 日志记录：调用成员记录器的方法，将指定信息录入日志，并同步显示在终端 112 | * 配置记录：按照固定格式将实验配置记录到日志，并同步显示在终端 113 | * 数据转移：将内存中的张量数据转移到配置的设备的内存/显存中 114 | * 正向调用：根据输入、模型隐藏值（可选）和标签（可选）输出预测值、模型隐藏值和损失值，如果没有输入模型隐藏值则将其初始化为等大小同设备的全零张量，如果没有输入标签则只输出预测值和模型隐藏值 115 | 116 | 在初始化成员对象之前会先设置全局随机数种子，保证实验结果可以复现。 117 | 118 | ```python 119 | import torch 120 | import torch.nn as nn 121 | import torchvision.transforms as trans 122 | import time 123 | import transformers as tfm 124 | import utils.nlp as unlp 125 | 126 | 127 | class ModelHandlerNLP(nn.Module): 128 | def __init__(self, config: unlp.config.ConfigObject): 129 | super().__init__() 130 | self.config = config 131 | tfm.set_seed(config.seed) 132 | self.recorder = unlp.recorder.Recorder(config.log_path) 133 | self.device = config.device 134 | self.model = config.model_class(**config.model_params).to(config.device) 135 | self.criterion = config.criterion_class(**config.criterion_params) 136 | 137 | def log(self, message): 138 | self.recorder.audit(message) 139 | 140 | def log_config(self): 141 | self.log(f"\n\n[+] exp starts from: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}") 142 | for config_key, config_value in self.config.params_dict.items(): 143 | if config_key.endswith("_class"): 144 | self.log(f"[+] {config_key.replace('_class', '')}: {config_value.__name__ if config_value is not None else None}") 145 | elif config_key.endswith("_params") and isinstance(config_value, dict): 146 | for param_key, param_value in config_value.items(): 147 | self.log(f" [-] {config_key.replace('_params', '')}.{param_key}: {param_value}") 148 | elif isinstance(config_value, trans.transforms.Compose): 149 | self.log(f"[+] {config_key}:") 150 | for index, value in enumerate(str(config_value).replace(" ", "").split("\n")[1:-1]): 151 | self.log(f" [-] {index:02d}: {value}") 152 | else: 153 | self.log(f"[+] {config_key}: {config_value}") 154 | 155 | def device_transfer(self, data): 156 | if isinstance(data, torch.Tensor): 157 | data = data.to(self.device) 158 | if isinstance(data, dict): 159 | data = {key: value.to(self.device) for key, value in data.items()} 160 | if isinstance(data, tuple): 161 | data = tuple([child.to(self.device) for child in data]) 162 | if isinstance(data, list): 163 | data = [child.to(self.device) for child in data] 164 | return data 165 | 166 | 167 | class ModelHandlerGenerator(ModelHandlerNLP): 168 | def __init__(self, config: unlp.config.ConfigObject): 169 | super().__init__(config) 170 | 171 | def forward(self, inputs: torch.Tensor, hiddens: tuple = None, targets: torch.Tensor = None): 172 | if hiddens is None: 173 | batch_size = inputs.shape[0] 174 | lstm_h0 = torch.zeros(size=(self.model.num_lstm_layers, batch_size, self.model.lstm_output_size), dtype=torch.float, requires_grad=False) 175 | lstm_c0 = torch.zeros(size=(self.model.num_lstm_layers, batch_size, self.model.lstm_output_size), dtype=torch.float, requires_grad=False) 176 | hiddens = (lstm_h0, lstm_c0) 177 | inputs = self.device_transfer(inputs) 178 | targets = self.device_transfer(targets) 179 | hiddens = self.device_transfer(hiddens) 180 | preds, hiddens = self.model(inputs, hiddens) 181 | 182 | # 相当于把batch内的多个样本拼接起来算损失函数 183 | if targets is not None: 184 | batch_size, sequence_length, vocab_size = preds.shape 185 | preds = preds.reshape(batch_size * sequence_length, vocab_size) 186 | targets = targets.reshape(batch_size * sequence_length) 187 | loss = self.criterion(preds, targets) 188 | self.recorder.update(preds, targets, loss) 189 | else: 190 | loss = None 191 | return preds, loss, hiddens 192 | ``` 193 | 194 | ## 04 记录器 195 | 196 | 这个类用于记录模型训练过程中的一些指标，以及日志管理的相关功能，成员对象包括： 197 | 198 | * 累计准确率 199 | * 累计损失值 200 | * 累计样本数 201 | * 日志记录器 202 | 203 | 成员方法包括： 204 | 205 | * 计算并更新累计准确率、损失值、样本数 206 | * 还原成员变量 207 | * 返回平均准确率和损失值 208 | * 日志录入 209 | 210 | ```python 211 | import numpy as np 212 | import sklearn.metrics as metrics 213 | import logging 214 | import os 215 | 216 | 217 | class Recorder: 218 | def __init__(self, logpath): 219 | self.accumulative_accuracy = 0.0 220 | self.accumulative_loss = 0.0 221 | self.accumulative_num_samples = 0 222 | self.logger = logging.getLogger(__name__) 223 | self.logger.setLevel(logging.DEBUG) 224 | self.logger.addHandler(logging.StreamHandler(stream=None)) 225 | if logpath is not None: 226 | if not os.path.exists(os.path.dirname(logpath)): 227 | os.makedirs(os.path.dirname(logpath)) 228 | logfile = open(logpath, "a", encoding="utf-8") 229 | logfile.close() 230 | self.logger.addHandler(logging.FileHandler(filename=logpath, mode="a")) 231 | 232 | def update(self, preds, targets, loss): 233 | assert len(preds) == len(targets) 234 | num_samples = len(preds) 235 | preds = np.array([pred.argmax() for pred in preds.detach().cpu().numpy()]) 236 | targets = targets.detach().cpu().numpy() 237 | self.accumulative_accuracy += metrics.accuracy_score(y_pred=preds, y_true=targets) * num_samples 238 | self.accumulative_loss += loss * num_samples 239 | self.accumulative_num_samples += num_samples 240 | 241 | def clear(self): 242 | self.accumulative_accuracy = 0.0 243 | self.accumulative_loss = 0.0 244 | self.accumulative_num_samples = 0 245 | 246 | def accuracy(self): 247 | accuracy = self.accumulative_accuracy / self.accumulative_num_samples 248 | loss = self.accumulative_loss / self.accumulative_num_samples 249 | return accuracy, loss 250 | 251 | def audit(self, msg): 252 | self.logger.debug(msg) 253 | ``` 254 | 255 | ## 05 模型训练 256 | 257 | 用于训练和评估模型的类，成员对象包括： 258 | 259 | * 模型操作器 260 | * 优化器 261 | * 学习率调整策略（可选） 262 | 263 | 成员方法包括： 264 | 265 | * 训练：训练模型一轮，会调用模型操作器（ModelHandler）的记录器（Recorder）计算训练时的准确率和损失，并在本轮结束时返回训练报告并重置记录器 266 | * 验证：验证模型，没有反向传播过程，并且不计算梯度以节省显存和算力 267 | * 生成：仅针对NLP生成任务，根据输入的起始token以及最大输出长度输出模型生成序列，如果生成了结束符则提前终止生成 268 | 269 | ```python 270 | import torch 271 | import tqdm 272 | import utils.nlp as unlp 273 | 274 | 275 | class Trainer: 276 | def __init__(self, handler: [unlp.handler.ModelHandlerNLP]): 277 | self.handler = handler 278 | self.config = handler.config 279 | self.optimizer = handler.config.optimizer_class(handler.model.parameters(), **handler.config.optimizer_params) 280 | self.scheduler = handler.config.scheduler_class(self.optimizer, **handler.config.scheduler_params) if handler.config.scheduler_class is not None else None 281 | 282 | def train(self, loader): 283 | self.handler.train() 284 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] training", delay=0.2, leave=False, ascii="->"): 285 | preds, loss = self.handler(inputs, targets) 286 | self.handler.recorder.update(preds, targets, loss) 287 | self.optimizer.zero_grad() 288 | loss.backward() 289 | self.optimizer.step() 290 | accuracy, loss = self.handler.recorder.accuracy() 291 | self.handler.recorder.clear() 292 | if self.scheduler is not None: 293 | self.scheduler.step() 294 | report = {"loss": loss, "accuracy": accuracy} 295 | return report 296 | 297 | @torch.no_grad() 298 | def validate(self, loader): 299 | self.handler.eval() 300 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] validating", delay=0.2, leave=False, ascii="->"): 301 | preds, loss = self.handler(inputs, targets) 302 | self.handler.recorder.update(preds, targets, loss) 303 | accuracy, loss = self.handler.recorder.accuracy() 304 | self.handler.recorder.clear() 305 | report = {"loss": loss, "accuracy": accuracy} 306 | return report 307 | 308 | @torch.no_grad() 309 | def generate(self, input_tokens: list, output_length: int): 310 | self.handler.eval() 311 | start_token = 8291 312 | end_token = 8290 313 | if input_tokens[0] != start_token: 314 | input_tokens.insert(0, start_token) 315 | output_tokens = input_tokens 316 | inputs = torch.tensor(input_tokens).unsqueeze(0) 317 | outputs, _, hiddens = self.handler(inputs=inputs, hiddens=None) 318 | for _ in range(output_length - len(input_tokens)): 319 | preds = outputs[0][-1].argmax(axis=0) 320 | output_tokens.append(int(preds.item())) 321 | if preds.item() == end_token: 322 | break 323 | else: 324 | inputs = preds.reshape(1, 1) 325 | outputs, _, hiddens = self.handler(inputs=inputs, hiddens=hiddens) 326 | return output_tokens 327 | ``` 328 | 329 | ## 06 实验主函数 330 | 331 | 配置、对象初始化、训练和评估。 332 | 333 | 模型的嵌入层输出特征为512维，隐藏值为1024维，使用3层LSTM，层与层之间的dropout率为50%。使用原始数据中的前640条，最大长度为50，批大小为32，共训练20轮，AdamW优化器初始学习率为0.002，L2正则参数设为0.0001，不额外调整学习率。每次通过让模型生成以“风”开头的序列来评估模型质量。 334 | 335 | ```python 336 | import torch 337 | import torch.nn as nn 338 | import torch.optim as optim 339 | import torch.utils.data as tdata 340 | import utils.nlp as unlp 341 | 342 | if __name__ == "__main__": 343 | config = unlp.config.ConfigObject() 344 | 345 | config.model_class = unlp.nnmodels.LstmGnerator 346 | config.model_params = {"vocab_size": 8293, "lstm_input_size": 512, "lstm_output_size": 1024, "num_lstm_layers": 3, "lstm_dropout": 0.5} 347 | config.device = "cuda:0" 348 | config.criterion_class = nn.CrossEntropyLoss 349 | config.criterion_params = {} 350 | config.dataset_class = unlp.dataset.DatasetPoemGenerator 351 | config.dataset_params = {"sequence_length": 50, "use_samples": 640} 352 | config.log_path = f"./logs/poem({config.dataset_params['use_samples']}).lstm.log" 353 | config.seed = 0 354 | config.batch_size = 32 355 | config.num_epochs = 20 356 | config.optimizer_class = optim.AdamW 357 | config.optimizer_params = {"lr": 0.002, "weight_decay": 1e-4} 358 | config.scheduler_class = None 359 | config.checkpoint_path = f"./checkpoints/poem({config.dataset_params['use_samples']}).lstm.pt" 360 | 361 | handler = unlp.handler.ModelHandlerGenerator(config) 362 | handler.log_config() 363 | 364 | dataset = config.dataset_class(**config.dataset_params) 365 | trn_loader = tdata.DataLoader(dataset, batch_size=config.batch_size, shuffle=True) 366 | trainer = unlp.trainer.Trainer(handler) 367 | 368 | best_accuracy = 0.0 369 | best_generation = "" 370 | for index in range(config.num_epochs): 371 | handler.log(" " + "=" * 40) 372 | report = trainer.train(trn_loader, index) 373 | tokens = trainer.generate(input_tokens=[dataset.encode(x) for x in "风"], output_length=50) 374 | generation_sample = "".join(dataset.decode(x) for x in tokens) 375 | handler.log(f" [{index + 1:03d}] {generation_sample}") 376 | if report["accuracy"] > best_accuracy: 377 | best_accuracy = report["accuracy"] 378 | best_generation = generation_sample 379 | if config.checkpoint_path is not None: 380 | torch.save(handler.model.state_dict, config.checkpoint_path) 381 | handler.log(f"[=] best-acc: {best_accuracy:.2%}") 382 | handler.log(f"[=] best-generation: {best_generation}") 383 | ``` 384 | 385 | ## 07 实验结果 386 | 387 | 实验日志如下，包含了本次实验的配置以及实验结果，最高训练准确率时生成内容为：“风阁何馆霜，散宫散幽林。高阁霭新节，高光洒华襟。徒髴趂芳席，终与閒人魂。” 388 | 389 | ``` 390 | [+] exp starts from: 2023-04-08 23:56:52 391 | [+] model: LstmGnerator 392 | [-] model.vocab_size: 8293 393 | [-] model.lstm_input_size: 512 394 | [-] model.lstm_output_size: 1024 395 | [-] model.num_lstm_layers: 3 396 | [-] model.lstm_dropout: 0.5 397 | [+] device: cuda:0 398 | [+] criterion: CrossEntropyLoss 399 | [+] dataset: DatasetPoemGenerator 400 | [-] dataset.sequence_length: 50 401 | [-] dataset.use_samples: 640 402 | [+] log_path: ./logs/poem(640).lstm.log 403 | [+] seed: 0 404 | [+] batch_size: 32 405 | [+] num_epochs: 20 406 | [+] optimizer: AdamW 407 | [-] optimizer.lr: 0.002 408 | [-] optimizer.weight_decay: 0.0001 409 | [+] scheduler: None 410 | [+] checkpoint_path: ./checkpoints/poem(640).lstm.pt 411 | ======================================== 412 | [001] trn-loss: 7.3294 --- trn-acc: 11.80% 413 | [001] 风，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，， 414 | ======================================== 415 | [002] trn-loss: 5.6176 --- trn-acc: 22.90% 416 | [002] 风山 417 | ======================================== 418 | [003] trn-loss: 5.3750 --- trn-acc: 24.41% 419 | [003] 风山不不，山山山山。 420 | ======================================== 421 | [004] trn-loss: 5.2521 --- trn-acc: 24.57% 422 | [004] 风山不山，山人不中。 423 | ======================================== 424 | [005] trn-loss: 5.1551 --- trn-acc: 24.60% 425 | [005] 风人一山，山人一山。 426 | ======================================== 427 | [006] trn-loss: 5.0807 --- trn-acc: 24.71% 428 | [006] 风年不不远，不山一山时。 429 | ======================================== 430 | [007] trn-loss: 5.0085 --- trn-acc: 24.67% 431 | [007] 风年不已郡，晨山复自秋。 432 | ======================================== 433 | [008] trn-loss: 4.9615 --- trn-acc: 24.79% 434 | [008] 风子一已子，春山不不秋。 435 | ======================================== 436 | [009] trn-loss: 4.8971 --- trn-acc: 24.87% 437 | [009] 风山一山人，山山一山门。 438 | ======================================== 439 | [010] trn-loss: 4.8190 --- trn-acc: 24.94% 440 | [010] 风月生云凉，晨是独中行。 441 | ======================================== 442 | [011] trn-loss: 4.7500 --- trn-acc: 25.11% 443 | [011] 风年不已郡，西人独幽时。 444 | ======================================== 445 | [012] trn-loss: 4.6528 --- trn-acc: 25.32% 446 | [012] 风藩不已久，幽林亦未欣。还来无已远，高此独未施。 447 | ======================================== 448 | [013] trn-loss: 4.5453 --- trn-acc: 25.54% 449 | [013] 风年欲山去，山山出幽门。还从方所攀，高来一所疎。 450 | ======================================== 451 | [014] trn-loss: 4.4194 --- trn-acc: 25.86% 452 | [014] 风子滴云兮岐边之，一年青人一炜然。立人不可脱琼去，欲有先人满人藓。 453 | ======================================== 454 | [015] trn-loss: 4.3017 --- trn-acc: 26.41% 455 | [015] 风子寒云度，西山在幽里。始见无芳里，高月亦已同。始见心已永，高是已幽情。 456 | ======================================== 457 | [016] trn-loss: 4.1723 --- trn-acc: 26.85% 458 | [016] 风树春城暮，晨木亦相同。还来何人动，高树夜南川。端居无相见，高人何南眠。 459 | ======================================== 460 | [017] trn-loss: 4.0198 --- trn-acc: 27.71% 461 | [017] 风年郡郡郡，青月已清襟。还见南海散，还是清城曙。还见南海散，还见清城曲。 462 | ======================================== 463 | [018] trn-loss: 3.8670 --- trn-acc: 28.71% 464 | [018] 风阁非京构，晨咏一雾襟。还君飘所职，欲复独纷持。还然须海巅，高书复归前。还怀飘园气，高书复归归。 465 | ======================================== 466 | [019] trn-loss: 3.6855 --- trn-acc: 30.17% 467 | [019] 风阁澄芳燕，西为已伊门。诸门已已永，高里已华曙。还怀故园郡，独见此田时。 468 | ======================================== 469 | [020] trn-loss: 3.5179 --- trn-acc: 31.45% 470 | [020] 风阁何馆霜，散宫散幽林。高阁霭新节，高光洒华襟。徒髴趂芳席，终与閒人魂。 471 | [=] best-acc: 31.45% 472 | [=] best-generation: 风阁何馆霜，散宫散幽林。高阁霭新节，高光洒华襟。徒髴趂芳席，终与閒人魂。 473 | ``` 474 | 475 | -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp3/实验三.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp3/实验三.pdf -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp3/实验三.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp3/实验三.pptx -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp4/实验四.md: -------------------------------------------------------------------------------- 1 | # 实验四 2 | 3 | 202218018670003 马天行 4 | 5 | ## 00 概述 6 | 7 | 实验内容是使用手工TextCNN网络实现电影评论分类任务，文件结构等信息可以在前置文档《实验项目结构概述》中看到。 8 | 9 | 完整代码和文档见仓库：https://github.com/Noitolar/CourseDL.git 10 | 11 | ## 01 数据集和预处理 12 | 13 | 根据数据集内容生成字典，根据课程网站上给出的中文wiki预训练word2vec模型生成可以加载到PyTorch模型中进行微调的权重矩阵。如果出现不在预训练模型中的词，就将其嵌入词向量初始化为随机分布的等长度向量。 14 | 15 | ```python 16 | import os 17 | import tqdm 18 | import torch 19 | import torch.utils.data as tdata 20 | import gensim 21 | import numpy as np 22 | 23 | 24 | class DatasetSentimentClassifier(tdata.Dataset): 25 | def __init__(self, from_file, from_vocab, sequence_length=64): 26 | npz_data = np.load(from_vocab, allow_pickle=True) 27 | self.vocab_encode = npz_data["vocab_encode"].item() 28 | self.sequence_length = sequence_length 29 | self.sentences = [] 30 | self.targets = [] 31 | self.load_data(from_file) 32 | 33 | def load_data(self, from_file): 34 | with open(from_file, "r", encoding="utf-8") as file: 35 | for line in tqdm.tqdm(file.readlines(), desc=f"[+] reading \"{from_file}\"", delay=0.2, leave=False, ascii="->"): 36 | elements = line.strip().split() 37 | if len(elements) < 2: 38 | continue 39 | self.targets.append(int(elements[0])) 40 | sentence = elements[1:] 41 | if len(sentence) > self.sequence_length: 42 | sentence = sentence[:self.sequence_length] 43 | else: 44 | sentence.extend(["_PAD_"] * (self.sequence_length - len(sentence))) 45 | self.sentences.append(sentence) 46 | 47 | def __getitem__(self, index): 48 | sentence = torch.tensor(np.array([self.vocab_encode[word] for word in self.sentences[index]])) 49 | target = torch.tensor(self.targets[index]) 50 | return sentence, target 51 | 52 | def __len__(self): 53 | return len(self.sentences) 54 | 55 | @staticmethod 56 | def build_w2v(from_dir, to_file, from_pretrained_embeddings_model): 57 | w2v_model = gensim.models.KeyedVectors.load_word2vec_format(from_pretrained_embeddings_model, binary=True) 58 | vocab_encode = {"_PAD_": 0} 59 | embed_size = w2v_model.vector_size 60 | embeddings = np.zeros(shape=(1, embed_size)) 61 | # embeddings = np.random.uniform(-1, 1, size=(1, embed_size)) 62 | for file_name in [name for name in os.listdir(from_dir) if name.endswith(".txt")]: 63 | with open(f"{from_dir}/{file_name}", "r", encoding="utf-8") as file: 64 | for line in tqdm.tqdm(file.readlines(), desc=f"[+] reading \"{file_name}\"", delay=0.2, leave=False, ascii="->"): 65 | for word in line.strip().split()[1:]: 66 | if word not in vocab_encode.keys(): 67 | vocab_encode[word] = len(vocab_encode) 68 | try: 69 | embeddings = np.vstack([embeddings, w2v_model[word].reshape(1, embed_size)]) 70 | except KeyError: 71 | embeddings = np.vstack([embeddings, np.random.uniform(-1, 1, size=(1, embed_size))]) 72 | np.savez(to_file, **{"vocab_encode": vocab_encode, "embeddings": embeddings}) 73 | 74 | @staticmethod 75 | def get_embeddings_weight(): 76 | return torch.tensor(np.load("./datasets/movie/vocab.npz", allow_pickle=True)["embeddings"], dtype=torch.float32) 77 | ``` 78 | 79 | ## 02 神经网络模型 80 | 81 | 模型由一至多个并行的卷积网络组成： 82 | 83 | * 嵌入层：加载自Ginsim格式的word2vec模型，从外部配置是否冻结参数 84 | * 卷积层：一至多个不同卷积核大小的卷积层（这里的卷积核大小仅代表第0维度，第1维度固定为嵌入层词向量长度）将输出的结果分别进行ReLU激活以及一维最大池化（池化核大小为经过卷积后的词向量长度）之后输出 85 | * 分类层：将卷积层的一至多个输出拼接在一起后经过dropout进行全连接层做分类 86 | 87 | ```python 88 | import torch 89 | import torch.nn as nn 90 | import torch.nn.functional as func 91 | 92 | 93 | class TextConvClassifier(nn.Module): 94 | def __init__(self, num_classes, dropout_rate, conv_out_channelses, kernel_sizes, pretrained_embeddings, freeze_embeddings=False): 95 | super().__init__() 96 | self.embeddings = nn.Embedding.from_pretrained(pretrained_embeddings, freeze=freeze_embeddings) 97 | self.embed_size = int(pretrained_embeddings.shape[-1]) 98 | self.parallel_conv_layers = nn.ModuleList([nn.Conv2d(1, conv_out_channels, (kernel_size, self.embed_size)) for conv_out_channels, kernel_size in zip(conv_out_channelses, kernel_sizes)]) 99 | # self.bn = nn.BatchNorm2d(conv_out_channels) 100 | self.dropout = nn.Dropout(dropout_rate) 101 | self.flatten = nn.Flatten() 102 | self.fc = nn.Linear(sum(conv_out_channelses), num_classes) 103 | 104 | def forward(self, inputs): 105 | outputs = self.embeddings(inputs).unsqueeze(dim=1) 106 | outputs = [conv_layer(outputs).squeeze(dim=3) for conv_layer in self.parallel_conv_layers] 107 | outputs = [func.relu(output) for output in outputs] 108 | outputs = [func.max_pool1d(output, output.size(dim=2)).squeeze(dim=2) for output in outputs] 109 | outputs = torch.cat(outputs, dim=1) 110 | outputs = self.dropout(outputs) 111 | outputs = self.flatten(outputs) 112 | outputs = self.fc(outputs) 113 | return outputs 114 | ``` 115 | 116 | ## 03 模型操作 117 | 118 | 这个类用于对模型进行一些高级操作，和实验三的模型操作器继承自同一个父类，成员包括： 119 | 120 | * 配置项 121 | * 记录器（用于记录训练过程中的指标以及进行日志管理） 122 | * 设备 123 | * 神经网络模型 124 | * 损失函数 125 | 126 | 类方法包括： 127 | 128 | * 日志记录：调用成员记录器的方法，将指定信息录入日志，并同步显示在终端 129 | * 配置记录：按照固定格式将实验配置记录到日志，并同步显示在终端 130 | * 数据转移：将内存中的张量数据转移到配置的设备的内存/显存中 131 | * 正向调用：根据输入和标签输出预测值和损失值，如果没有输入标签则只输出预测值 132 | 133 | 在初始化成员对象之前会先设置全局随机数种子，保证实验结果可以复现。 134 | 135 | ```python 136 | import torch 137 | import torch.nn as nn 138 | import torchvision.transforms as trans 139 | import time 140 | import transformers as tfm 141 | import utils.nlp as unlp 142 | 143 | 144 | class ModelHandlerNLP(nn.Module): 145 | ... 146 | 147 | 148 | class ModelHandlerClassifier(ModelHandlerNLP): 149 | def __init__(self, config: unlp.config.ConfigObject): 150 | super().__init__(config) 151 | 152 | def forward(self, inputs: torch.Tensor, targets=None): 153 | inputs = self.device_transfer(inputs) 154 | targets = self.device_transfer(targets) 155 | preds = self.model(inputs) 156 | loss = self.criterion(preds, targets) if targets is not None else None 157 | return preds, loss 158 | ``` 159 | 160 | ## 04 记录器 161 | 162 | 第04节到第05节实验三完全一致，如果不想看可以直接跳转到第06节。 163 | 164 | 这个类用于记录模型训练过程中的一些指标，以及日志管理的相关功能，成员对象包括： 165 | 166 | * 累计准确率 167 | * 累计损失值 168 | * 累计样本数 169 | * 日志记录器 170 | 171 | 成员方法包括： 172 | 173 | * 计算并更新累计准确率、损失值、样本数 174 | * 还原成员变量 175 | * 返回平均准确率和损失值 176 | * 日志录入 177 | 178 | ```python 179 | import numpy as np 180 | import sklearn.metrics as metrics 181 | import logging 182 | import os 183 | 184 | 185 | class Recorder: 186 | def __init__(self, logpath): 187 | self.accumulative_accuracy = 0.0 188 | self.accumulative_loss = 0.0 189 | self.accumulative_num_samples = 0 190 | self.logger = logging.getLogger(__name__) 191 | self.logger.setLevel(logging.DEBUG) 192 | self.logger.addHandler(logging.StreamHandler(stream=None)) 193 | if logpath is not None: 194 | if not os.path.exists(os.path.dirname(logpath)): 195 | os.makedirs(os.path.dirname(logpath)) 196 | logfile = open(logpath, "a", encoding="utf-8") 197 | logfile.close() 198 | self.logger.addHandler(logging.FileHandler(filename=logpath, mode="a")) 199 | 200 | def update(self, preds, targets, loss): 201 | assert len(preds) == len(targets) 202 | num_samples = len(preds) 203 | preds = np.array([pred.argmax() for pred in preds.detach().cpu().numpy()]) 204 | targets = targets.detach().cpu().numpy() 205 | self.accumulative_accuracy += metrics.accuracy_score(y_pred=preds, y_true=targets) * num_samples 206 | self.accumulative_loss += loss * num_samples 207 | self.accumulative_num_samples += num_samples 208 | 209 | def clear(self): 210 | self.accumulative_accuracy = 0.0 211 | self.accumulative_loss = 0.0 212 | self.accumulative_num_samples = 0 213 | 214 | def accuracy(self): 215 | accuracy = self.accumulative_accuracy / self.accumulative_num_samples 216 | loss = self.accumulative_loss / self.accumulative_num_samples 217 | return accuracy, loss 218 | 219 | def audit(self, msg): 220 | self.logger.debug(msg) 221 | ``` 222 | 223 | ## 05 模型训练 224 | 225 | 用于训练和评估模型的类，成员对象包括： 226 | 227 | * 模型操作器 228 | * 优化器 229 | * 学习率调整策略（可选） 230 | 231 | 成员方法包括： 232 | 233 | * 训练：训练模型一轮，会调用模型操作器（ModelHandler）的记录器（Recorder）计算训练时的准确率和损失，并在本轮结束时返回训练报告并重置记录器 234 | * 验证：验证模型，没有反向传播过程，并且不计算梯度以节省显存和算力 235 | * 生成：仅针对NLP生成任务，根据输入的起始token以及最大输出长度输出模型生成序列，如果生成了结束符则提前终止生成 236 | 237 | ```python 238 | import torch 239 | import tqdm 240 | import utils.nlp as unlp 241 | 242 | 243 | class Trainer: 244 | def __init__(self, handler: [unlp.handler.ModelHandlerNLP]): 245 | self.handler = handler 246 | self.config = handler.config 247 | self.optimizer = handler.config.optimizer_class(handler.model.parameters(), **handler.config.optimizer_params) 248 | self.scheduler = handler.config.scheduler_class(self.optimizer, **handler.config.scheduler_params) if handler.config.scheduler_class is not None else None 249 | 250 | def train(self, loader): 251 | self.handler.train() 252 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] training", delay=0.2, leave=False, ascii="->"): 253 | preds, loss = self.handler(inputs, targets) 254 | self.handler.recorder.update(preds, targets, loss) 255 | self.optimizer.zero_grad() 256 | loss.backward() 257 | self.optimizer.step() 258 | accuracy, loss = self.handler.recorder.accuracy() 259 | self.handler.recorder.clear() 260 | if self.scheduler is not None: 261 | self.scheduler.step() 262 | report = {"loss": loss, "accuracy": accuracy} 263 | return report 264 | 265 | @torch.no_grad() 266 | def validate(self, loader): 267 | self.handler.eval() 268 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] validating", delay=0.2, leave=False, ascii="->"): 269 | preds, loss = self.handler(inputs, targets) 270 | self.handler.recorder.update(preds, targets, loss) 271 | accuracy, loss = self.handler.recorder.accuracy() 272 | self.handler.recorder.clear() 273 | report = {"loss": loss, "accuracy": accuracy} 274 | return report 275 | 276 | @torch.no_grad() 277 | def generate(self, input_tokens: list, output_length: int): 278 | self.handler.eval() 279 | start_token = 8291 280 | end_token = 8290 281 | if input_tokens[0] != start_token: 282 | input_tokens.insert(0, start_token) 283 | output_tokens = input_tokens 284 | inputs = torch.tensor(input_tokens).unsqueeze(0) 285 | outputs, _, hiddens = self.handler(inputs=inputs, hiddens=None) 286 | for _ in range(output_length - len(input_tokens)): 287 | preds = outputs[0][-1].argmax(axis=0) 288 | output_tokens.append(int(preds.item())) 289 | if preds.item() == end_token: 290 | break 291 | else: 292 | inputs = preds.reshape(1, 1) 293 | outputs, _, hiddens = self.handler(inputs=inputs, hiddens=hiddens) 294 | return output_tokens 295 | ``` 296 | 297 | ## 06 实验主函数 298 | 299 | 配置、对象初始化、训练和评估。进行实验之前首先生成Embeddings权重矩阵保存至本地。 300 | 301 | 模型使用四种尺寸的卷积核：2、4、6、8，其对应的卷积和数量为：32、32、24、16，dropout率为0.1，允许更新来自word2vec的嵌入层模型。使用全量原始数据，最大长度为80，批大小为32，共训练4轮，AdamW优化器初始学习率为0.001，L2正则参数设为0.0001，在第二轮后学习率调整为原来的十分之一。 302 | 303 | ```python 304 | import torch 305 | import torch.nn as nn 306 | import torch.optim as optim 307 | import torch.utils.data as tdata 308 | import utils.nlp as unlp 309 | 310 | if __name__ == "__main__": 311 | config = unlp.config.ConfigObject() 312 | 313 | config.model_class = unlp.nnmodels.TextConvClassifier 314 | config.model_params = {"num_classes": 2, "dropout_rate": 0.1, "kernel_sizes": [2, 4, 6, 8], "conv_out_channelses": [32, 32, 24, 16], 315 | "freeze_embeddings": False, "pretrained_embeddings": unlp.dataset.DatasetSentimentClassifier.get_embeddings_weight()} 316 | config.device = "cuda:0" 317 | config.criterion_class = nn.CrossEntropyLoss 318 | config.criterion_params = {} 319 | config.log_path = "./logs/movie.conv.log" 320 | 321 | config.dataset = "movie" 322 | config.seed = 0 323 | config.sequence_length = 80 324 | config.batch_size = 32 325 | config.num_epochs = 4 326 | config.optimizer_class = optim.AdamW 327 | config.optimizer_params = {"lr": 0.001, "weight_decay": 1e-4} 328 | config.scheduler_class = optim.lr_scheduler.StepLR 329 | config.scheduler_params = {"step_size": 2, "gamma": 0.1} 330 | config.checkpoint_path = "./checkpoints/movie.conv.pt" 331 | 332 | handler = unlp.handler.ModelHandlerClassifier(config) 333 | handler.log_config() 334 | 335 | # unlp.dataset.DatasetSentimentClassifier.build_w2v(from_dir="./datasets/movie", to_file="./datasets/movie/vocab.npz", from_pretrained_embeddings_model="./datasets/movie/wiki_word2vec_50.bin") 336 | trn_set = unlp.dataset.DatasetSentimentClassifier(from_file="./datasets/movie/train.txt", from_vocab="./datasets/movie/vocab.npz", sequence_length=config.sequence_length) 337 | val_set = unlp.dataset.DatasetSentimentClassifier(from_file="./datasets/movie/validation.txt", from_vocab="./datasets/movie/vocab.npz", sequence_length=config.sequence_length) 338 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 339 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 340 | trainer = unlp.trainer.Trainer(handler) 341 | 342 | best_val_accuracy = 0.0 343 | for epoch in range(config.num_epochs): 344 | handler.log(" " + "=" * 40) 345 | trn_report = trainer.train(trn_loader) 346 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 347 | val_report = trainer.validate(val_loader) 348 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 349 | if val_report["accuracy"] > best_val_accuracy: 350 | best_val_accuracy = val_report["accuracy"] 351 | if config.checkpoint_path is not None: 352 | torch.save(handler.model.state_dict, config.checkpoint_path) 353 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 354 | ``` 355 | 356 | ## 07 实验结果 357 | 358 | 实验日志如下，包含了本次实验的配置以及实验结果，最高验证集准确率为85.10%。 359 | 360 | ``` 361 | [+] exp starts from: 2023-04-16 20:22:06 362 | [+] model: TextConvClassifier 363 | [-] model.num_classes: 2 364 | [-] model.dropout_rate: 0.1 365 | [-] model.kernel_sizes: [2, 4, 6, 8] 366 | [-] model.conv_out_channelses: [32, 32, 24, 16] 367 | [-] model.freeze_embeddings: False 368 | [-] model.pretrained_embeddings: tensor([ 369 | [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000], 370 | [ 0.1642, -0.1085, 0.1950, ..., -0.4986, 0.2594, -0.2626], 371 | [ 0.3688, -0.1484, 0.0068, ..., -0.5649, 0.3491, -0.5529], 372 | ..., 373 | [-0.3737, -0.5502, -0.2598, ..., 0.4632, 0.4255, 0.0466], 374 | [ 0.2875, 0.2226, 0.6066, ..., -0.1017, 0.1671, 0.1913], 375 | [ 0.5021, -0.4564, 0.2155, ..., -0.3437, 0.1961, -0.0354]]) 376 | [+] device: cuda:0 377 | [+] criterion: CrossEntropyLoss 378 | [+] log_path: ./logs/movie.conv.log 379 | [+] dataset: movie 380 | [+] seed: 0 381 | [+] sequence_length: 80 382 | [+] batch_size: 32 383 | [+] num_epochs: 4 384 | [+] optimizer: AdamW 385 | [-] optimizer.lr: 0.001 386 | [-] optimizer.weight_decay: 0.0001 387 | [+] scheduler: StepLR 388 | [-] scheduler.step_size: 2 389 | [-] scheduler.gamma: 0.1 390 | [+] checkpoint_path: ./checkpoints/movie.conv.pt 391 | ======================================== 392 | [001] trn-loss: 0.4782 --- trn-acc: 76.64% 393 | [001] val-loss: 0.4220 --- val-acc: 80.78% 394 | ======================================== 395 | [002] trn-loss: 0.2956 --- trn-acc: 87.64% 396 | [002] val-loss: 0.3614 --- val-acc: 84.79% 397 | ======================================== 398 | [003] trn-loss: 0.1672 --- trn-acc: 94.45% 399 | [003] val-loss: 0.3645 --- val-acc: 85.10% 400 | ======================================== 401 | [004] trn-loss: 0.1507 --- trn-acc: 94.93% 402 | [004] val-loss: 0.3716 --- val-acc: 85.01% 403 | [=] best-val-acc: 85.10% 404 | ``` 405 | 406 | -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp4/实验四.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp4/实验四.pdf -------------------------------------------------------------------------------- /docs/马天行.深度学习.Exp4/实验四.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Noitolar/CourseDL/79d8fb8b30c4d9d3de269a9c735983b61a0bba98/docs/马天行.深度学习.Exp4/实验四.pptx -------------------------------------------------------------------------------- /logs/dogs_vs_cats.resnet18.log: -------------------------------------------------------------------------------- 1 | 2 | 3 | [+] exp starts from: 2023-04-08 23:34:21 4 | [+] model: Resnet18Classifier 5 | [-] model.num_classes: 2 6 | [-] model.num_channels: 3 7 | [-] model.from_pretrained: imagenet 8 | [+] device: cuda:0 9 | [+] criterion: CrossEntropyLoss 10 | [+] log_path: ./logs/dogs_vs_cats.resnet18.log 11 | [+] dataset: dogs_vs_cats 12 | [+] seed: 0 13 | [+] trn_preprocess: 14 | [-] 00: Resize(size=(256,256),interpolation=bilinear,max_size=None,antialias=None) 15 | [-] 01: RandomCrop(size=(224,224),padding=None) 16 | [-] 02: RandomHorizontalFlip(p=0.5) 17 | [-] 03: ToTensor() 18 | [-] 04: Normalize(mean=(0.485,0.456,0.406),std=(0.229,0.224,0.2225)) 19 | [+] val_preprocess: 20 | [-] 00: Resize(size=(256,256),interpolation=bilinear,max_size=None,antialias=None) 21 | [-] 01: ToTensor() 22 | [-] 02: Normalize(mean=(0.485,0.456,0.406),std=(0.229,0.224,0.2225)) 23 | [+] batch_size: 32 24 | [+] num_epochs: 8 25 | [+] optimizer: SGD 26 | [-] optimizer.lr: 0.001 27 | [-] optimizer.momentum: 0.9 28 | [-] optimizer.nesterov: True 29 | [+] scheduler: StepLR 30 | [-] scheduler.step_size: 4 31 | [-] scheduler.gamma: 0.1 32 | [+] checkpoint_path: ./checkpoints/dogs_vs_cats.resnet18.pt 33 | ======================================== 34 | [001] trn-loss: 0.2173 --- trn-acc: 90.49% 35 | [001] val-loss: 0.1256 --- val-acc: 94.80% 36 | ======================================== 37 | [002] trn-loss: 0.0862 --- trn-acc: 96.55% 38 | [002] val-loss: 0.0705 --- val-acc: 97.36% 39 | ======================================== 40 | [003] trn-loss: 0.0627 --- trn-acc: 97.63% 41 | [003] val-loss: 0.0622 --- val-acc: 97.52% 42 | ======================================== 43 | [004] trn-loss: 0.0517 --- trn-acc: 97.99% 44 | [004] val-loss: 0.0604 --- val-acc: 97.84% 45 | ======================================== 46 | [005] trn-loss: 0.0369 --- trn-acc: 98.67% 47 | [005] val-loss: 0.0514 --- val-acc: 98.08% 48 | ======================================== 49 | [006] trn-loss: 0.0308 --- trn-acc: 98.85% 50 | [006] val-loss: 0.0474 --- val-acc: 98.32% 51 | ======================================== 52 | [007] trn-loss: 0.0301 --- trn-acc: 98.88% 53 | [007] val-loss: 0.0498 --- val-acc: 98.32% 54 | ======================================== 55 | [008] trn-loss: 0.0297 --- trn-acc: 98.92% 56 | [008] val-loss: 0.0419 --- val-acc: 98.56% 57 | [=] best-val-acc: 98.56% 58 | -------------------------------------------------------------------------------- /logs/mnist.conv.log: -------------------------------------------------------------------------------- 1 | 2 | 3 | [+] exp starts from: 2023-04-08 23:32:41 4 | [+] model: SimpleConvClassifier 5 | [-] model.num_classes: 10 6 | [-] model.num_channels: 1 7 | [+] device: cuda:0 8 | [+] criterion: CrossEntropyLoss 9 | [+] log_path: ./logs/mnist.conv.log 10 | [+] dataset: mnist 11 | [+] seed: 0 12 | [+] trn_preprocess: 13 | [-] 00: ToTensor() 14 | [-] 01: Normalize(mean=[0.485],std=[0.229]) 15 | [+] val_preprocess: 16 | [-] 00: ToTensor() 17 | [-] 01: Normalize(mean=[0.485],std=[0.229]) 18 | [+] batch_size: 32 19 | [+] num_epochs: 8 20 | [+] optimizer: SGD 21 | [-] optimizer.lr: 0.001 22 | [-] optimizer.momentum: 0.9 23 | [-] optimizer.nesterov: True 24 | [+] scheduler: MultiStepLR 25 | [-] scheduler.milestones: [4, 6] 26 | [-] scheduler.gamma: 0.1 27 | [+] checkpoint_path: ./checkpoints/mnist.conv.pt 28 | ======================================== 29 | [001] trn-loss: 0.1085 --- trn-acc: 96.72% 30 | [001] val-loss: 0.0428 --- val-acc: 98.63% 31 | ======================================== 32 | [002] trn-loss: 0.0394 --- trn-acc: 98.84% 33 | [002] val-loss: 0.0339 --- val-acc: 98.94% 34 | ======================================== 35 | [003] trn-loss: 0.0271 --- trn-acc: 99.21% 36 | [003] val-loss: 0.0311 --- val-acc: 98.94% 37 | ======================================== 38 | [004] trn-loss: 0.0196 --- trn-acc: 99.45% 39 | [004] val-loss: 0.0317 --- val-acc: 99.02% 40 | ======================================== 41 | [005] trn-loss: 0.0118 --- trn-acc: 99.77% 42 | [005] val-loss: 0.0266 --- val-acc: 99.12% 43 | ======================================== 44 | [006] trn-loss: 0.0103 --- trn-acc: 99.80% 45 | [006] val-loss: 0.0261 --- val-acc: 99.17% 46 | ======================================== 47 | [007] trn-loss: 0.0095 --- trn-acc: 99.83% 48 | [007] val-loss: 0.0262 --- val-acc: 99.18% 49 | ======================================== 50 | [008] trn-loss: 0.0096 --- trn-acc: 99.84% 51 | [008] val-loss: 0.0260 --- val-acc: 99.17% 52 | [=] best-val-acc: 99.18% 53 | -------------------------------------------------------------------------------- /logs/movie.conv.log: -------------------------------------------------------------------------------- 1 | 2 | 3 | [+] exp starts from: 2023-04-16 20:22:06 4 | [+] model: TextConvClassifier 5 | [-] model.num_classes: 2 6 | [-] model.dropout_rate: 0.1 7 | [-] model.kernel_sizes: [2, 4, 6, 8] 8 | [-] model.conv_out_channelses: [32, 32, 24, 16] 9 | [-] model.freeze_embeddings: False 10 | [-] model.pretrained_embeddings: tensor([ 11 | [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000], 12 | [ 0.1642, -0.1085, 0.1950, ..., -0.4986, 0.2594, -0.2626], 13 | [ 0.3688, -0.1484, 0.0068, ..., -0.5649, 0.3491, -0.5529], 14 | ..., 15 | [-0.3737, -0.5502, -0.2598, ..., 0.4632, 0.4255, 0.0466], 16 | [ 0.2875, 0.2226, 0.6066, ..., -0.1017, 0.1671, 0.1913], 17 | [ 0.5021, -0.4564, 0.2155, ..., -0.3437, 0.1961, -0.0354]]) 18 | [+] device: cuda:0 19 | [+] criterion: CrossEntropyLoss 20 | [+] log_path: ./logs/movie.conv.log 21 | [+] dataset: movie 22 | [+] seed: 0 23 | [+] sequence_length: 80 24 | [+] batch_size: 32 25 | [+] num_epochs: 4 26 | [+] optimizer: AdamW 27 | [-] optimizer.lr: 0.001 28 | [-] optimizer.weight_decay: 0.0001 29 | [+] scheduler: StepLR 30 | [-] scheduler.step_size: 2 31 | [-] scheduler.gamma: 0.1 32 | [+] checkpoint_path: ./checkpoints/movie.conv.pt 33 | ======================================== 34 | [001] trn-loss: 0.4782 --- trn-acc: 76.64% 35 | [001] val-loss: 0.4220 --- val-acc: 80.78% 36 | ======================================== 37 | [002] trn-loss: 0.2956 --- trn-acc: 87.64% 38 | [002] val-loss: 0.3614 --- val-acc: 84.79% 39 | ======================================== 40 | [003] trn-loss: 0.1672 --- trn-acc: 94.45% 41 | [003] val-loss: 0.3645 --- val-acc: 85.10% 42 | ======================================== 43 | [004] trn-loss: 0.1507 --- trn-acc: 94.93% 44 | [004] val-loss: 0.3716 --- val-acc: 85.01% 45 | [=] best-val-acc: 85.10% 46 | -------------------------------------------------------------------------------- /logs/poem(640).lstm.log: -------------------------------------------------------------------------------- 1 | 2 | 3 | [+] exp starts from: 2023-04-08 23:56:52 4 | [+] model: LstmGnerator 5 | [-] model.vocab_size: 8293 6 | [-] model.lstm_input_size: 512 7 | [-] model.lstm_output_size: 1024 8 | [-] model.num_lstm_layers: 3 9 | [-] model.lstm_dropout: 0.5 10 | [+] device: cuda:0 11 | [+] criterion: CrossEntropyLoss 12 | [+] dataset: DatasetPoemGenerator 13 | [-] dataset.sequence_length: 50 14 | [-] dataset.use_samples: 640 15 | [+] log_path: ./logs/poem(640).lstm.log 16 | [+] seed: 0 17 | [+] batch_size: 32 18 | [+] num_epochs: 20 19 | [+] optimizer: AdamW 20 | [-] optimizer.lr: 0.002 21 | [-] optimizer.weight_decay: 0.0001 22 | [+] scheduler: None 23 | [+] checkpoint_path: ./checkpoints/poem(640).lstm.pt 24 | ======================================== 25 | [001] trn-loss: 7.3294 --- trn-acc: 11.80% 26 | [001] 风，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，，， 27 | ======================================== 28 | [002] trn-loss: 5.6176 --- trn-acc: 22.90% 29 | [002] 风山 30 | ======================================== 31 | [003] trn-loss: 5.3750 --- trn-acc: 24.41% 32 | [003] 风山不不，山山山山。 33 | ======================================== 34 | [004] trn-loss: 5.2521 --- trn-acc: 24.57% 35 | [004] 风山不山，山人不中。 36 | ======================================== 37 | [005] trn-loss: 5.1551 --- trn-acc: 24.60% 38 | [005] 风人一山，山人一山。 39 | ======================================== 40 | [006] trn-loss: 5.0807 --- trn-acc: 24.71% 41 | [006] 风年不不远，不山一山时。 42 | ======================================== 43 | [007] trn-loss: 5.0085 --- trn-acc: 24.67% 44 | [007] 风年不已郡，晨山复自秋。 45 | ======================================== 46 | [008] trn-loss: 4.9615 --- trn-acc: 24.79% 47 | [008] 风子一已子，春山不不秋。 48 | ======================================== 49 | [009] trn-loss: 4.8971 --- trn-acc: 24.87% 50 | [009] 风山一山人，山山一山门。 51 | ======================================== 52 | [010] trn-loss: 4.8190 --- trn-acc: 24.94% 53 | [010] 风月生云凉，晨是独中行。 54 | ======================================== 55 | [011] trn-loss: 4.7500 --- trn-acc: 25.11% 56 | [011] 风年不已郡，西人独幽时。 57 | ======================================== 58 | [012] trn-loss: 4.6528 --- trn-acc: 25.32% 59 | [012] 风藩不已久，幽林亦未欣。还来无已远，高此独未施。 60 | ======================================== 61 | [013] trn-loss: 4.5453 --- trn-acc: 25.54% 62 | [013] 风年欲山去，山山出幽门。还从方所攀，高来一所疎。 63 | ======================================== 64 | [014] trn-loss: 4.4194 --- trn-acc: 25.86% 65 | [014] 风子滴云兮岐边之，一年青人一炜然。立人不可脱琼去，欲有先人满人藓。 66 | ======================================== 67 | [015] trn-loss: 4.3017 --- trn-acc: 26.41% 68 | [015] 风子寒云度，西山在幽里。始见无芳里，高月亦已同。始见心已永，高是已幽情。 69 | ======================================== 70 | [016] trn-loss: 4.1723 --- trn-acc: 26.85% 71 | [016] 风树春城暮，晨木亦相同。还来何人动，高树夜南川。端居无相见，高人何南眠。 72 | ======================================== 73 | [017] trn-loss: 4.0198 --- trn-acc: 27.71% 74 | [017] 风年郡郡郡，青月已清襟。还见南海散，还是清城曙。还见南海散，还见清城曲。 75 | ======================================== 76 | [018] trn-loss: 3.8670 --- trn-acc: 28.71% 77 | [018] 风阁非京构，晨咏一雾襟。还君飘所职，欲复独纷持。还然须海巅，高书复归前。还怀飘园气，高书复归归。 78 | ======================================== 79 | [019] trn-loss: 3.6855 --- trn-acc: 30.17% 80 | [019] 风阁澄芳燕，西为已伊门。诸门已已永，高里已华曙。还怀故园郡，独见此田时。 81 | ======================================== 82 | [020] trn-loss: 3.5179 --- trn-acc: 31.45% 83 | [020] 风阁何馆霜，散宫散幽林。高阁霭新节，高光洒华襟。徒髴趂芳席，终与閒人魂。 84 | [=] best-acc: 31.45% 85 | [=] best-poem: 风阁何馆霜，散宫散幽林。高阁霭新节，高光洒华襟。徒髴趂芳席，终与閒人魂。 86 | -------------------------------------------------------------------------------- /task_01.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.utils.data as tdata 5 | import torchvision.transforms as trans 6 | import utils.cv as ucv 7 | 8 | if __name__ == "__main__": 9 | config = ucv.config.ConfigObject() 10 | 11 | config.model_class = ucv.nnmodels.SimpleConvClassifier 12 | config.model_params = {"num_classes": 10, "num_channels": 1} 13 | config.device = "cuda:0" 14 | config.criterion_class = nn.CrossEntropyLoss 15 | config.criterion_params = {} 16 | config.log_path = "./logs/mnist.conv.log" 17 | 18 | config.dataset = "mnist" 19 | config.seed = 0 20 | config.trn_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 21 | config.val_preprocess = config.trn_preprocess 22 | config.batch_size = 32 23 | config.num_epochs = 8 24 | config.optimizer_class = optim.SGD 25 | config.optimizer_params = {"lr": 0.001, "momentum": 0.9, "nesterov": True} 26 | config.scheduler_class = optim.lr_scheduler.MultiStepLR 27 | config.scheduler_params = {"milestones": [4, 6], "gamma": 0.1} 28 | config.checkpoint_path = "./checkpoints/mnist.conv.pt" 29 | 30 | handler = ucv.handler.ModelHandlerVanilla(config) 31 | handler.log_config() 32 | 33 | trn_set, val_set = ucv.dataset.get_dataset(config) 34 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 35 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 36 | trainer = ucv.trainer.Trainer(handler) 37 | 38 | best_val_accuracy = 0.0 39 | for epoch in range(config.num_epochs): 40 | handler.log(" " + "=" * 40) 41 | trn_report = trainer.train(trn_loader) 42 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 43 | val_report = trainer.validate(val_loader) 44 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 45 | if val_report["accuracy"] > best_val_accuracy: 46 | best_val_accuracy = val_report["accuracy"] 47 | if config.checkpoint_path is not None: 48 | torch.save(handler.model.state_dict, config.checkpoint_path) 49 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 50 | -------------------------------------------------------------------------------- /task_02.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.utils.data as tdata 5 | import torchvision.transforms as trans 6 | import utils.cv as ucv 7 | 8 | if __name__ == "__main__": 9 | config = ucv.config.ConfigObject() 10 | 11 | config.model_class = ucv.nnmodels.Resnet18Classifier 12 | config.model_params = {"num_classes": 2, "num_channels": 3, "from_pretrained": "imagenet"} 13 | config.device = "cuda:0" 14 | config.criterion_class = nn.CrossEntropyLoss 15 | config.criterion_params = {} 16 | config.log_path = "./logs/dogs_vs_cats.resnet18.log" 17 | 18 | config.dataset = "dogs_vs_cats" 19 | config.seed = 0 20 | config.trn_preprocess = trans.Compose([ 21 | trans.Resize((256, 256)), 22 | trans.RandomCrop((224, 224)), 23 | trans.RandomHorizontalFlip(), 24 | trans.ToTensor(), 25 | trans.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.2225)) 26 | ]) 27 | config.val_preprocess = trans.Compose([ 28 | trans.Resize((256, 256)), 29 | trans.ToTensor(), 30 | trans.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.2225)) 31 | ]) 32 | config.batch_size = 32 33 | config.num_epochs = 8 34 | config.optimizer_class = optim.SGD 35 | config.optimizer_params = {"lr": 0.001, "momentum": 0.9, "nesterov": True} 36 | config.scheduler_class = optim.lr_scheduler.StepLR 37 | config.scheduler_params = {"step_size": 4, "gamma": 0.1} 38 | config.checkpoint_path = "./checkpoints/dogs_vs_cats.resnet18.pt" 39 | 40 | handler = ucv.handler.ModelHandlerVanilla(config) 41 | handler.log_config() 42 | 43 | trn_set, val_set = ucv.dataset.get_dataset(config) 44 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 45 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 46 | trainer = ucv.trainer.Trainer(handler) 47 | 48 | best_val_accuracy = 0.0 49 | for epoch in range(config.num_epochs): 50 | handler.log(" " + "=" * 40) 51 | trn_report = trainer.train(trn_loader) 52 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 53 | val_report = trainer.validate(val_loader) 54 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 55 | if val_report["accuracy"] > best_val_accuracy: 56 | best_val_accuracy = val_report["accuracy"] 57 | if config.checkpoint_path is not None: 58 | torch.save(handler.model.state_dict, config.checkpoint_path) 59 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 60 | -------------------------------------------------------------------------------- /task_03.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.utils.data as tdata 5 | import utils.nlp as unlp 6 | 7 | if __name__ == "__main__": 8 | config = unlp.config.ConfigObject() 9 | 10 | config.model_class = unlp.nnmodels.LstmGnerator 11 | config.model_params = {"vocab_size": 8293, "lstm_input_size": 512, "lstm_output_size": 1024, "num_lstm_layers": 3, "lstm_dropout": 0.5} 12 | config.device = "cuda:0" 13 | config.criterion_class = nn.CrossEntropyLoss 14 | config.criterion_params = {} 15 | config.dataset_class = unlp.dataset.DatasetPoemGenerator 16 | config.dataset_params = {"sequence_length": 50, "use_samples": 640} 17 | config.log_path = f"./logs/poem({config.dataset_params['use_samples']}).lstm.log" 18 | config.seed = 0 19 | config.batch_size = 32 20 | config.num_epochs = 20 21 | config.optimizer_class = optim.AdamW 22 | config.optimizer_params = {"lr": 0.002, "weight_decay": 1e-4} 23 | config.scheduler_class = None 24 | config.checkpoint_path = f"./checkpoints/poem({config.dataset_params['use_samples']}).lstm.pt" 25 | 26 | handler = unlp.handler.ModelHandlerGenerator(config) 27 | handler.log_config() 28 | 29 | dataset = config.dataset_class(**config.dataset_params) 30 | trn_loader = tdata.DataLoader(dataset, batch_size=config.batch_size, shuffle=True) 31 | trainer = unlp.trainer.Trainer(handler) 32 | 33 | best_accuracy = 0.0 34 | best_generation = "" 35 | for index in range(config.num_epochs): 36 | handler.log(" " + "=" * 40) 37 | report = trainer.train(trn_loader, index) 38 | tokens = trainer.generate(input_tokens=[dataset.encode(x) for x in "风"], output_length=50) 39 | generation_sample = "".join(dataset.decode(x) for x in tokens) 40 | handler.log(f" [{index + 1:03d}] {generation_sample}") 41 | if report["accuracy"] > best_accuracy: 42 | best_accuracy = report["accuracy"] 43 | best_generation = generation_sample 44 | if config.checkpoint_path is not None: 45 | torch.save(handler.model.state_dict, config.checkpoint_path) 46 | handler.log(f"[=] best-acc: {best_accuracy:.2%}") 47 | handler.log(f"[=] best-generation: {best_generation}") 48 | -------------------------------------------------------------------------------- /task_04.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.utils.data as tdata 5 | import utils.nlp as unlp 6 | 7 | if __name__ == "__main__": 8 | config = unlp.config.ConfigObject() 9 | 10 | config.model_class = unlp.nnmodels.TextConvClassifier 11 | config.model_params = {"num_classes": 2, "dropout_rate": 0.1, "kernel_sizes": [2, 4, 6, 8], "conv_out_channelses": [32, 32, 24, 16], 12 | "freeze_embeddings": False, "pretrained_embeddings": unlp.dataset.DatasetSentimentClassifier.get_embeddings_weight()} 13 | config.device = "cuda:0" 14 | config.criterion_class = nn.CrossEntropyLoss 15 | config.criterion_params = {} 16 | config.log_path = "./logs/movie.conv.log" 17 | 18 | config.dataset = "movie" 19 | config.seed = 0 20 | config.sequence_length = 80 21 | config.batch_size = 32 22 | config.num_epochs = 4 23 | config.optimizer_class = optim.AdamW 24 | config.optimizer_params = {"lr": 0.001, "weight_decay": 1e-4} 25 | config.scheduler_class = optim.lr_scheduler.StepLR 26 | config.scheduler_params = {"step_size": 2, "gamma": 0.1} 27 | config.checkpoint_path = "./checkpoints/movie.conv.pt" 28 | 29 | handler = unlp.handler.ModelHandlerClassifier(config) 30 | handler.log_config() 31 | 32 | # unlp.dataset.DatasetSentimentClassifier.build_w2v(from_dir="./datasets/movie", to_file="./datasets/movie/vocab.npz", from_pretrained_embeddings_model="./datasets/movie/wiki_word2vec_50.bin") 33 | trn_set = unlp.dataset.DatasetSentimentClassifier(from_file="./datasets/movie/train.txt", from_vocab="./datasets/movie/vocab.npz", sequence_length=config.sequence_length) 34 | val_set = unlp.dataset.DatasetSentimentClassifier(from_file="./datasets/movie/validation.txt", from_vocab="./datasets/movie/vocab.npz", sequence_length=config.sequence_length) 35 | trn_loader = tdata.DataLoader(trn_set, batch_size=config.batch_size, shuffle=True) 36 | val_loader = tdata.DataLoader(val_set, batch_size=config.batch_size * 8) 37 | trainer = unlp.trainer.Trainer(handler) 38 | 39 | best_val_accuracy = 0.0 40 | for epoch in range(config.num_epochs): 41 | handler.log(" " + "=" * 40) 42 | trn_report = trainer.train(trn_loader) 43 | handler.log(f" [{epoch + 1:03d}] trn-loss: {trn_report['loss']:.4f} --- trn-acc: {trn_report['accuracy']:.2%}") 44 | val_report = trainer.validate(val_loader) 45 | handler.log(f" [{epoch + 1:03d}] val-loss: {val_report['loss']:.4f} --- val-acc: {val_report['accuracy']:.2%}") 46 | if val_report["accuracy"] > best_val_accuracy: 47 | best_val_accuracy = val_report["accuracy"] 48 | if config.checkpoint_path is not None: 49 | torch.save(handler.model.state_dict, config.checkpoint_path) 50 | handler.log(f"[=] best-val-acc: {best_val_accuracy:.2%}") 51 | -------------------------------------------------------------------------------- /task_05.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.utils.data as tdata 5 | import torchvision.transforms as trans 6 | import utils.cv as ucv 7 | 8 | if __name__ == "__main__": 9 | config = ucv.config.ConfigObject() 10 | 11 | -------------------------------------------------------------------------------- /task_plot.py: -------------------------------------------------------------------------------- 1 | import re 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import scipy.interpolate as interpolate 5 | 6 | 7 | def plot(from_log, num_interpolations=0): 8 | trn_loss = [] 9 | trn_acc = [] 10 | val_loss = [] 11 | val_acc = [] 12 | with open(from_log, "r", encoding="utf-8") as log: 13 | pattern = re.compile(r"\d+[.\d]*") 14 | for line in log.readlines(): 15 | if "---" not in line: 16 | continue 17 | datas = list(map(float, re.findall(pattern, line))) 18 | if "trn" in line: 19 | trn_loss.append(datas[1]) 20 | trn_acc.append(datas[2]) 21 | elif "val" in line: 22 | val_loss.append(datas[1]) 23 | val_acc.append(datas[2]) 24 | 25 | plt.figure(figsize=(12, 9)) 26 | plt.ylim((0, 1.05 * max(max(trn_loss), max(val_loss)))) 27 | plt.plot(*smooth(np.array(trn_loss), num_interpolations), label="trn-loss", linewidth=3, color="#86cabf") 28 | plt.plot(*smooth(np.array(val_loss), num_interpolations), label="val-loss", linewidth=3, color="#fa8e7a") 29 | plt.grid() 30 | plt.legend() 31 | plt.show() 32 | plt.close() 33 | 34 | plt.figure(figsize=(12, 9)) 35 | plt.ylim((0.99 * min(min(trn_acc), min(val_acc))), 1.01 * max(max(trn_acc), max(val_acc))) 36 | plt.plot(*smooth(np.array(trn_acc), num_interpolations), label="trn-acc", linewidth=3, color="#86cabf") 37 | plt.plot(*smooth(np.array(val_acc), num_interpolations), label="val-acc", linewidth=3, color="#fa8e7a") 38 | plt.grid() 39 | plt.legend() 40 | plt.show() 41 | plt.close() 42 | 43 | 44 | def smooth(lst, num_interpolations): 45 | if len(lst) < 0: 46 | return None, None 47 | x = np.linspace(1, len(lst), len(lst)) 48 | y = np.array(lst) 49 | if num_interpolations <= 0: 50 | return x, y 51 | x_smooth = np.linspace(min(x), max(x), num_interpolations) 52 | smoothing_spline = interpolate.make_smoothing_spline(x, y) 53 | y_smooth = smoothing_spline(x_smooth) 54 | return x_smooth, y_smooth 55 | 56 | 57 | if __name__ == "__main__": 58 | plot("./logs/movie.conv.log", 0) 59 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["cv", "nlp"] 2 | from utils import * 3 | -------------------------------------------------------------------------------- /utils/cv/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "config", 3 | "nnmodels", 4 | "handler", 5 | "dataset", 6 | "trainer", 7 | "recorder", 8 | ] 9 | 10 | from utils.cv import * 11 | -------------------------------------------------------------------------------- /utils/cv/config.py: -------------------------------------------------------------------------------- 1 | class ConfigObject(object): 2 | def __init__(self, params_dict=None): 3 | self.params_dict = params_dict if params_dict is not None else dict() 4 | 5 | def __setattr__(self, key, value): 6 | if key == "params_dict": 7 | object.__setattr__(self, key, value) 8 | else: 9 | self.params_dict[key] = value 10 | 11 | def __getattr__(self, key): 12 | return self.params_dict[key] 13 | -------------------------------------------------------------------------------- /utils/cv/dataset.py: -------------------------------------------------------------------------------- 1 | import torchvision.datasets as vdatasets 2 | import torchvision.transforms as trans 3 | import torch.utils.data as tdata 4 | import os 5 | import cv2 6 | import utils.cv as ucv 7 | 8 | 9 | def get_dataset(config: ucv.config.ConfigObject): 10 | if config.dataset == "mnist": 11 | if config.trn_preprocess is None: 12 | config.trn_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 13 | if config.val_preprocess is None: 14 | config.val_preprocess = trans.Compose([trans.ToTensor(), trans.Normalize(mean=[0.485], std=[0.229])]) 15 | trn_set = vdatasets.MNIST(root=f"./datasets/mnist", train=True, transform=config.trn_preprocess, download=True) 16 | val_set = vdatasets.MNIST(root=f"./datasets/mnist", train=False, transform=config.val_preprocess, download=True) 17 | return trn_set, val_set 18 | elif config.dataset == "cifar10": 19 | if config.trn_preprocess is None: 20 | config.trn_preprocess = trans.Compose([ 21 | trans.Resize((224, 224)), 22 | trans.RandomHorizontalFlip(p=0.5), 23 | trans.ToTensor(), 24 | trans.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) 25 | ]) 26 | if config.val_preprocess is None: 27 | config.val_preprocess = trans.Compose([ 28 | trans.ToTensor(), 29 | trans.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) 30 | ]) 31 | trn_set = vdatasets.CIFAR10(root=f"./datasets/cifar10", train=True, transform=config.trn_preprocess, download=True) 32 | val_set = vdatasets.CIFAR10(root=f"./datasets/cifar10", train=False, transform=config.val_preprocess, download=True) 33 | return trn_set, val_set 34 | elif config.dataset == "cifar100": 35 | if config.trn_preprocess is None: 36 | config.trn_preprocess = trans.Compose([ 37 | trans.Resize((224, 224)), 38 | trans.RandomHorizontalFlip(p=0.5), 39 | trans.ToTensor(), 40 | trans.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) 41 | ]) 42 | if config.val_preprocess is None: 43 | config.val_preprocess = trans.Compose([ 44 | trans.ToTensor(), 45 | trans.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) 46 | ]) 47 | trn_set = vdatasets.CIFAR100(root=f"./datasets/cifar100", train=True, transform=config.trn_preprocess, download=True) 48 | val_set = vdatasets.CIFAR100(root=f"./datasets/cifar100", train=False, transform=config.val_preprocess, download=True) 49 | return trn_set, val_set 50 | elif config.dataset == "license_plate": 51 | pass 52 | else: 53 | assert config.trn_preprocess is not None 54 | assert config.val_preprocess is not None 55 | trn_set = vdatasets.ImageFolder(root=f"./datasets/{config.dataset}/train", transform=config.trn_preprocess) 56 | val_set = vdatasets.ImageFolder(root=f"./datasets/{config.dataset}/validation", transform=config.val_preprocess) 57 | return trn_set, val_set 58 | 59 | 60 | class LprDataset(tdata.Dataset): 61 | def __init__(self, image_dir, image_preprocess): 62 | self.image_paths = [f"{image_dir}/{image_name}" for image_name in os.listdir(image_dir)] 63 | self.provinces = ["皖", "沪", "津", "渝", "冀", "晋", "蒙", "辽", "吉", "黑", "苏", "浙", "京", "闽", "赣", "鲁", "豫", "鄂", "湘", "粤", "桂", "琼", "川", "贵", "云", "藏", "陕", "甘", "青", "宁", "新", "警", "学", "O"] 64 | self.alphabets = ["A", "B", "C", "D", "E", "F", "G", "H", "J", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "O"] 65 | self.ads = ["A", "B", "C", "D", "E", "F", "G", "H", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] 66 | 67 | def __len__(self): 68 | return len(self.image_paths) 69 | 70 | def __getitem__(self, index): 71 | image = cv2.image 72 | -------------------------------------------------------------------------------- /utils/cv/handler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.transforms as trans 4 | import time 5 | import transformers as tfm 6 | import utils.cv as ucv 7 | 8 | 9 | class ModelHandlerCv(nn.Module): 10 | def __init__(self, config: ucv.config.ConfigObject): 11 | super().__init__() 12 | self.config = config 13 | tfm.set_seed(config.seed) 14 | self.recorder = ucv.recorder.Recorder(config.log_path) 15 | self.device = config.device 16 | self.model = config.model_class(**config.model_params).to(config.device) 17 | self.criterion = config.criterion_class(**config.criterion_params) 18 | 19 | def log(self, message): 20 | self.recorder.audit(message) 21 | 22 | def log_config(self): 23 | self.log(f"\n\n[+] exp starts from: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}") 24 | for config_key, config_value in self.config.params_dict.items(): 25 | if config_value is None: 26 | continue 27 | elif config_key.endswith("_class"): 28 | self.log(f"[+] {config_key.replace('_class', '')}: {config_value.__name__}") 29 | elif config_key.endswith("_params") and isinstance(config_value, dict): 30 | for param_key, param_value in config_value.items(): 31 | self.log(f" [-] {config_key.replace('_params', '')}.{param_key}: {param_value}") 32 | elif isinstance(config_value, trans.transforms.Compose): 33 | self.log(f"[+] {config_key}:") 34 | for index, value in enumerate(str(config_value).replace(" ", "").split("\n")[1:-1]): 35 | self.log(f" [-] {index:02d}: {value}") 36 | else: 37 | self.log(f"[+] {config_key}: {config_value}") 38 | 39 | def device_transfer(self, data): 40 | if isinstance(data, torch.Tensor): 41 | data = data.to(self.device) 42 | if isinstance(data, dict): 43 | data = {key: value.to(self.device) for key, value in data.items()} 44 | return data 45 | 46 | 47 | class ModelHandlerVanilla(ModelHandlerCv): 48 | def __init__(self, config: ucv.config.ConfigObject): 49 | super().__init__(config) 50 | 51 | def forward(self, inputs: torch.Tensor, targets=None): 52 | inputs = self.device_transfer(inputs) 53 | targets = self.device_transfer(targets) 54 | preds = self.model(inputs) 55 | loss = self.criterion(preds, targets) if targets is not None else None 56 | return preds, loss 57 | 58 | 59 | class ModelHandlerLpr(ModelHandlerCv): 60 | def __init__(self, config: ucv.config.ConfigObject): 61 | super().__init__(config) 62 | 63 | def forward(self, inputs: torch.Tensor, targets=None): 64 | inputs = self.device_transfer(inputs) 65 | targets = self.device_transfer(targets) 66 | preds = self.model(inputs) 67 | loss = self.criterion(preds, targets) if targets is not None else None 68 | return preds, loss 69 | -------------------------------------------------------------------------------- /utils/cv/nnmodels.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.models as vmodels 4 | 5 | 6 | class SimpleLinearClassifier(nn.Module): 7 | def __init__(self, num_classes, input_size=784): 8 | super().__init__() 9 | self.flatten = nn.Flatten() 10 | self.layer1 = nn.Sequential(nn.Linear(input_size, 2048), nn.ReLU()) 11 | self.layer2 = nn.Sequential(nn.Linear(2048, 300), nn.ReLU()) 12 | self.dropout = nn.Dropout(0.5) 13 | self.fc = nn.Linear(300, num_classes) 14 | 15 | def forward(self, inputs: torch.Tensor): 16 | outputs = self.flatten(inputs) 17 | outputs = self.layer1(outputs) 18 | outputs = self.layer2(outputs) 19 | outputs = self.dropout(outputs) 20 | outputs = self.fc(outputs) 21 | return outputs 22 | 23 | 24 | class SimpleConvClassifier(nn.Module): 25 | def __init__(self, num_classes, num_channels): 26 | super().__init__() 27 | self.layer1 = self.build_layer(num_channels, 16) 28 | self.layer2 = self.build_layer(16, 32) 29 | self.flatten = nn.Flatten() 30 | self.fc = nn.Linear(32 * 7 * 7, num_classes) 31 | 32 | @staticmethod 33 | def build_layer(conv_in_channels, conv_out_channels, conv_kernel_size=5, conv_stride=1, conv_padding=2, pool_kernel_size=2): 34 | layer = nn.Sequential( 35 | nn.Conv2d(conv_in_channels, conv_out_channels, conv_kernel_size, conv_stride, conv_padding), 36 | nn.ReLU(), nn.BatchNorm2d(conv_out_channels), nn.MaxPool2d(pool_kernel_size)) 37 | return layer 38 | 39 | def forward(self, inputs): 40 | outputs = self.layer1(inputs) 41 | outputs = self.layer2(outputs) 42 | outputs = self.flatten(outputs) 43 | outputs = self.fc(outputs) 44 | return outputs 45 | 46 | 47 | class Resnet18Classifier(nn.Module): 48 | def __init__(self, num_classes, num_channels=3, from_pretrained=None): 49 | super().__init__() 50 | if from_pretrained == "default": 51 | self.resnet = vmodels.resnet18(weights=vmodels.ResNet18_Weights.DEFAULT) 52 | elif from_pretrained == "imagenet": 53 | self.resnet = vmodels.resnet18(weights=vmodels.ResNet18_Weights.IMAGENET1K_V1) 54 | else: 55 | self.resnet = vmodels.resnet18(weights=None) 56 | self.resnet.conv1 = nn.Conv2d(num_channels, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) 57 | self.resnet.fc = nn.Linear(512, num_classes) 58 | 59 | def forward(self, inputs: torch.Tensor): 60 | outputs = self.resnet(inputs) 61 | return outputs 62 | 63 | 64 | class LprNet(nn.Module): 65 | def __init__(self, num_classes=66, dropout=0.5): 66 | super().__init__() 67 | self.backbone = nn.Sequential( 68 | nn.Conv2d(in_channels=3, out_channels=64, kernel_size=(3, 3)), 69 | nn.BatchNorm2d(num_features=64), 70 | nn.ReLU(), 71 | nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 1, 1)), 72 | self.build_basic_block(in_channels=64, out_channels=128), 73 | nn.BatchNorm2d(num_features=128), 74 | nn.ReLU(), 75 | nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(2, 1, 2)), 76 | self.build_basic_block(in_channels=64, out_channels=256), 77 | nn.BatchNorm2d(num_features=256), 78 | nn.ReLU(), 79 | self.build_basic_block(in_channels=256, out_channels=256), 80 | nn.BatchNorm2d(num_features=256), 81 | nn.ReLU(), 82 | nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(4, 1, 2)), 83 | nn.Dropout(dropout), 84 | nn.Conv2d(in_channels=64, out_channels=256, kernel_size=(1, 4)), 85 | nn.BatchNorm2d(num_features=256), 86 | nn.ReLU(), 87 | nn.Dropout(dropout), 88 | nn.Conv2d(in_channels=256, out_channels=num_classes, kernel_size=(13, 1)), 89 | nn.BatchNorm2d(num_features=num_classes), 90 | nn.ReLU()) 91 | self.extractor = nn.Sequential(nn.Linear(num_classes * 8, 128), nn.ReLU()) 92 | self.container = nn.Conv2d(in_channels=num_classes + 128, out_channels=num_classes, kernel_size=(1, 1)) 93 | 94 | @staticmethod 95 | def build_basic_block(in_channels, out_channels): 96 | return nn.Sequential( 97 | nn.Conv2d(in_channels, out_channels // 4, kernel_size=(1, 1)), 98 | nn.ReLU(), 99 | nn.Conv2d(out_channels // 4, out_channels // 4, kernel_size=(3, 1), padding=(1, 0)), 100 | nn.ReLU(), 101 | nn.Conv2d(out_channels // 4, out_channels // 4, kernel_size=(1, 3), padding=(0, 1)), 102 | nn.ReLU(), 103 | nn.Conv2d(out_channels // 4, out_channels, kernel_size=(1, 1)) 104 | ) 105 | 106 | def forward(self, inputs): 107 | inputs = self.backbone(inputs) 108 | pattern = inputs.flatten(1, -1) 109 | pattern = self.extractor(pattern) 110 | pattern = torch.reshape(pattern, [-1, 128, 1, 1]) 111 | pattern = pattern.repeat(1, 1, 1, inputs.size()[-1]) 112 | inputs = torch.cat([inputs, pattern], dim=1) 113 | inputs = self.container(inputs) 114 | logits = inputs.squeeze(2) 115 | return logits 116 | -------------------------------------------------------------------------------- /utils/cv/recorder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sklearn.metrics as metrics 3 | import logging 4 | import os 5 | 6 | 7 | class Recorder: 8 | def __init__(self, logpath): 9 | self.accumulative_accuracy = 0.0 10 | self.accumulative_loss = 0.0 11 | self.accumulative_num_samples = 0 12 | self.logger = logging.getLogger(__name__) 13 | self.logger.setLevel(logging.DEBUG) 14 | self.logger.addHandler(logging.StreamHandler(stream=None)) 15 | if logpath is not None: 16 | if not os.path.exists(os.path.dirname(logpath)): 17 | os.makedirs(os.path.dirname(logpath)) 18 | logfile = open(logpath, "a", encoding="utf-8") 19 | logfile.close() 20 | self.logger.addHandler(logging.FileHandler(filename=logpath, mode="a")) 21 | 22 | def update(self, preds, targets, loss): 23 | assert len(preds) == len(targets) 24 | num_samples = len(preds) 25 | preds = np.array([pred.argmax() for pred in preds.detach().cpu().numpy()]) 26 | targets = targets.detach().cpu().numpy() 27 | self.accumulative_accuracy += metrics.accuracy_score(y_pred=preds, y_true=targets) * num_samples 28 | self.accumulative_loss += loss * num_samples 29 | self.accumulative_num_samples += num_samples 30 | 31 | def clear(self): 32 | self.accumulative_accuracy = 0.0 33 | self.accumulative_loss = 0.0 34 | self.accumulative_num_samples = 0 35 | 36 | def accuracy(self): 37 | accuracy = self.accumulative_accuracy / self.accumulative_num_samples 38 | loss = self.accumulative_loss / self.accumulative_num_samples 39 | return accuracy, loss 40 | 41 | def audit(self, msg): 42 | self.logger.debug(msg) 43 | -------------------------------------------------------------------------------- /utils/cv/trainer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import tqdm 3 | 4 | 5 | class Trainer: 6 | def __init__(self, handler): 7 | self.handler = handler 8 | self.optimizer = handler.config.optimizer_class(handler.model.parameters(), **handler.config.optimizer_params) 9 | self.scheduler = handler.config.scheduler_class(self.optimizer, **handler.config.scheduler_params) if handler.config.scheduler_class is not None else None 10 | 11 | def train(self, loader): 12 | self.handler.train() 13 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] training", delay=0.2, leave=False, ascii="->"): 14 | preds, loss = self.handler(inputs, targets) 15 | self.handler.recorder.update(preds, targets, loss) 16 | self.optimizer.zero_grad() 17 | loss.backward() 18 | self.optimizer.step() 19 | accuracy, loss = self.handler.recorder.accuracy() 20 | self.handler.recorder.clear() 21 | if self.scheduler is not None: 22 | self.scheduler.step() 23 | report = {"loss": loss, "accuracy": accuracy} 24 | return report 25 | 26 | @torch.no_grad() 27 | def validate(self, loader): 28 | self.handler.eval() 29 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] validating", delay=0.2, leave=False, ascii="->"): 30 | preds, loss = self.handler(inputs, targets) 31 | self.handler.recorder.update(preds, targets, loss) 32 | accuracy, loss = self.handler.recorder.accuracy() 33 | self.handler.recorder.clear() 34 | report = {"loss": loss, "accuracy": accuracy} 35 | return report 36 | -------------------------------------------------------------------------------- /utils/nlp/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "config", 3 | "recorder", 4 | "nnmodels", 5 | "handler", 6 | "dataset", 7 | "trainer" 8 | ] 9 | 10 | from utils.nlp import * 11 | -------------------------------------------------------------------------------- /utils/nlp/config.py: -------------------------------------------------------------------------------- 1 | class ConfigObject(object): 2 | def __init__(self, params_dict=None): 3 | self.params_dict = params_dict if params_dict is not None else dict() 4 | 5 | def __setattr__(self, key, value): 6 | if key == "params_dict": 7 | object.__setattr__(self, key, value) 8 | else: 9 | self.params_dict[key] = value 10 | 11 | def __getattr__(self, key): 12 | return self.params_dict[key] 13 | -------------------------------------------------------------------------------- /utils/nlp/dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tqdm 3 | import torch 4 | import torch.utils.data as tdata 5 | import gensim 6 | import numpy as np 7 | 8 | 9 | class DatasetPoemGenerator(tdata.Dataset): 10 | def __init__(self, sequence_length=50, use_samples=-1): 11 | npz_data = np.load(f"./datasets/poem/tang.npz", allow_pickle=True) 12 | self.vocab = {"encode": npz_data["word2ix"].item(), "decode": npz_data["ix2word"].item()} 13 | if use_samples == -1: 14 | self.sentences = npz_data["data"] 15 | else: 16 | self.sentences = npz_data["data"][:use_samples] 17 | self.sequence_length = sequence_length 18 | self.preprocess() 19 | 20 | def preprocess(self): 21 | new_sentences = [] 22 | for sentence in self.sentences: 23 | new_sentence = [token for token in sentence if token != 8292] 24 | if len(new_sentence) < self.sequence_length: 25 | new_sentence.extend([8292] * (self.sequence_length - len(new_sentence))) 26 | else: 27 | new_sentence = new_sentence[:self.sequence_length] 28 | new_sentences.append(new_sentence) 29 | self.sentences = np.array(new_sentences) 30 | self.sentences = torch.tensor(self.sentences, dtype=torch.long) 31 | 32 | def encode(self, character: str): 33 | return self.vocab["encode"][character] 34 | 35 | def decode(self, token: int): 36 | return self.vocab["decode"][token] 37 | 38 | def __getitem__(self, index): 39 | sentence = self.sentences[index, :-1] 40 | target = self.sentences[index, 1:] 41 | return sentence, target 42 | 43 | def __len__(self): 44 | return len(self.sentences) 45 | 46 | 47 | class DatasetSentimentClassifier(tdata.Dataset): 48 | 49 | def __init__(self, from_file, from_vocab, sequence_length=64): 50 | npz_data = np.load(from_vocab, allow_pickle=True) 51 | self.vocab_encode = npz_data["vocab_encode"].item() 52 | self.sequence_length = sequence_length 53 | self.sentences = [] 54 | self.targets = [] 55 | self.load_data(from_file) 56 | 57 | def load_data(self, from_file): 58 | with open(from_file, "r", encoding="utf-8") as file: 59 | for line in tqdm.tqdm(file.readlines(), desc=f"[+] reading \"{from_file}\"", delay=0.2, leave=False, ascii="->"): 60 | elements = line.strip().split() 61 | if len(elements) < 2: 62 | continue 63 | self.targets.append(int(elements[0])) 64 | sentence = elements[1:] 65 | if len(sentence) > self.sequence_length: 66 | sentence = sentence[:self.sequence_length] 67 | else: 68 | sentence.extend(["_PAD_"] * (self.sequence_length - len(sentence))) 69 | self.sentences.append(sentence) 70 | 71 | def __getitem__(self, index): 72 | sentence = torch.tensor(np.array([self.vocab_encode[word] for word in self.sentences[index]])) 73 | target = torch.tensor(self.targets[index]) 74 | return sentence, target 75 | 76 | def __len__(self): 77 | return len(self.sentences) 78 | 79 | @staticmethod 80 | def build_w2v(from_dir, to_file, from_pretrained_embeddings_model): 81 | w2v_model = gensim.models.KeyedVectors.load_word2vec_format(from_pretrained_embeddings_model, binary=True) 82 | vocab_encode = {"_PAD_": 0} 83 | embed_size = w2v_model.vector_size 84 | embeddings = np.zeros(shape=(1, embed_size)) 85 | # embeddings = np.random.uniform(-1, 1, size=(1, embed_size)) 86 | for file_name in [name for name in os.listdir(from_dir) if name.endswith(".txt")]: 87 | with open(f"{from_dir}/{file_name}", "r", encoding="utf-8") as file: 88 | for line in tqdm.tqdm(file.readlines(), desc=f"[+] reading \"{file_name}\"", delay=0.2, leave=False, ascii="->"): 89 | for word in line.strip().split()[1:]: 90 | if word not in vocab_encode.keys(): 91 | vocab_encode[word] = len(vocab_encode) 92 | try: 93 | embeddings = np.vstack([embeddings, w2v_model[word].reshape(1, embed_size)]) 94 | except KeyError: 95 | embeddings = np.vstack([embeddings, np.random.uniform(-1, 1, size=(1, embed_size))]) 96 | np.savez(to_file, **{"vocab_encode": vocab_encode, "embeddings": embeddings}) 97 | 98 | @staticmethod 99 | def get_embeddings_weight(): 100 | return torch.tensor(np.load("./datasets/movie/vocab.npz", allow_pickle=True)["embeddings"], dtype=torch.float32) 101 | -------------------------------------------------------------------------------- /utils/nlp/handler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.transforms as trans 4 | import time 5 | import transformers as tfm 6 | import utils.nlp as unlp 7 | 8 | 9 | class ModelHandlerNLP(nn.Module): 10 | def __init__(self, config: unlp.config.ConfigObject): 11 | super().__init__() 12 | self.config = config 13 | tfm.set_seed(config.seed) 14 | self.recorder = unlp.recorder.Recorder(config.log_path) 15 | self.device = config.device 16 | self.model = config.model_class(**config.model_params).to(config.device) 17 | self.criterion = config.criterion_class(**config.criterion_params) 18 | 19 | def log(self, message): 20 | self.recorder.audit(message) 21 | 22 | def log_config(self): 23 | self.log(f"\n\n[+] exp starts from: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}") 24 | for config_key, config_value in self.config.params_dict.items(): 25 | if config_key.endswith("_class"): 26 | self.log(f"[+] {config_key.replace('_class', '')}: {config_value.__name__ if config_value is not None else None}") 27 | elif config_key.endswith("_params") and isinstance(config_value, dict): 28 | for param_key, param_value in config_value.items(): 29 | self.log(f" [-] {config_key.replace('_params', '')}.{param_key}: {param_value}") 30 | elif isinstance(config_value, trans.transforms.Compose): 31 | self.log(f"[+] {config_key}:") 32 | for index, value in enumerate(str(config_value).replace(" ", "").split("\n")[1:-1]): 33 | self.log(f" [-] {index:02d}: {value}") 34 | else: 35 | self.log(f"[+] {config_key}: {config_value}") 36 | 37 | def device_transfer(self, data): 38 | if isinstance(data, torch.Tensor): 39 | data = data.to(self.device) 40 | if isinstance(data, dict): 41 | data = {key: value.to(self.device) for key, value in data.items()} 42 | if isinstance(data, tuple): 43 | data = tuple([child.to(self.device) for child in data]) 44 | if isinstance(data, list): 45 | data = [child.to(self.device) for child in data] 46 | return data 47 | 48 | 49 | class ModelHandlerGenerator(ModelHandlerNLP): 50 | def __init__(self, config: unlp.config.ConfigObject): 51 | super().__init__(config) 52 | 53 | def forward(self, inputs: torch.Tensor, hiddens: tuple = None, targets: torch.Tensor = None): 54 | if hiddens is None: 55 | batch_size = inputs.shape[0] 56 | lstm_h0 = torch.zeros(size=(self.model.num_lstm_layers, batch_size, self.model.lstm_output_size), dtype=torch.float, requires_grad=False) 57 | lstm_c0 = torch.zeros(size=(self.model.num_lstm_layers, batch_size, self.model.lstm_output_size), dtype=torch.float, requires_grad=False) 58 | hiddens = (lstm_h0, lstm_c0) 59 | inputs = self.device_transfer(inputs) 60 | targets = self.device_transfer(targets) 61 | hiddens = self.device_transfer(hiddens) 62 | preds, hiddens = self.model(inputs, hiddens) 63 | 64 | # 相当于把batch内的多个样本拼接起来算损失函数 65 | if targets is not None: 66 | batch_size, sequence_length, vocab_size = preds.shape 67 | preds = preds.reshape(batch_size * sequence_length, vocab_size) 68 | targets = targets.reshape(batch_size * sequence_length) 69 | loss = self.criterion(preds, targets) 70 | self.recorder.update(preds, targets, loss) 71 | else: 72 | loss = None 73 | return preds, loss, hiddens 74 | 75 | 76 | class ModelHandlerClassifier(ModelHandlerNLP): 77 | def __init__(self, config: unlp.config.ConfigObject): 78 | super().__init__(config) 79 | 80 | def forward(self, inputs: torch.Tensor, targets=None): 81 | inputs = self.device_transfer(inputs) 82 | targets = self.device_transfer(targets) 83 | preds = self.model(inputs) 84 | loss = self.criterion(preds, targets) if targets is not None else None 85 | return preds, loss 86 | -------------------------------------------------------------------------------- /utils/nlp/nnmodels.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as func 4 | 5 | 6 | class LstmGnerator(nn.Module): 7 | def __init__(self, vocab_size, embed_size, lstm_output_size, num_lstm_layers=1, lstm_dropout=0.0): 8 | super().__init__() 9 | self.lstm_output_size = lstm_output_size 10 | self.num_lstm_layers = num_lstm_layers 11 | # 在windows上多层lstm使用dropout或导致driver shutdown告警，应该是torch的问题 12 | self.embedding = nn.Embedding(vocab_size, embed_size) 13 | self.lstm = nn.LSTM(embed_size, lstm_output_size, num_layers=num_lstm_layers, batch_first=True, dropout=lstm_dropout) 14 | self.fc = nn.Sequential( 15 | nn.Linear(lstm_output_size, 2048), 16 | nn.Tanh(), 17 | nn.Linear(2048, vocab_size)) 18 | 19 | def forward(self, inputs, hiddens): 20 | outputs = self.embedding(inputs) 21 | # lstm_outputs.shape: (batch_size, sequence_length, vocab_size) 22 | # lstm_hiddens: (lstm_h0, lstm_c0) 23 | outputs, lstm_hiddens = self.lstm(outputs, hiddens) 24 | outputs = self.fc(outputs) 25 | return outputs, lstm_hiddens 26 | 27 | 28 | class TextConvClassifier(nn.Module): 29 | def __init__(self, num_classes, dropout_rate, conv_out_channelses, kernel_sizes, pretrained_embeddings, freeze_embeddings=False): 30 | super().__init__() 31 | self.embeddings = nn.Embedding.from_pretrained(pretrained_embeddings, freeze=freeze_embeddings) 32 | self.embed_size = int(pretrained_embeddings.shape[-1]) 33 | self.parallel_conv_layers = nn.ModuleList([nn.Conv2d(1, conv_out_channels, (kernel_size, self.embed_size)) for conv_out_channels, kernel_size in zip(conv_out_channelses, kernel_sizes)]) 34 | # self.bn = nn.BatchNorm2d(conv_out_channels) 35 | self.dropout = nn.Dropout(dropout_rate) 36 | self.flatten = nn.Flatten() 37 | self.fc = nn.Linear(sum(conv_out_channelses), num_classes) 38 | 39 | def forward(self, inputs): 40 | outputs = self.embeddings(inputs).unsqueeze(dim=1) 41 | outputs = [conv_layer(outputs).squeeze(dim=3) for conv_layer in self.parallel_conv_layers] 42 | outputs = [func.relu(output) for output in outputs] 43 | outputs = [func.max_pool1d(output, output.size(dim=2)).squeeze(dim=2) for output in outputs] 44 | outputs = torch.cat(outputs, dim=1) 45 | outputs = self.dropout(outputs) 46 | outputs = self.flatten(outputs) 47 | outputs = self.fc(outputs) 48 | return outputs 49 | -------------------------------------------------------------------------------- /utils/nlp/recorder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sklearn.metrics as metrics 3 | import logging 4 | import os 5 | 6 | 7 | class Recorder: 8 | def __init__(self, logpath): 9 | self.accumulative_accuracy = 0.0 10 | self.accumulative_loss = 0.0 11 | self.accumulative_num_samples = 0 12 | self.logger = logging.getLogger(__name__) 13 | self.logger.setLevel(logging.DEBUG) 14 | self.logger.addHandler(logging.StreamHandler(stream=None)) 15 | if logpath is not None: 16 | if not os.path.exists(os.path.dirname(logpath)): 17 | os.makedirs(os.path.dirname(logpath)) 18 | logfile = open(logpath, "a", encoding="utf-8") 19 | logfile.close() 20 | self.logger.addHandler(logging.FileHandler(filename=logpath, mode="a")) 21 | 22 | def update(self, preds, targets, loss): 23 | assert len(preds) == len(targets) 24 | num_samples = len(preds) 25 | preds = np.array([pred.argmax() for pred in preds.detach().cpu().numpy()]) 26 | targets = targets.detach().cpu().numpy() 27 | self.accumulative_accuracy += metrics.accuracy_score(y_pred=preds, y_true=targets) * num_samples 28 | self.accumulative_loss += loss * num_samples 29 | self.accumulative_num_samples += num_samples 30 | 31 | def clear(self): 32 | self.accumulative_accuracy = 0.0 33 | self.accumulative_loss = 0.0 34 | self.accumulative_num_samples = 0 35 | 36 | def accuracy(self): 37 | accuracy = self.accumulative_accuracy / self.accumulative_num_samples 38 | loss = self.accumulative_loss / self.accumulative_num_samples 39 | return accuracy, loss 40 | 41 | def audit(self, msg): 42 | self.logger.debug(msg) 43 | -------------------------------------------------------------------------------- /utils/nlp/trainer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import tqdm 3 | import utils.nlp as unlp 4 | 5 | 6 | class Trainer: 7 | def __init__(self, handler: [unlp.handler.ModelHandlerNLP]): 8 | self.handler = handler 9 | self.config = handler.config 10 | self.optimizer = handler.config.optimizer_class(handler.model.parameters(), **handler.config.optimizer_params) 11 | self.scheduler = handler.config.scheduler_class(self.optimizer, **handler.config.scheduler_params) if handler.config.scheduler_class is not None else None 12 | 13 | def train(self, loader): 14 | self.handler.train() 15 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] training", delay=0.2, leave=False, ascii="->"): 16 | preds, loss = self.handler(inputs, targets) 17 | self.handler.recorder.update(preds, targets, loss) 18 | self.optimizer.zero_grad() 19 | loss.backward() 20 | self.optimizer.step() 21 | accuracy, loss = self.handler.recorder.accuracy() 22 | self.handler.recorder.clear() 23 | if self.scheduler is not None: 24 | self.scheduler.step() 25 | report = {"loss": loss, "accuracy": accuracy} 26 | return report 27 | 28 | @torch.no_grad() 29 | def validate(self, loader): 30 | self.handler.eval() 31 | for inputs, targets in tqdm.tqdm(loader, desc=f" [-] validating", delay=0.2, leave=False, ascii="->"): 32 | preds, loss = self.handler(inputs, targets) 33 | self.handler.recorder.update(preds, targets, loss) 34 | accuracy, loss = self.handler.recorder.accuracy() 35 | self.handler.recorder.clear() 36 | report = {"loss": loss, "accuracy": accuracy} 37 | return report 38 | 39 | @torch.no_grad() 40 | def generate(self, input_tokens: list, output_length: int): 41 | self.handler.eval() 42 | start_token = 8291 43 | end_token = 8290 44 | if input_tokens[0] != start_token: 45 | input_tokens.insert(0, start_token) 46 | output_tokens = input_tokens 47 | inputs = torch.tensor(input_tokens).unsqueeze(0) 48 | outputs, _, hiddens = self.handler(inputs=inputs, hiddens=None) 49 | for _ in range(output_length - len(input_tokens)): 50 | preds = outputs[0][-1].argmax(axis=0) 51 | output_tokens.append(int(preds.item())) 52 | if preds.item() == end_token: 53 | break 54 | else: 55 | inputs = preds.reshape(1, 1) 56 | outputs, _, hiddens = self.handler(inputs=inputs, hiddens=hiddens) 57 | return output_tokens 58 | --------------------------------------------------------------------------------