├── 1_basic_of_deep_learning_with_pytorch ├── dynamic_computational_graph_and_gradient_descent.ipynb ├── optimization_and_least_square.ipynb ├── tensor_broadcast_computing.ipynb ├── tensor_create_index.ipynb ├── tensor_index_slice_merge.ipynb └── tensor_linear_algebra.ipynb ├── 2_build_a_neural_network_from_scratch ├── deep_neural_network.ipynb ├── introductory_dl_and_pytorch.ipynb └── single_layer_neural_network.ipynb ├── 3_training_and_prediction_of_neural_network ├── building_and_importing_of_basic_neural_network.ipynb ├── learning_of_neural_network.ipynb └── loss_function_of_neural_network.ipynb ├── 4_stability_and_optimization_of_neural_network ├── data_normalization_and_batch_norm.ipynb ├── gradient_instability_and_dead_relu_problem.ipynb ├── learning_rate_scheduling.ipynb ├── modeling_objective_and_model_underfitting_in_deep_learning.ipynb └── xavier_and_kaiming_weight_initialization.ipynb ├── 5_deep_vision_and_convolutional_neural_network ├── classical_convolutional_neural_networks_and_model_architecture_evaluation.ipynb └── image_processing_and_convolutional_neural_network.ipynb ├── README.md ├── assets ├── 2_layer_grad_compute.png ├── alexnet_structure.png ├── bn_lr_learning_curve.png ├── conv_big_small_kernel.png ├── conv_group_demo.png ├── convolution_depthwise_separable.png ├── convolution_first.png ├── convolution_init.png ├── convolution_last.png ├── convolution_second.png ├── cross_entropy_parts.png ├── dataset_dataloader_process.png ├── decreasing_receptive_field.png ├── derivative_and_grad.png ├── dilation_convolution_demo.gif ├── dropout_demo.png ├── fc_kernel1_in_cnn.png ├── googlenet_inception_v1.png ├── googlenet_layer_param.png ├── googlenet_structure.png ├── grad_descent_compare.png ├── gradient_disappear_violin.png ├── handson_linear_regression_nn.png ├── handson_softmax_nn.png ├── invariance_of_statue.png ├── kernel_size_1_bottleneck.png ├── lenet5_structure.png ├── loss_func_demo.png ├── machine_learning_process.png ├── model_complexity_prediction_error_of_train_and_test.png ├── model_depth_performance_data.png ├── model_number.png ├── momentum_demo.png ├── multi_classification_one_hot.png ├── multi_nn.png ├── nin_structure.png ├── opencv_laplacian_sobel_detection.png ├── pytorch-logo.png ├── relu_curve.png ├── relu_derivative.png ├── repetitive_receptive_field.png ├── resnet_5_versions.png ├── resnet_conv_in_x.png ├── resnet_residual_unit.png ├── rgb_channels.png ├── seaborn_violin_plot_illustration.png ├── sigmoid_curve.png ├── sigmoid_nn.png ├── sigmoid_stack_compare.png ├── sigmoid_stacked_compare.png ├── sign_classification.png ├── sign_curve.png ├── single_lr_nn.png ├── softmax_nn.png ├── sse_3d_image.png ├── statistical_analysis_process.png ├── tanh_curve.png ├── tanh_derivative.png ├── tanh_stacked_compare.png ├── tensorboard_linear_model_structure.png ├── tensorboard_process.png ├── tensorboard_scalar_demo.png ├── vgg16_structure.png ├── weight_bias_res_number.png └── xor_multi_nn.png ├── layers ├── __init__.py ├── linears.py └── losses.py ├── requirements.txt └── utils ├── __init__.py ├── data_gen_split.py ├── datasets.py ├── receptive_field.py ├── train_utils.py └── visualization_utils.py /1_basic_of_deep_learning_with_pytorch/tensor_create_index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [ 10 | { 11 | "data": { 12 | "text/plain": [ 13 | "'1.11.0'" 14 | ] 15 | }, 16 | "execution_count": 1, 17 | "metadata": {}, 18 | "output_type": "execute_result" 19 | } 20 | ], 21 | "source": [ 22 | "# -*- coding: utf-8 -*-\n", 23 | "\n", 24 | "'''\n", 25 | "@Author : Corley Tang\n", 26 | "@contact : cutercorleytd@gmail.com\n", 27 | "@Github : https://github.com/corleytd\n", 28 | "@Time : 2023-01-09 17:18\n", 29 | "@Project : Hands-on Deep Learning with PyTorch-tensor_create_index\n", 30 | "张量（Tensor）的创建和索引\n", 31 | "'''\n", 32 | "\n", 33 | "# 导入所需的库\n", 34 | "import numpy as np\n", 35 | "import torch\n", 36 | "\n", 37 | "# 查看版本\n", 38 | "torch.__version__" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "## 1.张量的类型和转化" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "data": { 55 | "text/plain": [ 56 | "tensor([1, 2])" 57 | ] 58 | }, 59 | "execution_count": 2, 60 | "metadata": {}, 61 | "output_type": "execute_result" 62 | } 63 | ], 64 | "source": [ 65 | "# 使用张量创建函数创建张量\n", 66 | "t1 = torch.tensor([1, 2]) # 列表\n", 67 | "t1" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 3, 73 | "metadata": {}, 74 | "outputs": [ 75 | { 76 | "data": { 77 | "text/plain": [ 78 | "tensor([1, 2])" 79 | ] 80 | }, 81 | "execution_count": 3, 82 | "metadata": {}, 83 | "output_type": "execute_result" 84 | } 85 | ], 86 | "source": [ 87 | "torch.tensor((1, 2)) # 元组" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "data": { 97 | "text/plain": [ 98 | "tensor([1, 2], dtype=torch.int32)" 99 | ] 100 | }, 101 | "execution_count": 4, 102 | "metadata": {}, 103 | "output_type": "execute_result" 104 | } 105 | ], 106 | "source": [ 107 | "a = np.array((1, 2)) # 数据\n", 108 | "t2 = torch.tensor(a) # dtype指定张量的数据类型\n", 109 | "t2" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 5, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "data": { 119 | "text/plain": [ 120 | "(dtype('int32'), torch.int64, torch.int32)" 121 | ] 122 | }, 123 | "execution_count": 5, 124 | "metadata": {}, 125 | "output_type": "execute_result" 126 | } 127 | ], 128 | "source": [ 129 | "# 查看整型的默认类型：张量默认长整型、Array默认整型\n", 130 | "a.dtype, t1.dtype, t2.dtype" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 6, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "text/plain": [ 141 | "(dtype('float64'), torch.float64, torch.float32)" 142 | ] 143 | }, 144 | "execution_count": 6, 145 | "metadata": {}, 146 | "output_type": "execute_result" 147 | } 148 | ], 149 | "source": [ 150 | "# 查看浮点型的默认类型：张量默认单精度浮点型、Array默认双精度浮点型\n", 151 | "np.array([1.1, 2.2]).dtype, torch.tensor(np.array([1.1, 2.2])).dtype, torch.tensor([1.1, 2.2]).dtype" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 7, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "data": { 161 | "text/plain": [ 162 | "(tensor([ True, False]), torch.bool)" 163 | ] 164 | }, 165 | "execution_count": 7, 166 | "metadata": {}, 167 | "output_type": "execute_result" 168 | } 169 | ], 170 | "source": [ 171 | "# 布尔型\n", 172 | "t3 = torch.tensor([True, False])\n", 173 | "t3, t3.dtype" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 8, 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/plain": [ 184 | "(tensor([1.+2.j, 2.+3.j]), torch.complex64)" 185 | ] 186 | }, 187 | "execution_count": 8, 188 | "metadata": {}, 189 | "output_type": "execute_result" 190 | } 191 | ], 192 | "source": [ 193 | "# 复数型\n", 194 | "t4 = torch.tensor([1 + 2j, 2 + 3j])\n", 195 | "t4, t4.dtype" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 9, 201 | "metadata": {}, 202 | "outputs": [ 203 | { 204 | "data": { 205 | "text/plain": [ 206 | "tensor([1, 2], dtype=torch.int16)" 207 | ] 208 | }, 209 | "execution_count": 9, 210 | "metadata": {}, 211 | "output_type": "execute_result" 212 | } 213 | ], 214 | "source": [ 215 | "# 指定类型创建张量\n", 216 | "torch.tensor([1, 2], dtype=torch.int16)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 10, 222 | "metadata": {}, 223 | "outputs": [ 224 | { 225 | "data": { 226 | "text/plain": [ 227 | "(tensor([1.1000, 2.0000]), torch.float32)" 228 | ] 229 | }, 230 | "execution_count": 10, 231 | "metadata": {}, 232 | "output_type": "execute_result" 233 | } 234 | ], 235 | "source": [ 236 | "# 张量类型的隐式转化\n", 237 | "# 整型和浮点型\n", 238 | "t5 = torch.tensor([1.1, 2])\n", 239 | "t5, t5.dtype" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 11, 245 | "metadata": {}, 246 | "outputs": [ 247 | { 248 | "data": { 249 | "text/plain": [ 250 | "(tensor([1., 2., 0.]), torch.float32)" 251 | ] 252 | }, 253 | "execution_count": 11, 254 | "metadata": {}, 255 | "output_type": "execute_result" 256 | } 257 | ], 258 | "source": [ 259 | "# 布尔型和数值型\n", 260 | "t6 = torch.tensor([True, 2., False])\n", 261 | "t6, t6.dtype" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 12, 267 | "metadata": {}, 268 | "outputs": [ 269 | { 270 | "data": { 271 | "text/plain": [ 272 | "(tensor([1, 2]),\n", 273 | " tensor([1., 2.]),\n", 274 | " tensor([1., 2.], dtype=torch.float64),\n", 275 | " tensor([1, 2], dtype=torch.int16),\n", 276 | " tensor([1, 2]))" 277 | ] 278 | }, 279 | "execution_count": 12, 280 | "metadata": {}, 281 | "output_type": "execute_result" 282 | } 283 | ], 284 | "source": [ 285 | "# 张量类型的显式转化\n", 286 | "t1, t1.float(), t1.double(), t1.short(), t1 # 不会改变原有张量的类型" 287 | ] 288 | }, 289 | { 290 | "cell_type": "markdown", 291 | "metadata": {}, 292 | "source": [ 293 | "## 2.张量的维度和形变" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": 13, 299 | "metadata": {}, 300 | "outputs": [ 301 | { 302 | "data": { 303 | "text/plain": [ 304 | "(1, torch.Size([2]), torch.Size([2]))" 305 | ] 306 | }, 307 | "execution_count": 13, 308 | "metadata": {}, 309 | "output_type": "execute_result" 310 | } 311 | ], 312 | "source": [ 313 | "# 查看张量的维度和形状\n", 314 | "t1.ndim, t1.shape, t1.size()" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 14, 320 | "metadata": {}, 321 | "outputs": [ 322 | { 323 | "data": { 324 | "text/plain": [ 325 | "(2, 2)" 326 | ] 327 | }, 328 | "execution_count": 14, 329 | "metadata": {}, 330 | "output_type": "execute_result" 331 | } 332 | ], 333 | "source": [ 334 | "# 查看低1维度的元素个数、元素总个数\n", 335 | "len(t1), t1.numel()" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 15, 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "data": { 345 | "text/plain": [ 346 | "tensor([[1, 2, 3],\n", 347 | " [4, 5, 6]])" 348 | ] 349 | }, 350 | "execution_count": 15, 351 | "metadata": {}, 352 | "output_type": "execute_result" 353 | } 354 | ], 355 | "source": [ 356 | "# 创建二维张量\n", 357 | "t7 = torch.tensor([[1, 2, 3], [4, 5, 6]])\n", 358 | "t7" 359 | ] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": 16, 364 | "metadata": {}, 365 | "outputs": [ 366 | { 367 | "data": { 368 | "text/plain": [ 369 | "(2, torch.Size([2, 3]), torch.Size([2, 3]), 2, 6)" 370 | ] 371 | }, 372 | "execution_count": 16, 373 | "metadata": {}, 374 | "output_type": "execute_result" 375 | } 376 | ], 377 | "source": [ 378 | "t7.ndim, t7.shape, t7.size(), len(t7), t7.numel()" 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": 17, 384 | "metadata": {}, 385 | "outputs": [ 386 | { 387 | "data": { 388 | "text/plain": [ 389 | "(tensor(1), 0, torch.Size([]), 1)" 390 | ] 391 | }, 392 | "execution_count": 17, 393 | "metadata": {}, 394 | "output_type": "execute_result" 395 | } 396 | ], 397 | "source": [ 398 | "# “零”维张量\n", 399 | "t8 = torch.tensor(1)\n", 400 | "t8, t8.ndim, t8.shape, t8.numel()" 401 | ] 402 | }, 403 | { 404 | "cell_type": "code", 405 | "execution_count": 18, 406 | "metadata": {}, 407 | "outputs": [ 408 | { 409 | "data": { 410 | "text/plain": [ 411 | "(tensor([1]), 1, torch.Size([1]), 1)" 412 | ] 413 | }, 414 | "execution_count": 18, 415 | "metadata": {}, 416 | "output_type": "execute_result" 417 | } 418 | ], 419 | "source": [ 420 | "# 一维张量\n", 421 | "t9 = torch.tensor([1])\n", 422 | "t9, t9.ndim, t9.shape, t9.numel()" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": 19, 428 | "metadata": {}, 429 | "outputs": [ 430 | { 431 | "data": { 432 | "text/plain": [ 433 | "(array([[1, 2, 2],\n", 434 | " [3, 4, 4]]),\n", 435 | " array([[5, 6, 6],\n", 436 | " [7, 8, 8]]),\n", 437 | " tensor([[[1, 2, 2],\n", 438 | " [3, 4, 4]],\n", 439 | " \n", 440 | " [[5, 6, 6],\n", 441 | " [7, 8, 8]]], dtype=torch.int32))" 442 | ] 443 | }, 444 | "execution_count": 19, 445 | "metadata": {}, 446 | "output_type": "execute_result" 447 | } 448 | ], 449 | "source": [ 450 | "# 高维张量\n", 451 | "a1 = np.array([[1, 2, 2], [3, 4, 4]])\n", 452 | "a2 = np.array([[5, 6, 6], [7, 8, 8]])\n", 453 | "t10 = torch.tensor(np.array([a1, a2]))\n", 454 | "a1, a2, t10" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": 20, 460 | "metadata": {}, 461 | "outputs": [ 462 | { 463 | "data": { 464 | "text/plain": [ 465 | "(3, torch.Size([2, 2, 3]), 2, 12)" 466 | ] 467 | }, 468 | "execution_count": 20, 469 | "metadata": {}, 470 | "output_type": "execute_result" 471 | } 472 | ], 473 | "source": [ 474 | "t10.ndim, t10.shape, len(t10), t10.numel()" 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "execution_count": 21, 480 | "metadata": {}, 481 | "outputs": [ 482 | { 483 | "data": { 484 | "text/plain": [ 485 | "(tensor([[1, 2, 3],\n", 486 | " [4, 5, 6]]),\n", 487 | " tensor([1, 2, 3, 4, 5, 6]))" 488 | ] 489 | }, 490 | "execution_count": 21, 491 | "metadata": {}, 492 | "output_type": "execute_result" 493 | } 494 | ], 495 | "source": [ 496 | "# 张量的形变\n", 497 | "# flatten拉平：将任意维度张量转化为一维张量\n", 498 | "t7, t7.flatten()" 499 | ] 500 | }, 501 | { 502 | "cell_type": "code", 503 | "execution_count": 22, 504 | "metadata": {}, 505 | "outputs": [ 506 | { 507 | "data": { 508 | "text/plain": [ 509 | "(tensor([[[1, 2, 2],\n", 510 | " [3, 4, 4]],\n", 511 | " \n", 512 | " [[5, 6, 6],\n", 513 | " [7, 8, 8]]], dtype=torch.int32),\n", 514 | " tensor([1, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8], dtype=torch.int32))" 515 | ] 516 | }, 517 | "execution_count": 22, 518 | "metadata": {}, 519 | "output_type": "execute_result" 520 | } 521 | ], 522 | "source": [ 523 | "t10, t10.flatten()" 524 | ] 525 | }, 526 | { 527 | "cell_type": "code", 528 | "execution_count": 23, 529 | "metadata": {}, 530 | "outputs": [ 531 | { 532 | "data": { 533 | "text/plain": [ 534 | "(tensor(1), tensor([1]), 1)" 535 | ] 536 | }, 537 | "execution_count": 23, 538 | "metadata": {}, 539 | "output_type": "execute_result" 540 | } 541 | ], 542 | "source": [ 543 | "t8, t8.flatten(), t8.flatten().ndim" 544 | ] 545 | }, 546 | { 547 | "cell_type": "code", 548 | "execution_count": 24, 549 | "metadata": {}, 550 | "outputs": [ 551 | { 552 | "data": { 553 | "text/plain": [ 554 | "(tensor([1, 2]),\n", 555 | " tensor([[1],\n", 556 | " [2]]),\n", 557 | " torch.Size([2]),\n", 558 | " torch.Size([2, 1]),\n", 559 | " 1,\n", 560 | " 2)" 561 | ] 562 | }, 563 | "execution_count": 24, 564 | "metadata": {}, 565 | "output_type": "execute_result" 566 | } 567 | ], 568 | "source": [ 569 | "# reshape方法：任意变形\n", 570 | "t1, t1.reshape(2, 1), t1.shape, t1.reshape(2, 1).shape, t1.ndim, t1.reshape(2, 1).ndim" 571 | ] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "execution_count": 25, 576 | "metadata": {}, 577 | "outputs": [ 578 | { 579 | "data": { 580 | "text/plain": [ 581 | "(tensor([1, 2]), tensor([1, 2]), tensor([1, 2]), 1)" 582 | ] 583 | }, 584 | "execution_count": 25, 585 | "metadata": {}, 586 | "output_type": "execute_result" 587 | } 588 | ], 589 | "source": [ 590 | "# 转为一维张量\n", 591 | "t1, t1.reshape(2), t1.reshape(2, ), t1.reshape(2).ndim" 592 | ] 593 | }, 594 | { 595 | "cell_type": "code", 596 | "execution_count": 26, 597 | "metadata": {}, 598 | "outputs": [ 599 | { 600 | "data": { 601 | "text/plain": [ 602 | "(tensor([1, 2]), tensor([[1, 2]]), 2)" 603 | ] 604 | }, 605 | "execution_count": 26, 606 | "metadata": {}, 607 | "output_type": "execute_result" 608 | } 609 | ], 610 | "source": [ 611 | "# 转为二维张量\n", 612 | "t1, t1.reshape(1, 2), t1.reshape(1, 2).ndim" 613 | ] 614 | }, 615 | { 616 | "cell_type": "code", 617 | "execution_count": 27, 618 | "metadata": {}, 619 | "outputs": [ 620 | { 621 | "data": { 622 | "text/plain": [ 623 | "(tensor([1, 2]),\n", 624 | " tensor([[[1, 2]]]),\n", 625 | " torch.Size([1, 1, 2]),\n", 626 | " tensor([[[1],\n", 627 | " [2]]]),\n", 628 | " 3)" 629 | ] 630 | }, 631 | "execution_count": 27, 632 | "metadata": {}, 633 | "output_type": "execute_result" 634 | } 635 | ], 636 | "source": [ 637 | "# 转为三维张量\n", 638 | "t1, t1.reshape(1, 1, 2), t1.reshape(1, 1, 2).shape, t1.reshape(1, 2, 1), t1.reshape(1, 2, 1).ndim" 639 | ] 640 | }, 641 | { 642 | "cell_type": "code", 643 | "execution_count": 28, 644 | "metadata": {}, 645 | "outputs": [ 646 | { 647 | "data": { 648 | "text/plain": [ 649 | "tensor([1, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8], dtype=torch.int32)" 650 | ] 651 | }, 652 | "execution_count": 28, 653 | "metadata": {}, 654 | "output_type": "execute_result" 655 | } 656 | ], 657 | "source": [ 658 | "# 用reshape方法拉平高维张量\n", 659 | "t10.reshape(-1)" 660 | ] 661 | }, 662 | { 663 | "cell_type": "markdown", 664 | "metadata": {}, 665 | "source": [ 666 | "## 3.特殊张量的创建" 667 | ] 668 | }, 669 | { 670 | "cell_type": "code", 671 | "execution_count": 29, 672 | "metadata": {}, 673 | "outputs": [ 674 | { 675 | "data": { 676 | "text/plain": [ 677 | "tensor([[0., 0., 0.],\n", 678 | " [0., 0., 0.]])" 679 | ] 680 | }, 681 | "execution_count": 29, 682 | "metadata": {}, 683 | "output_type": "execute_result" 684 | } 685 | ], 686 | "source": [ 687 | "## 1.特殊取值的张量\n", 688 | "# 全0张量，默认浮点型\n", 689 | "torch.zeros([2, 3])" 690 | ] 691 | }, 692 | { 693 | "cell_type": "code", 694 | "execution_count": 30, 695 | "metadata": {}, 696 | "outputs": [ 697 | { 698 | "data": { 699 | "text/plain": [ 700 | "tensor([[[1., 1., 1., 1.],\n", 701 | " [1., 1., 1., 1.],\n", 702 | " [1., 1., 1., 1.]],\n", 703 | "\n", 704 | " [[1., 1., 1., 1.],\n", 705 | " [1., 1., 1., 1.],\n", 706 | " [1., 1., 1., 1.]]])" 707 | ] 708 | }, 709 | "execution_count": 30, 710 | "metadata": {}, 711 | "output_type": "execute_result" 712 | } 713 | ], 714 | "source": [ 715 | "# 全1张量\n", 716 | "torch.ones([2, 3, 4])" 717 | ] 718 | }, 719 | { 720 | "cell_type": "code", 721 | "execution_count": 31, 722 | "metadata": {}, 723 | "outputs": [ 724 | { 725 | "data": { 726 | "text/plain": [ 727 | "tensor([[1., 0., 0., 0., 0., 0.],\n", 728 | " [0., 1., 0., 0., 0., 0.],\n", 729 | " [0., 0., 1., 0., 0., 0.],\n", 730 | " [0., 0., 0., 1., 0., 0.],\n", 731 | " [0., 0., 0., 0., 1., 0.],\n", 732 | " [0., 0., 0., 0., 0., 1.]])" 733 | ] 734 | }, 735 | "execution_count": 31, 736 | "metadata": {}, 737 | "output_type": "execute_result" 738 | } 739 | ], 740 | "source": [ 741 | "# 单位矩阵\n", 742 | "torch.eye(6)" 743 | ] 744 | }, 745 | { 746 | "cell_type": "code", 747 | "execution_count": 32, 748 | "metadata": {}, 749 | "outputs": [ 750 | { 751 | "data": { 752 | "text/plain": [ 753 | "(tensor([1, 2]),\n", 754 | " tensor([[1, 0],\n", 755 | " [0, 2]]))" 756 | ] 757 | }, 758 | "execution_count": 32, 759 | "metadata": {}, 760 | "output_type": "execute_result" 761 | } 762 | ], 763 | "source": [ 764 | "# 对角矩阵\n", 765 | "# torch.diag([1, 2]) # TypeError\n", 766 | "t1, torch.diag(t1)" 767 | ] 768 | }, 769 | { 770 | "cell_type": "code", 771 | "execution_count": 33, 772 | "metadata": {}, 773 | "outputs": [ 774 | { 775 | "data": { 776 | "text/plain": [ 777 | "tensor([[0.7381, 0.4472, 0.6182],\n", 778 | " [0.7681, 0.2479, 0.0711]])" 779 | ] 780 | }, 781 | "execution_count": 33, 782 | "metadata": {}, 783 | "output_type": "execute_result" 784 | } 785 | ], 786 | "source": [ 787 | "# rand：服从0-1均匀分布的张量\n", 788 | "torch.rand(2, 3)" 789 | ] 790 | }, 791 | { 792 | "cell_type": "code", 793 | "execution_count": 34, 794 | "metadata": {}, 795 | "outputs": [ 796 | { 797 | "data": { 798 | "text/plain": [ 799 | "tensor([[ 1.7518, 0.3649, -1.2904],\n", 800 | " [ 0.9718, 0.6282, -0.8616]])" 801 | ] 802 | }, 803 | "execution_count": 34, 804 | "metadata": {}, 805 | "output_type": "execute_result" 806 | } 807 | ], 808 | "source": [ 809 | "# randn：服从标准正态分布的张量\n", 810 | "torch.randn(2, 3)" 811 | ] 812 | }, 813 | { 814 | "cell_type": "code", 815 | "execution_count": 35, 816 | "metadata": {}, 817 | "outputs": [ 818 | { 819 | "data": { 820 | "text/plain": [ 821 | "tensor([[2.2268, 1.3118, 7.0009],\n", 822 | " [4.7417, 2.2390, 6.2416]])" 823 | ] 824 | }, 825 | "execution_count": 35, 826 | "metadata": {}, 827 | "output_type": "execute_result" 828 | } 829 | ], 830 | "source": [ 831 | "# normal：服从指定正态分布的张量\n", 832 | "torch.normal(2, 3, size=(2, 3))" 833 | ] 834 | }, 835 | { 836 | "cell_type": "code", 837 | "execution_count": 36, 838 | "metadata": {}, 839 | "outputs": [ 840 | { 841 | "data": { 842 | "text/plain": [ 843 | "tensor([[86, 91, 56, 72],\n", 844 | " [21, 15, 25, 83],\n", 845 | " [74, 27, 40, 55]])" 846 | ] 847 | }, 848 | "execution_count": 36, 849 | "metadata": {}, 850 | "output_type": "execute_result" 851 | } 852 | ], 853 | "source": [ 854 | "# randint：整数随机采样结果\n", 855 | "torch.randint(1, 100, size=[3, 4]) # 左开右闭" 856 | ] 857 | }, 858 | { 859 | "cell_type": "code", 860 | "execution_count": 37, 861 | "metadata": {}, 862 | "outputs": [ 863 | { 864 | "data": { 865 | "text/plain": [ 866 | "(tensor([0, 1, 2, 3, 4]), tensor([1, 4, 7]))" 867 | ] 868 | }, 869 | "execution_count": 37, 870 | "metadata": {}, 871 | "output_type": "execute_result" 872 | } 873 | ], 874 | "source": [ 875 | "# arange/linspace：生成数列\n", 876 | "torch.arange(5), torch.arange(1, 10, 3) # 左开右闭" 877 | ] 878 | }, 879 | { 880 | "cell_type": "code", 881 | "execution_count": 38, 882 | "metadata": {}, 883 | "outputs": [ 884 | { 885 | "data": { 886 | "text/plain": [ 887 | "tensor([ 1., 4., 7., 10.])" 888 | ] 889 | }, 890 | "execution_count": 38, 891 | "metadata": {}, 892 | "output_type": "execute_result" 893 | } 894 | ], 895 | "source": [ 896 | "torch.linspace(1, 10, 4) # 左右都包含" 897 | ] 898 | }, 899 | { 900 | "cell_type": "code", 901 | "execution_count": 39, 902 | "metadata": {}, 903 | "outputs": [ 904 | { 905 | "data": { 906 | "text/plain": [ 907 | "tensor([[6.7356e+22, 5.4170e-05, 1.6785e-07],\n", 908 | " [2.0432e+20, 3.2915e-09, 2.1060e+23]])" 909 | ] 910 | }, 911 | "execution_count": 39, 912 | "metadata": {}, 913 | "output_type": "execute_result" 914 | } 915 | ], 916 | "source": [ 917 | "# empty：生成未初始化的指定形状矩阵\n", 918 | "torch.empty(2, 3)" 919 | ] 920 | }, 921 | { 922 | "cell_type": "code", 923 | "execution_count": 40, 924 | "metadata": {}, 925 | "outputs": [ 926 | { 927 | "data": { 928 | "text/plain": [ 929 | "tensor([[3.1400, 3.1400, 3.1400, 3.1400, 3.1400],\n", 930 | " [3.1400, 3.1400, 3.1400, 3.1400, 3.1400],\n", 931 | " [3.1400, 3.1400, 3.1400, 3.1400, 3.1400],\n", 932 | " [3.1400, 3.1400, 3.1400, 3.1400, 3.1400]])" 933 | ] 934 | }, 935 | "execution_count": 40, 936 | "metadata": {}, 937 | "output_type": "execute_result" 938 | } 939 | ], 940 | "source": [ 941 | "# full：根据指定形状，填充指定数值\n", 942 | "torch.full([4, 5], 3.14)" 943 | ] 944 | }, 945 | { 946 | "cell_type": "code", 947 | "execution_count": 41, 948 | "metadata": {}, 949 | "outputs": [ 950 | { 951 | "data": { 952 | "text/plain": [ 953 | "(tensor([1, 2]), tensor([5, 5]))" 954 | ] 955 | }, 956 | "execution_count": 41, 957 | "metadata": {}, 958 | "output_type": "execute_result" 959 | } 960 | ], 961 | "source": [ 962 | "## 2.指定形状的张量\n", 963 | "t1, torch.full_like(t1, 5)" 964 | ] 965 | }, 966 | { 967 | "cell_type": "code", 968 | "execution_count": 42, 969 | "metadata": {}, 970 | "outputs": [ 971 | { 972 | "data": { 973 | "text/plain": [ 974 | "(tensor([1.1000, 2.0000]),\n", 975 | " tensor([-0.7440, -0.2482]),\n", 976 | " tensor([[[1, 2, 2],\n", 977 | " [3, 4, 4]],\n", 978 | " \n", 979 | " [[5, 6, 6],\n", 980 | " [7, 8, 8]]], dtype=torch.int32),\n", 981 | " tensor([[[16, 27, 29],\n", 982 | " [67, 80, 41]],\n", 983 | " \n", 984 | " [[64, 74, 76],\n", 985 | " [15, 28, 83]]], dtype=torch.int32))" 986 | ] 987 | }, 988 | "execution_count": 42, 989 | "metadata": {}, 990 | "output_type": "execute_result" 991 | } 992 | ], 993 | "source": [ 994 | "# torch.randn_like(t1) # RuntimeError，_like类型转化需要注意转化前后数据类型一致的问题\n", 995 | "t5, torch.randn_like(t5), t10, torch.randint_like(t10, 1, 100)" 996 | ] 997 | }, 998 | { 999 | "cell_type": "code", 1000 | "execution_count": 43, 1001 | "metadata": {}, 1002 | "outputs": [ 1003 | { 1004 | "data": { 1005 | "text/plain": [ 1006 | "(tensor([1.1000, 2.0000]), tensor([0., 0.]), tensor([1., 1.]))" 1007 | ] 1008 | }, 1009 | "execution_count": 43, 1010 | "metadata": {}, 1011 | "output_type": "execute_result" 1012 | } 1013 | ], 1014 | "source": [ 1015 | "t5, torch.zeros_like(t5), torch.ones_like(t5)" 1016 | ] 1017 | }, 1018 | { 1019 | "cell_type": "markdown", 1020 | "metadata": {}, 1021 | "source": [ 1022 | "## 4.张量和其他相关类型之间的转化方法" 1023 | ] 1024 | }, 1025 | { 1026 | "cell_type": "code", 1027 | "execution_count": 44, 1028 | "metadata": {}, 1029 | "outputs": [ 1030 | { 1031 | "data": { 1032 | "text/plain": [ 1033 | "(tensor([1, 2]), array([1, 2], dtype=int64), array([1, 2], dtype=int64))" 1034 | ] 1035 | }, 1036 | "execution_count": 44, 1037 | "metadata": {}, 1038 | "output_type": "execute_result" 1039 | } 1040 | ], 1041 | "source": [ 1042 | "# .numpy方法和np.array函数：张量转化为数组\n", 1043 | "t1, t1.numpy(), np.array(t1)" 1044 | ] 1045 | }, 1046 | { 1047 | "cell_type": "code", 1048 | "execution_count": 45, 1049 | "metadata": {}, 1050 | "outputs": [ 1051 | { 1052 | "data": { 1053 | "text/plain": [ 1054 | "([1, 2], [tensor(1), tensor(2)])" 1055 | ] 1056 | }, 1057 | "execution_count": 45, 1058 | "metadata": {}, 1059 | "output_type": "execute_result" 1060 | } 1061 | ], 1062 | "source": [ 1063 | "# .tolist方法和list函数：张量转化为列表\n", 1064 | "t1.tolist(), list(t1)" 1065 | ] 1066 | }, 1067 | { 1068 | "cell_type": "code", 1069 | "execution_count": 46, 1070 | "metadata": {}, 1071 | "outputs": [ 1072 | { 1073 | "data": { 1074 | "text/plain": [ 1075 | "(tensor(1), 1)" 1076 | ] 1077 | }, 1078 | "execution_count": 46, 1079 | "metadata": {}, 1080 | "output_type": "execute_result" 1081 | } 1082 | ], 1083 | "source": [ 1084 | "# .item()方法：将0维张量转化为数值\n", 1085 | "t8, t8.item()" 1086 | ] 1087 | }, 1088 | { 1089 | "cell_type": "markdown", 1090 | "metadata": {}, 1091 | "source": [ 1092 | "## 5.张量的深拷贝" 1093 | ] 1094 | }, 1095 | { 1096 | "cell_type": "code", 1097 | "execution_count": 47, 1098 | "metadata": {}, 1099 | "outputs": [ 1100 | { 1101 | "data": { 1102 | "text/plain": [ 1103 | "(tensor(3),\n", 1104 | " tensor([ 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]),\n", 1105 | " tensor([ 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]))" 1106 | ] 1107 | }, 1108 | "execution_count": 47, 1109 | "metadata": {}, 1110 | "output_type": "execute_result" 1111 | } 1112 | ], 1113 | "source": [ 1114 | "t11 = torch.arange(1, 20, 2)\n", 1115 | "t12 = t11 # 浅拷贝，t12和t11二者指向相同的对象\n", 1116 | "t11[1], t11, t12" 1117 | ] 1118 | }, 1119 | { 1120 | "cell_type": "code", 1121 | "execution_count": 48, 1122 | "metadata": {}, 1123 | "outputs": [ 1124 | { 1125 | "data": { 1126 | "text/plain": [ 1127 | "(tensor([ 1, 30, 5, 7, 9, 11, 13, 15, 17, 19]),\n", 1128 | " tensor([ 1, 30, 5, 7, 9, 11, 13, 15, 17, 19]))" 1129 | ] 1130 | }, 1131 | "execution_count": 48, 1132 | "metadata": {}, 1133 | "output_type": "execute_result" 1134 | } 1135 | ], 1136 | "source": [ 1137 | "t11[1] = 30\n", 1138 | "t11, t12" 1139 | ] 1140 | }, 1141 | { 1142 | "cell_type": "code", 1143 | "execution_count": 49, 1144 | "metadata": {}, 1145 | "outputs": [ 1146 | { 1147 | "data": { 1148 | "text/plain": [ 1149 | "(tensor([ 1, 30, 50, 7, 9, 11, 13, 15, 17, 19]),\n", 1150 | " tensor([ 1, 30, 50, 7, 9, 11, 13, 15, 17, 19]))" 1151 | ] 1152 | }, 1153 | "execution_count": 49, 1154 | "metadata": {}, 1155 | "output_type": "execute_result" 1156 | } 1157 | ], 1158 | "source": [ 1159 | "t12[2] = 50\n", 1160 | "t11, t12" 1161 | ] 1162 | }, 1163 | { 1164 | "cell_type": "code", 1165 | "execution_count": 50, 1166 | "metadata": {}, 1167 | "outputs": [ 1168 | { 1169 | "data": { 1170 | "text/plain": [ 1171 | "(tensor(7), tensor([ 1, 30, 50, 7, 9, 11, 13, 15, 17, 19]))" 1172 | ] 1173 | }, 1174 | "execution_count": 50, 1175 | "metadata": {}, 1176 | "output_type": "execute_result" 1177 | } 1178 | ], 1179 | "source": [ 1180 | "t13 = t11.clone() # 深拷贝，t13指向的对象与t11不同，相当于创建了一个新的对象，只是数值相同\n", 1181 | "t13[3], t13" 1182 | ] 1183 | }, 1184 | { 1185 | "cell_type": "code", 1186 | "execution_count": 51, 1187 | "metadata": {}, 1188 | "outputs": [ 1189 | { 1190 | "data": { 1191 | "text/plain": [ 1192 | "(tensor([ 1, 30, 50, 70, 9, 11, 13, 15, 17, 19]),\n", 1193 | " tensor([ 1, 30, 50, 70, 9, 11, 13, 15, 17, 19]),\n", 1194 | " tensor([ 1, 30, 50, 7, 9, 11, 13, 15, 17, 19]))" 1195 | ] 1196 | }, 1197 | "execution_count": 51, 1198 | "metadata": {}, 1199 | "output_type": "execute_result" 1200 | } 1201 | ], 1202 | "source": [ 1203 | "t11[3] = 70\n", 1204 | "t11, t12, t13" 1205 | ] 1206 | } 1207 | ], 1208 | "metadata": { 1209 | "kernelspec": { 1210 | "display_name": "Python 3 (ipykernel)", 1211 | "language": "python", 1212 | "name": "python3" 1213 | }, 1214 | "language_info": { 1215 | "codemirror_mode": { 1216 | "name": "ipython", 1217 | "version": 3 1218 | }, 1219 | "file_extension": ".py", 1220 | "mimetype": "text/x-python", 1221 | "name": "python", 1222 | "nbconvert_exporter": "python", 1223 | "pygments_lexer": "ipython3", 1224 | "version": "3.9.13" 1225 | }, 1226 | "varInspector": { 1227 | "cols": { 1228 | "lenName": 16, 1229 | "lenType": 16, 1230 | "lenVar": 40 1231 | }, 1232 | "kernels_config": { 1233 | "python": { 1234 | "delete_cmd_postfix": "", 1235 | "delete_cmd_prefix": "del ", 1236 | "library": "var_list.py", 1237 | "varRefreshCmd": "print(var_dic_list())" 1238 | }, 1239 | "r": { 1240 | "delete_cmd_postfix": ") ", 1241 | "delete_cmd_prefix": "rm(", 1242 | "library": "var_list.r", 1243 | "varRefreshCmd": "cat(var_dic_list()) " 1244 | } 1245 | }, 1246 | "types_to_exclude": [ 1247 | "module", 1248 | "function", 1249 | "builtin_function_or_method", 1250 | "instance", 1251 | "_Feature" 1252 | ], 1253 | "window_display": false 1254 | } 1255 | }, 1256 | "nbformat": 4, 1257 | "nbformat_minor": 1 1258 | } 1259 | -------------------------------------------------------------------------------- /1_basic_of_deep_learning_with_pytorch/tensor_index_slice_merge.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "# -*- coding: utf-8 -*-\n", 12 | "\n", 13 | "'''\n", 14 | "@Author : Corley Tang\n", 15 | "@contact : cutercorleytd@gmail.com\n", 16 | "@Github : https://github.com/corleytd\n", 17 | "@Time : 2023-01-10 15:03\n", 18 | "@Project : Hands-on Deep Learning with PyTorch-tensor_index_slice_merge\n", 19 | "张量的索引、分片、合并以及维度调整\n", 20 | "'''\n", 21 | "\n", 22 | "# 导入所需的库\n", 23 | "import torch" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "source": [ 29 | "## 1.张量的符号索引" 30 | ], 31 | "metadata": { 32 | "collapsed": false 33 | } 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "outputs": [ 39 | { 40 | "data": { 41 | "text/plain": "tensor([ 0, 3, 6, 9, 12, 15, 18, 21, 24, 27])" 42 | }, 43 | "execution_count": 2, 44 | "metadata": {}, 45 | "output_type": "execute_result" 46 | } 47 | ], 48 | "source": [ 49 | "# 1.一维张量索引\n", 50 | "t1 = torch.arange(0, 30, 3)\n", 51 | "t1" 52 | ], 53 | "metadata": { 54 | "collapsed": false 55 | } 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": "(tensor(0), tensor(9))" 64 | }, 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "output_type": "execute_result" 68 | } 69 | ], 70 | "source": [ 71 | "t1[0], t1[3] # 张量索引的结果还是张量" 72 | ], 73 | "metadata": { 74 | "collapsed": false 75 | } 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 4, 80 | "outputs": [ 81 | { 82 | "data": { 83 | "text/plain": "tensor([ 6, 9, 12])" 84 | }, 85 | "execution_count": 4, 86 | "metadata": {}, 87 | "output_type": "execute_result" 88 | } 89 | ], 90 | "source": [ 91 | "# 切片\n", 92 | "t1[2:5] # 左包含右不包含" 93 | ], 94 | "metadata": { 95 | "collapsed": false 96 | } 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 5, 101 | "outputs": [ 102 | { 103 | "data": { 104 | "text/plain": "tensor([ 3, 9, 15, 21])" 105 | }, 106 | "execution_count": 5, 107 | "metadata": {}, 108 | "output_type": "execute_result" 109 | } 110 | ], 111 | "source": [ 112 | "# 切片，带索引间隔\n", 113 | "# t1[8:1:-1] # ValueError，在张量的索引中，step必须大于0\n", 114 | "t1[1:8:2]" 115 | ], 116 | "metadata": { 117 | "collapsed": false 118 | } 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 6, 123 | "outputs": [ 124 | { 125 | "data": { 126 | "text/plain": "(tensor([ 3, 9, 15, 21, 27]), tensor([ 0, 6, 12, 18]))" 127 | }, 128 | "execution_count": 6, 129 | "metadata": {}, 130 | "output_type": "execute_result" 131 | } 132 | ], 133 | "source": [ 134 | "t1[1::2], t1[:8:2]" 135 | ], 136 | "metadata": { 137 | "collapsed": false 138 | } 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "source": [], 143 | "metadata": { 144 | "collapsed": false 145 | } 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 7, 150 | "outputs": [ 151 | { 152 | "data": { 153 | "text/plain": "tensor([[ 1, 2, 3, 4, 5, 6],\n [ 7, 8, 9, 10, 11, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 23, 24]])" 154 | }, 155 | "execution_count": 7, 156 | "metadata": {}, 157 | "output_type": "execute_result" 158 | } 159 | ], 160 | "source": [ 161 | "# 2.二维张量索引\n", 162 | "t2 = torch.arange(1, 25).reshape(4, 6)\n", 163 | "t2" 164 | ], 165 | "metadata": { 166 | "collapsed": false 167 | } 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 8, 172 | "outputs": [ 173 | { 174 | "data": { 175 | "text/plain": "(tensor(9),\n tensor([ 7, 9, 11]),\n tensor([ 7, 9, 11]),\n tensor([ 2, 14]),\n tensor([[ 1, 3, 5],\n [13, 15, 17]]))" 176 | }, 177 | "execution_count": 8, 178 | "metadata": {}, 179 | "output_type": "execute_result" 180 | } 181 | ], 182 | "source": [ 183 | "t2[1, 2], t2[1, ::2], t2[1, [0, 2, 4]], t2[[0, 2], 1], t2[::2, ::2]" 184 | ], 185 | "metadata": { 186 | "collapsed": false 187 | } 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 9, 192 | "outputs": [ 193 | { 194 | "data": { 195 | "text/plain": "tensor([[[ 1, 2, 3, 4, 5, 6],\n [ 7, 8, 9, 10, 11, 12],\n [ 13, 14, 15, 16, 17, 18],\n [ 19, 20, 21, 22, 23, 24],\n [ 25, 26, 27, 28, 29, 30]],\n\n [[ 31, 32, 33, 34, 35, 36],\n [ 37, 38, 39, 40, 41, 42],\n [ 43, 44, 45, 46, 47, 48],\n [ 49, 50, 51, 52, 53, 54],\n [ 55, 56, 57, 58, 59, 60]],\n\n [[ 61, 62, 63, 64, 65, 66],\n [ 67, 68, 69, 70, 71, 72],\n [ 73, 74, 75, 76, 77, 78],\n [ 79, 80, 81, 82, 83, 84],\n [ 85, 86, 87, 88, 89, 90]],\n\n [[ 91, 92, 93, 94, 95, 96],\n [ 97, 98, 99, 100, 101, 102],\n [103, 104, 105, 106, 107, 108],\n [109, 110, 111, 112, 113, 114],\n [115, 116, 117, 118, 119, 120]]])" 196 | }, 197 | "execution_count": 9, 198 | "metadata": {}, 199 | "output_type": "execute_result" 200 | } 201 | ], 202 | "source": [ 203 | "# 3.三维张量的索引\n", 204 | "t3 = torch.arange(1, 121).reshape(4, 5, 6)\n", 205 | "t3" 206 | ], 207 | "metadata": { 208 | "collapsed": false 209 | } 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 10, 214 | "outputs": [ 215 | { 216 | "data": { 217 | "text/plain": "(tensor(83),\n tensor([[62, 64, 66],\n [74, 76, 78],\n [86, 88, 90]]),\n tensor([[[ 9, 11],\n [21, 23]],\n \n [[69, 71],\n [81, 83]]]))" 218 | }, 219 | "execution_count": 10, 220 | "metadata": {}, 221 | "output_type": "execute_result" 222 | } 223 | ], 224 | "source": [ 225 | "t3[2, 3, 4], t3[2, ::2, 1::2], t3[::2, 1::2, 2::2]" 226 | ], 227 | "metadata": { 228 | "collapsed": false 229 | } 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "source": [ 234 | "## 2.张量的函数索引" 235 | ], 236 | "metadata": { 237 | "collapsed": false 238 | } 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": 11, 243 | "outputs": [ 244 | { 245 | "data": { 246 | "text/plain": "(1, tensor([ 0, 3, 6, 9, 12, 15, 18, 21, 24, 27]))" 247 | }, 248 | "execution_count": 11, 249 | "metadata": {}, 250 | "output_type": "execute_result" 251 | } 252 | ], 253 | "source": [ 254 | "t1.ndim, t1" 255 | ], 256 | "metadata": { 257 | "collapsed": false 258 | } 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": 12, 263 | "outputs": [ 264 | { 265 | "data": { 266 | "text/plain": "tensor([3, 9])" 267 | }, 268 | "execution_count": 12, 269 | "metadata": {}, 270 | "output_type": "execute_result" 271 | } 272 | ], 273 | "source": [ 274 | "# 使用index_select函数进行索引\n", 275 | "indices = torch.tensor([1, 3])\n", 276 | "torch.index_select(t1, 0, indices) # 也可以用torch.index_select(t1, -1, indices)，等价于t1[[1, 2, 5]]" 277 | ], 278 | "metadata": { 279 | "collapsed": false 280 | } 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": 13, 285 | "outputs": [ 286 | { 287 | "data": { 288 | "text/plain": "(torch.Size([4, 6]),\n tensor([[ 1, 2, 3, 4, 5, 6],\n [ 7, 8, 9, 10, 11, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 23, 24]]))" 289 | }, 290 | "execution_count": 13, 291 | "metadata": {}, 292 | "output_type": "execute_result" 293 | } 294 | ], 295 | "source": [ 296 | "t2.shape, t2" 297 | ], 298 | "metadata": { 299 | "collapsed": false 300 | } 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": 14, 305 | "outputs": [ 306 | { 307 | "data": { 308 | "text/plain": "(tensor([[ 7, 8, 9, 10, 11, 12],\n [19, 20, 21, 22, 23, 24]]),\n tensor([[ 2, 4],\n [ 8, 10],\n [14, 16],\n [20, 22]]))" 309 | }, 310 | "execution_count": 14, 311 | "metadata": {}, 312 | "output_type": "execute_result" 313 | } 314 | ], 315 | "source": [ 316 | "torch.index_select(t2, 0, indices), torch.index_select(t2, 1, indices)" 317 | ], 318 | "metadata": { 319 | "collapsed": false 320 | } 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "source": [ 325 | "## 3.torch.view()方法\n", 326 | "PyTorch中的view()方法会返回一个类似视图的结果，该结果和原张量对象共享一块数据存储空间，并且通过view()方法，还可以改变对象结构，生成一个不同结构，但共享一个存储空间的张量。当然，共享一个存储空间，也就代表者是**浅拷贝**的关系，修改其中一个，另一个也会同步进行更改。视图的核心作用就是节省空间，张量的切分和合并得到的结果都是视图，而不是生成的新的对象。" 327 | ], 328 | "metadata": { 329 | "collapsed": false 330 | } 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": 15, 335 | "outputs": [ 336 | { 337 | "data": { 338 | "text/plain": "(tensor([[ 1, 2, 3, 4, 5, 6],\n [ 7, 8, 9, 10, 11, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 23, 24]]),\n tensor([[ 1, 2, 3, 4, 5, 6, 7, 8],\n [ 9, 10, 11, 12, 13, 14, 15, 16],\n [17, 18, 19, 20, 21, 22, 23, 24]]))" 339 | }, 340 | "execution_count": 15, 341 | "metadata": {}, 342 | "output_type": "execute_result" 343 | } 344 | ], 345 | "source": [ 346 | "t2_view_2 = t2.view(3, 8) # 构建一个数据相同、但形状不同的“视图”，两者指向同一个对象\n", 347 | "t2, t2_view_2" 348 | ], 349 | "metadata": { 350 | "collapsed": false 351 | } 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 16, 356 | "outputs": [ 357 | { 358 | "data": { 359 | "text/plain": "(tensor([[ 1, 2, 3, 4, 5, 6],\n [20, 8, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 23, 24]]),\n tensor([[ 1, 2, 3, 4, 5, 6, 20, 8],\n [20, 10, 20, 12, 13, 14, 15, 16],\n [17, 18, 19, 20, 21, 22, 23, 24]]))" 360 | }, 361 | "execution_count": 16, 362 | "metadata": {}, 363 | "output_type": "execute_result" 364 | } 365 | ], 366 | "source": [ 367 | "t2[1, ::2] = 20 # 两者会同步发生改变\n", 368 | "t2, t2_view_2" 369 | ], 370 | "metadata": { 371 | "collapsed": false 372 | } 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 17, 377 | "outputs": [ 378 | { 379 | "data": { 380 | "text/plain": "(tensor([[ 1, 2, 3, 4, 5, 6],\n [20, 8, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 23, 24]]),\n tensor([[[ 1, 2, 3, 4],\n [ 5, 6, 20, 8]],\n \n [[20, 10, 20, 12],\n [13, 14, 15, 16]],\n \n [[17, 18, 19, 20],\n [21, 22, 23, 24]]]))" 381 | }, 382 | "execution_count": 17, 383 | "metadata": {}, 384 | "output_type": "execute_result" 385 | } 386 | ], 387 | "source": [ 388 | "t2_view_3 = t2.view(3, 2, 4)\n", 389 | "t2, t2_view_3" 390 | ], 391 | "metadata": { 392 | "collapsed": false 393 | } 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": 18, 398 | "outputs": [ 399 | { 400 | "data": { 401 | "text/plain": "(tensor([[ 1, 2, 3, 4, 5, 6],\n [30, 30, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]]),\n tensor([[[ 1, 2, 3, 4],\n [ 5, 6, 30, 30]],\n \n [[20, 10, 20, 12],\n [13, 14, 15, 16]],\n \n [[17, 18, 19, 20],\n [21, 22, 30, 30]]]))" 402 | }, 403 | "execution_count": 18, 404 | "metadata": {}, 405 | "output_type": "execute_result" 406 | } 407 | ], 408 | "source": [ 409 | "t2_view_3[::2, 1, 2:] = 30\n", 410 | "t2, t2_view_3" 411 | ], 412 | "metadata": { 413 | "collapsed": false 414 | } 415 | }, 416 | { 417 | "cell_type": "markdown", 418 | "source": [ 419 | "## 4.张量的分片函数" 420 | ], 421 | "metadata": { 422 | "collapsed": false 423 | } 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": 19, 428 | "outputs": [ 429 | { 430 | "data": { 431 | "text/plain": "(tensor([[ 1, 2, 3, 4, 5, 6],\n [30, 30, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]]),\n (tensor([[1, 2, 3, 4, 5, 6]]),\n tensor([[30, 30, 20, 10, 20, 12]]),\n tensor([[13, 14, 15, 16, 17, 18]]),\n tensor([[19, 20, 21, 22, 30, 30]])),\n tensor([1, 2, 3, 4, 5, 6]))" 432 | }, 433 | "execution_count": 19, 434 | "metadata": {}, 435 | "output_type": "execute_result" 436 | } 437 | ], 438 | "source": [ 439 | "## 1.分块——chunk函数：按照某维度，对张量进行均匀切分，并且返回结果是原张量的视图，不改变维度\n", 440 | "t2_chunked = torch.chunk(t2, 4, dim=0)\n", 441 | "t2, t2_chunked, t2_chunked[0][0]" 442 | ], 443 | "metadata": { 444 | "collapsed": false 445 | } 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": 20, 450 | "outputs": [ 451 | { 452 | "data": { 453 | "text/plain": "(tensor([[10, 2, 10, 4, 10, 6],\n [30, 30, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]]),\n (tensor([[10, 2, 10, 4, 10, 6]]),\n tensor([[30, 30, 20, 10, 20, 12]]),\n tensor([[13, 14, 15, 16, 17, 18]]),\n tensor([[19, 20, 21, 22, 30, 30]])),\n tensor([10, 2, 10, 4, 10, 6]))" 454 | }, 455 | "execution_count": 20, 456 | "metadata": {}, 457 | "output_type": "execute_result" 458 | } 459 | ], 460 | "source": [ 461 | "t2_chunked[0][0][::2] = 10 # 两者会同步发生改变\n", 462 | "t2, t2_chunked, t2_chunked[0][0]" 463 | ], 464 | "metadata": { 465 | "collapsed": false 466 | } 467 | }, 468 | { 469 | "cell_type": "code", 470 | "execution_count": 21, 471 | "outputs": [ 472 | { 473 | "data": { 474 | "text/plain": "((tensor([[10, 2, 10, 4, 10, 6],\n [30, 30, 20, 10, 20, 12]]),\n tensor([[13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]])),\n (tensor([[10],\n [30],\n [13],\n [19]]),\n tensor([[ 2],\n [30],\n [14],\n [20]]),\n tensor([[10],\n [20],\n [15],\n [21]]),\n tensor([[ 4],\n [10],\n [16],\n [22]]),\n tensor([[10],\n [20],\n [17],\n [30]]),\n tensor([[ 6],\n [12],\n [18],\n [30]])))" 475 | }, 476 | "execution_count": 21, 477 | "metadata": {}, 478 | "output_type": "execute_result" 479 | } 480 | ], 481 | "source": [ 482 | "torch.chunk(t2, 3, dim=0), torch.chunk(t2, 7, dim=1) # 当原张量不能均分时，chunk不会报错，但会返回其他均分的结果，即次一级均分结果或者非等分结果" 483 | ], 484 | "metadata": { 485 | "collapsed": false 486 | } 487 | }, 488 | { 489 | "cell_type": "code", 490 | "execution_count": 22, 491 | "outputs": [ 492 | { 493 | "data": { 494 | "text/plain": "(tensor([[10, 2, 10, 4, 10, 6],\n [30, 30, 20, 10, 20, 12]]),\n tensor([[13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]]))" 495 | }, 496 | "execution_count": 22, 497 | "metadata": {}, 498 | "output_type": "execute_result" 499 | } 500 | ], 501 | "source": [ 502 | "# 2.拆分——split函数：split既能进行均分，也能进行自定义切分，返回结果也是view\n", 503 | "# 均分\n", 504 | "t2_splitted = torch.split(t2, 2, 0) # 第二个参数只输入一个数时表示均分\n", 505 | "t2_splitted" 506 | ], 507 | "metadata": { 508 | "collapsed": false 509 | } 510 | }, 511 | { 512 | "cell_type": "code", 513 | "execution_count": 23, 514 | "outputs": [ 515 | { 516 | "data": { 517 | "text/plain": "((tensor([[10, 2, 10, 4, 10, 6],\n [30, 30, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18]]),\n tensor([[19, 20, 21, 22, 30, 30]])),\n (tensor([[10, 2, 10, 4, 10, 6]]),\n tensor([[30, 30, 20, 10, 20, 12],\n [13, 14, 15, 16, 17, 18]]),\n tensor([[19, 20, 21, 22, 30, 30]])),\n (tensor([[10],\n [30],\n [13],\n [19]]),\n tensor([[ 2, 10],\n [30, 20],\n [14, 15],\n [20, 21]]),\n tensor([[ 4],\n [10],\n [16],\n [22]]),\n tensor([[10, 6],\n [20, 12],\n [17, 18],\n [30, 30]])))" 518 | }, 519 | "execution_count": 23, 520 | "metadata": {}, 521 | "output_type": "execute_result" 522 | } 523 | ], 524 | "source": [ 525 | "# torch.split(t2, [3, 2], 0) # RuntimeError，当第二个参数位输入一个序列时，序列的各数值之和必须等于对应维度下形状分量的取值\n", 526 | "torch.split(t2, [3, 1], 0), torch.split(t2, [1, 2, 1], 0), torch.split(t2, [1, 2, 1, 2], 1) # 第二个参数输入一个序列时表示按照序列数值进行切分" 527 | ], 528 | "metadata": { 529 | "collapsed": false 530 | } 531 | }, 532 | { 533 | "cell_type": "code", 534 | "execution_count": 24, 535 | "outputs": [ 536 | { 537 | "data": { 538 | "text/plain": "(tensor([[10, 2, 10, 4, 10, 6],\n [30, 30, 40, 10, 40, 12],\n [13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]]),\n (tensor([[10, 2, 10, 4, 10, 6],\n [30, 30, 40, 10, 40, 12]]),\n tensor([[13, 14, 15, 16, 17, 18],\n [19, 20, 21, 22, 30, 30]])))" 539 | }, 540 | "execution_count": 24, 541 | "metadata": {}, 542 | "output_type": "execute_result" 543 | } 544 | ], 545 | "source": [ 546 | "t2_splitted[0][1:, 2::2] = 40\n", 547 | "t2, t2_splitted # view进行修改，原对象同步修改" 548 | ], 549 | "metadata": { 550 | "collapsed": false 551 | } 552 | }, 553 | { 554 | "cell_type": "markdown", 555 | "source": [ 556 | "## 5.张量的合并操作\n", 557 | "张量的合并操作类似于列表的追加元素，可以拼接、也可以堆叠。" 558 | ], 559 | "metadata": { 560 | "collapsed": false 561 | } 562 | }, 563 | { 564 | "cell_type": "code", 565 | "execution_count": 25, 566 | "outputs": [ 567 | { 568 | "data": { 569 | "text/plain": "(tensor([[0., 0., 0.],\n [0., 0., 0.]]),\n tensor([[1., 1., 1.],\n [1., 1., 1.]]),\n tensor([[0., 0., 0.],\n [0., 0., 0.],\n [0., 0., 0.]]))" 570 | }, 571 | "execution_count": 25, 572 | "metadata": {}, 573 | "output_type": "execute_result" 574 | } 575 | ], 576 | "source": [ 577 | "# 1.拼接——cat函数：实现张量的拼接\n", 578 | "a = torch.zeros(2, 3)\n", 579 | "b = torch.ones(2, 3)\n", 580 | "c = torch.zeros(3, 3)\n", 581 | "a, b, c" 582 | ], 583 | "metadata": { 584 | "collapsed": false 585 | } 586 | }, 587 | { 588 | "cell_type": "code", 589 | "execution_count": 26, 590 | "outputs": [ 591 | { 592 | "data": { 593 | "text/plain": "tensor([[0., 0., 0.],\n [0., 0., 0.],\n [1., 1., 1.],\n [1., 1., 1.]])" 594 | }, 595 | "execution_count": 26, 596 | "metadata": {}, 597 | "output_type": "execute_result" 598 | } 599 | ], 600 | "source": [ 601 | "torch.cat([a, b]) # 按行进行拼接，dim参数默认为0" 602 | ], 603 | "metadata": { 604 | "collapsed": false 605 | } 606 | }, 607 | { 608 | "cell_type": "code", 609 | "execution_count": 27, 610 | "outputs": [ 611 | { 612 | "data": { 613 | "text/plain": "tensor([[0., 0., 0., 1., 1., 1.],\n [0., 0., 0., 1., 1., 1.]])" 614 | }, 615 | "execution_count": 27, 616 | "metadata": {}, 617 | "output_type": "execute_result" 618 | } 619 | ], 620 | "source": [ 621 | "# torch.cat([a, c], dim=1) # RuntimeError，对应维度形状不匹配无法进行拼接\n", 622 | "torch.cat([a, b], dim=1) # 按列进行拼接" 623 | ], 624 | "metadata": { 625 | "collapsed": false 626 | } 627 | }, 628 | { 629 | "cell_type": "code", 630 | "execution_count": 28, 631 | "outputs": [ 632 | { 633 | "data": { 634 | "text/plain": "(torch.Size([2, 3]), torch.Size([2, 3]), torch.Size([3, 3]))" 635 | }, 636 | "execution_count": 28, 637 | "metadata": {}, 638 | "output_type": "execute_result" 639 | } 640 | ], 641 | "source": [ 642 | "# 2.堆叠——stack函数：堆叠不是将元素拆分重装，而是简单地将各参与堆叠的对象分装到一个更高维度的张量\n", 643 | "a.shape, b.shape, c.shape" 644 | ], 645 | "metadata": { 646 | "collapsed": false 647 | } 648 | }, 649 | { 650 | "cell_type": "code", 651 | "execution_count": 29, 652 | "outputs": [ 653 | { 654 | "data": { 655 | "text/plain": "(torch.Size([2, 2, 3]),\n tensor([[[0., 0., 0.],\n [0., 0., 0.]],\n \n [[1., 1., 1.],\n [1., 1., 1.]]]))" 656 | }, 657 | "execution_count": 29, 658 | "metadata": {}, 659 | "output_type": "execute_result" 660 | } 661 | ], 662 | "source": [ 663 | "# torch.stack([a, c]) # RuntimeError，被堆叠的张量形状应该一致\n", 664 | "ab_stacked = torch.stack([a, b]) # 堆叠之后，生成1个三维张量\n", 665 | "ab_stacked.shape, ab_stacked" 666 | ], 667 | "metadata": { 668 | "collapsed": false 669 | } 670 | }, 671 | { 672 | "cell_type": "markdown", 673 | "source": [ 674 | "二者区别：\n", 675 | "- 拼接之后维度不变，堆叠之后维度升高\n", 676 | "- 拼接是把一个个元素单独提取出来之后再放到二维张量中，而堆叠则是直接将两个二维张量封装到一个三维张量中\n", 677 | "- 堆叠的要求更高，参与堆叠的张量必须形状完全相同\n", 678 | "## 6.张量维度变换\n", 679 | "通过reshape方法， 能够灵活调整张量的形状，而在实际操作张量进行计算时，往往需要另外进行降维和升维的操作：\n", 680 | "- 当我们需要除去不必要的维度时，可以使用squeeze函数\n", 681 | "- 需要手动升维时，则可采用unsqueeze函数" 682 | ], 683 | "metadata": { 684 | "collapsed": false 685 | } 686 | }, 687 | { 688 | "cell_type": "code", 689 | "execution_count": 30, 690 | "outputs": [ 691 | { 692 | "data": { 693 | "text/plain": "(4,\n torch.Size([1, 1, 3, 1]),\n tensor([[[[1.],\n [1.],\n [1.]]]]))" 694 | }, 695 | "execution_count": 30, 696 | "metadata": {}, 697 | "output_type": "execute_result" 698 | } 699 | ], 700 | "source": [ 701 | "t4 = torch.ones(1, 1, 3, 1) # 1个包含1个三维的四维张量，三维张量只包含1个三行一列的二维张量\n", 702 | "t4.ndim, t4.shape, t4" 703 | ], 704 | "metadata": { 705 | "collapsed": false 706 | } 707 | }, 708 | { 709 | "cell_type": "code", 710 | "execution_count": 31, 711 | "outputs": [ 712 | { 713 | "data": { 714 | "text/plain": "(torch.Size([3]), tensor([1., 1., 1.]))" 715 | }, 716 | "execution_count": 31, 717 | "metadata": {}, 718 | "output_type": "execute_result" 719 | } 720 | ], 721 | "source": [ 722 | "# 1.squeeze函数：删除不必要的维度\n", 723 | "t4_squeezed = torch.squeeze(t4) # 去除为1的维度，等价于t4.squeeze()\n", 724 | "t4_squeezed.shape, t4_squeezed" 725 | ], 726 | "metadata": { 727 | "collapsed": false 728 | } 729 | }, 730 | { 731 | "cell_type": "code", 732 | "execution_count": 32, 733 | "outputs": [ 734 | { 735 | "data": { 736 | "text/plain": "(6,\n torch.Size([1, 1, 3, 2, 1, 2]),\n tensor([[[[[[1., 1.]],\n \n [[1., 1.]]],\n \n \n [[[1., 1.]],\n \n [[1., 1.]]],\n \n \n [[[1., 1.]],\n \n [[1., 1.]]]]]]))" 737 | }, 738 | "execution_count": 32, 739 | "metadata": {}, 740 | "output_type": "execute_result" 741 | } 742 | ], 743 | "source": [ 744 | "t5 = torch.ones(1, 1, 3, 2, 1, 2)\n", 745 | "t5.ndim, t5.shape, t5" 746 | ], 747 | "metadata": { 748 | "collapsed": false 749 | } 750 | }, 751 | { 752 | "cell_type": "code", 753 | "execution_count": 33, 754 | "outputs": [ 755 | { 756 | "data": { 757 | "text/plain": "(torch.Size([3, 2, 2]),\n tensor([[[1., 1.],\n [1., 1.]],\n \n [[1., 1.],\n [1., 1.]],\n \n [[1., 1.],\n [1., 1.]]]))" 758 | }, 759 | "execution_count": 33, 760 | "metadata": {}, 761 | "output_type": "execute_result" 762 | } 763 | ], 764 | "source": [ 765 | "t5_squeezed = torch.squeeze(t5)\n", 766 | "t5_squeezed.shape, t5_squeezed" 767 | ], 768 | "metadata": { 769 | "collapsed": false 770 | } 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": 34, 775 | "outputs": [ 776 | { 777 | "data": { 778 | "text/plain": "(torch.Size([1, 2, 1, 3]),\n tensor([[[[5, 5, 5]],\n \n [[5, 5, 5]]]]))" 779 | }, 780 | "execution_count": 34, 781 | "metadata": {}, 782 | "output_type": "execute_result" 783 | } 784 | ], 785 | "source": [ 786 | "# 2.unsqueeze函数：手动升维\n", 787 | "t6 = torch.full((1, 2, 1, 3), 5)\n", 788 | "t6.shape, t6" 789 | ], 790 | "metadata": { 791 | "collapsed": false 792 | } 793 | }, 794 | { 795 | "cell_type": "code", 796 | "execution_count": 35, 797 | "outputs": [ 798 | { 799 | "data": { 800 | "text/plain": "(torch.Size([1, 1, 2, 1, 3]),\n tensor([[[[[5, 5, 5]],\n \n [[5, 5, 5]]]]]))" 801 | }, 802 | "execution_count": 35, 803 | "metadata": {}, 804 | "output_type": "execute_result" 805 | } 806 | ], 807 | "source": [ 808 | "t6_unsqueezed = torch.unsqueeze(t6, dim=0) # 在指定维度上升维，等价于t6.unsqueeze(0)\n", 809 | "t6_unsqueezed.shape, t6_unsqueezed" 810 | ], 811 | "metadata": { 812 | "collapsed": false 813 | } 814 | }, 815 | { 816 | "cell_type": "code", 817 | "execution_count": 36, 818 | "outputs": [ 819 | { 820 | "data": { 821 | "text/plain": "(torch.Size([1, 2, 1, 1, 3]), torch.Size([1, 2, 1, 3, 1]))" 822 | }, 823 | "execution_count": 36, 824 | "metadata": {}, 825 | "output_type": "execute_result" 826 | } 827 | ], 828 | "source": [ 829 | "torch.unsqueeze(t6, dim=2).shape, torch.unsqueeze(t6, dim=4).shape" 830 | ], 831 | "metadata": { 832 | "collapsed": false 833 | } 834 | } 835 | ], 836 | "metadata": { 837 | "kernelspec": { 838 | "display_name": "Python 3", 839 | "language": "python", 840 | "name": "python3" 841 | }, 842 | "language_info": { 843 | "codemirror_mode": { 844 | "name": "ipython", 845 | "version": 2 846 | }, 847 | "file_extension": ".py", 848 | "mimetype": "text/x-python", 849 | "name": "python", 850 | "nbconvert_exporter": "python", 851 | "pygments_lexer": "ipython2", 852 | "version": "2.7.6" 853 | } 854 | }, 855 | "nbformat": 4, 856 | "nbformat_minor": 0 857 | } 858 | -------------------------------------------------------------------------------- /1_basic_of_deep_learning_with_pytorch/tensor_linear_algebra.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "# -*- coding: utf-8 -*-\n", 12 | "\n", 13 | "'''\n", 14 | "@Author : Corley Tang\n", 15 | "@contact : cutercorleytd@gmail.com\n", 16 | "@Github : https://github.com/corleytd\n", 17 | "@Time : 2023-01-11 19:14\n", 18 | "@Project : Hands-on Deep Learning with PyTorch-tensor_linear_algebra\n", 19 | "张量的线性代数运算\n", 20 | "'''\n", 21 | "\n", 22 | "# 导入所需的库\n", 23 | "import warnings\n", 24 | "\n", 25 | "import torch\n", 26 | "from matplotlib import pyplot as plt\n", 27 | "\n", 28 | "warnings.filterwarnings('ignore', category=UserWarning)" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "source": [ 34 | "## 1.BLAS和LAPACK概览\n", 35 | "BLAS（Basic Linear Algeria Subprograms）和LAPACK（Linear Algeria Package）模块提供了完整的线性代数基本方法，函数种类较多，因此此处进行简单分类如下：\n", 36 | "- **矩阵的形变及特殊矩阵的构造方法**：包括矩阵的转置、对角矩阵的创建、单位矩阵的创建、上/下三角矩阵的创建等\n", 37 | "- **矩阵的基本运算**：包括矩阵乘法、向量内积、矩阵和向量的乘法等，当然，此处还包含了高维张量的基本运算，将着重探讨矩阵的基本运算拓展至三维张量中的基本方法\n", 38 | "- **矩阵的线性代数运算**：包括矩阵的迹、矩阵的秩、逆矩阵的求解、伴随矩阵和广义逆矩阵等\n", 39 | "- **矩阵分解运算**：特征分解和SVD分解（奇异值分解）等\n", 40 | "\n", 41 | "矩阵的两种理解方式：\n", 42 | "- 高维空间中的数据点的集合\n", 43 | "- 方程组的简写形式\n", 44 | "\n", 45 | "## 2.矩阵的形变及特殊矩阵构造方法\n", 46 | "矩阵的形变方法其实也就是二维张量的形变方法，在实际线性代数运算过程中，经常涉及一些特殊矩阵，如单位矩阵、对角矩阵等。" 47 | ], 48 | "metadata": { 49 | "collapsed": false 50 | } 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 2, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "text/plain": "(tensor([[83, 86, 80, 28],\n [91, 59, 75, 69],\n [55, 93, 35, 30]]),\n tensor([[83, 91, 55],\n [86, 59, 93],\n [80, 75, 35],\n [28, 69, 30]]),\n tensor([[83, 91, 55],\n [86, 59, 93],\n [80, 75, 35],\n [28, 69, 30]]))" 59 | }, 60 | "execution_count": 2, 61 | "metadata": {}, 62 | "output_type": "execute_result" 63 | } 64 | ], 65 | "source": [ 66 | "# 转置\n", 67 | "t1 = torch.randint(1, 100, size=(3, 4))\n", 68 | "t1, torch.t(t1), t1.t()" 69 | ], 70 | "metadata": { 71 | "collapsed": false 72 | } 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 3, 77 | "outputs": [ 78 | { 79 | "data": { 80 | "text/plain": "tensor([[1., 0., 0., 0., 0.],\n [0., 1., 0., 0., 0.],\n [0., 0., 1., 0., 0.],\n [0., 0., 0., 1., 0.],\n [0., 0., 0., 0., 1.]])" 81 | }, 82 | "execution_count": 3, 83 | "metadata": {}, 84 | "output_type": "execute_result" 85 | } 86 | ], 87 | "source": [ 88 | "# 单位矩阵\n", 89 | "torch.eye(5)" 90 | ], 91 | "metadata": { 92 | "collapsed": false 93 | } 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 4, 98 | "outputs": [ 99 | { 100 | "data": { 101 | "text/plain": "(tensor([[ 9, 0, 0, 0, 0],\n [ 0, 13, 0, 0, 0],\n [ 0, 0, 10, 0, 0],\n [ 0, 0, 0, 7, 0],\n [ 0, 0, 0, 0, 6]]),\n tensor([[ 9, 0, 0, 0, 0],\n [ 0, 13, 0, 0, 0],\n [ 0, 0, 10, 0, 0],\n [ 0, 0, 0, 7, 0],\n [ 0, 0, 0, 0, 6]]),\n tensor([[ 0, 0, 9, 0, 0, 0, 0],\n [ 0, 0, 0, 13, 0, 0, 0],\n [ 0, 0, 0, 0, 10, 0, 0],\n [ 0, 0, 0, 0, 0, 7, 0],\n [ 0, 0, 0, 0, 0, 0, 6],\n [ 0, 0, 0, 0, 0, 0, 0],\n [ 0, 0, 0, 0, 0, 0, 0]]),\n tensor([[ 0, 0, 0, 0, 0, 0, 0],\n [ 0, 0, 0, 0, 0, 0, 0],\n [ 9, 0, 0, 0, 0, 0, 0],\n [ 0, 13, 0, 0, 0, 0, 0],\n [ 0, 0, 10, 0, 0, 0, 0],\n [ 0, 0, 0, 7, 0, 0, 0],\n [ 0, 0, 0, 0, 6, 0, 0]]))" 102 | }, 103 | "execution_count": 4, 104 | "metadata": {}, 105 | "output_type": "execute_result" 106 | } 107 | ], 108 | "source": [ 109 | "# 对角矩阵，对角矩阵上移，对角矩阵下移\n", 110 | "t2 = torch.randint(1, 20, size=(5,))\n", 111 | "torch.diag(t2), t2.diag(), torch.diag(t2, diagonal=2), torch.diag(t2, -2)" 112 | ], 113 | "metadata": { 114 | "collapsed": false 115 | } 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 5, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "text/plain": "(tensor([[21, 8, 20, 7],\n [10, 26, 27, 6],\n [28, 27, 17, 1],\n [12, 5, 19, 1]]),\n tensor([[21, 8, 20, 7],\n [ 0, 26, 27, 6],\n [ 0, 0, 17, 1],\n [ 0, 0, 0, 1]]),\n tensor([[21, 8, 20, 7],\n [10, 26, 27, 6],\n [ 0, 27, 17, 1],\n [ 0, 0, 19, 1]]),\n tensor([[ 0, 8, 20, 7],\n [ 0, 0, 27, 6],\n [ 0, 0, 0, 1],\n [ 0, 0, 0, 0]]))" 124 | }, 125 | "execution_count": 5, 126 | "metadata": {}, 127 | "output_type": "execute_result" 128 | } 129 | ], 130 | "source": [ 131 | "# 上三角矩阵，上三角矩阵左下偏移，上三角矩阵右上偏移\n", 132 | "t3 = torch.randint(1, 30, size=(4, 4))\n", 133 | "t3, torch.triu(t3), t3.triu(-1), t3.triu(1)" 134 | ], 135 | "metadata": { 136 | "collapsed": false 137 | } 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 6, 142 | "outputs": [ 143 | { 144 | "data": { 145 | "text/plain": "(tensor([[21, 8, 20, 7],\n [10, 26, 27, 6],\n [28, 27, 17, 1],\n [12, 5, 19, 1]]),\n tensor([[21, 0, 0, 0],\n [10, 26, 0, 0],\n [28, 27, 17, 0],\n [12, 5, 19, 1]]),\n tensor([[ 0, 0, 0, 0],\n [10, 0, 0, 0],\n [28, 27, 0, 0],\n [12, 5, 19, 0]]),\n tensor([[21, 8, 0, 0],\n [10, 26, 27, 0],\n [28, 27, 17, 1],\n [12, 5, 19, 1]]))" 146 | }, 147 | "execution_count": 6, 148 | "metadata": {}, 149 | "output_type": "execute_result" 150 | } 151 | ], 152 | "source": [ 153 | "# 下三角矩阵，下三角矩阵左下偏移，下三角矩阵右上偏移\n", 154 | "t3, torch.tril(t3), torch.tril(t3, -1), t3.tril(1)" 155 | ], 156 | "metadata": { 157 | "collapsed": false 158 | } 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "source": [ 163 | "## 3.矩阵的基本运算\n", 164 | "矩阵不同于普通的二维数组，其具备一定的线性代数含义，而这些特殊的性质，其实就主要体现在矩阵的基本运算上。" 165 | ], 166 | "metadata": { 167 | "collapsed": false 168 | } 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 7, 173 | "outputs": [ 174 | { 175 | "data": { 176 | "text/plain": "(tensor(435), tensor(435))" 177 | }, 178 | "execution_count": 7, 179 | "metadata": {}, 180 | "output_type": "execute_result" 181 | } 182 | ], 183 | "source": [ 184 | "# 1.dot、vdot——点积计算：只能作用于一维张量，且对于数值型对象，二者计算结果并没有区别，两种函数只在进行复数运算时会有区别\n", 185 | "# torch.dot(t1, t1) # RuntimeError，只支持一维张量\n", 186 | "torch.dot(t2, t2), t2.vdot(t2)" 187 | ], 188 | "metadata": { 189 | "collapsed": false 190 | } 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 8, 195 | "outputs": [ 196 | { 197 | "data": { 198 | "text/plain": "(torch.Size([3, 4]), torch.Size([4, 4]))" 199 | }, 200 | "execution_count": 8, 201 | "metadata": {}, 202 | "output_type": "execute_result" 203 | } 204 | ], 205 | "source": [ 206 | "# 2.mm——矩阵乘法\n", 207 | "t1.shape, t3.shape" 208 | ], 209 | "metadata": { 210 | "collapsed": false 211 | } 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 9, 216 | "outputs": [ 217 | { 218 | "data": { 219 | "text/plain": "tensor([[5179, 5200, 5874, 1205],\n [5429, 4632, 5999, 1135],\n [3425, 3953, 4776, 1008]])" 220 | }, 221 | "execution_count": 9, 222 | "metadata": {}, 223 | "output_type": "execute_result" 224 | } 225 | ], 226 | "source": [ 227 | "# 矩阵乘法\n", 228 | "torch.mm(t1, t3)" 229 | ], 230 | "metadata": { 231 | "collapsed": false 232 | } 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 10, 237 | "outputs": [ 238 | { 239 | "data": { 240 | "text/plain": "tensor([[6889, 7396, 6400, 784],\n [8281, 3481, 5625, 4761],\n [3025, 8649, 1225, 900]])" 241 | }, 242 | "execution_count": 10, 243 | "metadata": {}, 244 | "output_type": "execute_result" 245 | } 246 | ], 247 | "source": [ 248 | "# 对应位置相乘\n", 249 | "t1 * t1" 250 | ], 251 | "metadata": { 252 | "collapsed": false 253 | } 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 11, 258 | "outputs": [ 259 | { 260 | "data": { 261 | "text/plain": "(tensor([3148, 3277, 2572]),\n tensor([[3148],\n [3277],\n [2572]]),\n tensor([3148, 3277, 2572]))" 262 | }, 263 | "execution_count": 11, 264 | "metadata": {}, 265 | "output_type": "execute_result" 266 | } 267 | ], 268 | "source": [ 269 | "# 3.mv——矩阵和向量相乘：可以看作先将向量转化为列向量再相乘，需要矩阵的列数和向量的元素个数保持相同，应用场景较多，例如线性方程组求解\n", 270 | "t4 = torch.randint(1, 20, size=(4,))\n", 271 | "torch.mv(t1, t4), torch.mm(t1, t4.reshape(-1, 1)), torch.mm(t1, t4.reshape(-1, 1)).flatten()" 272 | ], 273 | "metadata": { 274 | "collapsed": false 275 | } 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 12, 280 | "outputs": [ 281 | { 282 | "data": { 283 | "text/plain": "(torch.Size([3, 2, 2]), torch.Size([3, 2, 3]))" 284 | }, 285 | "execution_count": 12, 286 | "metadata": {}, 287 | "output_type": "execute_result" 288 | } 289 | ], 290 | "source": [ 291 | "# 4.bmm——批量矩阵相乘：三维张量的矩阵相乘，是三维张量内部各对应位置的矩阵相乘，需要两个矩阵第一维大小相同、二三维满足矩阵乘法的条件\n", 292 | "t5 = torch.randint(1, 20, size=(3, 2, 2))\n", 293 | "t6 = torch.randint(1, 20, size=(3, 2, 3))\n", 294 | "t5.shape, t6.shape" 295 | ], 296 | "metadata": { 297 | "collapsed": false 298 | } 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 13, 303 | "outputs": [ 304 | { 305 | "data": { 306 | "text/plain": "(torch.Size([3, 2, 3]),\n tensor([[[334, 194, 394],\n [340, 188, 292]],\n \n [[163, 157, 74],\n [178, 166, 76]],\n \n [[290, 262, 296],\n [375, 295, 360]]]))" 307 | }, 308 | "execution_count": 13, 309 | "metadata": {}, 310 | "output_type": "execute_result" 311 | } 312 | ], 313 | "source": [ 314 | "t56_bnn = torch.bmm(t5, t6)\n", 315 | "t56_bnn.shape, t56_bnn" 316 | ], 317 | "metadata": { 318 | "collapsed": false 319 | } 320 | }, 321 | { 322 | "cell_type": "code", 323 | "execution_count": 14, 324 | "outputs": [ 325 | { 326 | "data": { 327 | "text/plain": "(torch.Size([4]), torch.Size([3, 4]), torch.Size([4, 4]))" 328 | }, 329 | "execution_count": 14, 330 | "metadata": {}, 331 | "output_type": "execute_result" 332 | } 333 | ], 334 | "source": [ 335 | "# 5.addmm——矩阵相乘再相加\n", 336 | "t4.shape, t1.shape, t3.shape" 337 | ], 338 | "metadata": { 339 | "collapsed": false 340 | } 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": 15, 345 | "outputs": [ 346 | { 347 | "data": { 348 | "text/plain": "(tensor([[5191, 5214, 5882, 1216],\n [5441, 4646, 6007, 1146],\n [3437, 3967, 4784, 1019]]),\n tensor([[5191, 5214, 5882, 1216],\n [5441, 4646, 6007, 1146],\n [3437, 3967, 4784, 1019]]),\n tensor([[15561, 15628, 17638, 3637],\n [16311, 13924, 18013, 3427],\n [10299, 11887, 14344, 3046]]))" 349 | }, 350 | "execution_count": 15, 351 | "metadata": {}, 352 | "output_type": "execute_result" 353 | } 354 | ], 355 | "source": [ 356 | "torch.addmm(t4, t1, t3), t4 + torch.mm(t1, t3), torch.addmm(t4, t1, t3, beta=2, alpha=3)" 357 | ], 358 | "metadata": { 359 | "collapsed": false 360 | } 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 16, 365 | "outputs": [ 366 | { 367 | "data": { 368 | "text/plain": "(tensor([[793, 619, 776],\n [894, 668, 733]]),\n tensor([[793, 619, 776],\n [894, 668, 733]]))" 369 | }, 370 | "execution_count": 16, 371 | "metadata": {}, 372 | "output_type": "execute_result" 373 | } 374 | ], 375 | "source": [ 376 | "# 6.addbmm——批量矩阵相乘再相加，会对批量相乘后的三维张量第一个维度求和\n", 377 | "t7 = torch.randint(1, 20, size=(2, 3))\n", 378 | "torch.addbmm(t7, t5, t6), t7 + torch.bmm(t5, t6).sum(0)" 379 | ], 380 | "metadata": { 381 | "collapsed": false 382 | } 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "source": [ 387 | "# 4.矩阵的线性代数运算" 388 | ], 389 | "metadata": { 390 | "collapsed": false 391 | } 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": 17, 396 | "outputs": [ 397 | { 398 | "data": { 399 | "text/plain": "(tensor(65), tensor(65), tensor(177))" 400 | }, 401 | "execution_count": 17, 402 | "metadata": {}, 403 | "output_type": "execute_result" 404 | } 405 | ], 406 | "source": [ 407 | "# 1.trace——矩阵的迹：矩阵对角线元素之和，并不一定要求是方阵\n", 408 | "torch.trace(t3), t3.trace(), t1.trace()" 409 | ], 410 | "metadata": { 411 | "collapsed": false 412 | } 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 18, 417 | "outputs": [ 418 | { 419 | "data": { 420 | "text/plain": "(tensor(2), tensor(1))" 421 | }, 422 | "execution_count": 18, 423 | "metadata": {}, 424 | "output_type": "execute_result" 425 | } 426 | ], 427 | "source": [ 428 | "# 2.rank——矩阵的秩：矩阵中行或列的极大线性无关数，且矩阵中行、列极大无关数总是相同的，任何矩阵的秩都是唯一值，满秩指的是方阵（行数和列数相同的矩阵）中行数、列数和秩相同，满秩矩阵有线性唯一解等重要特性，而其他矩阵也能通过求解秩来降维\n", 429 | "t8 = torch.tensor([[1., 3], [2, 6]])\n", 430 | "torch.matrix_rank(t7.to(torch.float)), torch.matrix_rank(t8)" 431 | ], 432 | "metadata": { 433 | "collapsed": false 434 | } 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": 19, 439 | "outputs": [ 440 | { 441 | "data": { 442 | "text/plain": "(tensor(66746.), tensor(-0.))" 443 | }, 444 | "execution_count": 19, 445 | "metadata": {}, 446 | "output_type": "execute_result" 447 | } 448 | ], 449 | "source": [ 450 | "# 3.det——矩阵的行列式：矩阵的一个基本性质或者属性，通过行列式的计算，可以知道矩阵是否可逆，从而进一步求解矩阵所对应的线性方程，是矩阵进行线性变换的伸缩因子\n", 451 | "t3 = t3.float()\n", 452 | "torch.det(t3), t8.det()" 453 | ], 454 | "metadata": { 455 | "collapsed": false 456 | } 457 | }, 458 | { 459 | "cell_type": "code", 460 | "execution_count": 20, 461 | "outputs": [ 462 | { 463 | "data": { 464 | "text/plain": "
", 465 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgJUlEQVR4nO3de3BU9f3/8ddCcIOYXTVDko1ECN6AIBeTKgGpreGicVI7Q+sNhaIyRlEqmSgGO0Wn1mjVDlJtEApWGgVHA51Y7lYSlAmVYLAoEWmJJMTETLDdDViW2/n94Zf8XE0gm4S8yfp8zJw/9uRzcj57BjhP9pzddTmO4wgAAMBID+sJAACA7zdiBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmIqynkBbHD9+XJ9//rliYmLkcrmspwMAANrAcRw1NTUpMTFRPXq0/vpHt4iRzz//XElJSdbTAAAA7VBTU6N+/fq1+vNuESMxMTGSvn4yHo/HeDYAAKAtAoGAkpKSms/jrekWMXLi0ozH4yFGAADoZk51iwU3sAIAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMNUtPvQMAAB0vmPHHb1f9aUamg4pLiZaVyafr549uv474MJ+ZaS2tla33367YmNjdfbZZ2vEiBHatm3bSbcpLS1VamqqoqOjNXDgQC1YsKDdEwYAAB239qM6Xf30O7p10Rb9cvl23bpoi65++h2t/aiuy+cSVoz85z//0ZgxY9SrVy+tWbNGO3fu1HPPPadzzz231W2qqqqUmZmpsWPHqqKiQnPmzNHMmTNVVFTU0bkDAIB2WPtRne4t/EB1/kMh6+v9h3Rv4QddHiQux3Gctg5+5JFHtHnzZr377rtt3sHs2bNVXFysysrK5nXZ2dn68MMPVVZW1qbfEQgE5PV65ff7+W4aAAA64NhxR1c//c53QuQEl6QEb7Tem31thy/ZtPX8HdYrI8XFxUpLS9PPf/5zxcXFaeTIkVq0aNFJtykrK9OECRNC1k2cOFHl5eU6cuRIi9sEg0EFAoGQBQAAdNz7VV+2GiKS5Eiq8x/S+1VfdtmcwoqRPXv2qKCgQJdcconWrVun7OxszZw5U0uXLm11m/r6esXHx4esi4+P19GjR9XY2NjiNvn5+fJ6vc1LUlJSONMEAACtaGhqPUTaM64zhBUjx48f1xVXXKEnn3xSI0eO1D333KPp06eroKDgpNt9+6uDT1wZau0rhfPy8uT3+5uXmpqacKYJAABaERcT3anjOkNYMeLz+TRkyJCQdYMHD1Z1dXWr2yQkJKi+vj5kXUNDg6KiohQbG9viNm63Wx6PJ2QBAAAdd2Xy+fJ5o9Xa3SAuST7v12/z7SphxciYMWO0a9eukHWffvqp+vfv3+o26enp2rBhQ8i69evXKy0tTb169Qpn9wAAoIN69nBpbtbXLyx8O0hOPJ6bNaRLP28krBiZNWuWtmzZoieffFL/+te/9Nprr2nhwoWaMWNG85i8vDxNmTKl+XF2drb27t2rnJwcVVZWasmSJVq8eLFyc3M771kAAIA2u26oTwW3X6EEb+ilmARvtApuv0LXDfV16XzCemuvJP3tb39TXl6edu/ereTkZOXk5Gj69OnNP//FL36hzz77TCUlJc3rSktLNWvWLH388cdKTEzU7NmzlZ2d3eZ98tZeAAA63+n+BNa2nr/DjhELxAgAAN3PafmcEQAAgM5GjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMBVWjDz22GNyuVwhS0JCQqvjS0pKvjPe5XLpk08+6fDEAQBAZIgKd4OUlBS9/fbbzY979ux5ym127dolj8fT/Lhv377h7hYAAESosGMkKirqpK+GtCQuLk7nnntuuLsCAADfA2HfM7J7924lJiYqOTlZt9xyi/bs2XPKbUaOHCmfz6eMjAxt3LjxlOODwaACgUDIAgAAIlNYMXLVVVdp6dKlWrdunRYtWqT6+nqNHj1a+/fvb3G8z+fTwoULVVRUpBUrVuiyyy5TRkaGNm3adNL95Ofny+v1Ni9JSUnhTBMAAHQjLsdxnPZufPDgQV100UV6+OGHlZOT06ZtsrKy5HK5VFxc3OqYYDCoYDDY/DgQCCgpKUl+vz/k3hMAAHDmCgQC8nq9pzx/d+itvX369NHll1+u3bt3t3mbUaNGnXK82+2Wx+MJWQAAQGTqUIwEg0FVVlbK5/O1eZuKioqwxgMAgMgW1rtpcnNzlZWVpQsvvFANDQ164oknFAgENHXqVElSXl6eamtrtXTpUknSvHnzNGDAAKWkpOjw4cMqLCxUUVGRioqKOv+ZAACAbimsGNm3b59uvfVWNTY2qm/fvho1apS2bNmi/v37S5Lq6upUXV3dPP7w4cPKzc1VbW2tevfurZSUFK1atUqZmZmd+ywAAEC31aEbWLtKW2+AAQAAZ44uuYEVAACgo4gRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmwoqRxx57TC6XK2RJSEg46TalpaVKTU1VdHS0Bg4cqAULFnRowgAAILJEhbtBSkqK3n777ebHPXv2bHVsVVWVMjMzNX36dBUWFmrz5s2677771LdvX02aNKl9MwYAABEl7BiJioo65ashJyxYsEAXXnih5s2bJ0kaPHiwysvL9eyzzxIjAABAUjvuGdm9e7cSExOVnJysW265RXv27Gl1bFlZmSZMmBCybuLEiSovL9eRI0da3S4YDCoQCIQsAAAgMoUVI1dddZWWLl2qdevWadGiRaqvr9fo0aO1f//+FsfX19crPj4+ZF18fLyOHj2qxsbGVveTn58vr9fbvCQlJYUzTQAA0I2EFSPXX3+9Jk2apMsvv1zjxo3TqlWrJEmvvPJKq9u4XK6Qx47jtLj+m/Ly8uT3+5uXmpqacKYJAAC6kbDvGfmmPn366PLLL9fu3btb/HlCQoLq6+tD1jU0NCgqKkqxsbGt/l632y23292RqQEAgG6iQ58zEgwGVVlZKZ/P1+LP09PTtWHDhpB169evV1pamnr16tWRXQMAgAgRVozk5uaqtLRUVVVV+sc//qGf/exnCgQCmjp1qqSvL69MmTKleXx2drb27t2rnJwcVVZWasmSJVq8eLFyc3M791kAAIBuK6zLNPv27dOtt96qxsZG9e3bV6NGjdKWLVvUv39/SVJdXZ2qq6ubxycnJ2v16tWaNWuWXnzxRSUmJmr+/Pm8rRcAADRzOSfuKD2DBQIBeb1e+f1+eTwe6+kAAIA2aOv5m++mAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmOhQj+fn5crlcevDBB1sdU1JSIpfL9Z3lk08+6ciuAQBAhIhq74Zbt27VwoULNWzYsDaN37VrlzweT/Pjvn37tnfXAAAggrTrlZEDBw5o8uTJWrRokc4777w2bRMXF6eEhITmpWfPnu3ZNQAAiDDtipEZM2bohhtu0Lhx49q8zciRI+Xz+ZSRkaGNGzeedGwwGFQgEAhZAABAZAr7Ms3y5cv1wQcfaOvWrW0a7/P5tHDhQqWmpioYDOovf/mLMjIyVFJSoh/+8IctbpOfn6/HH3883KkBAIBuyOU4jtPWwTU1NUpLS9P69es1fPhwSdKPfvQjjRgxQvPmzWvzTrOysuRyuVRcXNziz4PBoILBYPPjQCCgpKQk+f3+kPtOAADAmSsQCMjr9Z7y/B3WZZpt27apoaFBqampioqKUlRUlEpLSzV//nxFRUXp2LFjbfo9o0aN0u7du1v9udvtlsfjCVkAAEBkCusyTUZGhnbs2BGybtq0aRo0aJBmz57d5ptSKyoq5PP5wtk1AACIUGHFSExMjIYOHRqyrk+fPoqNjW1en5eXp9raWi1dulSSNG/ePA0YMEApKSk6fPiwCgsLVVRUpKKiok56CgAAoDtr9+eMtKaurk7V1dXNjw8fPqzc3FzV1taqd+/eSklJ0apVq5SZmdnZuwYAAN1QWDewWmnrDTAAAODMcVpuYAUAAOhsxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFMdipH8/Hy5XC49+OCDJx1XWlqq1NRURUdHa+DAgVqwYEFHdgsAACJIu2Nk69atWrhwoYYNG3bScVVVVcrMzNTYsWNVUVGhOXPmaObMmSoqKmrvrgEAQARpV4wcOHBAkydP1qJFi3TeeeeddOyCBQt04YUXat68eRo8eLDuvvtu3XnnnXr22WfbNWEAABBZ2hUjM2bM0A033KBx48adcmxZWZkmTJgQsm7ixIkqLy/XkSNHWtwmGAwqEAiELAAAIDKFHSPLly/XBx98oPz8/DaNr6+vV3x8fMi6+Ph4HT16VI2NjS1uk5+fL6/X27wkJSWFO00AANBNhBUjNTU1+uUvf6nCwkJFR0e3eTuXyxXy2HGcFtefkJeXJ7/f37zU1NSEM00AANCNRIUzeNu2bWpoaFBqamrzumPHjmnTpk164YUXFAwG1bNnz5BtEhISVF9fH7KuoaFBUVFRio2NbXE/brdbbrc7nKkBAIBuKqwYycjI0I4dO0LWTZs2TYMGDdLs2bO/EyKSlJ6errfeeitk3fr165WWlqZevXq1Y8oAACCShBUjMTExGjp0aMi6Pn36KDY2tnl9Xl6eamtrtXTpUklSdna2XnjhBeXk5Gj69OkqKyvT4sWLtWzZsk56CgAAoDvr9E9graurU3V1dfPj5ORkrV69WiUlJRoxYoR+85vfaP78+Zo0aVJn7xoAAHRDLufE3aRnsEAgIK/XK7/fL4/HYz0dAADQBm09f/PdNAAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAVFgxUlBQoGHDhsnj8cjj8Sg9PV1r1qxpdXxJSYlcLtd3lk8++aTDEwcAAJEhKpzB/fr101NPPaWLL75YkvTKK6/oxhtvVEVFhVJSUlrdbteuXfJ4PM2P+/bt287pAgCASBNWjGRlZYU8/u1vf6uCggJt2bLlpDESFxenc889t10TBAAAka3d94wcO3ZMy5cv18GDB5Wenn7SsSNHjpTP51NGRoY2btx4yt8dDAYVCARCFgAAEJnCjpEdO3bonHPOkdvtVnZ2tlauXKkhQ4a0ONbn82nhwoUqKirSihUrdNlllykjI0ObNm066T7y8/Pl9Xqbl6SkpHCnCQAAugmX4zhOOBscPnxY1dXV+u9//6uioiL96U9/UmlpaatB8m1ZWVlyuVwqLi5udUwwGFQwGGx+HAgElJSUJL/fH3LvCQAAOHMFAgF5vd5Tnr/DumdEks4666zmG1jT0tK0detWPf/883rppZfatP2oUaNUWFh40jFut1tutzvcqQEAgG6ow58z4jhOyKsYp1JRUSGfz9fR3QIAgAgR1isjc+bM0fXXX6+kpCQ1NTVp+fLlKikp0dq1ayVJeXl5qq2t1dKlSyVJ8+bN04ABA5SSkqLDhw+rsLBQRUVFKioq6vxnAgAAuqWwYuSLL77QHXfcobq6Onm9Xg0bNkxr167V+PHjJUl1dXWqrq5uHn/48GHl5uaqtrZWvXv3VkpKilatWqXMzMzOfRYAAKDbCvsGVgttvQEGAACcOdp6/ua7aQAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmIqynoCVY8cdvV/1pRqaDikuJlpXJp+vnj1c1tMCAOB7J6xXRgoKCjRs2DB5PB55PB6lp6drzZo1J92mtLRUqampio6O1sCBA7VgwYIOTbgzrP2oTlc//Y5uXbRFv1y+Xbcu2qKrn35Haz+qs54aAADfO2HFSL9+/fTUU0+pvLxc5eXluvbaa3XjjTfq448/bnF8VVWVMjMzNXbsWFVUVGjOnDmaOXOmioqKOmXy7bH2ozrdW/iB6vyHQtbX+w/p3sIPCBIAALqYy3EcpyO/4Pzzz9czzzyju+666zs/mz17toqLi1VZWdm8Ljs7Wx9++KHKysravI9AICCv1yu/3y+Px9PuuR477ujqp9/5Toic4JKU4I3We7Ov5ZINAAAd1Nbzd7tvYD127JiWL1+ugwcPKj09vcUxZWVlmjBhQsi6iRMnqry8XEeOHGn1dweDQQUCgZClM7xf9WWrISJJjqQ6/yG9X/Vlp+wPAACcWtgxsmPHDp1zzjlyu93Kzs7WypUrNWTIkBbH1tfXKz4+PmRdfHy8jh49qsbGxlb3kZ+fL6/X27wkJSWFO80WNTS1HiLtGQcAADou7Bi57LLLtH37dm3ZskX33nuvpk6dqp07d7Y63uUKvdxx4qrQt9d/U15envx+f/NSU1MT7jRbFBcT3anjAABAx4X91t6zzjpLF198sSQpLS1NW7du1fPPP6+XXnrpO2MTEhJUX18fsq6hoUFRUVGKjY1tdR9ut1tutzvcqZ3Slcnny+eNVr3/kFq6UebEPSNXJp/f6fsGAAAt6/CHnjmOo2Aw2OLP0tPTtWHDhpB169evV1pamnr16tXRXYetZw+X5mZ9fUnp26/LnHg8N2sIN68CANCFwoqROXPm6N1339Vnn32mHTt26NFHH1VJSYkmT54s6evLK1OmTGken52drb179yonJ0eVlZVasmSJFi9erNzc3M59FmG4bqhPBbdfoQRv6KWYBG+0Cm6/QtcN9RnNDACA76ewLtN88cUXuuOOO1RXVyev16thw4Zp7dq1Gj9+vCSprq5O1dXVzeOTk5O1evVqzZo1Sy+++KISExM1f/58TZo0qXOfRZiuG+rT+CEJfAIrAABngA5/zkhX6KzPGQEAAF3ntH/OCAAAQGcgRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmAr7W3stnPiQ2EAgYDwTAADQVifO26f6sPduESNNTU2SpKSkJOOZAACAcDU1Ncnr9bb6827x3TTHjx/X559/rpiYGLlcnfdldoFAQElJSaqpqeE7b04zjnXX4Dh3DY5z1+A4d43TeZwdx1FTU5MSExPVo0frd4Z0i1dGevTooX79+p223+/xePiD3kU41l2D49w1OM5dg+PcNU7XcT7ZKyIncAMrAAAwRYwAAABT3+sYcbvdmjt3rtxut/VUIh7HumtwnLsGx7lrcJy7xplwnLvFDawAACByfa9fGQEAAPaIEQAAYIoYAQAApogRAABgKqJjZNOmTcrKylJiYqJcLpf++te/nnKb0tJSpaamKjo6WgMHDtSCBQtO/0S7uXCP84oVKzR+/Hj17dtXHo9H6enpWrduXddMthtrz5/nEzZv3qyoqCiNGDHitM0vUrTnOAeDQT366KPq37+/3G63LrroIi1ZsuT0T7aba8+xfvXVVzV8+HCdffbZ8vl8mjZtmvbv33/6J9tN5efn6wc/+IFiYmIUFxenn/70p9q1a9cpt+vqc2FEx8jBgwc1fPhwvfDCC20aX1VVpczMTI0dO1YVFRWaM2eOZs6cqaKiotM80+4t3OO8adMmjR8/XqtXr9a2bdv04x//WFlZWaqoqDjNM+3ewj3OJ/j9fk2ZMkUZGRmnaWaRpT3H+aabbtLf//53LV68WLt27dKyZcs0aNCg0zjLyBDusX7vvfc0ZcoU3XXXXfr444/1xhtvaOvWrbr77rtP80y7r9LSUs2YMUNbtmzRhg0bdPToUU2YMEEHDx5sdRuTc6HzPSHJWbly5UnHPPzww86gQYNC1t1zzz3OqFGjTuPMIktbjnNLhgwZ4jz++OOdP6EIFc5xvvnmm51f/epXzty5c53hw4ef1nlFmrYc5zVr1jher9fZv39/10wqQrXlWD/zzDPOwIEDQ9bNnz/f6dev32mcWWRpaGhwJDmlpaWtjrE4F0b0KyPhKisr04QJE0LWTZw4UeXl5Tpy5IjRrCLf8ePH1dTUpPPPP996KhHn5Zdf1r///W/NnTvXeioRq7i4WGlpafrd736nCy64QJdeeqlyc3P1v//9z3pqEWf06NHat2+fVq9eLcdx9MUXX+jNN9/UDTfcYD21bsPv90vSSf+9tTgXdosvyusq9fX1io+PD1kXHx+vo0ePqrGxUT6fz2hmke25557TwYMHddNNN1lPJaLs3r1bjzzyiN59911FRfFX/XTZs2eP3nvvPUVHR2vlypVqbGzUfffdpy+//JL7RjrZ6NGj9eqrr+rmm2/WoUOHdPToUf3kJz/RH/7wB+updQuO4ygnJ0dXX321hg4d2uo4i3Mhr4x8i8vlCnns/N8H1H57PTrHsmXL9Nhjj+n1119XXFyc9XQixrFjx3Tbbbfp8ccf16WXXmo9nYh2/PhxuVwuvfrqq7ryyiuVmZmp3//+9/rzn//MqyOdbOfOnZo5c6Z+/etfa9u2bVq7dq2qqqqUnZ1tPbVu4f7779c///lPLVu27JRju/pcyH+XviEhIUH19fUh6xoaGhQVFaXY2FijWUWu119/XXfddZfeeOMNjRs3zno6EaWpqUnl5eWqqKjQ/fffL+nrk6bjOIqKitL69et17bXXGs8yMvh8Pl1wwQUhX5M+ePBgOY6jffv26ZJLLjGcXWTJz8/XmDFj9NBDD0mShg0bpj59+mjs2LF64oknePX6JB544AEVFxdr06ZN6tev30nHWpwLiZFvSE9P11tvvRWybv369UpLS1OvXr2MZhWZli1bpjvvvFPLli3jeu9p4PF4tGPHjpB1f/zjH/XOO+/ozTffVHJystHMIs+YMWP0xhtv6MCBAzrnnHMkSZ9++ql69Ohxyn/0EZ6vvvrqO5cce/bsKen//88doRzH0QMPPKCVK1eqpKSkTX/3Lc6FEX2Z5sCBA9q+fbu2b98u6eu3K23fvl3V1dWSpLy8PE2ZMqV5fHZ2tvbu3aucnBxVVlZqyZIlWrx4sXJzcy2m322Ee5yXLVumKVOm6LnnntOoUaNUX1+v+vr65hur0LJwjnOPHj00dOjQkCUuLk7R0dEaOnSo+vTpY/U0znjh/nm+7bbbFBsbq2nTpmnnzp3atGmTHnroId15553q3bu3xVPoNsI91llZWVqxYoUKCgq0Z88ebd68WTNnztSVV16pxMREi6dwxpsxY4YKCwv12muvKSYmpvnf229eQjwjzoWn7X06Z4CNGzc6kr6zTJ061XEcx5k6dapzzTXXhGxTUlLijBw50jnrrLOcAQMGOAUFBV0/8W4m3ON8zTXXnHQ8WtaeP8/fxFt726Y9x7mystIZN26c07t3b6dfv35OTk6O89VXX3X95LuZ9hzr+fPnO0OGDHF69+7t+Hw+Z/Lkyc6+ffu6fvLdREvHV5Lz8ssvN485E86Frv+bLAAAgImIvkwDAADOfMQIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMPX/AIErHVXwkgNZAAAAAElFTkSuQmCC\n" 466 | }, 467 | "metadata": {}, 468 | "output_type": "display_data" 469 | } 470 | ], 471 | "source": [ 472 | "# 4.线性方程组的矩阵表达形式——逆矩阵\n", 473 | "plt.plot(t8[:, 0], t8[:, 1], 'o')\n", 474 | "plt.show()" 475 | ], 476 | "metadata": { 477 | "collapsed": false 478 | } 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": 21, 483 | "outputs": [ 484 | { 485 | "data": { 486 | "text/plain": "(tensor([[ 0.0288, -0.0381, 0.0284, -0.0017],\n [-0.0227, 0.0290, 0.0228, -0.0377],\n [-0.0195, 0.0143, -0.0213, 0.0719],\n [ 0.1381, 0.0402, -0.0505, -0.1573]]),\n tensor([[ 0.0288, -0.0381, 0.0284, -0.0017],\n [-0.0227, 0.0290, 0.0228, -0.0377],\n [-0.0195, 0.0143, -0.0213, 0.0719],\n [ 0.1381, 0.0402, -0.0505, -0.1573]]))" 487 | }, 488 | "execution_count": 21, 489 | "metadata": {}, 490 | "output_type": "execute_result" 491 | } 492 | ], 493 | "source": [ 494 | "# inverse函数：逆矩阵\n", 495 | "t3_inverse = torch.inverse(t3)\n", 496 | "t3_inverse, t3.inverse()" 497 | ], 498 | "metadata": { 499 | "collapsed": false 500 | } 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": 22, 505 | "outputs": [ 506 | { 507 | "data": { 508 | "text/plain": "(tensor([[ 1.0000e+00, 0.0000e+00, 2.9802e-08, 2.3842e-07],\n [-1.1921e-07, 1.0000e+00, 8.9407e-08, 2.3842e-07],\n [-2.0862e-07, -5.9605e-08, 1.0000e+00, 3.5763e-07],\n [-8.1956e-08, 0.0000e+00, -5.2154e-08, 1.0000e+00]]),\n tensor([[ 1.0000e+00, 0.0000e+00, 0.0000e+00, -1.4901e-08],\n [ 5.9605e-08, 1.0000e+00, -2.2352e-08, -1.4901e-08],\n [ 0.0000e+00, 0.0000e+00, 1.0000e+00, 0.0000e+00],\n [ 1.1921e-07, 2.0862e-07, 0.0000e+00, 1.0000e+00]]))" 509 | }, 510 | "execution_count": 22, 511 | "metadata": {}, 512 | "output_type": "execute_result" 513 | } 514 | ], 515 | "source": [ 516 | "torch.mm(t3, t3_inverse), torch.mm(t3_inverse, t3)" 517 | ], 518 | "metadata": { 519 | "collapsed": false 520 | } 521 | }, 522 | { 523 | "cell_type": "code", 524 | "execution_count": 23, 525 | "outputs": [ 526 | { 527 | "data": { 528 | "text/plain": "tensor([ 0.3661, -0.1311, -0.0135, 1.5187])" 529 | }, 530 | "execution_count": 23, 531 | "metadata": {}, 532 | "output_type": "execute_result" 533 | } 534 | ], 535 | "source": [ 536 | "# t3 * x = t9 -> x = t3_reverse * t9\n", 537 | "t9 = torch.randint(1, 20, size=(t3.size(0),), dtype=torch.float)\n", 538 | "res = torch.mv(t3_inverse, t9)\n", 539 | "res # y = res[0] * x1 + res[1] * x2 + res[2] * x3 + res[4]" 540 | ], 541 | "metadata": { 542 | "collapsed": false 543 | } 544 | }, 545 | { 546 | "cell_type": "markdown", 547 | "source": [ 548 | "## 5.矩阵的分解\n", 549 | "矩阵的分解也是矩阵运算中的常规计算，矩阵分解也有很多种类，常见的例如**QR分解**、**LU分解**、**特征分解**、**SVD分解**等等，虽然大多数情况下，矩阵分解都是在形式上将矩阵拆分成几种特殊矩阵的乘积，但本质上，矩阵的分解是去探索矩阵更深层次的一些属性。" 550 | ], 551 | "metadata": { 552 | "collapsed": false 553 | } 554 | }, 555 | { 556 | "cell_type": "code", 557 | "execution_count": 24, 558 | "outputs": [ 559 | { 560 | "data": { 561 | "text/plain": "(torch.return_types.eig(\n eigenvalues=tensor([[64.2417, 0.0000],\n [14.9217, 0.0000],\n [-7.0817, 4.4134],\n [-7.0817, -4.4134]]),\n eigenvectors=tensor([[-0.4384, 0.6602, 0.0846, -0.1704],\n [-0.5893, -0.6935, 0.2164, -0.1932],\n [-0.6032, -0.0239, -0.4340, 0.3352],\n [-0.3110, 0.2874, 0.7609, 0.0000]])),\n torch.return_types.eig(\n eigenvalues=tensor([[64.2417, 0.0000],\n [14.9217, 0.0000],\n [-7.0817, 4.4134],\n [-7.0817, -4.4134]]),\n eigenvectors=tensor([[-0.4384, 0.6602, 0.0846, -0.1704],\n [-0.5893, -0.6935, 0.2164, -0.1932],\n [-0.6032, -0.0239, -0.4340, 0.3352],\n [-0.3110, 0.2874, 0.7609, 0.0000]])))" 562 | }, 563 | "execution_count": 24, 564 | "metadata": {}, 565 | "output_type": "execute_result" 566 | } 567 | ], 568 | "source": [ 569 | "# 1.eig——特征分解：A = V * diag(λ) * V-1，只能作用于方阵，输出值分别表示特征值和Q矩阵\n", 570 | "torch.eig(t3, eigenvectors=True), t3.float().eig(eigenvectors=True)" 571 | ], 572 | "metadata": { 573 | "collapsed": false 574 | } 575 | }, 576 | { 577 | "cell_type": "code", 578 | "execution_count": 25, 579 | "outputs": [ 580 | { 581 | "data": { 582 | "text/plain": "(tensor([[-0.2609, 0.7254, -0.0909, -0.6304],\n [-0.6536, -0.3113, 0.6650, -0.1836],\n [-0.6885, -0.1319, -0.6748, 0.2306],\n [-0.1751, 0.5995, 0.3069, 0.7181]]),\n tensor([1.3195e+02, 2.3625e+01, 1.4831e+01, 3.4246e-07]),\n tensor([[-2.5310e-01, 6.6131e-01, -7.0613e-01, 1.6477e-07],\n [-2.9214e-01, -1.2077e-01, -8.3913e-03, -9.4868e-01],\n [-2.8722e-01, 6.4562e-01, 7.0759e-01, -1.9252e-08],\n [-8.7641e-01, -3.6230e-01, -2.5174e-02, 3.1623e-01]]))" 583 | }, 584 | "execution_count": 25, 585 | "metadata": {}, 586 | "output_type": "execute_result" 587 | } 588 | ], 589 | "source": [ 590 | "# 2.SVD——奇异值分解：A = U Σ VT，U、V是正交矩阵\n", 591 | "t3[:, -1] = t3[:, 1] * 3\n", 592 | "u, s, v = torch.svd(t3)\n", 593 | "u, s, v" 594 | ], 595 | "metadata": { 596 | "collapsed": false 597 | } 598 | }, 599 | { 600 | "cell_type": "code", 601 | "execution_count": 26, 602 | "outputs": [ 603 | { 604 | "data": { 605 | "text/plain": "(tensor([[1.3195e+02, 0.0000e+00, 0.0000e+00, 0.0000e+00],\n [0.0000e+00, 2.3625e+01, 0.0000e+00, 0.0000e+00],\n [0.0000e+00, 0.0000e+00, 1.4831e+01, 0.0000e+00],\n [0.0000e+00, 0.0000e+00, 0.0000e+00, 3.4246e-07]]),\n tensor([[21.0000, 8.0000, 20.0000, 24.0000],\n [10.0000, 26.0000, 27.0000, 78.0000],\n [28.0000, 27.0000, 17.0000, 81.0000],\n [12.0000, 5.0000, 19.0000, 15.0000]]))" 606 | }, 607 | "execution_count": 26, 608 | "metadata": {}, 609 | "output_type": "execute_result" 610 | } 611 | ], 612 | "source": [ 613 | "# 验证SVD\n", 614 | "torch.diag(s), torch.mm(torch.mm(u, torch.diag(s)), v.t())" 615 | ], 616 | "metadata": { 617 | "collapsed": false 618 | } 619 | }, 620 | { 621 | "cell_type": "code", 622 | "execution_count": 27, 623 | "outputs": [ 624 | { 625 | "data": { 626 | "text/plain": "tensor([[21.0000, 8.0000, 20.0000, 24.0000],\n [10.0000, 26.0000, 27.0000, 78.0000],\n [28.0000, 27.0000, 17.0000, 81.0000],\n [12.0000, 5.0000, 19.0000, 15.0000]])" 627 | }, 628 | "execution_count": 27, 629 | "metadata": {}, 630 | "output_type": "execute_result" 631 | } 632 | ], 633 | "source": [ 634 | "# 根据SVD结果进行降维：4 -> 3\n", 635 | "u_reduced = u[:, [0, 1, 2]]\n", 636 | "s_reduced = s[[0, 1, 2]]\n", 637 | "v_reduced = v[:, [0, 1, 2]].t()\n", 638 | "torch.mm(u_reduced * s_reduced, v_reduced) # 得到的新矩阵与原矩阵t3（几乎）一致" 639 | ], 640 | "metadata": { 641 | "collapsed": false 642 | } 643 | } 644 | ], 645 | "metadata": { 646 | "kernelspec": { 647 | "display_name": "Python 3", 648 | "language": "python", 649 | "name": "python3" 650 | }, 651 | "language_info": { 652 | "codemirror_mode": { 653 | "name": "ipython", 654 | "version": 2 655 | }, 656 | "file_extension": ".py", 657 | "mimetype": "text/x-python", 658 | "name": "python", 659 | "nbconvert_exporter": "python", 660 | "pygments_lexer": "ipython2", 661 | "version": "2.7.6" 662 | } 663 | }, 664 | "nbformat": 4, 665 | "nbformat_minor": 0 666 | } 667 | -------------------------------------------------------------------------------- /2_build_a_neural_network_from_scratch/introductory_dl_and_pytorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "> @Author : Corley Tang\n", 7 | "\n", 8 | "> @contact : cutercorleytd@gmail.com\n", 9 | "\n", 10 | "> @Github : https://github.com/corleytd\n", 11 | "\n", 12 | "> @Time : 2023-01-15 17:00\n", 13 | "\n", 14 | "> @Project : Hands-on Deep Learning with PyTorch-introductory_dl_and_pytorch\n", 15 | "\n", 16 | "> 认识深度学习和PyTorch\n", 17 | "\n", 18 | "## 1.深度学习、机器学习、人工智能与神经网络\n", 19 | "人工智能及相关领域的发展历程：\n", 20 | "- 预测未来，需要在某个领域深入的“只适合经验”\n", 21 | "- 经验和知识是无法在一夜之间获得\n", 22 | "- 依靠概率计算、模式拟合（pattern fit）等方法,数学家们创造了许多数学模型（线性回归、贝叶斯模型等），这些模型能够根据历史数据学习出某种规律，并依赖这些规律实现预测\n", 23 | "- 将能够根据历史数据实现预测或得出某种结果的计算步骤或计算方法称之为算法\n", 24 | "- 计算机科学家将计算机与数学结合，实现算法的规模化，提升”学习“和”预测“的效率\n", 25 | "- 哲学家与心理学家将大脑的思考机制复现，实现人工神经网络（ANN，1943年），来制造和人相似的智慧\n", 26 | "- 1950年，提出“图灵测试”\n", 27 | "- 1956年，提出人“人工智能”和“机器学习”概念\n", 28 | "- 神经网络因为预测效果差、数据需求大、计算时间长、**根本无法达到人类智力水平**，遭遇寒冬\n", 29 | "- 到新世纪，神经网络算法的大规模应用的条件逐渐成熟\n", 30 | " - 神经网络算法有了长足的进步\n", 31 | " - 全球数据量激增\n", 32 | " - 芯片、云技术迅猛发展，算力提升\n", 33 | "- 2016年，提出“深度学习”概念\n", 34 | "\n", 35 | "一般看来，**人工智能包含机器学习，机器学习包含深度学习**。\n", 36 | "\n", 37 | "## 2.深度学习前沿研究成果及酷炫应用展示\n", 38 | "AI的有趣应用如下：\n", 39 | "- 古代人物图像复原\n", 40 | "- 4K影像上色复原\n", 41 | "- 风格迁移\n", 42 | "- 动作迁移\n", 43 | "- 物体识别\n", 44 | "- AI作画\n", 45 | "- 机器翻译\n", 46 | "- AI动效\n", 47 | "- AI视频增强技术\n", 48 | "- AI图像增强技术\n", 49 | "- 语言模型\n", 50 | "- ……\n", 51 | "\n", 52 | "## 3.机器学习中的基本概念\n", 53 | "深度学习中的概念与传统机器学习中有所区别。\n", 54 | "（1）样本、特征与标签\n", 55 | "- 样本：在高维张量的操作中，一般不再区分”行列“，而是认为每个索引对应的对象就是一个样本\n", 56 | "- 特征：张量包含的内容就是特征，一般称特征所在的张量叫做特征张量\n", 57 | "- 标签：一般与数据集市分开的\n", 58 | "\n", 59 | "（2）分类与回归\n", 60 | "标签是机器学习中非常重要的一个概念，不同的标签指向了不同的问题，应用最广泛的问题主要是两类：\n", 61 | "- 分类：标签有限且互斥，标签是类别、表现为离散型变量（Categorical）\n", 62 | "- 回归：模型输出是具体数字，标签是浮点数，表现为连续型变量（Continuous）\n", 63 | "\n", 64 | "（3）有监督与无监督\n", 65 | "- 有监督学习：有标签的任务，从已知的历史数据中进行学习，然后去预测渴望了解的东西，如KNN、决策树、支持向量机、线性回归、逻辑回归和大部分神经网络等\n", 66 | "- 无监督学习：无标签的任务，一般作为作为辅助算法增强有监督算法的学习效果，包括聚类分析、协同过滤和变分自编码器等\n", 67 | "- 半监督学习\n", 68 | "- 强化学习\n", 69 | "\n", 70 | "（4）模型的评估标准\n", 71 | "- **模型预测效果**：模型进行判断/预测的效果一定是追求的核心目标，对于不同类型的算法，有不同的模型评估指标，我们依据这些评估指标来衡量模型的判断/预测效果\n", 72 | "- **运算速度**：能够同时处理大量数据、可以在超短时间内极速学习、可以实时进行预测是机器学习的重要优势。如果算法的运算速度太慢，也不利于调优和实验，同时可能需要占用更多的计算和储存资源，带来更高的成本。在模型效果不错的情况下保障运算速度较快，是机器学习中重要的一环\n", 73 | "- 可解释性：需要向人们解释算法的预测结果，否则相关人员河南接受。深度学习对模型的可解释性要求较低\n", 74 | "- 服务于业务：只有服务于业务，或服务于推动人类认知的研究，算法才会具有商业价值\n", 75 | "\n", 76 | "## 4.深入认识PyTorch框架\n", 77 | "PyTorch框架的优势：\n", 78 | "- 天生支持巨量数据和巨大神经网络的高速运算\n", 79 | "- 灵活性高，足以释放神经网络的潜力，并且在保留灵活性的同时，又有Python语法简单易学的优势\n", 80 | "- 支持研究环境与生产环境无缝切换，调试成本很低\n", 81 | "\n", 82 | "PyTorch的基本架构：\n", 83 | "- 原生Torch库——用于构建灵活神经网络的模块\n", 84 | " - 运行基础\n", 85 | " - tensor\n", 86 | " - autograd\n", 87 | " - 数据预处理utils\n", 88 | " - 数据导入与处理data、datasets\n", 89 | " - 可视化tensorboard\n", 90 | " - 预训练模型model_zoo\n", 91 | " - 神经网络基本元素nn\n", 92 | " - 各种层Module\n", 93 | " - 损失函数与激活函数functional\n", 94 | " - 优化算法optim\n", 95 | " - 运算性能\n", 96 | " - 分布式训练torchelastic\n", 97 | " - GPU训练cuda\n", 98 | " - 生产环境部署JIT\n", 99 | "- 成熟AI领域用以辅助具体行业应用的模块\n", 100 | " - 计算机视觉torchvision\n", 101 | " - 常用数据集datasets\n", 102 | " - 常用模型models\n", 103 | " - 图像数据预处理transform\n", 104 | " - 自然语言处理torchtext\n", 105 | " - 数据预处理data\n", 106 | " - 常用数据集datasets\n", 107 | " - 语音处理torchaudio\n", 108 | " - 常用数据集datasets\n", 109 | " - 声音数据预处理transform\n", 110 | " - 常用模型models\n", 111 | " - 常用函数functional" 112 | ], 113 | "metadata": { 114 | "collapsed": false 115 | } 116 | } 117 | ], 118 | "metadata": { 119 | "kernelspec": { 120 | "display_name": "Python 3", 121 | "language": "python", 122 | "name": "python3" 123 | }, 124 | "language_info": { 125 | "codemirror_mode": { 126 | "name": "ipython", 127 | "version": 2 128 | }, 129 | "file_extension": ".py", 130 | "mimetype": "text/x-python", 131 | "name": "python", 132 | "nbconvert_exporter": "python", 133 | "pygments_lexer": "ipython2", 134 | "version": "2.7.6" 135 | } 136 | }, 137 | "nbformat": 4, 138 | "nbformat_minor": 0 139 | } 140 | -------------------------------------------------------------------------------- /2_build_a_neural_network_from_scratch/single_layer_neural_network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "# -*- coding: utf-8 -*-\n", 12 | "\n", 13 | "'''\n", 14 | "@Author : Corley Tang\n", 15 | "@contact : cutercorleytd@gmail.com\n", 16 | "@Github : https://github.com/corleytd\n", 17 | "@Time : 2023-01-15 18:29\n", 18 | "@Project : Hands-on Deep Learning with PyTorch-single_layer_neural_network\n", 19 | "单层神经网络\n", 20 | "'''\n", 21 | "\n", 22 | "# 导入所需的库\n", 23 | "import warnings\n", 24 | "\n", 25 | "import torch\n", 26 | "from torch import nn\n", 27 | "from torch.nn import functional as F\n", 28 | "\n", 29 | "warnings.filterwarnings('ignore', category=UserWarning)" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "source": [ 35 | "## 1.PyTorch中的单层回归网络：线性回归\n", 36 | "### 理论基础\n", 37 | "线性回归的任务是构造一个预测函数来映射输入的特征矩阵X和标签值y的线性关系。这个预测函数的图像是一条直线，所以线性回归的求解就是对直线的拟合过程。这个预测函数的本质就是我们需要构建的模型，而**构造预测函数的核心就是找出模型的权重向量**，也就是求解线性方程组的参数。\n", 38 | "一个单层线性回归的神威网络示意图如下：\n", 39 | "![single_lr_nn](../assets/single_lr_nn.png)\n", 40 | "### tensor实现单层神经网络的正向传播" 41 | ], 42 | "metadata": { 43 | "collapsed": false 44 | } 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 2, 49 | "outputs": [ 50 | { 51 | "data": { 52 | "text/plain": "(tensor([[1., 0., 0.],\n [1., 0., 1.],\n [1., 1., 0.],\n [1., 1., 1.]]),\n tensor([-0.2000, -0.0500, -0.0500, 0.1000]),\n tensor([-0.2000, 0.1500, 0.1500]))" 53 | }, 54 | "execution_count": 2, 55 | "metadata": {}, 56 | "output_type": "execute_result" 57 | } 58 | ], 59 | "source": [ 60 | "# torch.tensor：会根据输入的数据类型来确定tensor的数据类型\n", 61 | "# torch.Tensor：无论输入的数据类型是什么，张量的数据类型都是float32，不建议使用\n", 62 | "X = torch.tensor([[1., 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]])\n", 63 | "z = torch.tensor([-0.2, -0.05, -0.05, 0.1])\n", 64 | "w = torch.tensor([-0.2, 0.15, 0.15])\n", 65 | "X, z, w" 66 | ], 67 | "metadata": { 68 | "collapsed": false 69 | } 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "outputs": [], 75 | "source": [ 76 | "# 线性回归函数\n", 77 | "def linear_regression(X, w):\n", 78 | " z_hat = torch.mv(X, w) # 矩阵乘向量\n", 79 | " return z_hat" 80 | ], 81 | "metadata": { 82 | "collapsed": false 83 | } 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 4, 88 | "outputs": [ 89 | { 90 | "data": { 91 | "text/plain": "tensor([-0.2000, -0.0500, -0.0500, 0.1000])" 92 | }, 93 | "execution_count": 4, 94 | "metadata": {}, 95 | "output_type": "execute_result" 96 | } 97 | ], 98 | "source": [ 99 | "# 调用\n", 100 | "z_hat = linear_regression(X, w)\n", 101 | "z_hat" 102 | ], 103 | "metadata": { 104 | "collapsed": false 105 | } 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "source": [ 110 | "### PyTorch新手避坑指南\n", 111 | "1.PyTorch的静态性\n", 112 | "- 很多函数要求两个张量的数据类型必须一致\n", 113 | "- 许多函数都不接受浮点型的分类标签，但也有许多函数要求真实标签的类型必须与预测值的类型一致，因此标签的类型定义总是一个容易踩坑的地方\n", 114 | "- PyTorch中许多函数不接受一维张量，但同时也有许多函数不接受二维标签，因此在生成标签时可以默认生成二维标签" 115 | ], 116 | "metadata": { 117 | "collapsed": false 118 | } 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 5, 123 | "outputs": [ 124 | { 125 | "data": { 126 | "text/plain": "tensor([ True, False, False, False])" 127 | }, 128 | "execution_count": 5, 129 | "metadata": {}, 130 | "output_type": "execute_result" 131 | } 132 | ], 133 | "source": [ 134 | "# 2.精度问题\n", 135 | "z == z_hat # 看起来预测值和真实值相等，实际上不等" 136 | ], 137 | "metadata": { 138 | "collapsed": false 139 | } 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 6, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "text/plain": "tensor(8.3267e-17)" 148 | }, 149 | "execution_count": 6, 150 | "metadata": {}, 151 | "output_type": "execute_result" 152 | } 153 | ], 154 | "source": [ 155 | "(z_hat - z).square().sum() # SSE的值也不为0" 156 | ], 157 | "metadata": { 158 | "collapsed": false 159 | } 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 7, 164 | "outputs": [ 165 | { 166 | "data": { 167 | "text/plain": "(tensor([-0.200000002980232238769531250000, -0.049999997019767761230468750000,\n -0.049999997019767761230468750000, 0.100000008940696716308593750000]),\n tensor([-0.200000002980232238769531250000, -0.050000000745058059692382812500,\n -0.050000000745058059692382812500, 0.100000001490116119384765625000]))" 168 | }, 169 | "execution_count": 7, 170 | "metadata": {}, 171 | "output_type": "execute_result" 172 | } 173 | ], 174 | "source": [ 175 | "# 设置显示精度，来查看比较预测值和真实值\n", 176 | "torch.set_printoptions(precision=30) # 看小数点后30位\n", 177 | "\n", 178 | "z_hat, z # 两者有精度的差异：float32本身带来的精确度损失，mv函数内部计算带来的微笑精度问题" 179 | ], 180 | "metadata": { 181 | "collapsed": false 182 | } 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 8, 187 | "outputs": [ 188 | { 189 | "data": { 190 | "text/plain": "(torch.Size([300, 68, 64, 64]), 83558400, tensor(83558400.))" 191 | }, 192 | "execution_count": 8, 193 | "metadata": {}, 194 | "output_type": "execute_result" 195 | } 196 | ], 197 | "source": [ 198 | "# 放大精度问题\n", 199 | "preds = torch.ones(300, 68, 64, 64)\n", 200 | "preds.shape, preds.numel(), preds.sum()" 201 | ], 202 | "metadata": { 203 | "collapsed": false 204 | } 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 9, 209 | "outputs": [ 210 | { 211 | "data": { 212 | "text/plain": "tensor(83558328.)" 213 | }, 214 | "execution_count": 9, 215 | "metadata": {}, 216 | "output_type": "execute_result" 217 | } 218 | ], 219 | "source": [ 220 | "preds *= 0.1\n", 221 | "preds.sum() * 10 # 两者计算结果不同，带来了较明显的精度问题" 222 | ], 223 | "metadata": { 224 | "collapsed": false 225 | } 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 10, 230 | "outputs": [ 231 | { 232 | "data": { 233 | "text/plain": "(torch.Size([300, 68, 64, 64]),\n 83558400,\n tensor(83558400., dtype=torch.float64))" 234 | }, 235 | "execution_count": 10, 236 | "metadata": {}, 237 | "output_type": "execute_result" 238 | } 239 | ], 240 | "source": [ 241 | "# 要缓解精度问题，可以使用float64位\n", 242 | "preds = torch.ones(300, 68, 64, 64, dtype=torch.float64)\n", 243 | "preds.shape, preds.numel(), preds.sum()" 244 | ], 245 | "metadata": { 246 | "collapsed": false 247 | } 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": 11, 252 | "outputs": [ 253 | { 254 | "data": { 255 | "text/plain": "tensor(83558400.000000059604644775390625000000, dtype=torch.float64)" 256 | }, 257 | "execution_count": 11, 258 | "metadata": {}, 259 | "output_type": "execute_result" 260 | } 261 | ], 262 | "source": [ 263 | "preds *= 0.1\n", 264 | "preds.sum() * 10 # 两者计算结果不同，但是精度问题得到了很大程度的缓解" 265 | ], 266 | "metadata": { 267 | "collapsed": false 268 | } 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 12, 273 | "outputs": [ 274 | { 275 | "data": { 276 | "text/plain": "True" 277 | }, 278 | "execution_count": 12, 279 | "metadata": {}, 280 | "output_type": "execute_result" 281 | } 282 | ], 283 | "source": [ 284 | "# 无视微小区别，比较两个张量是否一致\n", 285 | "torch.allclose(z_hat, z)" 286 | ], 287 | "metadata": { 288 | "collapsed": false 289 | } 290 | }, 291 | { 292 | "cell_type": "markdown", 293 | "source": [ 294 | "### torch.nn.Linear实现单层回归神经网络的正向传播\n", 295 | "torch.nn是包含了构筑神经网络结构基本元素的包，这个包中可以找到任意的神经网络层，这些神经网络层都是nn.Module这个大类的子类，torch.nn.Linear就是神经网络中的”线性层“。" 296 | ], 297 | "metadata": { 298 | "collapsed": false 299 | } 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": 13, 304 | "outputs": [ 305 | { 306 | "data": { 307 | "text/plain": "(tensor([[0., 0.],\n [0., 1.],\n [1., 0.],\n [1., 1.]]),\n tensor([-0.2000, -0.0500, -0.0500, 0.1000]))" 308 | }, 309 | "execution_count": 13, 310 | "metadata": {}, 311 | "output_type": "execute_result" 312 | } 313 | ], 314 | "source": [ 315 | "# 重新定义数据\n", 316 | "X = torch.tensor([[0., 0], [0, 1], [1, 0], [1, 1]]) # 输入Linear层的数据不需要添加x0\n", 317 | "z = torch.tensor([-0.2, -0.05, -0.05, 0.1])\n", 318 | "torch.set_printoptions(precision=4)\n", 319 | "X, z" 320 | ], 321 | "metadata": { 322 | "collapsed": false 323 | } 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 14, 328 | "outputs": [ 329 | { 330 | "data": { 331 | "text/plain": "(Linear(in_features=2, out_features=1, bias=True),\n Parameter containing:\n tensor([[ 0.6152, -0.1616]], requires_grad=True),\n Parameter containing:\n tensor([0.1680], requires_grad=True))" 332 | }, 333 | "execution_count": 14, 334 | "metadata": {}, 335 | "output_type": "execute_result" 336 | } 337 | ], 338 | "source": [ 339 | "# 实例化Linear层对象，默认带bias\n", 340 | "linear = nn.Linear(2, 1) # 参数：上一层的神经元个数（特征个数），这一层的神经元个数\n", 341 | "linear, linear.weight, linear.bias # 查看Linear对象及随机生成的权重和偏差" 342 | ], 343 | "metadata": { 344 | "collapsed": false 345 | } 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 15, 350 | "outputs": [ 351 | { 352 | "data": { 353 | "text/plain": "tensor([[0.1680],\n [0.0063],\n [0.7832],\n [0.6215]], grad_fn=)" 354 | }, 355 | "execution_count": 15, 356 | "metadata": {}, 357 | "output_type": "execute_result" 358 | } 359 | ], 360 | "source": [ 361 | "# 前向传播，得到预测值\n", 362 | "z_hat = linear(X)\n", 363 | "z_hat" 364 | ], 365 | "metadata": { 366 | "collapsed": false 367 | } 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": 16, 372 | "outputs": [ 373 | { 374 | "data": { 375 | "text/plain": "(Linear(in_features=2, out_features=1, bias=False),\n Parameter containing:\n tensor([[-0.1931, 0.4021]], requires_grad=True),\n None)" 376 | }, 377 | "execution_count": 16, 378 | "metadata": {}, 379 | "output_type": "execute_result" 380 | } 381 | ], 382 | "source": [ 383 | "# 实例化Linear层对象，不带bias\n", 384 | "linear = nn.Linear(2, 1, bias=False)\n", 385 | "linear, linear.weight, linear.bias" 386 | ], 387 | "metadata": { 388 | "collapsed": false 389 | } 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": 17, 394 | "outputs": [ 395 | { 396 | "data": { 397 | "text/plain": "tensor([[ 0.0000],\n [ 0.4021],\n [-0.1931],\n [ 0.2090]], grad_fn=)" 398 | }, 399 | "execution_count": 17, 400 | "metadata": {}, 401 | "output_type": "execute_result" 402 | } 403 | ], 404 | "source": [ 405 | "z_hat = linear(X) # 神经网络不需要定义输入层\n", 406 | "z_hat" 407 | ], 408 | "metadata": { 409 | "collapsed": false 410 | } 411 | }, 412 | { 413 | "cell_type": "code", 414 | "execution_count": 18, 415 | "outputs": [ 416 | { 417 | "data": { 418 | "text/plain": "(Linear(in_features=2, out_features=1, bias=True),\n Parameter containing:\n tensor([[-0.1004, 0.3112]], requires_grad=True),\n Parameter containing:\n tensor([0.6338], requires_grad=True))" 419 | }, 420 | "execution_count": 18, 421 | "metadata": {}, 422 | "output_type": "execute_result" 423 | } 424 | ], 425 | "source": [ 426 | "# 设置随机数种子，控制每次生成的权重为固定值\n", 427 | "torch.random.manual_seed(2023) # 人为设置随机数种子\n", 428 | "linear = nn.Linear(2, 1)\n", 429 | "linear, linear.weight, linear.bias" 430 | ], 431 | "metadata": { 432 | "collapsed": false 433 | } 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 19, 438 | "outputs": [ 439 | { 440 | "data": { 441 | "text/plain": "(Linear(in_features=2, out_features=1, bias=True),\n Parameter containing:\n tensor([[-0.1004, 0.3112]], requires_grad=True),\n Parameter containing:\n tensor([0.6338], requires_grad=True))" 442 | }, 443 | "execution_count": 19, 444 | "metadata": {}, 445 | "output_type": "execute_result" 446 | } 447 | ], 448 | "source": [ 449 | "torch.random.manual_seed(2023)\n", 450 | "linear = nn.Linear(2, 1)\n", 451 | "linear, linear.weight, linear.bias # 可复现" 452 | ], 453 | "metadata": { 454 | "collapsed": false 455 | } 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": 20, 460 | "outputs": [ 461 | { 462 | "data": { 463 | "text/plain": "(Linear(in_features=2, out_features=1, bias=True),\n Parameter containing:\n tensor([[-0.0288, 0.0585]], requires_grad=True),\n Parameter containing:\n tensor([0.6938], requires_grad=True))" 464 | }, 465 | "execution_count": 20, 466 | "metadata": {}, 467 | "output_type": "execute_result" 468 | } 469 | ], 470 | "source": [ 471 | "linear = nn.Linear(2, 1)\n", 472 | "linear, linear.weight, linear.bias # 与前面不一样，因为种子固定的是随机数，而不是随机数本身" 473 | ], 474 | "metadata": { 475 | "collapsed": false 476 | } 477 | }, 478 | { 479 | "cell_type": "markdown", 480 | "source": [ 481 | "## 2.二分类神经网络：逻辑回归\n", 482 | "### 理论基础\n", 483 | "线性关系描述的是变量之间的线性关系，但实际上变量之间更多的是曲线关系，为了更好地拟合曲线关系，在线性方程中引入**联系函数**，称为**广义线性回归**，包括对数线性回归、S形函数回归等，最经典的还是对数几率回归， 即逻辑回归，其在线性方程中引入Sigmoid函数，函数公式为$\\sigma=\\operatorname{Sigmoid}(z)=\\frac{1}{1+e^{-z}}$，函数图像如下：\n", 484 | "![sigmoid_curve](../assets/sigmoid_curve.png)\n", 485 | "\n", 486 | "可以看到，Sigmoid函数有一些特殊的性质：\n", 487 | "- 当自变量z趋近正无穷时，因变量σ趋近于1，而当z趋近负无穷时，σ趋近于0，这使得Sigmoid函数能够将任何实数映射到(0,1)区间\n", 488 | "- Sigmoid的导数在0点时最大（这一点的斜率最大），所以它可以快速将数据从0的附近排开，让数据点到远离自变量取0的地方去，让sigmoid函数拥有将连续性变量z转化为离散型变量σ的力量，这也就是化回归算法为分类算法的力量，具体给σ设定阈值即可，例如0.5\n", 489 | "- σ取对数几率后的结果就是线性回归的z，因此这个算法被称为对数几率回归，也就是逻辑回归\n", 490 | "\n", 491 | "### tensor实现二分类神经网络的正向传播\n", 492 | "Sigmoid二分类神经网络示意图如下：\n", 493 | "\n", 494 | "![sigmoid_nn](../assets/sigmoid_nn.png)" 495 | ], 496 | "metadata": { 497 | "collapsed": false 498 | } 499 | }, 500 | { 501 | "cell_type": "code", 502 | "execution_count": 21, 503 | "outputs": [ 504 | { 505 | "data": { 506 | "text/plain": "(tensor([[1., 0., 0.],\n [1., 0., 1.],\n [1., 1., 0.],\n [1., 1., 1.]]),\n tensor([0., 0., 0., 1.]),\n tensor([-0.2000, 0.1500, 0.1500]))" 507 | }, 508 | "execution_count": 21, 509 | "metadata": {}, 510 | "output_type": "execute_result" 511 | } 512 | ], 513 | "source": [ 514 | "# 具体实现\n", 515 | "X = torch.tensor([[1., 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]])\n", 516 | "and_gate = torch.tensor([0., 0, 0, 1])\n", 517 | "w = torch.tensor([-0.2, 0.15, 0.15])\n", 518 | "X, and_gate, w # 与门数据" 519 | ], 520 | "metadata": { 521 | "collapsed": false 522 | } 523 | }, 524 | { 525 | "cell_type": "code", 526 | "execution_count": 22, 527 | "outputs": [], 528 | "source": [ 529 | "# 自定义逻辑回归\n", 530 | "def logistic_regression(X, w, prob=0.5):\n", 531 | " z_hat = X @ w\n", 532 | " # sigma = torch.sigmoid(z_hat) # 自带的Sigmoid函数\n", 533 | " sigma = 1 / (1 + torch.exp(-z_hat))\n", 534 | " preds = (sigma >= prob).float() # 等价于torch.tensor([float(t) for t in (sigma >= prob)])\n", 535 | " return sigma, preds" 536 | ], 537 | "metadata": { 538 | "collapsed": false 539 | } 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": 23, 544 | "outputs": [ 545 | { 546 | "data": { 547 | "text/plain": "(tensor([0.4502, 0.4875, 0.4875, 0.5250]), tensor([0., 0., 0., 1.]))" 548 | }, 549 | "execution_count": 23, 550 | "metadata": {}, 551 | "output_type": "execute_result" 552 | } 553 | ], 554 | "source": [ 555 | "sigma, preds = logistic_regression(X, w)\n", 556 | "sigma, preds" 557 | ], 558 | "metadata": { 559 | "collapsed": false 560 | } 561 | }, 562 | { 563 | "cell_type": "code", 564 | "execution_count": 24, 565 | "outputs": [ 566 | { 567 | "data": { 568 | "text/plain": "tensor([True, True, True, True])" 569 | }, 570 | "execution_count": 24, 571 | "metadata": {}, 572 | "output_type": "execute_result" 573 | } 574 | ], 575 | "source": [ 576 | "# 比较预测值和真实值\n", 577 | "preds == and_gate" 578 | ], 579 | "metadata": { 580 | "collapsed": false 581 | } 582 | }, 583 | { 584 | "cell_type": "markdown", 585 | "source": [ 586 | "### 其他可以将连续型数据分割为离散型数据的函数\n", 587 | "除了Sigmoid函数，还有许多其他的函数可以被用来将连续型数据分割为离散型数据。\n", 588 | "**1.符号函数sign**\n", 589 | "公式为：\n", 590 | "$$y=\\left\\{\\begin{aligned}\n", 591 | "1 & \\text { if } z>0 \\\\\n", 592 | "0 & \\text { if } z=0 \\\\\n", 593 | "-1 & \\text { if } z<0\n", 594 | "\\end{aligned}\\right.$$\n", 595 | "函数图像如下：\n", 596 | "\n", 597 | "![sign_curve](../assets/sign_curve.png)\n", 598 | "\n", 599 | "由于函数的取值是间断的，符号函数也被称为**阶跃函数**，表示在0的两端，函数的结果y是从-1直接阶跃到了1。因为输出结果直接是0、1、-1这样的类别，所以可以直接用于分类。在二分类中，符号函数也可以忽略中间的0，直接分为0和1两类，如下：\n", 600 | "$$y=\\left\\{\\begin{array}{ll}\n", 601 | "1 & \\text { if } z>0 \\\\\n", 602 | "0 & \\text { if } z \\leq 0\n", 603 | "\\end{array}\\right.$$\n", 604 | "即\n", 605 | "$$y=\\left\\{\\begin{array}{ll}\n", 606 | "1 & \\text { if } w_{1} x_{1}+w_{2} x_{2}+b>0 \\\\\n", 607 | "0 & \\text { if } w_{1} x_{1}+w_{2} x_{2}+b \\leq 0\n", 608 | "\\end{array}\\right.$$" 609 | ], 610 | "metadata": { 611 | "collapsed": false 612 | } 613 | }, 614 | { 615 | "cell_type": "code", 616 | "execution_count": 25, 617 | "outputs": [], 618 | "source": [ 619 | "# 用阶跃函数实现二分类\n", 620 | "def logistic_regression_with_sign(X, w):\n", 621 | " z_hat = X @ w\n", 622 | " preds = (z_hat >= 0).float()\n", 623 | " return z_hat, preds" 624 | ], 625 | "metadata": { 626 | "collapsed": false 627 | } 628 | }, 629 | { 630 | "cell_type": "code", 631 | "execution_count": 26, 632 | "outputs": [ 633 | { 634 | "data": { 635 | "text/plain": "(tensor([-0.2000, -0.0500, -0.0500, 0.1000]), tensor([0., 0., 0., 1.]))" 636 | }, 637 | "execution_count": 26, 638 | "metadata": {}, 639 | "output_type": "execute_result" 640 | } 641 | ], 642 | "source": [ 643 | "# 调用\n", 644 | "z_hat, preds = logistic_regression_with_sign(X, w)\n", 645 | "z_hat, preds" 646 | ], 647 | "metadata": { 648 | "collapsed": false 649 | } 650 | }, 651 | { 652 | "cell_type": "markdown", 653 | "source": [ 654 | "**2.ReLU**\n", 655 | "\n", 656 | "ReLU(Rectified Linear Unit)函数又名整流线型单元函数，提供了一个很简单的非线性变换：当输入的自变量大于0时，直接输出该值，当输入的自变量小于等于0时，输出0。公式如下：\n", 657 | "$$\\operatorname{Re} L U: \\sigma=\\left\\{\\begin{array}{ll}\n", 658 | "z & (z>0) \\\\\n", 659 | "0 & (z \\leq 0)\n", 660 | "\\end{array}\\right.$$\n", 661 | "可以看到，ReLU函数的本质就是max(0,z)，从输入的数值中选择较大的那个值进行输出，以达到保留正数元素、将负元素清零的作用。函数图像如下：\n", 662 | "![relu_curve](../assets/relu_curve.png)\n", 663 | "\n", 664 | "ReLU函数导数的图像如下：\n", 665 | "\n", 666 | "![relu_derivative](../assets/relu_derivative.png)\n", 667 | "\n", 668 | "可以看到，ReLU函数的导数就是阶跃函数，当输入 为正数时，ReLU函数的导数为1，当z为负数时，ReLU函数的导数为0，当输入为0时，ReLU函数不可导。\n", 669 | "\n", 670 | "**3.Tanh**\n", 671 | "\n", 672 | "Tanh（hyperbolic tangent）是双曲正切函数，性质与Sigmoid相似，能够将数值压缩到(-1, 1)区间内。公式如下：\n", 673 | "$$\\tanh : \\sigma=\\frac{e^{2 z}-1}{e^{2 z}+1}$$\n", 674 | "函数图像如下：\n", 675 | "\n", 676 | "![tanh_curve](../assets/tanh_curve.png)\n", 677 | "\n", 678 | "显然，Tanh的图像和Sigmoid函数很像，区别在于，Sigmoid函数的范围是在(0, 1)之间，Tanh却是在坐标系的原点(0, 0)点上中心对称。Tanh函数的导数表达式为$\\tanh ^{\\prime}(z)=1-\\tanh ^{2}(z)$，图像如下：\n", 679 | "\n", 680 | "![tanh_derivative](../assets/tanh_derivative.png)\n", 681 | "\n", 682 | "可以看到，当输入的z约接近于0，Tanh函数导数也越接近最大值1，当输入越偏离0时，Tanh函数的导数越接近于0，也起到了类似Sigmoid函数的将将数据从0的附近排开的作用，从而将连续型变量转变为类别变量、实现二分类任务。这些函数是最常见的二分类转化函数，他们在神经网络的结构中有着不可替代的作用，也被称为**激活函数**。\n", 683 | "### torch.functional实现单层二分类神经网络的正向传播\n", 684 | "逻辑回归与线性回归的唯一区别，就是在线性回归的结果之后套上了Sigmoid函数，因此只要让nn.Linear的输出结果再经过sigmoid函数，就可以实现逻辑回归的正向传播。在PyTorch中，几乎总是从nn.functional中调用相关函数。" 685 | ], 686 | "metadata": { 687 | "collapsed": false 688 | } 689 | }, 690 | { 691 | "cell_type": "code", 692 | "execution_count": 27, 693 | "outputs": [ 694 | { 695 | "data": { 696 | "text/plain": "tensor([[0., 0.],\n [0., 1.],\n [1., 0.],\n [1., 1.]])" 697 | }, 698 | "execution_count": 27, 699 | "metadata": {}, 700 | "output_type": "execute_result" 701 | } 702 | ], 703 | "source": [ 704 | "X = torch.tensor([[0., 0], [0, 1], [1, 0], [1, 1]])\n", 705 | "X" 706 | ], 707 | "metadata": { 708 | "collapsed": false 709 | } 710 | }, 711 | { 712 | "cell_type": "code", 713 | "execution_count": 28, 714 | "outputs": [ 715 | { 716 | "data": { 717 | "text/plain": "(tensor([[-0.4478],\n [-0.8462],\n [-0.5771],\n [-0.9754]], grad_fn=),\n [0, 0, 0, 0])" 718 | }, 719 | "execution_count": 28, 720 | "metadata": {}, 721 | "output_type": "execute_result" 722 | } 723 | ], 724 | "source": [ 725 | "# 实例化和调用线性层\n", 726 | "prob = 0.5 # 概率阈值\n", 727 | "linear = nn.Linear(2, 1)\n", 728 | "z_hat = linear(X)\n", 729 | "sigma = F.sigmoid(z_hat) # 等价于torch.sigmoid(z_hat)\n", 730 | "y = (sigma >= prob).int().squeeze().tolist()\n", 731 | "z_hat, y" 732 | ], 733 | "metadata": { 734 | "collapsed": false 735 | } 736 | }, 737 | { 738 | "cell_type": "code", 739 | "execution_count": 29, 740 | "outputs": [ 741 | { 742 | "data": { 743 | "text/plain": "tensor([[-1.],\n [-1.],\n [-1.],\n [-1.]], grad_fn=)" 744 | }, 745 | "execution_count": 29, 746 | "metadata": {}, 747 | "output_type": "execute_result" 748 | } 749 | ], 750 | "source": [ 751 | "# 使用符号函数\n", 752 | "torch.sign(z_hat) # F.sign(z_hat)：AttributeError，符号函数只能使用torch调用，相比于神经网络元素，更像一个数学公式" 753 | ], 754 | "metadata": { 755 | "collapsed": false 756 | } 757 | }, 758 | { 759 | "cell_type": "code", 760 | "execution_count": 30, 761 | "outputs": [ 762 | { 763 | "data": { 764 | "text/plain": "(tensor([[0.],\n [0.],\n [0.],\n [0.]], grad_fn=),\n tensor([[0.],\n [0.],\n [0.],\n [0.]], grad_fn=))" 765 | }, 766 | "execution_count": 30, 767 | "metadata": {}, 768 | "output_type": "execute_result" 769 | } 770 | ], 771 | "source": [ 772 | "# 使用ReLU函数\n", 773 | "torch.relu(z_hat), F.relu(z_hat)" 774 | ], 775 | "metadata": { 776 | "collapsed": false 777 | } 778 | }, 779 | { 780 | "cell_type": "code", 781 | "execution_count": 31, 782 | "outputs": [ 783 | { 784 | "data": { 785 | "text/plain": "(tensor([[-0.4201],\n [-0.6891],\n [-0.5205],\n [-0.7511]], grad_fn=),\n tensor([[-0.4201],\n [-0.6891],\n [-0.5205],\n [-0.7511]], grad_fn=))" 786 | }, 787 | "execution_count": 31, 788 | "metadata": {}, 789 | "output_type": "execute_result" 790 | } 791 | ], 792 | "source": [ 793 | "# 使用Tanh函数\n", 794 | "torch.tanh(z_hat), F.tanh(z_hat) # Sigmoid、ReLU和Tanh等函数可以使用functional来调用，因为它们是对神经网络元素更有用的功能" 795 | ], 796 | "metadata": { 797 | "collapsed": false 798 | } 799 | }, 800 | { 801 | "cell_type": "markdown", 802 | "source": [ 803 | "## 3.多分类神经网络：Softmax回归\n", 804 | "### 认识Softmax函数\n", 805 | "实际生活中，出了二分类问题，还有很多多分类问题，例如手写数字识别就是一个十分类问题。在机器学习中，我们会使用二分类算法的Many-vs-Many（多对多）和One-vs-Rest（一对多）模式来进行多分类，但在深度学习中并不适用，此时有更好的方法——Softmax回归。Softmax函数是深度学习基础中的基础，它是神经网络进行多分类时，默认放在输出层中处理数据的函数，公式如下：\n", 806 | "$$\\sigma_{k}=\\operatorname{Softmax}\\left(z_{k}\\right)=\\frac{e^{z_{k}}}{\\sum^{K} e^{z}}$$\n", 807 | "可以看到，Softmax函数的分子是多分类状况下某一个标签类别的回归结果的指数函数，分母是多分类状况下所有标签类别的回归结果的指数函数之和，因此Softmax函数的结果代表了样本的结果为类别k的概率。\n", 808 | "例如对于一个三分类任务，神经网络结构如下：\n", 809 | "![softmax_nn](../assets/softmax_nn.png)\n", 810 | "\n", 811 | "在多分类中，神经元的个数与标签类别的个数是一致的，如果是十分类，在输出层上就会存在十个神经元，分别输出十个不同的概率。上图中，样本的预测标签就是输出的3个概率中最大的概率对应的标签类别。\n", 812 | "### PyTorch中的Softmax函数\n", 813 | "Softmax函数可以将多分类的结果转变为概率，但它需要的计算量非常巨大：由于Softmax的分子和分母中都带有e为底的指数函数，所以在计算中如果线性回归输出的z非常大，则$e^z$会非常大，甚至为inf，导致**溢出**，使得Softmax操作失效。" 814 | ], 815 | "metadata": { 816 | "collapsed": false 817 | } 818 | }, 819 | { 820 | "cell_type": "code", 821 | "execution_count": 32, 822 | "outputs": [ 823 | { 824 | "data": { 825 | "text/plain": "(tensor([0.6652, 0.2447, 0.0900]),\n tensor([nan, nan, nan]),\n tensor([nan, nan, nan]))" 826 | }, 827 | "execution_count": 32, 828 | "metadata": {}, 829 | "output_type": "execute_result" 830 | } 831 | ], 832 | "source": [ 833 | "# 大数据量时的Softmax失效\n", 834 | "z1 = torch.tensor([10., 9, 8])\n", 835 | "z2 = torch.tensor([105., 100, 95])\n", 836 | "z3 = torch.tensor([1010., 1000, 990])\n", 837 | "z1.exp() / z1.exp().sum(), z2.exp() / z2.exp().sum(), z3.exp() / z3.exp().sum() # Softmax的自定义实现" 838 | ], 839 | "metadata": { 840 | "collapsed": false 841 | } 842 | }, 843 | { 844 | "cell_type": "code", 845 | "execution_count": 33, 846 | "outputs": [ 847 | { 848 | "data": { 849 | "text/plain": "(tensor([0.6652, 0.2447, 0.0900]),\n tensor([9.9326e-01, 6.6925e-03, 4.5094e-05]),\n tensor([9.9995e-01, 4.5398e-05, 2.0611e-09]))" 850 | }, 851 | "execution_count": 33, 852 | "metadata": {}, 853 | "output_type": "execute_result" 854 | } 855 | ], 856 | "source": [ 857 | "# 使用torch.softmax时，需要传入维度的索引，指定在哪一个维度进行Softmax\n", 858 | "F.softmax(z1), torch.softmax(z2, 0), F.softmax(z3, -1) # Softmax函数可以通过functional和torch两种方式调用，只不过参数要求不完全一致" 859 | ], 860 | "metadata": { 861 | "collapsed": false 862 | } 863 | }, 864 | { 865 | "cell_type": "markdown", 866 | "source": [ 867 | "显然，PyTorch自带的Softmax使用了一些巧妙的手段来解决溢出问题。同时，输入的z越大，经过Softmax之后的概率值也越大，无论是否使用Softmax，都可以判断出样本被预测为哪一类，这是因为$e^z$是单调递增函数。所以在如果不需要具体概率、只需要分类出的类别结果时，此时可以不在最后一层加Softmax，一般的处理方式是在训练时加Softmax、测时时不加Softmax" 868 | ], 869 | "metadata": { 870 | "collapsed": false 871 | } 872 | }, 873 | { 874 | "cell_type": "code", 875 | "execution_count": 34, 876 | "outputs": [ 877 | { 878 | "data": { 879 | "text/plain": "(tensor([[[0.8668, 0.0453, 0.4683, 0.0177, 0.4223],\n [0.5761, 0.7054, 0.9094, 0.2119, 0.0171],\n [0.1554, 0.8438, 0.9362, 0.4223, 0.2595],\n [0.1554, 0.9362, 0.0453, 0.0453, 0.4879]],\n \n [[0.1173, 0.0453, 0.0634, 0.0177, 0.1554],\n [0.2119, 0.2595, 0.0453, 0.2119, 0.9362],\n [0.4223, 0.0420, 0.0171, 0.4223, 0.0351],\n [0.4223, 0.0466, 0.0453, 0.9094, 0.0243]],\n \n [[0.0159, 0.9094, 0.4683, 0.9647, 0.4223],\n [0.2119, 0.0351, 0.0453, 0.5761, 0.0466],\n [0.4223, 0.1142, 0.0466, 0.1554, 0.7054],\n [0.4223, 0.0171, 0.9094, 0.0453, 0.4879]]]),\n tensor([[[0.6572, 0.0279, 0.0628, 0.0130, 0.0347],\n [0.2418, 0.2060, 0.4643, 0.2619, 0.0128],\n [0.0120, 0.2060, 0.4643, 0.7120, 0.2562],\n [0.0889, 0.5601, 0.0085, 0.0130, 0.6964]],\n \n [[0.1966, 0.1966, 0.1749, 0.0104, 0.0164],\n [0.1966, 0.5344, 0.4754, 0.2097, 0.8945],\n [0.0723, 0.0723, 0.1749, 0.5701, 0.0445],\n [0.5344, 0.1966, 0.1749, 0.2097, 0.0445]],\n \n [[0.0321, 0.9205, 0.2245, 0.4191, 0.0237],\n [0.2369, 0.0169, 0.0826, 0.4191, 0.0237],\n [0.0871, 0.0458, 0.0826, 0.1542, 0.4763],\n [0.6439, 0.0169, 0.6103, 0.0077, 0.4763]]]),\n tensor([[[0.7979, 0.0397, 0.1080, 0.0146, 0.0397],\n [0.1734, 0.1734, 0.4713, 0.1734, 0.0086],\n [0.0067, 0.1336, 0.3631, 0.3631, 0.1336],\n [0.0623, 0.4604, 0.0084, 0.0084, 0.4604]],\n \n [[0.5637, 0.2074, 0.0763, 0.0763, 0.0763],\n [0.0802, 0.0802, 0.0295, 0.2179, 0.5923],\n [0.0438, 0.0161, 0.0161, 0.8801, 0.0438],\n [0.4309, 0.0583, 0.0215, 0.4309, 0.0583]],\n \n [[0.0083, 0.4538, 0.0614, 0.4538, 0.0226],\n [0.1080, 0.0146, 0.0397, 0.7979, 0.0397],\n [0.0328, 0.0328, 0.0328, 0.2425, 0.6591],\n [0.2076, 0.0103, 0.2076, 0.0103, 0.5642]]]))" 880 | }, 881 | "execution_count": 34, 882 | "metadata": {}, 883 | "output_type": "execute_result" 884 | } 885 | ], 886 | "source": [ 887 | "# 不同维度的Softmax操作：指定在哪一个维度进行操作，结果就是将哪一个维度上的大小作为类别数、计算Softmax概率\n", 888 | "t1 = torch.randint(1, 6, size=(3, 4, 5), dtype=torch.float)\n", 889 | "torch.softmax(t1, dim=0), F.softmax(t1, 1), t1.softmax(-1)" 890 | ], 891 | "metadata": { 892 | "collapsed": false 893 | } 894 | }, 895 | { 896 | "cell_type": "markdown", 897 | "source": [ 898 | "### 使用Linear层和functional实现多分类神经网络\n", 899 | "使用线性层和Softmax实现多分类神经网络需要确定以下几点：\n", 900 | "- 线性层的初始化参数：in_features=2, out_features=3\n", 901 | "- Softmax的使用维度：最后一个维度\n", 902 | "- 输出值的维度：(4, 3)" 903 | ], 904 | "metadata": { 905 | "collapsed": false 906 | } 907 | }, 908 | { 909 | "cell_type": "code", 910 | "execution_count": 35, 911 | "outputs": [ 912 | { 913 | "data": { 914 | "text/plain": "(tensor([[ 0.4729, 0.4893, -0.4421],\n [-0.2022, 0.4964, -0.9949],\n [ 0.0325, 1.1336, -0.9684],\n [-0.6426, 1.1407, -1.5212]], grad_fn=),\n tensor([[0.4137, 0.4206, 0.1657],\n [0.2887, 0.5806, 0.1307],\n [0.2286, 0.6874, 0.0840],\n [0.1358, 0.8078, 0.0564]], grad_fn=))" 915 | }, 916 | "execution_count": 35, 917 | "metadata": {}, 918 | "output_type": "execute_result" 919 | } 920 | ], 921 | "source": [ 922 | "# 具体实现\n", 923 | "X = torch.tensor([[0., 0], [0, 1], [1, 0], [1, 1]])\n", 924 | "linear = nn.Linear(2, 3)\n", 925 | "z_hat = linear(X)\n", 926 | "sigma = F.softmax(z_hat, -1)\n", 927 | "z_hat, sigma" 928 | ], 929 | "metadata": { 930 | "collapsed": false 931 | } 932 | } 933 | ], 934 | "metadata": { 935 | "kernelspec": { 936 | "display_name": "Python 3", 937 | "language": "python", 938 | "name": "python3" 939 | }, 940 | "language_info": { 941 | "codemirror_mode": { 942 | "name": "ipython", 943 | "version": 2 944 | }, 945 | "file_extension": ".py", 946 | "mimetype": "text/x-python", 947 | "name": "python", 948 | "nbconvert_exporter": "python", 949 | "pygments_lexer": "ipython2", 950 | "version": "2.7.6" 951 | } 952 | }, 953 | "nbformat": 4, 954 | "nbformat_minor": 0 955 | } 956 | -------------------------------------------------------------------------------- /3_training_and_prediction_of_neural_network/loss_function_of_neural_network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [ 10 | { 11 | "data": { 12 | "text/plain": "" 13 | }, 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "# -*- coding: utf-8 -*-\n", 21 | "\n", 22 | "'''\n", 23 | "@Author : Corley Tang\n", 24 | "@contact : cutercorleytd@gmail.com\n", 25 | "@Github : https://github.com/corleytd\n", 26 | "@Time : 2023-01-18 11:07\n", 27 | "@Project : Hands-on Deep Learning with PyTorch-loss_function_of_neural_network\n", 28 | "神经网络的损失函数\n", 29 | "'''\n", 30 | "\n", 31 | "# 导入所需的库\n", 32 | "import torch\n", 33 | "from torch import nn\n", 34 | "from torch.nn import functional as F\n", 35 | "\n", 36 | "# 设置随机种子\n", 37 | "torch.manual_seed(20230118)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "source": [ 43 | "前面已经实现了从0搭建一个神经网络的过程，并且完成了正向传播，得到了模型的输出，接下来需要完成模型的训练即学习过程，让模型的输出尽可能接近真实标签。\n", 44 | "## 1.机器学习中的优化思想\n", 45 | "之前，的过程虽然已经得到了模型的输出，但是并不能让神经网络的输出结果与真实值尽可能接近。以线性回归为例，其核心是构造一个预测函数来你和特征矩阵X与标签y之间的线性关系，即找出模型的权重w和b，使得线性回归的输出结果与真实值接近。神经网络也是一样，核心人物就是求得一组最合适的权重，使得神经网络的输出结果与真实值尽可能接近，这个找出权重w和b的过程就是**学习**，也称**训练**或**建模**。\n", 46 | "**模型的目标**：求解一组最适合的权重向量 ，令神经网络的输出结果与真实值尽量接近。\n", 47 | "用来评价w和b是否合适、输出结果与真实值之间差异的标准，就叫做损失函数。一个模型的训练过程如下：\n", 48 | "1. 提出基本模型，明确目标\n", 49 | "2. 确定损失函数/目标函数\n", 50 | " 定义的评估指标用以衡量模型权重为w的情况下预测结果与真实结果的差异，当真实值与预测值差异越大时，可以认为神经网络学习过程中丢失了许多信息，丢失的这部分被形象地称为**损失**，评估真实值与预测值差异和模型学习过程中产生的损失的函数就称为**损失函数**，在数学上，表示为以需要求解的权重向量w为自变量的函数L(w)：\n", 51 | " - 如果损失函数的值很小，则说明模型预测值与真实值很接近，模型在数据集上表现优异，权重优秀\n", 52 | " - 如果损失函数的值大，则说明模型预测值与真实值差异很大，模型在数据集上表现差劲，权重糟糕\n", 53 | "\n", 54 | " 目标是希望损失函数越小越好，因此将问题转变为求解函数L(w)的最小值所对应的自变量w，求解复杂函数就需要复杂的数学工具，主要有两部分：\n", 55 | " - 将损失函数L(w)转变成凸函数的数学方法，常见的有拉格朗日变换等——深度学习中很少用到\n", 56 | " - 在凸函数上求解L(w)的最小值对应的w的方法，也就是以梯度下降为代表的优化算法\n", 57 | "3. 确定适合的优化算法\n", 58 | "4. 利用优化算法最小化损失函数，求解最佳权重——训练\n", 59 | "## 2.回归：误差平方和SSE\n", 60 | "对于回归类神经网络而言，最常见的损失函数是误差平方和SSE（Sum of the Squared Errors），还有全部样本的平均损失MSE（均方误差，Mean squared Error），公式如下：\n", 61 | "$$S S E=\\sum_{i=1}^{m}\\left(z_{i}-\\hat{z}_{i}\\right)^{2} \\\\\n", 62 | "M S E=\\frac{1}{m} \\sum_{i=1}^{m}\\left(z_{i}-\\hat{z}_{i}\\right)^{2}$$\n", 63 | "PyTorch中MSE损失函数的调用如下。" 64 | ], 65 | "metadata": { 66 | "collapsed": false 67 | } 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 2, 72 | "outputs": [ 73 | { 74 | "data": { 75 | "text/plain": "(tensor(2.2906), tensor(2.2906))" 76 | }, 77 | "execution_count": 2, 78 | "metadata": {}, 79 | "output_type": "execute_result" 80 | } 81 | ], 82 | "source": [ 83 | "# 使用MSE损失函数\n", 84 | "y = torch.randn(50)\n", 85 | "y_hat = torch.randn(50)\n", 86 | "criterion = nn.MSELoss() # 实例化MSE类\n", 87 | "loss = criterion(y_hat, y) # 调用，预测值在前、真实值在后\n", 88 | "loss, (y - y_hat).square().sum() / y.size(0)" 89 | ], 90 | "metadata": { 91 | "collapsed": false 92 | } 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 3, 97 | "outputs": [ 98 | { 99 | "data": { 100 | "text/plain": "tensor(114.5323)" 101 | }, 102 | "execution_count": 3, 103 | "metadata": {}, 104 | "output_type": "execute_result" 105 | } 106 | ], 107 | "source": [ 108 | "# reduction参数：（1）mean：MSE（默认），（2）sum：SSE，（3）：none：逐元素计算\n", 109 | "criterion = nn.MSELoss(reduction='sum')\n", 110 | "loss = criterion(y_hat, y)\n", 111 | "loss" 112 | ], 113 | "metadata": { 114 | "collapsed": false 115 | } 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 4, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "text/plain": "tensor([1.3855e+00, 7.2827e-01, 3.5886e-01, 2.1885e-01, 1.0606e+00, 6.6579e+00,\n 1.3619e-01, 1.7970e+00, 7.2628e+00, 3.5205e-01, 9.6028e-01, 3.8422e+00,\n 5.1975e-01, 3.7674e-02, 1.7306e-01, 6.5780e+00, 1.1181e+01, 6.1518e-02,\n 2.4976e+00, 1.3350e+00, 6.3254e+00, 5.4917e-01, 1.9191e-01, 3.4457e+00,\n 5.3008e+00, 1.1419e+00, 5.1255e-03, 2.4853e-03, 1.2540e-01, 1.3146e-02,\n 1.6541e-01, 8.0619e-01, 9.8022e-02, 9.2769e+00, 3.0899e+00, 3.8199e-02,\n 1.2461e+00, 7.2920e+00, 1.6648e+00, 9.8241e+00, 1.8780e-02, 1.7243e+00,\n 8.4382e-03, 3.7473e+00, 1.4622e-01, 8.1785e-01, 4.6945e-01, 5.6041e+00,\n 2.1994e-01, 4.0292e+00])" 124 | }, 125 | "execution_count": 4, 126 | "metadata": {}, 127 | "output_type": "execute_result" 128 | } 129 | ], 130 | "source": [ 131 | "criterion = nn.MSELoss(reduction='none')\n", 132 | "loss = criterion(y_hat, y)\n", 133 | "loss" 134 | ], 135 | "metadata": { 136 | "collapsed": false 137 | } 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "source": [ 142 | "## 3.二分类交叉熵损失函数\n", 143 | "在进行分类任务时，一个很好的评价指标是准确率，但是准确率不能进行求导，而且只关注结果、不关注过程。二分类神经网络常用的损失函数是二分类交叉熵损失函数（Binary Cross EntropyLoss），也叫对数损失。BCE被广泛地使用在任何输出结果是二分类的神经网络中，同时不止限于单层神经网络，还可被拓展到多分类中。在全部样本上的MCE公式如下：\n", 144 | "$$L(w)=-\\sum_{i=1}^{m}\\left(y_{i} * \\ln \\left(\\sigma_{i}\\right)+\\left(1-y_{i}\\right) * \\ln \\left(1-\\sigma_{i}\\right)\\right)$$\n", 145 | "二分类交叉熵损失函数的目标，就是求解出使L(w)最小的w取值，是由极大似然估计推导出来的，其基本方法是寻找相应的权重 ，使得目标事件的发生概率最大，即希望每个样本的分类都正确。为了达成让模型拟合好、损失小的目的，我们每时每刻都希望$P(\\hat{y}_{i}\\mid{x}_{i}, w)$的值等于1，即每时每刻都在追求$P(\\hat{y}_{i}\\mid{x}_{i}, w)$的最大值。\n", 146 | "### 用Tensor实现BCELoss" 147 | ], 148 | "metadata": { 149 | "collapsed": false 150 | } 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 5, 155 | "outputs": [ 156 | { 157 | "data": { 158 | "text/plain": "(torch.Size([3000000, 20]), torch.Size([20, 1]), torch.Size([3000000, 1]))" 159 | }, 160 | "execution_count": 5, 161 | "metadata": {}, 162 | "output_type": "execute_result" 163 | } 164 | ], 165 | "source": [ 166 | "# 构造数据\n", 167 | "data_size = 3000000 # 样本量\n", 168 | "X = torch.rand(data_size, 20) # 特征张量\n", 169 | "w = torch.randn(20, 1) # 权重\n", 170 | "y = torch.randint(2, size=(data_size, 1), dtype=torch.float) # 标签\n", 171 | "X.shape, w.shape, y.shape" 172 | ], 173 | "metadata": { 174 | "collapsed": false 175 | } 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 6, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/plain": "torch.Size([3000000, 1])" 184 | }, 185 | "execution_count": 6, 186 | "metadata": {}, 187 | "output_type": "execute_result" 188 | } 189 | ], 190 | "source": [ 191 | "# 前向传播\n", 192 | "z_hat = X @ w\n", 193 | "sigma = z_hat.sigmoid()\n", 194 | "sigma.shape" 195 | ], 196 | "metadata": { 197 | "collapsed": false 198 | } 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": 7, 203 | "outputs": [ 204 | { 205 | "data": { 206 | "text/plain": "torch.Size([3000000, 1])" 207 | }, 208 | "execution_count": 7, 209 | "metadata": {}, 210 | "output_type": "execute_result" 211 | } 212 | ], 213 | "source": [ 214 | "# 自定义BCELoss：loss = -(y * ln(σ) + (1 - y) *ln(1 - σ))\n", 215 | "loss = -(y * torch.log(sigma) + (1 - y) * torch.log(1 - sigma)) # 计算单个样本的BCE损失\n", 216 | "loss.shape" 217 | ], 218 | "metadata": { 219 | "collapsed": false 220 | } 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 8, 225 | "outputs": [ 226 | { 227 | "data": { 228 | "text/plain": "(tensor(3830514.), tensor(1.2768))" 229 | }, 230 | "execution_count": 8, 231 | "metadata": {}, 232 | "output_type": "execute_result" 233 | } 234 | ], 235 | "source": [ 236 | "# 计算总损失和平均损失\n", 237 | "loss_sum = loss.sum()\n", 238 | "loss_mean = loss.mean()\n", 239 | "loss_sum, loss_mean" 240 | ], 241 | "metadata": { 242 | "collapsed": false 243 | } 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "source": [ 248 | "PyTorch提供了BCE损失的2个类：\n", 249 | "- BCELoss中只有交叉熵函数，没有sigmoid层，需要输入sigma与真实标签，且顺序不能变化、sigma必须在前\n", 250 | "- BCEWithLogitsLoss内置了Sigmoid函数与交叉熵函数，会自动计算输入值的sigmoid值，需要输入z_hat与真实标签，且顺序不能变化\n", 251 | "\n", 252 | "这两个函数都要求预测值与真实标签的结构（shape）及数据类型必须相同，否则不能正常运行。\n", 253 | "BCE有2个损失函数，推荐使用BCEWithLogitsLoss，是因为自己直接调用PyTorch的sigmoid()函数存在精度问题，当数据量变大、数据本身也变大时，BCELoss产生的结果可能有精度的损失，BCEWithLogitsLoss中的实现进行了优化，内置的Sigmoid函数可以让精度问题被缩小（因为将指数运算包含在了内部），以维持算法运行时的稳定性，带来更高的精度、更快的速度，因此在精度和速度要求更高的场景中，尽可能选择BCEWithLogitsLoss，如果需要计算准确率等指标，需要sigma，则必须选择BCELoss。" 254 | ], 255 | "metadata": { 256 | "collapsed": false 257 | } 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 9, 262 | "outputs": [ 263 | { 264 | "data": { 265 | "text/plain": "tensor(1.2768)" 266 | }, 267 | "execution_count": 9, 268 | "metadata": {}, 269 | "output_type": "execute_result" 270 | } 271 | ], 272 | "source": [ 273 | "# 使用BCELoss\n", 274 | "criterion = nn.BCELoss()\n", 275 | "criterion(sigma, y)" 276 | ], 277 | "metadata": { 278 | "collapsed": false 279 | } 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 10, 284 | "outputs": [ 285 | { 286 | "data": { 287 | "text/plain": "tensor(1.2768)" 288 | }, 289 | "execution_count": 10, 290 | "metadata": {}, 291 | "output_type": "execute_result" 292 | } 293 | ], 294 | "source": [ 295 | "# 使用BCEWithLogitsLoss\n", 296 | "criterion = nn.BCEWithLogitsLoss()\n", 297 | "criterion(z_hat, y) # 3个结果一致" 298 | ], 299 | "metadata": { 300 | "collapsed": false 301 | } 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 11, 306 | "outputs": [ 307 | { 308 | "data": { 309 | "text/plain": "tensor([[2.3507],\n [0.0568],\n [2.2575],\n ...,\n [0.1016],\n [0.4998],\n [0.1642]])" 310 | }, 311 | "execution_count": 11, 312 | "metadata": {}, 313 | "output_type": "execute_result" 314 | } 315 | ], 316 | "source": [ 317 | "# reduction参数：none\n", 318 | "criterion = nn.BCELoss(reduction='none') # 单个样本\n", 319 | "criterion(sigma, y)" 320 | ], 321 | "metadata": { 322 | "collapsed": false 323 | } 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 12, 328 | "outputs": [ 329 | { 330 | "data": { 331 | "text/plain": "tensor(3830514.)" 332 | }, 333 | "execution_count": 12, 334 | "metadata": {}, 335 | "output_type": "execute_result" 336 | } 337 | ], 338 | "source": [ 339 | "# reduction参数：sum\n", 340 | "criterion = nn.BCELoss(reduction='sum') # 损失综合\n", 341 | "criterion(sigma, y)" 342 | ], 343 | "metadata": { 344 | "collapsed": false 345 | } 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 13, 350 | "outputs": [ 351 | { 352 | "data": { 353 | "text/plain": "(tensor(1.2768), tensor(1.2768))" 354 | }, 355 | "execution_count": 13, 356 | "metadata": {}, 357 | "output_type": "execute_result" 358 | } 359 | ], 360 | "source": [ 361 | "# 也可以调用functional模块中的对应计算函数（不建议）\n", 362 | "F.binary_cross_entropy(sigma, y), F.binary_cross_entropy_with_logits(z_hat, y)" 363 | ], 364 | "metadata": { 365 | "collapsed": false 366 | } 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "source": [ 371 | "## 4.多分类交叉熵损失函数\n", 372 | "### 由二分类推广到多分类\n", 373 | "二分类交叉熵损失可以被推广到多分类上，但在实际处理时有所区别，一个主要问题在于多酚分类的真实标签可能是任意整数，无法使用y和(1-y)这样的结构来构建似然函数，要让多分类的标签也可以使用0和1来表示，就需要引入**哑变量**，举例如下：\n", 374 | "![multi_classification_one_hot](../assets/multi_classification_one_hot.png)\n", 375 | "\n", 376 | "可以看到，原本的真实标签y是含有1、2、3三个分类的列向量，现在将其变成了标签矩阵，每个样本对应一个向量，使用1标注出样本的真实标签的位置、使用0表示样本的真实标签不是这个标签，此时标签矩阵的结构是和Softmax函数输出的概率矩阵的结构一致，并且一一对应，这就是独热编码（One-Hot Encoding）。最终得到的多酚类交叉熵损失公式如下：\n", 377 | "$$L(w)=-\\sum_{i=1}^{m} y_{i(k=j)} \\ln \\sigma_{i}$$\n", 378 | "可以看到，二分类交叉熵损失其实是多分类交叉熵损失的一种特殊情况。交叉熵函数十分特殊，从计算结果来看，对数操作其实只对Softmax函数的结果σ起效，因此在实际操作中，把这样的函数单独提出来称为LogSoftmax，PyTorch中对应nn.logsoftmax类；同时把对数之外的操作，包括乘标签、加和、取负等过程统称为负对数似然函数（Negative Log Likelihood function），在PyTorch中对应nn.NLLLoss。因此在计算损失函数时，不再需要单独使用Softmax函数。如下：\n", 379 | "![cross_entropy_parts](../assets/cross_entropy_parts.png)\n", 380 | "### 用PyTorch实现多分类交叉熵损失\n", 381 | "用PyTorch实现交叉熵函数时，有2种方式。分别如下。" 382 | ], 383 | "metadata": { 384 | "collapsed": false 385 | } 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": 14, 390 | "outputs": [ 391 | { 392 | "data": { 393 | "text/plain": "(torch.Size([3000000, 20]), torch.Size([20, 3]), torch.Size([3000000]))" 394 | }, 395 | "execution_count": 14, 396 | "metadata": {}, 397 | "output_type": "execute_result" 398 | } 399 | ], 400 | "source": [ 401 | "# 1.调用LogSoftmax和NLLLoss实现\n", 402 | "# 构造数据\n", 403 | "w = torch.randn(20, 3) # 权重\n", 404 | "y = torch.randint(3, size=(data_size,)) # 标签\n", 405 | "X.shape, w.shape, y.shape" 406 | ], 407 | "metadata": { 408 | "collapsed": false 409 | } 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": 15, 414 | "outputs": [ 415 | { 416 | "data": { 417 | "text/plain": "torch.Size([3000000, 3])" 418 | }, 419 | "execution_count": 15, 420 | "metadata": {}, 421 | "output_type": "execute_result" 422 | } 423 | ], 424 | "source": [ 425 | "# 前向传播\n", 426 | "z_hat = X @ w\n", 427 | "z_hat.shape" 428 | ], 429 | "metadata": { 430 | "collapsed": false 431 | } 432 | }, 433 | { 434 | "cell_type": "code", 435 | "execution_count": 16, 436 | "outputs": [ 437 | { 438 | "data": { 439 | "text/plain": "tensor([[-1.8821e-04, -8.8193e+00, -1.0120e+01],\n [-3.5889e-03, -5.7479e+00, -7.8416e+00],\n [-3.0617e-02, -4.2700e+00, -4.1245e+00],\n ...,\n [-3.1070e-02, -3.8467e+00, -4.6839e+00],\n [-2.1988e-03, -6.1772e+00, -9.0269e+00],\n [-1.0573e-01, -3.9636e+00, -2.5091e+00]])" 440 | }, 441 | "execution_count": 16, 442 | "metadata": {}, 443 | "output_type": "execute_result" 444 | } 445 | ], 446 | "source": [ 447 | "# LogSoftmax\n", 448 | "logsm = nn.LogSoftmax(-1)\n", 449 | "log_sigma = logsm(z_hat)\n", 450 | "log_sigma" 451 | ], 452 | "metadata": { 453 | "collapsed": false 454 | } 455 | }, 456 | { 457 | "cell_type": "code", 458 | "execution_count": 17, 459 | "outputs": [ 460 | { 461 | "data": { 462 | "text/plain": "tensor(3.8036)" 463 | }, 464 | "execution_count": 17, 465 | "metadata": {}, 466 | "output_type": "execute_result" 467 | } 468 | ], 469 | "source": [ 470 | "# NLLLoss\n", 471 | "criterion = nn.NLLLoss()\n", 472 | "loss = criterion(log_sigma, y)\n", 473 | "loss" 474 | ], 475 | "metadata": { 476 | "collapsed": false 477 | } 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": 18, 482 | "outputs": [ 483 | { 484 | "data": { 485 | "text/plain": "tensor(3.8036)" 486 | }, 487 | "execution_count": 18, 488 | "metadata": {}, 489 | "output_type": "execute_result" 490 | } 491 | ], 492 | "source": [ 493 | "# 2.直接调用CrossEntropyLoss\n", 494 | "criterion = nn.CrossEntropyLoss()\n", 495 | "loss = criterion(z_hat, y)\n", 496 | "loss # 两种方式结果一致" 497 | ], 498 | "metadata": { 499 | "collapsed": false 500 | } 501 | }, 502 | { 503 | "cell_type": "markdown", 504 | "source": [ 505 | "从上面可以看到，无论时二分类还是多分类，PyTorch都提供了包含输出层激活函数和不包含输出层激活函数的类两种选择，在实际神经网络建模中，类可以被放入定义好的Model类中去构建神经网络的结构，是否包含激活函数由用户根据需要自行选择：\n", 506 | "- 重视展示网络结构和灵活性，应该使用不包含输出层激活函数的类\n", 507 | "- 重视稳定性和运算精度，使用包含输出层激活函数的类" 508 | ], 509 | "metadata": { 510 | "collapsed": false 511 | } 512 | } 513 | ], 514 | "metadata": { 515 | "kernelspec": { 516 | "display_name": "Python 3", 517 | "language": "python", 518 | "name": "python3" 519 | }, 520 | "language_info": { 521 | "codemirror_mode": { 522 | "name": "ipython", 523 | "version": 2 524 | }, 525 | "file_extension": ".py", 526 | "mimetype": "text/x-python", 527 | "name": "python", 528 | "nbconvert_exporter": "python", 529 | "pygments_lexer": "ipython2", 530 | "version": "2.7.6" 531 | } 532 | }, 533 | "nbformat": 4, 534 | "nbformat_minor": 0 535 | } 536 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![pytorch-logo](assets/pytorch-logo.png) 2 | 3 | ## PyTorch深度学习实战 4 | 5 | 作为Meta开源的深度学习框架，[PyTorch](https://github.com/pytorch/pytorch/)在近几年的发展中被越来越多的人使用，不论是**学术界**还是**工业界**、学生还是上班族，PyTorch被越来越多的人追捧。比于TensorFlow的**静态计算图**，PyTorch的**动态图**可以带来更大的灵活性，提供了各种张量操作并通过自动求导可以自动进行梯度计算，方便构建各种神经网络模型，同时支持使用**GPU/TPU加速计算**。本仓库提供了使用PyTorch进行深度学习的最佳实践，从深度学习环境搭建与张量基础入手，从0到1自由构筑和训练神经网络，通过优化网络保障高分结果和运行效率，同时着力于深度架构落地实践，最后通过一线CV（和NLP）企业级应用提升实战能力。项目以Jupyter Notebook为主，兼顾了理论基础和最佳实现，而不只是空洞的代码，适合小白入门；通过若干案例和大项目巩固实战和项目迁移能力；并提供多种优化手段助力论文和比赛提分。 6 | 7 | ### 环境 8 | 9 | 所有代码都是在以下环境中编写和调试： 10 | 11 | - Python 3.9.13 12 | - PyTorch 1.11.0 13 | - CudaToolkit 1.11.1 14 | - CUDA 1.11 15 | - Conda 22.11.1 16 | 17 | 完整的环境依赖可查看[requirements.txt](requirements.txt)，只需要安装主要的库即可： 18 | 19 | ```shell 20 | conda create -n handsondlbase python=3.9.10 -y 21 | conda install pytorch==1.11.0 cudatoolkit=11.1 -c pytorch -c conda-forge -y 22 | conda install pandas matplotlib seaborn jupyter scikit-learn tensorboard -y 23 | conda install torchvision=0.12.0 -c pytorch --no-deps -y 24 | pip install tqdm opencv-python 25 | ``` 26 | 27 | 如果不需要GPU版，也可以不安装cudatoolkit、直接安装PyTorch。 28 | 29 | ### 目录 30 | 31 | 1. 深度学习环境和PyTorch基础 32 | - [张量的创建和索引](1_basic_of_deep_learning_with_pytorch/tensor_create_index.ipynb) 33 | - [张量的索引、分片、合并以及维度调整](1_basic_of_deep_learning_with_pytorch/tensor_index_slice_merge.ipynb) 34 | - [张量的广播和科学运算](1_basic_of_deep_learning_with_pytorch/tensor_broadcast_computing.ipynb) 35 | - [张量的线性代数运算](1_basic_of_deep_learning_with_pytorch/tensor_linear_algebra.ipynb) 36 | - [基本优化思想与最小二乘法](1_basic_of_deep_learning_with_pytorch/optimization_and_least_square.ipynb) 37 | - [动态计算图与梯度下降入门](1_basic_of_deep_learning_with_pytorch/dynamic_computational_graph_and_gradient_descent.ipynb) 38 | 2. 从0搭建神经网络 39 | - [认识深度学习和PyTorch](2_build_a_neural_network_from_scratch/introductory_dl_and_pytorch.ipynb) 40 | - [单层神经网络](2_build_a_neural_network_from_scratch/single_layer_neural_network.ipynb) 41 | - [深层神经网络](2_build_a_neural_network_from_scratch/deep_neural_network.ipynb) 42 | 3. 神经网络的训练和预测 43 | - [神经网络的损失函数](3_training_and_prediction_of_neural_network/loss_function_of_neural_network.ipynb) 44 | - [神经网络的学习](3_training_and_prediction_of_neural_network/learning_of_neural_network.ipynb) 45 | - [深度学习基础网络的手动搭建与快速实现](3_training_and_prediction_of_neural_network/building_and_importing_of_basic_neural_network.ipynb) 46 | 4. 神经网络训练的稳定性与优化 47 | - [深度学习建模目标与模型欠拟合](4_stability_and_optimization_of_neural_network/modeling_objective_and_model_underfitting_in_deep_learning.ipynb) 48 | - [梯度不平稳性与Dead ReLU Problem](4_stability_and_optimization_of_neural_network/gradient_instability_and_dead_relu_problem.ipynb) 49 | - [Xavier与Kaiming参数初始化](4_stability_and_optimization_of_neural_network/xavier_and_kaiming_weight_initialization.ipynb) 50 | - [数据归一化与Batch Norm](4_stability_and_optimization_of_neural_network/data_normalization_and_batch_norm.ipynb) 51 | - [学习率调度](4_stability_and_optimization_of_neural_network/learning_rate_scheduling.ipynb) 52 | 5. 深度视觉与卷积神经网络 53 | - [图像处理与卷积神经网络](5_deep_vision_and_convolutional_neural_network/image_processing_and_convolutional_neural_network.ipynb) 54 | - [经典卷积神经网络与模型评估](5_deep_vision_and_convolutional_neural_network/classical_convolutional_neural_networks_and_model_architecture_evaluation.ipynb) 55 | 56 | ### 运行结果示例 57 | 58 | 一些实际运行的效果示例如下： 59 | 60 | - SSE损失的3维图像 61 | 62 | ![sse_3d_image](assets/sse_3d_image.png) 63 | - 导数与梯度 64 | 65 | ![derivative_and_grad](assets/derivative_and_grad.png) 66 | - TensorBoard可视化示例 67 | 68 | ![tensorboard_linear_model_structure](assets/tensorboard_linear_model_structure.png) 69 | - Sigmoid激活函数堆叠效应 70 | 71 | ![sigmoid_stack_compare](assets/sigmoid_stack_compare.png) 72 | - 存在梯度消失的模型的各层梯度小提琴图 73 | 74 | ![gradient_disappear_violin](assets/gradient_disappear_violin.png) 75 | - 带BN的模型的学习率的U型学习曲线 76 | 77 | ![bn_lr_learning_curve](assets/bn_lr_learning_curve.png) 78 | - OpenCV使用拉普拉斯算子和索贝尔算子进行边缘检测 79 | 80 | ![opencv_laplacian_sobel_detection](assets/opencv_laplacian_sobel_detection.png) 81 | 82 | ### 持续更新中…… 83 | 84 | ### 交流与反馈 85 | 86 | 欢迎您通过Github Issues来提交问题、报告与建议： 87 | 88 | - 个人主页：[https://github.com/corleytd](https://github.com/corleytd) 89 | - 个人邮箱：[cutercorleytd@gmail.com](mailto:cutercorleytd@gmail.com) 90 | -------------------------------------------------------------------------------- /assets/2_layer_grad_compute.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/2_layer_grad_compute.png -------------------------------------------------------------------------------- /assets/alexnet_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/alexnet_structure.png -------------------------------------------------------------------------------- /assets/bn_lr_learning_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/bn_lr_learning_curve.png -------------------------------------------------------------------------------- /assets/conv_big_small_kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/conv_big_small_kernel.png -------------------------------------------------------------------------------- /assets/conv_group_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/conv_group_demo.png -------------------------------------------------------------------------------- /assets/convolution_depthwise_separable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/convolution_depthwise_separable.png -------------------------------------------------------------------------------- /assets/convolution_first.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/convolution_first.png -------------------------------------------------------------------------------- /assets/convolution_init.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/convolution_init.png -------------------------------------------------------------------------------- /assets/convolution_last.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/convolution_last.png -------------------------------------------------------------------------------- /assets/convolution_second.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/convolution_second.png -------------------------------------------------------------------------------- /assets/cross_entropy_parts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/cross_entropy_parts.png -------------------------------------------------------------------------------- /assets/dataset_dataloader_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/dataset_dataloader_process.png -------------------------------------------------------------------------------- /assets/decreasing_receptive_field.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/decreasing_receptive_field.png -------------------------------------------------------------------------------- /assets/derivative_and_grad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/derivative_and_grad.png -------------------------------------------------------------------------------- /assets/dilation_convolution_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/dilation_convolution_demo.gif -------------------------------------------------------------------------------- /assets/dropout_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/dropout_demo.png -------------------------------------------------------------------------------- /assets/fc_kernel1_in_cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/fc_kernel1_in_cnn.png -------------------------------------------------------------------------------- /assets/googlenet_inception_v1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/googlenet_inception_v1.png -------------------------------------------------------------------------------- /assets/googlenet_layer_param.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/googlenet_layer_param.png -------------------------------------------------------------------------------- /assets/googlenet_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/googlenet_structure.png -------------------------------------------------------------------------------- /assets/grad_descent_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/grad_descent_compare.png -------------------------------------------------------------------------------- /assets/gradient_disappear_violin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/gradient_disappear_violin.png -------------------------------------------------------------------------------- /assets/handson_linear_regression_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/handson_linear_regression_nn.png -------------------------------------------------------------------------------- /assets/handson_softmax_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/handson_softmax_nn.png -------------------------------------------------------------------------------- /assets/invariance_of_statue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/invariance_of_statue.png -------------------------------------------------------------------------------- /assets/kernel_size_1_bottleneck.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/kernel_size_1_bottleneck.png -------------------------------------------------------------------------------- /assets/lenet5_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/lenet5_structure.png -------------------------------------------------------------------------------- /assets/loss_func_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/loss_func_demo.png -------------------------------------------------------------------------------- /assets/machine_learning_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/machine_learning_process.png -------------------------------------------------------------------------------- /assets/model_complexity_prediction_error_of_train_and_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/model_complexity_prediction_error_of_train_and_test.png -------------------------------------------------------------------------------- /assets/model_depth_performance_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/model_depth_performance_data.png -------------------------------------------------------------------------------- /assets/model_number.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/model_number.png -------------------------------------------------------------------------------- /assets/momentum_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/momentum_demo.png -------------------------------------------------------------------------------- /assets/multi_classification_one_hot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/multi_classification_one_hot.png -------------------------------------------------------------------------------- /assets/multi_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/multi_nn.png -------------------------------------------------------------------------------- /assets/nin_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/nin_structure.png -------------------------------------------------------------------------------- /assets/opencv_laplacian_sobel_detection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/opencv_laplacian_sobel_detection.png -------------------------------------------------------------------------------- /assets/pytorch-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/pytorch-logo.png -------------------------------------------------------------------------------- /assets/relu_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/relu_curve.png -------------------------------------------------------------------------------- /assets/relu_derivative.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/relu_derivative.png -------------------------------------------------------------------------------- /assets/repetitive_receptive_field.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/repetitive_receptive_field.png -------------------------------------------------------------------------------- /assets/resnet_5_versions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/resnet_5_versions.png -------------------------------------------------------------------------------- /assets/resnet_conv_in_x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/resnet_conv_in_x.png -------------------------------------------------------------------------------- /assets/resnet_residual_unit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/resnet_residual_unit.png -------------------------------------------------------------------------------- /assets/rgb_channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/rgb_channels.png -------------------------------------------------------------------------------- /assets/seaborn_violin_plot_illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/seaborn_violin_plot_illustration.png -------------------------------------------------------------------------------- /assets/sigmoid_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sigmoid_curve.png -------------------------------------------------------------------------------- /assets/sigmoid_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sigmoid_nn.png -------------------------------------------------------------------------------- /assets/sigmoid_stack_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sigmoid_stack_compare.png -------------------------------------------------------------------------------- /assets/sigmoid_stacked_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sigmoid_stacked_compare.png -------------------------------------------------------------------------------- /assets/sign_classification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sign_classification.png -------------------------------------------------------------------------------- /assets/sign_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sign_curve.png -------------------------------------------------------------------------------- /assets/single_lr_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/single_lr_nn.png -------------------------------------------------------------------------------- /assets/softmax_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/softmax_nn.png -------------------------------------------------------------------------------- /assets/sse_3d_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/sse_3d_image.png -------------------------------------------------------------------------------- /assets/statistical_analysis_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/statistical_analysis_process.png -------------------------------------------------------------------------------- /assets/tanh_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/tanh_curve.png -------------------------------------------------------------------------------- /assets/tanh_derivative.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/tanh_derivative.png -------------------------------------------------------------------------------- /assets/tanh_stacked_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/tanh_stacked_compare.png -------------------------------------------------------------------------------- /assets/tensorboard_linear_model_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/tensorboard_linear_model_structure.png -------------------------------------------------------------------------------- /assets/tensorboard_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/tensorboard_process.png -------------------------------------------------------------------------------- /assets/tensorboard_scalar_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/tensorboard_scalar_demo.png -------------------------------------------------------------------------------- /assets/vgg16_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/vgg16_structure.png -------------------------------------------------------------------------------- /assets/weight_bias_res_number.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/weight_bias_res_number.png -------------------------------------------------------------------------------- /assets/xor_multi_nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donggeai/Hands-on-Deep-Learning-with-PyTorch/647f3cc604edfec71ab4ead32214b27b9174f5cd/assets/xor_multi_nn.png -------------------------------------------------------------------------------- /layers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-03-21 18:20 8 | @Project : Hands-on Deep Learning with PyTorch-__init__.py 9 | ''' 10 | -------------------------------------------------------------------------------- /layers/linears.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-03-23 12:37 8 | @Project : Hands-on Deep Learning with PyTorch-linears 9 | ''' 10 | 11 | import torch 12 | from torch import nn 13 | from torch.nn import functional as F 14 | 15 | 16 | # 3层ReLU线性模型 17 | class L3ReLULR(nn.Module): 18 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 19 | super().__init__() 20 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 21 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 22 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 23 | self.linear4 = nn.Linear(hidden_dim, out_features, bias=bias) 24 | 25 | def forward(self, x): 26 | out = self.linear1(x) 27 | out = F.relu(out) 28 | out = self.linear2(out) 29 | out = F.relu(out) 30 | out = self.linear3(out) 31 | out = F.relu(out) 32 | out = self.linear4(out) 33 | return out 34 | 35 | 36 | # 1层Sigmoid线性模型 37 | class SigmoidLR(nn.Module): 38 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 39 | super().__init__() 40 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 41 | self.linear2 = nn.Linear(hidden_dim, out_features, bias=bias) 42 | 43 | def forward(self, x): 44 | out = self.linear1(x) 45 | out = torch.sigmoid(out) 46 | out = self.linear2(out) 47 | return out 48 | 49 | 50 | # 2层Sigmoid线性模型 51 | class L2SigmoidLR(nn.Module): 52 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 53 | super().__init__() 54 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 55 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 56 | self.linear3 = nn.Linear(hidden_dim, out_features, bias=bias) 57 | 58 | def forward(self, x): 59 | out = self.linear1(x) 60 | out = torch.sigmoid(out) 61 | out = self.linear2(out) 62 | out = torch.sigmoid(out) 63 | out = self.linear3(out) 64 | return out 65 | 66 | 67 | # 3层Sigmoid线性模型 68 | class L3SigmoidLR(nn.Module): 69 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 70 | super().__init__() 71 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 72 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 73 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 74 | self.linear4 = nn.Linear(hidden_dim, out_features, bias=bias) 75 | 76 | def forward(self, x): 77 | out = self.linear1(x) 78 | out = torch.sigmoid(out) 79 | out = self.linear2(out) 80 | out = torch.sigmoid(out) 81 | out = self.linear3(out) 82 | out = torch.sigmoid(out) 83 | out = self.linear4(out) 84 | return out 85 | 86 | 87 | # 4层Sigmoid线性模型 88 | class L4SigmoidLR(nn.Module): 89 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 90 | super().__init__() 91 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 92 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 93 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 94 | self.linear4 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 95 | self.linear5 = nn.Linear(hidden_dim, out_features, bias=bias) 96 | 97 | def forward(self, x): 98 | out = self.linear1(x) 99 | out = torch.sigmoid(out) 100 | out = self.linear2(out) 101 | out = torch.sigmoid(out) 102 | out = self.linear3(out) 103 | out = torch.sigmoid(out) 104 | out = self.linear4(out) 105 | out = torch.sigmoid(out) 106 | out = self.linear5(out) 107 | return out 108 | 109 | 110 | # 2层Tanh线性模型 111 | class L2TanhLR(nn.Module): 112 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 113 | super().__init__() 114 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 115 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 116 | self.linear3 = nn.Linear(hidden_dim, out_features, bias=bias) 117 | 118 | def forward(self, x): 119 | out = self.linear1(x) 120 | out = torch.tanh(out) 121 | out = self.linear2(out) 122 | out = torch.tanh(out) 123 | out = self.linear3(out) 124 | return out 125 | 126 | 127 | # 3层Tanh线性模型 128 | class L3TanhLR(nn.Module): 129 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 130 | super().__init__() 131 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 132 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 133 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 134 | self.linear4 = nn.Linear(hidden_dim, out_features, bias=bias) 135 | 136 | def forward(self, x): 137 | out = self.linear1(x) 138 | out = torch.tanh(out) 139 | out = self.linear2(out) 140 | out = torch.tanh(out) 141 | out = self.linear3(out) 142 | out = torch.tanh(out) 143 | out = self.linear4(out) 144 | return out 145 | 146 | 147 | # 4层Tanh线性模型 148 | class L4TanhLR(nn.Module): 149 | def __init__(self, in_features=2, hidden_dim=4, out_features=1, bias=True): 150 | super().__init__() 151 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 152 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 153 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 154 | self.linear4 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 155 | self.linear5 = nn.Linear(hidden_dim, out_features, bias=bias) 156 | 157 | def forward(self, x): 158 | out = self.linear1(x) 159 | out = torch.tanh(out) 160 | out = self.linear2(out) 161 | out = torch.tanh(out) 162 | out = self.linear3(out) 163 | out = torch.tanh(out) 164 | out = self.linear4(out) 165 | out = torch.tanh(out) 166 | out = self.linear5(out) 167 | return out 168 | 169 | 170 | # 2层激活函数带BN线性模型 171 | class L2LRWithBN(nn.Module): 172 | def __init__(self, activation=F.relu, in_features=2, hidden_dim=4, out_features=1, bias=True, bn_mode=None, 173 | bn_momentum=0.1): 174 | super().__init__() 175 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 176 | self.bn1 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 177 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 178 | self.bn2 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 179 | self.linear3 = nn.Linear(hidden_dim, out_features, bias=bias) 180 | self.activation = activation 181 | self.bn_mode = bn_mode 182 | 183 | def forward(self, x): 184 | if self.bn_mode == 'pre': # 前置BN 185 | out = self.bn1(self.linear1(x)) 186 | out = self.activation(out) 187 | out = self.bn2(self.linear2(out)) 188 | out = self.activation(out) 189 | out = self.linear3(out) 190 | elif self.bn_mode == 'post': # 后置BN 191 | out = self.linear1(x) 192 | out = self.activation(out) 193 | out = self.linear2(self.bn1(out)) 194 | out = self.activation(out) 195 | out = self.linear3(self.bn2(out)) 196 | else: # 不使用BN层 197 | out = self.linear1(x) 198 | out = self.activation(out) 199 | out = self.linear2(out) 200 | out = self.activation(out) 201 | out = self.linear3(out) 202 | return out 203 | 204 | 205 | # 3层激活函数带BN线性模型 206 | class L3LRWithBN(nn.Module): 207 | def __init__(self, activation=F.relu, in_features=2, hidden_dim=4, out_features=1, bias=True, bn_mode=None, 208 | bn_momentum=0.1): 209 | super().__init__() 210 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 211 | self.bn1 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 212 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 213 | self.bn2 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 214 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 215 | self.bn3 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 216 | self.linear4 = nn.Linear(hidden_dim, out_features, bias=bias) 217 | self.activation = activation 218 | self.bn_mode = bn_mode 219 | 220 | def forward(self, x): 221 | if self.bn_mode == 'pre': # 前置BN 222 | out = self.bn1(self.linear1(x)) 223 | out = self.activation(out) 224 | out = self.bn2(self.linear2(out)) 225 | out = self.activation(out) 226 | out = self.bn3(self.linear3(out)) 227 | out = self.activation(out) 228 | out = self.linear4(out) 229 | elif self.bn_mode == 'post': # 后置BN 230 | out = self.linear1(x) 231 | out = self.activation(out) 232 | out = self.linear2(self.bn1(out)) 233 | out = self.activation(out) 234 | out = self.linear3(self.bn2(out)) 235 | out = self.activation(out) 236 | out = self.linear4(self.bn3(out)) 237 | else: # 不使用BN层 238 | out = self.linear1(x) 239 | out = self.activation(out) 240 | out = self.linear2(out) 241 | out = self.activation(out) 242 | out = self.linear3(out) 243 | out = self.activation(out) 244 | out = self.linear4(out) 245 | return out 246 | 247 | 248 | # 4层激活函数带BN线性模型 249 | class L4LRWithBN(nn.Module): 250 | def __init__(self, activation=F.relu, in_features=2, hidden_dim=4, out_features=1, bias=True, bn_mode=None, 251 | bn_momentum=0.1): 252 | super().__init__() 253 | self.linear1 = nn.Linear(in_features, hidden_dim, bias=bias) 254 | self.bn1 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 255 | self.linear2 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 256 | self.bn2 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 257 | self.linear3 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 258 | self.bn3 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 259 | self.linear4 = nn.Linear(hidden_dim, hidden_dim, bias=bias) 260 | self.bn4 = nn.BatchNorm1d(hidden_dim, momentum=bn_momentum) 261 | self.linear5 = nn.Linear(hidden_dim, out_features, bias=bias) 262 | self.activation = activation 263 | self.bn_mode = bn_mode 264 | 265 | def forward(self, x): 266 | if self.bn_mode == 'pre': # 前置BN 267 | out = self.bn1(self.linear1(x)) 268 | out = self.activation(out) 269 | out = self.bn2(self.linear2(out)) 270 | out = self.activation(out) 271 | out = self.bn3(self.linear3(out)) 272 | out = self.activation(out) 273 | out = self.bn4(self.linear4(out)) 274 | out = self.activation(out) 275 | out = self.linear5(out) 276 | elif self.bn_mode == 'post': # 后置BN 277 | out = self.linear1(x) 278 | out = self.activation(out) 279 | out = self.linear2(self.bn1(out)) 280 | out = self.activation(out) 281 | out = self.linear3(self.bn2(out)) 282 | out = self.activation(out) 283 | out = self.linear4(self.bn3(out)) 284 | out = self.activation(out) 285 | out = self.linear5(self.bn4(out)) 286 | else: # 不使用BN层 287 | out = self.linear1(x) 288 | out = self.activation(out) 289 | out = self.linear2(out) 290 | out = self.activation(out) 291 | out = self.linear3(out) 292 | out = self.activation(out) 293 | out = self.linear4(out) 294 | out = self.activation(out) 295 | out = self.linear5(out) 296 | return out 297 | -------------------------------------------------------------------------------- /layers/losses.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-03-21 18:20 8 | @Project : Hands-on Deep Learning with PyTorch-losses 9 | ''' 10 | 11 | 12 | def loss_with_loader(model, data_loader, criterion): 13 | ''' 14 | 对整个数据集计算损失 15 | :param model: 模型 16 | :param data_loader: 加载好的数据 17 | :param criterion: 损失函数 18 | :return: 损失值 19 | ''' 20 | data = data_loader.dataset 21 | X = data[:][0] 22 | y = data[:][1] 23 | y_hat = model(X) 24 | return criterion(y_hat, y) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # This file may be used to create an environment using: 2 | # $ conda create --name --file 3 | # platform: win-64 4 | absl-py=1.4.0=pyhd8ed1ab_0 5 | aiohttp=3.8.4=py39ha55989b_0 6 | aiosignal=1.3.1=pyhd8ed1ab_0 7 | anyio=3.6.2=pyhd8ed1ab_0 8 | argon2-cffi=21.3.0=pyhd8ed1ab_0 9 | argon2-cffi-bindings=21.2.0=py39ha55989b_3 10 | arrow=1.2.3=pypi_0 11 | asttokens=2.2.1=pyhd8ed1ab_0 12 | async-timeout=4.0.2=pyhd8ed1ab_0 13 | attrs=22.2.0=pyh71513ae_0 14 | backcall=0.2.0=pyh9f0ad1d_0 15 | backports=1.0=pyhd8ed1ab_3 16 | backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 17 | beautifulsoup4=4.11.2=pyha770c72_0 18 | blas=2.116=mkl 19 | blas-devel=3.9.0=16_win64_mkl 20 | bleach=6.0.0=pyhd8ed1ab_0 21 | blinker=1.5=pyhd8ed1ab_0 22 | brotli=1.0.9=hcfcfb64_8 23 | brotli-bin=1.0.9=hcfcfb64_8 24 | brotlipy=0.7.0=py39ha55989b_1005 25 | bzip2=1.0.8=h8ffe710_4 26 | c-ares=1.18.1=h8ffe710_0 27 | ca-certificates=2022.12.7=h5b45459_0 28 | cachetools=5.3.0=pyhd8ed1ab_0 29 | certifi=2022.12.7=pyhd8ed1ab_0 30 | cffi=1.15.1=py39h68f70e3_3 31 | charset-normalizer=2.1.1=pyhd8ed1ab_0 32 | click=8.1.3=win_pyhd8ed1ab_2 33 | colorama=0.4.6=pyhd8ed1ab_0 34 | comm=0.1.2=pyhd8ed1ab_0 35 | contourpy=1.0.7=py39h1f6ef14_0 36 | cryptography=39.0.2=py39hb6bd5e6_0 37 | cudatoolkit=11.1.1=hb074779_11 38 | cycler=0.11.0=pyhd8ed1ab_0 39 | debugpy=1.6.6=py39h99910a6_0 40 | decorator=5.1.1=pyhd8ed1ab_0 41 | defusedxml=0.7.1=pyhd8ed1ab_0 42 | entrypoints=0.4=pyhd8ed1ab_0 43 | executing=1.2.0=pyhd8ed1ab_0 44 | flit-core=3.8.0=pyhd8ed1ab_0 45 | fonttools=4.39.2=py39ha55989b_0 46 | fqdn=1.5.1=pypi_0 47 | freetype=2.12.1=h546665d_1 48 | frozenlist=1.3.3=py39ha55989b_0 49 | gettext=0.21.1=h5728263_0 50 | glib=2.74.1=h12be248_1 51 | glib-tools=2.74.1=h12be248_1 52 | google-auth=2.16.2=pyh1a96a4e_0 53 | google-auth-oauthlib=0.4.6=pyhd8ed1ab_0 54 | grpcio=1.52.1=py39hd0003e1_1 55 | gst-plugins-base=1.22.0=h001b923_2 56 | gstreamer=1.22.0=h6b5321d_2 57 | icu=70.1=h0e60522_0 58 | idna=3.4=pyhd8ed1ab_0 59 | importlib-metadata=6.1.0=pyha770c72_0 60 | importlib-resources=5.12.0=pyhd8ed1ab_0 61 | importlib_metadata=6.1.0=hd8ed1ab_0 62 | importlib_resources=5.12.0=pyhd8ed1ab_0 63 | intel-openmp=2023.0.0=h57928b3_25922 64 | ipykernel=6.21.3=pyh025b116_0 65 | ipython=8.11.0=pyh08f2357_0 66 | ipython_genutils=0.2.0=py_1 67 | ipywidgets=8.0.4=pyhd8ed1ab_0 68 | isoduration=20.11.0=pypi_0 69 | jedi=0.18.2=pyhd8ed1ab_0 70 | jinja2=3.1.2=pyhd8ed1ab_1 71 | joblib=1.2.0=pyhd8ed1ab_0 72 | jsonpointer=2.3=pypi_0 73 | jsonschema=4.17.3=pyhd8ed1ab_0 74 | jupyter=1.0.0=py39hcbf5309_8 75 | jupyter-contrib-core=0.4.2=pypi_0 76 | jupyter-nbextensions-configurator=0.6.1=pypi_0 77 | jupyter_client=8.0.3=pyhd8ed1ab_0 78 | jupyter_console=6.6.3=pyhd8ed1ab_0 79 | jupyter_core=5.3.0=py39hcbf5309_0 80 | jupyter_events=0.6.3=pyhd8ed1ab_0 81 | jupyter_server=2.5.0=pyhd8ed1ab_0 82 | jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 83 | jupyterlab_pygments=0.2.2=pyhd8ed1ab_0 84 | jupyterlab_widgets=3.0.5=pyhd8ed1ab_0 85 | kiwisolver=1.4.4=py39h1f6ef14_1 86 | krb5=1.20.1=heb0366b_0 87 | lcms2=2.15=h3e3b177_1 88 | lerc=4.0.0=h63175ca_0 89 | libabseil=20230125.0=cxx17_h63175ca_1 90 | libblas=3.9.0=16_win64_mkl 91 | libbrotlicommon=1.0.9=hcfcfb64_8 92 | libbrotlidec=1.0.9=hcfcfb64_8 93 | libbrotlienc=1.0.9=hcfcfb64_8 94 | libcblas=3.9.0=16_win64_mkl 95 | libclang=15.0.7=default_h77d9078_1 96 | libclang13=15.0.7=default_h77d9078_1 97 | libdeflate=1.17=hcfcfb64_0 98 | libffi=3.4.2=h8ffe710_5 99 | libglib=2.74.1=he8f3873_1 100 | libgrpc=1.52.1=h32da247_1 101 | libhwloc=2.9.0=h51c2c0f_0 102 | libiconv=1.17=h8ffe710_0 103 | libjpeg-turbo=2.1.5.1=hcfcfb64_0 104 | liblapack=3.9.0=16_win64_mkl 105 | liblapacke=3.9.0=16_win64_mkl 106 | libogg=1.3.4=h8ffe710_1 107 | libpng=1.6.39=h19919ed_0 108 | libprotobuf=3.21.12=h12be248_0 109 | libsodium=1.0.18=h8d14728_1 110 | libsqlite=3.40.0=hcfcfb64_0 111 | libtiff=4.5.0=hc3b8658_5 112 | libuv=1.44.2=h8ffe710_0 113 | libvorbis=1.3.7=h0e60522_0 114 | libwebp-base=1.3.0=hcfcfb64_0 115 | libxcb=1.13=hcd874cb_1004 116 | libxml2=2.10.3=hc3477c8_3 117 | libzlib=1.2.13=hcfcfb64_4 118 | m2w64-gcc-libgfortran=5.3.0=6 119 | m2w64-gcc-libs=5.3.0=7 120 | m2w64-gcc-libs-core=5.3.0=7 121 | m2w64-gmp=6.1.0=2 122 | m2w64-libwinpthread-git=5.0.0.4634.697f757=2 123 | markdown=3.4.1=pyhd8ed1ab_0 124 | markupsafe=2.1.2=py39ha55989b_0 125 | matplotlib=3.7.1=py39hcbf5309_0 126 | matplotlib-base=3.7.1=py39haf65ace_0 127 | matplotlib-inline=0.1.6=pyhd8ed1ab_0 128 | mistune=2.0.5=pyhd8ed1ab_0 129 | mkl=2022.1.0=h6a75c08_874 130 | mkl-devel=2022.1.0=h57928b3_875 131 | mkl-include=2022.1.0=h6a75c08_874 132 | msys2-conda-epoch=20160418=1 133 | multidict=6.0.4=py39ha55989b_0 134 | munkres=1.1.4=pyh9f0ad1d_0 135 | nbclassic=0.5.3=pyhb4ecaf3_3 136 | nbclient=0.7.2=pyhd8ed1ab_0 137 | nbconvert=7.2.9=pyhd8ed1ab_0 138 | nbconvert-core=7.2.9=pyhd8ed1ab_0 139 | nbconvert-pandoc=7.2.9=pyhd8ed1ab_0 140 | nbformat=5.7.3=pyhd8ed1ab_0 141 | nest-asyncio=1.5.6=pyhd8ed1ab_0 142 | notebook=6.5.3=pyha770c72_0 143 | notebook-shim=0.2.2=pyhd8ed1ab_0 144 | numpy=1.24.2=py39h16ffa76_0 145 | oauthlib=3.2.2=pyhd8ed1ab_0 146 | opencv-python=4.7.0.72=pypi_0 147 | openjpeg=2.5.0=ha2aaf27_2 148 | openssl=3.1.0=hcfcfb64_0 149 | packaging=23.0=pyhd8ed1ab_0 150 | pandas=1.5.3=py39h2ba5b7c_0 151 | pandoc=3.1.1=h57928b3_0 152 | pandocfilters=1.5.0=pyhd8ed1ab_0 153 | parso=0.8.3=pyhd8ed1ab_0 154 | patsy=0.5.3=pyhd8ed1ab_0 155 | pcre2=10.40=h17e33f8_0 156 | pickleshare=0.7.5=py_1003 157 | pillow=9.4.0=py39haa1d754_2 158 | pip=23.0.1=pyhd8ed1ab_0 159 | pkgutil-resolve-name=1.3.10=pyhd8ed1ab_0 160 | platformdirs=3.1.1=pyhd8ed1ab_0 161 | ply=3.11=py_1 162 | pooch=1.7.0=pyhd8ed1ab_0 163 | prometheus_client=0.16.0=pyhd8ed1ab_0 164 | prompt-toolkit=3.0.38=pyha770c72_0 165 | prompt_toolkit=3.0.38=hd8ed1ab_0 166 | protobuf=4.21.12=py39h99910a6_0 167 | psutil=5.9.4=py39ha55989b_0 168 | pthread-stubs=0.4=hcd874cb_1001 169 | pthreads-win32=2.9.1=hfa6e2cd_3 170 | pure_eval=0.2.2=pyhd8ed1ab_0 171 | pyasn1=0.4.8=py_0 172 | pyasn1-modules=0.2.7=py_0 173 | pycparser=2.21=pyhd8ed1ab_0 174 | pygments=2.14.0=pyhd8ed1ab_0 175 | pyjwt=2.6.0=pyhd8ed1ab_0 176 | pyopenssl=23.0.0=pyhd8ed1ab_0 177 | pyparsing=3.0.9=pyhd8ed1ab_0 178 | pyqt=5.15.7=py39hb77abff_3 179 | pyqt5-sip=12.11.0=py39h99910a6_3 180 | pyrsistent=0.19.3=py39ha55989b_0 181 | pysocks=1.7.1=pyh0701188_6 182 | python=3.9.10=hcf16a7b_2_cpython 183 | python-dateutil=2.8.2=pyhd8ed1ab_0 184 | python-fastjsonschema=2.16.3=pyhd8ed1ab_0 185 | python-json-logger=2.0.7=pyhd8ed1ab_0 186 | python_abi=3.9=3_cp39 187 | pytorch=1.11.0=py3.9_cuda11.1_cudnn8_0 188 | pytorch-mutex=1.0=cuda 189 | pytz=2022.7.1=pyhd8ed1ab_0 190 | pyu2f=0.1.5=pyhd8ed1ab_0 191 | pywin32=304=py39h99910a6_2 192 | pywinpty=2.0.10=py39h99910a6_0 193 | pyyaml=6.0=py39ha55989b_5 194 | pyzmq=25.0.1=py39hea35a22_0 195 | qt-main=5.15.8=h88fe7eb_7 196 | qtconsole=5.4.1=pyhd8ed1ab_0 197 | qtconsole-base=5.4.1=pyha770c72_0 198 | qtpy=2.3.0=pyhd8ed1ab_0 199 | re2=2023.02.02=h63175ca_0 200 | requests=2.28.2=pyhd8ed1ab_0 201 | requests-oauthlib=1.3.1=pyhd8ed1ab_0 202 | rfc3339-validator=0.1.4=pyhd8ed1ab_0 203 | rfc3986-validator=0.1.1=pyh9f0ad1d_0 204 | rsa=4.9=pyhd8ed1ab_0 205 | scikit-learn=1.2.2=py39h6fe01c0_0 206 | scipy=1.10.1=py39hfbf2dce_0 207 | seaborn=0.12.2=hd8ed1ab_0 208 | seaborn-base=0.12.2=pyhd8ed1ab_0 209 | send2trash=1.8.0=pyhd8ed1ab_0 210 | setuptools=67.6.0=pyhd8ed1ab_0 211 | sip=6.7.7=py39h99910a6_0 212 | six=1.16.0=pyh6c4a22f_0 213 | sniffio=1.3.0=pyhd8ed1ab_0 214 | soupsieve=2.3.2.post1=pyhd8ed1ab_0 215 | sqlite=3.40.0=hcfcfb64_0 216 | stack_data=0.6.2=pyhd8ed1ab_0 217 | statsmodels=0.13.5=py39hc266a54_2 218 | tbb=2021.8.0=h91493d7_0 219 | tensorboard=2.12.0=pyhd8ed1ab_0 220 | tensorboard-data-server=0.7.0=py39hcbf5309_0 221 | tensorboard-plugin-wit=1.8.1=pyhd8ed1ab_0 222 | terminado=0.15.0=py39hcbf5309_0 223 | threadpoolctl=3.1.0=pyh8a188c0_0 224 | tinycss2=1.2.1=pyhd8ed1ab_0 225 | tk=8.6.12=h8ffe710_0 226 | toml=0.10.2=pyhd8ed1ab_0 227 | torchinfo=1.7.2=pyhd8ed1ab_0 228 | torchvision=0.13.1=cpu_py39h378ed51_0 229 | tornado=6.2=py39ha55989b_1 230 | tqdm=4.65.0=pypi_0 231 | traitlets=5.9.0=pyhd8ed1ab_0 232 | typing-extensions=4.5.0=hd8ed1ab_0 233 | typing_extensions=4.5.0=pyha770c72_0 234 | tzdata=2022g=h191b570_0 235 | ucrt=10.0.22621.0=h57928b3_0 236 | unicodedata2=15.0.0=py39ha55989b_0 237 | uri-template=1.2.0=pypi_0 238 | urllib3=1.26.15=pyhd8ed1ab_0 239 | vc=14.3=hb6edc58_10 240 | vs2015_runtime=14.34.31931=h4c5c07a_10 241 | wcwidth=0.2.6=pyhd8ed1ab_0 242 | webcolors=1.12=pypi_0 243 | webencodings=0.5.1=py_1 244 | websocket-client=1.5.1=pyhd8ed1ab_0 245 | werkzeug=2.2.3=pyhd8ed1ab_0 246 | wheel=0.40.0=pyhd8ed1ab_0 247 | widgetsnbextension=4.0.5=pyhd8ed1ab_0 248 | win_inet_pton=1.1.0=pyhd8ed1ab_6 249 | winpty=0.4.3=4 250 | xorg-libxau=1.0.9=hcd874cb_0 251 | xorg-libxdmcp=1.1.3=hcd874cb_0 252 | xz=5.2.6=h8d14728_0 253 | yaml=0.2.5=h8ffe710_2 254 | yarl=1.8.2=py39ha55989b_0 255 | zeromq=4.3.4=h0e60522_1 256 | zipp=3.15.0=pyhd8ed1ab_0 257 | zlib=1.2.13=hcfcfb64_4 258 | zstd=1.5.2=h12be248_6 259 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-01-27 23:35 8 | @Project : Hands-on Deep Learning with PyTorch-__init__.py 9 | ''' 10 | -------------------------------------------------------------------------------- /utils/data_gen_split.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-01-27 23:35 8 | @Project : Hands-on Deep Learning with PyTorch-data_gen_split 9 | 数据的生成与切分 10 | ''' 11 | 12 | import random 13 | 14 | import torch 15 | from torch.utils.data import random_split, DataLoader 16 | 17 | 18 | def gen_reg_data(num_samples=1000, w=(1, 1, 0), deg=1, delta=0.01, bias=True): 19 | ''' 20 | 创建回归类数据集 21 | :param num_samples: 数据集样本量 22 | :param w: 特征系数向量（包含截距） 23 | :param deg: 多项式关系的最高次项 24 | :param delta: 扰动项系数 25 | :param bias: 是否需要截距 26 | :return: 生成的特征张量和标签 27 | ''' 28 | if bias: 29 | num_features = len(w) - 1 # 特征数 30 | w_true = torch.tensor(w[:-1], dtype=torch.float) # 特征系数 31 | b_true = torch.tensor(w[-1], dtype=torch.float) # 截距 32 | features_true = torch.randn(num_samples, num_features) # 特征张量 33 | if num_features == 1: # 若输入特征只有1个，则不能使用矩阵乘法 34 | labels_true = torch.pow(features_true, deg) * w_true + b_true 35 | else: 36 | labels_true = torch.mv(features_true.pow(deg), w_true) + b_true 37 | features = torch.cat((features_true, torch.ones(features_true.shape[0], 1)), 1) # 在特征张量的最后添加1列1 38 | labels = labels_true + torch.randn_like(labels_true) * delta 39 | else: 40 | num_features = len(w) 41 | w_true = torch.tensor(w, dtype=torch.float) 42 | features = torch.randn(num_samples, num_features) 43 | if num_features == 1: 44 | labels_true = torch.pow(features, deg) * w_true 45 | else: 46 | labels_true = torch.mv(features.pow(deg), w_true) 47 | labels = labels_true + torch.randn_like(labels_true) * delta 48 | return features, labels 49 | 50 | 51 | def gen_cls_data(num_features=2, num_samples=500, num_classes=2, dispersions=(4, 2), bias=False): 52 | ''' 53 | 生成分类数据 54 | :param num_features: 特征数量 55 | :param num_samples: 每个类别样本数量 56 | :param num_classes: 类别数 57 | :param dispersions: 数据分布离散程度，表示(数组均值, 数组标准差) 58 | :param bias: 建立回归模型时是否需要带入截距 59 | :return: 特征张量和标签，分别是浮点型和长整型 60 | ''' 61 | label_ = torch.empty(num_samples, 1) # 每一类标签的参考 62 | mean_, std_ = dispersions # 每一类特征张量的均值和方差 63 | features, labels = [], [] # 存储每一类别的特征张量和标签 64 | k = mean_ * (num_classes - 1) / 2 # 每一类特征张量均值的惩罚因子，实现对分布离散程度的控制 65 | 66 | for i in range(num_classes): 67 | cur_features = torch.normal(i * mean_ - k, std_, size=(num_samples, num_features)) # 每一类特征张量 68 | cur_labels = torch.full_like(label_, i) # 每一类标签 69 | features.append(cur_features) 70 | labels.append(cur_labels) 71 | 72 | # 合并数据 73 | features = torch.cat(features).float() 74 | labels = torch.cat(labels).long() 75 | 76 | # 有截距 77 | if bias: 78 | features = torch.cat((features, torch.ones(features.size(0), 1)), -1) 79 | 80 | return features, labels 81 | 82 | 83 | def data_split_batches(batch_size, features, labels, shuffle=True): 84 | ''' 85 | 数据切分 86 | :param batch_size: 分配的大小，即每个小批量包含多少样本 87 | :param features: 特征张量 88 | :param labels: 标签 89 | :param shuffle: 是否随机打乱 90 | :return: 由切分后的特征和标签组成的列表 91 | ''' 92 | num_samples = labels.size(0) 93 | indices = list(range(num_samples)) 94 | if shuffle: # 随机打乱 95 | random.shuffle(indices) 96 | batches = [] 97 | for idx in range(0, num_samples, batch_size): 98 | slice_indices = torch.tensor(indices[idx:idx + batch_size]) 99 | batches.append((torch.index_select(features, 0, slice_indices), torch.index_select(labels, 0, slice_indices))) 100 | 101 | return batches 102 | 103 | 104 | def split_load_data(dataset_class, features, labels, batch_size=16, train_ratio=0.8): 105 | ''' 106 | 数据封装、切分和加载 107 | :param features: 特征 108 | :param labels: 标签 109 | :param batch_size: 批大小 110 | :param train_ratio: 训练集数据占比 111 | :return: 加载好的训练集和测试集 112 | ''' 113 | data = dataset_class(features, labels) 114 | num_samples = len(features) 115 | num_train = int(num_samples * train_ratio) 116 | num_test = num_samples - num_train 117 | train_data, test_data = random_split(data, [num_train, num_test]) 118 | train_loader = DataLoader(train_data, batch_size, shuffle=True) 119 | test_loader = DataLoader(test_data, batch_size * 2) 120 | return train_loader, test_loader 121 | -------------------------------------------------------------------------------- /utils/datasets.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-03-21 18:01 8 | @Project : Hands-on Deep Learning with PyTorch-datasets 9 | ''' 10 | 11 | from torch.utils.data import Dataset 12 | 13 | 14 | class GenDataset(Dataset): 15 | '''自定义生成数据Dataset''' 16 | def __init__(self, features, labels): 17 | self.features = features 18 | self.labels = labels 19 | 20 | def __getitem__(self, index): 21 | return self.features[index], self.labels[index] 22 | 23 | def __len__(self): 24 | return self.features.shape[0] -------------------------------------------------------------------------------- /utils/receptive_field.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import numpy as np 4 | import torch 5 | import torch.nn as nn 6 | from torch.autograd import Variable 7 | 8 | 9 | def check_same(stride): 10 | if isinstance(stride, (list, tuple)): 11 | assert (len(stride) == 2 and stride[0] == stride[1]) or ( 12 | len(stride) == 3 and stride[0] == stride[1] and stride[1] == stride[2]) 13 | stride = stride[0] 14 | return stride 15 | 16 | 17 | def receptive_field(model, input_size, batch_size=-1, device='cuda'): 18 | ''' 19 | :parameter 20 | 'input_size': tuple of (Channel, Height, Width) 21 | :return OrderedDict of `Layername`->OrderedDict of receptive field stats {'j':,'r':,'start':,'conv_stage':,'output_shape':,} 22 | 'j' for 'jump' denotes how many pixels do the receptive fields of spatially neighboring units in the feature tensor 23 | do not overlap in one direction. 24 | i.e. shift one unit in this feature map == how many pixels shift in the input image in one direction. 25 | 'r' for 'receptive_field' is the spatial range of the receptive field in one direction. 26 | 'start' denotes the center of the receptive field for the first unit (start) in on direction of the feature tensor. 27 | Convention is to use half a pixel as the center for a range. center for `slice(0,5)` is 2.5. 28 | ''' 29 | 30 | def register_hook(module): 31 | 32 | def hook(module, input, output): 33 | class_name = str(module.__class__).split('.')[-1].split('\'')[0] 34 | module_idx = len(receptive_field) 35 | m_key = '%i' % module_idx 36 | p_key = '%i' % (module_idx - 1) 37 | receptive_field[m_key] = OrderedDict() 38 | receptive_field[m_key]['layer_cls'] = class_name 39 | 40 | if not receptive_field['0']['conv_stage']: 41 | print('Enter in deconv_stage') 42 | receptive_field[m_key]['j'] = 0 43 | receptive_field[m_key]['r'] = 0 44 | receptive_field[m_key]['start'] = 0 45 | else: 46 | p_j = receptive_field[p_key]['j'] 47 | p_r = receptive_field[p_key]['r'] 48 | p_start = receptive_field[p_key]['start'] 49 | 50 | if class_name == 'Conv2d' or class_name == 'MaxPool2d' or class_name == 'AvgPool2d' or class_name == 'Conv3d' or class_name == 'MaxPool3d': 51 | kernel_size = module.kernel_size 52 | stride = module.stride 53 | padding = module.padding 54 | 55 | if class_name == 'AvgPool2d': 56 | # Avg Pooling does not have dilation, set it to 1 (no dilation) 57 | dilation = 1 58 | else: 59 | dilation = module.dilation 60 | 61 | kernel_size, stride, padding, dilation = map(check_same, [kernel_size, stride, padding, dilation]) 62 | receptive_field[m_key]['j'] = p_j * stride 63 | receptive_field[m_key]['r'] = int(p_r + ((kernel_size - 1) * dilation) * p_j) 64 | receptive_field[m_key]['start'] = p_start + ((kernel_size - 1) / 2 - padding) * p_j 65 | elif class_name == 'BatchNorm2d' or class_name in ['ReLU', 'Tanh', 'Sigmoid'] or class_name == 'Bottleneck' or class_name == 'BatchNorm3d': 66 | receptive_field[m_key]['j'] = p_j 67 | receptive_field[m_key]['r'] = p_r 68 | receptive_field[m_key]['start'] = p_start 69 | elif class_name == 'ConvTranspose2d' or class_name == 'ConvTranspose3d': 70 | receptive_field['0']['conv_stage'] = False 71 | receptive_field[m_key]['j'] = 0 72 | receptive_field[m_key]['r'] = 0 73 | receptive_field[m_key]['start'] = 0 74 | else: 75 | raise ValueError('module {} not ok'.format(class_name)) 76 | pass 77 | receptive_field[m_key]['input_shape'] = list(input[0].size()) # only one 78 | receptive_field[m_key]['input_shape'][0] = batch_size 79 | if isinstance(output, (list, tuple)): 80 | # list/tuple 81 | receptive_field[m_key]['output_shape'] = [ 82 | [-1] + list(o.size())[1:] for o in output 83 | ] 84 | else: 85 | # tensor 86 | receptive_field[m_key]['output_shape'] = list(output.size()) 87 | receptive_field[m_key]['output_shape'][0] = batch_size 88 | 89 | if ( 90 | not isinstance(module, nn.Sequential) 91 | and not isinstance(module, nn.ModuleList) 92 | and not (module == model) 93 | and not isinstance(module, nn.Linear) 94 | ): 95 | hooks.append(module.register_forward_hook(hook)) 96 | 97 | device = device.lower() 98 | assert device in [ 99 | 'cuda', 100 | 'cpu', 101 | ], 'Input device is not valid, please specify "cuda" or "cpu"' 102 | 103 | if device == 'cuda' and torch.cuda.is_available(): 104 | dtype = torch.cuda.FloatTensor 105 | else: 106 | dtype = torch.FloatTensor 107 | 108 | # check if there are multiple inputs to the network 109 | if isinstance(input_size[0], (list, tuple)): 110 | x = [Variable(torch.rand(2, *in_size)).type(dtype) for in_size in input_size] 111 | else: 112 | x = Variable(torch.rand(2, *input_size)).type(dtype) 113 | 114 | # create properties 115 | receptive_field = OrderedDict() 116 | receptive_field['0'] = OrderedDict() 117 | receptive_field['0']['layer_cls'] = 'Input' 118 | receptive_field['0']['j'] = 1.0 119 | receptive_field['0']['r'] = 1 120 | receptive_field['0']['start'] = 0.5 121 | receptive_field['0']['conv_stage'] = True 122 | receptive_field['0']['output_shape'] = list(x.size()) 123 | receptive_field['0']['output_shape'][0] = batch_size 124 | hooks = [] 125 | 126 | # register hook 127 | model.apply(register_hook) 128 | 129 | # make a forward pass 130 | model(x) 131 | 132 | # remove these hooks 133 | for h in hooks: 134 | h.remove() 135 | 136 | print('------------------------------------------------------------------------------') 137 | line_new = '{:>23} {:>10} {:>10} {:>10} {:>15} '.format('Layer (type)', 'map size', 'start', 'jump', 138 | 'receptive_field') 139 | print(line_new) 140 | print('==============================================================================') 141 | total_params = 0 142 | total_output = 0 143 | trainable_params = 0 144 | for layer in receptive_field: 145 | # input_shape, output_shape, trainable, nb_params 146 | assert 'start' in receptive_field[layer], layer 147 | if len(receptive_field[layer]['output_shape']) == 4 or len(receptive_field[layer]['output_shape']) == 5: 148 | line_new = '{:7} {:15} {:>10} {:>10} {:>10} {:>15} '.format( 149 | '', 150 | f'{layer} ({receptive_field[layer]["layer_cls"]})', 151 | str(receptive_field[layer]['output_shape'][2:]), 152 | str(receptive_field[layer]['start']), 153 | str(receptive_field[layer]['j']), 154 | format(str(receptive_field[layer]['r'])) 155 | ) 156 | print(line_new) 157 | else: 158 | break 159 | 160 | print('==============================================================================') 161 | # add input_shape 162 | receptive_field['input_size'] = input_size 163 | return receptive_field 164 | 165 | 166 | def receptive_field_for_unit(receptive_field_dict, layer, unit_position): 167 | '''Utility function to calculate the receptive field for a specific unit in a layer 168 | using the dictionary calculated above 169 | :parameter 170 | 'layer': layer name, should be a key in the result dictionary 171 | 'unit_position': spatial coordinate of the unit (H, W) 172 | ``` 173 | alexnet = models.alexnet() 174 | model = alexnet.features.to('cuda') 175 | receptive_field_dict = receptive_field(model, (3, 224, 224)) 176 | receptive_field_for_unit(receptive_field_dict, '8', (6,6)) 177 | ``` 178 | Out: [(62.0, 161.0), (62.0, 161.0)] 179 | ''' 180 | input_shape = receptive_field_dict['input_size'] 181 | if layer in receptive_field_dict: 182 | rf_stats = receptive_field_dict[layer] 183 | assert len(unit_position) == 2 or len(unit_position) == 3 184 | feat_map_lim = rf_stats['output_shape'][2:] 185 | if np.any([unit_position[idx] < 0 or 186 | unit_position[idx] >= feat_map_lim[idx] 187 | for idx in range(len(unit_position))]): 188 | if len(unit_position) == 2: 189 | raise Exception( 190 | 'Unit position outside spatial extent of the feature tensor ((H, W) = (%d, %d)) ' % tuple( 191 | feat_map_lim)) 192 | else: 193 | raise Exception( 194 | 'Unit position outside spatial extent of the feature tensor ((D, H, W) = (%d, %d, %d)) ' % tuple( 195 | feat_map_lim)) 196 | # X, Y = tuple(unit_position) 197 | rf_range = [(rf_stats['start'] + idx * rf_stats['j'] - rf_stats['r'] / 2, 198 | rf_stats['start'] + idx * rf_stats['j'] + rf_stats['r'] / 2) for idx in unit_position] 199 | if len(unit_position) == 2: 200 | if len(input_shape) == 2: 201 | limit = input_shape 202 | else: # input shape is (channel, H, W) 203 | limit = input_shape[1:3] 204 | rf_range = [(max(0, rf_range[axis][0]), min(limit[axis], rf_range[axis][1])) for axis in range(2)] 205 | else: 206 | if len(input_shape) == 3: 207 | limit = input_shape 208 | else: # input shape is (channel, D, H, W) 209 | limit = input_shape[1:4] 210 | rf_range = [(max(0, rf_range[axis][0]), min(limit[axis], rf_range[axis][1])) for axis in range(3)] 211 | 212 | print('Receptive field size for layer %s, unit_position %s, is \n %s' % (layer, unit_position, rf_range)) 213 | return rf_range 214 | else: 215 | raise KeyError('Layer name incorrect, or not included in the model.') 216 | -------------------------------------------------------------------------------- /utils/train_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-03-21 18:10 8 | @Project : Hands-on Deep Learning with PyTorch-train_utils 9 | ''' 10 | 11 | import torch 12 | from torch import nn 13 | from torch import optim 14 | from tqdm import tqdm 15 | 16 | from layers.losses import loss_with_loader 17 | 18 | 19 | def fit_one_epoch(model, criterion, optimizer, data_loader, cla=False): 20 | ''' 21 | 模型训练一轮 22 | :param model: 待训练的模型 23 | :param criterion: 损失函数 24 | :param optimizer: 优化算法 25 | :param data_loader: 数据 26 | :param cla: 是否是分类任务 27 | :return: None 28 | ''' 29 | for X, y in data_loader: 30 | optimizer.zero_grad() 31 | y_hat = model(X) 32 | if cla: 33 | y = y.int() 34 | loss = criterion(y_hat, y) 35 | loss.backward() 36 | optimizer.step() 37 | 38 | 39 | def fit(model, criterion, optimizer, data_loader, cla=False, num_epochs=5): 40 | ''' 41 | 模型训练 42 | :param model: 待训练的模型 43 | :param criterion: 损失函数 44 | :param optimizer: 优化算法 45 | :param data_loader: 数据 46 | :param cla: 是否是分类任务 47 | :param num_epochs: 训练轮数 48 | :return: None 49 | ''' 50 | for _ in tqdm(range(num_epochs), desc='Epoch'): 51 | fit_one_epoch(model, criterion, optimizer, data_loader, cla) 52 | 53 | 54 | def train_test_model(model, train_loader, test_loader, criterion=nn.MSELoss, optimizer_class=optim.SGD, 55 | evaluation=loss_with_loader, lr=0.03, num_epochs=20, cla=False): 56 | ''' 57 | 训练，并记录训练过程中训练集和测试集上的损失变化 58 | :param model: 待训练的模型 59 | :param train_loader: 训练数据集 60 | :param test_loader: 测试数据集 61 | :param criterion: 损失函数 62 | :param optimizer_class: 优化器 63 | :param evaluation: 数据集损失计算函数 64 | :param lr: 学习率 65 | :param num_epochs: 训练轮数 66 | :param cla: 是否是分类任务 67 | :return: 训练损失和测试损失 68 | ''' 69 | criterion = criterion() 70 | optimizer = optimizer_class(model.parameters(), lr=lr) 71 | train_losses, test_losses = [], [] 72 | for _ in tqdm(range(num_epochs), desc='Epoch'): 73 | model.train() 74 | fit_one_epoch(model, criterion, optimizer, train_loader, cla) 75 | model.eval() 76 | train_losses.append(evaluation(model, train_loader, criterion).item()) 77 | test_losses.append(evaluation(model, test_loader, criterion).item()) 78 | return train_losses, test_losses 79 | 80 | 81 | def compare_models(model_list, train_loader, test_loader, criterion=nn.MSELoss, optimizer_class=optim.SGD, 82 | evaluation=loss_with_loader, lr=0.03, num_epochs=20, cla=False): 83 | ''' 84 | 模型对比 85 | :param model_list: 模型对象列表 86 | :param train_loader: 训练集 87 | :param test_loader: 测试集 88 | :param criterion: 损失函数 89 | :param optimizer_class: 优化器 90 | :param evaluation: 数据集损失计算函数 91 | :param lr: 学习率 92 | :param num_epochs: 训练轮数 93 | :param cla: 是否是分类任务 94 | :return: 训练损失和测试损失 95 | ''' 96 | criterion = criterion() 97 | 98 | train_losses, test_losses = torch.zeros(len(model_list), num_epochs), torch.zeros(len(model_list), num_epochs) 99 | 100 | # 训练模型 101 | for epoch in tqdm(range(num_epochs), desc='Epoch'): 102 | for idx, model in enumerate(model_list): 103 | model.train() 104 | optimizer = optimizer_class(model.parameters(), lr=lr) 105 | fit_one_epoch(model, criterion, optimizer, train_loader, cla=cla) 106 | model.eval() 107 | train_losses[idx, epoch] = evaluation(model, train_loader, criterion).item() 108 | test_losses[idx, epoch] = evaluation(model, test_loader, criterion).item() 109 | 110 | return train_losses, test_losses 111 | -------------------------------------------------------------------------------- /utils/visualization_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | @Author : Corley Tang 5 | @contact : cutercorleytd@gmail.com 6 | @Github : https://github.com/corleytd 7 | @Time : 2023-03-25 11:34 8 | @Project : Hands-on Deep Learning with PyTorch-visualization_utils 9 | ''' 10 | 11 | import numpy as np 12 | import seaborn as sns 13 | from matplotlib import pyplot as plt 14 | from torch import nn 15 | 16 | 17 | def violin_plot_layers(model, is_grad=True): 18 | ''' 19 | 绘制模型参数或梯度的小提琴图 20 | :param model: 模型 21 | :param is_grad: 是否画梯度 22 | :return: None 23 | ''' 24 | layer_params = [] 25 | 26 | # 记录梯度 27 | for idx, module in enumerate(model.modules()): 28 | if isinstance(module, nn.Linear): 29 | if is_grad: 30 | layer_param = module.weight.grad.reshape(-1, 1).numpy() # 每一层的梯度 31 | else: 32 | layer_param = module.weight.detach().reshape(-1, 1).numpy() # 每一层的权重 33 | index = np.full_like(layer_param, idx) # 对层进行标号 34 | layer = np.concatenate((layer_param, index), -1) 35 | layer_params.append(layer) 36 | 37 | # 拼接各层 38 | layer_params = np.concatenate(layer_params, 0) 39 | 40 | # 绘制图像 41 | ax = sns.violinplot(y=layer_params[:, 0], x=layer_params[:, 1]) 42 | ax.set(xlabel='layer', title='grad' if is_grad else 'weight') 43 | plt.show() 44 | --------------------------------------------------------------------------------