├── .gitignore ├── LICENSE ├── README.md ├── requirements.txt ├── res ├── deeplearning-with-tensorflow-book.jpg └── keras-dataset.png └── src ├── ch01 ├── 1.5.2-tf1.py ├── 1.5.2-tf2.py ├── 1.5.3-autograd.py ├── 1.5.3-cpu-gpu-time.svg └── 1.5.3-cpu-gpu.py ├── ch02 ├── 2.3-linear-model.py ├── data.csv └── 模型训练MSE下降曲线.png ├── ch03 ├── 3.8-non-linear-nn.py └── MNIST数据集的训练误差曲线.png ├── ch04 ├── 4.10-forward-prop.py ├── MNIST数据集的前向传播训练误差曲线.png └── ch04-TensorFlow基础.ipynb ├── ch05 ├── 5.8-mnist-tensorflow.py ├── MNIST测试准确率曲线.svg ├── MNIST训练误差曲线.svg └── ch05-TensorFlow进阶.ipynb ├── ch06 ├── 6.8-auto-mpg-efficency.py ├── MEA变化曲线.svg ├── ch06-神经网络.ipynb └── 特征之间的两两分布.svg ├── ch07 ├── 7.9-backward-prop.py ├── ch07-反向传播算法.ipynb ├── 数据集分布.svg ├── 网络测试准确率.svg └── 训练误差曲线.svg ├── ch08 ├── 8.2-model-assembly-training-and-testing.py ├── ch08-Keras高层接口.ipynb ├── checkpoint ├── model-savedmodel │ ├── saved_model.pb │ └── variables │ │ ├── variables.data-00000-of-00002 │ │ ├── variables.data-00001-of-00002 │ │ └── variables.index ├── model.h5 ├── weights.ckpt.data-00000-of-00002 ├── weights.ckpt.data-00001-of-00002 ├── weights.ckpt.index └── 模型性能.png ├── ch09 ├── 9.8-over-fitting-and-under-fitting.py ├── ch09-过拟合.ipynb ├── lena512color.png ├── lena512color.tiff └── output_dir │ ├── dropout │ ├── Dropout_0.png │ ├── Dropout_1.png │ ├── Dropout_2.png │ ├── Dropout_3.png │ └── Dropout_4.png │ ├── network_layers │ ├── 网络容量_2.png │ ├── 网络容量_3.png │ ├── 网络容量_4.png │ ├── 网络容量_5.png │ └── 网络容量_6.png │ ├── regularizers │ ├── 正则化_0.001.svg │ ├── 正则化_0.1.svg │ ├── 正则化_0.12.svg │ ├── 正则化_0.13.svg │ ├── 正则化_1e-05.svg │ ├── 正则化网络权值_0.001.svg │ ├── 正则化网络权值_0.1.svg │ ├── 正则化网络权值_0.12.svg │ ├── 正则化网络权值_0.13.svg │ └── 正则化网络权值_1e-05.svg │ └── 月牙形状二分类数据集分布.svg ├── ch10 ├── 10.10-cifar10-vgg13-compile.py ├── 10.10-cifar10-vgg13-output-1582916285.1641374.log ├── 10.10-cifar10-vgg13.py ├── 10.14-cifar10-resnet18.py ├── 10.4-LeNet-5.py ├── ch10-卷积神经网络.ipynb ├── cifar10-vgg13-accuracy.png └── resnet.py ├── ch11 ├── 11.11-sentiment-analysis-cell-GRU.py ├── 11.11-sentiment-analysis-cell-LSTM.py ├── 11.11-sentiment-analysis-layer-GRU.py ├── 11.11-sentiment-analysis-layer-LSTM.py ├── 11.12-sentiment-analysis-layer-LSTM-pretrain.py ├── 11.5-sentiment-analysis-RNN.py └── ch11-循环神经网络.ipynb ├── ch12 ├── 12.2-fashion-mnist-autoencoder.py ├── 12.5-fashion-mnist-vae.py ├── ae_images │ └── rec_epoch_99.png └── vae_images │ ├── epoch_99_rec.png │ └── epoch_99_sampled.png ├── ch13 ├── 13.3-faces-dcgan.py ├── 13.8-faces-wgan-gp.py ├── dataset.py └── gan.py ├── ch14 ├── 14.1.2-cartpole-v1.py ├── 14.1.5-cartpole-policy.py ├── 14.1.5-cartpole-policy.svg ├── 14.3.6-cartpole-ppo.py ├── 14.3.6-cartpole-ppo.svg ├── 14.4.7-cartpole-dqn.py └── 14.5.3-cartpole-a3c.py └── ch15 ├── pokemon.py ├── resnet.py ├── train_scratch_dense_net.py ├── train_transfer_dense_net.py └── train_transfer_dense_net.svg /.gitignore: -------------------------------------------------------------------------------- 1 | /venv 2 | /.idea -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow深度学习练习代码 2 |   龙龙（龙曲良）老师的《TensorFlow深度学习》是TensorFlow2.0入门教材之一。 3 |   本书共15章，大体上可分为4个部份：第1-3章为第1部分，主要介绍人工智能的初步认知，并引出相关问题；第4-5章为第2部分，主要介绍TensorFlow相关基础，为后续算法实现铺垫；第6-9章为第3部分，主要介绍神经网络的核心理论和共性知识，让读者理解深度学习的本质；第10-15章为模型算法应用部分，主要介绍常见的算法与模型，让读者能够学有所用。 4 |   **申明：** 所有的代码都来源于《TensorFlow深度学习》，github地址：https://github.com/dragen1860/Deep-Learning-with-TensorFlow-book。 5 | 6 | ## 使用说明 7 | 1. 本练习代码是搭配龙龙老师的《TensorFlow深度学习》一书。 8 | 2. 关于本笔记中的练习代码，已经消缺了书中代码的错误，可以很方便地执行程序。 9 | 3. 关于书中的很多图，已经写好了生成数据图的代码，在对应的目录下也有数据图。 10 | 4. 关于书中很多用jupyter notebook写的代码示例，也在对应的目录下有对应章节的ipynb文档。 11 | 5. 关于python包的版本问题，请详见requirements.txt文件，笔者采用的tensorflow-gpu==2.0.0，可以使用cpu版本，但是运行会特别慢。 12 | 6. keras模型与数据下载地址：链接：https://pan.baidu.com/s/1Rt6KYWUAQ8MWKY9UVVDtmQ 提取码：wedp 13 | 7. 相关数据集和gym包，百度网盘的下载地址：链接：https://pan.baidu.com/s/1fZ748Xz3WrgQnIaxGsrZLQ，提取码：ea6u 14 | 15 | 16 |   使用windows平台的tensorflow，将keras中的datasets和models放入到C:\\Users\\{pcUserName}\\.keras路径下，其他的数据包，在对应的练习代码中有说明。 17 | 18 | ## 选用的《TensorFlow深度学习》版本 19 | 20 | 21 | 22 | > 书名：TensorFlow深度学习
23 | > 作者：龙龙老师
24 | > 版次：2019年12月05日测试版第2版
25 | 26 | 电子书（带书签-无水印版）的百度网盘地址：链接：https://pan.baidu.com/s/1CPXZSrqVTJWHc3cYXIYjNg，提取码：mrhw 27 | 28 | ## 主要贡献者（按首字母排名） 29 | [@胡锐锋-天国之影-Relph](https://github.com/Relph1119) 30 | 31 | ## 总结 32 |   本书总共用了16天（2020年2月14日-2020年3月1日）阅读完，对TensorFlow和Keras的使用有很大的收获，其中第11、13章和第15章的scratch训练，由于电脑的显卡不好，不能完成练习，但其他章节的练习均已完成。 33 | 34 | **注意：** 如果出现以下这个错误，说明显卡的显存太低，可以将代码和数据集放到Google Colab上面执行。 35 | > tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[500,500,500] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [Op:Sub] name: sub/ 36 | 37 | ## LICENSE 38 | [GNU General Public License v3.0](https://github.com/relph1119/deeplearning-with-tensorflow-notes/blob/master/LICENSE) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.9.0 2 | astor==0.8.1 3 | attrs==19.3.0 4 | bleach==3.1.0 5 | cachetools==4.0.0 6 | certifi==2019.11.28 7 | chardet==3.0.4 8 | cloudpickle==1.3.0 9 | colorama==0.4.3 10 | cycler==0.10.0 11 | decorator==4.4.1 12 | defusedxml==0.6.0 13 | entrypoints==0.3 14 | future==0.18.2 15 | gast==0.2.2 16 | google-auth==1.11.1 17 | google-auth-oauthlib==0.4.1 18 | google-pasta==0.1.8 19 | grpcio==1.27.1 20 | -e egg=gym&subdirectory=venv\lib\site-packages\gym 21 | h5py==2.10.0 22 | idna==2.8 23 | imageio==2.8.0 24 | importlib-metadata==1.5.0 25 | ipykernel==5.1.4 26 | ipython==5.0.0 27 | ipython-genutils==0.2.0 28 | ipywidgets==7.5.1 29 | Jinja2==2.11.1 30 | joblib==0.14.1 31 | jsonschema==3.2.0 32 | jupyter==1.0.0 33 | jupyter-client==5.3.4 34 | jupyter-console==6.1.0 35 | jupyter-contrib-core==0.3.3 36 | jupyter-core==4.6.2 37 | jupyter-nbextensions-configurator==0.4.1 38 | Keras-Applications==1.0.8 39 | Keras-Preprocessing==1.1.0 40 | kiwisolver==1.1.0 41 | Markdown==3.2.1 42 | MarkupSafe==1.1.1 43 | matplotlib==3.1.3 44 | mistune==0.8.4 45 | nbconvert==5.6.1 46 | nbformat==5.0.4 47 | notebook==6.0.3 48 | numpy==1.18.1 49 | oauthlib==3.1.0 50 | opt-einsum==3.1.0 51 | pandas==1.0.1 52 | pandocfilters==1.4.2 53 | pickleshare==0.7.5 54 | Pillow==7.0.0 55 | prometheus-client==0.7.1 56 | prompt-toolkit==1.0.18 57 | protobuf==3.11.3 58 | pyasn1==0.4.8 59 | pyasn1-modules==0.2.8 60 | pyglet==1.5.0 61 | Pygments==2.5.2 62 | pyparsing==2.4.6 63 | pyrsistent==0.15.7 64 | python-dateutil==2.8.1 65 | pytz==2019.3 66 | pywin32==227 67 | pywinpty==0.5.7 68 | PyYAML==5.3 69 | pyzmq==18.1.1 70 | qtconsole==4.6.0 71 | requests==2.22.0 72 | requests-oauthlib==1.3.0 73 | rsa==4.0 74 | scikit-learn==0.22.1 75 | scipy==1.4.1 76 | seaborn==0.10.0 77 | Send2Trash==1.5.0 78 | simplegeneric==0.8.1 79 | six==1.14.0 80 | tensorboard==2.0.2 81 | tensorflow-estimator==2.0.1 82 | tensorflow-gpu==2.0.0 83 | termcolor==1.1.0 84 | terminado==0.8.3 85 | testpath==0.4.4 86 | tornado==6.0.3 87 | traitlets==4.3.3 88 | urllib3==1.25.8 89 | wcwidth==0.1.8 90 | webencodings==0.5.1 91 | Werkzeug==1.0.0 92 | widgetsnbextension==3.5.1 93 | win-unicode-console==0.5 94 | wrapt==1.11.2 95 | zipp==2.2.0 96 | -------------------------------------------------------------------------------- /res/deeplearning-with-tensorflow-book.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/res/deeplearning-with-tensorflow-book.jpg -------------------------------------------------------------------------------- /res/keras-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/res/keras-dataset.png -------------------------------------------------------------------------------- /src/ch01/1.5.2-tf1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 1.5.2-tf1.py 6 | @time: 2020/2/14 15:30 7 | @desc: 1.5.2 TensorFlow 1.x的代码 8 | """ 9 | 10 | import tensorflow.compat.v1 as tf 11 | tf.disable_v2_behavior() 12 | 13 | # 1.创建计算图阶段，此处代码需要使用 tf 1.x 版本运行 14 | # 创建 2 个输入端子， 并指定类型和名字 15 | a_ph = tf.placeholder(tf.float32, name='variable_a') 16 | b_ph = tf.placeholder(tf.float32, name='variable_b') 17 | 18 | # 创建输出端子的运算操作，并命名 19 | c_op = tf.add(a_ph, b_ph, name='variable_c') 20 | 21 | 22 | # 2.运行计算图阶段，此处代码需要使用 tf 1.x 版本运行 23 | # 创建运行环境 24 | sess = tf.InteractiveSession() 25 | # 初始化步骤也需要作为操作运行 26 | init = tf.global_variables_initializer() 27 | sess.run(init) # 运行初始化操作，完成初始化 28 | # 运行输出端子，需要给输入端子赋值 29 | c_numpy = sess.run(c_op, feed_dict={a_ph: 2., b_ph: 4.}) 30 | # 运算完输出端子才能得到数值类型的 c_numpy 31 | print('a+b=',c_numpy) -------------------------------------------------------------------------------- /src/ch01/1.5.2-tf2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 1.5.2-tf2.py 6 | @time: 2020/2/14 15:34 7 | @desc: 1.5.2 TensorFlow 2的代码 8 | """ 9 | 10 | import tensorflow as tf 11 | # 此处代码需要使用 tf 2 版本运行 12 | # 1.创建输入张量，并赋初始值 13 | a = tf.constant(2.) 14 | b = tf.constant(4.) 15 | # 2.直接计算， 并打印结果 16 | 17 | print('a+b=', a+b) -------------------------------------------------------------------------------- /src/ch01/1.5.3-autograd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 1.5.3-autograd.py 6 | @time: 2020/2/14 18:02 7 | @desc: 1.5.3 功能演示-自动梯度的代码 8 | """ 9 | 10 | import tensorflow as tf 11 | 12 | # 创建 4 个张量，并赋值 13 | a = tf.constant(1.) 14 | b = tf.constant(2.) 15 | c = tf.constant(3.) 16 | w = tf.constant(4.) 17 | 18 | with tf.GradientTape() as tape: # 构建梯度环境 19 | tape.watch([w]) # 将 w 加入梯度跟踪列表 20 | # 构建计算过程，函数表达式 21 | y = a * w ** 2 + b * w + c 22 | 23 | # 自动求导 24 | [dy_dw] = tape.gradient(y, [w]) 25 | print(dy_dw) # 打印出导数 26 | -------------------------------------------------------------------------------- /src/ch01/1.5.3-cpu-gpu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 1.5.3-cpu-gpu.py 6 | @time: 2020/2/14 17:56 7 | @desc: 1.5.3 功能演示-加速计算的代码 8 | """ 9 | 10 | import matplotlib 11 | from matplotlib import pyplot as plt 12 | 13 | # Default parameters for plots 14 | matplotlib.rcParams['font.size'] = 20 15 | matplotlib.rcParams['figure.titlesize'] = 20 16 | matplotlib.rcParams['figure.figsize'] = [9, 7] 17 | matplotlib.rcParams['font.family'] = ['STKaiti'] 18 | matplotlib.rcParams['axes.unicode_minus'] = False 19 | 20 | import tensorflow as tf 21 | import timeit 22 | 23 | # 设置 GPU 显存使用方式 24 | # 获取 GPU 设备列表 25 | gpus = tf.config.experimental.list_physical_devices('GPU') 26 | if gpus: 27 | try: 28 | # 设置 GPU 为增长式占用 29 | for gpu in gpus: 30 | tf.config.experimental.set_memory_growth(gpu, True) 31 | except RuntimeError as e: 32 | # 打印异常 33 | print(e) 34 | 35 | cpu_data = [] 36 | gpu_data = [] 37 | for n in range(8): 38 | n = 10 ** n 39 | # 创建在CPU上运算的2个矩阵 40 | with tf.device('/cpu:0'): 41 | cpu_a = tf.random.normal([1, n]) 42 | cpu_b = tf.random.normal([n, 1]) 43 | print(cpu_a.device, cpu_b.device) 44 | # 创建使用GPU运算的2个矩阵 45 | with tf.device('/gpu:0'): 46 | gpu_a = tf.random.normal([1, n]) 47 | gpu_b = tf.random.normal([n, 1]) 48 | print(gpu_a.device, gpu_b.device) 49 | 50 | 51 | def cpu_run(): 52 | with tf.device('/cpu:0'): 53 | c = tf.matmul(cpu_a, cpu_b) 54 | return c 55 | 56 | 57 | def gpu_run(): 58 | with tf.device('/gpu:0'): 59 | c = tf.matmul(gpu_a, gpu_b) 60 | return c 61 | 62 | 63 | # 第一次计算需要热身，避免将初始化阶段时间结算在内 64 | cpu_time = timeit.timeit(cpu_run, number=10) 65 | gpu_time = timeit.timeit(gpu_run, number=10) 66 | print('warmup:', cpu_time, gpu_time) 67 | # 正式计算10次，取平均时间 68 | cpu_time = timeit.timeit(cpu_run, number=10) 69 | gpu_time = timeit.timeit(gpu_run, number=10) 70 | print('run time:', cpu_time, gpu_time) 71 | cpu_data.append(cpu_time / 10) 72 | gpu_data.append(gpu_time / 10) 73 | 74 | del cpu_a, cpu_b, gpu_a, gpu_b 75 | 76 | x = [10 ** i for i in range(8)] 77 | cpu_data = [1000 * i for i in cpu_data] 78 | gpu_data = [1000 * i for i in gpu_data] 79 | plt.plot(x, cpu_data, 'C1') 80 | plt.plot(x, cpu_data, color='C1', marker='s', label='CPU') 81 | plt.plot(x, gpu_data, 'C0') 82 | plt.plot(x, gpu_data, color='C0', marker='^', label='GPU') 83 | 84 | plt.gca().set_xscale('log') 85 | plt.gca().set_yscale('log') 86 | plt.ylim([0, 100]) 87 | plt.xlabel('矩阵大小n:(1xn)@(nx1)') 88 | plt.ylabel('运算时间(ms)') 89 | plt.legend() 90 | plt.savefig('1.5.3-cpu-gpu-time.svg') 91 | -------------------------------------------------------------------------------- /src/ch02/2.3-linear-model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 2.3-linear-model.py 6 | @time: 2020/2/14 19:13 7 | @desc: 2.3 线性模型实战的代码 8 | """ 9 | 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | 13 | # Default parameters for plots 14 | plt.rcParams['font.size'] = 16 15 | plt.rcParams['font.family'] = ['STKaiti'] 16 | plt.rcParams['axes.unicode_minus'] = False 17 | 18 | # def load_data(seed): 19 | # np.random.seed(seed) 20 | # # 保存样本集的列表 21 | # data = [] 22 | # # 循环采样 100 个点 23 | # for i in range(100): 24 | # # 随机采样输入 x 25 | # x = np.random.uniform(-10., 10.) 26 | # # 采样高斯噪声 27 | # eps = np.random.normal(0., 0.01) 28 | # # 得到模型的输出 29 | # y = 1.477 * x + 0.089 + eps 30 | # # 保存样本点 31 | # data.append([x, y]) 32 | # 33 | # # 转换为 2D Numpy 数组 34 | # return np.array(data) 35 | 36 | 37 | def mse(b, w, points): 38 | # 根据当前的 w,b 参数计算均方差损失 39 | total_error = 0 40 | 41 | # 循环迭代所有点 42 | for i in range(0, len(points)): 43 | # 获得 i 号点的输入 x 44 | x = points[i, 0] 45 | # 获得 i 号点的输出 y 46 | y = points[i, 1] 47 | # 计算差的平方，并累加 48 | total_error += (y - (w * x + b)) ** 2 49 | 50 | # 将累加的误差求平均，得到均方差 51 | return total_error / float(len(points)) 52 | 53 | 54 | def step_gradient(b_current, w_current, points, lr): 55 | # 计算误差函数在所有点上的导数，并更新 w,b 56 | b_gradient = 0 57 | w_gradient = 0 58 | # 总样本数 59 | m = float(len(points)) 60 | for i in range(0, len(points)): 61 | x = points[i, 0] 62 | y = points[i, 1] 63 | # 误差函数对 b 的导数： grad_b = 2(wx+b-y)，参考公式(2.3) 64 | b_gradient += (2 / m) * ((w_current * x + b_current) - y) 65 | # 误差函数对 w 的导数： grad_w = 2(wx+b-y)*x，参考公式(2.2) 66 | w_gradient += (2 / m) * x * ((w_current * x + b_current) - y) 67 | 68 | # 根据梯度下降算法更新 w',b',其中 lr 为学习率 69 | new_b = b_current - (lr * b_gradient) 70 | new_w = w_current - (lr * w_gradient) 71 | 72 | return [new_b, new_w] 73 | 74 | 75 | def gradient_descent(points, starting_b, starting_w, lr, num_iterations): 76 | losses = [] 77 | 78 | # 循环更新 w,b 多次 79 | # b 的初始值 80 | b = starting_b 81 | # w 的初始值 82 | w = starting_w 83 | # 根据梯度下降算法更新多次 84 | for step in range(num_iterations): 85 | # 计算梯度并更新一次 86 | b, w = step_gradient(b, w, np.array(points), lr) 87 | # 计算当前的均方差，用于监控训练进度 88 | loss = mse(b, w, points) 89 | losses.append(loss) 90 | # 打印误差和实时的 w,b 值 91 | if step % 50 == 0: 92 | print(f"iteration:{step}, loss:{loss}, w:{w}, b:{b}") 93 | 94 | # 返回最后一次的 w,b 95 | return [b, w], losses 96 | 97 | 98 | def main(): 99 | # 加载训练集数据，这些数据是通过真实模型添加观测误差采样得到的 100 | data = np.genfromtxt("data.csv", delimiter=",") 101 | # 学习率 102 | lr = 0.0001 103 | # 初始化 b 为 0 104 | initial_b = 0 105 | # 初始化 w 为 0 106 | initial_w = 0 107 | num_iterations = 1000 108 | # 训练优化 1000 次，返回最优 w*,b*和训练 Loss 的下降过程 109 | [b, w], losses = gradient_descent(data, initial_b, initial_w, lr, num_iterations) 110 | # 计算最优数值解 w,b 上的均方差 111 | loss = mse(b, w, data) 112 | print(f'Final loss:{loss}, w:{w}, b:{b}') 113 | 114 | x = [i for i in range(0, 1000)] 115 | # 绘制曲线 116 | plt.plot(x, losses, 'C1') 117 | plt.plot(x, losses, color='C1', label='均方差') 118 | plt.xlabel('Epoch') 119 | plt.ylabel('MSE') 120 | plt.legend() 121 | plt.savefig('模型训练MSE下降曲线.png') 122 | plt.close() 123 | 124 | 125 | if __name__ == '__main__': 126 | main() 127 | -------------------------------------------------------------------------------- /src/ch02/data.csv: -------------------------------------------------------------------------------- 1 | 32.502345269453031,31.70700584656992 2 | 53.426804033275019,68.77759598163891 3 | 61.530358025636438,62.562382297945803 4 | 47.475639634786098,71.546632233567777 5 | 59.813207869512318,87.230925133687393 6 | 55.142188413943821,78.211518270799232 7 | 52.211796692214001,79.64197304980874 8 | 39.299566694317065,59.171489321869508 9 | 48.10504169176825,75.331242297063056 10 | 52.550014442733818,71.300879886850353 11 | 45.419730144973755,55.165677145959123 12 | 54.351634881228918,82.478846757497919 13 | 44.164049496773352,62.008923245725825 14 | 58.16847071685779,75.392870425994957 15 | 56.727208057096611,81.43619215887864 16 | 48.955888566093719,60.723602440673965 17 | 44.687196231480904,82.892503731453715 18 | 60.297326851333466,97.379896862166078 19 | 45.618643772955828,48.847153317355072 20 | 38.816817537445637,56.877213186268506 21 | 66.189816606752601,83.878564664602763 22 | 65.41605174513407,118.59121730252249 23 | 47.48120860786787,57.251819462268969 24 | 41.57564261748702,51.391744079832307 25 | 51.84518690563943,75.380651665312357 26 | 59.370822011089523,74.765564032151374 27 | 57.31000343834809,95.455052922574737 28 | 63.615561251453308,95.229366017555307 29 | 46.737619407976972,79.052406169565586 30 | 50.556760148547767,83.432071421323712 31 | 52.223996085553047,63.358790317497878 32 | 35.567830047746632,41.412885303700563 33 | 42.436476944055642,76.617341280074044 34 | 58.16454011019286,96.769566426108199 35 | 57.504447615341789,74.084130116602523 36 | 45.440530725319981,66.588144414228594 37 | 61.89622268029126,77.768482417793024 38 | 33.093831736163963,50.719588912312084 39 | 36.436009511386871,62.124570818071781 40 | 37.675654860850742,60.810246649902211 41 | 44.555608383275356,52.682983366387781 42 | 43.318282631865721,58.569824717692867 43 | 50.073145632289034,82.905981485070512 44 | 43.870612645218372,61.424709804339123 45 | 62.997480747553091,115.24415280079529 46 | 32.669043763467187,45.570588823376085 47 | 40.166899008703702,54.084054796223612 48 | 53.575077531673656,87.994452758110413 49 | 33.864214971778239,52.725494375900425 50 | 64.707138666121296,93.576118692658241 51 | 38.119824026822805,80.166275447370964 52 | 44.502538064645101,65.101711570560326 53 | 40.599538384552318,65.562301260400375 54 | 41.720676356341293,65.280886920822823 55 | 51.088634678336796,73.434641546324301 56 | 55.078095904923202,71.13972785861894 57 | 41.377726534895203,79.102829683549857 58 | 62.494697427269791,86.520538440347153 59 | 49.203887540826003,84.742697807826218 60 | 41.102685187349664,59.358850248624933 61 | 41.182016105169822,61.684037524833627 62 | 50.186389494880601,69.847604158249183 63 | 52.378446219236217,86.098291205774103 64 | 50.135485486286122,59.108839267699643 65 | 33.644706006191782,69.89968164362763 66 | 39.557901222906828,44.862490711164398 67 | 56.130388816875467,85.498067778840223 68 | 57.362052133238237,95.536686846467219 69 | 60.269214393997906,70.251934419771587 70 | 35.678093889410732,52.721734964774988 71 | 31.588116998132829,50.392670135079896 72 | 53.66093226167304,63.642398775657753 73 | 46.682228649471917,72.247251068662365 74 | 43.107820219102464,57.812512976181402 75 | 70.34607561504933,104.25710158543822 76 | 44.492855880854073,86.642020318822006 77 | 57.50453330326841,91.486778000110135 78 | 36.930076609191808,55.231660886212836 79 | 55.805733357942742,79.550436678507609 80 | 38.954769073377065,44.847124242467601 81 | 56.901214702247074,80.207523139682763 82 | 56.868900661384046,83.14274979204346 83 | 34.33312470421609,55.723489260543914 84 | 59.04974121466681,77.634182511677864 85 | 57.788223993230673,99.051414841748269 86 | 54.282328705967409,79.120646274680027 87 | 51.088719898979143,69.588897851118475 88 | 50.282836348230731,69.510503311494389 89 | 44.211741752090113,73.687564318317285 90 | 38.005488008060688,61.366904537240131 91 | 32.940479942618296,67.170655768995118 92 | 53.691639571070056,85.668203145001542 93 | 68.76573426962166,114.85387123391394 94 | 46.230966498310252,90.123572069967423 95 | 68.319360818255362,97.919821035242848 96 | 50.030174340312143,81.536990783015028 97 | 49.239765342753763,72.111832469615663 98 | 50.039575939875988,85.232007342325673 99 | 48.149858891028863,66.224957888054632 100 | 25.128484647772304,53.454394214850524 -------------------------------------------------------------------------------- /src/ch02/模型训练MSE下降曲线.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch02/模型训练MSE下降曲线.png -------------------------------------------------------------------------------- /src/ch03/3.8-non-linear-nn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 3.8-non-linear-nn.py 6 | @time: 2020/2/14 20:23 7 | @desc: 3.8 手写数字图片识别体验的代码 8 | """ 9 | 10 | import os 11 | 12 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 13 | 14 | import matplotlib.pyplot as plt 15 | import tensorflow as tf 16 | from tensorflow import keras 17 | from tensorflow.keras import layers, optimizers, datasets 18 | 19 | plt.rcParams['font.size'] = 16 20 | plt.rcParams['font.family'] = ['STKaiti'] 21 | plt.rcParams['axes.unicode_minus'] = False 22 | 23 | # 加载 MNIST 数据集 24 | (x, y), (x_val, y_val) = datasets.mnist.load_data() 25 | # 转换为浮点张量， 并缩放到-1~1 26 | x = tf.convert_to_tensor(x, dtype=tf.float32) / 255. 27 | # 转换为整形张量 28 | y = tf.convert_to_tensor(y, dtype=tf.int32) 29 | # one-hot 编码 30 | y = tf.one_hot(y, depth=10) 31 | print(x.shape, y.shape) 32 | # 构建数据集对象 33 | train_dataset = tf.data.Dataset.from_tensor_slices((x, y)) 34 | # 批量训练 35 | train_dataset = train_dataset.batch(200) 36 | 37 | # 利用 Sequential 容器封装 3 个网络层，前网络层的输出默认作为下一层的输入 38 | model = keras.Sequential([ 39 | layers.Dense(512, activation='relu'), 40 | layers.Dense(256, activation='relu'), 41 | layers.Dense(10)]) 42 | 43 | optimizer = optimizers.SGD(learning_rate=0.001) 44 | 45 | 46 | def train_epoch(epoch): 47 | # Step4.loop 48 | for step, (x, y) in enumerate(train_dataset): 49 | # 构建梯度记录环境 50 | with tf.GradientTape() as tape: 51 | # [b, 28, 28] => [b, 784] 52 | x = tf.reshape(x, (-1, 28 * 28)) 53 | # Step1. 得到模型输出 output [b, 784] => [b, 10] 54 | out = model(x) 55 | # Step2. compute loss 56 | # 计算每个样本的平均误差， [b] 57 | loss = tf.reduce_sum(tf.square(out - y)) / x.shape[0] 58 | 59 | # Step3. optimize and update w1, w2, w3, b1, b2, b3 60 | # 计算参数的梯度 w1, w2, w3, b1, b2, b3 61 | grads = tape.gradient(loss, model.trainable_variables) 62 | # w' = w - lr * grad 63 | # 更新网络参数 64 | optimizer.apply_gradients(zip(grads, model.trainable_variables)) 65 | 66 | if step % 100 == 0: 67 | print(epoch, step, 'loss:', loss.numpy()) 68 | 69 | return loss.numpy() 70 | 71 | 72 | def train(): 73 | losses = [] 74 | 75 | for epoch in range(50): 76 | loss = train_epoch(epoch) 77 | losses.append(loss) 78 | 79 | x = [i for i in range(0, 50)] 80 | # 绘制曲线 81 | plt.plot(x, losses, color='blue', marker='s', label='训练误差') 82 | plt.xlabel('Epoch') 83 | plt.ylabel('MSE') 84 | plt.legend() 85 | plt.savefig('MNIST数据集的训练误差曲线.png') 86 | plt.close() 87 | 88 | 89 | if __name__ == '__main__': 90 | train() 91 | -------------------------------------------------------------------------------- /src/ch03/MNIST数据集的训练误差曲线.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch03/MNIST数据集的训练误差曲线.png -------------------------------------------------------------------------------- /src/ch04/4.10-forward-prop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 4.10-forward-prop.py 6 | @time: 2020/2/14 23:47 7 | @desc: 4.10 前向传播实战的示例代码 8 | """ 9 | 10 | import matplotlib.pyplot as plt 11 | import tensorflow as tf 12 | import tensorflow.keras.datasets as datasets 13 | 14 | plt.rcParams['font.size'] = 16 15 | plt.rcParams['font.family'] = ['STKaiti'] 16 | plt.rcParams['axes.unicode_minus'] = False 17 | 18 | 19 | def load_data(): 20 | # 加载 MNIST 数据集 21 | (x, y), (x_val, y_val) = datasets.mnist.load_data() 22 | # 转换为浮点张量， 并缩放到-1~1 23 | x = tf.convert_to_tensor(x, dtype=tf.float32) / 255. 24 | # 转换为整形张量 25 | y = tf.convert_to_tensor(y, dtype=tf.int32) 26 | # one-hot 编码 27 | y = tf.one_hot(y, depth=10) 28 | 29 | # 改变视图， [b, 28, 28] => [b, 28*28] 30 | x = tf.reshape(x, (-1, 28 * 28)) 31 | 32 | # 构建数据集对象 33 | train_dataset = tf.data.Dataset.from_tensor_slices((x, y)) 34 | # 批量训练 35 | train_dataset = train_dataset.batch(200) 36 | return train_dataset 37 | 38 | 39 | def init_paramaters(): 40 | # 每层的张量都需要被优化，故使用 Variable 类型，并使用截断的正太分布初始化权值张量 41 | # 偏置向量初始化为 0 即可 42 | # 第一层的参数 43 | w1 = tf.Variable(tf.random.truncated_normal([784, 256], stddev=0.1)) 44 | b1 = tf.Variable(tf.zeros([256])) 45 | # 第二层的参数 46 | w2 = tf.Variable(tf.random.truncated_normal([256, 128], stddev=0.1)) 47 | b2 = tf.Variable(tf.zeros([128])) 48 | # 第三层的参数 49 | w3 = tf.Variable(tf.random.truncated_normal([128, 10], stddev=0.1)) 50 | b3 = tf.Variable(tf.zeros([10])) 51 | return w1, b1, w2, b2, w3, b3 52 | 53 | 54 | def train_epoch(epoch, train_dataset, w1, b1, w2, b2, w3, b3, lr=0.001): 55 | for step, (x, y) in enumerate(train_dataset): 56 | with tf.GradientTape() as tape: 57 | # 第一层计算， [b, 784]@[784, 256] + [256] => [b, 256] + [256] => [b,256] + [b, 256] 58 | h1 = x @ w1 + tf.broadcast_to(b1, (x.shape[0], 256)) 59 | h1 = tf.nn.relu(h1) # 通过激活函数 60 | 61 | # 第二层计算， [b, 256] => [b, 128] 62 | h2 = h1 @ w2 + b2 63 | h2 = tf.nn.relu(h2) 64 | # 输出层计算， [b, 128] => [b, 10] 65 | out = h2 @ w3 + b3 66 | 67 | # 计算网络输出与标签之间的均方差， mse = mean(sum(y-out)^2) 68 | # [b, 10] 69 | loss = tf.square(y - out) 70 | # 误差标量， mean: scalar 71 | loss = tf.reduce_mean(loss) 72 | 73 | # 自动梯度，需要求梯度的张量有[w1, b1, w2, b2, w3, b3] 74 | grads = tape.gradient(loss, [w1, b1, w2, b2, w3, b3]) 75 | 76 | # 梯度更新， assign_sub 将当前值减去参数值，原地更新 77 | w1.assign_sub(lr * grads[0]) 78 | b1.assign_sub(lr * grads[1]) 79 | w2.assign_sub(lr * grads[2]) 80 | b2.assign_sub(lr * grads[3]) 81 | w3.assign_sub(lr * grads[4]) 82 | b3.assign_sub(lr * grads[5]) 83 | 84 | if step % 100 == 0: 85 | print(epoch, step, 'loss:', loss.numpy()) 86 | 87 | return loss.numpy() 88 | 89 | 90 | def train(epochs): 91 | losses = [] 92 | train_dataset = load_data() 93 | w1, b1, w2, b2, w3, b3 = init_paramaters() 94 | for epoch in range(epochs): 95 | loss = train_epoch(epoch, train_dataset, w1, b1, w2, b2, w3, b3, lr=0.001) 96 | losses.append(loss) 97 | 98 | x = [i for i in range(0, epochs)] 99 | # 绘制曲线 100 | plt.plot(x, losses, color='blue', marker='s', label='训练') 101 | plt.xlabel('Epoch') 102 | plt.ylabel('MSE') 103 | plt.legend() 104 | plt.savefig('MNIST数据集的前向传播训练误差曲线.png') 105 | plt.close() 106 | 107 | 108 | if __name__ == '__main__': 109 | train(epochs=20) 110 | -------------------------------------------------------------------------------- /src/ch04/MNIST数据集的前向传播训练误差曲线.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch04/MNIST数据集的前向传播训练误差曲线.png -------------------------------------------------------------------------------- /src/ch05/5.8-mnist-tensorflow.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 5.8-mnist-tensorflow.py 6 | @time: 2020/2/18 14:25 7 | @desc: 5.8 MNIST测试实战代码 8 | """ 9 | import os 10 | 11 | import tensorflow as tf 12 | from matplotlib import pyplot as plt 13 | from tensorflow.keras import datasets 14 | 15 | # Default parameters for plots 16 | plt.rcParams['font.size'] = 16 17 | plt.rcParams['font.family'] = ['STKaiti'] 18 | plt.rcParams['axes.unicode_minus'] = False 19 | 20 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 21 | 22 | 23 | def preprocess(x, y): 24 | # [b, 28, 28], [b] 25 | print(x.shape, y.shape) 26 | x = tf.cast(x, dtype=tf.float32) / 255. 27 | x = tf.reshape(x, [-1, 28 * 28]) 28 | y = tf.cast(y, dtype=tf.int32) 29 | y = tf.one_hot(y, depth=10) 30 | 31 | return x, y 32 | 33 | 34 | def load_dataset(): 35 | (x, y), (x_test, y_test) = datasets.mnist.load_data() 36 | print('x:', x.shape, 'y:', y.shape, 'x test:', x_test.shape, 'y test:', y_test) 37 | 38 | batchsz = 512 39 | train_db = tf.data.Dataset.from_tensor_slices((x, y)) 40 | train_db = train_db.shuffle(1000) 41 | train_db = train_db.batch(batchsz) 42 | train_db = train_db.map(preprocess) 43 | train_db = train_db.repeat(20) 44 | 45 | test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 46 | test_db = test_db.shuffle(1000).batch(batchsz).map(preprocess) 47 | x, y = next(iter(train_db)) 48 | print('train sample:', x.shape, y.shape) 49 | 50 | return train_db, test_db 51 | 52 | 53 | def init_parameters(): 54 | # 784 => 512 55 | w1, b1 = tf.Variable(tf.random.normal([784, 256], stddev=0.1)), tf.Variable(tf.zeros([256])) 56 | # 512 => 256 57 | w2, b2 = tf.Variable(tf.random.normal([256, 128], stddev=0.1)), tf.Variable(tf.zeros([128])) 58 | # 256 => 10 59 | w3, b3 = tf.Variable(tf.random.normal([128, 10], stddev=0.1)), tf.Variable(tf.zeros([10])) 60 | return w1, b1, w2, b2, w3, b3 61 | 62 | 63 | def train(train_db, test_db, w1, b1, w2, b2, w3, b3, lr=1e-2): 64 | accs, losses = [], [] 65 | for step, (x, y) in enumerate(train_db): 66 | 67 | # [b, 28, 28] => [b, 784] 68 | x = tf.reshape(x, (-1, 784)) 69 | 70 | with tf.GradientTape() as tape: 71 | 72 | # layer1. 73 | h1 = x @ w1 + b1 74 | h1 = tf.nn.relu(h1) 75 | # layer2 76 | h2 = h1 @ w2 + b2 77 | h2 = tf.nn.relu(h2) 78 | # output 79 | out = h2 @ w3 + b3 80 | # out = tf.nn.relu(out) 81 | 82 | # compute loss 83 | # [b, 10] - [b, 10] 84 | loss = tf.square(y - out) 85 | # [b, 10] => scalar 86 | loss = tf.reduce_mean(loss) 87 | 88 | grads = tape.gradient(loss, [w1, b1, w2, b2, w3, b3]) 89 | for p, g in zip([w1, b1, w2, b2, w3, b3], grads): 90 | p.assign_sub(lr * g) 91 | 92 | # print 93 | if step % 80 == 0: 94 | print(step, 'loss:', float(loss)) 95 | losses.append(float(loss)) 96 | 97 | if step % 80 == 0: 98 | # evaluate/test 99 | total, total_correct = 0., 0 100 | 101 | for x, y in test_db: 102 | # layer1. 103 | h1 = x @ w1 + b1 104 | h1 = tf.nn.relu(h1) 105 | # layer2 106 | h2 = h1 @ w2 + b2 107 | h2 = tf.nn.relu(h2) 108 | # output 109 | out = h2 @ w3 + b3 110 | # [b, 10] => [b] 111 | pred = tf.argmax(out, axis=1) 112 | # convert one_hot y to number y 113 | y = tf.argmax(y, axis=1) 114 | # bool type 115 | correct = tf.equal(pred, y) 116 | # bool tensor => int tensor => numpy 117 | total_correct += tf.reduce_sum(tf.cast(correct, dtype=tf.int32)).numpy() 118 | total += x.shape[0] 119 | 120 | print(step, 'Evaluate Acc:', total_correct / total) 121 | 122 | accs.append(total_correct / total) 123 | return accs, losses 124 | 125 | 126 | def main(): 127 | train_db, test_db = load_dataset() 128 | w1, b1, w2, b2, w3, b3 = init_parameters() 129 | 130 | accs, losses = train(train_db, test_db, w1, b1, w2, b2, w3, b3) 131 | 132 | plt.figure() 133 | x = [i * 80 for i in range(len(losses))] 134 | plt.plot(x, losses, color='C0', marker='s', label='训练') 135 | plt.ylabel('MSE') 136 | plt.xlabel('Step') 137 | plt.legend() 138 | plt.savefig('MNIST训练误差曲线.svg') 139 | plt.close() 140 | 141 | plt.figure() 142 | plt.plot(x, accs, color='C1', marker='s', label='测试') 143 | plt.ylabel('准确率') 144 | plt.xlabel('Step') 145 | plt.legend() 146 | plt.savefig('MNIST测试准确率曲线.svg') 147 | plt.close() 148 | 149 | if __name__ == '__main__': 150 | main() 151 | -------------------------------------------------------------------------------- /src/ch06/6.8-auto-mpg-efficency.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 6.8-auto-mpg-efficency.py 6 | @time: 2020/2/24 12:43 7 | @desc: 6.8 汽车油耗预测实战的代码 8 | """ 9 | 10 | import os 11 | 12 | import matplotlib.pyplot as plt 13 | import pandas as pd 14 | import seaborn as sns 15 | import tensorflow as tf 16 | from tensorflow import keras 17 | from tensorflow.keras import layers, losses 18 | 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | def load_dataset(): 23 | # 在线下载汽车效能数据集 24 | dataset_path = keras.utils.get_file("auto-mpg.data", 25 | "http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data") 26 | 27 | # 效能（公里数每加仑），气缸数，排量，马力，重量 28 | # 加速度，型号年份，产地 29 | column_names = ['MPG', 'Cylinders', 'Displacement', 'Horsepower', 'Weight', 30 | 'Acceleration', 'Model Year', 'Origin'] 31 | raw_dataset = pd.read_csv(dataset_path, names=column_names, 32 | na_values="?", comment='\t', 33 | sep=" ", skipinitialspace=True) 34 | 35 | dataset = raw_dataset.copy() 36 | return dataset 37 | 38 | 39 | def preprocess_dataset(dataset): 40 | dataset = dataset.copy() 41 | # 统计空白数据,并清除 42 | dataset = dataset.dropna() 43 | 44 | # 处理类别型数据，其中origin列代表了类别1,2,3,分布代表产地：美国、欧洲、日本 45 | # 其弹出这一列 46 | origin = dataset.pop('Origin') 47 | # 根据origin列来写入新列 48 | dataset['USA'] = (origin == 1) * 1.0 49 | dataset['Europe'] = (origin == 2) * 1.0 50 | dataset['Japan'] = (origin == 3) * 1.0 51 | 52 | # 切分为训练集和测试集 53 | train_dataset = dataset.sample(frac=0.8, random_state=0) 54 | test_dataset = dataset.drop(train_dataset.index) 55 | 56 | return train_dataset, test_dataset 57 | 58 | 59 | def get_train_stats(train_dataset): 60 | # 查看训练集的输入X的统计数据 61 | train_stats = train_dataset.describe() 62 | train_stats.pop("MPG") 63 | train_stats = train_stats.transpose() 64 | return train_stats 65 | 66 | 67 | def norm(x, train_stats): 68 | """ 69 | 标准化数据 70 | :param x: 71 | :param train_stats: get_train_stats(train_dataset) 72 | :return: 73 | """ 74 | return (x - train_stats['mean']) / train_stats['std'] 75 | 76 | 77 | class Network(keras.Model): 78 | # 回归网络 79 | def __init__(self): 80 | super(Network, self).__init__() 81 | # 创建3个全连接层 82 | self.fc1 = layers.Dense(64, activation='relu') 83 | self.fc2 = layers.Dense(64, activation='relu') 84 | self.fc3 = layers.Dense(1) 85 | 86 | def call(self, inputs): 87 | # 依次通过3个全连接层 88 | x = self.fc1(inputs) 89 | x = self.fc2(x) 90 | x = self.fc3(x) 91 | 92 | return x 93 | 94 | 95 | def build_model(): 96 | # 创建网络 97 | model = Network() 98 | model.build(input_shape=(4, 9)) 99 | model.summary() 100 | return model 101 | 102 | 103 | def train(model, train_db, optimizer, normed_test_data, test_labels): 104 | train_mae_losses = [] 105 | test_mae_losses = [] 106 | for epoch in range(200): 107 | for step, (x, y) in enumerate(train_db): 108 | 109 | with tf.GradientTape() as tape: 110 | out = model(x) 111 | loss = tf.reduce_mean(losses.MSE(y, out)) 112 | mae_loss = tf.reduce_mean(losses.MAE(y, out)) 113 | 114 | if step % 10 == 0: 115 | print(epoch, step, float(loss)) 116 | 117 | grads = tape.gradient(loss, model.trainable_variables) 118 | optimizer.apply_gradients(zip(grads, model.trainable_variables)) 119 | 120 | train_mae_losses.append(float(mae_loss)) 121 | out = model(tf.constant(normed_test_data.values)) 122 | test_mae_losses.append(tf.reduce_mean(losses.MAE(test_labels, out))) 123 | 124 | return train_mae_losses, test_mae_losses 125 | 126 | 127 | def plot(train_mae_losses, test_mae_losses): 128 | plt.figure() 129 | plt.xlabel('Epoch') 130 | plt.ylabel('MAE') 131 | plt.plot(train_mae_losses, label='Train') 132 | plt.plot(test_mae_losses, label='Test') 133 | plt.legend() 134 | # plt.ylim([0,10]) 135 | plt.legend() 136 | plt.savefig('MEA变化曲线.svg') 137 | 138 | 139 | def main(): 140 | dataset = load_dataset() 141 | train_dataset, test_dataset = preprocess_dataset(dataset) 142 | # 统计数据 143 | sns_plot = sns.pairplot(train_dataset[["Cylinders", "Displacement", "Weight", "MPG"]], 144 | diag_kind="kde") 145 | sns_plot.savefig("特征之间的两两分布.svg") 146 | 147 | train_stats = get_train_stats(train_dataset) 148 | 149 | # 移动MPG油耗效能这一列为真实标签Y 150 | train_labels = train_dataset.pop('MPG') 151 | test_labels = test_dataset.pop('MPG') 152 | 153 | # 进行标准化 154 | normed_train_data = norm(train_dataset, train_stats) 155 | normed_test_data = norm(test_dataset, train_stats) 156 | 157 | model = build_model() 158 | optimizer = tf.keras.optimizers.RMSprop(0.001) 159 | train_db = tf.data.Dataset.from_tensor_slices((normed_train_data.values, train_labels.values)) 160 | train_db = train_db.shuffle(100).batch(32) 161 | 162 | train_mae_losses, test_mae_losses = train(model, train_db, optimizer, normed_test_data, test_labels) 163 | plot(train_mae_losses, test_mae_losses) 164 | 165 | 166 | if __name__ == '__main__': 167 | main() 168 | -------------------------------------------------------------------------------- /src/ch07/7.9-backward-prop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 7.9-backward-prop.py 6 | @time: 2020/2/24 17:32 7 | @desc: 7.9 反向传播算法实战的代码 8 | """ 9 | 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | import seaborn as sns 13 | from sklearn.datasets import make_moons 14 | from sklearn.model_selection import train_test_split 15 | 16 | plt.rcParams['font.size'] = 16 17 | plt.rcParams['font.family'] = ['STKaiti'] 18 | plt.rcParams['axes.unicode_minus'] = False 19 | 20 | 21 | def load_dataset(): 22 | # 采样点数 23 | N_SAMPLES = 2000 24 | # 测试数量比率 25 | TEST_SIZE = 0.3 26 | # 利用工具函数直接生成数据集 27 | X, y = make_moons(n_samples=N_SAMPLES, noise=0.2, random_state=100) 28 | # 将 2000 个点按着 7:3 分割为训练集和测试集 29 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=TEST_SIZE, random_state=42) 30 | return X, y, X_train, X_test, y_train, y_test 31 | 32 | 33 | def make_plot(X, y, plot_name, XX=None, YY=None, preds=None, dark=False): 34 | # 绘制数据集的分布， X 为 2D 坐标， y 为数据点的标签 35 | if (dark): 36 | plt.style.use('dark_background') 37 | else: 38 | sns.set_style("whitegrid") 39 | plt.figure(figsize=(16, 12)) 40 | axes = plt.gca() 41 | axes.set(xlabel="$x_1$", ylabel="$x_2$") 42 | plt.title(plot_name, fontsize=30) 43 | plt.subplots_adjust(left=0.20) 44 | plt.subplots_adjust(right=0.80) 45 | if XX is not None and YY is not None and preds is not None: 46 | plt.contourf(XX, YY, preds.reshape(XX.shape), 25, alpha=1, cmap=plt.cm.Spectral) 47 | plt.contour(XX, YY, preds.reshape(XX.shape), levels=[.5], cmap="Greys", vmin=0, vmax=.6) 48 | # 绘制散点图，根据标签区分颜色 49 | plt.scatter(X[:, 0], X[:, 1], c=y.ravel(), s=40, cmap=plt.cm.Spectral, edgecolors='none') 50 | plt.savefig('数据集分布.svg') 51 | plt.close() 52 | 53 | 54 | class Layer: 55 | # 全连接网络层 56 | def __init__(self, n_input, n_neurons, activation=None, weights=None, 57 | bias=None): 58 | """ 59 | :param int n_input: 输入节点数 60 | :param int n_neurons: 输出节点数 61 | :param str activation: 激活函数类型 62 | :param weights: 权值张量，默认类内部生成 63 | :param bias: 偏置，默认类内部生成 64 | """ 65 | # 通过正态分布初始化网络权值，初始化非常重要，不合适的初始化将导致网络不收敛 66 | self.weights = weights if weights is not None else np.random.randn(n_input, n_neurons) * np.sqrt(1 / n_neurons) 67 | self.bias = bias if bias is not None else np.random.rand(n_neurons) * 0.1 68 | self.activation = activation # 激活函数类型，如’sigmoid’ 69 | self.last_activation = None # 激活函数的输出值o 70 | self.error = None # 用于计算当前层的delta 变量的中间变量 71 | self.delta = None # 记录当前层的delta 变量，用于计算梯度 72 | 73 | # 网络层的前向传播函数实现如下，其中last_activation 变量用于保存当前层的输出值： 74 | def activate(self, x): 75 | # 前向传播函数 76 | r = np.dot(x, self.weights) + self.bias # X@W+b 77 | # 通过激活函数，得到全连接层的输出o 78 | self.last_activation = self._apply_activation(r) 79 | return self.last_activation 80 | 81 | # 上述代码中的self._apply_activation 函数实现了不同类型的激活函数的前向计算过程， 82 | # 尽管此处我们只使用Sigmoid 激活函数一种。代码如下： 83 | def _apply_activation(self, r): 84 | # 计算激活函数的输出 85 | if self.activation is None: 86 | return r # 无激活函数，直接返回 87 | # ReLU 激活函数 88 | elif self.activation == 'relu': 89 | return np.maximum(r, 0) 90 | # tanh 激活函数 91 | elif self.activation == 'tanh': 92 | return np.tanh(r) 93 | # sigmoid 激活函数 94 | elif self.activation == 'sigmoid': 95 | return 1 / (1 + np.exp(-r)) 96 | return r 97 | 98 | # 针对于不同类型的激活函数，它们的导数计算实现如下： 99 | def apply_activation_derivative(self, r): 100 | # 计算激活函数的导数 101 | # 无激活函数，导数为1 102 | if self.activation is None: 103 | return np.ones_like(r) 104 | # ReLU 函数的导数实现 105 | elif self.activation == 'relu': 106 | grad = np.array(r, copy=True) 107 | grad[r > 0] = 1. 108 | grad[r <= 0] = 0. 109 | return grad 110 | # tanh 函数的导数实现 111 | elif self.activation == 'tanh': 112 | return 1 - r ** 2 113 | # Sigmoid 函数的导数实现 114 | elif self.activation == 'sigmoid': 115 | return r * (1 - r) 116 | return r 117 | 118 | 119 | # 神经网络模型 120 | class NeuralNetwork: 121 | def __init__(self): 122 | self._layers = [] # 网络层对象列表 123 | 124 | def add_layer(self, layer): 125 | # 追加网络层 126 | self._layers.append(layer) 127 | 128 | # 网络的前向传播只需要循环调各个网络层对象的前向计算函数即可，代码如下： 129 | # 前向传播 130 | def feed_forward(self, X): 131 | for layer in self._layers: 132 | # 依次通过各个网络层 133 | X = layer.activate(X) 134 | return X 135 | 136 | def backpropagation(self, X, y, learning_rate): 137 | # 反向传播算法实现 138 | # 前向计算，得到输出值 139 | output = self.feed_forward(X) 140 | for i in reversed(range(len(self._layers))): # 反向循环 141 | layer = self._layers[i] # 得到当前层对象 142 | # 如果是输出层 143 | if layer == self._layers[-1]: # 对于输出层 144 | layer.error = y - output # 计算2 分类任务的均方差的导数 145 | # 关键步骤：计算最后一层的delta，参考输出层的梯度公式 146 | layer.delta = layer.error * layer.apply_activation_derivative(output) 147 | else: # 如果是隐藏层 148 | next_layer = self._layers[i + 1] # 得到下一层对象 149 | layer.error = np.dot(next_layer.weights, next_layer.delta) 150 | # 关键步骤：计算隐藏层的delta，参考隐藏层的梯度公式 151 | layer.delta = layer.error * layer.apply_activation_derivative(layer.last_activation) 152 | 153 | # 循环更新权值 154 | for i in range(len(self._layers)): 155 | layer = self._layers[i] 156 | # o_i 为上一网络层的输出 157 | o_i = np.atleast_2d(X if i == 0 else self._layers[i - 1].last_activation) 158 | # 梯度下降算法，delta 是公式中的负数，故这里用加号 159 | layer.weights += layer.delta * o_i.T * learning_rate 160 | 161 | def train(self, X_train, X_test, y_train, y_test, learning_rate, max_epochs): 162 | # 网络训练函数 163 | # one-hot 编码 164 | y_onehot = np.zeros((y_train.shape[0], 2)) 165 | y_onehot[np.arange(y_train.shape[0]), y_train] = 1 166 | 167 | # 将One-hot 编码后的真实标签与网络的输出计算均方误差，并调用反向传播函数更新网络参数，循环迭代训练集1000 遍即可 168 | mses = [] 169 | accuracys = [] 170 | for i in range(max_epochs + 1): # 训练1000 个epoch 171 | for j in range(len(X_train)): # 一次训练一个样本 172 | self.backpropagation(X_train[j], y_onehot[j], learning_rate) 173 | if i % 10 == 0: 174 | # 打印出MSE Loss 175 | mse = np.mean(np.square(y_onehot - self.feed_forward(X_train))) 176 | mses.append(mse) 177 | accuracy = self.accuracy(self.predict(X_test), y_test.flatten()) 178 | accuracys.append(accuracy) 179 | print('Epoch: #%s, MSE: %f' % (i, float(mse))) 180 | # 统计并打印准确率 181 | print('Accuracy: %.2f%%' % (accuracy * 100)) 182 | return mses, accuracys 183 | 184 | def predict(self, X): 185 | return self.feed_forward(X) 186 | 187 | def accuracy(self, X, y): 188 | return np.sum(np.equal(np.argmax(X, axis=1), y)) / y.shape[0] 189 | 190 | 191 | def main(): 192 | X, y, X_train, X_test, y_train, y_test = load_dataset() 193 | # 调用 make_plot 函数绘制数据的分布，其中 X 为 2D 坐标， y 为标签 194 | make_plot(X, y, "Classification Dataset Visualization ") 195 | plt.show() 196 | nn = NeuralNetwork() # 实例化网络类 197 | nn.add_layer(Layer(2, 25, 'sigmoid')) # 隐藏层 1, 2=>25 198 | nn.add_layer(Layer(25, 50, 'sigmoid')) # 隐藏层 2, 25=>50 199 | nn.add_layer(Layer(50, 25, 'sigmoid')) # 隐藏层 3, 50=>25 200 | nn.add_layer(Layer(25, 2, 'sigmoid')) # 输出层, 25=>2 201 | mses, accuracys = nn.train(X_train, X_test, y_train, y_test, 0.01, 1000) 202 | 203 | x = [i for i in range(0, 101, 10)] 204 | 205 | # 绘制MES曲线 206 | plt.title("MES Loss") 207 | plt.plot(x, mses[:11], color='blue') 208 | plt.xlabel('Epoch') 209 | plt.ylabel('MSE') 210 | plt.savefig('训练误差曲线.svg') 211 | plt.close() 212 | 213 | # 绘制Accuracy曲线 214 | plt.title("Accuracy") 215 | plt.plot(x, accuracys[:11], color='blue') 216 | plt.xlabel('Epoch') 217 | plt.ylabel('Accuracy') 218 | plt.savefig('网络测试准确率.svg') 219 | plt.close() 220 | 221 | 222 | if __name__ == '__main__': 223 | main() 224 | -------------------------------------------------------------------------------- /src/ch08/8.2-model-assembly-training-and-testing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 8.2-model-assembly-training-and-testing.py 6 | @time: 2020/2/25 13:37 7 | @desc: 8.2 模型装配、训练与测试的示例代码， 8 | 由于书中得到的准确值太差了，笔者调整了代码，使得准确率提高了97% 9 | """ 10 | 11 | import matplotlib.pyplot as plt 12 | import tensorflow as tf 13 | from tensorflow.keras import layers, Sequential, losses, optimizers, datasets 14 | 15 | plt.rcParams['font.size'] = 16 16 | plt.rcParams['font.family'] = ['STKaiti'] 17 | plt.rcParams['axes.unicode_minus'] = False 18 | 19 | 20 | def build_network(): 21 | # 创建 5 层的全连接网络 22 | network = Sequential([layers.Flatten(input_shape=(28, 28)), 23 | layers.Dense(256, activation='relu'), 24 | layers.Dense(128, activation='relu'), 25 | layers.Dense(64, activation='relu'), 26 | layers.Dense(32, activation='relu'), 27 | layers.Dense(10)]) 28 | network.summary() 29 | 30 | # 模型装配 31 | # 采用 Adam 优化器，学习率为 0.01;采用交叉熵损失函数，包含 Softmax 32 | # kears sparse_categorical_crossentropy说明： 33 | # from_logits=False，output为经过softmax输出的概率值。 34 | # from_logits=True，output为经过网络直接输出的logits张量。 35 | network.compile(optimizer=optimizers.Adam(learning_rate=0.01), 36 | loss=losses.CategoricalCrossentropy(from_logits=True), 37 | metrics=['accuracy'] # 设置测量指标为准确率 38 | ) 39 | 40 | return network 41 | 42 | 43 | def preprocess(x, y): 44 | # [b, 28, 28], [b] 45 | x = tf.cast(x, dtype=tf.float32) / 255. 46 | y = tf.cast(y, dtype=tf.int32) 47 | y = tf.one_hot(y, depth=10) 48 | 49 | return x, y 50 | 51 | 52 | def load_dataset(): 53 | (x, y), (x_test, y_test) = datasets.mnist.load_data() 54 | 55 | batchsz = 512 56 | train_db = tf.data.Dataset.from_tensor_slices((x, y)) 57 | train_db = train_db.shuffle(1000).map(preprocess).batch(batchsz) 58 | 59 | test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 60 | test_db = test_db.shuffle(1000).map(preprocess).batch(batchsz) 61 | 62 | return train_db, test_db 63 | 64 | 65 | def train(network, train_db, test_db, epochs=5): 66 | # 指定训练集为 train_db，验证集为 val_db,训练 5 个 epochs，每 2 个 epoch 验证一次 67 | # 返回训练轨迹信息保存在 history 对象中 68 | history = network.fit(train_db, epochs=epochs, validation_data=test_db, validation_freq=2) 69 | 70 | print(history.history) 71 | return network, history 72 | 73 | 74 | def test_one_data(network, test_db): 75 | # 加载一个 batch 的测试数据 76 | x, y = next(iter(test_db)) 77 | print('predict x:', x.shape) # 打印当前 batch 的形状 78 | out = network.predict(x) # 模型预测，预测结果保存在 out 中 79 | print(out) 80 | 81 | 82 | def test_model(network, test_db): 83 | # 模型测试，测试在 db_test 上的性能表现 84 | network.evaluate(test_db) 85 | 86 | 87 | def main(): 88 | epochs = 30 89 | train_db, test_db = load_dataset() 90 | network = build_network() 91 | network, history = train(network, train_db, test_db, epochs) 92 | test_one_data(network, test_db) 93 | test_model(network, test_db) 94 | 95 | x = range(epochs) 96 | 97 | plt.figure(figsize=(10, 6)) 98 | plt.subplots_adjust(wspace=0.5) 99 | plt.subplot(1, 2, 1) 100 | # 绘制MES曲线 101 | plt.title("训练误差曲线") 102 | plt.plot(x, history.history['loss'], color='blue') 103 | plt.xlabel('Epoch') 104 | plt.ylabel('MSE') 105 | 106 | # 绘制Accuracy曲线 107 | plt.subplot(1, 2, 2) 108 | plt.title("准确率") 109 | plt.plot(x, history.history['accuracy'], color='blue') 110 | plt.xlabel('Epoch') 111 | plt.ylabel('Accuracy') 112 | 113 | plt.savefig('模型性能.png') 114 | plt.close() 115 | 116 | 117 | if __name__ == '__main__': 118 | main() 119 | -------------------------------------------------------------------------------- /src/ch08/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "weights.ckpt" 2 | all_model_checkpoint_paths: "weights.ckpt" 3 | -------------------------------------------------------------------------------- /src/ch08/model-savedmodel/saved_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/model-savedmodel/saved_model.pb -------------------------------------------------------------------------------- /src/ch08/model-savedmodel/variables/variables.data-00000-of-00002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/model-savedmodel/variables/variables.data-00000-of-00002 -------------------------------------------------------------------------------- /src/ch08/model-savedmodel/variables/variables.data-00001-of-00002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/model-savedmodel/variables/variables.data-00001-of-00002 -------------------------------------------------------------------------------- /src/ch08/model-savedmodel/variables/variables.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/model-savedmodel/variables/variables.index -------------------------------------------------------------------------------- /src/ch08/model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/model.h5 -------------------------------------------------------------------------------- /src/ch08/weights.ckpt.data-00000-of-00002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/weights.ckpt.data-00000-of-00002 -------------------------------------------------------------------------------- /src/ch08/weights.ckpt.data-00001-of-00002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/weights.ckpt.data-00001-of-00002 -------------------------------------------------------------------------------- /src/ch08/weights.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/weights.ckpt.index -------------------------------------------------------------------------------- /src/ch08/模型性能.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch08/模型性能.png -------------------------------------------------------------------------------- /src/ch09/9.8-over-fitting-and-under-fitting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 9.8-over-fitting-and-under-fitting.py 6 | @time: 2020/2/25 21:14 7 | @desc: 9.8 过拟合问题实战的代码 8 | from mpl_toolkits.mplot3d import Axes3D 这个必须添加，解决3d报错问题 9 | """ 10 | 11 | import matplotlib.pyplot as plt 12 | # 导入数据集生成工具 13 | import numpy as np 14 | import seaborn as sns 15 | from sklearn.datasets import make_moons 16 | from sklearn.model_selection import train_test_split 17 | from tensorflow.keras import layers, Sequential, regularizers 18 | from mpl_toolkits.mplot3d import Axes3D 19 | 20 | plt.rcParams['font.size'] = 16 21 | plt.rcParams['font.family'] = ['STKaiti'] 22 | plt.rcParams['axes.unicode_minus'] = False 23 | 24 | OUTPUT_DIR = 'output_dir' 25 | N_EPOCHS = 500 26 | 27 | 28 | def load_dataset(): 29 | # 采样点数 30 | N_SAMPLES = 1000 31 | # 测试数量比率 32 | TEST_SIZE = None 33 | 34 | # 从 moon 分布中随机采样 1000 个点，并切分为训练集-测试集 35 | X, y = make_moons(n_samples=N_SAMPLES, noise=0.25, random_state=100) 36 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=TEST_SIZE, random_state=42) 37 | return X, y, X_train, X_test, y_train, y_test 38 | 39 | 40 | def make_plot(X, y, plot_name, file_name, XX=None, YY=None, preds=None, dark=False, output_dir=OUTPUT_DIR): 41 | # 绘制数据集的分布， X 为 2D 坐标， y 为数据点的标签 42 | if dark: 43 | plt.style.use('dark_background') 44 | else: 45 | sns.set_style("whitegrid") 46 | axes = plt.gca() 47 | axes.set_xlim([-2, 3]) 48 | axes.set_ylim([-1.5, 2]) 49 | axes.set(xlabel="$x_1$", ylabel="$x_2$") 50 | plt.title(plot_name, fontsize=20, fontproperties='SimHei') 51 | plt.subplots_adjust(left=0.20) 52 | plt.subplots_adjust(right=0.80) 53 | if XX is not None and YY is not None and preds is not None: 54 | plt.contourf(XX, YY, preds.reshape(XX.shape), 25, alpha=0.08, cmap=plt.cm.Spectral) 55 | plt.contour(XX, YY, preds.reshape(XX.shape), levels=[.5], cmap="Greys", vmin=0, vmax=.6) 56 | # 绘制散点图，根据标签区分颜色m=markers 57 | markers = ['o' if i == 1 else 's' for i in y.ravel()] 58 | mscatter(X[:, 0], X[:, 1], c=y.ravel(), s=20, cmap=plt.cm.Spectral, edgecolors='none', m=markers, ax=axes) 59 | # 保存矢量图 60 | plt.savefig(output_dir + '/' + file_name) 61 | plt.close() 62 | 63 | 64 | def mscatter(x, y, ax=None, m=None, **kw): 65 | import matplotlib.markers as mmarkers 66 | if not ax: ax = plt.gca() 67 | sc = ax.scatter(x, y, **kw) 68 | if (m is not None) and (len(m) == len(x)): 69 | paths = [] 70 | for marker in m: 71 | if isinstance(marker, mmarkers.MarkerStyle): 72 | marker_obj = marker 73 | else: 74 | marker_obj = mmarkers.MarkerStyle(marker) 75 | path = marker_obj.get_path().transformed( 76 | marker_obj.get_transform()) 77 | paths.append(path) 78 | sc.set_paths(paths) 79 | return sc 80 | 81 | 82 | def network_layers_influence(X_train, y_train): 83 | # 构建 5 种不同层数的网络 84 | for n in range(5): 85 | # 创建容器 86 | model = Sequential() 87 | # 创建第一层 88 | model.add(layers.Dense(8, input_dim=2, activation='relu')) 89 | # 添加 n 层，共 n+2 层 90 | for _ in range(n): 91 | model.add(layers.Dense(32, activation='relu')) 92 | # 创建最末层 93 | model.add(layers.Dense(1, activation='sigmoid')) 94 | # 模型装配与训练 95 | model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) 96 | model.fit(X_train, y_train, epochs=N_EPOCHS, verbose=1) 97 | # 绘制不同层数的网络决策边界曲线 98 | # 可视化的 x 坐标范围为[-2, 3] 99 | xx = np.arange(-2, 3, 0.01) 100 | # 可视化的 y 坐标范围为[-1.5, 2] 101 | yy = np.arange(-1.5, 2, 0.01) 102 | # 生成 x-y 平面采样网格点，方便可视化 103 | XX, YY = np.meshgrid(xx, yy) 104 | preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()]) 105 | title = "网络层数：{0}".format(2 + n) 106 | file = "网络容量_%i.png" % (2 + n) 107 | make_plot(X_train, y_train, title, file, XX, YY, preds, output_dir=OUTPUT_DIR + '/network_layers') 108 | 109 | 110 | def dropout_influence(X_train, y_train): 111 | # 构建 5 种不同数量 Dropout 层的网络 112 | for n in range(5): 113 | # 创建容器 114 | model = Sequential() 115 | # 创建第一层 116 | model.add(layers.Dense(8, input_dim=2, activation='relu')) 117 | counter = 0 118 | # 网络层数固定为 5 119 | for _ in range(5): 120 | model.add(layers.Dense(64, activation='relu')) 121 | # 添加 n 个 Dropout 层 122 | if counter < n: 123 | counter += 1 124 | model.add(layers.Dropout(rate=0.5)) 125 | 126 | # 输出层 127 | model.add(layers.Dense(1, activation='sigmoid')) 128 | # 模型装配 129 | model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) 130 | # 训练 131 | model.fit(X_train, y_train, epochs=N_EPOCHS, verbose=1) 132 | # 绘制不同 Dropout 层数的决策边界曲线 133 | # 可视化的 x 坐标范围为[-2, 3] 134 | xx = np.arange(-2, 3, 0.01) 135 | # 可视化的 y 坐标范围为[-1.5, 2] 136 | yy = np.arange(-1.5, 2, 0.01) 137 | # 生成 x-y 平面采样网格点，方便可视化 138 | XX, YY = np.meshgrid(xx, yy) 139 | preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()]) 140 | title = "无Dropout层" if n == 0 else "{0}层 Dropout层".format(n) 141 | file = "Dropout_%i.png" % n 142 | make_plot(X_train, y_train, title, file, XX, YY, preds, output_dir=OUTPUT_DIR + '/dropout') 143 | 144 | 145 | def build_model_with_regularization(_lambda): 146 | # 创建带正则化项的神经网络 147 | model = Sequential() 148 | model.add(layers.Dense(8, input_dim=2, activation='relu')) # 不带正则化项 149 | # 2-4层均是带 L2 正则化项 150 | model.add(layers.Dense(256, activation='relu', kernel_regularizer=regularizers.l2(_lambda))) 151 | model.add(layers.Dense(256, activation='relu', kernel_regularizer=regularizers.l2(_lambda))) 152 | model.add(layers.Dense(256, activation='relu', kernel_regularizer=regularizers.l2(_lambda))) 153 | # 输出层 154 | model.add(layers.Dense(1, activation='sigmoid')) 155 | model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # 模型装配 156 | return model 157 | 158 | 159 | def plot_weights_matrix(model, layer_index, plot_name, file_name, output_dir=OUTPUT_DIR): 160 | # 绘制权值范围函数 161 | # 提取指定层的权值矩阵 162 | weights = model.layers[layer_index].get_weights()[0] 163 | shape = weights.shape 164 | # 生成和权值矩阵等大小的网格坐标 165 | X = np.array(range(shape[1])) 166 | Y = np.array(range(shape[0])) 167 | X, Y = np.meshgrid(X, Y) 168 | # 绘制3D图 169 | fig = plt.figure() 170 | ax = fig.gca(projection='3d') 171 | ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0)) 172 | ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0)) 173 | ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0)) 174 | plt.title(plot_name, fontsize=20, fontproperties='SimHei') 175 | # 绘制权值矩阵范围 176 | ax.plot_surface(X, Y, weights, cmap=plt.get_cmap('rainbow'), linewidth=0) 177 | # 设置坐标轴名 178 | ax.set_xlabel('网格x坐标', fontsize=16, rotation=0, fontproperties='SimHei') 179 | ax.set_ylabel('网格y坐标', fontsize=16, rotation=0, fontproperties='SimHei') 180 | ax.set_zlabel('权值', fontsize=16, rotation=90, fontproperties='SimHei') 181 | # 保存矩阵范围图 182 | plt.savefig(output_dir + "/" + file_name + ".svg") 183 | plt.close(fig) 184 | 185 | 186 | def regularizers_influence(X_train, y_train): 187 | for _lambda in [1e-5, 1e-3, 1e-1, 0.12, 0.13]: # 设置不同的正则化系数 188 | # 创建带正则化项的模型 189 | model = build_model_with_regularization(_lambda) 190 | # 模型训练 191 | model.fit(X_train, y_train, epochs=N_EPOCHS, verbose=1) 192 | # 绘制权值范围 193 | layer_index = 2 194 | plot_title = "正则化系数：{}".format(_lambda) 195 | file_name = "正则化网络权值_" + str(_lambda) 196 | # 绘制网络权值范围图 197 | plot_weights_matrix(model, layer_index, plot_title, file_name, output_dir=OUTPUT_DIR + '/regularizers') 198 | # 绘制不同正则化系数的决策边界线 199 | # 可视化的 x 坐标范围为[-2, 3] 200 | xx = np.arange(-2, 3, 0.01) 201 | # 可视化的 y 坐标范围为[-1.5, 2] 202 | yy = np.arange(-1.5, 2, 0.01) 203 | # 生成 x-y 平面采样网格点，方便可视化 204 | XX, YY = np.meshgrid(xx, yy) 205 | preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()]) 206 | title = "正则化系数：{}".format(_lambda) 207 | file = "正则化_%g.svg" % _lambda 208 | make_plot(X_train, y_train, title, file, XX, YY, preds, output_dir=OUTPUT_DIR + '/regularizers') 209 | 210 | 211 | def main(): 212 | X, y, X_train, X_test, y_train, y_test = load_dataset() 213 | # 绘制数据集分布 214 | make_plot(X, y, None, "月牙形状二分类数据集分布.svg") 215 | # 网络层数的影响 216 | network_layers_influence(X_train, y_train) 217 | # Dropout的影响 218 | dropout_influence(X_train, y_train) 219 | # 正则化的影响 220 | regularizers_influence(X_train, y_train) 221 | 222 | 223 | if __name__ == '__main__': 224 | main() 225 | -------------------------------------------------------------------------------- /src/ch09/lena512color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/lena512color.png -------------------------------------------------------------------------------- /src/ch09/lena512color.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/lena512color.tiff -------------------------------------------------------------------------------- /src/ch09/output_dir/dropout/Dropout_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/dropout/Dropout_0.png -------------------------------------------------------------------------------- /src/ch09/output_dir/dropout/Dropout_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/dropout/Dropout_1.png -------------------------------------------------------------------------------- /src/ch09/output_dir/dropout/Dropout_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/dropout/Dropout_2.png -------------------------------------------------------------------------------- /src/ch09/output_dir/dropout/Dropout_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/dropout/Dropout_3.png -------------------------------------------------------------------------------- /src/ch09/output_dir/dropout/Dropout_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/dropout/Dropout_4.png -------------------------------------------------------------------------------- /src/ch09/output_dir/network_layers/网络容量_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/network_layers/网络容量_2.png -------------------------------------------------------------------------------- /src/ch09/output_dir/network_layers/网络容量_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/network_layers/网络容量_3.png -------------------------------------------------------------------------------- /src/ch09/output_dir/network_layers/网络容量_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/network_layers/网络容量_4.png -------------------------------------------------------------------------------- /src/ch09/output_dir/network_layers/网络容量_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/network_layers/网络容量_5.png -------------------------------------------------------------------------------- /src/ch09/output_dir/network_layers/网络容量_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch09/output_dir/network_layers/网络容量_6.png -------------------------------------------------------------------------------- /src/ch10/10.10-cifar10-vgg13-compile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 10.10-cifar10-vgg13.py 6 | @time: 2020/2/28 1:11 7 | @desc: 10.10 CIFAR10与VGG13实战的代码（装配版本） 8 | """ 9 | 10 | import os 11 | import sys 12 | import time 13 | 14 | import matplotlib.pyplot as plt 15 | import tensorflow as tf 16 | from tensorflow.keras import layers, Sequential, losses, optimizers, datasets 17 | 18 | plt.rcParams['font.size'] = 16 19 | plt.rcParams['font.family'] = ['STKaiti'] 20 | plt.rcParams['axes.unicode_minus'] = False 21 | 22 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 23 | tf.random.set_seed(2345) 24 | 25 | os.environ['TF_ENABLE_GPU_GARBAGE_COLLECTION'] = 'false' 26 | sys.stdout = open('10.10-cifar10-vgg13-output-'+str(time.time())+'.log', mode='w', encoding='utf-8') 27 | 28 | # 获取 GPU 设备列表 29 | gpus = tf.config.experimental.list_physical_devices('GPU') 30 | if gpus: 31 | try: 32 | # 设置 GPU 为增长式占用 33 | for gpu in gpus: 34 | tf.config.experimental.set_memory_growth(gpu, True) 35 | except RuntimeError as e: 36 | # 打印异常 37 | print(e) 38 | 39 | 40 | def load_dataset(): 41 | # 在线下载，加载 CIFAR10 数据集 42 | (x, y), (x_test, y_test) = datasets.cifar10.load_data() 43 | # 删除 y 的一个维度， [b,1] => [b] 44 | y = tf.squeeze(y, axis=1) 45 | y_test = tf.squeeze(y_test, axis=1) 46 | # 打印训练集和测试集的形状 47 | print(x.shape, y.shape, x_test.shape, y_test.shape) 48 | # 构建训练集对象，随机打乱，预处理，批量化 49 | train_db = tf.data.Dataset.from_tensor_slices((x, y)) 50 | train_db = train_db.shuffle(1000).map(preprocess).batch(128) 51 | # 构建测试集对象，预处理，批量化 52 | test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 53 | test_db = test_db.map(preprocess).batch(64) 54 | # 从训练集中采样一个 Batch，并观察 55 | sample = next(iter(train_db)) 56 | print('sample:', sample[0].shape, sample[1].shape, tf.reduce_min(sample[0]), tf.reduce_max(sample[0])) 57 | return train_db, test_db 58 | 59 | 60 | def preprocess(x, y): 61 | # [0~1] 62 | x = 2 * tf.cast(x, dtype=tf.float32) / 255. - 1 63 | y = tf.cast(y, dtype=tf.int32) 64 | y = tf.one_hot(y, depth=10) 65 | return x, y 66 | 67 | 68 | def build_network(): 69 | # 先创建包含多网络层的列表 70 | conv_layers = [ 71 | # Conv-Conv-Pooling 单元 1 72 | # 64 个 3x3 卷积核, 输入输出同大小 73 | layers.Conv2D(64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 74 | layers.Conv2D(64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 75 | # 高宽减半 76 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 77 | 78 | # Conv-Conv-Pooling 单元 2,输出通道提升至 128，高宽大小减半 79 | layers.Conv2D(128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 80 | layers.Conv2D(128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 81 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 82 | 83 | # Conv-Conv-Pooling 单元 3,输出通道提升至 256，高宽大小减半 84 | layers.Conv2D(256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 85 | layers.Conv2D(256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 86 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 87 | 88 | # Conv-Conv-Pooling 单元 4,输出通道提升至 512，高宽大小减半 89 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 90 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 91 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 92 | 93 | # Conv-Conv-Pooling 单元 5,输出通道提升至 512，高宽大小减半 94 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 95 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 96 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same') 97 | 98 | ] 99 | 100 | fc_layers = [ 101 | layers.Flatten(), 102 | layers.Dense(256, activation=tf.nn.relu), 103 | layers.Dense(128, activation=tf.nn.relu), 104 | layers.Dense(10, activation=None), 105 | ] 106 | 107 | conv_layers.extend(fc_layers) 108 | network = Sequential(conv_layers) 109 | network.build(input_shape=[None, 32, 32, 3]) 110 | network.summary() 111 | network.compile(optimizer=optimizers.Adam(lr=1e-4), 112 | loss=losses.CategoricalCrossentropy(from_logits=True), 113 | metrics=['accuracy'] # 设置测量指标为准确率 114 | ) 115 | 116 | return network 117 | 118 | 119 | def train(network, train_db, test_db, epoch_num): 120 | # 指定训练集为train_db，验证集为test_db，每个epoch 验证一次 121 | # 返回训练轨迹信息保存在 history 对象中 122 | history = network.fit(train_db, epochs=epoch_num, validation_data=test_db) 123 | 124 | print(history.history) 125 | return network, history 126 | 127 | 128 | def predict(network, test_db): 129 | # 模型测试，测试在 test_db 上的性能表现 130 | network.evaluate(test_db) 131 | 132 | 133 | def main(): 134 | epoch_num = 50 135 | 136 | train_db, test_db = load_dataset() 137 | network = build_network() 138 | network, history = train(network, train_db, test_db, epoch_num) 139 | predict(network, test_db) 140 | 141 | x = range(epoch_num) 142 | plt.title("准确率") 143 | plt.plot(x, history.history['accuracy'], color='blue') 144 | plt.xlabel('Epoch') 145 | plt.ylabel('Accuracy') 146 | 147 | plt.savefig('cifar10-vgg13-accuracy.png') 148 | 149 | 150 | if __name__ == '__main__': 151 | main() 152 | -------------------------------------------------------------------------------- /src/ch10/10.10-cifar10-vgg13.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 10.10-cifar10-vgg13.py 6 | @time: 2020/2/28 1:11 7 | @desc: 10.10 CIFAR10与VGG13实战的代码 8 | """ 9 | 10 | import os 11 | 12 | import matplotlib.pyplot as plt 13 | import tensorflow as tf 14 | from tensorflow.keras import layers, optimizers, datasets, Sequential 15 | 16 | plt.rcParams['font.size'] = 16 17 | plt.rcParams['font.family'] = ['STKaiti'] 18 | plt.rcParams['axes.unicode_minus'] = False 19 | 20 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 21 | tf.random.set_seed(2345) 22 | 23 | os.environ['TF_ENABLE_GPU_GARBAGE_COLLECTION'] = 'false' 24 | 25 | # 获取 GPU 设备列表 26 | gpus = tf.config.experimental.list_physical_devices('GPU') 27 | if gpus: 28 | try: 29 | # 设置 GPU 为增长式占用 30 | for gpu in gpus: 31 | tf.config.experimental.set_memory_growth(gpu, True) 32 | except RuntimeError as e: 33 | # 打印异常 34 | print(e) 35 | 36 | 37 | def load_dataset(): 38 | # 在线下载，加载 CIFAR10 数据集 39 | (x, y), (x_test, y_test) = datasets.cifar10.load_data() 40 | # 删除 y 的一个维度， [b,1] => [b] 41 | y = tf.squeeze(y, axis=1) 42 | y_test = tf.squeeze(y_test, axis=1) 43 | # 打印训练集和测试集的形状 44 | print(x.shape, y.shape, x_test.shape, y_test.shape) 45 | # 构建训练集对象，随机打乱，预处理，批量化 46 | train_db = tf.data.Dataset.from_tensor_slices((x, y)) 47 | train_db = train_db.shuffle(1000).map(preprocess).batch(128) 48 | # 构建测试集对象，预处理，批量化 49 | test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 50 | test_db = test_db.map(preprocess).batch(64) 51 | # 从训练集中采样一个 Batch，并观察 52 | sample = next(iter(train_db)) 53 | print('sample:', sample[0].shape, sample[1].shape, tf.reduce_min(sample[0]), tf.reduce_max(sample[0])) 54 | return train_db, test_db 55 | 56 | 57 | def preprocess(x, y): 58 | # [0~1] 59 | x = 2 * tf.cast(x, dtype=tf.float32) / 255. - 1 60 | y = tf.cast(y, dtype=tf.int32) 61 | return x, y 62 | 63 | 64 | def build_network(): 65 | # 先创建包含多网络层的列表 66 | conv_layers = [ # 5 units of conv + max pooling 67 | # Conv-Conv-Pooling 单元 1 68 | # 64 个 3x3 卷积核, 输入输出同大小 69 | layers.Conv2D(64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 70 | layers.Conv2D(64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 71 | # 高宽减半 72 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 73 | 74 | # Conv-Conv-Pooling 单元 2,输出通道提升至 128，高宽大小减半 75 | layers.Conv2D(128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 76 | layers.Conv2D(128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 77 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 78 | 79 | # Conv-Conv-Pooling 单元 3,输出通道提升至 256，高宽大小减半 80 | layers.Conv2D(256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 81 | layers.Conv2D(256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 82 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 83 | 84 | # Conv-Conv-Pooling 单元 4,输出通道提升至 512，高宽大小减半 85 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 86 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 87 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same'), 88 | 89 | # Conv-Conv-Pooling 单元 5,输出通道提升至 512，高宽大小减半 90 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 91 | layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu), 92 | layers.MaxPool2D(pool_size=[2, 2], strides=2, padding='same') 93 | 94 | ] 95 | 96 | # 利用前面创建的层列表构建网络容器 97 | conv_net = Sequential(conv_layers) 98 | 99 | # 创建 3 层全连接层子网络 100 | fc_net = Sequential([ 101 | layers.Dense(256, activation=tf.nn.relu), 102 | layers.Dense(128, activation=tf.nn.relu), 103 | layers.Dense(10, activation=None), 104 | ]) 105 | 106 | # build2 个子网络，并打印网络参数信息 107 | conv_net.build(input_shape=[None, 32, 32, 3]) 108 | fc_net.build(input_shape=[None, 512]) 109 | conv_net.summary() 110 | fc_net.summary() 111 | 112 | return conv_net, fc_net 113 | 114 | 115 | def train(conv_net, fc_net, train_db, optimizer, variables, epoch): 116 | for step, (x, y) in enumerate(train_db): 117 | with tf.GradientTape() as tape: 118 | # [b, 32, 32, 3] => [b, 1, 1, 512] 119 | out = conv_net(x) 120 | # flatten, => [b, 512] 121 | out = tf.reshape(out, [-1, 512]) 122 | # [b, 512] => [b, 10] 123 | logits = fc_net(out) 124 | # [b] => [b, 10] 125 | y_onehot = tf.one_hot(y, depth=10) 126 | # compute loss 127 | loss = tf.losses.categorical_crossentropy(y_onehot, logits, from_logits=True) 128 | loss = tf.reduce_mean(loss) 129 | 130 | # 对所有参数求梯度 131 | grads = tape.gradient(loss, variables) 132 | # 自动更新 133 | optimizer.apply_gradients(zip(grads, variables)) 134 | 135 | if step % 100 == 0: 136 | print(epoch, step, 'loss:', float(loss)) 137 | 138 | return conv_net, fc_net 139 | 140 | 141 | def predict(conv_net, fc_net, test_db, epoch): 142 | total_num = 0 143 | total_correct = 0 144 | for x, y in test_db: 145 | out = conv_net(x) 146 | out = tf.reshape(out, [-1, 512]) 147 | logits = fc_net(out) 148 | prob = tf.nn.softmax(logits, axis=1) 149 | pred = tf.argmax(prob, axis=1) 150 | pred = tf.cast(pred, dtype=tf.int32) 151 | 152 | correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) 153 | correct = tf.reduce_sum(correct) 154 | 155 | total_num += x.shape[0] 156 | total_correct += int(correct) 157 | 158 | acc = total_correct / total_num 159 | print(epoch, 'acc:', acc) 160 | return acc 161 | 162 | 163 | def main(): 164 | epoch_num = 50 165 | 166 | train_db, test_db = load_dataset() 167 | conv_net, fc_net = build_network() 168 | optimizer = optimizers.Adam(lr=1e-4) 169 | 170 | # 列表合并，合并 2 个子网络的参数 171 | variables = conv_net.trainable_variables + fc_net.trainable_variables 172 | 173 | accs = [] 174 | for epoch in range(epoch_num): 175 | conv_net, fc_net = train(conv_net, fc_net, train_db, optimizer, variables, epoch) 176 | acc = predict(conv_net, fc_net, test_db, epoch) 177 | accs.append(acc) 178 | 179 | x = range(epoch_num) 180 | plt.title("准确率") 181 | plt.plot(x, accs, color='blue') 182 | plt.xlabel('Epoch') 183 | plt.ylabel('Accuracy') 184 | 185 | plt.savefig('cifar10-vgg13-accuracy.png') 186 | 187 | 188 | if __name__ == '__main__': 189 | main() 190 | -------------------------------------------------------------------------------- /src/ch10/10.14-cifar10-resnet18.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 10.14-cifar10-resnet18.py 6 | @time: 2020/2/28 12:22 7 | @desc: 10.14 CIFAR10与RESNET18实战的代码 8 | """ 9 | 10 | import os 11 | 12 | import matplotlib.pyplot as plt 13 | import tensorflow as tf 14 | from tensorflow.keras import optimizers, datasets 15 | 16 | from src.ch10.resnet import resnet18 17 | 18 | plt.rcParams['font.size'] = 16 19 | plt.rcParams['font.family'] = ['STKaiti'] 20 | plt.rcParams['axes.unicode_minus'] = False 21 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 22 | tf.random.set_seed(2345) 23 | 24 | os.environ['TF_ENABLE_GPU_GARBAGE_COLLECTION'] = 'false' 25 | 26 | # 获取 GPU 设备列表 27 | gpus = tf.config.experimental.list_physical_devices('GPU') 28 | if gpus: 29 | try: 30 | # 设置 GPU 为增长式占用 31 | for gpu in gpus: 32 | tf.config.experimental.set_memory_growth(gpu, True) 33 | except RuntimeError as e: 34 | # 打印异常 35 | print(e) 36 | 37 | 38 | def preprocess(x, y): 39 | # 将数据映射到-1~1 40 | x = 2 * tf.cast(x, dtype=tf.float32) / 255. - 1 41 | # 类型转换 42 | y = tf.cast(y, dtype=tf.int32) 43 | return x, y 44 | 45 | 46 | def load_dataset(): 47 | # 加载数据集 48 | (x, y), (x_test, y_test) = datasets.cifar10.load_data() 49 | # 删除不必要的维度 50 | y = tf.squeeze(y, axis=1) 51 | y_test = tf.squeeze(y_test, axis=1) 52 | print(x.shape, y.shape, x_test.shape, y_test.shape) 53 | 54 | # 构建训练集 55 | train_db = tf.data.Dataset.from_tensor_slices((x, y)) 56 | # 随机打散，预处理，批量化 57 | train_db = train_db.shuffle(1000).map(preprocess).batch(64) 58 | 59 | # 构建测试集 60 | test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 61 | # 随机打散，预处理，批量化 62 | test_db = test_db.map(preprocess).batch(64) 63 | # 采样一个样本 64 | sample = next(iter(train_db)) 65 | print('sample:', sample[0].shape, sample[1].shape, 66 | tf.reduce_min(sample[0]), tf.reduce_max(sample[0])) 67 | return train_db, test_db 68 | 69 | 70 | def predict(model, test_db): 71 | total_num = 0 72 | total_correct = 0 73 | for x, y in test_db: 74 | logits = model(x) 75 | prob = tf.nn.softmax(logits, axis=1) 76 | pred = tf.argmax(prob, axis=1) 77 | pred = tf.cast(pred, dtype=tf.int32) 78 | 79 | correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) 80 | correct = tf.reduce_sum(correct) 81 | 82 | total_num += x.shape[0] 83 | total_correct += int(correct) 84 | 85 | acc = total_correct / total_num 86 | return acc 87 | 88 | 89 | def train(epoch, model, optimizer, train_db): 90 | for step, (x, y) in enumerate(train_db): 91 | 92 | with tf.GradientTape() as tape: 93 | # [b, 32, 32, 3] => [b, 10],前向传播 94 | logits = model(x) 95 | # [b] => [b, 10],one-hot编码 96 | y_onehot = tf.one_hot(y, depth=10) 97 | # 计算交叉熵 98 | loss = tf.losses.categorical_crossentropy(y_onehot, logits, from_logits=True) 99 | loss = tf.reduce_mean(loss) 100 | # 计算梯度信息 101 | grads = tape.gradient(loss, model.trainable_variables) 102 | # 更新网络参数 103 | optimizer.apply_gradients(zip(grads, model.trainable_variables)) 104 | 105 | if step % 50 == 0: 106 | print(epoch, step, 'loss:', float(loss)) 107 | 108 | return model 109 | 110 | 111 | def main(): 112 | epoch_num = 50 113 | 114 | train_db, test_db = load_dataset() 115 | # [b, 32, 32, 3] => [b, 1, 1, 512] 116 | # ResNet18网络 117 | model = resnet18() 118 | model.build(input_shape=(None, 32, 32, 3)) 119 | # 统计网络参数 120 | model.summary() 121 | # 构建优化器 122 | optimizer = optimizers.Adam(lr=1e-4) 123 | 124 | accs = [] 125 | # 训练epoch 126 | for epoch in range(epoch_num): 127 | model = train(epoch, model, optimizer, train_db) 128 | acc = predict(model, test_db) 129 | print(epoch, 'acc:', acc) 130 | accs.append(acc) 131 | 132 | x = range(epoch_num) 133 | plt.title("准确率") 134 | plt.plot(x, accs, color='blue') 135 | plt.xlabel('Epoch') 136 | plt.ylabel('Accuracy') 137 | 138 | plt.savefig('cifar10-resnet18-accuracy.png') 139 | 140 | 141 | if __name__ == '__main__': 142 | main() 143 | -------------------------------------------------------------------------------- /src/ch10/10.4-LeNet-5.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 10.4-LeNet-5.py 6 | @time: 2020/2/27 1:50 7 | @desc: 10.4 LeNet-5实战的代码 8 | """ 9 | import tensorflow as tf 10 | # 导入误差计算，优化器模块 11 | from tensorflow.keras import Sequential, layers, losses, datasets, optimizers 12 | 13 | # 获取 GPU 设备列表 14 | gpus = tf.config.experimental.list_physical_devices('GPU') 15 | if gpus: 16 | try: 17 | # 设置 GPU 为增长式占用 18 | for gpu in gpus: 19 | tf.config.experimental.set_memory_growth(gpu, True) 20 | except RuntimeError as e: 21 | # 打印异常 22 | print(e) 23 | 24 | 25 | def preprocess(x, y): 26 | # [b, 28, 28], [b] 27 | print(x.shape, y.shape) 28 | x = tf.cast(x, dtype=tf.float32) / 255. 29 | x = tf.reshape(x, [-1, 28, 28]) 30 | y = tf.cast(y, dtype=tf.int64) 31 | 32 | return x, y 33 | 34 | 35 | def load_dataset(): 36 | (x, y), (x_test, y_test) = datasets.mnist.load_data() 37 | 38 | batchsz = 128 39 | train_db = tf.data.Dataset.from_tensor_slices((x, y)) 40 | train_db = train_db.shuffle(1000).batch(batchsz).map(preprocess) 41 | 42 | test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 43 | test_db = test_db.shuffle(1000).batch(batchsz).map(preprocess) 44 | 45 | return train_db, test_db 46 | 47 | 48 | def build_network(): 49 | network = Sequential([ # 网络容器 50 | layers.Conv2D(6, kernel_size=3, strides=1), # 第一个卷积层, 6 个 3x3 卷积核 51 | layers.MaxPooling2D(pool_size=2, strides=2), # 高宽各减半的池化层 52 | layers.ReLU(), # 激活函数 53 | layers.Conv2D(16, kernel_size=3, strides=1), # 第二个卷积层, 16 个 3x3 卷积核 54 | layers.MaxPooling2D(pool_size=2, strides=2), # 高宽各减半的池化层 55 | layers.ReLU(), # 激活函数 56 | layers.Flatten(), # 打平层，方便全连接层处理 57 | layers.Dense(120, activation='relu'), # 全连接层， 120 个节点 58 | layers.Dense(84, activation='relu'), # 全连接层， 84 节点 59 | layers.Dense(10) # 全连接层， 10 个节点 60 | ]) 61 | # build 一次网络模型，给输入 X 的形状，其中 4 为随意给的 batchsz 62 | network.build(input_shape=(4, 28, 28, 1)) 63 | # 统计网络信息 64 | network.summary() 65 | return network 66 | 67 | 68 | def train(train_db, network, criteon, optimizer, epoch_num): 69 | for epoch in range(epoch_num): 70 | correct, total, loss = 0, 0, 0 71 | for step, (x, y) in enumerate(train_db): 72 | # 构建梯度记录环境 73 | with tf.GradientTape() as tape: 74 | # 插入通道维度， =>[b,28,28,1] 75 | x = tf.expand_dims(x, axis=3) 76 | # 前向计算，获得 10 类别的概率分布， [b, 784] => [b, 10] 77 | out = network(x) 78 | pred = tf.argmax(out, axis=-1) 79 | # 真实标签 one-hot 编码， [b] => [b, 10] 80 | y_onehot = tf.one_hot(y, depth=10) 81 | # 计算交叉熵损失函数，标量 82 | loss += criteon(y_onehot, out) 83 | # 统计预测正确数量 84 | correct += float(tf.reduce_sum(tf.cast(tf.equal(pred, y), tf.float32))) 85 | # 统计预测样本总数 86 | total += x.shape[0] 87 | 88 | 89 | # 自动计算梯度 90 | grads = tape.gradient(loss, network.trainable_variables) 91 | # 自动更新参数 92 | optimizer.apply_gradients(zip(grads, network.trainable_variables)) 93 | 94 | print(epoch, 'loss=', float(loss), 'acc=', correct / total) 95 | 96 | return network 97 | 98 | 99 | def predict(test_db, network): 100 | # 记录预测正确的数量，总样本数量 101 | correct, total = 0, 0 102 | for x, y in test_db: # 遍历所有训练集样本 103 | # 插入通道维度， =>[b,28,28,1] 104 | x = tf.expand_dims(x, axis=3) 105 | # 前向计算，获得 10 类别的预测分布， [b, 784] => [b, 10] 106 | out = network(x) 107 | # 真实的流程时先经过 softmax，再 argmax 108 | # 但是由于 softmax 不改变元素的大小相对关系，故省去 109 | pred = tf.argmax(out, axis=-1) 110 | y = tf.cast(y, tf.int64) 111 | # 统计预测正确数量 112 | correct += float(tf.reduce_sum(tf.cast(tf.equal(pred, y), tf.float32))) 113 | # 统计预测样本总数 114 | total += x.shape[0] 115 | 116 | # 计算准确率 117 | print('test acc:', correct / total) 118 | 119 | 120 | def main(): 121 | epoch_num = 30 122 | 123 | train_db, test_db = load_dataset() 124 | network = build_network() 125 | # 创建损失函数的类，在实际计算时直接调用类实例即可 126 | criteon = losses.CategoricalCrossentropy(from_logits=True) 127 | optimizer = optimizers.RMSprop(0.001) 128 | network = train(train_db, network, criteon, optimizer, epoch_num) 129 | predict(test_db, network) 130 | 131 | 132 | if __name__ == '__main__': 133 | main() 134 | -------------------------------------------------------------------------------- /src/ch10/ch10-卷积神经网络.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "collapsed": true 7 | }, 8 | "source": [ 9 | "# 卷积神经网络\n", 10 | "\n", 11 | "## 全连接网络的问题" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "name": "stdout", 21 | "output_type": "stream", 22 | "text": [ 23 | "Model: \"sequential\"\n", 24 | "_________________________________________________________________\n", 25 | "Layer (type) Output Shape Param # \n", 26 | "=================================================================\n", 27 | "dense (Dense) multiple 200960 \n", 28 | "_________________________________________________________________\n", 29 | "dense_1 (Dense) multiple 65792 \n", 30 | "_________________________________________________________________\n", 31 | "dense_2 (Dense) multiple 65792 \n", 32 | "_________________________________________________________________\n", 33 | "dense_3 (Dense) multiple 2570 \n", 34 | "=================================================================\n", 35 | "Total params: 335,114\n", 36 | "Trainable params: 335,114\n", 37 | "Non-trainable params: 0\n", 38 | "_________________________________________________________________\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "from tensorflow import keras\n", 45 | "from tensorflow.keras import layers,Sequential,losses,optimizers,datasets\n", 46 | "\n", 47 | "# 获取所有 GPU 设备列表\n", 48 | "gpus = tf.config.experimental.list_physical_devices('GPU0')\n", 49 | "if gpus:\n", 50 | " try:\n", 51 | " # 设置 GPU 为增长式占用\n", 52 | " for gpu in gpus:\n", 53 | " tf.config.experimental.set_memory_growth(gpu, True)\n", 54 | " except RuntimeError as e:\n", 55 | " # 打印异常\n", 56 | " print(e)\n", 57 | "\n", 58 | "# 创建 4 层全连接网络\n", 59 | "model = keras.Sequential([\n", 60 | " layers.Dense(256, activation='relu'),\n", 61 | " layers.Dense(256, activation='relu'),\n", 62 | " layers.Dense(256, activation='relu'),\n", 63 | " layers.Dense(10),\n", 64 | "])\n", 65 | "# build 模型，并打印模型信息\n", 66 | "model.build(input_shape=(4, 784))\n", 67 | "model.summary()" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "**卷积运算：** \n", 75 | "在信号处理领域，1D连续信号的卷积运算被定义两个函数的积分：函数$f(\\tau)$、函数$g(\\tau)$，其中$g(\\tau)$经过了翻转$g(-\\tau)$和平移后变成$g(n-\\tau)$。卷积的“卷”是指翻转平移操作，“积”是指积分运算，1D连续卷积定义为：$$(f\\otimes g)(n)=\\int_{-\\infty}^{\\infty}f(\\tau)g(n-\\tau)d\\tau$$离散卷积将积分运算换成累加运算：$$(f\\otimes g)(n)=\\sum_{\\tau=-\\infty}^{\\infty}f(\\tau)g(n-\\tau)$$" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "卷积神经层的输出尺寸$[b,h',w',c_{out}]$由卷积核的数量$c_{out}$，卷积核的大小$k$，步长$s$，填充数$p$（只考虑上下填充数量$p_h$相同，左右填充数量$p_w$相同的情况）以及输入$X$的高宽$h/w$共同决定， 它们之间的数学关系可以表达为：$$h'=\\left\\lfloor \\frac{h+2 \\cdot p_h - k}{s} \\right\\rfloor + 1 \\\\ w'=\\left\\lfloor\\frac{w+2 \\cdot p_w - k}{s} \\right\\rfloor + 1$$其中$p_h$、$p_w$分别表示高、宽方向的填充数量，$\\lfloor \\cdot \\rfloor$表示向下取整。" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "## 卷积层实现\n", 90 | "\n", 91 | "### 自定义权值" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 2, 97 | "metadata": { 98 | "scrolled": true 99 | }, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "TensorShape([2, 3, 3, 4])" 105 | ] 106 | }, 107 | "execution_count": 2, 108 | "metadata": {}, 109 | "output_type": "execute_result" 110 | } 111 | ], 112 | "source": [ 113 | "# 模拟输入， 3 通道，高宽为 5\n", 114 | "x = tf.random.normal([2,5,5,3]) \n", 115 | "# 需要根据[k,k,cin,cout]格式创建 W 张量， 4 个 3x3 大小卷积核\n", 116 | "w = tf.random.normal([3,3,3,4])\n", 117 | "# 步长为 1, padding 为 0,\n", 118 | "out = tf.nn.conv2d(x,w,strides=1,padding=[[0,0],[0,0],[0,0],[0,0]])\n", 119 | "out.shape" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "上下左右各填充一个单位，则 padding 参数设置为[[0,0],[1,1],[1,1],[0,0]]" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 3, 132 | "metadata": {}, 133 | "outputs": [ 134 | { 135 | "data": { 136 | "text/plain": [ 137 | "TensorShape([2, 5, 5, 4])" 138 | ] 139 | }, 140 | "execution_count": 3, 141 | "metadata": {}, 142 | "output_type": "execute_result" 143 | } 144 | ], 145 | "source": [ 146 | "x = tf.random.normal([2,5,5,3]) # 模拟输入， 3 通道，高宽为 5\n", 147 | "# 需要根据[k,k,cin,cout]格式创建， 4 个 3x3 大小卷积核\n", 148 | "w = tf.random.normal([3,3,3,4])\n", 149 | "# 步长为 1, padding 为 1,\n", 150 | "out = tf.nn.conv2d(x,w,strides=1,padding=[[0,0],[1,1],[1,1],[0,0]])\n", 151 | "out.shape" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 4, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "data": { 161 | "text/plain": [ 162 | "TensorShape([2, 5, 5, 4])" 163 | ] 164 | }, 165 | "execution_count": 4, 166 | "metadata": {}, 167 | "output_type": "execute_result" 168 | } 169 | ], 170 | "source": [ 171 | "x = tf.random.normal([2,5,5,3]) # 模拟输入， 3 通道，高宽为 5\n", 172 | "w = tf.random.normal([3,3,3,4]) # 4 个 3x3 大小的卷积核\n", 173 | "# 步长为,padding 设置为输出、输入同大小\n", 174 | "# 需要注意的是, padding=same 只有在 strides=1 时才是同大小\n", 175 | "out = tf.nn.conv2d(x,w,strides=1,padding='SAME')\n", 176 | "out.shape" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 5, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "data": { 186 | "text/plain": [ 187 | "TensorShape([2, 2, 2, 4])" 188 | ] 189 | }, 190 | "execution_count": 5, 191 | "metadata": {}, 192 | "output_type": "execute_result" 193 | } 194 | ], 195 | "source": [ 196 | "x = tf.random.normal([2,5,5,3])\n", 197 | "w = tf.random.normal([3,3,3,4])\n", 198 | "# 高宽先 padding 成可以整除 3 的最小整数 6，然后 6 按 3 倍减少，得到 2x2\n", 199 | "out = tf.nn.conv2d(x,w,strides=3,padding='SAME')\n", 200 | "out.shape" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 6, 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "data": { 210 | "text/plain": [ 211 | "TensorShape([2, 2, 2, 4])" 212 | ] 213 | }, 214 | "execution_count": 6, 215 | "metadata": {}, 216 | "output_type": "execute_result" 217 | } 218 | ], 219 | "source": [ 220 | "# 根据[cout]格式创建偏置向量\n", 221 | "b = tf.zeros([4])\n", 222 | "# 在卷积输出上叠加偏置向量，它会自动 broadcasting 为[b,h',w',cout]\n", 223 | "out = out + b\n", 224 | "out.shape" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "### 卷积层类" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 7, 237 | "metadata": {}, 238 | "outputs": [ 239 | { 240 | "data": { 241 | "text/plain": [ 242 | "TensorShape([2, 5, 5, 4])" 243 | ] 244 | }, 245 | "execution_count": 7, 246 | "metadata": {}, 247 | "output_type": "execute_result" 248 | } 249 | ], 250 | "source": [ 251 | "layer = layers.Conv2D(4,kernel_size=3,strides=1,padding='SAME')\n", 252 | "out = layer(x) # 前向计算\n", 253 | "out.shape # 输出张量的 shape" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 8, 259 | "metadata": { 260 | "scrolled": true 261 | }, 262 | "outputs": [ 263 | { 264 | "data": { 265 | "text/plain": [ 266 | "[,\n", 331 | " ]" 332 | ] 333 | }, 334 | "execution_count": 8, 335 | "metadata": {}, 336 | "output_type": "execute_result" 337 | } 338 | ], 339 | "source": [ 340 | "# 返回所有待优化张量列表\n", 341 | "layer.trainable_variables" 342 | ] 343 | }, 344 | { 345 | "cell_type": "markdown", 346 | "metadata": {}, 347 | "source": [ 348 | "## 卷积层变种\n", 349 | "\n", 350 | "### 空洞卷积" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 9, 356 | "metadata": {}, 357 | "outputs": [ 358 | { 359 | "data": { 360 | "text/plain": [ 361 | "TensorShape([1, 3, 3, 1])" 362 | ] 363 | }, 364 | "execution_count": 9, 365 | "metadata": {}, 366 | "output_type": "execute_result" 367 | } 368 | ], 369 | "source": [ 370 | "x = tf.random.normal([1,7,7,1]) # 模拟输入\n", 371 | "# 空洞卷积， 1 个 3x3 的卷积核\n", 372 | "layer = layers.Conv2D(1,kernel_size=3,strides=1,dilation_rate=2)\n", 373 | "out = layer(x) # 前向计算\n", 374 | "out.shape" 375 | ] 376 | }, 377 | { 378 | "cell_type": "markdown", 379 | "metadata": {}, 380 | "source": [ 381 | "### 转置卷积" 382 | ] 383 | }, 384 | { 385 | "cell_type": "code", 386 | "execution_count": 10, 387 | "metadata": {}, 388 | "outputs": [ 389 | { 390 | "data": { 391 | "text/plain": [ 392 | "" 398 | ] 399 | }, 400 | "execution_count": 10, 401 | "metadata": {}, 402 | "output_type": "execute_result" 403 | } 404 | ], 405 | "source": [ 406 | "# 创建 X 矩阵，高宽为 5x5\n", 407 | "x = tf.range(25)+1\n", 408 | "# Reshape 为合法维度的张量\n", 409 | "x = tf.reshape(x,[1,5,5,1])\n", 410 | "x = tf.cast(x, tf.float32)\n", 411 | "# 创建固定内容的卷积核矩阵\n", 412 | "w = tf.constant([[-1,2,-3.],[4,-5,6],[-7,8,-9]])\n", 413 | "# 调整为合法维度的张量\n", 414 | "w = tf.expand_dims(w,axis=2)\n", 415 | "w = tf.expand_dims(w,axis=3)\n", 416 | "# 进行普通卷积运算\n", 417 | "out = tf.nn.conv2d(x,w,strides=2,padding='VALID')\n", 418 | "out" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": 11, 424 | "metadata": { 425 | "scrolled": true 426 | }, 427 | "outputs": [ 428 | { 429 | "data": { 430 | "text/plain": [ 431 | "" 461 | ] 462 | }, 463 | "execution_count": 11, 464 | "metadata": {}, 465 | "output_type": "execute_result" 466 | } 467 | ], 468 | "source": [ 469 | "# 普通卷积的输出作为转置卷积的输入，进行转置卷积运算\n", 470 | "xx = tf.nn.conv2d_transpose(out, w, strides=2,\n", 471 | " padding='VALID',\n", 472 | " output_shape=[1,5,5,1])\n", 473 | "xx" 474 | ] 475 | }, 476 | { 477 | "cell_type": "code", 478 | "execution_count": 12, 479 | "metadata": {}, 480 | "outputs": [ 481 | { 482 | "data": { 483 | "text/plain": [ 484 | "TensorShape([1, 2, 2, 1])" 485 | ] 486 | }, 487 | "execution_count": 12, 488 | "metadata": {}, 489 | "output_type": "execute_result" 490 | } 491 | ], 492 | "source": [ 493 | "x = tf.random.normal([1,6,6,1])\n", 494 | "# 6x6 的输入经过普通卷积\n", 495 | "out = tf.nn.conv2d(x,w,strides=2,padding='VALID')\n", 496 | "out.shape" 497 | ] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": 13, 502 | "metadata": { 503 | "scrolled": true 504 | }, 505 | "outputs": [ 506 | { 507 | "data": { 508 | "text/plain": [ 509 | "" 551 | ] 552 | }, 553 | "execution_count": 13, 554 | "metadata": {}, 555 | "output_type": "execute_result" 556 | } 557 | ], 558 | "source": [ 559 | "# 恢复出 6x6 大小\n", 560 | "xx = tf.nn.conv2d_transpose(out, w, strides=2,\n", 561 | " padding='VALID',\n", 562 | " output_shape=[1,6,6,1])\n", 563 | "xx" 564 | ] 565 | }, 566 | { 567 | "cell_type": "markdown", 568 | "metadata": {}, 569 | "source": [ 570 | "**转置卷积实现**" 571 | ] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "execution_count": 14, 576 | "metadata": {}, 577 | "outputs": [ 578 | { 579 | "data": { 580 | "text/plain": [ 581 | "" 587 | ] 588 | }, 589 | "execution_count": 14, 590 | "metadata": {}, 591 | "output_type": "execute_result" 592 | } 593 | ], 594 | "source": [ 595 | "# 创建 4x4 大小的输入\n", 596 | "x = tf.range(16)+1\n", 597 | "x = tf.reshape(x,[1,4,4,1])\n", 598 | "x = tf.cast(x, tf.float32)\n", 599 | "# 创建 3x3 卷积核\n", 600 | "w = tf.constant([[-1,2,-3.],[4,-5,6],[-7,8,-9]])\n", 601 | "w = tf.expand_dims(w,axis=2)\n", 602 | "w = tf.expand_dims(w,axis=3)\n", 603 | "# 普通卷积运算\n", 604 | "out = tf.nn.conv2d(x,w,strides=1,padding='VALID')\n", 605 | "out" 606 | ] 607 | }, 608 | { 609 | "cell_type": "markdown", 610 | "metadata": {}, 611 | "source": [ 612 | "在保持strides=1， padding='VALID'，卷积核不变的情况下，我们通过卷积核w与输出out的转置卷积运算尝试恢复与输入x相同大小的高宽张量" 613 | ] 614 | }, 615 | { 616 | "cell_type": "code", 617 | "execution_count": 15, 618 | "metadata": {}, 619 | "outputs": [ 620 | { 621 | "data": { 622 | "text/plain": [ 623 | "" 628 | ] 629 | }, 630 | "execution_count": 15, 631 | "metadata": {}, 632 | "output_type": "execute_result" 633 | } 634 | ], 635 | "source": [ 636 | "# 恢复 4x4 大小的输入\n", 637 | "xx = tf.nn.conv2d_transpose(out, w, strides=1, padding='VALID',\n", 638 | " output_shape=[1,4,4,1])\n", 639 | "tf.squeeze(xx)" 640 | ] 641 | }, 642 | { 643 | "cell_type": "code", 644 | "execution_count": 16, 645 | "metadata": { 646 | "scrolled": true 647 | }, 648 | "outputs": [ 649 | { 650 | "data": { 651 | "text/plain": [ 652 | "" 672 | ] 673 | }, 674 | "execution_count": 16, 675 | "metadata": {}, 676 | "output_type": "execute_result" 677 | } 678 | ], 679 | "source": [ 680 | "# 创建转置卷积类\n", 681 | "layer = layers.Conv2DTranspose(1,kernel_size=3,strides=1,padding='VALID')\n", 682 | "xx2 = layer(out) # 通过转置卷积层\n", 683 | "xx2" 684 | ] 685 | }, 686 | { 687 | "cell_type": "markdown", 688 | "metadata": {}, 689 | "source": [ 690 | "### 分离卷积" 691 | ] 692 | }, 693 | { 694 | "cell_type": "markdown", 695 | "metadata": {}, 696 | "source": [ 697 | "那么采用分离卷积有什么优势呢？一个很明显的优势在于， 同样的输入和输出，采用Separable Convolution 的参数量约是普通卷积的$\\displaystyle \\frac{1}{3}$。" 698 | ] 699 | }, 700 | { 701 | "cell_type": "markdown", 702 | "metadata": {}, 703 | "source": [ 704 | "## 深度残差网络\n", 705 | "\n", 706 | "### ResBlock实现" 707 | ] 708 | }, 709 | { 710 | "cell_type": "code", 711 | "execution_count": 17, 712 | "metadata": {}, 713 | "outputs": [], 714 | "source": [ 715 | "class BasicBlock(layers.Layer):\n", 716 | " # 残差模块类\n", 717 | " def __init__(self, filter_num, stride=1):\n", 718 | " super(BasicBlock, self).__init__()\n", 719 | " # f(x)包含了 2 个普通卷积层，创建卷积层 1\n", 720 | " self.conv1 = layers.Conv2D(filter_num, (3, 3), strides=stride, padding='same')\n", 721 | " self.bn1 = layers.BatchNormalization()\n", 722 | " self.relu = layers.Activation('relu')\n", 723 | " # 创建卷积层 2\n", 724 | " self.conv2 = layers.Conv2D(filter_num, (3, 3), strides=1, padding='same')\n", 725 | " self.bn2 = layers.BatchNormalization()\n", 726 | " \n", 727 | " if stride != 1: # 插入 identity 层\n", 728 | " self.downsample = Sequential()\n", 729 | " self.downsample.add(layers.Conv2D(filter_num, (1, 1), strides=stride))\n", 730 | " else: # 否则，直接连接\n", 731 | " self.downsample = lambda x:x\n", 732 | " \n", 733 | " def call(self, inputs, training=None):\n", 734 | " # 前向传播函数\n", 735 | " out = self.conv1(inputs) # 通过第一个卷积层\n", 736 | " out = self.bn1(out)\n", 737 | " out = self.relu(out)\n", 738 | " out = self.conv2(out) # 通过第二个卷积层\n", 739 | " out = self.bn2(out)\n", 740 | " # 输入通过 identity()转换\n", 741 | " identity = self.downsample(inputs)\n", 742 | " # f(x)+x 运算\n", 743 | " output = layers.add([out, identity])\n", 744 | " # 再通过激活函数并返回\n", 745 | " output = tf.nn.relu(output)\n", 746 | " return output" 747 | ] 748 | }, 749 | { 750 | "cell_type": "code", 751 | "execution_count": null, 752 | "metadata": {}, 753 | "outputs": [], 754 | "source": [] 755 | } 756 | ], 757 | "metadata": { 758 | "kernelspec": { 759 | "display_name": "Python 3", 760 | "language": "python", 761 | "name": "python3" 762 | }, 763 | "language_info": { 764 | "codemirror_mode": { 765 | "name": "ipython", 766 | "version": 3 767 | }, 768 | "file_extension": ".py", 769 | "mimetype": "text/x-python", 770 | "name": "python", 771 | "nbconvert_exporter": "python", 772 | "pygments_lexer": "ipython3", 773 | "version": "3.7.2" 774 | }, 775 | "toc": { 776 | "base_numbering": 1, 777 | "nav_menu": {}, 778 | "number_sections": true, 779 | "sideBar": true, 780 | "skip_h1_title": true, 781 | "title_cell": "Table of Contents", 782 | "title_sidebar": "Contents", 783 | "toc_cell": false, 784 | "toc_position": {}, 785 | "toc_section_display": true, 786 | "toc_window_display": false 787 | } 788 | }, 789 | "nbformat": 4, 790 | "nbformat_minor": 1 791 | } 792 | -------------------------------------------------------------------------------- /src/ch10/cifar10-vgg13-accuracy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch10/cifar10-vgg13-accuracy.png -------------------------------------------------------------------------------- /src/ch10/resnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: resnet.py 6 | @time: 2020/2/28 12:25 7 | @desc: ResNet模型的代码 8 | """ 9 | 10 | import tensorflow as tf 11 | from tensorflow import keras 12 | from tensorflow.keras import layers, Sequential 13 | 14 | 15 | class BasicBlock(layers.Layer): 16 | # 残差模块 17 | def __init__(self, filter_num, stride=1): 18 | super(BasicBlock, self).__init__() 19 | # 第一个卷积单元 20 | self.conv1 = layers.Conv2D(filter_num, (3, 3), strides=stride, padding='same') 21 | self.bn1 = layers.BatchNormalization() 22 | self.relu = layers.Activation('relu') 23 | # 第二个卷积单元 24 | self.conv2 = layers.Conv2D(filter_num, (3, 3), strides=1, padding='same') 25 | self.bn2 = layers.BatchNormalization() 26 | 27 | if stride != 1: 28 | # 通过1x1卷积完成shape匹配 29 | self.downsample = Sequential() 30 | self.downsample.add(layers.Conv2D(filter_num, (1, 1), strides=stride)) 31 | else: 32 | # shape匹配，直接短接 33 | self.downsample = lambda x: x 34 | 35 | def call(self, inputs, training=None): 36 | 37 | # [b, h, w, c]，通过第一个卷积单元 38 | out = self.conv1(inputs) 39 | out = self.bn1(out) 40 | out = self.relu(out) 41 | # 通过第二个卷积单元 42 | out = self.conv2(out) 43 | out = self.bn2(out) 44 | # 通过identity模块 45 | identity = self.downsample(inputs) 46 | # 2条路径输出直接相加 47 | output = layers.add([out, identity]) 48 | output = tf.nn.relu(output) # 激活函数 49 | 50 | return output 51 | 52 | 53 | class ResNet(keras.Model): 54 | # 通用的ResNet实现类 55 | def __init__(self, layer_dims, num_classes=10): # [2, 2, 2, 2] 56 | super(ResNet, self).__init__() 57 | # 根网络，预处理 58 | self.stem = Sequential([layers.Conv2D(64, (3, 3), strides=(1, 1)), 59 | layers.BatchNormalization(), 60 | layers.Activation('relu'), 61 | layers.MaxPool2D(pool_size=(2, 2), strides=(1, 1), padding='same') 62 | ]) 63 | # 堆叠4个Block，每个block包含了多个BasicBlock,设置步长不一样 64 | self.layer1 = self.build_resblock(64, layer_dims[0]) 65 | self.layer2 = self.build_resblock(128, layer_dims[1], stride=2) 66 | self.layer3 = self.build_resblock(256, layer_dims[2], stride=2) 67 | self.layer4 = self.build_resblock(512, layer_dims[3], stride=2) 68 | 69 | # 通过Pooling层将高宽降低为1x1 70 | self.avgpool = layers.GlobalAveragePooling2D() 71 | # 最后连接一个全连接层分类 72 | self.fc = layers.Dense(num_classes) 73 | 74 | def call(self, inputs, training=None): 75 | # 通过根网络 76 | x = self.stem(inputs) 77 | # 一次通过4个模块 78 | x = self.layer1(x) 79 | x = self.layer2(x) 80 | x = self.layer3(x) 81 | x = self.layer4(x) 82 | 83 | # 通过池化层 84 | x = self.avgpool(x) 85 | # 通过全连接层 86 | x = self.fc(x) 87 | 88 | return x 89 | 90 | def build_resblock(self, filter_num, blocks, stride=1): 91 | # 辅助函数，堆叠filter_num个BasicBlock 92 | res_blocks = Sequential() 93 | # 只有第一个BasicBlock的步长可能不为1，实现下采样 94 | res_blocks.add(BasicBlock(filter_num, stride)) 95 | 96 | for _ in range(1, blocks): # 其他BasicBlock步长都为1 97 | res_blocks.add(BasicBlock(filter_num, stride=1)) 98 | 99 | return res_blocks 100 | 101 | 102 | def resnet18(): 103 | # 通过调整模块内部BasicBlock的数量和配置实现不同的ResNet 104 | return ResNet([2, 2, 2, 2]) 105 | 106 | 107 | def resnet34(): 108 | # 通过调整模块内部BasicBlock的数量和配置实现不同的ResNet 109 | return ResNet([3, 4, 6, 3]) 110 | -------------------------------------------------------------------------------- /src/ch11/11.11-sentiment-analysis-cell-GRU.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 11.11-sentiment-analysis-cell-GRU.py 6 | @time: 2020/2/28 16:25 7 | @desc: 11.5 GRU情感分类问题实战的代码（cell方式） 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow import keras 15 | from tensorflow.keras import layers, losses, optimizers, Sequential 16 | 17 | tf.random.set_seed(22) 18 | np.random.seed(22) 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | def load_dataset(batchsz, total_words, max_review_len): 23 | # 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词 24 | (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words) 25 | print(x_train.shape, len(x_train[0]), y_train.shape) 26 | print(x_test.shape, len(x_test[0]), y_test.shape) 27 | 28 | # x_train:[b, 80] 29 | # x_test: [b, 80] 30 | # 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充 31 | x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len) 32 | x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len) 33 | # 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch 34 | db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 35 | db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True) 36 | db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 37 | db_test = db_test.batch(batchsz, drop_remainder=True) 38 | print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train)) 39 | print('x_test shape:', x_test.shape) 40 | return db_train, db_test 41 | 42 | 43 | class MyRNN(keras.Model): 44 | # Cell方式构建多层网络 45 | def __init__(self, units, batchsz, total_words, embedding_len, max_review_len): 46 | super(MyRNN, self).__init__() 47 | # [b, 64]，构建Cell初始化状态向量，重复使用 48 | self.state0 = [tf.zeros([batchsz, units])] 49 | self.state1 = [tf.zeros([batchsz, units])] 50 | # 词向量编码 [b, 80] => [b, 80, 100] 51 | self.embedding = layers.Embedding(total_words, embedding_len, 52 | input_length=max_review_len) 53 | # 构建2个Cell 54 | self.rnn_cell0 = layers.GRUCell(units, dropout=0.5) 55 | self.rnn_cell1 = layers.GRUCell(units, dropout=0.5) 56 | # 构建分类网络，用于将CELL的输出特征进行分类，2分类 57 | # [b, 80, 100] => [b, 64] => [b, 1] 58 | self.outlayer = Sequential([ 59 | layers.Dense(units), 60 | layers.Dropout(rate=0.5), 61 | layers.ReLU(), 62 | layers.Dense(1)]) 63 | 64 | def call(self, inputs, training=None): 65 | x = inputs # [b, 80] 66 | # embedding: [b, 80] => [b, 80, 100] 67 | x = self.embedding(x) 68 | # rnn cell compute,[b, 80, 100] => [b, 64] 69 | state0 = self.state0 70 | state1 = self.state1 71 | for word in tf.unstack(x, axis=1): # word: [b, 100] 72 | out0, state0 = self.rnn_cell0(word, state0, training) 73 | out1, state1 = self.rnn_cell1(out0, state1, training) 74 | # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1] 75 | x = self.outlayer(out1, training) 76 | # p(y is pos|x) 77 | prob = tf.sigmoid(x) 78 | 79 | return prob 80 | 81 | 82 | def main(): 83 | batchsz = 128 # 批量大小 84 | total_words = 10000 # 词汇表大小N_vocab 85 | embedding_len = 100 # 词向量特征长度f 86 | max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充 87 | 88 | db_train, db_test = load_dataset(batchsz, total_words, max_review_len) 89 | 90 | units = 64 # RNN状态向量长度f 91 | epochs = 20 # 训练epochs 92 | 93 | model = MyRNN(units, batchsz, total_words, embedding_len, max_review_len) 94 | # 装配 95 | model.compile(optimizer=optimizers.RMSprop(0.001), 96 | loss=losses.BinaryCrossentropy(), 97 | metrics=['accuracy'], 98 | experimental_run_tf_function=False) 99 | # 训练和验证 100 | model.fit(db_train, epochs=epochs, validation_data=db_test) 101 | # 测试 102 | model.evaluate(db_test) 103 | 104 | 105 | if __name__ == '__main__': 106 | main() 107 | -------------------------------------------------------------------------------- /src/ch11/11.11-sentiment-analysis-cell-LSTM.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 11.11-sentiment-analysis-cell-LSTM.py 6 | @time: 2020/2/28 16:25 7 | @desc: 11.11 LSTM情感分类问题实战的代码（cell方式） 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow import keras 15 | from tensorflow.keras import layers, losses, optimizers, Sequential 16 | 17 | tf.random.set_seed(22) 18 | np.random.seed(22) 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | def load_dataset(batchsz, total_words, max_review_len): 23 | # 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词 24 | (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words) 25 | print(x_train.shape, len(x_train[0]), y_train.shape) 26 | print(x_test.shape, len(x_test[0]), y_test.shape) 27 | 28 | # x_train:[b, 80] 29 | # x_test: [b, 80] 30 | # 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充 31 | x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len) 32 | x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len) 33 | # 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch 34 | db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 35 | db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True) 36 | db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 37 | db_test = db_test.batch(batchsz, drop_remainder=True) 38 | print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train)) 39 | print('x_test shape:', x_test.shape) 40 | return db_train, db_test 41 | 42 | 43 | class MyRNN(keras.Model): 44 | # Cell方式构建多层网络 45 | def __init__(self, units, batchsz, total_words, embedding_len, max_review_len): 46 | super(MyRNN, self).__init__() 47 | # [b, 64]，构建Cell初始化状态向量，重复使用 48 | self.state0 = [tf.zeros([batchsz, units]), tf.zeros([batchsz, units])] 49 | self.state1 = [tf.zeros([batchsz, units]), tf.zeros([batchsz, units])] 50 | # 词向量编码 [b, 80] => [b, 80, 100] 51 | self.embedding = layers.Embedding(total_words, embedding_len, 52 | input_length=max_review_len) 53 | # 构建2个Cell 54 | self.rnn_cell0 = layers.LSTMCell(units, dropout=0.5) 55 | self.rnn_cell1 = layers.LSTMCell(units, dropout=0.5) 56 | # 构建分类网络，用于将CELL的输出特征进行分类，2分类 57 | # [b, 80, 100] => [b, 64] => [b, 1] 58 | self.outlayer = Sequential([ 59 | layers.Dense(units), 60 | layers.Dropout(rate=0.5), 61 | layers.ReLU(), 62 | layers.Dense(1)]) 63 | 64 | def call(self, inputs, training=None): 65 | x = inputs # [b, 80] 66 | # embedding: [b, 80] => [b, 80, 100] 67 | x = self.embedding(x) 68 | # rnn cell compute,[b, 80, 100] => [b, 64] 69 | state0 = self.state0 70 | state1 = self.state1 71 | for word in tf.unstack(x, axis=1): # word: [b, 100] 72 | out0, state0 = self.rnn_cell0(word, state0, training) 73 | out1, state1 = self.rnn_cell1(out0, state1, training) 74 | # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1] 75 | x = self.outlayer(out1, training) 76 | # p(y is pos|x) 77 | prob = tf.sigmoid(x) 78 | 79 | return prob 80 | 81 | 82 | def main(): 83 | batchsz = 128 # 批量大小 84 | total_words = 10000 # 词汇表大小N_vocab 85 | embedding_len = 100 # 词向量特征长度f 86 | max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充 87 | 88 | db_train, db_test = load_dataset(batchsz, total_words, max_review_len) 89 | 90 | units = 64 # RNN状态向量长度f 91 | epochs = 20 # 训练epochs 92 | 93 | model = MyRNN(units, batchsz, total_words, embedding_len, max_review_len) 94 | # 装配 95 | model.compile(optimizer=optimizers.RMSprop(0.001), 96 | loss=losses.BinaryCrossentropy(), 97 | metrics=['accuracy'], 98 | experimental_run_tf_function=False) 99 | # 训练和验证 100 | model.fit(db_train, epochs=epochs, validation_data=db_test) 101 | # 测试 102 | model.evaluate(db_test) 103 | 104 | 105 | if __name__ == '__main__': 106 | main() 107 | -------------------------------------------------------------------------------- /src/ch11/11.11-sentiment-analysis-layer-GRU.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 11.11-sentiment-analysis-layer-GRU.py 6 | @time: 2020/2/28 16:25 7 | @desc: 11.11 GRU情感分类问题实战的代码（layer方式） 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow import keras 15 | from tensorflow.keras import layers, losses, optimizers, Sequential 16 | 17 | tf.random.set_seed(22) 18 | np.random.seed(22) 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | def load_dataset(batchsz, total_words, max_review_len): 23 | # 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词 24 | (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words) 25 | print(x_train.shape, len(x_train[0]), y_train.shape) 26 | print(x_test.shape, len(x_test[0]), y_test.shape) 27 | 28 | # x_train:[b, 80] 29 | # x_test: [b, 80] 30 | # 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充 31 | x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len) 32 | x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len) 33 | # 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch 34 | db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 35 | db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True) 36 | db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 37 | db_test = db_test.batch(batchsz, drop_remainder=True) 38 | print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train)) 39 | print('x_test shape:', x_test.shape) 40 | return db_train, db_test 41 | 42 | 43 | class MyRNN(keras.Model): 44 | # Cell方式构建多层网络 45 | def __init__(self, units, total_words, embedding_len, max_review_len): 46 | super(MyRNN, self).__init__() 47 | # 词向量编码 [b, 80] => [b, 80, 100] 48 | self.embedding = layers.Embedding(total_words, embedding_len, 49 | input_length=max_review_len) 50 | # 构建RNN 51 | self.rnn = Sequential([ 52 | layers.GRU(units, dropout=0.5, return_sequences=True), 53 | layers.GRU(units, dropout=0.5) 54 | ]) 55 | 56 | # 构建分类网络，用于将CELL的输出特征进行分类，2分类 57 | # [b, 80, 100] => [b, 64] => [b, 1] 58 | self.outlayer = Sequential([ 59 | layers.Dense(32), 60 | layers.Dropout(rate=0.5), 61 | layers.ReLU(), 62 | layers.Dense(1)]) 63 | 64 | def call(self, inputs, training=None): 65 | x = inputs # [b, 80] 66 | # embedding: [b, 80] => [b, 80, 100] 67 | x = self.embedding(x) 68 | # rnn cell compute,[b, 80, 100] => [b, 64] 69 | x = self.rnn(x) 70 | # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1] 71 | x = self.outlayer(x, training) 72 | # p(y is pos|x) 73 | prob = tf.sigmoid(x) 74 | 75 | return prob 76 | 77 | 78 | def main(): 79 | batchsz = 128 # 批量大小 80 | total_words = 10000 # 词汇表大小N_vocab 81 | embedding_len = 100 # 词向量特征长度f 82 | max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充 83 | 84 | db_train, db_test = load_dataset(batchsz, total_words, max_review_len) 85 | 86 | units = 32 # RNN状态向量长度f 87 | epochs = 20 # 训练epochs 88 | 89 | model = MyRNN(units, total_words, embedding_len, max_review_len) 90 | # 装配 91 | model.compile(optimizer=optimizers.RMSprop(0.001), 92 | loss=losses.BinaryCrossentropy(), 93 | metrics=['accuracy'], 94 | experimental_run_tf_function=False) 95 | # 训练和验证 96 | model.fit(db_train, epochs=epochs, validation_data=db_test) 97 | # 测试 98 | model.evaluate(db_test) 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /src/ch11/11.11-sentiment-analysis-layer-LSTM.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 11.11-sentiment-analysis-layer-LSTM.py 6 | @time: 2020/2/28 16:25 7 | @desc: 11.11 LSTM情感分类问题实战的代码（layer方式） 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow import keras 15 | from tensorflow.keras import layers, losses, optimizers, Sequential 16 | 17 | tf.random.set_seed(22) 18 | np.random.seed(22) 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | def load_dataset(batchsz, total_words, max_review_len): 23 | # 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词 24 | (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words) 25 | print(x_train.shape, len(x_train[0]), y_train.shape) 26 | print(x_test.shape, len(x_test[0]), y_test.shape) 27 | 28 | # x_train:[b, 80] 29 | # x_test: [b, 80] 30 | # 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充 31 | x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len) 32 | x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len) 33 | # 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch 34 | db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 35 | db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True) 36 | db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 37 | db_test = db_test.batch(batchsz, drop_remainder=True) 38 | print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train)) 39 | print('x_test shape:', x_test.shape) 40 | return db_train, db_test 41 | 42 | 43 | class MyRNN(keras.Model): 44 | # Cell方式构建多层网络 45 | def __init__(self, units, total_words, embedding_len, max_review_len): 46 | super(MyRNN, self).__init__() 47 | # 词向量编码 [b, 80] => [b, 80, 100] 48 | self.embedding = layers.Embedding(total_words, embedding_len, 49 | input_length=max_review_len) 50 | # 构建RNN 51 | self.rnn = Sequential([ 52 | layers.LSTM(units, dropout=0.5, return_sequences=True), 53 | layers.LSTM(units, dropout=0.5) 54 | ]) 55 | 56 | # 构建分类网络，用于将CELL的输出特征进行分类，2分类 57 | # [b, 80, 100] => [b, 64] => [b, 1] 58 | self.outlayer = Sequential([ 59 | layers.Dense(32), 60 | layers.Dropout(rate=0.5), 61 | layers.ReLU(), 62 | layers.Dense(1)]) 63 | 64 | def call(self, inputs, training=None): 65 | x = inputs # [b, 80] 66 | # embedding: [b, 80] => [b, 80, 100] 67 | x = self.embedding(x) 68 | # rnn cell compute,[b, 80, 100] => [b, 64] 69 | x = self.rnn(x) 70 | # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1] 71 | x = self.outlayer(x, training) 72 | # p(y is pos|x) 73 | prob = tf.sigmoid(x) 74 | 75 | return prob 76 | 77 | 78 | def main(): 79 | batchsz = 128 # 批量大小 80 | total_words = 10000 # 词汇表大小N_vocab 81 | embedding_len = 100 # 词向量特征长度f 82 | max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充 83 | 84 | db_train, db_test = load_dataset(batchsz, total_words, max_review_len) 85 | 86 | units = 32 # RNN状态向量长度f 87 | epochs = 20 # 训练epochs 88 | 89 | model = MyRNN(units, total_words, embedding_len, max_review_len) 90 | # 装配 91 | model.compile(optimizer=optimizers.RMSprop(0.001), 92 | loss=losses.BinaryCrossentropy(), 93 | metrics=['accuracy'], 94 | experimental_run_tf_function=False) 95 | # 训练和验证 96 | model.fit(db_train, epochs=epochs, validation_data=db_test) 97 | # 测试 98 | model.evaluate(db_test) 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /src/ch11/11.12-sentiment-analysis-layer-LSTM-pretrain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 11.12-sentiment-analysis-layer-LSTM-pretrain.py 6 | @time: 2020/2/28 16:25 7 | @desc: 11.12 预训练的词向量的代码（LSTM layer方式） 8 | 将glove.6B文件夹放到ch11目录下，读取的glove路径为/ch11/glove.6B/glove.6B.100d.txt 9 | """ 10 | 11 | import os 12 | 13 | import numpy as np 14 | import tensorflow as tf 15 | from tensorflow import keras 16 | from tensorflow.keras import layers, losses, optimizers, Sequential 17 | 18 | tf.random.set_seed(22) 19 | np.random.seed(22) 20 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 21 | 22 | 23 | def load_dataset(batchsz, total_words, max_review_len, embedding_len): 24 | # 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词 25 | (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words) 26 | print(x_train.shape, len(x_train[0]), y_train.shape) 27 | print(x_test.shape, len(x_test[0]), y_test.shape) 28 | 29 | # x_train:[b, 80] 30 | # x_test: [b, 80] 31 | # 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充 32 | x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len) 33 | x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len) 34 | # 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch 35 | db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 36 | db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True) 37 | db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 38 | db_test = db_test.batch(batchsz, drop_remainder=True) 39 | print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train)) 40 | print('x_test shape:', x_test.shape) 41 | return db_train, db_test 42 | 43 | 44 | def pretrain_embeding_matrix(embedding_len, total_words): 45 | # 数字编码表 46 | word_index = keras.datasets.imdb.get_word_index() 47 | 48 | word_index = {k: (v + 3) for k, v in word_index.items()} 49 | word_index[""] = 0 50 | word_index[""] = 1 51 | word_index[""] = 2 # unknown 52 | word_index[""] = 3 53 | 54 | print('Indexing word vectors.') 55 | embeddings_index = {} 56 | GLOVE_DIR = r'.\glove.6B' 57 | with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'), encoding='utf-8') as f: 58 | for line in f: 59 | values = line.split() 60 | word = values[0] 61 | coefs = np.asarray(values[1:], dtype='float32') 62 | embeddings_index[word] = coefs 63 | print('Found %s word vectors.' % len(embeddings_index)) 64 | MAX_NUM_WORDS = total_words 65 | # prepare embedding matrix 66 | num_words = min(MAX_NUM_WORDS, len(word_index)) 67 | embedding_matrix = np.zeros((num_words, embedding_len)) 68 | applied_vec_count = 0 69 | for word, i in word_index.items(): 70 | if i >= MAX_NUM_WORDS: 71 | continue 72 | embedding_vector = embeddings_index.get(word) 73 | # print(word,embedding_vector) 74 | if embedding_vector is not None: 75 | # words not found in embedding index will be all-zeros. 76 | embedding_matrix[i] = embedding_vector 77 | applied_vec_count += 1 78 | 79 | return embedding_matrix 80 | 81 | 82 | class MyRNN(keras.Model): 83 | # Cell方式构建多层网络 84 | def __init__(self, units, total_words, embedding_len, max_review_len, embedding_matrix): 85 | super(MyRNN, self).__init__() 86 | # 词向量编码 [b, 80] => [b, 80, 100] 87 | # 创建 Embedding 层 88 | self.embedding = layers.Embedding(total_words, embedding_len, 89 | input_length=max_review_len, 90 | trainable=False) 91 | self.embedding.build(input_shape=(None, max_review_len)) 92 | # 利用 GloVe 模型初始化 Embedding 层 93 | self.embedding.set_weights([embedding_matrix]) # 初始化 94 | # 构建RNN 95 | self.rnn = Sequential([ 96 | layers.LSTM(units, dropout=0.5, return_sequences=True), 97 | layers.LSTM(units, dropout=0.5) 98 | ]) 99 | 100 | # 构建分类网络，用于将CELL的输出特征进行分类，2分类 101 | # [b, 80, 100] => [b, 64] => [b, 1] 102 | self.outlayer = Sequential([ 103 | layers.Dense(32), 104 | layers.Dropout(rate=0.5), 105 | layers.ReLU(), 106 | layers.Dense(1)]) 107 | 108 | def call(self, inputs, training=None): 109 | x = inputs # [b, 80] 110 | # embedding: [b, 80] => [b, 80, 100] 111 | x = self.embedding(x) 112 | # rnn cell compute,[b, 80, 100] => [b, 64] 113 | x = self.rnn(x) 114 | # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1] 115 | x = self.outlayer(x, training) 116 | # p(y is pos|x) 117 | prob = tf.sigmoid(x) 118 | 119 | return prob 120 | 121 | 122 | def main(): 123 | batchsz = 128 # 批量大小 124 | total_words = 10000 # 词汇表大小N_vocab 125 | embedding_len = 100 # 词向量特征长度f 126 | max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充 127 | 128 | db_train, db_test = load_dataset(batchsz, total_words, max_review_len, embedding_len) 129 | 130 | units = 32 # RNN状态向量长度f 131 | epochs = 20 # 训练epochs 132 | embedding_matrix = pretrain_embeding_matrix(embedding_len, total_words) 133 | 134 | model = MyRNN(units, total_words, embedding_len, max_review_len, embedding_matrix) 135 | # 装配 136 | model.compile(optimizer=optimizers.RMSprop(0.001), 137 | loss=losses.BinaryCrossentropy(), 138 | metrics=['accuracy'], 139 | experimental_run_tf_function=False) 140 | # 训练和验证 141 | model.fit(db_train, epochs=epochs, validation_data=db_test) 142 | # 测试 143 | model.evaluate(db_test) 144 | 145 | 146 | if __name__ == '__main__': 147 | main() 148 | -------------------------------------------------------------------------------- /src/ch11/11.5-sentiment-analysis-RNN.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 11.5-sentiment-analysis-RNN.py 6 | @time: 2020/2/28 16:25 7 | @desc: 11.5 RNN情感分类问题实战的代码 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow import keras 15 | from tensorflow.keras import layers, losses, optimizers, Sequential 16 | 17 | tf.random.set_seed(22) 18 | np.random.seed(22) 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | def load_dataset(batchsz, total_words, max_review_len): 23 | # 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词 24 | (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words) 25 | print(x_train.shape, len(x_train[0]), y_train.shape) 26 | print(x_test.shape, len(x_test[0]), y_test.shape) 27 | 28 | # x_train:[b, 80] 29 | # x_test: [b, 80] 30 | # 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充 31 | x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len) 32 | x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len) 33 | # 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch 34 | db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 35 | db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True) 36 | db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 37 | db_test = db_test.batch(batchsz, drop_remainder=True) 38 | print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train)) 39 | print('x_test shape:', x_test.shape) 40 | return db_train, db_test 41 | 42 | 43 | class MyRNN(keras.Model): 44 | # Cell方式构建多层网络 45 | def __init__(self, units, batchsz, total_words, embedding_len, max_review_len): 46 | super(MyRNN, self).__init__() 47 | # [b, 64]，构建Cell初始化状态向量，重复使用 48 | self.state0 = [tf.zeros([batchsz, units])] 49 | self.state1 = [tf.zeros([batchsz, units])] 50 | # 词向量编码 [b, 80] => [b, 80, 100] 51 | self.embedding = layers.Embedding(total_words, embedding_len, 52 | input_length=max_review_len) 53 | # 构建2个Cell 54 | self.rnn_cell0 = layers.SimpleRNNCell(units, dropout=0.5) 55 | self.rnn_cell1 = layers.SimpleRNNCell(units, dropout=0.5) 56 | # 构建分类网络，用于将CELL的输出特征进行分类，2分类 57 | # [b, 80, 100] => [b, 64] => [b, 1] 58 | self.outlayer = Sequential([ 59 | layers.Dense(units), 60 | layers.Dropout(rate=0.5), 61 | layers.ReLU(), 62 | layers.Dense(1)]) 63 | 64 | def call(self, inputs, training=None): 65 | x = inputs # [b, 80] 66 | # embedding: [b, 80] => [b, 80, 100] 67 | x = self.embedding(x) 68 | # rnn cell compute,[b, 80, 100] => [b, 64] 69 | state0 = self.state0 70 | state1 = self.state1 71 | for word in tf.unstack(x, axis=1): # word: [b, 100] 72 | out0, state0 = self.rnn_cell0(word, state0, training) 73 | out1, state1 = self.rnn_cell1(out0, state1, training) 74 | # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1] 75 | x = self.outlayer(out1, training) 76 | # p(y is pos|x) 77 | prob = tf.sigmoid(x) 78 | 79 | return prob 80 | 81 | 82 | def main(): 83 | batchsz = 128 # 批量大小 84 | total_words = 10000 # 词汇表大小N_vocab 85 | embedding_len = 100 # 词向量特征长度f 86 | max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充 87 | 88 | db_train, db_test = load_dataset(batchsz, total_words, max_review_len) 89 | 90 | units = 64 # RNN状态向量长度f 91 | epochs = 20 # 训练epochs 92 | 93 | model = MyRNN(units, batchsz, total_words, embedding_len, max_review_len) 94 | # 装配 95 | model.compile(optimizer=optimizers.RMSprop(0.001), 96 | loss=losses.BinaryCrossentropy(), 97 | metrics=['accuracy'], 98 | experimental_run_tf_function=False) 99 | # 训练和验证 100 | model.fit(db_train, epochs=epochs, validation_data=db_test) 101 | # 测试 102 | model.evaluate(db_test) 103 | 104 | 105 | if __name__ == '__main__': 106 | main() 107 | -------------------------------------------------------------------------------- /src/ch12/12.2-fashion-mnist-autoencoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 12.2-fashion-mnist-autoencoder.py 6 | @time: 2020/2/29 16:38 7 | @desc: 12.2 Fashion MNIST 图片重建实战的代码 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from PIL import Image 15 | from tensorflow import keras 16 | from tensorflow.keras import Sequential, layers 17 | 18 | tf.random.set_seed(22) 19 | np.random.seed(22) 20 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 21 | 22 | 23 | def save_images(imgs, name): 24 | # 创建 280x280 大小图片阵列 25 | new_im = Image.new('L', (280, 280)) 26 | 27 | index = 0 28 | # 10 行图片阵列 29 | for i in range(0, 280, 28): 30 | # 10 列图片阵列 31 | for j in range(0, 280, 28): 32 | im = imgs[index] 33 | im = Image.fromarray(im, mode='L') 34 | # 写入对应位置 35 | new_im.paste(im, (i, j)) 36 | index += 1 37 | # 保存图片阵列 38 | new_im.save(name) 39 | 40 | 41 | def load_dataset(batchsz): 42 | # 加载 Fashion MNIST 图片数据集 43 | (x_train, y_train), (x_test, y_test) = keras.datasets.fashion_mnist.load_data() 44 | # 归一化 45 | x_train, x_test = x_train.astype(np.float32) / 255., x_test.astype(np.float32) / 255. 46 | # 只需要通过图片数据即可构建数据集对象，不需要标签 47 | train_db = tf.data.Dataset.from_tensor_slices(x_train) 48 | train_db = train_db.shuffle(batchsz * 5).batch(batchsz) 49 | # 构建测试集对象 50 | test_db = tf.data.Dataset.from_tensor_slices(x_test) 51 | test_db = test_db.batch(batchsz) 52 | return train_db, test_db 53 | 54 | 55 | class AE(keras.Model): 56 | # 自编码器模型类，包含了 Encoder 和 Decoder2 个子网络 57 | def __init__(self, h_dim): 58 | super(AE, self).__init__() 59 | 60 | # 创建 Encoders 网络，实现在自编码器类的初始化函数中 61 | self.encoder = Sequential([ 62 | layers.Dense(256, activation=tf.nn.relu), 63 | layers.Dense(128, activation=tf.nn.relu), 64 | layers.Dense(h_dim) 65 | ]) 66 | 67 | # 创建 Decoders 网络 68 | self.decoder = Sequential([ 69 | layers.Dense(128, activation=tf.nn.relu), 70 | layers.Dense(256, activation=tf.nn.relu), 71 | layers.Dense(784) 72 | ]) 73 | 74 | def call(self, inputs, training=None): 75 | # 前向传播函数 76 | # 编码获得隐藏向量 h,[b, 784] => [b, 20] 77 | h = self.encoder(inputs) 78 | # 解码获得重建图片， [b, 20] => [b, 784] 79 | x_hat = self.decoder(h) 80 | 81 | return x_hat 82 | 83 | 84 | def build_model(h_dim): 85 | # 创建网络对象 86 | model = AE(h_dim) 87 | # 指定输入大小 88 | model.build(input_shape=(None, 784)) 89 | # 打印网络信息 90 | model.summary() 91 | return model 92 | 93 | 94 | def train(train_db, model, optimizer, epoch): 95 | # 遍历训练集 96 | for step, x in enumerate(train_db): 97 | 98 | # [b, 28, 28] => [b, 784] 99 | # 打平， [b, 28, 28] => [b, 784] 100 | x = tf.reshape(x, [-1, 784]) 101 | # 构建梯度记录器 102 | with tf.GradientTape() as tape: 103 | # 前向计算获得重建的图片 104 | x_rec_logits = model(x) 105 | # 计算重建图片与输入之间的损失函数 106 | rec_loss = tf.losses.binary_crossentropy(x, x_rec_logits, from_logits=True) 107 | # 计算均值 108 | rec_loss = tf.reduce_mean(rec_loss) 109 | # 自动求导，包含了2个子网络的梯度 110 | grads = tape.gradient(rec_loss, model.trainable_variables) 111 | # 自动更新，同时更新2个子网络 112 | optimizer.apply_gradients(zip(grads, model.trainable_variables)) 113 | 114 | if step % 100 == 0: 115 | # 间隔性打印训练误差 116 | print(epoch, step, float(rec_loss)) 117 | 118 | return model 119 | 120 | 121 | def evaluation(test_db, model, epoch): 122 | # evaluation 123 | # 重建图片，从测试集采样一批图片 124 | x = next(iter(test_db)) 125 | # 打平并送入自编码器 126 | logits = model(tf.reshape(x, [-1, 784])) 127 | # 将输出转换为像素值，使用sigmoid函数 128 | x_hat = tf.sigmoid(logits) 129 | # 恢复为 28x28,[b, 784] => [b, 28, 28] 130 | x_hat = tf.reshape(x_hat, [-1, 28, 28]) 131 | 132 | # 输入的前50张+重建的前50张图片合并， [b, 28, 28] => [2b, 28, 28] 133 | x_concat = tf.concat([x[:50], x_hat[:50]], axis=0) 134 | # 恢复为0~255范围 135 | x_concat = x_concat.numpy() * 255. 136 | # 转换为整型 137 | x_concat = x_concat.astype(np.uint8) 138 | # 保存图片 139 | save_images(x_concat, './ae_images/rec_epoch_%d.png' % epoch) 140 | 141 | 142 | def main(): 143 | h_dim = 20 144 | batchsz = 512 145 | lr = 1e-3 146 | 147 | train_db, test_db = load_dataset(batchsz) 148 | model = build_model(h_dim) 149 | # 创建优化器，并设置学习率 150 | optimizer = tf.optimizers.Adam(lr=lr) 151 | # 训练100个Epoch 152 | for epoch in range(100): 153 | model = train(train_db, model, optimizer, epoch) 154 | evaluation(test_db, model, epoch) 155 | 156 | 157 | if __name__ == '__main__': 158 | main() 159 | -------------------------------------------------------------------------------- /src/ch12/12.5-fashion-mnist-vae.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 12.5-fashion-mnist-vae.py 6 | @time: 2020/2/29 16:38 7 | @desc: 12.5 VAE图片生成实战的代码 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from PIL import Image 15 | from tensorflow import keras 16 | from tensorflow.keras import layers 17 | 18 | tf.random.set_seed(22) 19 | np.random.seed(22) 20 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 21 | 22 | 23 | def save_images(imgs, name): 24 | # 创建 280x280 大小图片阵列 25 | new_im = Image.new('L', (280, 280)) 26 | 27 | index = 0 28 | # 10 行图片阵列 29 | for i in range(0, 280, 28): 30 | # 10 列图片阵列 31 | for j in range(0, 280, 28): 32 | im = imgs[index] 33 | im = Image.fromarray(im, mode='L') 34 | # 写入对应位置 35 | new_im.paste(im, (i, j)) 36 | index += 1 37 | # 保存图片阵列 38 | new_im.save(name) 39 | 40 | 41 | def load_dataset(batchsz): 42 | # 加载 Fashion MNIST 图片数据集 43 | (x_train, y_train), (x_test, y_test) = keras.datasets.fashion_mnist.load_data() 44 | # 归一化 45 | x_train, x_test = x_train.astype(np.float32) / 255., x_test.astype(np.float32) / 255. 46 | # 只需要通过图片数据即可构建数据集对象，不需要标签 47 | train_db = tf.data.Dataset.from_tensor_slices(x_train) 48 | train_db = train_db.shuffle(batchsz * 5).batch(batchsz) 49 | # 构建测试集对象 50 | test_db = tf.data.Dataset.from_tensor_slices(x_test) 51 | test_db = test_db.batch(batchsz) 52 | return train_db, test_db 53 | 54 | 55 | class VAE(keras.Model): 56 | # 自编码器模型类，包含了 Encoder 和 Decoder2 个子网络 57 | def __init__(self, z_dim): 58 | super(VAE, self).__init__() 59 | 60 | # Encoder 网络 61 | self.fc1 = layers.Dense(128) 62 | self.fc2 = layers.Dense(z_dim) # 均值输出 63 | self.fc3 = layers.Dense(z_dim) # 方差输出 64 | # Decoder 网络 65 | self.fc4 = layers.Dense(128) 66 | self.fc5 = layers.Dense(784) 67 | 68 | def encoder(self, x): 69 | # 获得编码器的均值和方差 70 | h = tf.nn.relu(self.fc1(x)) 71 | # 均值向量 72 | mu = self.fc2(h) 73 | # 方差的 log 向量 74 | log_var = self.fc3(h) 75 | return mu, log_var 76 | 77 | def decoder(self, z): 78 | # 根据隐藏变量 z 生成图片数据 79 | out = tf.nn.relu(self.fc4(z)) 80 | out = self.fc5(out) 81 | # 返回图片数据， 784 向量 82 | return out 83 | 84 | def reparameterize(self, mu, log_var): 85 | # reparameterize 技巧，从正态分布采样 epsion 86 | eps = tf.random.normal(tf.shape(log_var)) 87 | # 计算标准差 88 | std = tf.exp(log_var) ** 0.5 89 | # reparameterize 技巧 90 | z = mu + std * eps 91 | return z 92 | 93 | def call(self, inputs, training=None): 94 | # 前向计算 95 | # 编码器[b, 784] => [b, z_dim], [b, z_dim] 96 | mu, log_var = self.encoder(inputs) 97 | # 采样 reparameterization trick 98 | z = self.reparameterize(mu, log_var) 99 | # 通过解码器生成 100 | x_hat = self.decoder(z) 101 | # 返回生成样本，及其均值与方差 102 | return x_hat, mu, log_var 103 | 104 | 105 | def build_model(z_dim): 106 | # 创建网络对象 107 | model = VAE(z_dim) 108 | # 指定输入大小 109 | model.build(input_shape=(None, 784)) 110 | return model 111 | 112 | 113 | def train(train_db, model, optimizer, epoch): 114 | # 遍历训练集 115 | for step, x in enumerate(train_db): 116 | 117 | # [b, 28, 28] => [b, 784] 118 | # 打平， [b, 28, 28] => [b, 784] 119 | x = tf.reshape(x, [-1, 784]) 120 | # 构建梯度记录器 121 | with tf.GradientTape() as tape: 122 | # 前向计算获得重建的图片 123 | x_rec_logits, mu, log_var = model(x) 124 | # 重建损失值计算 125 | rec_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=x, logits=x_rec_logits) 126 | rec_loss = tf.reduce_sum(rec_loss) / x.shape[0] 127 | 128 | # 计算 KL 散度 N(mu, var) VS N(0, 1) 129 | # 公式参考： https://stats.stackexchange.com/questions/7440/kldivergence-between-two-univariate-gaussians 130 | kl_div = -0.5 * (log_var + 1 - mu ** 2 - tf.exp(log_var)) 131 | kl_div = tf.reduce_sum(kl_div) / x.shape[0] 132 | # 合并误差项 133 | loss = rec_loss + 1. * kl_div 134 | 135 | # 自动求导 136 | grads = tape.gradient(loss, model.trainable_variables) 137 | # 自动更新 138 | optimizer.apply_gradients(zip(grads, model.trainable_variables)) 139 | 140 | if step % 100 == 0: 141 | # 打印训练误差 142 | print("epoch=%s," % epoch, "step=%s," % step, "kl_div=%f," % float(kl_div), "rec_loss=%f" % float(rec_loss)) 143 | 144 | return model 145 | 146 | 147 | def evaluation(test_db, model, epoch, batchsz, z_dim): 148 | z = tf.random.normal((batchsz, z_dim)) 149 | # 仅通过解码器生成图片 150 | logits = model.decoder(z) 151 | # 转换为像素范围 152 | x_hat = tf.sigmoid(logits) 153 | x_hat = tf.reshape(x_hat, [-1, 28, 28]).numpy() * 255. 154 | x_hat = x_hat.astype(np.uint8) 155 | save_images(x_hat, './vae_images/epoch_%d_sampled.png' % epoch) # 保存生成图片 156 | 157 | # 重建图片，从测试集采样图片 158 | x = next(iter(test_db)) 159 | # 打平并送入自编码器 160 | logits, _, _ = model(tf.reshape(x, [-1, 784])) 161 | # 将输出转换为像素值 162 | x_hat = tf.sigmoid(logits) 163 | # 恢复为 28x28,[b, 784] => [b, 28, 28] 164 | x_hat = tf.reshape(x_hat, [-1, 28, 28]) 165 | # 输入的前 50 张+重建的前 50 张图片合并， [b, 28, 28] => [2b, 28, 28] 166 | x_concat = tf.concat([x[:50], x_hat[:50]], axis=0) 167 | x_concat = x_concat.numpy() * 255. # 恢复为 0~255 范围 168 | x_concat = x_concat.astype(np.uint8) 169 | save_images(x_concat, './vae_images/epoch_%d_rec.png' % epoch) # 保存重建图片 170 | 171 | 172 | def main(): 173 | z_dim = 10 174 | batchsz = 512 175 | lr = 1e-3 176 | 177 | train_db, test_db = load_dataset(batchsz) 178 | model = build_model(z_dim) 179 | # 创建优化器，并设置学习率 180 | optimizer = tf.optimizers.Adam(lr=lr) 181 | # 训练100个Epoch 182 | for epoch in range(100): 183 | model = train(train_db, model, optimizer, epoch) 184 | evaluation(test_db, model, epoch, batchsz, z_dim) 185 | 186 | 187 | if __name__ == '__main__': 188 | main() 189 | -------------------------------------------------------------------------------- /src/ch12/ae_images/rec_epoch_99.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch12/ae_images/rec_epoch_99.png -------------------------------------------------------------------------------- /src/ch12/vae_images/epoch_99_rec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch12/vae_images/epoch_99_rec.png -------------------------------------------------------------------------------- /src/ch12/vae_images/epoch_99_sampled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Relph1119/deeplearning-with-tensorflow-notes/77d7a04bc933a14d6d86cba4f9fd54fd0d05ddbe/src/ch12/vae_images/epoch_99_sampled.png -------------------------------------------------------------------------------- /src/ch13/13.3-faces-dcgan.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 13.3-faces-dcgan.py 6 | @time: 2020/2/29 20:51 7 | @desc: 13.3 DCGAN实战的代码 8 | 将face文件夹放到ch13目录下，读取路径为/ch13/face/*. 9 | """ 10 | 11 | import glob 12 | import os 13 | 14 | import numpy as np 15 | import tensorflow as tf 16 | from PIL import Image 17 | from tensorflow import keras 18 | 19 | from src.ch13.dataset import make_anime_dataset 20 | from src.ch13.gan import Generator, Discriminator 21 | 22 | os.environ['TF_ENABLE_GPU_GARBAGE_COLLECTION'] = 'false' 23 | 24 | # 获取 GPU 设备列表 25 | gpus = tf.config.experimental.list_physical_devices('GPU') 26 | if gpus: 27 | try: 28 | # 设置 GPU 为增长式占用 29 | for gpu in gpus: 30 | tf.config.experimental.set_memory_growth(gpu, True) 31 | except RuntimeError as e: 32 | # 打印异常 33 | print(e) 34 | 35 | 36 | def save_result(val_out, val_block_size, image_path, color_mode): 37 | def preprocess(img): 38 | img = ((img + 1.0) * 127.5).astype(np.uint8) 39 | # img = img.astype(np.uint8) 40 | return img 41 | 42 | preprocesed = preprocess(val_out) 43 | final_image = np.array([]) 44 | single_row = np.array([]) 45 | for b in range(val_out.shape[0]): 46 | # concat image into a row 47 | if single_row.size == 0: 48 | single_row = preprocesed[b, :, :, :] 49 | else: 50 | single_row = np.concatenate((single_row, preprocesed[b, :, :, :]), axis=1) 51 | 52 | # concat image row to final_image 53 | if (b + 1) % val_block_size == 0: 54 | if final_image.size == 0: 55 | final_image = single_row 56 | else: 57 | final_image = np.concatenate((final_image, single_row), axis=0) 58 | 59 | # reset single row 60 | single_row = np.array([]) 61 | 62 | if final_image.shape[2] == 1: 63 | final_image = np.squeeze(final_image, axis=2) 64 | Image.fromarray(final_image).save(image_path) 65 | 66 | 67 | def celoss_ones(logits): 68 | # 计算属于与标签为1的交叉熵 69 | y = tf.ones_like(logits) 70 | loss = keras.losses.binary_crossentropy(y, logits, from_logits=True) 71 | return tf.reduce_mean(loss) 72 | 73 | 74 | def celoss_zeros(logits): 75 | # 计算属于与便签为0的交叉熵 76 | y = tf.zeros_like(logits) 77 | loss = keras.losses.binary_crossentropy(y, logits, from_logits=True) 78 | return tf.reduce_mean(loss) 79 | 80 | 81 | def d_loss_fn(generator, discriminator, batch_z, batch_x, is_training): 82 | # 计算判别器的误差函数 83 | # 采样生成图片 84 | fake_image = generator(batch_z, is_training) 85 | # 判定生成图片 86 | d_fake_logits = discriminator(fake_image, is_training) 87 | # 判定真实图片 88 | d_real_logits = discriminator(batch_x, is_training) 89 | # 真实图片与1之间的误差 90 | d_loss_real = celoss_ones(d_real_logits) 91 | # 生成图片与0之间的误差 92 | d_loss_fake = celoss_zeros(d_fake_logits) 93 | # 合并误差 94 | loss = d_loss_fake + d_loss_real 95 | 96 | return loss 97 | 98 | 99 | def g_loss_fn(generator, discriminator, batch_z, is_training): 100 | # 采样生成图片 101 | fake_image = generator(batch_z, is_training) 102 | # 在训练生成网络时，需要迫使生成图片判定为真 103 | d_fake_logits = discriminator(fake_image, is_training) 104 | # 计算生成图片与1之间的误差 105 | loss = celoss_ones(d_fake_logits) 106 | 107 | return loss 108 | 109 | 110 | def main(): 111 | tf.random.set_seed(3333) 112 | np.random.seed(3333) 113 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 114 | 115 | z_dim = 100 # 隐藏向量z的长度 116 | epochs = 3000000 # 训练步数 117 | batch_size = 64 # batch size 118 | learning_rate = 0.0002 119 | is_training = True 120 | 121 | # 获取数据集路径 122 | img_path = glob.glob(r'.\faces\*.jpg') 123 | print('images num:', len(img_path)) 124 | # 构建数据集对象 125 | dataset, img_shape, _ = make_anime_dataset(img_path, batch_size, resize=64) 126 | print(dataset, img_shape) 127 | sample = next(iter(dataset)) # 采样 128 | print(sample.shape, tf.reduce_max(sample).numpy(), 129 | tf.reduce_min(sample).numpy()) 130 | dataset = dataset.repeat(100) # 重复循环 131 | db_iter = iter(dataset) 132 | 133 | discriminator, generator = build_network(z_dim) 134 | # 分别为生成器和判别器创建优化器 135 | g_optimizer = keras.optimizers.Adam(learning_rate=learning_rate, beta_1=0.5) 136 | d_optimizer = keras.optimizers.Adam(learning_rate=learning_rate, beta_1=0.5) 137 | 138 | generator.load_weights('generator.ckpt') 139 | discriminator.load_weights('discriminator.ckpt') 140 | print('Loaded ckpt!!') 141 | 142 | d_losses, g_losses = [], [] 143 | for epoch in range(epochs): # 训练epochs次 144 | train(batch_size, d_losses, d_optimizer, db_iter, discriminator, epoch, g_losses, g_optimizer, generator, 145 | is_training, z_dim) 146 | 147 | 148 | def train(batch_size, d_losses, d_optimizer, db_iter, discriminator, epoch, g_losses, g_optimizer, generator, 149 | is_training, z_dim): 150 | # 1. 训练判别器 151 | for _ in range(1): 152 | # 采样隐藏向量 153 | batch_z = tf.random.normal([batch_size, z_dim]) 154 | batch_x = next(db_iter) # 采样真实图片 155 | # 判别器前向计算 156 | with tf.GradientTape() as tape: 157 | d_loss = d_loss_fn(generator, discriminator, batch_z, batch_x, is_training) 158 | grads = tape.gradient(d_loss, discriminator.trainable_variables) 159 | d_optimizer.apply_gradients(zip(grads, discriminator.trainable_variables)) 160 | # 2. 训练生成器 161 | # 采样隐藏向量 162 | batch_z = tf.random.normal([batch_size, z_dim]) 163 | batch_x = next(db_iter) # 采样真实图片 164 | # 生成器前向计算 165 | with tf.GradientTape() as tape: 166 | g_loss = g_loss_fn(generator, discriminator, batch_z, is_training) 167 | grads = tape.gradient(g_loss, generator.trainable_variables) 168 | g_optimizer.apply_gradients(zip(grads, generator.trainable_variables)) 169 | if epoch % 100 == 0: 170 | print(epoch, 'd-loss:', float(d_loss), 'g-loss:', float(g_loss)) 171 | # 可视化 172 | z = tf.random.normal([100, z_dim]) 173 | fake_image = generator(z, training=False) 174 | img_path = os.path.join('./gan_images', 'gan-%d.png' % epoch) 175 | save_result(fake_image.numpy(), 10, img_path, color_mode='P') 176 | 177 | d_losses.append(float(d_loss)) 178 | g_losses.append(float(g_loss)) 179 | 180 | if epoch % 10000 == 1: 181 | generator.save_weights('generator.ckpt') 182 | discriminator.save_weights('discriminator.ckpt') 183 | 184 | 185 | def build_network(z_dim): 186 | # 创建生成器 187 | generator = Generator() 188 | generator.build(input_shape=(4, z_dim)) 189 | # 创建判别器 190 | discriminator = Discriminator() 191 | discriminator.build(input_shape=(4, 64, 64, 3)) 192 | return discriminator, generator 193 | 194 | 195 | if __name__ == '__main__': 196 | main() 197 | -------------------------------------------------------------------------------- /src/ch13/13.8-faces-wgan-gp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 13.8-faces-wgan-gp.py 6 | @time: 2020/2/29 20:51 7 | @desc: 13.8 WGAN-GP实战的代码 8 | 将face文件夹放到ch13目录下，读取路径为/ch13/face/*. 9 | """ 10 | 11 | import glob 12 | import os 13 | 14 | import numpy as np 15 | import tensorflow as tf 16 | from PIL import Image 17 | from tensorflow import keras 18 | 19 | from src.ch13.dataset import make_anime_dataset 20 | from src.ch13.gan import Generator, Discriminator 21 | 22 | os.environ['TF_ENABLE_GPU_GARBAGE_COLLECTION'] = 'false' 23 | 24 | # 获取 GPU 设备列表 25 | gpus = tf.config.experimental.list_physical_devices('GPU') 26 | if gpus: 27 | try: 28 | # 设置 GPU 为增长式占用 29 | for gpu in gpus: 30 | tf.config.experimental.set_memory_growth(gpu, True) 31 | except RuntimeError as e: 32 | # 打印异常 33 | print(e) 34 | 35 | 36 | def save_result(val_out, val_block_size, image_path, color_mode): 37 | def preprocess(img): 38 | img = ((img + 1.0) * 127.5).astype(np.uint8) 39 | # img = img.astype(np.uint8) 40 | return img 41 | 42 | preprocesed = preprocess(val_out) 43 | final_image = np.array([]) 44 | single_row = np.array([]) 45 | for b in range(val_out.shape[0]): 46 | # concat image into a row 47 | if single_row.size == 0: 48 | single_row = preprocesed[b, :, :, :] 49 | else: 50 | single_row = np.concatenate((single_row, preprocesed[b, :, :, :]), axis=1) 51 | 52 | # concat image row to final_image 53 | if (b + 1) % val_block_size == 0: 54 | if final_image.size == 0: 55 | final_image = single_row 56 | else: 57 | final_image = np.concatenate((final_image, single_row), axis=0) 58 | 59 | # reset single row 60 | single_row = np.array([]) 61 | 62 | if final_image.shape[2] == 1: 63 | final_image = np.squeeze(final_image, axis=2) 64 | Image.fromarray(final_image).save(image_path) 65 | 66 | 67 | def celoss_ones(logits): 68 | # 计算属于与标签为1的交叉熵 69 | y = tf.ones_like(logits) 70 | loss = keras.losses.binary_crossentropy(y, logits, from_logits=True) 71 | return tf.reduce_mean(loss) 72 | 73 | 74 | def celoss_zeros(logits): 75 | # 计算属于与便签为0的交叉熵 76 | y = tf.zeros_like(logits) 77 | loss = keras.losses.binary_crossentropy(y, logits, from_logits=True) 78 | return tf.reduce_mean(loss) 79 | 80 | 81 | def gradient_penalty(discriminator, batch_x, fake_image): 82 | # 梯度惩罚项计算函数 83 | batchsz = batch_x.shape[0] 84 | # 每个样本均随机采样 t,用于插值 85 | t = tf.random.uniform([batchsz, 1, 1, 1]) 86 | # 自动扩展为 x 的形状， [b, 1, 1, 1] => [b, h, w, c] 87 | t = tf.broadcast_to(t, batch_x.shape) 88 | # 在真假图片之间做线性插值 89 | interplate = t * batch_x + (1 - t) * fake_image 90 | # 在梯度环境中计算 D 对插值样本的梯度 91 | with tf.GradientTape() as tape: 92 | tape.watch([interplate]) # 加入梯度观察列表 93 | d_interplote_logits = discriminator(interplate) 94 | grads = tape.gradient(d_interplote_logits, interplate) 95 | 96 | # 计算每个样本的梯度的范数:[b, h, w, c] => [b, -1] 97 | grads = tf.reshape(grads, [grads.shape[0], -1]) 98 | gp = tf.norm(grads, axis=1) # [b] 99 | # 计算梯度惩罚项 100 | gp = tf.reduce_mean((gp - 1.) ** 2) 101 | return gp 102 | 103 | 104 | def d_loss_fn(generator, discriminator, batch_z, batch_x, is_training): 105 | # 计算 D 的损失函数 106 | fake_image = generator(batch_z, is_training) # 假样本 107 | d_fake_logits = discriminator(fake_image, is_training) # 假样本的输出 108 | d_real_logits = discriminator(batch_x, is_training) # 真样本的输出 109 | # 计算梯度惩罚项 110 | gp = gradient_penalty(discriminator, batch_x, fake_image) 111 | # WGAN-GP D 损失函数的定义，这里并不是计算交叉熵，而是直接最大化正样本的输出 112 | # 最小化假样本的输出和梯度惩罚项 113 | loss = tf.reduce_mean(d_fake_logits) - tf.reduce_mean(d_real_logits) + 10. * gp 114 | return loss, gp 115 | 116 | 117 | def g_loss_fn(generator, discriminator, batch_z, is_training): 118 | # 生成器的损失函数 119 | fake_image = generator(batch_z, is_training) 120 | d_fake_logits = discriminator(fake_image, is_training) 121 | # WGAN-GP G 损失函数，最大化假样本的输出值 122 | loss = - tf.reduce_mean(d_fake_logits) 123 | return loss 124 | 125 | 126 | def main(): 127 | tf.random.set_seed(3333) 128 | np.random.seed(3333) 129 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 130 | 131 | z_dim = 100 # 隐藏向量z的长度 132 | epochs = 3000000 # 训练步数 133 | batch_size = 64 # batch size 134 | learning_rate = 0.0002 135 | is_training = True 136 | 137 | # 获取数据集路径 138 | img_path = glob.glob(r'.\faces\*.jpg') 139 | print('images num:', len(img_path)) 140 | # 构建数据集对象 141 | dataset, img_shape, _ = make_anime_dataset(img_path, batch_size, resize=64) 142 | print(dataset, img_shape) 143 | sample = next(iter(dataset)) # 采样 144 | print(sample.shape, tf.reduce_max(sample).numpy(), 145 | tf.reduce_min(sample).numpy()) 146 | dataset = dataset.repeat(100) # 重复循环 147 | db_iter = iter(dataset) 148 | 149 | discriminator, generator = build_network(z_dim) 150 | # 分别为生成器和判别器创建优化器 151 | g_optimizer = keras.optimizers.Adam(learning_rate=learning_rate, beta_1=0.5) 152 | d_optimizer = keras.optimizers.Adam(learning_rate=learning_rate, beta_1=0.5) 153 | 154 | generator.load_weights('generator.ckpt') 155 | discriminator.load_weights('discriminator.ckpt') 156 | print('Loaded ckpt!!') 157 | 158 | d_losses, g_losses = [], [] 159 | for epoch in range(epochs): # 训练epochs次 160 | train(batch_size, d_losses, d_optimizer, db_iter, discriminator, epoch, g_losses, g_optimizer, generator, 161 | is_training, z_dim) 162 | 163 | 164 | def train(batch_size, d_losses, d_optimizer, db_iter, discriminator, epoch, g_losses, g_optimizer, generator, 165 | is_training, z_dim): 166 | # 1. 训练判别器 167 | for _ in range(1): 168 | # 采样隐藏向量 169 | batch_z = tf.random.normal([batch_size, z_dim]) 170 | batch_x = next(db_iter) # 采样真实图片 171 | # 判别器前向计算 172 | with tf.GradientTape() as tape: 173 | d_loss, gp = d_loss_fn(generator, discriminator, batch_z, batch_x, is_training) 174 | grads = tape.gradient(d_loss, discriminator.trainable_variables) 175 | d_optimizer.apply_gradients(zip(grads, discriminator.trainable_variables)) 176 | 177 | # 2. 训练生成器 178 | # 采样隐藏向量 179 | batch_z = tf.random.normal([batch_size, z_dim]) 180 | batch_x = next(db_iter) # 采样真实图片 181 | # 生成器前向计算 182 | with tf.GradientTape() as tape: 183 | g_loss = g_loss_fn(generator, discriminator, batch_z, is_training) 184 | 185 | grads = tape.gradient(g_loss, generator.trainable_variables) 186 | g_optimizer.apply_gradients(zip(grads, generator.trainable_variables)) 187 | if epoch % 100 == 0: 188 | print(epoch, 'd-loss:', float(d_loss), 'g-loss:', float(g_loss)) 189 | # 可视化 190 | z = tf.random.normal([100, z_dim]) 191 | fake_image = generator(z, training=False) 192 | img_path = os.path.join('./gan_images', 'gan-%d.png' % epoch) 193 | save_result(fake_image.numpy(), 10, img_path, color_mode='P') 194 | 195 | d_losses.append(float(d_loss)) 196 | g_losses.append(float(g_loss)) 197 | 198 | if epoch % 10000 == 1: 199 | generator.save_weights('generator.ckpt') 200 | discriminator.save_weights('discriminator.ckpt') 201 | 202 | 203 | def build_network(z_dim): 204 | # 创建生成器 205 | generator = Generator() 206 | generator.build(input_shape=(4, z_dim)) 207 | # 创建判别器 208 | discriminator = Discriminator() 209 | discriminator.build(input_shape=(4, 64, 64, 3)) 210 | return discriminator, generator 211 | 212 | 213 | if __name__ == '__main__': 214 | main() 215 | -------------------------------------------------------------------------------- /src/ch13/dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: dataset.py 6 | @time: 2020/2/29 21:08 7 | @desc: 8 | """ 9 | 10 | import multiprocessing 11 | 12 | import tensorflow as tf 13 | 14 | 15 | def make_anime_dataset(img_paths, batch_size, resize=64, drop_remainder=True, shuffle=True, repeat=1): 16 | # @tf.function 17 | def _map_fn(img): 18 | img = tf.image.resize(img, [resize, resize]) 19 | # img = tf.image.random_crop(img,[resize, resize]) 20 | # img = tf.image.random_flip_left_right(img) 21 | # img = tf.image.random_flip_up_down(img) 22 | img = tf.clip_by_value(img, 0, 255) 23 | img = img / 127.5 - 1 # -1~1 24 | return img 25 | 26 | dataset = disk_image_batch_dataset(img_paths, 27 | batch_size, 28 | drop_remainder=drop_remainder, 29 | map_fn=_map_fn, 30 | shuffle=shuffle, 31 | repeat=repeat) 32 | img_shape = (resize, resize, 3) 33 | len_dataset = len(img_paths) // batch_size 34 | 35 | return dataset, img_shape, len_dataset 36 | 37 | 38 | def batch_dataset(dataset, 39 | batch_size, 40 | drop_remainder=True, 41 | n_prefetch_batch=1, 42 | filter_fn=None, 43 | map_fn=None, 44 | n_map_threads=None, 45 | filter_after_map=False, 46 | shuffle=True, 47 | shuffle_buffer_size=None, 48 | repeat=None): 49 | # set defaults 50 | if n_map_threads is None: 51 | n_map_threads = multiprocessing.cpu_count() 52 | if shuffle and shuffle_buffer_size is None: 53 | shuffle_buffer_size = max(batch_size * 128, 2048) # set the minimum buffer size as 2048 54 | 55 | # [*] it is efficient to conduct `shuffle` before `map`/`filter` because `map`/`filter` is sometimes costly 56 | if shuffle: 57 | dataset = dataset.shuffle(shuffle_buffer_size) 58 | 59 | if not filter_after_map: 60 | if filter_fn: 61 | dataset = dataset.filter(filter_fn) 62 | 63 | if map_fn: 64 | dataset = dataset.map(map_fn, num_parallel_calls=n_map_threads) 65 | 66 | else: # [*] this is slower 67 | if map_fn: 68 | dataset = dataset.map(map_fn, num_parallel_calls=n_map_threads) 69 | 70 | if filter_fn: 71 | dataset = dataset.filter(filter_fn) 72 | 73 | dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) 74 | 75 | dataset = dataset.repeat(repeat).prefetch(n_prefetch_batch) 76 | 77 | return dataset 78 | 79 | 80 | def memory_data_batch_dataset(memory_data, 81 | batch_size, 82 | drop_remainder=True, 83 | n_prefetch_batch=1, 84 | filter_fn=None, 85 | map_fn=None, 86 | n_map_threads=None, 87 | filter_after_map=False, 88 | shuffle=True, 89 | shuffle_buffer_size=None, 90 | repeat=None): 91 | """Batch dataset of memory data. 92 | Parameters 93 | ---------- 94 | memory_data : nested structure of tensors/ndarrays/lists 95 | """ 96 | dataset = tf.data.Dataset.from_tensor_slices(memory_data) 97 | dataset = batch_dataset(dataset, 98 | batch_size, 99 | drop_remainder=drop_remainder, 100 | n_prefetch_batch=n_prefetch_batch, 101 | filter_fn=filter_fn, 102 | map_fn=map_fn, 103 | n_map_threads=n_map_threads, 104 | filter_after_map=filter_after_map, 105 | shuffle=shuffle, 106 | shuffle_buffer_size=shuffle_buffer_size, 107 | repeat=repeat) 108 | return dataset 109 | 110 | 111 | def disk_image_batch_dataset(img_paths, 112 | batch_size, 113 | labels=None, 114 | drop_remainder=True, 115 | n_prefetch_batch=1, 116 | filter_fn=None, 117 | map_fn=None, 118 | n_map_threads=None, 119 | filter_after_map=False, 120 | shuffle=True, 121 | shuffle_buffer_size=None, 122 | repeat=None): 123 | """Batch dataset of disk image for PNG and JPEG. 124 | Parameters 125 | ---------- 126 | img_paths : 1d-tensor/ndarray/list of str 127 | labels : nested structure of tensors/ndarrays/lists 128 | """ 129 | if labels is None: 130 | memory_data = img_paths 131 | else: 132 | memory_data = (img_paths, labels) 133 | 134 | def parse_fn(path, *label): 135 | img = tf.io.read_file(path) 136 | img = tf.image.decode_jpeg(img, channels=3) # fix channels to 3 137 | return (img,) + label 138 | 139 | if map_fn: # fuse `map_fn` and `parse_fn` 140 | def map_fn_(*args): 141 | return map_fn(*parse_fn(*args)) 142 | else: 143 | map_fn_ = parse_fn 144 | 145 | dataset = memory_data_batch_dataset(memory_data, 146 | batch_size, 147 | drop_remainder=drop_remainder, 148 | n_prefetch_batch=n_prefetch_batch, 149 | filter_fn=filter_fn, 150 | map_fn=map_fn_, 151 | n_map_threads=n_map_threads, 152 | filter_after_map=filter_after_map, 153 | shuffle=shuffle, 154 | shuffle_buffer_size=shuffle_buffer_size, 155 | repeat=repeat) 156 | 157 | return dataset 158 | -------------------------------------------------------------------------------- /src/ch13/gan.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: gan.py 6 | @time: 2020/2/29 21:09 7 | @desc: 8 | """ 9 | 10 | import tensorflow as tf 11 | from tensorflow import keras 12 | from tensorflow.keras import layers 13 | 14 | 15 | class Generator(keras.Model): 16 | # 生成器网络类 17 | def __init__(self): 18 | super(Generator, self).__init__() 19 | filter = 64 20 | # 转置卷积层1,输出channel为filter*8,核大小4,步长1,不使用padding,不使用偏置 21 | self.conv1 = layers.Conv2DTranspose(filter * 8, 4, 1, 'valid', use_bias=False) 22 | self.bn1 = layers.BatchNormalization() 23 | # 转置卷积层2 24 | self.conv2 = layers.Conv2DTranspose(filter * 4, 4, 2, 'same', use_bias=False) 25 | self.bn2 = layers.BatchNormalization() 26 | # 转置卷积层3 27 | self.conv3 = layers.Conv2DTranspose(filter * 2, 4, 2, 'same', use_bias=False) 28 | self.bn3 = layers.BatchNormalization() 29 | # 转置卷积层4 30 | self.conv4 = layers.Conv2DTranspose(filter * 1, 4, 2, 'same', use_bias=False) 31 | self.bn4 = layers.BatchNormalization() 32 | # 转置卷积层5 33 | self.conv5 = layers.Conv2DTranspose(3, 4, 2, 'same', use_bias=False) 34 | 35 | def call(self, inputs, training=None): 36 | x = inputs # [z, 100] 37 | # Reshape乘4D张量，方便后续转置卷积运算:(b, 1, 1, 100) 38 | x = tf.reshape(x, (x.shape[0], 1, 1, x.shape[1])) 39 | x = tf.nn.relu(x) # 激活函数 40 | # 转置卷积-BN-激活函数:(b, 4, 4, 512) 41 | x = tf.nn.relu(self.bn1(self.conv1(x), training=training)) 42 | # 转置卷积-BN-激活函数:(b, 8, 8, 256) 43 | x = tf.nn.relu(self.bn2(self.conv2(x), training=training)) 44 | # 转置卷积-BN-激活函数:(b, 16, 16, 128) 45 | x = tf.nn.relu(self.bn3(self.conv3(x), training=training)) 46 | # 转置卷积-BN-激活函数:(b, 32, 32, 64) 47 | x = tf.nn.relu(self.bn4(self.conv4(x), training=training)) 48 | # 转置卷积-激活函数:(b, 64, 64, 3) 49 | x = self.conv5(x) 50 | x = tf.tanh(x) # 输出x范围-1~1,与预处理一致 51 | 52 | return x 53 | 54 | 55 | class Discriminator(keras.Model): 56 | # 判别器类 57 | def __init__(self): 58 | super(Discriminator, self).__init__() 59 | filter = 64 60 | # 卷积层1 61 | self.conv1 = layers.Conv2D(filter, 4, 2, 'valid', use_bias=False) 62 | self.bn1 = layers.BatchNormalization() 63 | # 卷积层2 64 | self.conv2 = layers.Conv2D(filter * 2, 4, 2, 'valid', use_bias=False) 65 | self.bn2 = layers.BatchNormalization() 66 | # 卷积层3 67 | self.conv3 = layers.Conv2D(filter * 4, 4, 2, 'valid', use_bias=False) 68 | self.bn3 = layers.BatchNormalization() 69 | # 卷积层4 70 | self.conv4 = layers.Conv2D(filter * 8, 3, 1, 'valid', use_bias=False) 71 | self.bn4 = layers.BatchNormalization() 72 | # 卷积层5 73 | self.conv5 = layers.Conv2D(filter * 16, 3, 1, 'valid', use_bias=False) 74 | self.bn5 = layers.BatchNormalization() 75 | # 全局池化层 76 | self.pool = layers.GlobalAveragePooling2D() 77 | # 特征打平层 78 | self.flatten = layers.Flatten() 79 | # 2分类全连接层 80 | self.fc = layers.Dense(1) 81 | 82 | def call(self, inputs, training=None): 83 | # 卷积-BN-激活函数:(4, 31, 31, 64) 84 | x = tf.nn.leaky_relu(self.bn1(self.conv1(inputs), training=training)) 85 | # 卷积-BN-激活函数:(4, 14, 14, 128) 86 | x = tf.nn.leaky_relu(self.bn2(self.conv2(x), training=training)) 87 | # 卷积-BN-激活函数:(4, 6, 6, 256) 88 | x = tf.nn.leaky_relu(self.bn3(self.conv3(x), training=training)) 89 | # 卷积-BN-激活函数:(4, 4, 4, 512) 90 | x = tf.nn.leaky_relu(self.bn4(self.conv4(x), training=training)) 91 | # 卷积-BN-激活函数:(4, 2, 2, 1024) 92 | x = tf.nn.leaky_relu(self.bn5(self.conv5(x), training=training)) 93 | # 卷积-BN-激活函数:(4, 1024) 94 | x = self.pool(x) 95 | # 打平 96 | x = self.flatten(x) 97 | # 输出，[b, 1024] => [b, 1] 98 | logits = self.fc(x) 99 | 100 | return logits 101 | 102 | 103 | def main(): 104 | d = Discriminator() 105 | g = Generator() 106 | 107 | x = tf.random.normal([2, 64, 64, 3]) 108 | z = tf.random.normal([2, 100]) 109 | 110 | prob = d(x) 111 | print(prob) 112 | x_hat = g(z) 113 | print(x_hat.shape) 114 | 115 | 116 | if __name__ == '__main__': 117 | main() 118 | -------------------------------------------------------------------------------- /src/ch14/14.1.2-cartpole-v1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 14.1.2-cartpole-v1.py 6 | @time: 2020/2/29 22:20 7 | @desc: 14.1.2 Gym平台-平衡杆游戏的交互代码 8 | 安装gym： 9 | 1. 将gym文件夹放入venv/Lib/site-packages目录下 10 | 2. cd venv/Lib/site-packages/gym 11 | 3. 执行命令pip install -e . 12 | 4. 检查gym包是否已经安装好 13 | """ 14 | 15 | # 导入 gym 游戏平台 16 | import gym 17 | 18 | # 创建平衡杆游戏环境 19 | env = gym.make("CartPole-v1") 20 | # 复位游戏，回到初始状态 21 | observation = env.reset() 22 | # 循环交互 1000 次 23 | for _ in range(1000): 24 | # 显示当前时间戳的游戏画面 25 | env.render() 26 | # 随机生成一个动作 27 | action = env.action_space.sample() 28 | # 与环境交互，返回新的状态，奖励，是否结束标志，其他信息 29 | observation, reward, done, info = env.step(action) 30 | # 游戏回合结束，复位状态 31 | if done: 32 | observation = env.reset() 33 | 34 | # 销毁游戏环境 35 | env.close() 36 | -------------------------------------------------------------------------------- /src/ch14/14.1.5-cartpole-policy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 14.1.5-cartpole-policy.py 6 | @time: 2020/2/29 22:37 7 | @desc: 14.1.5 平衡杆游戏实战的代码（策略网络Policy） 8 | """ 9 | 10 | import os 11 | 12 | import gym 13 | import matplotlib 14 | import numpy as np 15 | from matplotlib import pyplot as plt 16 | 17 | # Default parameters for plots 18 | matplotlib.rcParams['font.size'] = 18 19 | matplotlib.rcParams['figure.titlesize'] = 18 20 | matplotlib.rcParams['figure.figsize'] = [9, 7] 21 | matplotlib.rcParams['font.family'] = ['KaiTi'] 22 | matplotlib.rcParams['axes.unicode_minus'] = False 23 | 24 | import tensorflow as tf 25 | from tensorflow import keras 26 | from tensorflow.keras import layers, optimizers 27 | 28 | env = gym.make('CartPole-v1') # 创建游戏环境 29 | env.seed(2333) 30 | tf.random.set_seed(2333) 31 | np.random.seed(2333) 32 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 33 | 34 | 35 | class Policy(keras.Model): 36 | # 策略网络，生成动作的概率分布 37 | def __init__(self, learning_rate, gamma): 38 | super(Policy, self).__init__() 39 | self.data = [] # 存储轨迹 40 | # 输入为长度为4的向量，输出为左、右2个动作 41 | self.fc1 = layers.Dense(128, kernel_initializer='he_normal') 42 | self.fc2 = layers.Dense(2, kernel_initializer='he_normal') 43 | # 网络优化器 44 | self.optimizer = optimizers.Adam(lr=learning_rate) 45 | self.gamma = gamma 46 | 47 | def call(self, inputs, training=None): 48 | # 状态输入s的shape为向量：[4] 49 | x = tf.nn.relu(self.fc1(inputs)) 50 | x = tf.nn.softmax(self.fc2(x), axis=1) 51 | return x 52 | 53 | def put_data(self, item): 54 | """ 55 | 在交互时，将每个时间戳上的状态输入s[t]，动作分布输出a[t]， 56 | 环境奖励r[t]和新状态s[t+1]作为一个4元组item记录下来. 57 | """ 58 | # 记录r,log_P(a|s) 59 | self.data.append(item) 60 | 61 | def train_net(self, tape): 62 | # 计算梯度并更新策略网络参数。tape为梯度记录器 63 | # 终结状态的初始回报为0 64 | R = 0 65 | # 逆序取轨迹数据 66 | for r, log_prob in self.data[::-1]: 67 | # 累加计算每个时间戳上的回报 68 | R = r + self.gamma * R 69 | loss = -log_prob * R 70 | with tape.stop_recording(): 71 | # 优化策略网络 72 | grads = tape.gradient(loss, self.trainable_variables) 73 | self.optimizer.apply_gradients(zip(grads, self.trainable_variables)) 74 | 75 | # 清空轨迹 76 | self.data = [] 77 | 78 | 79 | def train(epoch, pi, print_interval, returns, score): 80 | s = env.reset() # 回到游戏初始状态，返回s0 81 | with tf.GradientTape(persistent=True) as tape: 82 | for t in range(501): # CartPole-v1 forced to terminates at 500 step. 83 | # 送入状态向量，获取策略 84 | s = tf.constant(s, dtype=tf.float32) 85 | # s: [4] => [1,4] 86 | s = tf.expand_dims(s, axis=0) 87 | # 动作分布:[1,2] 88 | prob = pi(s) 89 | # 从类别分布中采样1个动作, shape: [1] 90 | a = tf.random.categorical(tf.math.log(prob), 1)[0] 91 | # Tensor转数字 92 | a = int(a) 93 | s_prime, r, done, info = env.step(a) 94 | # 记录动作a和动作产生的奖励r 95 | # prob shape:[1,2] 96 | pi.put_data((r, tf.math.log(prob[0][a]))) 97 | # 刷新状态 98 | s = s_prime 99 | # 累积奖励 100 | score += r 101 | 102 | if epoch > 1000: 103 | env.render() 104 | 105 | # 当前episode终止 106 | if done: 107 | break 108 | 109 | # episode终止后，训练一次网络 110 | pi.train_net(tape) 111 | del tape 112 | 113 | if epoch % print_interval == 0 and epoch != 0: 114 | returns.append(score / print_interval) 115 | # 每20次的平均得分 116 | print(f"# of episode :{epoch}, avg score : {score / print_interval}") 117 | score = 0.0 118 | 119 | 120 | def main(): 121 | learning_rate = 0.0002 122 | gamma = 0.98 123 | 124 | pi = Policy(learning_rate, gamma) # 创建策略网络 125 | pi(tf.random.normal((4, 4))) 126 | pi.summary() 127 | score = 0.0 # 计分 128 | print_interval = 20 # 打印间隔 129 | returns = [0] 130 | epoch_num = 400 131 | 132 | for epoch in range(1, epoch_num + 1): 133 | train(epoch, pi, print_interval, returns, score) 134 | 135 | # 关闭环境 136 | env.close() 137 | 138 | plt.plot(np.arange(len(returns)) * print_interval, returns) 139 | plt.plot(np.arange(len(returns)) * print_interval, returns, 's') 140 | plt.xlabel('回合数') 141 | plt.ylabel('每20次的平均得分') 142 | plt.savefig('14.1.5-cartpole-policy.svg') 143 | 144 | 145 | if __name__ == '__main__': 146 | main() 147 | -------------------------------------------------------------------------------- /src/ch14/14.3.6-cartpole-ppo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 14.3.6-cartpole-ppo.py 6 | @time: 2020/2/29 23:01 7 | @desc: 14.3.6 PPO实战的代码 8 | """ 9 | 10 | import os 11 | from collections import namedtuple 12 | 13 | import gym 14 | import matplotlib 15 | import numpy as np 16 | import tensorflow as tf 17 | from matplotlib import pyplot as plt 18 | from tensorflow import keras 19 | from tensorflow.keras import layers, optimizers, losses 20 | 21 | matplotlib.rcParams['font.size'] = 18 22 | matplotlib.rcParams['figure.titlesize'] = 18 23 | matplotlib.rcParams['figure.figsize'] = [9, 7] 24 | matplotlib.rcParams['font.family'] = ['KaiTi'] 25 | matplotlib.rcParams['axes.unicode_minus'] = False 26 | 27 | env = gym.make('CartPole-v1') # 创建游戏环境 28 | env.seed(2222) 29 | tf.random.set_seed(2222) 30 | np.random.seed(2222) 31 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 32 | 33 | # 创建游戏环境 34 | env = gym.make('CartPole-v0').unwrapped 35 | Transition = namedtuple('Transition', ['state', 'action', 'a_log_prob', 'reward', 'next_state']) 36 | 37 | 38 | class Actor(keras.Model): 39 | def __init__(self): 40 | super(Actor, self).__init__() 41 | # 策略网络，也叫Actor网络，输出为概率分布pi(a|s) 42 | self.fc1 = layers.Dense(100, kernel_initializer='he_normal') 43 | self.fc2 = layers.Dense(2, kernel_initializer='he_normal') 44 | 45 | def call(self, inputs): 46 | x = tf.nn.relu(self.fc1(inputs)) 47 | x = self.fc2(x) 48 | x = tf.nn.softmax(x, axis=1) # 转换成概率 49 | return x 50 | 51 | 52 | class Critic(keras.Model): 53 | def __init__(self): 54 | super(Critic, self).__init__() 55 | # 偏置b的估值网络，也叫Critic网络，输出为v(s) 56 | self.fc1 = layers.Dense(100, kernel_initializer='he_normal') 57 | self.fc2 = layers.Dense(1, kernel_initializer='he_normal') 58 | 59 | def call(self, inputs): 60 | x = tf.nn.relu(self.fc1(inputs)) 61 | # 输出基准线 b 的估计 62 | x = self.fc2(x) 63 | return x 64 | 65 | 66 | class PPO(): 67 | # PPO算法主体 68 | def __init__(self, gamma, batch_size, epsilon): 69 | super(PPO, self).__init__() 70 | # 创建Actor网络 71 | self.actor = Actor() 72 | # 创建Critic网络 73 | self.critic = Critic() 74 | # 数据缓冲池 75 | self.buffer = [] 76 | # Actor优化器 77 | self.actor_optimizer = optimizers.Adam(1e-3) 78 | # Critic优化器 79 | self.critic_optimizer = optimizers.Adam(3e-3) 80 | self._gamma = gamma 81 | self._batch_size = batch_size 82 | self._epsilon = epsilon 83 | 84 | def select_action(self, s): 85 | # 送入状态向量，获取策略: [4] 86 | s = tf.constant(s, dtype=tf.float32) 87 | # s: [4] => [1,4] 88 | s = tf.expand_dims(s, axis=0) 89 | # 获取策略分布: [1, 2] 90 | prob = self.actor(s) 91 | # 从类别分布中采样1个动作, shape: [1] 92 | a = tf.random.categorical(tf.math.log(prob), 1)[0] 93 | # Tensor转数字 94 | a = int(a) 95 | # 返回动作及其概率 96 | return a, float(prob[0][a]) 97 | 98 | def get_value(self, s): 99 | # 送入状态向量，获取策略: [4] 100 | s = tf.constant(s, dtype=tf.float32) 101 | # s: [4] => [1,4] 102 | s = tf.expand_dims(s, axis=0) 103 | # 获取策略分布: [1, 2] 104 | v = self.critic(s)[0] 105 | return float(v) # 返回v(s) 106 | 107 | def store_transition(self, transition): 108 | # 存储采样数据 109 | self.buffer.append(transition) 110 | 111 | def optimize(self): 112 | # 优化网络主函数 113 | # 从缓存中取出样本数据，转换成Tensor 114 | state = tf.constant([t.state for t in self.buffer], dtype=tf.float32) 115 | action = tf.constant([t.action for t in self.buffer], dtype=tf.int32) 116 | action = tf.reshape(action, [-1, 1]) 117 | reward = [t.reward for t in self.buffer] 118 | old_action_log_prob = tf.constant([t.a_log_prob for t in self.buffer], dtype=tf.float32) 119 | old_action_log_prob = tf.reshape(old_action_log_prob, [-1, 1]) 120 | # 通过MC方法循环计算R(st) 121 | R = 0 122 | Rs = [] 123 | for r in reward[::-1]: 124 | R = r + self._gamma * R 125 | Rs.insert(0, R) 126 | Rs = tf.constant(Rs, dtype=tf.float32) 127 | # 对缓冲池数据大致迭代10遍 128 | for _ in range(round(10 * len(self.buffer) / self._batch_size)): 129 | # 随机从缓冲池采样batch size大小样本 130 | index = np.random.choice(np.arange(len(self.buffer)), self._batch_size, replace=False) 131 | # 构建梯度跟踪环境 132 | with tf.GradientTape() as tape1, tf.GradientTape() as tape2: 133 | # 取出R(st)，[b,1] 134 | v_target = tf.expand_dims(tf.gather(Rs, index, axis=0), axis=1) 135 | # 计算v(s)预测值，也就是偏置b，我们后面会介绍为什么写成v 136 | v = self.critic(tf.gather(state, index, axis=0)) 137 | delta = v_target - v # 计算优势值 138 | advantage = tf.stop_gradient(delta) # 断开梯度连接 139 | # 由于TF的gather_nd与pytorch的gather功能不一样，需要构造 140 | # gather_nd需要的坐标参数，indices:[b, 2] 141 | # pi_a = pi.gather(1, a) # pytorch只需要一行即可实现 142 | a = tf.gather(action, index, axis=0) # 取出batch的动作at 143 | # batch的动作分布pi(a|st) 144 | pi = self.actor(tf.gather(state, index, axis=0)) 145 | indices = tf.expand_dims(tf.range(a.shape[0]), axis=1) 146 | indices = tf.concat([indices, a], axis=1) 147 | pi_a = tf.gather_nd(pi, indices) # 动作的概率值pi(at|st), [b] 148 | pi_a = tf.expand_dims(pi_a, axis=1) # [b]=> [b,1] 149 | # 重要性采样 150 | ratio = (pi_a / tf.gather(old_action_log_prob, index, axis=0)) 151 | surr1 = ratio * advantage 152 | surr2 = tf.clip_by_value(ratio, 1 - self._epsilon, 1 + self._epsilon) * advantage 153 | # PPO误差函数 154 | policy_loss = -tf.reduce_mean(tf.minimum(surr1, surr2)) 155 | # 对于偏置v来说，希望与MC估计的R(st)越接近越好 156 | value_loss = losses.MSE(v_target, v) 157 | # 优化策略网络 158 | grads = tape1.gradient(policy_loss, self.actor.trainable_variables) 159 | self.actor_optimizer.apply_gradients(zip(grads, self.actor.trainable_variables)) 160 | # 优化偏置值网络 161 | grads = tape2.gradient(value_loss, self.critic.trainable_variables) 162 | self.critic_optimizer.apply_gradients(zip(grads, self.critic.trainable_variables)) 163 | 164 | self.buffer = [] # 清空已训练数据 165 | 166 | 167 | def train(agent, batch_size, epoch, returns, total, print_interval): 168 | # 复位环境 169 | state = env.reset() 170 | # 最多考虑500步 171 | for t in range(500): 172 | # 通过最新策略与环境交互 173 | action, action_prob = agent.select_action(state) 174 | next_state, reward, done, _ = env.step(action) 175 | # 构建样本并存储 176 | trans = Transition(state, action, action_prob, reward, next_state) 177 | agent.store_transition(trans) 178 | # 刷新状态 179 | state = next_state 180 | # 累积奖励 181 | total += reward 182 | # 合适的时间点训练网络 183 | if done: 184 | if len(agent.buffer) >= batch_size: 185 | # 训练网络 186 | agent.optimize() 187 | break 188 | 189 | # 每20个回合统计一次平均得分 190 | if epoch % print_interval == 0: 191 | returns.append(total / print_interval) 192 | print(f"# of episode :{epoch}, avg score : {total / print_interval}") 193 | total = 0 194 | 195 | 196 | def main(): 197 | gamma = 0.98 # 激励衰减因子 198 | epsilon = 0.2 # PPO误差超参数0.8~1.2 199 | batch_size = 32 # batch size 200 | epoch_num = 500 201 | print_interval = 20 202 | 203 | agent = PPO(gamma, batch_size, epsilon) 204 | # 统计总回报 205 | returns = [0] 206 | total = 0 # 一段时间内平均回报 207 | for epoch in range(1, epoch_num + 1): # 训练回合数 208 | train(agent, batch_size, epoch, returns, total, print_interval) 209 | 210 | print(np.array(returns)) 211 | plt.plot(np.arange(len(returns)) * 20, np.array(returns)) 212 | plt.plot(np.arange(len(returns)) * 20, np.array(returns), 's') 213 | plt.xlabel('回合数') 214 | plt.ylabel('每20次的平均得分') 215 | plt.savefig('14.3.6-cartpole-ppo.svg') 216 | 217 | 218 | if __name__ == '__main__': 219 | main() 220 | print("end") 221 | -------------------------------------------------------------------------------- /src/ch14/14.4.7-cartpole-dqn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 14.4.7-cartpole-dqn.py 6 | @time: 2020/2/29 23:39 7 | @desc: 14.4.7 DQN实战的代码 8 | """ 9 | 10 | import collections 11 | import os 12 | import random 13 | 14 | import gym 15 | import numpy as np 16 | import tensorflow as tf 17 | from tensorflow import keras 18 | from tensorflow.keras import layers, optimizers, losses 19 | 20 | env = gym.make('CartPole-v1') # 创建游戏环境 21 | env.seed(1234) 22 | tf.random.set_seed(1234) 23 | np.random.seed(1234) 24 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 25 | 26 | 27 | class ReplayBuffer(): 28 | # 经验回放池 29 | def __init__(self, buffer_limit): 30 | # 双向队列 31 | self.buffer = collections.deque(maxlen=buffer_limit) 32 | 33 | def put(self, transition): 34 | self.buffer.append(transition) 35 | 36 | def sample(self, n): 37 | # 从回放池采样n个5元组 38 | mini_batch = random.sample(self.buffer, n) 39 | s_lst, a_lst, r_lst, s_prime_lst, done_mask_lst = [], [], [], [], [] 40 | # 按类别进行整理 41 | for transition in mini_batch: 42 | s, a, r, s_prime, done_mask = transition 43 | s_lst.append(s) 44 | a_lst.append([a]) 45 | r_lst.append([r]) 46 | s_prime_lst.append(s_prime) 47 | done_mask_lst.append([done_mask]) 48 | # 转换成Tensor 49 | return tf.constant(s_lst, dtype=tf.float32), tf.constant(a_lst, dtype=tf.int32), \ 50 | tf.constant(r_lst, dtype=tf.float32), tf.constant(s_prime_lst, dtype=tf.float32), \ 51 | tf.constant(done_mask_lst, dtype=tf.float32) 52 | 53 | def size(self): 54 | return len(self.buffer) 55 | 56 | 57 | class Qnet(keras.Model): 58 | def __init__(self): 59 | # 创建Q网络，输入为状态向量，输出为动作的Q值 60 | super(Qnet, self).__init__() 61 | self.fc1 = layers.Dense(256, kernel_initializer='he_normal') 62 | self.fc2 = layers.Dense(256, kernel_initializer='he_normal') 63 | self.fc3 = layers.Dense(2, kernel_initializer='he_normal') 64 | 65 | def call(self, x, training=None): 66 | x = tf.nn.relu(self.fc1(x)) 67 | x = tf.nn.relu(self.fc2(x)) 68 | x = self.fc3(x) 69 | return x 70 | 71 | def sample_action(self, s, epsilon): 72 | # 送入状态向量，获取策略: [4] 73 | s = tf.constant(s, dtype=tf.float32) 74 | # s: [4] => [1,4] 75 | s = tf.expand_dims(s, axis=0) 76 | out = self(s)[0] 77 | coin = random.random() 78 | # 策略改进：e-贪心方式 79 | if coin < epsilon: 80 | # epsilon大的概率随机选取 81 | return random.randint(0, 1) 82 | else: # 选择Q值最大的动作 83 | return int(tf.argmax(out)) 84 | 85 | 86 | def train(q, q_target, memory, optimizer, batch_size, gamma): 87 | # 通过Q网络和影子网络来构造贝尔曼方程的误差， 88 | # 并只更新Q网络，影子网络的更新会滞后Q网络 89 | huber = losses.Huber() 90 | # 训练10次 91 | for i in range(10): 92 | # 从缓冲池采样 93 | s, a, r, s_prime, done_mask = memory.sample(batch_size) 94 | with tf.GradientTape() as tape: 95 | # s: [b, 4] 96 | q_out = q(s) # 得到Q(s,a)的分布 97 | # 由于TF的gather_nd与pytorch的gather功能不一样，需要构造 98 | # gather_nd需要的坐标参数，indices:[b, 2] 99 | # pi_a = pi.gather(1, a) # pytorch只需要一行即可实现 100 | indices = tf.expand_dims(tf.range(a.shape[0]), axis=1) 101 | indices = tf.concat([indices, a], axis=1) 102 | # 动作的概率值, [b] 103 | q_a = tf.gather_nd(q_out, indices) 104 | # [b]=> [b,1] 105 | q_a = tf.expand_dims(q_a, axis=1) 106 | # 得到Q(s',a)的最大值，它来自影子网络！ [b,4]=>[b,2]=>[b,1] 107 | max_q_prime = tf.reduce_max(q_target(s_prime), axis=1, keepdims=True) 108 | # 构造Q(s,a_t)的目标值，来自贝尔曼方程 109 | target = r + gamma * max_q_prime * done_mask 110 | # 计算Q(s,a_t)与目标值的误差 111 | loss = huber(q_a, target) 112 | # 更新网络，使得Q(s,a_t)估计符合贝尔曼方程 113 | grads = tape.gradient(loss, q.trainable_variables) 114 | optimizer.apply_gradients(zip(grads, q.trainable_variables)) 115 | 116 | 117 | def main(): 118 | # Hyperparameters 119 | learning_rate = 0.0002 120 | gamma = 0.99 121 | buffer_limit = 50000 122 | batch_size = 32 123 | # 训练次数 124 | epoch_num = 10000 125 | 126 | env = gym.make('CartPole-v1') # 创建环境 127 | q_net = Qnet() # 创建Q网络 128 | q_target = Qnet() # 创建影子网络 129 | q_net.build(input_shape=(2, 4)) 130 | q_target.build(input_shape=(2, 4)) 131 | for src, dest in zip(q_net.variables, q_target.variables): 132 | dest.assign(src) # 影子网络权值来自Q 133 | memory = ReplayBuffer(buffer_limit) # 创建回放池 134 | 135 | print_interval = 20 136 | score = 0.0 137 | optimizer = optimizers.Adam(lr=learning_rate) 138 | 139 | for epoch in range(epoch_num): 140 | # epsilon概率也会8%到1%衰减，越到后面越使用Q值最大的动作 141 | epsilon = max(0.01, 0.08 - 0.01 * (epoch / 200)) 142 | # 复位环境 143 | s = env.reset() 144 | # 一个回合最大时间戳 145 | for t in range(600): 146 | # 根据当前Q网络提取策略，并改进策略 147 | a = q_net.sample_action(s, epsilon) 148 | # 使用改进的策略与环境交互 149 | s_prime, r, done, info = env.step(a) 150 | # 结束标志掩码 151 | done_mask = 0.0 if done else 1.0 152 | # 保存5元组 153 | memory.put((s, a, r / 100.0, s_prime, done_mask)) 154 | # 刷新状态 155 | s = s_prime 156 | # 记录总得分 157 | score += r 158 | if done: # 回合结束 159 | break 160 | 161 | if memory.size() > 2000: # 缓冲池只有大于2000就可以训练 162 | train(q_net, q_target, memory, optimizer, batch_size, gamma) 163 | 164 | if epoch % print_interval == 0 and epoch != 0: 165 | for src, dest in zip(q_net.variables, q_target.variables): 166 | dest.assign(src) # 影子网络权值来自Q 167 | print("# of episode :{}, avg score : {:.1f}, buffer size : {}, " \ 168 | "epsilon : {:.1f}%" \ 169 | .format(epoch, score / print_interval, memory.size(), epsilon * 100)) 170 | score = 0.0 171 | env.close() 172 | 173 | 174 | if __name__ == '__main__': 175 | main() 176 | -------------------------------------------------------------------------------- /src/ch14/14.5.3-cartpole-a3c.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: 14.5.3-cartpole-a3c.py 6 | @time: 2020/2/29 23:50 7 | @desc: 14.5.3 A3C实战的代码 8 | """ 9 | 10 | import multiprocessing 11 | import os 12 | import threading 13 | from queue import Queue 14 | 15 | import gym 16 | import numpy as np 17 | import tensorflow as tf 18 | from matplotlib import pyplot as plt 19 | from tensorflow import keras 20 | from tensorflow.keras import layers, optimizers 21 | 22 | plt.rcParams['font.size'] = 18 23 | plt.rcParams['figure.titlesize'] = 18 24 | plt.rcParams['figure.figsize'] = [9, 7] 25 | plt.rcParams['font.family'] = ['KaiTi'] 26 | plt.rcParams['axes.unicode_minus'] = False 27 | 28 | tf.random.set_seed(1231) 29 | np.random.seed(1231) 30 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 31 | 32 | 33 | class ActorCritic(keras.Model): 34 | # Actor-Critic模型 35 | def __init__(self, state_size, action_size): 36 | super(ActorCritic, self).__init__() 37 | # 状态向量长度 38 | self.state_size = state_size 39 | # 动作数量 40 | self.action_size = action_size 41 | # 策略网络Actor 42 | self.dense1 = layers.Dense(128, activation='relu') 43 | self.policy_logits = layers.Dense(action_size) 44 | # V网络Critic 45 | self.dense2 = layers.Dense(128, activation='relu') 46 | self.values = layers.Dense(1) 47 | 48 | def call(self, inputs): 49 | # 获得策略分布Pi(a|s) 50 | x = self.dense1(inputs) 51 | logits = self.policy_logits(x) 52 | # 获得v(s) 53 | v = self.dense2(inputs) 54 | values = self.values(v) 55 | return logits, values 56 | 57 | 58 | def record(episode, 59 | episode_reward, 60 | worker_idx, 61 | global_ep_reward, 62 | result_queue, 63 | total_loss, 64 | num_steps): 65 | # 统计工具函数 66 | if global_ep_reward == 0: 67 | global_ep_reward = episode_reward 68 | else: 69 | global_ep_reward = global_ep_reward * 0.99 + episode_reward * 0.01 70 | print( 71 | f"{episode} | " 72 | f"Average Reward: {int(global_ep_reward)} | " 73 | f"Episode Reward: {int(episode_reward)} | " 74 | f"Loss: {int(total_loss / float(num_steps) * 1000) / 1000} | " 75 | f"Steps: {num_steps} | " 76 | f"Worker: {worker_idx}" 77 | ) 78 | result_queue.put(global_ep_reward) # 保存回报，传给主线程 79 | return global_ep_reward 80 | 81 | 82 | class Memory: 83 | def __init__(self): 84 | self.states = [] 85 | self.actions = [] 86 | self.rewards = [] 87 | 88 | def store(self, state, action, reward): 89 | self.states.append(state) 90 | self.actions.append(action) 91 | self.rewards.append(reward) 92 | 93 | def clear(self): 94 | self.states = [] 95 | self.actions = [] 96 | self.rewards = [] 97 | 98 | 99 | class Agent: 100 | # 智能体，包含了中央参数网络server 101 | def __init__(self): 102 | # server优化器，client不需要，直接从server拉取参数 103 | self.opt = optimizers.Adam(1e-3) 104 | # 中央模型，类似于参数服务器 105 | self.server = ActorCritic(4, 2) # 状态向量，动作数量 106 | self.server(tf.random.normal((2, 4))) 107 | 108 | def train(self): 109 | res_queue = Queue() # 共享队列 110 | # 创建各个交互环境 111 | workers = [Worker(self.server, self.opt, res_queue, i) 112 | for i in range(multiprocessing.cpu_count())] 113 | for i, worker in enumerate(workers): 114 | print("Starting worker {}".format(i)) 115 | worker.start() 116 | # 统计并绘制总回报曲线 117 | returns = [] 118 | while True: 119 | reward = res_queue.get() 120 | if reward is not None: 121 | returns.append(reward) 122 | else: # 结束标志 123 | break 124 | [w.join() for w in workers] # 等待线程退出 125 | 126 | print(returns) 127 | plt.figure() 128 | plt.plot(np.arange(len(returns)), returns) 129 | plt.xlabel('回合数') 130 | plt.ylabel('总奖励得分') 131 | plt.savefig('14.5.3-cartpole-a3c.svg') 132 | 133 | 134 | class Worker(threading.Thread): 135 | def __init__(self, server, opt, result_queue, idx): 136 | super(Worker, self).__init__() 137 | # 共享队列 138 | self.result_queue = result_queue 139 | # 中央模型 140 | self.server = server 141 | # 中央优化器 142 | self.opt = opt 143 | # 线程私有网络 144 | self.client = ActorCritic(4, 2) 145 | self.worker_idx = idx # 线程id 146 | self.env = gym.make('CartPole-v1').unwrapped 147 | self.ep_loss = 0.0 148 | 149 | def run(self): 150 | mem = Memory() # 每个worker自己维护一个memory 151 | for epi_counter in range(500): # 未达到最大回合数 152 | current_state = self.env.reset() # 复位client游戏状态 153 | mem.clear() 154 | ep_reward = 0. 155 | ep_steps = 0 156 | done = False 157 | while not done: 158 | # 获得Pi(a|s),未经softmax 159 | logits, _ = self.client(tf.constant(current_state[None, :], dtype=tf.float32)) 160 | probs = tf.nn.softmax(logits) 161 | # 随机采样动作 162 | action = np.random.choice(2, p=probs.numpy()[0]) 163 | # 交互 164 | new_state, reward, done, _ = self.env.step(action) 165 | # 累加奖励 166 | ep_reward += reward 167 | # 记录 168 | mem.store(current_state, action, reward) 169 | # 计算回合步数 170 | ep_steps += 1 171 | # 刷新状态 172 | current_state = new_state 173 | 174 | # 最长步数500 175 | if ep_steps >= 500 or done: 176 | # 计算当前client上的误差 177 | with tf.GradientTape() as tape: 178 | total_loss = self.compute_loss(done, new_state, mem) 179 | 180 | # 计算误差 181 | grads = tape.gradient(total_loss, self.client.trainable_weights) 182 | # 梯度提交到server，在server上更新梯度 183 | self.opt.apply_gradients(zip(grads, self.server.trainable_weights)) 184 | # 从server拉取最新的梯度 185 | self.client.set_weights(self.server.get_weights()) 186 | # 清空Memory 187 | mem.clear() 188 | # 统计此回合回报 189 | self.result_queue.put(ep_reward) 190 | print("epoch=%s," % epi_counter, "worker=%s," % self.worker_idx, "reward=%s" % ep_reward) 191 | break 192 | 193 | # 结束线程 194 | self.result_queue.put(None) 195 | 196 | def compute_loss(self, 197 | done, 198 | new_state, 199 | memory, 200 | gamma=0.99): 201 | if done: 202 | reward_sum = 0. # 终止状态的v(终止)=0 203 | else: 204 | reward_sum = self.client(tf.constant(new_state[None, :], dtype=tf.float32))[-1].numpy()[0] 205 | # 统计折扣回报 206 | discounted_rewards = [] 207 | for reward in memory.rewards[::-1]: # reverse buffer r 208 | reward_sum = reward + gamma * reward_sum 209 | discounted_rewards.append(reward_sum) 210 | discounted_rewards.reverse() 211 | # 获取状态的Pi(a|s)和v(s) 212 | logits, values = self.client(tf.constant(np.vstack(memory.states), dtype=tf.float32)) 213 | # 计算advantage = R() - v(s) 214 | advantage = tf.constant(np.array(discounted_rewards)[:, None], dtype=tf.float32) - values 215 | # Critic网络损失 216 | value_loss = advantage ** 2 217 | # 策略损失 218 | policy = tf.nn.softmax(logits) 219 | policy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits( 220 | labels=memory.actions, logits=logits) 221 | # 计算策略网络损失时，并不会计算V网络 222 | policy_loss = policy_loss * tf.stop_gradient(advantage) 223 | # Entropy Bonus 224 | entropy = tf.nn.softmax_cross_entropy_with_logits(labels=policy, 225 | logits=logits) 226 | policy_loss = policy_loss - 0.01 * entropy 227 | # 聚合各个误差 228 | total_loss = tf.reduce_mean((0.5 * value_loss + policy_loss)) 229 | return total_loss 230 | 231 | 232 | if __name__ == '__main__': 233 | master = Agent() 234 | master.train() 235 | -------------------------------------------------------------------------------- /src/ch15/pokemon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: pokemon.py 6 | @time: 2020/3/1 0:35 7 | @desc: 15.3 宝可梦数据集实战 8 | 将pokemon文件夹放到ch15目录下，读取的图片路径为/ch15/pokemon， 9 | 通过在ch15目录下执行tensorboard --logdir=logs，查看tensorboard可视化 10 | """ 11 | 12 | import csv 13 | import glob 14 | import os 15 | import random 16 | 17 | import tensorflow as tf 18 | 19 | 20 | def load_csv(root, filename, name2label): 21 | # 从csv文件返回images,labels列表 22 | # root:数据集根目录，filename:csv文件名， name2label:类别名编码表 23 | if not os.path.exists(os.path.join(root, filename)): 24 | # 如果csv文件不存在，则创建 25 | images = [] 26 | for name in name2label.keys(): # 遍历所有子目录，获得所有的图片 27 | # 只考虑后缀为png,jpg,jpeg的图片：'pokemon\\mewtwo\\00001.png 28 | images += glob.glob(os.path.join(root, name, '*.png')) 29 | images += glob.glob(os.path.join(root, name, '*.jpg')) 30 | images += glob.glob(os.path.join(root, name, '*.jpeg')) 31 | # 打印数据集信息：1167, 'pokemon\\bulbasaur\\00000000.png' 32 | print(len(images), images) 33 | random.shuffle(images) # 随机打散顺序 34 | # 创建csv文件，并存储图片路径及其label信息 35 | with open(os.path.join(root, filename), mode='w', newline='') as f: 36 | writer = csv.writer(f) 37 | for img in images: # 'pokemon\\bulbasaur\\00000000.png' 38 | name = img.split(os.sep)[-2] 39 | label = name2label[name] 40 | # 'pokemon\\bulbasaur\\00000000.png', 0 41 | writer.writerow([img, label]) 42 | print('written into csv file:', filename) 43 | 44 | # 此时已经有csv文件，直接读取 45 | images, labels = [], [] 46 | with open(os.path.join(root, filename)) as f: 47 | reader = csv.reader(f) 48 | for row in reader: 49 | # 'pokemon\\bulbasaur\\00000000.png', 0 50 | img, label = row 51 | label = int(label) 52 | images.append(img) 53 | labels.append(label) 54 | # 返回图片路径list和标签list 55 | return images, labels 56 | 57 | 58 | def load_pokemon(root, mode='train'): 59 | # 创建数字编码表 60 | name2label = {} # "sq...":0 61 | # 遍历根目录下的子文件夹，并排序，保证映射关系固定 62 | for name in sorted(os.listdir(os.path.join(root))): 63 | # 跳过非文件夹 64 | if not os.path.isdir(os.path.join(root, name)): 65 | continue 66 | # 给每个类别编码一个数字 67 | name2label[name] = len(name2label.keys()) 68 | 69 | # 读取Label信息 70 | # [file1,file2,], [3,1] 71 | images, labels = load_csv(root, 'images.csv', name2label) 72 | 73 | if mode == 'train': # 60% 74 | images = images[:int(0.6 * len(images))] 75 | labels = labels[:int(0.6 * len(labels))] 76 | elif mode == 'val': # 20% = 60%->80% 77 | images = images[int(0.6 * len(images)):int(0.8 * len(images))] 78 | labels = labels[int(0.6 * len(labels)):int(0.8 * len(labels))] 79 | else: # 20% = 80%->100% 80 | images = images[int(0.8 * len(images)):] 81 | labels = labels[int(0.8 * len(labels)):] 82 | 83 | return images, labels, name2label 84 | 85 | 86 | # 这里的mean和std根据真实的数据计算获得，比如ImageNet 87 | img_mean = tf.constant([0.485, 0.456, 0.406]) 88 | img_std = tf.constant([0.229, 0.224, 0.225]) 89 | 90 | 91 | def normalize(x, mean=img_mean, std=img_std): 92 | # 标准化 93 | # x: [224, 224, 3] 94 | # mean: [224, 224, 3], std: [3] 95 | x = (x - mean) / std 96 | return x 97 | 98 | 99 | def denormalize(x, mean=img_mean, std=img_std): 100 | # 标准化的逆过程 101 | x = x * std + mean 102 | return x 103 | 104 | 105 | def preprocess(x, y): 106 | # x: 图片的路径List，y：图片的数字编码List 107 | x = tf.io.read_file(x) # 根据路径读取图片 108 | x = tf.image.decode_jpeg(x, channels=3) # 图片解码 109 | x = tf.image.resize(x, [244, 244]) # 图片缩放 110 | 111 | # 数据增强 112 | # x = tf.image.random_flip_up_down(x) 113 | x = tf.image.random_flip_left_right(x) # 左右镜像 114 | x = tf.image.random_crop(x, [224, 224, 3]) # 随机裁剪 115 | # 转换成张量 116 | # x: [0,255]=> 0~1 117 | x = tf.cast(x, dtype=tf.float32) / 255. 118 | # 0~1 => D(0,1) 119 | x = normalize(x) # 标准化 120 | y = tf.convert_to_tensor(y) # 转换成张量 121 | 122 | return x, y 123 | 124 | 125 | def main(): 126 | import time 127 | 128 | # 加载pokemon数据集，指定加载训练集 129 | # 返回训练集的样本路径列表，标签数字列表和编码表字典 130 | images, labels, table = load_pokemon('pokemon', 'train') 131 | print('images:', len(images), images) 132 | print('labels:', len(labels), labels) 133 | print('table:', table) 134 | 135 | # images: string path 136 | # labels: number 137 | db = tf.data.Dataset.from_tensor_slices((images, labels)) 138 | db = db.shuffle(1000).map(preprocess).batch(32) 139 | 140 | # 创建TensorBoard对象 141 | writter = tf.summary.create_file_writer('logs') 142 | for step, (x, y) in enumerate(db): 143 | # x: [32, 224, 224, 3] 144 | # y: [32] 145 | with writter.as_default(): 146 | x = denormalize(x) # 反向normalize，方便可视化 147 | # 写入图片数据 148 | tf.summary.image('img', x, step=step, max_outputs=9) 149 | time.sleep(5) 150 | 151 | 152 | if __name__ == '__main__': 153 | main() 154 | -------------------------------------------------------------------------------- /src/ch15/resnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: resnet.py 6 | @time: 2020/3/1 0:35 7 | @desc: 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow import keras 15 | from tensorflow.keras import layers 16 | 17 | tf.random.set_seed(22) 18 | np.random.seed(22) 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | 22 | class ResnetBlock(keras.Model): 23 | 24 | def __init__(self, channels, strides=1): 25 | super(ResnetBlock, self).__init__() 26 | 27 | self.channels = channels 28 | self.strides = strides 29 | 30 | self.conv1 = layers.Conv2D(channels, 3, strides=strides, 31 | padding=[[0, 0], [1, 1], [1, 1], [0, 0]]) 32 | self.bn1 = keras.layers.BatchNormalization() 33 | self.conv2 = layers.Conv2D(channels, 3, strides=1, 34 | padding=[[0, 0], [1, 1], [1, 1], [0, 0]]) 35 | self.bn2 = keras.layers.BatchNormalization() 36 | 37 | if strides != 1: 38 | self.down_conv = layers.Conv2D(channels, 1, strides=strides, padding='valid') 39 | self.down_bn = tf.keras.layers.BatchNormalization() 40 | 41 | def call(self, inputs, training=None): 42 | residual = inputs 43 | 44 | x = self.conv1(inputs) 45 | x = tf.nn.relu(x) 46 | x = self.bn1(x, training=training) 47 | x = self.conv2(x) 48 | x = tf.nn.relu(x) 49 | x = self.bn2(x, training=training) 50 | 51 | # 残差连接 52 | if self.strides != 1: 53 | residual = self.down_conv(inputs) 54 | residual = tf.nn.relu(residual) 55 | residual = self.down_bn(residual, training=training) 56 | 57 | x = x + residual 58 | x = tf.nn.relu(x) 59 | return x 60 | 61 | 62 | class ResNet(keras.Model): 63 | 64 | def __init__(self, num_classes, initial_filters=16, **kwargs): 65 | super(ResNet, self).__init__(**kwargs) 66 | 67 | self.stem = layers.Conv2D(initial_filters, 3, strides=3, padding='valid') 68 | 69 | self.blocks = keras.models.Sequential([ 70 | ResnetBlock(initial_filters * 2, strides=3), 71 | ResnetBlock(initial_filters * 2, strides=1), 72 | # layers.Dropout(rate=0.5), 73 | 74 | ResnetBlock(initial_filters * 4, strides=3), 75 | ResnetBlock(initial_filters * 4, strides=1), 76 | 77 | ResnetBlock(initial_filters * 8, strides=2), 78 | ResnetBlock(initial_filters * 8, strides=1), 79 | 80 | ResnetBlock(initial_filters * 16, strides=2), 81 | ResnetBlock(initial_filters * 16, strides=1), 82 | ]) 83 | 84 | self.final_bn = layers.BatchNormalization() 85 | self.avg_pool = layers.GlobalMaxPool2D() 86 | self.fc = layers.Dense(num_classes) 87 | 88 | def call(self, inputs, training=None): 89 | # print('x:',inputs.shape) 90 | out = self.stem(inputs) 91 | out = tf.nn.relu(out) 92 | 93 | # print('stem:',out.shape) 94 | 95 | out = self.blocks(out, training=training) 96 | # print('res:',out.shape) 97 | 98 | out = self.final_bn(out, training=training) 99 | # out = tf.nn.relu(out) 100 | 101 | out = self.avg_pool(out) 102 | 103 | # print('avg_pool:',out.shape) 104 | out = self.fc(out) 105 | 106 | # print('out:',out.shape) 107 | 108 | return out 109 | 110 | 111 | def main(): 112 | num_classes = 5 113 | 114 | resnet18 = ResNet(num_classes) 115 | resnet18.build(input_shape=(4, 224, 224, 3)) 116 | resnet18.summary() 117 | 118 | 119 | if __name__ == '__main__': 120 | main() 121 | -------------------------------------------------------------------------------- /src/ch15/train_scratch_dense_net.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: train_scratch_dense_net.py 6 | @time: 2020/3/1 0:36 7 | @desc: 15.3.3 创建模型（从零训练 DenseNet 网络） 8 | """ 9 | 10 | import os 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from matplotlib import pyplot as plt 15 | from tensorflow import keras 16 | from tensorflow.keras import layers, optimizers, losses 17 | from tensorflow.keras.callbacks import EarlyStopping 18 | 19 | from src.ch15.pokemon import load_pokemon, normalize 20 | 21 | tf.random.set_seed(1234) 22 | np.random.seed(1234) 23 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 24 | 25 | plt.rcParams['font.size'] = 18 26 | plt.rcParams['figure.titlesize'] = 18 27 | plt.rcParams['figure.figsize'] = [9, 7] 28 | plt.rcParams['font.family'] = ['KaiTi'] 29 | plt.rcParams['axes.unicode_minus'] = False 30 | 31 | 32 | def preprocess(x, y): 33 | # x: 图片的路径，y：图片的数字编码 34 | x = tf.io.read_file(x) 35 | x = tf.image.decode_jpeg(x, channels=3) # RGBA 36 | x = tf.image.resize(x, [244, 244]) 37 | 38 | x = tf.image.random_flip_left_right(x) 39 | x = tf.image.random_flip_up_down(x) 40 | x = tf.image.random_crop(x, [224, 224, 3]) 41 | 42 | # x: [0,255]=> -1~1 43 | x = tf.cast(x, dtype=tf.float32) / 255. 44 | x = normalize(x) 45 | y = tf.convert_to_tensor(y) 46 | y = tf.one_hot(y, depth=5) 47 | 48 | return x, y 49 | 50 | 51 | def load_dataset(): 52 | batchsz = 32 53 | # 创建训练集Datset对象 54 | images, labels, table = load_pokemon('pokemon', mode='train') 55 | db_train = tf.data.Dataset.from_tensor_slices((images, labels)) 56 | db_train = db_train.shuffle(1000).map(preprocess).batch(batchsz) 57 | # 创建验证集Datset对象 58 | images2, labels2, table = load_pokemon('pokemon', mode='val') 59 | db_val = tf.data.Dataset.from_tensor_slices((images2, labels2)) 60 | db_val = db_val.map(preprocess).batch(batchsz) 61 | # 创建测试集Datset对象 62 | images3, labels3, table = load_pokemon('pokemon', mode='test') 63 | db_test = tf.data.Dataset.from_tensor_slices((images3, labels3)) 64 | db_test = db_test.map(preprocess).batch(batchsz) 65 | 66 | return db_train, db_val, db_test 67 | 68 | 69 | def build_net(): 70 | # 加载DenseNet网络模型，并去掉最后一层全连接层，最后一个池化层设置为max pooling 71 | net = keras.applications.DenseNet121(include_top=False, pooling='max') 72 | # 设置为 True，即 DenseNet 部分的参数也参与优化更新 73 | net.trainable = True 74 | newnet = keras.Sequential([ 75 | net, # 去掉最后一层的DenseNet121 76 | layers.Dense(1024, activation='relu'), # 追加全连接层 77 | layers.BatchNormalization(), # 追加BN层 78 | layers.Dropout(rate=0.5), # 追加Dropout层，防止过拟合 79 | layers.Dense(5) # 根据宝可梦数据的任务，设置最后一层输出节点数为5 80 | ]) 81 | newnet.build(input_shape=(4, 224, 224, 3)) 82 | newnet.summary() 83 | 84 | return newnet 85 | 86 | 87 | def main(): 88 | db_train, db_val, db_test = load_dataset() 89 | newnet = build_net() 90 | # 创建Early Stopping类，连续3次不下降则终止 91 | early_stopping = EarlyStopping( 92 | monitor='val_accuracy', 93 | min_delta=0.001, 94 | patience=3 95 | ) 96 | 97 | # 装配模型 98 | newnet.compile(optimizer=optimizers.Adam(lr=1e-3), 99 | loss=losses.CategoricalCrossentropy(from_logits=True), 100 | metrics=['accuracy']) 101 | # 训练模型，支持 early stopping 102 | history = newnet.fit(db_train, validation_data=db_val, validation_freq=1, epochs=100, 103 | callbacks=[early_stopping]) 104 | history = history.history 105 | print(history.keys()) 106 | print(history['val_accuracy']) 107 | print(history['accuracy']) 108 | test_acc = newnet.evaluate(db_test) 109 | 110 | plt.figure() 111 | returns = history['val_accuracy'] 112 | plt.plot(np.arange(len(returns)), returns, label='验证准确率') 113 | plt.plot(np.arange(len(returns)), returns, 's') 114 | returns = history['accuracy'] 115 | plt.plot(np.arange(len(returns)), returns, label='训练准确率') 116 | plt.plot(np.arange(len(returns)), returns, 's') 117 | 118 | plt.plot([len(returns) - 1], [test_acc[-1]], 'D', label='测试准确率') 119 | plt.legend() 120 | plt.xlabel('Epoch') 121 | plt.ylabel('准确率') 122 | plt.savefig('train_scratch_dense_net.svg') 123 | 124 | 125 | if __name__ == '__main__': 126 | main() 127 | -------------------------------------------------------------------------------- /src/ch15/train_transfer_dense_net.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | @author: HuRuiFeng 5 | @file: train_transfer_dense_net.py 6 | @time: 2020/3/1 0:37 7 | @desc: 15.4 迁移学习 8 | """ 9 | 10 | import os 11 | 12 | import matplotlib 13 | import numpy as np 14 | import tensorflow as tf 15 | from matplotlib import pyplot as plt 16 | from tensorflow import keras 17 | from tensorflow.keras import layers, optimizers, losses 18 | from tensorflow.keras.callbacks import EarlyStopping 19 | 20 | from src.ch15.pokemon import load_pokemon, normalize 21 | 22 | tf.random.set_seed(2222) 23 | np.random.seed(2222) 24 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 25 | 26 | matplotlib.rcParams['font.size'] = 18 27 | matplotlib.rcParams['figure.titlesize'] = 18 28 | matplotlib.rcParams['figure.figsize'] = [9, 7] 29 | matplotlib.rcParams['font.family'] = ['KaiTi'] 30 | matplotlib.rcParams['axes.unicode_minus'] = False 31 | 32 | 33 | def preprocess(x, y): 34 | # x: 图片的路径，y：图片的数字编码 35 | x = tf.io.read_file(x) 36 | x = tf.image.decode_jpeg(x, channels=3) # RGBA 37 | x = tf.image.resize(x, [244, 244]) 38 | 39 | x = tf.image.random_flip_left_right(x) 40 | x = tf.image.random_flip_up_down(x) 41 | x = tf.image.random_crop(x, [224, 224, 3]) 42 | 43 | # x: [0,255]=> -1~1 44 | x = tf.cast(x, dtype=tf.float32) / 255. 45 | x = normalize(x) 46 | y = tf.convert_to_tensor(y) 47 | y = tf.one_hot(y, depth=5) 48 | 49 | return x, y 50 | 51 | 52 | def load_dataset(): 53 | batchsz = 128 54 | # 创建训练集Datset对象 55 | images, labels, table = load_pokemon('pokemon', mode='train') 56 | db_train = tf.data.Dataset.from_tensor_slices((images, labels)) 57 | db_train = db_train.shuffle(1000).map(preprocess).batch(batchsz) 58 | # 创建验证集Datset对象 59 | images2, labels2, table = load_pokemon('pokemon', mode='val') 60 | db_val = tf.data.Dataset.from_tensor_slices((images2, labels2)) 61 | db_val = db_val.map(preprocess).batch(batchsz) 62 | # 创建测试集Datset对象 63 | images3, labels3, table = load_pokemon('pokemon', mode='test') 64 | db_test = tf.data.Dataset.from_tensor_slices((images3, labels3)) 65 | db_test = db_test.map(preprocess).batch(batchsz) 66 | 67 | return db_train, db_val, db_test 68 | 69 | 70 | def build_network(): 71 | # 加载DenseNet网络模型，并去掉最后一层全连接层，最后一个池化层设置为max pooling 72 | # 并使用预训练的参数初始化 73 | net = keras.applications.DenseNet121(weights='imagenet', include_top=False, pooling='max') 74 | # 设计为不参与优化，即DenseNet这部分参数固定不动 75 | net.trainable = False 76 | newnet = keras.Sequential([ 77 | net, # 去掉最后一层的DenseNet121 78 | layers.Dense(1024, activation='relu'), # 追加全连接层 79 | layers.BatchNormalization(), # 追加BN层 80 | layers.Dropout(rate=0.5), # 追加Dropout层，防止过拟合 81 | layers.Dense(5) # 根据宝可梦数据的任务，设置最后一层输出节点数为5 82 | ]) 83 | newnet.build(input_shape=(4, 224, 224, 3)) 84 | newnet.summary() 85 | return newnet 86 | 87 | 88 | def main(): 89 | db_train, db_val, db_test = load_dataset() 90 | newnet = build_network() 91 | # 创建Early Stopping类，连续3次不下降则终止 92 | early_stopping = EarlyStopping( 93 | monitor='val_accuracy', 94 | min_delta=0.001, 95 | patience=3 96 | ) 97 | 98 | newnet.compile(optimizer=optimizers.Adam(lr=1e-3), 99 | loss=losses.CategoricalCrossentropy(from_logits=True), 100 | metrics=['accuracy']) 101 | history = newnet.fit(db_train, validation_data=db_val, validation_freq=1, epochs=100, 102 | callbacks=[early_stopping]) 103 | history = history.history 104 | print(history.keys()) 105 | print(history['val_accuracy']) 106 | print(history['accuracy']) 107 | test_acc = newnet.evaluate(db_test) 108 | 109 | plt.figure() 110 | returns = history['val_accuracy'] 111 | plt.plot(np.arange(len(returns)), returns, label='验证准确率') 112 | plt.plot(np.arange(len(returns)), returns, 's') 113 | returns = history['accuracy'] 114 | plt.plot(np.arange(len(returns)), returns, label='训练准确率') 115 | plt.plot(np.arange(len(returns)), returns, 's') 116 | 117 | plt.plot([len(returns) - 1], [test_acc[-1]], 'D', label='测试准确率') 118 | plt.legend() 119 | plt.xlabel('Epoch') 120 | plt.ylabel('准确率') 121 | plt.savefig('train_transfer_dense_net.svg') 122 | 123 | 124 | if __name__ == '__main__': 125 | main() 126 | --------------------------------------------------------------------------------