9 |
10 | 
11 | 
12 | 
13 | 
14 | 
15 |
8 |
9 |
10 |
13 |
14 |
15 |
51 |
--------------------------------------------------------------------------------
/Tutorial/README.md:
--------------------------------------------------------------------------------
1 | ## 深度学习实战教程
2 |
3 | #### 说明
4 |
5 | * 该系列教程为[@hanbingtao](https://github.com/hanbt/learn_dl "悬停显示")所做，感觉很适合入门，顾转载到了个人网站。开源免费的好教程不多，望多多支持和推荐。
6 |
7 | * 可以到我的个人网站看该系列文章：[点击查看](https://cuijiahua.com/blog/dl/ "悬停显示")
8 |
9 | * 也可以到原作者的作业部落学习：[点击查看](https://www.zybuluo.com/hanbingtao/note/448086 "悬停显示")
--------------------------------------------------------------------------------
/Tutorial/lesson-1/perceptron.py:
--------------------------------------------------------------------------------
1 | """
2 | 作者：https://github.com/hanbt/learn_dl
3 | """
4 | from functools import reduce
5 |
6 | class Perceptron():
7 | def __init__(self, input_num, activator):
8 | '''
9 | 初始化感知器，设置输入参数的个数，以及激活函数。
10 | 激活函数的类型为double -> double
11 | '''
12 | self.activator = activator
13 | # 权重向量初始化为0
14 | self.weights = [0.0 for _ in range(input_num)]
15 | # 偏置项初始化为0
16 | self.bias = 0.0
17 | def __str__(self):
18 | '''
19 | 打印学习到的权重、偏置项
20 | '''
21 | return 'weights\t:%s\nbias\t:%f\n' % (self.weights, self.bias)
22 | def predict(self, input_vec):
23 | '''
24 | 输入向量，输出感知器的计算结果
25 | '''
26 | # 把input_vec[x1,x2,x3...]和weights[w1,w2,w3,...]打包在一起
27 | # 变成[(x1,w1),(x2,w2),(x3,w3),...]
28 | # 然后利用map函数计算[x1*w1, x2*w2, x3*w3]
29 | # 最后利用reduce求和
30 | return self.activator(
31 | reduce(lambda a, b: a + b,list(map(lambda x, w: x * w, input_vec, self.weights)), 0.0) + self.bias)
32 | def train(self, input_vecs, labels, iteration, rate):
33 | '''
34 | 输入训练数据：一组向量、与每个向量对应的label；以及训练轮数、学习率
35 | '''
36 | for i in range(iteration):
37 | self._one_iteration(input_vecs, labels, rate)
38 | def _one_iteration(self, input_vecs, labels, rate):
39 | '''
40 | 一次迭代，把所有的训练数据过一遍
41 | '''
42 | # 把输入和输出打包在一起，成为样本的列表[(input_vec, label), ...]
43 | # 而每个训练样本是(input_vec, label)
44 | samples = zip(input_vecs, labels)
45 | # 对每个样本，按照感知器规则更新权重
46 | for (input_vec, label) in samples:
47 | # 计算感知器在当前权重下的输出
48 | output = self.predict(input_vec)
49 | # 更新权重
50 | self._update_weights(input_vec, output, label, rate)
51 | def _update_weights(self, input_vec, output, label, rate):
52 | '''
53 | 按照感知器规则更新权重
54 | '''
55 | # 把input_vec[x1,x2,x3,...]和weights[w1,w2,w3,...]打包在一起
56 | # 变成[(x1,w1),(x2,w2),(x3,w3),...]
57 | # 然后利用感知器规则更新权重
58 | delta = label - output
59 | self.weights = list(map(lambda x, w: w + rate * delta * x, input_vec, self.weights))
60 | # 更新bias
61 | self.bias += rate * delta
62 |
63 | def f(x):
64 | '''
65 | 定义激活函数f
66 | '''
67 | return 1 if x > 0 else 0
68 | def get_training_dataset():
69 | '''
70 | 基于and真值表构建训练数据
71 | '''
72 | # 构建训练数据
73 | # 输入向量列表
74 | input_vecs = [[1,1], [0,0], [1,0], [0,1]]
75 | # 期望的输出列表，注意要与输入一一对应
76 | # [1,1] -> 1, [0,0] -> 0, [1,0] -> 0, [0,1] -> 0
77 | labels = [1, 0, 0, 0]
78 | return input_vecs, labels
79 | def train_and_perceptron():
80 | '''
81 | 使用and真值表训练感知器
82 | '''
83 | # 创建感知器，输入参数个数为2（因为and是二元函数），激活函数为f
84 | p = Perceptron(2, f)
85 | # 训练，迭代10轮, 学习速率为0.1
86 | input_vecs, labels = get_training_dataset()
87 | p.train(input_vecs, labels, 10, 0.1)
88 | #返回训练好的感知器
89 | return p
90 | if __name__ == '__main__':
91 | # 训练and感知器
92 | and_perception = train_and_perceptron()
93 | # 打印训练获得的权重
94 | print(and_perception)
95 | # 测试
96 | print('1 and 1 = %d' % and_perception.predict([1, 1]))
97 | print('0 and 0 = %d' % and_perception.predict([0, 0]))
98 | print('1 and 0 = %d' % and_perception.predict([1, 0]))
99 | print('0 and 1 = %d' % and_perception.predict([0, 1]))
--------------------------------------------------------------------------------
/Tutorial/lesson-2/linear_unit.py:
--------------------------------------------------------------------------------
1 | from perceptron import Perceptron
2 | #定义激活函数f
3 | f = lambda x: x
4 | class LinearUnit(Perceptron):
5 | def __init__(self, input_num):
6 | '''初始化线性单元，设置输入参数的个数'''
7 | Perceptron.__init__(self, input_num, f)
8 |
9 | def get_training_dataset():
10 | '''
11 | 捏造5个人的收入数据
12 | '''
13 | # 构建训练数据
14 | # 输入向量列表，每一项是工作年限
15 | input_vecs = [[5], [3], [8], [1.4], [10.1]]
16 | # 期望的输出列表，月薪，注意要与输入一一对应
17 | labels = [5500, 2300, 7600, 1800, 11400]
18 | return input_vecs, labels
19 | def train_linear_unit():
20 | '''
21 | 使用数据训练线性单元
22 | '''
23 | # 创建感知器，输入参数的特征数为1（工作年限）
24 | lu = LinearUnit(1)
25 | # 训练，迭代10轮, 学习速率为0.01
26 | input_vecs, labels = get_training_dataset()
27 | lu.train(input_vecs, labels, 10, 0.01)
28 | #返回训练好的线性单元
29 | return lu
30 | if __name__ == '__main__':
31 | '''训练线性单元'''
32 | linear_unit = train_linear_unit()
33 | # 打印训练获得的权重
34 | print(linear_unit)
35 | # 测试
36 | print('Work 3.4 years, monthly salary = %.2f' % linear_unit.predict([3.4]))
37 | print('Work 15 years, monthly salary = %.2f' % linear_unit.predict([15]))
38 | print('Work 1.5 years, monthly salary = %.2f' % linear_unit.predict([1.5]))
39 | print('Work 6.3 years, monthly salary = %.2f' % linear_unit.predict([6.3]))
--------------------------------------------------------------------------------
/Tutorial/lesson-2/perceptron.py:
--------------------------------------------------------------------------------
1 | """
2 | 作者：https://github.com/hanbt/learn_dl
3 | """
4 | from functools import reduce
5 |
6 | class Perceptron():
7 | def __init__(self, input_num, activator):
8 | '''
9 | 初始化感知器，设置输入参数的个数，以及激活函数。
10 | 激活函数的类型为double -> double
11 | '''
12 | self.activator = activator
13 | # 权重向量初始化为0
14 | self.weights = [0.0 for _ in range(input_num)]
15 | # 偏置项初始化为0
16 | self.bias = 0.0
17 | def __str__(self):
18 | '''
19 | 打印学习到的权重、偏置项
20 | '''
21 | return 'weights\t:%s\nbias\t:%f\n' % (self.weights, self.bias)
22 | def predict(self, input_vec):
23 | '''
24 | 输入向量，输出感知器的计算结果
25 | '''
26 | # 把input_vec[x1,x2,x3...]和weights[w1,w2,w3,...]打包在一起
27 | # 变成[(x1,w1),(x2,w2),(x3,w3),...]
28 | # 然后利用map函数计算[x1*w1, x2*w2, x3*w3]
29 | # 最后利用reduce求和
30 | return self.activator(
31 | reduce(lambda a, b: a + b,list(map(lambda x, w: x * w, input_vec, self.weights)), 0.0) + self.bias)
32 | def train(self, input_vecs, labels, iteration, rate):
33 | '''
34 | 输入训练数据：一组向量、与每个向量对应的label；以及训练轮数、学习率
35 | '''
36 | for i in range(iteration):
37 | self._one_iteration(input_vecs, labels, rate)
38 | def _one_iteration(self, input_vecs, labels, rate):
39 | '''
40 | 一次迭代，把所有的训练数据过一遍
41 | '''
42 | # 把输入和输出打包在一起，成为样本的列表[(input_vec, label), ...]
43 | # 而每个训练样本是(input_vec, label)
44 | samples = zip(input_vecs, labels)
45 | # 对每个样本，按照感知器规则更新权重
46 | for (input_vec, label) in samples:
47 | # 计算感知器在当前权重下的输出
48 | output = self.predict(input_vec)
49 | # 更新权重
50 | self._update_weights(input_vec, output, label, rate)
51 | def _update_weights(self, input_vec, output, label, rate):
52 | '''
53 | 按照感知器规则更新权重
54 | '''
55 | # 把input_vec[x1,x2,x3,...]和weights[w1,w2,w3,...]打包在一起
56 | # 变成[(x1,w1),(x2,w2),(x3,w3),...]
57 | # 然后利用感知器规则更新权重
58 | delta = label - output
59 | self.weights = list(map(lambda x, w: w + rate * delta * x, input_vec, self.weights))
60 | # 更新bias
61 | self.bias += rate * delta
62 |
63 | def f(x):
64 | '''
65 | 定义激活函数f
66 | '''
67 | return 1 if x > 0 else 0
68 | def get_training_dataset():
69 | '''
70 | 基于and真值表构建训练数据
71 | '''
72 | # 构建训练数据
73 | # 输入向量列表
74 | input_vecs = [[1,1], [0,0], [1,0], [0,1]]
75 | # 期望的输出列表，注意要与输入一一对应
76 | # [1,1] -> 1, [0,0] -> 0, [1,0] -> 0, [0,1] -> 0
77 | labels = [1, 0, 0, 0]
78 | return input_vecs, labels
79 | def train_and_perceptron():
80 | '''
81 | 使用and真值表训练感知器
82 | '''
83 | # 创建感知器，输入参数个数为2（因为and是二元函数），激活函数为f
84 | p = Perceptron(2, f)
85 | # 训练，迭代10轮, 学习速率为0.1
86 | input_vecs, labels = get_training_dataset()
87 | p.train(input_vecs, labels, 10, 0.1)
88 | #返回训练好的感知器
89 | return p
90 | if __name__ == '__main__':
91 | # 训练and感知器
92 | and_perception = train_and_perceptron()
93 | # 打印训练获得的权重
94 | print(and_perception)
95 | # 测试
96 | print('1 and 1 = %d' % and_perception.predict([1, 1]))
97 | print('0 and 0 = %d' % and_perception.predict([0, 0]))
98 | print('1 and 0 = %d' % and_perception.predict([1, 0]))
99 | print('0 and 1 = %d' % and_perception.predict([0, 1]))
--------------------------------------------------------------------------------
/Tutorial/lesson-3/bp.py:
--------------------------------------------------------------------------------
1 | #-*- coding:utf-8 -*-
2 | import random
3 | from functools import reduce
4 | from numpy import exp
5 |
6 | def sigmoid(inX):
7 | return 1.0 / (1 + exp(-inX))
8 |
9 | # 节点类，负责记录和维护节点自身信息以及与这个节点相关的上下游连接，实现输出值和误差项的计算。
10 | class Node():
11 | def __init__(self, layer_index, node_index):
12 | '''
13 | 构造节点对象。
14 | layer_index: 节点所属的层的编号
15 | node_index: 节点的编号
16 | '''
17 | self.layer_index = layer_index
18 | self.node_index = node_index
19 | self.downstream = []
20 | self.upstream = []
21 | self.output = 0
22 | self.delta = 0
23 |
24 | def set_output(self, output):
25 | '''
26 | 设置节点的输出值。如果节点属于输入层会用到这个函数。
27 | '''
28 | self.output = output
29 |
30 | def append_downstream_connection(self, conn):
31 | '''
32 | 添加一个到下游节点的连接
33 | '''
34 | self.downstream.append(conn)
35 |
36 | def append_upstream_connection(self, conn):
37 | '''
38 | 添加一个到上游节点的连接
39 | '''
40 | self.upstream.append(conn)
41 |
42 | def calc_output(self):
43 | '''
44 | 根据式1计算节点的输出
45 | '''
46 | # 每个节点的输出算法，N元一次方程求和
47 | output = reduce(lambda ret, conn: ret + conn.upstream_node.output * conn.weight, self.upstream, 0)
48 | # 结果放入激活函数
49 | self.output = sigmoid(output)
50 |
51 | def calc_hidden_layer_delta(self):
52 | '''
53 | 节点属于隐藏层时，根据式4计算delta
54 | '''
55 | downstream_delta = reduce(
56 | lambda ret, conn: ret + conn.downstream_node.delta * conn.weight,
57 | self.downstream, 0.0)
58 | self.delta = self.output * (1 - self.output) * downstream_delta
59 |
60 | def calc_output_layer_delta(self, label):
61 | '''
62 | 节点属于输出层时，根据式3计算delta
63 | '''
64 | self.delta = self.output * (1 - self.output) * (label - self.output)
65 |
66 | def __str__(self):
67 | '''
68 | 打印节点的信息
69 | '''
70 | node_str = '%u-%u: output: %f delta: %f' % (self.layer_index, self.node_index, self.output, self.delta)
71 | downstream_str = reduce(lambda ret, conn: ret + '\n\t' + str(conn), self.downstream, '')
72 | upstream_str = reduce(lambda ret, conn: ret + '\n\t' + str(conn), self.upstream, '')
73 | return node_str + '\n\tdownstream:' + downstream_str + '\n\tupstream:' + upstream_str
74 |
75 | class ConstNode():
76 | def __init__(self, layer_index, node_index):
77 | '''
78 | 构造节点对象。
79 | layer_index: 节点所属的层的编号
80 | node_index: 节点的编号
81 | '''
82 | self.layer_index = layer_index
83 | self.node_index = node_index
84 | self.downstream = []
85 | self.output = 1
86 | def append_downstream_connection(self, conn):
87 | '''
88 | 添加一个到下游节点的连接
89 | '''
90 | self.downstream.append(conn)
91 | def calc_hidden_layer_delta(self):
92 | '''
93 | 节点属于隐藏层时，根据式4计算delta
94 | '''
95 | downstream_delta = reduce(
96 | lambda ret, conn: ret + conn.downstream_node.delta * conn.weight,
97 | self.downstream, 0.0)
98 | self.delta = self.output * (1 - self.output) * downstream_delta
99 | def __str__(self):
100 | '''
101 | 打印节点的信息
102 | '''
103 | node_str = '%u-%u: output: 1' % (self.layer_index, self.node_index)
104 | downstream_str = reduce(lambda ret, conn: ret + '\n\t' + str(conn), self.downstream, '')
105 | return node_str + '\n\tdownstream:' + downstream_str
106 |
107 | class Layer():
108 | def __init__(self, layer_index, node_count):
109 | '''
110 | 初始化一层
111 | layer_index: 层编号
112 | node_count: 层所包含的节点个数
113 | '''
114 | self.layer_index = layer_index
115 | self.nodes = []
116 | for i in range(node_count):
117 | self.nodes.append(Node(layer_index, i))
118 | self.nodes.append(ConstNode(layer_index, node_count))
119 | def set_output(self, data):
120 | '''
121 | 设置层的输出。当层是输入层时会用到。
122 | '''
123 | for i in range(len(data)):
124 | self.nodes[i].set_output(data[i])
125 | def calc_output(self):
126 | '''
127 | 计算层的输出向量
128 | '''
129 | for node in self.nodes[:-1]:
130 | node.calc_output()
131 | def dump(self):
132 | '''
133 | 打印层的信息
134 | '''
135 | for node in self.nodes:
136 | print(node)
137 |
138 | class Connection():
139 | def __init__(self, upstream_node, downstream_node):
140 | '''
141 | 初始化连接，权重初始化为是一个很小的随机数
142 | upstream_node: 连接的上游节点
143 | downstream_node: 连接的下游节点
144 | '''
145 | self.upstream_node = upstream_node
146 | self.downstream_node = downstream_node
147 | self.weight = random.uniform(-0.1, 0.1)
148 | self.gradient = 0.0
149 | def calc_gradient(self):
150 | '''
151 | 计算梯度
152 | '''
153 | self.gradient = self.downstream_node.delta * self.upstream_node.output
154 | def get_gradient(self):
155 | '''
156 | 获取当前的梯度
157 | '''
158 | return self.gradient
159 | def update_weight(self, rate):
160 | '''
161 | 根据梯度下降算法更新权重
162 | '''
163 | self.calc_gradient()
164 | self.weight += rate * self.gradient
165 | def __str__(self):
166 | '''
167 | 打印连接信息
168 | '''
169 | return '(%u-%u) -> (%u-%u) = %f' % (
170 | self.upstream_node.layer_index,
171 | self.upstream_node.node_index,
172 | self.downstream_node.layer_index,
173 | self.downstream_node.node_index,
174 | self.weight)
175 |
176 | class Connections():
177 | def __init__(self):
178 | self.connections = []
179 | def add_connection(self, connection):
180 | self.connections.append(connection)
181 | def dump(self):
182 | for conn in self.connections:
183 | print(conn)
184 |
185 | class Network():
186 | def __init__(self, layers):
187 | '''
188 | 初始化一个全连接神经网络
189 | layers: 二维数组，描述神经网络每层节点数
190 | '''
191 | self.connections = Connections()
192 | self.layers = []
193 | layer_count = len(layers)
194 | node_count = 0;
195 | for i in range(layer_count):
196 | self.layers.append(Layer(i, layers[i]))
197 | for layer in range(layer_count - 1):
198 | connections = [Connection(upstream_node, downstream_node)
199 | for upstream_node in self.layers[layer].nodes
200 | for downstream_node in self.layers[layer + 1].nodes[:-1]]
201 | for conn in connections:
202 | self.connections.add_connection(conn)
203 | conn.downstream_node.append_upstream_connection(conn)
204 | conn.upstream_node.append_downstream_connection(conn)
205 | def train(self, labels, data_set, rate, iteration):
206 | '''
207 | 训练神经网络
208 | labels: 数组，训练样本标签。每个元素是一个样本的标签。
209 | data_set: 二维数组，训练样本特征。每个元素是一个样本的特征。
210 | '''
211 | for i in range(iteration):
212 | for d in range(len(data_set)):
213 | self.train_one_sample(labels[d], data_set[d], rate)
214 | def train_one_sample(self, label, sample, rate):
215 | '''
216 | 内部函数，用一个样本训练网络
217 | '''
218 | self.predict(sample)
219 | self.calc_delta(label)
220 | self.update_weight(rate)
221 | def calc_delta(self, label):
222 | '''
223 | 内部函数，计算每个节点的delta
224 | '''
225 | output_nodes = self.layers[-1].nodes
226 | for i in range(len(label)):
227 | output_nodes[i].calc_output_layer_delta(label[i])
228 | for layer in self.layers[-2::-1]:
229 | for node in layer.nodes:
230 | node.calc_hidden_layer_delta()
231 | def update_weight(self, rate):
232 | '''
233 | 内部函数，更新每个连接权重
234 | '''
235 | for layer in self.layers[:-1]:
236 | for node in layer.nodes:
237 | for conn in node.downstream:
238 | conn.update_weight(rate)
239 | def calc_gradient(self):
240 | '''
241 | 内部函数，计算每个连接的梯度
242 | '''
243 | for layer in self.layers[:-1]:
244 | for node in layer.nodes:
245 | for conn in node.downstream:
246 | conn.calc_gradient()
247 | def get_gradient(self, label, sample):
248 | '''
249 | 获得网络在一个样本下，每个连接上的梯度
250 | label: 样本标签
251 | sample: 样本输入
252 | '''
253 | self.predict(sample)
254 | self.calc_delta(label)
255 | self.calc_gradient()
256 | def predict(self, sample):
257 | '''
258 | 根据输入的样本预测输出值
259 | sample: 数组，样本的特征，也就是网络的输入向量
260 | '''
261 | self.layers[0].set_output(sample)
262 | for i in range(1, len(self.layers)):
263 | self.layers[i].calc_output()
264 | return list(map(lambda node: node.output, self.layers[-1].nodes[:-1]))
265 | def dump(self):
266 | '''
267 | 打印网络信息
268 | '''
269 | for layer in self.layers:
270 | layer.dump()
271 |
272 | def gradient_check(network, sample_feature, sample_label):
273 | '''
274 | 梯度检查
275 | network: 神经网络对象
276 | sample_feature: 样本的特征
277 | sample_label: 样本的标签
278 | '''
279 | # 计算网络误差
280 | network_error = lambda vec1, vec2: \
281 | 0.5 * reduce(lambda a, b: a + b,
282 | map(lambda v: (v[0] - v[1]) * (v[0] - v[1]),
283 | zip(vec1, vec2)))
284 | # 获取网络在当前样本下每个连接的梯度
285 | network.get_gradient(sample_feature, sample_label)
286 | # 对每个权重做梯度检查
287 | for conn in network.connections.connections:
288 | # 获取指定连接的梯度
289 | actual_gradient = conn.get_gradient()
290 | # 增加一个很小的值，计算网络的误差
291 | epsilon = 0.0001
292 | conn.weight += epsilon
293 | error1 = network_error(network.predict(sample_feature), sample_label)
294 | # 减去一个很小的值，计算网络的误差
295 | conn.weight -= 2 * epsilon # 刚才加过了一次，因此这里需要减去2倍
296 | error2 = network_error(network.predict(sample_feature), sample_label)
297 | # 根据式6计算期望的梯度值
298 | expected_gradient = (error2 - error1) / (2 * epsilon)
299 | # 打印
300 | print('expected gradient: \t%f\nactual gradient: \t%f' % (
301 | expected_gradient, actual_gradient))
--------------------------------------------------------------------------------
/Tutorial/lesson-3/fc.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | import numpy as np
3 |
4 | # 全连接层实现类
5 | class FullConnectedLayer():
6 | def __init__(self, input_size, output_size,
7 | activator):
8 | '''
9 | 构造函数
10 | input_size: 本层输入向量的维度
11 | output_size: 本层输出向量的维度
12 | activator: 激活函数
13 | '''
14 | self.input_size = input_size
15 | self.output_size = output_size
16 | self.activator = activator
17 | # 权重数组W
18 | self.W = np.random.uniform(-0.1, 0.1,
19 | (output_size, input_size))
20 | # 偏置项b
21 | self.b = np.zeros((output_size, 1))
22 | # 输出向量
23 | self.output = np.zeros((output_size, 1))
24 | def forward(self, input_array):
25 | '''
26 | 前向计算
27 | input_array: 输入向量，维度必须等于input_size
28 | '''
29 | # 式2
30 | self.input = input_array
31 | self.output = self.activator.forward(
32 | np.dot(self.W, input_array) + self.b)
33 | def backward(self, delta_array):
34 | '''
35 | 反向计算W和b的梯度
36 | delta_array: 从上一层传递过来的误差项
37 | '''
38 | # 式8
39 | print(delta_array.shape)
40 | print(self.activator.backward(self.input).shape)
41 | print(self.W.T.shape)
42 | self.delta = self.activator.backward(self.input) * np.dot(
43 | self.W.T, delta_array)
44 | self.W_grad = np.dot(delta_array, self.input.T)
45 | self.b_grad = delta_array
46 | def update(self, learning_rate):
47 | '''
48 | 使用梯度下降算法更新权重
49 | '''
50 | self.W += learning_rate * self.W_grad
51 | self.b += learning_rate * self.b_grad
52 |
53 | # Sigmoid激活函数类
54 | class SigmoidActivator():
55 | def forward(self, weighted_input):
56 | return 1.0 / (1.0 + np.exp(-weighted_input))
57 | def backward(self, output):
58 | return output * (1 - output)
59 |
60 | # 神经网络类
61 | class Network():
62 | def __init__(self, layers):
63 | '''
64 | 构造函数
65 | '''
66 | self.layers = []
67 | for i in range(len(layers) - 1):
68 | self.layers.append(
69 | FullConnectedLayer(
70 | layers[i], layers[i+1],
71 | SigmoidActivator()
72 | )
73 | )
74 | def predict(self, sample):
75 | '''
76 | 使用神经网络实现预测
77 | sample: 输入样本
78 | '''
79 | output = sample
80 | for layer in self.layers:
81 | layer.forward(output)
82 | output = layer.output
83 | return output
84 | def train(self, labels, data_set, rate, epoch):
85 | '''
86 | 训练函数
87 | labels: 样本标签
88 | data_set: 输入样本
89 | rate: 学习速率
90 | epoch: 训练轮数
91 | '''
92 | for i in range(epoch):
93 | for d in range(len(data_set)):
94 | self.train_one_sample(labels[d],
95 | data_set[d], rate)
96 | def train_one_sample(self, label, sample, rate):
97 | self.predict(sample)
98 | self.calc_gradient(label)
99 | self.update_weight(rate)
100 | def calc_gradient(self, label):
101 | delta = self.layers[-1].activator.backward(
102 | self.layers[-1].output
103 | ) * (label - self.layers[-1].output)
104 | for layer in self.layers[::-1]:
105 | layer.backward(delta)
106 | delta = layer.delta
107 | return delta
108 | def update_weight(self, rate):
109 | for layer in self.layers:
110 | layer.update(rate)
111 |
112 | def dump(self):
113 | for layer in self.layers:
114 | layer.dump()
115 |
116 | def loss(self, output, label):
117 | return 0.5 * ((label - output) * (label - output)).sum()
118 |
119 | def gradient_check(self, sample_feature, sample_label):
120 | '''
121 | 梯度检查
122 | network: 神经网络对象
123 | sample_feature: 样本的特征
124 | sample_label: 样本的标签
125 | '''
126 |
127 | # 获取网络在当前样本下每个连接的梯度
128 | self.predict(sample_feature)
129 | self.calc_gradient(sample_label)
130 |
131 | # 检查梯度
132 | epsilon = 10e-4
133 | for fc in self.layers:
134 | for i in range(fc.W.shape[0]):
135 | for j in range(fc.W.shape[1]):
136 | fc.W[i, j] += epsilon
137 | output = self.predict(sample_feature)
138 | err1 = self.loss(sample_label, output)
139 | fc.W[i, j] -= 2 * epsilon
140 | output = self.predict(sample_feature)
141 | err2 = self.loss(sample_label, output)
142 | expect_grad = (err1 - err2) / (2 * epsilon)
143 | fc.W[i, j] += epsilon
144 | print('weights(%d,%d): expected - actural %.4e - %.4e' % (
145 | i, j, expect_grad, fc.W_grad[i, j]))
146 |
147 | def transpose(args):
148 | return list(map(
149 | lambda arg: list(map(
150 | lambda line: np.array(line).reshape(len(line), 1)
151 | , arg))
152 | , args
153 | ))
154 |
155 | class Normalizer(object):
156 | def __init__(self):
157 | self.mask = [
158 | 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
159 | ]
160 |
161 | def norm(self, number):
162 | data = list(map(lambda m: 0.9 if number & m else 0.1, self.mask))
163 | return np.array(data).reshape(8, 1)
164 |
165 | def denorm(self, vec):
166 | binary = list(map(lambda i: 1 if i > 0.5 else 0, vec[:, 0]))
167 | for i in range(len(self.mask)):
168 | binary[i] = binary[i] * self.mask[i]
169 | return reduce(lambda x, y: x + y, binary)
170 |
171 | def train_data_set():
172 | normalizer = Normalizer()
173 | data_set = []
174 | labels = []
175 | for i in range(0, 256):
176 | n = normalizer.norm(i)
177 | data_set.append(n)
178 | labels.append(n)
179 | return labels, data_set
180 |
181 |
182 | def correct_ratio(network):
183 | normalizer = Normalizer()
184 | correct = 0.0;
185 | for i in range(256):
186 | if normalizer.denorm(network.predict(normalizer.norm(i))) == i:
187 | correct += 1.0
188 | print('correct_ratio: %.2f%%' % (correct / 256 * 100))
189 |
190 |
191 | def test():
192 | labels, data_set = transpose(train_data_set())
193 | net = Network([8, 3, 8])
194 | rate = 0.5
195 | mini_batch = 20
196 | epoch = 10
197 | for i in range(epoch):
198 | net.train(labels, data_set, rate, mini_batch)
199 | print('after epoch %d loss: %f' % (
200 | (i + 1),
201 | net.loss(labels[-1], net.predict(data_set[-1]))
202 | ))
203 | rate /= 2
204 | correct_ratio(net)
205 |
206 |
207 | def gradient_check():
208 | '''
209 | 梯度检查
210 | '''
211 | labels, data_set = transpose(train_data_set())
212 | net = Network([8, 3, 8])
213 | net.gradient_check(data_set[0], labels[0])
214 | return net
--------------------------------------------------------------------------------
/Tutorial/lesson-3/mnist.py:
--------------------------------------------------------------------------------
1 | #-*- coding:utf-8 -*-
2 | from bp import *
3 | from datetime import datetime
4 |
5 | # 数据加载器基类
6 | class Loader():
7 | def __init__(self, path, count):
8 | '''
9 | 初始化加载器
10 | path: 数据文件路径
11 | count: 文件中的样本个数
12 | '''
13 | self.path = path
14 | self.count = count
15 | def get_file_content(self):
16 | '''
17 | 读取文件内容
18 | '''
19 | f = open(self.path, 'rb')
20 | content = f.read()
21 | f.close()
22 | return content
23 |
24 | # 图像数据加载器
25 | class ImageLoader(Loader):
26 | def get_picture(self, content, index):
27 | '''
28 | 内部函数，从文件中获取图像
29 | '''
30 | start = index * 28 * 28 + 16
31 | picture = []
32 | for i in range(28):
33 | picture.append([])
34 | for j in range(28):
35 | picture[i].append(
36 | int(content[start + i * 28 + j]))
37 | return picture
38 | def get_one_sample(self, picture):
39 | '''
40 | 内部函数，将图像转化为样本的输入向量
41 | '''
42 | sample = []
43 | for i in range(28):
44 | for j in range(28):
45 | sample.append(picture[i][j])
46 | return sample
47 | def load(self):
48 | '''
49 | 加载数据文件，获得全部样本的输入向量
50 | '''
51 | content = self.get_file_content()
52 | data_set = []
53 | for index in range(self.count):
54 | data_set.append(
55 | self.get_one_sample(
56 | self.get_picture(content, index)))
57 | return data_set
58 | # 标签数据加载器
59 | class LabelLoader(Loader):
60 | def load(self):
61 | '''
62 | 加载数据文件，获得全部样本的标签向量
63 | '''
64 | content = self.get_file_content()
65 | labels = []
66 | for index in range(self.count):
67 | labels.append(self.norm(content[index + 8]))
68 | return labels
69 | def norm(self, label):
70 | '''
71 | 内部函数，将一个值转换为10维标签向量
72 | '''
73 | label_vec = []
74 | label_value = int(label)
75 | for i in range(10):
76 | if i == label_value:
77 | label_vec.append(0.9)
78 | else:
79 | label_vec.append(0.1)
80 | return label_vec
81 | def get_training_data_set():
82 | '''
83 | 获得训练数据集
84 | '''
85 | image_loader = ImageLoader('train-images.idx3-ubyte', 60000)
86 | label_loader = LabelLoader('train-labels.idx1-ubyte', 60000)
87 | return image_loader.load(), label_loader.load()
88 | def get_test_data_set():
89 | '''
90 | 获得测试数据集
91 | '''
92 | image_loader = ImageLoader('t10k-images.idx3-ubyte', 10000)
93 | label_loader = LabelLoader('t10k-labels.idx1-ubyte', 10000)
94 | return image_loader.load(), label_loader.load()
95 |
96 |
97 | def get_result(vec):
98 | max_value_index = 0
99 | max_value = 0
100 | for i in range(len(vec)):
101 | if vec[i] > max_value:
102 | max_value = vec[i]
103 | max_value_index = i
104 | return max_value_index
105 |
106 | def evaluate(network, test_data_set, test_labels):
107 | error = 0
108 | total = len(test_data_set)
109 | for i in range(total):
110 | label = get_result(test_labels[i])
111 | predict = get_result(network.predict(test_data_set[i]))
112 | if label != predict:
113 | error += 1
114 | return float(error) / float(total)
115 |
116 | def train_and_evaluate():
117 | last_error_ratio = 1.0
118 | epoch = 0
119 | train_data_set, train_labels = get_training_data_set()
120 | test_data_set, test_labels = get_test_data_set()
121 | network = Network([784, 300, 10])
122 | while True:
123 | epoch += 1
124 | network.train(train_labels, train_data_set, 0.3, 1)
125 | print('%s epoch %d finished' % (now(), epoch))
126 | if epoch % 10 == 0:
127 | error_ratio = evaluate(network, test_data_set, test_labels)
128 | print('%s after epoch %d, error ratio is %f' % (now(), epoch, error_ratio))
129 | if error_ratio > last_error_ratio:
130 | break
131 | else:
132 | last_error_ratio = error_ratio
133 |
134 | if __name__ == '__main__':
135 | train_and_evaluate()
--------------------------------------------------------------------------------
/Tutorial/lesson-4/activators.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 |
3 |
4 | import numpy as np
5 |
6 |
7 | class ReluActivator(object):
8 | def forward(self, weighted_input):
9 | #return weighted_input
10 | return max(0, weighted_input)
11 |
12 | def backward(self, output):
13 | return 1 if output > 0 else 0
14 |
15 |
16 | class IdentityActivator(object):
17 | def forward(self, weighted_input):
18 | return weighted_input
19 |
20 | def backward(self, output):
21 | return 1
22 |
23 |
24 | class SigmoidActivator(object):
25 | def forward(self, weighted_input):
26 | return 1.0 / (1.0 + np.exp(-weighted_input))
27 |
28 | def backward(self, output):
29 | return output * (1 - output)
30 |
31 |
32 | class TanhActivator(object):
33 | def forward(self, weighted_input):
34 | return 2.0 / (1.0 + np.exp(-2 * weighted_input)) - 1.0
35 |
36 | def backward(self, output):
37 | return 1 - output * output
--------------------------------------------------------------------------------
/Tutorial/lesson-4/cnn.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import numpy as np
3 | from activators import ReluActivator, IdentityActivator
4 |
5 |
6 | # 获取卷积区域
7 | def get_patch(input_array, i, j, filter_width,
8 | filter_height, stride):
9 | '''
10 | 从输入数组中获取本次卷积的区域，
11 | 自动适配输入为2D和3D的情况
12 | '''
13 | start_i = i * stride
14 | start_j = j * stride
15 | if input_array.ndim == 2:
16 | return input_array[
17 | start_i : start_i + filter_height,
18 | start_j : start_j + filter_width]
19 | elif input_array.ndim == 3:
20 | return input_array[:,
21 | start_i : start_i + filter_height,
22 | start_j : start_j + filter_width]
23 |
24 |
25 | # 获取一个2D区域的最大值所在的索引
26 | def get_max_index(array):
27 | max_i = 0
28 | max_j = 0
29 | max_value = array[0,0]
30 | for i in range(array.shape[0]):
31 | for j in range(array.shape[1]):
32 | if array[i,j] > max_value:
33 | max_value = array[i,j]
34 | max_i, max_j = i, j
35 | return max_i, max_j
36 |
37 |
38 | # 计算卷积
39 | def conv(input_array,
40 | kernel_array,
41 | output_array,
42 | stride, bias):
43 | '''
44 | 计算卷积，自动适配输入为2D和3D的情况
45 | '''
46 | channel_number = input_array.ndim
47 | output_width = output_array.shape[1]
48 | output_height = output_array.shape[0]
49 | kernel_width = kernel_array.shape[-1]
50 | kernel_height = kernel_array.shape[-2]
51 | for i in range(output_height):
52 | for j in range(output_width):
53 | output_array[i][j] = (
54 | get_patch(input_array, i, j, kernel_width,
55 | kernel_height, stride) * kernel_array
56 | ).sum() + bias
57 |
58 |
59 | # 为数组增加Zero padding
60 | def padding(input_array, zp):
61 | '''
62 | 为数组增加Zero padding，自动适配输入为2D和3D的情况
63 | '''
64 | if zp == 0:
65 | return input_array
66 | else:
67 | if input_array.ndim == 3:
68 | input_width = input_array.shape[2]
69 | input_height = input_array.shape[1]
70 | input_depth = input_array.shape[0]
71 | padded_array = np.zeros((
72 | input_depth,
73 | input_height + 2 * zp,
74 | input_width + 2 * zp))
75 | padded_array[:,
76 | zp : zp + input_height,
77 | zp : zp + input_width] = input_array
78 | return padded_array
79 | elif input_array.ndim == 2:
80 | input_width = input_array.shape[1]
81 | input_height = input_array.shape[0]
82 | padded_array = np.zeros((
83 | input_height + 2 * zp,
84 | input_width + 2 * zp))
85 | padded_array[zp : zp + input_height,
86 | zp : zp + input_width] = input_array
87 | return padded_array
88 |
89 |
90 | # 对numpy数组进行element wise操作
91 | def element_wise_op(array, op):
92 | for i in np.nditer(array,
93 | op_flags=['readwrite']):
94 | i[...] = op(i)
95 |
96 |
97 | class Filter(object):
98 | def __init__(self, width, height, depth):
99 | self.weights = np.random.uniform(-1e-4, 1e-4,
100 | (depth, height, width))
101 | self.bias = 0
102 | self.weights_grad = np.zeros(
103 | self.weights.shape)
104 | self.bias_grad = 0
105 |
106 | def __repr__(self):
107 | return 'filter weights:\n%s\nbias:\n%s' % (
108 | repr(self.weights), repr(self.bias))
109 |
110 | def get_weights(self):
111 | return self.weights
112 |
113 | def get_bias(self):
114 | return self.bias
115 |
116 | def update(self, learning_rate):
117 | self.weights -= learning_rate * self.weights_grad
118 | self.bias -= learning_rate * self.bias_grad
119 |
120 |
121 | class ConvLayer(object):
122 | def __init__(self, input_width, input_height,
123 | channel_number, filter_width,
124 | filter_height, filter_number,
125 | zero_padding, stride, activator,
126 | learning_rate):
127 | self.input_width = input_width
128 | self.input_height = input_height
129 | self.channel_number = channel_number
130 | self.filter_width = filter_width
131 | self.filter_height = filter_height
132 | self.filter_number = filter_number
133 | self.zero_padding = zero_padding
134 | self.stride = stride
135 | self.output_width = \
136 | ConvLayer.calculate_output_size(
137 | self.input_width, filter_width, zero_padding,
138 | stride)
139 | self.output_height = \
140 | ConvLayer.calculate_output_size(
141 | self.input_height, filter_height, zero_padding,
142 | stride)
143 | self.output_array = np.zeros((self.filter_number,
144 | self.output_height, self.output_width))
145 | self.filters = []
146 | for i in range(filter_number):
147 | self.filters.append(Filter(filter_width,
148 | filter_height, self.channel_number))
149 | self.activator = activator
150 | self.learning_rate = learning_rate
151 |
152 | def forward(self, input_array):
153 | '''
154 | 计算卷积层的输出
155 | 输出结果保存在self.output_array
156 | '''
157 | self.input_array = input_array
158 | self.padded_input_array = padding(input_array,
159 | self.zero_padding)
160 | for f in range(self.filter_number):
161 | filter = self.filters[f]
162 | conv(self.padded_input_array,
163 | filter.get_weights(), self.output_array[f],
164 | self.stride, filter.get_bias())
165 | element_wise_op(self.output_array,
166 | self.activator.forward)
167 |
168 | def backward(self, input_array, sensitivity_array,
169 | activator):
170 | '''
171 | 计算传递给前一层的误差项，以及计算每个权重的梯度
172 | 前一层的误差项保存在self.delta_array
173 | 梯度保存在Filter对象的weights_grad
174 | '''
175 | self.forward(input_array)
176 | self.bp_sensitivity_map(sensitivity_array,
177 | activator)
178 | self.bp_gradient(sensitivity_array)
179 |
180 | def update(self):
181 | '''
182 | 按照梯度下降，更新权重
183 | '''
184 | for filter in self.filters:
185 | filter.update(self.learning_rate)
186 |
187 | def bp_sensitivity_map(self, sensitivity_array,
188 | activator):
189 | '''
190 | 计算传递到上一层的sensitivity map
191 | sensitivity_array: 本层的sensitivity map
192 | activator: 上一层的激活函数
193 | '''
194 | # 处理卷积步长，对原始sensitivity map进行扩展
195 | expanded_array = self.expand_sensitivity_map(
196 | sensitivity_array)
197 | # full卷积，对sensitivitiy map进行zero padding
198 | # 虽然原始输入的zero padding单元也会获得残差
199 | # 但这个残差不需要继续向上传递，因此就不计算了
200 | expanded_width = expanded_array.shape[2]
201 | zp = int((self.input_width +
202 | self.filter_width - 1 - expanded_width) / 2)
203 | padded_array = padding(expanded_array, zp)
204 | # 初始化delta_array，用于保存传递到上一层的
205 | # sensitivity map
206 | self.delta_array = self.create_delta_array()
207 | # 对于具有多个filter的卷积层来说，最终传递到上一层的
208 | # sensitivity map相当于所有的filter的
209 | # sensitivity map之和
210 | for f in range(self.filter_number):
211 | filter = self.filters[f]
212 | # 将filter权重翻转180度
213 | flipped_weights = np.array(list(map(
214 | lambda i: np.rot90(i, 2),
215 | filter.get_weights())))
216 | # 计算与一个filter对应的delta_array
217 | delta_array = self.create_delta_array()
218 | for d in range(delta_array.shape[0]):
219 | conv(padded_array[f], flipped_weights[d],delta_array[d], 1, 0)
220 | self.delta_array += delta_array
221 | # 将计算结果与激活函数的偏导数做element-wise乘法操作
222 | derivative_array = np.array(self.input_array)
223 | element_wise_op(derivative_array,
224 | activator.backward)
225 | self.delta_array *= derivative_array
226 |
227 | def bp_gradient(self, sensitivity_array):
228 | # 处理卷积步长，对原始sensitivity map进行扩展
229 | expanded_array = self.expand_sensitivity_map(
230 | sensitivity_array)
231 | for f in range(self.filter_number):
232 | # 计算每个权重的梯度
233 | filter = self.filters[f]
234 | for d in range(filter.weights.shape[0]):
235 | conv(self.padded_input_array[d],
236 | expanded_array[f],
237 | filter.weights_grad[d], 1, 0)
238 | # 计算偏置项的梯度
239 | filter.bias_grad = expanded_array[f].sum()
240 |
241 | def expand_sensitivity_map(self, sensitivity_array):
242 | depth = sensitivity_array.shape[0]
243 | # 确定扩展后sensitivity map的大小
244 | # 计算stride为1时sensitivity map的大小
245 | expanded_width = (self.input_width -
246 | self.filter_width + 2 * self.zero_padding + 1)
247 | expanded_height = (self.input_height -
248 | self.filter_height + 2 * self.zero_padding + 1)
249 | # 构建新的sensitivity_map
250 | expand_array = np.zeros((depth, expanded_height,
251 | expanded_width))
252 | # 从原始sensitivity map拷贝误差值
253 | for i in range(self.output_height):
254 | for j in range(self.output_width):
255 | i_pos = i * self.stride
256 | j_pos = j * self.stride
257 | expand_array[:,i_pos,j_pos] = \
258 | sensitivity_array[:,i,j]
259 | return expand_array
260 |
261 | def create_delta_array(self):
262 | return np.zeros((self.channel_number,
263 | self.input_height, self.input_width))
264 |
265 | @staticmethod
266 | def calculate_output_size(input_size,
267 | filter_size, zero_padding, stride):
268 | return int((input_size - filter_size +
269 | 2 * zero_padding) / stride + 1)
270 |
271 |
272 | class MaxPoolingLayer(object):
273 | def __init__(self, input_width, input_height,
274 | channel_number, filter_width,
275 | filter_height, stride):
276 | self.input_width = input_width
277 | self.input_height = input_height
278 | self.channel_number = channel_number
279 | self.filter_width = filter_width
280 | self.filter_height = filter_height
281 | self.stride = stride
282 | self.output_width = int((input_width -
283 | filter_width) / self.stride + 1)
284 | self.output_height = int((input_height -
285 | filter_height) / self.stride + 1)
286 | self.output_array = np.zeros((self.channel_number,
287 | self.output_height, self.output_width))
288 |
289 | def forward(self, input_array):
290 | for d in range(self.channel_number):
291 | for i in range(self.output_height):
292 | for j in range(self.output_width):
293 | self.output_array[d,i,j] = (
294 | get_patch(input_array[d], i, j,
295 | self.filter_width,
296 | self.filter_height,
297 | self.stride).max())
298 |
299 | def backward(self, input_array, sensitivity_array):
300 | self.delta_array = np.zeros(input_array.shape)
301 | for d in range(self.channel_number):
302 | for i in range(self.output_height):
303 | for j in range(self.output_width):
304 | patch_array = get_patch(
305 | input_array[d], i, j,
306 | self.filter_width,
307 | self.filter_height,
308 | self.stride)
309 | k, l = get_max_index(patch_array)
310 | self.delta_array[d,
311 | i * self.stride + k,
312 | j * self.stride + l] = \
313 | sensitivity_array[d,i,j]
314 |
315 |
316 | def init_test():
317 | a = np.array(
318 | [[[0,1,1,0,2],
319 | [2,2,2,2,1],
320 | [1,0,0,2,0],
321 | [0,1,1,0,0],
322 | [1,2,0,0,2]],
323 | [[1,0,2,2,0],
324 | [0,0,0,2,0],
325 | [1,2,1,2,1],
326 | [1,0,0,0,0],
327 | [1,2,1,1,1]],
328 | [[2,1,2,0,0],
329 | [1,0,0,1,0],
330 | [0,2,1,0,1],
331 | [0,1,2,2,2],
332 | [2,1,0,0,1]]])
333 | b = np.array(
334 | [[[0,1,1],
335 | [2,2,2],
336 | [1,0,0]],
337 | [[1,0,2],
338 | [0,0,0],
339 | [1,2,1]]])
340 | cl = ConvLayer(5,5,3,3,3,2,1,2,IdentityActivator(),0.001)
341 | cl.filters[0].weights = np.array(
342 | [[[-1,1,0],
343 | [0,1,0],
344 | [0,1,1]],
345 | [[-1,-1,0],
346 | [0,0,0],
347 | [0,-1,0]],
348 | [[0,0,-1],
349 | [0,1,0],
350 | [1,-1,-1]]], dtype=np.float64)
351 | cl.filters[0].bias=1
352 | cl.filters[1].weights = np.array(
353 | [[[1,1,-1],
354 | [-1,-1,1],
355 | [0,-1,1]],
356 | [[0,1,0],
357 | [-1,0,-1],
358 | [-1,1,0]],
359 | [[-1,0,0],
360 | [-1,0,1],
361 | [-1,0,0]]], dtype=np.float64)
362 | return a, b, cl
363 |
364 |
365 | def test():
366 | a, b, cl = init_test()
367 | cl.forward(a)
368 | print(cl.output_array)
369 |
370 | def test_bp():
371 | a, b, cl = init_test()
372 | cl.backward(a, b, IdentityActivator())
373 | cl.update()
374 | print(cl.filters[0])
375 | print(cl.filters[1])
376 |
377 |
378 | def gradient_check():
379 | '''
380 | 梯度检查
381 | '''
382 | # 设计一个误差函数，取所有节点输出项之和
383 | error_function = lambda o: o.sum()
384 |
385 | # 计算forward值
386 | a, b, cl = init_test()
387 | cl.forward(a)
388 |
389 | # 求取sensitivity map
390 | sensitivity_array = np.ones(cl.output_array.shape,
391 | dtype=np.float64)
392 | # 计算梯度
393 | cl.backward(a, sensitivity_array,
394 | IdentityActivator())
395 | # 检查梯度
396 | epsilon = 10e-4
397 | for d in range(cl.filters[0].weights_grad.shape[0]):
398 | for i in range(cl.filters[0].weights_grad.shape[1]):
399 | for j in range(cl.filters[0].weights_grad.shape[2]):
400 | cl.filters[0].weights[d,i,j] += epsilon
401 | cl.forward(a)
402 | err1 = error_function(cl.output_array)
403 | cl.filters[0].weights[d,i,j] -= 2*epsilon
404 | cl.forward(a)
405 | err2 = error_function(cl.output_array)
406 | expect_grad = int((err1 - err2) / (2 * epsilon))
407 | cl.filters[0].weights[d,i,j] += epsilon
408 | print('weights(%d,%d,%d): expected - actural %f - %f' % (
409 | d, i, j, expect_grad, cl.filters[0].weights_grad[d,i,j]))
410 |
411 |
412 | def init_pool_test():
413 | a = np.array(
414 | [[[1,1,2,4],
415 | [5,6,7,8],
416 | [3,2,1,0],
417 | [1,2,3,4]],
418 | [[0,1,2,3],
419 | [4,5,6,7],
420 | [8,9,0,1],
421 | [3,4,5,6]]], dtype=np.float64)
422 |
423 | b = np.array(
424 | [[[1,2],
425 | [2,4]],
426 | [[3,5],
427 | [8,2]]], dtype=np.float64)
428 |
429 | mpl = MaxPoolingLayer(4,4,2,2,2,2)
430 |
431 | return a, b, mpl
432 |
433 |
434 | def test_pool():
435 | a, b, mpl = init_pool_test()
436 | mpl.forward(a)
437 | print('input array:\n%s\noutput array:\n%s' % (a,
438 | mpl.output_array))
439 |
440 |
441 | def test_pool_bp():
442 | a, b, mpl = init_pool_test()
443 | mpl.backward(a, b)
444 | print('input array:\n%s\nsensitivity array:\n%s\ndelta array:\n%s' % (
445 | a, b, mpl.delta_array))
446 |
--------------------------------------------------------------------------------
/Tutorial/lesson-5/activators.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 |
3 |
4 | import numpy as np
5 |
6 |
7 | class ReluActivator(object):
8 | def forward(self, weighted_input):
9 | #return weighted_input
10 | return max(0, weighted_input)
11 |
12 | def backward(self, output):
13 | return 1 if output > 0 else 0
14 |
15 |
16 | class IdentityActivator(object):
17 | def forward(self, weighted_input):
18 | return weighted_input
19 |
20 | def backward(self, output):
21 | return 1
22 |
23 |
24 | class SigmoidActivator(object):
25 | def forward(self, weighted_input):
26 | return 1.0 / (1.0 + np.exp(-weighted_input))
27 |
28 | def backward(self, output):
29 | return output * (1 - output)
30 |
31 |
32 | class TanhActivator(object):
33 | def forward(self, weighted_input):
34 | return 2.0 / (1.0 + np.exp(-2 * weighted_input)) - 1.0
35 |
36 | def backward(self, output):
37 | return 1 - output * output
--------------------------------------------------------------------------------
/Tutorial/lesson-5/cnn.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import numpy as np
3 | from activators import ReluActivator, IdentityActivator
4 |
5 |
6 | # 获取卷积区域
7 | def get_patch(input_array, i, j, filter_width,
8 | filter_height, stride):
9 | '''
10 | 从输入数组中获取本次卷积的区域，
11 | 自动适配输入为2D和3D的情况
12 | '''
13 | start_i = i * stride
14 | start_j = j * stride
15 | if input_array.ndim == 2:
16 | return input_array[
17 | start_i : start_i + filter_height,
18 | start_j : start_j + filter_width]
19 | elif input_array.ndim == 3:
20 | return input_array[:,
21 | start_i : start_i + filter_height,
22 | start_j : start_j + filter_width]
23 |
24 |
25 | # 获取一个2D区域的最大值所在的索引
26 | def get_max_index(array):
27 | max_i = 0
28 | max_j = 0
29 | max_value = array[0,0]
30 | for i in range(array.shape[0]):
31 | for j in range(array.shape[1]):
32 | if array[i,j] > max_value:
33 | max_value = array[i,j]
34 | max_i, max_j = i, j
35 | return max_i, max_j
36 |
37 |
38 | # 计算卷积
39 | def conv(input_array,
40 | kernel_array,
41 | output_array,
42 | stride, bias):
43 | '''
44 | 计算卷积，自动适配输入为2D和3D的情况
45 | '''
46 | channel_number = input_array.ndim
47 | output_width = output_array.shape[1]
48 | output_height = output_array.shape[0]
49 | kernel_width = kernel_array.shape[-1]
50 | kernel_height = kernel_array.shape[-2]
51 | for i in range(output_height):
52 | for j in range(output_width):
53 | output_array[i][j] = (
54 | get_patch(input_array, i, j, kernel_width,
55 | kernel_height, stride) * kernel_array
56 | ).sum() + bias
57 |
58 |
59 | # 为数组增加Zero padding
60 | def padding(input_array, zp):
61 | '''
62 | 为数组增加Zero padding，自动适配输入为2D和3D的情况
63 | '''
64 | if zp == 0:
65 | return input_array
66 | else:
67 | if input_array.ndim == 3:
68 | input_width = input_array.shape[2]
69 | input_height = input_array.shape[1]
70 | input_depth = input_array.shape[0]
71 | padded_array = np.zeros((
72 | input_depth,
73 | input_height + 2 * zp,
74 | input_width + 2 * zp))
75 | padded_array[:,
76 | zp : zp + input_height,
77 | zp : zp + input_width] = input_array
78 | return padded_array
79 | elif input_array.ndim == 2:
80 | input_width = input_array.shape[1]
81 | input_height = input_array.shape[0]
82 | padded_array = np.zeros((
83 | input_height + 2 * zp,
84 | input_width + 2 * zp))
85 | padded_array[zp : zp + input_height,
86 | zp : zp + input_width] = input_array
87 | return padded_array
88 |
89 |
90 | # 对numpy数组进行element wise操作
91 | def element_wise_op(array, op):
92 | for i in np.nditer(array,
93 | op_flags=['readwrite']):
94 | i[...] = op(i)
95 |
96 |
97 | class Filter(object):
98 | def __init__(self, width, height, depth):
99 | self.weights = np.random.uniform(-1e-4, 1e-4,
100 | (depth, height, width))
101 | self.bias = 0
102 | self.weights_grad = np.zeros(
103 | self.weights.shape)
104 | self.bias_grad = 0
105 |
106 | def __repr__(self):
107 | return 'filter weights:\n%s\nbias:\n%s' % (
108 | repr(self.weights), repr(self.bias))
109 |
110 | def get_weights(self):
111 | return self.weights
112 |
113 | def get_bias(self):
114 | return self.bias
115 |
116 | def update(self, learning_rate):
117 | self.weights -= learning_rate * self.weights_grad
118 | self.bias -= learning_rate * self.bias_grad
119 |
120 |
121 | class ConvLayer(object):
122 | def __init__(self, input_width, input_height,
123 | channel_number, filter_width,
124 | filter_height, filter_number,
125 | zero_padding, stride, activator,
126 | learning_rate):
127 | self.input_width = input_width
128 | self.input_height = input_height
129 | self.channel_number = channel_number
130 | self.filter_width = filter_width
131 | self.filter_height = filter_height
132 | self.filter_number = filter_number
133 | self.zero_padding = zero_padding
134 | self.stride = stride
135 | self.output_width = \
136 | ConvLayer.calculate_output_size(
137 | self.input_width, filter_width, zero_padding,
138 | stride)
139 | self.output_height = \
140 | ConvLayer.calculate_output_size(
141 | self.input_height, filter_height, zero_padding,
142 | stride)
143 | self.output_array = np.zeros((self.filter_number,
144 | self.output_height, self.output_width))
145 | self.filters = []
146 | for i in range(filter_number):
147 | self.filters.append(Filter(filter_width,
148 | filter_height, self.channel_number))
149 | self.activator = activator
150 | self.learning_rate = learning_rate
151 |
152 | def forward(self, input_array):
153 | '''
154 | 计算卷积层的输出
155 | 输出结果保存在self.output_array
156 | '''
157 | self.input_array = input_array
158 | self.padded_input_array = padding(input_array,
159 | self.zero_padding)
160 | for f in range(self.filter_number):
161 | filter = self.filters[f]
162 | conv(self.padded_input_array,
163 | filter.get_weights(), self.output_array[f],
164 | self.stride, filter.get_bias())
165 | element_wise_op(self.output_array,
166 | self.activator.forward)
167 |
168 | def backward(self, input_array, sensitivity_array,
169 | activator):
170 | '''
171 | 计算传递给前一层的误差项，以及计算每个权重的梯度
172 | 前一层的误差项保存在self.delta_array
173 | 梯度保存在Filter对象的weights_grad
174 | '''
175 | self.forward(input_array)
176 | self.bp_sensitivity_map(sensitivity_array,
177 | activator)
178 | self.bp_gradient(sensitivity_array)
179 |
180 | def update(self):
181 | '''
182 | 按照梯度下降，更新权重
183 | '''
184 | for filter in self.filters:
185 | filter.update(self.learning_rate)
186 |
187 | def bp_sensitivity_map(self, sensitivity_array,
188 | activator):
189 | '''
190 | 计算传递到上一层的sensitivity map
191 | sensitivity_array: 本层的sensitivity map
192 | activator: 上一层的激活函数
193 | '''
194 | # 处理卷积步长，对原始sensitivity map进行扩展
195 | expanded_array = self.expand_sensitivity_map(
196 | sensitivity_array)
197 | # full卷积，对sensitivitiy map进行zero padding
198 | # 虽然原始输入的zero padding单元也会获得残差
199 | # 但这个残差不需要继续向上传递，因此就不计算了
200 | expanded_width = expanded_array.shape[2]
201 | zp = int((self.input_width +
202 | self.filter_width - 1 - expanded_width) / 2)
203 | padded_array = padding(expanded_array, zp)
204 | # 初始化delta_array，用于保存传递到上一层的
205 | # sensitivity map
206 | self.delta_array = self.create_delta_array()
207 | # 对于具有多个filter的卷积层来说，最终传递到上一层的
208 | # sensitivity map相当于所有的filter的
209 | # sensitivity map之和
210 | for f in range(self.filter_number):
211 | filter = self.filters[f]
212 | # 将filter权重翻转180度
213 | flipped_weights = np.array(list(map(
214 | lambda i: np.rot90(i, 2),
215 | filter.get_weights())))
216 | # 计算与一个filter对应的delta_array
217 | delta_array = self.create_delta_array()
218 | for d in range(delta_array.shape[0]):
219 | conv(padded_array[f], flipped_weights[d],delta_array[d], 1, 0)
220 | self.delta_array += delta_array
221 | # 将计算结果与激活函数的偏导数做element-wise乘法操作
222 | derivative_array = np.array(self.input_array)
223 | element_wise_op(derivative_array,
224 | activator.backward)
225 | self.delta_array *= derivative_array
226 |
227 | def bp_gradient(self, sensitivity_array):
228 | # 处理卷积步长，对原始sensitivity map进行扩展
229 | expanded_array = self.expand_sensitivity_map(
230 | sensitivity_array)
231 | for f in range(self.filter_number):
232 | # 计算每个权重的梯度
233 | filter = self.filters[f]
234 | for d in range(filter.weights.shape[0]):
235 | conv(self.padded_input_array[d],
236 | expanded_array[f],
237 | filter.weights_grad[d], 1, 0)
238 | # 计算偏置项的梯度
239 | filter.bias_grad = expanded_array[f].sum()
240 |
241 | def expand_sensitivity_map(self, sensitivity_array):
242 | depth = sensitivity_array.shape[0]
243 | # 确定扩展后sensitivity map的大小
244 | # 计算stride为1时sensitivity map的大小
245 | expanded_width = (self.input_width -
246 | self.filter_width + 2 * self.zero_padding + 1)
247 | expanded_height = (self.input_height -
248 | self.filter_height + 2 * self.zero_padding + 1)
249 | # 构建新的sensitivity_map
250 | expand_array = np.zeros((depth, expanded_height,
251 | expanded_width))
252 | # 从原始sensitivity map拷贝误差值
253 | for i in range(self.output_height):
254 | for j in range(self.output_width):
255 | i_pos = i * self.stride
256 | j_pos = j * self.stride
257 | expand_array[:,i_pos,j_pos] = \
258 | sensitivity_array[:,i,j]
259 | return expand_array
260 |
261 | def create_delta_array(self):
262 | return np.zeros((self.channel_number,
263 | self.input_height, self.input_width))
264 |
265 | @staticmethod
266 | def calculate_output_size(input_size,
267 | filter_size, zero_padding, stride):
268 | return int((input_size - filter_size +
269 | 2 * zero_padding) / stride + 1)
270 |
271 |
272 | class MaxPoolingLayer(object):
273 | def __init__(self, input_width, input_height,
274 | channel_number, filter_width,
275 | filter_height, stride):
276 | self.input_width = input_width
277 | self.input_height = input_height
278 | self.channel_number = channel_number
279 | self.filter_width = filter_width
280 | self.filter_height = filter_height
281 | self.stride = stride
282 | self.output_width = int((input_width -
283 | filter_width) / self.stride + 1)
284 | self.output_height = int((input_height -
285 | filter_height) / self.stride + 1)
286 | self.output_array = np.zeros((self.channel_number,
287 | self.output_height, self.output_width))
288 |
289 | def forward(self, input_array):
290 | for d in range(self.channel_number):
291 | for i in range(self.output_height):
292 | for j in range(self.output_width):
293 | self.output_array[d,i,j] = (
294 | get_patch(input_array[d], i, j,
295 | self.filter_width,
296 | self.filter_height,
297 | self.stride).max())
298 |
299 | def backward(self, input_array, sensitivity_array):
300 | self.delta_array = np.zeros(input_array.shape)
301 | for d in range(self.channel_number):
302 | for i in range(self.output_height):
303 | for j in range(self.output_width):
304 | patch_array = get_patch(
305 | input_array[d], i, j,
306 | self.filter_width,
307 | self.filter_height,
308 | self.stride)
309 | k, l = get_max_index(patch_array)
310 | self.delta_array[d,
311 | i * self.stride + k,
312 | j * self.stride + l] = \
313 | sensitivity_array[d,i,j]
314 |
315 |
316 | def init_test():
317 | a = np.array(
318 | [[[0,1,1,0,2],
319 | [2,2,2,2,1],
320 | [1,0,0,2,0],
321 | [0,1,1,0,0],
322 | [1,2,0,0,2]],
323 | [[1,0,2,2,0],
324 | [0,0,0,2,0],
325 | [1,2,1,2,1],
326 | [1,0,0,0,0],
327 | [1,2,1,1,1]],
328 | [[2,1,2,0,0],
329 | [1,0,0,1,0],
330 | [0,2,1,0,1],
331 | [0,1,2,2,2],
332 | [2,1,0,0,1]]])
333 | b = np.array(
334 | [[[0,1,1],
335 | [2,2,2],
336 | [1,0,0]],
337 | [[1,0,2],
338 | [0,0,0],
339 | [1,2,1]]])
340 | cl = ConvLayer(5,5,3,3,3,2,1,2,IdentityActivator(),0.001)
341 | cl.filters[0].weights = np.array(
342 | [[[-1,1,0],
343 | [0,1,0],
344 | [0,1,1]],
345 | [[-1,-1,0],
346 | [0,0,0],
347 | [0,-1,0]],
348 | [[0,0,-1],
349 | [0,1,0],
350 | [1,-1,-1]]], dtype=np.float64)
351 | cl.filters[0].bias=1
352 | cl.filters[1].weights = np.array(
353 | [[[1,1,-1],
354 | [-1,-1,1],
355 | [0,-1,1]],
356 | [[0,1,0],
357 | [-1,0,-1],
358 | [-1,1,0]],
359 | [[-1,0,0],
360 | [-1,0,1],
361 | [-1,0,0]]], dtype=np.float64)
362 | return a, b, cl
363 |
364 |
365 | def test():
366 | a, b, cl = init_test()
367 | cl.forward(a)
368 | print(cl.output_array)
369 |
370 | def test_bp():
371 | a, b, cl = init_test()
372 | cl.backward(a, b, IdentityActivator())
373 | cl.update()
374 | print(cl.filters[0])
375 | print(cl.filters[1])
376 |
377 |
378 | def gradient_check():
379 | '''
380 | 梯度检查
381 | '''
382 | # 设计一个误差函数，取所有节点输出项之和
383 | error_function = lambda o: o.sum()
384 |
385 | # 计算forward值
386 | a, b, cl = init_test()
387 | cl.forward(a)
388 |
389 | # 求取sensitivity map
390 | sensitivity_array = np.ones(cl.output_array.shape,
391 | dtype=np.float64)
392 | # 计算梯度
393 | cl.backward(a, sensitivity_array,
394 | IdentityActivator())
395 | # 检查梯度
396 | epsilon = 10e-4
397 | for d in range(cl.filters[0].weights_grad.shape[0]):
398 | for i in range(cl.filters[0].weights_grad.shape[1]):
399 | for j in range(cl.filters[0].weights_grad.shape[2]):
400 | cl.filters[0].weights[d,i,j] += epsilon
401 | cl.forward(a)
402 | err1 = error_function(cl.output_array)
403 | cl.filters[0].weights[d,i,j] -= 2*epsilon
404 | cl.forward(a)
405 | err2 = error_function(cl.output_array)
406 | expect_grad = int((err1 - err2) / (2 * epsilon))
407 | cl.filters[0].weights[d,i,j] += epsilon
408 | print('weights(%d,%d,%d): expected - actural %f - %f' % (
409 | d, i, j, expect_grad, cl.filters[0].weights_grad[d,i,j]))
410 |
411 |
412 | def init_pool_test():
413 | a = np.array(
414 | [[[1,1,2,4],
415 | [5,6,7,8],
416 | [3,2,1,0],
417 | [1,2,3,4]],
418 | [[0,1,2,3],
419 | [4,5,6,7],
420 | [8,9,0,1],
421 | [3,4,5,6]]], dtype=np.float64)
422 |
423 | b = np.array(
424 | [[[1,2],
425 | [2,4]],
426 | [[3,5],
427 | [8,2]]], dtype=np.float64)
428 |
429 | mpl = MaxPoolingLayer(4,4,2,2,2,2)
430 |
431 | return a, b, mpl
432 |
433 |
434 | def test_pool():
435 | a, b, mpl = init_pool_test()
436 | mpl.forward(a)
437 | print('input array:\n%s\noutput array:\n%s' % (a,
438 | mpl.output_array))
439 |
440 |
441 | def test_pool_bp():
442 | a, b, mpl = init_pool_test()
443 | mpl.backward(a, b)
444 | print('input array:\n%s\nsensitivity array:\n%s\ndelta array:\n%s' % (
445 | a, b, mpl.delta_array))
446 |
--------------------------------------------------------------------------------
/Tutorial/lesson-5/rnn.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import numpy as np
3 | from cnn import element_wise_op
4 | from activators import ReluActivator, IdentityActivator
5 | from functools import reduce
6 |
7 |
8 | class RecurrentLayer():
9 | def __init__(self, input_width, state_width,
10 | activator, learning_rate):
11 | self.input_width = input_width
12 | self.state_width = state_width
13 | self.activator = activator
14 | self.learning_rate = learning_rate
15 | self.times = 0 # 当前时刻初始化为t0
16 | self.state_list = [] # 保存各个时刻的state
17 | self.state_list.append(np.zeros(
18 | (state_width, 1))) # 初始化s0
19 | self.U = np.random.uniform(-1e-4, 1e-4,
20 | (state_width, input_width)) # 初始化U
21 | self.W = np.random.uniform(-1e-4, 1e-4,
22 | (state_width, state_width)) # 初始化W
23 |
24 | def forward(self, input_array):
25 | '''
26 | 根据『式2』进行前向计算
27 | '''
28 | self.times += 1
29 | state = (np.dot(self.U, input_array) +
30 | np.dot(self.W, self.state_list[-1]))
31 | element_wise_op(state, self.activator.forward)
32 | self.state_list.append(state)
33 |
34 | def backward(self, sensitivity_array,
35 | activator):
36 | '''
37 | 实现BPTT算法
38 | '''
39 | self.calc_delta(sensitivity_array, activator)
40 | self.calc_gradient()
41 |
42 | def update(self):
43 | '''
44 | 按照梯度下降，更新权重
45 | '''
46 | self.W -= self.learning_rate * self.gradient
47 |
48 | def calc_delta(self, sensitivity_array, activator):
49 | self.delta_list = [] # 用来保存各个时刻的误差项
50 | for i in range(self.times):
51 | self.delta_list.append(np.zeros(
52 | (self.state_width, 1)))
53 | self.delta_list.append(sensitivity_array)
54 | # 迭代计算每个时刻的误差项
55 | for k in range(self.times - 1, 0, -1):
56 | self.calc_delta_k(k, activator)
57 |
58 | def calc_delta_k(self, k, activator):
59 | '''
60 | 根据k+1时刻的delta计算k时刻的delta
61 | '''
62 | state = self.state_list[k+1].copy()
63 | element_wise_op(self.state_list[k+1],
64 | activator.backward)
65 | self.delta_list[k] = np.dot(
66 | np.dot(self.delta_list[k+1].T, self.W),
67 | np.diag(state[:,0])).T
68 |
69 | def calc_gradient(self):
70 | self.gradient_list = [] # 保存各个时刻的权重梯度
71 | for t in range(self.times + 1):
72 | self.gradient_list.append(np.zeros(
73 | (self.state_width, self.state_width)))
74 | for t in range(self.times, 0, -1):
75 | self.calc_gradient_t(t)
76 | # 实际的梯度是各个时刻梯度之和
77 | self.gradient = reduce(
78 | lambda a, b: a + b, self.gradient_list,
79 | self.gradient_list[0]) # [0]被初始化为0且没有被修改过
80 |
81 | def calc_gradient_t(self, t):
82 | '''
83 | 计算每个时刻t权重的梯度
84 | '''
85 | gradient = np.dot(self.delta_list[t],
86 | self.state_list[t-1].T)
87 | self.gradient_list[t] = gradient
88 |
89 | def reset_state(self):
90 | self.times = 0 # 当前时刻初始化为t0
91 | self.state_list = [] # 保存各个时刻的state
92 | self.state_list.append(np.zeros(
93 | (self.state_width, 1))) # 初始化s0
94 |
95 |
96 | def data_set():
97 | x = [np.array([[1], [2], [3]]),
98 | np.array([[2], [3], [4]])]
99 | d = np.array([[1], [2]])
100 | return x, d
101 |
102 |
103 | def gradient_check():
104 | '''
105 | 梯度检查
106 | '''
107 | # 设计一个误差函数，取所有节点输出项之和
108 | error_function = lambda o: o.sum()
109 |
110 | rl = RecurrentLayer(3, 2, IdentityActivator(), 1e-3)
111 |
112 | # 计算forward值
113 | x, d = data_set()
114 | rl.forward(x[0])
115 | rl.forward(x[1])
116 |
117 | # 求取sensitivity map
118 | sensitivity_array = np.ones(rl.state_list[-1].shape,
119 | dtype=np.float64)
120 | # 计算梯度
121 | rl.backward(sensitivity_array, IdentityActivator())
122 |
123 | # 检查梯度
124 | epsilon = 10e-4
125 | for i in range(rl.W.shape[0]):
126 | for j in range(rl.W.shape[1]):
127 | rl.W[i,j] += epsilon
128 | rl.reset_state()
129 | rl.forward(x[0])
130 | rl.forward(x[1])
131 | err1 = error_function(rl.state_list[-1])
132 | rl.W[i,j] -= 2*epsilon
133 | rl.reset_state()
134 | rl.forward(x[0])
135 | rl.forward(x[1])
136 | err2 = error_function(rl.state_list[-1])
137 | expect_grad = (err1 - err2) / (2 * epsilon)
138 | rl.W[i,j] += epsilon
139 | print('weights(%d,%d): expected - actural %f - %f' % (
140 | i, j, expect_grad, rl.gradient[i,j]))
141 |
142 |
143 | def test():
144 | l = RecurrentLayer(3, 2, ReluActivator(), 1e-3)
145 | x, d = data_set()
146 | l.forward(x[0])
147 | l.forward(x[1])
148 | l.backward(d, ReluActivator())
149 | return l
--------------------------------------------------------------------------------
/Tutorial/lesson-6/activators.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 |
3 |
4 | import numpy as np
5 |
6 |
7 | class ReluActivator(object):
8 | def forward(self, weighted_input):
9 | #return weighted_input
10 | return max(0, weighted_input)
11 |
12 | def backward(self, output):
13 | return 1 if output > 0 else 0
14 |
15 |
16 | class IdentityActivator(object):
17 | def forward(self, weighted_input):
18 | return weighted_input
19 |
20 | def backward(self, output):
21 | return 1
22 |
23 |
24 | class SigmoidActivator(object):
25 | def forward(self, weighted_input):
26 | return 1.0 / (1.0 + np.exp(-weighted_input))
27 |
28 | def backward(self, output):
29 | return output * (1 - output)
30 |
31 |
32 | class TanhActivator(object):
33 | def forward(self, weighted_input):
34 | return 2.0 / (1.0 + np.exp(-2 * weighted_input)) - 1.0
35 |
36 | def backward(self, output):
37 | return 1 - output * output
--------------------------------------------------------------------------------
/Tutorial/lesson-6/cnn.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import numpy as np
3 | from activators import ReluActivator, IdentityActivator
4 |
5 |
6 | # 获取卷积区域
7 | def get_patch(input_array, i, j, filter_width,
8 | filter_height, stride):
9 | '''
10 | 从输入数组中获取本次卷积的区域，
11 | 自动适配输入为2D和3D的情况
12 | '''
13 | start_i = i * stride
14 | start_j = j * stride
15 | if input_array.ndim == 2:
16 | return input_array[
17 | start_i : start_i + filter_height,
18 | start_j : start_j + filter_width]
19 | elif input_array.ndim == 3:
20 | return input_array[:,
21 | start_i : start_i + filter_height,
22 | start_j : start_j + filter_width]
23 |
24 |
25 | # 获取一个2D区域的最大值所在的索引
26 | def get_max_index(array):
27 | max_i = 0
28 | max_j = 0
29 | max_value = array[0,0]
30 | for i in range(array.shape[0]):
31 | for j in range(array.shape[1]):
32 | if array[i,j] > max_value:
33 | max_value = array[i,j]
34 | max_i, max_j = i, j
35 | return max_i, max_j
36 |
37 |
38 | # 计算卷积
39 | def conv(input_array,
40 | kernel_array,
41 | output_array,
42 | stride, bias):
43 | '''
44 | 计算卷积，自动适配输入为2D和3D的情况
45 | '''
46 | channel_number = input_array.ndim
47 | output_width = output_array.shape[1]
48 | output_height = output_array.shape[0]
49 | kernel_width = kernel_array.shape[-1]
50 | kernel_height = kernel_array.shape[-2]
51 | for i in range(output_height):
52 | for j in range(output_width):
53 | output_array[i][j] = (
54 | get_patch(input_array, i, j, kernel_width,
55 | kernel_height, stride) * kernel_array
56 | ).sum() + bias
57 |
58 |
59 | # 为数组增加Zero padding
60 | def padding(input_array, zp):
61 | '''
62 | 为数组增加Zero padding，自动适配输入为2D和3D的情况
63 | '''
64 | if zp == 0:
65 | return input_array
66 | else:
67 | if input_array.ndim == 3:
68 | input_width = input_array.shape[2]
69 | input_height = input_array.shape[1]
70 | input_depth = input_array.shape[0]
71 | padded_array = np.zeros((
72 | input_depth,
73 | input_height + 2 * zp,
74 | input_width + 2 * zp))
75 | padded_array[:,
76 | zp : zp + input_height,
77 | zp : zp + input_width] = input_array
78 | return padded_array
79 | elif input_array.ndim == 2:
80 | input_width = input_array.shape[1]
81 | input_height = input_array.shape[0]
82 | padded_array = np.zeros((
83 | input_height + 2 * zp,
84 | input_width + 2 * zp))
85 | padded_array[zp : zp + input_height,
86 | zp : zp + input_width] = input_array
87 | return padded_array
88 |
89 |
90 | # 对numpy数组进行element wise操作
91 | def element_wise_op(array, op):
92 | for i in np.nditer(array,
93 | op_flags=['readwrite']):
94 | i[...] = op(i)
95 |
96 |
97 | class Filter(object):
98 | def __init__(self, width, height, depth):
99 | self.weights = np.random.uniform(-1e-4, 1e-4,
100 | (depth, height, width))
101 | self.bias = 0
102 | self.weights_grad = np.zeros(
103 | self.weights.shape)
104 | self.bias_grad = 0
105 |
106 | def __repr__(self):
107 | return 'filter weights:\n%s\nbias:\n%s' % (
108 | repr(self.weights), repr(self.bias))
109 |
110 | def get_weights(self):
111 | return self.weights
112 |
113 | def get_bias(self):
114 | return self.bias
115 |
116 | def update(self, learning_rate):
117 | self.weights -= learning_rate * self.weights_grad
118 | self.bias -= learning_rate * self.bias_grad
119 |
120 |
121 | class ConvLayer(object):
122 | def __init__(self, input_width, input_height,
123 | channel_number, filter_width,
124 | filter_height, filter_number,
125 | zero_padding, stride, activator,
126 | learning_rate):
127 | self.input_width = input_width
128 | self.input_height = input_height
129 | self.channel_number = channel_number
130 | self.filter_width = filter_width
131 | self.filter_height = filter_height
132 | self.filter_number = filter_number
133 | self.zero_padding = zero_padding
134 | self.stride = stride
135 | self.output_width = \
136 | ConvLayer.calculate_output_size(
137 | self.input_width, filter_width, zero_padding,
138 | stride)
139 | self.output_height = \
140 | ConvLayer.calculate_output_size(
141 | self.input_height, filter_height, zero_padding,
142 | stride)
143 | self.output_array = np.zeros((self.filter_number,
144 | self.output_height, self.output_width))
145 | self.filters = []
146 | for i in range(filter_number):
147 | self.filters.append(Filter(filter_width,
148 | filter_height, self.channel_number))
149 | self.activator = activator
150 | self.learning_rate = learning_rate
151 |
152 | def forward(self, input_array):
153 | '''
154 | 计算卷积层的输出
155 | 输出结果保存在self.output_array
156 | '''
157 | self.input_array = input_array
158 | self.padded_input_array = padding(input_array,
159 | self.zero_padding)
160 | for f in range(self.filter_number):
161 | filter = self.filters[f]
162 | conv(self.padded_input_array,
163 | filter.get_weights(), self.output_array[f],
164 | self.stride, filter.get_bias())
165 | element_wise_op(self.output_array,
166 | self.activator.forward)
167 |
168 | def backward(self, input_array, sensitivity_array,
169 | activator):
170 | '''
171 | 计算传递给前一层的误差项，以及计算每个权重的梯度
172 | 前一层的误差项保存在self.delta_array
173 | 梯度保存在Filter对象的weights_grad
174 | '''
175 | self.forward(input_array)
176 | self.bp_sensitivity_map(sensitivity_array,
177 | activator)
178 | self.bp_gradient(sensitivity_array)
179 |
180 | def update(self):
181 | '''
182 | 按照梯度下降，更新权重
183 | '''
184 | for filter in self.filters:
185 | filter.update(self.learning_rate)
186 |
187 | def bp_sensitivity_map(self, sensitivity_array,
188 | activator):
189 | '''
190 | 计算传递到上一层的sensitivity map
191 | sensitivity_array: 本层的sensitivity map
192 | activator: 上一层的激活函数
193 | '''
194 | # 处理卷积步长，对原始sensitivity map进行扩展
195 | expanded_array = self.expand_sensitivity_map(
196 | sensitivity_array)
197 | # full卷积，对sensitivitiy map进行zero padding
198 | # 虽然原始输入的zero padding单元也会获得残差
199 | # 但这个残差不需要继续向上传递，因此就不计算了
200 | expanded_width = expanded_array.shape[2]
201 | zp = int((self.input_width +
202 | self.filter_width - 1 - expanded_width) / 2)
203 | padded_array = padding(expanded_array, zp)
204 | # 初始化delta_array，用于保存传递到上一层的
205 | # sensitivity map
206 | self.delta_array = self.create_delta_array()
207 | # 对于具有多个filter的卷积层来说，最终传递到上一层的
208 | # sensitivity map相当于所有的filter的
209 | # sensitivity map之和
210 | for f in range(self.filter_number):
211 | filter = self.filters[f]
212 | # 将filter权重翻转180度
213 | flipped_weights = np.array(list(map(
214 | lambda i: np.rot90(i, 2),
215 | filter.get_weights())))
216 | # 计算与一个filter对应的delta_array
217 | delta_array = self.create_delta_array()
218 | for d in range(delta_array.shape[0]):
219 | conv(padded_array[f], flipped_weights[d],delta_array[d], 1, 0)
220 | self.delta_array += delta_array
221 | # 将计算结果与激活函数的偏导数做element-wise乘法操作
222 | derivative_array = np.array(self.input_array)
223 | element_wise_op(derivative_array,
224 | activator.backward)
225 | self.delta_array *= derivative_array
226 |
227 | def bp_gradient(self, sensitivity_array):
228 | # 处理卷积步长，对原始sensitivity map进行扩展
229 | expanded_array = self.expand_sensitivity_map(
230 | sensitivity_array)
231 | for f in range(self.filter_number):
232 | # 计算每个权重的梯度
233 | filter = self.filters[f]
234 | for d in range(filter.weights.shape[0]):
235 | conv(self.padded_input_array[d],
236 | expanded_array[f],
237 | filter.weights_grad[d], 1, 0)
238 | # 计算偏置项的梯度
239 | filter.bias_grad = expanded_array[f].sum()
240 |
241 | def expand_sensitivity_map(self, sensitivity_array):
242 | depth = sensitivity_array.shape[0]
243 | # 确定扩展后sensitivity map的大小
244 | # 计算stride为1时sensitivity map的大小
245 | expanded_width = (self.input_width -
246 | self.filter_width + 2 * self.zero_padding + 1)
247 | expanded_height = (self.input_height -
248 | self.filter_height + 2 * self.zero_padding + 1)
249 | # 构建新的sensitivity_map
250 | expand_array = np.zeros((depth, expanded_height,
251 | expanded_width))
252 | # 从原始sensitivity map拷贝误差值
253 | for i in range(self.output_height):
254 | for j in range(self.output_width):
255 | i_pos = i * self.stride
256 | j_pos = j * self.stride
257 | expand_array[:,i_pos,j_pos] = \
258 | sensitivity_array[:,i,j]
259 | return expand_array
260 |
261 | def create_delta_array(self):
262 | return np.zeros((self.channel_number,
263 | self.input_height, self.input_width))
264 |
265 | @staticmethod
266 | def calculate_output_size(input_size,
267 | filter_size, zero_padding, stride):
268 | return int((input_size - filter_size +
269 | 2 * zero_padding) / stride + 1)
270 |
271 |
272 | class MaxPoolingLayer(object):
273 | def __init__(self, input_width, input_height,
274 | channel_number, filter_width,
275 | filter_height, stride):
276 | self.input_width = input_width
277 | self.input_height = input_height
278 | self.channel_number = channel_number
279 | self.filter_width = filter_width
280 | self.filter_height = filter_height
281 | self.stride = stride
282 | self.output_width = int((input_width -
283 | filter_width) / self.stride + 1)
284 | self.output_height = int((input_height -
285 | filter_height) / self.stride + 1)
286 | self.output_array = np.zeros((self.channel_number,
287 | self.output_height, self.output_width))
288 |
289 | def forward(self, input_array):
290 | for d in range(self.channel_number):
291 | for i in range(self.output_height):
292 | for j in range(self.output_width):
293 | self.output_array[d,i,j] = (
294 | get_patch(input_array[d], i, j,
295 | self.filter_width,
296 | self.filter_height,
297 | self.stride).max())
298 |
299 | def backward(self, input_array, sensitivity_array):
300 | self.delta_array = np.zeros(input_array.shape)
301 | for d in range(self.channel_number):
302 | for i in range(self.output_height):
303 | for j in range(self.output_width):
304 | patch_array = get_patch(
305 | input_array[d], i, j,
306 | self.filter_width,
307 | self.filter_height,
308 | self.stride)
309 | k, l = get_max_index(patch_array)
310 | self.delta_array[d,
311 | i * self.stride + k,
312 | j * self.stride + l] = \
313 | sensitivity_array[d,i,j]
314 |
315 |
316 | def init_test():
317 | a = np.array(
318 | [[[0,1,1,0,2],
319 | [2,2,2,2,1],
320 | [1,0,0,2,0],
321 | [0,1,1,0,0],
322 | [1,2,0,0,2]],
323 | [[1,0,2,2,0],
324 | [0,0,0,2,0],
325 | [1,2,1,2,1],
326 | [1,0,0,0,0],
327 | [1,2,1,1,1]],
328 | [[2,1,2,0,0],
329 | [1,0,0,1,0],
330 | [0,2,1,0,1],
331 | [0,1,2,2,2],
332 | [2,1,0,0,1]]])
333 | b = np.array(
334 | [[[0,1,1],
335 | [2,2,2],
336 | [1,0,0]],
337 | [[1,0,2],
338 | [0,0,0],
339 | [1,2,1]]])
340 | cl = ConvLayer(5,5,3,3,3,2,1,2,IdentityActivator(),0.001)
341 | cl.filters[0].weights = np.array(
342 | [[[-1,1,0],
343 | [0,1,0],
344 | [0,1,1]],
345 | [[-1,-1,0],
346 | [0,0,0],
347 | [0,-1,0]],
348 | [[0,0,-1],
349 | [0,1,0],
350 | [1,-1,-1]]], dtype=np.float64)
351 | cl.filters[0].bias=1
352 | cl.filters[1].weights = np.array(
353 | [[[1,1,-1],
354 | [-1,-1,1],
355 | [0,-1,1]],
356 | [[0,1,0],
357 | [-1,0,-1],
358 | [-1,1,0]],
359 | [[-1,0,0],
360 | [-1,0,1],
361 | [-1,0,0]]], dtype=np.float64)
362 | return a, b, cl
363 |
364 |
365 | def test():
366 | a, b, cl = init_test()
367 | cl.forward(a)
368 | print(cl.output_array)
369 |
370 | def test_bp():
371 | a, b, cl = init_test()
372 | cl.backward(a, b, IdentityActivator())
373 | cl.update()
374 | print(cl.filters[0])
375 | print(cl.filters[1])
376 |
377 |
378 | def gradient_check():
379 | '''
380 | 梯度检查
381 | '''
382 | # 设计一个误差函数，取所有节点输出项之和
383 | error_function = lambda o: o.sum()
384 |
385 | # 计算forward值
386 | a, b, cl = init_test()
387 | cl.forward(a)
388 |
389 | # 求取sensitivity map
390 | sensitivity_array = np.ones(cl.output_array.shape,
391 | dtype=np.float64)
392 | # 计算梯度
393 | cl.backward(a, sensitivity_array,
394 | IdentityActivator())
395 | # 检查梯度
396 | epsilon = 10e-4
397 | for d in range(cl.filters[0].weights_grad.shape[0]):
398 | for i in range(cl.filters[0].weights_grad.shape[1]):
399 | for j in range(cl.filters[0].weights_grad.shape[2]):
400 | cl.filters[0].weights[d,i,j] += epsilon
401 | cl.forward(a)
402 | err1 = error_function(cl.output_array)
403 | cl.filters[0].weights[d,i,j] -= 2*epsilon
404 | cl.forward(a)
405 | err2 = error_function(cl.output_array)
406 | expect_grad = int((err1 - err2) / (2 * epsilon))
407 | cl.filters[0].weights[d,i,j] += epsilon
408 | print('weights(%d,%d,%d): expected - actural %f - %f' % (
409 | d, i, j, expect_grad, cl.filters[0].weights_grad[d,i,j]))
410 |
411 |
412 | def init_pool_test():
413 | a = np.array(
414 | [[[1,1,2,4],
415 | [5,6,7,8],
416 | [3,2,1,0],
417 | [1,2,3,4]],
418 | [[0,1,2,3],
419 | [4,5,6,7],
420 | [8,9,0,1],
421 | [3,4,5,6]]], dtype=np.float64)
422 |
423 | b = np.array(
424 | [[[1,2],
425 | [2,4]],
426 | [[3,5],
427 | [8,2]]], dtype=np.float64)
428 |
429 | mpl = MaxPoolingLayer(4,4,2,2,2,2)
430 |
431 | return a, b, mpl
432 |
433 |
434 | def test_pool():
435 | a, b, mpl = init_pool_test()
436 | mpl.forward(a)
437 | print('input array:\n%s\noutput array:\n%s' % (a,
438 | mpl.output_array))
439 |
440 |
441 | def test_pool_bp():
442 | a, b, mpl = init_pool_test()
443 | mpl.backward(a, b)
444 | print('input array:\n%s\nsensitivity array:\n%s\ndelta array:\n%s' % (
445 | a, b, mpl.delta_array))
446 |
--------------------------------------------------------------------------------
/Tutorial/lesson-6/lstm.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import matplotlib.pyplot as plt
3 | import numpy as np
4 | from cnn import element_wise_op
5 | from activators import SigmoidActivator, TanhActivator, IdentityActivator
6 |
7 | class LstmLayer():
8 | def __init__(self, input_width, state_width,
9 | learning_rate):
10 | self.input_width = input_width
11 | self.state_width = state_width
12 | self.learning_rate = learning_rate
13 | # 门的激活函数
14 | self.gate_activator = SigmoidActivator()
15 | # 输出的激活函数
16 | self.output_activator = TanhActivator()
17 | # 当前时刻初始化为t0
18 | self.times = 0
19 | # 各个时刻的单元状态向量c
20 | self.c_list = self.init_state_vec()
21 | # 各个时刻的输出向量h
22 | self.h_list = self.init_state_vec()
23 | # 各个时刻的遗忘门f
24 | self.f_list = self.init_state_vec()
25 | # 各个时刻的输入门i
26 | self.i_list = self.init_state_vec()
27 | # 各个时刻的输出门o
28 | self.o_list = self.init_state_vec()
29 | # 各个时刻的即时状态c~
30 | self.ct_list = self.init_state_vec()
31 | # 遗忘门权重矩阵Wfh, Wfx, 偏置项bf
32 | self.Wfh, self.Wfx, self.bf = (
33 | self.init_weight_mat())
34 | # 输入门权重矩阵Wfh, Wfx, 偏置项bf
35 | self.Wih, self.Wix, self.bi = (
36 | self.init_weight_mat())
37 | # 输出门权重矩阵Wfh, Wfx, 偏置项bf
38 | self.Woh, self.Wox, self.bo = (
39 | self.init_weight_mat())
40 | # 单元状态权重矩阵Wfh, Wfx, 偏置项bf
41 | self.Wch, self.Wcx, self.bc = (
42 | self.init_weight_mat())
43 |
44 | def init_state_vec(self):
45 | '''
46 | 初始化保存状态的向量
47 | '''
48 | state_vec_list = []
49 | state_vec_list.append(np.zeros(
50 | (self.state_width, 1)))
51 | return state_vec_list
52 |
53 | def init_weight_mat(self):
54 | '''
55 | 初始化权重矩阵
56 | '''
57 | Wh = np.random.uniform(-1e-4, 1e-4,
58 | (self.state_width, self.state_width))
59 | Wx = np.random.uniform(-1e-4, 1e-4,
60 | (self.state_width, self.input_width))
61 | b = np.zeros((self.state_width, 1))
62 | return Wh, Wx, b
63 |
64 | def forward(self, x):
65 | '''
66 | 根据式1-式6进行前向计算
67 | '''
68 | self.times += 1
69 | # 遗忘门
70 | fg = self.calc_gate(x, self.Wfx, self.Wfh,
71 | self.bf, self.gate_activator)
72 | self.f_list.append(fg)
73 | # 输入门
74 | ig = self.calc_gate(x, self.Wix, self.Wih,
75 | self.bi, self.gate_activator)
76 | self.i_list.append(ig)
77 | # 输出门
78 | og = self.calc_gate(x, self.Wox, self.Woh,
79 | self.bo, self.gate_activator)
80 | self.o_list.append(og)
81 | # 即时状态
82 | ct = self.calc_gate(x, self.Wcx, self.Wch,
83 | self.bc, self.output_activator)
84 | self.ct_list.append(ct)
85 | # 单元状态
86 | c = fg * self.c_list[self.times - 1] + ig * ct
87 | self.c_list.append(c)
88 | # 输出
89 | h = og * self.output_activator.forward(c)
90 | self.h_list.append(h)
91 |
92 | def calc_gate(self, x, Wx, Wh, b, activator):
93 | '''
94 | 计算门
95 | '''
96 | h = self.h_list[self.times - 1] # 上次的LSTM输出
97 | net = np.dot(Wh, h) + np.dot(Wx, x) + b
98 | gate = activator.forward(net)
99 | return gate
100 |
101 |
102 | def backward(self, x, delta_h, activator):
103 | '''
104 | 实现LSTM训练算法
105 | '''
106 | self.calc_delta(delta_h, activator)
107 | self.calc_gradient(x)
108 |
109 | def update(self):
110 | '''
111 | 按照梯度下降，更新权重
112 | '''
113 | self.Wfh -= self.learning_rate * self.Whf_grad
114 | self.Wfx -= self.learning_rate * self.Whx_grad
115 | self.bf -= self.learning_rate * self.bf_grad
116 | self.Wih -= self.learning_rate * self.Whi_grad
117 | self.Wix -= self.learning_rate * self.Whi_grad
118 | self.bi -= self.learning_rate * self.bi_grad
119 | self.Woh -= self.learning_rate * self.Wof_grad
120 | self.Wox -= self.learning_rate * self.Wox_grad
121 | self.bo -= self.learning_rate * self.bo_grad
122 | self.Wch -= self.learning_rate * self.Wcf_grad
123 | self.Wcx -= self.learning_rate * self.Wcx_grad
124 | self.bc -= self.learning_rate * self.bc_grad
125 |
126 | def calc_delta(self, delta_h, activator):
127 | # 初始化各个时刻的误差项
128 | self.delta_h_list = self.init_delta() # 输出误差项
129 | self.delta_o_list = self.init_delta() # 输出门误差项
130 | self.delta_i_list = self.init_delta() # 输入门误差项
131 | self.delta_f_list = self.init_delta() # 遗忘门误差项
132 | self.delta_ct_list = self.init_delta() # 即时输出误差项
133 |
134 | # 保存从上一层传递下来的当前时刻的误差项
135 | self.delta_h_list[-1] = delta_h
136 |
137 | # 迭代计算每个时刻的误差项
138 | for k in range(self.times, 0, -1):
139 | self.calc_delta_k(k)
140 |
141 | def init_delta(self):
142 | '''
143 | 初始化误差项
144 | '''
145 | delta_list = []
146 | for i in range(self.times + 1):
147 | delta_list.append(np.zeros(
148 | (self.state_width, 1)))
149 | return delta_list
150 |
151 | def calc_delta_k(self, k):
152 | '''
153 | 根据k时刻的delta_h，计算k时刻的delta_f、
154 | delta_i、delta_o、delta_ct，以及k-1时刻的delta_h
155 | '''
156 | # 获得k时刻前向计算的值
157 | ig = self.i_list[k]
158 | og = self.o_list[k]
159 | fg = self.f_list[k]
160 | ct = self.ct_list[k]
161 | c = self.c_list[k]
162 | c_prev = self.c_list[k-1]
163 | tanh_c = self.output_activator.forward(c)
164 | delta_k = self.delta_h_list[k]
165 |
166 | # 根据式9计算delta_o
167 | delta_o = (delta_k * tanh_c *
168 | self.gate_activator.backward(og))
169 | delta_f = (delta_k * og *
170 | (1 - tanh_c * tanh_c) * c_prev *
171 | self.gate_activator.backward(fg))
172 | delta_i = (delta_k * og *
173 | (1 - tanh_c * tanh_c) * ct *
174 | self.gate_activator.backward(ig))
175 | delta_ct = (delta_k * og *
176 | (1 - tanh_c * tanh_c) * ig *
177 | self.output_activator.backward(ct))
178 | delta_h_prev = (
179 | np.dot(delta_o.transpose(), self.Woh) +
180 | np.dot(delta_i.transpose(), self.Wih) +
181 | np.dot(delta_f.transpose(), self.Wfh) +
182 | np.dot(delta_ct.transpose(), self.Wch)
183 | ).transpose()
184 |
185 | # 保存全部delta值
186 | self.delta_h_list[k-1] = delta_h_prev
187 | self.delta_f_list[k] = delta_f
188 | self.delta_i_list[k] = delta_i
189 | self.delta_o_list[k] = delta_o
190 | self.delta_ct_list[k] = delta_ct
191 |
192 | def calc_gradient(self, x):
193 | # 初始化遗忘门权重梯度矩阵和偏置项
194 | self.Wfh_grad, self.Wfx_grad, self.bf_grad = (
195 | self.init_weight_gradient_mat())
196 | # 初始化输入门权重梯度矩阵和偏置项
197 | self.Wih_grad, self.Wix_grad, self.bi_grad = (
198 | self.init_weight_gradient_mat())
199 | # 初始化输出门权重梯度矩阵和偏置项
200 | self.Woh_grad, self.Wox_grad, self.bo_grad = (
201 | self.init_weight_gradient_mat())
202 | # 初始化单元状态权重梯度矩阵和偏置项
203 | self.Wch_grad, self.Wcx_grad, self.bc_grad = (
204 | self.init_weight_gradient_mat())
205 |
206 | # 计算对上一次输出h的权重梯度
207 | for t in range(self.times, 0, -1):
208 | # 计算各个时刻的梯度
209 | (Wfh_grad, bf_grad,
210 | Wih_grad, bi_grad,
211 | Woh_grad, bo_grad,
212 | Wch_grad, bc_grad) = (
213 | self.calc_gradient_t(t))
214 | # 实际梯度是各时刻梯度之和
215 | self.Wfh_grad += Wfh_grad
216 | self.bf_grad += bf_grad
217 | self.Wih_grad += Wih_grad
218 | self.bi_grad += bi_grad
219 | self.Woh_grad += Woh_grad
220 | self.bo_grad += bo_grad
221 | self.Wch_grad += Wch_grad
222 | self.bc_grad += bc_grad
223 |
224 | # 计算对本次输入x的权重梯度
225 | xt = x.transpose()
226 | self.Wfx_grad = np.dot(self.delta_f_list[-1], xt)
227 | self.Wix_grad = np.dot(self.delta_i_list[-1], xt)
228 | self.Wox_grad = np.dot(self.delta_o_list[-1], xt)
229 | self.Wcx_grad = np.dot(self.delta_ct_list[-1], xt)
230 |
231 | def init_weight_gradient_mat(self):
232 | '''
233 | 初始化权重矩阵
234 | '''
235 | Wh_grad = np.zeros((self.state_width,
236 | self.state_width))
237 | Wx_grad = np.zeros((self.state_width,
238 | self.input_width))
239 | b_grad = np.zeros((self.state_width, 1))
240 | return Wh_grad, Wx_grad, b_grad
241 |
242 | def calc_gradient_t(self, t):
243 | '''
244 | 计算每个时刻t权重的梯度
245 | '''
246 | h_prev = self.h_list[t-1].transpose()
247 | Wfh_grad = np.dot(self.delta_f_list[t], h_prev)
248 | bf_grad = self.delta_f_list[t]
249 | Wih_grad = np.dot(self.delta_i_list[t], h_prev)
250 | bi_grad = self.delta_f_list[t]
251 | Woh_grad = np.dot(self.delta_o_list[t], h_prev)
252 | bo_grad = self.delta_f_list[t]
253 | Wch_grad = np.dot(self.delta_ct_list[t], h_prev)
254 | bc_grad = self.delta_ct_list[t]
255 | return Wfh_grad, bf_grad, Wih_grad, bi_grad, \
256 | Woh_grad, bo_grad, Wch_grad, bc_grad
257 |
258 | def reset_state(self):
259 | # 当前时刻初始化为t0
260 | self.times = 0
261 | # 各个时刻的单元状态向量c
262 | self.c_list = self.init_state_vec()
263 | # 各个时刻的输出向量h
264 | self.h_list = self.init_state_vec()
265 | # 各个时刻的遗忘门f
266 | self.f_list = self.init_state_vec()
267 | # 各个时刻的输入门i
268 | self.i_list = self.init_state_vec()
269 | # 各个时刻的输出门o
270 | self.o_list = self.init_state_vec()
271 | # 各个时刻的即时状态c~
272 | self.ct_list = self.init_state_vec()
273 |
274 |
275 | def data_set():
276 | x = [np.array([[1], [2], [3]]),
277 | np.array([[2], [3], [4]])]
278 | d = np.array([[1], [2]])
279 | return x, d
280 |
281 |
282 | def gradient_check():
283 | '''
284 | 梯度检查
285 | '''
286 | # 设计一个误差函数，取所有节点输出项之和
287 | error_function = lambda o: o.sum()
288 |
289 | lstm = LstmLayer(3, 2, 1e-3)
290 |
291 | # 计算forward值
292 | x, d = data_set()
293 | lstm.forward(x[0])
294 | lstm.forward(x[1])
295 |
296 | # 求取sensitivity map
297 | sensitivity_array = np.ones(lstm.h_list[-1].shape,
298 | dtype=np.float64)
299 | # 计算梯度
300 | lstm.backward(x[1], sensitivity_array, IdentityActivator())
301 |
302 | # 检查梯度
303 | epsilon = 10e-4
304 | for i in range(lstm.Wfh.shape[0]):
305 | for j in range(lstm.Wfh.shape[1]):
306 | lstm.Wfh[i,j] += epsilon
307 | lstm.reset_state()
308 | lstm.forward(x[0])
309 | lstm.forward(x[1])
310 | err1 = error_function(lstm.h_list[-1])
311 | lstm.Wfh[i,j] -= 2*epsilon
312 | lstm.reset_state()
313 | lstm.forward(x[0])
314 | lstm.forward(x[1])
315 | err2 = error_function(lstm.h_list[-1])
316 | expect_grad = (err1 - err2) / (2 * epsilon)
317 | lstm.Wfh[i,j] += epsilon
318 | print('weights(%d,%d): expected - actural %.4e - %.4e' % (
319 | i, j, expect_grad, lstm.Wfh_grad[i,j]))
320 | return lstm
321 |
322 |
323 | def test():
324 | l = LstmLayer(3, 2, 1e-3)
325 | x, d = data_set()
326 | l.forward(x[0])
327 | l.forward(x[1])
328 | l.backward(x[1], d, IdentityActivator())
329 | return l
--------------------------------------------------------------------------------
/Tutorial/lesson-7/activators.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 |
3 |
4 | import numpy as np
5 |
6 |
7 | class ReluActivator(object):
8 | def forward(self, weighted_input):
9 | #return weighted_input
10 | return max(0, weighted_input)
11 |
12 | def backward(self, output):
13 | return 1 if output > 0 else 0
14 |
15 |
16 | class IdentityActivator(object):
17 | def forward(self, weighted_input):
18 | return weighted_input
19 |
20 | def backward(self, output):
21 | return 1
22 |
23 |
24 | class SigmoidActivator(object):
25 | def forward(self, weighted_input):
26 | return 1.0 / (1.0 + np.exp(-weighted_input))
27 |
28 | def backward(self, output):
29 | return output * (1 - output)
30 |
31 |
32 | class TanhActivator(object):
33 | def forward(self, weighted_input):
34 | return 2.0 / (1.0 + np.exp(-2 * weighted_input)) - 1.0
35 |
36 | def backward(self, output):
37 | return 1 - output * output
--------------------------------------------------------------------------------
/Tutorial/lesson-7/recursive.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: UTF-8 -*-
3 |
4 |
5 | import numpy as np
6 | from activators import IdentityActivator
7 |
8 |
9 | class TreeNode(object):
10 | def __init__(self, data, children=[], children_data=[]):
11 | self.parent = None
12 | self.children = children
13 | self.children_data = children_data
14 | self.data = data
15 | for child in children:
16 | child.parent = self
17 |
18 | # 递归神经网络实现
19 | class RecursiveLayer(object):
20 | def __init__(self, node_width, child_count,
21 | activator, learning_rate):
22 | '''
23 | 递归神经网络构造函数
24 | node_width: 表示每个节点的向量的维度
25 | child_count: 每个父节点有几个子节点
26 | activator: 激活函数对象
27 | learning_rate: 梯度下降算法学习率
28 | '''
29 | self.node_width = node_width
30 | self.child_count = child_count
31 | self.activator = activator
32 | self.learning_rate = learning_rate
33 | # 权重数组W
34 | self.W = np.random.uniform(-1e-4, 1e-4,
35 | (node_width, node_width * child_count))
36 | # 偏置项b
37 | self.b = np.zeros((node_width, 1))
38 | # 递归神经网络生成的树的根节点
39 | self.root = None
40 |
41 | def forward(self, *children):
42 | '''
43 | 前向计算
44 | '''
45 | children_data = self.concatenate(children)
46 | parent_data = self.activator.forward(
47 | np.dot(self.W, children_data) + self.b
48 | )
49 | self.root = TreeNode(parent_data, children
50 | , children_data)
51 |
52 | def backward(self, parent_delta):
53 | '''
54 | BPTS反向传播算法
55 | '''
56 | self.calc_delta(parent_delta, self.root)
57 | self.W_grad, self.b_grad = self.calc_gradient(self.root)
58 |
59 | def update(self):
60 | '''
61 | 使用SGD算法更新权重
62 | '''
63 | self.W -= self.learning_rate * self.W_grad
64 | self.b -= self.learning_rate * self.b_grad
65 |
66 | def reset_state(self):
67 | self.root = None
68 |
69 | def concatenate(self, tree_nodes):
70 | '''
71 | 将各个树节点中的数据拼接成一个长向量
72 | '''
73 | concat = np.zeros((0,1))
74 | for node in tree_nodes:
75 | concat = np.concatenate((concat, node.data))
76 | return concat
77 |
78 | def calc_delta(self, parent_delta, parent):
79 | '''
80 | 计算每个节点的delta
81 | '''
82 | parent.delta = parent_delta
83 | if parent.children:
84 | # 根据式2计算每个子节点的delta
85 | children_delta = np.dot(self.W.T, parent_delta) * (
86 | self.activator.backward(parent.children_data)
87 | )
88 | # slices = [(子节点编号，子节点delta起始位置，子节点delta结束位置)]
89 | slices = [(i, i * self.node_width,
90 | (i + 1) * self.node_width)
91 | for i in range(self.child_count)]
92 | # 针对每个子节点，递归调用calc_delta函数
93 | for s in slices:
94 | self.calc_delta(children_delta[s[1]:s[2]],
95 | parent.children[s[0]])
96 |
97 | def calc_gradient(self, parent):
98 | '''
99 | 计算每个节点权重的梯度，并将它们求和，得到最终的梯度
100 | '''
101 | W_grad = np.zeros((self.node_width,
102 | self.node_width * self.child_count))
103 | b_grad = np.zeros((self.node_width, 1))
104 | if not parent.children:
105 | return W_grad, b_grad
106 | parent.W_grad = np.dot(parent.delta, parent.children_data.T)
107 | parent.b_grad = parent.delta
108 | W_grad += parent.W_grad
109 | b_grad += parent.b_grad
110 | for child in parent.children:
111 | W, b = self.calc_gradient(child)
112 | W_grad += W
113 | b_grad += b
114 | return W_grad, b_grad
115 |
116 | def dump(self, **kwArgs):
117 | print('root.data: %s' % self.root.data)
118 | print('root.children_data: %s' % self.root.children_data)
119 | if kwArgs.has_key('dump_grad'):
120 | print('W_grad: %s' % self.W_grad)
121 | print('b_grad: %s' % self.b_grad)
122 |
123 |
124 | def data_set():
125 | children = [
126 | TreeNode(np.array([[1],[2]])),
127 | TreeNode(np.array([[3],[4]])),
128 | TreeNode(np.array([[5],[6]]))
129 | ]
130 | d = np.array([[0.5],[0.8]])
131 | return children, d
132 |
133 |
134 | def gradient_check():
135 | '''
136 | 梯度检查
137 | '''
138 | # 设计一个误差函数，取所有节点输出项之和
139 | error_function = lambda o: o.sum()
140 |
141 | rnn = RecursiveLayer(2, 2, IdentityActivator(), 1e-3)
142 |
143 | # 计算forward值
144 | x, d = data_set()
145 | rnn.forward(x[0], x[1])
146 | rnn.forward(rnn.root, x[2])
147 |
148 | # 求取sensitivity map
149 | sensitivity_array = np.ones((rnn.node_width, 1),
150 | dtype=np.float64)
151 | # 计算梯度
152 | rnn.backward(sensitivity_array)
153 |
154 | # 检查梯度
155 | epsilon = 10e-4
156 | for i in range(rnn.W.shape[0]):
157 | for j in range(rnn.W.shape[1]):
158 | rnn.W[i,j] += epsilon
159 | rnn.reset_state()
160 | rnn.forward(x[0], x[1])
161 | rnn.forward(rnn.root, x[2])
162 | err1 = error_function(rnn.root.data)
163 | rnn.W[i,j] -= 2*epsilon
164 | rnn.reset_state()
165 | rnn.forward(x[0], x[1])
166 | rnn.forward(rnn.root, x[2])
167 | err2 = error_function(rnn.root.data)
168 | expect_grad = (err1 - err2) / (2 * epsilon)
169 | rnn.W[i,j] += epsilon
170 | print('weights(%d,%d): expected - actural %.4e - %.4e' % (
171 | i, j, expect_grad, rnn.W_grad[i,j]))
172 | return rnn
173 |
174 |
175 | def test():
176 | children, d = data_set()
177 | rnn = RecursiveLayer(2, 2, IdentityActivator(), 1e-3)
178 | rnn.forward(children[0], children[1])
179 | rnn.dump()
180 | rnn.forward(rnn.root, children[2])
181 | rnn.dump()
182 | rnn.backward(d)
183 | rnn.dump(dump_grad='true')
184 | return rnn
--------------------------------------------------------------------------------
/face/video_mosaic.py:
--------------------------------------------------------------------------------
1 | # Author : Jack Cui
2 | # Website: https://cuijiahua.com/
3 | import cv2
4 | import face_recognition
5 | import matplotlib.pyplot as plt
6 | # %matplotlib inline # 在 jupyter 中使用的时候，去掉注释
7 |
8 | import subprocess
9 | import os
10 | from PIL import Image
11 |
12 | def video2mp3(file_name):
13 | """
14 | 将视频转为音频
15 | :param file_name: 传入视频文件的路径
16 | :return:
17 | """
18 | outfile_name = file_name.split('.')[0] + '.mp3'
19 | cmd = 'ffmpeg -i ' + file_name + ' -f mp3 ' + outfile_name
20 | print(cmd)
21 | subprocess.call(cmd, shell=True)
22 |
23 |
24 | def video_add_mp3(file_name, mp3_file):
25 | """
26 | 视频添加音频
27 | :param file_name: 传入视频文件的路径
28 | :param mp3_file: 传入音频文件的路径
29 | :return:
30 | """
31 | outfile_name = file_name.split('.')[0] + '-f.mp4'
32 | subprocess.call('ffmpeg -i ' + file_name
33 | + ' -i ' + mp3_file + ' -strict -2 -f mp4 '
34 | + outfile_name, shell=True)
35 |
36 | def mask_video(input_video, output_video, mask_path='mask.jpg'):
37 | # 打码图片
38 | mask = cv2.imread(mask_path)
39 | # 读取视频
40 | cap = cv2.VideoCapture(input_video)
41 | # 读取视频参数,fps、width、heigth
42 | CV_CAP_PROP_FPS = 5
43 | CV_CAP_PROP_FRAME_WIDTH = 3
44 | CV_CAP_PROP_FRAME_HEIGHT = 4
45 | v_fps = cap.get(CV_CAP_PROP_FPS)
46 | v_width = cap.get(CV_CAP_PROP_FRAME_WIDTH)
47 | v_height = cap.get(CV_CAP_PROP_FRAME_HEIGHT)
48 | # 设置写视频参数，格式为 mp4
49 | size = (int(v_width), int(v_height))
50 | fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
51 | out = cv2.VideoWriter(output_video,fourcc, v_fps, size)
52 |
53 | # 已知人脸
54 | known_image = face_recognition.load_image_file("tz.jpg")
55 | biden_encoding = face_recognition.face_encodings(known_image)[0]
56 | # 读取视频
57 | cap = cv2.VideoCapture(input_video)
58 | while(cap.isOpened()):
59 | ret, frame = cap.read()
60 | if ret:
61 | # 检测人脸
62 | face_locations = face_recognition.face_locations(frame)
63 | # 检测每一个人脸
64 | for (top_right_y, top_right_x, left_bottom_y,left_bottom_x) in face_locations:
65 | unknown_image = frame[top_right_y-50:left_bottom_y+50, left_bottom_x-50:top_right_x+50]
66 | unknown_encoding = face_recognition.face_encodings(unknown_image)[0]
67 | # 对比结果
68 | results = face_recognition.compare_faces([biden_encoding], unknown_encoding)
69 | # 是仝卓，就将打码贴图。
70 | if results[0] == True:
71 | mask = cv2.resize(mask, (top_right_x-left_bottom_x, left_bottom_y-top_right_y))
72 | frame[top_right_y:left_bottom_y, left_bottom_x:top_right_x] = mask
73 | # 写入视频
74 | out.write(frame)
75 | else:
76 | break
77 |
78 | if __name__ == '__main__':
79 | # 将音频保存为cut.mp3
80 | video2mp3(file_name='cut.mp4')
81 | # 处理视频，自动打码，输出视频为output.mp4
82 | mask_video(input_video='cut.mp4', output_video='output.mp4')
83 | # 为 output.mp4 处理好的视频添加声音
84 | video_add_mp3(file_name='output.mp4', mp3_file='cut.mp3')
85 |
--------------------------------------------------------------------------------
/mnist.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from tensorflow.examples.tutorials.mnist import input_data
3 | import os
4 |
5 | '''
6 | Website:http://cuijiahua.linear_com
7 | Modify:2018-01-23
8 | Author:Jack Cui
9 | '''
10 |
11 | os.environ["CUDA_VISIBLE_DEVICES"] = "0"
12 | config = tf.ConfigProto(allow_soft_placement = True)
13 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction = 0.33)
14 | config.gpu_options.allow_growth = True
15 |
16 |
17 | max_steps = 1000 # 最大迭代次数
18 | learning_rate = 0.001 # 学习率
19 | dropout = 0.9 # dropout时随机保留神经元的比例
20 | data_dir = './MNIST_DATA' # 样本数据存储的路径
21 | log_dir = './MNIST_LOG' # 输出日志保存的路径
22 |
23 | # 获取数据集，并采用采用one_hot热编码
24 | mnist = input_data.read_data_sets(data_dir,one_hot = True)
25 |
26 | sess = tf.InteractiveSession(config = config)
27 |
28 | with tf.name_scope('input'):
29 | x = tf.placeholder(tf.float32, [None, 784], name='x-input')
30 | y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')
31 |
32 | # 保存图像信息
33 | with tf.name_scope('input_reshape'):
34 | image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
35 | tf.summary.image('input', image_shaped_input, 10)
36 |
37 | # 初始化权重参数
38 | def weight_variable(shape):
39 | initial = tf.truncated_normal(shape, stddev = 0.1)
40 | return tf.Variable(initial)
41 |
42 | # 初始化偏执参数
43 | def bias_variable(shape):
44 | initial = tf.constant(0.1, shape = shape)
45 | return tf.Variable(initial)
46 |
47 | # 绘制参数变化
48 | def variable_summaries(var):
49 | with tf.name_scope('summaries'):
50 | # 计算参数的均值，并使用tf.summary.scaler记录
51 | mean = tf.reduce_mean(var)
52 | tf.summary.scalar('mean', mean)
53 |
54 | # 计算参数的标准差
55 | with tf.name_scope('stddev'):
56 | stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
57 | # 使用tf.summary.scaler记录记录下标准差，最大值，最小值
58 | tf.summary.scalar('stddev', stddev)
59 | tf.summary.scalar('max', tf.reduce_max(var))
60 | tf.summary.scalar('min', tf.reduce_min(var))
61 | # 用直方图记录参数的分布
62 | tf.summary.histogram('histogram', var)
63 |
64 | # 构建神经网络
65 | def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):
66 | # 设置命名空间
67 | with tf.name_scope(layer_name):
68 | # 调用之前的方法初始化权重w，并且调用参数信息的记录方法，记录w的信息
69 | with tf.name_scope('weights'):
70 | weights = weight_variable([input_dim, output_dim])
71 | variable_summaries(weights)
72 | # 调用之前的方法初始化权重b，并且调用参数信息的记录方法，记录b的信息
73 | with tf.name_scope('biases'):
74 | biases = bias_variable([output_dim])
75 | variable_summaries(biases)
76 | # 执行wx+b的线性计算，并且用直方图记录下来
77 | with tf.name_scope('linear_compute'):
78 | preactivate = tf.matmul(input_tensor, weights) + biases
79 | tf.summary.histogram('linear', preactivate)
80 | # 将线性输出经过激励函数，并将输出也用直方图记录下来
81 | activations = act(preactivate, name='activation')
82 | tf.summary.histogram('activations', activations)
83 | # 返回激励层的最终输出
84 | return activations
85 |
86 | hidden1 = nn_layer(x, 784, 500, 'layer1')
87 |
88 | # 创建dropout层
89 | with tf.name_scope('dropout'):
90 | keep_prob = tf.placeholder(tf.float32)
91 | tf.summary.scalar('dropout_keep_probability', keep_prob)
92 | dropped = tf.nn.dropout(hidden1, keep_prob)
93 |
94 |
95 | y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity)
96 |
97 | # 创建损失函数
98 | with tf.name_scope('loss'):
99 | # 计算交叉熵损失（每个样本都会有一个损失）
100 | diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
101 | with tf.name_scope('total'):
102 | # 计算所有样本交叉熵损失的均值
103 | cross_entropy = tf.reduce_mean(diff)
104 | tf.summary.scalar('loss', cross_entropy)
105 |
106 |
107 | # 使用AdamOptimizer优化器训练模型，最小化交叉熵损失
108 | with tf.name_scope('train'):
109 | train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
110 |
111 | # 计算准确率
112 | with tf.name_scope('accuracy'):
113 | with tf.name_scope('correct_prediction'):
114 | # 分别将预测和真实的标签中取出最大值的索引，弱相同则返回1(true),不同则返回0(false)
115 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
116 | with tf.name_scope('accuracy'):
117 | # 求均值即为准确率
118 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
119 |
120 |
121 | tf.summary.scalar('accuracy', accuracy)
122 |
123 | # summaries合并
124 | merged = tf.summary.merge_all()
125 | # 写到指定的磁盘路径中
126 | train_writer = tf.summary.FileWriter(log_dir + '/train', sess.graph)
127 | test_writer = tf.summary.FileWriter(log_dir + '/test')
128 |
129 | # 运行初始化所有变量
130 | tf.global_variables_initializer().run()
131 |
132 | def feed_dict(train):
133 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
134 | if train:
135 | xs, ys = mnist.train.next_batch(100)
136 | k = dropout
137 | else:
138 | xs, ys = mnist.test.images, mnist.test.labels
139 | k = 1.0
140 | return {x: xs, y_: ys, keep_prob: k}
141 |
142 | for i in range(max_steps):
143 | if i % 10 == 0: # 记录测试集的summary与accuracy
144 | summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False))
145 | test_writer.add_summary(summary, i)
146 | print('Accuracy at step %s: %s' % (i, acc))
147 | else: # 记录训练集的summary
148 | summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True))
149 | train_writer.add_summary(summary, i)
150 |
151 | train_writer.close()
152 | test_writer.close()
--------------------------------------------------------------------------------