3 |
2 |
3 |
10 | 
11 |
12 |
13 |
14 | 
17 | 
20 | 
23 |
27 |
28 | 【试读】本书的部分内容可在线阅读。
29 | https://www.ituring.com.cn/book/2863
30 |
31 | ## 文件夹的内容
32 |
33 | |文件夹名 |说明 |
34 | |:-- |:-- |
35 | |[dezero](/dezero) |DeZero的源代码|
36 | |[examples](/examples) |使用DeZero开发的示例|
37 | |[steps](/steps)|各步骤的代码文件（step01.py ~ step60.py）|
38 | |[tests](/tests)|DeZero的单元测试|
39 |
40 |
41 | ## 所需的外部库
42 |
43 | 本书使用的Python版本和外部库如下所示。
44 |
45 | - [Python 3](https://docs.python.org/3/)
46 | - [NumPy](https://numpy.org/)
47 | - [Matplotlib](https://matplotlib.org/)
48 |
49 | 另外DeZero还提供了可在NVIDIA的GPU上运行的可选功能。此时需要安装下面的库。
50 |
51 | - [CuPy](https://cupy.chainer.org/) （可选）
52 |
53 |
54 | ## 运行方法
55 |
56 | 本书所讲解的Python文件主要在[steps](/steps)文件夹中。
57 | 可以通过以下Python命令运行这些文件（可以在任何目录下运行Python命令）。
58 |
59 | ```
60 | $ python steps/step01.py
61 | $ python steps/step02.py
62 |
63 | $ cd steps
64 | $ python step31.py
65 | ```
66 |
67 | ## 代码示例
68 |
69 | DeZero的其他实现示例在[examples](/examples)。
70 |
71 | [
](/examples/tanh.py)[
](/examples/spiral.py)[
](/examples/mnist_colab_gpu.ipynb)
72 |
73 | [
](/examples/gan.py)[
](/examples/vae.py)[
](/examples/grad_cam.py)
74 |
75 | [
](/examples/style_transfer.py)[
](https://github.com/oreilly-japan/deep-learning-from-scratch-3/wiki/DeZero%E3%82%92iPhone%E3%81%A7%E5%8B%95%E3%81%8B%E3%81%99)
76 |
77 | ## 勘误信息
78 |
79 | 本书的勘误信息汇总在[勘误页面](https://www.ituring.com.cn/book/2863)。
80 |
81 | 如果您发现任何未列在勘误页面上的错字或其他错误，请在勘误页面[提交勘误](https://www.ituring.com.cn/book/2863)。
82 |
--------------------------------------------------------------------------------
/dezero/__init__.py:
--------------------------------------------------------------------------------
1 | # =============================================================================
2 | # 从step23.py到step32.py使用simple_core
3 | is_simple_core = False # True
4 | # =============================================================================
5 |
6 | if is_simple_core:
7 | from dezero.core_simple import Variable
8 | from dezero.core_simple import Function
9 | from dezero.core_simple import using_config
10 | from dezero.core_simple import no_grad
11 | from dezero.core_simple import as_array
12 | from dezero.core_simple import as_variable
13 | from dezero.core_simple import setup_variable
14 |
15 | else:
16 | from dezero.core import Variable
17 | from dezero.core import Parameter
18 | from dezero.core import Function
19 | from dezero.core import using_config
20 | from dezero.core import no_grad
21 | from dezero.core import test_mode
22 | from dezero.core import as_array
23 | from dezero.core import as_variable
24 | from dezero.core import setup_variable
25 | from dezero.core import Config
26 | from dezero.layers import Layer
27 | from dezero.models import Model
28 | from dezero.datasets import Dataset
29 | from dezero.dataloaders import DataLoader
30 | from dezero.dataloaders import SeqDataLoader
31 |
32 | import dezero.datasets
33 | import dezero.dataloaders
34 | import dezero.optimizers
35 | import dezero.functions
36 | import dezero.functions_conv
37 | import dezero.layers
38 | import dezero.utils
39 | import dezero.cuda
40 | import dezero.transforms
41 |
42 | setup_variable()
43 | __version__ = '0.0.13'
--------------------------------------------------------------------------------
/dezero/cuda.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | gpu_enable = True
3 | try:
4 | import cupy as cp
5 | cupy = cp
6 | except ImportError:
7 | gpu_enable = False
8 | from dezero import Variable
9 |
10 |
11 | def get_array_module(x):
12 | """Returns the array module for `x`.
13 |
14 | Args:
15 | x (dezero.Variable or numpy.ndarray or cupy.ndarray): Values to
16 | determine whether NumPy or CuPy should be used.
17 |
18 | Returns:
19 | module: `cupy` or `numpy` is returned based on the argument.
20 | """
21 | if isinstance(x, Variable):
22 | x = x.data
23 |
24 | if not gpu_enable:
25 | return np
26 | xp = cp.get_array_module(x)
27 | return xp
28 |
29 |
30 | def as_numpy(x):
31 | """Convert to `numpy.ndarray`.
32 |
33 | Args:
34 | x (`numpy.ndarray` or `cupy.ndarray`): Arbitrary object that can be
35 | converted to `numpy.ndarray`.
36 | Returns:
37 | `numpy.ndarray`: Converted array.
38 | """
39 | if isinstance(x, Variable):
40 | x = x.data
41 |
42 | if np.isscalar(x):
43 | return np.array(x)
44 | elif isinstance(x, np.ndarray):
45 | return x
46 | return cp.asnumpy(x)
47 |
48 |
49 | def as_cupy(x):
50 | """Convert to `cupy.ndarray`.
51 |
52 | Args:
53 | x (`numpy.ndarray` or `cupy.ndarray`): Arbitrary object that can be
54 | converted to `cupy.ndarray`.
55 | Returns:
56 | `cupy.ndarray`: Converted array.
57 | """
58 | if isinstance(x, Variable):
59 | x = x.data
60 |
61 | if not gpu_enable:
62 | raise Exception('CuPy cannot be loaded. Install CuPy!')
63 | return cp.asarray(x)
--------------------------------------------------------------------------------
/dezero/dataloaders.py:
--------------------------------------------------------------------------------
1 | import math
2 | pil_available = True
3 | try:
4 | from PIL import Image
5 | except:
6 | pil_available = False
7 | import numpy as np
8 | from dezero import cuda
9 |
10 |
11 | class DataLoader:
12 | def __init__(self, dataset, batch_size, shuffle=True, gpu=False):
13 | self.dataset = dataset
14 | self.batch_size = batch_size
15 | self.shuffle = shuffle
16 | self.data_size = len(dataset)
17 | self.max_iter = math.ceil(self.data_size / batch_size)
18 | self.gpu = gpu
19 |
20 | self.reset()
21 |
22 | def reset(self):
23 | self.iteration = 0
24 | if self.shuffle:
25 | self.index = np.random.permutation(len(self.dataset))
26 | else:
27 | self.index = np.arange(len(self.dataset))
28 |
29 | def __iter__(self):
30 | return self
31 |
32 | def __next__(self):
33 | if self.iteration >= self.max_iter:
34 | self.reset()
35 | raise StopIteration
36 |
37 | i, batch_size = self.iteration, self.batch_size
38 | batch_index = self.index[i * batch_size:(i + 1) * batch_size]
39 | batch = [self.dataset[i] for i in batch_index]
40 |
41 | xp = cuda.cupy if self.gpu else np
42 | x = xp.array([example[0] for example in batch])
43 | t = xp.array([example[1] for example in batch])
44 |
45 | self.iteration += 1
46 | return x, t
47 |
48 | def next(self):
49 | return self.__next__()
50 |
51 | def to_cpu(self):
52 | self.gpu = False
53 |
54 | def to_gpu(self):
55 | self.gpu = True
56 |
57 |
58 | class SeqDataLoader(DataLoader):
59 | def __init__(self, dataset, batch_size, gpu=False):
60 | super().__init__(dataset=dataset, batch_size=batch_size, shuffle=False,
61 | gpu=gpu)
62 |
63 | def __next__(self):
64 | if self.iteration >= self.max_iter:
65 | self.reset()
66 | raise StopIteration
67 |
68 | jump = self.data_size // self.batch_size
69 | batch_index = [(i * jump + self.iteration) % self.data_size for i in
70 | range(self.batch_size)]
71 | batch = [self.dataset[i] for i in batch_index]
72 |
73 | xp = cuda.cupy if self.gpu else np
74 | x = xp.array([example[0] for example in batch])
75 | t = xp.array([example[1] for example in batch])
76 |
77 | self.iteration += 1
78 | return x, t
--------------------------------------------------------------------------------
/dezero/optimizers.py:
--------------------------------------------------------------------------------
1 | import math
2 | from dezero import cuda, Parameter
3 |
4 |
5 | # =============================================================================
6 | # Optimizer (base class)
7 | # =============================================================================
8 | class Optimizer:
9 | def __init__(self):
10 | self.target = None
11 | self.hooks = []
12 |
13 | def setup(self, target):
14 | self.target = target
15 | return self
16 |
17 | def update(self):
18 | params = [p for p in self.target.params() if p.grad is not None]
19 |
20 | for f in self.hooks:
21 | f(params)
22 |
23 | for param in params:
24 | self.update_one(param)
25 |
26 | def update_one(self, param):
27 | raise NotImplementedError()
28 |
29 | def add_hook(self, f):
30 | self.hooks.append(f)
31 |
32 |
33 | # =============================================================================
34 | # Hook functions
35 | # =============================================================================
36 | class WeightDecay:
37 | def __init__(self, rate):
38 | self.rate = rate
39 |
40 | def __call__(self, params):
41 | for param in params:
42 | param.grad.data += self.rate * param.data
43 |
44 |
45 | class ClipGrad:
46 | def __init__(self, max_norm):
47 | self.max_norm = max_norm
48 |
49 | def __call__(self, params):
50 | total_norm = 0
51 | for param in params:
52 | total_norm += (param.grad.data ** 2).sum()
53 | total_norm = math.sqrt(float(total_norm))
54 |
55 | rate = self.max_norm / (total_norm + 1e-6)
56 | if rate < 1:
57 | for param in params:
58 | param.grad.data *= rate
59 |
60 |
61 | class FreezeParam:
62 | def __init__(self, *layers):
63 | self.freeze_params = []
64 | for l in layers:
65 | if isinstance(l, Parameter):
66 | self.freeze_params.append(l)
67 | else:
68 | for p in l.params():
69 | self.freeze_params.append(p)
70 |
71 | def __call__(self, params):
72 | for p in self.freeze_params:
73 | p.grad = None
74 |
75 |
76 |
77 | # =============================================================================
78 | # SGD / MomentumSGD / AdaGrad / AdaDelta / Adam
79 | # =============================================================================
80 | class SGD(Optimizer):
81 | def __init__(self, lr=0.01):
82 | super().__init__()
83 | self.lr = lr
84 |
85 | def update_one(self, param):
86 | param.data -= self.lr * param.grad.data
87 |
88 |
89 | class MomentumSGD(Optimizer):
90 | def __init__(self, lr=0.01, momentum=0.9):
91 | super().__init__()
92 | self.lr = lr
93 | self.momentum = momentum
94 | self.vs = {}
95 |
96 | def update_one(self, param):
97 | v_key = id(param)
98 | if v_key not in self.vs:
99 | xp = cuda.get_array_module(param.data)
100 | self.vs[v_key] = xp.zeros_like(param.data)
101 |
102 | v = self.vs[v_key]
103 | v *= self.momentum
104 | v -= self.lr * param.grad.data
105 | param.data += v
106 |
107 |
108 | class AdaGrad(Optimizer):
109 | def __init__(self, lr=0.001, eps=1e-8):
110 | super().__init__()
111 | self.lr = lr
112 | self.eps = eps
113 | self.hs = {}
114 |
115 | def update_one(self, param):
116 | xp = cuda.get_array_module(param.data)
117 |
118 | h_key = id(param)
119 | if h_key not in self.hs:
120 | self.hs[h_key] = xp.zeros_like(param.data)
121 |
122 | lr = self.lr
123 | eps = self.eps
124 | grad = param.grad.data
125 | h = self.hs[h_key]
126 |
127 | h += grad * grad
128 | param.data -= lr * grad / (xp.sqrt(h) + eps)
129 |
130 |
131 | class AdaDelta(Optimizer):
132 | def __init__(self, rho=0.95, eps=1e-6):
133 | super().__init__()
134 | self.rho = rho
135 | self.eps = eps
136 | self.msg = {}
137 | self.msdx = {}
138 |
139 | def update_one(self, param):
140 | xp = cuda.get_array_module(param.data)
141 |
142 | key = id(param)
143 | if key not in self.msg:
144 | self.msg[key] = xp.zeros_like(param.data)
145 | self.msdx[key] = xp.zeros_like(param.data)
146 |
147 | msg, msdx = self.msg[key], self.msdx[key]
148 | rho = self.rho
149 | eps = self.eps
150 | grad = param.grad.data
151 |
152 | msg *= rho
153 | msg += (1 - rho) * grad * grad
154 | dx = xp.sqrt((msdx + eps) / (msg + eps)) * grad
155 | msdx *= rho
156 | msdx += (1 - rho) * dx * dx
157 | param.data -= dx
158 |
159 |
160 | class Adam(Optimizer):
161 | def __init__(self, alpha=0.001, beta1=0.9, beta2=0.999, eps=1e-8):
162 | super().__init__()
163 | self.t = 0
164 | self.alpha = alpha
165 | self.beta1 = beta1
166 | self.beta2 = beta2
167 | self.eps = eps
168 | self.ms = {}
169 | self.vs = {}
170 |
171 | def update(self, *args, **kwargs):
172 | self.t += 1
173 | super().update(*args, **kwargs)
174 |
175 | @property
176 | def lr(self):
177 | fix1 = 1. - math.pow(self.beta1, self.t)
178 | fix2 = 1. - math.pow(self.beta2, self.t)
179 | return self.alpha * math.sqrt(fix2) / fix1
180 |
181 | def update_one(self, param):
182 | xp = cuda.get_array_module(param.data)
183 |
184 | key = id(param)
185 | if key not in self.ms:
186 | self.ms[key] = xp.zeros_like(param.data)
187 | self.vs[key] = xp.zeros_like(param.data)
188 |
189 | m, v = self.ms[key], self.vs[key]
190 | beta1, beta2, eps = self.beta1, self.beta2, self.eps
191 | grad = param.grad.data
192 |
193 | m += (1 - beta1) * (grad - m)
194 | v += (1 - beta2) * (grad * grad - v)
195 | param.data -= self.lr * m / (xp.sqrt(v) + eps)
--------------------------------------------------------------------------------
/dezero/transforms.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | try:
3 | import Image
4 | except ImportError:
5 | from PIL import Image
6 | from dezero.utils import pair
7 |
8 |
9 | class Compose:
10 | """Compose several transforms.
11 |
12 | Args:
13 | transforms (list): list of transforms
14 | """
15 | def __init__(self, transforms=[]):
16 | self.transforms = transforms
17 |
18 | def __call__(self, img):
19 | if not self.transforms:
20 | return img
21 | for t in self.transforms:
22 | img = t(img)
23 | return img
24 |
25 |
26 | # =============================================================================
27 | # Transforms for PIL Image
28 | # =============================================================================
29 | class Convert:
30 | def __init__(self, mode='RGB'):
31 | self.mode = mode
32 |
33 | def __call__(self, img):
34 | if self.mode == 'BGR':
35 | img = img.convert('RGB')
36 | r, g, b = img.split()
37 | img = Image.merge('RGB', (b, g, r))
38 | return img
39 | else:
40 | return img.convert(self.mode)
41 |
42 |
43 | class Resize:
44 | """Resize the input PIL image to the given size.
45 |
46 | Args:
47 | size (int or (int, int)): Desired output size
48 | mode (int): Desired interpolation.
49 | """
50 | def __init__(self, size, mode=Image.BILINEAR):
51 | self.size = pair(size)
52 | self.mode = mode
53 |
54 | def __call__(self, img):
55 | return img.resize(self.size, self.mode)
56 |
57 |
58 | class CenterCrop:
59 | """Resize the input PIL image to the given size.
60 |
61 | Args:
62 | size (int or (int, int)): Desired output size.
63 | mode (int): Desired interpolation.
64 | """
65 | def __init__(self, size):
66 | self.size = pair(size)
67 |
68 | def __call__(self, img):
69 | W, H = img.size
70 | OW, OH = self.size
71 | left = (W - OW) // 2
72 | right = W - ((W - OW) // 2 + (W - OW) % 2)
73 | up = (H - OH) // 2
74 | bottom = H - ((H - OH) // 2 + (H - OH) % 2)
75 | return img.crop((left, up, right, bottom))
76 |
77 |
78 | class ToArray:
79 | """Convert PIL Image to NumPy array."""
80 | def __init__(self, dtype=np.float32):
81 | self.dtype = dtype
82 |
83 | def __call__(self, img):
84 | if isinstance(img, np.ndarray):
85 | return img
86 | if isinstance(img, Image.Image):
87 | img = np.asarray(img)
88 | img = img.transpose(2, 0, 1)
89 | img = img.astype(self.dtype)
90 | return img
91 | else:
92 | raise TypeError
93 |
94 |
95 | class ToPIL:
96 | """Convert NumPy array to PIL Image."""
97 | def __call__(self, array):
98 | data = array.transpose(1, 2, 0)
99 | return Image.fromarray(data)
100 |
101 |
102 | class RandomHorizontalFlip:
103 | pass
104 |
105 |
106 | # =============================================================================
107 | # Transforms for NumPy ndarray
108 | # =============================================================================
109 | class Normalize:
110 | """Normalize a NumPy array with mean and standard deviation.
111 |
112 | Args:
113 | mean (float or sequence): mean for all values or sequence of means for
114 | each channel.
115 | std (float or sequence):
116 | """
117 | def __init__(self, mean=0, std=1):
118 | self.mean = mean
119 | self.std = std
120 |
121 | def __call__(self, array):
122 | mean, std = self.mean, self.std
123 |
124 | if not np.isscalar(mean):
125 | mshape = [1] * array.ndim
126 | mshape[0] = len(array) if len(self.mean) == 1 else len(self.mean)
127 | mean = np.array(self.mean, dtype=array.dtype).reshape(*mshape)
128 | if not np.isscalar(std):
129 | rshape = [1] * array.ndim
130 | rshape[0] = len(array) if len(self.std) == 1 else len(self.std)
131 | std = np.array(self.std, dtype=array.dtype).reshape(*rshape)
132 | return (array - mean) / std
133 |
134 |
135 | class Flatten:
136 | """Flatten a NumPy array.
137 | """
138 | def __call__(self, array):
139 | return array.flatten()
140 |
141 |
142 | class AsType:
143 | def __init__(self, dtype=np.float32):
144 | self.dtype = dtype
145 |
146 | def __call__(self, array):
147 | return array.astype(self.dtype)
148 |
149 |
150 | ToFloat = AsType
151 |
152 |
153 | class ToInt(AsType):
154 | def __init__(self, dtype=np.int):
155 | self.dtype = dtype
156 |
--------------------------------------------------------------------------------
/examples/gan.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 | import dezero
4 | import dezero.functions as F
5 | import dezero.layers as L
6 | from dezero import DataLoader
7 | from dezero.models import Sequential
8 | from dezero.optimizers import Adam
9 |
10 |
11 | use_gpu = dezero.cuda.gpu_enable
12 | max_epoch = 5
13 | batch_size = 128
14 | hidden_size = 62
15 |
16 | fc_channel, fc_height, fc_width = 128, 7, 7
17 |
18 | gen = Sequential(
19 | L.Linear(1024),
20 | L.BatchNorm(),
21 | F.relu,
22 | L.Linear(fc_channel * fc_height * fc_width),
23 | L.BatchNorm(),
24 | F.relu,
25 | lambda x: F.reshape(x, (-1, fc_channel, fc_height, fc_width)),
26 | L.Deconv2d(fc_channel // 2, kernel_size=4, stride=2, pad=1),
27 | L.BatchNorm(),
28 | F.relu,
29 | L.Deconv2d(1, kernel_size=4, stride=2, pad=1),
30 | F.sigmoid
31 | )
32 |
33 | dis = Sequential(
34 | L.Conv2d(64, kernel_size=4, stride=2, pad=1),
35 | F.leaky_relu,
36 | L.Conv2d(128, kernel_size=4, stride=2, pad=1),
37 | L.BatchNorm(),
38 | F.leaky_relu,
39 | F.flatten,
40 | L.Linear(1024),
41 | L.BatchNorm(),
42 | F.leaky_relu,
43 | L.Linear(1),
44 | F.sigmoid
45 | )
46 |
47 |
48 | def init_weight(dis, gen, hidden_size):
49 | # Input dummy data to initialize weights
50 | batch_size = 1
51 | z = np.random.rand(batch_size, hidden_size)
52 | fake_images = gen(z)
53 | dis(fake_images)
54 |
55 | for l in dis.layers + gen.layers:
56 | classname = l.__class__.__name__
57 | if classname.lower() in ('conv2d', 'linear', 'deconv2d'):
58 | l.W.data = 0.02 * np.random.randn(*l.W.data.shape)
59 |
60 | init_weight(dis, gen, hidden_size)
61 |
62 | opt_g = Adam(alpha=0.0002, beta1=0.5).setup(gen)
63 | opt_d = Adam(alpha=0.0002, beta1=0.5).setup(dis)
64 |
65 | transform = lambda x: (x / 255.0).astype(np.float32)
66 | train_set = dezero.datasets.MNIST(train=True, transform=transform)
67 | train_loader = DataLoader(train_set, batch_size)
68 |
69 | if use_gpu:
70 | gen.to_gpu()
71 | dis.to_gpu()
72 | train_loader.to_gpu()
73 | xp = dezero.cuda.cupy
74 | else:
75 | xp = np
76 |
77 | label_real = xp.ones(batch_size).astype(np.int)
78 | label_fake = xp.zeros(batch_size).astype(np.int)
79 | test_z = xp.random.randn(25, hidden_size).astype(np.float32)
80 |
81 |
82 | def generate_image():
83 | with dezero.test_mode():
84 | fake_images = gen(test_z)
85 |
86 | img = dezero.cuda.as_numpy(fake_images.data)
87 | plt.figure()
88 | for i in range(0, 25):
89 | ax = plt.subplot(5, 5, i+1)
90 | ax.axis('off')
91 | plt.imshow(img[i][0], 'gray')
92 | plt.show()
93 | #plt.savefig('gan_{}.png'.format(idx))
94 |
95 | for epoch in range(max_epoch):
96 | avg_loss_d = 0
97 | avg_loss_g = 0
98 | cnt = 0
99 |
100 | for x, t in train_loader:
101 | cnt += 1
102 | if len(t) != batch_size:
103 | continue
104 |
105 | # (1) Update discriminator
106 | z = xp.random.randn(batch_size, hidden_size).astype(np.float32)
107 | fake = gen(z)
108 | y_real = dis(x)
109 | y_fake = dis(fake.data)
110 | loss_d = F.binary_cross_entropy(y_real, label_real) + \
111 | F.binary_cross_entropy(y_fake, label_fake)
112 | gen.cleargrads()
113 | dis.cleargrads()
114 | loss_d.backward()
115 | opt_d.update()
116 |
117 | # (2) Update generator
118 | y_fake = dis(fake)
119 | loss_g = F.binary_cross_entropy(y_fake, label_real)
120 | gen.cleargrads()
121 | dis.cleargrads()
122 | loss_g.backward()
123 | opt_g.update()
124 |
125 | # Print loss & visualize generator
126 | avg_loss_g += loss_g.data
127 | avg_loss_d += loss_d.data
128 | interval = 100 if use_gpu else 5
129 | if cnt % interval == 0:
130 | epoch_detail = epoch + cnt / train_loader.max_iter
131 | print('epoch: {:.2f}, loss_g: {:.4f}, loss_d: {:.4f}'.format(
132 | epoch_detail, float(avg_loss_g/cnt), float(avg_loss_d/cnt)))
133 | generate_image()
--------------------------------------------------------------------------------
/examples/grad_cam.py:
--------------------------------------------------------------------------------
1 | """
2 | Simple implementation of Grad-CAM (https://arxiv.org/pdf/1610.02391.pdf)
3 | """
4 | import numpy as np
5 | from PIL import Image
6 | import cv2
7 | import dezero
8 | import dezero.functions as F
9 | from dezero.models import VGG16
10 |
11 |
12 | url = 'https://github.com/oreilly-japan/deep-learning-from-scratch-3/raw/images/zebra.jpg'
13 | img_path = dezero.utils.get_file(url)
14 | img = Image.open(img_path)
15 | img_size = img.size
16 |
17 | model = VGG16(pretrained=True)
18 | x = VGG16.preprocess(img)[np.newaxis] # preprocess for VGG
19 | y = model(x)
20 | last_conv_output = model.conv5_3.outputs[0]()
21 | predict_id = np.argmax(y.data)
22 | predict_output = y[0, predict_id]
23 |
24 | predict_output.backward(retain_grad=True)
25 | grads = last_conv_output.grad
26 | pooled_grads = F.average(grads, axis=(0, 2, 3))
27 |
28 | heatmap = last_conv_output.data[0]
29 | for c in range(heatmap.shape[0]):
30 | heatmap[c] *= pooled_grads[c].data
31 |
32 | heatmap = np.mean(heatmap, axis=0)
33 | heatmap = np.maximum(heatmap, 0)
34 | heatmap /= np.max(heatmap)
35 |
36 | # visualize the heatmap on image
37 | img = cv2.imread(img_path)
38 | heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
39 | heatmap = np.uint8(255 * heatmap)
40 | heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
41 | heatmap_on_img = heatmap * 0.4 + img
42 | cv2.imwrite('grad_cam.png', heatmap_on_img)
--------------------------------------------------------------------------------
/examples/mnist.py:
--------------------------------------------------------------------------------
1 | import dezero
2 | import dezero.functions as F
3 | from dezero import DataLoader
4 | from dezero.models import MLP
5 |
6 |
7 | max_epoch = 5
8 | batch_size = 100
9 | hidden_size = 1000
10 |
11 | train_set = dezero.datasets.MNIST(train=True)
12 | test_set = dezero.datasets.MNIST(train=False)
13 | train_loader = DataLoader(train_set, batch_size)
14 | test_loader = DataLoader(test_set, batch_size, shuffle=False)
15 |
16 | model = MLP((hidden_size, hidden_size, 10), activation=F.relu)
17 | optimizer = dezero.optimizers.Adam().setup(model)
18 | optimizer.add_hook(dezero.optimizers.WeightDecay(1e-4)) # Weight decay
19 |
20 | if dezero.cuda.gpu_enable:
21 | train_loader.to_gpu()
22 | test_loader.to_gpu()
23 | model.to_gpu()
24 |
25 | for epoch in range(max_epoch):
26 | sum_loss, sum_acc = 0, 0
27 |
28 | for x, t in train_loader:
29 | y = model(x)
30 | loss = F.softmax_cross_entropy(y, t)
31 | acc = F.accuracy(y, t)
32 | model.cleargrads()
33 | loss.backward()
34 | optimizer.update()
35 |
36 | sum_loss += float(loss.data) * len(t)
37 | sum_acc += float(acc.data) * len(t)
38 |
39 | print('epoch: {}'.format(epoch+1))
40 | print('train loss: {}, accuracy: {}'.format(
41 | sum_loss / len(train_set), sum_acc / len(train_set)))
42 |
43 | sum_loss, sum_acc = 0, 0
44 | with dezero.no_grad():
45 | for x, t in test_loader:
46 | y = model(x)
47 | loss = F.softmax_cross_entropy(y, t)
48 | acc = F.accuracy(y, t)
49 | sum_loss += float(loss.data) * len(t)
50 | sum_acc += float(acc.data) * len(t)
51 |
52 | print('test loss: {}, accuracy: {}'.format(
53 | sum_loss / len(test_set), sum_acc / len(test_set)))
--------------------------------------------------------------------------------
/examples/spiral.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 | import dezero
4 | from dezero import optimizers
5 | from dezero import Model
6 | import dezero.functions as F
7 | import dezero.layers as L
8 | from dezero import DataLoader
9 |
10 |
11 | max_epoch = 100
12 | batch_size = 30
13 | hidden_size = 10
14 | lr = 1.0
15 |
16 | train_set = dezero.datasets.Spiral(train=True)
17 | test_set = dezero.datasets.Spiral(train=False)
18 | train_loader = DataLoader(train_set, batch_size)
19 | test_loader = DataLoader(test_set, batch_size, shuffle=False)
20 |
21 | class TwoLayerNet(Model):
22 | def __init__(self, hidden_size, out_size):
23 | super().__init__()
24 | self.l1 = L.Linear(hidden_size)
25 | self.l2 = L.Linear(out_size)
26 | self.bn1 = L.BatchNorm()
27 |
28 | def forward(self, x):
29 | y = F.sigmoid(self.bn1(self.l1(x)))
30 | y = self.l2(y)
31 | return y
32 |
33 |
34 | model = TwoLayerNet(hidden_size, 3)
35 | optimizer = optimizers.SGD(lr).setup(model)
36 |
37 | for epoch in range(max_epoch):
38 | for x, t in train_loader:
39 | y = model(x)
40 | loss = F.softmax_cross_entropy(y, t)
41 | model.cleargrads()
42 | loss.backward()
43 | optimizer.update()
44 | if epoch % 10 == 0:
45 | print('loss:', loss.data)
46 |
47 | # Plot
48 | x = np.array([example[0] for example in train_set])
49 | t = np.array([example[1] for example in train_set])
50 | h = 0.001
51 | x_min, x_max = x[:, 0].min() - .1, x[:, 0].max() + .1
52 | y_min, y_max = x[:, 1].min() - .1, x[:, 1].max() + .1
53 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
54 | X = np.c_[xx.ravel(), yy.ravel()]
55 |
56 | with dezero.test_mode():
57 | score = model(X)
58 | predict_cls = np.argmax(score.data, axis=1)
59 | Z = predict_cls.reshape(xx.shape)
60 | plt.contourf(xx, yy, Z)
61 |
62 | N, CLS_NUM = 100, 3
63 | markers = ['o', 'x', '^']
64 | colors = ['orange', 'blue', 'green']
65 | for i in range(len(x)):
66 | c = t[i]
67 | plt.scatter(x[i][0], x[i][1],s=40, marker=markers[c], c=colors[c])
68 | plt.show()
--------------------------------------------------------------------------------
/examples/style_transfer.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | import numpy as np
3 | from PIL import Image
4 | import dezero
5 | import dezero.functions as F
6 | from dezero import Variable
7 | from dezero.models import VGG16
8 |
9 |
10 | use_gpu = dezero.cuda.gpu_enable
11 | lr = 5.0
12 | iterations = 2001
13 | model_input_size = (224, 224)
14 | style_weight = 1.0
15 | content_weight = 1e-4
16 | total_varitaion_weight = 1e-6
17 | content_layers = ['conv5_2']
18 | style_layers = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
19 | content_url = 'https://github.com/oreilly-japan/deep-learning-from-scratch-3/raw/images/zebra.jpg'
20 | style_url = 'https://raw.githubusercontent.com/jcjohnson/neural-style/master/examples/inputs/starry_night_google.jpg'
21 |
22 |
23 | class VGG16(VGG16):
24 | def extract(self, x):
25 | c1_1 = F.relu(self.conv1_1(x))
26 | c1_2 = F.relu(self.conv1_2(c1_1))
27 | p1 = F.average_pooling(c1_2, 2, 2)
28 | c2_1 = F.relu(self.conv2_1(p1))
29 | c2_2 = F.relu(self.conv2_2(c2_1))
30 | p2 = F.average_pooling(c2_2, 2, 2)
31 | c3_1 = F.relu(self.conv3_1(p2))
32 | c3_2 = F.relu(self.conv3_2(c3_1))
33 | c3_3 = F.relu(self.conv3_3(c3_2))
34 | p3 = F.average_pooling(c3_3, 2, 2)
35 | c4_1 = F.relu(self.conv4_1(p3))
36 | c4_2 = F.relu(self.conv4_2(c4_1))
37 | c4_3 = F.relu(self.conv4_3(c4_2))
38 | p4 = F.average_pooling(c4_3, 2, 2)
39 | c5_1 = F.relu(self.conv5_1(p4))
40 | c5_2 = F.relu(self.conv5_2(c5_1))
41 | c5_3 = F.relu(self.conv5_3(c5_2))
42 | return {'conv1_1':c1_1, 'conv1_2':c1_2, 'conv2_1':c2_1, 'conv2_2':c2_2,
43 | 'conv3_1':c3_1, 'conv3_2':c3_2, 'conv3_3':c3_3, 'conv4_1':c4_1,
44 | 'conv5_1':c5_1, 'conv5_2':c5_2, 'conv5_3':c5_3}
45 |
46 | # Setup for content & style image
47 | content_path = dezero.utils.get_file(content_url)
48 | style_path = dezero.utils.get_file(style_url)
49 | content_img = Image.open(content_path)
50 | content_size = content_img.size
51 | style_img = Image.open(style_path)
52 | content_img = VGG16.preprocess(content_img, size=model_input_size)[np.newaxis] # preprocess for VGG
53 | style_img = VGG16.preprocess(style_img, size=model_input_size)[np.newaxis]
54 | content_img, style_img = Variable(content_img), Variable(style_img)
55 |
56 | model = VGG16(pretrained=True)
57 | #gen_data = np.random.uniform(-20, 20, (1, 3, img_resize[0], img_resize[1])).astype(np.float32)
58 | gen_data = content_img.data.copy()
59 | gen_img = dezero.Parameter(gen_data)
60 | gen_model = dezero.models.Model()
61 | gen_model.param = gen_img
62 | optimizer = dezero.optimizers.AdaGrad(lr=lr).setup(gen_model)
63 |
64 | if use_gpu:
65 | model.to_gpu()
66 | gen_img.to_gpu()
67 | content_img.to_gpu()
68 | style_img.to_gpu()
69 |
70 |
71 | with dezero.no_grad():
72 | content_features = model.extract(content_img)
73 | style_features = model.extract(style_img)
74 |
75 |
76 | def deprocess_image(x, size=None):
77 | if use_gpu:
78 | x = dezero.cuda.as_numpy(x)
79 | if x.ndim == 4:
80 | x = np.squeeze(x)
81 | x = x.transpose((1,2,0))
82 | x += np.array([103.939, 116.779, 123.68])
83 | x = x[:,:,::-1] # BGR -> RGB
84 | x = np.clip(x, 0, 255).astype('uint8')
85 | img = Image.fromarray(x, mode="RGB")
86 | if size:
87 | img = img.resize(size)
88 | return img
89 |
90 |
91 | def gram_mat(x):
92 | N, C, H, W = x.shape
93 | features = x.reshape(C, -1)
94 | gram = F.matmul(features, features.T)
95 | return gram.reshape(1, C, C)
96 |
97 |
98 | def style_loss(style, comb):
99 | S = gram_mat(style)
100 | C = gram_mat(comb)
101 | N, ch, H, W = style.shape
102 | return F.mean_squared_error(S, C) / (4 * (ch * W * H)**2)
103 |
104 |
105 | def content_loss(base, comb):
106 | return F.mean_squared_error(base, comb) / 2
107 |
108 |
109 | def total_varitaion_loss(x):
110 | a = (x[:, :, :-1, :-1] - x[:, :, 1:, : -1]) ** 2
111 | b = (x[:, :, :-1, :-1] - x[:, :, : -1, 1:]) ** 2
112 | return F.sum(a + b)
113 |
114 |
115 | def loss_func(gen_features, content_features, style_features, gen_img):
116 | loss = 0
117 | # content loss
118 | for layer in content_features:
119 | loss += content_weight / len(content_layers) * \
120 | content_loss(content_features[layer], gen_features[layer])
121 | # style loss
122 | for layer in style_features:
123 | loss += style_weight / len(style_layers) * \
124 | style_loss(style_features[layer], gen_features[layer])
125 | # total variation loss
126 | loss += total_varitaion_weight * total_varitaion_loss(gen_img)
127 | return loss
128 |
129 |
130 | print_interval = 100 if use_gpu else 1
131 | for i in range(iterations):
132 | model.cleargrads()
133 | gen_img.cleargrad()
134 |
135 | gen_features = model.extract(gen_img)
136 | loss = loss_func(gen_features, content_features, style_features, gen_img)
137 | loss.backward()
138 | optimizer.update()
139 |
140 | if i % print_interval == 0:
141 | print('{} loss: {:.0f}'.format(i, float(loss.data)))
142 |
143 | if i % 100 == 0:
144 | img = deprocess_image(gen_img.data, content_size)
145 | plt.imshow(np.array(img))
146 | plt.show()
147 | #img.save("style_transfer_{}.png".format(str(i)))
--------------------------------------------------------------------------------
/examples/tanh.py:
--------------------------------------------------------------------------------
1 | '''
2 | Need the dot binary from the graphviz package (www.graphviz.org).
3 | '''
4 | import numpy as np
5 | from dezero import Variable
6 | from dezero.utils import plot_dot_graph
7 | import dezero.functions as F
8 |
9 | x = Variable(np.array(1.0))
10 | y = F.tanh(x)
11 | x.name = 'x'
12 | y.name = 'y'
13 | y.backward(create_graph=True)
14 |
15 | iters = 3
16 |
17 | for i in range(iters):
18 | gx = x.grad
19 | x.cleargrad()
20 | gx.backward(create_graph=True)
21 |
22 | gx = x.grad
23 | gx.name = 'gx' + str(iters + 1)
24 | plot_dot_graph(gx, verbose=False, to_file='tanh.png')
--------------------------------------------------------------------------------
/examples/vae.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from scipy.stats import norm
3 | import matplotlib.pyplot as plt
4 | import dezero
5 | import dezero.functions as F
6 | import dezero.layers as L
7 | from dezero import DataLoader
8 | from dezero.models import Model
9 | from dezero.optimizers import Adam
10 |
11 |
12 | use_gpu = dezero.cuda.gpu_enable
13 | max_epoch = 10
14 | batch_size = 16
15 | latent_size = 2
16 |
17 |
18 | class Encoder(Model):
19 | def __init__(self, latent_size):
20 | super().__init__()
21 | self.latent_size = latent_size
22 | self.conv1 = L.Conv2d(32, kernel_size=3, stride=1, pad=1)
23 | self.conv2 = L.Conv2d(64, kernel_size=3, stride=2, pad=1)
24 | self.conv3 = L.Conv2d(64, kernel_size=3, stride=1, pad=1)
25 | self.conv4 = L.Conv2d(64, kernel_size=3, stride=1, pad=1)
26 | self.linear1 = L.Linear(32)
27 | self.linear2 = L.Linear(latent_size)
28 | self.linear3 = L.Linear(latent_size)
29 |
30 | def forward(self, x):
31 | x = F.relu(self.conv1(x))
32 | x = F.relu(self.conv2(x))
33 | x = F.relu(self.conv3(x))
34 | x = F.relu(self.conv4(x))
35 | x = F.flatten(x)
36 | x = F.relu(self.linear1(x))
37 | z_mean = self.linear2(x)
38 | z_log_var = self.linear3(x)
39 | return z_mean, z_log_var
40 |
41 | def sampling(self, z_mean, z_log_var):
42 | batch_size = len(z_mean)
43 | xp = dezero.cuda.get_array_module(z_mean.data)
44 | epsilon = xp.random.randn(batch_size, self.latent_size)
45 | return z_mean + F.exp(z_log_var) * epsilon
46 |
47 |
48 | class Decoder(Model):
49 | def __init__(self):
50 | super().__init__()
51 | self.to_shape = (64, 14, 14) # (C, H, W)
52 | self.linear = L.Linear(np.prod(self.to_shape))
53 | self.deconv = L.Deconv2d(32, kernel_size=4, stride=2, pad=1)
54 | self.conv = L.Conv2d(1, kernel_size=3, stride=1, pad=1)
55 |
56 | def forward(self, x):
57 | x = F.relu(self.linear(x))
58 | x = F.reshape(x, (-1,) + self.to_shape) # reshape to (-1, C, H, W)
59 | x = F.relu(self.deconv(x))
60 | x = self.conv(x)
61 | x = F.sigmoid(x)
62 | return x
63 |
64 |
65 | class VAE(Model):
66 | def __init__(self, latent_size):
67 | super().__init__()
68 | self.encoder = Encoder(latent_size)
69 | self.decoder = Decoder()
70 |
71 | def forward(self, x, C=1.0, k=1):
72 | """Call loss function of VAE.
73 | The loss value is equal to ELBO (Evidence Lower Bound)
74 | multiplied by -1.
75 |
76 | Args:
77 | x (Variable or ndarray): Input variable.
78 | C (int): Usually this is 1.0. Can be changed to control the
79 | second term of ELBO bound, which works as regularization.
80 | k (int): Number of Monte Carlo samples used in encoded vector.
81 | """
82 | z_mean, z_log_var = self.encoder(x)
83 |
84 | rec_loss = 0
85 | for l in range(k):
86 | z = self.encoder.sampling(z_mean, z_log_var)
87 | y = self.decoder(z)
88 | rec_loss += F.binary_cross_entropy(F.flatten(y), F.flatten(x)) / k
89 |
90 | kl_loss = C * (z_mean ** 2 + F.exp(z_log_var) - z_log_var - 1) * 0.5
91 | kl_loss = F.sum(kl_loss) / len(x)
92 | return rec_loss + kl_loss
93 |
94 |
95 | def show_digits(epoch=0):
96 | """Display a 2D manifold of the digits"""
97 | n = 15 # 15x15 digits
98 | digit_size = 28
99 | figure = np.zeros((digit_size * n, digit_size * n))
100 | grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
101 | grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
102 |
103 | for i, yi in enumerate(grid_x):
104 | for j, xi in enumerate(grid_y):
105 | z_sample = np.array([[xi, yi]])
106 | if use_gpu:
107 | z_sample = dezero.cuda.as_cupy(z_sample)
108 | with dezero.no_grad():
109 | x_decoded = vae.decoder(z_sample)
110 | if use_gpu:
111 | x_decoded.data = dezero.cuda.as_numpy(x_decoded.data)
112 | digit = x_decoded.data.reshape(digit_size, digit_size)
113 | figure[i * digit_size: (i + 1) * digit_size,
114 | j * digit_size: (j + 1) * digit_size] = digit
115 |
116 | plt.figure(figsize=(10, 10))
117 | plt.axis('off')
118 | plt.imshow(figure, cmap='Greys_r')
119 | plt.show()
120 | #plt.savefig('vae_{}.png'.format(epoch))
121 |
122 |
123 | vae = VAE(latent_size)
124 | optimizer = Adam().setup(vae)
125 |
126 | transform = lambda x: (x / 255.0).astype(np.float32)
127 | train_set = dezero.datasets.MNIST(train=True, transform=transform)
128 | train_loader = DataLoader(train_set, batch_size)
129 |
130 | if use_gpu:
131 | vae.to_gpu()
132 | train_loader.to_gpu()
133 | xp = dezero.cuda.cupy
134 | else:
135 | xp = np
136 |
137 | for epoch in range(max_epoch):
138 | avg_loss = 0
139 | cnt = 0
140 |
141 | for x, t in train_loader:
142 | cnt += 1
143 |
144 | loss = vae(x)
145 | vae.cleargrads()
146 | loss.backward()
147 | optimizer.update()
148 |
149 | avg_loss += loss.data
150 | interval = 100 if use_gpu else 10
151 | if cnt % interval == 0:
152 | epoch_detail = epoch + cnt / train_loader.max_iter
153 | print('epoch: {:.2f}, loss: {:.4f}'.format(epoch_detail,
154 | float(avg_loss/cnt)))
155 |
156 | show_digits(epoch)
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 | from dezero import __version__
3 |
4 | setup(name='dezero',
5 | version=__version__,
6 | license='MIT License',
7 | install_requires=['numpy'],
8 | description='Deep Learning Framework from Zero',
9 | author='Koki Saitoh',
10 | author_email='koki0702@gmail.com',
11 | url='',
12 | packages=['dezero'],
13 | )
--------------------------------------------------------------------------------
/steps/step01.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 |
8 |
9 | data = np.array(1.0)
10 | x = Variable(data)
11 | print(x.data)
12 |
13 | x.data = np.array(2.0)
14 | print(x.data)
--------------------------------------------------------------------------------
/steps/step02.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 |
8 |
9 | class Function:
10 | def __call__(self, input):
11 | x = input.data
12 | y = self.forward(x)
13 | output = Variable(y)
14 | return output
15 |
16 | def forward(self, in_data):
17 | raise NotImplementedError()
18 |
19 |
20 | class Square(Function):
21 | def forward(self, x):
22 | return x ** 2
23 |
24 |
25 | x = Variable(np.array(10))
26 | f = Square()
27 | y = f(x)
28 | print(type(y))
29 | print(y.data)
--------------------------------------------------------------------------------
/steps/step03.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 |
8 |
9 | class Function:
10 | def __call__(self, input):
11 | x = input.data
12 | y = self.forward(x)
13 | output = Variable(y)
14 | return output
15 |
16 | def forward(self, x):
17 | raise NotImplementedError()
18 |
19 |
20 | class Square(Function):
21 | def forward(self, x):
22 | return x ** 2
23 |
24 |
25 | class Exp(Function):
26 | def forward(self, x):
27 | return np.exp(x)
28 |
29 |
30 | A = Square()
31 | B = Exp()
32 | C = Square()
33 |
34 | x = Variable(np.array(0.5))
35 | a = A(x)
36 | b = B(a)
37 | y = C(b)
38 | print(y.data)
--------------------------------------------------------------------------------
/steps/step04.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 |
8 |
9 | class Function:
10 | def __call__(self, input):
11 | x = input.data
12 | y = self.forward(x)
13 | output = Variable(y)
14 | self.input = input
15 | self.output = output
16 | return output
17 |
18 | def forward(self, x):
19 | raise NotImplementedError()
20 |
21 |
22 | class Square(Function):
23 | def forward(self, x):
24 | return x ** 2
25 |
26 |
27 | class Exp(Function):
28 | def forward(self, x):
29 | return np.exp(x)
30 |
31 |
32 | def numerical_diff(f, x, eps=1e-4):
33 | x0 = Variable(x.data - eps)
34 | x1 = Variable(x.data + eps)
35 | y0 = f(x0)
36 | y1 = f(x1)
37 | return (y1.data - y0.data) / (2 * eps)
38 |
39 |
40 | f = Square()
41 | x = Variable(np.array(2.0))
42 | dy = numerical_diff(f, x)
43 | print(dy)
44 |
45 |
46 | def f(x):
47 | A = Square()
48 | B = Exp()
49 | C = Square()
50 | return C(B(A(x)))
51 |
52 |
53 | x = Variable(np.array(0.5))
54 | dy = numerical_diff(f, x)
55 | print(dy)
--------------------------------------------------------------------------------
/steps/step05.py:
--------------------------------------------------------------------------------
1 | # No code
--------------------------------------------------------------------------------
/steps/step06.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 | self.grad = None
8 |
9 |
10 | class Function:
11 | def __call__(self, input):
12 | x = input.data
13 | y = self.forward(x)
14 | output = Variable(y)
15 | self.input = input
16 | return output
17 |
18 | def forward(self, x):
19 | raise NotImplementedError()
20 |
21 | def backward(self, gy):
22 | raise NotImplementedError()
23 |
24 |
25 | class Square(Function):
26 | def forward(self, x):
27 | y = x ** 2
28 | return y
29 |
30 | def backward(self, gy):
31 | x = self.input.data
32 | gx = 2 * x * gy
33 | return gx
34 |
35 |
36 | class Exp(Function):
37 | def forward(self, x):
38 | y = np.exp(x)
39 | return y
40 |
41 | def backward(self, gy):
42 | x = self.input.data
43 | gx = np.exp(x) * gy
44 | return gx
45 |
46 |
47 | A = Square()
48 | B = Exp()
49 | C = Square()
50 |
51 | x = Variable(np.array(0.5))
52 | a = A(x)
53 | b = B(a)
54 | y = C(b)
55 |
56 | y.grad = np.array(1.0)
57 | b.grad = C.backward(y.grad)
58 | a.grad = B.backward(b.grad)
59 | x.grad = A.backward(a.grad)
60 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step07.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 | self.grad = None
8 | self.creator = None
9 |
10 | def set_creator(self, func):
11 | self.creator = func
12 |
13 | def backward(self):
14 | f = self.creator # 1. Get a function
15 | if f is not None:
16 | x = f.input # 2. Get the function's input
17 | x.grad = f.backward(self.grad) # 3. Call the function's backward
18 | x.backward()
19 |
20 |
21 | class Function:
22 | def __call__(self, input):
23 | x = input.data
24 | y = self.forward(x)
25 | output = Variable(y)
26 | output.set_creator(self) # Set parent(function)
27 | self.input = input
28 | self.output = output # Set output
29 | return output
30 |
31 | def forward(self, x):
32 | raise NotImplementedError()
33 |
34 | def backward(self, gy):
35 | raise NotImplementedError()
36 |
37 |
38 | class Square(Function):
39 | def forward(self, x):
40 | y = x ** 2
41 | return y
42 |
43 | def backward(self, gy):
44 | x = self.input.data
45 | gx = 2 * x * gy
46 | return gx
47 |
48 |
49 | class Exp(Function):
50 | def forward(self, x):
51 | y = np.exp(x)
52 | return y
53 |
54 | def backward(self, gy):
55 | x = self.input.data
56 | gx = np.exp(x) * gy
57 | return gx
58 |
59 |
60 | A = Square()
61 | B = Exp()
62 | C = Square()
63 |
64 | x = Variable(np.array(0.5))
65 | a = A(x)
66 | b = B(a)
67 | y = C(b)
68 |
69 | # backward
70 | y.grad = np.array(1.0)
71 | y.backward()
72 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step08.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | self.data = data
7 | self.grad = None
8 | self.creator = None
9 |
10 | def set_creator(self, func):
11 | self.creator = func
12 |
13 | def backward(self):
14 | funcs = [self.creator]
15 | while funcs:
16 | f = funcs.pop() # 1. Get a function
17 | x, y = f.input, f.output # 2. Get the function's input/output
18 | x.grad = f.backward(y.grad) # 3. Call the function's backward
19 |
20 | if x.creator is not None:
21 | funcs.append(x.creator)
22 |
23 |
24 | class Function:
25 | def __call__(self, input):
26 | x = input.data
27 | y = self.forward(x)
28 | output = Variable(y)
29 | output.set_creator(self)
30 | self.input = input
31 | self.output = output
32 | return output
33 |
34 | def forward(self, x):
35 | raise NotImplementedError()
36 |
37 | def backward(self, gy):
38 | raise NotImplementedError()
39 |
40 |
41 | class Square(Function):
42 | def forward(self, x):
43 | y = x ** 2
44 | return y
45 |
46 | def backward(self, gy):
47 | x = self.input.data
48 | gx = 2 * x * gy
49 | return gx
50 |
51 |
52 | class Exp(Function):
53 | def forward(self, x):
54 | y = np.exp(x)
55 | return y
56 |
57 | def backward(self, gy):
58 | x = self.input.data
59 | gx = np.exp(x) * gy
60 | return gx
61 |
62 |
63 | A = Square()
64 | B = Exp()
65 | C = Square()
66 |
67 | x = Variable(np.array(0.5))
68 | a = A(x)
69 | b = B(a)
70 | y = C(b)
71 |
72 | # backward
73 | y.grad = np.array(1.0)
74 | y.backward()
75 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step09.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | if data is not None:
7 | if not isinstance(data, np.ndarray):
8 | raise TypeError('{} is not supported'.format(type(data)))
9 |
10 | self.data = data
11 | self.grad = None
12 | self.creator = None
13 |
14 | def set_creator(self, func):
15 | self.creator = func
16 |
17 | def backward(self):
18 | if self.grad is None:
19 | self.grad = np.ones_like(self.data)
20 |
21 | funcs = [self.creator]
22 | while funcs:
23 | f = funcs.pop()
24 | x, y = f.input, f.output
25 | x.grad = f.backward(y.grad)
26 |
27 | if x.creator is not None:
28 | funcs.append(x.creator)
29 |
30 |
31 | def as_array(x):
32 | if np.isscalar(x):
33 | return np.array(x)
34 | return x
35 |
36 |
37 | class Function:
38 | def __call__(self, input):
39 | x = input.data
40 | y = self.forward(x)
41 | output = Variable(as_array(y))
42 | output.set_creator(self)
43 | self.input = input
44 | self.output = output
45 | return output
46 |
47 | def forward(self, x):
48 | raise NotImplementedError()
49 |
50 | def backward(self, gy):
51 | raise NotImplementedError()
52 |
53 |
54 | class Square(Function):
55 | def forward(self, x):
56 | y = x ** 2
57 | return y
58 |
59 | def backward(self, gy):
60 | x = self.input.data
61 | gx = 2 * x * gy
62 | return gx
63 |
64 |
65 | class Exp(Function):
66 | def forward(self, x):
67 | y = np.exp(x)
68 | return y
69 |
70 | def backward(self, gy):
71 | x = self.input.data
72 | gx = np.exp(x) * gy
73 | return gx
74 |
75 |
76 | def square(x):
77 | return Square()(x)
78 |
79 |
80 | def exp(x):
81 | return Exp()(x)
82 |
83 |
84 | x = Variable(np.array(0.5))
85 | y = square(exp(square(x)))
86 | y.backward()
87 | print(x.grad)
88 |
89 |
90 | x = Variable(np.array(1.0)) # OK
91 | x = Variable(None) # OK
92 | x = Variable(1.0) # NG
--------------------------------------------------------------------------------
/steps/step10.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 |
4 |
5 | class Variable:
6 | def __init__(self, data):
7 | if data is not None:
8 | if not isinstance(data, np.ndarray):
9 | raise TypeError('{} is not supported'.format(type(data)))
10 |
11 | self.data = data
12 | self.grad = None
13 | self.creator = None
14 |
15 | def set_creator(self, func):
16 | self.creator = func
17 |
18 | def backward(self):
19 | if self.grad is None:
20 | self.grad = np.ones_like(self.data)
21 |
22 | funcs = [self.creator]
23 | while funcs:
24 | f = funcs.pop()
25 | x, y = f.input, f.output
26 | x.grad = f.backward(y.grad)
27 |
28 | if x.creator is not None:
29 | funcs.append(x.creator)
30 |
31 |
32 | def as_array(x):
33 | if np.isscalar(x):
34 | return np.array(x)
35 | return x
36 |
37 |
38 | class Function:
39 | def __call__(self, input):
40 | x = input.data
41 | y = self.forward(x)
42 | output = Variable(as_array(y))
43 | output.set_creator(self)
44 | self.input = input
45 | self.output = output
46 | return output
47 |
48 | def forward(self, x):
49 | raise NotImplementedError()
50 |
51 | def backward(self, gy):
52 | raise NotImplementedError()
53 |
54 |
55 | class Square(Function):
56 | def forward(self, x):
57 | y = x ** 2
58 | return y
59 |
60 | def backward(self, gy):
61 | x = self.input.data
62 | gx = 2 * x * gy
63 | return gx
64 |
65 |
66 | def square(x):
67 | return Square()(x)
68 |
69 |
70 | def numerical_diff(f, x, eps=1e-4):
71 | x0 = Variable(x.data - eps)
72 | x1 = Variable(x.data + eps)
73 | y0 = f(x0)
74 | y1 = f(x1)
75 | return (y1.data - y0.data) / (2 * eps)
76 |
77 |
78 | class SquareTest(unittest.TestCase):
79 | def test_forward(self):
80 | x = Variable(np.array(2.0))
81 | y = square(x)
82 | expected = np.array(4.0)
83 | self.assertEqual(y.data, expected)
84 |
85 | def test_backward(self):
86 | x = Variable(np.array(3.0))
87 | y = square(x)
88 | y.backward()
89 | expected = np.array(6.0)
90 | self.assertEqual(x.grad, expected)
91 |
92 | def test_gradient_check(self):
93 | x = Variable(np.random.rand(1))
94 | y = square(x)
95 | y.backward()
96 | num_grad = numerical_diff(square, x)
97 | flg = np.allclose(x.grad, num_grad)
98 | self.assertTrue(flg)
--------------------------------------------------------------------------------
/steps/step11.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | if data is not None:
7 | if not isinstance(data, np.ndarray):
8 | raise TypeError('{} is not supported'.format(type(data)))
9 |
10 | self.data = data
11 | self.grad = None
12 | self.creator = None
13 |
14 | def set_creator(self, func):
15 | self.creator = func
16 |
17 | def backward(self):
18 | if self.grad is None:
19 | self.grad = np.ones_like(self.data)
20 |
21 | funcs = [self.creator]
22 | while funcs:
23 | f = funcs.pop()
24 | x, y = f.input, f.output
25 | x.grad = f.backward(y.grad)
26 |
27 | if x.creator is not None:
28 | funcs.append(x.creator)
29 |
30 |
31 | def as_array(x):
32 | if np.isscalar(x):
33 | return np.array(x)
34 | return x
35 |
36 |
37 | class Function:
38 | def __call__(self, inputs):
39 | xs = [x.data for x in inputs] # Get data from Variable
40 | ys = self.forward(xs)
41 | outputs = [Variable(as_array(y)) for y in ys] # Wrap data
42 |
43 | for output in outputs:
44 | output.set_creator(self)
45 | self.inputs = inputs
46 | self.outputs = outputs
47 | return outputs
48 |
49 | def forward(self, xs):
50 | raise NotImplementedError()
51 |
52 | def backward(self, gys):
53 | raise NotImplementedError()
54 |
55 |
56 | class Add(Function):
57 | def forward(self, xs):
58 | x0, x1 = xs
59 | y = x0 + x1
60 | return (y,)
61 |
62 |
63 | xs = [Variable(np.array(2)), Variable(np.array(3))]
64 | f = Add()
65 | ys = f(xs)
66 | y = ys[0]
67 | print(y.data)
--------------------------------------------------------------------------------
/steps/step12.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | if data is not None:
7 | if not isinstance(data, np.ndarray):
8 | raise TypeError('{} is not supported'.format(type(data)))
9 |
10 | self.data = data
11 | self.grad = None
12 | self.creator = None
13 |
14 | def set_creator(self, func):
15 | self.creator = func
16 |
17 | def backward(self):
18 | if self.grad is None:
19 | self.grad = np.ones_like(self.data)
20 |
21 | funcs = [self.creator]
22 | while funcs:
23 | f = funcs.pop()
24 | x, y = f.input, f.output
25 | x.grad = f.backward(y.grad)
26 |
27 | if x.creator is not None:
28 | funcs.append(x.creator)
29 |
30 |
31 | def as_array(x):
32 | if np.isscalar(x):
33 | return np.array(x)
34 | return x
35 |
36 |
37 | class Function:
38 | def __call__(self, *inputs):
39 | xs = [x.data for x in inputs]
40 | ys = self.forward(*xs)
41 | if not isinstance(ys, tuple):
42 | ys = (ys,)
43 | outputs = [Variable(as_array(y)) for y in ys]
44 |
45 | for output in outputs:
46 | output.set_creator(self)
47 | self.inputs = inputs
48 | self.outputs = outputs
49 | return outputs if len(outputs) > 1 else outputs[0]
50 |
51 | def forward(self, xs):
52 | raise NotImplementedError()
53 |
54 | def backward(self, gys):
55 | raise NotImplementedError()
56 |
57 |
58 | class Add(Function):
59 | def forward(self, x0, x1):
60 | y = x0 + x1
61 | return y
62 |
63 |
64 | def add(x0, x1):
65 | return Add()(x0, x1)
66 |
67 |
68 | x0 = Variable(np.array(2))
69 | x1 = Variable(np.array(3))
70 | y = add(x0, x1)
71 | print(y.data)
--------------------------------------------------------------------------------
/steps/step13.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | if data is not None:
7 | if not isinstance(data, np.ndarray):
8 | raise TypeError('{} is not supported'.format(type(data)))
9 |
10 | self.data = data
11 | self.grad = None
12 | self.creator = None
13 |
14 | def set_creator(self, func):
15 | self.creator = func
16 |
17 | def backward(self):
18 | if self.grad is None:
19 | self.grad = np.ones_like(self.data)
20 |
21 | funcs = [self.creator]
22 | while funcs:
23 | f = funcs.pop()
24 | gys = [output.grad for output in f.outputs]
25 | gxs = f.backward(*gys)
26 | if not isinstance(gxs, tuple):
27 | gxs = (gxs,)
28 |
29 | for x, gx in zip(f.inputs, gxs):
30 | x.grad = gx
31 |
32 | if x.creator is not None:
33 | funcs.append(x.creator)
34 |
35 |
36 | def as_array(x):
37 | if np.isscalar(x):
38 | return np.array(x)
39 | return x
40 |
41 |
42 | class Function:
43 | def __call__(self, *inputs):
44 | xs = [x.data for x in inputs]
45 | ys = self.forward(*xs)
46 | if not isinstance(ys, tuple):
47 | ys = (ys,)
48 | outputs = [Variable(as_array(y)) for y in ys]
49 |
50 | for output in outputs:
51 | output.set_creator(self)
52 | self.inputs = inputs
53 | self.outputs = outputs
54 | return outputs if len(outputs) > 1 else outputs[0]
55 |
56 | def forward(self, xs):
57 | raise NotImplementedError()
58 |
59 | def backward(self, gys):
60 | raise NotImplementedError()
61 |
62 |
63 | class Square(Function):
64 | def forward(self, x):
65 | y = x ** 2
66 | return y
67 |
68 | def backward(self, gy):
69 | x = self.inputs[0].data
70 | gx = 2 * x * gy
71 | return gx
72 |
73 |
74 | def square(x):
75 | f = Square()
76 | return f(x)
77 |
78 |
79 | class Add(Function):
80 | def forward(self, x0, x1):
81 | y = x0 + x1
82 | return y
83 |
84 | def backward(self, gy):
85 | return gy, gy
86 |
87 |
88 | def add(x0, x1):
89 | return Add()(x0, x1)
90 |
91 |
92 | x = Variable(np.array(2.0))
93 | y = Variable(np.array(3.0))
94 |
95 | z = add(square(x), square(y))
96 | z.backward()
97 | print(z.data)
98 | print(x.grad)
99 | print(y.grad)
--------------------------------------------------------------------------------
/steps/step14.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | if data is not None:
7 | if not isinstance(data, np.ndarray):
8 | raise TypeError('{} is not supported'.format(type(data)))
9 |
10 | self.data = data
11 | self.grad = None
12 | self.creator = None
13 |
14 | def set_creator(self, func):
15 | self.creator = func
16 |
17 | def cleargrad(self):
18 | self.grad = None
19 |
20 | def backward(self):
21 | if self.grad is None:
22 | self.grad = np.ones_like(self.data)
23 |
24 | funcs = [self.creator]
25 | while funcs:
26 | f = funcs.pop()
27 | gys = [output.grad for output in f.outputs]
28 | gxs = f.backward(*gys)
29 | if not isinstance(gxs, tuple):
30 | gxs = (gxs,)
31 |
32 | for x, gx in zip(f.inputs, gxs):
33 | if x.grad is None:
34 | x.grad = gx
35 | else:
36 | x.grad = x.grad + gx
37 |
38 | if x.creator is not None:
39 | funcs.append(x.creator)
40 |
41 |
42 | def as_array(x):
43 | if np.isscalar(x):
44 | return np.array(x)
45 | return x
46 |
47 |
48 | class Function:
49 | def __call__(self, *inputs):
50 | xs = [x.data for x in inputs]
51 | ys = self.forward(*xs)
52 | if not isinstance(ys, tuple):
53 | ys = (ys,)
54 | outputs = [Variable(as_array(y)) for y in ys]
55 |
56 | for output in outputs:
57 | output.set_creator(self)
58 | self.inputs = inputs
59 | self.outputs = outputs
60 | return outputs if len(outputs) > 1 else outputs[0]
61 |
62 | def forward(self, xs):
63 | raise NotImplementedError()
64 |
65 | def backward(self, gys):
66 | raise NotImplementedError()
67 |
68 |
69 | class Add(Function):
70 | def forward(self, x0, x1):
71 | y = x0 + x1
72 | return y
73 |
74 | def backward(self, gy):
75 | return gy, gy
76 |
77 |
78 | def add(x0, x1):
79 | return Add()(x0, x1)
80 |
81 |
82 | x = Variable(np.array(3.0))
83 | y = add(x, x)
84 | y.backward()
85 | print(x.grad)
86 |
87 |
88 | x = Variable(np.array(3.0)) # or x.cleargrad()
89 | y = add(add(x, x), x)
90 | y.backward()
91 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step15.py:
--------------------------------------------------------------------------------
1 | # No code
2 |
--------------------------------------------------------------------------------
/steps/step16.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class Variable:
5 | def __init__(self, data):
6 | if data is not None:
7 | if not isinstance(data, np.ndarray):
8 | raise TypeError('{} is not supported'.format(type(data)))
9 |
10 | self.data = data
11 | self.grad = None
12 | self.creator = None
13 | self.generation = 0
14 |
15 | def set_creator(self, func):
16 | self.creator = func
17 | self.generation = func.generation + 1
18 |
19 | def cleargrad(self):
20 | self.grad = None
21 |
22 | def backward(self):
23 | if self.grad is None:
24 | self.grad = np.ones_like(self.data)
25 |
26 | funcs = []
27 | seen_set = set()
28 |
29 | def add_func(f):
30 | if f not in seen_set:
31 | funcs.append(f)
32 | seen_set.add(f)
33 | funcs.sort(key=lambda x: x.generation)
34 |
35 | add_func(self.creator)
36 |
37 | while funcs:
38 | f = funcs.pop()
39 | gys = [output.grad for output in f.outputs]
40 | gxs = f.backward(*gys)
41 | if not isinstance(gxs, tuple):
42 | gxs = (gxs,)
43 |
44 | for x, gx in zip(f.inputs, gxs):
45 | if x.grad is None:
46 | x.grad = gx
47 | else:
48 | x.grad = x.grad + gx
49 |
50 | if x.creator is not None:
51 | add_func(x.creator)
52 |
53 |
54 | def as_array(x):
55 | if np.isscalar(x):
56 | return np.array(x)
57 | return x
58 |
59 |
60 | class Function:
61 | def __call__(self, *inputs):
62 | xs = [x.data for x in inputs]
63 | ys = self.forward(*xs)
64 | if not isinstance(ys, tuple):
65 | ys = (ys,)
66 | outputs = [Variable(as_array(y)) for y in ys]
67 |
68 | self.generation = max([x.generation for x in inputs])
69 | for output in outputs:
70 | output.set_creator(self)
71 | self.inputs = inputs
72 | self.outputs = outputs
73 | return outputs if len(outputs) > 1 else outputs[0]
74 |
75 | def forward(self, xs):
76 | raise NotImplementedError()
77 |
78 | def backward(self, gys):
79 | raise NotImplementedError()
80 |
81 |
82 | class Square(Function):
83 | def forward(self, x):
84 | y = x ** 2
85 | return y
86 |
87 | def backward(self, gy):
88 | x = self.inputs[0].data
89 | gx = 2 * x * gy
90 | return gx
91 |
92 |
93 | def square(x):
94 | return Square()(x)
95 |
96 |
97 | class Add(Function):
98 | def forward(self, x0, x1):
99 | y = x0 + x1
100 | return y
101 |
102 | def backward(self, gy):
103 | return gy, gy
104 |
105 |
106 | def add(x0, x1):
107 | return Add()(x0, x1)
108 |
109 |
110 | x = Variable(np.array(2.0))
111 | a = square(x)
112 | y = add(square(a), square(a))
113 | y.backward()
114 |
115 | print(y.data)
116 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step17.py:
--------------------------------------------------------------------------------
1 | import weakref
2 | import numpy as np
3 |
4 |
5 | class Variable:
6 | def __init__(self, data):
7 | if data is not None:
8 | if not isinstance(data, np.ndarray):
9 | raise TypeError('{} is not supported'.format(type(data)))
10 |
11 | self.data = data
12 | self.grad = None
13 | self.creator = None
14 | self.generation = 0
15 |
16 | def set_creator(self, func):
17 | self.creator = func
18 | self.generation = func.generation + 1
19 |
20 | def cleargrad(self):
21 | self.grad = None
22 |
23 | def backward(self):
24 | if self.grad is None:
25 | self.grad = np.ones_like(self.data)
26 |
27 | funcs = []
28 | seen_set = set()
29 |
30 | def add_func(f):
31 | if f not in seen_set:
32 | funcs.append(f)
33 | seen_set.add(f)
34 | funcs.sort(key=lambda x: x.generation)
35 |
36 | add_func(self.creator)
37 |
38 | while funcs:
39 | f = funcs.pop()
40 | gys = [output().grad for output in f.outputs] # output is weakref
41 | gxs = f.backward(*gys)
42 | if not isinstance(gxs, tuple):
43 | gxs = (gxs,)
44 |
45 | for x, gx in zip(f.inputs, gxs):
46 | if x.grad is None:
47 | x.grad = gx
48 | else:
49 | x.grad = x.grad + gx
50 |
51 | if x.creator is not None:
52 | add_func(x.creator)
53 |
54 |
55 | def as_array(x):
56 | if np.isscalar(x):
57 | return np.array(x)
58 | return x
59 |
60 |
61 | class Function:
62 | def __call__(self, *inputs):
63 | xs = [x.data for x in inputs]
64 | ys = self.forward(*xs)
65 | if not isinstance(ys, tuple):
66 | ys = (ys,)
67 | outputs = [Variable(as_array(y)) for y in ys]
68 |
69 | self.generation = max([x.generation for x in inputs])
70 | for output in outputs:
71 | output.set_creator(self)
72 | self.inputs = inputs
73 | self.outputs = [weakref.ref(output) for output in outputs]
74 | return outputs if len(outputs) > 1 else outputs[0]
75 |
76 | def forward(self, xs):
77 | raise NotImplementedError()
78 |
79 | def backward(self, gys):
80 | raise NotImplementedError()
81 |
82 |
83 | class Square(Function):
84 | def forward(self, x):
85 | y = x ** 2
86 | return y
87 |
88 | def backward(self, gy):
89 | x = self.inputs[0].data
90 | gx = 2 * x * gy
91 | return gx
92 |
93 |
94 | def square(x):
95 | return Square()(x)
96 |
97 |
98 | for i in range(10):
99 | x = Variable(np.random.randn(10000)) # big data
100 | y = square(square(square(x)))
--------------------------------------------------------------------------------
/steps/step18.py:
--------------------------------------------------------------------------------
1 | import weakref
2 | import numpy as np
3 | import contextlib
4 |
5 |
6 | class Config:
7 | enable_backprop = True
8 |
9 |
10 | @contextlib.contextmanager
11 | def using_config(name, value):
12 | old_value = getattr(Config, name)
13 | setattr(Config, name, value)
14 | try:
15 | yield
16 | finally:
17 | setattr(Config, name, old_value)
18 |
19 |
20 | def no_grad():
21 | return using_config('enable_backprop', False)
22 |
23 |
24 | class Variable:
25 | def __init__(self, data):
26 | if data is not None:
27 | if not isinstance(data, np.ndarray):
28 | raise TypeError('{} is not supported'.format(type(data)))
29 |
30 | self.data = data
31 | self.grad = None
32 | self.creator = None
33 | self.generation = 0
34 |
35 | def set_creator(self, func):
36 | self.creator = func
37 | self.generation = func.generation + 1
38 |
39 | def cleargrad(self):
40 | self.grad = None
41 |
42 | def backward(self, retain_grad=False):
43 | if self.grad is None:
44 | self.grad = np.ones_like(self.data)
45 |
46 | funcs = []
47 | seen_set = set()
48 |
49 | def add_func(f):
50 | if f not in seen_set:
51 | funcs.append(f)
52 | seen_set.add(f)
53 | funcs.sort(key=lambda x: x.generation)
54 |
55 | add_func(self.creator)
56 |
57 | while funcs:
58 | f = funcs.pop()
59 | gys = [output().grad for output in f.outputs] # output is weakref
60 | gxs = f.backward(*gys)
61 | if not isinstance(gxs, tuple):
62 | gxs = (gxs,)
63 |
64 | for x, gx in zip(f.inputs, gxs):
65 | if x.grad is None:
66 | x.grad = gx
67 | else:
68 | x.grad = x.grad + gx
69 |
70 | if x.creator is not None:
71 | add_func(x.creator)
72 |
73 | if not retain_grad:
74 | for y in f.outputs:
75 | y().grad = None # y is weakref
76 |
77 |
78 | def as_array(x):
79 | if np.isscalar(x):
80 | return np.array(x)
81 | return x
82 |
83 |
84 | class Function:
85 | def __call__(self, *inputs):
86 | xs = [x.data for x in inputs]
87 | ys = self.forward(*xs)
88 | if not isinstance(ys, tuple):
89 | ys = (ys,)
90 | outputs = [Variable(as_array(y)) for y in ys]
91 |
92 | if Config.enable_backprop:
93 | self.generation = max([x.generation for x in inputs])
94 | for output in outputs:
95 | output.set_creator(self)
96 | self.inputs = inputs
97 | self.outputs = [weakref.ref(output) for output in outputs]
98 |
99 | return outputs if len(outputs) > 1 else outputs[0]
100 |
101 | def forward(self, xs):
102 | raise NotImplementedError()
103 |
104 | def backward(self, gys):
105 | raise NotImplementedError()
106 |
107 |
108 | class Square(Function):
109 | def forward(self, x):
110 | y = x ** 2
111 | return y
112 |
113 | def backward(self, gy):
114 | x = self.inputs[0].data
115 | gx = 2 * x * gy
116 | return gx
117 |
118 |
119 | def square(x):
120 | return Square()(x)
121 |
122 |
123 | class Add(Function):
124 | def forward(self, x0, x1):
125 | y = x0 + x1
126 | return y
127 |
128 | def backward(self, gy):
129 | return gy, gy
130 |
131 |
132 | def add(x0, x1):
133 | return Add()(x0, x1)
134 |
135 |
136 | x0 = Variable(np.array(1.0))
137 | x1 = Variable(np.array(1.0))
138 | t = add(x0, x1)
139 | y = add(x0, t)
140 | y.backward()
141 | print(y.grad, t.grad) # None None
142 | print(x0.grad, x1.grad) # 2.0 1.0
143 |
144 |
145 | with using_config('enable_backprop', False):
146 | x = Variable(np.array(2.0))
147 | y = square(x)
148 |
149 | with no_grad():
150 | x = Variable(np.array(2.0))
151 | y = square(x)
--------------------------------------------------------------------------------
/steps/step19.py:
--------------------------------------------------------------------------------
1 | import weakref
2 | import numpy as np
3 | import contextlib
4 |
5 |
6 | class Config:
7 | enable_backprop = True
8 |
9 |
10 | @contextlib.contextmanager
11 | def using_config(name, value):
12 | old_value = getattr(Config, name)
13 | setattr(Config, name, value)
14 | try:
15 | yield
16 | finally:
17 | setattr(Config, name, old_value)
18 |
19 |
20 | def no_grad():
21 | return using_config('enable_backprop', False)
22 |
23 |
24 | class Variable:
25 | def __init__(self, data, name=None):
26 | if data is not None:
27 | if not isinstance(data, np.ndarray):
28 | raise TypeError('{} is not supported'.format(type(data)))
29 |
30 | self.data = data
31 | self.name = name
32 | self.grad = None
33 | self.creator = None
34 | self.generation = 0
35 |
36 | @property
37 | def shape(self):
38 | return self.data.shape
39 |
40 | @property
41 | def ndim(self):
42 | return self.data.ndim
43 |
44 | @property
45 | def size(self):
46 | return self.data.size
47 |
48 | @property
49 | def dtype(self):
50 | return self.data.dtype
51 |
52 | def __len__(self):
53 | return len(self.data)
54 |
55 | def __repr__(self):
56 | if self.data is None:
57 | return 'variable(None)'
58 | p = str(self.data).replace('\n', '\n' + ' ' * 9)
59 | return 'variable(' + p + ')'
60 |
61 | def set_creator(self, func):
62 | self.creator = func
63 | self.generation = func.generation + 1
64 |
65 | def cleargrad(self):
66 | self.grad = None
67 |
68 | def backward(self, retain_grad=False):
69 | if self.grad is None:
70 | self.grad = np.ones_like(self.data)
71 |
72 | funcs = []
73 | seen_set = set()
74 |
75 | def add_func(f):
76 | if f not in seen_set:
77 | funcs.append(f)
78 | seen_set.add(f)
79 | funcs.sort(key=lambda x: x.generation)
80 |
81 | add_func(self.creator)
82 |
83 | while funcs:
84 | f = funcs.pop()
85 | gys = [output().grad for output in f.outputs] # output is weakref
86 | gxs = f.backward(*gys)
87 | if not isinstance(gxs, tuple):
88 | gxs = (gxs,)
89 |
90 | for x, gx in zip(f.inputs, gxs):
91 | if x.grad is None:
92 | x.grad = gx
93 | else:
94 | x.grad = x.grad + gx
95 |
96 | if x.creator is not None:
97 | add_func(x.creator)
98 |
99 | if not retain_grad:
100 | for y in f.outputs:
101 | y().grad = None # y is weakref
102 |
103 |
104 | def as_array(x):
105 | if np.isscalar(x):
106 | return np.array(x)
107 | return x
108 |
109 |
110 | class Function:
111 | def __call__(self, *inputs):
112 | xs = [x.data for x in inputs]
113 | ys = self.forward(*xs)
114 | if not isinstance(ys, tuple):
115 | ys = (ys,)
116 | outputs = [Variable(as_array(y)) for y in ys]
117 |
118 | if Config.enable_backprop:
119 | self.generation = max([x.generation for x in inputs])
120 | for output in outputs:
121 | output.set_creator(self)
122 | self.inputs = inputs
123 | self.outputs = [weakref.ref(output) for output in outputs]
124 |
125 | return outputs if len(outputs) > 1 else outputs[0]
126 |
127 | def forward(self, xs):
128 | raise NotImplementedError()
129 |
130 | def backward(self, gys):
131 | raise NotImplementedError()
132 |
133 |
134 | class Square(Function):
135 | def forward(self, x):
136 | y = x ** 2
137 | return y
138 |
139 | def backward(self, gy):
140 | x = self.inputs[0].data
141 | gx = 2 * x * gy
142 | return gx
143 |
144 |
145 | def square(x):
146 | return Square()(x)
147 |
148 |
149 | class Add(Function):
150 | def forward(self, x0, x1):
151 | y = x0 + x1
152 | return y
153 |
154 | def backward(self, gy):
155 | return gy, gy
156 |
157 |
158 | def add(x0, x1):
159 | return Add()(x0, x1)
160 |
161 |
162 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
163 | x.name = 'x'
164 |
165 | print(x.name)
166 | print(x.shape)
167 | print(x)
--------------------------------------------------------------------------------
/steps/step20.py:
--------------------------------------------------------------------------------
1 | import weakref
2 | import numpy as np
3 | import contextlib
4 |
5 |
6 | class Config:
7 | enable_backprop = True
8 |
9 |
10 | @contextlib.contextmanager
11 | def using_config(name, value):
12 | old_value = getattr(Config, name)
13 | setattr(Config, name, value)
14 | try:
15 | yield
16 | finally:
17 | setattr(Config, name, old_value)
18 |
19 |
20 | def no_grad():
21 | return using_config('enable_backprop', False)
22 |
23 |
24 | class Variable:
25 | def __init__(self, data, name=None):
26 | if data is not None:
27 | if not isinstance(data, np.ndarray):
28 | raise TypeError('{} is not supported'.format(type(data)))
29 |
30 | self.data = data
31 | self.name = name
32 | self.grad = None
33 | self.creator = None
34 | self.generation = 0
35 |
36 | @property
37 | def shape(self):
38 | return self.data.shape
39 |
40 | @property
41 | def ndim(self):
42 | return self.data.ndim
43 |
44 | @property
45 | def size(self):
46 | return self.data.size
47 |
48 | @property
49 | def dtype(self):
50 | return self.data.dtype
51 |
52 | def __len__(self):
53 | return len(self.data)
54 |
55 | def __repr__(self):
56 | if self.data is None:
57 | return 'variable(None)'
58 | p = str(self.data).replace('\n', '\n' + ' ' * 9)
59 | return 'variable(' + p + ')'
60 |
61 | def set_creator(self, func):
62 | self.creator = func
63 | self.generation = func.generation + 1
64 |
65 | def cleargrad(self):
66 | self.grad = None
67 |
68 | def backward(self, retain_grad=False):
69 | if self.grad is None:
70 | self.grad = np.ones_like(self.data)
71 |
72 | funcs = []
73 | seen_set = set()
74 |
75 | def add_func(f):
76 | if f not in seen_set:
77 | funcs.append(f)
78 | seen_set.add(f)
79 | funcs.sort(key=lambda x: x.generation)
80 |
81 | add_func(self.creator)
82 |
83 | while funcs:
84 | f = funcs.pop()
85 | gys = [output().grad for output in f.outputs] # output is weakref
86 | gxs = f.backward(*gys)
87 | if not isinstance(gxs, tuple):
88 | gxs = (gxs,)
89 |
90 | for x, gx in zip(f.inputs, gxs):
91 | if x.grad is None:
92 | x.grad = gx
93 | else:
94 | x.grad = x.grad + gx
95 |
96 | if x.creator is not None:
97 | add_func(x.creator)
98 |
99 | if not retain_grad:
100 | for y in f.outputs:
101 | y().grad = None # y is weakref
102 |
103 |
104 | def as_array(x):
105 | if np.isscalar(x):
106 | return np.array(x)
107 | return x
108 |
109 |
110 | class Function:
111 | def __call__(self, *inputs):
112 | xs = [x.data for x in inputs]
113 | ys = self.forward(*xs)
114 | if not isinstance(ys, tuple):
115 | ys = (ys,)
116 | outputs = [Variable(as_array(y)) for y in ys]
117 |
118 | if Config.enable_backprop:
119 | self.generation = max([x.generation for x in inputs])
120 | for output in outputs:
121 | output.set_creator(self)
122 | self.inputs = inputs
123 | self.outputs = [weakref.ref(output) for output in outputs]
124 |
125 | return outputs if len(outputs) > 1 else outputs[0]
126 |
127 | def forward(self, xs):
128 | raise NotImplementedError()
129 |
130 | def backward(self, gys):
131 | raise NotImplementedError()
132 |
133 |
134 | class Add(Function):
135 | def forward(self, x0, x1):
136 | y = x0 + x1
137 | return y
138 |
139 | def backward(self, gy):
140 | return gy, gy
141 |
142 |
143 | def add(x0, x1):
144 | return Add()(x0, x1)
145 |
146 |
147 | class Mul(Function):
148 | def forward(self, x0, x1):
149 | y = x0 * x1
150 | return y
151 |
152 | def backward(self, gy):
153 | x0, x1 = self.inputs[0].data, self.inputs[1].data
154 | return gy * x1, gy * x0
155 |
156 |
157 | def mul(x0, x1):
158 | return Mul()(x0, x1)
159 |
160 |
161 | Variable.__add__ = add
162 | Variable.__mul__ = mul
163 |
164 | a = Variable(np.array(3.0))
165 | b = Variable(np.array(2.0))
166 | c = Variable(np.array(1.0))
167 |
168 | # y = add(mul(a, b), c)
169 | y = a * b + c
170 | y.backward()
171 |
172 | print(y)
173 | print(a.grad)
174 | print(b.grad)
--------------------------------------------------------------------------------
/steps/step21.py:
--------------------------------------------------------------------------------
1 | import weakref
2 | import numpy as np
3 | import contextlib
4 |
5 |
6 | class Config:
7 | enable_backprop = True
8 |
9 |
10 | @contextlib.contextmanager
11 | def using_config(name, value):
12 | old_value = getattr(Config, name)
13 | setattr(Config, name, value)
14 | try:
15 | yield
16 | finally:
17 | setattr(Config, name, old_value)
18 |
19 |
20 | def no_grad():
21 | return using_config('enable_backprop', False)
22 |
23 |
24 | class Variable:
25 | __array_priority__ = 200
26 |
27 | def __init__(self, data, name=None):
28 | if data is not None:
29 | if not isinstance(data, np.ndarray):
30 | raise TypeError('{} is not supported'.format(type(data)))
31 |
32 | self.data = data
33 | self.name = name
34 | self.grad = None
35 | self.creator = None
36 | self.generation = 0
37 |
38 | @property
39 | def shape(self):
40 | return self.data.shape
41 |
42 | @property
43 | def ndim(self):
44 | return self.data.ndim
45 |
46 | @property
47 | def size(self):
48 | return self.data.size
49 |
50 | @property
51 | def dtype(self):
52 | return self.data.dtype
53 |
54 | def __len__(self):
55 | return len(self.data)
56 |
57 | def __repr__(self):
58 | if self.data is None:
59 | return 'variable(None)'
60 | p = str(self.data).replace('\n', '\n' + ' ' * 9)
61 | return 'variable(' + p + ')'
62 |
63 | def set_creator(self, func):
64 | self.creator = func
65 | self.generation = func.generation + 1
66 |
67 | def cleargrad(self):
68 | self.grad = None
69 |
70 | def backward(self, retain_grad=False):
71 | if self.grad is None:
72 | self.grad = np.ones_like(self.data)
73 |
74 | funcs = []
75 | seen_set = set()
76 |
77 | def add_func(f):
78 | if f not in seen_set:
79 | funcs.append(f)
80 | seen_set.add(f)
81 | funcs.sort(key=lambda x: x.generation)
82 |
83 | add_func(self.creator)
84 |
85 | while funcs:
86 | f = funcs.pop()
87 | gys = [output().grad for output in f.outputs] # output is weakref
88 | gxs = f.backward(*gys)
89 | if not isinstance(gxs, tuple):
90 | gxs = (gxs,)
91 |
92 | for x, gx in zip(f.inputs, gxs):
93 | if x.grad is None:
94 | x.grad = gx
95 | else:
96 | x.grad = x.grad + gx
97 |
98 | if x.creator is not None:
99 | add_func(x.creator)
100 |
101 | if not retain_grad:
102 | for y in f.outputs:
103 | y().grad = None # y is weakref
104 |
105 |
106 | def as_variable(obj):
107 | if isinstance(obj, Variable):
108 | return obj
109 | return Variable(obj)
110 |
111 |
112 | def as_array(x):
113 | if np.isscalar(x):
114 | return np.array(x)
115 | return x
116 |
117 |
118 | class Function:
119 | def __call__(self, *inputs):
120 | inputs = [as_variable(x) for x in inputs]
121 |
122 | xs = [x.data for x in inputs]
123 | ys = self.forward(*xs)
124 | if not isinstance(ys, tuple):
125 | ys = (ys,)
126 | outputs = [Variable(as_array(y)) for y in ys]
127 |
128 | if Config.enable_backprop:
129 | self.generation = max([x.generation for x in inputs])
130 | for output in outputs:
131 | output.set_creator(self)
132 | self.inputs = inputs
133 | self.outputs = [weakref.ref(output) for output in outputs]
134 |
135 | return outputs if len(outputs) > 1 else outputs[0]
136 |
137 | def forward(self, xs):
138 | raise NotImplementedError()
139 |
140 | def backward(self, gys):
141 | raise NotImplementedError()
142 |
143 |
144 | class Add(Function):
145 | def forward(self, x0, x1):
146 | y = x0 + x1
147 | return y
148 |
149 | def backward(self, gy):
150 | return gy, gy
151 |
152 |
153 | def add(x0, x1):
154 | x1 = as_array(x1)
155 | return Add()(x0, x1)
156 |
157 |
158 | class Mul(Function):
159 | def forward(self, x0, x1):
160 | y = x0 * x1
161 | return y
162 |
163 | def backward(self, gy):
164 | x0, x1 = self.inputs[0].data, self.inputs[1].data
165 | return gy * x1, gy * x0
166 |
167 |
168 | def mul(x0, x1):
169 | x1 = as_array(x1)
170 | return Mul()(x0, x1)
171 |
172 |
173 | Variable.__add__ = add
174 | Variable.__radd__ = add
175 | Variable.__mul__ = mul
176 | Variable.__rmul__ = mul
177 |
178 | x = Variable(np.array(2.0))
179 | y = x + np.array(3.0)
180 | print(y)
181 |
182 | y = x + 3.0
183 | print(y)
184 |
185 | y = 3.0 * x + 1.0
186 | print(y)
--------------------------------------------------------------------------------
/steps/step23.py:
--------------------------------------------------------------------------------
1 | # Add import path for the dezero directory.
2 | if '__file__' in globals():
3 | import os, sys
4 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
5 |
6 | import numpy as np
7 | from dezero import Variable
8 |
9 |
10 | x = Variable(np.array(1.0))
11 | y = (x + 3) ** 2
12 | y.backward()
13 |
14 | print(y)
15 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step24.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 |
7 |
8 | def sphere(x, y):
9 | z = x ** 2 + y ** 2
10 | return z
11 |
12 |
13 | def matyas(x, y):
14 | z = 0.26 * (x ** 2 + y ** 2) - 0.48 * x * y
15 | return z
16 |
17 |
18 | def goldstein(x, y):
19 | z = (1 + (x + y + 1)**2 * (19 - 14*x + 3*x**2 - 14*y + 6*x*y + 3*y**2)) * \
20 | (30 + (2*x - 3*y)**2 * (18 - 32*x + 12*x**2 + 48*y - 36*x*y + 27*y**2))
21 | return z
22 |
23 |
24 | x = Variable(np.array(1.0))
25 | y = Variable(np.array(1.0))
26 | z = goldstein(x, y) # sphere(x, y) / matyas(x, y)
27 | z.backward()
28 | print(x.grad, y.grad)
--------------------------------------------------------------------------------
/steps/step25.py:
--------------------------------------------------------------------------------
1 | # No code
--------------------------------------------------------------------------------
/steps/step26.py:
--------------------------------------------------------------------------------
1 | '''
2 | Need the dot binary from the graphviz package (www.graphviz.org).
3 | '''
4 | if '__file__' in globals():
5 | import os, sys
6 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
7 | import numpy as np
8 | from dezero import Variable
9 | from dezero.utils import plot_dot_graph
10 |
11 |
12 | def goldstein(x, y):
13 | z = (1 + (x + y + 1)**2 * (19 - 14*x + 3*x**2 - 14*y + 6*x*y + 3*y**2)) * \
14 | (30 + (2*x - 3*y)**2 * (18 - 32*x + 12*x**2 + 48*y - 36*x*y + 27*y**2))
15 | return z
16 |
17 |
18 | x = Variable(np.array(1.0))
19 | y = Variable(np.array(1.0))
20 | z = goldstein(x, y)
21 | z.backward()
22 |
23 | x.name = 'x'
24 | y.name = 'y'
25 | z.name = 'z'
26 | plot_dot_graph(z, verbose=False, to_file='goldstein.png')
--------------------------------------------------------------------------------
/steps/step27.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import math
6 | from dezero import Variable, Function
7 | from dezero.utils import plot_dot_graph
8 |
9 |
10 | class Sin(Function):
11 | def forward(self, x):
12 | y = np.sin(x)
13 | return y
14 |
15 | def backward(self, gy):
16 | x = self.inputs[0].data
17 | gx = gy * np.cos(x)
18 | return gx
19 |
20 |
21 | def sin(x):
22 | return Sin()(x)
23 |
24 |
25 | x = Variable(np.array(np.pi / 4))
26 | y = sin(x)
27 | y.backward()
28 | print('--- original sin ---')
29 | print(y.data)
30 | print(x.grad)
31 |
32 |
33 | def my_sin(x, threshold=0.0001):
34 | y = 0
35 | for i in range(100000):
36 | c = (-1) ** i / math.factorial(2 * i + 1)
37 | t = c * x ** (2 * i + 1)
38 | y = y + t
39 | if abs(t.data) < threshold:
40 | break
41 | return y
42 |
43 |
44 | x = Variable(np.array(np.pi / 4))
45 | y = my_sin(x) # , threshold=1e-150)
46 | y.backward()
47 | print('--- approximate sin ---')
48 | print(y.data)
49 | print(x.grad)
50 |
51 | x.name = 'x'
52 | y.name = 'y'
53 | plot_dot_graph(y, verbose=False, to_file='my_sin.png')
--------------------------------------------------------------------------------
/steps/step28.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | # import dezero's simple_core explicitly
7 | import dezero
8 | if not dezero.is_simple_core:
9 | from dezero.core_simple import Variable
10 | from dezero.core_simple import setup_variable
11 | setup_variable()
12 |
13 |
14 | def rosenbrock(x0, x1):
15 | y = 100 * (x1 - x0 ** 2) ** 2 + (x0 - 1) ** 2
16 | return y
17 |
18 |
19 | x0 = Variable(np.array(0.0))
20 | x1 = Variable(np.array(2.0))
21 | lr = 0.001
22 | iters = 1000
23 |
24 | for i in range(iters):
25 | print(x0, x1)
26 |
27 | y = rosenbrock(x0, x1)
28 |
29 | x0.cleargrad()
30 | x1.cleargrad()
31 | y.backward()
32 |
33 | x0.data -= lr * x0.grad
34 | x1.data -= lr * x1.grad
--------------------------------------------------------------------------------
/steps/step29.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | # import dezero's simple_core explicitly
7 | import dezero
8 | if not dezero.is_simple_core:
9 | from dezero.core_simple import Variable
10 | from dezero.core_simple import setup_variable
11 | setup_variable()
12 |
13 |
14 | def f(x):
15 | y = x ** 4 - 2 * x ** 2
16 | return y
17 |
18 |
19 | def gx2(x):
20 | return 12 * x ** 2 - 4
21 |
22 |
23 | x = Variable(np.array(2.0))
24 | iters = 10
25 |
26 | for i in range(iters):
27 | print(i, x)
28 |
29 | y = f(x)
30 | x.cleargrad()
31 | y.backward()
32 |
33 | x.data -= x.grad / gx2(x.data)
--------------------------------------------------------------------------------
/steps/step30.py:
--------------------------------------------------------------------------------
1 | # No code
--------------------------------------------------------------------------------
/steps/step31.py:
--------------------------------------------------------------------------------
1 | # No code
--------------------------------------------------------------------------------
/steps/step32.py:
--------------------------------------------------------------------------------
1 | # No code
--------------------------------------------------------------------------------
/steps/step33.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 |
7 | def f(x):
8 | y = x ** 4 - 2 * x ** 2
9 | return y
10 |
11 | x = Variable(np.array(2.0))
12 | iters = 10
13 |
14 | for i in range(iters):
15 | print(i, x)
16 |
17 | y = f(x)
18 | x.cleargrad()
19 | y.backward(create_graph=True)
20 |
21 | gx = x.grad
22 | x.cleargrad()
23 | gx.backward()
24 | gx2 = x.grad
25 |
26 | x.data -= gx.data / gx2.data
--------------------------------------------------------------------------------
/steps/step34.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import matplotlib.pyplot as plt
6 | from dezero import Variable
7 | import dezero.functions as F
8 |
9 | x = Variable(np.linspace(-7, 7, 200))
10 | y = F.sin(x)
11 | y.backward(create_graph=True)
12 |
13 | logs = [y.data]
14 |
15 | for i in range(3):
16 | logs.append(x.grad.data)
17 | gx = x.grad
18 | x.cleargrad()
19 | gx.backward(create_graph=True)
20 |
21 | labels = ["y=sin(x)", "y'", "y''", "y'''"]
22 | for i, v in enumerate(logs):
23 | plt.plot(x.data, logs[i], label=labels[i])
24 | plt.legend(loc='lower right')
25 | plt.show()
--------------------------------------------------------------------------------
/steps/step35.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | from dezero.utils import plot_dot_graph
7 | import dezero.functions as F
8 |
9 | x = Variable(np.array(1.0))
10 | y = F.tanh(x)
11 | x.name = 'x'
12 | y.name = 'y'
13 | y.backward(create_graph=True)
14 |
15 | iters = 1
16 |
17 | for i in range(iters):
18 | gx = x.grad
19 | x.cleargrad()
20 | gx.backward(create_graph=True)
21 |
22 | gx = x.grad
23 | gx.name = 'gx' + str(iters + 1)
24 | plot_dot_graph(gx, verbose=False, to_file='tanh.png')
--------------------------------------------------------------------------------
/steps/step36.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 |
7 | x = Variable(np.array(2.0))
8 | y = x ** 2
9 | y.backward(create_graph=True)
10 | gx = x.grad
11 | x.cleargrad()
12 |
13 | z = gx ** 3 + y
14 | z.backward()
15 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step37.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | import dezero.functions as F
7 |
8 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
9 | c = Variable(np.array([[10, 20, 30], [40, 50, 60]]))
10 | t = x + c
11 | y = F.sum(t)
12 |
13 | y.backward(retain_grad=True)
14 | print(y.grad)
15 | print(t.grad)
16 | print(x.grad)
17 | print(c.grad)
--------------------------------------------------------------------------------
/steps/step38.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | import dezero.functions as F
7 |
8 | x = Variable(np.array([[0, 1, 2], [3, 4, 5]]))
9 | y = F.reshape(x, (6,)) # y = x.reshape(6)
10 | y.backward(retain_grad=True)
11 | print(x.grad)
12 |
13 |
14 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
15 | y = F.transpose(x) # y = x.T
16 | y.backward()
17 | print(x.grad)
--------------------------------------------------------------------------------
/steps/step39.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | import dezero.functions as F
7 |
8 | x = Variable(np.array([1, 2, 3, 4, 5, 6]))
9 | y = F.sum(x)
10 | y.backward()
11 | print(y)
12 | print(x.grad)
13 |
14 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
15 | y = F.sum(x)
16 | y.backward()
17 | print(y)
18 | print(x.grad)
19 |
20 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
21 | y = F.sum(x, axis=0)
22 | y.backward()
23 | print(y)
24 | print(x.grad)
25 |
26 | x = Variable(np.random.randn(2, 3, 4, 5))
27 | y = x.sum(keepdims=True)
28 | print(y.shape)
--------------------------------------------------------------------------------
/steps/step40.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 |
7 | x0 = Variable(np.array([1, 2, 3]))
8 | x1 = Variable(np.array([10]))
9 | y = x0 + x1
10 | print(y)
11 |
12 | y.backward()
13 | print(x1.grad)
--------------------------------------------------------------------------------
/steps/step41.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | import dezero.functions as F
7 |
8 | x = Variable(np.random.randn(2, 3))
9 | w = Variable(np.random.randn(3, 4))
10 | y = F.matmul(x, w)
11 | y.backward()
12 |
13 | print(x.grad.shape)
14 | print(w.grad.shape)
--------------------------------------------------------------------------------
/steps/step42.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import matplotlib.pyplot as plt
6 | from dezero import Variable
7 | import dezero.functions as F
8 |
9 | # Generate toy dataset
10 | np.random.seed(0)
11 | x = np.random.rand(100, 1)
12 | y = 5 + 2 * x + np.random.rand(100, 1)
13 | x, y = Variable(x), Variable(y)
14 |
15 | W = Variable(np.zeros((1, 1)))
16 | b = Variable(np.zeros(1))
17 |
18 |
19 | def predict(x):
20 | y = F.matmul(x, W) + b
21 | return y
22 |
23 |
24 | def mean_squared_error(x0, x1):
25 | diff = x0 - x1
26 | return F.sum(diff ** 2) / len(diff)
27 |
28 |
29 | lr = 0.1
30 | iters = 100
31 |
32 | for i in range(iters):
33 | y_pred = predict(x)
34 | loss = mean_squared_error(y, y_pred)
35 |
36 | W.cleargrad()
37 | b.cleargrad()
38 | loss.backward()
39 |
40 | # Update .data attribute (No need grads when updating params)
41 | W.data -= lr * W.grad.data
42 | b.data -= lr * b.grad.data
43 | print(W, b, loss)
44 |
45 |
46 | # Plot
47 | plt.scatter(x.data, y.data, s=10)
48 | plt.xlabel('x')
49 | plt.ylabel('y')
50 | y_pred = predict(x)
51 | plt.plot(x.data, y_pred.data, color='r')
52 | plt.show()
--------------------------------------------------------------------------------
/steps/step43.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import matplotlib.pyplot as plt
6 | from dezero import Variable
7 | import dezero.functions as F
8 |
9 |
10 | np.random.seed(0)
11 | x = np.random.rand(100, 1)
12 | y = np.sin(2 * np.pi * x) + np.random.rand(100, 1)
13 |
14 | I, H, O = 1, 10, 1
15 | W1 = Variable(0.01 * np.random.randn(I, H))
16 | b1 = Variable(np.zeros(H))
17 | W2 = Variable(0.01 * np.random.randn(H, O))
18 | b2 = Variable(np.zeros(O))
19 |
20 |
21 | def predict(x):
22 | y = F.linear(x, W1, b1)
23 | y = F.sigmoid(y)
24 | y = F.linear(y, W2, b2)
25 | return y
26 |
27 |
28 | lr = 0.2
29 | iters = 10000
30 |
31 | for i in range(iters):
32 | y_pred = predict(x)
33 | loss = F.mean_squared_error(y, y_pred)
34 |
35 | W1.cleargrad()
36 | b1.cleargrad()
37 | W2.cleargrad()
38 | b2.cleargrad()
39 | loss.backward()
40 |
41 | W1.data -= lr * W1.grad.data
42 | b1.data -= lr * b1.grad.data
43 | W2.data -= lr * W2.grad.data
44 | b2.data -= lr * b2.grad.data
45 | if i % 1000 == 0:
46 | print(loss)
47 |
48 |
49 | # Plot
50 | plt.scatter(x, y, s=10)
51 | plt.xlabel('x')
52 | plt.ylabel('y')
53 | t = np.arange(0, 1, .01)[:, np.newaxis]
54 | y_pred = predict(t)
55 | plt.plot(t, y_pred.data, color='r')
56 | plt.show()
--------------------------------------------------------------------------------
/steps/step44.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import dezero.functions as F
6 | import dezero.layers as L
7 |
8 |
9 | np.random.seed(0)
10 | x = np.random.rand(100, 1)
11 | y = np.sin(2 * np.pi * x) + np.random.rand(100, 1)
12 |
13 | l1 = L.Linear(10)
14 | l2 = L.Linear(1)
15 |
16 |
17 | def predict(x):
18 | y = l1(x)
19 | y = F.sigmoid(y)
20 | y = l2(y)
21 | return y
22 |
23 |
24 | lr = 0.2
25 | iters = 10000
26 |
27 | for i in range(iters):
28 | y_pred = predict(x)
29 | loss = F.mean_squared_error(y, y_pred)
30 |
31 | l1.cleargrads()
32 | l2.cleargrads()
33 | loss.backward()
34 |
35 | for l in [l1, l2]:
36 | for p in l.params():
37 | p.data -= lr * p.grad.data
38 | if i % 1000 == 0:
39 | print(loss)
--------------------------------------------------------------------------------
/steps/step45.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Model
6 | import dezero.layers as L
7 | import dezero.functions as F
8 |
9 |
10 | np.random.seed(0)
11 | x = np.random.rand(100, 1)
12 | y = np.sin(2 * np.pi * x) + np.random.rand(100, 1)
13 |
14 | # Hyperparameters
15 | lr = 0.2
16 | max_iter = 10000
17 | hidden_size = 10
18 |
19 | # Model definition
20 | class TwoLayerNet(Model):
21 | def __init__(self, hidden_size, out_size):
22 | super().__init__()
23 | self.l1 = L.Linear(hidden_size)
24 | self.l2 = L.Linear(out_size)
25 |
26 | def forward(self, x):
27 | y = F.sigmoid(self.l1(x))
28 | y = self.l2(y)
29 | return y
30 |
31 |
32 | model = TwoLayerNet(hidden_size, 1)
33 |
34 | for i in range(max_iter):
35 | y_pred = model(x)
36 | loss = F.mean_squared_error(y, y_pred)
37 |
38 | model.cleargrads()
39 | loss.backward()
40 |
41 | for p in model.params():
42 | p.data -= lr * p.grad.data
43 | if i % 1000 == 0:
44 | print(loss)
--------------------------------------------------------------------------------
/steps/step46.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import optimizers
6 | import dezero.functions as F
7 | from dezero.models import MLP
8 |
9 |
10 | np.random.seed(0)
11 | x = np.random.rand(100, 1)
12 | y = np.sin(2 * np.pi * x) + np.random.rand(100, 1)
13 |
14 | lr = 0.2
15 | max_iter = 10000
16 | hidden_size = 10
17 |
18 | model = MLP((hidden_size, 1))
19 | optimizer = optimizers.SGD(lr).setup(model)
20 |
21 | for i in range(max_iter):
22 | y_pred = model(x)
23 | loss = F.mean_squared_error(y, y_pred)
24 |
25 | model.cleargrads()
26 | loss.backward()
27 |
28 | optimizer.update()
29 | if i % 1000 == 0:
30 | print(loss)
--------------------------------------------------------------------------------
/steps/step47.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | np.random.seed(0)
6 | from dezero import Variable, as_variable
7 | import dezero.functions as F
8 | from dezero.models import MLP
9 |
10 |
11 | def softmax1d(x):
12 | x = as_variable(x)
13 | y = F.exp(x)
14 | sum_y = F.sum(y)
15 | return y / sum_y
16 |
17 |
18 | model = MLP((10, 3))
19 |
20 | x = Variable(np.array([[0.2, -0.4]]))
21 | y = model(x)
22 | p = softmax1d(y)
23 | print(y)
24 | print(p)
25 |
26 | x = np.array([[0.2, -0.4], [0.3, 0.5], [1.3, -3.2], [2.1, 0.3]])
27 | t = np.array([2, 0, 1, 0])
28 |
29 | y = model(x)
30 | p = F.softmax_simple(y)
31 | print(y)
32 | print(p)
33 |
34 | loss = F.softmax_cross_entropy_simple(y, t)
35 | loss.backward()
36 | print(loss)
--------------------------------------------------------------------------------
/steps/step48.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import math
5 | import numpy as np
6 | import matplotlib.pyplot as plt
7 | import dezero
8 | from dezero import optimizers
9 | import dezero.functions as F
10 | from dezero.models import MLP
11 |
12 | # Hyperparameters
13 | max_epoch = 300
14 | batch_size = 30
15 | hidden_size = 10
16 | lr = 1.0
17 |
18 | x, t = dezero.datasets.get_spiral(train=True)
19 | model = MLP((hidden_size, 3))
20 | optimizer = optimizers.SGD(lr).setup(model)
21 |
22 | data_size = len(x)
23 | max_iter = math.ceil(data_size / batch_size)
24 |
25 | for epoch in range(max_epoch):
26 | # Shuffle index for data
27 | index = np.random.permutation(data_size)
28 | sum_loss = 0
29 |
30 | for i in range(max_iter):
31 | batch_index = index[i * batch_size:(i + 1) * batch_size]
32 | batch_x = x[batch_index]
33 | batch_t = t[batch_index]
34 |
35 | y = model(batch_x)
36 | loss = F.softmax_cross_entropy(y, batch_t)
37 | model.cleargrads()
38 | loss.backward()
39 | optimizer.update()
40 |
41 | sum_loss += float(loss.data) * len(batch_t)
42 |
43 | # Print loss every epoch
44 | avg_loss = sum_loss / data_size
45 | print('epoch %d, loss %.2f' % (epoch + 1, avg_loss))
46 |
47 | # Plot boundary area the model predict
48 | h = 0.001
49 | x_min, x_max = x[:, 0].min() - .1, x[:, 0].max() + .1
50 | y_min, y_max = x[:, 1].min() - .1, x[:, 1].max() + .1
51 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
52 | X = np.c_[xx.ravel(), yy.ravel()]
53 |
54 | with dezero.no_grad():
55 | score = model(X)
56 | predict_cls = np.argmax(score.data, axis=1)
57 | Z = predict_cls.reshape(xx.shape)
58 | plt.contourf(xx, yy, Z)
59 |
60 | # Plot data points of the dataset
61 | N, CLS_NUM = 100, 3
62 | markers = ['o', 'x', '^']
63 | colors = ['orange', 'blue', 'green']
64 | for i in range(len(x)):
65 | c = t[i]
66 | plt.scatter(x[i][0], x[i][1], s=40, marker=markers[c], c=colors[c])
67 | plt.show()
--------------------------------------------------------------------------------
/steps/step49.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import math
5 | import numpy as np
6 | import dezero
7 | import dezero.functions as F
8 | from dezero import optimizers
9 | from dezero.models import MLP
10 |
11 |
12 | max_epoch = 300
13 | batch_size = 30
14 | hidden_size = 10
15 | lr = 1.0
16 |
17 | train_set = dezero.datasets.Spiral(train=True)
18 | model = MLP((hidden_size, 3))
19 | optimizer = optimizers.SGD(lr).setup(model)
20 |
21 | data_size = len(train_set)
22 | max_iter = math.ceil(data_size / batch_size)
23 |
24 | for epoch in range(max_epoch):
25 | # Shuffle index for data
26 | index = np.random.permutation(data_size)
27 | sum_loss = 0
28 |
29 | for i in range(max_iter):
30 | # Create minibatch
31 | batch_index = index[i * batch_size:(i + 1) * batch_size]
32 | batch = [train_set[i] for i in batch_index]
33 | batch_x = np.array([example[0] for example in batch])
34 | batch_t = np.array([example[1] for example in batch])
35 |
36 | y = model(batch_x)
37 | loss = F.softmax_cross_entropy(y, batch_t)
38 | model.cleargrads()
39 | loss.backward()
40 | optimizer.update()
41 |
42 | sum_loss += float(loss.data) * len(batch_t)
43 |
44 | # Print loss every epoch
45 | avg_loss = sum_loss / data_size
46 | print('epoch %d, loss %.2f' % (epoch + 1, avg_loss))
47 |
--------------------------------------------------------------------------------
/steps/step50.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import dezero
5 | import dezero.functions as F
6 | from dezero import optimizers
7 | from dezero import DataLoader
8 | from dezero.models import MLP
9 |
10 |
11 | max_epoch = 300
12 | batch_size = 30
13 | hidden_size = 10
14 | lr = 1.0
15 |
16 | train_set = dezero.datasets.Spiral(train=True)
17 | test_set = dezero.datasets.Spiral(train=False)
18 | train_loader = DataLoader(train_set, batch_size)
19 | test_loader = DataLoader(test_set, batch_size, shuffle=False)
20 |
21 | model = MLP((hidden_size, 3))
22 | optimizer = optimizers.SGD(lr).setup(model)
23 |
24 | for epoch in range(max_epoch):
25 | sum_loss, sum_acc = 0, 0
26 |
27 | for x, t in train_loader:
28 | y = model(x)
29 | loss = F.softmax_cross_entropy(y, t)
30 | acc = F.accuracy(y, t)
31 | model.cleargrads()
32 | loss.backward()
33 | optimizer.update()
34 |
35 | sum_loss += float(loss.data) * len(t)
36 | sum_acc += float(acc.data) * len(t)
37 |
38 | print('epoch: {}'.format(epoch+1))
39 | print('train loss: {:.4f}, accuracy: {:.4f}'.format(
40 | sum_loss / len(train_set), sum_acc / len(train_set)))
41 |
42 | sum_loss, sum_acc = 0, 0
43 | with dezero.no_grad():
44 | for x, t in test_loader:
45 | y = model(x)
46 | loss = F.softmax_cross_entropy(y, t)
47 | acc = F.accuracy(y, t)
48 | sum_loss += float(loss.data) * len(t)
49 | sum_acc += float(acc.data) * len(t)
50 |
51 | print('test loss: {:.4f}, accuracy: {:.4f}'.format(
52 | sum_loss / len(test_set), sum_acc / len(test_set)))
53 |
--------------------------------------------------------------------------------
/steps/step51.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import dezero
5 | import dezero.functions as F
6 | from dezero import optimizers
7 | from dezero import DataLoader
8 | from dezero.models import MLP
9 |
10 |
11 | max_epoch = 5
12 | batch_size = 100
13 | hidden_size = 1000
14 |
15 | train_set = dezero.datasets.MNIST(train=True)
16 | test_set = dezero.datasets.MNIST(train=False)
17 | train_loader = DataLoader(train_set, batch_size)
18 | test_loader = DataLoader(test_set, batch_size, shuffle=False)
19 |
20 | model = MLP((hidden_size, 10))
21 | optimizer = optimizers.SGD().setup(model)
22 | #model = MLP((hidden_size, hidden_size, 10), activation=F.relu)
23 | #optimizer = optimizers.Adam().setup(model)
24 |
25 | for epoch in range(max_epoch):
26 | sum_loss, sum_acc = 0, 0
27 |
28 | for x, t in train_loader:
29 | y = model(x)
30 | loss = F.softmax_cross_entropy(y, t)
31 | acc = F.accuracy(y, t)
32 | model.cleargrads()
33 | loss.backward()
34 | optimizer.update()
35 |
36 | sum_loss += float(loss.data) * len(t)
37 | sum_acc += float(acc.data) * len(t)
38 |
39 | print('epoch: {}'.format(epoch+1))
40 | print('train loss: {:.4f}, accuracy: {:.4f}'.format(
41 | sum_loss / len(train_set), sum_acc / len(train_set)))
42 |
43 | sum_loss, sum_acc = 0, 0
44 | with dezero.no_grad():
45 | for x, t in test_loader:
46 | y = model(x)
47 | loss = F.softmax_cross_entropy(y, t)
48 | acc = F.accuracy(y, t)
49 | sum_loss += float(loss.data) * len(t)
50 | sum_acc += float(acc.data) * len(t)
51 |
52 | print('test loss: {:.4f}, accuracy: {:.4f}'.format(
53 | sum_loss / len(test_set), sum_acc / len(test_set)))
--------------------------------------------------------------------------------
/steps/step52.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import time
5 | import dezero
6 | import dezero.functions as F
7 | from dezero import optimizers
8 | from dezero import DataLoader
9 | from dezero.models import MLP
10 |
11 |
12 | max_epoch = 5
13 | batch_size = 100
14 |
15 | train_set = dezero.datasets.MNIST(train=True)
16 | train_loader = DataLoader(train_set, batch_size)
17 | model = MLP((1000, 10))
18 | optimizer = optimizers.SGD().setup(model)
19 |
20 | # GPU mode
21 | if dezero.cuda.gpu_enable:
22 | train_loader.to_gpu()
23 | model.to_gpu()
24 |
25 | for epoch in range(max_epoch):
26 | start = time.time()
27 | sum_loss = 0
28 |
29 | for x, t in train_loader:
30 | y = model(x)
31 | loss = F.softmax_cross_entropy(y, t)
32 | model.cleargrads()
33 | loss.backward()
34 | optimizer.update()
35 | sum_loss += float(loss.data) * len(t)
36 |
37 | elapsed_time = time.time() - start
38 | print('epoch: {}, loss: {:.4f}, time: {:.4f}[sec]'.format(
39 | epoch + 1, sum_loss / len(train_set), elapsed_time))
--------------------------------------------------------------------------------
/steps/step53.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import dezero
5 | import dezero.functions as F
6 | from dezero import optimizers
7 | from dezero import DataLoader
8 | from dezero.models import MLP
9 |
10 |
11 | max_epoch = 3
12 | batch_size = 100
13 |
14 | train_set = dezero.datasets.MNIST(train=True)
15 | train_loader = DataLoader(train_set, batch_size)
16 | model = MLP((1000, 10))
17 | optimizer = optimizers.SGD().setup(model)
18 |
19 | if os.path.exists('my_mlp.npz'):
20 | model.load_weights('my_mlp.npz')
21 |
22 | for epoch in range(max_epoch):
23 | sum_loss = 0
24 |
25 | for x, t in train_loader:
26 | y = model(x)
27 | loss = F.softmax_cross_entropy(y, t)
28 | model.cleargrads()
29 | loss.backward()
30 | optimizer.update()
31 | sum_loss += float(loss.data) * len(t)
32 |
33 | print('epoch: {}, loss: {:.4f}'.format(
34 | epoch + 1, sum_loss / len(train_set)))
35 |
36 | model.save_weights('my_mlp.npz')
--------------------------------------------------------------------------------
/steps/step54.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import test_mode
6 | import dezero.functions as F
7 |
8 | x = np.ones(5)
9 | print(x)
10 |
11 | # When training
12 | y = F.dropout(x)
13 | print(y)
14 |
15 | # When testing (predicting)
16 | with test_mode():
17 | y = F.dropout(x)
18 | print(y)
--------------------------------------------------------------------------------
/steps/step55.py:
--------------------------------------------------------------------------------
1 | def get_conv_outsize(input_size, kernel_size, stride, pad):
2 | return (input_size + pad * 2 - kernel_size) // stride + 1
3 |
4 |
5 | H, W = 4, 4 # Input size
6 | KH, KW = 3, 3 # Kernel size
7 | SH, SW = 1, 1 # Kernel stride
8 | PH, PW = 1, 1 # Padding size
9 |
10 | OH = get_conv_outsize(H, KH, SH, PH)
11 | OW = get_conv_outsize(W, KW, SW, PW)
12 | print(OH, OW)
--------------------------------------------------------------------------------
/steps/step56.py:
--------------------------------------------------------------------------------
1 | # No code
--------------------------------------------------------------------------------
/steps/step57.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from dezero import Variable
6 | import dezero.functions as F
7 |
8 |
9 | # im2col
10 | x1 = np.random.rand(1, 3, 7, 7)
11 | col1 = F.im2col(x1, kernel_size=5, stride=1, pad=0, to_matrix=True)
12 | print(col1.shape) # (9, 75)
13 |
14 | x2 = np.random.rand(10, 3, 7, 7) # 10个数据
15 | kernel_size = (5, 5)
16 | stride = (1, 1)
17 | pad = (0, 0)
18 | col2 = F.im2col(x2, kernel_size, stride, pad, to_matrix=True)
19 | print(col2.shape) # (90, 75)
20 |
21 |
22 | # conv2d
23 | N, C, H, W = 1, 5, 15, 15
24 | OC, (KH, KW) = 8, (3, 3)
25 | x = Variable(np.random.randn(N, C, H, W))
26 | W = np.random.randn(OC, C, KH, KW)
27 | y = F.conv2d_simple(x, W, b=None, stride=1, pad=1)
28 | y.backward()
29 | print(y.shape) # (1, 8, 15, 15)
30 | print(x.grad.shape) # (1, 5, 15, 15)
31 |
--------------------------------------------------------------------------------
/steps/step58.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | from PIL import Image
6 | import dezero
7 | from dezero.models import VGG16
8 |
9 |
10 | url = 'https://github.com/oreilly-japan/deep-learning-from-scratch-3/raw/images/zebra.jpg'
11 | img_path = dezero.utils.get_file(url)
12 | img = Image.open(img_path)
13 |
14 | x = VGG16.preprocess(img)
15 | x = x[np.newaxis]
16 |
17 | model = VGG16(pretrained=True)
18 | with dezero.test_mode():
19 | y = model(x)
20 | predict_id = np.argmax(y.data)
21 |
22 | model.plot(x, to_file='vgg.pdf')
23 | labels = dezero.datasets.ImageNet.labels()
24 | print(labels[predict_id])
--------------------------------------------------------------------------------
/steps/step59.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import matplotlib.pyplot as plt
6 | import dezero
7 | from dezero import Model
8 | import dezero.functions as F
9 | import dezero.layers as L
10 |
11 | # Hyperparameters
12 | max_epoch = 100
13 | hidden_size = 100
14 | bptt_length = 30
15 |
16 | train_set = dezero.datasets.SinCurve(train=True)
17 | seqlen = len(train_set)
18 |
19 |
20 | class SimpleRNN(Model):
21 | def __init__(self, hidden_size, out_size):
22 | super().__init__()
23 | self.rnn = L.RNN(hidden_size)
24 | self.fc = L.Linear(out_size)
25 |
26 | def reset_state(self):
27 | self.rnn.reset_state()
28 |
29 | def __call__(self, x):
30 | h = self.rnn(x)
31 | y = self.fc(h)
32 | return y
33 |
34 |
35 | model = SimpleRNN(hidden_size, 1)
36 | optimizer = dezero.optimizers.Adam().setup(model)
37 |
38 | # Start training.
39 | for epoch in range(max_epoch):
40 | model.reset_state()
41 | loss, count = 0, 0
42 |
43 | for x, t in train_set:
44 | x = x.reshape(1, 1)
45 | y = model(x)
46 | loss += F.mean_squared_error(y, t)
47 | count += 1
48 |
49 | if count % bptt_length == 0 or count == seqlen:
50 | model.cleargrads()
51 | loss.backward()
52 | loss.unchain_backward()
53 | optimizer.update()
54 |
55 | avg_loss = float(loss.data) / count
56 | print('| epoch %d | loss %f' % (epoch + 1, avg_loss))
57 |
58 | # Plot
59 | xs = np.cos(np.linspace(0, 4 * np.pi, 1000))
60 | model.reset_state()
61 | pred_list = []
62 |
63 | with dezero.no_grad():
64 | for x in xs:
65 | x = np.array(x).reshape(1, 1)
66 | y = model(x)
67 | pred_list.append(float(y.data))
68 |
69 | plt.plot(np.arange(len(xs)), xs, label='y=cos(x)')
70 | plt.plot(np.arange(len(xs)), pred_list, label='predict')
71 | plt.xlabel('x')
72 | plt.ylabel('y')
73 | plt.legend()
74 | plt.show()
--------------------------------------------------------------------------------
/steps/step60.py:
--------------------------------------------------------------------------------
1 | if '__file__' in globals():
2 | import os, sys
3 | sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
4 | import numpy as np
5 | import matplotlib.pyplot as plt
6 | import dezero
7 | from dezero import Model
8 | from dezero import SeqDataLoader
9 | import dezero.functions as F
10 | import dezero.layers as L
11 |
12 |
13 | max_epoch = 100
14 | batch_size = 30
15 | hidden_size = 100
16 | bptt_length = 30
17 |
18 | train_set = dezero.datasets.SinCurve(train=True)
19 | dataloader = SeqDataLoader(train_set, batch_size=batch_size)
20 | seqlen = len(train_set)
21 |
22 |
23 | class BetterRNN(Model):
24 | def __init__(self, hidden_size, out_size):
25 | super().__init__()
26 | self.rnn = L.LSTM(hidden_size)
27 | self.fc = L.Linear(out_size)
28 |
29 | def reset_state(self):
30 | self.rnn.reset_state()
31 |
32 | def __call__(self, x):
33 | y = self.rnn(x)
34 | y = self.fc(y)
35 | return y
36 |
37 | model = BetterRNN(hidden_size, 1)
38 | optimizer = dezero.optimizers.Adam().setup(model)
39 |
40 | for epoch in range(max_epoch):
41 | model.reset_state()
42 | loss, count = 0, 0
43 |
44 | for x, t in dataloader:
45 | y = model(x)
46 | loss += F.mean_squared_error(y, t)
47 | count += 1
48 |
49 | if count % bptt_length == 0 or count == seqlen:
50 | model.cleargrads()
51 | loss.backward()
52 | loss.unchain_backward()
53 | optimizer.update()
54 | avg_loss = float(loss.data) / count
55 | print('| epoch %d | loss %f' % (epoch + 1, avg_loss))
56 |
57 | # Plot
58 | xs = np.cos(np.linspace(0, 4 * np.pi, 1000))
59 | model.reset_state()
60 | pred_list = []
61 |
62 | with dezero.no_grad():
63 | for x in xs:
64 | x = np.array(x).reshape(1, 1)
65 | y = model(x)
66 | pred_list.append(float(y.data))
67 |
68 | plt.plot(np.arange(len(xs)), xs, label='y=cos(x)')
69 | plt.plot(np.arange(len(xs)), pred_list, label='predict')
70 | plt.xlabel('x')
71 | plt.ylabel('y')
72 | plt.legend()
73 | plt.show()
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_basic_math.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | from dezero.utils import gradient_check, array_equal
5 | import dezero.functions as F
6 |
7 |
8 | class TestAdd(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x0 = np.array([1, 2, 3])
12 | x1 = Variable(np.array([1, 2, 3]))
13 | y = x0 + x1
14 | res = y.data
15 | expected = np.array([2, 4, 6])
16 | self.assertTrue(array_equal(res, expected))
17 |
18 | def test_datatype(self):
19 | """是否不返回np.float64，而是返回0维的ndarray"""
20 | x = Variable(np.array(2.0))
21 | y = x ** 2
22 | self.assertFalse(np.isscalar(y))
23 |
24 | def test_backward1(self):
25 | x = Variable(np.random.randn(3, 3))
26 | y = np.random.randn(3, 3)
27 | f = lambda x: x + y
28 | self.assertTrue(gradient_check(f, x))
29 |
30 | def test_backward2(self):
31 | x = Variable(np.random.randn(3, 3))
32 | y = np.random.randn(3, 1)
33 | f = lambda x: x + y
34 | self.assertTrue(gradient_check(f, x))
35 |
36 | def test_backward3(self):
37 | x = np.random.randn(3, 3)
38 | y = np.random.randn(3, 1)
39 | self.assertTrue(gradient_check(F.add, x, y))
40 |
41 |
42 | class TestMul(unittest.TestCase):
43 |
44 | def test_forward1(self):
45 | x0 = np.array([1, 2, 3])
46 | x1 = Variable(np.array([1, 2, 3]))
47 | y = x0 * x1
48 | res = y.data
49 | expected = np.array([1, 4, 9])
50 | self.assertTrue(array_equal(res, expected))
51 |
52 | def test_backward1(self):
53 | x = np.random.randn(3, 3)
54 | y = np.random.randn(3, 3)
55 | f = lambda x: x * y
56 | self.assertTrue(gradient_check(f, x))
57 |
58 | def test_backward2(self):
59 | x = np.random.randn(3, 3)
60 | y = np.random.randn(3, 1)
61 | f = lambda x: x * y
62 | self.assertTrue(gradient_check(f, x))
63 |
64 | def test_backward3(self):
65 | x = np.random.randn(3, 3)
66 | y = np.random.randn(3, 1)
67 | f = lambda y: x * y
68 | self.assertTrue(gradient_check(f, x))
69 |
70 |
71 | class TestDiv(unittest.TestCase):
72 |
73 | def test_forward1(self):
74 | x0 = np.array([1, 2, 3])
75 | x1 = Variable(np.array([1, 2, 3]))
76 | y = x0 / x1
77 | res = y.data
78 | expected = np.array([1, 1, 1])
79 | self.assertTrue(array_equal(res, expected))
80 |
81 | def test_backward1(self):
82 | x = np.random.randn(3, 3)
83 | y = np.random.randn(3, 3)
84 | f = lambda x: x / y
85 | self.assertTrue(gradient_check(f, x))
86 |
87 | def test_backward2(self):
88 | x = np.random.randn(3, 3)
89 | y = np.random.randn(3, 1)
90 | f = lambda x: x / y
91 | self.assertTrue(gradient_check(f, x))
92 |
93 | def test_backward3(self):
94 | x = np.random.randn(3, 3)
95 | y = np.random.randn(3, 1)
96 | f = lambda x: x / y
97 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_broadcast.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check
6 |
7 |
8 | class TestBroadcast(unittest.TestCase):
9 |
10 | def test_shape_check(self):
11 | x = Variable(np.random.randn(1, 10))
12 | b = Variable(np.random.randn(10))
13 | y = x + b
14 | loss = F.sum(y)
15 | loss.backward()
16 | self.assertEqual(b.grad.shape, b.shape)
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_conv2d.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | import dezero.layers as L
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestConv2d_simple(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | n, c, h, w = 1, 5, 15, 15
13 | o, k, s, p = 8, (3, 3), (1, 1), (1, 1)
14 | x = np.random.randn(n, c, h, w).astype('f')
15 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
16 | b = None
17 | y = F.conv2d_simple(x, W, b, s, p)
18 | expected = CF.convolution_2d(x, W, b, s, p)
19 | self.assertTrue(array_allclose(expected.data, y.data))
20 |
21 | def test_forward2(self):
22 | n, c, h, w = 1, 5, 15, 15
23 | o, k, s, p = 8, (3, 3), (3, 1), (2, 1)
24 | x = np.random.randn(n, c, h, w).astype('f')
25 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
26 | b = None
27 | y = F.conv2d_simple(x, W, b, s, p)
28 | expected = CF.convolution_2d(x, W, b, s, p)
29 | self.assertTrue(array_allclose(expected.data, y.data))
30 |
31 | def test_forward3(self):
32 | n, c, h, w = 1, 5, 20, 15
33 | o, k, s, p = 3, (5, 3), 1, 3
34 | x = np.random.randn(n, c, h, w).astype('f')
35 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
36 | b = None
37 | y = F.conv2d_simple(x, W, b, s, p)
38 | expected = CF.convolution_2d(x, W, b, s, p)
39 | self.assertTrue(array_allclose(expected.data, y.data))
40 |
41 | def test_forward4(self):
42 | n, c, h, w = 1, 5, 20, 15
43 | o, k, s, p = 3, (5, 3), 1, 3
44 | x = np.random.randn(n, c, h, w).astype('f')
45 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
46 | b = np.random.randn(o).astype('f')
47 | y = F.conv2d_simple(x, W, b, s, p)
48 | expected = CF.convolution_2d(x, W, b, s, p)
49 | self.assertTrue(array_allclose(expected.data, y.data))
50 |
51 | def test_backward1(self):
52 | n, c, h, w = 1, 5, 20, 15
53 | o, k, s, p = 3, (5, 3), 1, 3
54 | x = np.random.randn(n, c, h, w)
55 | W = np.random.randn(o, c, k[0], k[1])
56 | b = np.random.randn(o)
57 | f = lambda x: F.conv2d_simple(x, W, b, s, p)
58 | self.assertTrue(gradient_check(f, x))
59 |
60 | def test_backward2(self):
61 | n, c, h, w = 1, 5, 20, 15
62 | o, k, s, p = 3, (5, 3), 1, 3
63 | x = np.random.randn(n, c, h, w)
64 | W = np.random.randn(o, c, k[0], k[1])
65 | b = np.random.randn(o)
66 | f = lambda b: F.conv2d_simple(x, W, b, s, p)
67 | self.assertTrue(gradient_check(f, b))
68 |
69 | def test_backward3(self):
70 | n, c, h, w = 1, 5, 20, 15
71 | o, k, s, p = 3, (5, 3), 1, 3
72 | x = np.random.randn(n, c, h, w)
73 | W = np.random.randn(o, c, k[0], k[1])
74 | b = np.random.randn(o)
75 | f = lambda W: F.conv2d_simple(x, W, b, s, p)
76 | self.assertTrue(gradient_check(f, W))
77 |
78 |
79 | class TestConv2d(unittest.TestCase):
80 |
81 | def test_forward1(self):
82 | n, c, h, w = 1, 5, 15, 15
83 | o, k, s, p = 8, (3, 3), (1, 1), (1, 1)
84 | x = np.random.randn(n, c, h, w).astype('f')
85 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
86 | b = None
87 | y = F.conv2d(x, W, b, s, p)
88 | expected = CF.convolution_2d(x, W, b, s, p)
89 | self.assertTrue(array_allclose(expected.data, y.data))
90 |
91 | def test_forward2(self):
92 | n, c, h, w = 1, 5, 15, 15
93 | o, k, s, p = 8, (3, 3), (3, 1), (2, 1)
94 | x = np.random.randn(n, c, h, w).astype('f')
95 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
96 | b = None
97 | y = F.conv2d(x, W, b, s, p)
98 | expected = CF.convolution_2d(x, W, b, s, p)
99 | self.assertTrue(array_allclose(expected.data, y.data))
100 |
101 | def test_forward3(self):
102 | n, c, h, w = 1, 5, 20, 15
103 | o, k, s, p = 3, (5, 3), 1, 3
104 | x = np.random.randn(n, c, h, w).astype('f')
105 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
106 | b = None
107 | y = F.conv2d(x, W, b, s, p)
108 | expected = CF.convolution_2d(x, W, b, s, p)
109 | self.assertTrue(array_allclose(expected.data, y.data))
110 |
111 | def test_forward4(self):
112 | n, c, h, w = 1, 5, 20, 15
113 | o, k, s, p = 3, (5, 3), 1, 3
114 | x = np.random.randn(n, c, h, w).astype('f')
115 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
116 | b = np.random.randn(o).astype('f')
117 | y = F.conv2d(x, W, b, s, p)
118 | expected = CF.convolution_2d(x, W, b, s, p)
119 | self.assertTrue(array_allclose(expected.data, y.data))
120 |
121 | def test_backward1(self):
122 | n, c, h, w = 1, 5, 20, 15
123 | o, k, s, p = 3, (5, 3), 1, 3
124 | x = np.random.randn(n, c, h, w)
125 | W = np.random.randn(o, c, k[0], k[1])
126 | b = np.random.randn(o)
127 | f = lambda x: F.conv2d(x, W, b, s, p)
128 | self.assertTrue(gradient_check(f, x))
129 |
130 | def test_backward2(self):
131 | n, c, h, w = 1, 5, 20, 15
132 | o, k, s, p = 3, (5, 3), 1, 3
133 | x = np.random.randn(n, c, h, w)
134 | W = np.random.randn(o, c, k[0], k[1])
135 | b = np.random.randn(o)
136 | f = lambda b: F.conv2d(x, W, b, s, p)
137 | self.assertTrue(gradient_check(f, b))
138 |
139 | def test_backward3(self):
140 | n, c, h, w = 1, 5, 20, 15
141 | o, k, s, p = 3, (5, 3), 1, 3
142 | x = np.random.randn(n, c, h, w)
143 | W = np.random.randn(o, c, k[0], k[1])
144 | b = np.random.randn(o)
145 | f = lambda W: F.conv2d(x, W, b, s, p)
146 | self.assertTrue(gradient_check(f, W))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_deconv2d.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | import dezero.layers as L
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestDeconv2d(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | n, c_i, c_o = 10, 1, 3
13 | h_i, w_i = 5, 10
14 | h_k, w_k = 10, 10
15 | h_p, w_p = 5, 5
16 | s_y, s_x = 5, 5
17 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i)).astype(np.float32)
18 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k)).astype(np.float32)
19 | b = np.random.uniform(0, 1, c_o).astype(np.float32)
20 |
21 | expected = CF.deconvolution_2d(x, W, b, stride=(s_y, s_x),
22 | pad=(h_p, w_p))
23 | y = F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
24 | self.assertTrue(array_allclose(expected.data, y.data))
25 |
26 | def test_forward2(self):
27 | n, c_i, c_o = 10, 1, 3
28 | h_i, w_i = 5, 10
29 | h_k, w_k = 10, 10
30 | h_p, w_p = 5, 5
31 | s_y, s_x = 5, 5
32 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i)).astype(np.float32)
33 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k)).astype(np.float32)
34 | b = None
35 | expected = CF.deconvolution_2d(x, W, b, stride=(s_y, s_x),
36 | pad=(h_p, w_p))
37 | y = F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
38 | self.assertTrue(array_allclose(expected.data, y.data))
39 |
40 | def test_backward1(self):
41 | n, c_i, c_o = 10, 1, 3
42 | h_i, w_i = 5, 10
43 | h_k, w_k = 10, 10
44 | h_p, w_p = 5, 5
45 | s_y, s_x = 5, 5
46 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i))
47 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k))
48 | b = None # np.random.uniform(0, 1, c_o).astype(np.float32)
49 | f = lambda x: F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
50 | self.assertTrue(gradient_check(f, x))
51 |
52 | def test_backward2(self):
53 | n, c_i, c_o = 10, 1, 3
54 | h_i, w_i = 5, 10
55 | h_k, w_k = 10, 10
56 | h_p, w_p = 5, 5
57 | s_y, s_x = 5, 5
58 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i))
59 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k))
60 | b = np.random.uniform(0, 1, c_o)
61 | f = lambda W: F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
62 | self.assertTrue(gradient_check(f, W))
63 |
64 | def test_backward3(self):
65 | n, c_i, c_o = 10, 1, 3
66 | h_i, w_i = 5, 10
67 | h_k, w_k = 10, 10
68 | h_p, w_p = 5, 5
69 | s_y, s_x = 5, 5
70 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i))
71 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k))
72 | b = np.random.uniform(0, 1, c_o)
73 | f = lambda b: F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
74 | self.assertTrue(gradient_check(f, b))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_dropout.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | import dezero
4 | from dezero import Variable
5 | import dezero.functions as F
6 | from dezero.utils import gradient_check, array_equal
7 |
8 |
9 | class TestDropout(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.random.randn(100, 100)
13 | y = F.dropout(Variable(x), dropout_ratio=0.0)
14 | res = array_equal(y.data, x)
15 | self.assertTrue(res)
16 |
17 | def test_forward2(self):
18 | x = np.random.randn(100, 100)
19 | with dezero.test_mode():
20 | y = F.dropout(x)
21 | res = array_equal(y.data, x)
22 | self.assertTrue(res)
23 |
24 | def test_backward1(self):
25 | x_data = np.random.randn(10, 10)
26 |
27 | def f(x):
28 | np.random.seed(0)
29 | return F.dropout(x, 0.5)
30 |
31 | self.assertTrue(gradient_check(f, x_data))
32 |
33 | def test_backward2(self):
34 | x_data = np.random.randn(10, 20)
35 |
36 | def f(x):
37 | np.random.seed(0)
38 | return F.dropout(x, 0.99)
39 |
40 | self.assertTrue(gradient_check(f, x_data))
41 |
42 | def test_backward3(self):
43 | x_data = np.random.randn(10, 10)
44 |
45 | def f(x):
46 | np.random.seed(0)
47 | return F.dropout(x, 0.0)
48 |
49 | self.assertTrue(gradient_check(f, x_data))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_getitem.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestGetitem(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x_data = np.arange(12).reshape((2, 2, 3))
12 | x = Variable(x_data)
13 | y = F.get_item(x, 0)
14 | self.assertTrue(array_allclose(y.data, x_data[0]))
15 |
16 | def test_forward1a(self):
17 | x_data = np.arange(12).reshape((2, 2, 3))
18 | x = Variable(x_data)
19 | y = x[0]
20 | self.assertTrue(array_allclose(y.data, x_data[0]))
21 |
22 | def test_forward2(self):
23 | x_data = np.arange(12).reshape((2, 2, 3))
24 | x = Variable(x_data)
25 | y = F.get_item(x, (0, 0, slice(0, 2, 1)))
26 | self.assertTrue(array_allclose(y.data, x_data[0, 0, 0:2:1]))
27 |
28 | def test_forward3(self):
29 | x_data = np.arange(12).reshape((2, 2, 3))
30 | x = Variable(x_data)
31 | y = F.get_item(x, (Ellipsis, 2))
32 | self.assertTrue(array_allclose(y.data, x_data[..., 2]))
33 |
34 | def test_backward1(self):
35 | x_data = np.array([[1, 2, 3], [4, 5, 6]])
36 | slices = 1
37 | f = lambda x: F.get_item(x, slices)
38 | gradient_check(f, x_data)
39 |
40 | def test_backward2(self):
41 | x_data = np.arange(12).reshape(4, 3)
42 | slices = slice(1, 3)
43 | f = lambda x: F.get_item(x, slices)
44 | gradient_check(f, x_data)
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_im2col.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_equal
6 | from dezero import utils
7 |
8 |
9 | class TestIm2col(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | n, c, h, w = 1, 1, 3, 3
13 | x = np.arange(n * c * h * w).reshape((n, c, h, w))
14 | y = F.im2col(x, 3, 3, 0, to_matrix=True)
15 | expected = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8]])
16 |
17 | res = array_equal(y.data, expected)
18 | self.assertTrue(res)
19 |
20 | def test_backward1(self):
21 | n, c, h, w = 1, 1, 3, 3
22 | x = np.arange(n * c * h * w).reshape((n, c, h, w))
23 | f = lambda x: F.im2col(x, 3, 3, 0, to_matrix=True)
24 | self.assertTrue(gradient_check(f, x))
25 |
26 | def test_backward2(self):
27 | n, c, h, w = 1, 1, 3, 3
28 | x = np.arange(n * c * h * w).reshape((n, c, h, w))
29 | f = lambda x: F.im2col(x, 3, 3, 0, to_matrix=False)
30 | self.assertTrue(gradient_check(f, x))
31 |
32 |
33 | class TestCol2in(unittest.TestCase):
34 |
35 | def test_backward1(self):
36 | n, c, h, w = 1, 1, 3, 3
37 | x = np.random.rand(1, 9)
38 | f = lambda x: F.col2im(x, (n, c, h, w), 3, 3, 0, to_matrix=True)
39 | self.assertTrue(gradient_check(f, x))
40 |
41 | def test_backward2(self):
42 | n, c, h, w = 1, 1, 3, 3
43 | x = np.random.rand(1, 1, 3, 3, 1, 1)
44 | f = lambda x: F.col2im(x, (n, c, h, w), 3, 3, 0, to_matrix=False)
45 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_linear.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import chainer
5 | import dezero.functions as F
6 | from dezero.utils import gradient_check, array_allclose
7 |
8 |
9 | class TestLinear(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
13 | w = Variable(x.data.T)
14 | b = None
15 | y = F.linear(x, w, b)
16 |
17 | res = y.data
18 | expected = np.array([[14, 32], [32, 77]])
19 | self.assertTrue(array_allclose(res, expected))
20 |
21 | def test_forward2(self):
22 | x = np.array([[1, 2, 3], [4, 5, 6]]).astype('f')
23 | W = x.T
24 | b = None
25 | y = F.linear(x, W, b)
26 |
27 | cy = chainer.functions.linear(x, W.T)
28 | self.assertTrue(array_allclose(y.data, cy.data))
29 |
30 | def test_forward3(self):
31 | layer = chainer.links.Linear(3, 2)
32 | layer.to_gpu()
33 | x = np.array([[1, 2, 3], [4, 5, 6]]).astype('f')
34 | W = layer.W.data.T
35 | b = layer.b.data
36 | y = F.linear(x, W, b)
37 |
38 | cy = layer(x)
39 | self.assertTrue(array_allclose(y.data, cy.data))
40 |
41 | def test_backward1(self):
42 | x = np.random.randn(3, 2)
43 | W = np.random.randn(2, 3)
44 | b = np.random.randn(3)
45 | f = lambda x: F.linear(x, W, b)
46 | self.assertTrue(gradient_check(f, x))
47 |
48 | def test_backward1(self):
49 | x = np.random.randn(3, 2)
50 | W = np.random.randn(2, 3)
51 | b = np.random.randn(3)
52 | f = lambda x: F.linear(x, W, b)
53 | self.assertTrue(gradient_check(f, x))
54 |
55 | def test_backward2(self):
56 | x = np.random.randn(100, 200)
57 | W = np.random.randn(200, 300)
58 | b = None
59 | f = lambda x: F.linear(x, W, b)
60 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_loss.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestMSE_simple(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x0 = np.array([0.0, 1.0, 2.0])
12 | x1 = np.array([0.0, 1.0, 2.0])
13 | expected = ((x0 - x1) ** 2).sum() / x0.size
14 | y = F.mean_squared_error_simple(x0, x1)
15 | self.assertTrue(array_allclose(y.data, expected))
16 |
17 | def test_backward1(self):
18 | x0 = np.random.rand(10)
19 | x1 = np.random.rand(10)
20 | f = lambda x0: F.mean_squared_error_simple(x0, x1)
21 | self.assertTrue(gradient_check(f, x0))
22 |
23 | def test_backward2(self):
24 | x0 = np.random.rand(100)
25 | x1 = np.random.rand(100)
26 | f = lambda x0: F.mean_squared_error_simple(x0, x1)
27 | self.assertTrue(gradient_check(f, x0))
28 |
29 |
30 | class TestMSE_simple(unittest.TestCase):
31 |
32 | def test_forward1(self):
33 | x0 = np.array([0.0, 1.0, 2.0])
34 | x1 = np.array([0.0, 1.0, 2.0])
35 | expected = ((x0 - x1) ** 2).sum() / x0.size
36 | y = F.mean_squared_error(x0, x1)
37 | self.assertTrue(array_allclose(y.data, expected))
38 |
39 | def test_backward1(self):
40 | x0 = np.random.rand(10)
41 | x1 = np.random.rand(10)
42 | f = lambda x0: F.mean_squared_error(x0, x1)
43 | self.assertTrue(gradient_check(f, x0))
44 |
45 | def test_backward2(self):
46 | x0 = np.random.rand(100)
47 | x1 = np.random.rand(100)
48 | f = lambda x0: F.mean_squared_error(x0, x1)
49 | self.assertTrue(gradient_check(f, x0))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_matmul.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestMatmul(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
12 | w = Variable(x.data.T)
13 | y = F.matmul(x, w)
14 | res = y.data
15 | expected = np.array([[14, 32], [32, 77]])
16 | self.assertTrue(array_allclose(res, expected))
17 |
18 | def test_backward1(self):
19 | x = np.random.randn(3, 2)
20 | w = np.random.randn(2, 3)
21 | f = lambda x: F.matmul(x, Variable(w))
22 | self.assertTrue(gradient_check(f, x))
23 |
24 | def test_backward2(self):
25 | x_data = np.random.randn(10, 1)
26 | w_data = np.random.randn(1, 5)
27 | f = lambda w: F.matmul(Variable(x_data), w)
28 | self.assertTrue(gradient_check(f, w_data))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_max.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestMax(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = Variable(np.random.rand(10))
12 | y = F.max(x)
13 | expected = np.max(x.data)
14 | self.assertTrue(array_allclose(y.data, expected))
15 |
16 | def test_forward2(self):
17 | shape = (10, 20, 30)
18 | axis = 1
19 | x = Variable(np.random.rand(*shape))
20 | y = F.max(x, axis=axis)
21 | expected = np.max(x.data, axis=axis)
22 | self.assertTrue(array_allclose(y.data, expected))
23 |
24 | def test_forward3(self):
25 | shape = (10, 20, 30)
26 | axis = (0, 1)
27 | x = Variable(np.random.rand(*shape))
28 | y = F.max(x, axis=axis)
29 | expected = np.max(x.data, axis=axis)
30 | self.assertTrue(array_allclose(y.data, expected))
31 |
32 | def test_forward4(self):
33 | shape = (10, 20, 30)
34 | axis = (0, 1)
35 | x = Variable(np.random.rand(*shape))
36 | y = F.max(x, axis=axis, keepdims=True)
37 | expected = np.max(x.data, axis=axis, keepdims=True)
38 | self.assertTrue(array_allclose(y.data, expected))
39 |
40 | def test_backward1(self):
41 | x_data = np.random.rand(10)
42 | f = lambda x: F.max(x)
43 | self.assertTrue(gradient_check(f, x_data))
44 |
45 | def test_backward2(self):
46 | x_data = np.random.rand(10, 10) * 100
47 | f = lambda x: F.max(x, axis=1)
48 | self.assertTrue(gradient_check(f, x_data))
49 |
50 | def test_backward3(self):
51 | x_data = np.random.rand(10, 20, 30) * 100
52 | f = lambda x: F.max(x, axis=(1, 2))
53 | self.assertTrue(gradient_check(f, x_data))
54 |
55 | def test_backward4(self):
56 | x_data = np.random.rand(10, 20, 20) * 100
57 | f = lambda x: F.sum(x, axis=None)
58 | self.assertTrue(gradient_check(f, x_data))
59 |
60 | def test_backward5(self):
61 | x_data = np.random.rand(10, 20, 20) * 100
62 | f = lambda x: F.sum(x, axis=None, keepdims=True)
63 | self.assertTrue(gradient_check(f, x_data))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_pooling.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | import dezero.functions as F
4 | from dezero.utils import gradient_check, array_allclose
5 | import chainer.functions as CF
6 |
7 |
8 | class TestPooling_simple(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | n, c, h, w = 1, 5, 16, 16
12 | ksize, stride, pad = 2, 2, 0
13 | x = np.random.randn(n, c, h, w).astype('f')
14 |
15 | y = F.pooling_simple(x, ksize, stride, pad)
16 | expected = CF.max_pooling_2d(x, ksize, stride, pad)
17 | self.assertTrue(array_allclose(expected.data, y.data))
18 |
19 | def test_forward2(self):
20 | n, c, h, w = 1, 5, 15, 15
21 | ksize, stride, pad = 2, 2, 0
22 | x = np.random.randn(n, c, h, w).astype('f')
23 |
24 | y = F.pooling_simple(x, ksize, stride, pad)
25 | expected = CF.max_pooling_2d(x, ksize, stride, pad, cover_all=False)
26 | self.assertTrue(array_allclose(expected.data, y.data))
27 |
28 | def test_backward1(self):
29 | n, c, h, w = 1, 5, 16, 16
30 | ksize, stride, pad = 2, 2, 0
31 | x = np.random.randn(n, c, h, w).astype('f') * 100
32 | f = lambda x: F.pooling_simple(x, ksize, stride, pad)
33 | self.assertTrue(gradient_check(f, x))
34 |
35 |
36 | class TestPooling(unittest.TestCase):
37 |
38 | def test_forward1(self):
39 | n, c, h, w = 1, 5, 16, 16
40 | ksize, stride, pad = 2, 2, 0
41 | x = np.random.randn(n, c, h, w).astype('f')
42 |
43 | y = F.pooling(x, ksize, stride, pad)
44 | expected = CF.max_pooling_2d(x, ksize, stride, pad)
45 | self.assertTrue(array_allclose(expected.data, y.data))
46 |
47 | def test_forward2(self):
48 | n, c, h, w = 1, 5, 15, 15
49 | ksize, stride, pad = 2, 2, 0
50 | x = np.random.randn(n, c, h, w).astype('f')
51 |
52 | y = F.pooling(x, ksize, stride, pad)
53 | expected = CF.max_pooling_2d(x, ksize, stride, pad, cover_all=False)
54 | self.assertTrue(array_allclose(expected.data, y.data))
55 |
56 | def test_backward1(self):
57 | n, c, h, w = 1, 5, 16, 16
58 | ksize, stride, pad = 2, 2, 0
59 | x = np.random.randn(n, c, h, w).astype('f') * 1000
60 | f = lambda x: F.pooling(x, ksize, stride, pad)
61 | self.assertTrue(gradient_check(f, x))
62 |
63 |
64 | class TestAveragePooling(unittest.TestCase):
65 |
66 | def test_forward1(self):
67 | n, c, h, w = 1, 5, 16, 16
68 | ksize, stride, pad = 2, 2, 0
69 | x = np.random.randn(n, c, h, w).astype('f')
70 |
71 | y = F.average_pooling(x, ksize, stride, pad)
72 | expected = CF.average_pooling_2d(x, ksize, stride, pad)
73 | self.assertTrue(array_allclose(expected.data, y.data))
74 |
75 | def test_forward2(self):
76 | n, c, h, w = 1, 5, 15, 15
77 | ksize, stride, pad = 2, 2, 0
78 | x = np.random.randn(n, c, h, w).astype('f')
79 |
80 | y = F.average_pooling(x, ksize, stride, pad)
81 | expected = CF.average_pooling_2d(x, ksize, stride, pad)
82 | self.assertTrue(array_allclose(expected.data, y.data))
83 |
84 | def test_backward1(self):
85 | n, c, h, w = 1, 5, 16, 16
86 | ksize, stride, pad = 2, 2, 0
87 | x = np.random.randn(n, c, h, w).astype('f') * 1000
88 | f = lambda x: F.average_pooling(x, ksize, stride, pad)
89 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_relu.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose, array_equal
6 |
7 |
8 | class TestRelu(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = np.array([[-1, 0], [2, -3], [-2, 1]], np.float32)
12 | res = F.relu(x)
13 | ans = np.array([[0, 0], [2, 0], [0, 1]], np.float32)
14 | self.assertTrue(array_allclose(res, ans))
15 |
16 | def test_backward1(self):
17 | x_data = np.array([[-1, 1, 2], [-1, 2, 4]])
18 | self.assertTrue(gradient_check(F.relu, x_data))
19 |
20 | def test_backward2(self):
21 | np.random.seed(0)
22 | x_data = np.random.rand(10, 10) * 100
23 | self.assertTrue(gradient_check(F.relu, x_data))
24 |
25 | def test_backward3(self):
26 | np.random.seed(0)
27 | x_data = np.random.rand(10, 10, 10) * 100
28 | self.assertTrue(gradient_check(F.relu, x_data))
29 |
30 |
31 | class TestLeakyRelu(unittest.TestCase):
32 |
33 | def test_forward1(self):
34 | x = np.array([[-1, 0], [2, -3], [-2, 1]], np.float32)
35 | res = F.leaky_relu(x)
36 | ans = np.array([[-0.2, 0.], [2., -0.6], [-0.4, 1.]], np.float32)
37 | self.assertTrue(array_allclose(res, ans))
38 |
39 | def test_backward1(self):
40 | x_data = np.array([[-1, 1, 2], [-1, 2, 4]])
41 | self.assertTrue(gradient_check(F.leaky_relu, x_data))
42 |
43 | def test_backward2(self):
44 | np.random.seed(0)
45 | x_data = np.random.rand(10, 10) * 100
46 | self.assertTrue(gradient_check(F.leaky_relu, x_data))
47 |
48 | def test_backward3(self):
49 | np.random.seed(0)
50 | x_data = np.random.rand(10, 10, 10) * 100
51 | self.assertTrue(gradient_check(F.leaky_relu, x_data))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_sigmoid.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestSigmoid(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
13 | y2 = CF.sigmoid(x)
14 | y = F.sigmoid(Variable(x))
15 | res = array_allclose(y.data, y2.data)
16 | self.assertTrue(res)
17 |
18 | def test_forward2(self):
19 | x = np.random.randn(10, 10).astype(np.float32)
20 | y2 = CF.sigmoid(x)
21 | y = F.sigmoid(Variable(x))
22 | res = array_allclose(y.data, y2.data)
23 | self.assertTrue(res)
24 |
25 | def test_backward1(self):
26 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
27 | self.assertTrue(gradient_check(F.sigmoid, x_data))
28 |
29 | def test_backward2(self):
30 | np.random.seed(0)
31 | x_data = np.random.rand(10, 10)
32 | self.assertTrue(gradient_check(F.sigmoid, x_data))
33 |
34 | def test_backward3(self):
35 | np.random.seed(0)
36 | x_data = np.random.rand(10, 10, 10)
37 | self.assertTrue(gradient_check(F.sigmoid, x_data))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_softmax.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestSoftmaxSimple(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
13 | y2 = CF.softmax(x, axis=1)
14 | y = F.softmax_simple(Variable(x))
15 | res = array_allclose(y.data, y2.data)
16 | self.assertTrue(res)
17 |
18 | def test_forward2(self):
19 | np.random.seed(0)
20 | x = np.random.rand(10, 10).astype('f')
21 | y2 = CF.softmax(x, axis=1)
22 | y = F.softmax_simple(Variable(x))
23 | res = array_allclose(y.data, y2.data)
24 | self.assertTrue(res)
25 |
26 | def test_forward3(self):
27 | np.random.seed(0)
28 | x = np.random.rand(10, 10, 10).astype('f')
29 | y2 = CF.softmax(x, axis=1)
30 | y = F.softmax_simple(Variable(x))
31 | res = array_allclose(y.data, y2.data)
32 | self.assertTrue(res)
33 |
34 | def test_backward1(self):
35 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
36 | f = lambda x: F.softmax_simple(x, axis=1)
37 | self.assertTrue(gradient_check(f, x_data))
38 |
39 | def test_backward2(self):
40 | np.random.seed(0)
41 | x_data = np.random.rand(10, 10)
42 | f = lambda x: F.softmax_simple(x, axis=1)
43 | self.assertTrue(gradient_check(f, x_data))
44 |
45 | def test_backward3(self):
46 | np.random.seed(0)
47 | x_data = np.random.rand(10, 10, 10)
48 | f = lambda x: F.softmax_simple(x, axis=1)
49 | self.assertTrue(gradient_check(f, x_data))
50 |
51 |
52 | class TestSoftmax(unittest.TestCase):
53 |
54 | def test_forward1(self):
55 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
56 | y2 = CF.softmax(x, axis=1)
57 | y = F.softmax(Variable(x))
58 | res = array_allclose(y.data, y2.data)
59 | self.assertTrue(res)
60 |
61 | def test_forward2(self):
62 | np.random.seed(0)
63 | x = np.random.rand(10, 10).astype('f')
64 | y2 = CF.softmax(x, axis=1)
65 | y = F.softmax(Variable(x))
66 | res = array_allclose(y.data, y2.data)
67 | self.assertTrue(res)
68 |
69 | def test_forward3(self):
70 | np.random.seed(0)
71 | x = np.random.rand(10, 10, 10).astype('f')
72 | y2 = CF.softmax(x, axis=1)
73 | y = F.softmax(Variable(x))
74 | res = array_allclose(y.data, y2.data)
75 | self.assertTrue(res)
76 |
77 | def test_backward1(self):
78 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
79 | f = lambda x: F.softmax(x, axis=1)
80 | self.assertTrue(gradient_check(f, x_data))
81 |
82 | def test_backward2(self):
83 | np.random.seed(0)
84 | x_data = np.random.rand(10, 10)
85 | f = lambda x: F.softmax(x, axis=1)
86 | self.assertTrue(gradient_check(f, x_data))
87 |
88 | def test_backward3(self):
89 | np.random.seed(0)
90 | x_data = np.random.rand(10, 10, 10)
91 | f = lambda x: F.softmax(x, axis=1)
92 | self.assertTrue(gradient_check(f, x_data))
93 |
94 |
95 | class TestSoftmax(unittest.TestCase):
96 |
97 | def test_forward1(self):
98 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
99 | y2 = CF.softmax(x, axis=1)
100 | y = F.softmax(Variable(x))
101 | res = array_allclose(y.data, y2.data)
102 | self.assertTrue(res)
103 |
104 | def test_forward2(self):
105 | np.random.seed(0)
106 | x = np.random.rand(10, 10).astype('f')
107 | y2 = CF.softmax(x, axis=1)
108 | y = F.softmax(Variable(x))
109 | res = array_allclose(y.data, y2.data)
110 | self.assertTrue(res)
111 |
112 | def test_forward3(self):
113 | np.random.seed(0)
114 | x = np.random.rand(10, 10, 10).astype('f')
115 | y2 = CF.softmax(x, axis=1)
116 | y = F.softmax(Variable(x))
117 | res = array_allclose(y.data, y2.data)
118 | self.assertTrue(res)
119 |
120 | def test_backward1(self):
121 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
122 | f = lambda x: F.softmax(x, axis=1)
123 | self.assertTrue(gradient_check(f, x_data))
124 |
125 | def test_backward2(self):
126 | np.random.seed(0)
127 | x_data = np.random.rand(10, 10)
128 | f = lambda x: F.softmax(x, axis=1)
129 | self.assertTrue(gradient_check(f, x_data))
130 |
131 | def test_backward3(self):
132 | np.random.seed(0)
133 | x_data = np.random.rand(10, 10, 10)
134 | f = lambda x: F.softmax(x, axis=1)
135 | self.assertTrue(gradient_check(f, x_data))
136 |
137 |
138 | class TestLogSoftmax(unittest.TestCase):
139 |
140 | def test_forward1(self):
141 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
142 | y = F.log_softmax(x)
143 | y2 = CF.log_softmax(x)
144 | res = array_allclose(y.data, y2.data)
145 | self.assertTrue(res)
146 |
147 | def test_backward1(self):
148 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]])
149 | f = lambda x: F.log_softmax(x)
150 | self.assertTrue(gradient_check(f, x))
151 |
152 | def test_backward2(self):
153 | x = np.random.randn(10, 10)
154 | f = lambda x: F.log_softmax(x)
155 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_softmax_cross_entropy.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestSoftmaxCrossEntropy(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
13 | t = np.array([3, 0]).astype(np.int32)
14 | y = F.softmax_cross_entropy(x, t)
15 | y2 = CF.softmax_cross_entropy(x, t)
16 | res = array_allclose(y.data, y2.data)
17 | self.assertTrue(res)
18 |
19 | def test_backward1(self):
20 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
21 | t = np.array([3, 0]).astype(np.int32)
22 | f = lambda x: F.softmax_cross_entropy(x, Variable(t))
23 | self.assertTrue(gradient_check(f, x))
24 |
25 | def test_backward2(self):
26 | N, CLS_NUM = 10, 10
27 | x = np.random.randn(N, CLS_NUM)
28 | t = np.random.randint(0, CLS_NUM, (N,))
29 | f = lambda x: F.softmax_cross_entropy(x, t)
30 | self.assertTrue(gradient_check(f, x))
31 |
32 | def test_backward3(self):
33 | N, CLS_NUM = 100, 10
34 | x = np.random.randn(N, CLS_NUM)
35 | t = np.random.randint(0, CLS_NUM, (N,))
36 | f = lambda x: F.softmax_cross_entropy(x, t)
37 | self.assertTrue(gradient_check(f, x))
38 |
39 |
40 | class TestSoftmaxCrossEntropy_simple(unittest.TestCase):
41 |
42 | def test_forward1(self):
43 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
44 | t = np.array([3, 0]).astype(np.int32)
45 | y = F.softmax_cross_entropy_simple(x, t)
46 | y2 = CF.softmax_cross_entropy(x, t)
47 | res = array_allclose(y.data, y2.data)
48 | self.assertTrue(res)
49 |
50 | def test_backward1(self):
51 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
52 | t = np.array([3, 0]).astype(np.int32)
53 | f = lambda x: F.softmax_cross_entropy_simple(x, Variable(t))
54 | self.assertTrue(gradient_check(f, x))
55 |
56 | def test_backward2(self):
57 | N, CLS_NUM = 10, 10
58 | x = np.random.randn(N, CLS_NUM)
59 | t = np.random.randint(0, CLS_NUM, (N,))
60 | f = lambda x: F.softmax_cross_entropy_simple(x, t)
61 | self.assertTrue(gradient_check(f, x))
62 |
63 | def test_backward3(self):
64 | N, CLS_NUM = 100, 10
65 | x = np.random.randn(N, CLS_NUM)
66 | t = np.random.randint(0, CLS_NUM, (N,))
67 | f = lambda x: F.softmax_cross_entropy_simple(x, t)
68 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_sum.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestSum(unittest.TestCase):
9 |
10 | def test_datatype(self):
11 | x = Variable(np.random.rand(10))
12 | y = F.sum(x)
13 | # 不返回np.float64，而是返回0维的ndarray
14 | self.assertFalse(np.isscalar(y))
15 |
16 | def test_forward1(self):
17 | x = Variable(np.array(2.0))
18 | y = F.sum(x)
19 | expected = np.sum(x.data)
20 | self.assertTrue(array_allclose(y.data, expected))
21 |
22 | def test_forward2(self):
23 | x = Variable(np.random.rand(10, 20, 30))
24 | y = F.sum(x, axis=1)
25 | expected = np.sum(x.data, axis=1)
26 | self.assertTrue(array_allclose(y.data, expected))
27 |
28 | def test_forward3(self):
29 | x = Variable(np.random.rand(10, 20, 30))
30 | y = F.sum(x, axis=1, keepdims=True)
31 | expected = np.sum(x.data, axis=1, keepdims=True)
32 | self.assertTrue(array_allclose(y.data, expected))
33 |
34 | def test_backward1(self):
35 | x_data = np.random.rand(10)
36 | f = lambda x: F.sum(x)
37 | self.assertTrue(gradient_check(f, x_data))
38 |
39 | def test_backward2(self):
40 | x_data = np.random.rand(10, 10)
41 | f = lambda x: F.sum(x, axis=1)
42 | self.assertTrue(gradient_check(f, x_data))
43 |
44 | def test_backward3(self):
45 | x_data = np.random.rand(10, 20, 20)
46 | f = lambda x: F.sum(x, axis=2)
47 | self.assertTrue(gradient_check(f, x_data))
48 |
49 | def test_backward4(self):
50 | x_data = np.random.rand(10, 20, 20)
51 | f = lambda x: F.sum(x, axis=None)
52 | self.assertTrue(gradient_check(f, x_data))
53 |
54 |
55 | class TestSumTo(unittest.TestCase):
56 |
57 | def test_forward1(self):
58 | x = Variable(np.random.rand(10))
59 | y = F.sum_to(x, (1,))
60 | expected = np.sum(x.data)
61 | self.assertTrue(array_allclose(y.data, expected))
62 |
63 | def test_forward2(self):
64 | x = Variable(np.array([[1., 2., 3.], [4., 5., 6.]]))
65 | y = F.sum_to(x, (1, 3))
66 | expected = np.sum(x.data, axis=0, keepdims=True)
67 | self.assertTrue(array_allclose(y.data, expected))
68 |
69 | def test_forward3(self):
70 | x = Variable(np.random.rand(10))
71 | y = F.sum_to(x, (10,))
72 | expected = x.data # 由于形状相同，不做任何处理
73 | self.assertTrue(array_allclose(y.data, expected))
74 |
75 | def test_backward1(self):
76 | x_data = np.random.rand(10)
77 | f = lambda x: F.sum_to(x, (1,))
78 | self.assertTrue(gradient_check(f, x_data))
79 |
80 | def test_backward2(self):
81 | x_data = np.random.rand(10, 10) * 10
82 | f = lambda x: F.sum_to(x, (10,))
83 | self.assertTrue(gradient_check(f, x_data))
84 |
85 | def test_backward3(self):
86 | x_data = np.random.rand(10, 20, 20) * 100
87 | f = lambda x: F.sum_to(x, (10,))
88 | self.assertTrue(gradient_check(f, x_data))
89 |
90 | def test_backward4(self):
91 | x_data = np.random.rand(10)
92 | f = lambda x: F.sum_to(x, (10,)) + 1
93 | self.assertTrue(gradient_check(f, x_data))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_transpose.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check
6 |
7 |
8 | class TestTranspose(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
12 | y = F.transpose(x)
13 | self.assertEqual(y.shape, (3, 2))
14 |
15 | def test_backward1(self):
16 | x = np.array([[1, 2, 3], [4, 5, 6]])
17 | self.assertTrue(gradient_check(F.transpose, x))
18 |
19 | def test_backward2(self):
20 | x = np.array([1, 2, 3])
21 | self.assertTrue(gradient_check(F.transpose, x))
22 |
23 | def test_backward3(self):
24 | x = np.random.randn(10, 5)
25 | self.assertTrue(gradient_check(F.transpose, x))
26 |
27 | def test_backward4(self):
28 | x = np.array([1, 2])
29 | self.assertTrue(gradient_check(F.transpose, x))
--------------------------------------------------------------------------------
/tests/gpu/gpu_test_vgg16.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import cupy as np # !! CUPY !!
3 | import chainer
4 | import dezero
5 | from dezero.utils import array_allclose
6 | from dezero.models import VGG16
7 |
8 |
9 | class TestVGG16(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.random.randn(1, 3, 224, 224).astype('f')
13 | _model = chainer.links.VGG16Layers(None)
14 | _model.to_gpu()
15 |
16 | with chainer.using_config('train', False):
17 | with chainer.using_config('enable_backprop', False):
18 | out_layer_name = 'fc8'
19 | _y = _model.forward(x, [out_layer_name])[out_layer_name]
20 |
21 | model = VGG16()
22 | layers = _model.available_layers
23 | for l in layers:
24 | if "conv" in l or "fc" in l:
25 | m1 = getattr(model, l)
26 | m2 = getattr(_model, l)
27 | m1.W.data = m2.W.data
28 | m1.b.data = m2.b.data
29 | if "fc" in l:
30 | m1.W.data = m1.W.data.T
31 | model.to_gpu()
32 |
33 | with dezero.test_mode():
34 | y = model(x)
35 |
36 | self.assertTrue(array_allclose(y.data, _y.data))
37 |
38 | def test_forward2(self):
39 | x = np.random.randn(1, 3, 224, 224).astype('f')
40 | model = VGG16()
41 | model.to_gpu()
42 | y = model(x)
43 | self.assertTrue(y.dtype == np.float32)
44 |
45 | def test_backward1(self):
46 | x = np.random.randn(2, 3, 224, 224).astype('f')
47 | _model = chainer.links.VGG16Layers(None)
48 | _model.to_gpu()
49 |
50 | with chainer.using_config('train', False):
51 | out_layer_name = 'fc8'
52 | _y = _model.forward(x, [out_layer_name])[out_layer_name]
53 | _y.grad = np.ones_like(_y.data)
54 | _y.backward()
55 |
56 | model = VGG16()
57 | layers = _model.available_layers
58 | for l in layers:
59 | if "conv" in l or "fc" in l:
60 | m1 = getattr(model, l)
61 | m2 = getattr(_model, l)
62 | m1.W.data = m2.W.data
63 | m1.b.data = m2.b.data
64 | if "fc" in l:
65 | m1.W.data = m1.W.data.T
66 | model.to_gpu()
67 |
68 | with dezero.test_mode():
69 | y = model(x)
70 | y.backward()
71 |
72 | layers = _model.available_layers
73 | for l in layers:
74 | if "conv" in l:
75 | m1 = getattr(model, l)
76 | m2 = getattr(_model, l)
77 | self.assertTrue(array_allclose(m1.W.data, m2.W.data))
78 | self.assertTrue(array_allclose(m1.b.data, m2.b.data))
79 | elif "fc" in l:
80 | m1 = getattr(model, l)
81 | m2 = getattr(_model, l)
82 | self.assertTrue(array_allclose(m1.W.data, m2.W.data.T))
83 | self.assertTrue(array_allclose(m1.b.data, m2.b.data))
--------------------------------------------------------------------------------
/tests/test_basic_math.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | from dezero.utils import gradient_check, array_equal
5 | import dezero.functions as F
6 |
7 |
8 | class TestAdd(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x0 = np.array([1, 2, 3])
12 | x1 = Variable(np.array([1, 2, 3]))
13 | y = x0 + x1
14 | res = y.data
15 | expected = np.array([2, 4, 6])
16 | self.assertTrue(array_equal(res, expected))
17 |
18 | def test_datatype(self):
19 | """是否不返回np.float64，而是返回0维的ndarray"""
20 | x = Variable(np.array(2.0))
21 | y = x ** 2
22 | self.assertFalse(np.isscalar(y))
23 |
24 | def test_backward1(self):
25 | x = Variable(np.random.randn(3, 3))
26 | y = np.random.randn(3, 3)
27 | f = lambda x: x + y
28 | self.assertTrue(gradient_check(f, x))
29 |
30 | def test_backward2(self):
31 | x = Variable(np.random.randn(3, 3))
32 | y = np.random.randn(3, 1)
33 | f = lambda x: x + y
34 | self.assertTrue(gradient_check(f, x))
35 |
36 | def test_backward3(self):
37 | x = np.random.randn(3, 3)
38 | y = np.random.randn(3, 1)
39 | self.assertTrue(gradient_check(F.add, x, y))
40 |
41 |
42 | class TestMul(unittest.TestCase):
43 |
44 | def test_forward1(self):
45 | x0 = np.array([1, 2, 3])
46 | x1 = Variable(np.array([1, 2, 3]))
47 | y = x0 * x1
48 | res = y.data
49 | expected = np.array([1, 4, 9])
50 | self.assertTrue(array_equal(res, expected))
51 |
52 | def test_backward1(self):
53 | x = np.random.randn(3, 3)
54 | y = np.random.randn(3, 3)
55 | f = lambda x: x * y
56 | self.assertTrue(gradient_check(f, x))
57 |
58 | def test_backward2(self):
59 | x = np.random.randn(3, 3)
60 | y = np.random.randn(3, 1)
61 | f = lambda x: x * y
62 | self.assertTrue(gradient_check(f, x))
63 |
64 | def test_backward3(self):
65 | x = np.random.randn(3, 3)
66 | y = np.random.randn(3, 1)
67 | f = lambda y: x * y
68 | self.assertTrue(gradient_check(f, x))
69 |
70 |
71 | class TestDiv(unittest.TestCase):
72 |
73 | def test_forward1(self):
74 | x0 = np.array([1, 2, 3])
75 | x1 = Variable(np.array([1, 2, 3]))
76 | y = x0 / x1
77 | res = y.data
78 | expected = np.array([1, 1, 1])
79 | self.assertTrue(array_equal(res, expected))
80 |
81 | def test_backward1(self):
82 | x = np.random.randn(3, 3)
83 | y = np.random.randn(3, 3)
84 | f = lambda x: x / y
85 | self.assertTrue(gradient_check(f, x))
86 |
87 | def test_backward2(self):
88 | x = np.random.randn(3, 3)
89 | y = np.random.randn(3, 1)
90 | f = lambda x: x / y
91 | self.assertTrue(gradient_check(f, x))
92 |
93 | def test_backward3(self):
94 | x = np.random.randn(3, 3)
95 | y = np.random.randn(3, 1)
96 | f = lambda x: x / y
97 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/test_broadcast.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check
6 |
7 |
8 | class TestBroadcast(unittest.TestCase):
9 |
10 | def test_shape_check(self):
11 | x = Variable(np.random.randn(1, 10))
12 | b = Variable(np.random.randn(10))
13 | y = x + b
14 | loss = F.sum(y)
15 | loss.backward()
16 | self.assertEqual(b.grad.shape, b.shape)
--------------------------------------------------------------------------------
/tests/test_conv2d.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import dezero.layers as L
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_equal
6 | import chainer.functions as CF
7 |
8 |
9 | class TestConv2d_simple(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | n, c, h, w = 1, 5, 15, 15
13 | o, k, s, p = 8, (3, 3), (1, 1), (1, 1)
14 | x = np.random.randn(n, c, h, w).astype('f')
15 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
16 | b = None
17 | y = F.conv2d_simple(x, W, b, s, p)
18 | expected = CF.convolution_2d(x, W, b, s, p)
19 | self.assertTrue(array_equal(expected.data, y.data))
20 |
21 | def test_forward2(self):
22 | n, c, h, w = 1, 5, 15, 15
23 | o, k, s, p = 8, (3, 3), (3, 1), (2, 1)
24 | x = np.random.randn(n, c, h, w).astype('f')
25 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
26 | b = None
27 | y = F.conv2d_simple(x, W, b, s, p)
28 | expected = CF.convolution_2d(x, W, b, s, p)
29 | self.assertTrue(array_equal(expected.data, y.data))
30 |
31 | def test_forward3(self):
32 | n, c, h, w = 1, 5, 20, 15
33 | o, k, s, p = 3, (5, 3), 1, 3
34 | x = np.random.randn(n, c, h, w).astype('f')
35 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
36 | b = None
37 | y = F.conv2d_simple(x, W, b, s, p)
38 | expected = CF.convolution_2d(x, W, b, s, p)
39 | self.assertTrue(array_equal(expected.data, y.data))
40 |
41 | def test_forward4(self):
42 | n, c, h, w = 1, 5, 20, 15
43 | o, k, s, p = 3, (5, 3), 1, 3
44 | x = np.random.randn(n, c, h, w).astype('f')
45 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
46 | b = np.random.randn(o).astype('f')
47 | y = F.conv2d_simple(x, W, b, s, p)
48 | expected = CF.convolution_2d(x, W, b, s, p)
49 | self.assertTrue(array_equal(expected.data, y.data))
50 |
51 | def test_backward1(self):
52 | n, c, h, w = 1, 5, 20, 15
53 | o, k, s, p = 3, (5, 3), 1, 3
54 | x = np.random.randn(n, c, h, w)
55 | W = np.random.randn(o, c, k[0], k[1])
56 | b = np.random.randn(o)
57 | f = lambda x: F.conv2d_simple(x, W, b, s, p)
58 | self.assertTrue(gradient_check(f, x))
59 |
60 | def test_backward2(self):
61 | n, c, h, w = 1, 5, 20, 15
62 | o, k, s, p = 3, (5, 3), 1, 3
63 | x = np.random.randn(n, c, h, w)
64 | W = np.random.randn(o, c, k[0], k[1])
65 | b = np.random.randn(o)
66 | f = lambda b: F.conv2d_simple(x, W, b, s, p)
67 | self.assertTrue(gradient_check(f, b))
68 |
69 | def test_backward3(self):
70 | n, c, h, w = 1, 5, 20, 15
71 | o, k, s, p = 3, (5, 3), 1, 3
72 | x = np.random.randn(n, c, h, w)
73 | W = np.random.randn(o, c, k[0], k[1])
74 | b = np.random.randn(o)
75 | f = lambda W: F.conv2d_simple(x, W, b, s, p)
76 | self.assertTrue(gradient_check(f, W))
77 |
78 |
79 | class TestConv2d(unittest.TestCase):
80 |
81 | def test_forward1(self):
82 | n, c, h, w = 1, 5, 15, 15
83 | o, k, s, p = 8, (3, 3), (1, 1), (1, 1)
84 | x = np.random.randn(n, c, h, w).astype('f')
85 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
86 | b = None
87 | y = F.conv2d(x, W, b, s, p)
88 | expected = CF.convolution_2d(x, W, b, s, p)
89 | self.assertTrue(array_equal(expected.data, y.data))
90 |
91 | def test_forward2(self):
92 | n, c, h, w = 1, 5, 15, 15
93 | o, k, s, p = 8, (3, 3), (3, 1), (2, 1)
94 | x = np.random.randn(n, c, h, w).astype('f')
95 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
96 | b = None
97 | y = F.conv2d(x, W, b, s, p)
98 | expected = CF.convolution_2d(x, W, b, s, p)
99 | self.assertTrue(array_equal(expected.data, y.data))
100 |
101 | def test_forward3(self):
102 | n, c, h, w = 1, 5, 20, 15
103 | o, k, s, p = 3, (5, 3), 1, 3
104 | x = np.random.randn(n, c, h, w).astype('f')
105 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
106 | b = None
107 | y = F.conv2d(x, W, b, s, p)
108 | expected = CF.convolution_2d(x, W, b, s, p)
109 | self.assertTrue(array_equal(expected.data, y.data))
110 |
111 | def test_forward4(self):
112 | n, c, h, w = 1, 5, 20, 15
113 | o, k, s, p = 3, (5, 3), 1, 3
114 | x = np.random.randn(n, c, h, w).astype('f')
115 | W = np.random.randn(o, c, k[0], k[1]).astype('f')
116 | b = np.random.randn(o).astype('f')
117 | y = F.conv2d(x, W, b, s, p)
118 | expected = CF.convolution_2d(x, W, b, s, p)
119 | self.assertTrue(array_equal(expected.data, y.data))
120 |
121 | def test_backward1(self):
122 | n, c, h, w = 1, 5, 20, 15
123 | o, k, s, p = 3, (5, 3), 1, 3
124 | x = np.random.randn(n, c, h, w)
125 | W = np.random.randn(o, c, k[0], k[1])
126 | b = np.random.randn(o)
127 | f = lambda x: F.conv2d(x, W, b, s, p)
128 | self.assertTrue(gradient_check(f, x))
129 |
130 | def test_backward2(self):
131 | n, c, h, w = 1, 5, 20, 15
132 | o, k, s, p = 3, (5, 3), 1, 3
133 | x = np.random.randn(n, c, h, w)
134 | W = np.random.randn(o, c, k[0], k[1])
135 | b = np.random.randn(o)
136 | f = lambda b: F.conv2d(x, W, b, s, p)
137 | self.assertTrue(gradient_check(f, b))
138 |
139 | def test_backward3(self):
140 | n, c, h, w = 1, 5, 20, 15
141 | o, k, s, p = 3, (5, 3), 1, 3
142 | x = np.random.randn(n, c, h, w)
143 | W = np.random.randn(o, c, k[0], k[1])
144 | b = np.random.randn(o)
145 | f = lambda W: F.conv2d(x, W, b, s, p)
146 | self.assertTrue(gradient_check(f, W))
--------------------------------------------------------------------------------
/tests/test_deconv2d.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import dezero.layers as L
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestDeconv2d(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | n, c_i, c_o = 10, 1, 3
13 | h_i, w_i = 5, 10
14 | h_k, w_k = 10, 10
15 | h_p, w_p = 5, 5
16 | s_y, s_x = 5, 5
17 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i)).astype(np.float32)
18 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k)).astype(np.float32)
19 | b = np.random.uniform(0, 1, c_o).astype(np.float32)
20 |
21 | expected = CF.deconvolution_2d(x, W, b, stride=(s_y, s_x),
22 | pad=(h_p, w_p))
23 | y = F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
24 | self.assertTrue(array_allclose(expected.data, y.data))
25 |
26 | def test_forward2(self):
27 | n, c_i, c_o = 10, 1, 3
28 | h_i, w_i = 5, 10
29 | h_k, w_k = 10, 10
30 | h_p, w_p = 5, 5
31 | s_y, s_x = 5, 5
32 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i)).astype(np.float32)
33 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k)).astype(np.float32)
34 | b = None
35 | expected = CF.deconvolution_2d(x, W, b, stride=(s_y, s_x),
36 | pad=(h_p, w_p))
37 | y = F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
38 | self.assertTrue(array_allclose(expected.data, y.data))
39 |
40 | def test_backward1(self):
41 | n, c_i, c_o = 10, 1, 3
42 | h_i, w_i = 5, 10
43 | h_k, w_k = 10, 10
44 | h_p, w_p = 5, 5
45 | s_y, s_x = 5, 5
46 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i))
47 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k))
48 | b = None # np.random.uniform(0, 1, c_o).astype(np.float32)
49 | f = lambda x: F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
50 | self.assertTrue(gradient_check(f, x))
51 |
52 | def test_backward2(self):
53 | n, c_i, c_o = 10, 1, 3
54 | h_i, w_i = 5, 10
55 | h_k, w_k = 10, 10
56 | h_p, w_p = 5, 5
57 | s_y, s_x = 5, 5
58 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i))
59 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k))
60 | b = np.random.uniform(0, 1, c_o)
61 | f = lambda W: F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
62 | self.assertTrue(gradient_check(f, W))
63 |
64 | def test_backward3(self):
65 | n, c_i, c_o = 10, 1, 3
66 | h_i, w_i = 5, 10
67 | h_k, w_k = 10, 10
68 | h_p, w_p = 5, 5
69 | s_y, s_x = 5, 5
70 | x = np.random.uniform(0, 1, (n, c_i, h_i, w_i))
71 | W = np.random.uniform(0, 1, (c_i, c_o, h_k, w_k))
72 | b = np.random.uniform(0, 1, c_o)
73 | f = lambda b: F.deconv2d(x, W, b, stride=(s_y, s_x), pad=(h_p, w_p))
74 | self.assertTrue(gradient_check(f, b))
--------------------------------------------------------------------------------
/tests/test_dropout.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import dezero
4 | from dezero import Variable
5 | import dezero.functions as F
6 | from dezero.utils import gradient_check, array_equal
7 |
8 |
9 | class TestDropout(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.random.randn(100, 100)
13 | y = F.dropout(Variable(x), dropout_ratio=0.0)
14 | res = array_equal(y.data, x)
15 | self.assertTrue(res)
16 |
17 | def test_forward2(self):
18 | x = np.random.randn(100, 100)
19 | with dezero.test_mode():
20 | y = F.dropout(x)
21 | res = array_equal(y.data, x)
22 | self.assertTrue(res)
23 |
24 | def test_backward1(self):
25 | x_data = np.random.randn(10, 10)
26 |
27 | def f(x):
28 | np.random.seed(0)
29 | return F.dropout(x, 0.5)
30 |
31 | self.assertTrue(gradient_check(f, x_data))
32 |
33 | def test_backward2(self):
34 | x_data = np.random.randn(10, 20)
35 |
36 | def f(x):
37 | np.random.seed(0)
38 | return F.dropout(x, 0.99)
39 |
40 | self.assertTrue(gradient_check(f, x_data))
41 |
42 | def test_backward3(self):
43 | x_data = np.random.randn(10, 10)
44 |
45 | def f(x):
46 | np.random.seed(0)
47 | return F.dropout(x, 0.0)
48 |
49 | self.assertTrue(gradient_check(f, x_data))
--------------------------------------------------------------------------------
/tests/test_getitem.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestGetitem(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x_data = np.arange(12).reshape((2, 2, 3))
12 | x = Variable(x_data)
13 | y = F.get_item(x, 0)
14 | self.assertTrue(array_allclose(y.data, x_data[0]))
15 |
16 | def test_forward1a(self):
17 | x_data = np.arange(12).reshape((2, 2, 3))
18 | x = Variable(x_data)
19 | y = x[0]
20 | self.assertTrue(array_allclose(y.data, x_data[0]))
21 |
22 | def test_forward2(self):
23 | x_data = np.arange(12).reshape((2, 2, 3))
24 | x = Variable(x_data)
25 | y = F.get_item(x, (0, 0, slice(0, 2, 1)))
26 | self.assertTrue(array_allclose(y.data, x_data[0, 0, 0:2:1]))
27 |
28 | def test_forward3(self):
29 | x_data = np.arange(12).reshape((2, 2, 3))
30 | x = Variable(x_data)
31 | y = F.get_item(x, (Ellipsis, 2))
32 | self.assertTrue(array_allclose(y.data, x_data[..., 2]))
33 |
34 | def test_backward1(self):
35 | x_data = np.array([[1, 2, 3], [4, 5, 6]])
36 | slices = 1
37 | f = lambda x: F.get_item(x, slices)
38 | gradient_check(f, x_data)
39 |
40 | def test_backward2(self):
41 | x_data = np.arange(12).reshape(4, 3)
42 | slices = slice(1, 3)
43 | f = lambda x: F.get_item(x, slices)
44 | gradient_check(f, x_data)
--------------------------------------------------------------------------------
/tests/test_im2col.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_equal
6 | from dezero import utils
7 |
8 |
9 | class TestIm2col(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | n, c, h, w = 1, 1, 3, 3
13 | x = np.arange(n * c * h * w).reshape((n, c, h, w))
14 | y = F.im2col(x, 3, 3, 0, to_matrix=True)
15 | expected = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8]])
16 |
17 | res = array_equal(y.data, expected)
18 | self.assertTrue(res)
19 |
20 | def test_backward1(self):
21 | n, c, h, w = 1, 1, 3, 3
22 | x = np.arange(n * c * h * w).reshape((n, c, h, w))
23 | f = lambda x: F.im2col(x, 3, 3, 0, to_matrix=True)
24 | self.assertTrue(gradient_check(f, x))
25 |
26 | def test_backward2(self):
27 | n, c, h, w = 1, 1, 3, 3
28 | x = np.arange(n * c * h * w).reshape((n, c, h, w))
29 | f = lambda x: F.im2col(x, 3, 3, 0, to_matrix=False)
30 | self.assertTrue(gradient_check(f, x))
31 |
32 |
33 | class TestCol2in(unittest.TestCase):
34 |
35 | def test_backward1(self):
36 | n, c, h, w = 1, 1, 3, 3
37 | x = np.random.rand(1, 9)
38 | f = lambda x: F.col2im(x, (n, c, h, w), 3, 3, 0, to_matrix=True)
39 | self.assertTrue(gradient_check(f, x))
40 |
41 | def test_backward2(self):
42 | n, c, h, w = 1, 1, 3, 3
43 | x = np.random.rand(1, 1, 3, 3, 1, 1)
44 | f = lambda x: F.col2im(x, (n, c, h, w), 3, 3, 0, to_matrix=False)
45 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/test_linear.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import chainer
5 | import dezero.functions as F
6 | from dezero.utils import gradient_check, array_allclose
7 |
8 |
9 | class TestLinear(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
13 | w = Variable(x.data.T)
14 | b = None
15 | y = F.linear(x, w, b)
16 |
17 | res = y.data
18 | expected = np.array([[14, 32], [32, 77]])
19 | self.assertTrue(array_allclose(res, expected))
20 |
21 | def test_forward2(self):
22 | x = np.array([[1, 2, 3], [4, 5, 6]]).astype('f')
23 | W = x.T
24 | b = None
25 | y = F.linear(x, W, b)
26 |
27 | cy = chainer.functions.linear(x, W.T)
28 | self.assertTrue(array_allclose(y.data, cy.data))
29 |
30 | def test_forward3(self):
31 | layer = chainer.links.Linear(3, 2)
32 | x = np.array([[1, 2, 3], [4, 5, 6]]).astype('f')
33 | W = layer.W.data.T
34 | b = layer.b.data
35 | y = F.linear(x, W, b)
36 |
37 | cy = layer(x)
38 | self.assertTrue(array_allclose(y.data, cy.data))
39 |
40 | def test_backward1(self):
41 | x = np.random.randn(3, 2)
42 | W = np.random.randn(2, 3)
43 | b = np.random.randn(3)
44 | f = lambda x: F.linear(x, W, b)
45 | self.assertTrue(gradient_check(f, x))
46 |
47 | def test_backward1(self):
48 | x = np.random.randn(3, 2)
49 | W = np.random.randn(2, 3)
50 | b = np.random.randn(3)
51 | f = lambda x: F.linear(x, W, b)
52 | self.assertTrue(gradient_check(f, x))
53 |
54 | def test_backward2(self):
55 | x = np.random.randn(100, 200)
56 | W = np.random.randn(200, 300)
57 | b = None
58 | f = lambda x: F.linear(x, W, b)
59 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/test_loss.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestMSE_simple(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x0 = np.array([0.0, 1.0, 2.0])
12 | x1 = np.array([0.0, 1.0, 2.0])
13 | expected = ((x0 - x1) ** 2).sum() / x0.size
14 | y = F.mean_squared_error_simple(x0, x1)
15 | self.assertTrue(array_allclose(y.data, expected))
16 |
17 | def test_backward1(self):
18 | x0 = np.random.rand(10)
19 | x1 = np.random.rand(10)
20 | f = lambda x0: F.mean_squared_error_simple(x0, x1)
21 | self.assertTrue(gradient_check(f, x0))
22 |
23 | def test_backward2(self):
24 | x0 = np.random.rand(100)
25 | x1 = np.random.rand(100)
26 | f = lambda x0: F.mean_squared_error_simple(x0, x1)
27 | self.assertTrue(gradient_check(f, x0))
28 |
29 |
30 | class TestMSE_simple(unittest.TestCase):
31 |
32 | def test_forward1(self):
33 | x0 = np.array([0.0, 1.0, 2.0])
34 | x1 = np.array([0.0, 1.0, 2.0])
35 | expected = ((x0 - x1) ** 2).sum() / x0.size
36 | y = F.mean_squared_error(x0, x1)
37 | self.assertTrue(array_allclose(y.data, expected))
38 |
39 | def test_backward1(self):
40 | x0 = np.random.rand(10)
41 | x1 = np.random.rand(10)
42 | f = lambda x0: F.mean_squared_error(x0, x1)
43 | self.assertTrue(gradient_check(f, x0))
44 |
45 | def test_backward2(self):
46 | x0 = np.random.rand(100)
47 | x1 = np.random.rand(100)
48 | f = lambda x0: F.mean_squared_error(x0, x1)
49 | self.assertTrue(gradient_check(f, x0))
--------------------------------------------------------------------------------
/tests/test_matmul.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestMatmul(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
12 | w = Variable(x.data.T)
13 | y = F.matmul(x, w)
14 | res = y.data
15 | expected = np.array([[14, 32], [32, 77]])
16 | self.assertTrue(array_allclose(res, expected))
17 |
18 | def test_backward1(self):
19 | x = np.random.randn(3, 2)
20 | w = np.random.randn(2, 3)
21 | f = lambda x: F.matmul(x, Variable(w))
22 | self.assertTrue(gradient_check(f, x))
23 |
24 | def test_backward2(self):
25 | x_data = np.random.randn(10, 1)
26 | w_data = np.random.randn(1, 5)
27 | f = lambda w: F.matmul(Variable(x_data), w)
28 | self.assertTrue(gradient_check(f, w_data))
--------------------------------------------------------------------------------
/tests/test_max.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestMax(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = Variable(np.random.rand(10))
12 | y = F.max(x)
13 | expected = np.max(x.data)
14 | self.assertTrue(array_allclose(y.data, expected))
15 |
16 | def test_forward2(self):
17 | shape = (10, 20, 30)
18 | axis = 1
19 | x = Variable(np.random.rand(*shape))
20 | y = F.max(x, axis=axis)
21 | expected = np.max(x.data, axis=axis)
22 | self.assertTrue(array_allclose(y.data, expected))
23 |
24 | def test_forward3(self):
25 | shape = (10, 20, 30)
26 | axis = (0, 1)
27 | x = Variable(np.random.rand(*shape))
28 | y = F.max(x, axis=axis)
29 | expected = np.max(x.data, axis=axis)
30 | self.assertTrue(array_allclose(y.data, expected))
31 |
32 | def test_forward4(self):
33 | shape = (10, 20, 30)
34 | axis = (0, 1)
35 | x = Variable(np.random.rand(*shape))
36 | y = F.max(x, axis=axis, keepdims=True)
37 | expected = np.max(x.data, axis=axis, keepdims=True)
38 | self.assertTrue(array_allclose(y.data, expected))
39 |
40 | def test_backward1(self):
41 | x_data = np.random.rand(10)
42 | f = lambda x: F.max(x)
43 | self.assertTrue(gradient_check(f, x_data))
44 |
45 | def test_backward2(self):
46 | x_data = np.random.rand(10, 10) * 100
47 | f = lambda x: F.max(x, axis=1)
48 | self.assertTrue(gradient_check(f, x_data))
49 |
50 | def test_backward3(self):
51 | x_data = np.random.rand(10, 20, 30) * 100
52 | f = lambda x: F.max(x, axis=(1, 2))
53 | self.assertTrue(gradient_check(f, x_data))
54 |
55 | def test_backward4(self):
56 | x_data = np.random.rand(10, 20, 20) * 100
57 | f = lambda x: F.sum(x, axis=None)
58 | self.assertTrue(gradient_check(f, x_data))
59 |
60 | def test_backward5(self):
61 | x_data = np.random.rand(10, 20, 20) * 100
62 | f = lambda x: F.sum(x, axis=None, keepdims=True)
63 | self.assertTrue(gradient_check(f, x_data))
--------------------------------------------------------------------------------
/tests/test_pooling.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import dezero.functions as F
4 | from dezero.utils import gradient_check, array_allclose
5 | import chainer.functions as CF
6 |
7 |
8 | class TestPooling_simple(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | n, c, h, w = 1, 5, 16, 16
12 | ksize, stride, pad = 2, 2, 0
13 | x = np.random.randn(n, c, h, w).astype('f')
14 |
15 | y = F.pooling_simple(x, ksize, stride, pad)
16 | expected = CF.max_pooling_2d(x, ksize, stride, pad)
17 | self.assertTrue(array_allclose(expected.data, y.data))
18 |
19 | def test_forward2(self):
20 | n, c, h, w = 1, 5, 15, 15
21 | ksize, stride, pad = 2, 2, 0
22 | x = np.random.randn(n, c, h, w).astype('f')
23 |
24 | y = F.pooling_simple(x, ksize, stride, pad)
25 | expected = CF.max_pooling_2d(x, ksize, stride, pad, cover_all=False)
26 | self.assertTrue(array_allclose(expected.data, y.data))
27 |
28 | def test_backward1(self):
29 | n, c, h, w = 1, 5, 16, 16
30 | ksize, stride, pad = 2, 2, 0
31 | x = np.random.randn(n, c, h, w).astype('f') * 100
32 | f = lambda x: F.pooling_simple(x, ksize, stride, pad)
33 | self.assertTrue(gradient_check(f, x))
34 |
35 |
36 | class TestPooling(unittest.TestCase):
37 |
38 | def test_forward1(self):
39 | n, c, h, w = 1, 5, 16, 16
40 | ksize, stride, pad = 2, 2, 0
41 | x = np.random.randn(n, c, h, w).astype('f')
42 |
43 | y = F.pooling(x, ksize, stride, pad)
44 | expected = CF.max_pooling_2d(x, ksize, stride, pad)
45 | self.assertTrue(array_allclose(expected.data, y.data))
46 |
47 | def test_forward2(self):
48 | n, c, h, w = 1, 5, 15, 15
49 | ksize, stride, pad = 2, 2, 0
50 | x = np.random.randn(n, c, h, w).astype('f')
51 |
52 | y = F.pooling(x, ksize, stride, pad)
53 | expected = CF.max_pooling_2d(x, ksize, stride, pad, cover_all=False)
54 | self.assertTrue(array_allclose(expected.data, y.data))
55 |
56 | def test_backward1(self):
57 | n, c, h, w = 1, 5, 16, 16
58 | ksize, stride, pad = 2, 2, 0
59 | x = np.random.randn(n, c, h, w).astype('f') * 1000
60 | f = lambda x: F.pooling(x, ksize, stride, pad)
61 | self.assertTrue(gradient_check(f, x))
62 |
63 |
64 | class TestAveragePooling(unittest.TestCase):
65 |
66 | def test_forward1(self):
67 | n, c, h, w = 1, 5, 16, 16
68 | ksize, stride, pad = 2, 2, 0
69 | x = np.random.randn(n, c, h, w).astype('f')
70 |
71 | y = F.average_pooling(x, ksize, stride, pad)
72 | expected = CF.average_pooling_2d(x, ksize, stride, pad)
73 | self.assertTrue(array_allclose(expected.data, y.data))
74 |
75 | def test_forward2(self):
76 | n, c, h, w = 1, 5, 15, 15
77 | ksize, stride, pad = 2, 2, 0
78 | x = np.random.randn(n, c, h, w).astype('f')
79 |
80 | y = F.average_pooling(x, ksize, stride, pad)
81 | expected = CF.average_pooling_2d(x, ksize, stride, pad)
82 | self.assertTrue(array_allclose(expected.data, y.data))
83 |
84 | def test_backward1(self):
85 | n, c, h, w = 1, 5, 16, 16
86 | ksize, stride, pad = 2, 2, 0
87 | x = np.random.randn(n, c, h, w).astype('f') * 1000
88 | f = lambda x: F.average_pooling(x, ksize, stride, pad)
89 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/test_relu.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose, array_equal
6 | import chainer.functions as CF
7 |
8 | class TestRelu(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = np.array([[-1, 0], [2, -3], [-2, 1]], np.float32)
12 | res = F.relu(x)
13 | ans = np.array([[0, 0], [2, 0], [0, 1]], np.float32)
14 | self.assertTrue(array_allclose(res, ans))
15 |
16 | def test_backward1(self):
17 | x_data = np.array([[-1, 1, 2], [-1, 2, 4]])
18 | self.assertTrue(gradient_check(F.relu, x_data))
19 |
20 | def test_backward2(self):
21 | np.random.seed(0)
22 | x_data = np.random.rand(10, 10) * 100
23 | self.assertTrue(gradient_check(F.relu, x_data))
24 |
25 | def test_backward3(self):
26 | np.random.seed(0)
27 | x_data = np.random.rand(10, 10, 10) * 100
28 | self.assertTrue(gradient_check(F.relu, x_data))
29 |
30 |
31 | class TestLeakyRelu(unittest.TestCase):
32 |
33 | def test_forward1(self):
34 | x = np.array([[-1, 0], [2, -3], [-2, 1]], np.float32)
35 | res = F.leaky_relu(x)
36 | ans = np.array([[-0.2, 0.], [2., -0.6], [-0.4, 1.]], np.float32)
37 | self.assertTrue(array_allclose(res, ans))
38 |
39 | def test_forward2(self):
40 | slope = 0.002
41 | x = np.random.randn(100)
42 | y2 = CF.leaky_relu(x, slope)
43 | y = F.leaky_relu(x, slope)
44 | res = array_allclose(y.data, y2.data)
45 | self.assertTrue(res)
46 |
47 | def test_backward1(self):
48 | x_data = np.array([[-1, 1, 2], [-1, 2, 4]])
49 | self.assertTrue(gradient_check(F.leaky_relu, x_data))
50 |
51 | def test_backward2(self):
52 | np.random.seed(0)
53 | x_data = np.random.rand(10, 10) * 100
54 | self.assertTrue(gradient_check(F.leaky_relu, x_data))
55 |
56 | def test_backward3(self):
57 | np.random.seed(0)
58 | x_data = np.random.rand(10, 10, 10) * 100
59 | self.assertTrue(gradient_check(F.leaky_relu, x_data))
--------------------------------------------------------------------------------
/tests/test_resnet.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import chainer
4 | import dezero
5 | from dezero.utils import array_allclose
6 | from dezero.models import VGG16
7 |
8 |
9 | class TestResnet152(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.random.randn(1, 3, 224, 224).astype('f')
13 | _model = chainer.links.ResNet152Layers(None)
14 |
15 | with chainer.using_config('train', False):
16 | with chainer.using_config('enable_backprop', False):
17 | out_layer_name = 'fc6'
18 | _y = _model.forward(x, [out_layer_name])[out_layer_name]
19 |
20 | print(_y.shape)
21 | """
22 | model = VGG16()
23 | layers = _model.available_layers
24 | for l in layers:
25 | if "conv" in l or "fc" in l:
26 | m1 = getattr(model, l)
27 | m2 = getattr(_model, l)
28 | m1.W.data = m2.W.data
29 | m1.b.data = m2.b.data
30 | if "fc" in l:
31 | m1.W.data = m1.W.data.T
32 |
33 | with dezero.test_mode():
34 | y = model(x)
35 |
36 | self.assertTrue(array_allclose(y.data, _y.data))
37 | """
38 |
39 |
40 | def test_forward2(self):
41 | x = np.random.randn(1, 3, 224, 224).astype('f')
42 | model = VGG16()
43 | y = model(x)
44 | self.assertTrue(y.dtype == np.float32)
45 |
46 | def test_backward1(self):
47 | x = np.random.randn(2, 3, 224, 224).astype('f')
48 | _model = chainer.links.VGG16Layers(None)
49 |
50 | with chainer.using_config('train', False):
51 | out_layer_name = 'fc8'
52 | _y = _model.forward(x, [out_layer_name])[out_layer_name]
53 | _y.grad = np.ones_like(_y.data)
54 | _y.backward()
55 |
56 | model = VGG16()
57 | layers = _model.available_layers
58 | for l in layers:
59 | if "conv" in l or "fc" in l:
60 | m1 = getattr(model, l)
61 | m2 = getattr(_model, l)
62 | m1.W.data = m2.W.data
63 | m1.b.data = m2.b.data
64 | if "fc" in l:
65 | m1.W.data = m1.W.data.T
66 |
67 | with dezero.test_mode():
68 | y = model(x)
69 | y.backward()
70 |
71 | layers = _model.available_layers
72 | for l in layers:
73 | if "conv" in l:
74 | m1 = getattr(model, l)
75 | m2 = getattr(_model, l)
76 | self.assertTrue(array_allclose(m1.W.data, m2.W.data))
77 | self.assertTrue(array_allclose(m1.b.data, m2.b.data))
78 | elif "fc" in l:
79 | m1 = getattr(model, l)
80 | m2 = getattr(_model, l)
81 | self.assertTrue(array_allclose(m1.W.data, m2.W.data.T))
82 | self.assertTrue(array_allclose(m1.b.data, m2.b.data))
--------------------------------------------------------------------------------
/tests/test_sigmoid.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestSigmoid(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
13 | y2 = CF.sigmoid(x)
14 | y = F.sigmoid(Variable(x))
15 | res = array_allclose(y.data, y2.data)
16 | self.assertTrue(res)
17 |
18 | def test_forward2(self):
19 | x = np.random.randn(10, 10).astype(np.float32)
20 | y2 = CF.sigmoid(x)
21 | y = F.sigmoid(Variable(x))
22 | res = array_allclose(y.data, y2.data)
23 | self.assertTrue(res)
24 |
25 | def test_backward1(self):
26 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
27 | self.assertTrue(gradient_check(F.sigmoid, x_data))
28 |
29 | def test_backward2(self):
30 | np.random.seed(0)
31 | x_data = np.random.rand(10, 10)
32 | self.assertTrue(gradient_check(F.sigmoid, x_data))
33 |
34 | def test_backward3(self):
35 | np.random.seed(0)
36 | x_data = np.random.rand(10, 10, 10)
37 | self.assertTrue(gradient_check(F.sigmoid, x_data))
--------------------------------------------------------------------------------
/tests/test_softmax.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestSoftmaxSimple(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
13 | y2 = CF.softmax(x, axis=1)
14 | y = F.softmax_simple(Variable(x))
15 | res = array_allclose(y.data, y2.data)
16 | self.assertTrue(res)
17 |
18 | def test_forward2(self):
19 | np.random.seed(0)
20 | x = np.random.rand(10, 10).astype('f')
21 | y2 = CF.softmax(x, axis=1)
22 | y = F.softmax_simple(Variable(x))
23 | res = array_allclose(y.data, y2.data)
24 | self.assertTrue(res)
25 |
26 | def test_forward3(self):
27 | np.random.seed(0)
28 | x = np.random.rand(10, 10, 10).astype('f')
29 | y2 = CF.softmax(x, axis=1)
30 | y = F.softmax_simple(Variable(x))
31 | res = array_allclose(y.data, y2.data)
32 | self.assertTrue(res)
33 |
34 | def test_backward1(self):
35 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
36 | f = lambda x: F.softmax_simple(x, axis=1)
37 | self.assertTrue(gradient_check(f, x_data))
38 |
39 | def test_backward2(self):
40 | np.random.seed(0)
41 | x_data = np.random.rand(10, 10)
42 | f = lambda x: F.softmax_simple(x, axis=1)
43 | self.assertTrue(gradient_check(f, x_data))
44 |
45 | def test_backward3(self):
46 | np.random.seed(0)
47 | x_data = np.random.rand(10, 10, 10)
48 | f = lambda x: F.softmax_simple(x, axis=1)
49 | self.assertTrue(gradient_check(f, x_data))
50 |
51 |
52 | class TestSoftmax(unittest.TestCase):
53 |
54 | def test_forward1(self):
55 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
56 | y2 = CF.softmax(x, axis=1)
57 | y = F.softmax(Variable(x))
58 | res = array_allclose(y.data, y2.data)
59 | self.assertTrue(res)
60 |
61 | def test_forward2(self):
62 | np.random.seed(0)
63 | x = np.random.rand(10, 10).astype('f')
64 | y2 = CF.softmax(x, axis=1)
65 | y = F.softmax(Variable(x))
66 | res = array_allclose(y.data, y2.data)
67 | self.assertTrue(res)
68 |
69 | def test_forward3(self):
70 | np.random.seed(0)
71 | x = np.random.rand(10, 10, 10).astype('f')
72 | y2 = CF.softmax(x, axis=1)
73 | y = F.softmax(Variable(x))
74 | res = array_allclose(y.data, y2.data)
75 | self.assertTrue(res)
76 |
77 | def test_backward1(self):
78 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
79 | f = lambda x: F.softmax(x, axis=1)
80 | self.assertTrue(gradient_check(f, x_data))
81 |
82 | def test_backward2(self):
83 | np.random.seed(0)
84 | x_data = np.random.rand(10, 10)
85 | f = lambda x: F.softmax(x, axis=1)
86 | self.assertTrue(gradient_check(f, x_data))
87 |
88 | def test_backward3(self):
89 | np.random.seed(0)
90 | x_data = np.random.rand(10, 10, 10)
91 | f = lambda x: F.softmax(x, axis=1)
92 | self.assertTrue(gradient_check(f, x_data))
93 |
94 |
95 | class TestSoftmax(unittest.TestCase):
96 |
97 | def test_forward1(self):
98 | x = np.array([[0, 1, 2], [0, 2, 4]], np.float32)
99 | y2 = CF.softmax(x, axis=1)
100 | y = F.softmax(Variable(x))
101 | res = array_allclose(y.data, y2.data)
102 | self.assertTrue(res)
103 |
104 | def test_forward2(self):
105 | np.random.seed(0)
106 | x = np.random.rand(10, 10).astype('f')
107 | y2 = CF.softmax(x, axis=1)
108 | y = F.softmax(Variable(x))
109 | res = array_allclose(y.data, y2.data)
110 | self.assertTrue(res)
111 |
112 | def test_forward3(self):
113 | np.random.seed(0)
114 | x = np.random.rand(10, 10, 10).astype('f')
115 | y2 = CF.softmax(x, axis=1)
116 | y = F.softmax(Variable(x))
117 | res = array_allclose(y.data, y2.data)
118 | self.assertTrue(res)
119 |
120 | def test_backward1(self):
121 | x_data = np.array([[0, 1, 2], [0, 2, 4]])
122 | f = lambda x: F.softmax(x, axis=1)
123 | self.assertTrue(gradient_check(f, x_data))
124 |
125 | def test_backward2(self):
126 | np.random.seed(0)
127 | x_data = np.random.rand(10, 10)
128 | f = lambda x: F.softmax(x, axis=1)
129 | self.assertTrue(gradient_check(f, x_data))
130 |
131 | def test_backward3(self):
132 | np.random.seed(0)
133 | x_data = np.random.rand(10, 10, 10)
134 | f = lambda x: F.softmax(x, axis=1)
135 | self.assertTrue(gradient_check(f, x_data))
136 |
137 |
138 | class TestLogSoftmax(unittest.TestCase):
139 |
140 | def test_forward1(self):
141 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
142 | y = F.log_softmax(x)
143 | y2 = CF.log_softmax(x)
144 | res = array_allclose(y.data, y2.data)
145 | self.assertTrue(res)
146 |
147 | def test_backward1(self):
148 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]])
149 | f = lambda x: F.log_softmax(x)
150 | self.assertTrue(gradient_check(f, x))
151 |
152 | def test_backward2(self):
153 | x = np.random.randn(10, 10)
154 | f = lambda x: F.log_softmax(x)
155 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/test_softmax_cross_entropy.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 | import chainer.functions as CF
7 |
8 |
9 | class TestSoftmaxCrossEntropy(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
13 | t = np.array([3, 0]).astype(np.int32)
14 | y = F.softmax_cross_entropy(x, t)
15 | y2 = CF.softmax_cross_entropy(x, t)
16 | res = array_allclose(y.data, y2.data)
17 | self.assertTrue(res)
18 |
19 | def test_backward1(self):
20 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
21 | t = np.array([3, 0]).astype(np.int32)
22 | f = lambda x: F.softmax_cross_entropy(x, Variable(t))
23 | self.assertTrue(gradient_check(f, x))
24 |
25 | def test_backward2(self):
26 | N, CLS_NUM = 10, 10
27 | x = np.random.randn(N, CLS_NUM)
28 | t = np.random.randint(0, CLS_NUM, (N,))
29 | f = lambda x: F.softmax_cross_entropy(x, t)
30 | self.assertTrue(gradient_check(f, x))
31 |
32 | def test_backward3(self):
33 | N, CLS_NUM = 100, 10
34 | x = np.random.randn(N, CLS_NUM)
35 | t = np.random.randint(0, CLS_NUM, (N,))
36 | f = lambda x: F.softmax_cross_entropy(x, t)
37 | self.assertTrue(gradient_check(f, x))
38 |
39 |
40 | class TestSoftmaxCrossEntropy_simple(unittest.TestCase):
41 |
42 | def test_forward1(self):
43 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
44 | t = np.array([3, 0]).astype(np.int32)
45 | y = F.softmax_cross_entropy_simple(x, t)
46 | y2 = CF.softmax_cross_entropy(x, t)
47 | res = array_allclose(y.data, y2.data)
48 | self.assertTrue(res)
49 |
50 | def test_backward1(self):
51 | x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32)
52 | t = np.array([3, 0]).astype(np.int32)
53 | f = lambda x: F.softmax_cross_entropy_simple(x, Variable(t))
54 | self.assertTrue(gradient_check(f, x))
55 |
56 | def test_backward2(self):
57 | N, CLS_NUM = 10, 10
58 | x = np.random.randn(N, CLS_NUM)
59 | t = np.random.randint(0, CLS_NUM, (N,))
60 | f = lambda x: F.softmax_cross_entropy_simple(x, t)
61 | self.assertTrue(gradient_check(f, x))
62 |
63 | def test_backward3(self):
64 | N, CLS_NUM = 100, 10
65 | x = np.random.randn(N, CLS_NUM)
66 | t = np.random.randint(0, CLS_NUM, (N,))
67 | f = lambda x: F.softmax_cross_entropy_simple(x, t)
68 | self.assertTrue(gradient_check(f, x))
--------------------------------------------------------------------------------
/tests/test_sum.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check, array_allclose
6 |
7 |
8 | class TestSum(unittest.TestCase):
9 |
10 | def test_datatype(self):
11 | x = Variable(np.random.rand(10))
12 | y = F.sum(x)
13 | # 不返回np.float64，而是返回0维的ndarray
14 | self.assertFalse(np.isscalar(y))
15 |
16 | def test_forward1(self):
17 | x = Variable(np.array(2.0))
18 | y = F.sum(x)
19 | expected = np.sum(x.data)
20 | self.assertTrue(array_allclose(y.data, expected))
21 |
22 | def test_forward2(self):
23 | x = Variable(np.random.rand(10, 20, 30))
24 | y = F.sum(x, axis=1)
25 | expected = np.sum(x.data, axis=1)
26 | self.assertTrue(array_allclose(y.data, expected))
27 |
28 | def test_forward3(self):
29 | x = Variable(np.random.rand(10, 20, 30))
30 | y = F.sum(x, axis=1, keepdims=True)
31 | expected = np.sum(x.data, axis=1, keepdims=True)
32 | self.assertTrue(array_allclose(y.data, expected))
33 |
34 | def test_backward1(self):
35 | x_data = np.random.rand(10)
36 | f = lambda x: F.sum(x)
37 | self.assertTrue(gradient_check(f, x_data))
38 |
39 | def test_backward2(self):
40 | x_data = np.random.rand(10, 10)
41 | f = lambda x: F.sum(x, axis=1)
42 | self.assertTrue(gradient_check(f, x_data))
43 |
44 | def test_backward3(self):
45 | x_data = np.random.rand(10, 20, 20)
46 | f = lambda x: F.sum(x, axis=2)
47 | self.assertTrue(gradient_check(f, x_data))
48 |
49 | def test_backward4(self):
50 | x_data = np.random.rand(10, 20, 20)
51 | f = lambda x: F.sum(x, axis=None)
52 | self.assertTrue(gradient_check(f, x_data))
53 |
54 |
55 | class TestSumTo(unittest.TestCase):
56 |
57 | def test_forward1(self):
58 | x = Variable(np.random.rand(10))
59 | y = F.sum_to(x, (1,))
60 | expected = np.sum(x.data)
61 | self.assertTrue(array_allclose(y.data, expected))
62 |
63 | def test_forward2(self):
64 | x = Variable(np.array([[1., 2., 3.], [4., 5., 6.]]))
65 | y = F.sum_to(x, (1, 3))
66 | expected = np.sum(x.data, axis=0, keepdims=True)
67 | self.assertTrue(array_allclose(y.data, expected))
68 |
69 | def test_forward3(self):
70 | x = Variable(np.random.rand(10))
71 | y = F.sum_to(x, (10,))
72 | expected = x.data # 由于形状相同，不做任何处理
73 | self.assertTrue(array_allclose(y.data, expected))
74 |
75 | def test_backward1(self):
76 | x_data = np.random.rand(10)
77 | f = lambda x: F.sum_to(x, (1,))
78 | self.assertTrue(gradient_check(f, x_data))
79 |
80 | def test_backward2(self):
81 | x_data = np.random.rand(10, 10) * 10
82 | f = lambda x: F.sum_to(x, (10,))
83 | self.assertTrue(gradient_check(f, x_data))
84 |
85 | def test_backward3(self):
86 | x_data = np.random.rand(10, 20, 20) * 100
87 | f = lambda x: F.sum_to(x, (10,))
88 | self.assertTrue(gradient_check(f, x_data))
89 |
90 | def test_backward4(self):
91 | x_data = np.random.rand(10)
92 | f = lambda x: F.sum_to(x, (10,)) + 1
93 | self.assertTrue(gradient_check(f, x_data))
--------------------------------------------------------------------------------
/tests/test_transpose.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | from dezero import Variable
4 | import dezero.functions as F
5 | from dezero.utils import gradient_check
6 |
7 |
8 | class TestTranspose(unittest.TestCase):
9 |
10 | def test_forward1(self):
11 | x = Variable(np.array([[1, 2, 3], [4, 5, 6]]))
12 | y = F.transpose(x)
13 | self.assertEqual(y.shape, (3, 2))
14 |
15 | def test_backward1(self):
16 | x = np.array([[1, 2, 3], [4, 5, 6]])
17 | self.assertTrue(gradient_check(F.transpose, x))
18 |
19 | def test_backward2(self):
20 | x = np.array([1, 2, 3])
21 | self.assertTrue(gradient_check(F.transpose, x))
22 |
23 | def test_backward3(self):
24 | x = np.random.randn(10, 5)
25 | self.assertTrue(gradient_check(F.transpose, x))
26 |
27 | def test_backward4(self):
28 | x = np.array([1, 2])
29 | self.assertTrue(gradient_check(F.transpose, x))
--------------------------------------------------------------------------------
/tests/test_vgg16.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import chainer
4 | import dezero
5 | from dezero.utils import array_allclose
6 | from dezero.models import VGG16
7 |
8 |
9 | class TestVGG16(unittest.TestCase):
10 |
11 | def test_forward1(self):
12 | x = np.random.randn(1, 3, 224, 224).astype('f')
13 | _model = chainer.links.VGG16Layers(None)
14 |
15 | with chainer.using_config('train', False):
16 | with chainer.using_config('enable_backprop', False):
17 | out_layer_name = 'fc8'
18 | _y = _model.forward(x, [out_layer_name])[out_layer_name]
19 |
20 | model = VGG16()
21 | layers = _model.available_layers
22 | for l in layers:
23 | if "conv" in l or "fc" in l:
24 | m1 = getattr(model, l)
25 | m2 = getattr(_model, l)
26 | m1.W.data = m2.W.data
27 | m1.b.data = m2.b.data
28 | if "fc" in l:
29 | m1.W.data = m1.W.data.T
30 |
31 | with dezero.test_mode():
32 | y = model(x)
33 |
34 | self.assertTrue(array_allclose(y.data, _y.data))
35 |
36 |
37 | def test_forward2(self):
38 | x = np.random.randn(1, 3, 224, 224).astype('f')
39 | model = VGG16()
40 | y = model(x)
41 | self.assertTrue(y.dtype == np.float32)
42 |
43 | def test_backward1(self):
44 | x = np.random.randn(2, 3, 224, 224).astype('f')
45 | _model = chainer.links.VGG16Layers(None)
46 |
47 | with chainer.using_config('train', False):
48 | out_layer_name = 'fc8'
49 | _y = _model.forward(x, [out_layer_name])[out_layer_name]
50 | _y.grad = np.ones_like(_y.data)
51 | _y.backward()
52 |
53 | model = VGG16()
54 | layers = _model.available_layers
55 | for l in layers:
56 | if "conv" in l or "fc" in l:
57 | m1 = getattr(model, l)
58 | m2 = getattr(_model, l)
59 | m1.W.data = m2.W.data
60 | m1.b.data = m2.b.data
61 | if "fc" in l:
62 | m1.W.data = m1.W.data.T
63 |
64 | with dezero.test_mode():
65 | y = model(x)
66 | y.backward()
67 |
68 | layers = _model.available_layers
69 | for l in layers:
70 | if "conv" in l:
71 | m1 = getattr(model, l)
72 | m2 = getattr(_model, l)
73 | self.assertTrue(array_allclose(m1.W.data, m2.W.data))
74 | self.assertTrue(array_allclose(m1.b.data, m2.b.data))
75 | elif "fc" in l:
76 | m1 = getattr(model, l)
77 | m2 = getattr(_model, l)
78 | self.assertTrue(array_allclose(m1.W.data, m2.W.data.T))
79 | self.assertTrue(array_allclose(m1.b.data, m2.b.data))
--------------------------------------------------------------------------------
/tests/test_weight_decay.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import numpy as np
3 | import dezero
4 | import dezero.functions as F
5 | from dezero.utils import array_allclose
6 |
7 |
8 | class TestWeightDecay(unittest.TestCase):
9 |
10 | def test_compare1(self):
11 | rate = 0.4
12 | x = np.random.rand(10, 2)
13 | t = np.zeros((10)).astype(int)
14 | layer = dezero.layers.Linear(in_size=2, out_size=3, nobias=True)
15 | layer.W.data = np.ones_like(layer.W.data)
16 | optimizer = dezero.optimizers.SGD().setup(layer)
17 | optimizer.add_hook(dezero.optimizers.WeightDecay(rate=rate))
18 |
19 | layer.cleargrads()
20 | y = layer(x)
21 | y = F.softmax_cross_entropy(y, t)
22 | y.backward()
23 | optimizer.update()
24 | W0 = layer.W.data.copy()
25 |
26 | layer.W.data = np.ones_like(layer.W.data)
27 | optimizer.hooks.clear()
28 | layer.cleargrads()
29 | y = layer(x)
30 | y = F.softmax_cross_entropy(y, t) + rate / 2 * (layer.W ** 2).sum()
31 | y.backward()
32 | optimizer.update()
33 | W1 = layer.W.data
34 | self.assertTrue(array_allclose(W0, W1))
35 |
--------------------------------------------------------------------------------