├── src ├── network │ ├── __init__.py │ ├── densenet.py │ ├── resnet.py │ ├── pyramid.py │ ├── resnext.py │ └── shakenet.py ├── mylib │ ├── functions │ │ ├── noise │ │ │ ├── __init__.py │ │ │ └── shake.py │ │ ├── connection │ │ │ ├── __init__.py │ │ │ └── grouped_convolution_2d.py │ │ └── __init__.py │ ├── links │ │ ├── connection │ │ │ ├── __init__.py │ │ │ └── grouped_convolution_2d.py │ │ └── __init__.py │ ├── training │ │ ├── trigger │ │ │ ├── __init__.py │ │ │ └── interval.py │ │ ├── __init__.py │ │ ├── extensions │ │ │ ├── log_report.py │ │ │ ├── plot_report.py │ │ │ ├── __init__.py │ │ │ ├── step_shift.py │ │ │ ├── network_size.py │ │ │ ├── cosine_shift.py │ │ │ ├── print_report.py │ │ │ ├── bestshot.py │ │ │ └── computational_graph.py │ │ └── updater.py │ ├── __init__.py │ └── datasets │ │ ├── __init__.py │ │ ├── mnist.py │ │ └── cifar.py ├── train.py └── report.py ├── result └── result.txt └── README.md /src/network/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/mylib/functions/noise/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/mylib/links/connection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/mylib/functions/connection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /result/result.txt: -------------------------------------------------------------------------------- 1 | Results are saved in this directory. 2 | -------------------------------------------------------------------------------- /src/mylib/training/trigger/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib.training.trigger.interval import IntervalTrigger -------------------------------------------------------------------------------- /src/mylib/links/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib.links.connection.grouped_convolution_2d import GroupedConvolution2D 2 | -------------------------------------------------------------------------------- /src/mylib/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib import datasets 2 | from mylib import training 3 | from mylib import functions 4 | from mylib import links 5 | -------------------------------------------------------------------------------- /src/mylib/training/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib.training import extensions 2 | from mylib.training import trigger 3 | 4 | from mylib.training.updater import StandardUpdater -------------------------------------------------------------------------------- /src/mylib/functions/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib.functions.connection.grouped_convolution_2d import grouped_convolution_2d 2 | from mylib.functions.noise.shake import shake_noise 3 | -------------------------------------------------------------------------------- /src/mylib/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib.datasets.cifar import get_cifar10 2 | from mylib.datasets.cifar import get_cifar100 3 | from mylib.datasets.mnist import get_mnist 4 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/log_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from chainer.training import extensions 5 | 6 | class LogReport(extensions.LogReport): 7 | def serialize(self, serializer): 8 | super().serialize(serializer) 9 | self._trigger.serialize(serializer['_trigger']) 10 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/plot_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from chainer.training import extensions 5 | 6 | class PlotReport(extensions.PlotReport): 7 | def serialize(self, serializer): 8 | super().serialize(serializer) 9 | self._trigger.serialize(serializer['_trigger']) 10 | -------------------------------------------------------------------------------- /src/mylib/datasets/mnist.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import chainer 5 | 6 | def get_mnist(): 7 | train_data, test_data = chainer.datasets.get_mnist() 8 | 9 | images, labels = train_data._datasets 10 | images = images.reshape(images.shape[0], 1, 28, 28) 11 | train_data._datasets = (images, labels) 12 | 13 | images, labels = test_data._datasets 14 | images = images.reshape(images.shape[0], 1, 28, 28) 15 | test_data._datasets = (images, labels) 16 | 17 | return train_data, test_data 18 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | from mylib.training.extensions.log_report import LogReport 2 | from mylib.training.extensions.plot_report import PlotReport 3 | from mylib.training.extensions.print_report import PrintReport 4 | 5 | from mylib.training.extensions.bestshot import Bestshot 6 | from mylib.training.extensions.computational_graph import dump_graph 7 | from mylib.training.extensions.network_size import dump_network_size 8 | 9 | from mylib.training.extensions.cosine_shift import CosineShift 10 | from mylib.training.extensions.step_shift import StepShift -------------------------------------------------------------------------------- /src/mylib/training/extensions/step_shift.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import chainer 5 | 6 | class StepShift(chainer.training.extension.Extension): 7 | def __init__(self, attr, value, period, optimizer=None): 8 | self._attr = attr 9 | self._value = value 10 | self._period = period 11 | self._optimizer = optimizer 12 | 13 | def initialize(self, trainer): 14 | self._update_value(trainer) 15 | 16 | def __call__(self, trainer): 17 | self._update_value(trainer) 18 | 19 | def _update_value(self, trainer): 20 | optimizer = self._optimizer or trainer.updater.get_optimizer('main') 21 | current = self._period - trainer.updater.epoch 22 | 23 | if current <= self._period // 4: 24 | value = self._value * 0.01 25 | elif current <= self._period // 2: 26 | value = self._value * 0.1 27 | else: 28 | value = self._value 29 | 30 | setattr(optimizer, self._attr, value) 31 | 32 | -------------------------------------------------------------------------------- /src/mylib/training/trigger/interval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import json 5 | import chainer 6 | 7 | class IntervalTrigger(object): 8 | def __init__(self, period, unit): 9 | assert unit == 'epoch' or unit == 'iteration' 10 | 11 | self._period = period 12 | self._unit = unit 13 | self._count = 0 14 | 15 | def __call__(self, trainer): 16 | updater = trainer.updater 17 | 18 | if self._unit == 'epoch': 19 | count = updater.epoch // self._period 20 | 21 | if count != self._count: 22 | self._count = count 23 | return True 24 | else: 25 | return False 26 | 27 | else: 28 | return updater.iteration > 0 and updater.iteration % self._period == 0 29 | 30 | def serialize(self, serializer): 31 | if isinstance(serializer, chainer.serializer.Serializer): 32 | serializer('_count', json.dumps(self._count)) 33 | else: 34 | self._count = json.loads(serializer('_count', '')) 35 | 36 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/network_size.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import functools 5 | import os 6 | import chainer 7 | 8 | def dump_network_size(filename='network_size.txt'): 9 | def trigger(trainer): 10 | return trainer.updater.iteration == 1 11 | 12 | @chainer.training.extension.make_extension(trigger=trigger) 13 | def dump_network_size(trainer): 14 | model = trainer.updater.get_optimizer('main').target 15 | network_size = {} 16 | 17 | for n, v in model.namedparams(): 18 | size = functools.reduce(lambda x, y: x * y, v.shape) 19 | names = n.split('/') 20 | names = ['/'] + ['/'.join(names[:i + 1]) for i in range(1, len(names))] 21 | 22 | for name in names: 23 | if name not in network_size: 24 | network_size[name] = 0 25 | 26 | network_size[name] += size 27 | 28 | network_size = [(n, s) for n, s in network_size.items()] 29 | network_size.sort(key=lambda x: x[0]) 30 | 31 | path = os.path.join(trainer.out, filename) 32 | 33 | with open(path, 'w') as handle: 34 | for v in network_size: 35 | handle.write("{}: {}\n".format(*v)) 36 | 37 | return dump_network_size 38 | 39 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/cosine_shift.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import math 5 | import chainer 6 | 7 | class CosineShift(chainer.training.extension.Extension): 8 | def __init__(self, attr, value, period, period_mult=1, optimizer=None): 9 | self._attr = attr 10 | self._value = value 11 | self._period = period 12 | self._period_mult = period_mult 13 | self._optimizer = optimizer 14 | 15 | if not hasattr(self._value, '__getitem__'): 16 | self._value = (self._value, 0) 17 | 18 | def initialize(self, trainer): 19 | self._update_value(trainer) 20 | 21 | def __call__(self, trainer): 22 | self._update_value(trainer) 23 | 24 | def _update_value(self, trainer): 25 | optimizer = self._optimizer or trainer.updater.get_optimizer('main') 26 | epoch = trainer.updater.epoch 27 | 28 | period_range = self._period 29 | period_start = 0 30 | period_end = period_range 31 | 32 | while period_end <= epoch: 33 | period_start = period_end 34 | period_range *= self._period_mult 35 | period_end += period_range 36 | 37 | n_max, n_min = self._value 38 | t_cur = epoch - period_start 39 | t_i = period_range 40 | value = n_min + 0.5 * (n_max - n_min) * (1 + math.cos((t_cur / t_i) * math.pi)) 41 | 42 | setattr(optimizer, self._attr, value) 43 | 44 | -------------------------------------------------------------------------------- /src/mylib/functions/noise/shake.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from chainer import cuda 5 | from chainer import configuration 6 | from chainer import function 7 | from chainer.utils import type_check 8 | 9 | class ShakeNoiseFunction(function.Function): 10 | def check_type_forward(self, in_types): 11 | type_check.expect(in_types.size() == 2) 12 | type_check.expect(in_types[0].dtype.kind == 'f') 13 | type_check.expect(in_types[1].dtype.kind == 'f') 14 | 15 | def forward(self, inputs): 16 | xp = cuda.get_array_module(*inputs) 17 | x1, x2 = inputs 18 | 19 | mask1 = xp.random.randint(0, 2, x1.shape[0]).astype(xp.float32) 20 | mask2 = 1 - mask1 21 | 22 | x1 = x1 * mask1[:, xp.newaxis, xp.newaxis, xp.newaxis] 23 | x2 = x2 * mask2[:, xp.newaxis, xp.newaxis, xp.newaxis] 24 | 25 | return x1 + x2, 26 | 27 | def backward(self, inputs, grad): 28 | xp = cuda.get_array_module(*inputs) 29 | g = grad[0] 30 | 31 | mask1 = xp.random.randint(0, 2, g.shape[0]).astype(xp.float32) 32 | mask2 = 1 - mask1 33 | 34 | g1 = g * mask1[:, xp.newaxis, xp.newaxis, xp.newaxis] 35 | g2 = g * mask2[:, xp.newaxis, xp.newaxis, xp.newaxis] 36 | 37 | return g1, g2 38 | 39 | 40 | def shake_noise(x1, x2): 41 | if configuration.config.train: 42 | return ShakeNoiseFunction()(x1, x2) 43 | else: 44 | return (x1 + x2) / 2 45 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/print_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import os 6 | from chainer.training import extensions 7 | from chainer.training.extensions import log_report as log_report_module 8 | from chainer.training.extensions import util 9 | 10 | class PrintReport(extensions.PrintReport): 11 | def __call__(self, trainer): 12 | if isinstance(self._out, str): 13 | self._out = open(os.path.join(trainer.out, self._out), 'w') 14 | 15 | out = self._out 16 | 17 | if self._header: 18 | out.write(self._header) 19 | out.flush() 20 | self._header = None 21 | 22 | log_report = self._log_report 23 | 24 | if isinstance(log_report, str): 25 | log_report = trainer.get_extension(log_report) 26 | elif not isinstance(log_report, log_report_module.LogReport): 27 | raise TypeError('log report has a wrong type %s' % type(log_report)) 28 | 29 | log = log_report.log 30 | log_len = self._log_len 31 | 32 | while len(log) > log_len: 33 | if out == sys.stdout: 34 | # delete the printed contents from the current cursor 35 | if os.name == 'nt': 36 | util.erase_console(0, 0) 37 | else: 38 | out.write('\033[J') 39 | 40 | self._print(log[log_len]) 41 | out.flush() 42 | 43 | log_len += 1 44 | 45 | self._log_len = log_len 46 | -------------------------------------------------------------------------------- /src/mylib/training/updater.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import math 5 | import chainer 6 | from chainer.dataset import convert 7 | 8 | class StandardUpdater(chainer.training.StandardUpdater): 9 | def __init__(self, iterator, optimizer, converter=convert.concat_examples, 10 | device=None, loss_func=None, procsize=None): 11 | super().__init__(iterator, optimizer, converter, device, loss_func) 12 | self.procsize = procsize 13 | 14 | def update_core(self): 15 | batch = self.get_iterator('main').next() 16 | optimizer = self.get_optimizer('main') 17 | loss_func = self.loss_func or optimizer.target 18 | 19 | repeats = max(math.ceil(len(batch) / self.procsize), 1) 20 | 21 | optimizer.target.cleargrads() 22 | 23 | for i in range(repeats): 24 | start = len(batch) * i // repeats 25 | end = len(batch) * (i + 1) // repeats 26 | in_arrays = self.converter(batch[start:end], self.device) 27 | 28 | with chainer.function.force_backprop_mode(): 29 | if isinstance(in_arrays, tuple): 30 | loss = loss_func(*in_arrays) 31 | elif isinstance(in_arrays, dict): 32 | loss = loss_func(**in_arrays) 33 | else: 34 | loss = loss_func(in_arrays) 35 | 36 | loss.backward() 37 | 38 | if repeats != 1: 39 | for _, v in optimizer.target.namedparams(): 40 | grad = v.grad 41 | 42 | if grad is not None: 43 | grad /= repeats 44 | v.grad = grad 45 | 46 | optimizer.update() 47 | 48 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/bestshot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import json 6 | import shutil 7 | import tempfile 8 | import chainer 9 | 10 | from chainer.serializers import npz 11 | 12 | class Bestshot(chainer.training.extension.Extension): 13 | def __init__(self, savefun=npz.save_npz, filename='bestshot.npz', trigger=(1, 'epoch'), 14 | key='validation/main/accuracy', comp='max'): 15 | self._savefun = savefun 16 | self._filename = filename 17 | self._trigger = chainer.training.trigger.get_trigger(trigger) 18 | self._key = key 19 | self._comp = comp 20 | self._value = None 21 | 22 | if self._comp == 'min': 23 | self._comp = lambda x, y: y - x 24 | elif self._comp == 'max': 25 | self._comp = lambda x, y: x - y 26 | 27 | self._init_summary() 28 | 29 | def __call__(self, trainer): 30 | if self._key in trainer.observation: 31 | self._summary.add({self._key: trainer.observation[self._key]}) 32 | 33 | if self._trigger(trainer): 34 | stats = self._summary.compute_mean() 35 | value = float(stats[self._key]) 36 | 37 | if self._value is None or self._comp(value, self._value) > 0: 38 | self._value = value 39 | self._save(trainer) 40 | 41 | self._init_summary() 42 | 43 | def _save(self, trainer): 44 | filename = self._filename.format(trainer) 45 | prefix = 'tmp' + filename 46 | 47 | fd, tmppath = tempfile.mkstemp(prefix=prefix, dir=trainer.out) 48 | 49 | try: 50 | self._savefun(tmppath, trainer) 51 | except Exception: 52 | os.close(fd) 53 | os.remove(tmppath) 54 | raise 55 | 56 | os.close(fd) 57 | shutil.move(tmppath, os.path.join(trainer.out, filename)) 58 | 59 | def serialize(self, serializer): 60 | self._value = json.loads(serializer('value', json.dumps(self._value))) 61 | self._trigger.serialize(serializer['trigger']) 62 | 63 | def _init_summary(self): 64 | self._summary = chainer.reporter.DictSummary() 65 | -------------------------------------------------------------------------------- /src/mylib/links/connection/grouped_convolution_2d.py: -------------------------------------------------------------------------------- 1 | from mylib.functions import grouped_convolution_2d 2 | from chainer import initializers 3 | from chainer import link 4 | from chainer.utils import argument 5 | from chainer import variable 6 | 7 | def _pair(x): 8 | if hasattr(x, '__getitem__'): 9 | return x 10 | return x, x 11 | 12 | 13 | class GroupedConvolution2D(link.Link): 14 | def __init__(self, in_channels, out_channels, units, ksize=None, stride=1, pad=0, 15 | nobias=False, initialW=None, initial_bias=None, **kwargs): 16 | super().__init__() 17 | 18 | argument.check_unexpected_kwargs(kwargs, deterministic="deterministic argument is not " 19 | "supported anymore. " 20 | "Use chainer.using_config('cudnn_deterministic', value) " 21 | "context where value is either `True` or `False`.") 22 | argument.assert_kwargs_empty(kwargs) 23 | 24 | if ksize is None: 25 | out_channels, ksize, in_channels = in_channels, out_channels, None 26 | 27 | self.units = units 28 | self.ksize = ksize 29 | self.stride = _pair(stride) 30 | self.pad = _pair(pad) 31 | self.out_channels = out_channels 32 | 33 | with self.init_scope(): 34 | W_initializer = initializers._get_initializer(initialW) 35 | self.W = variable.Parameter(W_initializer) 36 | 37 | if in_channels is not None: 38 | self._initialize_params(in_channels) 39 | 40 | if nobias: 41 | self.b = None 42 | else: 43 | if initial_bias is None: 44 | initial_bias = 0 45 | 46 | bias_initializer = initializers._get_initializer(initial_bias) 47 | self.b = variable.Parameter(bias_initializer, (units, out_channels // units)) 48 | 49 | def _initialize_params(self, in_channels): 50 | kh, kw = _pair(self.ksize) 51 | W_shape = (self.units, self.out_channels // self.units, in_channels // self.units, kh, kw) 52 | self.W.initialize(W_shape) 53 | 54 | def __call__(self, x): 55 | if self.W.data is None: 56 | self._initialize_params(x.shape[1]) 57 | 58 | return grouped_convolution_2d(x, self.W, self.b, self.stride, self.pad) 59 | 60 | -------------------------------------------------------------------------------- /src/mylib/datasets/cifar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import numpy 5 | import chainer 6 | 7 | class CifarDataset(chainer.dataset.DatasetMixin): 8 | def __init__(self, images, labels, pad=0, flip=False): 9 | self._pad = pad 10 | self._flip = flip 11 | 12 | if self._pad > 0: 13 | sides = ((0, 0), (0, 0), (self._pad, self._pad), (self._pad, self._pad)) 14 | images = numpy.pad(images, sides, 'constant', constant_values=0) 15 | 16 | self._images = images 17 | self._labels = labels 18 | 19 | def __len__(self): 20 | return len(self._images) 21 | 22 | def get_example(self, i): 23 | image = self._images[i] 24 | label = self._labels[i] 25 | 26 | if self._pad > 0: 27 | r = numpy.random.randint(0, self._pad * 2 + 1, 2) 28 | s = (image.shape[1] - self._pad * 2, image.shape[2] - self._pad * 2) 29 | 30 | image = image[:, r[0]:r[0] + s[0], r[1]:r[1] + s[1]] 31 | 32 | if self._flip and numpy.random.randint(2) == 1: 33 | image = image[:, :, ::-1] 34 | 35 | return image, label 36 | 37 | 38 | def get_cifar10(): 39 | '''This function creates a cifar10 data set. 40 | All images in this data set are normalized. 41 | In addition, train images are augmented by random-padding-clipping and flipping. 42 | (This augmentation is standard of cifat10 benchmarks in 2015-2017) 43 | ''' 44 | train, test = chainer.datasets.cifar.get_cifar10() 45 | 46 | train_dataset = CifarDataset(train._datasets[0], train._datasets[1], pad=4, flip=True) 47 | test_dataset = CifarDataset(test._datasets[0], test._datasets[1], pad=0, flip=False) 48 | 49 | return train_dataset, test_dataset 50 | 51 | 52 | def get_cifar100(): 53 | '''This function creates a cifar100 data set. 54 | All images in this data set are normalized. 55 | In addition, train images are augmented by random-padding-clipping and flipping. 56 | ''' 57 | train, test = chainer.datasets.cifar.get_cifar100() 58 | 59 | train_dataset = CifarDataset(train._datasets[0], train._datasets[1], pad=4, flip=True) 60 | test_dataset = CifarDataset(test._datasets[0], test._datasets[1], pad=0, flip=False) 61 | 62 | return train_dataset, test_dataset 63 | 64 | -------------------------------------------------------------------------------- /src/network/densenet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Residual Network (pre-activation)モジュール。 6 | Residual Block : BN - Conv(3x3) - BN - ReLU - Conv(3x3) - BN 7 | @author: Atsushi TAKEDA 8 | ''' 9 | import chainer 10 | 11 | 12 | def reshape(x, channels): 13 | if x.shape[1] < channels: 14 | xp = chainer.cuda.get_array_module(x) 15 | p = xp.zeros((x.shape[0], channels - x.shape[1], x.shape[2], x.shape[3]), dtype=x.dtype) 16 | x = chainer.functions.concat((x, p), axis=1) 17 | elif x.shape[1] > channels: 18 | x = x[:, :channels, :] 19 | 20 | return x 21 | 22 | 23 | class DenseUnit(chainer.Chain): 24 | def __init__(self, in_channels, out_channels): 25 | super().__init__(norm0=chainer.links.BatchNormalization(in_channels), 26 | conv1=chainer.links.Convolution2D(in_channels, out_channels, 3, pad=1), 27 | norm1=chainer.links.BatchNormalization(out_channels)) 28 | 29 | def __call__(self, x): 30 | x = self.norm0(x) 31 | x = chainer.functions.relu(x) 32 | x = self.conv1(x) 33 | x = self.norm1(x) 34 | 35 | return x 36 | 37 | 38 | class DenseBlock(chainer.ChainList): 39 | def __init__(self, in_channels, growth, depth): 40 | units = [DenseUnit(in_channels + growth * i, growth) for i in range(depth)] 41 | 42 | super().__init__(*units) 43 | 44 | def __call__(self, x): 45 | for layer in self: 46 | y = layer(x) 47 | x = chainer.functions.concat((x, y), axis=1) 48 | 49 | return x 50 | 51 | 52 | class Network(chainer.Chain): 53 | def __init__(self, category, params): 54 | depth, growth = params 55 | depth = (depth - 2) // 3 56 | 57 | super().__init__(input=chainer.links.Convolution2D(None, 16, 3, pad=1), 58 | norm=chainer.links.BatchNormalization(16), 59 | block1=DenseBlock(16 + growth * depth * 0, growth, depth), 60 | conv1=chainer.links.Convolution2D(16 + growth * depth * 1, 16 + growth * depth * 1, 1), 61 | block2=DenseBlock(16 + growth * depth * 1, growth, depth), 62 | conv2=chainer.links.Convolution2D(16 + growth * depth * 2, 16 + growth * depth * 2, 1), 63 | block3=DenseBlock(16 + growth * depth * 2, growth, depth), 64 | output=chainer.links.Linear(16 + growth * depth * 3, category)) 65 | 66 | def __call__(self, x): 67 | x = self.input(x) 68 | x = self.norm(x) 69 | 70 | x = self.block1(x) 71 | x = self.conv1(x) 72 | 73 | x = chainer.functions.average_pooling_2d(x, 2) 74 | x = self.block2(x) 75 | x = self.conv2(x) 76 | 77 | x = chainer.functions.average_pooling_2d(x, 2) 78 | x = self.block3(x) 79 | 80 | x = chainer.functions.relu(x) 81 | x = chainer.functions.average_pooling_2d(x, x.shape[2]) 82 | x = self.output(x) 83 | 84 | return x 85 | 86 | -------------------------------------------------------------------------------- /src/network/resnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Residual Network (pre-activation)モジュール。 6 | Residual Block : BN - Conv(3x3) - BN - ReLU - Conv(3x3) - BN 7 | @author: Atsushi TAKEDA 8 | ''' 9 | import chainer 10 | 11 | 12 | def reshape(x, channels): 13 | if x.shape[1] < channels: 14 | xp = chainer.cuda.get_array_module(x) 15 | p = xp.zeros((x.shape[0], channels - x.shape[1], x.shape[2], x.shape[3]), dtype=x.dtype) 16 | x = chainer.functions.concat((x, p), axis=1) 17 | elif x.shape[1] > channels: 18 | x = x[:, :channels, :] 19 | 20 | return x 21 | 22 | 23 | class ResidualUnit(chainer.Chain): 24 | def __init__(self, in_channels, out_channels): 25 | super().__init__(norm0=chainer.links.BatchNormalization(in_channels), 26 | conv1=chainer.links.Convolution2D(in_channels, out_channels, 3, pad=1), 27 | norm1=chainer.links.BatchNormalization(out_channels), 28 | conv2=chainer.links.Convolution2D(out_channels, out_channels, 3, pad=1), 29 | norm2=chainer.links.BatchNormalization(out_channels)) 30 | 31 | def __call__(self, x): 32 | x = self.norm0(x) 33 | x = self.conv1(x) 34 | x = self.norm1(x) 35 | x = chainer.functions.relu(x) 36 | x = self.conv2(x) 37 | x = self.norm2(x) 38 | 39 | return x 40 | 41 | 42 | class ResidualBlock(chainer.ChainList): 43 | def __init__(self, in_channels, out_channels, depth): 44 | units = [ResidualUnit(in_channels, out_channels)] 45 | units += [ResidualUnit(out_channels, out_channels) for _ in range(depth - 1)] 46 | 47 | super().__init__(*units) 48 | 49 | def __call__(self, x): 50 | for layer in self: 51 | y = layer(x) 52 | y += reshape(x, y.shape[1]) 53 | x = y 54 | 55 | return x 56 | 57 | 58 | class Network(chainer.Chain): 59 | def __init__(self, category, params): 60 | depth, width = params 61 | depth = (depth - 2) // 6 62 | width = width * 16 63 | 64 | super().__init__(input=chainer.links.Convolution2D(None, width, 3, pad=1), 65 | norm=chainer.links.BatchNormalization(width), 66 | block1=ResidualBlock(width * 1, width * 1, depth), 67 | block2=ResidualBlock(width * 1, width * 2, depth), 68 | block3=ResidualBlock(width * 2, width * 4, depth), 69 | output=chainer.links.Linear(width * 4, category)) 70 | 71 | def __call__(self, x): 72 | x = self.input(x) 73 | x = self.norm(x) 74 | 75 | x = self.block1(x) 76 | 77 | x = chainer.functions.average_pooling_2d(x, 2) 78 | x = self.block2(x) 79 | 80 | x = chainer.functions.average_pooling_2d(x, 2) 81 | x = self.block3(x) 82 | 83 | x = chainer.functions.relu(x) 84 | x = chainer.functions.average_pooling_2d(x, x.shape[2]) 85 | x = self.output(x) 86 | 87 | return x 88 | 89 | -------------------------------------------------------------------------------- /src/network/pyramid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Residual Network (pre-activation)モジュール。 6 | Residual Block : BN - Conv(3x3) - BN - ReLU - Conv(3x3) - BN 7 | @author: Atsushi TAKEDA 8 | ''' 9 | import chainer 10 | 11 | def reshape(x, channels): 12 | if x.shape[1] < channels: 13 | xp = chainer.cuda.get_array_module(x) 14 | p = xp.zeros((x.shape[0], channels - x.shape[1], x.shape[2], x.shape[3]), dtype=x.dtype) 15 | x = chainer.functions.concat((x, p), axis=1) 16 | elif x.shape[1] > channels: 17 | x = x[:, :channels, :] 18 | 19 | return x 20 | 21 | 22 | class ResidualUnit(chainer.Chain): 23 | def __init__(self, in_channels, out_channels): 24 | super().__init__(norm0=chainer.links.BatchNormalization(in_channels), 25 | conv1=chainer.links.Convolution2D(in_channels, out_channels, 3, pad=1), 26 | norm1=chainer.links.BatchNormalization(out_channels), 27 | conv2=chainer.links.Convolution2D(out_channels, out_channels, 3, pad=1), 28 | norm2=chainer.links.BatchNormalization(out_channels)) 29 | 30 | def __call__(self, x): 31 | x = self.norm0(x) 32 | x = self.conv1(x) 33 | x = self.norm1(x) 34 | x = chainer.functions.relu(x) 35 | x = self.conv2(x) 36 | x = self.norm2(x) 37 | 38 | return x 39 | 40 | 41 | class ResidualBlock(chainer.ChainList): 42 | def __init__(self, in_channels, out_channels, depth): 43 | channels = [int((in_channels * (depth - i) + out_channels * i) / depth) for i in range(depth + 1)] 44 | 45 | super().__init__(*[ResidualUnit(channels[i], channels[i + 1]) for i in range(depth)]) 46 | 47 | def __call__(self, x): 48 | for layer in self: 49 | y = layer(x) 50 | y += reshape(x, y.shape[1]) 51 | x = y 52 | 53 | return x 54 | 55 | 56 | class Network(chainer.Chain): 57 | def __init__(self, category, params): 58 | depth, alpha = params 59 | depth = (depth - 2) // 6 60 | 61 | super().__init__(input=chainer.links.Convolution2D(None, 16, 3, pad=1), 62 | norm=chainer.links.BatchNormalization(16), 63 | block1=ResidualBlock(16 + alpha * 0 // 3, 16 + alpha * 1 // 3, depth), 64 | block2=ResidualBlock(16 + alpha * 1 // 3, 16 + alpha * 2 // 3, depth), 65 | block3=ResidualBlock(16 + alpha * 2 // 3, 16 + alpha * 3 // 3, depth), 66 | output=chainer.links.Linear(16 + alpha, category)) 67 | 68 | def __call__(self, x): 69 | x = self.input(x) 70 | x = self.norm(x) 71 | 72 | x = self.block1(x) 73 | 74 | x = chainer.functions.average_pooling_2d(x, 2) 75 | x = self.block2(x) 76 | 77 | x = chainer.functions.average_pooling_2d(x, 2) 78 | x = self.block3(x) 79 | 80 | x = chainer.functions.relu(x) 81 | x = chainer.functions.average_pooling_2d(x, x.shape[2]) 82 | x = self.output(x) 83 | 84 | return x 85 | 86 | -------------------------------------------------------------------------------- /src/mylib/functions/connection/grouped_convolution_2d.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import cuda 3 | from chainer.utils import type_check 4 | from chainer.functions.connection.convolution_2d import Convolution2DFunction 5 | 6 | def _pair(x): 7 | if hasattr(x, '__getitem__'): 8 | return x 9 | return x, x 10 | 11 | 12 | class GroupedConvolution2DFunction(chainer.Function): 13 | def __init__(self, stride=1, pad=0): 14 | super().__init__() 15 | self.conv = Convolution2DFunction(stride=stride, pad=pad) 16 | 17 | def check_type_forward(self, in_types): 18 | n_in = in_types.size() 19 | type_check.expect(2 <= n_in, n_in <= 3) 20 | 21 | x_type = in_types[0] 22 | w_type = in_types[1] 23 | type_check.expect( 24 | x_type.dtype.kind == 'f', 25 | w_type.dtype.kind == 'f', 26 | x_type.ndim == 4, 27 | w_type.ndim == 5, 28 | x_type.shape[1] == w_type.shape[0] * w_type.shape[2], 29 | ) 30 | 31 | if type_check.eval(n_in) == 3: 32 | b_type = in_types[2] 33 | type_check.expect( 34 | b_type.dtype == x_type.dtype, 35 | b_type.ndim == 2, 36 | b_type.shape[0] == w_type.shape[0], 37 | b_type.shape[1] == w_type.shape[1], 38 | ) 39 | 40 | def forward(self, inputs): 41 | xp = cuda.get_array_module(*inputs) 42 | x, W = inputs[:2] 43 | b = inputs[2] if len(inputs) == 3 else None 44 | y = xp.empty((W.shape[0], x.shape[0], W.shape[1], x.shape[2], x.shape[3]), dtype=x.dtype) 45 | 46 | c = W.shape[2] 47 | 48 | for i in range(len(W)): 49 | new_inputs = [x[:, i * c:(i + 1) * c, :], W[i]] 50 | 51 | if b is not None: 52 | new_inputs.append(b[i]) 53 | 54 | y[i, :] = self.conv.forward(new_inputs)[0] 55 | 56 | y = xp.rollaxis(y, 1) 57 | y = y.reshape(y.shape[0], y.shape[1] * y.shape[2], *y.shape[3:]) 58 | 59 | return y, 60 | 61 | def backward(self, inputs, grad_outputs): 62 | xp = cuda.get_array_module(*inputs) 63 | x, W = inputs[:2] 64 | b = inputs[2] if len(inputs) == 3 else None 65 | gy = grad_outputs[0] 66 | cx = W.shape[2] 67 | cy = W.shape[1] 68 | 69 | gx = xp.zeros((W.shape[0], x.shape[0], W.shape[2], *x.shape[2:]), dtype=x.dtype) 70 | gW = xp.zeros_like(W) 71 | gb = None 72 | 73 | if b is not None: 74 | gb = xp.zeros_like(b) 75 | 76 | for i in range(len(W)): 77 | new_inputs = [x[:, i * cx:(i + 1) * cx, :], W[i]] 78 | new_grad_outputs = [gy[:, i * cy:(i + 1) * cy, :]] 79 | 80 | if b is not None: 81 | new_inputs.append(b[i]) 82 | 83 | g = self.conv.backward(new_inputs, new_grad_outputs) 84 | gx[i, :] = g[0] 85 | gW[i, :] = g[1] 86 | 87 | if gb is not None: 88 | gb[i, :] = g[2] 89 | 90 | gx = xp.rollaxis(gx, 1) 91 | gx = gx.reshape(gx.shape[0], gx.shape[1] * gx.shape[2], *gx.shape[3:]) 92 | 93 | if gb is None: 94 | return gx, gW 95 | else: 96 | return gx, gW, gb 97 | 98 | 99 | def grouped_convolution_2d(x, W, b=None, stride=1, pad=0): 100 | func = GroupedConvolution2DFunction(stride, pad) 101 | if b is None: 102 | return func(x, W) 103 | else: 104 | return func(x, W, b) 105 | 106 | -------------------------------------------------------------------------------- /src/network/resnext.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | ResNeXt Network (pre-activation)モジュール。 6 | Residual Block : BN - Conv(1x1) - BN - ReLU - ConvGroup(3x3) - BN - ReLU - Conv(1x1) - BN 7 | @author: Atsushi TAKEDA 8 | ''' 9 | import chainer 10 | import mylib 11 | 12 | def reshape(x, channels): 13 | if x.shape[1] < channels: 14 | xp = chainer.cuda.get_array_module(x) 15 | p = xp.zeros((x.shape[0], channels - x.shape[1], x.shape[2], x.shape[3]), dtype=x.dtype) 16 | x = chainer.functions.concat((x, p), axis=1) 17 | elif x.shape[1] > channels: 18 | x = x[:, :channels, :] 19 | 20 | return x 21 | 22 | 23 | class ResnextUnit(chainer.Chain): 24 | def __init__(self, in_channels, out_channels, units): 25 | super().__init__(norm0=chainer.links.BatchNormalization(in_channels), 26 | conv1=chainer.links.Convolution2D(in_channels, out_channels, 1), 27 | norm1=chainer.links.BatchNormalization(out_channels), 28 | conv2=mylib.links.GroupedConvolution2D(out_channels, out_channels, units, 3, pad=1), 29 | norm2=chainer.links.BatchNormalization(out_channels), 30 | conv3=chainer.links.Convolution2D(out_channels, out_channels, 1), 31 | norm3=chainer.links.BatchNormalization(out_channels)) 32 | 33 | def __call__(self, x): 34 | x = self.norm0(x) 35 | x = self.conv1(x) 36 | x = self.norm1(x) 37 | x = chainer.functions.relu(x) 38 | x = self.conv2(x) 39 | x = self.norm2(x) 40 | x = chainer.functions.relu(x) 41 | x = self.conv3(x) 42 | x = self.norm3(x) 43 | 44 | return x 45 | 46 | 47 | class ResnextBlock(chainer.ChainList): 48 | def __init__(self, in_channels, out_channels, units, depth): 49 | layers = [ResnextUnit(in_channels, out_channels, units)] 50 | layers += [ResnextUnit(out_channels, out_channels, units) for _ in range(depth - 1)] 51 | 52 | super().__init__(*layers) 53 | 54 | def __call__(self, x): 55 | for layer in self: 56 | y = layer(x) 57 | y += reshape(x, y.shape[1]) 58 | x = y 59 | 60 | return x 61 | 62 | 63 | class Network(chainer.Chain): 64 | def __init__(self, category, params): 65 | depth, width, units = params 66 | depth = (depth - 2) // 9 67 | width = width * units 68 | 69 | super().__init__(input=chainer.links.Convolution2D(None, width, 3, pad=1), 70 | norm=chainer.links.BatchNormalization(width), 71 | block1=ResnextBlock(width * 1, width * 1, units, depth), 72 | block2=ResnextBlock(width * 1, width * 2, units, depth), 73 | block3=ResnextBlock(width * 2, width * 4, units, depth), 74 | output=chainer.links.Linear(width * 4, category)) 75 | 76 | def __call__(self, x): 77 | x = self.input(x) 78 | x = self.norm(x) 79 | 80 | x = self.block1(x) 81 | 82 | x = chainer.functions.average_pooling_2d(x, 2) 83 | x = self.block2(x) 84 | 85 | x = chainer.functions.average_pooling_2d(x, 2) 86 | x = self.block3(x) 87 | 88 | x = chainer.functions.relu(x) 89 | x = chainer.functions.average_pooling_2d(x, x.shape[2]) 90 | x = self.output(x) 91 | 92 | return x 93 | 94 | -------------------------------------------------------------------------------- /src/network/shakenet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Shake-Shake Residual Network (pre-activation)モジュール。 6 | Residual Block : BN - Conv(3x3) - BN - ReLU - Conv(3x3) - BN 7 | @author: Atsushi TAKEDA 8 | ''' 9 | import chainer 10 | import mylib 11 | 12 | def reshape(x, channels): 13 | if x.shape[1] < channels: 14 | xp = chainer.cuda.get_array_module(x) 15 | p = xp.zeros((x.shape[0], channels - x.shape[1], x.shape[2], x.shape[3]), dtype=x.dtype) 16 | x = chainer.functions.concat((x, p), axis=1) 17 | elif x.shape[1] > channels: 18 | x = x[:, :channels, :] 19 | 20 | return x 21 | 22 | 23 | class ShakeLine(chainer.Chain): 24 | def __init__(self, in_channels, out_channels): 25 | super().__init__(norm0=chainer.links.BatchNormalization(in_channels), 26 | conv1=chainer.links.Convolution2D(in_channels, out_channels, 3, pad=1), 27 | norm1=chainer.links.BatchNormalization(out_channels), 28 | conv2=chainer.links.Convolution2D(out_channels, out_channels, 3, pad=1), 29 | norm2=chainer.links.BatchNormalization(out_channels)) 30 | 31 | def __call__(self, x): 32 | x = self.norm0(x) 33 | x = self.conv1(x) 34 | x = self.norm1(x) 35 | x = chainer.functions.relu(x) 36 | x = self.conv2(x) 37 | x = self.norm2(x) 38 | 39 | return x 40 | 41 | 42 | class ShakeUnit(chainer.Chain): 43 | def __init__(self, in_channels, out_channels): 44 | super().__init__(line1=ShakeLine(in_channels, out_channels), 45 | line2=ShakeLine(in_channels, out_channels)) 46 | 47 | def __call__(self, x): 48 | y1 = self.line1(x) 49 | y2 = self.line2(x) 50 | y = mylib.functions.shake_noise(y1, y2) 51 | 52 | return y 53 | 54 | 55 | class ShakeBlock(chainer.ChainList): 56 | def __init__(self, in_channels, out_channels, depth): 57 | units = [ShakeUnit(in_channels, out_channels)] 58 | units += [ShakeUnit(out_channels, out_channels) for _ in range(depth - 1)] 59 | 60 | super().__init__(*units) 61 | 62 | def __call__(self, x): 63 | for layer in self: 64 | y = layer(x) 65 | y += reshape(x, y.shape[1]) 66 | x = y 67 | 68 | return x 69 | 70 | 71 | class Network(chainer.Chain): 72 | def __init__(self, category, params): 73 | depth, width = params 74 | depth = (depth - 2) // 6 75 | width = width * 16 76 | 77 | super().__init__(input=chainer.links.Convolution2D(None, width, 3, pad=1), 78 | norm=chainer.links.BatchNormalization(width), 79 | block1=ShakeBlock(width * 1, width * 1, depth), 80 | block2=ShakeBlock(width * 1, width * 2, depth), 81 | block3=ShakeBlock(width * 2, width * 4, depth), 82 | output=chainer.links.Linear(width * 4, category)) 83 | 84 | def __call__(self, x): 85 | x = self.input(x) 86 | x = self.norm(x) 87 | 88 | x = self.block1(x) 89 | 90 | x = chainer.functions.average_pooling_2d(x, 2) 91 | x = self.block2(x) 92 | 93 | x = chainer.functions.average_pooling_2d(x, 2) 94 | x = self.block3(x) 95 | 96 | x = chainer.functions.relu(x) 97 | x = chainer.functions.average_pooling_2d(x, x.shape[2]) 98 | x = self.output(x) 99 | 100 | return x 101 | 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 画像認識用 Deep Convolution Neural Network 2 | 3 | ## 実行環境 4 | `python3`の環境で実行できます。 5 | 実行に必要なモジュール(動作確認バージョン)は以下の通り。 6 | 7 | - chainer (v2.0) 8 | - cupy (1.0.0.1) 9 | - matplotlib (2.0.2) 10 | 11 | ## Training 12 | 13 | `src/train.py`はCNNのパラメータの学習処理を実行します。 14 | 実行結果は`result/`に保存されます。 15 | 16 | 実行方法は以下の通り。 17 | ``` 18 | % python src/train.py DATASET NETWORK PARAMETERS [OPTIONS] 19 | ``` 20 | 21 | - DATASET : 使用する画像のデータセットを指定します。 22 | (mnist, cifar10 or cifar100) 23 | - NETWORK : 実行するCNNの名前を指定します。 24 | (resnet, pyramid, densenet, resnext) 25 | - PARAMETERS : CNNに渡すパラメータを指定します(後述)。 26 | 27 | また、次のオプションを指定できます。 28 | 29 | - `-l NAME` : 学習係数の変更方法を指定します(default: step)。 30 | step : 学習回数1/2と3/4のときに、学習係数を0.1倍します。 31 | cosine : Cosine Annealing without Restart [[1](#ref1)] 32 | restart : Cosine Annealing with Restart [[1](#ref1)] 33 | - `-r RATE` : 学習係数の初期値を指定します(default: 0.1)。 34 | - `-m MOMENTUM` : モーメンタムの値を指定します(default: 0.9)。 35 | - `-d DECAY` : Weight Decayの値を指定します(default: 0.0001)。 36 | - `-e EPOCH` : 学習回数を指定します(default: 300)。 37 | - `-b BATCH` : バッチサイズを指定します(default: 128)。 38 | - `-p SIZE` : 1回の計算に使う画像の数を指定します(default: 128)。 39 | - `-g GPU` : 使用するGPUのIDを指定します(default: -1)。 40 | - `--no-check` : 入力される行列の大きさのチェックを省略します。 41 | 42 | 次のCNNとパラメータを指定できます。 43 | ただし、論文中のオリジナルとは少し異なります(詳細はQiitaの記事)。 44 | 45 | - ResNet [[2](#ref2),[3](#ref3)] (ResBlochはSingle RELUです [[4](#ref4)]) 46 | `python src/train.py DATASET resnet DEPTH WIDTH` 47 | - `DEPTH` : CNNの層数 48 | - `WIDTH` : 畳み込み層のチャンネル数(論文中のkの値) 49 | - Pyramidal ResNet [[4](#ref4)] 50 | `python src/train.py DATASET pyramid DEPTH ALPHA` 51 | - `DEPTH` : CNNの層数 52 | - `ALPHA` : 論文中のαの値 53 | - DenseNet [[5](#ref5)] 54 | `python src/train.py DATASET densenet DEPTH WIDTH` 55 | - `DEPTH` : CNNの層数 56 | - `WIDTH` : 畳み込み層のチャンネル数(論文中のkの値) 57 | - ResNeXt [[6](#ref6)] 58 | `python src/train.py DATASET resnext DEPTH WIDTH UNIT` 59 | - `DEPTH` : CNNの層数 60 | - `WIDTH` : 畳み込み層のチャンネル数(論文中のdの値) 61 | - `UNIT` : 畳み込み層のユニット数(論文中のCの値) 62 | - ShakeNet [[7](#ref7)] 63 | `python src/train.py DATASET shakenet DEPTH WIDTH` 64 | - `DEPTH` : CNNの層数 65 | - `WIDTH` : 畳み込み層のチャンネル数(論文中のdの値) 66 | 67 | ## Report 68 | 69 | `src/report.py`は`result/`に保存された実行結果からグラフを生成します。 70 | 71 | 実行方法は以下の通り。 72 | ``` 73 | % python src/result.py 74 | ``` 75 | 76 | `src/report.py`は設定ファイル`result/meta.txt`を生成します。 77 | このファイルを編集することにより、グラフに含めるデータや名前を変更できます。 78 | 設定ファイルのフォーマットは以下の通りです。 79 | ``` 80 | DIRECTORY_1: FLAG_1/NAME_1 81 | DIRECTORY_2: FLAG_2/NAME_2 82 | DIRECTORY_3: FLAG_3/NAME_3 83 | ... 84 | ``` 85 | - DIRECTORY : データが保存されているディレクトリ名です 86 | - FLAG : グラフに含めるならTrueを指定します 87 | - NAME : グラフの凡例に表示する名前です 88 | 89 | ## References 90 | 91 | 1. [Loshchilov, Ilya, and Frank Hutter. "Sgdr: Stochastic gradient descent with warm restarts." (2016).](https://arxiv.org/abs/1608.03983) 92 | 2. [He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. (2016).](https://arxiv.org/abs/1512.03385) 93 | 3. [Zagoruyko, Sergey, and Nikos Komodakis. "Wide residual networks." arXiv preprint arXiv:1605.07146 (2016).](https://arxiv.org/abs/1605.07146) 94 | 4. [Han, Dongyoon, Jiwhan Kim, and Junmo Kim. "Deep pyramidal residual networks." arXiv preprint arXiv:1610.02915 (2016).](https://arxiv.org/abs/1610.02915) 95 | 5. [Huang, Gao, et al. "Densely connected convolutional networks." arXiv preprint arXiv:1608.06993 (2016).](https://arxiv.org/abs/1608.06993) 96 | 6. [Xie, Saining, et al. "Aggregated residual transformations for deep neural networks." arXiv preprint arXiv:1611.05431 (2016).](https://arxiv.org/abs/1611.05431) 97 | 7. [Gastaldi, Xavier. "Shake-Shake regularization." arXiv preprint arXiv:1705.07485 (2017).](https://arxiv.org/abs/1705.07485) 98 | -------------------------------------------------------------------------------- /src/mylib/training/extensions/computational_graph.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | 6 | from chainer import computational_graph 7 | from chainer import configuration 8 | from chainer.training import extension 9 | from chainer import variable 10 | 11 | 12 | _var_style = {'shape': 'octagon', 'fillcolor': '#E0E0E0', 'style': 'filled'} 13 | _func_style = {'shape': 'record', 'fillcolor': '#6495ED', 'style': 'filled'} 14 | 15 | 16 | def dump_graph(root_name, out_name='cg.dot', 17 | variable_style=None, function_style=None, remove_variable=False): 18 | """Returns a trainer extension to dump a computational graph. 19 | This extension dumps a computational graph. The graph is output in DOT 20 | language. 21 | It only dumps a graph at the first invocation. 22 | .. note:: 23 | As of v2.0.0, the computational graph is not kept by default. This 24 | extension changes this behavior until the first invocation. **It is 25 | strongly recommended to use it with the default trigger setting.** 26 | The detailed behavior of this extension since v2.0.0 is as follows. 27 | 1. In its initializer, it turns on the 28 | ``chainer.config.keep_graph_on_report`` flag. 29 | 2. At the first iteration, it dumps the graph using the graph held by 30 | the reported variable. 31 | 3. After dumping the graph, it turns off the flag (if it was originally 32 | turned off) so that any variable reported afterward does not hold 33 | a computational graph. 34 | When the ``keep_graph_on_report`` flag is turned on, the computational 35 | graph created by the updater is kept during the invocation of 36 | extensions. It will cause an unnecessarily large memory consumption 37 | when an extension also uses a large amount of memory, e.g. 38 | :class:`~chainer.training.extensions.Evaluator`. 39 | With the default setting, the ``dump_graph`` extension is called at the 40 | first iteration. Since :class:`~chainer.training.extensions.Evaluator` 41 | is not called at the first iteration in most cases, it does not cause 42 | any memory problem. 43 | Args: 44 | root_name (str): Name of the root of the computational graph. The 45 | root variable is retrieved by this name from the observation 46 | dictionary of the trainer. 47 | out_name (str): Output file name. 48 | variable_style (dict): Dot node style for variables. Each variable is 49 | rendered by an octagon by default. 50 | function_style (dict): Dot node style for functions. Each function is 51 | rendered by a rectangular by default. 52 | .. seealso:: 53 | See :func:`~chainer.computational_graph.build_computational_graph` 54 | for the ``variable_style`` and ``function_style`` arguments. 55 | """ 56 | def trigger(trainer): 57 | return trainer.updater.iteration == 1 58 | 59 | if variable_style is None: 60 | variable_style = _var_style 61 | if function_style is None: 62 | function_style = _func_style 63 | 64 | original_flag = [None] 65 | 66 | def initializer(_): 67 | original_flag[0] = configuration.config.keep_graph_on_report 68 | configuration.config.keep_graph_on_report = True 69 | 70 | @extension.make_extension(trigger=trigger, initializer=initializer) 71 | def dump_graph(trainer): 72 | try: 73 | var = trainer.observation[root_name] 74 | if not isinstance(var, variable.Variable): 75 | raise TypeError('root value is not a Variable') 76 | cg = computational_graph.build_computational_graph( 77 | [var], 78 | variable_style=variable_style, 79 | function_style=function_style, 80 | remove_variable=remove_variable 81 | ).dump() 82 | 83 | out_path = os.path.join(trainer.out, out_name) 84 | # TODO(beam2d): support outputting images by the dot command 85 | with open(out_path, 'w') as f: 86 | f.write(cg) 87 | finally: 88 | configuration.config.keep_graph_on_report = original_flag[0] 89 | 90 | return dump_graph 91 | 92 | -------------------------------------------------------------------------------- /src/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import argparse 6 | import chainer 7 | import mylib 8 | from chainer.training import extensions 9 | 10 | 11 | def create_network(name, category, params): 12 | '''create the specified network model''' 13 | module = __import__('network.{0}'.format(name), fromlist=['Network']) 14 | cls = getattr(module, 'Network') 15 | 16 | return cls(category, params) 17 | 18 | 19 | def load_dataset(name): 20 | '''load the specified datasets''' 21 | if name == 'mnist': 22 | return (10,) + mylib.datasets.get_mnist() 23 | elif name == 'cifar10': 24 | return (10,) + mylib.datasets.get_cifar10() 25 | elif name == 'cifar100': 26 | return (100,) + mylib.datasets.get_cifar100() 27 | 28 | 29 | def main(): 30 | parser = argparse.ArgumentParser(description='network trainer') 31 | parser.add_argument('dataset', metavar='DATASET', help='datasets name') 32 | parser.add_argument('network', metavar='NETWORK', help='network name') 33 | parser.add_argument('params', type=int, nargs='*', metavar='PARAMS', help='parameters') 34 | parser.add_argument('--learning', '-l', default='step', choices=('step', 'cosine', 'restart'), 35 | metavar='NAME', help='name of learning rate control') 36 | parser.add_argument('--rate', '-r', type=float, default=0.1, 37 | metavar='LEARNING_RATE', help='initial leaning rate') 38 | parser.add_argument('--momentum', '-m', type=float, default=0.9, 39 | metavar='MOMENTUM', help='momentum of SGD') 40 | parser.add_argument('--decay', '-d', type=float, default=0.0001, 41 | metavar='WEIGHT_DECAY', help='weight decay') 42 | parser.add_argument('--epoch', '-e', type=int, default=300, 43 | metavar='EPOCH', help='number of epochs for training') 44 | parser.add_argument('--batchsize', '-b', type=int, default=128, 45 | metavar='BATCH_SIZE', help='batch size of training') 46 | parser.add_argument('--procsize', '-p', type=int, default=None, 47 | metavar='DATA_SIZE', help='number of images at a training process') 48 | parser.add_argument('--gpu', '-g', type=int, default=-1, 49 | metavar='GPU_ID', help='GPU ID') 50 | parser.add_argument('--no-check', action='store_true', default=False, help='without type check of variables') 51 | args = parser.parse_args() 52 | 53 | if args.procsize is None: 54 | args.procsize = args.batchsize 55 | 56 | if args.no_check: 57 | chainer.config.type_check = False 58 | 59 | name = '{}-{}-{}-{}'.format(args.dataset, args.network, '-'.join([str(v) for v in args.params]), args.learning) 60 | base_dir = os.path.join(os.path.dirname(__file__), os.path.pardir) 61 | result_dir = os.path.normpath(os.path.join(base_dir, 'result', name)) 62 | 63 | # load data-set 64 | category, train_data, test_data = load_dataset(args.dataset) 65 | 66 | # create a neural network 67 | network = create_network(args.network, category, args.params) 68 | lossfun = chainer.functions.softmax_cross_entropy 69 | accfun = chainer.functions.accuracy 70 | classifier = chainer.links.Classifier(network, lossfun=lossfun, accfun=accfun) 71 | 72 | if args.gpu >= 0: 73 | chainer.cuda.get_device(args.gpu).use() 74 | classifier.to_gpu() 75 | 76 | # create optimizer 77 | optimizer = chainer.optimizers.MomentumSGD(lr=args.rate, momentum=args.momentum) 78 | optimizer.setup(classifier) 79 | optimizer.add_hook(chainer.optimizer.WeightDecay(args.decay)) 80 | 81 | # create data iterators 82 | train_iter = chainer.iterators.SerialIterator(train_data, args.batchsize, repeat=True, shuffle=True) 83 | test_iter = chainer.iterators.SerialIterator(test_data, args.procsize, repeat=False, shuffle=False) 84 | 85 | # create trainer 86 | updater = mylib.training.StandardUpdater(train_iter, optimizer, device=args.gpu, procsize=args.procsize) 87 | trainer = chainer.training.Trainer(updater, (args.epoch, 'epoch'), out=result_dir) 88 | 89 | # extension for evaluation 90 | trainer.extend(extensions.Evaluator(test_iter, classifier, device=args.gpu)) 91 | 92 | # extension for controlling learning rate 93 | if args.learning == 'step': 94 | trainer.extend(mylib.training.extensions.StepShift('lr', args.rate, args.epoch)) 95 | elif args.learning == 'cosine': 96 | trainer.extend(mylib.training.extensions.CosineShift('lr', args.rate, args.epoch, 1)) 97 | elif args.learning == 'restart': 98 | trainer.extend(mylib.training.extensions.CosineShift('lr', args.rate, 10, 2)) 99 | 100 | # extensions for logging 101 | plot_err_keys = ['main/loss', 'validation/main/loss'] 102 | plot_acc_keys = ['main/accuracy', 'validation/main/accuracy'] 103 | print_keys = ['epoch', 104 | 'main/loss', 'validation/main/loss', 105 | 'main/accuracy', 'validation/main/accuracy', 106 | 'elapsed_time'] 107 | trigger = mylib.training.trigger.IntervalTrigger 108 | 109 | trainer.extend(mylib.training.extensions.dump_graph('main/loss', out_name="variable.dot", remove_variable=False)) 110 | trainer.extend(mylib.training.extensions.dump_graph('main/loss', out_name="function.dot", remove_variable=True)) 111 | trainer.extend(mylib.training.extensions.dump_network_size(filename='size.txt')) 112 | 113 | trainer.extend(extensions.snapshot(filename='snapshot.npz'), trigger=trigger(1, 'epoch')) 114 | trainer.extend(mylib.training.extensions.Bestshot(filename='bestshot.npz', trigger=trigger(1, 'epoch'))) 115 | 116 | trainer.extend(mylib.training.extensions.LogReport(log_name='log.txt', trigger=trigger(1, 'epoch'))) 117 | trainer.extend(mylib.training.extensions.PrintReport(print_keys, log_report='LogReport')) 118 | trainer.extend(mylib.training.extensions.PrintReport(print_keys, log_report='LogReport', out='out.txt')) 119 | 120 | trainer.extend(mylib.training.extensions.PlotReport(plot_err_keys, 'epoch', file_name='loss.png', 121 | marker=None, trigger=trigger(1, 'epoch'))) 122 | trainer.extend(mylib.training.extensions.PlotReport(plot_acc_keys, 'epoch', file_name='accuracy.png', 123 | marker=None, trigger=trigger(1, 'epoch'))) 124 | 125 | # resume setting 126 | snapshot = os.path.join(result_dir, 'snapshot.npz') 127 | 128 | if os.path.isfile(snapshot): 129 | chainer.serializers.load_npz(snapshot, trainer) 130 | 131 | # start 132 | trainer.run() 133 | 134 | 135 | if __name__ == '__main__': 136 | main() 137 | 138 | -------------------------------------------------------------------------------- /src/report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | @author: Atsushi TAKEDA 6 | ''' 7 | 8 | import os 9 | import re 10 | import json 11 | import matplotlib.pyplot 12 | 13 | GRAPH_FIGURE_SIZE = (8, 5) 14 | GRAPH_FONT_SIZE = 12 15 | 16 | class Data(object): 17 | def __init__(self, path): 18 | self.path = path 19 | self._read_size() 20 | self._read_log() 21 | 22 | def _read_size(self): 23 | with open(os.path.join(self.path, 'size.txt'), 'r') as handle: 24 | for line in handle: 25 | if line[:3] == '/: ': 26 | self.size = int(line[3:]) 27 | return 28 | 29 | raise Exception('size data is not found') 30 | 31 | def _read_log(self): 32 | with open(os.path.join(self.path, 'log.txt'), 'r') as handle: 33 | self.log = json.load(handle) 34 | 35 | for v in self.log: 36 | v['main/error'] = 1.0 - v['main/accuracy'] 37 | v['validation/main/error'] = 1.0 - v['validation/main/accuracy'] 38 | 39 | if v['main/loss'] >= 1000: 40 | v['main/loss'] = float('nan') 41 | 42 | if v['validation/main/loss'] >= 1000: 43 | v['validation/main/loss'] = float('nan') 44 | 45 | def make_graph(self, path): 46 | self._make_graph([('main/loss', 'train loss'), ('validation/main/loss', 'test loss')], 47 | 'loss', '{}_loss.png'.format(path)) 48 | self._make_graph([('main/error', 'train error'), ('validation/main/error', 'test error')], 49 | 'error rate', '{}_error.png'.format(path)) 50 | 51 | def _make_graph(self, keys, label, file): 52 | figure = matplotlib.pyplot.figure(figsize=GRAPH_FIGURE_SIZE) 53 | plot = figure.add_subplot(1, 1, 1) 54 | 55 | x = [v['epoch'] for v in self.log] 56 | 57 | y_max = 0.001 58 | x_max = max(x) 59 | 60 | for key, name in keys: 61 | y = [v[key] for v in self.log] 62 | y_max = max(max(y) * 1.1, y_max) 63 | 64 | plot.plot(x, y, label=name, linewidth=1) 65 | 66 | plot.set_xlim(0, x_max) 67 | plot.set_ylim(0, y_max) 68 | plot.set_xlabel('epoch', fontsize=GRAPH_FONT_SIZE) 69 | plot.set_ylabel(label, fontsize=GRAPH_FONT_SIZE) 70 | plot.tick_params(labelsize=GRAPH_FONT_SIZE) 71 | plot.legend(loc='upper right', fontsize=GRAPH_FONT_SIZE) 72 | 73 | figure.savefig(file, bbox_inches='tight') 74 | matplotlib.pyplot.close() 75 | 76 | 77 | class Result(object): 78 | def __init__(self, path): 79 | self.path = path 80 | self.data_list = [] 81 | 82 | for meta in self._read_meta(): 83 | data = Data(os.path.join(self.path, meta[0])) 84 | data.flag = meta[1] 85 | data.name = meta[2] 86 | 87 | self.data_list.append(data) 88 | 89 | def _read_meta(self): 90 | meta_list = [] 91 | paths = [] 92 | 93 | regex = re.compile(r'^([^:]+):([^\/]+)\/(.*)$') 94 | 95 | if os.path.isfile(os.path.join(self.path, 'meta.txt')): 96 | with open(os.path.join(self.path, 'meta.txt'), 'r') as handle: 97 | for line in handle: 98 | m = regex.match(line) 99 | 100 | if not m: 101 | continue 102 | 103 | path = m.group(1).strip() 104 | flag = m.group(2).strip().lower() == 'true' 105 | name = m.group(3).strip() 106 | 107 | if not os.path.isdir(os.path.join(self.path, path)): 108 | continue 109 | 110 | meta_list.append((path, flag, name)) 111 | paths.append(path) 112 | 113 | for item in os.listdir(self.path): 114 | if not os.path.isdir(os.path.join(self.path, item)): 115 | continue 116 | 117 | if not os.path.isfile(os.path.join(self.path, item, 'log.txt')): 118 | continue 119 | 120 | if item in paths: 121 | continue 122 | 123 | meta_list.append((item, True, item)) 124 | 125 | with open(os.path.join(self.path, 'meta.txt'), 'w') as handle: 126 | for meta in meta_list: 127 | handle.write("{}: {}/{}\n".format(*meta)) 128 | 129 | return meta_list 130 | 131 | def make_graph(self): 132 | for data in self.data_list: 133 | if data.flag: 134 | data.make_graph(os.path.join(self.path, os.path.basename(data.path))) 135 | 136 | self._make_graph('main/loss', 'loss (train)', 'train_loss.png') 137 | self._make_graph('main/error', 'error rate (train)', 'train_error.png') 138 | self._make_graph('validation/main/loss', 'loss (validation)', 'test_loss.png') 139 | self._make_graph('validation/main/error', 'error rate (validation)', 'test_error1.png') 140 | self._make_graph('validation/main/error', 'error rate (validation)', 'test_error2.png', yrange='min') 141 | 142 | def _make_graph(self, key, name, file, yrange='max'): 143 | figure = matplotlib.pyplot.figure(figsize=GRAPH_FIGURE_SIZE) 144 | plot = figure.add_subplot(1, 1, 1) 145 | 146 | y_max = 0.001 147 | x_max = 0 148 | 149 | for data in self.data_list: 150 | if not data.flag: 151 | continue 152 | 153 | x = [v['epoch'] for v in data.log] 154 | y = [v[key] for v in data.log] 155 | 156 | x_max = max(max(x), x_max) 157 | 158 | if yrange == 'min': 159 | y_max = max(min(y) * 2.5, y_max) 160 | else: 161 | y_max = max(max(y) * 1.1, y_max) 162 | 163 | plot.plot(x, y, label=data.name, linewidth=1) 164 | 165 | plot.set_xlim(0, x_max) 166 | plot.set_ylim(0, y_max) 167 | plot.set_xlabel('epoch', fontsize=GRAPH_FONT_SIZE) 168 | plot.set_ylabel(name, fontsize=GRAPH_FONT_SIZE) 169 | plot.tick_params(labelsize=GRAPH_FONT_SIZE) 170 | plot.legend(loc='upper right', fontsize=GRAPH_FONT_SIZE) 171 | 172 | figure.savefig(os.path.join(self.path, file), bbox_inches='tight') 173 | matplotlib.pyplot.close() 174 | 175 | def __str__(self): 176 | text = [['name', 'size', 'loss(train)', 'error(train)', 'loss(test)', 'error(test)']] 177 | 178 | for data in self.data_list: 179 | name = data.name 180 | size = '{:.2f}m'.format(data.size / 1000000) 181 | train_loss = '{:.4f}'.format(min([v['main/loss'] for v in data.log])) 182 | train_error = '{:.4f}'.format(min([v['main/error'] for v in data.log])) 183 | test_loss = '{:.4f}'.format(min([v['validation/main/loss'] for v in data.log])) 184 | test_error = '{:.4f}'.format(min([v['validation/main/error'] for v in data.log])) 185 | 186 | text.append([name, size, train_loss, train_error, test_loss, test_error]) 187 | 188 | spaces = [0] * len(text[0]) 189 | 190 | for line in text: 191 | spaces = [max(spaces[i], len(v)) for i, v in enumerate(line)] 192 | 193 | fmt = ', '.join(['{{:{}{}}}'.format('' if i == 0 else '>', v) for i, v in enumerate(spaces)]) 194 | text = [fmt.format(*v) for v in text] 195 | 196 | return "\n".join(text) 197 | 198 | 199 | def main(): 200 | basedir = os.path.join(os.path.dirname(__file__), os.path.pardir) 201 | result = Result(os.path.normpath(os.path.join(basedir, 'result'))) 202 | 203 | result.make_graph() 204 | print(result) 205 | 206 | 207 | if __name__ == '__main__': 208 | main() 209 | --------------------------------------------------------------------------------