├── models ├── __init__.py ├── segnet.py ├── softmax_cross_entropy.py └── upsampling_2d.py ├── experiments ├── download.sh ├── batch_evaluate.py ├── train_end2end_nostd_nocw.sh ├── train_end2end.sh ├── train_end2end_noda.sh ├── batch_predict.py ├── train_end2end_msgd.sh ├── train_nostd_nocw.sh ├── train_nostd.sh ├── train_nocw.sh ├── train.sh ├── train_msgd.sh ├── train_noda.sh ├── train_wide.sh ├── train_origcw.sh ├── train_origcw_wide.sh └── README.md ├── lib ├── __init__.py ├── calc_mean.py ├── draw_loss.py ├── densecrf.py ├── train_utils.py ├── camvid.py └── cmd_options.py ├── tests ├── test_experiments.sh ├── test_camvid_dataset.py ├── test_upsampling_2d.py ├── test_softmax_cross_entropy.py └── test_segnet.py ├── LICENSE ├── docker ├── README.md └── Dockerfile ├── .gitignore ├── evaluate.py ├── predict.py ├── train.py └── README.md /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wget https://github.com/alexgkendall/SegNet-Tutorial/archive/master.zip 4 | unzip master.zip && rm -rf master.zip 5 | mv SegNet-Tutorial-master/CamVid data 6 | rm -rf mv SegNet-Tutorial-master 7 | -------------------------------------------------------------------------------- /experiments/batch_evaluate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import glob 5 | import os 6 | import subprocess 7 | 8 | for result_dir in glob.glob('results*/pred*'): 9 | if not os.path.isdir(result_dir): 10 | continue 11 | print(result_dir) 12 | cmd = ['python', 'evaluate.py', 13 | '--result_dir', result_dir] 14 | subprocess.call(cmd) 15 | print('-' * 20) 16 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2016 Shunta Saito 5 | 6 | from __future__ import absolute_import 7 | from __future__ import division 8 | from __future__ import print_function 9 | from __future__ import unicode_literals 10 | 11 | from lib import camvid 12 | from lib import cmd_options 13 | from lib import train_utils 14 | 15 | CamVid = camvid.CamVid 16 | 17 | get_args = cmd_options.get_args 18 | 19 | create_result_dir = train_utils.create_result_dir 20 | create_logger = train_utils.create_logger 21 | get_model = train_utils.get_model 22 | get_optimizer = train_utils.get_optimizer 23 | -------------------------------------------------------------------------------- /tests/test_experiments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | snapshot_epoch=2 epoch=2 bash experiments/train.sh > tests/log_train 4 | snapshot_epoch=2 epoch=2 bash experiments/train_nostd.sh > tests/log_train_nostd 5 | snapshot_epoch=2 epoch=2 bash experiments/train_nostd_nocw.sh > tests/log_train_nostd_nocw 6 | snapshot_epoch=2 epoch=2 bash experiments/train_noda.sh > tests/log_train_noda 7 | snapshot_epoch=2 epoch=2 bash experiments/train_nocw.sh > tests/log_train_nocw 8 | snapshot_epoch=2 epoch=2 bash experiments/train_msgd.sh > tests/log_train_msgd 9 | snapshot_epoch=2 epoch=2 bash experiments/train_end2end.sh > tests/log_train_end2end 10 | snapshot_epoch=2 epoch=2 bash experiments/train_end2end_noda.sh > tests/log_train_end2end_noda 11 | snapshot_epoch=2 epoch=2 bash experiments/train_end2end_msgd.sh > tests/log_train_end2end_msgd 12 | -------------------------------------------------------------------------------- /experiments/train_end2end_nostd_nocw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_end2end_NoSTD_NoCW_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=1000 17 | fi 18 | 19 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 20 | --seed 2016 \ 21 | --gpus ${gpu_id} \ 22 | --batchsize ${batchsize} \ 23 | --opt ${opt} \ 24 | --lr ${lr} \ 25 | --adam_alpha ${adam_alpha} \ 26 | --snapshot_epoch ${snapshot_epoch} \ 27 | --valid_freq ${snapshot_epoch} \ 28 | --result_dir ${result_dir} \ 29 | --n_encdec ${n_encdec} \ 30 | --train_depth ${n_encdec} \ 31 | --epoch ${epoch} \ 32 | --rotate \ 33 | --fliplr \ 34 | --finetune 35 | -------------------------------------------------------------------------------- /experiments/train_end2end.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_end2end_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=1000 17 | fi 18 | 19 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 20 | --seed 2016 \ 21 | --gpus ${gpu_id} \ 22 | --batchsize ${batchsize} \ 23 | --opt ${opt} \ 24 | --lr ${lr} \ 25 | --adam_alpha ${adam_alpha} \ 26 | --snapshot_epoch ${snapshot_epoch} \ 27 | --valid_freq ${snapshot_epoch} \ 28 | --result_dir ${result_dir} \ 29 | --n_encdec ${n_encdec} \ 30 | --train_depth ${n_encdec} \ 31 | --epoch ${epoch} \ 32 | --mean data/train_mean.npy \ 33 | --std data/train_std.npy \ 34 | --use_class_weight \ 35 | --rotate \ 36 | --fliplr \ 37 | --finetune 38 | -------------------------------------------------------------------------------- /experiments/train_end2end_noda.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_end2end_NoDA_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=1000 17 | fi 18 | 19 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 20 | --seed 2016 \ 21 | --gpus ${gpu_id} \ 22 | --batchsize ${batchsize} \ 23 | --opt ${opt} \ 24 | --lr ${lr} \ 25 | --adam_alpha ${adam_alpha} \ 26 | --snapshot_epoch ${snapshot_epoch} \ 27 | --valid_freq ${snapshot_epoch} \ 28 | --result_dir ${result_dir} \ 29 | --n_encdec ${n_encdec} \ 30 | --train_depth ${n_encdec} \ 31 | --epoch ${epoch} \ 32 | --mean data/train_mean.npy \ 33 | --std data/train_std.npy \ 34 | --use_class_weight \ 35 | --shift_jitter 0 \ 36 | --scale_jitter 0.0 \ 37 | --finetune 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Preferred Networks, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Create Docker image 2 | 3 | ## Install docker 4 | 5 | ``` 6 | sudo apt-get update 7 | sudo apt-get install apt-transport-https ca-certificates 8 | sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 9 | sudo sh -c "echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' >> /etc/apt/sources.list.d/docker.list" 10 | sudo apt-get update 11 | sudo apt-get purge lxc-docker 12 | sudo apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual 13 | sudo apt-get install -y docker-engine 14 | sudo service docker start 15 | sudo groupadd docker 16 | sudo gpasswd -a $USER docker 17 | ``` 18 | 19 | ## Install NVIDIA-Docker 20 | 21 | ``` 22 | wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.0-rc.3/nvidia-docker_1.0.0.rc.3-1_amd64.deb 23 | sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb 24 | ``` 25 | 26 | ## Build Docker image 27 | 28 | ``` 29 | sudo nvidia-docker build -t chainer-segnet . 30 | ``` 31 | 32 | ## Launch the docker container 33 | 34 | Move to the top of this project dirtree, then 35 | 36 | ``` 37 | sudo nvidia-docker run \ 38 | -v $PWD:/home/ubuntu/chainer-segnet \ 39 | -p 8888:8888 \ 40 | -ti chainer-segnet /usr/bin/zsh 41 | ``` 42 | 43 | ### For debug 44 | 45 | ``` 46 | conda install -c conda-forge gdb 47 | CFLAGS='-Wall -O0 -g' python setup.py install 48 | ``` 49 | -------------------------------------------------------------------------------- /experiments/batch_predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import glob 6 | import json 7 | import os 8 | import re 9 | import subprocess 10 | 11 | 12 | def pred(result_dir): 13 | args_fn = glob.glob('{}/arg*.json'.format(result_dir))[0] 14 | args = json.load(open(args_fn)) 15 | 16 | pmfn = sorted([(int(re.search('epoch_([0-9]+)', fn).groups()[0]), fn) 17 | for fn in glob.glob('{}/*.trainer'.format(result_dir))]) 18 | pmfn = pmfn[-1][1] 19 | pred_dname = result_dir + \ 20 | '/pred_' + os.path.splitext(os.path.basename(pmfn))[0] 21 | 22 | cmd = ['python', 'predict.py', 23 | '--train_depth', '4', 24 | '--saved_args', '{}/args_4.json'.format(result_dir), 25 | '--snapshot', pmfn, 26 | '--out_dir', pred_dname] 27 | if args['mean'] is not None: 28 | cmd += ['--mean', args['mean']] 29 | cmd += ['--std', args['std']] 30 | subprocess.call(cmd) 31 | 32 | 33 | if __name__ == '__main__': 34 | parser = argparse.ArgumentParser() 35 | parser.add_argument('--result_dir', type=str, default=None) 36 | args = parser.parse_args() 37 | 38 | if args.result_dir is None: 39 | for result_dir in glob.glob('results*'): 40 | if not os.path.isdir(result_dir): 41 | continue 42 | pred(result_dir) 43 | else: 44 | pred(args.result_dir) 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .sync-config.cson 3 | data 4 | results 5 | *.ipynb 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *,cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # IPython Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | 93 | # Rope project settings 94 | .ropeproject 95 | -------------------------------------------------------------------------------- /lib/calc_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import cv2 as cv 5 | import glob 6 | import numpy as np 7 | 8 | fns = glob.glob('data/train/*.png') 9 | n = len(fns) 10 | mean = None 11 | for img_fn in fns: 12 | img = cv.imread(img_fn) 13 | if mean is None: 14 | mean = img.astype(np.float64) 15 | else: 16 | mean += img 17 | mean /= n 18 | 19 | std = None 20 | for img_fn in fns: 21 | img = cv.imread(img_fn) 22 | if std is None: 23 | std = (img.astype(np.float64) - mean) ** 2 24 | else: 25 | std += (img.astype(np.float64) - mean) ** 2 26 | std /= n 27 | std = np.sqrt(std) 28 | 29 | np.save('data/train_mean', mean) 30 | np.save('data/train_std', std) 31 | cv.imwrite('data/train_mean.png', mean) 32 | cv.imwrite('data/train_std.png', std) 33 | 34 | class_ids = set() 35 | fns = glob.glob('data/trainannot/*.png') 36 | n = len(fns) 37 | classes = [0.0 for _ in range(12)] 38 | for lbl_fn in fns: 39 | lbl = cv.imread(lbl_fn, cv.IMREAD_GRAYSCALE) 40 | for lb_i in np.unique(lbl): 41 | class_ids.add(lb_i) 42 | for l in range(12): 43 | classes[l] += np.sum(lbl == l) 44 | class_freq = np.array(classes) 45 | class_freq /= np.sum(class_freq) 46 | print('Existing class IDs: {}'.format(class_ids)) 47 | 48 | with open('data/train_freq.csv', 'w') as fp: 49 | for i, freq in enumerate(class_freq): 50 | print(1.0 / freq, end=',' if i < len(class_freq) - 1 else '', file=fp) 51 | print('0.2595,0.1826,4.5640,0.1417,0.9051,0.3826,9.6446,1.8418,' 52 | '0.6823,6.2478,7.3614,0.0', file=open('data/train_origcw.csv', 'w')) 53 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 2 | ENV DEBIAN_FRONTEND "noninteractive" 3 | 4 | RUN apt-get update 5 | RUN apt-get -y \ 6 | -o Dpkg::Options::="--force-confdef" \ 7 | -o Dpkg::Options::="--force-confold" dist-upgrade 8 | RUN apt-get install -y zsh silversearcher-ag tmux git curl wget build-essential python-dev libgtk2.0-dev vim 9 | RUN useradd -m -d /home/ubuntu ubuntu 10 | RUN echo "ubuntu ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers 11 | RUN chown -R ubuntu:ubuntu /home/ubuntu 12 | RUN chsh -s /usr/bin/zsh ubuntu 13 | 14 | USER ubuntu 15 | WORKDIR /home/ubuntu 16 | ENV HOME /home/ubuntu 17 | 18 | RUN bash -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)" 19 | RUN echo 'export PATH=/usr/local/cuda/bin:$PATH' >> /home/ubuntu/.zshrc 20 | RUN echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> /home/ubuntu/.zshrc 21 | RUN echo 'export CPATH=$HOME/cuda/include:$CPATH' >> /home/ubuntu/.zshrc 22 | RUN echo 'export LIBRARY_PATH=$HOME/cuda/lib64:$LIBRARY_PATH' >> /home/ubuntu/.zshrc 23 | RUN echo 'export LD_LIBRARY_PATH=$HOME/cuda/lib64:$LD_LIBRARY_PATH' >> /home/ubuntu/.zshrc 24 | 25 | RUN git clone https://github.com/yyuu/pyenv.git /home/ubuntu/.pyenv 26 | RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> /home/ubuntu/.zshrc 27 | RUN echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /home/ubuntu/.zshrc 28 | RUN echo 'eval "$(pyenv init -)"' >> /home/ubuntu/.zshrc 29 | RUN ls -la ~/ 30 | RUN chown -R ubuntu:ubuntu /home/ubuntu 31 | 32 | ENV PYENV_ROOT $HOME/.pyenv 33 | ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH 34 | RUN eval "$(pyenv init -)" 35 | RUN pyenv install anaconda3-4.2.0 36 | RUN pyenv global anaconda3-4.2.0 37 | RUN pyenv rehash 38 | RUN conda install -c https://conda.binstar.org/menpo -y opencv3 39 | 40 | RUN git clone https://github.com/pfnet/chainer 41 | RUN cd chainer; python setup.py install 42 | -------------------------------------------------------------------------------- /experiments/train_end2end_msgd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=MomentumSGD 7 | lr=0.01 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_end2end_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --adam_alpha ${adam_alpha} \ 26 | --snapshot_epoch ${snapshot_epoch} \ 27 | --valid_freq ${snapshot_epoch} \ 28 | --result_dir ${result_dir} \ 29 | --n_encdec ${n_encdec} \ 30 | --train_depth ${n_encdec} \ 31 | --mean data/train_mean.npy \ 32 | --std data/train_std.npy \ 33 | --finetune \ 34 | --use_class_weight \ 35 | --rotate \ 36 | --fliplr \ 37 | --lr $1 \ 38 | --epoch $2 39 | } 40 | 41 | train () { 42 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 43 | --seed 2016 \ 44 | --gpus ${gpu_id} \ 45 | --batchsize ${batchsize} \ 46 | --opt ${opt} \ 47 | --adam_alpha ${adam_alpha} \ 48 | --snapshot_epoch ${snapshot_epoch} \ 49 | --valid_freq ${snapshot_epoch} \ 50 | --result_dir ${result_dir} \ 51 | --n_encdec ${n_encdec} \ 52 | --train_depth ${n_encdec} \ 53 | --mean data/train_mean.npy \ 54 | --std data/train_std.npy \ 55 | --finetune \ 56 | --use_class_weight \ 57 | --rotate \ 58 | --fliplr \ 59 | --lr $1 \ 60 | --epoch $2 \ 61 | --resume $3 62 | } 63 | 64 | init_train 0.01 ${epoch} 65 | train 0.001 `expr ${epoch} \* 2` ${result_dir}/encdec4_finetune_epoch_${epoch}.trainer 66 | train 0.0005 `expr ${epoch} \* 3` ${result_dir}/encdec4_finetune_epoch_`expr ${epoch} \* 2`.trainer 67 | train 0.0001 `expr ${epoch} \* 4` ${result_dir}/encdec4_finetune_epoch_`expr ${epoch} \* 3`.trainer 68 | train 0.00005 `expr ${epoch} \* 5` ${result_dir}/encdec4_finetune_epoch_`expr ${epoch} \* 4`.trainer 69 | -------------------------------------------------------------------------------- /lib/draw_loss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from collections import defaultdict 5 | 6 | import glob 7 | import json 8 | import numpy as np 9 | import os 10 | import re 11 | 12 | if True: 13 | import matplotlib 14 | matplotlib.use('Agg') 15 | import matplotlib.pyplot as plt 16 | 17 | 18 | def show_loss_curve(log_fn): 19 | train_loss = defaultdict(list) 20 | valid_loss = [] 21 | for l in json.load(open(log_fn)): 22 | train_loss[l['epoch']].append(l['main/loss']) 23 | if 'validation/main/loss' in l: 24 | valid_loss.append((l['epoch'], l['validation/main/loss'])) 25 | train_loss = np.asarray([[epoch, np.mean(train_loss[epoch])] 26 | for epoch in sorted(train_loss.keys())]) 27 | if len(valid_loss) > 0: 28 | valid_loss = np.asarray(valid_loss) 29 | 30 | d = re.search('encdec([0-9]+)', log_fn) 31 | if d: 32 | d = '_' + d.groups()[0] 33 | else: 34 | d = '_4' 35 | args = json.load(open('{}/args{}.json'.format(os.path.dirname(log_fn), d))) 36 | axes[0].plot( 37 | train_loss[:, 0], train_loss[:, 1], 38 | label='Train {}({}) CW:{}'.format( 39 | args['opt'], args['adam_alpha'] 40 | if args['opt'] == 'Adam' else args['lr'], 41 | args['use_class_weight'])) 42 | 43 | if len(valid_loss) > 0: 44 | axes[1].plot( 45 | valid_loss[:, 0], valid_loss[:, 1], 46 | label='Valid {}({}) CW:{}'.format( 47 | args['opt'], args['adam_alpha'] 48 | if args['opt'] == 'Adam' else args['lr'], 49 | args['use_class_weight'])) 50 | 51 | if __name__ == '__main__': 52 | fig, axes = plt.subplots(2) 53 | fig.set_size_inches(8, 10) 54 | for ax in axes: 55 | ax.set_ylabel('loss') 56 | ax.set_xlabel('epoch') 57 | 58 | for rdir in glob.glob('results*/log_encdec*'): 59 | show_loss_curve(rdir) 60 | axes[0].legend(loc='upper left', bbox_to_anchor=(1, 1)) 61 | axes[1].legend(loc='upper left', bbox_to_anchor=(1, 1)) 62 | 63 | plt.savefig('loss.png') 64 | -------------------------------------------------------------------------------- /tests/test_camvid_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2016 Shunta Saito 5 | 6 | from __future__ import absolute_import 7 | from __future__ import division 8 | from __future__ import print_function 9 | from __future__ import unicode_literals 10 | 11 | import cv2 as cv 12 | import numpy as np 13 | import os 14 | import sys 15 | 16 | if True: 17 | sys.path.insert(0, '.') 18 | from lib import CamVid 19 | 20 | Sky = [128, 128, 128] 21 | Building = [128, 0, 0] 22 | Pole = [192, 192, 128] 23 | Road_marking = [255, 69, 0] 24 | Road = [128, 64, 128] 25 | Pavement = [60, 40, 222] 26 | Tree = [128, 128, 0] 27 | SignSymbol = [192, 128, 128] 28 | Fence = [64, 64, 128] 29 | Car = [64, 0, 128] 30 | Pedestrian = [64, 64, 0] 31 | Bicyclist = [0, 128, 192] 32 | Unlabelled = [0, 0, 0] 33 | 34 | colors = [Sky, Building, Pole, Road, Pavement, Tree, SignSymbol, Fence, 35 | Car, Pedestrian, Bicyclist, Unlabelled] 36 | 37 | 38 | if __name__ == '__main__': 39 | img_dir = 'data/train' 40 | lbl_dir = 'data/trainannot' 41 | list_fn = 'data/train.txt' 42 | mean = 'data/train_mean.npy' 43 | std = 'data/train_std.npy' 44 | shift_jitter = 50 45 | scale_jitter = 0.2 46 | fliplr = True 47 | rotate = True 48 | rotate_max = 7 49 | scale = 1.0 50 | camvid = CamVid(img_dir, lbl_dir, list_fn, mean, std, shift_jitter, 51 | scale_jitter, fliplr, rotate, rotate_max, scale) 52 | 53 | out_dir = 'tests/out' 54 | if not os.path.exists(out_dir): 55 | os.mkdir(out_dir) 56 | for i in range(len(camvid)): 57 | img, lbl = camvid[i] 58 | lbl_ids = np.unique(lbl) 59 | img = img.transpose(1, 2, 0) 60 | print(img.shape, lbl.shape, lbl_ids) 61 | img -= img.reshape(-1, 3).min(axis=0) 62 | img /= img.reshape(-1, 3).max(axis=0) 63 | img *= 255 64 | cv.imwrite('{}/{}_img.png'.format(out_dir, i), img) 65 | out_lbl = np.zeros_like(img) 66 | for k in range(12): 67 | out_lbl[np.where(lbl == k)] = colors[k] 68 | cv.imwrite('{}/{}_lbl.png'.format(out_dir, i), out_lbl) 69 | assert len(lbl_ids) <= 12 70 | assert lbl_ids.min() >= -1 71 | assert lbl_ids.max() <= 11 72 | -------------------------------------------------------------------------------- /experiments/train_nostd_nocw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_NoSTD_NoCW_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --rotate \ 32 | --fliplr \ 33 | --epoch $1 \ 34 | --train_depth 1 35 | } 36 | 37 | train () { 38 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 39 | --seed 2016 \ 40 | --gpus ${gpu_id} \ 41 | --batchsize ${batchsize} \ 42 | --opt ${opt} \ 43 | --lr ${lr} \ 44 | --adam_alpha ${adam_alpha} \ 45 | --snapshot_epoch ${snapshot_epoch} \ 46 | --valid_freq ${snapshot_epoch} \ 47 | --result_dir ${result_dir} \ 48 | --n_encdec ${n_encdec} \ 49 | --rotate \ 50 | --fliplr \ 51 | --train_depth $1 \ 52 | --resume $2 \ 53 | --epoch $3 54 | } 55 | 56 | finetune () { 57 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 58 | --seed 2016 \ 59 | --gpus ${gpu_id} \ 60 | --batchsize ${batchsize} \ 61 | --opt ${opt} \ 62 | --lr ${lr} \ 63 | --adam_alpha ${adam_alpha} \ 64 | --snapshot_epoch ${snapshot_epoch} \ 65 | --valid_freq ${snapshot_epoch} \ 66 | --result_dir ${result_dir} \ 67 | --n_encdec ${n_encdec} \ 68 | --rotate \ 69 | --fliplr \ 70 | --train_depth $1 \ 71 | --resume $2 \ 72 | --epoch $3 \ 73 | --finetune 74 | } 75 | 76 | init_train ${epoch} 77 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 78 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 79 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 80 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 81 | -------------------------------------------------------------------------------- /experiments/train_nostd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_NoSTD_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --use_class_weight \ 32 | --rotate \ 33 | --fliplr \ 34 | --epoch $1 \ 35 | --train_depth 1 36 | } 37 | 38 | train () { 39 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 40 | --seed 2016 \ 41 | --gpus ${gpu_id} \ 42 | --batchsize ${batchsize} \ 43 | --opt ${opt} \ 44 | --lr ${lr} \ 45 | --adam_alpha ${adam_alpha} \ 46 | --snapshot_epoch ${snapshot_epoch} \ 47 | --valid_freq ${snapshot_epoch} \ 48 | --result_dir ${result_dir} \ 49 | --n_encdec ${n_encdec} \ 50 | --use_class_weight \ 51 | --rotate \ 52 | --fliplr \ 53 | --train_depth $1 \ 54 | --resume $2 \ 55 | --epoch $3 56 | } 57 | 58 | finetune () { 59 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 60 | --seed 2016 \ 61 | --gpus ${gpu_id} \ 62 | --batchsize ${batchsize} \ 63 | --opt ${opt} \ 64 | --lr ${lr} \ 65 | --adam_alpha ${adam_alpha} \ 66 | --snapshot_epoch ${snapshot_epoch} \ 67 | --valid_freq ${snapshot_epoch} \ 68 | --result_dir ${result_dir} \ 69 | --n_encdec ${n_encdec} \ 70 | --use_class_weight \ 71 | --rotate \ 72 | --fliplr \ 73 | --train_depth $1 \ 74 | --resume $2 \ 75 | --epoch $3 \ 76 | --finetune 77 | } 78 | 79 | init_train ${epoch} 80 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 81 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 82 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 83 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 84 | -------------------------------------------------------------------------------- /lib/densecrf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import cv2 as cv 6 | import glob 7 | import numpy as np 8 | import os 9 | import pydensecrf.densecrf as dcrf 10 | 11 | Sky = [128, 128, 128] 12 | Building = [128, 0, 0] 13 | Pole = [192, 192, 128] 14 | Road_marking = [255, 69, 0] 15 | Road = [128, 64, 128] 16 | Pavement = [60, 40, 222] 17 | Tree = [128, 128, 0] 18 | SignSymbol = [192, 128, 128] 19 | Fence = [64, 64, 128] 20 | Car = [64, 0, 128] 21 | Pedestrian = [64, 64, 0] 22 | Bicyclist = [0, 128, 192] 23 | Unlabelled = [0, 0, 0] 24 | 25 | colors = [Sky, Building, Pole, Road, Pavement, Tree, SignSymbol, Fence, 26 | Car, Pedestrian, Bicyclist, Unlabelled] 27 | 28 | if __name__ == '__main__': 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument('--result_dir', type=str) 31 | args = parser.parse_args() 32 | 33 | for fn in glob.glob('{}/*_full.npy'.format(args.result_dir)): 34 | img_fn = 'data/test/' + os.path.splitext( 35 | os.path.basename(fn))[0].replace('_full', '.png') 36 | im = cv.imread(img_fn).transpose(1, 0, 2) 37 | im = np.ascontiguousarray(im) 38 | 39 | lbl_fn = img_fn.replace('test', 'testannot') 40 | lbl = cv.imread(lbl_fn, cv.IMREAD_GRAYSCALE) 41 | annot = np.zeros((lbl.shape[0], lbl.shape[1], 3), dtype=np.uint8) 42 | for k in range(12): 43 | annot[np.where(lbl == k)] = colors[k] 44 | lbl_fn = '{}/{}_annot.png'.format( 45 | args.result_dir, os.path.splitext(os.path.basename(fn))[0]) 46 | cv.imwrite(lbl_fn, annot) 47 | 48 | unary = -np.log(np.load(fn)) 49 | unary = unary.transpose(1, 0, 2) 50 | width, height, nlabels = unary.shape 51 | unary = unary.transpose(2, 0, 1).reshape(nlabels, -1) 52 | unary = np.ascontiguousarray(unary) 53 | 54 | d = dcrf.DenseCRF2D(width, height, nlabels) 55 | d.setUnaryEnergy(unary) 56 | d.addPairwiseBilateral(sxy=5, srgb=3, rgbim=im, compat=5) 57 | 58 | q = d.inference(5) 59 | mask = np.argmax(q, axis=0).reshape(width, height).transpose(1, 0) 60 | print(np.unique(mask)) 61 | 62 | out_lbl = np.zeros((height, width, 3), dtype=np.uint8) 63 | for k in range(12): 64 | out_lbl[np.where(mask == k)] = colors[k] 65 | fn = '{}/{}_densecrf.png'.format( 66 | args.result_dir, os.path.splitext(os.path.basename(fn))[0]) 67 | cv.imwrite(fn, out_lbl) 68 | break 69 | -------------------------------------------------------------------------------- /experiments/train_nocw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_NoCW_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --mean data/train_mean.npy \ 32 | --std data/train_std.npy \ 33 | --rotate \ 34 | --fliplr \ 35 | --epoch $1 \ 36 | --train_depth 1 37 | } 38 | 39 | train () { 40 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 41 | --seed 2016 \ 42 | --gpus ${gpu_id} \ 43 | --batchsize ${batchsize} \ 44 | --opt ${opt} \ 45 | --lr ${lr} \ 46 | --adam_alpha ${adam_alpha} \ 47 | --snapshot_epoch ${snapshot_epoch} \ 48 | --valid_freq ${snapshot_epoch} \ 49 | --result_dir ${result_dir} \ 50 | --n_encdec ${n_encdec} \ 51 | --mean data/train_mean.npy \ 52 | --std data/train_std.npy \ 53 | --rotate \ 54 | --fliplr \ 55 | --train_depth $1 \ 56 | --resume $2 \ 57 | --epoch $3 58 | } 59 | 60 | finetune () { 61 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 62 | --seed 2016 \ 63 | --gpus ${gpu_id} \ 64 | --batchsize ${batchsize} \ 65 | --opt ${opt} \ 66 | --lr ${lr} \ 67 | --adam_alpha ${adam_alpha} \ 68 | --snapshot_epoch ${snapshot_epoch} \ 69 | --valid_freq ${snapshot_epoch} \ 70 | --result_dir ${result_dir} \ 71 | --n_encdec ${n_encdec} \ 72 | --mean data/train_mean.npy \ 73 | --std data/train_std.npy \ 74 | --rotate \ 75 | --fliplr \ 76 | --train_depth $1 \ 77 | --resume $2 \ 78 | --epoch $3 \ 79 | --finetune 80 | } 81 | 82 | init_train ${epoch} 83 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 84 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 85 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 86 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 87 | -------------------------------------------------------------------------------- /experiments/train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --mean data/train_mean.npy \ 32 | --std data/train_std.npy \ 33 | --use_class_weight \ 34 | --rotate \ 35 | --fliplr \ 36 | --epoch $1 \ 37 | --train_depth 1 38 | } 39 | 40 | train () { 41 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 42 | --seed 2016 \ 43 | --gpus ${gpu_id} \ 44 | --batchsize ${batchsize} \ 45 | --opt ${opt} \ 46 | --lr ${lr} \ 47 | --adam_alpha ${adam_alpha} \ 48 | --snapshot_epoch ${snapshot_epoch} \ 49 | --valid_freq ${snapshot_epoch} \ 50 | --result_dir ${result_dir} \ 51 | --n_encdec ${n_encdec} \ 52 | --mean data/train_mean.npy \ 53 | --std data/train_std.npy \ 54 | --use_class_weight \ 55 | --rotate \ 56 | --fliplr \ 57 | --train_depth $1 \ 58 | --resume $2 \ 59 | --epoch $3 60 | } 61 | 62 | finetune () { 63 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 64 | --seed 2016 \ 65 | --gpus ${gpu_id} \ 66 | --batchsize ${batchsize} \ 67 | --opt ${opt} \ 68 | --lr ${lr} \ 69 | --adam_alpha ${adam_alpha} \ 70 | --snapshot_epoch ${snapshot_epoch} \ 71 | --valid_freq ${snapshot_epoch} \ 72 | --result_dir ${result_dir} \ 73 | --n_encdec ${n_encdec} \ 74 | --mean data/train_mean.npy \ 75 | --std data/train_std.npy \ 76 | --use_class_weight \ 77 | --rotate \ 78 | --fliplr \ 79 | --train_depth $1 \ 80 | --resume $2 \ 81 | --epoch $3 \ 82 | --finetune 83 | } 84 | 85 | init_train ${epoch} 86 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 87 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 88 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 89 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 90 | -------------------------------------------------------------------------------- /experiments/train_msgd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=MomentumSGD 7 | lr=0.0001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --mean data/train_mean.npy \ 32 | --std data/train_std.npy \ 33 | --use_class_weight \ 34 | --rotate \ 35 | --fliplr \ 36 | --epoch $1 \ 37 | --train_depth 1 38 | } 39 | 40 | train () { 41 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 42 | --seed 2016 \ 43 | --gpus ${gpu_id} \ 44 | --batchsize ${batchsize} \ 45 | --opt ${opt} \ 46 | --lr ${lr} \ 47 | --adam_alpha ${adam_alpha} \ 48 | --snapshot_epoch ${snapshot_epoch} \ 49 | --valid_freq ${snapshot_epoch} \ 50 | --result_dir ${result_dir} \ 51 | --n_encdec ${n_encdec} \ 52 | --mean data/train_mean.npy \ 53 | --std data/train_std.npy \ 54 | --use_class_weight \ 55 | --rotate \ 56 | --fliplr \ 57 | --train_depth $1 \ 58 | --resume $2 \ 59 | --epoch $3 60 | } 61 | 62 | finetune () { 63 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 64 | --seed 2016 \ 65 | --gpus ${gpu_id} \ 66 | --batchsize ${batchsize} \ 67 | --opt ${opt} \ 68 | --lr ${lr} \ 69 | --adam_alpha ${adam_alpha} \ 70 | --snapshot_epoch ${snapshot_epoch} \ 71 | --valid_freq ${snapshot_epoch} \ 72 | --result_dir ${result_dir} \ 73 | --n_encdec ${n_encdec} \ 74 | --mean data/train_mean.npy \ 75 | --std data/train_std.npy \ 76 | --use_class_weight \ 77 | --rotate \ 78 | --fliplr \ 79 | --train_depth $1 \ 80 | --resume $2 \ 81 | --epoch $3 \ 82 | --finetune 83 | } 84 | 85 | init_train ${epoch} 86 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 87 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 88 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 89 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 90 | -------------------------------------------------------------------------------- /experiments/train_noda.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_NoDA_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --mean data/train_mean.npy \ 32 | --std data/train_std.npy \ 33 | --use_class_weight \ 34 | --train_depth 1 \ 35 | --shift_jitter 0 \ 36 | --scale_jitter 0.0 \ 37 | --epoch $1 38 | } 39 | 40 | train () { 41 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 42 | --seed 2016 \ 43 | --gpus ${gpu_id} \ 44 | --batchsize ${batchsize} \ 45 | --opt ${opt} \ 46 | --lr ${lr} \ 47 | --adam_alpha ${adam_alpha} \ 48 | --snapshot_epoch ${snapshot_epoch} \ 49 | --valid_freq ${snapshot_epoch} \ 50 | --result_dir ${result_dir} \ 51 | --n_encdec ${n_encdec} \ 52 | --mean data/train_mean.npy \ 53 | --std data/train_std.npy \ 54 | --use_class_weight \ 55 | --shift_jitter 0 \ 56 | --scale_jitter 0.0 \ 57 | --train_depth $1 \ 58 | --resume $2 \ 59 | --epoch $3 60 | } 61 | 62 | finetune () { 63 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 64 | --seed 2016 \ 65 | --gpus ${gpu_id} \ 66 | --batchsize ${batchsize} \ 67 | --opt ${opt} \ 68 | --lr ${lr} \ 69 | --adam_alpha ${adam_alpha} \ 70 | --snapshot_epoch ${snapshot_epoch} \ 71 | --valid_freq ${snapshot_epoch} \ 72 | --result_dir ${result_dir} \ 73 | --n_encdec ${n_encdec} \ 74 | --mean data/train_mean.npy \ 75 | --std data/train_std.npy \ 76 | --use_class_weight \ 77 | --shift_jitter 0 \ 78 | --scale_jitter 0.0 \ 79 | --train_depth $1 \ 80 | --resume $2 \ 81 | --epoch $3 \ 82 | --finetune 83 | } 84 | 85 | init_train ${epoch} 86 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 87 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 88 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 89 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 90 | -------------------------------------------------------------------------------- /experiments/train_wide.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=6 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | n_mid=128 11 | result_dir=results_n-mid-${n_mid}_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 12 | 13 | if [ -z ${snapshot_epoch} ]; then 14 | snapshot_epoch=10 15 | fi 16 | if [ -z ${epoch} ]; then 17 | epoch=300 18 | fi 19 | 20 | init_train () { 21 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 22 | --seed 2016 \ 23 | --gpus ${gpu_id} \ 24 | --batchsize ${batchsize} \ 25 | --opt ${opt} \ 26 | --lr ${lr} \ 27 | --adam_alpha ${adam_alpha} \ 28 | --snapshot_epoch ${snapshot_epoch} \ 29 | --valid_freq ${snapshot_epoch} \ 30 | --result_dir ${result_dir} \ 31 | --n_encdec ${n_encdec} \ 32 | --mean data/train_mean.npy \ 33 | --std data/train_std.npy \ 34 | --use_class_weight \ 35 | --rotate \ 36 | --fliplr \ 37 | --n_mid ${n_mid} \ 38 | --epoch $1 \ 39 | --train_depth 1 40 | } 41 | 42 | train () { 43 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 44 | --seed 2016 \ 45 | --gpus ${gpu_id} \ 46 | --batchsize ${batchsize} \ 47 | --opt ${opt} \ 48 | --lr ${lr} \ 49 | --adam_alpha ${adam_alpha} \ 50 | --snapshot_epoch ${snapshot_epoch} \ 51 | --valid_freq ${snapshot_epoch} \ 52 | --result_dir ${result_dir} \ 53 | --n_encdec ${n_encdec} \ 54 | --mean data/train_mean.npy \ 55 | --std data/train_std.npy \ 56 | --use_class_weight \ 57 | --rotate \ 58 | --fliplr \ 59 | --n_mid ${n_mid} \ 60 | --train_depth $1 \ 61 | --resume $2 \ 62 | --epoch $3 63 | } 64 | 65 | finetune () { 66 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 67 | --seed 2016 \ 68 | --gpus ${gpu_id} \ 69 | --batchsize ${batchsize} \ 70 | --opt ${opt} \ 71 | --lr ${lr} \ 72 | --adam_alpha ${adam_alpha} \ 73 | --snapshot_epoch ${snapshot_epoch} \ 74 | --valid_freq ${snapshot_epoch} \ 75 | --result_dir ${result_dir} \ 76 | --n_encdec ${n_encdec} \ 77 | --mean data/train_mean.npy \ 78 | --std data/train_std.npy \ 79 | --use_class_weight \ 80 | --rotate \ 81 | --fliplr \ 82 | --n_mid ${n_mid} \ 83 | --train_depth $1 \ 84 | --resume $2 \ 85 | --epoch $3 \ 86 | --finetune 87 | } 88 | 89 | init_train ${epoch} 90 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 91 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 92 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 93 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 94 | -------------------------------------------------------------------------------- /experiments/train_origcw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | batchsize=16 6 | opt=Adam 7 | lr=0.001 8 | adam_alpha=0.0001 9 | n_encdec=4 10 | result_dir=results_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 11 | 12 | if [ -z ${snapshot_epoch} ]; then 13 | snapshot_epoch=10 14 | fi 15 | if [ -z ${epoch} ]; then 16 | epoch=300 17 | fi 18 | 19 | init_train () { 20 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 21 | --seed 2016 \ 22 | --gpus ${gpu_id} \ 23 | --batchsize ${batchsize} \ 24 | --opt ${opt} \ 25 | --lr ${lr} \ 26 | --adam_alpha ${adam_alpha} \ 27 | --snapshot_epoch ${snapshot_epoch} \ 28 | --valid_freq ${snapshot_epoch} \ 29 | --result_dir ${result_dir} \ 30 | --n_encdec ${n_encdec} \ 31 | --mean data/train_mean.npy \ 32 | --std data/train_std.npy \ 33 | --use_class_weight \ 34 | --rotate \ 35 | --fliplr \ 36 | --class_weight data/train_origcw.csv \ 37 | --epoch $1 \ 38 | --train_depth 1 39 | } 40 | 41 | train () { 42 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 43 | --seed 2016 \ 44 | --gpus ${gpu_id} \ 45 | --batchsize ${batchsize} \ 46 | --opt ${opt} \ 47 | --lr ${lr} \ 48 | --adam_alpha ${adam_alpha} \ 49 | --snapshot_epoch ${snapshot_epoch} \ 50 | --valid_freq ${snapshot_epoch} \ 51 | --result_dir ${result_dir} \ 52 | --n_encdec ${n_encdec} \ 53 | --mean data/train_mean.npy \ 54 | --std data/train_std.npy \ 55 | --use_class_weight \ 56 | --rotate \ 57 | --fliplr \ 58 | --class_weight data/train_origcw.csv \ 59 | --train_depth $1 \ 60 | --resume $2 \ 61 | --epoch $3 62 | } 63 | 64 | finetune () { 65 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 66 | --seed 2016 \ 67 | --gpus ${gpu_id} \ 68 | --batchsize ${batchsize} \ 69 | --opt ${opt} \ 70 | --lr ${lr} \ 71 | --adam_alpha ${adam_alpha} \ 72 | --snapshot_epoch ${snapshot_epoch} \ 73 | --valid_freq ${snapshot_epoch} \ 74 | --result_dir ${result_dir} \ 75 | --n_encdec ${n_encdec} \ 76 | --mean data/train_mean.npy \ 77 | --std data/train_std.npy \ 78 | --use_class_weight \ 79 | --rotate \ 80 | --fliplr \ 81 | --class_weight data/train_origcw.csv \ 82 | --train_depth $1 \ 83 | --resume $2 \ 84 | --epoch $3 \ 85 | --finetune 86 | } 87 | 88 | init_train ${epoch} 89 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 90 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 91 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 92 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 93 | -------------------------------------------------------------------------------- /experiments/train_origcw_wide.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seed=2016 4 | gpu_id=0 5 | opt=Adam 6 | lr=0.001 7 | adam_alpha=0.0001 8 | n_encdec=4 9 | result_dir=results_origCW_n-mid-128_opt-${opt}_lr-${lr}_alpha-${adam_alpha}_`date "+%Y-%m-%d_%H%M%S"` 10 | 11 | if [ -z ${snapshot_epoch} ]; then 12 | snapshot_epoch=10 13 | fi 14 | if [ -z ${epoch} ]; then 15 | epoch=300 16 | fi 17 | 18 | init_train () { 19 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 20 | --seed 2016 \ 21 | --gpus ${gpu_id} \ 22 | --batchsize 6 \ 23 | --opt ${opt} \ 24 | --lr ${lr} \ 25 | --adam_alpha ${adam_alpha} \ 26 | --snapshot_epoch ${snapshot_epoch} \ 27 | --valid_freq ${snapshot_epoch} \ 28 | --result_dir ${result_dir} \ 29 | --n_encdec ${n_encdec} \ 30 | --mean data/train_mean.npy \ 31 | --std data/train_std.npy \ 32 | --use_class_weight \ 33 | --rotate \ 34 | --fliplr \ 35 | --class_weight data/train_origcw.csv \ 36 | --n_mid 128 \ 37 | --epoch $1 \ 38 | --train_depth 1 39 | } 40 | 41 | train () { 42 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 43 | --seed 2016 \ 44 | --gpus ${gpu_id} \ 45 | --batchsize $4 \ 46 | --opt ${opt} \ 47 | --lr ${lr} \ 48 | --adam_alpha ${adam_alpha} \ 49 | --snapshot_epoch ${snapshot_epoch} \ 50 | --valid_freq ${snapshot_epoch} \ 51 | --result_dir ${result_dir} \ 52 | --n_encdec ${n_encdec} \ 53 | --mean data/train_mean.npy \ 54 | --std data/train_std.npy \ 55 | --use_class_weight \ 56 | --rotate \ 57 | --fliplr \ 58 | --class_weight data/train_origcw.csv \ 59 | --n_mid 128 \ 60 | --train_depth $1 \ 61 | --resume $2 \ 62 | --epoch $3 63 | } 64 | 65 | finetune () { 66 | CHAINER_SEED=${seed} CHAINER_TYPE_CHECK=0 python train.py \ 67 | --seed 2016 \ 68 | --gpus ${gpu_id} \ 69 | --batchsize $4 \ 70 | --opt ${opt} \ 71 | --lr ${lr} \ 72 | --adam_alpha ${adam_alpha} \ 73 | --snapshot_epoch ${snapshot_epoch} \ 74 | --valid_freq ${snapshot_epoch} \ 75 | --result_dir ${result_dir} \ 76 | --n_encdec ${n_encdec} \ 77 | --mean data/train_mean.npy \ 78 | --std data/train_std.npy \ 79 | --use_class_weight \ 80 | --rotate \ 81 | --fliplr \ 82 | --class_weight data/train_origcw.csv \ 83 | --n_mid 128 \ 84 | --train_depth $1 \ 85 | --resume $2 \ 86 | --epoch $3 \ 87 | --finetune 88 | } 89 | 90 | init_train ${epoch} 91 | train 2 ${result_dir}/encdec1_epoch_${epoch}.trainer `expr ${epoch} \* 2` 4 92 | train 3 ${result_dir}/encdec2_epoch_`expr ${epoch} \* 2`.trainer `expr ${epoch} \* 3` 4 93 | train 4 ${result_dir}/encdec3_epoch_`expr ${epoch} \* 3`.trainer `expr ${epoch} \* 4` 4 94 | finetune 4 ${result_dir}/encdec4_epoch_`expr ${epoch} \* 4`.trainer `expr ${epoch} \* 5` 4 95 | -------------------------------------------------------------------------------- /tests/test_upsampling_2d.py: -------------------------------------------------------------------------------- 1 | from chainer import cuda 2 | from chainer import gradient_check 3 | from chainer import testing 4 | from chainer.testing import attr 5 | from chainer.testing import condition 6 | from chainer.utils import conv 7 | from models import upsampling_2d 8 | 9 | import chainer.functions as F 10 | import numpy 11 | import unittest 12 | 13 | 14 | @testing.parameterize( 15 | {'in_shape': (4, 3, 6, 8)}, 16 | {'in_shape': (4, 3, 5, 7)}, 17 | ) 18 | class TestUpsampling2D(unittest.TestCase): 19 | 20 | def setUp(self): 21 | self.x = numpy.random.uniform(-1, 1, self.in_shape).astype('f') 22 | self.p = F.MaxPooling2D(2, 2, use_cudnn=False) 23 | self.pooled_y = self.p(self.x) 24 | self.gy = numpy.random.uniform( 25 | -1, 1, self.in_shape).astype(numpy.float32) 26 | 27 | def check_forward(self, y): 28 | y = upsampling_2d.upsampling_2d( 29 | self.pooled_y, self.p.indexes, ksize=(self.p.kh, self.p.kw), 30 | stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw), 31 | outsize=self.in_shape[2:], cover_all=self.p.cover_all) 32 | if isinstance(y.data, numpy.ndarray): 33 | y = conv.im2col_cpu(y.data, self.p.kh, self.p.kw, 34 | self.p.sy, self.p.sx, self.p.ph, self.p.pw) 35 | else: 36 | y = conv.im2col_gpu(y.data, self.p.kh, self.p.kw, 37 | self.p.sy, self.p.sx, self.p.ph, self.p.pw) 38 | for i in numpy.ndindex(y.shape): 39 | n, c, ky, kx, oy, ox = i 40 | up_y = y[n, c, ky, kx, oy, ox] 41 | if ky * y.shape[3] + kx == self.p.indexes[n, c, oy, ox]: 42 | in_y = self.pooled_y.data[n, c, oy, ox] 43 | testing.assert_allclose(in_y, up_y) 44 | else: 45 | testing.assert_allclose(up_y, 0) 46 | 47 | @condition.retry(3) 48 | def test_forward_cpu(self): 49 | self.pooled_y.to_cpu() 50 | self.check_forward(self.pooled_y) 51 | 52 | @attr.gpu 53 | @condition.retry(3) 54 | def test_forward_gpu(self): 55 | self.pooled_y.to_gpu() 56 | self.check_forward(self.pooled_y) 57 | 58 | def check_backward(self, x_data, y_grad): 59 | func = upsampling_2d.Upsampling2D( 60 | self.p.indexes, ksize=(self.p.kh, self.p.kw), 61 | stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw), 62 | outsize=self.in_shape[2:], cover_all=self.p.cover_all) 63 | gradient_check.check_backward(func, x_data, y_grad) 64 | 65 | @condition.retry(3) 66 | def test_backward_cpu(self): 67 | self.check_backward(self.pooled_y.data, self.gy) 68 | 69 | @attr.gpu 70 | @condition.retry(3) 71 | def test_backward_gpu(self): 72 | self.check_backward(cuda.to_gpu( 73 | self.pooled_y.data), cuda.to_gpu(self.gy)) 74 | 75 | testing.run_module(__name__, __file__) 76 | -------------------------------------------------------------------------------- /lib/train_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Shunta Saito 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | from __future__ import unicode_literals 7 | from chainer import optimizers 8 | 9 | import chainer 10 | import imp 11 | import logging 12 | import os 13 | import shutil 14 | import sys 15 | import time 16 | 17 | 18 | def create_result_dir(model_name): 19 | result_dir = 'results/{}_{}'.format( 20 | model_name, time.strftime('%Y-%m-%d_%H-%M-%S')) 21 | if os.path.exists(result_dir): 22 | result_dir += '_{}'.format(time.clock()) 23 | if not os.path.exists(result_dir): 24 | os.makedirs(result_dir) 25 | return result_dir 26 | 27 | 28 | def create_logger(args, result_dir): 29 | root = logging.getLogger() 30 | root.setLevel(logging.DEBUG) 31 | msg_format = '%(asctime)s [%(levelname)s] %(message)s' 32 | formatter = logging.Formatter(msg_format) 33 | ch = logging.StreamHandler(sys.stdout) 34 | ch.setLevel(logging.DEBUG) 35 | ch.setFormatter(formatter) 36 | root.addHandler(ch) 37 | fileHandler = logging.FileHandler("{}/stdout.log".format(result_dir)) 38 | fileHandler.setFormatter(formatter) 39 | root.addHandler(fileHandler) 40 | logging.info(sys.version_info) 41 | logging.info('chainer version: {}'.format(chainer.__version__)) 42 | logging.info('cuda: {}, cudnn: {}'.format( 43 | chainer.cuda.available, chainer.cuda.cudnn_enabled)) 44 | logging.info(args) 45 | 46 | 47 | def get_model( 48 | model_file, model_name, loss_file, loss_name, class_weight, n_encdec, 49 | n_classes, in_channel, n_mid, train_depth=None, result_dir=None): 50 | model = imp.load_source(model_name, model_file) 51 | model = getattr(model, model_name) 52 | loss = imp.load_source(loss_name, loss_file) 53 | loss = getattr(loss, loss_name) 54 | 55 | # Initialize 56 | model = model(n_encdec, n_classes, in_channel, n_mid) 57 | if train_depth: 58 | model = loss(model, class_weight, train_depth) 59 | 60 | # Copy files 61 | if result_dir is not None: 62 | base_fn = os.path.basename(model_file) 63 | dst = '{}/{}'.format(result_dir, base_fn) 64 | if not os.path.exists(dst): 65 | shutil.copy(model_file, dst) 66 | base_fn = os.path.basename(loss_file) 67 | dst = '{}/{}'.format(result_dir, base_fn) 68 | if not os.path.exists(dst): 69 | shutil.copy(loss_file, dst) 70 | 71 | return model 72 | 73 | 74 | def get_optimizer(opt, lr=None, adam_alpha=None, adam_beta1=None, 75 | adam_beta2=None, adam_eps=None, weight_decay=None): 76 | if opt == 'MomentumSGD': 77 | optimizer = optimizers.MomentumSGD(lr=lr, momentum=0.9) 78 | elif opt == 'Adam': 79 | optimizer = optimizers.Adam( 80 | alpha=adam_alpha, beta1=adam_beta1, 81 | beta2=adam_beta2, eps=adam_eps) 82 | elif opt == 'AdaGrad': 83 | optimizer = optimizers.AdaGrad(lr=lr) 84 | elif opt == 'RMSprop': 85 | optimizer = optimizers.RMSprop(lr=lr) 86 | else: 87 | raise Exception('No optimizer is selected') 88 | 89 | # The first model as the master model 90 | if opt == 'MomentumSGD': 91 | optimizer.decay = weight_decay 92 | 93 | return optimizer 94 | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import cv2 as cv 6 | import glob 7 | import numpy as np 8 | import os 9 | 10 | if __name__ == '__main__': 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('--result_dir', type=str) 13 | parser.add_argument('--anno_dir', type=str, default='data/testannot') 14 | parser.add_argument('--n_classes', type=int, default=12) 15 | parser.add_argument('--unknown_class', type=int, default=11) 16 | args = parser.parse_args() 17 | 18 | anno_image_fns = sorted(glob.glob('{}/*.png'.format(args.anno_dir))) 19 | result_npy_fns = sorted(glob.glob('{}/*.npy'.format(args.result_dir))) 20 | 21 | print('anno_imag_fns n = {}'.format(len(anno_image_fns))) 22 | print('result_npy_fns n = {}'.format(len(result_npy_fns))) 23 | assert len(result_npy_fns) == len(anno_image_fns) 24 | 25 | totalpoints = 0 26 | cf = np.zeros((len(anno_image_fns), args.n_classes, args.n_classes)) 27 | globalacc = 0 28 | 29 | for i, (anno_img_fn, result_npy_fn) in enumerate( 30 | zip(anno_image_fns, result_npy_fns)): 31 | assert os.path.splitext(os.path.basename(anno_img_fn))[0] == \ 32 | os.path.splitext(os.path.basename(result_npy_fn))[0] 33 | anno = cv.imread(anno_img_fn, cv.IMREAD_GRAYSCALE) 34 | pred = np.load(result_npy_fn) 35 | 36 | totalpoints += np.sum(anno != args.unknown_class) 37 | 38 | for j in range(args.n_classes): 39 | if j == args.unknown_class: 40 | continue 41 | c1 = anno == j 42 | for k in range(args.n_classes): 43 | c1p = pred == k 44 | index = c1 * c1p 45 | cf[i, j, k] += np.sum(index) 46 | if j == k: 47 | globalacc += np.sum(index) 48 | 49 | cf = np.sum(cf, axis=0) 50 | 51 | # Compute confusion matrix 52 | conf = np.zeros((args.n_classes, args.n_classes)) 53 | for i in range(args.n_classes): 54 | if i != args.unknown_class and cf[i, :].sum() > 0: 55 | conf[i, :] = cf[i, :] / cf[i, :].sum() 56 | globalacc /= float(totalpoints) 57 | 58 | # Compute intersection over union for each class and its mean 59 | iou = np.zeros((args.n_classes,)) 60 | for i in range(args.n_classes): 61 | if i != args.unknown_class and conf[i, :].sum() > 0: 62 | n_true = cf[i, :].sum() 63 | n_positive = cf[:, i].sum() 64 | n_true_positive = cf[i, i] 65 | iou[i] = n_true_positive / float(n_true + n_positive - n_true_positive) 66 | 67 | print('Global acc = {}'.format(globalacc)) 68 | 69 | class_average_acc = np.diag(conf).sum() / float(args.n_classes) 70 | print('Class average acc = {}'.format(class_average_acc)) 71 | 72 | classes = [ 73 | 'Sky', 'Building', 'Pole', 'Road', 'Pavement', 'Tree', 'SignSymbol', 74 | 'Fence', 'Car', 'Pedestrian', 'Bicyclist', 'Unlabelled' 75 | ] 76 | per_class_results = {} 77 | for class_id, conf in enumerate(np.diag(conf)): 78 | per_class_results[classes[class_id]] = conf 79 | 80 | # Table titles 81 | print('|', end='') 82 | for class_name in [ 83 | 'Building', 'Tree', 'Sky', 'Car', 'SignSymbol', 'Road', 'Pedestrian', 84 | 'Fence', 'Pole', 'Pavement', 'Bicyclist']: 85 | print(' {} |'.format(class_name), end='') 86 | print(' Class avg. | Global avg. | IoU |') 87 | 88 | # Table borders 89 | for _ in range(len(classes) + 3): 90 | print(':-:|', end='') 91 | print('') 92 | 93 | # Values 94 | print('|', end='') 95 | for class_name in [ 96 | 'Building', 'Tree', 'Sky', 'Car', 'SignSymbol', 'Road', 'Pedestrian', 97 | 'Fence', 'Pole', 'Pavement', 'Bicyclist']: 98 | print(' {:.1f} |'.format(per_class_results[class_name] * 100), end='') 99 | print(' {:.1f} | {:.1f} | {:.1f} |'.format( 100 | class_average_acc * 100, globalacc * 100, 101 | iou.sum() / float(args.n_classes) * 100)) 102 | -------------------------------------------------------------------------------- /lib/camvid.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Shunta Saito 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | from __future__ import unicode_literals 7 | from chainer.dataset import dataset_mixin 8 | from os.path import basename as bn 9 | 10 | import cv2 as cv 11 | import numpy as np 12 | import os 13 | import re 14 | 15 | 16 | def _get_img_id(fn): 17 | bname = os.path.basename(fn) 18 | return re.search('([a-zA-Z\-]+_[0-9]+_[0-9]+)', bname).groups()[0] 19 | 20 | 21 | class CamVid(dataset_mixin.DatasetMixin): 22 | 23 | def __init__(self, img_dir, lbl_dir, list_fn, mean, std, shift_jitter=0, 24 | scale_jitter=0, fliplr=False, rotate=None, rotate_max=7, 25 | scale=1.0, ignore_labels=[11]): 26 | self.scale = scale 27 | self.lbl_fns = [] 28 | self.img_fns = [] 29 | for line in open(list_fn): 30 | img_fn, lbl_fn = line.split() 31 | self.img_fns.append('{}/{}'.format(img_dir, bn(img_fn))) 32 | self.lbl_fns.append('{}/{}'.format(lbl_dir, bn(lbl_fn))) 33 | self.mean = None if mean is None else np.load(mean) 34 | self.std = None if std is None else np.load(std) 35 | self.shift_jitter = shift_jitter 36 | self.scale_jitter = scale_jitter 37 | self.fliplr = fliplr 38 | self.rotate = rotate 39 | self.rotate_max = rotate_max 40 | self.ignore_labels = ignore_labels 41 | 42 | def __len__(self): 43 | return len(self.img_fns) 44 | 45 | def get_example(self, i): 46 | img = cv.imread(self.img_fns[i]).astype(np.float) 47 | if self.mean is not None: 48 | img -= self.mean 49 | if self.std is not None: 50 | img /= self.std 51 | if self.mean is None and self.std is None: 52 | img /= 255.0 53 | if self.scale != 1.0: 54 | img = cv.resize(img, None, fx=self.scale, fy=self.scale, 55 | interpolation=cv.INTER_NEAREST) 56 | 57 | lbl = cv.imread(self.lbl_fns[i], cv.IMREAD_GRAYSCALE).astype(np.int32) 58 | if self.scale != 1.0: 59 | lbl = cv.resize(lbl, None, fx=self.scale, fy=self.scale, 60 | interpolation=cv.INTER_NEAREST) 61 | if self.ignore_labels is not None: 62 | for ignore_l in self.ignore_labels: 63 | lbl[np.where(lbl == ignore_l)] = -1 64 | img_size = (lbl.shape[1], lbl.shape[0]) # W, H 65 | 66 | if self.shift_jitter != 0: 67 | s = np.random.randint(-self.shift_jitter, self.shift_jitter) 68 | cy, cx = img.shape[0] // 2 + s, img.shape[1] // 2 + s 69 | hh = (img.shape[0] - 2 * self.shift_jitter) // 2 70 | hw = (img.shape[1] - 2 * self.shift_jitter) // 2 71 | img = img[cy - hh:cy + hh, cx - hw:cx + hw] 72 | lbl = lbl[cy - hh:cy + hh, cx - hw:cx + hw] 73 | img = cv.resize(img, img_size, interpolation=cv.INTER_NEAREST) 74 | lbl = cv.resize(lbl, img_size, interpolation=cv.INTER_NEAREST) 75 | 76 | if self.scale_jitter != 0.0: 77 | h, w = img.shape[:2] 78 | s = np.random.uniform(1 - self.scale_jitter, 1 + self.scale_jitter) 79 | img = cv.resize(img, None, fx=s, fy=s, 80 | interpolation=cv.INTER_NEAREST) 81 | lbl = cv.resize(lbl, None, fx=s, fy=s, 82 | interpolation=cv.INTER_NEAREST) 83 | img = cv.resize(img, img_size, interpolation=cv.INTER_NEAREST) 84 | lbl = cv.resize(lbl, img_size, interpolation=cv.INTER_NEAREST) 85 | 86 | if self.rotate: 87 | h, w = img.shape[:2] 88 | s = np.clip(np.random.normal(), -self.rotate_max, self.rotate_max) 89 | mat = cv.getRotationMatrix2D((w // 2, h // 2), s, 1) 90 | img = cv.warpAffine(img, mat, (w, h), flags=cv.INTER_NEAREST) 91 | lbl = cv.warpAffine(lbl.astype(np.float), mat, 92 | (w, h), flags=cv.INTER_NEAREST, borderValue=-1) 93 | lbl = lbl.astype(np.int32) 94 | 95 | if self.fliplr and np.random.randint(0, 2) == 1: 96 | img = cv.flip(img, 1) 97 | lbl = cv.flip(lbl, 1) 98 | 99 | lbl = lbl.astype(np.int32) 100 | img = img.transpose(2, 0, 1).astype(np.float32) 101 | 102 | return img, lbl 103 | -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2016 Shunta Saito 5 | 6 | import argparse 7 | import chainer 8 | import chainer.functions as F 9 | import cv2 as cv 10 | import glob 11 | import imp 12 | import json 13 | import numpy as np 14 | import os 15 | import six 16 | 17 | Sky = [128, 128, 128] 18 | Building = [128, 0, 0] 19 | Pole = [192, 192, 128] 20 | Road_marking = [255, 69, 0] 21 | Road = [128, 64, 128] 22 | Pavement = [60, 40, 222] 23 | Tree = [128, 128, 0] 24 | SignSymbol = [192, 128, 128] 25 | Fence = [64, 64, 128] 26 | Car = [64, 0, 128] 27 | Pedestrian = [64, 64, 0] 28 | Bicyclist = [0, 128, 192] 29 | Unlabelled = [0, 0, 0] 30 | 31 | colors = [Sky, Building, Pole, Road, Pavement, Tree, SignSymbol, Fence, 32 | Car, Pedestrian, Bicyclist, Unlabelled] 33 | 34 | if __name__ == '__main__': 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument('--train_depth', type=int) 37 | parser.add_argument('--saved_args', type=str, default='results/args.json') 38 | parser.add_argument('--snapshot', type=str) 39 | parser.add_argument('--test_img_dir', type=str, default='data/test') 40 | parser.add_argument('--out_dir', type=str, default='results/prediction') 41 | parser.add_argument('--gpu', type=int, default=0) 42 | parser.add_argument('--mean', type=str, default=None) 43 | parser.add_argument('--std', type=str, default=None) 44 | parser.add_argument('--scale', type=float, default=1.0) 45 | parser.add_argument('--save_output', action='store_true', default=False) 46 | args = parser.parse_args() 47 | 48 | if not os.path.exists(args.out_dir): 49 | os.makedirs(args.out_dir) 50 | 51 | loaded_args = json.load(open(args.saved_args)) 52 | model = imp.load_source( 53 | loaded_args['model_name'], loaded_args['model_file']) 54 | model = getattr(model, loaded_args['model_name'])( 55 | loaded_args['n_encdec'], loaded_args['n_classes'], 56 | loaded_args['in_channel'], loaded_args['n_mid']) 57 | 58 | # Load parameters 59 | param = np.load(args.snapshot) 60 | prefix = 'updater/model:main/predictor' 61 | for key, arr in six.iteritems(param): 62 | if prefix in key: 63 | key = key.replace(prefix, '') 64 | names = [k for k in key.split('/') if len(k) > 0] 65 | link = model 66 | for name in names[:-1]: 67 | link = link.__dict__[name] 68 | if isinstance(link.__dict__[names[-1]], chainer.Variable): 69 | link.__dict__[names[-1]] = chainer.Variable(arr, volatile='on') 70 | else: 71 | link.__dict__[names[-1]] = arr 72 | 73 | if args.gpu >= 0: 74 | model.to_gpu(args.gpu) 75 | print('Model is transferred to GPU: {}'.format(args.gpu)) 76 | model.train = False 77 | 78 | mean = None if args.mean is None else np.load(args.mean) 79 | std = None if args.std is None else np.load(args.std) 80 | 81 | print('mean = {}, std = {}'.format(mean, std)) 82 | 83 | for img_fn in sorted(glob.glob('{}/*.png'.format(args.test_img_dir))): 84 | # Load & prepare image 85 | img = cv.imread(img_fn).astype(np.float) 86 | if mean is not None: 87 | img -= mean 88 | if std is not None: 89 | img /= std 90 | if mean is None and std is None: 91 | img /= 255.0 92 | if args.scale != 1.0: 93 | img = cv.resize(img, None, fx=args.scale, fy=args.scale) 94 | img = np.expand_dims(img, 0).transpose(0, 3, 1, 2) 95 | img = img.astype(np.float32) 96 | if args.gpu >= 0: 97 | with chainer.cuda.Device(args.gpu): 98 | img = chainer.cuda.cupy.asarray(img) 99 | img_var = chainer.Variable(img, volatile='on') 100 | 101 | # Forward 102 | ret = model(img_var, depth=args.train_depth) 103 | ret = F.softmax(ret).data[0].transpose(1, 2, 0) 104 | if args.gpu >= 0: 105 | with chainer.cuda.Device(args.gpu): 106 | ret = chainer.cuda.to_cpu(ret) 107 | if args.save_output: 108 | np.save('{}/{}_full'.format( 109 | args.out_dir, 110 | os.path.splitext(os.path.basename(img_fn))[0]), ret) 111 | 112 | # Create output 113 | out = np.zeros((ret.shape[0], ret.shape[1], 3), dtype=np.uint8) 114 | mask = np.argmax(ret, axis=2) 115 | for y in range(mask.shape[0]): 116 | for x in range(mask.shape[1]): 117 | out[y, x] = colors[mask[y, x]] 118 | 119 | base_fn = os.path.basename(img_fn) 120 | out_fn = '{}/{}'.format(args.out_dir, base_fn) 121 | cv.imwrite(out_fn, out) 122 | out_fn = '{}/{}'.format( 123 | args.out_dir, 124 | os.path.splitext(os.path.basename(out_fn))[0]) 125 | np.save(out_fn, mask) 126 | print(img_fn, img.shape, out_fn) 127 | -------------------------------------------------------------------------------- /models/segnet.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Shunta Saito 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | from __future__ import unicode_literals 7 | from chainer import cuda 8 | from chainer import reporter 9 | from models.softmax_cross_entropy import softmax_cross_entropy 10 | from models.upsampling_2d import upsampling_2d 11 | 12 | import chainer 13 | import chainer.functions as F 14 | import chainer.links as L 15 | import logging 16 | import math 17 | import numpy as np 18 | import six 19 | 20 | 21 | class EncDec(chainer.Chain): 22 | 23 | def __init__(self, in_channel, n_mid=64): 24 | w = math.sqrt(2) 25 | super(EncDec, self).__init__( 26 | enc=L.Convolution2D(in_channel, n_mid, 7, 1, 3, w), 27 | bn_m=L.BatchNormalization(n_mid), 28 | dec=L.Convolution2D(n_mid, n_mid, 7, 1, 3, w), 29 | bn_o=L.BatchNormalization(n_mid), 30 | ) 31 | self.p = F.MaxPooling2D(2, 2, use_cudnn=False) 32 | self.inside = None 33 | 34 | def upsampling_2d(self, pooler, x, outsize): 35 | return upsampling_2d( 36 | x, pooler.indexes, ksize=(pooler.kh, pooler.kw), 37 | stride=(pooler.sy, pooler.sx), pad=(pooler.ph, pooler.pw), 38 | outsize=outsize) 39 | 40 | def __call__(self, x, train=True): 41 | # Encode 42 | h = self.p(F.relu(self.bn_m(self.enc(x), test=not train))) 43 | 44 | # Run the inside network 45 | if self.inside is not None: 46 | h = self.inside(h, train) 47 | 48 | # Decode 49 | h = self.dec(self.upsampling_2d(self.p, h, x.shape[2:])) 50 | return self.bn_o(h, test=not train) 51 | 52 | 53 | class SegNet(chainer.Chain): 54 | 55 | """SegNet model architecture. 56 | 57 | This class has all Links to construct SegNet model and also optimizers to 58 | optimize each part (Encoder-Decoder pair or conv_cls) of SegNet. 59 | """ 60 | 61 | def __init__(self, n_encdec=4, n_classes=12, in_channel=3, n_mid=64): 62 | assert n_encdec >= 1 63 | w = math.sqrt(2) 64 | super(SegNet, self).__init__( 65 | conv_cls=L.Convolution2D(n_mid, n_classes, 1, 1, 0, w)) 66 | 67 | # Create and add EncDecs 68 | for i in six.moves.range(1, n_encdec + 1): 69 | name = 'encdec{}'.format(i) 70 | self.add_link(name, EncDec(n_mid if i > 1 else in_channel, n_mid)) 71 | for d in six.moves.range(1, n_encdec): 72 | encdec = getattr(self, 'encdec{}'.format(d)) 73 | encdec.inside = getattr(self, 'encdec{}'.format(d + 1)) 74 | setattr(self, 'encdec{}'.format(d), encdec) 75 | 76 | self.n_encdec = n_encdec 77 | self.n_classes = n_classes 78 | self.train = True 79 | 80 | def is_registered_link(self, name): 81 | return name in self._children 82 | 83 | def remove_link(self, name): 84 | """Remove a link that has the given name from this model 85 | 86 | Optimizer sees ``~Chain.namedparams()`` to know which parameters should 87 | be updated. And inside of ``namedparams()``, ``self._children`` is 88 | called to get names of all links included in the Chain. 89 | """ 90 | self._children.remove(name) 91 | 92 | def recover_link(self, name): 93 | self._children.append(name) 94 | 95 | def __call__(self, x, depth=1): 96 | assert 1 <= depth <= self.n_encdec 97 | h = F.local_response_normalization(x, 5, 1, 0.0005, 0.75) 98 | 99 | # Unchain the inner EncDecs after the given depth 100 | encdec = getattr(self, 'encdec{}'.format(depth)) 101 | encdec.inside = None 102 | 103 | h = self.encdec1(h, train=self.train) 104 | h = self.conv_cls(h) 105 | return h 106 | 107 | 108 | class SegNetLoss(chainer.Chain): 109 | 110 | def __init__(self, model, class_weight=None, train_depth=1): 111 | super(SegNetLoss, self).__init__(predictor=model) 112 | if class_weight is not None: 113 | logging.info('class_weight: {}'.format(class_weight)) 114 | if not isinstance(class_weight, np.ndarray): 115 | class_weight = np.asarray(class_weight, dtype=np.float32) 116 | self.class_weight = class_weight 117 | assert len(self.class_weight) == model.n_classes 118 | self.train_depth = train_depth 119 | 120 | def __call__(self, x, t): 121 | self.y = self.predictor(x, self.train_depth) 122 | if hasattr(self, 'class_weight'): 123 | if isinstance(x.data, cuda.cupy.ndarray) \ 124 | and not isinstance(self.class_weight, cuda.cupy.ndarray): 125 | self.class_weight = cuda.to_gpu( 126 | self.class_weight, device=x.data.device) 127 | self.loss = softmax_cross_entropy( 128 | self.y, t, class_weight=self.class_weight) 129 | else: 130 | self.loss = F.softmax_cross_entropy(self.y, t) 131 | reporter.report({'loss': self.loss}, self) 132 | return self.loss 133 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2016 Shunta Saito 5 | 6 | from __future__ import absolute_import 7 | from __future__ import division 8 | from __future__ import print_function 9 | from __future__ import unicode_literals 10 | from chainer import iterators 11 | from chainer import serializers 12 | from chainer import training 13 | from chainer.training import extensions 14 | from chainer.training import updater 15 | from lib import CamVid 16 | from lib import create_logger 17 | from lib import create_result_dir 18 | from lib import get_args 19 | from lib import get_model 20 | from lib import get_optimizer 21 | 22 | import json 23 | import logging 24 | import os 25 | 26 | 27 | @training.make_extension(default_name='recover_links') 28 | def recover_links(trainer): 29 | model = trainer.updater.get_optimizer('main').target 30 | n_encdec = model.predictor.n_encdec 31 | train_depth = model.train_depth 32 | for d in range(1, n_encdec + 1): 33 | if d != train_depth: 34 | model.predictor.recover_link('encdec{}'.format(d)) 35 | model.predictor.recover_link('conv_cls') 36 | 37 | 38 | @training.make_extension(default_name='remove_links') 39 | def remove_links(trainer): 40 | model = trainer.updater.get_optimizer('main').target 41 | n_encdec = model.predictor.n_encdec 42 | train_depth = model.train_depth 43 | for d in range(1, n_encdec + 1): 44 | if d != train_depth: 45 | if model.predictor.is_registered_link('encdec{}'.format(d)): 46 | model.predictor.remove_link('encdec{}'.format(d)) 47 | if train_depth > 1 and model.predictor.is_registered_link('conv_cls'): 48 | model.predictor.remove_link('conv_cls') 49 | 50 | if __name__ == '__main__': 51 | args = get_args() 52 | if args.result_dir is None: 53 | result_dir = create_result_dir(args.model_name) 54 | else: 55 | if not os.path.exists(args.result_dir): 56 | os.makedirs(args.result_dir) 57 | result_dir = args.result_dir 58 | json.dump(vars(args), open('{}/args_{}.json'.format( 59 | result_dir, args.train_depth), 'w')) 60 | create_logger(args, result_dir) 61 | 62 | # Instantiate model 63 | model = get_model( 64 | args.model_file, args.model_name, args.loss_file, args.loss_name, 65 | args.class_weight if args.use_class_weight else None, args.n_encdec, 66 | args.n_classes, args.in_channel, args.n_mid, args.train_depth, 67 | result_dir) 68 | 69 | # Initialize optimizer 70 | optimizer = get_optimizer( 71 | args.opt, args.lr, adam_alpha=args.adam_alpha, 72 | adam_beta1=args.adam_beta1, adam_beta2=args.adam_beta2, 73 | adam_eps=args.adam_eps, weight_decay=args.weight_decay) 74 | optimizer.setup(model) 75 | 76 | # Prepare devices 77 | devices = {} 78 | for gid in [int(i) for i in args.gpus.split(',')]: 79 | if 'main' not in devices: 80 | devices['main'] = gid 81 | else: 82 | devices['gpu{}'.format(gid)] = gid 83 | 84 | # Setting up datasets 85 | train = CamVid( 86 | args.train_img_dir, args.train_lbl_dir, args.train_list_fn, args.mean, 87 | args.std, args.shift_jitter, args.scale_jitter, args.fliplr, 88 | args.rotate, args.rotate_max, args.scale, args.ignore_labels) 89 | valid = CamVid( 90 | args.valid_img_dir, args.valid_lbl_dir, args.valid_list_fn, args.mean, 91 | args.std, ignore_labels=args.ignore_labels) 92 | logging.info('train: {}, valid: {}'.format(len(train), len(valid))) 93 | 94 | # Create iterators 95 | train_iter = iterators.MultiprocessIterator( 96 | train, args.batchsize, n_prefetch=10) 97 | valid_iter = iterators.SerialIterator(valid, args.valid_batchsize, 98 | repeat=False, shuffle=False) 99 | 100 | # Create updater 101 | updater = updater.ParallelUpdater(train_iter, optimizer, devices=devices) 102 | trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=result_dir) 103 | if args.resume is not None: 104 | serializers.load_npz(args.resume, trainer) 105 | 106 | # Add Evaluator 107 | trainer.extend( 108 | extensions.Evaluator(valid_iter, model, device=devices['main']), 109 | trigger=(args.valid_freq, 'epoch')) 110 | 111 | # Add dump_graph 112 | graph_name = 'encdec{}.dot'.format(args.train_depth) \ 113 | if not args.finetune else 'encdec_finetune.dot' 114 | trainer.extend(extensions.dump_graph('main/loss', out_name=graph_name)) 115 | 116 | # Add snapshot_object 117 | if not args.finetune: 118 | save_fn = 'encdec{}'.format(args.train_depth) + \ 119 | '_epoch_{.updater.epoch}.trainer' 120 | else: 121 | save_fn = 'encdec4_finetune_epoch_' + '{.updater.epoch}.trainer' 122 | trainer.extend(extensions.snapshot( 123 | filename=save_fn, trigger=(args.snapshot_epoch, 'epoch')), 124 | priority=0, invoke_before_training=False) 125 | 126 | # Add Logger 127 | if not args.finetune: 128 | log_fn = 'log_encdec{}.0'.format(args.train_depth) 129 | else: 130 | log_fn = 'log_encdec_finetune.0' 131 | if os.path.exists('{}/{}'.format(result_dir, log_fn)): 132 | n = int(log_fn.split('.')[-1]) 133 | log_fn = log_fn.replace(str(n), str(n + 1)) 134 | trainer.extend(extensions.ProgressBar()) 135 | if args.show_log_iter: 136 | log_trigger = args.show_log_iter, 'iteration' 137 | else: 138 | log_trigger = 1, 'epoch' 139 | trainer.extend(extensions.LogReport(trigger=log_trigger, log_name=log_fn)) 140 | trainer.extend(extensions.PrintReport( 141 | ['epoch', 'iteration', 'main/loss', 'validation/main/loss'])) 142 | 143 | # Add remover and recoverer 144 | if not args.finetune: 145 | trainer.extend(remove_links, trigger=(args.snapshot_epoch, 'epoch'), 146 | priority=500, invoke_before_training=True) 147 | trainer.extend(recover_links, trigger=(args.snapshot_epoch, 'epoch'), 148 | priority=400, invoke_before_training=False) 149 | 150 | trainer.run() 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SegNet 2 | 3 | SegNet implementation & experiments written in Chainer 4 | 5 | This is an unofficial implementation of SegNet. This implementation doesn't use L-BFGS for optimization. This uses Adam with the default settings. 6 | 7 | ## Requirements 8 | 9 | - Python 2.7.12+, 3.5.1+ 10 | - Chainer 1.17.0+ 11 | - scikit-learn 0.17.1 12 | - NumPy 1.11.0+ 13 | - six 1.10.0 14 | - OpenCV 3.1.0 15 | - `conda install -c https://conda.binstar.org/menpo opencv3` 16 | - Graphviz (To execute tests) 17 | - `sudo apt-get install -y graphviz` 18 | 19 | ## Download Dataset 20 | 21 | ``` 22 | bash experiments/download.sh 23 | ``` 24 | 25 | This shell script performs download CamVid dataset from [SegNet-Tutorial](https://github.com/alexgkendall/SegNet-Tutorial) repository owned by the original auther of the SegNet paper. 26 | 27 | ## Calculate dataset mean 28 | 29 | ``` 30 | python lib/calc_mean.py 31 | ``` 32 | 33 | It produces `train_mean.npy` and `train_std.npy` to normalize inputs during training and also `train_freq.csv` to weight the softmax cross entropy loss. 34 | 35 | ## Training 36 | 37 | ``` 38 | CUDA_VISIBLE_DEVICES=0 bash experiments/train.sh 39 | ``` 40 | 41 | You can specify which GPU you want to use by using `CUDA_VISIBLE_DEVICES` environment variable. Or if you directory use `train.py` instead of prepared training shell scripts in `experimetns` dir, you can easily specify the GPU ID by using `--gpu` argment. 42 | 43 | ### About train.sh 44 | 45 | To use the preliminarily calculated coefficients to weight the softmax cross entropy in class-wise manner, add `--use_class_weight` option to the above command. 46 | 47 | What the shell script `train.sh` do is a simple sequential training process. Once the first training for a most outer encoder-decoder pair, start the training for the next inner pair from the saved model state of the previous training process. 48 | 49 | If you would like to change the training settings and hyper parameters, please see the output of `python train.py --help` to check the argments it can take. 50 | 51 | ## Prediction 52 | 53 | Use `predict.py` to create prediction results on the test dataset. The below script executes the prediction script for all result dirs. 54 | 55 | ``` 56 | python experiments/batch_predict.py 57 | ``` 58 | 59 | ## Evaluation 60 | 61 | ``` 62 | python experiments/batch_evaluate.py 63 | ``` 64 | 65 | # Results 66 | 67 | The below table shows the evaluation results. Each column means: 68 | 69 | - **Class weight**: It means the weight for softmax cross entropy. If class weight calculated from training labels using `lib/calc_mean.py`, it shows `Yes`. If a set of class weights copied from the original implementation (from a caffe protobuf file) are used, it shows `Original`. If no class weight is used, it shows `No`. 70 | - **Standardization**: It means mean subtraction and stddev division. 71 | - **Data Aug.**: Data augmentation by random rotation and flipping left/right, and random translation and scale jittering. 72 | - **# conv channels**: The number of convolution filters used for all convolutional layers in the SegNet model. 73 | - **End-to-End**: `Pairwise` means the way to train the SegNet in encoder-decoder pairwise manner. `Finetune` means that the model was finetuned after the pairwise training of encoder-decorder pairs in end-to-end manner. `End-to-End` means that the model was trained in end-to-end manner from the beggining to the end. 74 | 75 | **Please find the more detailed results here: [`experiments/README.md`](https://github.com/mitmul/chainer-segnet/tree/master/experiments/README.md)** 76 | 77 | The bold numbers are better than the paper result, and bold and italic ones are the top scores in this table. 78 | 79 | | Model | Opt | Class weight | Standardization | Data Aug. | # conv channels | End-to-End | Class avg. | Global avg. | 80 | |:-----:|:---:|:------------:|:---------------:|:---------:|:---------------:|:----------:|:----------:|:-----------:| 81 | | SegNet - 4 layer (from paper) | L-BFGS | Original | ? | ? | 64 | Pairwise | 62.9 | 84.3 | 82 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | Yes | 128 | Pairwise | _**69.8**_ | **86.0** | 83 | | chainer-segnet | Adam (alpha=0.0001) | Original | Yes | Yes | 64 | Pairwise | **68.6** | 82.2 | 84 | | chainer-segnet | Adam (alpha=0.0001) | Original | Yes | Yes | 64 | Finetune | **68.5** | 83.3 | 85 | | chainer-segnet | Adam (alpha=0.0001) | Yes | No | Yes | 64 | Pairwise | **68.0** | 82.3 | 86 | | chainer-segnet | Adam (alpha=0.0001) | Original | Yes | Yes | 128 | Pairwise | **67.3** | **86.5** | 87 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | Yes | 128 | Finetune | **67.3** | **86.4** | 88 | | chainer-segnet | Adam (alpha=0.0001) | Yes | No | Yes | 64 | Finetune | **66.9** | 83.5 | 89 | | chainer-segnet | Adam (alpha=0.0001) | Original | Yes | Yes | 128 | Finetune | **66.3** | **86.2** | 90 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | Yes | 64 | Finetune | **65.5** | 82.9 | 91 | | chainer-segnet | Adam (alpha=0.0001) | Original | Yes | Yes | 64 | Finetune | **65.1** | 80.5 | 92 | | chainer-segnet | Adam (alpha=0.0001) | Original | Yes | Yes | 64 | Pairwise | **64.8** | 79.8 | 93 | | chainer-segnet | MomentumSGD (lr=0.0001) | Yes | Yes | Yes | 64 | Pairwise | **64.8** | 76.9 | 94 | | chainer-segnet | MomentumSGD (lr=0.0001) | Yes | Yes | Yes | 64 | Finetune | **64.7** | 79.8 | 95 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | Yes | 64 | Pairwise | **64.4** | 81.1 | 96 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | Yes | 64 | End-to-End | 62.6 | 82.3 | 97 | | chainer-segnet | Adam (alpha=0.0001) | No | No | Yes | 64 | Pairwise | 58.9 | _**86.9**_ | 98 | | chainer-segnet | Adam (alpha=0.0001) | No | Yes | Yes | 64 | Finetune | 58.0 | **85.5** | 99 | | chainer-segnet | Adam (alpha=0.0001) | No | No | Yes | 64 | Finetune | 57.2 | **87.0** | 100 | | chainer-segnet | Adam (alpha=0.0001) | No | Yes | Yes | 64 | Pairwise | 56.3 | **85.8** | 101 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | No | 64 | Pairwise | 56.2 | 83.9 | 102 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | No | 64 | Finetune | 54.1 | 83.3 | 103 | | chainer-segnet | Adam (alpha=0.0001) | Yes | Yes | No | 64 | End-to-End | 47.0 | 80.6 | 104 | 105 | ## Discussion 106 | 107 | - Several models exceeded the accuracy described in the original paper. 108 | - Larger number of channels leads better results. 109 | - The original class weights seem to be better than the ones calculated using `lib/calc_mean.py` in this repository. 110 | - Finetuning the model after enc-dec pairwise training improves global average accuracy but it decreases the class average accuracy in many cases. 111 | - Pairwise (w/ or w/o finetuning) is almost always better than completely end-to-end training. 112 | - Data augmentation is necessary. 113 | - Standardization decreases both accuracy (class avg. and global avg.) in several cases. 114 | 115 | # License 116 | 117 | MIT License (see LICENSE file). 118 | 119 | # Reference 120 | 121 | > Vijay Badrinarayanan, Alex Kendall and Roberto Cipolla "SegNet: A Deep Convolutional Encoder-Decoder Architecture for Image Segmentation." arXiv preprint arXiv:1511.00561, 2015\. [PDF](http://arxiv.org/abs/1511.00561) 122 | 123 | ## Official Implementation with Caffe 124 | 125 | - [alexgkendall/SegNet-Tutorial](https://github.com/alexgkendall/SegNet-Tutorial) 126 | - [alexgkendall/caffe-segnet](https://github.com/alexgkendall/caffe-segnet) 127 | -------------------------------------------------------------------------------- /lib/cmd_options.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Shunta Saito 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | from __future__ import unicode_literals 7 | 8 | import argparse 9 | import chainer 10 | import numpy 11 | import random 12 | 13 | 14 | def get_args(): 15 | parser = argparse.ArgumentParser() 16 | 17 | # Training settings 18 | parser.add_argument( 19 | '--model_file', type=str, default='models/segnet.py', 20 | help='The model filename with .py extension') 21 | parser.add_argument( 22 | '--model_name', type=str, default='SegNet', 23 | help='The model class name') 24 | parser.add_argument( 25 | '--result_dir', type=str, default=None, 26 | help='The directory path to store files') 27 | parser.add_argument( 28 | '--loss_file', type=str, default='models/segnet.py', 29 | help='The filename that contains the definition of loss function') 30 | parser.add_argument( 31 | '--loss_name', type=str, default='SegNetLoss', 32 | help='The name of the loss function') 33 | parser.add_argument( 34 | '--resume', type=str, default=None, 35 | help='Saved trainer state to be used for resuming') 36 | parser.add_argument( 37 | '--trunk', type=str, 38 | help='The pre-trained trunk param filename with .npz extension') 39 | parser.add_argument( 40 | '--epoch', type=int, default=1000, 41 | help='When the trianing will finish') 42 | parser.add_argument( 43 | '--gpus', type=str, default='0', 44 | help='GPU Ids to be used') 45 | parser.add_argument( 46 | '--batchsize', type=int, default=8, 47 | help='minibatch size') 48 | parser.add_argument( 49 | '--snapshot_epoch', type=int, default=10, 50 | help='The current learnt parameters in the model is saved every' 51 | 'this epoch') 52 | parser.add_argument( 53 | '--valid_freq', type=int, default=10, 54 | help='Perform test every this epoch (0 means no test)') 55 | parser.add_argument( 56 | '--valid_batchsize', type=int, default=16, 57 | help='The mini-batch size during validation loop') 58 | parser.add_argument( 59 | '--show_log_iter', type=int, default=None, 60 | help='Show loss value per this iterations') 61 | 62 | # Settings 63 | parser.add_argument( 64 | '--train_depth', type=int, default=1, 65 | help='The depth of an EncDec pair in the SegNet model to be trained.' 66 | 'This means the number of inside pairs of Encoder-Decoder.') 67 | parser.add_argument( 68 | '--shift_jitter', type=int, default=50, 69 | help='Shift jitter amount for data augmentation (typically 50)') 70 | parser.add_argument( 71 | '--scale_jitter', type=float, default=0.2, 72 | help='Scale jitter amount for data augmentation ' 73 | '(typically 0.2 for 0.8~1.2)') 74 | parser.add_argument( 75 | '--fliplr', action='store_true', default=False, 76 | help='Perform LR flipping during training for data augmentation') 77 | parser.add_argument( 78 | '--rotate', action='store_true', default=False, 79 | help='Perform rotation for data argumentation') 80 | parser.add_argument( 81 | '--rotate_max', type=float, default=7, 82 | help='Maximum roatation angle(degree) for data augmentation') 83 | parser.add_argument( 84 | '--scale', type=float, default=1.0, 85 | help='Scale for the input images and label images') 86 | parser.add_argument( 87 | '--n_classes', type=int, default=12, 88 | help='The number of classes that the model predicts') 89 | parser.add_argument( 90 | '--n_encdec', type=int, default=4, 91 | help='The number of Encoder-Decoder pairs that are included in the' 92 | ' Segnet model') 93 | parser.add_argument( 94 | '--in_channel', type=int, default=3, 95 | help='The number of channels of the input image') 96 | parser.add_argument( 97 | '--n_mid', type=int, default=64, 98 | help='The number of channels of convolutions in EncDec unit') 99 | parser.add_argument( 100 | '--mean', type=str, default=None, 101 | help='Mean npy over the training data') 102 | parser.add_argument( 103 | '--std', type=str, default=None, 104 | help='Stddev npy over the training data') 105 | parser.add_argument( 106 | '--use_class_weight', action='store_true', default=False, 107 | help='If it\'s given, the loss is weighted during training.') 108 | parser.add_argument( 109 | '--class_weight', type=str, default='data/train_freq.csv', 110 | help='The path to the file that contains inverse class frequency ' 111 | 'calculated using calc_mean.py') 112 | parser.add_argument( 113 | '--ignore_labels', type=str, default='11', 114 | help='The label id that is ignored during training') 115 | parser.add_argument( 116 | '--finetune', action='store_true', default=False, 117 | help='Train whole encoder-decorder pairs at a time.') 118 | 119 | # Dataset paths 120 | parser.add_argument( 121 | '--train_img_dir', type=str, default='data/train', 122 | help='Full path to images for trianing') 123 | parser.add_argument( 124 | '--valid_img_dir', type=str, default='data/val', 125 | help='Full path to images for validation') 126 | parser.add_argument( 127 | '--train_lbl_dir', type=str, default='data/trainannot', 128 | help='Full path to labels for trianing') 129 | parser.add_argument( 130 | '--valid_lbl_dir', type=str, default='data/valannot', 131 | help='Full path to labels for validation') 132 | parser.add_argument( 133 | '--train_list_fn', type=str, default='data/train.txt', 134 | help='Full path to the file list of training images') 135 | parser.add_argument( 136 | '--valid_list_fn', type=str, default='data/val.txt', 137 | help='Full path to the file list of validation images') 138 | 139 | # Optimization settings 140 | parser.add_argument( 141 | '--opt', type=str, default='MomentumSGD', 142 | choices=['MomentumSGD', 'Adam', 'AdaGrad', 'RMSprop'], 143 | help='Optimization method') 144 | parser.add_argument('--weight_decay', type=float, default=0.0005) 145 | parser.add_argument('--adam_alpha', type=float, default=0.001) 146 | parser.add_argument('--adam_beta1', type=float, default=0.9) 147 | parser.add_argument('--adam_beta2', type=float, default=0.999) 148 | parser.add_argument('--adam_eps', type=float, default=1e-8) 149 | parser.add_argument('--lr', type=float, default=0.01) 150 | parser.add_argument( 151 | '--lr_decay_freq', type=int, default=10, 152 | help='The learning rate will be decreased every this epoch') 153 | parser.add_argument( 154 | '--lr_decay_ratio', type=float, default=0.1, 155 | help='When the learning rate is decreased, this number will be' 156 | 'multiplied') 157 | parser.add_argument('--seed', type=int, default=1701) 158 | 159 | parser.add_argument('-f') # To call this from jupyter notebook 160 | 161 | args = parser.parse_args() 162 | args.class_weight = [ 163 | float(w) for w in open(args.class_weight).readline().split(',')] 164 | 165 | xp = chainer.cuda.cupy if chainer.cuda.available else numpy 166 | random.seed(args.seed) 167 | xp.random.seed(args.seed) 168 | numpy.random.seed(args.seed) 169 | args.ignore_labels = [int(l) for l in args.ignore_labels.split(',')] 170 | 171 | return args 172 | -------------------------------------------------------------------------------- /tests/test_softmax_cross_entropy.py: -------------------------------------------------------------------------------- 1 | from chainer import cuda 2 | from chainer import gradient_check 3 | from chainer import testing 4 | from chainer.testing import attr 5 | from chainer.testing import condition 6 | from models import softmax_cross_entropy 7 | 8 | import chainer 9 | import mock 10 | import numpy 11 | import six 12 | import unittest 13 | 14 | 15 | @testing.parameterize(*(testing.product({ 16 | 'shape': [None, (2, 3), (2, 3, 2), (2, 3, 2, 2)], 17 | 'cache_score': [True, False], 18 | 'normalize': [True, False], 19 | 'ignore_index': [None, (slice(None),), (0,), (0, 1), (0, 1, 0)], 20 | 'dtype': [numpy.float32], 21 | 'weight_apply': [False, True], 22 | }) + testing.product({ 23 | 'shape': [None, (2, 3), (2, 3, 2), (2, 3, 2, 2)], 24 | 'cache_score': [False], 25 | 'normalize': [True], 26 | 'ignore_index': [(0, 1)], 27 | 'dtype': [numpy.float16, numpy.float32, numpy.float64], 28 | 'weight_apply': [False, True], 29 | }))) 30 | class TestSoftmaxCrossEntropy(unittest.TestCase): 31 | 32 | def setUp(self): 33 | if self.shape is None: 34 | if self.dtype == numpy.float16: 35 | self.x = numpy.array([[-5, 1]], dtype=self.dtype) 36 | else: 37 | self.x = numpy.array([[-1000, 1]], dtype=self.dtype) 38 | self.t = numpy.array([0], dtype=numpy.int32) 39 | else: 40 | self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype) 41 | out_shape = (self.shape[0],) + self.shape[2:] 42 | self.t = numpy.random.randint( 43 | 0, self.shape[1], out_shape).astype(numpy.int32) 44 | if (self.ignore_index is not None and 45 | len(self.ignore_index) <= self.t.ndim): 46 | self.t[self.ignore_index] = -1 47 | self.check_forward_options = {} 48 | self.check_backward_options = {'dtype': numpy.float64} 49 | if self.dtype == numpy.float16: 50 | self.check_forward_options = {'atol': 5e-4, 'rtol': 5e-3} 51 | self.check_backward_options = { 52 | 'dtype': numpy.float64, 'atol': 5e-4, 'rtol': 5e-3} 53 | if self.weight_apply: 54 | self.class_weight = numpy.random.uniform( 55 | 0, 10, (self.x.shape[1],)).astype(self.dtype) 56 | else: 57 | self.class_weight = None 58 | 59 | def check_forward(self, x_data, t_data, class_weight, use_cudnn=True): 60 | x = chainer.Variable(x_data) 61 | t = chainer.Variable(t_data) 62 | loss = softmax_cross_entropy.softmax_cross_entropy( 63 | x, t, use_cudnn=use_cudnn, normalize=self.normalize, 64 | cache_score=self.cache_score, class_weight=class_weight) 65 | self.assertEqual(loss.data.shape, ()) 66 | self.assertEqual(loss.data.dtype, self.dtype) 67 | self.assertEqual(hasattr(loss.creator, 'y'), self.cache_score) 68 | loss_value = float(cuda.to_cpu(loss.data)) 69 | 70 | # Compute expected value 71 | loss_expect = 0.0 72 | count = 0 73 | x = numpy.rollaxis(self.x, 1, self.x.ndim).reshape( 74 | (self.t.size, self.x.shape[1])) 75 | t = self.t.ravel() 76 | for xi, ti in six.moves.zip(x, t): 77 | if ti == -1: 78 | continue 79 | log_z = numpy.ufunc.reduce(numpy.logaddexp, xi) 80 | if class_weight is None: 81 | loss_expect -= (xi - log_z)[ti] 82 | else: 83 | loss_expect -= (xi - log_z)[ti] * class_weight[ti] 84 | count += 1 85 | 86 | if self.normalize: 87 | if count == 0: 88 | loss_expect = 0.0 89 | else: 90 | loss_expect /= count 91 | else: 92 | loss_expect /= len(t_data) 93 | 94 | testing.assert_allclose( 95 | loss_expect, loss_value, **self.check_forward_options) 96 | 97 | @condition.retry(3) 98 | def test_forward_cpu(self): 99 | self.check_forward(self.x, self.t, self.class_weight) 100 | 101 | @attr.gpu 102 | @condition.retry(3) 103 | def test_forward_gpu(self): 104 | self.check_forward( 105 | cuda.to_gpu(self.x), cuda.to_gpu(self.t), 106 | None if not self.weight_apply else cuda.to_gpu(self.class_weight)) 107 | 108 | @attr.gpu 109 | @condition.retry(3) 110 | def test_forward_gpu_no_cudnn(self): 111 | self.check_forward( 112 | cuda.to_gpu(self.x), cuda.to_gpu(self.t), 113 | None if not self.weight_apply else cuda.to_gpu(self.class_weight), 114 | False) 115 | 116 | def check_backward(self, x_data, t_data, class_weight, use_cudnn=True): 117 | func = softmax_cross_entropy.SoftmaxCrossEntropy( 118 | use_cudnn=use_cudnn, cache_score=self.cache_score, 119 | class_weight=class_weight) 120 | gradient_check.check_backward( 121 | func, (x_data, t_data), None, eps=0.02, 122 | **self.check_backward_options) 123 | 124 | @condition.retry(3) 125 | def test_backward_cpu(self): 126 | self.check_backward(self.x, self.t, self.class_weight) 127 | 128 | @attr.gpu 129 | @condition.retry(3) 130 | def test_backward_gpu(self): 131 | self.check_backward( 132 | cuda.to_gpu(self.x), cuda.to_gpu(self.t), 133 | None if not self.weight_apply else cuda.to_gpu(self.class_weight)) 134 | 135 | @attr.gpu 136 | @condition.retry(3) 137 | def test_backward_gpu_no_cudnn(self): 138 | self.check_backward( 139 | cuda.to_gpu(self.x), cuda.to_gpu(self.t), 140 | None if not self.weight_apply else cuda.to_gpu(self.class_weight), 141 | False) 142 | 143 | 144 | @testing.parameterize( 145 | {'t_value': -2, 'valid': False}, 146 | {'t_value': 3, 'valid': False}, 147 | {'t_value': -1, 'valid': True}, # -1 is ignore_label 148 | ) 149 | class TestSoftmaxCrossEntropyValueCheck(unittest.TestCase): 150 | 151 | def setUp(self): 152 | self.x = numpy.random.uniform(-1, 1, (2, 2)).astype(numpy.float32) 153 | # `0` is required to avoid NaN 154 | self.t = numpy.array([self.t_value, 0], dtype=numpy.int32) 155 | self.original_debug = chainer.is_debug() 156 | chainer.set_debug(True) 157 | 158 | def tearDown(self): 159 | chainer.set_debug(self.original_debug) 160 | 161 | def check_value_check(self, x_data, t_data, use_cudnn): 162 | x = chainer.Variable(x_data) 163 | t = chainer.Variable(t_data) 164 | 165 | if self.valid: 166 | # Check if it throws nothing 167 | softmax_cross_entropy.softmax_cross_entropy(x, t, use_cudnn) 168 | else: 169 | with self.assertRaises(ValueError): 170 | softmax_cross_entropy.softmax_cross_entropy(x, t, use_cudnn) 171 | 172 | def test_value_check_cpu(self): 173 | self.check_value_check(self.x, self.t, False) 174 | 175 | @attr.gpu 176 | def test_value_check_gpu(self): 177 | self.check_value_check(self.x, self.t, False) 178 | 179 | @attr.gpu 180 | def test_value_check_gpu_cudnn(self): 181 | self.check_value_check(cuda.to_gpu(self.x), cuda.to_gpu(self.t), True) 182 | 183 | 184 | @testing.parameterize(*testing.product({ 185 | 'use_cudnn': [True, False], 186 | 'dtype': [numpy.float16, numpy.float32, numpy.float64], 187 | })) 188 | @attr.cudnn 189 | class TestSoftmaxCrossEntropyCudnnCall(unittest.TestCase): 190 | 191 | def setUp(self): 192 | self.x = cuda.cupy.random.uniform(-1, 1, (4, 3)).astype(self.dtype) 193 | self.t = cuda.cupy.random.randint(0, 3, (4,)).astype(numpy.int32) 194 | 195 | def forward(self): 196 | x = chainer.Variable(self.x) 197 | t = chainer.Variable(self.t) 198 | return softmax_cross_entropy.softmax_cross_entropy(x, t, self.use_cudnn) 199 | 200 | @unittest.skipIf(cuda.cudnn_enabled and 201 | cuda.cudnn.cudnn.getVersion() < 3000, 202 | 'Only cudnn ver>=3 supports softmax-log') 203 | def test_call_cudnn_forward(self): 204 | with mock.patch('cupy.cudnn.cudnn.softmaxForward') as func: 205 | self.forward() 206 | self.assertEqual(func.called, self.use_cudnn) 207 | 208 | # Note that SoftmaxCrossEntropy does not use cudnn on backward 209 | 210 | 211 | testing.run_module(__name__, __file__) 212 | -------------------------------------------------------------------------------- /tests/test_segnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2016 Shunta Saito 5 | 6 | from chainer import cuda 7 | from chainer import optimizers 8 | from chainer import testing 9 | from chainer import Variable 10 | from chainer.computational_graph import build_computational_graph 11 | from models import segnet 12 | 13 | import copy 14 | import numpy as np 15 | import os 16 | import re 17 | import six 18 | import subprocess 19 | import unittest 20 | 21 | _var_style = {'shape': 'octagon', 'fillcolor': '#E0E0E0', 'style': 'filled'} 22 | _func_style = {'shape': 'record', 'fillcolor': '#6495ED', 'style': 'filled'} 23 | 24 | if cuda.available: 25 | xp = cuda.cupy 26 | 27 | 28 | @testing.parameterize(*testing.product({ 29 | 'n_encdec': [1, 3, 4, 5], 30 | 'n_classes': [2, 3], 31 | 'n_mid': [2, 16, 32], 32 | 'x_shape': [(4, 3, 5, 7), (2, 2, 4, 6)], 33 | })) 34 | class TestSegNet(unittest.TestCase): 35 | 36 | def setUp(self): 37 | pass 38 | 39 | def get_xt(self): 40 | x = np.random.uniform(-1, 1, self.x_shape) 41 | x = Variable(x.astype(np.float32)) 42 | t = np.random.randint( 43 | 0, self.n_classes, 44 | (self.x_shape[0], self.x_shape[2], self.x_shape[3])) 45 | t = Variable(t.astype(np.int32)) 46 | return x, t 47 | 48 | def test_remove_link(self): 49 | opt = optimizers.MomentumSGD(lr=0.01) 50 | # Update each depth 51 | for depth in six.moves.range(1, self.n_encdec + 1): 52 | model = segnet.SegNet(self.n_encdec, self.n_classes, 53 | self.x_shape[1], self.n_mid) 54 | model = segnet.SegNetLoss( 55 | model, class_weight=None, train_depth=depth) 56 | opt.setup(model) 57 | 58 | # Deregister non-target links from opt 59 | if depth > 1: 60 | model.predictor.remove_link('conv_cls') 61 | for d in range(1, self.n_encdec + 1): 62 | if d != depth: 63 | model.predictor.remove_link('encdec{}'.format(d)) 64 | 65 | for name, link in model.namedparams(): 66 | if depth > 1: 67 | self.assertTrue( 68 | 'encdec{}'.format(depth) in name) 69 | else: 70 | self.assertTrue( 71 | 'encdec{}'.format(depth) in name or 'conv_cls' in name) 72 | 73 | def test_backward(self): 74 | opt = optimizers.MomentumSGD(lr=0.01) 75 | # Update each depth 76 | for depth in six.moves.range(1, self.n_encdec + 1): 77 | model = segnet.SegNet(self.n_encdec, self.n_classes, 78 | self.x_shape[1], self.n_mid) 79 | model = segnet.SegNetLoss( 80 | model, class_weight=None, train_depth=depth) 81 | opt.setup(model) 82 | 83 | # Deregister non-target links from opt 84 | if depth > 1: 85 | model.predictor.remove_link('conv_cls') 86 | for d in range(1, self.n_encdec + 1): 87 | if d != depth: 88 | model.predictor.remove_link('encdec{}'.format(d)) 89 | 90 | # Keep the initial values 91 | prev_params = { 92 | 'conv_cls': copy.deepcopy(model.predictor.conv_cls.W.data)} 93 | for d in range(1, self.n_encdec + 1): 94 | name = '/encdec{}/enc/W'.format(d) 95 | encdec = getattr(model.predictor, 'encdec{}'.format(d)) 96 | prev_params[name] = copy.deepcopy(encdec.enc.W.data) 97 | self.assertTrue(prev_params[name] is not encdec.enc.W.data) 98 | 99 | # Update the params 100 | x, t = self.get_xt() 101 | loss = model(x, t) 102 | loss.data *= 1E20 103 | model.cleargrads() 104 | loss.backward() 105 | opt.update() 106 | 107 | for d in range(1, self.n_encdec + 1): 108 | # The weight only in the target layer should be updated 109 | c = self.assertFalse if d == depth else self.assertTrue 110 | encdec = getattr(opt.target.predictor, 'encdec{}'.format(d)) 111 | self.assertTrue(hasattr(encdec, 'enc')) 112 | self.assertTrue(hasattr(encdec.enc, 'W')) 113 | self.assertTrue('/encdec{}/enc/W'.format(d) in prev_params) 114 | c(np.array_equal(encdec.enc.W.data, 115 | prev_params['/encdec{}/enc/W'.format(d)]), 116 | msg='depth:{} d:{} diff:{}'.format( 117 | depth, d, np.sum(encdec.enc.W.data - 118 | prev_params['/encdec{}/enc/W'.format(d)]))) 119 | if depth == 1: 120 | # The weight in the last layer should be updated 121 | self.assertFalse(np.allclose(model.predictor.conv_cls.W.data, 122 | prev_params['conv_cls'])) 123 | 124 | cg = build_computational_graph( 125 | [loss], 126 | variable_style=_var_style, 127 | function_style=_func_style 128 | ).dump() 129 | 130 | fn = 'tests/SegNet_bw_depth-{}_{}.dot'.format(self.n_encdec, depth) 131 | if os.path.exists(fn): 132 | continue 133 | with open(fn, 'w') as f: 134 | f.write(cg) 135 | subprocess.call( 136 | 'dot -Tpng {} -o {}'.format( 137 | fn, fn.replace('.dot', '.png')), shell=True) 138 | 139 | for name, param in model.namedparams(): 140 | encdec_depth = re.search('encdec([0-9]+)', name) 141 | if encdec_depth: 142 | ed = int(encdec_depth.groups()[0]) 143 | self.assertEqual(ed, depth) 144 | 145 | def test_save_normal_graphs(self): 146 | x = np.random.uniform(-1, 1, self.x_shape) 147 | x = Variable(x.astype(np.float32)) 148 | 149 | for depth in six.moves.range(1, self.n_encdec + 1): 150 | model = segnet.SegNet( 151 | n_encdec=self.n_encdec, in_channel=self.x_shape[1]) 152 | y = model(x, depth) 153 | cg = build_computational_graph( 154 | [y], 155 | variable_style=_var_style, 156 | function_style=_func_style 157 | ).dump() 158 | for e in range(1, self.n_encdec + 1): 159 | self.assertTrue('encdec{}'.format(e) in model._children) 160 | 161 | fn = 'tests/SegNet_x_depth-{}_{}.dot'.format(self.n_encdec, depth) 162 | if os.path.exists(fn): 163 | continue 164 | with open(fn, 'w') as f: 165 | f.write(cg) 166 | subprocess.call( 167 | 'dot -Tpng {} -o {}'.format( 168 | fn, fn.replace('.dot', '.png')), shell=True) 169 | 170 | def test_save_loss_graphs_no_class_weight(self): 171 | x = np.random.uniform(-1, 1, self.x_shape) 172 | x = Variable(x.astype(np.float32)) 173 | t = np.random.randint( 174 | 0, 12, (self.x_shape[0], self.x_shape[2], self.x_shape[3])) 175 | t = Variable(t.astype(np.int32)) 176 | 177 | for depth in six.moves.range(1, self.n_encdec + 1): 178 | model = segnet.SegNet(n_encdec=self.n_encdec, n_classes=12, 179 | in_channel=self.x_shape[1]) 180 | model = segnet.SegNetLoss( 181 | model, class_weight=None, train_depth=depth) 182 | y = model(x, t) 183 | cg = build_computational_graph( 184 | [y], 185 | variable_style=_var_style, 186 | function_style=_func_style 187 | ).dump() 188 | for e in range(1, self.n_encdec + 1): 189 | self.assertTrue( 190 | 'encdec{}'.format(e) in model.predictor._children) 191 | 192 | fn = 'tests/SegNet_xt_depth-{}_{}.dot'.format(self.n_encdec, depth) 193 | if os.path.exists(fn): 194 | continue 195 | with open(fn, 'w') as f: 196 | f.write(cg) 197 | subprocess.call( 198 | 'dot -Tpng {} -o {}'.format( 199 | fn, fn.replace('.dot', '.png')), shell=True) 200 | -------------------------------------------------------------------------------- /models/softmax_cross_entropy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Shunta Saito 2 | 3 | from chainer import cuda 4 | from chainer import function 5 | from chainer.functions.activation import log_softmax 6 | from chainer.utils import type_check 7 | 8 | import chainer 9 | import numpy 10 | import six 11 | 12 | 13 | class SoftmaxCrossEntropy(function.Function): 14 | 15 | """Softmax activation followed by a cross entropy loss.""" 16 | 17 | ignore_label = -1 18 | normalize = True 19 | 20 | def __init__(self, use_cudnn=True, normalize=True, cache_score=True, 21 | class_weight=None): 22 | self.use_cudnn = use_cudnn 23 | self.normalize = normalize 24 | self.cache_score = cache_score 25 | self.class_weight = class_weight 26 | if class_weight is not None: 27 | assert self.class_weight.ndim == 1 28 | assert self.class_weight.dtype.kind == 'f' 29 | 30 | def check_type_forward(self, in_types): 31 | type_check.expect(in_types.size() == 2) 32 | x_type, t_type = in_types 33 | 34 | type_check.expect( 35 | x_type.dtype.kind == 'f', 36 | t_type.dtype == numpy.int32, 37 | t_type.ndim == x_type.ndim - 1, 38 | 39 | x_type.shape[0] == t_type.shape[0], 40 | x_type.shape[2:] == t_type.shape[1:], 41 | ) 42 | 43 | def _check_input_values(self, x, t): 44 | if not (((0 <= t) & 45 | (t < x.shape[1])) | 46 | (t == self.ignore_label)).all(): 47 | msg = ('Each label `t` need to satisfy ' 48 | '`0 <= t < x.shape[1] or t == %d`' % self.ignore_label) 49 | raise ValueError(msg) 50 | 51 | def forward_cpu(self, inputs): 52 | x, t = inputs 53 | if chainer.is_debug(): 54 | self._check_input_values(x, t) 55 | 56 | log_y = log_softmax._log_softmax(x, self.use_cudnn) 57 | if self.cache_score: 58 | self.y = numpy.exp(log_y) 59 | if self.class_weight is not None: 60 | if self.class_weight.shape != x.shape: 61 | shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)] 62 | self.class_weight = numpy.broadcast_to( 63 | self.class_weight.reshape(shape), x.shape) 64 | log_y *= self.class_weight 65 | log_yd = numpy.rollaxis(log_y, 1) 66 | log_yd = log_yd.reshape(len(log_yd), -1) 67 | log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)] 68 | 69 | # deal with the case where the SoftmaxCrossEntropy is 70 | # unpickled from the old version 71 | if self.normalize: 72 | count = (t != self.ignore_label).sum() 73 | else: 74 | count = len(x) 75 | self._coeff = 1.0 / max(count, 1) 76 | 77 | y = (log_p * (t.ravel() != self.ignore_label)).sum(keepdims=True) \ 78 | * (-self._coeff) 79 | return y.reshape(()), 80 | 81 | def forward_gpu(self, inputs): 82 | cupy = cuda.cupy 83 | x, t = inputs 84 | if chainer.is_debug(): 85 | self._check_input_values(x, t) 86 | 87 | log_y = log_softmax._log_softmax(x, self.use_cudnn) 88 | if self.cache_score: 89 | self.y = cupy.exp(log_y) 90 | if self.class_weight is not None: 91 | shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)] 92 | log_y *= cupy.broadcast_to( 93 | self.class_weight.reshape(shape), x.shape) 94 | if self.normalize: 95 | coeff = cupy.maximum(1, (t != self.ignore_label).sum()) 96 | else: 97 | coeff = max(1, len(t)) 98 | self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype) 99 | 100 | log_y = cupy.rollaxis(log_y, 1, log_y.ndim) 101 | ret = cuda.reduce( 102 | 'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out', 103 | 't == -1 ? T(0) : log_y[_j * n_channel + t]', 104 | 'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd' 105 | )(t, log_y.reduced_view(), log_y.shape[-1], self._coeff) 106 | return ret, 107 | 108 | def backward_cpu(self, inputs, grad_outputs): 109 | x, t = inputs 110 | gloss = grad_outputs[0] 111 | if hasattr(self, 'y'): 112 | y = self.y.copy() 113 | else: 114 | y = log_softmax._log_softmax(x, self.use_cudnn) 115 | numpy.exp(y, out=y) 116 | if y.ndim == 2: 117 | gx = y 118 | gx[numpy.arange(len(t)), numpy.maximum(t, 0)] -= 1 119 | if self.class_weight is not None: 120 | c = self.class_weight[ 121 | numpy.arange(len(t)), numpy.maximum(t, 0)] 122 | gx *= numpy.broadcast_to(numpy.expand_dims(c, 1), gx.shape) 123 | gx *= (t != self.ignore_label).reshape((len(t), 1)) 124 | else: 125 | # in the case where y.ndim is higher than 2, 126 | # we think that a current implementation is inefficient 127 | # because it yields two provisional arrays for indexing. 128 | n_unit = t.size // len(t) 129 | gx = y.reshape(y.shape[0], y.shape[1], -1) 130 | fst_index = numpy.arange(t.size) // n_unit 131 | trd_index = numpy.arange(t.size) % n_unit 132 | gx[fst_index, numpy.maximum(t.ravel(), 0), trd_index] -= 1 133 | if self.class_weight is not None: 134 | c = self.class_weight.reshape(gx.shape) 135 | c = c[fst_index, numpy.maximum(t.ravel(), 0), trd_index] 136 | c = c.reshape(y.shape[0], 1, -1) 137 | gx *= numpy.broadcast_to(c, gx.shape) 138 | gx *= (t != self.ignore_label).reshape((len(t), 1, -1)) 139 | gx = gx.reshape(y.shape) 140 | gx *= gloss * self._coeff 141 | return gx, None 142 | 143 | def backward_gpu(self, inputs, grad_outputs): 144 | cupy = cuda.cupy 145 | x, t = inputs 146 | if hasattr(self, 'y'): 147 | y = self.y 148 | else: 149 | y = log_softmax._log_softmax(x, self.use_cudnn) 150 | cupy.exp(y, out=y) 151 | gloss = grad_outputs[0] 152 | n_unit = t.size // len(t) 153 | coeff = gloss * self._coeff 154 | if self.class_weight is None: 155 | gx = cuda.elementwise( 156 | 'T y, S t, raw T coeff, S n_channel, S n_unit', 157 | 'T gx', 158 | ''' 159 | const int c = (i / n_unit % n_channel); 160 | gx = (t == -1) ? 0 : (coeff[0] * (y - (c == t))); 161 | ''', 162 | 'softmax_crossent_bwd')( 163 | y, cupy.expand_dims(t, 1), coeff, x.shape[1], n_unit) 164 | else: 165 | gx = cuda.elementwise( 166 | 'T y, raw T w, S t, raw T coeff, S n_channel, S n_unit', 167 | 'T gx', 168 | ''' 169 | const int c = (i / n_unit % n_channel); 170 | gx = t == -1 ? 0 : coeff[0] * (y - (c == t)) * w[t]; 171 | ''', 172 | 'softmax_crossent_bwd')( 173 | y, self.class_weight, cupy.expand_dims(t, 1), coeff, 174 | x.shape[1], n_unit) 175 | return gx, None 176 | 177 | 178 | def softmax_cross_entropy( 179 | x, t, use_cudnn=True, normalize=True, cache_score=True, 180 | class_weight=None): 181 | """Computes cross entropy loss for pre-softmax activations. 182 | 183 | Args: 184 | x (~chainer.Variable): Variable holding a multidimensional array whose 185 | element indicates unnormalized log probability: the first axis of 186 | the variable represents the number of samples, and the second axis 187 | represents the number of classes. While this function computes 188 | a usual softmax cross entropy if the number of dimensions is equal 189 | to 2, it computes a cross entropy of the replicated softmax if the 190 | number of dimensions is greater than 2. 191 | t (~chainer.Variable): Variable holding an int32 vector of ground truth 192 | labels. If ``t[i] == -1``, corresponding ``x[i]`` is ignored. 193 | normalize (bool): If ``True``, this function normalizes the cross 194 | entropy loss across all instances. If ``False``, it only 195 | normalizes along a batch size. 196 | cache_score (bool): When it is ``True``, the function stores result 197 | of forward computation to use it on backward computation. It 198 | reduces computational cost though consumes more memory. 199 | class_weight (~numpy.ndarray or ~cupy.ndarray): An array that contains 200 | constant weights that will be multiplied with the loss values along 201 | with the second dimension. The shape of this array should be 202 | ``(x.shape[1],)``. 203 | 204 | Returns: 205 | Variable: A variable holding a scalar array of the cross entropy loss. 206 | 207 | .. note:: 208 | 209 | This function is differentiable only by ``x``. 210 | 211 | """ 212 | return SoftmaxCrossEntropy( 213 | use_cudnn, normalize, cache_score, class_weight)(x, t) 214 | -------------------------------------------------------------------------------- /models/upsampling_2d.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Shunta Saito 2 | 3 | from chainer import cuda 4 | from chainer.functions.pooling import pooling_2d 5 | from chainer.utils import conv 6 | from chainer.utils import type_check 7 | 8 | import numpy 9 | import six 10 | 11 | 12 | class Upsampling2D(pooling_2d.Pooling2D): 13 | 14 | """Upsampling over a set of 2d planes w/ indices used for max pooling.""" 15 | 16 | def __init__(self, indexes, ksize, stride=None, pad=0, outsize=None, 17 | cover_all=True): 18 | super(Upsampling2D, self).__init__(ksize, stride, pad, cover_all) 19 | self.indexes = indexes 20 | self.outh, self.outw = (None, None) if outsize is None else outsize 21 | 22 | def check_type_forward(self, in_types): 23 | n_in = in_types.size() 24 | type_check.expect(n_in == 1) 25 | x_type = in_types[0] 26 | 27 | type_check.expect( 28 | x_type.dtype.kind == 'f', 29 | x_type.ndim == 4, 30 | x_type.shape == self.indexes.shape, 31 | ) 32 | 33 | if self.outh is not None: 34 | expected_h = conv.get_conv_outsize( 35 | self.outh, self.kh, self.sy, self.ph, cover_all=self.cover_all) 36 | type_check.expect(x_type.shape[2] == expected_h) 37 | if self.outw is not None: 38 | expected_w = conv.get_conv_outsize( 39 | self.outw, self.kw, self.sx, self.pw, cover_all=self.cover_all) 40 | type_check.expect(x_type.shape[3] == expected_w) 41 | 42 | def forward_cpu(self, x): 43 | n, c, h, w = x[0].shape 44 | if self.outh is None: 45 | self.outh = conv.get_deconv_outsize( 46 | h, self.kh, self.sy, self.ph, cover_all=self.cover_all) 47 | if self.outw is None: 48 | self.outw = conv.get_deconv_outsize( 49 | w, self.kw, self.sx, self.pw, cover_all=self.cover_all) 50 | 51 | up_y = numpy.zeros((n, c, self.outh, self.outw), dtype=numpy.float32) 52 | up_y = conv.im2col_cpu( 53 | up_y, self.kh, self.kw, self.sy, self.sx, self.ph, self.pw, 54 | cover_all=self.cover_all) 55 | for n in six.moves.range(up_y.shape[0]): 56 | for c in six.moves.range(up_y.shape[1]): 57 | for oy in six.moves.range(up_y.shape[4]): 58 | for ox in six.moves.range(up_y.shape[5]): 59 | ky = self.indexes[n, c, oy, ox] // up_y.shape[3] 60 | kx = self.indexes[n, c, oy, ox] % up_y.shape[3] 61 | up_y[n, c, ky, kx, oy, ox] = x[0][n, c, oy, ox] 62 | up_y = conv.col2im_cpu(up_y, self.sy, self.sx, self.ph, 63 | self.pw, self.outh, self.outw) 64 | return up_y, 65 | 66 | def forward_gpu(self, x): 67 | xp = cuda.cupy 68 | n, c, h, w = x[0].shape 69 | if self.outh is None: 70 | self.outh = conv.get_deconv_outsize( 71 | h, self.kh, self.sy, self.ph, cover_all=self.cover_all) 72 | if self.outw is None: 73 | self.outw = conv.get_deconv_outsize( 74 | w, self.kw, self.sx, self.pw, cover_all=self.cover_all) 75 | up_y = xp.zeros((n, c, self.outh, self.outw), dtype=numpy.float32) 76 | up_y = conv.im2col_gpu( 77 | up_y, self.kh, self.kw, self.sy, self.sx, self.ph, self.pw, 78 | cover_all=self.cover_all) 79 | up_y = up_y.transpose(0, 1, 4, 5, 2, 3) 80 | n, c, oy, ox, ky, kx = up_y.shape 81 | indexes = xp.asarray(self.indexes, dtype=numpy.int32) 82 | xp.ElementwiseKernel( 83 | 'int32 index, float32 x, int32 n, int32 c, int32 oy, int32 ox,' 84 | 'int32 ky, int32 kx', 'raw float32 up_y', 85 | ''' 86 | int yn = i / c / oy / ox; 87 | int yc = (i / oy / ox) % c; 88 | int yoy = (i / ox) % oy; 89 | int yox = i % ox; 90 | up_y[yn * c * oy * ox * ky * kx + \ 91 | yc * oy * ox * ky * kx + \ 92 | yoy * ox * ky * kx + \ 93 | yox * ky * kx + \ 94 | index] = x; 95 | ''', 96 | 'upsampling_2d_fwd')(indexes, x[0], n, c, oy, ox, ky, kx, up_y) 97 | up_y = up_y.transpose(0, 1, 4, 5, 2, 3) 98 | up_y = conv.col2im_gpu(up_y, self.sy, self.sx, self.ph, self.pw, 99 | self.outh, self.outw) 100 | return up_y, 101 | 102 | def backward_cpu(self, x, gy): 103 | gcol = conv.im2col_cpu( 104 | gy[0], self.kh, self.kw, self.sy, self.sx, self.ph, self.pw, 105 | cover_all=self.cover_all) 106 | 107 | gcol = gcol.transpose(0, 1, 4, 5, 2, 3) 108 | n, c, oy, ox, ky, kx = gcol.shape 109 | gcol = gcol.reshape((n, c, oy, ox, ky * kx)) 110 | gx = numpy.empty((n, c, oy, ox), dtype=x[0].dtype) 111 | for n in six.moves.range(gcol.shape[0]): 112 | for c in six.moves.range(gcol.shape[1]): 113 | for oy in six.moves.range(gcol.shape[2]): 114 | for ox in six.moves.range(gcol.shape[3]): 115 | gx[n, c, oy, ox] = \ 116 | gcol[n, c, oy, ox][self.indexes[n, c, oy, ox]] 117 | return gx, 118 | 119 | def backward_gpu(self, x, gy): 120 | xp = cuda.cupy 121 | gcol = conv.im2col_gpu( 122 | gy[0], self.kh, self.kw, self.sy, self.sx, self.ph, self.pw, 123 | cover_all=self.cover_all) 124 | 125 | gcol = gcol.transpose(0, 1, 4, 5, 2, 3) 126 | n, c, oy, ox, ky, kx = gcol.shape 127 | gcol = gcol.reshape((n, c, oy, ox, ky * kx)) 128 | indexes = xp.asarray(self.indexes, dtype=numpy.int32) 129 | gx = xp.empty((n, c, oy, ox), dtype=x[0].dtype) 130 | xp.ElementwiseKernel( 131 | 'int32 indexes, raw float32 gcol, int32 n, int32 c, int32 oy,' 132 | 'int32 ox, int32 ky, int32 kx', 133 | 'raw float32 gx', 134 | ''' 135 | int ind_n = i / c / oy / ox; 136 | int ind_c = (i / oy / ox) % c; 137 | int ind_oy = (i / ox) % oy; 138 | int ind_ox = i % ox; 139 | int gcol_ky = indexes / kx; 140 | int gcol_kx = indexes % kx; 141 | float top_gx = gcol[ind_n * c * oy * ox * ky * kx + \ 142 | ind_c * oy * ox * ky * kx + \ 143 | ind_oy * ox * ky * kx + \ 144 | ind_ox * ky * kx + \ 145 | gcol_ky * kx + \ 146 | gcol_kx]; 147 | gx[ind_n * c * oy * ox + \ 148 | ind_c * oy * ox + \ 149 | ind_oy * ox + \ 150 | ind_ox] = top_gx; 151 | ''', 152 | 'upsampling_2d_bwd')(indexes, gcol, n, c, oy, ox, ky, kx, gx) 153 | 154 | return gx, 155 | 156 | 157 | def upsampling_2d( 158 | x, indexes, ksize, stride=None, pad=0, outsize=None, cover_all=True): 159 | """Upsampling using pooling indices. 160 | 161 | This function produces an upsampled image using pooling indices. 162 | 163 | .. admonition:: Example 164 | 165 | It should be noted that you need to specify ``use_cudnn=False`` when 166 | you create MaxPooling2D object because if cuDNN used for operating 167 | max pooling, ``indexes`` is never created and stored in the 168 | MaxPooling2D object. 169 | 170 | >>> p = F.MaxPooling2D(2, 2, use_cudnn=False) 171 | >>> x = numpy.arange(1, 37).reshape(1, 1, 6, 6).astype('f') 172 | >>> x = chainer.Variable(x) 173 | >>> x.data 174 | array([[[[ 1., 2., 3., 4., 5., 6.], 175 | [ 7., 8., 9., 10., 11., 12.], 176 | [ 13., 14., 15., 16., 17., 18.], 177 | [ 19., 20., 21., 22., 23., 24.], 178 | [ 25., 26., 27., 28., 29., 30.], 179 | [ 31., 32., 33., 34., 35., 36.]]]], dtype=float32) 180 | 181 | This is the original ``x`` before max pooling. 182 | 183 | >>> pooled_x = p(x) 184 | >>> pooled_x.data 185 | array([[[[ 8., 10., 12.], 186 | [ 20., 22., 24.], 187 | [ 32., 34., 36.]]]], dtype=float32) 188 | 189 | This is the output of the max pooling operation. ``upsampling_2d`` 190 | needs ``indexes`` array stored in the max pooling object ``p``. 191 | 192 | >>> upsampled_x = F.upsampling_2d( 193 | ... pooled_x, p.indexes, p.kh, p.sy, p.ph, x.shape[2:]) 194 | >>> upsampled_x.shape 195 | (1, 1, 6, 6) 196 | >>> upsampled_x.data 197 | array([[[[ 0., 0., 0., 0., 0., 0.], 198 | [ 0., 8., 0., 10., 0., 12.], 199 | [ 0., 0., 0., 0., 0., 0.], 200 | [ 0., 20., 0., 22., 0., 24.], 201 | [ 0., 0., 0., 0., 0., 0.], 202 | [ 0., 32., 0., 34., 0., 36.]]]], dtype=float32) 203 | 204 | Args: 205 | x (~chainer.Variable): Input variable. 206 | indexes (~numpy.ndarray or ~cupy.ndarray): Index array that was used 207 | to calculate x with MaxPooling2D. 208 | ksize (int or (int, int)): ksize attribtue of MaxPooling2D object that 209 | is used to calculate x 210 | stride (int or (int, int)): stride attribtue of MaxPooling2D object 211 | that is used to calculate x 212 | pad (int or (int, int)): pad attribtue of MaxPooling2D object that is 213 | used to calculate x 214 | outsize ((int, int)): Expected output size (height, width). 215 | cover_all (bool): Whether cover_all is used in the MaxPooling2D object 216 | or not. 217 | 218 | Returns: 219 | ~chainer.Variable: Output variable. 220 | """ 221 | return Upsampling2D(indexes, ksize, stride, pad, outsize, cover_all)(x) 222 | -------------------------------------------------------------------------------- /experiments/README.md: -------------------------------------------------------------------------------- 1 | # Experimental results 2 | 3 | ## results_opt-MomentumSGD_lr-0.0001_alpha-0.0001_2016-12-26_113051/pred_encdec4_finetune_epoch_1500 4 | 5 | - Global acc = 0.7983081285353912 6 | - Class average acc = 0.6466110597737679 7 | 8 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 9 | |:-----:|:---:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:|:----:| 10 | | 72.6 | 64.3 | 93.9 | 88.2 | 52.1 | 90.0 | 78.3 | 58.1 | 55.8 | 69.5 | 53.0 | 64.7 | 79.8 | 43.4 | 11 | 12 | -------------------- 13 | 14 | ## results_opt-MomentumSGD_lr-0.0001_alpha-0.0001_2016-12-26_113051/pred_encdec4_epoch_1200 15 | 16 | - Global acc = 0.7685316236205066 17 | - Class average acc = 0.647568823415377 18 | 19 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 20 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 21 | | 65.8 | 65.6 | 92.2 | 86.7 | 51.4 | 86.7 | 80.3 | 61.5 | 60.8 | 66.9 | 59.2 | 64.8 | 76.9 | 40.9 | 22 | 23 | -------------------- 24 | 25 | ## results_NoCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_113042/pred_encdec4_finetune_epoch_1500 26 | 27 | - Global acc = 0.8552323784459714 28 | - Class average acc = 0.5797935453845088 29 | 30 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 31 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 32 | | 88.7 | 73.4 | 96.6 | 82.4 | 42.9 | 96.8 | 45.4 | 35.4 | 29.7 | 63.1 | 41.3 | 58.0 | 85.5 | 48.4 | 33 | 34 | -------------------- 35 | 36 | ## results_NoCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_113042/pred_encdec4_epoch_1200 37 | 38 | - Global acc = 0.8579101065504839 39 | - Class average acc = 0.5626453561537881 40 | 41 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 42 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 43 | | 89.7 | 78.9 | 94.4 | 83.2 | 38.0 | 97.1 | 27.8 | 36.9 | 23.5 | 61.6 | 44.0 | 56.3 | 85.8 | 47.5 | 44 | 45 | -------------------- 46 | 47 | ## results_end2end_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_175131/pred_encdec4_finetune_epoch_1000 48 | 49 | - Global acc = 0.8228313492566198 50 | - Class average acc = 0.6255555466943951 51 | 52 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 53 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 54 | | 77.6 | 68.7 | 92.5 | 84.6 | 47.3 | 89.5 | 74.1 | 43.0 | 54.7 | 85.1 | 33.6 | 62.6 | 82.3 | 45.8 | 55 | 56 | -------------------- 57 | 58 | ## results_end2end_noda_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_140246/pred_encdec4_finetune_epoch_1000 59 | 60 | - Global acc = 0.8060916680813461 61 | - Class average acc = 0.4699122638760181 62 | 63 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 64 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 65 | | 86.8 | 56.9 | 93.4 | 72.4 | 9.6 | 95.5 | 24.4 | 25.2 | 24.4 | 58.5 | 16.8 | 47.0 | 80.6 | 38.4 | 66 | 67 | -------------------- 68 | 69 | ## results_NoSTD_NoCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_185951/pred_encdec4_finetune_epoch_1500 70 | 71 | - Global acc = 0.8703864305232749 72 | - Class average acc = 0.5716801498323112 73 | 74 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 75 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 76 | | 90.2 | 81.0 | 96.4 | 87.1 | 28.2 | 96.1 | 47.3 | 33.5 | 25.6 | 68.5 | 32.1 | 57.2 | 87.0 | 49.2 | 77 | 78 | -------------------- 79 | 80 | ## results_NoSTD_NoCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_185951/pred_encdec4_epoch_1200 81 | 82 | - Global acc = 0.8686509009255065 83 | - Class average acc = 0.589028498135272 84 | 85 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 86 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 87 | | 87.9 | 79.7 | 96.6 | 86.9 | 36.9 | 95.3 | 48.9 | 33.5 | 28.9 | 74.6 | 37.6 | 58.9 | 86.9 | 49.4 | 88 | 89 | -------------------- 90 | 91 | ## results_n-mid-128_opt-Adam_lr-0.001_alpha-0.0001_2016-12-27_195321/pred_encdec4_epoch_1200 92 | 93 | - Global acc = 0.8601476375291243 94 | - Class average acc = 0.69750120240245 95 | 96 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 97 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 98 | | 77.1 | 83.9 | 94.5 | 86.7 | 56.6 | 92.8 | 79.7 | 57.2 | 55.4 | 89.0 | 64.1 | 69.8 | 86.0 | 51.3 | 99 | 100 | -------------------- 101 | 102 | ## results_n-mid-128_opt-Adam_lr-0.001_alpha-0.0001_2016-12-27_195321/pred_encdec4_finetune_epoch-1500 103 | 104 | - Global acc = 0.8635421968486442 105 | - Class average acc = 0.672926891245397 106 | 107 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 108 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 109 | | 81.1 | 82.7 | 95.1 | 86.4 | 53.8 | 94.9 | 71.0 | 49.0 | 55.4 | 78.9 | 59.4 | 67.3 | 86.4 | 52.1 | 110 | 111 | -------------------- 112 | 113 | ## results_origCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-28_170243/pred_encdec4_finetune_epoch_1500 114 | 115 | - Global acc = 0.8050820608384763 116 | - Class average acc = 0.650921577769523 117 | 118 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 119 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 120 | | 74.5 | 67.6 | 91.3 | 83.9 | 64.5 | 91.5 | 82.7 | 52.2 | 57.5 | 69.0 | 46.4 | 65.1 | 80.5 | 44.7 | 121 | 122 | -------------------- 123 | 124 | ## results_origCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-28_170243/pred_encdec4_epoch_1200 125 | 126 | - Global acc = 0.798311899292278 127 | - Class average acc = 0.6479471421499633 128 | 129 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 130 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 131 | | 72.8 | 72.7 | 90.9 | 89.3 | 59.3 | 88.9 | 80.5 | 45.8 | 62.7 | 66.7 | 47.8 | 64.8 | 79.8 | 44.0 | 132 | 133 | -------------------- 134 | 135 | ## results_origCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-28_170410/pred_encdec4_epoch_1200 136 | 137 | - Global acc = 0.8218020875891555 138 | - Class average acc = 0.686322810739146 139 | 140 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 141 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 142 | | 66.7 | 79.2 | 93.8 | 87.3 | 72.6 | 93.8 | 74.6 | 54.7 | 64.1 | 78.8 | 58.0 | 68.6 | 82.2 | 48.3 | 143 | 144 | -------------------- 145 | 146 | ## results_origCW_opt-Adam_lr-0.001_alpha-0.0001_2016-12-28_170410/pred_encdec4_finetune_epoch_1500 147 | 148 | - Global acc = 0.8333214657799809 149 | - Class average acc = 0.685018129484241 150 | 151 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 152 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 153 | | 71.1 | 78.9 | 94.3 | 89.8 | 66.8 | 92.1 | 70.5 | 55.9 | 60.1 | 83.3 | 59.3 | 68.5 | 83.3 | 48.6 | 154 | 155 | -------------------- 156 | 157 | ## results_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_110810/pred_encdec4_finetune_epoch_1500 158 | 159 | - Global acc = 0.8290296987653379 160 | - Class average acc = 0.6549302124831567 161 | 162 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 163 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 164 | | 76.6 | 75.3 | 93.9 | 88.7 | 51.5 | 91.6 | 77.5 | 53.1 | 57.2 | 73.7 | 46.8 | 65.5 | 82.9 | 47.3 | 165 | 166 | -------------------- 167 | 168 | ## results_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_110810/pred_encdec4_epoch_1200 169 | 170 | - Global acc = 0.8108681064914433 171 | - Class average acc = 0.643636264400267 172 | 173 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 174 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 175 | | 76.8 | 67.8 | 92.5 | 89.9 | 45.7 | 89.0 | 79.0 | 55.8 | 54.2 | 73.2 | 48.7 | 64.4 | 81.1 | 45.2 | 176 | 177 | -------------------- 178 | 179 | ## results_origCW_n-mid-128_opt-Adam_lr-0.001_alpha-0.0001_2016-12-29_092620/pred_encdec4_finetune_epoch_1500 180 | 181 | - Global acc = 0.8615318927130821 182 | - Class average acc = 0.6632390609343596 183 | 184 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 185 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 186 | | 81.4 | 82.6 | 93.5 | 84.7 | 50.1 | 93.1 | 71.8 | 47.0 | 52.2 | 85.9 | 53.7 | 66.3 | 86.2 | 50.2 | 187 | 188 | -------------------- 189 | 190 | ## results_origCW_n-mid-128_opt-Adam_lr-0.001_alpha-0.0001_2016-12-29_092620/pred_encdec4_epoch_1200 191 | 192 | - Global acc = 0.8650127662073652 193 | - Class average acc = 0.6732021562132452 194 | 195 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 196 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 197 | | 81.1 | 79.6 | 95.9 | 84.6 | 52.4 | 95.5 | 78.8 | 52.4 | 57.0 | 80.6 | 49.9 | 67.3 | 86.5 | 52.1 | 198 | 199 | -------------------- 200 | 201 | ## results_NoDA_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_112544/pred_encdec4_finetune_epoch_1500 202 | 203 | - Global acc = 0.8328385764545624 204 | - Class average acc = 0.5406803050062763 205 | 206 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 207 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 208 | | 85.9 | 64.5 | 95.3 | 77.8 | 19.5 | 96.0 | 45.5 | 39.3 | 31.6 | 67.0 | 26.6 | 54.1 | 83.3 | 43.7 | 209 | 210 | -------------------- 211 | 212 | ## results_NoDA_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_112544/pred_encdec4_epoch_1200 213 | 214 | - Global acc = 0.839120915698867 215 | - Class average acc = 0.5619635454617927 216 | 217 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 218 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 219 | | 84.6 | 68.9 | 95.8 | 82.9 | 25.3 | 94.8 | 49.9 | 34.4 | 34.0 | 71.1 | 32.7 | 56.2 | 83.9 | 45.0 | 220 | 221 | -------------------- 222 | 223 | ## results_NoSTD_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_190006/pred_encdec4_finetune_epoch_1500 224 | 225 | - Global acc = 0.8349017744949465 226 | - Class average acc = 0.668564998740226 227 | 228 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 229 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 230 | | 74.4 | 75.8 | 94.1 | 88.6 | 60.2 | 90.8 | 74.0 | 52.5 | 59.6 | 85.9 | 46.5 | 66.9 | 83.5 | 47.6 | 231 | 232 | -------------------- 233 | 234 | ## results_NoSTD_opt-Adam_lr-0.001_alpha-0.0001_2016-12-26_190006/pred_encdec4_epoch_1200 235 | 236 | - Global acc = 0.82314434790532 237 | - Class average acc = 0.6798899868582285 238 | 239 | | Building | Tree | Sky | Car | SignSymbol | Road | Pedestrian | Fence | Pole | Pavement | Bicyclist | Class avg. | Global avg. | IoU | 240 | |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:| 241 | | 69.8 | 78.1 | 93.4 | 88.7 | 59.2 | 91.0 | 78.4 | 56.8 | 61.7 | 82.5 | 56.3 | 68.0 | 82.3 | 46.6 | 242 | --------------------------------------------------------------------------------