├── models ├── deeplab_details │ ├── __init__.py │ ├── sync_batchnorm │ │ ├── __init__.py │ │ ├── comm.py │ │ └── batchnorm.py │ ├── backbone │ │ ├── __init__.py │ │ ├── mobilenet.py │ │ ├── resnet.py │ │ ├── xception.py │ │ └── drn.py │ ├── decoder.py │ ├── deeplab.py │ └── aspp.py ├── __init__.py ├── simple_conv.py ├── common.py ├── resnet.py ├── deeplab.py ├── mobilenetv2.py └── mobilenetv3.py ├── tensorboard_events ├── MobileNetV2_035_dy_tiny_300ep │ └── events.out.tfevents.1616111307.theta.10525.0 ├── DeepLabV3+_x0.5_dy_pascal │ ├── events.out.tfevents.1616394553.17bec54ce095.59.0 │ ├── events.out.tfevents.1616396175.17bec54ce095.59.1 │ ├── events.out.tfevents.1616397390.d6698bb24c52.59.0 │ ├── events.out.tfevents.1616434156.aaf44e041659.58.0 │ └── events.out.tfevents.1616372319.c5e4336cf6d4.1444.0 ├── MoblieNetV2_baseline │ └── events.out.tfevents.1616141574.a7a84a2bec98.573 (10).0 ├── DeepLabV3+_x1.0_dy_pascal_sbd │ └── events.out.tfevents.1616452269.9bc1026ea589.59.1 ├── DeepLabV3+_x0.5_baseline_pascal │ ├── events.out.tfevents.1616441878.aaf44e041659.58.2 │ └── events.out.tfevents.1616485358.adffa85e925e.58.0 ├── MobileNetV2_035_dy_tiny_t6020_300ep │ └── events.out.tfevents.1616241245.theta.16175.0 ├── MobileNetV2_035_dy_tiny_t10040_300ep │ ├── events.out.tfevents.1616295195.theta.27968.0 │ └── config.yaml ├── DeepLabV3+_x1.0_baseline_pascal_sbd │ ├── events.out.tfevents.1616475592.9bc1026ea589.59.2 │ ├── events.out.tfevents.1616492304.8929e58b1f44.59.0 │ ├── events.out.tfevents.1616492681.8929e58b1f44.59.1 │ └── events.out.tfevents.1616484234.9bc1026ea589.10523.0 └── MobileNetV2_035_dy_tiny_t6060_300ep │ └── events.out.tfevents.1616295624.18f1170a923a.71.0 ├── .gitignore ├── configs ├── config.yaml ├── MobileNetV2_DY_05.yaml ├── MobileNetV3_DY_1_Tiny.yaml ├── MobileNetV3_DY_05_Tiny.yaml ├── MobileNetV2_Vanilla_05.yaml ├── MobileNetV3_Vanilla_05_Tiny.yaml ├── MobileNetV3_Vanilla_1_Tiny.yaml ├── MobileNetV2_DY_035.yaml ├── MobileNetV3_DY_05_Imagenette.yaml ├── MobileNetV3_DY_1_Imagenette.yaml ├── MobileNetV3_DY_025_Imagenette.yaml ├── MobileNetV3_Vanilla_1_Imagenette.yaml ├── MobileNetV3_Vanilla_025_Imagenette.yaml ├── MobileNetV3_Vanilla_05_Imagenette.yaml ├── MobileNetV2_Vanilla_035.yaml ├── deeplabv3plus.yaml ├── MobileNetV2_DY_035_60_20.yaml ├── MobileNetV2_DY_035_60_60.yaml ├── deeplabv3plus_0.5.yaml ├── Resnet_DY_1.yaml ├── Resnet_DY_025.yaml ├── Resnet_DY_05.yaml ├── Resnet_Vanilla_1.yaml ├── Resnet_Vanilla_05.yaml ├── Resnet_Vanilla_025.yaml ├── dy_deeplabv3plus.yaml ├── Resnet_DY_05_smooth.yaml ├── dy_deeplabv3plus_0.5.yaml ├── Resnet_DY_Leaky_05_smooth.yaml ├── Resnet_Vanilla_05_smooth.yaml └── Resnet_DY_Improve_05.yaml ├── data ├── mnist_dataset.py ├── __init__.py ├── pascalvoc2012_dataset.py ├── imagenette_dataset.py ├── tinyimagenet_dataset.py └── sb_dataset.py ├── LICENSE ├── utils ├── utils.py ├── segmentation_metrics.py ├── options.py └── parsable_options.py ├── environment.yml ├── notebooks ├── DyConv_training.ipynb └── DyConv_inspect.ipynb ├── inspect_attention.py ├── train.py ├── dynamic_convolutions.py └── README.md /models/deeplab_details/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorboard_events/MobileNetV2_035_dy_tiny_300ep/events.out.tfevents.1616111307.theta.10525.0: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | datasets/ 3 | experiments/ 4 | __pycache__/ 5 | .ipynb_checkpoints 6 | weights/ -------------------------------------------------------------------------------- /configs/config.yaml: -------------------------------------------------------------------------------- 1 | model_class: 'SimpleConvNet' 2 | dataset_class: 'MNIST_dataset' 3 | save_freq: 1 4 | batch_size: 16 5 | num_workers: 2 6 | device: 'cpu' 7 | -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616394553.17bec54ce095.59.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616394553.17bec54ce095.59.0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616396175.17bec54ce095.59.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616396175.17bec54ce095.59.1 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616397390.d6698bb24c52.59.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616397390.d6698bb24c52.59.0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616434156.aaf44e041659.58.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616434156.aaf44e041659.58.0 -------------------------------------------------------------------------------- /tensorboard_events/MoblieNetV2_baseline/events.out.tfevents.1616141574.a7a84a2bec98.573 (10).0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/MoblieNetV2_baseline/events.out.tfevents.1616141574.a7a84a2bec98.573 (10).0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616372319.c5e4336cf6d4.1444.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_dy_pascal/events.out.tfevents.1616372319.c5e4336cf6d4.1444.0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x1.0_dy_pascal_sbd/events.out.tfevents.1616452269.9bc1026ea589.59.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x1.0_dy_pascal_sbd/events.out.tfevents.1616452269.9bc1026ea589.59.1 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_baseline_pascal/events.out.tfevents.1616441878.aaf44e041659.58.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_baseline_pascal/events.out.tfevents.1616441878.aaf44e041659.58.2 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x0.5_baseline_pascal/events.out.tfevents.1616485358.adffa85e925e.58.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x0.5_baseline_pascal/events.out.tfevents.1616485358.adffa85e925e.58.0 -------------------------------------------------------------------------------- /tensorboard_events/MobileNetV2_035_dy_tiny_t6020_300ep/events.out.tfevents.1616241245.theta.16175.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/MobileNetV2_035_dy_tiny_t6020_300ep/events.out.tfevents.1616241245.theta.16175.0 -------------------------------------------------------------------------------- /tensorboard_events/MobileNetV2_035_dy_tiny_t10040_300ep/events.out.tfevents.1616295195.theta.27968.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/MobileNetV2_035_dy_tiny_t10040_300ep/events.out.tfevents.1616295195.theta.27968.0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616475592.9bc1026ea589.59.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616475592.9bc1026ea589.59.2 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616492304.8929e58b1f44.59.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616492304.8929e58b1f44.59.0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616492681.8929e58b1f44.59.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616492681.8929e58b1f44.59.1 -------------------------------------------------------------------------------- /tensorboard_events/MobileNetV2_035_dy_tiny_t6060_300ep/events.out.tfevents.1616295624.18f1170a923a.71.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/MobileNetV2_035_dy_tiny_t6060_300ep/events.out.tfevents.1616295624.18f1170a923a.71.0 -------------------------------------------------------------------------------- /tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616484234.9bc1026ea589.10523.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TArdelean/DynamicConvolution/HEAD/tensorboard_events/DeepLabV3+_x1.0_baseline_pascal_sbd/events.out.tfevents.1616484234.9bc1026ea589.10523.0 -------------------------------------------------------------------------------- /models/deeplab_details/sync_batchnorm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : __init__.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | from .batchnorm import SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d -------------------------------------------------------------------------------- /configs/MobileNetV2_DY_05.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV2' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV2_DY_05' 9 | checkpoint_path: None 10 | max_epoch: 300 11 | save_freq: 5 12 | batch_size: 600 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.05, 0.9, 0, 4e-05) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (300, 0) 18 | device: 'cuda' 19 | criterion: 'NLLLoss' 20 | criterion_args: () 21 | model_extra_args: (200, 0.5) -------------------------------------------------------------------------------- /configs/MobileNetV3_DY_1_Tiny.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV3_DY_1_Tiny' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (1,) -------------------------------------------------------------------------------- /data/mnist_dataset.py: -------------------------------------------------------------------------------- 1 | from torchvision import datasets, transforms 2 | 3 | 4 | transform = transforms.Compose([ 5 | transforms.ToTensor(), 6 | transforms.Normalize((0.1307,), (0.3081,)) 7 | ]) 8 | 9 | 10 | def MNIST_dataset(stage="train", download=True, root='datasets/'): 11 | if stage == "train": 12 | return datasets.MNIST(root, train=True, download=download, 13 | transform=transform) 14 | else: 15 | return datasets.MNIST(root, train=False, download=download, 16 | transform=transform) 17 | -------------------------------------------------------------------------------- /configs/MobileNetV3_DY_05_Tiny.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV3_DY_05_Tiny' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5,) -------------------------------------------------------------------------------- /configs/MobileNetV2_Vanilla_05.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV2' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV2_Vanilla_05' 9 | checkpoint_path: None 10 | max_epoch: 300 11 | save_freq: 5 12 | batch_size: 600 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.05, 0.9, 0, 4e-05) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (300, 0) 18 | device: 'cuda' 19 | criterion: 'NLLLoss' 20 | criterion_args: () 21 | model_extra_args: (200, 0.5) -------------------------------------------------------------------------------- /configs/MobileNetV3_Vanilla_05_Tiny.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV3_Vanilla_05_Tiny' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5,) -------------------------------------------------------------------------------- /configs/MobileNetV3_Vanilla_1_Tiny.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV3_Vanilla_1_Tiny' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (1,) -------------------------------------------------------------------------------- /configs/MobileNetV2_DY_035.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV2' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV2_DY_035' 9 | checkpoint_path: None 10 | max_epoch: 300 11 | save_freq: 5 12 | batch_size: 600 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.05, 0.9, 0, 4e-05) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (300, 0) 18 | device: 'cuda' 19 | checkpoints_dir: 'experiments/MobileNetV2_dy' 20 | criterion: 'NLLLoss' 21 | criterion_args: () -------------------------------------------------------------------------------- /configs/MobileNetV3_DY_05_Imagenette.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'Imagenette_dataset' 8 | experiment_name: 'MobileNetV3_DY_05_Imagenette' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5, 10, 2) -------------------------------------------------------------------------------- /configs/MobileNetV3_DY_1_Imagenette.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'Imagenette_dataset' 8 | experiment_name: 'MobileNetV3_DY_1_Imagenette' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (1, 10, 2) -------------------------------------------------------------------------------- /configs/MobileNetV3_DY_025_Imagenette.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'Imagenette_dataset' 8 | experiment_name: 'MobileNetV3_DY_025_Imagenette' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.25, 10, 2) -------------------------------------------------------------------------------- /configs/MobileNetV3_Vanilla_1_Imagenette.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'Imagenette_dataset' 8 | experiment_name: 'MobileNetV3_Vanilla_1_Imagenette' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (1, 10, 2) -------------------------------------------------------------------------------- /configs/MobileNetV3_Vanilla_025_Imagenette.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'Imagenette_dataset' 8 | experiment_name: 'MobileNetV3_Vanilla_025_Imagenette' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.25, 10, 2) -------------------------------------------------------------------------------- /configs/MobileNetV3_Vanilla_05_Imagenette.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV3' 7 | dataset_class: 'Imagenette_dataset' 8 | experiment_name: 'MobileNetV3_Vanilla_05_Imagenette' 9 | checkpoint_path: None 10 | max_epoch: 200 11 | save_freq: 5 12 | batch_size: 256 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 3e-5) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (200, 0) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5, 10, 2) -------------------------------------------------------------------------------- /configs/MobileNetV2_Vanilla_035.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV2' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'MobileNetV2_Vanilla_035' 9 | checkpoint_path: None 10 | max_epoch: 300 11 | save_freq: 5 12 | batch_size: 600 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.05, 0.9, 0, 4e-05) 16 | scheduler: 'CosineAnnealingLR' 17 | scheduler_args: (300, 0) 18 | device: 'cuda' 19 | checkpoints_dir: 'experiments/MobileNetV2_Vanilla_035' 20 | criterion: 'NLLLoss' 21 | criterion_args: () -------------------------------------------------------------------------------- /models/deeplab_details/backbone/__init__.py: -------------------------------------------------------------------------------- 1 | from ...deeplab_details.backbone import resnet, xception, drn, mobilenet 2 | 3 | def build_backbone(backbone, output_stride, BatchNorm, wm=1.0): 4 | if backbone == 'resnet': 5 | return resnet.ResNet101(output_stride, BatchNorm) 6 | elif backbone == 'xception': 7 | return xception.AlignedXception(output_stride, BatchNorm) 8 | elif backbone == 'drn': 9 | return drn.drn_d_54(BatchNorm) 10 | elif backbone == 'mobilenet': 11 | return mobilenet.MobileNetV2(output_stride, BatchNorm) # width_mult=wm 12 | else: 13 | raise NotImplementedError 14 | -------------------------------------------------------------------------------- /configs/deeplabv3plus.yaml: -------------------------------------------------------------------------------- 1 | model_class: 'DeepLab' 2 | model_extra_args: (0.007, 'mobilenet') 3 | experiment_name: 'deeplabv3plus_baseline' 4 | dataset_class: 'PascalVOC2012_dataset' 5 | dataset_extra_args: (False,) # use SBD 6 | download_dataset: True 7 | save_freq: 5 8 | is_classification: False 9 | n_classes: 21 10 | use_dynamic : False 11 | max_epoch: 100 12 | batch_size: 16 13 | optimizer : "SGD" 14 | optimizer_args : (0.007, 0.9, 0, 5e-4) 15 | scheduler : "CosineAnnealingLR" 16 | scheduler_args : (100, 0.000001) 17 | criterion: "CrossEntropyLoss" 18 | criterion_args: (None, True, 255) 19 | batch_average: False 20 | num_workers: 2 21 | device : "cuda" -------------------------------------------------------------------------------- /configs/MobileNetV2_DY_035_60_20.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (60, 1, 20) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV2' 7 | model_extra_args: () 8 | dataset_class: 'TinyImageNet_dataset' 9 | experiment_name: 'MobileNetV2_DY_035_60_20' 10 | checkpoint_path: None 11 | max_epoch: 300 12 | save_freq: 5 13 | batch_size: 600 14 | num_workers: 4 15 | optimizer: 'SGD' 16 | optimizer_args: (0.05, 0.9, 0, 4e-05) 17 | scheduler: 'CosineAnnealingLR' 18 | scheduler_args: (300, 0) 19 | criterion: 'CrossEntropyLoss' 20 | criterion_args: '' 21 | device: 'cuda' 22 | checkpoints_dir: 'experiments/MobileNetV2_DY_035_60_20' 23 | -------------------------------------------------------------------------------- /configs/MobileNetV2_DY_035_60_60.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (60, 1, 60) 5 | experiments: 'experiments' 6 | model_class: 'MobileNetV2' 7 | model_extra_args: () 8 | dataset_class: 'TinyImageNet_dataset' 9 | experiment_name: 'MobileNetV2_DY_035_60_60' 10 | checkpoint_path: None 11 | max_epoch: 300 12 | save_freq: 5 13 | batch_size: 600 14 | num_workers: 4 15 | optimizer: 'SGD' 16 | optimizer_args: (0.05, 0.9, 0, 4e-05) 17 | scheduler: 'CosineAnnealingLR' 18 | scheduler_args: (300, 0) 19 | criterion: 'CrossEntropyLoss' 20 | criterion_args: '' 21 | device: 'cuda' 22 | checkpoints_dir: 'experiments/MobileNetV2_DY_035_60_60' 23 | -------------------------------------------------------------------------------- /configs/deeplabv3plus_0.5.yaml: -------------------------------------------------------------------------------- 1 | model_class: 'DeepLab' 2 | model_extra_args: (0.007, 'mobilenet', 0.5) 3 | experiment_name: 'deeplabv3plus_baseline_0.5' 4 | dataset_class: 'PascalVOC2012_dataset' 5 | dataset_extra_args: (True,) # use SBD 6 | download_dataset: True 7 | save_freq: 5 8 | is_classification: False 9 | n_classes: 21 10 | use_dynamic : False 11 | max_epoch: 100 12 | batch_size: 16 13 | optimizer : "SGD" 14 | optimizer_args : (0.007, 0.9, 0, 5e-4) 15 | scheduler : "CosineAnnealingLR" 16 | scheduler_args : (100, 0.000001) 17 | criterion: "CrossEntropyLoss" 18 | criterion_args: (None, True, 255) 19 | batch_average: True 20 | num_workers: 2 21 | device : "cuda" -------------------------------------------------------------------------------- /configs/Resnet_DY_1.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_DY_1' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (1.0,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DY_1' 23 | -------------------------------------------------------------------------------- /configs/Resnet_DY_025.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_DY_025' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.25,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DY_025' 23 | -------------------------------------------------------------------------------- /configs/Resnet_DY_05.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_DY_05' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DY_05' 23 | -------------------------------------------------------------------------------- /configs/Resnet_Vanilla_1.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_Vanilla_1' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (1.0,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_Vanilla_1' 23 | -------------------------------------------------------------------------------- /configs/Resnet_Vanilla_05.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_Vanilla_05' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_Vanilla_05' 23 | -------------------------------------------------------------------------------- /configs/Resnet_Vanilla_025.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_Vanilla_025' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.25,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_Vanilla_025' 23 | -------------------------------------------------------------------------------- /configs/dy_deeplabv3plus.yaml: -------------------------------------------------------------------------------- 1 | model_class: 'DeepLab' 2 | model_extra_args: (0.007, 'mobilenet') 3 | experiment_name: 'deeplabv3plus_dy' 4 | dataset_class: 'PascalVOC2012_dataset' 5 | dataset_extra_args: (False,) # use SBD 6 | download_dataset: True 7 | save_freq: 5 8 | is_classification: False 9 | n_classes: 21 10 | use_dynamic: True 11 | nof_kernels: 4 12 | reduce: 4 13 | temperature: (30, 1, 10) 14 | max_epoch: 100 15 | batch_size: 16 16 | optimizer : "SGD" 17 | optimizer_args : (0.007, 0.9, 0, 5e-4) 18 | scheduler : "CosineAnnealingLR" 19 | scheduler_args : (100, 0.000001) 20 | criterion: "CrossEntropyLoss" 21 | criterion_args: (None, True, 255) 22 | batch_average: False 23 | num_workers: 2 24 | device : "cuda" -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from utils.options import Options 4 | from .common import Conv2dWrapper 5 | from dynamic_convolutions import dynamic_convolution_generator 6 | from .simple_conv import SimpleConvNet 7 | from .mobilenetv3 import MobileNetV3 8 | from .mobilenetv2 import MobileNetV2 9 | from .resnet import ResNet10 10 | from .deeplab import DeepLab 11 | 12 | def create_model(opt: Options): 13 | current_module = sys.modules[__name__] 14 | model_class = getattr(current_module, opt.model_class) 15 | conv_layer = dynamic_convolution_generator(opt.nof_kernels, opt.reduce) if opt.use_dynamic else Conv2dWrapper 16 | return model_class(conv_layer, *opt.model_extra_args).to(opt.device) 17 | -------------------------------------------------------------------------------- /configs/Resnet_DY_05_smooth.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_DY_05_smooth' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'SmoothNLLLoss' 19 | criterion_args: (0.1,) 20 | device: 'cuda' 21 | model_extra_args: (0.5,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DY_05_smooth' 23 | -------------------------------------------------------------------------------- /configs/dy_deeplabv3plus_0.5.yaml: -------------------------------------------------------------------------------- 1 | model_class: 'DeepLab' 2 | model_extra_args: (0.007, 'mobilenet', 0.5) 3 | experiment_name: 'deeplabv3plus_dy_0.5' 4 | dataset_class: 'PascalVOC2012_dataset' 5 | dataset_extra_args: (True,) # use SBD 6 | download_dataset: True 7 | save_freq: 5 8 | is_classification: False 9 | n_classes: 21 10 | use_dynamic: True 11 | nof_kernels: 4 12 | reduce: 4 13 | temperature: (30, 1, 10) 14 | max_epoch: 100 15 | batch_size: 16 16 | optimizer : "SGD" 17 | optimizer_args : (0.007, 0.9, 0, 5e-4) 18 | scheduler : "CosineAnnealingLR" 19 | scheduler_args : (100, 0.000001) 20 | criterion: "CrossEntropyLoss" 21 | criterion_args: (None, True, 255) 22 | batch_average: True 23 | num_workers: 2 24 | device : "cuda" -------------------------------------------------------------------------------- /tensorboard_events/MobileNetV2_035_dy_tiny_t10040_300ep/config.yaml: -------------------------------------------------------------------------------- 1 | : '' 2 | use_dynamic: True 3 | nof_kernels: 4 4 | reduce: 4 5 | temperature: (100, 1, 40) 6 | experiments: 'experiments' 7 | model_class: 'MobileNetV2' 8 | model_extra_args: () 9 | dataset_class: 'TinyImageNet_dataset' 10 | experiment_name: 'MobileNetV2_035_dy_tiny_t10040_300ep' 11 | checkpoint_path: None 12 | max_epoch: 300 13 | save_freq: 5 14 | batch_size: 600 15 | num_workers: 4 16 | optimizer: 'SGD' 17 | optimizer_args: (0.05, 0.9, 0, 4e-05) 18 | scheduler: 'CosineAnnealingLR' 19 | scheduler_args: (300, 0) 20 | criterion: 'CrossEntropyLoss' 21 | criterion_args: '' 22 | device: 'cuda' 23 | checkpoints_dir: 'experiments/MobileNetV2_035_dy_tiny_t10040_300ep' 24 | -------------------------------------------------------------------------------- /configs/Resnet_DY_Leaky_05_smooth.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_DYLeaky_05_smooth' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'SmoothNLLLoss' 19 | criterion_args: (0.1,) 20 | device: 'cuda' 21 | model_extra_args: (0.5,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DYLeaky_05_smooth' 23 | -------------------------------------------------------------------------------- /configs/Resnet_Vanilla_05_smooth.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: False 2 | nof_kernels: 4 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_Vanilla_05_smooth' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'SmoothNLLLoss' 19 | criterion_args: (0.1,) 20 | device: 'cuda' 21 | model_extra_args: (0.5,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_Vanilla_05_smooth' 23 | -------------------------------------------------------------------------------- /configs/Resnet_DY_Improve_05.yaml: -------------------------------------------------------------------------------- 1 | use_dynamic: True 2 | nof_kernels: [2, 2, 2, 3, 2, 3, 6, 3, 6, 9, 6] 3 | reduce: 4 4 | temperature: (30, 1, 10) 5 | experiments: '/content/drive/MyDrive/DynamicConvolution/experiments' 6 | model_class: 'ResNet10' 7 | dataset_class: 'TinyImageNet_dataset' 8 | experiment_name: 'Resnet_DY_Improve_05' 9 | checkpoint_path: None 10 | max_epoch: 100 11 | save_freq: 5 12 | batch_size: 64 13 | num_workers: 2 14 | optimizer: 'SGD' 15 | optimizer_args: (0.1, 0.9, 0, 0.0001) 16 | scheduler: 'StepLR' 17 | scheduler_args: (30, 0.1) 18 | criterion: 'NLLLoss' 19 | criterion_args: () 20 | device: 'cuda' 21 | model_extra_args: (0.5,) 22 | checkpoints_dir: '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DY_Improve_05' 23 | -------------------------------------------------------------------------------- /data/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import torch.utils.data 4 | 5 | from utils.options import Options 6 | from .mnist_dataset import MNIST_dataset 7 | from .tinyimagenet_dataset import TinyImageNet_dataset 8 | from .imagenette_dataset import Imagenette_dataset 9 | from .pascalvoc2012_dataset import PascalVOC2012_dataset 10 | from .sb_dataset import SB_dataset 11 | 12 | def create_data_loader(opt: Options, stage): 13 | current_module = sys.modules[__name__] 14 | dataset_getter = getattr(current_module, opt.dataset_class) 15 | dataset = dataset_getter(stage, *opt.dataset_extra_args, download=opt.download_dataset, root=opt.dataset_root) 16 | return torch.utils.data.DataLoader(dataset, batch_size=opt.batch_size, num_workers=opt.num_workers, 17 | shuffle=(stage == "train")) 18 | -------------------------------------------------------------------------------- /models/simple_conv.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | # Simple example for testing from https://github.com/pytorch/examples/blob/master/mnist/main.py 6 | from models.common import BaseModel 7 | 8 | 9 | class SimpleConvNet(BaseModel): 10 | def __init__(self, ConvLayer, c_in=1, out_dim=10): 11 | super().__init__(ConvLayer) 12 | self.conv1 = nn.Conv2d(c_in, 32, 3, 1) # First convolution is always non-dynamic 13 | self.conv2 = self.ConvLayer(32, 64, 3, 1) 14 | self.dropout1 = nn.Dropout(0.25) 15 | self.dropout2 = nn.Dropout(0.5) 16 | self.fc1 = nn.Linear(9216, 128) 17 | self.fc2 = nn.Linear(128, out_dim) 18 | 19 | def forward(self, x, temperature): 20 | x = self.conv1(x) 21 | x = F.relu(x) 22 | x = self.conv2(x, temperature) 23 | x = F.relu(x) 24 | x = F.max_pool2d(x, 2) 25 | x = self.dropout1(x) 26 | x = torch.flatten(x, 1) 27 | x = self.fc1(x) 28 | x = F.relu(x) 29 | x = self.dropout2(x) 30 | x = self.fc2(x) 31 | output = F.log_softmax(x, dim=1) 32 | return output 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Andrei-Timotei Ardelean, Andreea Dogaru, Alexey Larionov, Saian Protasov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | import numpy as np 5 | from utils.options import Options 6 | 7 | 8 | def load_checkpoint(path, model, optimizer=None, scheduler=None, device="cpu"): 9 | if path: 10 | checkpoint = torch.load(path, map_location=device) 11 | model.load_state_dict(checkpoint["model"]) 12 | if optimizer: 13 | optimizer.load_state_dict(checkpoint["optimizer"]) 14 | if scheduler: 15 | scheduler.load_state_dict(checkpoint["scheduler"]) 16 | epoch = checkpoint["epoch"] 17 | else: 18 | epoch = 0 19 | if optimizer: 20 | if scheduler: 21 | return model, epoch, optimizer, scheduler 22 | else: 23 | return model, epoch, optimizer 24 | else: 25 | return model, epoch 26 | 27 | 28 | def save_checkpoint(model, optimizer, scheduler, epoch, opt: Options): 29 | torch.save({ 30 | 'model': model.state_dict(), 31 | 'optimizer': optimizer.state_dict(), 32 | 'scheduler': scheduler.state_dict(), 33 | 'epoch': epoch, 34 | }, os.path.join(opt.checkpoints_dir, f"{opt.experiment_name}_{epoch}.pth")) 35 | 36 | 37 | def profile(model, inputs, repeats=1000): 38 | # Reference for counting flops: http://www.bnikolic.co.uk/blog/python/flops/2019/10/01/pytorch-count-flops.html 39 | from pypapi import papi_high 40 | from pypapi import events as papi_events 41 | 42 | papi_high.start_counters([ 43 | papi_events.PAPI_SP_OPS, 44 | ]) 45 | model.forward(*inputs) 46 | flops = papi_high.stop_counters()[0] / 1000000.0 47 | 48 | from time import perf_counter 49 | times = [] 50 | for _ in range(repeats): 51 | t = perf_counter() 52 | model.forward(*inputs) 53 | times.append(perf_counter() - t) 54 | params = sum(p.numel() for p in model.parameters()) / 1000000.0 55 | times = np.array(times) * 1000 56 | return {"params(M)": params, "flops(M)": flops, 57 | "inf_time_mean(ms)": np.mean(times), "inf_time_std(ms)": np.std(times)} 58 | -------------------------------------------------------------------------------- /utils/segmentation_metrics.py: -------------------------------------------------------------------------------- 1 | 2 | # ALL ACKNOWLEDGMENT GOES TO THE PAPER & REPOSITORY AUTHORS 3 | # https://github.com/jfzhang95/pytorch-deeplab-xception 4 | 5 | import numpy as np 6 | 7 | class SegmentationEvaluator(object): 8 | def __init__(self, num_class): 9 | self.num_class = num_class 10 | self.confusion_matrix = np.zeros((self.num_class,)*2) 11 | 12 | def Pixel_Accuracy(self): 13 | Acc = np.diag(self.confusion_matrix).sum() / self.confusion_matrix.sum() 14 | return Acc 15 | 16 | def Pixel_Accuracy_Class(self): 17 | Acc = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1) 18 | Acc = np.nanmean(Acc) 19 | return Acc 20 | 21 | def Mean_Intersection_over_Union(self): 22 | MIoU = np.diag(self.confusion_matrix) / ( 23 | np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) - 24 | np.diag(self.confusion_matrix)) 25 | MIoU = np.nanmean(MIoU) 26 | return MIoU 27 | 28 | def Frequency_Weighted_Intersection_over_Union(self): 29 | freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix) 30 | iu = np.diag(self.confusion_matrix) / ( 31 | np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) - 32 | np.diag(self.confusion_matrix)) 33 | 34 | FWIoU = (freq[freq > 0] * iu[freq > 0]).sum() 35 | return FWIoU 36 | 37 | def _generate_matrix(self, gt_image, pre_image): 38 | mask = (gt_image >= 0) & (gt_image < self.num_class) 39 | label = self.num_class * gt_image[mask].astype('int') + pre_image[mask] 40 | count = np.bincount(label, minlength=self.num_class**2) 41 | confusion_matrix = count.reshape(self.num_class, self.num_class) 42 | return confusion_matrix 43 | 44 | def add_batch(self, gt_image, pre_image): 45 | assert gt_image.shape == pre_image.shape 46 | self.confusion_matrix += self._generate_matrix(gt_image, pre_image) 47 | 48 | def reset(self): 49 | self.confusion_matrix = np.zeros((self.num_class,) * 2) 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /utils/options.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from utils.parsable_options import ParsableOptions 4 | 5 | 6 | class Options(ParsableOptions): 7 | 8 | # noinspection PyAttributeOutsideInit 9 | def initialize(self): 10 | self.use_dynamic = False 11 | self.nof_kernels = 4 # Parameter is ignored if not using dynamic 12 | self.reduce = 4 # Dimension reduction in hidden layer for attention in dynamic convolutions 13 | self.temperature = (30, 1, 10) # Temperature parameters: (initial_value, final_value, final_epoch) 14 | self.experiments = "experiments" 15 | self.model_class = "" 16 | self.model_extra_args = () # Model specific arguments (e.g. width multiplier) 17 | self.dataset_class = "" # Can also be a function which returns a dataset instance 18 | self.experiment_name = "attempt" 19 | self.checkpoint_path = None # e.g. "experiments/attempt/attempt_4.pth" 20 | self.max_epoch = 10 21 | self.save_freq = 1 22 | self.batch_size = 16 23 | self.num_workers = 2 24 | self.optimizer = "SGD" 25 | self.optimizer_args = (0.001, 0.9) # e.g. (lr, momentum) 26 | self.scheduler = "StepLR" 27 | self.scheduler_args = (30, 0.1) # e.g. (step_size, gamma) for StepLR 28 | self.criterion = "SmoothNLLLoss" 29 | self.criterion_args = (0.1,) 30 | self.device = "cpu" 31 | self.batch_average = False # normalize training loss by batch size 32 | self.is_classification = True # otherwise segmentation 33 | self.n_classes = None # NOTE(alexey-larionov): added for segmentation model and dataset 34 | self.config_path = "" 35 | self.download_dataset = True 36 | self.dataset_root = './datasets' 37 | self.dataset_extra_args = () 38 | 39 | # noinspection PyAttributeOutsideInit 40 | def proc(self): 41 | super().proc() 42 | self.checkpoints_dir = os.path.join(self.experiments, self.experiment_name) 43 | if not os.path.exists(self.checkpoints_dir): 44 | os.makedirs(self.checkpoints_dir) 45 | self.print_to_file() 46 | 47 | def print_to_file(self, **kwargs): 48 | super(Options, self).print_to_file( 49 | os.path.join(self.checkpoints_dir, "config.yaml")) 50 | -------------------------------------------------------------------------------- /data/pascalvoc2012_dataset.py: -------------------------------------------------------------------------------- 1 | 2 | # ALL ACKNOWLEDGMENT GOES TO THE PAPER & REPOSITORY AUTHORS 3 | # https://github.com/jfzhang95/pytorch-deeplab-xception 4 | 5 | from torchvision import transforms 6 | from torchvision import datasets 7 | 8 | from .sb_dataset import * 9 | 10 | import torch 11 | import random 12 | import numpy as np 13 | 14 | from PIL import Image, ImageOps, ImageFilter 15 | 16 | def PascalVOC2012_dataset(stage="train", use_sbd_dataset=False, download=True, root='datasets/'): 17 | if stage == "train": 18 | voc_train = datasets.VOCSegmentation(root, year='2012', image_set='train', download=download, 19 | transforms=CustomCompose([ 20 | CustomRandomHorizontalFlip(), 21 | CustomRandomScaleCrop(base_size=513, crop_size=513), 22 | CustomRandomGaussianBlur(), 23 | # NOTE: original repo has args parameter 24 | # CustomRandomScaleCrop(base_size=args.base_size, crop_size=args.crop_size), 25 | CustomNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), 26 | CustomToTensor(), 27 | ])) 28 | if use_sbd_dataset: 29 | sbd_train = SB_dataset(stage, download=download) 30 | print('Merging PascalVOC2012 and SB datasets') 31 | return torch.utils.data.ConcatDataset([voc_train, sbd_train]) 32 | else: 33 | return voc_train 34 | else: 35 | return datasets.VOCSegmentation(root, year='2012', image_set='val', download=download, 36 | transforms=CustomCompose([ 37 | CustomFixScaleCrop(crop_size=513), 38 | CustomNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), 39 | CustomToTensor(), 40 | ])) 41 | 42 | if __name__ == '__main__': 43 | from torch.utils.data import DataLoader 44 | import matplotlib.pyplot as plt 45 | 46 | voc_train = PascalVOC2012_dataset(stage='train', use_sbd_dataset=False, download=False) 47 | dataloader = DataLoader(voc_train, batch_size=3, shuffle=True, num_workers=0) 48 | print('Created loader') 49 | for ii, sample in enumerate(dataloader): 50 | img, gt = sample 51 | for jj in range(img.size()[0]): 52 | plt.figure() 53 | plt.subplot(211) 54 | plt.imshow(img[jj].numpy().transpose((1, 2, 0))) 55 | plt.subplot(212) 56 | plt.imshow(gt[jj].numpy()) 57 | break 58 | 59 | plt.show(block=True) -------------------------------------------------------------------------------- /data/imagenette_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from torchvision import transforms 4 | from torchvision.datasets import ImageFolder 5 | from torchvision.datasets.utils import verify_str_arg 6 | from torchvision.datasets.utils import download_and_extract_archive 7 | 8 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 9 | std=[0.229, 0.224, 0.225]) 10 | 11 | 12 | def Imagenette_dataset(stage="train", download=True, root='datasets/'): 13 | if stage == "train": 14 | return ImagenetteDataset(root, split="train", download=download, 15 | transform=transforms.Compose([ 16 | transforms.RandomAffine(15, None, (0.9, 1.1)), 17 | transforms.RandomResizedCrop(224, scale=(0.25, 1.0)), 18 | transforms.RandomHorizontalFlip(), 19 | transforms.ColorJitter(0.3, 0.3, 0.2, 0.1), 20 | transforms.GaussianBlur(5, (0.1, 0.5)), 21 | transforms.ToTensor(), 22 | normalize 23 | ])) 24 | else: 25 | return ImagenetteDataset(root, split="val", download=download, 26 | transform=transforms.Compose([ 27 | transforms.CenterCrop(224), 28 | transforms.ToTensor(), 29 | normalize 30 | ])) 31 | 32 | 33 | class ImagenetteDataset(ImageFolder): 34 | """ 35 | Dataset for Imagenette: a subset of 10 easily classified classes from Imagenet 36 | """ 37 | base_folder = 'imagenette2-320' 38 | zip_md5 = '90528d7ca1a48142e341f4ef8d21d0de' 39 | splits = ('train', 'val') 40 | filename = 'imagenette2-320.tgz' 41 | url = 'https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz' 42 | 43 | def __init__(self, root, split='train', download=False, **kwargs): 44 | self.data_root = os.path.expanduser(root) 45 | self.split = verify_str_arg(split, "split", self.splits) 46 | 47 | if download: 48 | self.download() 49 | 50 | if not self._check_exists(): 51 | raise RuntimeError('Dataset not found.' + 52 | ' You can use download=True to download it') 53 | super().__init__(self.split_folder, **kwargs) 54 | 55 | @property 56 | def dataset_folder(self): 57 | return os.path.join(self.data_root, self.base_folder) 58 | 59 | @property 60 | def split_folder(self): 61 | return os.path.join(self.dataset_folder, self.split) 62 | 63 | def _check_exists(self): 64 | return os.path.exists(self.split_folder) 65 | 66 | def extra_repr(self): 67 | return "Split: {split}".format(**self.__dict__) 68 | 69 | def download(self): 70 | if self._check_exists(): 71 | return 72 | download_and_extract_archive( 73 | self.url, self.data_root, filename=self.filename, 74 | remove_finished=True) 75 | -------------------------------------------------------------------------------- /models/deeplab_details/decoder.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from ..deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 6 | 7 | from dynamic_convolutions import DynamicConvolution, TempModule 8 | from models.common import BaseModel, CustomSequential 9 | 10 | class Decoder(TempModule): 11 | def __init__(self, num_classes, backbone, BatchNorm, ConvLayer, wm=1.0): 12 | super(Decoder, self).__init__() 13 | if backbone == 'resnet' or backbone == 'drn': 14 | low_level_inplanes = 256 15 | elif backbone == 'xception': 16 | low_level_inplanes = 128 17 | elif backbone == 'mobilenet': 18 | low_level_inplanes = 24 19 | else: 20 | raise NotImplementedError 21 | 22 | self.conv1 = ConvLayer(low_level_inplanes, int(48*wm), 1, bias=False) 23 | self.bn1 = BatchNorm(int(48*wm)) 24 | # self.conv1 = ConvLayer(int(low_level_inplanes*wm), int(48*wm), 1, bias=False) 25 | # self.bn1 = BatchNorm(int(48*wm)) 26 | self.relu = nn.ReLU() 27 | self.last_conv = CustomSequential(ConvLayer(int(304*wm), int(256*wm), kernel_size=3, stride=1, padding=1, bias=False), 28 | BatchNorm(int(256*wm)), 29 | nn.ReLU(), 30 | nn.Dropout(0.5), 31 | ConvLayer(int(256*wm), int(256*wm), kernel_size=3, stride=1, padding=1, bias=False), 32 | BatchNorm(int(256*wm)), 33 | nn.ReLU(), 34 | nn.Dropout(0.1), 35 | ConvLayer(int(256*wm), num_classes, kernel_size=1, stride=1)) 36 | self._init_weight() 37 | 38 | 39 | def forward(self, x, low_level_feat, temperature): 40 | low_level_feat = self.conv1(low_level_feat, temperature) 41 | low_level_feat = self.bn1(low_level_feat) 42 | low_level_feat = self.relu(low_level_feat) 43 | 44 | x = F.interpolate(x, size=low_level_feat.size()[2:], mode='bilinear', align_corners=True) 45 | x = torch.cat((x, low_level_feat), dim=1) 46 | x = self.last_conv(x, temperature) 47 | 48 | return x 49 | 50 | def _init_weight(self): 51 | for m in self.modules(): 52 | if isinstance(m, nn.Conv2d): 53 | torch.nn.init.kaiming_normal_(m.weight) 54 | elif isinstance(m, SynchronizedBatchNorm2d): 55 | m.weight.data.fill_(1) 56 | m.bias.data.zero_() 57 | elif isinstance(m, nn.BatchNorm2d): 58 | m.weight.data.fill_(1) 59 | m.bias.data.zero_() 60 | if isinstance(m, DynamicConvolution): 61 | for i_kernel in range(m.nof_kernels): 62 | nn.init.kaiming_normal_(m.kernels_weights[i_kernel], mode='fan_out') 63 | if m.kernels_bias is not None: 64 | nn.init.zeros_(m.kernels_bias) 65 | 66 | def build_decoder(num_classes, backbone, BatchNorm, ConvLayer, wm=1.0): 67 | return Decoder(num_classes, backbone, BatchNorm, ConvLayer, wm=wm) -------------------------------------------------------------------------------- /models/common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.nn import * 4 | from collections import OrderedDict 5 | from typing import Any, Iterable, Iterator, Mapping, Optional, TYPE_CHECKING, overload, Tuple, TypeVar, Union 6 | T = TypeVar('T', bound=Module) 7 | 8 | class Conv2dWrapper(nn.Conv2d): 9 | """ 10 | Wrapper for pytorch Conv2d class which can take additional parameters(like temperature) and ignores them. 11 | """ 12 | 13 | def __init__(self, *args, **kwargs): 14 | super().__init__(*args, **kwargs) 15 | 16 | def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: 17 | return super().forward(x) 18 | 19 | 20 | class TempModule(nn.Module): 21 | def __init__(self): 22 | super().__init__() 23 | 24 | def forward(self, x, temperature) -> torch.Tensor: 25 | return x 26 | 27 | 28 | class BaseModel(TempModule): 29 | def __init__(self, ConvLayer): 30 | super().__init__() 31 | self.ConvLayer = ConvLayer 32 | 33 | 34 | class TemperatureScheduler: 35 | def __init__(self, initial_value, final_value=None, final_epoch=None): 36 | self.initial_value = initial_value 37 | self.final_value = final_value if final_value else initial_value 38 | self.final_epoch = final_epoch if final_epoch else 1 39 | self.step = 0 if self.final_epoch == 1 else (final_value - initial_value) / (final_epoch - 1) 40 | 41 | def get(self, crt_epoch=None): 42 | crt_epoch = crt_epoch if crt_epoch else self.final_epoch 43 | return self.initial_value + (min(crt_epoch, self.final_epoch) - 1) * self.step 44 | 45 | 46 | class CustomSequential(TempModule): 47 | """Sequential container that supports passing temperature to TempModule""" 48 | 49 | def __init__(self, *args): 50 | super().__init__() 51 | self.layers = nn.ModuleList(args) 52 | 53 | def forward(self, x, temperature): 54 | for layer in self.layers: 55 | if isinstance(layer, TempModule): 56 | x = layer(x, temperature) 57 | else: 58 | x = layer(x) 59 | return x 60 | 61 | def __getitem__(self, idx): 62 | return CustomSequential(*list(self.layers[idx])) 63 | # if isinstance(idx, slice): 64 | # return self.__class__(OrderedDict(list(self.layers.items())[idx])) 65 | # else: 66 | # return self._get_item_by_idx(self.layers.values(), idx) 67 | 68 | 69 | # Implementation inspired from 70 | # https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/master/train.py#L38 and 71 | # https://github.com/pytorch/pytorch/issues/7455 72 | class SmoothNLLLoss(nn.Module): 73 | def __init__(self, smoothing=0.0, dim=-1): 74 | super().__init__() 75 | self.smoothing = smoothing 76 | self.dim = dim 77 | 78 | def forward(self, prediction, target): 79 | with torch.no_grad(): 80 | smooth_target = torch.zeros_like(prediction) 81 | n_class = prediction.size(self.dim) 82 | smooth_target.fill_(self.smoothing / (n_class - 1)) 83 | smooth_target.scatter_(1, target.unsqueeze(1), 1 - self.smoothing) 84 | return torch.mean(torch.sum(-smooth_target * prediction, dim=self.dim)) 85 | -------------------------------------------------------------------------------- /models/deeplab_details/deeplab.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from ..deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 5 | from ..deeplab_details.aspp import build_aspp 6 | from ..deeplab_details.decoder import build_decoder 7 | from ..deeplab_details.backbone import build_backbone 8 | 9 | class DeepLab(nn.Module): 10 | def __init__(self, backbone='resnet', output_stride=16, num_classes=21, 11 | sync_bn=True, freeze_bn=False): 12 | super(DeepLab, self).__init__() 13 | if backbone == 'drn': 14 | output_stride = 8 15 | 16 | if sync_bn == True: 17 | BatchNorm = SynchronizedBatchNorm2d 18 | else: 19 | BatchNorm = nn.BatchNorm2d 20 | 21 | self.backbone = build_backbone(backbone, output_stride, BatchNorm) 22 | self.aspp = build_aspp(backbone, output_stride, BatchNorm) 23 | self.decoder = build_decoder(num_classes, backbone, BatchNorm) 24 | 25 | self.freeze_bn = freeze_bn 26 | 27 | def forward(self, input): 28 | x, low_level_feat = self.backbone(input) 29 | x = self.aspp(x) 30 | x = self.decoder(x, low_level_feat) 31 | x = F.interpolate(x, size=input.size()[2:], mode='bilinear', align_corners=True) 32 | 33 | return x 34 | 35 | def freeze_bn(self): 36 | for m in self.modules(): 37 | if isinstance(m, SynchronizedBatchNorm2d): 38 | m.eval() 39 | elif isinstance(m, nn.BatchNorm2d): 40 | m.eval() 41 | 42 | def get_1x_lr_params(self): 43 | modules = [self.backbone] 44 | for i in range(len(modules)): 45 | for m in modules[i].named_modules(): 46 | if self.freeze_bn: 47 | if isinstance(m[1], nn.Conv2d): 48 | for p in m[1].parameters(): 49 | if p.requires_grad: 50 | yield p 51 | else: 52 | if isinstance(m[1], nn.Conv2d) or isinstance(m[1], SynchronizedBatchNorm2d) \ 53 | or isinstance(m[1], nn.BatchNorm2d): 54 | for p in m[1].parameters(): 55 | if p.requires_grad: 56 | yield p 57 | 58 | def get_10x_lr_params(self): 59 | modules = [self.aspp, self.decoder] 60 | for i in range(len(modules)): 61 | for m in modules[i].named_modules(): 62 | if self.freeze_bn: 63 | if isinstance(m[1], nn.Conv2d): 64 | for p in m[1].parameters(): 65 | if p.requires_grad: 66 | yield p 67 | else: 68 | if isinstance(m[1], nn.Conv2d) or isinstance(m[1], SynchronizedBatchNorm2d) \ 69 | or isinstance(m[1], nn.BatchNorm2d): 70 | for p in m[1].parameters(): 71 | if p.requires_grad: 72 | yield p 73 | 74 | if __name__ == "__main__": 75 | model = DeepLab(backbone='mobilenet', output_stride=16) 76 | model.eval() 77 | input = torch.rand(1, 3, 513, 513) 78 | output = model(input) 79 | print(output.size()) 80 | 81 | 82 | -------------------------------------------------------------------------------- /utils/parsable_options.py: -------------------------------------------------------------------------------- 1 | import numbers 2 | import argparse 3 | 4 | 5 | class ParsableOptions: 6 | def __init__(self, suppress_parse=False, config_file_arg=None): 7 | """ 8 | Base class for parsable options 9 | :param suppress_parse: Whether to parse CLI arguments. Useful when passing external options 10 | :param config_file_arg: Allows reading existing configuration from file. 11 | """ 12 | if config_file_arg is not None: 13 | self.__setattr__(config_file_arg, "") 14 | self._config_file_arg = config_file_arg 15 | self.initialize() 16 | if not suppress_parse: 17 | self.parse() 18 | self.proc() 19 | 20 | def initialize(self): 21 | """ Method were all fields should be initialized. Variables starting with _ will not be parsed""" 22 | pass 23 | 24 | def proc(self): 25 | """ Post processing, after the options have been parsed """ 26 | pass 27 | 28 | @staticmethod 29 | def good_instance(val): 30 | return isinstance(val, str) or (isinstance(val, numbers.Number) and not isinstance(val, bool)) 31 | 32 | def parse(self): 33 | parser = argparse.ArgumentParser() 34 | for name, val in vars(self).items(): 35 | if name.startswith("_"): 36 | continue 37 | like = type(val) if ParsableOptions.good_instance(val) else eval 38 | parser.add_argument(f'--{name}', type=like, default=argparse.SUPPRESS, help="It is obvious") 39 | 40 | args = parser.parse_args() 41 | if self._config_file_arg is not None: 42 | config = getattr(args, self._config_file_arg, "") 43 | if config: 44 | self.load_from_file(config) 45 | for name, val in vars(self).items(): 46 | if name.startswith("_"): 47 | continue 48 | if name in args: 49 | attr = getattr(args, name) 50 | self.__setattr__(name, attr) 51 | 52 | def print_to_file(self, file_path, include_hidden=False): 53 | """ Prints options to a config file in a human readable format """ 54 | with open(file_path, "w") as f: 55 | for name, val in vars(self).items(): 56 | if name.startswith("_") and not include_hidden: 57 | continue 58 | val_format = str(val) if not isinstance(val, str) else f"'{val}'" 59 | f.write(f"{name}: {val_format}\n") 60 | 61 | def load_from_file(self, file_path, include_hidden=False): 62 | """ Load options from config file""" 63 | with open(file_path, "r") as f: 64 | lines = f.readlines() 65 | for line in lines: # First and last lines are {} 66 | tokens = line.split(":") # Getting rid of \t and \n 67 | identifier = tokens[0].strip() 68 | value = tokens[1].strip() 69 | if not hasattr(self, identifier): 70 | print(f"Warning: redundant option {identifier}") 71 | continue 72 | if identifier.startswith("_") and not include_hidden: 73 | continue 74 | if value.startswith("'"): 75 | parsed_value = value[1:-1] 76 | else: 77 | parsed_value = eval(value) 78 | self.__setattr__(identifier, parsed_value) 79 | return self 80 | -------------------------------------------------------------------------------- /models/resnet.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import torch.nn.functional as F 3 | 4 | from models.common import BaseModel, CustomSequential 5 | 6 | # Resnet implementation based on https://github.com/kuangliu/pytorch-cifar/blob/master/models/resnet.py 7 | 8 | 9 | class BasicBlock(BaseModel): 10 | expansion = 1 11 | 12 | def __init__(self, ConvLayer, in_planes, planes, stride=1): 13 | super(BasicBlock, self).__init__(ConvLayer) 14 | self.conv1 = self.ConvLayer( 15 | in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 16 | self.bn1 = nn.BatchNorm2d(planes) 17 | self.conv2 = self.ConvLayer(planes, planes, kernel_size=3, 18 | stride=1, padding=1, bias=False) 19 | self.bn2 = nn.BatchNorm2d(planes) 20 | 21 | self.shortcut = CustomSequential() 22 | if stride != 1 or in_planes != self.expansion*planes: 23 | self.shortcut = CustomSequential( 24 | self.ConvLayer(in_planes, self.expansion*planes, 25 | kernel_size=1, stride=stride, bias=False), 26 | nn.BatchNorm2d(self.expansion*planes) 27 | ) 28 | 29 | def forward(self, x, temperature): 30 | out = F.relu(self.bn1(self.conv1(x, temperature))) 31 | out = self.bn2(self.conv2(out, temperature)) 32 | out += self.shortcut(x, temperature) 33 | out = F.relu(out) 34 | return out 35 | 36 | 37 | class ResNet(BaseModel): 38 | def __init__(self, ConvLayer, block, num_blocks, width_multiplier=1.0, num_classes=200): 39 | super(ResNet, self).__init__(ConvLayer) 40 | nf = int(64 * width_multiplier) 41 | self.in_planes = nf 42 | 43 | self.conv1 = nn.Conv2d(3, self.in_planes, kernel_size=3, stride=1, padding=1, bias=False) # First convolution non-dynamic 44 | self.bn1 = nn.BatchNorm2d(self.in_planes) 45 | self.layer1 = self._make_layer(block, nf, num_blocks[0], stride=1) 46 | self.layer2 = self._make_layer(block, 2 * nf, num_blocks[1], stride=2) 47 | self.layer3 = self._make_layer(block, 4 * nf, num_blocks[2], stride=2) 48 | self.layer4 = self._make_layer(block, 8 * nf, num_blocks[3], stride=2) 49 | self.linear = nn.Linear(8 * nf * block.expansion, num_classes) 50 | self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) 51 | 52 | def _make_layer(self, block, planes, num_blocks, stride): 53 | strides = [stride] + [1]*(num_blocks-1) 54 | layers = [] 55 | for stride in strides: 56 | layers.append(block(self.ConvLayer, self.in_planes, planes, stride)) 57 | self.in_planes = planes * block.expansion 58 | return CustomSequential(*layers) 59 | 60 | def forward(self, x, temperature): 61 | out = F.relu(self.bn1(self.conv1(x))) 62 | out = self.layer1(out, temperature) 63 | out = self.layer2(out, temperature) 64 | out = self.layer3(out, temperature) 65 | out = self.layer4(out, temperature) 66 | out = self.avg_pool(out) 67 | out = out.view(out.size(0), -1) 68 | out = self.linear(out) 69 | out = F.log_softmax(out, dim=1) 70 | return out 71 | 72 | 73 | def ResNet10(ConvLayer, width_multiplier=1.0, num_classes=200): 74 | return ResNet(ConvLayer, BasicBlock, [1, 1, 1, 1], width_multiplier, num_classes=num_classes) 75 | 76 | 77 | def ResNet18(ConvLayer, width_multiplier=1.0, num_classes=200): 78 | return ResNet(ConvLayer, BasicBlock, [2, 2, 2, 2], width_multiplier, num_classes=num_classes) 79 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: dyconv 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - _libgcc_mutex=0.1 8 | - argon2-cffi=20.1.0 9 | - async_generator=1.10 10 | - attrs=20.3.0 11 | - backcall=0.2.0 12 | - backports=1.0 13 | - backports.functools_lru_cache=1.6.1 14 | - blas=1.0 15 | - bleach=3.3.0 16 | - ca-certificates=2021.1.19 17 | - certifi=2020.12.5 18 | - cffi=1.14.5 19 | - cudatoolkit=10.2.89 20 | - cycler=0.10.0 21 | - dbus=1.13.18 22 | - decorator=4.4.2 23 | - defusedxml=0.7.1 24 | - entrypoints=0.3 25 | - expat=2.2.10 26 | - fontconfig=2.13.1 27 | - freetype=2.10.4 28 | - glib=2.67.4 29 | - gst-plugins-base=1.14.0 30 | - gstreamer=1.14.0 31 | - icu=58.2 32 | - importlib-metadata=3.7.3 33 | - intel-openmp=2020.2 34 | - ipykernel=5.5.0 35 | - ipython=7.21.0 36 | - ipython_genutils=0.2.0 37 | - jedi=0.18.0 38 | - jinja2=2.11.3 39 | - jpeg=9b 40 | - jsonschema=3.2.0 41 | - jupyter_client=6.1.12 42 | - jupyter_core=4.7.1 43 | - jupyterlab_pygments=0.1.2 44 | - kiwisolver=1.3.1 45 | - lcms2=2.11 46 | - ld_impl_linux-64=2.33.1 47 | - libffi=3.3 48 | - libgcc-ng=9.1.0 49 | - libpng=1.6.37 50 | - libsodium=1.0.18 51 | - libstdcxx-ng=9.1.0 52 | - libtiff=4.2.0 53 | - libuuid=1.0.3 54 | - libuv=1.40.0 55 | - libwebp-base=1.2.0 56 | - libxcb=1.14 57 | - libxml2=2.9.10 58 | - lz4-c=1.9.3 59 | - markupsafe=1.1.1 60 | - matplotlib=3.3.4 61 | - matplotlib-base=3.3.4 62 | - mistune=0.8.4 63 | - mkl=2020.2 64 | - mkl-service=2.3.0 65 | - mkl_fft=1.3.0 66 | - mkl_random=1.1.1 67 | - nbclient=0.5.3 68 | - nbconvert=6.0.7 69 | - nbformat=5.1.2 70 | - ncurses=6.2 71 | - nest-asyncio=1.4.3 72 | - ninja=1.10.2 73 | - notebook=6.2.0 74 | - numpy=1.19.2 75 | - numpy-base=1.19.2 76 | - olefile=0.46 77 | - openssl=1.1.1j 78 | - packaging=20.9 79 | - pandas=1.2.3 80 | - pandoc=2.12 81 | - pandocfilters=1.4.2 82 | - parso=0.8.1 83 | - pcre=8.44 84 | - pexpect=4.8.0 85 | - pickleshare=0.7.5 86 | - pillow=8.1.2 87 | - pip=21.0.1 88 | - prometheus_client=0.9.0 89 | - prompt-toolkit=3.0.17 90 | - ptyprocess=0.7.0 91 | - pycparser=2.20 92 | - pygments=2.8.1 93 | - pyparsing=2.4.7 94 | - pyqt=5.9.2 95 | - pyrsistent=0.17.3 96 | - python=3.8.8 97 | - python-dateutil=2.8.1 98 | - python_abi=3.8 99 | - pytorch=1.7.0 100 | - pytz=2021.1 101 | - pyzmq=19.0.2 102 | - qt=5.9.7 103 | - readline=8.1 104 | - send2trash=1.5.0 105 | - setuptools=52.0.0 106 | - sip=4.19.13 107 | - six=1.15.0 108 | - sqlite=3.35.2 109 | - terminado=0.9.3 110 | - testpath=0.4.4 111 | - tk=8.6.10 112 | - torchvision=0.8.0 113 | - tornado=6.1 114 | - tqdm=4.59.0 115 | - traitlets=5.0.5 116 | - typing_extensions=3.7.4.3 117 | - wcwidth=0.2.5 118 | - webencodings=0.5.1 119 | - wheel=0.36.2 120 | - xz=5.2.5 121 | - zeromq=4.3.4 122 | - zipp=3.4.1 123 | - zlib=1.2.11 124 | - zstd=1.4.5 125 | - pip: 126 | - absl-py==0.12.0 127 | - cachetools==4.2.1 128 | - chardet==4.0.0 129 | - google-auth==1.28.0 130 | - google-auth-oauthlib==0.4.3 131 | - grpcio==1.36.1 132 | - idna==2.10 133 | - ipywidgets==7.6.3 134 | - jupyterlab-widgets==1.0.0 135 | - markdown==3.3.4 136 | - oauthlib==3.1.0 137 | - protobuf==3.15.6 138 | - pyasn1==0.4.8 139 | - pyasn1-modules==0.2.8 140 | - python-papi==5.5.1.5 141 | - requests==2.25.1 142 | - requests-oauthlib==1.3.0 143 | - rsa==4.7.2 144 | - tensorboard==2.4.1 145 | - tensorboard-plugin-wit==1.8.0 146 | - urllib3==1.26.4 147 | - werkzeug==1.0.1 148 | - widgetsnbextension==3.5.1 149 | -------------------------------------------------------------------------------- /notebooks/DyConv_training.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\"Open" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# Dynamic Convolution\n", 15 | "\n", 16 | "Unofficial implementation of Dynamic Convolutions. Approach from paper [Dynamic Convolution: Attention over Convolution Kernels](https://arxiv.org/pdf/1912.03458.pdf)." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Set-up drive for saving the experiments" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "from google.colab import drive\n", 33 | "drive.mount('/content/drive')\n", 34 | "\n", 35 | "root_dir = '/content/drive/MyDrive/DynamicConvolution'" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "## Clone the repo" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "!git clone https://github.com/TArdelean/DynamicConvolution.git $root_dir" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "## Load experiment configuration" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "import os\n", 68 | "import sys\n", 69 | "sys.path.append(root_dir)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": { 76 | "scrolled": true 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "from utils.options import Options\n", 81 | "\n", 82 | "opt = Options(config_file_arg=\"config_path\", suppress_parse=True)\n", 83 | "opt.load_from_file(os.path.join(root_dir, \"configs/config.yaml\")).proc() " 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "## Launch TensorBoard " 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "%load_ext tensorboard\n", 100 | "%tensorboard --logdir $opt.experiments" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "## Start Training" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "from train import main" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "main(opt)" 126 | ] 127 | } 128 | ], 129 | "metadata": { 130 | "kernelspec": { 131 | "display_name": "Python 3", 132 | "language": "python", 133 | "name": "python3" 134 | }, 135 | "language_info": { 136 | "codemirror_mode": { 137 | "name": "ipython", 138 | "version": 3 139 | }, 140 | "file_extension": ".py", 141 | "mimetype": "text/x-python", 142 | "name": "python", 143 | "nbconvert_exporter": "python", 144 | "pygments_lexer": "ipython3", 145 | "version": "3.8.8" 146 | } 147 | }, 148 | "nbformat": 4, 149 | "nbformat_minor": 4 150 | } 151 | -------------------------------------------------------------------------------- /inspect_attention.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torch import nn 4 | import torch.nn.functional as F 5 | 6 | import models 7 | import data 8 | from dynamic_convolutions import DynamicConvolution, FlexibleKernelsDynamicConvolution 9 | from train import test_accuracy as test 10 | from utils.options import Options 11 | 12 | 13 | def load_my_state_dict(model, state_dict): 14 | own_state = model.state_dict() 15 | for name, param in state_dict.items(): 16 | if name not in own_state: 17 | continue 18 | if isinstance(param, nn.Parameter): 19 | param = param.data 20 | own_state[name].copy_(param) 21 | 22 | 23 | attentions_register = {} 24 | 25 | 26 | class InspectDynamicConvolution(DynamicConvolution): 27 | def __init__(self, *args, **kwargs): 28 | super().__init__(*args, **kwargs) 29 | 30 | def forward(self, x, temperature=1): 31 | batch_size = x.shape[0] 32 | 33 | alphas = self.attention(x, temperature) 34 | self.register(x, alphas) 35 | 36 | agg_weights = torch.sum( 37 | torch.mul(self.kernels_weights.unsqueeze(0), alphas.view(batch_size, -1, 1, 1, 1, 1)), dim=1) 38 | # Group the weights for each batch to conv2 all at once 39 | agg_weights = agg_weights.view(-1, 40 | *agg_weights.shape[-3:]) # batch_size*out_c X in_c X kernel_size X kernel_size 41 | if self.kernels_bias is not None: 42 | agg_bias = torch.sum(torch.mul(self.kernels_bias.unsqueeze(0), alphas.view(batch_size, -1, 1)), dim=1) 43 | agg_bias = agg_bias.view(-1) 44 | else: 45 | agg_bias = None 46 | x_grouped = x.view(1, -1, *x.shape[-2:]) # 1 X batch_size*out_c X H X W 47 | 48 | out = F.conv2d(x_grouped, agg_weights, agg_bias, groups=self.groups * batch_size, 49 | **self.conv_args) # 1 X batch_size*out_C X H' x W' 50 | out = out.view(batch_size, -1, *out.shape[-2:]) # batch_size X out_C X H' x W' 51 | 52 | return out 53 | 54 | def register(self, x, attention): 55 | resolution = x.shape[-2:] 56 | key = "x".join(map(str, resolution)) 57 | if key not in attentions_register: 58 | attentions_register[key] = [] 59 | attentions_register[key].append(attention.detach().cpu().clone()) 60 | 61 | 62 | def inspect_convolution_generator(nof_kernels, reduce): 63 | return FlexibleKernelsDynamicConvolution(InspectDynamicConvolution, nof_kernels, reduce) 64 | 65 | 66 | def compute_entropy(prob_dict): 67 | def one_resolution(prob): 68 | cat = torch.cat(prob) 69 | return torch.distributions.Categorical(cat).entropy().mean() 70 | return {k: one_resolution(prob) for k, prob in prob_dict.items()} 71 | 72 | 73 | def get_inspect_model(opt: Options): 74 | existing_dict = torch.load(opt.checkpoint_path, map_location=opt.device)['model'] 75 | 76 | model_class = getattr(models, opt.model_class) 77 | conv_layer = inspect_convolution_generator(opt.nof_kernels, opt.reduce) 78 | model = model_class(conv_layer, *opt.model_extra_args).to(opt.device) 79 | model.load_state_dict(existing_dict) 80 | return model 81 | 82 | 83 | def main(): 84 | experiment_name = "Resnet_DY_05" 85 | epoch = 100 86 | 87 | experiment_path = os.path.join("experiments", experiment_name) 88 | opt = Options(suppress_parse=True) 89 | opt.load_from_file(os.path.join(experiment_path, "config.yaml")) 90 | opt.checkpoint_path = os.path.join(experiment_path, f"{experiment_name}_{epoch}.pth") 91 | assert opt.use_dynamic 92 | 93 | model = get_inspect_model(opt) 94 | 95 | test_dl = data.create_data_loader(opt, "test") 96 | test_score = test(model, opt.temperature[1], test_dl, opt.device) 97 | 98 | entropy = compute_entropy(attentions_register) 99 | 100 | print(test_score) 101 | print(entropy) 102 | 103 | 104 | if __name__ == '__main__': 105 | main() 106 | -------------------------------------------------------------------------------- /models/deeplab.py: -------------------------------------------------------------------------------- 1 | 2 | # ALL ACKNOWLEDGMENT GOES TO THE PAPER & REPOSITORY AUTHORS 3 | # https://github.com/jfzhang95/pytorch-deeplab-xception 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | from .deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 9 | from .deeplab_details.aspp import build_aspp 10 | from .deeplab_details.decoder import build_decoder 11 | from .deeplab_details.backbone import build_backbone 12 | 13 | from dynamic_convolutions import DynamicConvolution, TempModule 14 | from models.common import BaseModel, CustomSequential 15 | 16 | # DeepLabV3+ model from paper "Encoder-Decoder with Atrous Separable Convolution for Semantic Image Segmentation" (2018) 17 | # https://paperswithcode.com/paper/encoder-decoder-with-atrous-separable 18 | 19 | # With their model they achieved 89% of mean intersection-over-union score 20 | # on PascalVOC-2012, which makes it second best model as of now (the best model at 21 | # the moment is based on enormous amount of self training, and also no source 22 | # code available) 23 | 24 | __all__ = ['DeepLab', 'deeplab'] 25 | 26 | class DeepLab(BaseModel): 27 | def __init__(self, ConvLayer, lr=0.007, backbone='resnet', wm=1.0, output_stride=16, num_classes=21, 28 | sync_bn=True, freeze_bn=False): 29 | super().__init__(ConvLayer) 30 | if backbone == 'drn': 31 | output_stride = 8 32 | 33 | if sync_bn == True: 34 | BatchNorm = SynchronizedBatchNorm2d 35 | else: 36 | BatchNorm = nn.BatchNorm2d 37 | 38 | self.backbone = build_backbone(backbone, output_stride, BatchNorm, wm=wm) 39 | self.aspp = build_aspp(backbone, output_stride, BatchNorm, ConvLayer, wm=wm) 40 | self.decoder = build_decoder(num_classes, backbone, BatchNorm, ConvLayer, wm=wm) 41 | self.lr = lr 42 | self.freeze_bn = freeze_bn 43 | 44 | def forward(self, input, temperature): 45 | x, low_level_feat = self.backbone(input) 46 | x = self.aspp(x, temperature) 47 | x = self.decoder(x, low_level_feat, temperature) 48 | x = F.interpolate(x, size=input.size()[2:], mode='bilinear', align_corners=True) 49 | 50 | return x 51 | 52 | def freeze_bn(self): 53 | for m in self.modules(): 54 | if isinstance(m, SynchronizedBatchNorm2d): 55 | m.eval() 56 | elif isinstance(m, nn.BatchNorm2d): 57 | m.eval() 58 | 59 | def get_1x_lr_params(self): 60 | modules = [self.backbone] 61 | for i in range(len(modules)): 62 | for m in modules[i].named_modules(): 63 | if self.freeze_bn: 64 | if isinstance(m[1], (nn.Conv2d, DynamicConvolution)): 65 | for p in m[1].parameters(): 66 | if p.requires_grad: 67 | yield p 68 | else: 69 | if isinstance(m[1], (nn.Conv2d, DynamicConvolution)) or isinstance(m[1], SynchronizedBatchNorm2d) \ 70 | or isinstance(m[1], nn.BatchNorm2d): 71 | for p in m[1].parameters(): 72 | if p.requires_grad: 73 | yield p 74 | 75 | def get_10x_lr_params(self): 76 | modules = [self.aspp, self.decoder] 77 | for i in range(len(modules)): 78 | for m in modules[i].named_modules(): 79 | if self.freeze_bn: 80 | if isinstance(m[1], (nn.Conv2d, DynamicConvolution)): 81 | for p in m[1].parameters(): 82 | if p.requires_grad: 83 | yield p 84 | else: 85 | if isinstance(m[1], (nn.Conv2d, DynamicConvolution)) or isinstance(m[1], SynchronizedBatchNorm2d) \ 86 | or isinstance(m[1], nn.BatchNorm2d): 87 | for p in m[1].parameters(): 88 | if p.requires_grad: 89 | yield p 90 | def parameters(self): 91 | return [{'params': self.get_1x_lr_params(), 'lr': self.lr}, 92 | {'params': self.get_10x_lr_params(), 'lr': self.lr * 10}] 93 | 94 | if __name__ == "__main__": 95 | model = DeepLab(backbone='mobilenet', output_stride=16) 96 | model.eval() 97 | input = torch.rand(1, 3, 513, 513) 98 | output = model(input) 99 | print(output.size()) 100 | 101 | 102 | -------------------------------------------------------------------------------- /data/tinyimagenet_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | from torchvision import transforms 5 | from torchvision.datasets import ImageFolder 6 | from torchvision.datasets.utils import verify_str_arg 7 | from torchvision.datasets.utils import download_and_extract_archive 8 | 9 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 10 | std=[0.229, 0.224, 0.225]) 11 | 12 | 13 | def TinyImageNet_dataset(stage="train", download=True, root='datasets/'): 14 | if stage == "train": 15 | return TinyImageNetDataset(root, split="train", download=download, 16 | transform=transforms.Compose([ 17 | transforms.RandomAffine(15, None, (0.9, 1.1)), 18 | transforms.RandomResizedCrop(56, scale=(0.25, 1.0)), 19 | transforms.RandomHorizontalFlip(), 20 | transforms.ColorJitter(0.3, 0.3, 0.2, 0.1), 21 | transforms.GaussianBlur(5, (0.1, 0.5)), 22 | transforms.ToTensor(), 23 | normalize, 24 | ])) 25 | else: 26 | return TinyImageNetDataset(root, split="val", download=download, 27 | transform=transforms.Compose([ 28 | transforms.CenterCrop(56), 29 | transforms.ToTensor(), 30 | normalize 31 | ])) 32 | 33 | 34 | class TinyImageNetDataset(ImageFolder): 35 | """ 36 | Dataset for TinyImageNet-200 37 | credits: https://gist.github.com/lromor/bcfc69dcf31b2f3244358aea10b7a11b 38 | """ 39 | base_folder = 'tiny-imagenet-200' 40 | zip_md5 = '90528d7ca1a48142e341f4ef8d21d0de' 41 | splits = ('train', 'val') 42 | filename = 'tiny-imagenet-200.zip' 43 | url = 'http://cs231n.stanford.edu/tiny-imagenet-200.zip' 44 | 45 | def __init__(self, root, split='train', download=False, **kwargs): 46 | self.data_root = os.path.expanduser(root) 47 | self.split = verify_str_arg(split, "split", self.splits) 48 | 49 | if download: 50 | self.download() 51 | 52 | if not self._check_exists(): 53 | raise RuntimeError('Dataset not found.' + 54 | ' You can use download=True to download it') 55 | super().__init__(self.split_folder, **kwargs) 56 | 57 | @property 58 | def dataset_folder(self): 59 | return os.path.join(self.data_root, self.base_folder) 60 | 61 | @property 62 | def split_folder(self): 63 | return os.path.join(self.dataset_folder, self.split) 64 | 65 | def _check_exists(self): 66 | return os.path.exists(self.split_folder) 67 | 68 | def extra_repr(self): 69 | return "Split: {split}".format(**self.__dict__) 70 | 71 | def download(self): 72 | if self._check_exists(): 73 | return 74 | download_and_extract_archive( 75 | self.url, self.data_root, filename=self.filename, 76 | remove_finished=True, md5=self.zip_md5) 77 | assert 'val' in self.splits 78 | normalize_tin_val_folder_structure( 79 | os.path.join(self.dataset_folder, 'val')) 80 | 81 | 82 | def normalize_tin_val_folder_structure(path, 83 | images_folder='images', 84 | annotations_file='val_annotations.txt'): 85 | # Check if files/annotations are still there to see 86 | # if we already run reorganize the folder structure. 87 | images_folder = os.path.join(path, images_folder) 88 | annotations_file = os.path.join(path, annotations_file) 89 | 90 | # Exists 91 | if not os.path.exists(images_folder) \ 92 | and not os.path.exists(annotations_file): 93 | if not os.listdir(path): 94 | raise RuntimeError('Validation folder is empty.') 95 | return 96 | 97 | # Parse the annotations 98 | with open(annotations_file) as f: 99 | for line in f: 100 | values = line.split() 101 | img = values[0] 102 | label = values[1] 103 | img_file = os.path.join(images_folder, values[0]) 104 | label_folder = os.path.join(path, label) 105 | os.makedirs(label_folder, exist_ok=True) 106 | try: 107 | shutil.move(img_file, os.path.join(label_folder, img)) 108 | except FileNotFoundError: 109 | continue 110 | 111 | os.sync() 112 | assert not os.listdir(images_folder) 113 | shutil.rmtree(images_folder) 114 | os.remove(annotations_file) 115 | os.sync() 116 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data 2 | from torch.utils.tensorboard import SummaryWriter 3 | 4 | import models 5 | from torch import nn 6 | import data 7 | from models import common 8 | from models.common import TemperatureScheduler, SmoothNLLLoss 9 | from utils.options import Options 10 | from utils.segmentation_metrics import SegmentationEvaluator 11 | import sys 12 | import numpy as np 13 | from functools import partial 14 | 15 | from tqdm import tqdm 16 | 17 | from utils.utils import load_checkpoint, save_checkpoint 18 | 19 | 20 | def train_epoch(epoch: int, model: nn.Module, criterion: nn.Module, temperature: TemperatureScheduler, 21 | optimizer: torch.optim.Optimizer, loader: torch.utils.data.DataLoader, device: torch.device, 22 | writer: SummaryWriter = None, batch_average=False): 23 | model.train() 24 | print(f"Training epoch {epoch}") 25 | t_bar = tqdm(loader) 26 | iteration = (epoch-1) * len(loader.dataset) 27 | for b_idx, batch in enumerate(t_bar): 28 | in_data, target = batch 29 | in_data, target = in_data.to(device), target.to(device) 30 | out = model(in_data, temperature.get(epoch)) 31 | loss = criterion(out, target) 32 | if batch_average: 33 | loss /= loader.batch_size 34 | optimizer.zero_grad() 35 | loss.backward() 36 | optimizer.step() 37 | iteration += len(in_data) 38 | t_bar.set_description(f"Current loss={loss.item():.3f}", refresh=True) 39 | if writer: 40 | writer.add_scalar("Loss/train", loss.item(), iteration) 41 | 42 | 43 | # classification accuracy validation score 44 | def test_accuracy(model: nn.Module, temperature: float, loader: torch.utils.data.DataLoader, device: torch.device): 45 | model.eval() 46 | correct = 0 47 | with torch.no_grad(): 48 | for batch in loader: 49 | in_data, target = batch 50 | in_data, target = in_data.to(device), target.to(device) 51 | output = model(in_data, temperature) 52 | pred = output.argmax(dim=1, keepdim=True) 53 | correct += pred.eq(target.view_as(pred)).sum().item() 54 | 55 | 56 | return correct / len(loader.dataset) 57 | 58 | # segmentation validation scores 59 | def test_segmentation(model: nn.Module, temperature: float, loader: torch.utils.data.DataLoader, device: torch.device, n_classes: int): 60 | model.eval() 61 | evaluator = SegmentationEvaluator(n_classes) 62 | with torch.no_grad(): 63 | for batch in loader: 64 | in_data, target = batch 65 | in_data, target = in_data.to(device), target.to(device) 66 | with torch.no_grad(): 67 | output = model(in_data, temperature) 68 | target = target.cpu().numpy() 69 | pred = output.cpu().numpy() 70 | pred = np.argmax(pred, axis=1) 71 | evaluator.add_batch(target, pred) 72 | mIoU = evaluator.Mean_Intersection_over_Union() 73 | #fwIoU = evaluator.Frequency_Weighted_Intersection_over_Union() 74 | #px_accuracy = evaluator.Pixel_Accuracy() 75 | #px_accuracy_class = evaluator.Pixel_Accuracy_Class() 76 | return mIoU 77 | 78 | def main(opt: Options): 79 | model = models.create_model(opt) 80 | print("Training with network:") 81 | print(model) 82 | writer = SummaryWriter(opt.checkpoints_dir) 83 | 84 | train_dl = data.create_data_loader(opt, "train") 85 | test_dl = data.create_data_loader(opt, "test") 86 | criterion = getattr(common, opt.criterion)(*opt.criterion_args) 87 | temperature = TemperatureScheduler(*opt.temperature) 88 | optimizer = getattr(torch.optim, opt.optimizer)(model.parameters(), *opt.optimizer_args) 89 | scheduler = getattr(torch.optim.lr_scheduler, opt.scheduler)(optimizer, *opt.scheduler_args) 90 | device = torch.device(opt.device) 91 | model, epoch, optimizer, scheduler = load_checkpoint(opt.checkpoint_path, model, optimizer, scheduler, device) 92 | 93 | if opt.is_classification: 94 | test_metric = test_accuracy 95 | metric_name = 'Accuracy' 96 | else: # segmentation 97 | test_metric = partial(test_segmentation, n_classes=opt.n_classes) 98 | metric_name = 'mIoU' 99 | 100 | print('Setting up complete, starting training') 101 | for ep in range(epoch + 1, opt.max_epoch+1): 102 | train_epoch(ep, model, criterion, temperature, optimizer, train_dl, device, writer, batch_average=opt.batch_average) 103 | test_score = test_metric(model, temperature.get(ep), test_dl, device) 104 | writer.add_scalar(f"{metric_name}/test", test_score, ep * len(test_dl.dataset)) 105 | print(f"Test {metric_name} after {ep} epochs = {test_score}") 106 | scheduler.step() 107 | if ep % opt.save_freq == 0: 108 | save_checkpoint(model, optimizer, scheduler, ep, opt) 109 | 110 | 111 | if __name__ == '__main__': 112 | options = Options(config_file_arg="config_path") 113 | main(options) 114 | -------------------------------------------------------------------------------- /models/deeplab_details/sync_batchnorm/comm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : comm.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import queue 12 | import collections 13 | import threading 14 | 15 | __all__ = ['FutureResult', 'SlavePipe', 'SyncMaster'] 16 | 17 | 18 | class FutureResult(object): 19 | """A thread-safe future implementation. Used only as one-to-one pipe.""" 20 | 21 | def __init__(self): 22 | self._result = None 23 | self._lock = threading.Lock() 24 | self._cond = threading.Condition(self._lock) 25 | 26 | def put(self, result): 27 | with self._lock: 28 | assert self._result is None, 'Previous result has\'t been fetched.' 29 | self._result = result 30 | self._cond.notify() 31 | 32 | def get(self): 33 | with self._lock: 34 | if self._result is None: 35 | self._cond.wait() 36 | 37 | res = self._result 38 | self._result = None 39 | return res 40 | 41 | 42 | _MasterRegistry = collections.namedtuple('MasterRegistry', ['result']) 43 | _SlavePipeBase = collections.namedtuple('_SlavePipeBase', ['identifier', 'queue', 'result']) 44 | 45 | 46 | class SlavePipe(_SlavePipeBase): 47 | """Pipe for master-slave communication.""" 48 | 49 | def run_slave(self, msg): 50 | self.queue.put((self.identifier, msg)) 51 | ret = self.result.get() 52 | self.queue.put(True) 53 | return ret 54 | 55 | 56 | class SyncMaster(object): 57 | """An abstract `SyncMaster` object. 58 | - During the replication, as the data parallel will trigger an callback of each module, all slave devices should 59 | call `register(id)` and obtain an `SlavePipe` to communicate with the master. 60 | - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected, 61 | and passed to a registered callback. 62 | - After receiving the messages, the master device should gather the information and determine to message passed 63 | back to each slave devices. 64 | """ 65 | 66 | def __init__(self, master_callback): 67 | """ 68 | Args: 69 | master_callback: a callback to be invoked after having collected messages from slave devices. 70 | """ 71 | self._master_callback = master_callback 72 | self._queue = queue.Queue() 73 | self._registry = collections.OrderedDict() 74 | self._activated = False 75 | 76 | def __getstate__(self): 77 | return {'master_callback': self._master_callback} 78 | 79 | def __setstate__(self, state): 80 | self.__init__(state['master_callback']) 81 | 82 | def register_slave(self, identifier): 83 | """ 84 | Register an slave device. 85 | Args: 86 | identifier: an identifier, usually is the device id. 87 | Returns: a `SlavePipe` object which can be used to communicate with the master device. 88 | """ 89 | if self._activated: 90 | assert self._queue.empty(), 'Queue is not clean before next initialization.' 91 | self._activated = False 92 | self._registry.clear() 93 | future = FutureResult() 94 | self._registry[identifier] = _MasterRegistry(future) 95 | return SlavePipe(identifier, self._queue, future) 96 | 97 | def run_master(self, master_msg): 98 | """ 99 | Main entry for the master device in each forward pass. 100 | The messages were first collected from each devices (including the master device), and then 101 | an callback will be invoked to compute the message to be sent back to each devices 102 | (including the master device). 103 | Args: 104 | master_msg: the message that the master want to send to itself. This will be placed as the first 105 | message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example. 106 | Returns: the message to be sent back to the master device. 107 | """ 108 | self._activated = True 109 | 110 | intermediates = [(0, master_msg)] 111 | for i in range(self.nr_slaves): 112 | intermediates.append(self._queue.get()) 113 | 114 | results = self._master_callback(intermediates) 115 | assert results[0][0] == 0, 'The first result should belongs to the master.' 116 | 117 | for i, res in results: 118 | if i == 0: 119 | continue 120 | self._registry[i].result.put(res) 121 | 122 | for i in range(self.nr_slaves): 123 | assert self._queue.get() is True 124 | 125 | return results[0][1] 126 | 127 | @property 128 | def nr_slaves(self): 129 | return len(self._registry) 130 | -------------------------------------------------------------------------------- /dynamic_convolutions.py: -------------------------------------------------------------------------------- 1 | from collections import Iterable 2 | import itertools 3 | 4 | import torch 5 | import math 6 | import torch.nn.functional as F 7 | from torch.nn import init 8 | from torch.nn.modules.utils import _pair 9 | from torch import nn 10 | from models.common import TempModule 11 | 12 | 13 | class AttentionLayer(nn.Module): 14 | def __init__(self, c_dim, hidden_dim, nof_kernels): 15 | super().__init__() 16 | self.global_pooling = nn.Sequential(nn.AdaptiveAvgPool2d(1), nn.Flatten()) 17 | self.to_scores = nn.Sequential(nn.Linear(c_dim, hidden_dim), 18 | nn.ReLU(inplace=True), 19 | nn.Linear(hidden_dim, nof_kernels)) 20 | 21 | def forward(self, x, temperature=1): 22 | out = self.global_pooling(x) 23 | scores = self.to_scores(out) 24 | return F.softmax(scores / temperature, dim=-1) 25 | 26 | 27 | class DynamicConvolution(TempModule): 28 | def __init__(self, nof_kernels, reduce, in_channels, out_channels, kernel_size, 29 | stride=1, padding=0, dilation=1, groups=1, bias=True): 30 | """ 31 | Implementation of Dynamic convolution layer 32 | :param in_channels: number of input channels. 33 | :param out_channels: number of output channels. 34 | :param kernel_size: size of the kernel. 35 | :param groups: controls the connections between inputs and outputs. 36 | in_channels and out_channels must both be divisible by groups. 37 | :param nof_kernels: number of kernels to use. 38 | :param reduce: Refers to the size of the hidden layer in attention: hidden = in_channels // reduce 39 | :param bias: If True, convolutions also have a learnable bias 40 | """ 41 | super().__init__() 42 | self.in_channels = in_channels 43 | self.out_channels = out_channels 44 | 45 | self.groups = groups 46 | self.conv_args = {'stride': stride, 'padding': padding, 'dilation': dilation} 47 | self.nof_kernels = nof_kernels 48 | self.attention = AttentionLayer(in_channels, max(1, in_channels // reduce), nof_kernels) 49 | self.kernel_size = _pair(kernel_size) 50 | self.kernels_weights = nn.Parameter(torch.Tensor( 51 | nof_kernels, out_channels, in_channels // self.groups, *self.kernel_size), requires_grad=True) 52 | if bias: 53 | self.kernels_bias = nn.Parameter(torch.Tensor(nof_kernels, out_channels), requires_grad=True) 54 | else: 55 | self.register_parameter('kernels_bias', None) 56 | self.initialize_parameters() 57 | 58 | def initialize_parameters(self): 59 | for i_kernel in range(self.nof_kernels): 60 | init.kaiming_uniform_(self.kernels_weights[i_kernel], a=math.sqrt(5)) 61 | if self.kernels_bias is not None: 62 | bound = 1 / math.sqrt(self.kernels_weights[0, 0].numel()) 63 | nn.init.uniform_(self.kernels_bias, -bound, bound) 64 | 65 | def forward(self, x, temperature=1): 66 | batch_size = x.shape[0] 67 | 68 | alphas = self.attention(x, temperature) 69 | agg_weights = torch.sum( 70 | torch.mul(self.kernels_weights.unsqueeze(0), alphas.view(batch_size, -1, 1, 1, 1, 1)), dim=1) 71 | # Group the weights for each batch to conv2 all at once 72 | agg_weights = agg_weights.view(-1, *agg_weights.shape[-3:]) # batch_size*out_c X in_c X kernel_size X kernel_size 73 | if self.kernels_bias is not None: 74 | agg_bias = torch.sum(torch.mul(self.kernels_bias.unsqueeze(0), alphas.view(batch_size, -1, 1)), dim=1) 75 | agg_bias = agg_bias.view(-1) 76 | else: 77 | agg_bias = None 78 | x_grouped = x.view(1, -1, *x.shape[-2:]) # 1 X batch_size*out_c X H X W 79 | 80 | out = F.conv2d(x_grouped, agg_weights, agg_bias, groups=self.groups * batch_size, 81 | **self.conv_args) # 1 X batch_size*out_C X H' x W' 82 | out = out.view(batch_size, -1, *out.shape[-2:]) # batch_size X out_C X H' x W' 83 | 84 | return out 85 | 86 | 87 | class FlexibleKernelsDynamicConvolution: 88 | def __init__(self, Base, nof_kernels, reduce): 89 | if isinstance(nof_kernels, Iterable): 90 | self.nof_kernels_it = iter(nof_kernels) 91 | else: 92 | self.nof_kernels_it = itertools.cycle([nof_kernels]) 93 | self.Base = Base 94 | self.reduce = reduce 95 | 96 | def __call__(self, *args, **kwargs): 97 | return self.Base(next(self.nof_kernels_it), self.reduce, *args, **kwargs) 98 | 99 | 100 | def dynamic_convolution_generator(nof_kernels, reduce): 101 | return FlexibleKernelsDynamicConvolution(DynamicConvolution, nof_kernels, reduce) 102 | 103 | 104 | if __name__ == '__main__': 105 | torch.manual_seed(41) 106 | t = torch.randn(1, 3, 16, 16) 107 | conv = DynamicConvolution(3, 1, in_channels=3, out_channels=8, kernel_size=3, padding=1, bias=True) 108 | print(conv(t, 10).sum()) 109 | -------------------------------------------------------------------------------- /models/deeplab_details/aspp.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from ..deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 6 | 7 | from dynamic_convolutions import DynamicConvolution, TempModule 8 | from models.common import BaseModel, CustomSequential 9 | 10 | class _ASPPModule(TempModule): 11 | def __init__(self, inplanes, planes, kernel_size, padding, dilation, BatchNorm, ConvLayer): 12 | super(_ASPPModule, self).__init__() 13 | self.atrous_conv = ConvLayer(inplanes, planes, kernel_size=kernel_size, 14 | stride=1, padding=padding, dilation=dilation, bias=False) 15 | self.bn = BatchNorm(planes) 16 | self.relu = nn.ReLU() 17 | 18 | self._init_weight() 19 | 20 | def forward(self, x, temperature): 21 | x = self.atrous_conv(x, temperature) 22 | x = self.bn(x) 23 | 24 | return self.relu(x) 25 | 26 | def _init_weight(self): 27 | for m in self.modules(): 28 | if isinstance(m, nn.Conv2d): 29 | torch.nn.init.kaiming_normal_(m.weight) 30 | elif isinstance(m, SynchronizedBatchNorm2d): 31 | m.weight.data.fill_(1) 32 | m.bias.data.zero_() 33 | elif isinstance(m, nn.BatchNorm2d): 34 | m.weight.data.fill_(1) 35 | m.bias.data.zero_() 36 | if isinstance(m, DynamicConvolution): 37 | for i_kernel in range(m.nof_kernels): 38 | nn.init.kaiming_normal_(m.kernels_weights[i_kernel], mode='fan_out') 39 | if m.kernels_bias is not None: 40 | nn.init.zeros_(m.kernels_bias) 41 | 42 | class ASPP(TempModule): 43 | def __init__(self, backbone, output_stride, BatchNorm, ConvLayer, wm=1.0): 44 | super(ASPP, self).__init__() 45 | if backbone == 'drn': 46 | inplanes = 512 47 | elif backbone == 'mobilenet': 48 | inplanes = 320 49 | else: 50 | inplanes = 2048 51 | if output_stride == 16: 52 | dilations = [1, 6, 12, 18] 53 | elif output_stride == 8: 54 | dilations = [1, 12, 24, 36] 55 | else: 56 | raise NotImplementedError 57 | 58 | inplanes_wm = inplanes #int(inplanes * wm) 59 | planes_wm = int(256 * wm) 60 | self.aspp1 = _ASPPModule(inplanes_wm, planes_wm, 1, padding=0, dilation=dilations[0], BatchNorm=BatchNorm, ConvLayer=ConvLayer) 61 | self.aspp2 = _ASPPModule(inplanes_wm, planes_wm, 3, padding=dilations[1], dilation=dilations[1], BatchNorm=BatchNorm, ConvLayer=ConvLayer) 62 | self.aspp3 = _ASPPModule(inplanes_wm, planes_wm, 3, padding=dilations[2], dilation=dilations[2], BatchNorm=BatchNorm, ConvLayer=ConvLayer) 63 | self.aspp4 = _ASPPModule(inplanes_wm, planes_wm, 3, padding=dilations[3], dilation=dilations[3], BatchNorm=BatchNorm, ConvLayer=ConvLayer) 64 | 65 | self.global_avg_pool = CustomSequential(nn.AdaptiveAvgPool2d((1, 1)), 66 | ConvLayer(inplanes_wm, int(256*wm), 1, stride=1, bias=False), 67 | BatchNorm(int(256*wm)), 68 | nn.ReLU()) 69 | self.conv1 = ConvLayer(int(1280 * wm), int(256 * wm), 1, bias=False) 70 | self.bn1 = BatchNorm(int(256 * wm)) 71 | self.relu = nn.ReLU() 72 | self.dropout = nn.Dropout(0.5) 73 | self._init_weight() 74 | 75 | def forward(self, x, temperature): 76 | x1 = self.aspp1(x, temperature) 77 | x2 = self.aspp2(x, temperature) 78 | x3 = self.aspp3(x, temperature) 79 | x4 = self.aspp4(x, temperature) 80 | x5 = self.global_avg_pool(x, temperature) 81 | x5 = F.interpolate(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) 82 | x = torch.cat((x1, x2, x3, x4, x5), dim=1) 83 | 84 | x = self.conv1(x, temperature) 85 | x = self.bn1(x) 86 | x = self.relu(x) 87 | 88 | return self.dropout(x) 89 | 90 | def _init_weight(self): 91 | for m in self.modules(): 92 | if isinstance(m, nn.Conv2d): 93 | # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 94 | # m.weight.data.normal_(0, math.sqrt(2. / n)) 95 | torch.nn.init.kaiming_normal_(m.weight) 96 | elif isinstance(m, SynchronizedBatchNorm2d): 97 | m.weight.data.fill_(1) 98 | m.bias.data.zero_() 99 | elif isinstance(m, nn.BatchNorm2d): 100 | m.weight.data.fill_(1) 101 | m.bias.data.zero_() 102 | if isinstance(m, DynamicConvolution): 103 | for i_kernel in range(m.nof_kernels): 104 | nn.init.kaiming_normal_(m.kernels_weights[i_kernel], mode='fan_out') 105 | if m.kernels_bias is not None: 106 | nn.init.zeros_(m.kernels_bias) 107 | 108 | 109 | def build_aspp(backbone, output_stride, BatchNorm, ConvLayer, wm=1.0): 110 | return ASPP(backbone, output_stride, BatchNorm, ConvLayer, wm=wm) -------------------------------------------------------------------------------- /models/deeplab_details/backbone/mobilenet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | import math 5 | from ...deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 6 | import torch.utils.model_zoo as model_zoo 7 | 8 | def conv_bn(inp, oup, stride, BatchNorm): 9 | return nn.Sequential( 10 | nn.Conv2d(inp, oup, 3, stride, 1, bias=False), 11 | BatchNorm(oup), 12 | nn.ReLU6(inplace=True) 13 | ) 14 | 15 | 16 | def fixed_padding(inputs, kernel_size, dilation): 17 | kernel_size_effective = kernel_size + (kernel_size - 1) * (dilation - 1) 18 | pad_total = kernel_size_effective - 1 19 | pad_beg = pad_total // 2 20 | pad_end = pad_total - pad_beg 21 | padded_inputs = F.pad(inputs, (pad_beg, pad_end, pad_beg, pad_end)) 22 | return padded_inputs 23 | 24 | 25 | class InvertedResidual(nn.Module): 26 | def __init__(self, inp, oup, stride, dilation, expand_ratio, BatchNorm): 27 | super(InvertedResidual, self).__init__() 28 | self.stride = stride 29 | assert stride in [1, 2] 30 | 31 | hidden_dim = round(inp * expand_ratio) 32 | self.use_res_connect = self.stride == 1 and inp == oup 33 | self.kernel_size = 3 34 | self.dilation = dilation 35 | 36 | if expand_ratio == 1: 37 | self.conv = nn.Sequential( 38 | # dw 39 | nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 0, dilation, groups=hidden_dim, bias=False), 40 | BatchNorm(hidden_dim), 41 | nn.ReLU6(inplace=True), 42 | # pw-linear 43 | nn.Conv2d(hidden_dim, oup, 1, 1, 0, 1, 1, bias=False), 44 | BatchNorm(oup), 45 | ) 46 | else: 47 | self.conv = nn.Sequential( 48 | # pw 49 | nn.Conv2d(inp, hidden_dim, 1, 1, 0, 1, bias=False), 50 | BatchNorm(hidden_dim), 51 | nn.ReLU6(inplace=True), 52 | # dw 53 | nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 0, dilation, groups=hidden_dim, bias=False), 54 | BatchNorm(hidden_dim), 55 | nn.ReLU6(inplace=True), 56 | # pw-linear 57 | nn.Conv2d(hidden_dim, oup, 1, 1, 0, 1, bias=False), 58 | BatchNorm(oup), 59 | ) 60 | 61 | def forward(self, x): 62 | x_pad = fixed_padding(x, self.kernel_size, dilation=self.dilation) 63 | if self.use_res_connect: 64 | x = x + self.conv(x_pad) 65 | else: 66 | x = self.conv(x_pad) 67 | return x 68 | 69 | 70 | class MobileNetV2(nn.Module): 71 | def __init__(self, output_stride=8, BatchNorm=None, width_mult=1., pretrained=True): 72 | super(MobileNetV2, self).__init__() 73 | block = InvertedResidual 74 | input_channel = 32 75 | current_stride = 1 76 | rate = 1 77 | interverted_residual_setting = [ 78 | # t, c, n, s 79 | [1, 16, 1, 1], 80 | [6, 24, 2, 2], 81 | [6, 32, 3, 2], 82 | [6, 64, 4, 2], 83 | [6, 96, 3, 1], 84 | [6, 160, 3, 2], 85 | [6, 320, 1, 1], 86 | ] 87 | 88 | # building first layer 89 | input_channel = int(input_channel * width_mult) 90 | self.features = [conv_bn(3, input_channel, 2, BatchNorm)] 91 | current_stride *= 2 92 | # building inverted residual blocks 93 | for t, c, n, s in interverted_residual_setting: 94 | if current_stride == output_stride: 95 | stride = 1 96 | dilation = rate 97 | rate *= s 98 | else: 99 | stride = s 100 | dilation = 1 101 | current_stride *= s 102 | output_channel = int(c * width_mult) 103 | for i in range(n): 104 | if i == 0: 105 | self.features.append(block(input_channel, output_channel, stride, dilation, t, BatchNorm)) 106 | else: 107 | self.features.append(block(input_channel, output_channel, 1, dilation, t, BatchNorm)) 108 | input_channel = output_channel 109 | self.features = nn.Sequential(*self.features) 110 | self._initialize_weights() 111 | 112 | if pretrained: 113 | self._load_pretrained_model() 114 | 115 | self.low_level_features = self.features[0:4] 116 | self.high_level_features = self.features[4:] 117 | 118 | def forward(self, x): 119 | low_level_feat = self.low_level_features(x) 120 | x = self.high_level_features(low_level_feat) 121 | return x, low_level_feat 122 | 123 | def _load_pretrained_model(self): 124 | pretrain_dict = model_zoo.load_url('http://jeff95.me/models/mobilenet_v2-6a65762b.pth') 125 | model_dict = {} 126 | state_dict = self.state_dict() 127 | for k, v in pretrain_dict.items(): 128 | if k in state_dict: 129 | model_dict[k] = v 130 | state_dict.update(model_dict) 131 | self.load_state_dict(state_dict) 132 | 133 | def _initialize_weights(self): 134 | for m in self.modules(): 135 | if isinstance(m, nn.Conv2d): 136 | # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 137 | # m.weight.data.normal_(0, math.sqrt(2. / n)) 138 | torch.nn.init.kaiming_normal_(m.weight) 139 | elif isinstance(m, SynchronizedBatchNorm2d): 140 | m.weight.data.fill_(1) 141 | m.bias.data.zero_() 142 | elif isinstance(m, nn.BatchNorm2d): 143 | m.weight.data.fill_(1) 144 | m.bias.data.zero_() 145 | 146 | if __name__ == "__main__": 147 | input = torch.rand(1, 3, 512, 512) 148 | model = MobileNetV2(output_stride=16, BatchNorm=nn.BatchNorm2d) 149 | output, low_level_feat = model(input) 150 | print(output.size()) 151 | print(low_level_feat.size()) 152 | -------------------------------------------------------------------------------- /models/mobilenetv2.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code adapted from here: https://github.com/d-li14/mobilenetv2.pytorch/blob/master/models/imagenet/mobilenetv2.py 3 | """ 4 | 5 | import torch 6 | import torch.nn as nn 7 | import math 8 | 9 | from models.common import BaseModel, CustomSequential 10 | from dynamic_convolutions import DynamicConvolution, TempModule, dynamic_convolution_generator 11 | 12 | 13 | __all__ = ['mobilenetv2', 'MobileNetV2'] 14 | 15 | 16 | def _make_divisible(v, divisor, min_value=None): 17 | """ 18 | This function is taken from the original tf repo. 19 | It ensures that all layers have a channel number that is divisible by 8 20 | It can be seen here: 21 | https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py 22 | :param v: 23 | :param divisor: 24 | :param min_value: 25 | :return: 26 | """ 27 | if min_value is None: 28 | min_value = divisor 29 | new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) 30 | # Make sure that round down does not go down by more than 10%. 31 | if new_v < 0.9 * v: 32 | new_v += divisor 33 | return new_v 34 | 35 | 36 | def conv_3x3_bn(inp, oup, stride, conv=nn.Conv2d): 37 | return CustomSequential( 38 | conv(inp, oup, 3, stride, 1, bias=False), 39 | nn.BatchNorm2d(oup), 40 | nn.ReLU6(inplace=True) 41 | ) 42 | 43 | 44 | def conv_1x1_bn(inp, oup, conv=nn.Conv2d): 45 | return CustomSequential( 46 | conv(inp, oup, 1, 1, 0, bias=False), 47 | nn.BatchNorm2d(oup), 48 | nn.ReLU6(inplace=True) 49 | ) 50 | 51 | 52 | class InvertedResidual(TempModule): 53 | def __init__(self, inp, oup, stride, expand_ratio, conv=nn.Conv2d): 54 | super(InvertedResidual, self).__init__() 55 | assert stride in [1, 2] 56 | 57 | hidden_dim = round(inp * expand_ratio) 58 | self.identity = stride == 1 and inp == oup 59 | 60 | if expand_ratio == 1: 61 | self.conv = CustomSequential( 62 | # dw 63 | conv(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), 64 | nn.BatchNorm2d(hidden_dim), 65 | nn.ReLU6(inplace=True), 66 | # pw-linear 67 | conv(hidden_dim, oup, 1, 1, 0, bias=False), 68 | nn.BatchNorm2d(oup), 69 | ) 70 | else: 71 | self.conv = CustomSequential( 72 | # pw 73 | conv(inp, hidden_dim, 1, 1, 0, bias=False), 74 | nn.BatchNorm2d(hidden_dim), 75 | nn.ReLU6(inplace=True), 76 | # dw 77 | conv(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), 78 | nn.BatchNorm2d(hidden_dim), 79 | nn.ReLU6(inplace=True), 80 | # pw-linear 81 | conv(hidden_dim, oup, 1, 1, 0, bias=False), 82 | nn.BatchNorm2d(oup), 83 | ) 84 | 85 | def forward(self, x, temperature): 86 | if self.identity: 87 | return x + self.conv(x, temperature) 88 | else: 89 | return self.conv(x, temperature) 90 | 91 | 92 | class MobileNetV2(BaseModel): 93 | def __init__(self, conv, num_classes=200, width_multiplier=0.35): 94 | super(MobileNetV2, self).__init__(conv) 95 | # setting of inverted residual blocks 96 | self.cfgs = [ 97 | # t, c, n, s 98 | [1, 16, 1, 1], 99 | [6, 24, 2, 2], 100 | [6, 32, 3, 2], 101 | [6, 64, 4, 2], 102 | [6, 96, 3, 1], 103 | [6, 160, 3, 2], 104 | [6, 320, 1, 1], 105 | ] 106 | 107 | # building first layer 108 | input_channel = _make_divisible(32 * width_multiplier, 4 if width_multiplier == 0.1 else 8) 109 | layers = [conv_3x3_bn(3, input_channel, 2, conv=nn.Conv2d)] 110 | # building inverted residual blocks 111 | block = InvertedResidual 112 | for t, c, n, s in self.cfgs: 113 | output_channel = _make_divisible(c * width_multiplier, 4 if width_multiplier == 0.1 else 8) 114 | for i in range(n): 115 | layers.append(block(input_channel, output_channel, s if i == 0 else 1, t, conv=self.ConvLayer)) 116 | input_channel = output_channel 117 | self.features = CustomSequential(*layers) 118 | # building last several layers 119 | output_channel = _make_divisible(1280 * width_multiplier, 4 if width_multiplier == 0.1 else 8) if width_multiplier > 1.0 else 1280 120 | self.conv = conv_1x1_bn(input_channel, output_channel, conv=self.ConvLayer) 121 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 122 | self.classifier = nn.Linear(output_channel, num_classes) 123 | 124 | self._initialize_weights() 125 | 126 | def forward(self, x, temperature): 127 | x = self.features(x, temperature) 128 | x = self.conv(x, temperature) 129 | x = self.avgpool(x) 130 | x = x.view(x.size(0), -1) 131 | x = self.classifier(x) 132 | return x 133 | 134 | def _initialize_weights(self): 135 | for m in self.modules(): 136 | if isinstance(m, nn.Conv2d): 137 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 138 | m.weight.data.normal_(0, math.sqrt(2. / n)) 139 | if m.bias is not None: 140 | m.bias.data.zero_() 141 | elif isinstance(m, DynamicConvolution): 142 | for i_kernel in range(m.nof_kernels): 143 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 144 | m.kernels_weights[i_kernel].data.normal_(0, math.sqrt(2. / n)) 145 | if m.kernels_bias is not None: 146 | m.kernels_bias.data.zeros_() 147 | elif isinstance(m, nn.BatchNorm2d): 148 | m.weight.data.fill_(1) 149 | m.bias.data.zero_() 150 | elif isinstance(m, nn.Linear): 151 | m.weight.data.normal_(0, 0.01) 152 | m.bias.data.zero_() 153 | 154 | def mobilenetv2(**kwargs): 155 | """ 156 | Constructs a MobileNet V2 model 157 | """ 158 | return MobileNetV2(**kwargs) 159 | 160 | 161 | if __name__ == '__main__': 162 | x = torch.rand(1, 3, 64, 64) 163 | conv_layer = dynamic_convolution_generator(4, 4) 164 | model = MobileNetV2(conv_layer) 165 | x = model(x, 30) 166 | print(x.size()) -------------------------------------------------------------------------------- /models/deeplab_details/backbone/resnet.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | import torch.utils.model_zoo as model_zoo 4 | from ...deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 5 | 6 | class Bottleneck(nn.Module): 7 | expansion = 4 8 | 9 | def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None, BatchNorm=None): 10 | super(Bottleneck, self).__init__() 11 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 12 | self.bn1 = BatchNorm(planes) 13 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, 14 | dilation=dilation, padding=dilation, bias=False) 15 | self.bn2 = BatchNorm(planes) 16 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 17 | self.bn3 = BatchNorm(planes * 4) 18 | self.relu = nn.ReLU(inplace=True) 19 | self.downsample = downsample 20 | self.stride = stride 21 | self.dilation = dilation 22 | 23 | def forward(self, x): 24 | residual = x 25 | 26 | out = self.conv1(x) 27 | out = self.bn1(out) 28 | out = self.relu(out) 29 | 30 | out = self.conv2(out) 31 | out = self.bn2(out) 32 | out = self.relu(out) 33 | 34 | out = self.conv3(out) 35 | out = self.bn3(out) 36 | 37 | if self.downsample is not None: 38 | residual = self.downsample(x) 39 | 40 | out += residual 41 | out = self.relu(out) 42 | 43 | return out 44 | 45 | class ResNet(nn.Module): 46 | 47 | def __init__(self, block, layers, output_stride, BatchNorm, pretrained=True): 48 | self.inplanes = 64 49 | super(ResNet, self).__init__() 50 | blocks = [1, 2, 4] 51 | if output_stride == 16: 52 | strides = [1, 2, 2, 1] 53 | dilations = [1, 1, 1, 2] 54 | elif output_stride == 8: 55 | strides = [1, 2, 1, 1] 56 | dilations = [1, 1, 2, 4] 57 | else: 58 | raise NotImplementedError 59 | 60 | # Modules 61 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 62 | bias=False) 63 | self.bn1 = BatchNorm(64) 64 | self.relu = nn.ReLU(inplace=True) 65 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 66 | 67 | self.layer1 = self._make_layer(block, 64, layers[0], stride=strides[0], dilation=dilations[0], BatchNorm=BatchNorm) 68 | self.layer2 = self._make_layer(block, 128, layers[1], stride=strides[1], dilation=dilations[1], BatchNorm=BatchNorm) 69 | self.layer3 = self._make_layer(block, 256, layers[2], stride=strides[2], dilation=dilations[2], BatchNorm=BatchNorm) 70 | self.layer4 = self._make_MG_unit(block, 512, blocks=blocks, stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm) 71 | # self.layer4 = self._make_layer(block, 512, layers[3], stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm) 72 | self._init_weight() 73 | 74 | if pretrained: 75 | self._load_pretrained_model() 76 | 77 | def _make_layer(self, block, planes, blocks, stride=1, dilation=1, BatchNorm=None): 78 | downsample = None 79 | if stride != 1 or self.inplanes != planes * block.expansion: 80 | downsample = nn.Sequential( 81 | nn.Conv2d(self.inplanes, planes * block.expansion, 82 | kernel_size=1, stride=stride, bias=False), 83 | BatchNorm(planes * block.expansion), 84 | ) 85 | 86 | layers = [] 87 | layers.append(block(self.inplanes, planes, stride, dilation, downsample, BatchNorm)) 88 | self.inplanes = planes * block.expansion 89 | for i in range(1, blocks): 90 | layers.append(block(self.inplanes, planes, dilation=dilation, BatchNorm=BatchNorm)) 91 | 92 | return nn.Sequential(*layers) 93 | 94 | def _make_MG_unit(self, block, planes, blocks, stride=1, dilation=1, BatchNorm=None): 95 | downsample = None 96 | if stride != 1 or self.inplanes != planes * block.expansion: 97 | downsample = nn.Sequential( 98 | nn.Conv2d(self.inplanes, planes * block.expansion, 99 | kernel_size=1, stride=stride, bias=False), 100 | BatchNorm(planes * block.expansion), 101 | ) 102 | 103 | layers = [] 104 | layers.append(block(self.inplanes, planes, stride, dilation=blocks[0]*dilation, 105 | downsample=downsample, BatchNorm=BatchNorm)) 106 | self.inplanes = planes * block.expansion 107 | for i in range(1, len(blocks)): 108 | layers.append(block(self.inplanes, planes, stride=1, 109 | dilation=blocks[i]*dilation, BatchNorm=BatchNorm)) 110 | 111 | return nn.Sequential(*layers) 112 | 113 | def forward(self, input): 114 | x = self.conv1(input) 115 | x = self.bn1(x) 116 | x = self.relu(x) 117 | x = self.maxpool(x) 118 | 119 | x = self.layer1(x) 120 | low_level_feat = x 121 | x = self.layer2(x) 122 | x = self.layer3(x) 123 | x = self.layer4(x) 124 | return x, low_level_feat 125 | 126 | def _init_weight(self): 127 | for m in self.modules(): 128 | if isinstance(m, nn.Conv2d): 129 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 130 | m.weight.data.normal_(0, math.sqrt(2. / n)) 131 | elif isinstance(m, SynchronizedBatchNorm2d): 132 | m.weight.data.fill_(1) 133 | m.bias.data.zero_() 134 | elif isinstance(m, nn.BatchNorm2d): 135 | m.weight.data.fill_(1) 136 | m.bias.data.zero_() 137 | 138 | def _load_pretrained_model(self): 139 | pretrain_dict = model_zoo.load_url('https://download.pytorch.org/models/resnet101-5d3b4d8f.pth') 140 | model_dict = {} 141 | state_dict = self.state_dict() 142 | for k, v in pretrain_dict.items(): 143 | if k in state_dict: 144 | model_dict[k] = v 145 | state_dict.update(model_dict) 146 | self.load_state_dict(state_dict) 147 | 148 | def ResNet101(output_stride, BatchNorm, pretrained=True): 149 | """Constructs a ResNet-101 model. 150 | Args: 151 | pretrained (bool): If True, returns a model pre-trained on ImageNet 152 | """ 153 | model = ResNet(Bottleneck, [3, 4, 23, 3], output_stride, BatchNorm, pretrained=pretrained) 154 | return model 155 | 156 | if __name__ == "__main__": 157 | import torch 158 | model = ResNet101(BatchNorm=nn.BatchNorm2d, pretrained=True, output_stride=8) 159 | input = torch.rand(1, 3, 512, 512) 160 | output, low_level_feat = model(input) 161 | print(output.size()) 162 | print(low_level_feat.size()) -------------------------------------------------------------------------------- /data/sb_dataset.py: -------------------------------------------------------------------------------- 1 | 2 | # ALL ACKNOWLEDGMENT GOES TO THE PAPER & REPOSITORY AUTHORS 3 | # https://github.com/jfzhang95/pytorch-deeplab-xception 4 | 5 | from torchvision import transforms 6 | from torchvision import datasets 7 | 8 | import torch 9 | import random 10 | import numpy as np 11 | 12 | from PIL import Image, ImageOps, ImageFilter 13 | 14 | def SB_dataset(stage="train", download=True, root='datasets/SBD'): 15 | if stage == "train": 16 | return datasets.SBDataset(root, image_set='train_noval', 17 | download=download, mode='segmentation', 18 | transforms=CustomCompose([ 19 | CustomRandomHorizontalFlip(), 20 | # NOTE: original repo has args parameter 21 | # CustomRandomScaleCrop(base_size=args.base_size, crop_size=args.crop_size), 22 | CustomRandomScaleCrop(base_size=513, crop_size=513), 23 | CustomRandomGaussianBlur(), 24 | CustomNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), 25 | CustomToTensor(), 26 | ])) 27 | else: 28 | return datasets.SBDataset(root, image_set='val', 29 | download=download, mode='segmentation', 30 | transforms=CustomCompose([ 31 | CustomFixScaleCrop(crop_size=513), 32 | CustomNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), 33 | CustomToTensor(), 34 | ])) 35 | 36 | class CustomRandomGaussianBlur(object): 37 | def __call__(self, img, mask): 38 | #img, mask = sample 39 | if random.random() < 0.5: 40 | img = img.filter(ImageFilter.GaussianBlur( 41 | radius=random.random())) 42 | 43 | return img, mask 44 | 45 | 46 | class CustomRandomScaleCrop(object): 47 | def __init__(self, base_size, crop_size, fill=0): 48 | self.base_size = base_size 49 | self.crop_size = crop_size 50 | self.fill = fill 51 | 52 | def __call__(self, img, mask): 53 | #img, mask = sample 54 | # random scale (short edge) 55 | short_size = random.randint(int(self.base_size * 0.5), int(self.base_size * 2.0)) 56 | w, h = img.size 57 | if h > w: 58 | ow = short_size 59 | oh = int(1.0 * h * ow / w) 60 | else: 61 | oh = short_size 62 | ow = int(1.0 * w * oh / h) 63 | img = img.resize((ow, oh), Image.BILINEAR) 64 | mask = mask.resize((ow, oh), Image.NEAREST) 65 | # pad crop 66 | if short_size < self.crop_size: 67 | padh = self.crop_size - oh if oh < self.crop_size else 0 68 | padw = self.crop_size - ow if ow < self.crop_size else 0 69 | img = ImageOps.expand(img, border=(0, 0, padw, padh), fill=0) 70 | mask = ImageOps.expand(mask, border=(0, 0, padw, padh), fill=self.fill) 71 | # random crop crop_size 72 | w, h = img.size 73 | x1 = random.randint(0, w - self.crop_size) 74 | y1 = random.randint(0, h - self.crop_size) 75 | img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) 76 | mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) 77 | 78 | return img, mask 79 | 80 | class CustomFixScaleCrop(object): 81 | def __init__(self, crop_size): 82 | self.crop_size = crop_size 83 | 84 | def __call__(self, img, mask): 85 | #img, mask = sample 86 | w, h = img.size 87 | if w > h: 88 | oh = self.crop_size 89 | ow = int(1.0 * w * oh / h) 90 | else: 91 | ow = self.crop_size 92 | oh = int(1.0 * h * ow / w) 93 | img = img.resize((ow, oh), Image.BILINEAR) 94 | mask = mask.resize((ow, oh), Image.NEAREST) 95 | # center crop 96 | w, h = img.size 97 | x1 = int(round((w - self.crop_size) / 2.)) 98 | y1 = int(round((h - self.crop_size) / 2.)) 99 | img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) 100 | mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) 101 | 102 | return img, mask 103 | 104 | class CustomToTensor(object): 105 | """Convert ndarrays in sample to Tensors.""" 106 | 107 | def __call__(self, img, mask): 108 | # swap color axis because 109 | # numpy image: H x W x C 110 | # torch image: C X H X W 111 | #img, mask = sample 112 | img = np.array(img).astype(np.float32).transpose((2, 0, 1)) 113 | mask = np.array(mask).astype(np.float32) 114 | 115 | img = torch.from_numpy(img).float() 116 | mask = torch.LongTensor(mask) 117 | return img, mask 118 | 119 | 120 | class CustomRandomHorizontalFlip(object): 121 | def __call__(self, img, mask): 122 | #img, mask = sample 123 | if random.random() < 0.5: 124 | img = img.transpose(Image.FLIP_LEFT_RIGHT) 125 | mask = mask.transpose(Image.FLIP_LEFT_RIGHT) 126 | 127 | return img, mask 128 | 129 | class CustomNormalize(object): 130 | """Normalize a tensor image with mean and standard deviation. 131 | Args: 132 | mean (tuple): means for each channel. 133 | std (tuple): standard deviations for each channel. 134 | """ 135 | def __init__(self, mean=(0., 0., 0.), std=(1., 1., 1.)): 136 | self.mean = np.array(mean) 137 | self.std = np.array(std) 138 | 139 | def __call__(self, img, mask): 140 | #img, mask = sample 141 | img = np.array(img).astype(np.float32) 142 | mask = np.array(mask).astype(np.float32) 143 | img /= 255.0 144 | img -= self.mean 145 | img /= self.std 146 | 147 | return img, mask 148 | 149 | class CustomCompose(object): 150 | def __init__(self, transforms): 151 | self.transforms = transforms 152 | 153 | def __call__(self, image, target): 154 | for t in self.transforms: 155 | image, target = t(image, target) 156 | return image, target 157 | 158 | if __name__ == '__main__': 159 | from torch.utils.data import DataLoader 160 | import matplotlib.pyplot as plt 161 | 162 | sbd_train = SB_dataset(stage='train', download=False) 163 | print('Created dataset') 164 | dataloader = DataLoader(sbd_train, batch_size=2, shuffle=True, num_workers=0) 165 | print('Created loader') 166 | for ii, sample in enumerate(dataloader): 167 | img, gt = sample 168 | for jj in range(img.size()[0]): 169 | plt.figure() 170 | plt.subplot(211) 171 | plt.imshow(img[jj].numpy().transpose((1, 2, 0))) 172 | plt.subplot(212) 173 | plt.imshow(gt[jj].numpy()) 174 | break 175 | plt.show(block=True) -------------------------------------------------------------------------------- /models/mobilenetv3.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code adapted from here: https://github.com/kuan-wang/pytorch-mobilenet-v3/blob/master/mobilenetv3.py 3 | """ 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | __all__ = ['MobileNetV3', 'mobilenetv3'] 10 | 11 | from dynamic_convolutions import DynamicConvolution, TempModule 12 | from models.common import BaseModel, CustomSequential 13 | 14 | 15 | def conv_bn(inp, oup, stride, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU): 16 | return CustomSequential( 17 | conv_layer(inp, oup, 3, stride, 1, bias=False), 18 | norm_layer(oup), 19 | nlin_layer(inplace=True) 20 | ) 21 | 22 | 23 | def conv_1x1_bn(inp, oup, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU): 24 | return CustomSequential( 25 | conv_layer(inp, oup, 1, 1, 0, bias=False), 26 | norm_layer(oup), 27 | nlin_layer(inplace=True) 28 | ) 29 | 30 | 31 | class Hswish(nn.Module): 32 | def __init__(self, inplace=True): 33 | super(Hswish, self).__init__() 34 | self.inplace = inplace 35 | 36 | def forward(self, x): 37 | return x * F.relu6(x + 3., inplace=self.inplace) / 6. 38 | 39 | 40 | class Hsigmoid(nn.Module): 41 | def __init__(self, inplace=True): 42 | super(Hsigmoid, self).__init__() 43 | self.inplace = inplace 44 | 45 | def forward(self, x): 46 | return F.relu6(x + 3., inplace=self.inplace) / 6. 47 | 48 | 49 | class SEModule(nn.Module): 50 | def __init__(self, channel, reduction=4): 51 | super(SEModule, self).__init__() 52 | self.avg_pool = nn.AdaptiveAvgPool2d(1) 53 | self.fc = nn.Sequential( 54 | nn.Linear(channel, channel // reduction, bias=False), 55 | nn.ReLU(inplace=True), 56 | nn.Linear(channel // reduction, channel, bias=False), 57 | Hsigmoid() 58 | # nn.Sigmoid() 59 | ) 60 | 61 | def forward(self, x): 62 | b, c, _, _ = x.size() 63 | y = self.avg_pool(x).view(b, c) 64 | y = self.fc(y).view(b, c, 1, 1) 65 | return x * y.expand_as(x) 66 | 67 | 68 | class Identity(nn.Module): 69 | def __init__(self, channel): 70 | super(Identity, self).__init__() 71 | 72 | def forward(self, x): 73 | return x 74 | 75 | 76 | def make_divisible(x, divisible_by=8): 77 | import numpy as np 78 | return int(np.ceil(x * 1. / divisible_by) * divisible_by) 79 | 80 | 81 | class MobileBottleneck(TempModule): 82 | def __init__(self, inp, oup, kernel, stride, exp, se=False, nl='RE', conv_layer=nn.Conv2d): 83 | super().__init__() 84 | assert stride in [1, 2] 85 | assert kernel in [3, 5] 86 | padding = (kernel - 1) // 2 87 | self.use_res_connect = stride == 1 and inp == oup 88 | 89 | norm_layer = nn.BatchNorm2d 90 | if nl == 'RE': 91 | nlin_layer = nn.ReLU # or ReLU6 92 | elif nl == 'HS': 93 | nlin_layer = Hswish 94 | else: 95 | raise NotImplementedError 96 | if se: 97 | SELayer = SEModule 98 | else: 99 | SELayer = Identity 100 | 101 | self.conv = CustomSequential( 102 | # pw 103 | conv_layer(inp, exp, 1, 1, 0, bias=False), 104 | norm_layer(exp), 105 | nlin_layer(inplace=True), 106 | # dw 107 | conv_layer(exp, exp, kernel, stride, padding, groups=exp, bias=False), 108 | norm_layer(exp), 109 | SELayer(exp), 110 | nlin_layer(inplace=True), 111 | # pw-linear 112 | conv_layer(exp, oup, 1, 1, 0, bias=False), 113 | norm_layer(oup), 114 | ) 115 | 116 | def forward(self, x, temperature): 117 | if self.use_res_connect: 118 | return x + self.conv(x, temperature) 119 | else: 120 | return self.conv(x, temperature) 121 | 122 | 123 | class MobileNetV3_(BaseModel): 124 | def __init__(self, ConvLayer, n_class=200, dropout=0.2, mode='small', width_multiplier=1.0, s=1): 125 | super(MobileNetV3_, self).__init__(ConvLayer) 126 | input_channel = 16 127 | last_channel = 1280 128 | if mode == 'large': 129 | # refer to Table 1 in paper 130 | mobile_setting = [ 131 | # k, exp, c, se, nl, s, 132 | [3, 16, 16, False, 'RE', 1], 133 | [3, 64, 24, False, 'RE', 2], 134 | [3, 72, 24, False, 'RE', 1], 135 | [5, 72, 40, True, 'RE', 2], 136 | [5, 120, 40, True, 'RE', 1], 137 | [5, 120, 40, True, 'RE', 1], 138 | [3, 240, 80, False, 'HS', 2], 139 | [3, 200, 80, False, 'HS', 1], 140 | [3, 184, 80, False, 'HS', 1], 141 | [3, 184, 80, False, 'HS', 1], 142 | [3, 480, 112, True, 'HS', 1], 143 | [3, 672, 112, True, 'HS', 1], 144 | [5, 672, 160, True, 'HS', 2], 145 | [5, 960, 160, True, 'HS', 1], 146 | [5, 960, 160, True, 'HS', 1], 147 | ] 148 | elif mode == 'small': 149 | # refer to Table 2 in paper 150 | mobile_setting = [ 151 | # k, exp, c, se, nl, s, 152 | [3, 16, 16, True, 'RE', s], 153 | [3, 72, 24, False, 'RE', 2], 154 | [3, 88, 24, False, 'RE', 1], 155 | [5, 96, 40, True, 'HS', 2], 156 | [5, 240, 40, True, 'HS', 1], 157 | [5, 240, 40, True, 'HS', 1], 158 | [5, 120, 48, True, 'HS', 1], 159 | [5, 144, 48, True, 'HS', 1], 160 | [5, 288, 96, True, 'HS', 2], 161 | [5, 576, 96, True, 'HS', 1], 162 | [5, 576, 96, True, 'HS', 1], 163 | ] 164 | else: 165 | raise NotImplementedError 166 | 167 | # building first layer 168 | last_channel = make_divisible(last_channel * width_multiplier) if width_multiplier > 1.0 else last_channel 169 | self.features = [conv_bn(3, input_channel, s, nlin_layer=Hswish)] 170 | self.classifier = [] 171 | 172 | # building mobile blocks 173 | for k, exp, c, se, nl, s in mobile_setting: 174 | output_channel = make_divisible(c * width_multiplier) 175 | exp_channel = make_divisible(exp * width_multiplier) 176 | self.features.append( 177 | MobileBottleneck(input_channel, output_channel, k, s, exp_channel, se, nl, self.ConvLayer)) 178 | input_channel = output_channel 179 | 180 | # building last several layers 181 | if mode == 'large': 182 | last_conv = make_divisible(960 * width_multiplier) 183 | self.features.append(conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish, conv_layer=self.ConvLayer)) 184 | self.features.append(nn.AdaptiveAvgPool2d(1)) 185 | self.features.append(self.ConvLayer(last_conv, last_channel, 1, 1, 0)) 186 | self.features.append(Hswish(inplace=True)) 187 | elif mode == 'small': 188 | last_conv = make_divisible(576 * width_multiplier) 189 | self.features.append(conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish, conv_layer=self.ConvLayer)) 190 | # self.features.append(SEModule(last_conv)) # refer to paper Table2, but I think this is a mistake 191 | self.features.append(nn.AdaptiveAvgPool2d(1)) 192 | self.features.append(self.ConvLayer(last_conv, last_channel, 1, 1, 0)) 193 | self.features.append(Hswish(inplace=True)) 194 | else: 195 | raise NotImplementedError 196 | 197 | # make it Sequential 198 | self.features = CustomSequential(*self.features) 199 | 200 | # building classifier 201 | self.classifier = CustomSequential( 202 | nn.Dropout(p=dropout), # refer to paper section 6 203 | nn.Linear(last_channel, n_class), 204 | nn.LogSoftmax(dim=-1) 205 | ) 206 | 207 | self._initialize_weights() 208 | 209 | def forward(self, x, temperature): 210 | x = self.features(x, temperature) 211 | x = x.mean(3).mean(2) 212 | x = self.classifier(x, temperature) 213 | return x 214 | 215 | def _initialize_weights(self): 216 | # weight initialization 217 | for m in self.modules(): 218 | if isinstance(m, nn.Conv2d): 219 | nn.init.kaiming_normal_(m.weight, mode='fan_out') 220 | if m.bias is not None: 221 | nn.init.zeros_(m.bias) 222 | if isinstance(m, DynamicConvolution): 223 | for i_kernel in range(m.nof_kernels): 224 | nn.init.kaiming_normal_(m.kernels_weights[i_kernel], mode='fan_out') 225 | if m.kernels_bias is not None: 226 | nn.init.zeros_(m.kernels_bias) 227 | elif isinstance(m, nn.BatchNorm2d): 228 | nn.init.ones_(m.weight) 229 | nn.init.zeros_(m.bias) 230 | elif isinstance(m, nn.Linear): 231 | nn.init.normal_(m.weight, 0, 0.01) 232 | if m.bias is not None: 233 | nn.init.zeros_(m.bias) 234 | 235 | 236 | def MobileNetV3(ConvLayer, width_multiplier=1.0, num_classes=200, stride=1): 237 | return MobileNetV3_(ConvLayer, n_class=num_classes, width_multiplier=width_multiplier, s=stride) 238 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dynamic Convolution 2 | 3 | Unofficial implementation of Dynamic Convolutions. Approach from paper 4 | [Dynamic Convolution: Attention over Convolution Kernels](https://arxiv.org/pdf/1912.03458.pdf). 5 | 6 | Source code of the Project for the Machine Learning course at Skoltech 2021. 7 | 8 | 9 | ### Team members 10 | 11 | - Timotei Ardelean 12 | - Andreea Dogaru 13 | - Alexey Larionov 14 | - Oleg Maslov 15 | - Saian Protasov 16 | 17 | ### Brief repository overview 18 | 19 | * [`configs/`](configs/) - configuration files to be used with `train.py` to train different combinations of models / datasets / optimizers / schedulers and their parameters 20 | * [`data/`](data/) - datasets downloading and setting up: 21 | * `imagenette_dataset.py` - Imagenette dataset ([download](https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz)), which is a small ImageNet subset of 10 classes. 22 | * `tinyimagenet_dataset.py` - Tiny ImageNet dataset ([download](http://cs231n.stanford.edu/tiny-imagenet-200.zip)), which is a small ImageNet subset of 200 classes with 600 images per class. 23 | * `pascalvoc2012_dataset.py` - Pascal VOC 2012 segmentation dataset ([official page](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/)), used to train and evaluate DeepLabV3+ 24 | * `sb_dataset.py` - Semantic Boundaries segmentation dataset ([official page](http://home.bharathh.info/pubs/codes/SBD/download.html)), used to train DeepLabV3+ 25 | * `mnist_dataset.py` - MNIST dataset, provided to test correctness of installation with a trivial classification CNN 26 | * [`datasets/`](datasets/) -default storage of downloaded and extracted datasets (can be configured to be stored somewhere else) 27 | * [`experiments/`](experiments/) - default storage of checkpoints and `tensorboard` logs of experiments (can be configured to be stored somewhere else) 28 | * [`models/`](models/) - different models' source code on which we apply Dynamic Convolutions 29 | * `common.py` - base classes that allow integration of Dynamic Convolutions into existing models. Models derive from `BaseModel` class which allows to construct a model with a custom convolutional layer class (either `nn.Conv2d` or our `DynamicConvolution`), submodules inside models derive from `TempModule` class, which allows pass a `temperature` argument to `forward()` method 30 | * `deeplab.py` and `deeplab_details/` folder - [DeepLabV3+](https://github.com/jfzhang95/pytorch-deeplab-xception) 31 | * `mobilenetv3.py` - MobileNetV3 32 | * `mobilenetv2.py` - MobileNetV2 33 | * `resnet.py` - general ResNet, ResNet10 and ResNet18 34 | * `simple_conv.py` - a trivial classification CNN provided to test correctness of installation 35 | * [`tensorboard_events/`](tensorboard_events/) - a collection of training logs: loss and scores progression over epochs, made to be parsed/visualized in TensorBoard library 36 | * [`notebooks/`](notebooks/) - check out section [below](#notebooks) 37 | * [`utils/`](utils/) - framework auxiliary code for parsing options from configuration files and command line, loading and storing of checkpoints, custom metrics, losses 38 | * 👉[`dynamic_convolutions.py`](dynamic_convolutions.py) - implementation of `DynamicConvolution` class, i.e. a drop-in replacement for `nn.Conv2d` with learnable per-sample attention 39 | * `inspect_attention.py` - methods to extract and analyze Dynamic Convolution state in trained models 40 | * 👉 [`train.py`](train.py) - entry point for training of models, using a configuration file (or other options, see **Reproduce training** section) 41 | 42 | 43 | ### Requirements 44 | A GPU is recommended to perform the experiments. 45 | The code is set up to be effortlessly run using [Google Colab](colab.research.google.com). 46 | Main prerequisites are: 47 | 48 | - [`pytorch`](http://pytorch.org/), [`torchvision`](https://github.com/pytorch/vision) 49 | - `numpy`, `tensorboard`, `pillow`, `tqdm` 50 | 51 | For convenience, a ready to use conda [environment](environment.yml) is provided. 52 | To create a new python environment with all the required packages, you can run: 53 | ```shell 54 | conda env create -f environment.yml 55 | conda activate dyconv 56 | ``` 57 | 58 | ### Training setup 59 | The training process can be started with the provided script: 60 | ```shell 61 | python train.py --config_path "configs/.yaml" 62 | ``` 63 | where `.yaml` refers to a configuration file ([`configs/`](configs)) facilitating experiment reproducibility. Among the available training setups there are: 64 | * `MobileNetV2_[type]_[width-multiplier]_[temperature]` - various setups of MobileNetV2 architecture 65 | * `MobileNetV3_[type]_[width-multiplier]_[dataset]` - various setups of MobileNetV3 architecture 66 | * `deeplabv3plus.yaml`, `dy_deeplabv3plus.yaml` - DeepLabV3+ with MobileNetV2 backbone trained on Pascal VOC 2012 dataset (and Dynamic Convolution variant) 67 | * `deeplabv3plus_0.5.yaml`, `dy_deeplabv3plus_0.5.yaml` - baseline of DeepLabV3+ with MobileNetV2 backbone and x0.5 width convolutions, trained on a combination of Pascal VOC 2012 and Semantic Boundaries datasets (and Dynamic Convolution variant) 68 | * `Resnet_[type]_[width-multiplier]` - various setups of ResNet architecture 69 | * `config.yaml` - toy example of classifying MNIST digits with a trivial CNN, provided to test correctness of installation 70 | 71 | Besides providing the config file, you can also pass options as command line arguments, which will override those used in the config. For the full list of available (and default) options refer to [`utils/options.py`](utils/options.py) file. 72 | ```shell 73 | python train.py --config_path "configs/.yaml" --device "cpu" --batch_size 100 74 | ``` 75 | 76 | TensorBoard facilitates tracking the training process and comparing experiments: 77 | ```shell 78 | tensorboard --logdir experiments 79 | ``` 80 | 81 | ### Notebooks 82 | 83 | The following Google Colab compatible Jupyter notebooks are available: 84 | - [`notebooks/DyConv_training.ipynb`](notebooks/DyConv_training.ipynb) - Self-explanatory training procedure 85 | - [`notebooks/DyConv_profiling.ipynb`](notebooks/DyConv_profiling.ipynb) - Profile number of parameters, FLOPS and inference time 86 | - [`notebooks/DyConv_inspect.ipynb`](notebooks/DyConv_inspect.ipynb) - Inspect the distribution of attentions computed by Dynamic Convolutions 87 | - [`notebooks/DyConv_inspect_segmentation.ipynb`](notebooks/DyConv_inspect_segmentation.ipynb) - Inspect learning progress and statistics of segmentation models training (DeepLabV3+) 88 | 89 | ### Experimental results 90 | 91 | #### Classification 92 | 93 | - Dataset: Tiny ImageNet 94 | 95 | | Network | Number of parameters | Accuracy | Config file | 96 | | :--- | :---: | :---: | :---: | 97 | | ResNet-10 | 5.00M | 56.86 | `Resnet_Vanilla_1.yaml` | 98 | | DY-ResNet-10 | 19.80M | **58.17** | `Resnet_DY_1.yaml` | 99 | | ResNet-10 x 0.5 | 1.27M | 52.78 | `Resnet_Vanilla_05.yaml` | 100 | | DY-ResNet-10 x 0.5 | 4.97M | 53.92 | `Resnet_DY_05.yaml` | 101 | | DY-ResNet-10 x 0.5 (**) | 4.97M | 55.38 |`Resnet_DY_Leaky_05_smooth.yaml`| 102 | | ResNet-10 x 0.25 | 0.33M | 46.17 | `Resnet_Vanilla_025.yaml` | 103 | | DY-ResNet-10 x 0.25 | 1.25M | 48.05 | `Resnet_DY_025.yaml` | 104 | | MobileNetV2 x 0.35 | 0.65M | 34.05 | `MobileNetV2_Vanilla_035.yaml`| 105 | | DY-MobileNetV2 x 0.35 | 2.09M | 31.91 | `MobileNetV2_DY_035.yaml`| 106 | | MobileNetV3 x 0.5 | 0.87M | 53.36 | `MobileNetV3_Vanilla_05_Tiny.yaml`| 107 | | DY-MobileNetV3 x 0.5 | 2.52M | 50.95 | `MobileNetV3_DY_05_Tiny.yaml` | 108 | | MobileNetV3 | 1.91M | 55.53 | `MobileNetV3_Vanilla_1_Tiny.yaml` | 109 | | DY-MobileNetV3 | 6.04M | 53.59 | `MobileNetV3_DY_1_Tiny.yaml` | 110 | 111 | - Dataset: Imagenette 112 | 113 | | Network | Number of parameters | Accuracy | Config file | 114 | | :--- | :---: | :---: | :---: | 115 | | MobileNetV3 x 0.25 | 0.27M | 85.07 | `MobileNetV3_Vanilla_025_Imagenette.yaml` | 116 | | DY-MobileNetV3 x 0.25 | 0.99M | 85.24 | `MobileNetV3_DY_025_Imagenette.yaml` | 117 | | MobileNetV3 x 0.5 | 0.63M | 87.31 | `MobileNetV3_Vanilla_05_Imagenette.yaml` | 118 | | DY-MobileNetV3 x 0.5 | 2.28M | 87.89 | `MobileNetV3_DY_05_Imagenette.yaml` | 119 | | MobileNetV3 | 1.67M | 88.40 | `MobileNetV3_Vanilla_1_Imagenette.yaml` | 120 | | DY-MobileNetV3 | 5.80M | **89.32** | `MobileNetV3_DY_1_Imagenette.yaml` | 121 | 122 | #### Segmentation 123 | 124 | - Dataset: Pascal VOC 2012 125 | 126 | | Network | Number of parameters | mIoU | Config file | 127 | | :--- | :---: | :---: | :---: | 128 | | DeepLabV3+ | 5.81M | 65.09 | `deeplabv3plus.yaml`| 129 | | DY-DeepLabV3+ | 18.4M | **65.45** | `dy_deeplabv3plus.yaml`| 130 | 131 | 132 | - Dataset: Pascal VOC 2012 + SBD (only used for training) 133 | 134 | | Network | Number of parameters | mIoU | Config file | 135 | | :--- | :---: | :---: | :---: | 136 | | DeepLabV3+ x 0.5 | 3.40M | 70.23 | `deeplabv3plus_0.5.yaml` | 137 | | DY-DeepLabV3+ x 0.5 | 8.44M | **71.09** | `dy_deeplabv3plus_0.5.yaml` | 138 | -------------------------------------------------------------------------------- /models/deeplab_details/backbone/xception.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | import torch.utils.model_zoo as model_zoo 6 | from ...deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 7 | 8 | def fixed_padding(inputs, kernel_size, dilation): 9 | kernel_size_effective = kernel_size + (kernel_size - 1) * (dilation - 1) 10 | pad_total = kernel_size_effective - 1 11 | pad_beg = pad_total // 2 12 | pad_end = pad_total - pad_beg 13 | padded_inputs = F.pad(inputs, (pad_beg, pad_end, pad_beg, pad_end)) 14 | return padded_inputs 15 | 16 | 17 | class SeparableConv2d(nn.Module): 18 | def __init__(self, inplanes, planes, kernel_size=3, stride=1, dilation=1, bias=False, BatchNorm=None): 19 | super(SeparableConv2d, self).__init__() 20 | 21 | self.conv1 = nn.Conv2d(inplanes, inplanes, kernel_size, stride, 0, dilation, 22 | groups=inplanes, bias=bias) 23 | self.bn = BatchNorm(inplanes) 24 | self.pointwise = nn.Conv2d(inplanes, planes, 1, 1, 0, 1, 1, bias=bias) 25 | 26 | def forward(self, x): 27 | x = fixed_padding(x, self.conv1.kernel_size[0], dilation=self.conv1.dilation[0]) 28 | x = self.conv1(x) 29 | x = self.bn(x) 30 | x = self.pointwise(x) 31 | return x 32 | 33 | 34 | class Block(nn.Module): 35 | def __init__(self, inplanes, planes, reps, stride=1, dilation=1, BatchNorm=None, 36 | start_with_relu=True, grow_first=True, is_last=False): 37 | super(Block, self).__init__() 38 | 39 | if planes != inplanes or stride != 1: 40 | self.skip = nn.Conv2d(inplanes, planes, 1, stride=stride, bias=False) 41 | self.skipbn = BatchNorm(planes) 42 | else: 43 | self.skip = None 44 | 45 | self.relu = nn.ReLU(inplace=True) 46 | rep = [] 47 | 48 | filters = inplanes 49 | if grow_first: 50 | rep.append(self.relu) 51 | rep.append(SeparableConv2d(inplanes, planes, 3, 1, dilation, BatchNorm=BatchNorm)) 52 | rep.append(BatchNorm(planes)) 53 | filters = planes 54 | 55 | for i in range(reps - 1): 56 | rep.append(self.relu) 57 | rep.append(SeparableConv2d(filters, filters, 3, 1, dilation, BatchNorm=BatchNorm)) 58 | rep.append(BatchNorm(filters)) 59 | 60 | if not grow_first: 61 | rep.append(self.relu) 62 | rep.append(SeparableConv2d(inplanes, planes, 3, 1, dilation, BatchNorm=BatchNorm)) 63 | rep.append(BatchNorm(planes)) 64 | 65 | if stride != 1: 66 | rep.append(self.relu) 67 | rep.append(SeparableConv2d(planes, planes, 3, 2, BatchNorm=BatchNorm)) 68 | rep.append(BatchNorm(planes)) 69 | 70 | if stride == 1 and is_last: 71 | rep.append(self.relu) 72 | rep.append(SeparableConv2d(planes, planes, 3, 1, BatchNorm=BatchNorm)) 73 | rep.append(BatchNorm(planes)) 74 | 75 | if not start_with_relu: 76 | rep = rep[1:] 77 | 78 | self.rep = nn.Sequential(*rep) 79 | 80 | def forward(self, inp): 81 | x = self.rep(inp) 82 | 83 | if self.skip is not None: 84 | skip = self.skip(inp) 85 | skip = self.skipbn(skip) 86 | else: 87 | skip = inp 88 | 89 | x = x + skip 90 | 91 | return x 92 | 93 | 94 | class AlignedXception(nn.Module): 95 | """ 96 | Modified Alighed Xception 97 | """ 98 | def __init__(self, output_stride, BatchNorm, 99 | pretrained=True): 100 | super(AlignedXception, self).__init__() 101 | 102 | if output_stride == 16: 103 | entry_block3_stride = 2 104 | middle_block_dilation = 1 105 | exit_block_dilations = (1, 2) 106 | elif output_stride == 8: 107 | entry_block3_stride = 1 108 | middle_block_dilation = 2 109 | exit_block_dilations = (2, 4) 110 | else: 111 | raise NotImplementedError 112 | 113 | 114 | # Entry flow 115 | self.conv1 = nn.Conv2d(3, 32, 3, stride=2, padding=1, bias=False) 116 | self.bn1 = BatchNorm(32) 117 | self.relu = nn.ReLU(inplace=True) 118 | 119 | self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1, bias=False) 120 | self.bn2 = BatchNorm(64) 121 | 122 | self.block1 = Block(64, 128, reps=2, stride=2, BatchNorm=BatchNorm, start_with_relu=False) 123 | self.block2 = Block(128, 256, reps=2, stride=2, BatchNorm=BatchNorm, start_with_relu=False, 124 | grow_first=True) 125 | self.block3 = Block(256, 728, reps=2, stride=entry_block3_stride, BatchNorm=BatchNorm, 126 | start_with_relu=True, grow_first=True, is_last=True) 127 | 128 | # Middle flow 129 | self.block4 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 130 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 131 | self.block5 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 132 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 133 | self.block6 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 134 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 135 | self.block7 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 136 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 137 | self.block8 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 138 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 139 | self.block9 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 140 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 141 | self.block10 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 142 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 143 | self.block11 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 144 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 145 | self.block12 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 146 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 147 | self.block13 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 148 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 149 | self.block14 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 150 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 151 | self.block15 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 152 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 153 | self.block16 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 154 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 155 | self.block17 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 156 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 157 | self.block18 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 158 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 159 | self.block19 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, 160 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) 161 | 162 | # Exit flow 163 | self.block20 = Block(728, 1024, reps=2, stride=1, dilation=exit_block_dilations[0], 164 | BatchNorm=BatchNorm, start_with_relu=True, grow_first=False, is_last=True) 165 | 166 | self.conv3 = SeparableConv2d(1024, 1536, 3, stride=1, dilation=exit_block_dilations[1], BatchNorm=BatchNorm) 167 | self.bn3 = BatchNorm(1536) 168 | 169 | self.conv4 = SeparableConv2d(1536, 1536, 3, stride=1, dilation=exit_block_dilations[1], BatchNorm=BatchNorm) 170 | self.bn4 = BatchNorm(1536) 171 | 172 | self.conv5 = SeparableConv2d(1536, 2048, 3, stride=1, dilation=exit_block_dilations[1], BatchNorm=BatchNorm) 173 | self.bn5 = BatchNorm(2048) 174 | 175 | # Init weights 176 | self._init_weight() 177 | 178 | # Load pretrained model 179 | if pretrained: 180 | self._load_pretrained_model() 181 | 182 | def forward(self, x): 183 | # Entry flow 184 | x = self.conv1(x) 185 | x = self.bn1(x) 186 | x = self.relu(x) 187 | 188 | x = self.conv2(x) 189 | x = self.bn2(x) 190 | x = self.relu(x) 191 | 192 | x = self.block1(x) 193 | # add relu here 194 | x = self.relu(x) 195 | low_level_feat = x 196 | x = self.block2(x) 197 | x = self.block3(x) 198 | 199 | # Middle flow 200 | x = self.block4(x) 201 | x = self.block5(x) 202 | x = self.block6(x) 203 | x = self.block7(x) 204 | x = self.block8(x) 205 | x = self.block9(x) 206 | x = self.block10(x) 207 | x = self.block11(x) 208 | x = self.block12(x) 209 | x = self.block13(x) 210 | x = self.block14(x) 211 | x = self.block15(x) 212 | x = self.block16(x) 213 | x = self.block17(x) 214 | x = self.block18(x) 215 | x = self.block19(x) 216 | 217 | # Exit flow 218 | x = self.block20(x) 219 | x = self.relu(x) 220 | x = self.conv3(x) 221 | x = self.bn3(x) 222 | x = self.relu(x) 223 | 224 | x = self.conv4(x) 225 | x = self.bn4(x) 226 | x = self.relu(x) 227 | 228 | x = self.conv5(x) 229 | x = self.bn5(x) 230 | x = self.relu(x) 231 | 232 | return x, low_level_feat 233 | 234 | def _init_weight(self): 235 | for m in self.modules(): 236 | if isinstance(m, nn.Conv2d): 237 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 238 | m.weight.data.normal_(0, math.sqrt(2. / n)) 239 | elif isinstance(m, SynchronizedBatchNorm2d): 240 | m.weight.data.fill_(1) 241 | m.bias.data.zero_() 242 | elif isinstance(m, nn.BatchNorm2d): 243 | m.weight.data.fill_(1) 244 | m.bias.data.zero_() 245 | 246 | 247 | def _load_pretrained_model(self): 248 | pretrain_dict = model_zoo.load_url('http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth') 249 | model_dict = {} 250 | state_dict = self.state_dict() 251 | 252 | for k, v in pretrain_dict.items(): 253 | if k in state_dict: 254 | if 'pointwise' in k: 255 | v = v.unsqueeze(-1).unsqueeze(-1) 256 | if k.startswith('block11'): 257 | model_dict[k] = v 258 | model_dict[k.replace('block11', 'block12')] = v 259 | model_dict[k.replace('block11', 'block13')] = v 260 | model_dict[k.replace('block11', 'block14')] = v 261 | model_dict[k.replace('block11', 'block15')] = v 262 | model_dict[k.replace('block11', 'block16')] = v 263 | model_dict[k.replace('block11', 'block17')] = v 264 | model_dict[k.replace('block11', 'block18')] = v 265 | model_dict[k.replace('block11', 'block19')] = v 266 | elif k.startswith('block12'): 267 | model_dict[k.replace('block12', 'block20')] = v 268 | elif k.startswith('bn3'): 269 | model_dict[k] = v 270 | model_dict[k.replace('bn3', 'bn4')] = v 271 | elif k.startswith('conv4'): 272 | model_dict[k.replace('conv4', 'conv5')] = v 273 | elif k.startswith('bn4'): 274 | model_dict[k.replace('bn4', 'bn5')] = v 275 | else: 276 | model_dict[k] = v 277 | state_dict.update(model_dict) 278 | self.load_state_dict(state_dict) 279 | 280 | 281 | 282 | if __name__ == "__main__": 283 | import torch 284 | model = AlignedXception(BatchNorm=nn.BatchNorm2d, pretrained=True, output_stride=16) 285 | input = torch.rand(1, 3, 512, 512) 286 | output, low_level_feat = model(input) 287 | print(output.size()) 288 | print(low_level_feat.size()) 289 | -------------------------------------------------------------------------------- /models/deeplab_details/sync_batchnorm/batchnorm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : batchnorm.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import collections 12 | 13 | import torch 14 | import torch.nn.functional as F 15 | 16 | from torch.nn.modules.batchnorm import _BatchNorm 17 | from torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast 18 | 19 | from .comm import SyncMaster 20 | 21 | __all__ = ['SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', 'SynchronizedBatchNorm3d'] 22 | 23 | 24 | def _sum_ft(tensor): 25 | """sum over the first and last dimention""" 26 | return tensor.sum(dim=0).sum(dim=-1) 27 | 28 | 29 | def _unsqueeze_ft(tensor): 30 | """add new dementions at the front and the tail""" 31 | return tensor.unsqueeze(0).unsqueeze(-1) 32 | 33 | 34 | _ChildMessage = collections.namedtuple('_ChildMessage', ['sum', 'ssum', 'sum_size']) 35 | _MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std']) 36 | 37 | 38 | class _SynchronizedBatchNorm(_BatchNorm): 39 | def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True): 40 | super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine) 41 | 42 | self._sync_master = SyncMaster(self._data_parallel_master) 43 | 44 | self._is_parallel = False 45 | self._parallel_id = None 46 | self._slave_pipe = None 47 | 48 | def forward(self, input): 49 | # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation. 50 | if not (self._is_parallel and self.training): 51 | return F.batch_norm( 52 | input, self.running_mean, self.running_var, self.weight, self.bias, 53 | self.training, self.momentum, self.eps) 54 | 55 | # Resize the input to (B, C, -1). 56 | input_shape = input.size() 57 | input = input.view(input.size(0), self.num_features, -1) 58 | 59 | # Compute the sum and square-sum. 60 | sum_size = input.size(0) * input.size(2) 61 | input_sum = _sum_ft(input) 62 | input_ssum = _sum_ft(input ** 2) 63 | 64 | # Reduce-and-broadcast the statistics. 65 | if self._parallel_id == 0: 66 | mean, inv_std = self._sync_master.run_master(_ChildMessage(input_sum, input_ssum, sum_size)) 67 | else: 68 | mean, inv_std = self._slave_pipe.run_slave(_ChildMessage(input_sum, input_ssum, sum_size)) 69 | 70 | # Compute the output. 71 | if self.affine: 72 | # MJY:: Fuse the multiplication for speed. 73 | output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias) 74 | else: 75 | output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std) 76 | 77 | # Reshape it. 78 | return output.view(input_shape) 79 | 80 | def __data_parallel_replicate__(self, ctx, copy_id): 81 | self._is_parallel = True 82 | self._parallel_id = copy_id 83 | 84 | # parallel_id == 0 means master device. 85 | if self._parallel_id == 0: 86 | ctx.sync_master = self._sync_master 87 | else: 88 | self._slave_pipe = ctx.sync_master.register_slave(copy_id) 89 | 90 | def _data_parallel_master(self, intermediates): 91 | """Reduce the sum and square-sum, compute the statistics, and broadcast it.""" 92 | 93 | # Always using same "device order" makes the ReduceAdd operation faster. 94 | # Thanks to:: Tete Xiao (http://tetexiao.com/) 95 | intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device()) 96 | 97 | to_reduce = [i[1][:2] for i in intermediates] 98 | to_reduce = [j for i in to_reduce for j in i] # flatten 99 | target_gpus = [i[1].sum.get_device() for i in intermediates] 100 | 101 | sum_size = sum([i[1].sum_size for i in intermediates]) 102 | sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce) 103 | mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size) 104 | 105 | broadcasted = Broadcast.apply(target_gpus, mean, inv_std) 106 | 107 | outputs = [] 108 | for i, rec in enumerate(intermediates): 109 | outputs.append((rec[0], _MasterMessage(*broadcasted[i * 2:i * 2 + 2]))) 110 | 111 | return outputs 112 | 113 | def _compute_mean_std(self, sum_, ssum, size): 114 | """Compute the mean and standard-deviation with sum and square-sum. This method 115 | also maintains the moving average on the master device.""" 116 | assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.' 117 | mean = sum_ / size 118 | sumvar = ssum - sum_ * mean 119 | unbias_var = sumvar / (size - 1) 120 | bias_var = sumvar / size 121 | 122 | self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data 123 | self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data 124 | 125 | return mean, bias_var.clamp(self.eps) ** -0.5 126 | 127 | 128 | class SynchronizedBatchNorm1d(_SynchronizedBatchNorm): 129 | r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a 130 | mini-batch. 131 | .. math:: 132 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 133 | This module differs from the built-in PyTorch BatchNorm1d as the mean and 134 | standard-deviation are reduced across all devices during training. 135 | For example, when one uses `nn.DataParallel` to wrap the network during 136 | training, PyTorch's implementation normalize the tensor on each device using 137 | the statistics only on that device, which accelerated the computation and 138 | is also easy to implement, but the statistics might be inaccurate. 139 | Instead, in this synchronized version, the statistics will be computed 140 | over all training samples distributed on multiple devices. 141 | 142 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 143 | as the built-in PyTorch implementation. 144 | The mean and standard-deviation are calculated per-dimension over 145 | the mini-batches and gamma and beta are learnable parameter vectors 146 | of size C (where C is the input size). 147 | During training, this layer keeps a running estimate of its computed mean 148 | and variance. The running sum is kept with a default momentum of 0.1. 149 | During evaluation, this running mean/variance is used for normalization. 150 | Because the BatchNorm is done over the `C` dimension, computing statistics 151 | on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm 152 | Args: 153 | num_features: num_features from an expected input of size 154 | `batch_size x num_features [x width]` 155 | eps: a value added to the denominator for numerical stability. 156 | Default: 1e-5 157 | momentum: the value used for the running_mean and running_var 158 | computation. Default: 0.1 159 | affine: a boolean value that when set to ``True``, gives the layer learnable 160 | affine parameters. Default: ``True`` 161 | Shape: 162 | - Input: :math:`(N, C)` or :math:`(N, C, L)` 163 | - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input) 164 | Examples: 165 | >>> # With Learnable Parameters 166 | >>> m = SynchronizedBatchNorm1d(100) 167 | >>> # Without Learnable Parameters 168 | >>> m = SynchronizedBatchNorm1d(100, affine=False) 169 | >>> input = torch.autograd.Variable(torch.randn(20, 100)) 170 | >>> output = m(input) 171 | """ 172 | 173 | def _check_input_dim(self, input): 174 | if input.dim() != 2 and input.dim() != 3: 175 | raise ValueError('expected 2D or 3D input (got {}D input)' 176 | .format(input.dim())) 177 | super(SynchronizedBatchNorm1d, self)._check_input_dim(input) 178 | 179 | 180 | class SynchronizedBatchNorm2d(_SynchronizedBatchNorm): 181 | r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch 182 | of 3d inputs 183 | .. math:: 184 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 185 | This module differs from the built-in PyTorch BatchNorm2d as the mean and 186 | standard-deviation are reduced across all devices during training. 187 | For example, when one uses `nn.DataParallel` to wrap the network during 188 | training, PyTorch's implementation normalize the tensor on each device using 189 | the statistics only on that device, which accelerated the computation and 190 | is also easy to implement, but the statistics might be inaccurate. 191 | Instead, in this synchronized version, the statistics will be computed 192 | over all training samples distributed on multiple devices. 193 | 194 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 195 | as the built-in PyTorch implementation. 196 | The mean and standard-deviation are calculated per-dimension over 197 | the mini-batches and gamma and beta are learnable parameter vectors 198 | of size C (where C is the input size). 199 | During training, this layer keeps a running estimate of its computed mean 200 | and variance. The running sum is kept with a default momentum of 0.1. 201 | During evaluation, this running mean/variance is used for normalization. 202 | Because the BatchNorm is done over the `C` dimension, computing statistics 203 | on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm 204 | Args: 205 | num_features: num_features from an expected input of 206 | size batch_size x num_features x height x width 207 | eps: a value added to the denominator for numerical stability. 208 | Default: 1e-5 209 | momentum: the value used for the running_mean and running_var 210 | computation. Default: 0.1 211 | affine: a boolean value that when set to ``True``, gives the layer learnable 212 | affine parameters. Default: ``True`` 213 | Shape: 214 | - Input: :math:`(N, C, H, W)` 215 | - Output: :math:`(N, C, H, W)` (same shape as input) 216 | Examples: 217 | >>> # With Learnable Parameters 218 | >>> m = SynchronizedBatchNorm2d(100) 219 | >>> # Without Learnable Parameters 220 | >>> m = SynchronizedBatchNorm2d(100, affine=False) 221 | >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45)) 222 | >>> output = m(input) 223 | """ 224 | 225 | def _check_input_dim(self, input): 226 | if input.dim() != 4: 227 | raise ValueError('expected 4D input (got {}D input)' 228 | .format(input.dim())) 229 | super(SynchronizedBatchNorm2d, self)._check_input_dim(input) 230 | 231 | 232 | class SynchronizedBatchNorm3d(_SynchronizedBatchNorm): 233 | r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch 234 | of 4d inputs 235 | .. math:: 236 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 237 | This module differs from the built-in PyTorch BatchNorm3d as the mean and 238 | standard-deviation are reduced across all devices during training. 239 | For example, when one uses `nn.DataParallel` to wrap the network during 240 | training, PyTorch's implementation normalize the tensor on each device using 241 | the statistics only on that device, which accelerated the computation and 242 | is also easy to implement, but the statistics might be inaccurate. 243 | Instead, in this synchronized version, the statistics will be computed 244 | over all training samples distributed on multiple devices. 245 | 246 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 247 | as the built-in PyTorch implementation. 248 | The mean and standard-deviation are calculated per-dimension over 249 | the mini-batches and gamma and beta are learnable parameter vectors 250 | of size C (where C is the input size). 251 | During training, this layer keeps a running estimate of its computed mean 252 | and variance. The running sum is kept with a default momentum of 0.1. 253 | During evaluation, this running mean/variance is used for normalization. 254 | Because the BatchNorm is done over the `C` dimension, computing statistics 255 | on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm 256 | or Spatio-temporal BatchNorm 257 | Args: 258 | num_features: num_features from an expected input of 259 | size batch_size x num_features x depth x height x width 260 | eps: a value added to the denominator for numerical stability. 261 | Default: 1e-5 262 | momentum: the value used for the running_mean and running_var 263 | computation. Default: 0.1 264 | affine: a boolean value that when set to ``True``, gives the layer learnable 265 | affine parameters. Default: ``True`` 266 | Shape: 267 | - Input: :math:`(N, C, D, H, W)` 268 | - Output: :math:`(N, C, D, H, W)` (same shape as input) 269 | Examples: 270 | >>> # With Learnable Parameters 271 | >>> m = SynchronizedBatchNorm3d(100) 272 | >>> # Without Learnable Parameters 273 | >>> m = SynchronizedBatchNorm3d(100, affine=False) 274 | >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10)) 275 | >>> output = m(input) 276 | """ 277 | 278 | def _check_input_dim(self, input): 279 | if input.dim() != 5: 280 | raise ValueError('expected 5D input (got {}D input)' 281 | .format(input.dim())) 282 | super(SynchronizedBatchNorm3d, self)._check_input_dim(input) -------------------------------------------------------------------------------- /models/deeplab_details/backbone/drn.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import math 3 | import torch.utils.model_zoo as model_zoo 4 | from ...deeplab_details.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d 5 | 6 | webroot = 'http://dl.yf.io/drn/' 7 | 8 | model_urls = { 9 | 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 10 | 'drn-c-26': webroot + 'drn_c_26-ddedf421.pth', 11 | 'drn-c-42': webroot + 'drn_c_42-9d336e8c.pth', 12 | 'drn-c-58': webroot + 'drn_c_58-0a53a92c.pth', 13 | 'drn-d-22': webroot + 'drn_d_22-4bd2f8ea.pth', 14 | 'drn-d-38': webroot + 'drn_d_38-eebb45f0.pth', 15 | 'drn-d-54': webroot + 'drn_d_54-0e0534ff.pth', 16 | 'drn-d-105': webroot + 'drn_d_105-12b40979.pth' 17 | } 18 | 19 | 20 | def conv3x3(in_planes, out_planes, stride=1, padding=1, dilation=1): 21 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 22 | padding=padding, bias=False, dilation=dilation) 23 | 24 | 25 | class BasicBlock(nn.Module): 26 | expansion = 1 27 | 28 | def __init__(self, inplanes, planes, stride=1, downsample=None, 29 | dilation=(1, 1), residual=True, BatchNorm=None): 30 | super(BasicBlock, self).__init__() 31 | self.conv1 = conv3x3(inplanes, planes, stride, 32 | padding=dilation[0], dilation=dilation[0]) 33 | self.bn1 = BatchNorm(planes) 34 | self.relu = nn.ReLU(inplace=True) 35 | self.conv2 = conv3x3(planes, planes, 36 | padding=dilation[1], dilation=dilation[1]) 37 | self.bn2 = BatchNorm(planes) 38 | self.downsample = downsample 39 | self.stride = stride 40 | self.residual = residual 41 | 42 | def forward(self, x): 43 | residual = x 44 | 45 | out = self.conv1(x) 46 | out = self.bn1(out) 47 | out = self.relu(out) 48 | 49 | out = self.conv2(out) 50 | out = self.bn2(out) 51 | 52 | if self.downsample is not None: 53 | residual = self.downsample(x) 54 | if self.residual: 55 | out += residual 56 | out = self.relu(out) 57 | 58 | return out 59 | 60 | 61 | class Bottleneck(nn.Module): 62 | expansion = 4 63 | 64 | def __init__(self, inplanes, planes, stride=1, downsample=None, 65 | dilation=(1, 1), residual=True, BatchNorm=None): 66 | super(Bottleneck, self).__init__() 67 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 68 | self.bn1 = BatchNorm(planes) 69 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, 70 | padding=dilation[1], bias=False, 71 | dilation=dilation[1]) 72 | self.bn2 = BatchNorm(planes) 73 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 74 | self.bn3 = BatchNorm(planes * 4) 75 | self.relu = nn.ReLU(inplace=True) 76 | self.downsample = downsample 77 | self.stride = stride 78 | 79 | def forward(self, x): 80 | residual = x 81 | 82 | out = self.conv1(x) 83 | out = self.bn1(out) 84 | out = self.relu(out) 85 | 86 | out = self.conv2(out) 87 | out = self.bn2(out) 88 | out = self.relu(out) 89 | 90 | out = self.conv3(out) 91 | out = self.bn3(out) 92 | 93 | if self.downsample is not None: 94 | residual = self.downsample(x) 95 | 96 | out += residual 97 | out = self.relu(out) 98 | 99 | return out 100 | 101 | 102 | class DRN(nn.Module): 103 | 104 | def __init__(self, block, layers, arch='D', 105 | channels=(16, 32, 64, 128, 256, 512, 512, 512), 106 | BatchNorm=None): 107 | super(DRN, self).__init__() 108 | self.inplanes = channels[0] 109 | self.out_dim = channels[-1] 110 | self.arch = arch 111 | 112 | if arch == 'C': 113 | self.conv1 = nn.Conv2d(3, channels[0], kernel_size=7, stride=1, 114 | padding=3, bias=False) 115 | self.bn1 = BatchNorm(channels[0]) 116 | self.relu = nn.ReLU(inplace=True) 117 | 118 | self.layer1 = self._make_layer( 119 | BasicBlock, channels[0], layers[0], stride=1, BatchNorm=BatchNorm) 120 | self.layer2 = self._make_layer( 121 | BasicBlock, channels[1], layers[1], stride=2, BatchNorm=BatchNorm) 122 | 123 | elif arch == 'D': 124 | self.layer0 = nn.Sequential( 125 | nn.Conv2d(3, channels[0], kernel_size=7, stride=1, padding=3, 126 | bias=False), 127 | BatchNorm(channels[0]), 128 | nn.ReLU(inplace=True) 129 | ) 130 | 131 | self.layer1 = self._make_conv_layers( 132 | channels[0], layers[0], stride=1, BatchNorm=BatchNorm) 133 | self.layer2 = self._make_conv_layers( 134 | channels[1], layers[1], stride=2, BatchNorm=BatchNorm) 135 | 136 | self.layer3 = self._make_layer(block, channels[2], layers[2], stride=2, BatchNorm=BatchNorm) 137 | self.layer4 = self._make_layer(block, channels[3], layers[3], stride=2, BatchNorm=BatchNorm) 138 | self.layer5 = self._make_layer(block, channels[4], layers[4], 139 | dilation=2, new_level=False, BatchNorm=BatchNorm) 140 | self.layer6 = None if layers[5] == 0 else \ 141 | self._make_layer(block, channels[5], layers[5], dilation=4, 142 | new_level=False, BatchNorm=BatchNorm) 143 | 144 | if arch == 'C': 145 | self.layer7 = None if layers[6] == 0 else \ 146 | self._make_layer(BasicBlock, channels[6], layers[6], dilation=2, 147 | new_level=False, residual=False, BatchNorm=BatchNorm) 148 | self.layer8 = None if layers[7] == 0 else \ 149 | self._make_layer(BasicBlock, channels[7], layers[7], dilation=1, 150 | new_level=False, residual=False, BatchNorm=BatchNorm) 151 | elif arch == 'D': 152 | self.layer7 = None if layers[6] == 0 else \ 153 | self._make_conv_layers(channels[6], layers[6], dilation=2, BatchNorm=BatchNorm) 154 | self.layer8 = None if layers[7] == 0 else \ 155 | self._make_conv_layers(channels[7], layers[7], dilation=1, BatchNorm=BatchNorm) 156 | 157 | self._init_weight() 158 | 159 | def _init_weight(self): 160 | for m in self.modules(): 161 | if isinstance(m, nn.Conv2d): 162 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 163 | m.weight.data.normal_(0, math.sqrt(2. / n)) 164 | elif isinstance(m, SynchronizedBatchNorm2d): 165 | m.weight.data.fill_(1) 166 | m.bias.data.zero_() 167 | elif isinstance(m, nn.BatchNorm2d): 168 | m.weight.data.fill_(1) 169 | m.bias.data.zero_() 170 | 171 | 172 | def _make_layer(self, block, planes, blocks, stride=1, dilation=1, 173 | new_level=True, residual=True, BatchNorm=None): 174 | assert dilation == 1 or dilation % 2 == 0 175 | downsample = None 176 | if stride != 1 or self.inplanes != planes * block.expansion: 177 | downsample = nn.Sequential( 178 | nn.Conv2d(self.inplanes, planes * block.expansion, 179 | kernel_size=1, stride=stride, bias=False), 180 | BatchNorm(planes * block.expansion), 181 | ) 182 | 183 | layers = list() 184 | layers.append(block( 185 | self.inplanes, planes, stride, downsample, 186 | dilation=(1, 1) if dilation == 1 else ( 187 | dilation // 2 if new_level else dilation, dilation), 188 | residual=residual, BatchNorm=BatchNorm)) 189 | self.inplanes = planes * block.expansion 190 | for i in range(1, blocks): 191 | layers.append(block(self.inplanes, planes, residual=residual, 192 | dilation=(dilation, dilation), BatchNorm=BatchNorm)) 193 | 194 | return nn.Sequential(*layers) 195 | 196 | def _make_conv_layers(self, channels, convs, stride=1, dilation=1, BatchNorm=None): 197 | modules = [] 198 | for i in range(convs): 199 | modules.extend([ 200 | nn.Conv2d(self.inplanes, channels, kernel_size=3, 201 | stride=stride if i == 0 else 1, 202 | padding=dilation, bias=False, dilation=dilation), 203 | BatchNorm(channels), 204 | nn.ReLU(inplace=True)]) 205 | self.inplanes = channels 206 | return nn.Sequential(*modules) 207 | 208 | def forward(self, x): 209 | if self.arch == 'C': 210 | x = self.conv1(x) 211 | x = self.bn1(x) 212 | x = self.relu(x) 213 | elif self.arch == 'D': 214 | x = self.layer0(x) 215 | 216 | x = self.layer1(x) 217 | x = self.layer2(x) 218 | 219 | x = self.layer3(x) 220 | low_level_feat = x 221 | 222 | x = self.layer4(x) 223 | x = self.layer5(x) 224 | 225 | if self.layer6 is not None: 226 | x = self.layer6(x) 227 | 228 | if self.layer7 is not None: 229 | x = self.layer7(x) 230 | 231 | if self.layer8 is not None: 232 | x = self.layer8(x) 233 | 234 | return x, low_level_feat 235 | 236 | 237 | class DRN_A(nn.Module): 238 | 239 | def __init__(self, block, layers, BatchNorm=None): 240 | self.inplanes = 64 241 | super(DRN_A, self).__init__() 242 | self.out_dim = 512 * block.expansion 243 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 244 | bias=False) 245 | self.bn1 = BatchNorm(64) 246 | self.relu = nn.ReLU(inplace=True) 247 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 248 | self.layer1 = self._make_layer(block, 64, layers[0], BatchNorm=BatchNorm) 249 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2, BatchNorm=BatchNorm) 250 | self.layer3 = self._make_layer(block, 256, layers[2], stride=1, 251 | dilation=2, BatchNorm=BatchNorm) 252 | self.layer4 = self._make_layer(block, 512, layers[3], stride=1, 253 | dilation=4, BatchNorm=BatchNorm) 254 | 255 | self._init_weight() 256 | 257 | def _init_weight(self): 258 | for m in self.modules(): 259 | if isinstance(m, nn.Conv2d): 260 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 261 | m.weight.data.normal_(0, math.sqrt(2. / n)) 262 | elif isinstance(m, SynchronizedBatchNorm2d): 263 | m.weight.data.fill_(1) 264 | m.bias.data.zero_() 265 | elif isinstance(m, nn.BatchNorm2d): 266 | m.weight.data.fill_(1) 267 | m.bias.data.zero_() 268 | 269 | def _make_layer(self, block, planes, blocks, stride=1, dilation=1, BatchNorm=None): 270 | downsample = None 271 | if stride != 1 or self.inplanes != planes * block.expansion: 272 | downsample = nn.Sequential( 273 | nn.Conv2d(self.inplanes, planes * block.expansion, 274 | kernel_size=1, stride=stride, bias=False), 275 | BatchNorm(planes * block.expansion), 276 | ) 277 | 278 | layers = [] 279 | layers.append(block(self.inplanes, planes, stride, downsample, BatchNorm=BatchNorm)) 280 | self.inplanes = planes * block.expansion 281 | for i in range(1, blocks): 282 | layers.append(block(self.inplanes, planes, 283 | dilation=(dilation, dilation, ), BatchNorm=BatchNorm)) 284 | 285 | return nn.Sequential(*layers) 286 | 287 | def forward(self, x): 288 | x = self.conv1(x) 289 | x = self.bn1(x) 290 | x = self.relu(x) 291 | x = self.maxpool(x) 292 | 293 | x = self.layer1(x) 294 | x = self.layer2(x) 295 | x = self.layer3(x) 296 | x = self.layer4(x) 297 | 298 | return x 299 | 300 | def drn_a_50(BatchNorm, pretrained=True): 301 | model = DRN_A(Bottleneck, [3, 4, 6, 3], BatchNorm=BatchNorm) 302 | if pretrained: 303 | model.load_state_dict(model_zoo.load_url(model_urls['resnet50'])) 304 | return model 305 | 306 | 307 | def drn_c_26(BatchNorm, pretrained=True): 308 | model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 1, 1], arch='C', BatchNorm=BatchNorm) 309 | if pretrained: 310 | pretrained = model_zoo.load_url(model_urls['drn-c-26']) 311 | del pretrained['fc.weight'] 312 | del pretrained['fc.bias'] 313 | model.load_state_dict(pretrained) 314 | return model 315 | 316 | 317 | def drn_c_42(BatchNorm, pretrained=True): 318 | model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 1, 1], arch='C', BatchNorm=BatchNorm) 319 | if pretrained: 320 | pretrained = model_zoo.load_url(model_urls['drn-c-42']) 321 | del pretrained['fc.weight'] 322 | del pretrained['fc.bias'] 323 | model.load_state_dict(pretrained) 324 | return model 325 | 326 | 327 | def drn_c_58(BatchNorm, pretrained=True): 328 | model = DRN(Bottleneck, [1, 1, 3, 4, 6, 3, 1, 1], arch='C', BatchNorm=BatchNorm) 329 | if pretrained: 330 | pretrained = model_zoo.load_url(model_urls['drn-c-58']) 331 | del pretrained['fc.weight'] 332 | del pretrained['fc.bias'] 333 | model.load_state_dict(pretrained) 334 | return model 335 | 336 | 337 | def drn_d_22(BatchNorm, pretrained=True): 338 | model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 1, 1], arch='D', BatchNorm=BatchNorm) 339 | if pretrained: 340 | pretrained = model_zoo.load_url(model_urls['drn-d-22']) 341 | del pretrained['fc.weight'] 342 | del pretrained['fc.bias'] 343 | model.load_state_dict(pretrained) 344 | return model 345 | 346 | 347 | def drn_d_24(BatchNorm, pretrained=True): 348 | model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 2, 2], arch='D', BatchNorm=BatchNorm) 349 | if pretrained: 350 | pretrained = model_zoo.load_url(model_urls['drn-d-24']) 351 | del pretrained['fc.weight'] 352 | del pretrained['fc.bias'] 353 | model.load_state_dict(pretrained) 354 | return model 355 | 356 | 357 | def drn_d_38(BatchNorm, pretrained=True): 358 | model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 1, 1], arch='D', BatchNorm=BatchNorm) 359 | if pretrained: 360 | pretrained = model_zoo.load_url(model_urls['drn-d-38']) 361 | del pretrained['fc.weight'] 362 | del pretrained['fc.bias'] 363 | model.load_state_dict(pretrained) 364 | return model 365 | 366 | 367 | def drn_d_40(BatchNorm, pretrained=True): 368 | model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 2, 2], arch='D', BatchNorm=BatchNorm) 369 | if pretrained: 370 | pretrained = model_zoo.load_url(model_urls['drn-d-40']) 371 | del pretrained['fc.weight'] 372 | del pretrained['fc.bias'] 373 | model.load_state_dict(pretrained) 374 | return model 375 | 376 | 377 | def drn_d_54(BatchNorm, pretrained=True): 378 | model = DRN(Bottleneck, [1, 1, 3, 4, 6, 3, 1, 1], arch='D', BatchNorm=BatchNorm) 379 | if pretrained: 380 | pretrained = model_zoo.load_url(model_urls['drn-d-54']) 381 | del pretrained['fc.weight'] 382 | del pretrained['fc.bias'] 383 | model.load_state_dict(pretrained) 384 | return model 385 | 386 | 387 | def drn_d_105(BatchNorm, pretrained=True): 388 | model = DRN(Bottleneck, [1, 1, 3, 4, 23, 3, 1, 1], arch='D', BatchNorm=BatchNorm) 389 | if pretrained: 390 | pretrained = model_zoo.load_url(model_urls['drn-d-105']) 391 | del pretrained['fc.weight'] 392 | del pretrained['fc.bias'] 393 | model.load_state_dict(pretrained) 394 | return model 395 | 396 | if __name__ == "__main__": 397 | import torch 398 | model = drn_a_50(BatchNorm=nn.BatchNorm2d, pretrained=True) 399 | input = torch.rand(1, 3, 512, 512) 400 | output, low_level_feat = model(input) 401 | print(output.size()) 402 | print(low_level_feat.size()) 403 | -------------------------------------------------------------------------------- /notebooks/DyConv_inspect.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "DyConv_inspect.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [] 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "code", 22 | "metadata": { 23 | "colab": { 24 | "base_uri": "https://localhost:8080/" 25 | }, 26 | "id": "DfNFwlN6yKTL", 27 | "outputId": "490062d8-23ca-4c88-b912-34494b85603a" 28 | }, 29 | "source": [ 30 | "from google.colab import drive\n", 31 | "drive.mount('/content/drive')\n", 32 | "\n", 33 | "root_dir = '/content/drive/MyDrive/DynamicConvolution'" 34 | ], 35 | "execution_count": 1, 36 | "outputs": [ 37 | { 38 | "output_type": "stream", 39 | "text": [ 40 | "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n" 41 | ], 42 | "name": "stdout" 43 | } 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "metadata": { 49 | "id": "fHkeeKVnyUh1" 50 | }, 51 | "source": [ 52 | "import os\n", 53 | "import sys\n", 54 | "sys.path.append(root_dir)" 55 | ], 56 | "execution_count": 2, 57 | "outputs": [] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "metadata": { 62 | "colab": { 63 | "base_uri": "https://localhost:8080/" 64 | }, 65 | "id": "HFcpC9UjyU56", 66 | "outputId": "224a794b-38f7-4a8d-8675-1cf498aefb4b" 67 | }, 68 | "source": [ 69 | "from utils.options import Options\n", 70 | "import inspect_attention\n", 71 | "from inspect_attention import *\n", 72 | "\n", 73 | "\n", 74 | "experiment_name = \"Resnet_DY_05\"\n", 75 | "epoch = 100\n", 76 | "\n", 77 | "experiment_path = os.path.join(root_dir, \"experiments\", experiment_name)\n", 78 | "\n", 79 | "opt = Options(config_file_arg=\"config_path\", suppress_parse=True)\n", 80 | "opt.load_from_file(os.path.join(experiment_path, \"config.yaml\"))\n", 81 | "opt.experiments = os.path.join(root_dir, \"experiments\")\n", 82 | "opt.checkpoint_path = os.path.join(experiment_path, f\"{experiment_name}_{epoch}.pth\")\n", 83 | "\n", 84 | "assert opt.use_dynamic" 85 | ], 86 | "execution_count": 3, 87 | "outputs": [ 88 | { 89 | "output_type": "stream", 90 | "text": [ 91 | "Warning: redundant option checkpoints_dir\n" 92 | ], 93 | "name": "stdout" 94 | } 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "metadata": { 100 | "colab": { 101 | "base_uri": "https://localhost:8080/" 102 | }, 103 | "id": "-PCFH_ajyd1w", 104 | "outputId": "2b21c327-717b-4b3b-c312-b7bf00da9e08" 105 | }, 106 | "source": [ 107 | "opt.__dict__" 108 | ], 109 | "execution_count": 4, 110 | "outputs": [ 111 | { 112 | "output_type": "execute_result", 113 | "data": { 114 | "text/plain": [ 115 | "{'_config_file_arg': 'config_path',\n", 116 | " 'batch_size': 64,\n", 117 | " 'checkpoint_path': '/content/drive/MyDrive/DynamicConvolution/experiments/Resnet_DY_05/Resnet_DY_05_100.pth',\n", 118 | " 'config_path': '',\n", 119 | " 'criterion': 'SmoothNLLLoss',\n", 120 | " 'criterion_args': (0.1,),\n", 121 | " 'dataset_class': 'TinyImageNet_dataset',\n", 122 | " 'device': 'cuda',\n", 123 | " 'experiment_name': 'Resnet_DY_05',\n", 124 | " 'experiments': '/content/drive/MyDrive/DynamicConvolution/experiments',\n", 125 | " 'max_epoch': 100,\n", 126 | " 'model_class': 'ResNet10',\n", 127 | " 'model_extra_args': (0.5,),\n", 128 | " 'nof_kernels': 4,\n", 129 | " 'num_workers': 2,\n", 130 | " 'optimizer': 'SGD',\n", 131 | " 'optimizer_args': (0.1, 0.9, 0, 0.0001),\n", 132 | " 'reduce': 4,\n", 133 | " 'save_freq': 5,\n", 134 | " 'scheduler': 'StepLR',\n", 135 | " 'scheduler_args': (30, 0.1),\n", 136 | " 'temperature': (30, 1, 10),\n", 137 | " 'use_dynamic': True}" 138 | ] 139 | }, 140 | "metadata": { 141 | "tags": [] 142 | }, 143 | "execution_count": 4 144 | } 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "metadata": { 150 | "id": "6o7bywvRWBtc" 151 | }, 152 | "source": [ 153 | "model = get_inspect_model(opt)" 154 | ], 155 | "execution_count": 5, 156 | "outputs": [] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "metadata": { 161 | "id": "Un5l_kwyZnXy" 162 | }, 163 | "source": [ 164 | "# train_dl = data.create_data_loader(opt, \"train\")\n", 165 | "# inspect_attention.attentions_register = {}\n", 166 | "# test_score = test(model, opt.temperature[1], train_dl, opt.device)\n", 167 | "# print(\"Train score:\", test_score)" 168 | ], 169 | "execution_count": 6, 170 | "outputs": [] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "M6P3QwD4z4E5", 176 | "colab": { 177 | "base_uri": "https://localhost:8080/" 178 | }, 179 | "outputId": "d5b998ff-9e51-433e-9a3f-3c09f3e8017e" 180 | }, 181 | "source": [ 182 | "test_dl = data.create_data_loader(opt, \"test\")\n", 183 | "inspect_attention.attentions_register = {}\n", 184 | "test_score = test(model, opt.temperature[1], test_dl, opt.device)\n", 185 | "print(\"Test score:\", test_score)" 186 | ], 187 | "execution_count": 7, 188 | "outputs": [ 189 | { 190 | "output_type": "stream", 191 | "text": [ 192 | "Test score: 0.5392\n" 193 | ], 194 | "name": "stdout" 195 | } 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "metadata": { 201 | "colab": { 202 | "base_uri": "https://localhost:8080/" 203 | }, 204 | "id": "ywe6bDAyA2kf", 205 | "outputId": "d90da4ec-1906-49be-a893-c324dbfa3aec" 206 | }, 207 | "source": [ 208 | "compute_entropy(inspect_attention.attentions_register)" 209 | ], 210 | "execution_count": 8, 211 | "outputs": [ 212 | { 213 | "output_type": "execute_result", 214 | "data": { 215 | "text/plain": [ 216 | "{'14x14': tensor(1.0643),\n", 217 | " '28x28': tensor(1.2968),\n", 218 | " '56x56': tensor(1.3518),\n", 219 | " '7x7': tensor(1.0041)}" 220 | ] 221 | }, 222 | "metadata": { 223 | "tags": [] 224 | }, 225 | "execution_count": 8 226 | } 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "metadata": { 232 | "id": "sLHm-OwM6C6A" 233 | }, 234 | "source": [ 235 | "import pandas as pd\n", 236 | "import matplotlib.pyplot as plt\n", 237 | "import seaborn as sns\n", 238 | "\n", 239 | "def to_pandas(register):\n", 240 | " labels = []\n", 241 | " values = []\n", 242 | " for resolution, t_list in register.items():\n", 243 | " t = torch.cat(t_list)\n", 244 | " labels.extend([resolution] * len(t))\n", 245 | " values.extend(torch.distributions.Categorical(t).entropy().tolist())\n", 246 | " return pd.DataFrame({'Resolution': labels, 'Entropy': values})" 247 | ], 248 | "execution_count": 9, 249 | "outputs": [] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "metadata": { 254 | "colab": { 255 | "base_uri": "https://localhost:8080/", 256 | "height": 456 257 | }, 258 | "id": "JX-dv4J97X5-", 259 | "outputId": "fd7a9b4c-d61e-4f8f-8225-adc1fa7f3056" 260 | }, 261 | "source": [ 262 | "df = to_pandas(inspect_attention.attentions_register)\n", 263 | "fig, _ = plt.subplots(figsize = (12, 7))\n", 264 | "b = sns.boxplot(data=df, x=\"Resolution\", y=\"Entropy\")\n", 265 | "\n", 266 | "b.set_xlabel(\"Resolution\",fontsize=20)\n", 267 | "b.set_ylabel(\"Entropy\",fontsize=20)\n", 268 | "b.tick_params(labelsize=16)" 269 | ], 270 | "execution_count": 10, 271 | "outputs": [ 272 | { 273 | "output_type": "display_data", 274 | "data": { 275 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuMAAAG3CAYAAAAeilvYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde5hdZXn38e89yQhFVCQTUQkQMVjFilZTi1VjQBMcW1Droa1Vt29VsFaitR6w9RCEeqjWQ6JFsB6mHlt9bQFlagIIQV4oBgmpYJVYAkQFMwFUBHHC3O8faw3M7EzmsOfw7Jn5fq5rXTvr2etw77DI/ObZz3pWZCaSJEmSZl5H6QIkSZKk+cowLkmSJBViGJckSZIKMYxLkiRJhRjGJUmSpEIWli6gpK6urly6dGnpMiRJkjSHXXnllX2ZuXik9+Z1GF+6dCmbN28uXYYkSZLmsIi4YW/vOUxFkiRJKsQwLkmSJBViGJckSZIKMYxLkiRJhRjGJUmSpEIM45IkSVIhhnFJkiSpkOJhPCKWRMT6iLgsIu6MiIyIpS0c50/rfXdMfZWSJEnS1CsexoFlwIuB24BLWjlARBwAfAS4eQrrkiRJkqZVO4TxTZl5UGY+B/hKi8f4B+Bq4JtTV5YkSZI0vYqH8cwcmMz+EfFU4KXAX01NRZIkSdLMKB7GJyMiOoGzgA9k5rbS9UiSJEkTMavDOPBWYB/gvePdISJOjIjNEbF5586d01eZJEmSNIZZG8YjYhnwd8DrMvPX490vM8/KzOWZuXzx4sXTV6AkSZI0hoWlC5iEdcCFwOX1bCoA9wOiXr87M+8qVp0kSZI0htkcxo8EDqOaErHZbcBHgTfMaEUFrFu3jm3byg6X37Gjmtp9yZIlResAWLZsGWvWrCldhmiPaxPa5/r02mwv7XB9tsu1CV6fUkmzOYz/KbBvU9spwJOAFwHT+vCfdviHHKp/zO+6q+wXAIPnL10HVH8fpf+7+EOtvbTDdSmNxGtTErRJGI+IF9Z/fFL92h0RO4GdmXlxvc1uoCczXwmQmZePcJxXUA1PuWi6a962bRtX/fe1DOx34HSfagwLoeMBRSuIBQnALwrXAfCLu+GWH5V79lPHnbcWO3e7aZdfSAbrWLduXeFK1E7a4fr02pQEbRLG2fNhP/9Uv14MrKz/vKBe2sbAfgfy6yP/qHQZaiP7Xvv10iVIkmaxdvrmHcoPo5oP3za3RRjPzJiibV4xJQWNw44dO+i48+eGLw3TcecuduzYXboMSZImxWFUM6ctwrgkSZLaYwgVOIxqJhnGW7RkyRJuuXuhw1Q0zL7Xfp0lSx5augxJkjRLzNqH/kiSJEmznWFckiRJKsRhKtIc1C5347eD6667DmifcZilzYeZCSRpNjGMS3PQtm3b+OH3vsuh+99TupTi7tdffQH46+3fKVxJeTfe0Vazw0qSMIxLc9ah+9/D25ffUboMtZHTN+9fugRJUhPD+CR03Hmr84wD8etfAJD7PrBwJeVVT+B0NhVJkjQ+hvEWLVu2rHQJbeO6634JwBGPNITCQ702JEnSuBnGW+QNUPfxwQCSJEmtcWpDSZIkqRDDuCRJklSIYVySJEkqxDHj0hy0Y8cOfvXLBU5lp2Fu+OUC7r9jR+kyJElD2DMuSZIkFWLPuDQHLVmyhF/v/qkP/dEwp2/en32XLCldhiRpCHvGJUmSpEIM45IkSVIhhnFJkiSpEMO4Jm379u1s2bKFD3zgA6VLkSRJmlUM45q022+/HYBzzz23cCWSJEmzi2Fck/Le97532Lq945IkSePn1Iaz3Lp169i2bVux82/ZsmXY+rnnnstNN91UqBpYtmwZa9asKXZ+SZKkibBnXJIkSSrEnvFZrnQv8IoVK/ZoW7duXYFKJEmSZh/DuCRJEuWHfraT6667Dijf6dcupnMYrGFcmqNuvGMBp2/ev3QZxd1yZzUa76D9BgpXUt6NdyzgUaWLkNrYtm3b+J8tW3ho6ULawOA45tub7g2bj26e5uMbxqU5aNmyZaVLaBu/qXt39l16ROFKynsUXhvSWB4KvJIoXYbayKfIaT2+YVyag/xa8T6DfxfeyyBJakfOpiJJkiQVYhiXJEmSCnGYiiRpxjhbxX2crWI4H9qm+cowLkmaMdu2beOqa66CA0pX0gbqCX6u+vFVZetoB7eXLkAqxzAuSZpZB8DASqea1H06LnLUrOYvr35JkiSpEMO4JEmSVIhhXJIkSSrEMC5JkiQVUjyMR8SSiFgfEZdFxJ0RkRGxdBz7PSoiPhoRWyPijoj4aUScExGPn/6qJUmSpMkrHsaBZcCLgduASyaw32rgGKAHOB54LbAYuDwinjTVRUqSJElTrR2mNtyUmQcBRMSrqEL2eHwZ+Hhm5mBDRFwIbAdeD7x8iuuUJEmSplTxMJ6ZLU02m5l9I7T9PCJ+CBw86cIkSZKkadYOw1SmTEQcCPwO8P3StUiSJEljmVNhHFgPBPCRvW0QESdGxOaI2Lxz586Zq0ySJElqMmfCeES8DXgJ8LrM3La37TLzrMxcnpnLFy9ePHMFSpIkSU3mRBiPiNcA7wHenpmfLl2PJEmSNB6zPoxHxMuAfwL+MTP/vnQ9kiRJ0njN6jAeEc8HPgP8c2a+qXQ9kiRJ0kQUn9oQICJeWP9x8GE93RGxE9iZmRfX2+wGejLzlfX6CuBLwNXAZyPi6CGHvDszr5qZ6iVJkqTWtEUYB77StP5P9evFwMr6zwvqZdCxwD7AE4FLm/a/AVg6pRVKkiRJU6wtwnhmxkS3ycy1wNppKkmSJEmadrN6zLgkSZI0mxnGJUmSpEIM45IkSVIhhnFJkiSpEMO4pDlt27ZtbNmyhbe//e2lS5EkaQ+GcUlz2h133AHApk2bClciSdKeDOOS5qxTTjll2Lq945KkdtMW84xLmnvWrVvHtm3bitawZcuWYeubNm1izZo1RWpZtmxZsXNLktqXPeOSJElSIfaMS5oW7dALvGLFij3a1q1bV6ASSZJGZs+4JEmSVIg945KkGbNjxw74OXRcZF+QhrgdduSO0lVIRfivoSRJklSIPeOSpBmzZMkSdsZOBlYOlC5FbaTjog6WHLykdBlSEfaMS5IkSYXYMy5JkkR1T8MvgU+RpUtRG/kpcMeO6bunwZ5xSZIkqRB7xiVJkqjuabi9r49XEqVLURv5FMkBS6bvngZ7xiVJkqRCDOOSJElSIYZxSZIkqRDDuCRJklSIYVySJEkqxDAuSZIkFWIYlyRJkgoxjEuSJEmFGMYlSZKkQgzjkiRJUiGGcUmSJKkQw7gkSZJUiGFckiRJKsQwLkmSJBViGJckSZIKMYxLkiRJhRjGJUmSpEIM45IkSVIhhnFJkiSpkOJhPCKWRMT6iLgsIu6MiIyIpePctyMi3hYR2yPi1xFxdUS8YHorliRJkqZG8TAOLANeDNwGXDLBfU8D1gIfA7qBy4GvRMRzprJASZIkaTosLF0AsCkzDwKIiFcBq8ezU0Q8BHgT8L7M/GDd/K2IWAa8DzhvOoqVJEmSpkrxnvHMHGhx1+OA+wGfb2r/PPC4iHjEpAqTJEmSplnxMD4JjwXuBrY1tV9Tvx45s+VIkiRJEzObw/iBwO2ZmU3ttw55fw8RcWJEbI6IzTt37pzWAiVJkqTRzOYw3pLMPCszl2fm8sWLF5cuR5IkSfPYbA7jtwEHREQ0tQ/2iN+KJEmS1MZmcxi/BtgHeGRT++BY8WtnthxJkiRpYmZzGP9PoB/486b2lwLfy8zrZ74kSZIkafzaYZ5xIuKF9R+fVL92R8ROYGdmXlxvsxvoycxXAmTmzyLiQ8DbIuKXwHeBPwGOBU6Y0Q8gSZIktaAtwjjwlab1f6pfLwZW1n9eUC9D/R1wB/B64KHAD4AXZ+bXp6dMSZIkaeq0RRjPzOabMMe1TWbeA5xeL5IkSdKsMpvHjEuSJEmzmmFckiRJKsQwLkmSJBViGJckSZIKMYxrUhYsWDDquiRJkvauLWZT0ex1zz33jLouSXu4HTousi+IO+rX/YtW0R5uBw4uXYRUhmFck7JgwYJhAdyecUmjWbZsWekS2sZ1110HwBEHH1G4kjZwsNeG5i/DuCbFnnFJE7FmzZrSJbSNwb+LdevWFa5EUkl+TyhJkiQVYhiXJEmSCjGMS5IkSYUYxiVJkqRCDOOSJElSIYZxSZIkqRDDuCRJklSIYVySJEkqxDAuSZIkFWIYlyRJkgoxjGtSDjrooFHXJUmStHeGcU3KkiVLhq0fcsghhSqRJEmafQzjmpQrr7xy2PrmzZsLVSJJkjT7GMYlSZKkQgzjkiRJUiGGcU3KU57ylGHrf/AHf1CoEkmSpNnHMK5J2WeffUZdlyRJ0t4ZxjUp3/72t4etX3LJJYUqkSRJmn0M45qUzBx1XZIkSXtnGNekPP3pTx+2vmLFikKVSJIkzT6GcU2KY8YlSZJaZxjXpGzatGnY+sUXX1yoEkmSpNnHMK5JOeigg0ZdlyRJ0t4ZxjUpt9xyy6jrkiRJ2rsJh/GIeOJ0FKLZafXq1UQEABHBcccdV7giSZKk2aOVnvHNEfFfEfEXEbHflFekWaXRaNDZ2QlAZ2cnjUajcEWSJEmzRyth/BvAE4FPAj+JiPUR8bipLUuzRVdXF93d3UQEz3nOc1i0aFHpkiRJkmaNCYfxzDweeARwGvAL4K+ALRFxaUS8PCKc226eaTQaHHXUUfaKS5IkTdDCVnbKzB3A2oh4N/CHwInAs4GjgQ9HxL8AZ2Xm96esUrWtrq4u1q9fX7oMSZIm7WbgU/g06V31q993V9fEAdN4/JbC+KDMHADOBc6NiCXAK4HXAGuANRFxCfCxzPzqpCuVJEmaRsuWLStdQtvYed11ABxwxBGFKynvAKb32phUGG9yJHAU1S9RAfQBTweeHhFbgBdk5vbmnSLiEODDwKp6v/OBN2TmjWOdMCIOpRoucwywGLgJ+DfgvZn5qyn4TJIkaZ5Ys2ZN6RLaxuDfxbp16wpXMvdNap7xiHhIRJwSET8CeoHnARcBfww8FFgGnAk8AfinEfbfD7gQeDTQAF4GHAF8KyLuP8a5708V3FcA7wCeA/wz8DfApyfzuSRJkqSZ0FLPeEQ8EzgJeC7QCdwGfAQ4IzO3Ddn0euC19U2dLx7hUK8GDgd+e3C/iNgKXFcf/0OjlPFUquB+XGZuqNu+FREHAm+KiP0y885WPp8kSZI0E1p56M91wAbghcDVwF8AB2fm3zQF8aGuA0bq6T4BuHzofpl5PXApVdAfzf3q1180td9O9blijP01Rfr6+jj55JPZtWvX2BtLkiTpXq0MUzkY+Czwe5n55Mz8bGb+eox9vkA1rrvZY4HvjdB+DdUY9NGcTxXy3x8RR0bE/hFxLPB64BOOGZ85PT09bN26lZ6entKlSJIkzSqthPGHZ+YrM/PK8e6QmTdl5sUjvHUg1RCXZrcCDx7jmL8Gnkb1Ga4BfglcAHwdeN3e9ouIEyNic0Rs3rlz5zg/gfamr6+P3t5eMpPe3l57xyVJkiaglYf+3D4dhUxUROwL/CvwEKobP58BvBn4E+Dje9svM8/KzOWZuXzx4sUzUutc1tPTQ2Y1H+vAwIC945IkSRPQ8mwqEfHnEXFBRNwaEbvr1/Mj4s8ncJjbGLkHfG895kO9ElgJPCczP5+ZmzLzg1SzqbwmIh4/gTrUoo0bN9Lf3w9Af38/GzZsGGMPSZIkDWrlBs7OiDgb+BeqceAPAHbWr8cC/xIRZ0dE5zgOdw3VuPFmRwLXjrHv44DbMvNHTe1X1K+PGcf5NUmrVq2is7P6T93Z2cnq1asLVyRJkjR7tNIz/jbgeOC/qML4vpn5MGBfqjB+BfBHwFvHcaxzgKMj4vDBhohYSjVt4Tlj7Hsz8OCIaH4k0u/Xrz8ex/k1SY1Gg4hq4pqOjg4ajUbhiiRJkmaPVsL4y4FtwMrMvDgz7wHIzHsy8yKqoSP/C7xiHMf6JLAdODsinhsRJwBnUz1J88zBjSLisHoozDuH7PtZqps2z4uIRkQcExFvBj4IXEk1PaKmWVdXF93d3UQE3d3dLFq0qHRJkiRJs0YrYXwJcHZm/makNzPzbqpAffBYB6qnHzwW+CHwOaopEK8Hjs3MO4ZsGsCCofVm5nbgaGALcDpwHtVDhM4CVmXmwEQ/mFpz/PHHs99++3HCCSeULkUa5qCDDhp1XZKk0lp5AudPqJ66OZrOersxZeaNwAvG2GY7IzzEJzOvZeQne2oGnXvuudx5552cc845vPGNbyxdjnSvW2+9ddR1SZJKa6Vn/IvACyPigSO9GREHUD2d8wuTKUyzg/OMq50NzvSzt3VJkkprJYy/G9gMXBERL4mIJfUMK0vqaQ0vp7qJ87SpLFTtyXnGJUmSWtfKMJW76tegGufdLIAjgF8PzrJRy8xs5XxqYyPNM+5QFUmSpPFpJRxfAuRUF6LZadWqVZx33nn09/c7z7gkSdIETTiMZ+bKaahDs1Sj0aC3txdwnnG1n4i4dxjV4LokSe2klTHj0r26uro45phjADjmmGOcZ1xtZWgQH2ldkqTSJjWGu37k/aOBA4CfA9/PTKcrmGfuvvvuYa+SJEkan5Z6xiPigRHxCeB2qofuXARcBdweEZ+opzfUPNDX18emTZsAuPjii53aUJIkaQImHMbr+cUvBU4EdlPd0Plv9Wt/3f7tvc1DrrnlzDPPZGCgetjpwMAAZ555ZuGKpPsccsgho65LklRaKz3jbwMeC5wBHJaZKzPzz+obOw8DPg4cWW+nOe78888ftr5x48ZClUh7ete73jVs/dRTTy1UiSRJI2sljP8xcHlm/lVm3j70jcz8eWaeDFzGGI+419zQPDuFs1WonRx44IHD1h/84AcXqkSSpJG1EsYPoxojPpqLAb8Pngee+cxnDlt/1rOeVagSaU/Nw6YcRiVJajethPFfAQ8ZY5vFwJ0tHFuzzEknnXRvb3hEcNJJJxWuSLqPw6gkSe2ulTD+HeBFEXHESG9GxCOBF9fbaR4YGsaldjJ4c/He1iVJKq2VMP4BYH/gOxFxWkQcGxGPiYhjIuJUqhC+P/DBqSxU7amnp4eOjuoy6ujooKenp3BF0n0WLFgw6rokSaVNOIxn5gXAa4F9gb8FNgLfA84H3gHcH3hdZp6/14Nozti4cSO7d+8GYPfu3WzYsKFwRdJ9Bq/Nva1LklRaSw/9ycwzgUcB7wT+Hbiwfn0H8KjMPGPKKlRbW7VqFZ2dnQB0dnayevXqwhVJ99l///1HXZckqbRWHvrzzoh4WWbemJl/n5kvzMxV9evfZ+YN01Go2lOj0bh3rHhHRweNRqNwRdJ91q5dO2z9tNNOK1OIJEl70UrP+NuBx011IZqdurq66O7uJiLo7u5m0aJFpUuS7vXkJz/53t7w/fffnyc96UmFK5IkabhWwviPAR91r3s1Gg2OOuooe8XVltauXUtHR4e94pKktrSwhX3+HTghIn4rM++a6oI0+3R1dbF+/frSZUgjevKTn8xFF11UugxJkkbUSs/4u4DbgP+IiN+Z4nokaUpdccUVrFy5kiuvvLJ0KZIk7aGVMH418DDgWcDVEfGriLg+Iv63afnR1JYqSRO3du1aBgYGeMc73lG6FEmS9tBKGO8A+oEb6+VndXs0LS1NmyhJU+WKK67gjjvuAOCOO+6wd1yS1HYmPGY8M5dOQx2SNOWapzZ8xzvewXnnnVemGEmSRmDvtaQ5a7BXfG/rkiSV1spDfy6MiJePsc1LI+LC1suSpMnzCZySpHbXSs/4SmDpGNscBjyjhWNL0pR54xvfOGz9zW9+c6FKJEka2XQNU/ktYPc0HVuSxuXqq68etn7VVVcVqkSSpJG1GsZzpMaoHAY8B7ip5aokaQps3Lhx2PqGDRsKVSJJ0sjGFcYjYiAi7omIe+qmtYPrQxeq3vD/BZ4AfHmaapakcVm1ahWdnZ0AdHZ2snr16sIVSZI03HinNtzEfb3hK6jmF98+wnb3ALuAC4B/nmxxkjQZjUaD3t5eADo6Omg0GoUrkiRpuHGF8cxcOfjniBgAPpOZ756uoiRpKnR1ddHd3c0555xDd3c3ixYtKl2SJEnDTPihP8AjgNunuhBJmg6NRoPt27fbKy5JakutPIHzhukoRJKmQ1dXF+vXry9dhiRJI2qlZ5yI6ASeCzwZeDCwYITNMjNfOYnaJGnS+vr6OPXUU1m7dq3DVCRJbWfCYTwiHg5sBB4NxCibJmAYl1RUT08PW7dupaenZ4+HAEmSVFor84z/I/AYqqkLjwWOoBpH3rwcPkU1SlJL+vr66O3tJTPp7e1l165dpUuSJGmYVsL4amBTZv55Zl6UmT/KzBtGWsZzsIg4JCK+GhE/j4hfRMTXIuLQ8RYTEY+JiK9ERF9E3BURP4iI17fwuSTNMT09PWRWs7IODAzQ09NTuCJJkoZrJYzvC/zXVJw8IvYDLqQa8tIAXkbV0/6tiLj/OPZfXteyD/Aqqid//iMjj2GXNM9s3LiR/v5+APr7+30CpySp7bRyA+f3gMOm6PyvphrO8tuZuQ0gIrYC1wEnAR/a244R0QH8C3BBZj5/yFvfmqLaJM1yq1at4hvf+Aa7d+9m4cKFPoFTktR2WukZ/wBwQkQcOQXnPwG4fDCIA2Tm9cClVLO1jGYl1dj1vQZ2SfNbo9FgYGAAqIapONe4JKndtNIz/jPgXOD/RcRHgSvZy0OAMnPTGMd6LHD2CO3XAC8aY9+n1a/7RsTlwJOA26huLH1rZt41xv6SJElSUa2E8Yuopi0M4B31n/dmrLHbB1IF6Ga3Us1fPpqH16//CnwMOAVYDrwbOAR4/kg7RcSJwIkAhx467vtEJc1CPT09dHR0MDAwQEdHh9MbSpLaTith/N2MHsBnyuAQm89n5jvrP18UEQuA90XEYzLz+807ZeZZwFkAy5cvb4fPIWmabNy4kd27dwOwe/duNmzYYBiXJLWVCYfxzFw7hee/jZF7wPfWYz7U4ITBG5vaNwDvA34X2COMS5o/Vq1axXnnnUd/fz+dnZ3ewClJajut3MA5la6hGjfe7Ejg2nHsO5qBliqSNGc0Gg0iqgcFd3R0eAOnJKntjCuMR8SKCT6I5/ER8fJxbHoOcHRE3Pu0zohYCjy1fm80vcDdwHFN7c+uXzePq1hJc1ZXVxfd3d1EBN3d3SxatKh0SZIkDTPenvFvAa8Y2hARb42IvT1b+nnAZ8Zx3E8C24GzI+K5EXEC1ewqNwFnDjnXYRGxOyIGx4aTmbuA9wKviYj3RMSzIuIU4J1Az9DpEiXNX41Gg6OOOspecUlSWxrvmPEYoW1f4IDJnDwzfxURxwIfBj5Xn+cC4A2ZeUfT+Rew5y8P7wZ+CbwWeBPwU6p50E+bTF2S5o6uri7Wr19fugxJkkbUymwqUyozbwReMMY22xnhF4LMTKqH/vjgH0mSJM06pW/glCRJkuYtw7gkSZJUiGFckiRJKmQiYdynVUqSJElTaCJhfG1E3DO4UE0hyNC25vckqbS+vj5OPvlkdu3a20yskiSVM5EwHhNcJKm4np4etm7dSk9PT+lSJEnaw7jCeGZ2tLAsmO7iJWk0fX199Pb2kpn09vbaOy5JajvewClpzurp6aF6HAEMDAzYOy5JajuGcUlz1saNG+nv7wegv7+fDRs2FK5IkqThDOOS5qxVq1bR2dkJQGdnJ6tXry5ckSRJwxnGJc1ZjUaDiOp+8o6ODhqNRuGKJEkazjAuac7q6uqiu7ubiKC7u5tFixaVLkmSpGEWli5AkqZTo9Fg+/bt9opLktqSPeOS5rSuri7Wr19vr7jazg9+8AO2bNnCG97whtKlSCrIMC5JUgF33XUXAN/97ncLVyKpJMO4JEkz7HWve92wdXvHpfnLMeOS5rS+vj5OPfVU1q5d61AV3WvdunVs27at2Pm3bt06bP273/0ua9asKVQNLFu2rOj5pfnMnnFJc1pPTw9bt2716ZuSpLZkz7ikOauvr4/e3l4yk97eXhqNhr3jAijeC7xixYo92tatW1egEkml2TMuac7q6ekhMwEYGBiwd1yS1HYM45LmrI0bN9Lf3w9Af38/GzZsKFyRJEnDGcYlzVmrVq0iIgCICFavXl24IkmShjOMS5qzjj/++HuHqWQmJ5xwQuGKJEkazjAuac4699xzh/WMn3POOYUrkiRpOMO4pDlr48aNw3rGHTMuSWo3hnFJc9aqVavo7OwEoLOz0zHjkqS2YxiXNGc1Go17/xwRw9YlSWoHhnFJc1ZXVxcHH3wwAA9/+MN94I8kqe0YxiXNWX19ffzkJz8B4Cc/+Qm7du0qXJEkScMZxiXNWT09PQwMDAA+gVOS1J4M45LmrI0bN7J7924Adu/e7WwqkqS2YxiXNGc9/elPH7a+YsWKQpVIkjQyw7gkSZJUiGFc0px1ySWXDFvftGlToUokSRqZYVzSnOUwFUlSuzOMS5IkSYUYxiXNWQ5TkSS1u+JhPCIOiYivRsTPI+IXEfG1iDi0heOcEhEZEd+ejjolzT6rVq1i4cKFACxcuJDVq1cXrkiSpOGKhvGI2A+4EHg00ABeBhwBfCsi7j+B4xwOvB342XTUKWl2ajQadHRU/8wtWLCARqNRuCJJkoYr3TP+auBw4HmZ+R+ZeTZwAnAYcNIEjnMG8AXg+1NfoqTZqquri+7ubiKC7u5uFi1aVLokSZKGKR3GTwAuz8xtgw2ZeT1wKfDc8RwgIl4CPBF427RUKGlWazQaHHXUUfaKS5LaUukw/ljgeyO0XwMcOdbOEfFg4MPAWzLz1imuTdIc0NXVxfr16+0VlyS1pdJh/EDgthHabwUePI79PwD8EPjseE8YESdGxOaI2Lxz587x7iZJkiRNudJhvGUR8XTg5cBfZmaOd7/MPCszl2fm8sWLF09fgZIkSdIYFhY+/22M3AO+tx7zoc4EPgXsiIgD6raFwIJ6/a7MvHvKKpUkSZKmWOkwfg3VuPFmRwLXjrHvY+rlNSO8dxvw18BHJlWdJEmSNI1Kh/FzgA9GxOGZ+b8AEbEUeCpwyhj7HjNC20eABcDJwLYR3pckSZLaRlN9LgIAABp4SURBVOkw/kngdcDZEfF2IIHTgJuohqEAEBGHAT8C3p2Z7wbIzIuaDxYRtwMLR3pPkiRJajdFb+DMzF8Bx1LNiPI5qgf3XA8cm5l3DNk0qHq8Z+0Np5IkSVKz4uE2M2/MzBdk5gMz8wGZ+bzM3N60zfbMjMxcO8axVmbm06azXkmzS19fHyeffDK7du0qXYokSXsoHsYlaTr19PSwdetWenp6SpciSdIeDOOS5qy+vj56e3vJTHp7e+0dlyS1HcO4pDmrp6eHwWeCDQwM2DsuSWo7hnFJc9bGjRvp7+8HoL+/nw0bNhSuSJKk4QzjkuasVatW0dnZCUBnZyerV68uXJEkScMZxiXNWY1Gg4gAoKOjg0ajUbgiSZKGM4xLmrO6urro7u4mIuju7mbRokWlS5IkaZjST+CUpGnVaDTYvn27veKSpLZkGJc0p3V1dbF+/frSZUiSNCKHqUiSJEmFGMYlSZKkQgzjkiRJUiGGcUmSJKkQw7gkSZJUiGFckiRJKsQwLkmSJBViGJckSZIKMYxLkiRJhRjGJUmSpEIM45LmtL6+Pk4++WR27dpVuhRJkvZgGJc0p/X09LB161Z6enpKlyJJ0h4M45LmrL6+Pnp7e8lMent77R2XJLUdw7ikOaunp4fMBGBgYMDecUlS2zGMS5qzNm7cSH9/PwD9/f1s2LChcEWSJA1nGJc0Z61atYrOzk4AOjs7Wb16deGKJEkazjAuac5qNBpEBAAdHR00Go3CFUmSNJxhXNKc1dXVRXd3NxFBd3c3ixYtKl2SJEnDLCxdgCRNp0ajwfbt2+0VlyS1JcO4pDmtq6uL9evXly5DkqQROUxFkiRJKsQwLkmSJBViGJckSZIKMYxLkiRJhRjGJUmSpEIM45IkSVIhhnFJkiSpEMO4JEmSVIhhXJIkSSqkeBiPiEMi4qsR8fOI+EVEfC0iDh3Hfssj4qyI+J+IuDMiboyIL0TEI2aibkmSJGmyiobxiNgPuBB4NNAAXgYcAXwrIu4/xu5/CjwWWAd0A6cATwQ2R8Qh01a0JEmSNEUWFj7/q4HDgd/OzG0AEbEVuA44CfjQKPu+PzN3Dm2IiEuB6+vjvnNaKpYkSZKmSOlhKicAlw8GcYDMvB64FHjuaDs2B/G67QZgJ3DwFNcpSZIkTbnSYfyxwPdGaL8GOHKiB4uIxwAPAb4/ybokSZKkaVc6jB8I3DZC+63AgydyoIhYCHyCqmf8U6Nsd2JEbI6IzTt37tG5LkmSJM2Y0mF8Kn0M+APgpZk5UsAHIDPPyszlmbl88eLFM1edJEmS1KT0DZy3MXIP+N56zEcUEe8DTgQamblhimqTJEmSplXpMH4N1bjxZkcC147nABHxd8BbgZMz83NTWJskSZI0rUoPUzkHODoiDh9siIilwFPr90YVEWuA04G/y8yPTVONkiRJ0rQoHcY/CWwHzo6I50bECcDZwE3AmYMbRcRhEbE7It45pO1PgY8A/wlcGBFHD1kmPBOLJEmSNNOKDlPJzF9FxLHAh4HPAQFcALwhM+8YsmkACxj+y8Oz6/Zn18tQFwMrp6lsSZIkaUqUHjNOZt4IvGCMbbZTBe+hba8AXjFddUmSJM1XN998MzfffDNf+tKX+LM/+7PS5cxppYepSJIkqc3cfPPNAJxxxhmFK5n7DOOSJEm61+c+N3xyui996UuFKpkfig9TkSRJUmXdunVs27ataA1btmwZtn7GGWdw2WWXFall2bJlrFmzpsi5Z4o945IkSVIh9oxLkiS1iXboBV6xYsUebevWrStQyfxgz7gkSZJUiGFckqQZtnDhwlHXJc0fhnFJkmbY7t27R12XNH8YxiVJkqRCDOOSJElSIYZxSZIkqRDDuCRJklSIYVySJEkqxDAuSZIkFWIYlyRphi1YsGDUdUnzh2FckqQZ5kN/JA0yjEuSNMNWrlw5bP2YY44pU4ik4gzjkiRJUiGGcUmSZtimTZuGrV988cWFKpFUmmFckqQZdtBBB426Lmn+MIxLkjTDbrnlllHXJc0fhnFJkmbYihUrhq0/4xnPKFSJpNIM45IkSVIhhnFJkmbYJZdcMmy9+YZOSfOHYVySpBm2atWqex/0s3DhQlavXl24IkmlGMYlSZphjUaDjo7qR/CCBQtoNBqFK5JUimFckqQZ1tXVRXd3NxFBd3c3ixYtKl2SpEIM45IkFXD88cez3377ccIJJ5QuRRpmwYIFo65rahnGJUkq4Nxzz+XOO+/knHPOKV2KNMw999wz6rqmlmFckqQZ1tfXR29vL5lJb28vu3btKl2SpEIM45IkzbCenh4yE4CBgQF6enoKVySpFMO4JEkzbOPGjfT39wPQ39/Phg0bClckqRTDuCRJM2zVqlV0dnYC0NnZ6Tzj0jxmGJckaYY1Gg0iAoCOjg7nGZfmMcO4JEkzzHnGJQ1aWLoASZLmo0ajwfbt2+0Vl+Y5w7gkSQV0dXWxfv360mVIKsxhKpIkSVIhxcN4RBwSEV+NiJ9HxC8i4msRceg49903Ij4QET+NiLsi4rKIWDHdNUuSJM1VgzcX721dU6toGI+I/YALgUcDDeBlwBHAtyLi/uM4xKeAVwPvBP4I+CnwzYh4wvRULEmSNLc1T7V53HHHFapkfijdM/5q4HDgeZn5H5l5NnACcBhw0mg7RsTjgZcAf52Zn8zMC4AXAzcC757esiVJkuamk046adR1Ta3SYfwE4PLM3DbYkJnXA5cCzx3Hvv3Avw7ZdzfwZeC4iNhn6suVJEma27q6uu7tDX/2s5/t1JvTrHQYfyzwvRHarwGOHMe+12fmnSPsez9g2eTLkyRJmn9OOukkHv/4x9srPgNKT214IHDbCO23Ag+exL6D7+8hIk4ETgQ49NBx3ScqSZI0rzj15swp3TM+4zLzrMxcnpnLFy9eXLocSZIkzWOlw/htjNwDvrde7/HuC/f1kEuSJEltqXQYv4Zq7HezI4Frx7HvI+rpEZv3/Q2wbc9dJEmSpPZROoyfAxwdEYcPNkTEUuCp9XujORfoBF40ZN+FwJ8AGzLz7qkuVpIkSZpKpcP4J4HtwNkR8dyIOAE4G7gJOHNwo4g4LCJ2R8Q7B9sy8yqqaQ0/EhGviohnUk1r+AjgXTP4GSRJkqSWFA3jmfkr4Fjgh8DngC8A1wPHZuYdQzYNYAF71vt/gM8ApwPfAA4Bnp2Z353m0iVJkqRJKz21IZl5I/CCMbbZThXIm9vvAt5YL5IkSdKsUnqYiiRJkjRvGcYlSZKkQgzjkiRJUiGGcUmSJKmQyMzSNRQTETuBG0rXMUd0AX2li5D2wutT7cprU+3M63PqHJaZi0d6Y16HcU2diNicmctL1yGNxOtT7cprU+3M63NmOExFkiRJKsQwLkmSJBViGNdUOat0AdIovD7Vrrw21c68PmeAY8YlSZKkQuwZlyRJkgoxjEuSJEmFGMbnkYhYGRE5wnL7CNseHRH/GRG3R8SvIuK/I+JPWzjn2r2c8z/2sv3LI+I7EXFnfe5vR8TjWvm8mj0i4oUR8X8j4oaIuCsifhAR742IBzRt99iI+FpE/KS+Lq+JiDdFxMIWzvnqiDgvIn5cH+t7EfHmiLjfCNs+NSI2RMTPIuKXEfHdiPiLyXxmzR4RsSQi1kfEZfW/TRkRS8fY55R6u2+3eM7fiYgzI+LKiPhNRIxrTGlEfKI+7+dbOa/mpoi4aC8/izMi/nMCx1k6ynGylZwgmPAPMM0Ja4DvDFnfPfTNiPhD4N+BLwIvAX4DHAnsO4lzPg24Z8j6rc0bRMR7gDcA/wC8BdgPeHL9qrntTcCNwN8CO4DfBdYCx0TEH2TmQEQ8HLgI+DHVddIHPJPqelkMvHWC53wnsBH4NLCL6ho9jeqae9HgRhFxFHA+cDnwauBO4IXApyJin8w8Y+IfV7PMMuDFwJXAJcDq0TaOiMOBtwM/m8Q5nwQ8B9gM3A08ZawdIuKpwEuBX0zivJqbXgs8sKntKcCHgHMmcJyfMvK1eDrVv6HfbKm6ec4bOOeRiFgJfAtYlZnn72WbBwA/Ar6YmW+YgnOuBd4FdGbm7lG2ewpwKfDHmTlir7nmrohYnJk7m9peDvQAz8zMCyPiROBM4Lcz84dDtvsy8IzMfNgUnPOdwKnAIzPzf+u291D9snBgZt4xZNvLADJzzJCk2S0iOjJzoP7zq4BPAo/IzO172f6bwHbgt4GFmfm0SZ7zdODvMjNG2b4TuAr4AnAS8O3MfOlEz6v5IyI+RfXL28Myc48OsgkcZz/gZuCbmfmisbbXnhymomYvoupl/MfRNoqIz9XDSA4b0vbwiNgZEV9p4bx/CVxvEJ+fmkNxbfDbm4Pr18HhI829frcz5N+yiDit/lr/94a03b8e+nLZ4JCWcZ5z8Lz9wF1N2/4c/w2dFwZD8XhExEuAJwJv28v7470+x33O2puBBcAHJ7if5qE6QL8IODczb42Ijnooy/aIeNCQ7R5XDx38wCiH+2PgAVSdJ2qBP0jmpy9ExD0RsSsivhgRhw5572lUQ0geF9U48d0RcVNEvCsiFgzZ7rVUX+1/ISIWREQH8Dmqr/BfPcI5b6rPeUNEvD8ifqvp/acBV0fEW+oxvLvrMbz+lj1/PaN+/X79+hWqoSkfi4hHRMQDI+L5wMsY/svjqVRf7X8xIvav2z4OPBR4yWjf0NTnHAB+OKTts/XruvoXzgMi4tVUQ2Q+3NpH01wUEQ+muibeMkpP42Suz72ddxnVsJjXZmb/xCvXPPR8hgTo+pe/l9ZtZwLUP6e/DFwD/N0ox2pQDcka99hzDeeY8fnl51Sh5WKq3sXfpRqje1lE/G5m/gx4ONUY7S9SjZ+9EngW8A7gAOCvATLzlxHxZ1RDS95JNabxGcDKzBx6Q+g24BSqr0+TaqzlX1P1HK0ast3Dga66pjcDO4ETgX+LiOdl5tlT+jehthYRBwPvBs7PzM0AmXlLPZzpbOB/600TWJuZ/zC4b2burnsntwAfr29OalAFnetHOedRwOuBT2fmLUOO9716iNe/U/0SClVP+Wsy88tT8oE1V3yA6he5z+5tg1avzzGcAXwtM7/V4v6af15OFaB7Bxsyc0c9DOtr9VCrpwCHAk/MzN+MdJD63+pjgY+28oukapnpMo8XqlC8Gzi9Xt9AFXDe2LTdGVQ3cj6oqf2Uev9+4NRxnvP19TmeNaTtN3XbE4e0dQDfA/6r9N+Ty8wtwP5UPYc/AZYMaV8MbAWuAF4ArKQK7L8B3jrCcf60vqZ+DfSMcc6HAdfX19sDmt47gurm0m8Cf0TVI76uvub/vPTfl8vMLsCr6utqaVP70+tr8XeGtF1ENXZ7pONM5Po8vfpxPeJ7L6UaqnXQkLbtwOdL/125tOdC1fl1D/Chvbz/ifq6TOD/jHGsU+rtjir9uWbz4jCVeS4zv0vVkzM4fnFX/bqxadMNQCfw2Kb2L1L9j5hUX7WOx5fq198b0rYLuLWuZ7C2AeAC4AnjPK5mufpr0XOBw4HjMnPHkLffAiyt2/9vZl6Ume+k6o08LSK6mg73Darrah9GGU4SEYuorveoj/3Lpk3eQxW8/ygzv56ZF2TmGuDfgI/WQ7SkM4FPATvqoUwHUH37vKBe36dp+3Fdn6Oph7l8CHg/cPeQ83YAnfV6Z4ufR3PXS6mukb2N8e6hui5/RvUzfjQvB7Zk5tapK2/+8YeIBg1Oq3PNGNvde1NRHUJ6qKai+wXVb9OtnHOs8zrlzzxQh4avAsuB52Tmfzdt8jhgW2be1tR+BdUvisua2j9OdUPbj4AzY4S5yCPigVQ93ouovqn58QilPQ64Ovcci3tFvd9DxvpsmhceA7wGuG3I8lTg6PrPf9m0/ZjX5zh0UX1j9J6m8x5CNRXjbcAftnBczW0Nqn/Trm5+o76x89NU3xI+CHjf3g5S34T8GLxxc9IM4/NcRCynmn7rirppcDaT45o2fTbV11bfG9L2NqobL18C/AXw/Ig4aRyn/fP69Yohbf8OHFjXM1hbB9W48qFzomsOqv9bf4Fq7OHzMvPyETa7GVhW3yQ31O/Xr/cG6XpM7suo7jv4E6p7EU5rOud+VL2TjwBWZ+a2vZR3M/CE2PNhQL9P9f9Ey1OCaU45ZoTlaqp/M4+h+kUTGN/1OU437+W8t1DNjX8M0NJDhzQ31T9jj2TvAfqjVLNJPZfq28jXR0RzHhjUoBqmOlbvucZSepyMy8wtVGHndKppiI4F/oZqdoobga4h232GalaUt1DdvPk+qvFla4ds8/tUX92/fUjbx4FfAY8Z0nYV1Q2bzwG6qb5S7Qd6m2rbF7gWuIHqh1Q3VUC/h2qe6eJ/fy7Tem2eQfUNyOlUPYlDlyX1NkfX1853qHr9nkkVYH5DdfPa4LEeQXWz8j8PaXtzfS0dM6Stl+qbnpNHOOfiIdu9sK7tm1Q/oFYDH6vbRhxz6TL3lvo6eOGQa/Uv6/VnjLLPRTSNGZ/A9bnfkHN+tT7n4PryMWrdjmPGXUZYuO9+l4eM8N4L6uvspUPavk71S99Dmra9X50fzin9mebCUrwAlxn8j131ZG+tfxD0AzcBZ1FN+D90u/vVoeimOuj8EHj9kPcfSPXV6sVAx5D2fYH/pgrg+9RtX663vZOqF/FaqplZ9hmhvocBn6fqafw1cBlVj2XxvzuXab82t3PfvQfNy9oh2x0NnEf1FLhfUQ1vejvwW/X7C+vr5gfA/YfsF1T3PewAFtVteztfAq9oqq+7DlY7gV9SzYTxWmBB6b87lxm7Rvd2rVw0yj4XMSSMT/D6XDrKOT87Rq3bMYy7NC1Uw/l2Us0t3vzeIfXP3s83tS+u/709j/pBkXX78+tr8QWlP9dcWHwCpyRJklSIY8YlSZKkQgzjkiRJUiGGcUmSJKkQw7gkSZJUiGFckiRJKsQwLkmSJBViGJckjSoilkZERsRnZ+BcGREXTfd5JKldGMYlaQbUIXPock9E3BoRF0XEKyIiStc4EyJie0RsL12HJLWLhaULkKR55tT6tRNYRvUku2cAy4HXlSqqjTyG6om9kjQv+AROSZoBEVE9Uz0zmtqfCmyieiT6IzPz+gLljSoilgLXAz2Z+YpJHms7QGYunWRZkjQnOExFkgrKzEuB/6EK409qfj8ifj8ivhoRN0fEbyLipog4MyIePsK2h0fEWRGxLSLuqofB/HdEfCIiFjVtu09EnFK/f2dE/CIiLomIF4+39nqIzYg9OvXQm4yIV9TrK+ttDwMOaxqy89kh+404ZjwiHhQR742IH0TEryPitoj4ZkQ8a4RtV9bHWRsRT4iIb0TE7fXnvDgi/mC8n1GSppvDVCSpffQPXYmIvwDOAu4GzgFuAo4AXgUcHxFHZ+aN9bYPA74DPBA4D/i/wL7AI4CXAR8DdtXb3g/4JtXwmP8BPg7sB7wQ+NeIeEJm/u0Uf7btVEN03lCvf2TIe1tG2zEiDgAuBY6k+owfAbqAFwMbIuIvM/PMEXZdDrwFuAz4Z+BQ4AXABfVn/EHLn0aSpohhXJIKiogVwKOB3wBXDGl/FPAJqhD7jMz88ZD3nglsAD5KNeYcqiB9IPCGzPxo0znuDwwMafobqiDeC5yQmbvr7U6ta3hbRHw9M//fVH3OzNwOrB3sKc/MtRPY/f1UQfws4DVZj6+MiPcDm4F1EfHN+hxD/SHwfzLzs4MNEXES1d/r64HXtvBRJGlKOUxFkmZQPXRibUT8fUT8K3A+1RCVN2XmT4ds+pdUN3m+fmgQB8jMC6h6yo+PiAc0neKu5nNm5q8yc2j7XwAJvHEwiNfb/Qw4rV59VWufcGrVvfgvBe4A3pZDbnTKzOuAdcD9gJePsPulQ4N47dPAbuDJ01KwJE2QPeOSNLPe1bSewCsz8zNN7U+pX58REb83wnEeAiwAHgVcSRXO3wN8PCKOoxqGcilw7dAAW4f3ZcCPM/N/RjjuhfXr747/I02r36YaQnNpZt46wvsXAm9n5Ho3NzdkZn9E3AI8eEqrlKQWGcYlaQYNzqZSDx15CvAp4BMRcUNmXjhk08EbLt88xiH3r497Q0Q8GVgLPBv44/r9myLig5m5rl5/UP36U0Y22H7AOD7OTJhMvbfvZZ/dVL/ISFJxDlORpALqoSPnA8dTBcOeiNhvyCY/r18flJkxynLxkGN+PzP/hCrILwdOofp3/qMR8cqm4z50L6U9rGm70QwARMRIHTtTFeansl5JajuGcUkqKDO3Ap8ElgB/PeSty+vXp7dwzN2ZeWVmvh/4s7r5efV7vwR+BBwcEUeMsPsx9et3x3Gq2+rXQ0Z4b/le9rmHifVK/4DqIUCPr2dVaTaReiWp7RjGJam806mmL3xTRAyOZf4Y1VSHH65nVhkmIu4XEU8fsv6kiHhQ83bAQfXr0KdafprqptEPRMS9wTgiuoB3DNlmLIOzv7y6qbZnct8vAc12AYsj4rfGcXwy8zfAF4AHcN/NpYPneSSwhurv6XPjOZ4ktRvHjEtSYZn544gYnG7vLVSzhvxPPc/4p4FrIuI/gR9SzbByKFWP+U6qaRGhmkv8pIj4NlXP923AI6mGwdzN8Hm9Pwh0A88Fro6I86huknwR1Y2h/5CZ3x5H6Z+hGtP+toh4PHAt1Q2l3cC/U83p3ewC4PeA/4yITXVtV2fmuaOc55T6876uvpn1W9w3z/gDgNe145NLJWk8DOOS1B7eS9XDvCYiPpKZt2T+//buGCeoIAqg6J3KsCQ1wVBA5wbcgw0dPR1hFyRo4gpcgrEhYQtYWNDR0IzFaEKBCZVjcU79fzLlLd68mVdjjJvWXvCj6qR6qH5UX6rPT/6/rl5Vb1oveR5Ud9Wn6nLOefvnwznn4xjjuDqtPlQfW5cab1p7yq9fcuA5588xxrvqojps7S7/Xh23Hht6LsbPW/Pk76u3/Z6Xr/4a43PO+zHG6+qsdTH1tLXC8Vt1Mef8+pLzAvyPxpONVwAAwD9kZhwAADYR4wAAsIkYBwCATcQ4AABsIsYBAGATMQ4AAJuIcQAA2ESMAwDAJmIcAAA2+QVp9ye2FO10rgAAAABJRU5ErkJggg==\n", 276 | "text/plain": [ 277 | "
" 278 | ] 279 | }, 280 | "metadata": { 281 | "tags": [], 282 | "needs_background": "light" 283 | } 284 | } 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "metadata": { 290 | "id": "8G61e7Qi7nju" 291 | }, 292 | "source": [ 293 | "fig.savefig(\"plot.pdf\", bbox_inches='tight')" 294 | ], 295 | "execution_count": 11, 296 | "outputs": [] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "metadata": { 301 | "id": "tN6L0WvP5InD" 302 | }, 303 | "source": [ 304 | "" 305 | ], 306 | "execution_count": 11, 307 | "outputs": [] 308 | } 309 | ] 310 | } --------------------------------------------------------------------------------