├── dataset ├── __init__.py ├── __pycache__ │ ├── CamVid.cpython-36.pyc │ └── __init__.cpython-36.pyc └── CamVid.py ├── detection_models ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-311.pyc │ ├── networks.cpython-311.pyc │ └── antialiasing.cpython-311.pyc ├── sync_batchnorm │ ├── __pycache__ │ │ ├── comm.cpython-311.pyc │ │ ├── __init__.cpython-311.pyc │ │ ├── batchnorm.cpython-311.pyc │ │ └── replicate.cpython-311.pyc │ ├── __init__.py │ ├── unittest.py │ ├── batchnorm_reimpl.py │ ├── replicate.py │ ├── comm.py │ └── batchnorm.py ├── antialiasing.py └── networks.py ├── demo.png ├── test.png ├── test_label.png ├── examples ├── eg1.jpg ├── eg2.jpg ├── obj1.jpg ├── obj2.jpg ├── eg2gif.gif └── objgif.gif ├── tfboard_loss.jpg ├── tfboard_miou.jpg ├── tfboard_precision.jpg ├── input_images └── input_image.png ├── output_masks ├── input │ └── input_image.png └── mask │ └── input_image.png ├── __pycache__ └── scratch_detection.cpython-311.pyc ├── detection_util ├── __pycache__ │ └── util.cpython-311.pyc └── util.py ├── .idea ├── vcs.xml ├── modules.xml ├── misc.xml ├── bisenet.iml ├── deployment.xml └── workspace.xml ├── src ├── __pycache__ │ └── pipeline_stable_diffusion_controlnet_inpaint.cpython-311.pyc └── pipeline_stable_diffusion_controlnet_inpaint.py ├── loss.py ├── README.md ├── demo.py ├── obrem.py ├── resnet.py ├── eval.py ├── rest.py ├── scratch_detection.py ├── train.py ├── utils.py ├── model.py └── LICENSE.md /dataset/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /detection_models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/demo.png -------------------------------------------------------------------------------- /test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/test.png -------------------------------------------------------------------------------- /test_label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/test_label.png -------------------------------------------------------------------------------- /examples/eg1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/examples/eg1.jpg -------------------------------------------------------------------------------- /examples/eg2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/examples/eg2.jpg -------------------------------------------------------------------------------- /examples/obj1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/examples/obj1.jpg -------------------------------------------------------------------------------- /examples/obj2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/examples/obj2.jpg -------------------------------------------------------------------------------- /tfboard_loss.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/tfboard_loss.jpg -------------------------------------------------------------------------------- /tfboard_miou.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/tfboard_miou.jpg -------------------------------------------------------------------------------- /examples/eg2gif.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/examples/eg2gif.gif -------------------------------------------------------------------------------- /examples/objgif.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/examples/objgif.gif -------------------------------------------------------------------------------- /tfboard_precision.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/tfboard_precision.jpg -------------------------------------------------------------------------------- /input_images/input_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/input_images/input_image.png -------------------------------------------------------------------------------- /output_masks/input/input_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/output_masks/input/input_image.png -------------------------------------------------------------------------------- /output_masks/mask/input_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/output_masks/mask/input_image.png -------------------------------------------------------------------------------- /dataset/__pycache__/CamVid.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/dataset/__pycache__/CamVid.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/scratch_detection.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/__pycache__/scratch_detection.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/dataset/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /detection_util/__pycache__/util.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_util/__pycache__/util.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/__pycache__/networks.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/__pycache__/networks.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/__pycache__/antialiasing.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/__pycache__/antialiasing.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/__pycache__/comm.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/sync_batchnorm/__pycache__/comm.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/sync_batchnorm/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/__pycache__/batchnorm.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/sync_batchnorm/__pycache__/batchnorm.cpython-311.pyc -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/__pycache__/replicate.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/detection_models/sync_batchnorm/__pycache__/replicate.cpython-311.pyc -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/__pycache__/pipeline_stable_diffusion_controlnet_inpaint.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/HEAD/src/__pycache__/pipeline_stable_diffusion_controlnet_inpaint.cpython-311.pyc -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/bisenet.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : __init__.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | from .batchnorm import set_sbn_eps_mode 12 | from .batchnorm import SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d 13 | from .batchnorm import patch_sync_batchnorm, convert_model 14 | from .replicate import DataParallelWithCallback, patch_replication_callback 15 | -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : unittest.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import unittest 12 | import torch 13 | 14 | 15 | class TorchTestCase(unittest.TestCase): 16 | def assertTensorClose(self, x, y): 17 | adiff = float((x - y).abs().max()) 18 | if (y == 0).all(): 19 | rdiff = 'NaN' 20 | else: 21 | rdiff = float((adiff / y).abs().max()) 22 | 23 | message = ( 24 | 'Tensor close check failed\n' 25 | 'adiff={}\n' 26 | 'rdiff={}\n' 27 | ).format(adiff, rdiff) 28 | self.assertTrue(torch.allclose(x, y, atol=1e-5, rtol=1e-3), message) 29 | 30 | -------------------------------------------------------------------------------- /loss.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | def flatten(tensor): 6 | """Flattens a given tensor such that the channel axis is first. 7 | The shapes are transformed as follows: 8 | (N, C, D, H, W) -> (C, N * D * H * W) 9 | """ 10 | C = tensor.size(1) 11 | # new axis order 12 | axis_order = (1, 0) + tuple(range(2, tensor.dim())) 13 | # Transpose: (N, C, D, H, W) -> (C, N, D, H, W) 14 | transposed = tensor.permute(axis_order) 15 | # Flatten: (C, N, D, H, W) -> (C, N * D * H * W) 16 | return transposed.contiguous().view(C, -1) 17 | 18 | 19 | class DiceLoss(nn.Module): 20 | def __init__(self): 21 | super().__init__() 22 | self.epsilon = 1e-5 23 | 24 | def forward(self, output, target): 25 | assert output.size() == target.size(), "'input' and 'target' must have the same shape" 26 | output = F.softmax(output, dim=1) 27 | output = flatten(output) 28 | target = flatten(target) 29 | # intersect = (output * target).sum(-1).sum() + self.epsilon 30 | # denominator = ((output + target).sum(-1)).sum() + self.epsilon 31 | 32 | intersect = (output * target).sum(-1) 33 | denominator = (output + target).sum(-1) 34 | dice = intersect / denominator 35 | dice = torch.mean(dice) 36 | return 1 - dice 37 | # return 1 - 2. * intersect / denominator 38 | -------------------------------------------------------------------------------- /.idea/deployment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 43 | -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/batchnorm_reimpl.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # File : batchnorm_reimpl.py 4 | # Author : acgtyrant 5 | # Date : 11/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.init as init 14 | 15 | __all__ = ['BatchNorm2dReimpl'] 16 | 17 | 18 | class BatchNorm2dReimpl(nn.Module): 19 | """ 20 | A re-implementation of batch normalization, used for testing the numerical 21 | stability. 22 | 23 | Author: acgtyrant 24 | See also: 25 | https://github.com/vacancy/Synchronized-BatchNorm-PyTorch/issues/14 26 | """ 27 | def __init__(self, num_features, eps=1e-5, momentum=0.1): 28 | super().__init__() 29 | 30 | self.num_features = num_features 31 | self.eps = eps 32 | self.momentum = momentum 33 | self.weight = nn.Parameter(torch.empty(num_features)) 34 | self.bias = nn.Parameter(torch.empty(num_features)) 35 | self.register_buffer('running_mean', torch.zeros(num_features)) 36 | self.register_buffer('running_var', torch.ones(num_features)) 37 | self.reset_parameters() 38 | 39 | def reset_running_stats(self): 40 | self.running_mean.zero_() 41 | self.running_var.fill_(1) 42 | 43 | def reset_parameters(self): 44 | self.reset_running_stats() 45 | init.uniform_(self.weight) 46 | init.zeros_(self.bias) 47 | 48 | def forward(self, input_): 49 | batchsize, channels, height, width = input_.size() 50 | numel = batchsize * height * width 51 | input_ = input_.permute(1, 0, 2, 3).contiguous().view(channels, numel) 52 | sum_ = input_.sum(1) 53 | sum_of_square = input_.pow(2).sum(1) 54 | mean = sum_ / numel 55 | sumvar = sum_of_square - sum_ * mean 56 | 57 | self.running_mean = ( 58 | (1 - self.momentum) * self.running_mean 59 | + self.momentum * mean.detach() 60 | ) 61 | unbias_var = sumvar / (numel - 1) 62 | self.running_var = ( 63 | (1 - self.momentum) * self.running_var 64 | + self.momentum * unbias_var.detach() 65 | ) 66 | 67 | bias_var = sumvar / numel 68 | inv_std = 1 / (bias_var + self.eps).pow(0.5) 69 | output = ( 70 | (input_ - mean.unsqueeze(1)) * inv_std.unsqueeze(1) * 71 | self.weight.unsqueeze(1) + self.bias.unsqueeze(1)) 72 | 73 | return output.view(channels, batchsize, height, width).permute(1, 0, 2, 3).contiguous() 74 | 75 | -------------------------------------------------------------------------------- /detection_models/antialiasing.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import torch 5 | import torch.nn.parallel 6 | import numpy as np 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Downsample(nn.Module): 12 | # https://github.com/adobe/antialiased-cnns 13 | 14 | def __init__(self, pad_type="reflect", filt_size=3, stride=2, channels=None, pad_off=0): 15 | super(Downsample, self).__init__() 16 | self.filt_size = filt_size 17 | self.pad_off = pad_off 18 | self.pad_sizes = [ 19 | int(1.0 * (filt_size - 1) / 2), 20 | int(np.ceil(1.0 * (filt_size - 1) / 2)), 21 | int(1.0 * (filt_size - 1) / 2), 22 | int(np.ceil(1.0 * (filt_size - 1) / 2)), 23 | ] 24 | self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes] 25 | self.stride = stride 26 | self.off = int((self.stride - 1) / 2.0) 27 | self.channels = channels 28 | 29 | # print('Filter size [%i]'%filt_size) 30 | if self.filt_size == 1: 31 | a = np.array([1.0,]) 32 | elif self.filt_size == 2: 33 | a = np.array([1.0, 1.0]) 34 | elif self.filt_size == 3: 35 | a = np.array([1.0, 2.0, 1.0]) 36 | elif self.filt_size == 4: 37 | a = np.array([1.0, 3.0, 3.0, 1.0]) 38 | elif self.filt_size == 5: 39 | a = np.array([1.0, 4.0, 6.0, 4.0, 1.0]) 40 | elif self.filt_size == 6: 41 | a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0]) 42 | elif self.filt_size == 7: 43 | a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0]) 44 | 45 | filt = torch.Tensor(a[:, None] * a[None, :]) 46 | filt = filt / torch.sum(filt) 47 | self.register_buffer("filt", filt[None, None, :, :].repeat((self.channels, 1, 1, 1))) 48 | 49 | self.pad = get_pad_layer(pad_type)(self.pad_sizes) 50 | 51 | def forward(self, inp): 52 | if self.filt_size == 1: 53 | if self.pad_off == 0: 54 | return inp[:, :, :: self.stride, :: self.stride] 55 | else: 56 | return self.pad(inp)[:, :, :: self.stride, :: self.stride] 57 | else: 58 | return F.conv2d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1]) 59 | 60 | 61 | def get_pad_layer(pad_type): 62 | if pad_type in ["refl", "reflect"]: 63 | PadLayer = nn.ReflectionPad2d 64 | elif pad_type in ["repl", "replicate"]: 65 | PadLayer = nn.ReplicationPad2d 66 | elif pad_type == "zero": 67 | PadLayer = nn.ZeroPad2d 68 | else: 69 | print("Pad type [%s] not recognized" % pad_type) 70 | return PadLayer 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UnpromptedControl 2 | 3 | **By sponsoring me, you're not just supporting my work - you're helping to create a more collaborative, innovative open source community 💖 [sponsor](https://github.com/sponsors/vijishmadhavan?o=sd&sc=t).** 4 | 5 | [Get more updates on Twitter](https://twitter.com/Vijish68859437) 6 | 7 | ControlNet is a highly regarded tool for guiding StableDiffusion models, and it has been widely acknowledged for its effectiveness. In this repository, A simple hack that allows for the restoration or removal of objects without requiring user prompts. By leveraging this approach, the workflow can be significantly streamlined, leading to enhanced process efficiency. 8 | 9 | ## No-prompt 10 | 11 | [](https://colab.research.google.com/github/vijishmadhavan/UnpromptedControl/blob/master/UnpromptedControl.ipynb) 12 | 13 | ![restore Result](examples/eg2gif.gif) 14 | ![restore Result](examples/objgif.gif) 15 | ## Image Restoration 16 | 17 | In this image restoration is accomplished using the controlnet-canny and stable-diffusion-2-inpainting techniques, with only "" blank input prompts. Additionally, for automatic scratch segmentation, the FT_Epoch_latest.pt model is being used. However, if the segmentation output is not satisfactory, it is possible to manually sketch and refine the mask to achieve better results. As ControlNet model is trained on pairs of images, one of which has missing parts, and it learns to predict the missing parts based on the content of the complete image. 18 | 19 | ![restore Result](examples/eg1.jpg) 20 | 21 | ![restore Result](examples/eg2.jpg) 22 | 23 | ## Object Removal 24 | 25 | Automatically removing objects from images is a challenging task that requires a combination of computer vision and deep learning techniques. This code leverages the power of OpenCV inpainting, deep learning-based image restoration, and blending techniques to achieve this task automatically, without the need for user prompts. The ControlNetModel and StableDiffusionInpaintPipeline models play a crucial role in guiding the inpainting process and restoring the image to a more natural-looking state. Overall, this code provides an efficient and effective way to remove unwanted objects from images and produce natural-looking results that are consistent with the surrounding image content. 26 | 27 | **"Surely, it has its limitations and might fail with certain images, especially those of faces, and may require some back and forth. To obtain good results, we need to mask not only the object but also its shadow."** 28 | 29 | 30 | ![restore Result](examples/obj2.jpg) 31 | ![restore Result](examples/obj1.jpg) 32 | 33 | ## Limitation 34 | 35 | - Limited Generalization: The algorithm currently has limitations when it comes to processing images of people's faces and bodies. It may not work as expected for these types of images, and additional work is needed to improve its performance in these areas. 36 | 37 | - When it comes to removing an object from an image, it's important to consider the surrounding environment and any elements that may be affected by the removal process. In some cases, removing an object may require the removal of a large area surrounding the object, including its shadows. 38 | 39 | - To obtain good results, we need to mask not only the object but also its shadow. 40 | 41 | ## Acknowledgements 42 | 43 | https://github.com/microsoft/Bringing-Old-Photos-Back-to-Life (Segmentation) 44 | 45 | https://huggingface.co/thibaud/controlnet-sd21 46 | 47 | https://github.com/lllyasviel/ControlNet 48 | 49 | 50 | -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/replicate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : replicate.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import functools 12 | 13 | from torch.nn.parallel.data_parallel import DataParallel 14 | 15 | __all__ = [ 16 | 'CallbackContext', 17 | 'execute_replication_callbacks', 18 | 'DataParallelWithCallback', 19 | 'patch_replication_callback' 20 | ] 21 | 22 | 23 | class CallbackContext(object): 24 | pass 25 | 26 | 27 | def execute_replication_callbacks(modules): 28 | """ 29 | Execute an replication callback `__data_parallel_replicate__` on each module created by original replication. 30 | 31 | The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` 32 | 33 | Note that, as all modules are isomorphism, we assign each sub-module with a context 34 | (shared among multiple copies of this module on different devices). 35 | Through this context, different copies can share some information. 36 | 37 | We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback 38 | of any slave copies. 39 | """ 40 | master_copy = modules[0] 41 | nr_modules = len(list(master_copy.modules())) 42 | ctxs = [CallbackContext() for _ in range(nr_modules)] 43 | 44 | for i, module in enumerate(modules): 45 | for j, m in enumerate(module.modules()): 46 | if hasattr(m, '__data_parallel_replicate__'): 47 | m.__data_parallel_replicate__(ctxs[j], i) 48 | 49 | 50 | class DataParallelWithCallback(DataParallel): 51 | """ 52 | Data Parallel with a replication callback. 53 | 54 | An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by 55 | original `replicate` function. 56 | The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` 57 | 58 | Examples: 59 | > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) 60 | > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) 61 | # sync_bn.__data_parallel_replicate__ will be invoked. 62 | """ 63 | 64 | def replicate(self, module, device_ids): 65 | modules = super(DataParallelWithCallback, self).replicate(module, device_ids) 66 | execute_replication_callbacks(modules) 67 | return modules 68 | 69 | 70 | def patch_replication_callback(data_parallel): 71 | """ 72 | Monkey-patch an existing `DataParallel` object. Add the replication callback. 73 | Useful when you have customized `DataParallel` implementation. 74 | 75 | Examples: 76 | > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) 77 | > sync_bn = DataParallel(sync_bn, device_ids=[0, 1]) 78 | > patch_replication_callback(sync_bn) 79 | # this is equivalent to 80 | > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) 81 | > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) 82 | """ 83 | 84 | assert isinstance(data_parallel, DataParallel) 85 | 86 | old_replicate = data_parallel.replicate 87 | 88 | @functools.wraps(old_replicate) 89 | def new_replicate(module, device_ids): 90 | modules = old_replicate(module, device_ids) 91 | execute_replication_callbacks(modules) 92 | return modules 93 | 94 | data_parallel.replicate = new_replicate 95 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import argparse 3 | from model.build_BiSeNet import BiSeNet 4 | import os 5 | import torch 6 | import cv2 7 | from imgaug import augmenters as iaa 8 | from PIL import Image 9 | from torchvision import transforms 10 | import numpy as np 11 | from utils import reverse_one_hot, get_label_info, colour_code_segmentation 12 | 13 | def predict_on_image(model, args): 14 | # pre-processing on image 15 | image = cv2.imread(args.data, -1) 16 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 17 | resize = iaa.Scale({'height': args.crop_height, 'width': args.crop_width}) 18 | resize_det = resize.to_deterministic() 19 | image = resize_det.augment_image(image) 20 | image = Image.fromarray(image).convert('RGB') 21 | image = transforms.ToTensor()(image) 22 | image = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image).unsqueeze(0) 23 | # read csv label path 24 | label_info = get_label_info(args.csv_path) 25 | # predict 26 | model.eval() 27 | predict = model(image).squeeze() 28 | predict = reverse_one_hot(predict) 29 | predict = colour_code_segmentation(np.array(predict), label_info) 30 | predict = cv2.resize(np.uint8(predict), (960, 720)) 31 | cv2.imwrite(args.save_path, cv2.cvtColor(np.uint8(predict), cv2.COLOR_RGB2BGR)) 32 | 33 | def main(params): 34 | # basic parameters 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument('--image', action='store_true', default=False, help='predict on image') 37 | parser.add_argument('--video', action='store_true', default=False, help='predict on video') 38 | parser.add_argument('--checkpoint_path', type=str, default=None, help='The path to the pretrained weights of model') 39 | parser.add_argument('--context_path', type=str, default="resnet101", help='The context path model you are using.') 40 | parser.add_argument('--num_classes', type=int, default=12, help='num of object classes (with void)') 41 | parser.add_argument('--data', type=str, default=None, help='Path to image or video for prediction') 42 | parser.add_argument('--crop_height', type=int, default=720, help='Height of cropped/resized input image to network') 43 | parser.add_argument('--crop_width', type=int, default=960, help='Width of cropped/resized input image to network') 44 | parser.add_argument('--cuda', type=str, default='0', help='GPU ids used for training') 45 | parser.add_argument('--use_gpu', type=bool, default=True, help='Whether to user gpu for training') 46 | parser.add_argument('--csv_path', type=str, default=None, required=True, help='Path to label info csv file') 47 | parser.add_argument('--save_path', type=str, default=None, required=True, help='Path to save predict image') 48 | 49 | 50 | args = parser.parse_args(params) 51 | 52 | # build model 53 | os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda 54 | model = BiSeNet(args.num_classes, args.context_path) 55 | if torch.cuda.is_available() and args.use_gpu: 56 | model = torch.nn.DataParallel(model).cuda() 57 | 58 | # load pretrained model if exists 59 | print('load model from %s ...' % args.checkpoint_path) 60 | model.module.load_state_dict(torch.load(args.checkpoint_path)) 61 | print('Done!') 62 | 63 | # predict on image 64 | if args.image: 65 | predict_on_image(model, args) 66 | 67 | # predict on video 68 | if args.video: 69 | pass 70 | 71 | if __name__ == '__main__': 72 | params = [ 73 | '--image', 74 | '--data', 'exp.png', 75 | '--checkpoint_path', '/path/to/ckpt', 76 | '--cuda', '0', 77 | '--csv_path', '/data/sqy/CamVid/class_dict.csv', 78 | '--save_path', 'demo.png', 79 | '--context_path', 'resnet18' 80 | ] 81 | main(params) -------------------------------------------------------------------------------- /obrem.py: -------------------------------------------------------------------------------- 1 | #best object removal model 2 | 3 | import gradio as gr 4 | import numpy as np 5 | import torch 6 | from src.pipeline_stable_diffusion_controlnet_inpaint import * 7 | 8 | from diffusers import StableDiffusionInpaintPipeline, ControlNetModel, DEISMultistepScheduler 9 | from diffusers.utils import load_image 10 | from PIL import Image 11 | import cv2 12 | 13 | 14 | controlnet = ControlNetModel.from_pretrained("thepowefuldeez/sd21-controlnet-canny", torch_dtype=torch.float16) 15 | 16 | pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( 17 | "stabilityai/stable-diffusion-2-inpainting", controlnet=controlnet, torch_dtype=torch.float16 18 | ) 19 | 20 | pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config) 21 | 22 | # speed up diffusion process with faster scheduler and memory optimization 23 | # remove following line if xformers is not installed 24 | pipe.enable_xformers_memory_efficient_attention() 25 | pipe.to('cuda') 26 | 27 | def resize_image(image, target_size): 28 | width, height = image.size 29 | aspect_ratio = float(width) / float(height) 30 | if width > height: 31 | new_width = target_size 32 | new_height = int(target_size / aspect_ratio) 33 | else: 34 | new_width = int(target_size * aspect_ratio) 35 | new_height = target_size 36 | return image.resize((new_width, new_height), Image.BICUBIC) 37 | def predict(input_dict): 38 | # Get the drawn input image and mask 39 | image = input_dict["image"].convert("RGB") 40 | input_image = input_dict["mask"].convert("RGB") 41 | input_image = resize_image(input_image, 768) 42 | image = resize_image(image, 768) 43 | 44 | # Convert images to numpy arrays 45 | image_np = np.array(image) 46 | input_image_np = np.array(input_image) 47 | 48 | # Convert input_image_np to grayscale and normalize to [0, 1] range 49 | mask_np = cv2.cvtColor(input_image_np, cv2.COLOR_RGB2GRAY) / 255.0 50 | 51 | # Apply OpenCV inpainting 52 | inpainted_image_np = cv2.inpaint(image_np, (mask_np * 255).astype(np.uint8), 3, cv2.INPAINT_TELEA) 53 | 54 | # Blend the original image and the inpainted image using the mask 55 | blended_image_np = image_np * (1 - mask_np)[:, :, None] + inpainted_image_np * mask_np[:, :, None] 56 | 57 | # Convert the blended image back to a PIL Image 58 | blended_image = Image.fromarray(np.uint8(blended_image_np)) 59 | 60 | # Process the blended image 61 | blended_image_np = np.array(blended_image) 62 | low_threshold = 800 63 | high_threshold = 900 64 | canny = cv2.Canny(blended_image_np, low_threshold, high_threshold) 65 | canny = canny[:, :, None] 66 | canny = np.concatenate([canny, canny, canny], axis=2) 67 | canny_image = Image.fromarray(canny) 68 | canny_image.save("canny.png") 69 | 70 | 71 | generator = torch.manual_seed(0) 72 | output = pipe( 73 | prompt="", 74 | num_inference_steps=20, 75 | generator=generator, 76 | image=blended_image_np, 77 | control_image=canny_image, 78 | controlnet_conditioning_scale=0.9, 79 | mask_image=input_image 80 | ).images[0] 81 | 82 | return output 83 | image_blocks = gr.Blocks() 84 | 85 | with image_blocks as demo: 86 | with gr.Row(): 87 | with gr.Column(): 88 | # Allow user to draw on the input image 89 | input_image = gr.Image(source='upload', tool='sketch', elem_id="input_image_upload", type="pil", label="Upload & Draw on Image") 90 | # Allow user to draw the mask 91 | #mask = gr.Image(source='upload', tool='sketch', elem_id="mask_upload", type="pil", label="Draw Mask") 92 | #prompt = gr.Textbox(label='Your prompt (what you want to add in place of what you are removing)') 93 | btn = gr.Button("Run") 94 | with gr.Column(): 95 | result = gr.Image(label="Result") 96 | btn.click(fn=predict, inputs=[input_image], outputs=result) 97 | demo.launch(share=True) 98 | -------------------------------------------------------------------------------- /resnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.utils.model_zoo as modelzoo 8 | 9 | # from modules.bn import InPlaceABNSync as BatchNorm2d 10 | 11 | resnet18_url = 'https://download.pytorch.org/models/resnet18-5c106cde.pth' 12 | 13 | 14 | def conv3x3(in_planes, out_planes, stride=1): 15 | """3x3 convolution with padding""" 16 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 17 | padding=1, bias=False) 18 | 19 | 20 | class BasicBlock(nn.Module): 21 | def __init__(self, in_chan, out_chan, stride=1): 22 | super(BasicBlock, self).__init__() 23 | self.conv1 = conv3x3(in_chan, out_chan, stride) 24 | self.bn1 = nn.BatchNorm2d(out_chan) 25 | self.conv2 = conv3x3(out_chan, out_chan) 26 | self.bn2 = nn.BatchNorm2d(out_chan) 27 | self.relu = nn.ReLU(inplace=True) 28 | self.downsample = None 29 | if in_chan != out_chan or stride != 1: 30 | self.downsample = nn.Sequential( 31 | nn.Conv2d(in_chan, out_chan, 32 | kernel_size=1, stride=stride, bias=False), 33 | nn.BatchNorm2d(out_chan), 34 | ) 35 | 36 | def forward(self, x): 37 | residual = self.conv1(x) 38 | residual = F.relu(self.bn1(residual)) 39 | residual = self.conv2(residual) 40 | residual = self.bn2(residual) 41 | 42 | shortcut = x 43 | if self.downsample is not None: 44 | shortcut = self.downsample(x) 45 | 46 | out = shortcut + residual 47 | out = self.relu(out) 48 | return out 49 | 50 | 51 | def create_layer_basic(in_chan, out_chan, bnum, stride=1): 52 | layers = [BasicBlock(in_chan, out_chan, stride=stride)] 53 | for i in range(bnum-1): 54 | layers.append(BasicBlock(out_chan, out_chan, stride=1)) 55 | return nn.Sequential(*layers) 56 | 57 | 58 | class Resnet18(nn.Module): 59 | def __init__(self): 60 | super(Resnet18, self).__init__() 61 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 62 | bias=False) 63 | self.bn1 = nn.BatchNorm2d(64) 64 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 65 | self.layer1 = create_layer_basic(64, 64, bnum=2, stride=1) 66 | self.layer2 = create_layer_basic(64, 128, bnum=2, stride=2) 67 | self.layer3 = create_layer_basic(128, 256, bnum=2, stride=2) 68 | self.layer4 = create_layer_basic(256, 512, bnum=2, stride=2) 69 | self.init_weight() 70 | 71 | def forward(self, x): 72 | x = self.conv1(x) 73 | x = F.relu(self.bn1(x)) 74 | x = self.maxpool(x) 75 | 76 | x = self.layer1(x) 77 | feat8 = self.layer2(x) # 1/8 78 | feat16 = self.layer3(feat8) # 1/16 79 | feat32 = self.layer4(feat16) # 1/32 80 | return feat8, feat16, feat32 81 | 82 | def init_weight(self): 83 | state_dict = modelzoo.load_url(resnet18_url) 84 | self_state_dict = self.state_dict() 85 | for k, v in state_dict.items(): 86 | if 'fc' in k: continue 87 | self_state_dict.update({k: v}) 88 | self.load_state_dict(self_state_dict) 89 | 90 | def get_params(self): 91 | wd_params, nowd_params = [], [] 92 | for name, module in self.named_modules(): 93 | if isinstance(module, (nn.Linear, nn.Conv2d)): 94 | wd_params.append(module.weight) 95 | if not module.bias is None: 96 | nowd_params.append(module.bias) 97 | elif isinstance(module, nn.BatchNorm2d): 98 | nowd_params += list(module.parameters()) 99 | return wd_params, nowd_params 100 | 101 | 102 | if __name__ == "__main__": 103 | net = Resnet18() 104 | x = torch.randn(16, 3, 224, 224) 105 | out = net(x) 106 | print(out[0].size()) 107 | print(out[1].size()) 108 | print(out[2].size()) 109 | net.get_params() 110 | -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- 1 | from dataset.CamVid import CamVid 2 | import torch 3 | import argparse 4 | import os 5 | from torch.utils.data import DataLoader 6 | from model.build_BiSeNet import BiSeNet 7 | import numpy as np 8 | from utils import reverse_one_hot, compute_global_accuracy, fast_hist, per_class_iu, cal_miou 9 | import tqdm 10 | 11 | 12 | def eval(model,dataloader, args, csv_path): 13 | print('start test!') 14 | with torch.no_grad(): 15 | model.eval() 16 | precision_record = [] 17 | tq = tqdm.tqdm(total=len(dataloader) * args.batch_size) 18 | tq.set_description('test') 19 | hist = np.zeros((args.num_classes, args.num_classes)) 20 | for i, (data, label) in enumerate(dataloader): 21 | tq.update(args.batch_size) 22 | if torch.cuda.is_available() and args.use_gpu: 23 | data = data.cuda() 24 | label = label.cuda() 25 | predict = model(data).squeeze() 26 | predict = reverse_one_hot(predict) 27 | predict = np.array(predict) 28 | # predict = colour_code_segmentation(np.array(predict), label_info) 29 | 30 | label = label.squeeze() 31 | if args.loss == 'dice': 32 | label = reverse_one_hot(label) 33 | label = np.array(label) 34 | # label = colour_code_segmentation(np.array(label), label_info) 35 | 36 | precision = compute_global_accuracy(predict, label) 37 | hist += fast_hist(label.flatten(), predict.flatten(), args.num_classes) 38 | precision_record.append(precision) 39 | precision = np.mean(precision_record) 40 | miou_list = per_class_iu(hist)[:-1] 41 | miou_dict, miou = cal_miou(miou_list, csv_path) 42 | print('IoU for each class:') 43 | for key in miou_dict: 44 | print('{}:{},'.format(key, miou_dict[key])) 45 | tq.close() 46 | print('precision for test: %.3f' % precision) 47 | print('mIoU for validation: %.3f' % miou) 48 | return precision 49 | 50 | def main(params): 51 | # basic parameters 52 | parser = argparse.ArgumentParser() 53 | parser.add_argument('--checkpoint_path', type=str, default=None, required=True, help='The path to the pretrained weights of model') 54 | parser.add_argument('--crop_height', type=int, default=720, help='Height of cropped/resized input image to network') 55 | parser.add_argument('--crop_width', type=int, default=960, help='Width of cropped/resized input image to network') 56 | parser.add_argument('--data', type=str, default='/path/to/data', help='Path of training data') 57 | parser.add_argument('--batch_size', type=int, default=1, help='Number of images in each batch') 58 | parser.add_argument('--context_path', type=str, default="resnet101", help='The context path model you are using.') 59 | parser.add_argument('--cuda', type=str, default='0', help='GPU ids used for training') 60 | parser.add_argument('--use_gpu', type=bool, default=True, help='Whether to user gpu for training') 61 | parser.add_argument('--num_classes', type=int, default=32, help='num of object classes (with void)') 62 | parser.add_argument('--loss', type=str, default='dice', help='loss function, dice or crossentropy') 63 | args = parser.parse_args(params) 64 | 65 | # create dataset and dataloader 66 | test_path = os.path.join(args.data, 'test') 67 | # test_path = os.path.join(args.data, 'train') 68 | test_label_path = os.path.join(args.data, 'test_labels') 69 | # test_label_path = os.path.join(args.data, 'train_labels') 70 | csv_path = os.path.join(args.data, 'class_dict.csv') 71 | dataset = CamVid(test_path, test_label_path, csv_path, scale=(args.crop_height, args.crop_width), mode='test') 72 | dataloader = DataLoader( 73 | dataset, 74 | batch_size=1, 75 | shuffle=True, 76 | num_workers=4, 77 | ) 78 | 79 | # build model 80 | os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda 81 | model = BiSeNet(args.num_classes, args.context_path) 82 | if torch.cuda.is_available() and args.use_gpu: 83 | model = torch.nn.DataParallel(model).cuda() 84 | 85 | # load pretrained model if exists 86 | print('load model from %s ...' % args.checkpoint_path) 87 | model.module.load_state_dict(torch.load(args.checkpoint_path)) 88 | print('Done!') 89 | 90 | # get label info 91 | # label_info = get_label_info(csv_path) 92 | # test 93 | eval(model, dataloader, args, csv_path) 94 | 95 | 96 | if __name__ == '__main__': 97 | params = [ 98 | '--checkpoint_path', 'path/to/ckpt', 99 | '--data', '/path/to/CamVid', 100 | '--cuda', '0', 101 | '--context_path', 'resnet18', 102 | '--num_classes', '12' 103 | ] 104 | main(params) -------------------------------------------------------------------------------- /rest.py: -------------------------------------------------------------------------------- 1 | #best restoration model 2 | 3 | import gradio as gr 4 | import numpy as np 5 | import torch 6 | from src.pipeline_stable_diffusion_controlnet_inpaint import * 7 | from scratch_detection import ScratchDetection 8 | 9 | from diffusers import StableDiffusionInpaintPipeline, ControlNetModel, DEISMultistepScheduler 10 | from diffusers.utils import load_image 11 | from PIL import Image 12 | import cv2 13 | import time 14 | import os 15 | 16 | device = "cuda" 17 | 18 | # load control net and stable diffusion v1-5 19 | controlnet = ControlNetModel.from_pretrained("thepowefuldeez/sd21-controlnet-canny", torch_dtype=torch.float16) 20 | 21 | pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( 22 | "stabilityai/stable-diffusion-2-inpainting", controlnet=controlnet, torch_dtype=torch.float16 23 | ) 24 | 25 | pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config) 26 | 27 | # speed up diffusion process with faster scheduler and memory optimization 28 | # remove following line if xformers is not installed 29 | pipe.enable_xformers_memory_efficient_attention() 30 | pipe.to('cuda') 31 | 32 | def combine_masks(mask1, mask2): 33 | mask1_np = np.array(mask1) 34 | mask2_np = np.array(mask2) 35 | combined_mask_np = np.maximum(mask1_np, mask2_np) 36 | combined_mask = Image.fromarray(combined_mask_np) 37 | return combined_mask 38 | 39 | if not os.path.exists("input_images"): 40 | os.makedirs("input_images") 41 | 42 | def generate_scratch_mask(input_dict): 43 | # Save the input image to a directory 44 | input_image = input_dict["image"].convert("RGB") 45 | input_image_path = "input_images/input_image.png" 46 | input_image_resized = resize_image(input_image, 768) 47 | input_image_resized.save(input_image_path) 48 | 49 | test_path = "input_images" 50 | output_dir = "output_masks" 51 | scratch_detector = ScratchDetection(test_path, output_dir, input_size="scale_256", gpu=0) 52 | scratch_detector.run() 53 | mask_image = scratch_detector.get_mask_image("input_image.png") 54 | 55 | # Resize the mask to match the input image size 56 | mask_image = mask_image.resize(input_image.size, Image.BICUBIC) 57 | 58 | # Apply dilation to make the lines bigger 59 | kernel = np.ones((5, 5), np.uint8) 60 | mask_image_np = np.array(mask_image) 61 | mask_image_np_dilated = cv2.dilate(mask_image_np, kernel, iterations=2) 62 | mask_image_dilated = Image.fromarray(mask_image_np_dilated) 63 | 64 | return mask_image_dilated 65 | 66 | def resize_image(image, target_size): 67 | width, height = image.size 68 | aspect_ratio = float(width) / float(height) 69 | if width > height: 70 | new_width = target_size 71 | new_height = int(target_size / aspect_ratio) 72 | else: 73 | new_width = int(target_size * aspect_ratio) 74 | new_height = target_size 75 | return image.resize((new_width, new_height), Image.BICUBIC) 76 | 77 | with gr.Blocks() as demo: 78 | with gr.Row(): 79 | input_image = gr.Image(source='upload', tool='sketch', elem_id="input_image_upload", type="pil", label="Upload & Draw on Image") 80 | mask_image = gr.Image(label="mask") 81 | output_image = gr.Image(label="output") 82 | with gr.Row(): 83 | generate_mask_button = gr.Button("Generate Scratch Mask") 84 | submit = gr.Button("Inpaint") 85 | 86 | def inpaint(input_dict, mask): 87 | image = input_dict["image"].convert("RGB") 88 | draw_mask = input_dict["mask"].convert("RGB") 89 | 90 | image = resize_image(image, 768) 91 | 92 | mask = Image.fromarray(mask) 93 | mask = resize_image(mask, 768) 94 | draw_mask = resize_image(draw_mask, 768) 95 | 96 | image = np.array(image) 97 | low_threshold = 100 98 | high_threshold = 200 99 | canny = cv2.Canny(image, low_threshold, high_threshold) 100 | canny = canny[:, :, None] 101 | canny = np.concatenate([canny, canny, canny], axis=2) 102 | canny_image = Image.fromarray(canny) 103 | generator = torch.manual_seed(0) 104 | 105 | # Combine drawn mask and generated mask 106 | combined_mask = combine_masks(draw_mask, mask) 107 | 108 | output = pipe( 109 | prompt="", 110 | num_inference_steps=20, 111 | generator=generator, 112 | image=image, 113 | control_image=canny_image, 114 | controlnet_conditioning_scale=0, 115 | mask_image=combined_mask 116 | ).images[0] 117 | return output 118 | 119 | generate_mask_button.click(generate_scratch_mask, inputs=[input_image], outputs=[mask_image]) 120 | submit.click(inpaint, inputs=[input_image, mask_image], outputs=[output_image]) 121 | demo.launch(share=True) 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/comm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : comm.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import queue 12 | import collections 13 | import threading 14 | 15 | __all__ = ['FutureResult', 'SlavePipe', 'SyncMaster'] 16 | 17 | 18 | class FutureResult(object): 19 | """A thread-safe future implementation. Used only as one-to-one pipe.""" 20 | 21 | def __init__(self): 22 | self._result = None 23 | self._lock = threading.Lock() 24 | self._cond = threading.Condition(self._lock) 25 | 26 | def put(self, result): 27 | with self._lock: 28 | assert self._result is None, 'Previous result has\'t been fetched.' 29 | self._result = result 30 | self._cond.notify() 31 | 32 | def get(self): 33 | with self._lock: 34 | if self._result is None: 35 | self._cond.wait() 36 | 37 | res = self._result 38 | self._result = None 39 | return res 40 | 41 | 42 | _MasterRegistry = collections.namedtuple('MasterRegistry', ['result']) 43 | _SlavePipeBase = collections.namedtuple('_SlavePipeBase', ['identifier', 'queue', 'result']) 44 | 45 | 46 | class SlavePipe(_SlavePipeBase): 47 | """Pipe for master-slave communication.""" 48 | 49 | def run_slave(self, msg): 50 | self.queue.put((self.identifier, msg)) 51 | ret = self.result.get() 52 | self.queue.put(True) 53 | return ret 54 | 55 | 56 | class SyncMaster(object): 57 | """An abstract `SyncMaster` object. 58 | 59 | - During the replication, as the data parallel will trigger an callback of each module, all slave devices should 60 | call `register(id)` and obtain an `SlavePipe` to communicate with the master. 61 | - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected, 62 | and passed to a registered callback. 63 | - After receiving the messages, the master device should gather the information and determine to message passed 64 | back to each slave devices. 65 | """ 66 | 67 | def __init__(self, master_callback): 68 | """ 69 | 70 | Args: 71 | master_callback: a callback to be invoked after having collected messages from slave devices. 72 | """ 73 | self._master_callback = master_callback 74 | self._queue = queue.Queue() 75 | self._registry = collections.OrderedDict() 76 | self._activated = False 77 | 78 | def __getstate__(self): 79 | return {'master_callback': self._master_callback} 80 | 81 | def __setstate__(self, state): 82 | self.__init__(state['master_callback']) 83 | 84 | def register_slave(self, identifier): 85 | """ 86 | Register an slave device. 87 | 88 | Args: 89 | identifier: an identifier, usually is the device id. 90 | 91 | Returns: a `SlavePipe` object which can be used to communicate with the master device. 92 | 93 | """ 94 | if self._activated: 95 | assert self._queue.empty(), 'Queue is not clean before next initialization.' 96 | self._activated = False 97 | self._registry.clear() 98 | future = FutureResult() 99 | self._registry[identifier] = _MasterRegistry(future) 100 | return SlavePipe(identifier, self._queue, future) 101 | 102 | def run_master(self, master_msg): 103 | """ 104 | Main entry for the master device in each forward pass. 105 | The messages were first collected from each devices (including the master device), and then 106 | an callback will be invoked to compute the message to be sent back to each devices 107 | (including the master device). 108 | 109 | Args: 110 | master_msg: the message that the master want to send to itself. This will be placed as the first 111 | message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example. 112 | 113 | Returns: the message to be sent back to the master device. 114 | 115 | """ 116 | self._activated = True 117 | 118 | intermediates = [(0, master_msg)] 119 | for i in range(self.nr_slaves): 120 | intermediates.append(self._queue.get()) 121 | 122 | results = self._master_callback(intermediates) 123 | assert results[0][0] == 0, 'The first result should belongs to the master.' 124 | 125 | for i, res in results: 126 | if i == 0: 127 | continue 128 | self._registry[i].result.put(res) 129 | 130 | for i in range(self.nr_slaves): 131 | assert self._queue.get() is True 132 | 133 | return results[0][1] 134 | 135 | @property 136 | def nr_slaves(self): 137 | return len(self._registry) 138 | -------------------------------------------------------------------------------- /dataset/CamVid.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import glob 3 | import os 4 | from torchvision import transforms 5 | import cv2 6 | from PIL import Image 7 | import pandas as pd 8 | import numpy as np 9 | from imgaug import augmenters as iaa 10 | import imgaug as ia 11 | from utils import get_label_info, one_hot_it, RandomCrop, reverse_one_hot, one_hot_it_v11, one_hot_it_v11_dice 12 | import random 13 | 14 | def augmentation(): 15 | # augment images with spatial transformation: Flip, Affine, Rotation, etc... 16 | # see https://github.com/aleju/imgaug for more details 17 | pass 18 | 19 | 20 | def augmentation_pixel(): 21 | # augment images with pixel intensity transformation: GaussianBlur, Multiply, etc... 22 | pass 23 | 24 | class CamVid(torch.utils.data.Dataset): 25 | def __init__(self, image_path, label_path, csv_path, scale, loss='dice', mode='train'): 26 | super().__init__() 27 | self.mode = mode 28 | self.image_list = [] 29 | if not isinstance(image_path, list): 30 | image_path = [image_path] 31 | for image_path_ in image_path: 32 | self.image_list.extend(glob.glob(os.path.join(image_path_, '*.png'))) 33 | self.image_list.sort() 34 | self.label_list = [] 35 | if not isinstance(label_path, list): 36 | label_path = [label_path] 37 | for label_path_ in label_path: 38 | self.label_list.extend(glob.glob(os.path.join(label_path_, '*.png'))) 39 | self.label_list.sort() 40 | # self.image_name = [x.split('/')[-1].split('.')[0] for x in self.image_list] 41 | # self.label_list = [os.path.join(label_path, x + '_L.png') for x in self.image_list] 42 | self.fliplr = iaa.Fliplr(0.5) 43 | self.label_info = get_label_info(csv_path) 44 | # resize 45 | # self.resize_label = transforms.Resize(scale, Image.NEAREST) 46 | # self.resize_img = transforms.Resize(scale, Image.BILINEAR) 47 | # normalization 48 | self.to_tensor = transforms.Compose([ 49 | transforms.ToTensor(), 50 | transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), 51 | ]) 52 | # self.crop = transforms.RandomCrop(scale, pad_if_needed=True) 53 | self.image_size = scale 54 | self.scale = [0.5, 1, 1.25, 1.5, 1.75, 2] 55 | self.loss = loss 56 | 57 | def __getitem__(self, index): 58 | # load image and crop 59 | seed = random.random() 60 | img = Image.open(self.image_list[index]) 61 | # random crop image 62 | # ===================================== 63 | # w,h = img.size 64 | # th, tw = self.scale 65 | # i = random.randint(0, h - th) 66 | # j = random.randint(0, w - tw) 67 | # img = F.crop(img, i, j, th, tw) 68 | # ===================================== 69 | 70 | scale = random.choice(self.scale) 71 | scale = (int(self.image_size[0] * scale), int(self.image_size[1] * scale)) 72 | 73 | # randomly resize image and random crop 74 | # ===================================== 75 | if self.mode == 'train': 76 | img = transforms.Resize(scale, Image.BILINEAR)(img) 77 | img = RandomCrop(self.image_size, seed, pad_if_needed=True)(img) 78 | # ===================================== 79 | 80 | img = np.array(img) 81 | # load label 82 | label = Image.open(self.label_list[index]) 83 | 84 | 85 | # crop the corresponding label 86 | # ===================================== 87 | # label = F.crop(label, i, j, th, tw) 88 | # ===================================== 89 | 90 | # randomly resize label and random crop 91 | # ===================================== 92 | if self.mode == 'train': 93 | label = transforms.Resize(scale, Image.NEAREST)(label) 94 | label = RandomCrop(self.image_size, seed, pad_if_needed=True)(label) 95 | # ===================================== 96 | 97 | label = np.array(label) 98 | 99 | 100 | # augment image and label 101 | if self.mode == 'train': 102 | seq_det = self.fliplr.to_deterministic() 103 | img = seq_det.augment_image(img) 104 | label = seq_det.augment_image(label) 105 | 106 | 107 | # image -> [C, H, W] 108 | img = Image.fromarray(img) 109 | img = self.to_tensor(img).float() 110 | 111 | if self.loss == 'dice': 112 | # label -> [num_classes, H, W] 113 | label = one_hot_it_v11_dice(label, self.label_info).astype(np.uint8) 114 | 115 | label = np.transpose(label, [2, 0, 1]).astype(np.float32) 116 | # label = label.astype(np.float32) 117 | label = torch.from_numpy(label) 118 | 119 | return img, label 120 | 121 | elif self.loss == 'crossentropy': 122 | label = one_hot_it_v11(label, self.label_info).astype(np.uint8) 123 | # label = label.astype(np.float32) 124 | label = torch.from_numpy(label).long() 125 | 126 | return img, label 127 | 128 | def __len__(self): 129 | return len(self.image_list) 130 | 131 | 132 | if __name__ == '__main__': 133 | # data = CamVid('/path/to/CamVid/train', '/path/to/CamVid/train_labels', '/path/to/CamVid/class_dict.csv', (640, 640)) 134 | data = CamVid(['/data/sqy/CamVid/train', '/data/sqy/CamVid/val'], 135 | ['/data/sqy/CamVid/train_labels', '/data/sqy/CamVid/val_labels'], '/data/sqy/CamVid/class_dict.csv', 136 | (720, 960), loss='crossentropy', mode='val') 137 | from model.build_BiSeNet import BiSeNet 138 | from utils import reverse_one_hot, get_label_info, colour_code_segmentation, compute_global_accuracy 139 | 140 | label_info = get_label_info('/data/sqy/CamVid/class_dict.csv') 141 | for i, (img, label) in enumerate(data): 142 | print(label.size()) 143 | print(torch.max(label)) 144 | 145 | -------------------------------------------------------------------------------- /scratch_detection.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import gc 3 | import json 4 | import os 5 | import time 6 | import warnings 7 | 8 | import numpy as np 9 | import torch 10 | import torch.nn.functional as F 11 | import torchvision as tv 12 | from PIL import Image, ImageFile 13 | 14 | from detection_models import networks 15 | from detection_util.util import * 16 | 17 | warnings.filterwarnings("ignore", category=UserWarning) 18 | 19 | ImageFile.LOAD_TRUNCATED_IMAGES = True 20 | 21 | 22 | def data_transforms(img, full_size, method=Image.BICUBIC): 23 | if full_size == "full_size": 24 | ow, oh = img.size 25 | h = int(round(oh / 16) * 16) 26 | w = int(round(ow / 16) * 16) 27 | if (h == oh) and (w == ow): 28 | return img 29 | return img.resize((w, h), method) 30 | 31 | elif full_size == "scale_256": 32 | ow, oh = img.size 33 | pw, ph = ow, oh 34 | if ow < oh: 35 | ow = 256 36 | oh = ph / pw * 256 37 | else: 38 | oh = 256 39 | ow = pw / ph * 256 40 | 41 | h = int(round(oh / 16) * 16) 42 | w = int(round(ow / 16) * 16) 43 | if (h == ph) and (w == pw): 44 | return img 45 | return img.resize((w, h), method) 46 | 47 | 48 | def scale_tensor(img_tensor, default_scale=256): 49 | _, _, w, h = img_tensor.shape 50 | if w < h: 51 | ow = default_scale 52 | oh = h / w * default_scale 53 | else: 54 | oh = default_scale 55 | ow = w / h * default_scale 56 | 57 | oh = int(round(oh / 16) * 16) 58 | ow = int(round(ow / 16) * 16) 59 | 60 | return F.interpolate(img_tensor, [ow, oh], mode="bilinear") 61 | 62 | 63 | def blend_mask(img, mask): 64 | 65 | np_img = np.array(img).astype("float") 66 | 67 | return Image.fromarray((np_img * (1 - mask) + mask * 255.0).astype("uint8")).convert("RGB") 68 | 69 | def process_images(test_path, output_dir, input_size="scale_256", gpu=0): 70 | print("initializing the dataloader") 71 | 72 | # Initialize the model 73 | model = networks.UNet( 74 | in_channels=1, 75 | out_channels=1, 76 | depth=4, 77 | conv_num=2, 78 | wf=6, 79 | padding=True, 80 | batch_norm=True, 81 | up_mode="upsample", 82 | with_tanh=False, 83 | sync_bn=True, 84 | antialiasing=True, 85 | ) 86 | 87 | ## load model 88 | checkpoint_path = os.path.join(os.path.dirname(__file__), "FT_Epoch_latest.pt") 89 | checkpoint = torch.load(checkpoint_path, map_location="cpu") 90 | model.load_state_dict(checkpoint["model_state"]) 91 | print("model weights loaded") 92 | 93 | if gpu >= 0: 94 | model.to(gpu) 95 | else: 96 | model.cpu() 97 | model.eval() 98 | 99 | ## dataloader and transformation 100 | print("directory of testing image: " + test_path) 101 | imagelist = os.listdir(test_path) 102 | imagelist.sort() 103 | total_iter = 0 104 | 105 | P_matrix = {} 106 | save_url = os.path.join(output_dir) 107 | mkdir_if_not(save_url) 108 | 109 | input_dir = os.path.join(save_url, "input") 110 | output_dir = os.path.join(save_url, "mask") 111 | mkdir_if_not(input_dir) 112 | mkdir_if_not(output_dir) 113 | 114 | idx = 0 115 | 116 | results = [] 117 | for image_name in imagelist: 118 | 119 | idx += 1 120 | 121 | print("processing", image_name) 122 | 123 | scratch_file = os.path.join(test_path, image_name) 124 | if not os.path.isfile(scratch_file): 125 | print("Skipping non-file %s" % image_name) 126 | continue 127 | scratch_image = Image.open(scratch_file).convert("RGB") 128 | w, h = scratch_image.size 129 | 130 | transformed_image_PIL = data_transforms(scratch_image, input_size) 131 | scratch_image = transformed_image_PIL.convert("L") 132 | scratch_image = tv.transforms.ToTensor()(scratch_image) 133 | scratch_image = tv.transforms.Normalize([0.5], [0.5])(scratch_image) 134 | scratch_image = torch.unsqueeze(scratch_image, 0) 135 | _, _, ow, oh = scratch_image.shape 136 | scratch_image_scale = scale_tensor(scratch_image) 137 | 138 | if gpu >= 0: 139 | scratch_image_scale = scratch_image_scale.to(gpu) 140 | else: 141 | scratch_image_scale = scratch_image_scale.cpu() 142 | with torch.no_grad(): 143 | P = torch.sigmoid(model(scratch_image_scale)) 144 | 145 | P = P.data.cpu() 146 | P = F.interpolate(P, [ow, oh], mode="nearest") 147 | 148 | tv.utils.save_image( 149 | (P >= 0.4).float(), 150 | os.path.join( 151 | output_dir, 152 | image_name[:-4] + ".png", 153 | ), 154 | nrow=1, 155 | padding=0, 156 | normalize=True, 157 | ) 158 | transformed_image_PIL.save(os.path.join(input_dir, image_name[:-4] + ".png")) 159 | gc.collect() 160 | torch.cuda.empty_cache() 161 | 162 | # Wrap the scratch detection in a class 163 | class ScratchDetection: 164 | def __init__(self, test_path, output_dir, input_size="scale_256", gpu=0): 165 | self.test_path = test_path 166 | self.output_dir = output_dir 167 | self.input_size = input_size 168 | self.gpu = gpu 169 | 170 | def run(self): 171 | process_images(self.test_path, self.output_dir, self.input_size, self.gpu) 172 | 173 | # Add a function to get the mask image from the output directory 174 | def get_mask_image(self, image_name): 175 | mask_image_path = os.path.join(self.output_dir, "mask", image_name) 176 | return Image.open(mask_image_path) 177 | 178 | # Keep the __main__ part, but modify it to use the new ScratchDetection class 179 | if __name__ == "__main__": 180 | parser = argparse.ArgumentParser() 181 | parser.add_argument("--GPU", type=int, default=0) 182 | parser.add_argument("--test_path", type=str, default=".") 183 | parser.add_argument("--output_dir", type=str, default=".") 184 | parser.add_argument("--input_size", type=str, default="scale_256", help="resize_256|full_size|scale_256") 185 | args = parser.parse_args() 186 | 187 | scratch_detector = ScratchDetection(args.test_path, args.output_dir, args.input_size, args.GPU) 188 | scratch_detector.run() 189 | -------------------------------------------------------------------------------- /detection_util/util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import os 5 | import sys 6 | import time 7 | import shutil 8 | import platform 9 | import numpy as np 10 | from datetime import datetime 11 | 12 | import torch 13 | import torchvision as tv 14 | import torch.backends.cudnn as cudnn 15 | 16 | # from torch.utils.tensorboard import SummaryWriter 17 | 18 | import yaml 19 | import matplotlib.pyplot as plt 20 | from easydict import EasyDict as edict 21 | import torchvision.utils as vutils 22 | 23 | 24 | ##### option parsing ###### 25 | def print_options(config_dict): 26 | print("------------ Options -------------") 27 | for k, v in sorted(config_dict.items()): 28 | print("%s: %s" % (str(k), str(v))) 29 | print("-------------- End ----------------") 30 | 31 | 32 | def save_options(config_dict): 33 | from time import gmtime, strftime 34 | 35 | file_dir = os.path.join(config_dict["checkpoint_dir"], config_dict["name"]) 36 | mkdir_if_not(file_dir) 37 | file_name = os.path.join(file_dir, "opt.txt") 38 | with open(file_name, "wt") as opt_file: 39 | opt_file.write(os.path.basename(sys.argv[0]) + " " + strftime("%Y-%m-%d %H:%M:%S", gmtime()) + "\n") 40 | opt_file.write("------------ Options -------------\n") 41 | for k, v in sorted(config_dict.items()): 42 | opt_file.write("%s: %s\n" % (str(k), str(v))) 43 | opt_file.write("-------------- End ----------------\n") 44 | 45 | 46 | def config_parse(config_file, options, save=True): 47 | with open(config_file, "r") as stream: 48 | config_dict = yaml.safe_load(stream) 49 | config = edict(config_dict) 50 | 51 | for option_key, option_value in vars(options).items(): 52 | config_dict[option_key] = option_value 53 | config[option_key] = option_value 54 | 55 | if config.debug_mode: 56 | config_dict["num_workers"] = 0 57 | config.num_workers = 0 58 | config.batch_size = 2 59 | if isinstance(config.gpu_ids, str): 60 | config.gpu_ids = [int(x) for x in config.gpu_ids.split(",")][0] 61 | 62 | print_options(config_dict) 63 | if save: 64 | save_options(config_dict) 65 | 66 | return config 67 | 68 | 69 | ###### utility ###### 70 | def to_np(x): 71 | return x.cpu().numpy() 72 | 73 | 74 | def prepare_device(use_gpu, gpu_ids): 75 | if use_gpu: 76 | cudnn.benchmark = True 77 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 78 | if isinstance(gpu_ids, str): 79 | gpu_ids = [int(x) for x in gpu_ids.split(",")] 80 | torch.cuda.set_device(gpu_ids[0]) 81 | device = torch.device("cuda:" + str(gpu_ids[0])) 82 | else: 83 | torch.cuda.set_device(gpu_ids) 84 | device = torch.device("cuda:" + str(gpu_ids)) 85 | print("running on GPU {}".format(gpu_ids)) 86 | else: 87 | device = torch.device("cpu") 88 | print("running on CPU") 89 | 90 | return device 91 | 92 | 93 | ###### file system ###### 94 | def get_dir_size(start_path="."): 95 | total_size = 0 96 | for dirpath, dirnames, filenames in os.walk(start_path): 97 | for f in filenames: 98 | fp = os.path.join(dirpath, f) 99 | total_size += os.path.getsize(fp) 100 | return total_size 101 | 102 | 103 | def mkdir_if_not(dir_path): 104 | if not os.path.exists(dir_path): 105 | os.makedirs(dir_path) 106 | 107 | 108 | ##### System related ###### 109 | class Timer: 110 | def __init__(self, msg): 111 | self.msg = msg 112 | self.start_time = None 113 | 114 | def __enter__(self): 115 | self.start_time = time.time() 116 | 117 | def __exit__(self, exc_type, exc_value, exc_tb): 118 | elapse = time.time() - self.start_time 119 | print(self.msg % elapse) 120 | 121 | 122 | ###### interactive ###### 123 | def get_size(start_path="."): 124 | total_size = 0 125 | for dirpath, dirnames, filenames in os.walk(start_path): 126 | for f in filenames: 127 | fp = os.path.join(dirpath, f) 128 | total_size += os.path.getsize(fp) 129 | return total_size 130 | 131 | 132 | def clean_tensorboard(directory): 133 | tensorboard_list = os.listdir(directory) 134 | SIZE_THRESH = 100000 135 | for tensorboard in tensorboard_list: 136 | tensorboard = os.path.join(directory, tensorboard) 137 | if get_size(tensorboard) < SIZE_THRESH: 138 | print("deleting the empty tensorboard: ", tensorboard) 139 | # 140 | if os.path.isdir(tensorboard): 141 | shutil.rmtree(tensorboard) 142 | else: 143 | os.remove(tensorboard) 144 | 145 | 146 | def prepare_tensorboard(config, experiment_name=datetime.now().strftime("%Y-%m-%d %H-%M-%S")): 147 | tensorboard_directory = os.path.join(config.checkpoint_dir, config.name, "tensorboard_logs") 148 | mkdir_if_not(tensorboard_directory) 149 | clean_tensorboard(tensorboard_directory) 150 | tb_writer = SummaryWriter(os.path.join(tensorboard_directory, experiment_name), flush_secs=10) 151 | 152 | # try: 153 | # shutil.copy('outputs/opt.txt', tensorboard_directory) 154 | # except: 155 | # print('cannot find file opt.txt') 156 | return tb_writer 157 | 158 | 159 | def tb_loss_logger(tb_writer, iter_index, loss_logger): 160 | for tag, value in loss_logger.items(): 161 | tb_writer.add_scalar(tag, scalar_value=value.item(), global_step=iter_index) 162 | 163 | 164 | def tb_image_logger(tb_writer, iter_index, images_info, config): 165 | ### Save and write the output into the tensorboard 166 | tb_logger_path = os.path.join(config.output_dir, config.name, config.train_mode) 167 | mkdir_if_not(tb_logger_path) 168 | for tag, image in images_info.items(): 169 | if tag == "test_image_prediction" or tag == "image_prediction": 170 | continue 171 | image = tv.utils.make_grid(image.cpu()) 172 | image = torch.clamp(image, 0, 1) 173 | tb_writer.add_image(tag, img_tensor=image, global_step=iter_index) 174 | tv.transforms.functional.to_pil_image(image).save( 175 | os.path.join(tb_logger_path, "{:06d}_{}.jpg".format(iter_index, tag)) 176 | ) 177 | 178 | 179 | def tb_image_logger_test(epoch, iter, images_info, config): 180 | 181 | url = os.path.join(config.output_dir, config.name, config.train_mode, "val_" + str(epoch)) 182 | if not os.path.exists(url): 183 | os.makedirs(url) 184 | scratch_img = images_info["test_scratch_image"].data.cpu() 185 | if config.norm_input: 186 | scratch_img = (scratch_img + 1.0) / 2.0 187 | scratch_img = torch.clamp(scratch_img, 0, 1) 188 | gt_mask = images_info["test_mask_image"].data.cpu() 189 | predict_mask = images_info["test_scratch_prediction"].data.cpu() 190 | 191 | predict_hard_mask = (predict_mask.data.cpu() >= 0.5).float() 192 | 193 | imgs = torch.cat((scratch_img, predict_hard_mask, gt_mask), 0) 194 | img_grid = vutils.save_image( 195 | imgs, os.path.join(url, str(iter) + ".jpg"), nrow=len(scratch_img), padding=0, normalize=True 196 | ) 197 | 198 | 199 | def imshow(input_image, title=None, to_numpy=False): 200 | inp = input_image 201 | if to_numpy or type(input_image) is torch.Tensor: 202 | inp = input_image.numpy() 203 | 204 | fig = plt.figure() 205 | if inp.ndim == 2: 206 | fig = plt.imshow(inp, cmap="gray", clim=[0, 255]) 207 | else: 208 | fig = plt.imshow(np.transpose(inp, [1, 2, 0]).astype(np.uint8)) 209 | plt.axis("off") 210 | fig.axes.get_xaxis().set_visible(False) 211 | fig.axes.get_yaxis().set_visible(False) 212 | plt.title(title) 213 | 214 | 215 | ###### vgg preprocessing ###### 216 | def vgg_preprocess(tensor): 217 | # input is RGB tensor which ranges in [0,1] 218 | # output is BGR tensor which ranges in [0,255] 219 | tensor_bgr = torch.cat((tensor[:, 2:3, :, :], tensor[:, 1:2, :, :], tensor[:, 0:1, :, :]), dim=1) 220 | # tensor_bgr = tensor[:, [2, 1, 0], ...] 221 | tensor_bgr_ml = tensor_bgr - torch.Tensor([0.40760392, 0.45795686, 0.48501961]).type_as(tensor_bgr).view( 222 | 1, 3, 1, 1 223 | ) 224 | tensor_rst = tensor_bgr_ml * 255 225 | return tensor_rst 226 | 227 | 228 | def torch_vgg_preprocess(tensor): 229 | # pytorch version normalization 230 | # note that both input and output are RGB tensors; 231 | # input and output ranges in [0,1] 232 | # normalize the tensor with mean and variance 233 | tensor_mc = tensor - torch.Tensor([0.485, 0.456, 0.406]).type_as(tensor).view(1, 3, 1, 1) 234 | tensor_mc_norm = tensor_mc / torch.Tensor([0.229, 0.224, 0.225]).type_as(tensor_mc).view(1, 3, 1, 1) 235 | return tensor_mc_norm 236 | 237 | 238 | def network_gradient(net, gradient_on=True): 239 | if gradient_on: 240 | for param in net.parameters(): 241 | param.requires_grad = True 242 | else: 243 | for param in net.parameters(): 244 | param.requires_grad = False 245 | return net 246 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from torch.utils.data import Dataset 3 | from torch.utils.data import DataLoader 4 | from dataset.CamVid import CamVid 5 | import os 6 | from model.build_BiSeNet import BiSeNet 7 | import torch 8 | from tensorboardX import SummaryWriter 9 | import tqdm 10 | import numpy as np 11 | from utils import poly_lr_scheduler 12 | from utils import reverse_one_hot, compute_global_accuracy, fast_hist, \ 13 | per_class_iu 14 | from loss import DiceLoss 15 | 16 | 17 | def val(args, model, dataloader): 18 | print('start val!') 19 | # label_info = get_label_info(csv_path) 20 | with torch.no_grad(): 21 | model.eval() 22 | precision_record = [] 23 | hist = np.zeros((args.num_classes, args.num_classes)) 24 | for i, (data, label) in enumerate(dataloader): 25 | if torch.cuda.is_available() and args.use_gpu: 26 | data = data.cuda() 27 | label = label.cuda() 28 | 29 | # get RGB predict image 30 | predict = model(data).squeeze() 31 | predict = reverse_one_hot(predict) 32 | predict = np.array(predict) 33 | 34 | # get RGB label image 35 | label = label.squeeze() 36 | if args.loss == 'dice': 37 | label = reverse_one_hot(label) 38 | label = np.array(label) 39 | 40 | # compute per pixel accuracy 41 | 42 | precision = compute_global_accuracy(predict, label) 43 | hist += fast_hist(label.flatten(), predict.flatten(), args.num_classes) 44 | 45 | # there is no need to transform the one-hot array to visual RGB array 46 | # predict = colour_code_segmentation(np.array(predict), label_info) 47 | # label = colour_code_segmentation(np.array(label), label_info) 48 | precision_record.append(precision) 49 | precision = np.mean(precision_record) 50 | # miou = np.mean(per_class_iu(hist)) 51 | miou_list = per_class_iu(hist)[:-1] 52 | # miou_dict, miou = cal_miou(miou_list, csv_path) 53 | miou = np.mean(miou_list) 54 | print('precision per pixel for test: %.3f' % precision) 55 | print('mIoU for validation: %.3f' % miou) 56 | # miou_str = '' 57 | # for key in miou_dict: 58 | # miou_str += '{}:{},\n'.format(key, miou_dict[key]) 59 | # print('mIoU for each class:') 60 | # print(miou_str) 61 | return precision, miou 62 | 63 | 64 | def train(args, model, optimizer, dataloader_train, dataloader_val): 65 | writer = SummaryWriter(comment=''.format(args.optimizer, args.context_path)) 66 | if args.loss == 'dice': 67 | loss_func = DiceLoss() 68 | elif args.loss == 'crossentropy': 69 | loss_func = torch.nn.CrossEntropyLoss() 70 | max_miou = 0 71 | step = 0 72 | for epoch in range(args.num_epochs): 73 | lr = poly_lr_scheduler(optimizer, args.learning_rate, iter=epoch, max_iter=args.num_epochs) 74 | model.train() 75 | tq = tqdm.tqdm(total=len(dataloader_train) * args.batch_size) 76 | tq.set_description('epoch %d, lr %f' % (epoch, lr)) 77 | loss_record = [] 78 | for i, (data, label) in enumerate(dataloader_train): 79 | if torch.cuda.is_available() and args.use_gpu: 80 | data = data.cuda() 81 | label = label.cuda() 82 | output, output_sup1, output_sup2 = model(data) 83 | loss1 = loss_func(output, label) 84 | loss2 = loss_func(output_sup1, label) 85 | loss3 = loss_func(output_sup2, label) 86 | loss = loss1 + loss2 + loss3 87 | tq.update(args.batch_size) 88 | tq.set_postfix(loss='%.6f' % loss) 89 | optimizer.zero_grad() 90 | loss.backward() 91 | optimizer.step() 92 | step += 1 93 | writer.add_scalar('loss_step', loss, step) 94 | loss_record.append(loss.item()) 95 | tq.close() 96 | loss_train_mean = np.mean(loss_record) 97 | writer.add_scalar('epoch/loss_epoch_train', float(loss_train_mean), epoch) 98 | print('loss for train : %f' % (loss_train_mean)) 99 | if epoch % args.checkpoint_step == 0 and epoch != 0: 100 | if not os.path.isdir(args.save_model_path): 101 | os.mkdir(args.save_model_path) 102 | torch.save(model.module.state_dict(), 103 | os.path.join(args.save_model_path, 'latest_dice_loss.pth')) 104 | 105 | if epoch % args.validation_step == 0: 106 | precision, miou = val(args, model, dataloader_val) 107 | if miou > max_miou: 108 | max_miou = miou 109 | torch.save(model.module.state_dict(), 110 | os.path.join(args.save_model_path, 'best_dice_loss.pth')) 111 | writer.add_scalar('epoch/precision_val', precision, epoch) 112 | writer.add_scalar('epoch/miou val', miou, epoch) 113 | 114 | 115 | def main(params): 116 | # basic parameters 117 | parser = argparse.ArgumentParser() 118 | parser.add_argument('--num_epochs', type=int, default=300, help='Number of epochs to train for') 119 | parser.add_argument('--epoch_start_i', type=int, default=0, help='Start counting epochs from this number') 120 | parser.add_argument('--checkpoint_step', type=int, default=1, help='How often to save checkpoints (epochs)') 121 | parser.add_argument('--validation_step', type=int, default=1, help='How often to perform validation (epochs)') 122 | parser.add_argument('--dataset', type=str, default="CamVid", help='Dataset you are using.') 123 | parser.add_argument('--crop_height', type=int, default=720, help='Height of cropped/resized input image to network') 124 | parser.add_argument('--crop_width', type=int, default=960, help='Width of cropped/resized input image to network') 125 | parser.add_argument('--batch_size', type=int, default=1, help='Number of images in each batch') 126 | parser.add_argument('--context_path', type=str, default="resnet101", 127 | help='The context path model you are using, resnet18, resnet101.') 128 | parser.add_argument('--learning_rate', type=float, default=0.01, help='learning rate used for train') 129 | parser.add_argument('--data', type=str, default='', help='path of training data') 130 | parser.add_argument('--num_workers', type=int, default=4, help='num of workers') 131 | parser.add_argument('--num_classes', type=int, default=32, help='num of object classes (with void)') 132 | parser.add_argument('--cuda', type=str, default='0', help='GPU ids used for training') 133 | parser.add_argument('--use_gpu', type=bool, default=True, help='whether to user gpu for training') 134 | parser.add_argument('--pretrained_model_path', type=str, default=None, help='path to pretrained model') 135 | parser.add_argument('--save_model_path', type=str, default=None, help='path to save model') 136 | parser.add_argument('--optimizer', type=str, default='rmsprop', help='optimizer, support rmsprop, sgd, adam') 137 | parser.add_argument('--loss', type=str, default='dice', help='loss function, dice or crossentropy') 138 | 139 | args = parser.parse_args(params) 140 | 141 | # create dataset and dataloader 142 | train_path = [os.path.join(args.data, 'train'), os.path.join(args.data, 'val')] 143 | train_label_path = [os.path.join(args.data, 'train_labels'), os.path.join(args.data, 'val_labels')] 144 | test_path = os.path.join(args.data, 'test') 145 | test_label_path = os.path.join(args.data, 'test_labels') 146 | csv_path = os.path.join(args.data, 'class_dict.csv') 147 | dataset_train = CamVid(train_path, train_label_path, csv_path, scale=(args.crop_height, args.crop_width), 148 | loss=args.loss, mode='train') 149 | dataloader_train = DataLoader( 150 | dataset_train, 151 | batch_size=args.batch_size, 152 | shuffle=True, 153 | num_workers=args.num_workers, 154 | drop_last=True 155 | ) 156 | dataset_val = CamVid(test_path, test_label_path, csv_path, scale=(args.crop_height, args.crop_width), 157 | loss=args.loss, mode='test') 158 | dataloader_val = DataLoader( 159 | dataset_val, 160 | # this has to be 1 161 | batch_size=1, 162 | shuffle=True, 163 | num_workers=args.num_workers 164 | ) 165 | 166 | # build model 167 | os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda 168 | model = BiSeNet(args.num_classes, args.context_path) 169 | if torch.cuda.is_available() and args.use_gpu: 170 | model = torch.nn.DataParallel(model).cuda() 171 | 172 | # build optimizer 173 | if args.optimizer == 'rmsprop': 174 | optimizer = torch.optim.RMSprop(model.parameters(), args.learning_rate) 175 | elif args.optimizer == 'sgd': 176 | optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=0.9, weight_decay=1e-4) 177 | elif args.optimizer == 'adam': 178 | optimizer = torch.optim.Adam(model.parameters(), args.learning_rate) 179 | else: # rmsprop 180 | print('not supported optimizer \n') 181 | return None 182 | 183 | # load pretrained model if exists 184 | if args.pretrained_model_path is not None: 185 | print('load model from %s ...' % args.pretrained_model_path) 186 | model.module.load_state_dict(torch.load(args.pretrained_model_path)) 187 | print('Done!') 188 | 189 | # train 190 | train(args, model, optimizer, dataloader_train, dataloader_val) 191 | 192 | # val(args, model, dataloader_val, csv_path) 193 | 194 | 195 | if __name__ == '__main__': 196 | params = [ 197 | '--num_epochs', '1000', 198 | '--learning_rate', '2.5e-2', 199 | '--data', '/path/to/CamVid', 200 | '--num_workers', '8', 201 | '--num_classes', '12', 202 | '--cuda', '0', 203 | '--batch_size', '2', # 6 for resnet101, 12 for resnet18 204 | '--save_model_path', './checkpoints_18_sgd', 205 | '--context_path', 'resnet18', # only support resnet18 and resnet101 206 | '--optimizer', 'sgd', 207 | 208 | ] 209 | main(params) 210 | 211 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | from torch.nn import functional as F 4 | from PIL import Image 5 | import numpy as np 6 | import pandas as pd 7 | import random 8 | import numbers 9 | import torchvision 10 | 11 | def poly_lr_scheduler(optimizer, init_lr, iter, lr_decay_iter=1, 12 | max_iter=300, power=0.9): 13 | """Polynomial decay of learning rate 14 | :param init_lr is base learning rate 15 | :param iter is a current iteration 16 | :param lr_decay_iter how frequently decay occurs, default is 1 17 | :param max_iter is number of maximum iterations 18 | :param power is a polymomial power 19 | 20 | """ 21 | # if iter % lr_decay_iter or iter > max_iter: 22 | # return optimizer 23 | 24 | lr = init_lr*(1 - iter/max_iter)**power 25 | optimizer.param_groups[0]['lr'] = lr 26 | return lr 27 | # return lr 28 | 29 | def get_label_info(csv_path): 30 | # return label -> {label_name: [r_value, g_value, b_value, ...} 31 | ann = pd.read_csv(csv_path) 32 | label = {} 33 | for iter, row in ann.iterrows(): 34 | label_name = row['name'] 35 | r = row['r'] 36 | g = row['g'] 37 | b = row['b'] 38 | class_11 = row['class_11'] 39 | label[label_name] = [int(r), int(g), int(b), class_11] 40 | return label 41 | 42 | def one_hot_it(label, label_info): 43 | # return semantic_map -> [H, W] 44 | semantic_map = np.zeros(label.shape[:-1]) 45 | for index, info in enumerate(label_info): 46 | color = label_info[info] 47 | # colour_map = np.full((label.shape[0], label.shape[1], label.shape[2]), colour, dtype=int) 48 | equality = np.equal(label, color) 49 | class_map = np.all(equality, axis=-1) 50 | semantic_map[class_map] = index 51 | # semantic_map.append(class_map) 52 | # semantic_map = np.stack(semantic_map, axis=-1) 53 | return semantic_map 54 | 55 | 56 | def one_hot_it_v11(label, label_info): 57 | # return semantic_map -> [H, W, class_num] 58 | semantic_map = np.zeros(label.shape[:-1]) 59 | # from 0 to 11, and 11 means void 60 | class_index = 0 61 | for index, info in enumerate(label_info): 62 | color = label_info[info][:3] 63 | class_11 = label_info[info][3] 64 | if class_11 == 1: 65 | # colour_map = np.full((label.shape[0], label.shape[1], label.shape[2]), colour, dtype=int) 66 | equality = np.equal(label, color) 67 | class_map = np.all(equality, axis=-1) 68 | # semantic_map[class_map] = index 69 | semantic_map[class_map] = class_index 70 | class_index += 1 71 | else: 72 | equality = np.equal(label, color) 73 | class_map = np.all(equality, axis=-1) 74 | semantic_map[class_map] = 11 75 | return semantic_map 76 | 77 | def one_hot_it_v11_dice(label, label_info): 78 | # return semantic_map -> [H, W, class_num] 79 | semantic_map = [] 80 | void = np.zeros(label.shape[:2]) 81 | for index, info in enumerate(label_info): 82 | color = label_info[info][:3] 83 | class_11 = label_info[info][3] 84 | if class_11 == 1: 85 | # colour_map = np.full((label.shape[0], label.shape[1], label.shape[2]), colour, dtype=int) 86 | equality = np.equal(label, color) 87 | class_map = np.all(equality, axis=-1) 88 | # semantic_map[class_map] = index 89 | semantic_map.append(class_map) 90 | else: 91 | equality = np.equal(label, color) 92 | class_map = np.all(equality, axis=-1) 93 | void[class_map] = 1 94 | semantic_map.append(void) 95 | semantic_map = np.stack(semantic_map, axis=-1).astype(np.float) 96 | return semantic_map 97 | 98 | def reverse_one_hot(image): 99 | """ 100 | Transform a 2D array in one-hot format (depth is num_classes), 101 | to a 2D array with only 1 channel, where each pixel value is 102 | the classified class key. 103 | 104 | # Arguments 105 | image: The one-hot format image 106 | 107 | # Returns 108 | A 2D array with the same width and height as the input, but 109 | with a depth size of 1, where each pixel value is the classified 110 | class key. 111 | """ 112 | # w = image.shape[0] 113 | # h = image.shape[1] 114 | # x = np.zeros([w,h,1]) 115 | 116 | # for i in range(0, w): 117 | # for j in range(0, h): 118 | # index, value = max(enumerate(image[i, j, :]), key=operator.itemgetter(1)) 119 | # x[i, j] = index 120 | image = image.permute(1, 2, 0) 121 | x = torch.argmax(image, dim=-1) 122 | return x 123 | 124 | 125 | def colour_code_segmentation(image, label_values): 126 | """ 127 | Given a 1-channel array of class keys, colour code the segmentation results. 128 | 129 | # Arguments 130 | image: single channel array where each value represents the class key. 131 | label_values 132 | 133 | # Returns 134 | Colour coded image for segmentation visualization 135 | """ 136 | 137 | # w = image.shape[0] 138 | # h = image.shape[1] 139 | # x = np.zeros([w,h,3]) 140 | # colour_codes = label_values 141 | # for i in range(0, w): 142 | # for j in range(0, h): 143 | # x[i, j, :] = colour_codes[int(image[i, j])] 144 | label_values = [label_values[key][:3] for key in label_values if label_values[key][3] == 1] 145 | label_values.append([0, 0, 0]) 146 | colour_codes = np.array(label_values) 147 | x = colour_codes[image.astype(int)] 148 | 149 | return x 150 | 151 | def compute_global_accuracy(pred, label): 152 | pred = pred.flatten() 153 | label = label.flatten() 154 | total = len(label) 155 | count = 0.0 156 | for i in range(total): 157 | if pred[i] == label[i]: 158 | count = count + 1.0 159 | return float(count) / float(total) 160 | 161 | def fast_hist(a, b, n): 162 | ''' 163 | a and b are predict and mask respectively 164 | n is the number of classes 165 | ''' 166 | k = (a >= 0) & (a < n) 167 | return np.bincount(n * a[k].astype(int) + b[k], minlength=n ** 2).reshape(n, n) 168 | 169 | 170 | def per_class_iu(hist): 171 | epsilon = 1e-5 172 | return (np.diag(hist) + epsilon) / (hist.sum(1) + hist.sum(0) - np.diag(hist) + epsilon) 173 | 174 | class RandomCrop(object): 175 | """Crop the given PIL Image at a random location. 176 | 177 | Args: 178 | size (sequence or int): Desired output size of the crop. If size is an 179 | int instead of sequence like (h, w), a square crop (size, size) is 180 | made. 181 | padding (int or sequence, optional): Optional padding on each border 182 | of the image. Default is 0, i.e no padding. If a sequence of length 183 | 4 is provided, it is used to pad left, top, right, bottom borders 184 | respectively. 185 | pad_if_needed (boolean): It will pad the image if smaller than the 186 | desired size to avoid raising an exception. 187 | """ 188 | 189 | def __init__(self, size, seed, padding=0, pad_if_needed=False): 190 | if isinstance(size, numbers.Number): 191 | self.size = (int(size), int(size)) 192 | else: 193 | self.size = size 194 | self.padding = padding 195 | self.pad_if_needed = pad_if_needed 196 | self.seed = seed 197 | 198 | @staticmethod 199 | def get_params(img, output_size, seed): 200 | """Get parameters for ``crop`` for a random crop. 201 | 202 | Args: 203 | img (PIL Image): Image to be cropped. 204 | output_size (tuple): Expected output size of the crop. 205 | 206 | Returns: 207 | tuple: params (i, j, h, w) to be passed to ``crop`` for random crop. 208 | """ 209 | random.seed(seed) 210 | w, h = img.size 211 | th, tw = output_size 212 | if w == tw and h == th: 213 | return 0, 0, h, w 214 | i = random.randint(0, h - th) 215 | j = random.randint(0, w - tw) 216 | return i, j, th, tw 217 | 218 | def __call__(self, img): 219 | """ 220 | Args: 221 | img (PIL Image): Image to be cropped. 222 | 223 | Returns: 224 | PIL Image: Cropped image. 225 | """ 226 | if self.padding > 0: 227 | img = torchvision.transforms.functional.pad(img, self.padding) 228 | 229 | # pad the width if needed 230 | if self.pad_if_needed and img.size[0] < self.size[1]: 231 | img = torchvision.transforms.functional.pad(img, (int((1 + self.size[1] - img.size[0]) / 2), 0)) 232 | # pad the height if needed 233 | if self.pad_if_needed and img.size[1] < self.size[0]: 234 | img = torchvision.transforms.functional.pad(img, (0, int((1 + self.size[0] - img.size[1]) / 2))) 235 | 236 | i, j, h, w = self.get_params(img, self.size, self.seed) 237 | 238 | return torchvision.transforms.functional.crop(img, i, j, h, w) 239 | 240 | def __repr__(self): 241 | return self.__class__.__name__ + '(size={0}, padding={1})'.format(self.size, self.padding) 242 | 243 | def cal_miou(miou_list, csv_path): 244 | # return label -> {label_name: [r_value, g_value, b_value, ...} 245 | ann = pd.read_csv(csv_path) 246 | miou_dict = {} 247 | cnt = 0 248 | for iter, row in ann.iterrows(): 249 | label_name = row['name'] 250 | class_11 = int(row['class_11']) 251 | if class_11 == 1: 252 | miou_dict[label_name] = miou_list[cnt] 253 | cnt += 1 254 | return miou_dict, np.mean(miou_list) 255 | 256 | class OHEM_CrossEntroy_Loss(nn.Module): 257 | def __init__(self, threshold, keep_num): 258 | super(OHEM_CrossEntroy_Loss, self).__init__() 259 | self.threshold = threshold 260 | self.keep_num = keep_num 261 | self.loss_function = nn.CrossEntropyLoss(reduction='none') 262 | 263 | def forward(self, output, target): 264 | loss = self.loss_function(output, target).view(-1) 265 | loss, loss_index = torch.sort(loss, descending=True) 266 | threshold_in_keep_num = loss[self.keep_num] 267 | if threshold_in_keep_num > self.threshold: 268 | loss = loss[loss>self.threshold] 269 | else: 270 | loss = loss[:self.keep_num] 271 | return torch.mean(loss) 272 | 273 | def group_weight(weight_group, module, norm_layer, lr): 274 | group_decay = [] 275 | group_no_decay = [] 276 | for m in module.modules(): 277 | if isinstance(m, nn.Linear): 278 | group_decay.append(m.weight) 279 | if m.bias is not None: 280 | group_no_decay.append(m.bias) 281 | elif isinstance(m, (nn.Conv2d, nn.Conv3d)): 282 | group_decay.append(m.weight) 283 | if m.bias is not None: 284 | group_no_decay.append(m.bias) 285 | elif isinstance(m, norm_layer) or isinstance(m, nn.GroupNorm): 286 | if m.weight is not None: 287 | group_no_decay.append(m.weight) 288 | if m.bias is not None: 289 | group_no_decay.append(m.bias) 290 | 291 | assert len(list(module.parameters())) == len(group_decay) + len( 292 | group_no_decay) 293 | weight_group.append(dict(params=group_decay, lr=lr)) 294 | weight_group.append(dict(params=group_no_decay, weight_decay=.0, lr=lr)) 295 | return weight_group 296 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import torchvision 9 | 10 | from resnet import Resnet18 11 | # from modules.bn import InPlaceABNSync as BatchNorm2d 12 | 13 | 14 | class ConvBNReLU(nn.Module): 15 | def __init__(self, in_chan, out_chan, ks=3, stride=1, padding=1, *args, **kwargs): 16 | super(ConvBNReLU, self).__init__() 17 | self.conv = nn.Conv2d(in_chan, 18 | out_chan, 19 | kernel_size = ks, 20 | stride = stride, 21 | padding = padding, 22 | bias = False) 23 | self.bn = nn.BatchNorm2d(out_chan) 24 | self.init_weight() 25 | 26 | def forward(self, x): 27 | x = self.conv(x) 28 | x = F.relu(self.bn(x)) 29 | return x 30 | 31 | def init_weight(self): 32 | for ly in self.children(): 33 | if isinstance(ly, nn.Conv2d): 34 | nn.init.kaiming_normal_(ly.weight, a=1) 35 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 36 | 37 | class BiSeNetOutput(nn.Module): 38 | def __init__(self, in_chan, mid_chan, n_classes, *args, **kwargs): 39 | super(BiSeNetOutput, self).__init__() 40 | self.conv = ConvBNReLU(in_chan, mid_chan, ks=3, stride=1, padding=1) 41 | self.conv_out = nn.Conv2d(mid_chan, n_classes, kernel_size=1, bias=False) 42 | self.init_weight() 43 | 44 | def forward(self, x): 45 | x = self.conv(x) 46 | x = self.conv_out(x) 47 | return x 48 | 49 | def init_weight(self): 50 | for ly in self.children(): 51 | if isinstance(ly, nn.Conv2d): 52 | nn.init.kaiming_normal_(ly.weight, a=1) 53 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 54 | 55 | def get_params(self): 56 | wd_params, nowd_params = [], [] 57 | for name, module in self.named_modules(): 58 | if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): 59 | wd_params.append(module.weight) 60 | if not module.bias is None: 61 | nowd_params.append(module.bias) 62 | elif isinstance(module, nn.BatchNorm2d): 63 | nowd_params += list(module.parameters()) 64 | return wd_params, nowd_params 65 | 66 | 67 | class AttentionRefinementModule(nn.Module): 68 | def __init__(self, in_chan, out_chan, *args, **kwargs): 69 | super(AttentionRefinementModule, self).__init__() 70 | self.conv = ConvBNReLU(in_chan, out_chan, ks=3, stride=1, padding=1) 71 | self.conv_atten = nn.Conv2d(out_chan, out_chan, kernel_size= 1, bias=False) 72 | self.bn_atten = nn.BatchNorm2d(out_chan) 73 | self.sigmoid_atten = nn.Sigmoid() 74 | self.init_weight() 75 | 76 | def forward(self, x): 77 | feat = self.conv(x) 78 | atten = F.avg_pool2d(feat, feat.size()[2:]) 79 | atten = self.conv_atten(atten) 80 | atten = self.bn_atten(atten) 81 | atten = self.sigmoid_atten(atten) 82 | out = torch.mul(feat, atten) 83 | return out 84 | 85 | def init_weight(self): 86 | for ly in self.children(): 87 | if isinstance(ly, nn.Conv2d): 88 | nn.init.kaiming_normal_(ly.weight, a=1) 89 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 90 | 91 | 92 | class ContextPath(nn.Module): 93 | def __init__(self, *args, **kwargs): 94 | super(ContextPath, self).__init__() 95 | self.resnet = Resnet18() 96 | self.arm16 = AttentionRefinementModule(256, 128) 97 | self.arm32 = AttentionRefinementModule(512, 128) 98 | self.conv_head32 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1) 99 | self.conv_head16 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1) 100 | self.conv_avg = ConvBNReLU(512, 128, ks=1, stride=1, padding=0) 101 | 102 | self.init_weight() 103 | 104 | def forward(self, x): 105 | H0, W0 = x.size()[2:] 106 | feat8, feat16, feat32 = self.resnet(x) 107 | H8, W8 = feat8.size()[2:] 108 | H16, W16 = feat16.size()[2:] 109 | H32, W32 = feat32.size()[2:] 110 | 111 | avg = F.avg_pool2d(feat32, feat32.size()[2:]) 112 | avg = self.conv_avg(avg) 113 | avg_up = F.interpolate(avg, (H32, W32), mode='nearest') 114 | 115 | feat32_arm = self.arm32(feat32) 116 | feat32_sum = feat32_arm + avg_up 117 | feat32_up = F.interpolate(feat32_sum, (H16, W16), mode='nearest') 118 | feat32_up = self.conv_head32(feat32_up) 119 | 120 | feat16_arm = self.arm16(feat16) 121 | feat16_sum = feat16_arm + feat32_up 122 | feat16_up = F.interpolate(feat16_sum, (H8, W8), mode='nearest') 123 | feat16_up = self.conv_head16(feat16_up) 124 | 125 | return feat8, feat16_up, feat32_up # x8, x8, x16 126 | 127 | def init_weight(self): 128 | for ly in self.children(): 129 | if isinstance(ly, nn.Conv2d): 130 | nn.init.kaiming_normal_(ly.weight, a=1) 131 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 132 | 133 | def get_params(self): 134 | wd_params, nowd_params = [], [] 135 | for name, module in self.named_modules(): 136 | if isinstance(module, (nn.Linear, nn.Conv2d)): 137 | wd_params.append(module.weight) 138 | if not module.bias is None: 139 | nowd_params.append(module.bias) 140 | elif isinstance(module, nn.BatchNorm2d): 141 | nowd_params += list(module.parameters()) 142 | return wd_params, nowd_params 143 | 144 | 145 | ### This is not used, since I replace this with the resnet feature with the same size 146 | class SpatialPath(nn.Module): 147 | def __init__(self, *args, **kwargs): 148 | super(SpatialPath, self).__init__() 149 | self.conv1 = ConvBNReLU(3, 64, ks=7, stride=2, padding=3) 150 | self.conv2 = ConvBNReLU(64, 64, ks=3, stride=2, padding=1) 151 | self.conv3 = ConvBNReLU(64, 64, ks=3, stride=2, padding=1) 152 | self.conv_out = ConvBNReLU(64, 128, ks=1, stride=1, padding=0) 153 | self.init_weight() 154 | 155 | def forward(self, x): 156 | feat = self.conv1(x) 157 | feat = self.conv2(feat) 158 | feat = self.conv3(feat) 159 | feat = self.conv_out(feat) 160 | return feat 161 | 162 | def init_weight(self): 163 | for ly in self.children(): 164 | if isinstance(ly, nn.Conv2d): 165 | nn.init.kaiming_normal_(ly.weight, a=1) 166 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 167 | 168 | def get_params(self): 169 | wd_params, nowd_params = [], [] 170 | for name, module in self.named_modules(): 171 | if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): 172 | wd_params.append(module.weight) 173 | if not module.bias is None: 174 | nowd_params.append(module.bias) 175 | elif isinstance(module, nn.BatchNorm2d): 176 | nowd_params += list(module.parameters()) 177 | return wd_params, nowd_params 178 | 179 | 180 | class FeatureFusionModule(nn.Module): 181 | def __init__(self, in_chan, out_chan, *args, **kwargs): 182 | super(FeatureFusionModule, self).__init__() 183 | self.convblk = ConvBNReLU(in_chan, out_chan, ks=1, stride=1, padding=0) 184 | self.conv1 = nn.Conv2d(out_chan, 185 | out_chan//4, 186 | kernel_size = 1, 187 | stride = 1, 188 | padding = 0, 189 | bias = False) 190 | self.conv2 = nn.Conv2d(out_chan//4, 191 | out_chan, 192 | kernel_size = 1, 193 | stride = 1, 194 | padding = 0, 195 | bias = False) 196 | self.relu = nn.ReLU(inplace=True) 197 | self.sigmoid = nn.Sigmoid() 198 | self.init_weight() 199 | 200 | def forward(self, fsp, fcp): 201 | fcat = torch.cat([fsp, fcp], dim=1) 202 | feat = self.convblk(fcat) 203 | atten = F.avg_pool2d(feat, feat.size()[2:]) 204 | atten = self.conv1(atten) 205 | atten = self.relu(atten) 206 | atten = self.conv2(atten) 207 | atten = self.sigmoid(atten) 208 | feat_atten = torch.mul(feat, atten) 209 | feat_out = feat_atten + feat 210 | return feat_out 211 | 212 | def init_weight(self): 213 | for ly in self.children(): 214 | if isinstance(ly, nn.Conv2d): 215 | nn.init.kaiming_normal_(ly.weight, a=1) 216 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 217 | 218 | def get_params(self): 219 | wd_params, nowd_params = [], [] 220 | for name, module in self.named_modules(): 221 | if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): 222 | wd_params.append(module.weight) 223 | if not module.bias is None: 224 | nowd_params.append(module.bias) 225 | elif isinstance(module, nn.BatchNorm2d): 226 | nowd_params += list(module.parameters()) 227 | return wd_params, nowd_params 228 | 229 | 230 | class BiSeNet(nn.Module): 231 | def __init__(self, n_classes, *args, **kwargs): 232 | super(BiSeNet, self).__init__() 233 | self.cp = ContextPath() 234 | ## here self.sp is deleted 235 | self.ffm = FeatureFusionModule(256, 256) 236 | self.conv_out = BiSeNetOutput(256, 256, n_classes) 237 | self.conv_out16 = BiSeNetOutput(128, 64, n_classes) 238 | self.conv_out32 = BiSeNetOutput(128, 64, n_classes) 239 | self.init_weight() 240 | 241 | def forward(self, x): 242 | H, W = x.size()[2:] 243 | feat_res8, feat_cp8, feat_cp16 = self.cp(x) # here return res3b1 feature 244 | feat_sp = feat_res8 # use res3b1 feature to replace spatial path feature 245 | feat_fuse = self.ffm(feat_sp, feat_cp8) 246 | 247 | feat_out = self.conv_out(feat_fuse) 248 | feat_out16 = self.conv_out16(feat_cp8) 249 | feat_out32 = self.conv_out32(feat_cp16) 250 | 251 | feat_out = F.interpolate(feat_out, (H, W), mode='bilinear', align_corners=True) 252 | feat_out16 = F.interpolate(feat_out16, (H, W), mode='bilinear', align_corners=True) 253 | feat_out32 = F.interpolate(feat_out32, (H, W), mode='bilinear', align_corners=True) 254 | return feat_out, feat_out16, feat_out32 255 | 256 | def init_weight(self): 257 | for ly in self.children(): 258 | if isinstance(ly, nn.Conv2d): 259 | nn.init.kaiming_normal_(ly.weight, a=1) 260 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 261 | 262 | def get_params(self): 263 | wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params = [], [], [], [] 264 | for name, child in self.named_children(): 265 | child_wd_params, child_nowd_params = child.get_params() 266 | if isinstance(child, FeatureFusionModule) or isinstance(child, BiSeNetOutput): 267 | lr_mul_wd_params += child_wd_params 268 | lr_mul_nowd_params += child_nowd_params 269 | else: 270 | wd_params += child_wd_params 271 | nowd_params += child_nowd_params 272 | return wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params 273 | 274 | 275 | if __name__ == "__main__": 276 | net = BiSeNet(19) 277 | net.cuda() 278 | net.eval() 279 | in_ten = torch.randn(16, 3, 640, 480).cuda() 280 | out, out16, out32 = net(in_ten) 281 | print(out.shape) 282 | 283 | net.get_params() 284 | -------------------------------------------------------------------------------- /detection_models/networks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from detection_models.sync_batchnorm import DataParallelWithCallback 8 | from detection_models.antialiasing import Downsample 9 | 10 | 11 | class UNet(nn.Module): 12 | def __init__( 13 | self, 14 | in_channels=3, 15 | out_channels=3, 16 | depth=5, 17 | conv_num=2, 18 | wf=6, 19 | padding=True, 20 | batch_norm=True, 21 | up_mode="upsample", 22 | with_tanh=False, 23 | sync_bn=True, 24 | antialiasing=True, 25 | ): 26 | """ 27 | Implementation of 28 | U-Net: Convolutional Networks for Biomedical Image Segmentation 29 | (Ronneberger et al., 2015) 30 | https://arxiv.org/abs/1505.04597 31 | Using the default arguments will yield the exact version used 32 | in the original paper 33 | Args: 34 | in_channels (int): number of input channels 35 | out_channels (int): number of output channels 36 | depth (int): depth of the network 37 | wf (int): number of filters in the first layer is 2**wf 38 | padding (bool): if True, apply padding such that the input shape 39 | is the same as the output. 40 | This may introduce artifacts 41 | batch_norm (bool): Use BatchNorm after layers with an 42 | activation function 43 | up_mode (str): one of 'upconv' or 'upsample'. 44 | 'upconv' will use transposed convolutions for 45 | learned upsampling. 46 | 'upsample' will use bilinear upsampling. 47 | """ 48 | super().__init__() 49 | assert up_mode in ("upconv", "upsample") 50 | self.padding = padding 51 | self.depth = depth - 1 52 | prev_channels = in_channels 53 | 54 | self.first = nn.Sequential( 55 | *[nn.ReflectionPad2d(3), nn.Conv2d(in_channels, 2 ** wf, kernel_size=7), nn.LeakyReLU(0.2, True)] 56 | ) 57 | prev_channels = 2 ** wf 58 | 59 | self.down_path = nn.ModuleList() 60 | self.down_sample = nn.ModuleList() 61 | for i in range(depth): 62 | if antialiasing and depth > 0: 63 | self.down_sample.append( 64 | nn.Sequential( 65 | *[ 66 | nn.ReflectionPad2d(1), 67 | nn.Conv2d(prev_channels, prev_channels, kernel_size=3, stride=1, padding=0), 68 | nn.BatchNorm2d(prev_channels), 69 | nn.LeakyReLU(0.2, True), 70 | Downsample(channels=prev_channels, stride=2), 71 | ] 72 | ) 73 | ) 74 | else: 75 | self.down_sample.append( 76 | nn.Sequential( 77 | *[ 78 | nn.ReflectionPad2d(1), 79 | nn.Conv2d(prev_channels, prev_channels, kernel_size=4, stride=2, padding=0), 80 | nn.BatchNorm2d(prev_channels), 81 | nn.LeakyReLU(0.2, True), 82 | ] 83 | ) 84 | ) 85 | self.down_path.append( 86 | UNetConvBlock(conv_num, prev_channels, 2 ** (wf + i + 1), padding, batch_norm) 87 | ) 88 | prev_channels = 2 ** (wf + i + 1) 89 | 90 | self.up_path = nn.ModuleList() 91 | for i in reversed(range(depth)): 92 | self.up_path.append( 93 | UNetUpBlock(conv_num, prev_channels, 2 ** (wf + i), up_mode, padding, batch_norm) 94 | ) 95 | prev_channels = 2 ** (wf + i) 96 | 97 | if with_tanh: 98 | self.last = nn.Sequential( 99 | *[nn.ReflectionPad2d(1), nn.Conv2d(prev_channels, out_channels, kernel_size=3), nn.Tanh()] 100 | ) 101 | else: 102 | self.last = nn.Sequential( 103 | *[nn.ReflectionPad2d(1), nn.Conv2d(prev_channels, out_channels, kernel_size=3)] 104 | ) 105 | 106 | if sync_bn: 107 | self = DataParallelWithCallback(self) 108 | 109 | def forward(self, x): 110 | x = self.first(x) 111 | 112 | blocks = [] 113 | for i, down_block in enumerate(self.down_path): 114 | blocks.append(x) 115 | x = self.down_sample[i](x) 116 | x = down_block(x) 117 | 118 | for i, up in enumerate(self.up_path): 119 | x = up(x, blocks[-i - 1]) 120 | 121 | return self.last(x) 122 | 123 | 124 | class UNetConvBlock(nn.Module): 125 | def __init__(self, conv_num, in_size, out_size, padding, batch_norm): 126 | super(UNetConvBlock, self).__init__() 127 | block = [] 128 | 129 | for _ in range(conv_num): 130 | block.append(nn.ReflectionPad2d(padding=int(padding))) 131 | block.append(nn.Conv2d(in_size, out_size, kernel_size=3, padding=0)) 132 | if batch_norm: 133 | block.append(nn.BatchNorm2d(out_size)) 134 | block.append(nn.LeakyReLU(0.2, True)) 135 | in_size = out_size 136 | 137 | self.block = nn.Sequential(*block) 138 | 139 | def forward(self, x): 140 | out = self.block(x) 141 | return out 142 | 143 | 144 | class UNetUpBlock(nn.Module): 145 | def __init__(self, conv_num, in_size, out_size, up_mode, padding, batch_norm): 146 | super(UNetUpBlock, self).__init__() 147 | if up_mode == "upconv": 148 | self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=2, stride=2) 149 | elif up_mode == "upsample": 150 | self.up = nn.Sequential( 151 | nn.Upsample(mode="bilinear", scale_factor=2, align_corners=False), 152 | nn.ReflectionPad2d(1), 153 | nn.Conv2d(in_size, out_size, kernel_size=3, padding=0), 154 | ) 155 | 156 | self.conv_block = UNetConvBlock(conv_num, in_size, out_size, padding, batch_norm) 157 | 158 | def center_crop(self, layer, target_size): 159 | _, _, layer_height, layer_width = layer.size() 160 | diff_y = (layer_height - target_size[0]) // 2 161 | diff_x = (layer_width - target_size[1]) // 2 162 | return layer[:, :, diff_y : (diff_y + target_size[0]), diff_x : (diff_x + target_size[1])] 163 | 164 | def forward(self, x, bridge): 165 | up = self.up(x) 166 | crop1 = self.center_crop(bridge, up.shape[2:]) 167 | out = torch.cat([up, crop1], 1) 168 | out = self.conv_block(out) 169 | 170 | return out 171 | 172 | 173 | class UnetGenerator(nn.Module): 174 | """Create a Unet-based generator""" 175 | 176 | def __init__(self, input_nc, output_nc, num_downs, ngf=64, norm_type="BN", use_dropout=False): 177 | """Construct a Unet generator 178 | Parameters: 179 | input_nc (int) -- the number of channels in input images 180 | output_nc (int) -- the number of channels in output images 181 | num_downs (int) -- the number of downsamplings in UNet. For example, # if |num_downs| == 7, 182 | image of size 128x128 will become of size 1x1 # at the bottleneck 183 | ngf (int) -- the number of filters in the last conv layer 184 | norm_layer -- normalization layer 185 | We construct the U-Net from the innermost layer to the outermost layer. 186 | It is a recursive process. 187 | """ 188 | super().__init__() 189 | if norm_type == "BN": 190 | norm_layer = nn.BatchNorm2d 191 | elif norm_type == "IN": 192 | norm_layer = nn.InstanceNorm2d 193 | else: 194 | raise NameError("Unknown norm layer") 195 | 196 | # construct unet structure 197 | unet_block = UnetSkipConnectionBlock( 198 | ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True 199 | ) # add the innermost layer 200 | for i in range(num_downs - 5): # add intermediate layers with ngf * 8 filters 201 | unet_block = UnetSkipConnectionBlock( 202 | ngf * 8, 203 | ngf * 8, 204 | input_nc=None, 205 | submodule=unet_block, 206 | norm_layer=norm_layer, 207 | use_dropout=use_dropout, 208 | ) 209 | # gradually reduce the number of filters from ngf * 8 to ngf 210 | unet_block = UnetSkipConnectionBlock( 211 | ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer 212 | ) 213 | unet_block = UnetSkipConnectionBlock( 214 | ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer 215 | ) 216 | unet_block = UnetSkipConnectionBlock( 217 | ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer 218 | ) 219 | self.model = UnetSkipConnectionBlock( 220 | output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer 221 | ) # add the outermost layer 222 | 223 | def forward(self, input): 224 | return self.model(input) 225 | 226 | 227 | class UnetSkipConnectionBlock(nn.Module): 228 | """Defines the Unet submodule with skip connection. 229 | 230 | -------------------identity---------------------- 231 | |-- downsampling -- |submodule| -- upsampling --| 232 | """ 233 | 234 | def __init__( 235 | self, 236 | outer_nc, 237 | inner_nc, 238 | input_nc=None, 239 | submodule=None, 240 | outermost=False, 241 | innermost=False, 242 | norm_layer=nn.BatchNorm2d, 243 | use_dropout=False, 244 | ): 245 | """Construct a Unet submodule with skip connections. 246 | Parameters: 247 | outer_nc (int) -- the number of filters in the outer conv layer 248 | inner_nc (int) -- the number of filters in the inner conv layer 249 | input_nc (int) -- the number of channels in input images/features 250 | submodule (UnetSkipConnectionBlock) -- previously defined submodules 251 | outermost (bool) -- if this module is the outermost module 252 | innermost (bool) -- if this module is the innermost module 253 | norm_layer -- normalization layer 254 | user_dropout (bool) -- if use dropout layers. 255 | """ 256 | super().__init__() 257 | self.outermost = outermost 258 | use_bias = norm_layer == nn.InstanceNorm2d 259 | if input_nc is None: 260 | input_nc = outer_nc 261 | downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4, stride=2, padding=1, bias=use_bias) 262 | downrelu = nn.LeakyReLU(0.2, True) 263 | downnorm = norm_layer(inner_nc) 264 | uprelu = nn.LeakyReLU(0.2, True) 265 | upnorm = norm_layer(outer_nc) 266 | 267 | if outermost: 268 | upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1) 269 | down = [downconv] 270 | up = [uprelu, upconv, nn.Tanh()] 271 | model = down + [submodule] + up 272 | elif innermost: 273 | upconv = nn.ConvTranspose2d(inner_nc, outer_nc, kernel_size=4, stride=2, padding=1, bias=use_bias) 274 | down = [downrelu, downconv] 275 | up = [uprelu, upconv, upnorm] 276 | model = down + up 277 | else: 278 | upconv = nn.ConvTranspose2d( 279 | inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1, bias=use_bias 280 | ) 281 | down = [downrelu, downconv, downnorm] 282 | up = [uprelu, upconv, upnorm] 283 | 284 | if use_dropout: 285 | model = down + [submodule] + up + [nn.Dropout(0.5)] 286 | else: 287 | model = down + [submodule] + up 288 | 289 | self.model = nn.Sequential(*model) 290 | 291 | def forward(self, x): 292 | if self.outermost: 293 | return self.model(x) 294 | else: # add skip connections 295 | return torch.cat([x, self.model(x)], 1) 296 | 297 | 298 | # ============================================ 299 | # Network testing 300 | # ============================================ 301 | if __name__ == "__main__": 302 | from torchsummary import summary 303 | 304 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 305 | 306 | model = UNet_two_decoders( 307 | in_channels=3, 308 | out_channels1=3, 309 | out_channels2=1, 310 | depth=4, 311 | conv_num=1, 312 | wf=6, 313 | padding=True, 314 | batch_norm=True, 315 | up_mode="upsample", 316 | with_tanh=False, 317 | ) 318 | model.to(device) 319 | 320 | model_pix2pix = UnetGenerator(3, 3, 5, ngf=64, norm_type="BN", use_dropout=False) 321 | model_pix2pix.to(device) 322 | 323 | print("customized unet:") 324 | summary(model, (3, 256, 256)) 325 | 326 | print("cyclegan unet:") 327 | summary(model_pix2pix, (3, 256, 256)) 328 | 329 | x = torch.zeros(1, 3, 256, 256).requires_grad_(True).cuda() 330 | g = make_dot(model(x)) 331 | g.render("models/Digraph.gv", view=False) 332 | 333 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # Creative Commons Attribution-NonCommercial 4.0 International 2 | 3 | Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. 4 | 5 | ### Using Creative Commons Public Licenses 6 | 7 | Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. 8 | 9 | * __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). 10 | 11 | * __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). 12 | 13 | ## Creative Commons Attribution-NonCommercial 4.0 International Public License 14 | 15 | By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. 16 | 17 | ### Section 1 – Definitions. 18 | 19 | a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. 20 | 21 | b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. 22 | 23 | c. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. 24 | 25 | d. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. 26 | 27 | e. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. 28 | 29 | f. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. 30 | 31 | g. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. 32 | 33 | h. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. 34 | 35 | i. __NonCommercial__ means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. 36 | 37 | j. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. 38 | 39 | k. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. 40 | 41 | l. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. 42 | 43 | ### Section 2 – Scope. 44 | 45 | a. ___License grant.___ 46 | 47 | 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: 48 | 49 | A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and 50 | 51 | B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only. 52 | 53 | 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. 54 | 55 | 3. __Term.__ The term of this Public License is specified in Section 6(a). 56 | 57 | 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. 58 | 59 | 5. __Downstream recipients.__ 60 | 61 | A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. 62 | 63 | B. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. 64 | 65 | 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). 66 | 67 | b. ___Other rights.___ 68 | 69 | 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. 70 | 71 | 2. Patent and trademark rights are not licensed under this Public License. 72 | 73 | 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes. 74 | 75 | ### Section 3 – License Conditions. 76 | 77 | Your exercise of the Licensed Rights is expressly made subject to the following conditions. 78 | 79 | a. ___Attribution.___ 80 | 81 | 1. If You Share the Licensed Material (including in modified form), You must: 82 | 83 | A. retain the following if it is supplied by the Licensor with the Licensed Material: 84 | 85 | i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); 86 | 87 | ii. a copyright notice; 88 | 89 | iii. a notice that refers to this Public License; 90 | 91 | iv. a notice that refers to the disclaimer of warranties; 92 | 93 | v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; 94 | 95 | B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and 96 | 97 | C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. 98 | 99 | 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. 100 | 101 | 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. 102 | 103 | 4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License. 104 | 105 | ### Section 4 – Sui Generis Database Rights. 106 | 107 | Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: 108 | 109 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; 110 | 111 | b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and 112 | 113 | c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. 114 | 115 | For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. 116 | 117 | ### Section 5 – Disclaimer of Warranties and Limitation of Liability. 118 | 119 | a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ 120 | 121 | b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ 122 | 123 | c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. 124 | 125 | ### Section 6 – Term and Termination. 126 | 127 | a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. 128 | 129 | b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: 130 | 131 | 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or 132 | 133 | 2. upon express reinstatement by the Licensor. 134 | 135 | For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. 136 | 137 | c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. 138 | 139 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. 140 | 141 | ### Section 7 – Other Terms and Conditions. 142 | 143 | a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. 144 | 145 | b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. 146 | 147 | ### Section 8 – Interpretation. 148 | 149 | a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. 150 | 151 | b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. 152 | 153 | c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. 154 | 155 | d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. 156 | 157 | > Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. 158 | > 159 | > Creative Commons may be contacted at creativecommons.org 160 | -------------------------------------------------------------------------------- /detection_models/sync_batchnorm/batchnorm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : batchnorm.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import collections 12 | import contextlib 13 | 14 | import torch 15 | import torch.nn.functional as F 16 | 17 | from torch.nn.modules.batchnorm import _BatchNorm 18 | 19 | try: 20 | from torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast 21 | except ImportError: 22 | ReduceAddCoalesced = Broadcast = None 23 | 24 | try: 25 | from jactorch.parallel.comm import SyncMaster 26 | from jactorch.parallel.data_parallel import JacDataParallel as DataParallelWithCallback 27 | except ImportError: 28 | from .comm import SyncMaster 29 | from .replicate import DataParallelWithCallback 30 | 31 | __all__ = [ 32 | 'set_sbn_eps_mode', 33 | 'SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', 'SynchronizedBatchNorm3d', 34 | 'patch_sync_batchnorm', 'convert_model' 35 | ] 36 | 37 | 38 | SBN_EPS_MODE = 'clamp' 39 | 40 | 41 | def set_sbn_eps_mode(mode): 42 | global SBN_EPS_MODE 43 | assert mode in ('clamp', 'plus') 44 | SBN_EPS_MODE = mode 45 | 46 | 47 | def _sum_ft(tensor): 48 | """sum over the first and last dimention""" 49 | return tensor.sum(dim=0).sum(dim=-1) 50 | 51 | 52 | def _unsqueeze_ft(tensor): 53 | """add new dimensions at the front and the tail""" 54 | return tensor.unsqueeze(0).unsqueeze(-1) 55 | 56 | 57 | _ChildMessage = collections.namedtuple('_ChildMessage', ['sum', 'ssum', 'sum_size']) 58 | _MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std']) 59 | 60 | 61 | class _SynchronizedBatchNorm(_BatchNorm): 62 | def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True): 63 | assert ReduceAddCoalesced is not None, 'Can not use Synchronized Batch Normalization without CUDA support.' 64 | 65 | super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine, 66 | track_running_stats=track_running_stats) 67 | 68 | if not self.track_running_stats: 69 | import warnings 70 | warnings.warn('track_running_stats=False is not supported by the SynchronizedBatchNorm.') 71 | 72 | self._sync_master = SyncMaster(self._data_parallel_master) 73 | 74 | self._is_parallel = False 75 | self._parallel_id = None 76 | self._slave_pipe = None 77 | 78 | def forward(self, input): 79 | # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation. 80 | if not (self._is_parallel and self.training): 81 | return F.batch_norm( 82 | input, self.running_mean, self.running_var, self.weight, self.bias, 83 | self.training, self.momentum, self.eps) 84 | 85 | # Resize the input to (B, C, -1). 86 | input_shape = input.size() 87 | assert input.size(1) == self.num_features, 'Channel size mismatch: got {}, expect {}.'.format(input.size(1), self.num_features) 88 | input = input.view(input.size(0), self.num_features, -1) 89 | 90 | # Compute the sum and square-sum. 91 | sum_size = input.size(0) * input.size(2) 92 | input_sum = _sum_ft(input) 93 | input_ssum = _sum_ft(input ** 2) 94 | 95 | # Reduce-and-broadcast the statistics. 96 | if self._parallel_id == 0: 97 | mean, inv_std = self._sync_master.run_master(_ChildMessage(input_sum, input_ssum, sum_size)) 98 | else: 99 | mean, inv_std = self._slave_pipe.run_slave(_ChildMessage(input_sum, input_ssum, sum_size)) 100 | 101 | # Compute the output. 102 | if self.affine: 103 | # MJY:: Fuse the multiplication for speed. 104 | output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias) 105 | else: 106 | output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std) 107 | 108 | # Reshape it. 109 | return output.view(input_shape) 110 | 111 | def __data_parallel_replicate__(self, ctx, copy_id): 112 | self._is_parallel = True 113 | self._parallel_id = copy_id 114 | 115 | # parallel_id == 0 means master device. 116 | if self._parallel_id == 0: 117 | ctx.sync_master = self._sync_master 118 | else: 119 | self._slave_pipe = ctx.sync_master.register_slave(copy_id) 120 | 121 | def _data_parallel_master(self, intermediates): 122 | """Reduce the sum and square-sum, compute the statistics, and broadcast it.""" 123 | 124 | # Always using same "device order" makes the ReduceAdd operation faster. 125 | # Thanks to:: Tete Xiao (http://tetexiao.com/) 126 | intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device()) 127 | 128 | to_reduce = [i[1][:2] for i in intermediates] 129 | to_reduce = [j for i in to_reduce for j in i] # flatten 130 | target_gpus = [i[1].sum.get_device() for i in intermediates] 131 | 132 | sum_size = sum([i[1].sum_size for i in intermediates]) 133 | sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce) 134 | mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size) 135 | 136 | broadcasted = Broadcast.apply(target_gpus, mean, inv_std) 137 | 138 | outputs = [] 139 | for i, rec in enumerate(intermediates): 140 | outputs.append((rec[0], _MasterMessage(*broadcasted[i*2:i*2+2]))) 141 | 142 | return outputs 143 | 144 | def _compute_mean_std(self, sum_, ssum, size): 145 | """Compute the mean and standard-deviation with sum and square-sum. This method 146 | also maintains the moving average on the master device.""" 147 | assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.' 148 | mean = sum_ / size 149 | sumvar = ssum - sum_ * mean 150 | unbias_var = sumvar / (size - 1) 151 | bias_var = sumvar / size 152 | 153 | if hasattr(torch, 'no_grad'): 154 | with torch.no_grad(): 155 | self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data 156 | self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data 157 | else: 158 | self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data 159 | self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data 160 | 161 | if SBN_EPS_MODE == 'clamp': 162 | return mean, bias_var.clamp(self.eps) ** -0.5 163 | elif SBN_EPS_MODE == 'plus': 164 | return mean, (bias_var + self.eps) ** -0.5 165 | else: 166 | raise ValueError('Unknown EPS mode: {}.'.format(SBN_EPS_MODE)) 167 | 168 | 169 | class SynchronizedBatchNorm1d(_SynchronizedBatchNorm): 170 | r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a 171 | mini-batch. 172 | 173 | .. math:: 174 | 175 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 176 | 177 | This module differs from the built-in PyTorch BatchNorm1d as the mean and 178 | standard-deviation are reduced across all devices during training. 179 | 180 | For example, when one uses `nn.DataParallel` to wrap the network during 181 | training, PyTorch's implementation normalize the tensor on each device using 182 | the statistics only on that device, which accelerated the computation and 183 | is also easy to implement, but the statistics might be inaccurate. 184 | Instead, in this synchronized version, the statistics will be computed 185 | over all training samples distributed on multiple devices. 186 | 187 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 188 | as the built-in PyTorch implementation. 189 | 190 | The mean and standard-deviation are calculated per-dimension over 191 | the mini-batches and gamma and beta are learnable parameter vectors 192 | of size C (where C is the input size). 193 | 194 | During training, this layer keeps a running estimate of its computed mean 195 | and variance. The running sum is kept with a default momentum of 0.1. 196 | 197 | During evaluation, this running mean/variance is used for normalization. 198 | 199 | Because the BatchNorm is done over the `C` dimension, computing statistics 200 | on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm 201 | 202 | Args: 203 | num_features: num_features from an expected input of size 204 | `batch_size x num_features [x width]` 205 | eps: a value added to the denominator for numerical stability. 206 | Default: 1e-5 207 | momentum: the value used for the running_mean and running_var 208 | computation. Default: 0.1 209 | affine: a boolean value that when set to ``True``, gives the layer learnable 210 | affine parameters. Default: ``True`` 211 | 212 | Shape:: 213 | - Input: :math:`(N, C)` or :math:`(N, C, L)` 214 | - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input) 215 | 216 | Examples: 217 | >>> # With Learnable Parameters 218 | >>> m = SynchronizedBatchNorm1d(100) 219 | >>> # Without Learnable Parameters 220 | >>> m = SynchronizedBatchNorm1d(100, affine=False) 221 | >>> input = torch.autograd.Variable(torch.randn(20, 100)) 222 | >>> output = m(input) 223 | """ 224 | 225 | def _check_input_dim(self, input): 226 | if input.dim() != 2 and input.dim() != 3: 227 | raise ValueError('expected 2D or 3D input (got {}D input)' 228 | .format(input.dim())) 229 | 230 | 231 | class SynchronizedBatchNorm2d(_SynchronizedBatchNorm): 232 | r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch 233 | of 3d inputs 234 | 235 | .. math:: 236 | 237 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 238 | 239 | This module differs from the built-in PyTorch BatchNorm2d as the mean and 240 | standard-deviation are reduced across all devices during training. 241 | 242 | For example, when one uses `nn.DataParallel` to wrap the network during 243 | training, PyTorch's implementation normalize the tensor on each device using 244 | the statistics only on that device, which accelerated the computation and 245 | is also easy to implement, but the statistics might be inaccurate. 246 | Instead, in this synchronized version, the statistics will be computed 247 | over all training samples distributed on multiple devices. 248 | 249 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 250 | as the built-in PyTorch implementation. 251 | 252 | The mean and standard-deviation are calculated per-dimension over 253 | the mini-batches and gamma and beta are learnable parameter vectors 254 | of size C (where C is the input size). 255 | 256 | During training, this layer keeps a running estimate of its computed mean 257 | and variance. The running sum is kept with a default momentum of 0.1. 258 | 259 | During evaluation, this running mean/variance is used for normalization. 260 | 261 | Because the BatchNorm is done over the `C` dimension, computing statistics 262 | on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm 263 | 264 | Args: 265 | num_features: num_features from an expected input of 266 | size batch_size x num_features x height x width 267 | eps: a value added to the denominator for numerical stability. 268 | Default: 1e-5 269 | momentum: the value used for the running_mean and running_var 270 | computation. Default: 0.1 271 | affine: a boolean value that when set to ``True``, gives the layer learnable 272 | affine parameters. Default: ``True`` 273 | 274 | Shape:: 275 | - Input: :math:`(N, C, H, W)` 276 | - Output: :math:`(N, C, H, W)` (same shape as input) 277 | 278 | Examples: 279 | >>> # With Learnable Parameters 280 | >>> m = SynchronizedBatchNorm2d(100) 281 | >>> # Without Learnable Parameters 282 | >>> m = SynchronizedBatchNorm2d(100, affine=False) 283 | >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45)) 284 | >>> output = m(input) 285 | """ 286 | 287 | def _check_input_dim(self, input): 288 | if input.dim() != 4: 289 | raise ValueError('expected 4D input (got {}D input)' 290 | .format(input.dim())) 291 | 292 | 293 | class SynchronizedBatchNorm3d(_SynchronizedBatchNorm): 294 | r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch 295 | of 4d inputs 296 | 297 | .. math:: 298 | 299 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 300 | 301 | This module differs from the built-in PyTorch BatchNorm3d as the mean and 302 | standard-deviation are reduced across all devices during training. 303 | 304 | For example, when one uses `nn.DataParallel` to wrap the network during 305 | training, PyTorch's implementation normalize the tensor on each device using 306 | the statistics only on that device, which accelerated the computation and 307 | is also easy to implement, but the statistics might be inaccurate. 308 | Instead, in this synchronized version, the statistics will be computed 309 | over all training samples distributed on multiple devices. 310 | 311 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 312 | as the built-in PyTorch implementation. 313 | 314 | The mean and standard-deviation are calculated per-dimension over 315 | the mini-batches and gamma and beta are learnable parameter vectors 316 | of size C (where C is the input size). 317 | 318 | During training, this layer keeps a running estimate of its computed mean 319 | and variance. The running sum is kept with a default momentum of 0.1. 320 | 321 | During evaluation, this running mean/variance is used for normalization. 322 | 323 | Because the BatchNorm is done over the `C` dimension, computing statistics 324 | on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm 325 | or Spatio-temporal BatchNorm 326 | 327 | Args: 328 | num_features: num_features from an expected input of 329 | size batch_size x num_features x depth x height x width 330 | eps: a value added to the denominator for numerical stability. 331 | Default: 1e-5 332 | momentum: the value used for the running_mean and running_var 333 | computation. Default: 0.1 334 | affine: a boolean value that when set to ``True``, gives the layer learnable 335 | affine parameters. Default: ``True`` 336 | 337 | Shape:: 338 | - Input: :math:`(N, C, D, H, W)` 339 | - Output: :math:`(N, C, D, H, W)` (same shape as input) 340 | 341 | Examples: 342 | >>> # With Learnable Parameters 343 | >>> m = SynchronizedBatchNorm3d(100) 344 | >>> # Without Learnable Parameters 345 | >>> m = SynchronizedBatchNorm3d(100, affine=False) 346 | >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10)) 347 | >>> output = m(input) 348 | """ 349 | 350 | def _check_input_dim(self, input): 351 | if input.dim() != 5: 352 | raise ValueError('expected 5D input (got {}D input)' 353 | .format(input.dim())) 354 | 355 | 356 | @contextlib.contextmanager 357 | def patch_sync_batchnorm(): 358 | import torch.nn as nn 359 | 360 | backup = nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d 361 | 362 | nn.BatchNorm1d = SynchronizedBatchNorm1d 363 | nn.BatchNorm2d = SynchronizedBatchNorm2d 364 | nn.BatchNorm3d = SynchronizedBatchNorm3d 365 | 366 | yield 367 | 368 | nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d = backup 369 | 370 | 371 | def convert_model(module): 372 | """Traverse the input module and its child recursively 373 | and replace all instance of torch.nn.modules.batchnorm.BatchNorm*N*d 374 | to SynchronizedBatchNorm*N*d 375 | 376 | Args: 377 | module: the input module needs to be convert to SyncBN model 378 | 379 | Examples: 380 | >>> import torch.nn as nn 381 | >>> import torchvision 382 | >>> # m is a standard pytorch model 383 | >>> m = torchvision.models.resnet18(True) 384 | >>> m = nn.DataParallel(m) 385 | >>> # after convert, m is using SyncBN 386 | >>> m = convert_model(m) 387 | """ 388 | if isinstance(module, torch.nn.DataParallel): 389 | mod = module.module 390 | mod = convert_model(mod) 391 | mod = DataParallelWithCallback(mod, device_ids=module.device_ids) 392 | return mod 393 | 394 | mod = module 395 | for pth_module, sync_module in zip([torch.nn.modules.batchnorm.BatchNorm1d, 396 | torch.nn.modules.batchnorm.BatchNorm2d, 397 | torch.nn.modules.batchnorm.BatchNorm3d], 398 | [SynchronizedBatchNorm1d, 399 | SynchronizedBatchNorm2d, 400 | SynchronizedBatchNorm3d]): 401 | if isinstance(module, pth_module): 402 | mod = sync_module(module.num_features, module.eps, module.momentum, module.affine) 403 | mod.running_mean = module.running_mean 404 | mod.running_var = module.running_var 405 | if module.affine: 406 | mod.weight.data = module.weight.data.clone().detach() 407 | mod.bias.data = module.bias.data.clone().detach() 408 | 409 | for name, child in module.named_children(): 410 | mod.add_module(name, convert_model(child)) 411 | 412 | return mod 413 | -------------------------------------------------------------------------------- /src/pipeline_stable_diffusion_controlnet_inpaint.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import PIL.Image 3 | import numpy as np 4 | 5 | from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import * 6 | 7 | EXAMPLE_DOC_STRING = """ 8 | Examples: 9 | ```py 10 | >>> # !pip install opencv-python transformers accelerate 11 | >>> from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, UniPCMultistepScheduler 12 | >>> from diffusers.utils import load_image 13 | >>> import numpy as np 14 | >>> import torch 15 | >>> import cv2 16 | >>> from PIL import Image 17 | >>> # download an image 18 | >>> image = load_image( 19 | ... "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" 20 | ... ) 21 | >>> image = np.array(image) 22 | >>> mask_image = load_image( 23 | ... "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" 24 | ... ) 25 | >>> mask_image = np.array(mask_image) 26 | >>> # get canny image 27 | >>> canny_image = cv2.Canny(image, 100, 200) 28 | >>> canny_image = canny_image[:, :, None] 29 | >>> canny_image = np.concatenate([canny_image, canny_image, canny_image], axis=2) 30 | >>> canny_image = Image.fromarray(canny_image) 31 | >>> # load control net and stable diffusion v1-5 32 | >>> controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16) 33 | >>> pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( 34 | ... "runwayml/stable-diffusion-inpainting", controlnet=controlnet, torch_dtype=torch.float16 35 | ... ) 36 | >>> # speed up diffusion process with faster scheduler and memory optimization 37 | >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) 38 | >>> # remove following line if xformers is not installed 39 | >>> pipe.enable_xformers_memory_efficient_attention() 40 | >>> pipe.enable_model_cpu_offload() 41 | >>> # generate image 42 | >>> generator = torch.manual_seed(0) 43 | >>> image = pipe( 44 | ... "futuristic-looking doggo", 45 | ... num_inference_steps=20, 46 | ... generator=generator, 47 | ... image=image, 48 | ... control_image=canny_image, 49 | ... mask_image=mask_image 50 | ... ).images[0] 51 | ``` 52 | """ 53 | 54 | 55 | def prepare_mask_and_masked_image(image, mask): 56 | """ 57 | Prepares a pair (image, mask) to be consumed by the Stable Diffusion pipeline. This means that those inputs will be 58 | converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the 59 | ``image`` and ``1`` for the ``mask``. 60 | The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be 61 | binarized (``mask > 0.5``) and cast to ``torch.float32`` too. 62 | Args: 63 | image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. 64 | It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` 65 | ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. 66 | mask (_type_): The mask to apply to the image, i.e. regions to inpaint. 67 | It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` 68 | ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. 69 | Raises: 70 | ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask 71 | should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. 72 | TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not 73 | (ot the other way around). 74 | Returns: 75 | tuple[torch.Tensor]: The pair (mask, masked_image) as ``torch.Tensor`` with 4 76 | dimensions: ``batch x channels x height x width``. 77 | """ 78 | if isinstance(image, torch.Tensor): 79 | if not isinstance(mask, torch.Tensor): 80 | raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not") 81 | 82 | # Batch single image 83 | if image.ndim == 3: 84 | assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)" 85 | image = image.unsqueeze(0) 86 | 87 | # Batch and add channel dim for single mask 88 | if mask.ndim == 2: 89 | mask = mask.unsqueeze(0).unsqueeze(0) 90 | 91 | # Batch single mask or add channel dim 92 | if mask.ndim == 3: 93 | # Single batched mask, no channel dim or single mask not batched but channel dim 94 | if mask.shape[0] == 1: 95 | mask = mask.unsqueeze(0) 96 | 97 | # Batched masks no channel dim 98 | else: 99 | mask = mask.unsqueeze(1) 100 | 101 | assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" 102 | assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" 103 | assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" 104 | 105 | # Check image is in [-1, 1] 106 | if image.min() < -1 or image.max() > 1: 107 | raise ValueError("Image should be in [-1, 1] range") 108 | 109 | # Check mask is in [0, 1] 110 | if mask.min() < 0 or mask.max() > 1: 111 | raise ValueError("Mask should be in [0, 1] range") 112 | 113 | # Binarize mask 114 | mask[mask < 0.5] = 0 115 | mask[mask >= 0.5] = 1 116 | 117 | # Image as float32 118 | image = image.to(dtype=torch.float32) 119 | elif isinstance(mask, torch.Tensor): 120 | raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") 121 | else: 122 | # preprocess image 123 | if isinstance(image, (PIL.Image.Image, np.ndarray)): 124 | image = [image] 125 | 126 | if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): 127 | image = [np.array(i.convert("RGB"))[None, :] for i in image] 128 | image = np.concatenate(image, axis=0) 129 | elif isinstance(image, list) and isinstance(image[0], np.ndarray): 130 | image = np.concatenate([i[None, :] for i in image], axis=0) 131 | 132 | image = image.transpose(0, 3, 1, 2) 133 | image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 134 | 135 | # preprocess mask 136 | if isinstance(mask, (PIL.Image.Image, np.ndarray)): 137 | mask = [mask] 138 | 139 | if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): 140 | mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) 141 | mask = mask.astype(np.float32) / 255.0 142 | elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): 143 | mask = np.concatenate([m[None, None, :] for m in mask], axis=0) 144 | 145 | mask[mask < 0.5] = 0 146 | mask[mask >= 0.5] = 1 147 | mask = torch.from_numpy(mask) 148 | 149 | masked_image = image * (mask < 0.5) 150 | 151 | return mask, masked_image 152 | 153 | class StableDiffusionControlNetInpaintPipeline(StableDiffusionControlNetPipeline): 154 | r""" 155 | Pipeline for text-guided image inpainting using Stable Diffusion with ControlNet guidance. 156 | This model inherits from [`StableDiffusionControlNetPipeline`]. Check the superclass documentation for the generic methods the 157 | library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) 158 | Args: 159 | vae ([`AutoencoderKL`]): 160 | Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. 161 | text_encoder ([`CLIPTextModel`]): 162 | Frozen text-encoder. Stable Diffusion uses the text portion of 163 | [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically 164 | the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. 165 | tokenizer (`CLIPTokenizer`): 166 | Tokenizer of class 167 | [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). 168 | unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. 169 | controlnet ([`ControlNetModel`]): 170 | Provides additional conditioning to the unet during the denoising process 171 | scheduler ([`SchedulerMixin`]): 172 | A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of 173 | [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. 174 | safety_checker ([`StableDiffusionSafetyChecker`]): 175 | Classification module that estimates whether generated images could be considered offensive or harmful. 176 | Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. 177 | feature_extractor ([`CLIPFeatureExtractor`]): 178 | Model that extracts features from generated images to be used as inputs for the `safety_checker`. 179 | """ 180 | 181 | def prepare_mask_latents( 182 | self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance 183 | ): 184 | # resize the mask to latents shape as we concatenate the mask to the latents 185 | # we do that before converting to dtype to avoid breaking in case we're using cpu_offload 186 | # and half precision 187 | mask = torch.nn.functional.interpolate( 188 | mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) 189 | ) 190 | mask = mask.to(device=device, dtype=dtype) 191 | 192 | masked_image = masked_image.to(device=device, dtype=dtype) 193 | 194 | # encode the mask image into latents space so we can concatenate it to the latents 195 | if isinstance(generator, list): 196 | masked_image_latents = [ 197 | self.vae.encode(masked_image[i : i + 1]).latent_dist.sample(generator=generator[i]) 198 | for i in range(batch_size) 199 | ] 200 | masked_image_latents = torch.cat(masked_image_latents, dim=0) 201 | else: 202 | masked_image_latents = self.vae.encode(masked_image).latent_dist.sample(generator=generator) 203 | masked_image_latents = self.vae.config.scaling_factor * masked_image_latents 204 | 205 | # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method 206 | if mask.shape[0] < batch_size: 207 | if not batch_size % mask.shape[0] == 0: 208 | raise ValueError( 209 | "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to" 210 | f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number" 211 | " of masks that you pass is divisible by the total requested batch size." 212 | ) 213 | mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1) 214 | if masked_image_latents.shape[0] < batch_size: 215 | if not batch_size % masked_image_latents.shape[0] == 0: 216 | raise ValueError( 217 | "The passed images and the required batch size don't match. Images are supposed to be duplicated" 218 | f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed." 219 | " Make sure the number of images that you pass is divisible by the total requested batch size." 220 | ) 221 | masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1) 222 | 223 | mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask 224 | masked_image_latents = ( 225 | torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents 226 | ) 227 | 228 | # aligning device to prevent device errors when concating it with the latent model input 229 | masked_image_latents = masked_image_latents.to(device=device, dtype=dtype) 230 | return mask, masked_image_latents 231 | 232 | @torch.no_grad() 233 | @replace_example_docstring(EXAMPLE_DOC_STRING) 234 | def __call__( 235 | self, 236 | prompt: Union[str, List[str]] = None, 237 | image: Union[torch.FloatTensor, PIL.Image.Image] = None, 238 | control_image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]] = None, 239 | mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None, 240 | height: Optional[int] = None, 241 | width: Optional[int] = None, 242 | num_inference_steps: int = 50, 243 | guidance_scale: float = 7.5, 244 | negative_prompt: Optional[Union[str, List[str]]] = None, 245 | num_images_per_prompt: Optional[int] = 1, 246 | eta: float = 0.0, 247 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, 248 | latents: Optional[torch.FloatTensor] = None, 249 | prompt_embeds: Optional[torch.FloatTensor] = None, 250 | negative_prompt_embeds: Optional[torch.FloatTensor] = None, 251 | output_type: Optional[str] = "pil", 252 | return_dict: bool = True, 253 | callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, 254 | callback_steps: int = 1, 255 | cross_attention_kwargs: Optional[Dict[str, Any]] = None, 256 | controlnet_conditioning_scale: float = 1.0, 257 | ): 258 | r""" 259 | Function invoked when calling the pipeline for generation. 260 | Args: 261 | prompt (`str` or `List[str]`, *optional*): 262 | The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. 263 | instead. 264 | image (`PIL.Image.Image`): 265 | `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will 266 | be masked out with `mask_image` and repainted according to `prompt`. 267 | control_image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]` or `List[PIL.Image.Image]`): 268 | The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If 269 | the type is specified as `Torch.FloatTensor`, it is passed to ControlNet as is. PIL.Image.Image` can 270 | also be accepted as an image. The control image is automatically resized to fit the output image. 271 | mask_image (`PIL.Image.Image`): 272 | `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be 273 | repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted 274 | to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) 275 | instead of 3, so the expected shape would be `(B, H, W, 1)`. 276 | height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): 277 | The height in pixels of the generated image. 278 | width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): 279 | The width in pixels of the generated image. 280 | num_inference_steps (`int`, *optional*, defaults to 50): 281 | The number of denoising steps. More denoising steps usually lead to a higher quality image at the 282 | expense of slower inference. 283 | guidance_scale (`float`, *optional*, defaults to 7.5): 284 | Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). 285 | `guidance_scale` is defined as `w` of equation 2. of [Imagen 286 | Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > 287 | 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, 288 | usually at the expense of lower image quality. 289 | negative_prompt (`str` or `List[str]`, *optional*): 290 | The prompt or prompts not to guide the image generation. If not defined, one has to pass 291 | `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. 292 | Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). 293 | num_images_per_prompt (`int`, *optional*, defaults to 1): 294 | The number of images to generate per prompt. 295 | eta (`float`, *optional*, defaults to 0.0): 296 | Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to 297 | [`schedulers.DDIMScheduler`], will be ignored for others. 298 | generator (`torch.Generator` or `List[torch.Generator]`, *optional*): 299 | One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) 300 | to make generation deterministic. 301 | latents (`torch.FloatTensor`, *optional*): 302 | Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image 303 | generation. Can be used to tweak the same generation with different prompts. If not provided, a latents 304 | tensor will ge generated by sampling using the supplied random `generator`. 305 | prompt_embeds (`torch.FloatTensor`, *optional*): 306 | Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not 307 | provided, text embeddings will be generated from `prompt` input argument. 308 | negative_prompt_embeds (`torch.FloatTensor`, *optional*): 309 | Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt 310 | weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input 311 | argument. 312 | output_type (`str`, *optional*, defaults to `"pil"`): 313 | The output format of the generate image. Choose between 314 | [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. 315 | return_dict (`bool`, *optional*, defaults to `True`): 316 | Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a 317 | plain tuple. 318 | callback (`Callable`, *optional*): 319 | A function that will be called every `callback_steps` steps during inference. The function will be 320 | called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. 321 | callback_steps (`int`, *optional*, defaults to 1): 322 | The frequency at which the `callback` function will be called. If not specified, the callback will be 323 | called at every step. 324 | cross_attention_kwargs (`dict`, *optional*): 325 | A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under 326 | `self.processor` in 327 | [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py). 328 | controlnet_conditioning_scale (`float`, *optional*, defaults to 1.0): 329 | The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added 330 | to the residual in the original unet. 331 | Examples: 332 | Returns: 333 | [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: 334 | [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. 335 | When returning a tuple, the first element is a list with the generated images, and the second element is a 336 | list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" 337 | (nsfw) content, according to the `safety_checker`. 338 | """ 339 | # 0. Default height and width to unet 340 | height, width = self._default_height_width(height, width, control_image) 341 | 342 | # 1. Check inputs. Raise error if not correct 343 | self.check_inputs( 344 | prompt, control_image, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds 345 | ) 346 | 347 | # 2. Define call parameters 348 | if prompt is not None and isinstance(prompt, str): 349 | batch_size = 1 350 | elif prompt is not None and isinstance(prompt, list): 351 | batch_size = len(prompt) 352 | else: 353 | batch_size = prompt_embeds.shape[0] 354 | 355 | device = self._execution_device 356 | # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) 357 | # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` 358 | # corresponds to doing no classifier free guidance. 359 | do_classifier_free_guidance = guidance_scale > 1.0 360 | 361 | # 3. Encode input prompt 362 | prompt_embeds = self._encode_prompt( 363 | prompt, 364 | device, 365 | num_images_per_prompt, 366 | do_classifier_free_guidance, 367 | negative_prompt, 368 | prompt_embeds=prompt_embeds, 369 | negative_prompt_embeds=negative_prompt_embeds, 370 | ) 371 | 372 | # 4. Prepare image 373 | control_image = self.prepare_image( 374 | control_image, 375 | width, 376 | height, 377 | batch_size * num_images_per_prompt, 378 | num_images_per_prompt, 379 | device, 380 | self.controlnet.dtype, 381 | ) 382 | 383 | if do_classifier_free_guidance: 384 | control_image = torch.cat([control_image] * 2) 385 | 386 | # 5. Prepare timesteps 387 | self.scheduler.set_timesteps(num_inference_steps, device=device) 388 | timesteps = self.scheduler.timesteps 389 | 390 | # 6. Prepare latent variables 391 | num_channels_latents = self.controlnet.in_channels 392 | latents = self.prepare_latents( 393 | batch_size * num_images_per_prompt, 394 | num_channels_latents, 395 | height, 396 | width, 397 | prompt_embeds.dtype, 398 | device, 399 | generator, 400 | latents, 401 | ) 402 | 403 | # EXTRA: prepare mask latents 404 | mask, masked_image = prepare_mask_and_masked_image(image, mask_image) 405 | mask, masked_image_latents = self.prepare_mask_latents( 406 | mask, 407 | masked_image, 408 | batch_size * num_images_per_prompt, 409 | height, 410 | width, 411 | prompt_embeds.dtype, 412 | device, 413 | generator, 414 | do_classifier_free_guidance, 415 | ) 416 | 417 | # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline 418 | extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) 419 | 420 | # 8. Denoising loop 421 | num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order 422 | with self.progress_bar(total=num_inference_steps) as progress_bar: 423 | for i, t in enumerate(timesteps): 424 | # expand the latents if we are doing classifier free guidance 425 | latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents 426 | latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) 427 | 428 | down_block_res_samples, mid_block_res_sample = self.controlnet( 429 | latent_model_input, 430 | t, 431 | encoder_hidden_states=prompt_embeds, 432 | controlnet_cond=control_image, 433 | return_dict=False, 434 | ) 435 | 436 | down_block_res_samples = [ 437 | down_block_res_sample * controlnet_conditioning_scale 438 | for down_block_res_sample in down_block_res_samples 439 | ] 440 | mid_block_res_sample *= controlnet_conditioning_scale 441 | 442 | # predict the noise residual 443 | latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1) 444 | noise_pred = self.unet( 445 | latent_model_input, 446 | t, 447 | encoder_hidden_states=prompt_embeds, 448 | cross_attention_kwargs=cross_attention_kwargs, 449 | down_block_additional_residuals=down_block_res_samples, 450 | mid_block_additional_residual=mid_block_res_sample, 451 | ).sample 452 | 453 | # perform guidance 454 | if do_classifier_free_guidance: 455 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) 456 | noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) 457 | 458 | # compute the previous noisy sample x_t -> x_t-1 459 | latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample 460 | 461 | # call the callback, if provided 462 | if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): 463 | progress_bar.update() 464 | if callback is not None and i % callback_steps == 0: 465 | callback(i, t, latents) 466 | 467 | # If we do sequential model offloading, let's offload unet and controlnet 468 | # manually for max memory savings 469 | if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: 470 | self.unet.to("cpu") 471 | self.controlnet.to("cpu") 472 | torch.cuda.empty_cache() 473 | 474 | if output_type == "latent": 475 | image = latents 476 | has_nsfw_concept = None 477 | elif output_type == "pil": 478 | # 8. Post-processing 479 | image = self.decode_latents(latents) 480 | 481 | # 9. Run safety checker 482 | image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) 483 | 484 | # 10. Convert to PIL 485 | image = self.numpy_to_pil(image) 486 | else: 487 | # 8. Post-processing 488 | image = self.decode_latents(latents) 489 | 490 | # 9. Run safety checker 491 | image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) 492 | 493 | # Offload last model to CPU 494 | if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: 495 | self.final_offload_hook.offload() 496 | 497 | if not return_dict: 498 | return (image, has_nsfw_concept) 499 | 500 | return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 198 | 199 | 200 | 201 | bounding 202 | quokka_segmentation_map 203 | quokka_heatmap 204 | SegmentationMapOnImage 205 | interpolation 206 | winname 207 | 208 | 209 | 210 | 212 | 213 | 232 | 233 | 234 | 235 | 236 | true 237 | DEFINITION_ORDER 238 | 239 | 240 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 |