├── .gitignore
├── README.md
├── detectron2_gradcam.py
├── gradcam.py
├── img
    ├── grad_cam++.png
    ├── grad_cam.png
    └── input.jpg
└── main.py


/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | pip-wheel-metadata/
 24 | share/python-wheels/
 25 | *.egg-info/
 26 | .installed.cfg
 27 | *.egg
 28 | MANIFEST
 29 | 
 30 | # PyInstaller
 31 | #  Usually these files are written by a python script from a template
 32 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 33 | *.manifest
 34 | *.spec
 35 | 
 36 | # Installer logs
 37 | pip-log.txt
 38 | pip-delete-this-directory.txt
 39 | 
 40 | # Unit test / coverage reports
 41 | htmlcov/
 42 | .tox/
 43 | .nox/
 44 | .coverage
 45 | .coverage.*
 46 | .cache
 47 | nosetests.xml
 48 | coverage.xml
 49 | *.cover
 50 | *.py,cover
 51 | .hypothesis/
 52 | .pytest_cache/
 53 | 
 54 | # Translations
 55 | *.mo
 56 | *.pot
 57 | 
 58 | # Django stuff:
 59 | *.log
 60 | local_settings.py
 61 | db.sqlite3
 62 | db.sqlite3-journal
 63 | 
 64 | # Flask stuff:
 65 | instance/
 66 | .webassets-cache
 67 | 
 68 | # Scrapy stuff:
 69 | .scrapy
 70 | 
 71 | # Sphinx documentation
 72 | docs/_build/
 73 | 
 74 | # PyBuilder
 75 | target/
 76 | 
 77 | # Jupyter Notebook
 78 | .ipynb_checkpoints
 79 | 
 80 | # IPython
 81 | profile_default/
 82 | ipython_config.py
 83 | 
 84 | # pyenv
 85 | .python-version
 86 | 
 87 | # pipenv
 88 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 89 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 90 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 91 | #   install all needed dependencies.
 92 | #Pipfile.lock
 93 | 
 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 95 | __pypackages__/
 96 | 
 97 | # Celery stuff
 98 | celerybeat-schedule
 99 | celerybeat.pid
100 | 
101 | # SageMath parsed files
102 | *.sage.py
103 | 
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 | 
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 | 
117 | # Rope project settings
118 | .ropeproject
119 | 
120 | # mkdocs documentation
121 | /site
122 | 
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 | 
128 | # Pyre type checker
129 | .pyre/
130 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # detectron2-GradCAM
 2 | This repo helps you to perform [GradCAM](https://arxiv.org/abs/1610.02391) and [GradCAM++](https://arxiv.org/abs/1710.11063) on the [detectron2](https://github.com/facebookresearch/detectron2) model zoo. It follows other GradCAM implementations but also handles the detectron2 API specific model details. Be sure to have the latest detectron2 version installed. 
 3 | 
 4 | There is also [this](https://github.com/yizt/Grad-CAM.pytorch) repo to do GradCAM in detectron2. It advises you to make changes to the detectron2 build which I think is not a good idea..
 5 | 
 6 | | Original        | GradCAM (horse)           | GradCAM++  (horse)           |
 7 | | ------------- |:-------------:| :-------------:|
 8 | | <img src="https://github.com/alexriedel1/detectron2-GradCAM/blob/main/img/input.jpg" alt="drawing" width="400"/>| <img src="https://github.com/alexriedel1/detectron2-GradCAM/blob/main/img/grad_cam.png" alt="drawing" width="400"/> | <img src="https://github.com/alexriedel1/detectron2-GradCAM/blob/main/img/grad_cam++.png" alt="drawing" width="400"/> |
 9 | 
10 | 
11 | For doing this, check the `main.py` and change the `img_path`, the `config_file` and the `model_file` according to your needs. 
12 | 
13 | For ResNet50 models the layer `backbone.bottom_up.res5.2.conv3` will be a good choice for the classification explanation. For larger or smaller models, change the layer accordingly via `layer_name`.
14 | 
15 | 
16 | For your custom models, either write your own config.yaml or edit [`cfg_list`](https://github.com/alexriedel1/detectron2-GradCAM/blob/main/main.py#L15)
17 | 
18 | 
19 | There's also a Colab with everything set up: [GradCam Detecteron2](https://colab.research.google.com/drive/15GN0juUurMPCDHA3tGp6nJ4mxUiSHknZ)
20 | 


--------------------------------------------------------------------------------
/detectron2_gradcam.py:
--------------------------------------------------------------------------------
  1 | from gradcam import GradCAM, GradCamPlusPlus
  2 | import detectron2.data.transforms as T
  3 | import torch
  4 | from detectron2.checkpoint import DetectionCheckpointer
  5 | from detectron2.config import get_cfg
  6 | from detectron2.data import DatasetCatalog, MetadataCatalog
  7 | from detectron2.data.detection_utils import read_image
  8 | from detectron2.modeling import build_model
  9 | from detectron2.data.datasets import register_coco_instances
 10 | 
 11 | class Detectron2GradCAM():
 12 |   """
 13 |       Attributes
 14 |     ----------
 15 |     config_file : str
 16 |         detectron2 model config file path
 17 |     cfg_list : list
 18 |         List of additional model configurations
 19 |     root_dir : str [optional]
 20 |         directory of coco.josn and dataset images for custom dataset registration
 21 |     custom_dataset : str [optional]
 22 |         Name of the custom dataset to register
 23 |     """
 24 |   def __init__(self, config_file, cfg_list, img_path, root_dir=None, custom_dataset=None):
 25 |       # load config from file
 26 |       cfg = get_cfg()
 27 |       cfg.merge_from_file(config_file)
 28 | 
 29 |       if custom_dataset:
 30 |           register_coco_instances(custom_dataset, {}, root_dir + "coco.json", root_dir)
 31 |           cfg.DATASETS.TRAIN = (custom_dataset,)
 32 |           MetadataCatalog.get(custom_dataset)
 33 |           DatasetCatalog.get(custom_dataset)
 34 | 
 35 |       if torch.cuda.is_available():
 36 |           cfg.MODEL.DEVICE = "cuda"
 37 |       else:
 38 |           cfg.MODEL.DEVICE = "cpu"
 39 | 
 40 |       cfg.merge_from_list(cfg_list)
 41 |       cfg.freeze()
 42 |       self.cfg =  cfg
 43 |       self._set_input_image(img_path)
 44 | 
 45 |   def _set_input_image(self, img_path):
 46 |       self.image = read_image(img_path, format="BGR")
 47 |       self.image_height, self.image_width = self.image.shape[:2]
 48 |       transform_gen = T.ResizeShortestEdge(
 49 |           [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST
 50 |       )
 51 |       transformed_img = transform_gen.get_transform(self.image).apply_image(self.image)
 52 |       self.input_tensor = torch.as_tensor(transformed_img.astype("float32").transpose(2, 0, 1)).requires_grad_(True)
 53 |   
 54 |   def get_cam(self, target_instance, layer_name, grad_cam_instance):
 55 |       """
 56 |       Calls the GradCAM instance
 57 | 
 58 |       Parameters
 59 |       ----------
 60 |       img : str
 61 |           Path to inference image
 62 |       target_instance : int
 63 |           The target instance index
 64 |       layer_name : str
 65 |           Convolutional layer to perform GradCAM on
 66 |       grad_cam_type : str
 67 |           GradCAM or GradCAM++ (for multiple instances of the same object, GradCAM++ can be favorable)
 68 | 
 69 |       Returns
 70 |       -------
 71 |       image_dict : dict
 72 |         {"image" : <image>, "cam" : <cam>, "output" : <output>, "label" : <label>}
 73 |         <image> original input image
 74 |         <cam> class activation map resized to original image shape
 75 |         <output> instances object generated by the model
 76 |         <label> label of the 
 77 |       cam_orig : numpy.ndarray
 78 |         unprocessed raw cam
 79 |       """
 80 |       model = build_model(self.cfg)
 81 |       checkpointer = DetectionCheckpointer(model)
 82 |       checkpointer.load(self.cfg.MODEL.WEIGHTS)
 83 | 
 84 |       input_image_dict = {"image": self.input_tensor, "height": self.image_height, "width": self.image_width}
 85 |       grad_cam = grad_cam_instance(model, layer_name)
 86 |     
 87 |       with grad_cam as cam:
 88 |         cam, cam_orig, output = cam(input_image_dict, target_instance=target_instance)
 89 |       
 90 |       output_dict = self.get_output_dict(cam, output, target_instance)
 91 |       
 92 |       return output_dict, cam_orig
 93 |     
 94 |   def get_output_dict(self, cam, output, target_instance):
 95 |       image_dict = {}
 96 |       image_dict["image"] =  self.image
 97 |       image_dict["cam"] = cam
 98 |       image_dict["output"] = output
 99 |       image_dict["label"] = MetadataCatalog.get(self.cfg.DATASETS.TRAIN[0]).thing_classes[output["instances"].pred_classes[target_instance]]
100 |       return image_dict
101 | 


--------------------------------------------------------------------------------
/gradcam.py:
--------------------------------------------------------------------------------
  1 | import cv2
  2 | import numpy as np
  3 | 
  4 | class GradCAM():
  5 |     """
  6 |     Class to implement the GradCam function with it's necessary Pytorch hooks.
  7 | 
  8 |     Attributes
  9 |     ----------
 10 |     model : detectron2 GeneralizedRCNN Model
 11 |         A model using the detectron2 API for inferencing
 12 |     layer_name : str
 13 |         name of the convolutional layer to perform GradCAM with
 14 |     """
 15 | 
 16 |     def __init__(self, model, target_layer_name):
 17 |         self.model = model
 18 |         self.target_layer_name = target_layer_name
 19 |         self.activations = None
 20 |         self.gradient = None
 21 |         self.model.eval()
 22 |         self.activations_grads = []
 23 |         self._register_hook()
 24 |         self.model.zero_grad()
 25 | 
 26 |     def _get_activations_hook(self, module, input, output):
 27 |         self.activations = output
 28 | 
 29 |     def _get_grads_hook(self, module, input_grad, output_grad):
 30 |         self.gradient = output_grad[0]
 31 | 
 32 |     def _register_hook(self):
 33 |         for (name, module) in self.model.named_modules():
 34 |             if name == self.target_layer_name:
 35 |                 self.activations_grads.append(module.register_forward_hook(self._get_activations_hook))
 36 |                 self.activations_grads.append(module.register_backward_hook(self._get_grads_hook))
 37 |                 return True
 38 |         print(f"Layer {self.target_layer_name} not found in Model!")
 39 | 
 40 |     def _release_activations_grads(self):
 41 |       for handle in self.activations_grads:
 42 |             handle.remove()
 43 |     
 44 |     def _postprocess_cam(self, raw_cam, img_width, img_height):
 45 |         cam_orig = np.sum(raw_cam, axis=0)  # [H,W]
 46 |         cam_orig = np.maximum(cam_orig, 0)  # ReLU
 47 |         cam_orig -= np.min(cam_orig)
 48 |         cam_orig /= np.max(cam_orig)
 49 |         cam = cv2.resize(cam_orig, (img_width, img_height))
 50 |         return cam, cam_orig
 51 |     
 52 |     def _forward_backward_pass(self, inputs, target_instance):
 53 |         output = self.model.forward([inputs])[0]
 54 |         if target_instance == None:
 55 |           target_instance =  np.argmax(output['instances'].scores.cpu().data.numpy(), axis=-1)
 56 |         assert len(output['instances']) >= target_instance, f"Only {len(output['instances'])} objects found but you request object number {target_instance}"
 57 |         score = output['instances'].scores[target_instance]
 58 |         score.backward()
 59 |         return output
 60 |     
 61 | 
 62 |     def __enter__(self):
 63 |         return self
 64 | 
 65 |     def __exit__(self, exc_type, exc_value, exc_tb):
 66 |         self._release_activations_grads()
 67 | 
 68 |     def __call__(self, inputs, target_instance):
 69 |         """
 70 |         Calls the GradCAM++ instance
 71 | 
 72 |         Parameters
 73 |         ----------
 74 |         inputs : dict
 75 |             The input in the standard detectron2 model input format
 76 |             https://detectron2.readthedocs.io/en/latest/tutorials/models.html#model-input-format
 77 | 
 78 |         target_instance : int, optional
 79 |             The target category index. If `None` the highest scoring class will be selected
 80 | 
 81 |         Returns
 82 |         -------
 83 |         cam : np.array()
 84 |           Gradient weighted class activation map
 85 |         output : list
 86 |           list of Instance objects representing the detectron2 model output
 87 |         """
 88 | 
 89 |         output = self._forward_backward_pass(inputs, target_instance)
 90 |         gradient = self.gradient[0].cpu().data.numpy()  # [C,H,W]
 91 |         activations = self.activations[0].cpu().data.numpy()  # [C,H,W]
 92 |         weight = np.mean(gradient, axis=(1, 2))  # [C]
 93 | 
 94 |         cam = activations * weight[:, np.newaxis, np.newaxis]  # [C,H,W]
 95 |         cam, cam_orig = self._postprocess_cam(cam, inputs["width"], inputs["height"])
 96 | 
 97 |         return cam, cam_orig, output
 98 | 
 99 | class GradCamPlusPlus(GradCAM):
100 |     """
101 |     Subclass to implement the GradCam++ function with it's necessary PyTorch hooks.
102 |     ...
103 | 
104 |     Attributes
105 |     ----------
106 |     model : detectron2 GeneralizedRCNN Model
107 |         A model using the detectron2 API for inferencing
108 |     target_layer_name : str
109 |         name of the convolutional layer to perform GradCAM++ with
110 | 
111 |     """
112 |     def __init__(self, model, target_layer_name):
113 |         super().__init__(model, target_layer_name)
114 | 
115 |     def __call__(self, inputs, target_instance):
116 |         """
117 |         Calls the GradCAM++ instance
118 | 
119 |         Parameters
120 |         ----------
121 |         inputs : dict
122 |             The input in the standard detectron2 model input format
123 |             https://detectron2.readthedocs.io/en/latest/tutorials/models.html#model-input-format
124 | 
125 |         target_instance : int, optional
126 |             The target category index. If `None` the highest scoring class will be selected
127 | 
128 |         Returns
129 |         -------
130 |         cam : np.array()
131 |           Gradient weighted class activation map
132 |         output : list
133 |           list of Instance objects representing the detectron2 model output
134 |         """
135 | 
136 |         output = self._forward_backward_pass(inputs, target_instance)
137 |         gradient = self.gradient[0].cpu().data.numpy()  # [C,H,W]
138 |         activations = self.activations[0].cpu().data.numpy()  # [C,H,W]
139 | 
140 |         #from https://github.com/jacobgil/pytorch-grad-cam/blob/master/pytorch_grad_cam/grad_cam_plusplus.py
141 |         grads_power_2 = gradient**2
142 |         grads_power_3 = grads_power_2 * gradient
143 |         # Equation 19 in https://arxiv.org/abs/1710.11063
144 |         sum_activations = np.sum(activations, axis=(1, 2))
145 |         eps = 0.000001
146 |         aij = grads_power_2 / (2 * grads_power_2 +
147 |                                sum_activations[:, None, None] * grads_power_3 + eps)
148 |         # Now bring back the ReLU from eq.7 in the paper,
149 |         # And zero out aijs where the activations are 0
150 |         aij = np.where(gradient != 0, aij, 0)
151 | 
152 |         weights = np.maximum(gradient, 0) * aij
153 |         weight = np.sum(weights, axis=(1, 2))
154 | 
155 |         cam = activations * weight[:, np.newaxis, np.newaxis]  # [C,H,W]
156 |         cam, cam_orig = self._postprocess_cam(cam, inputs["width"], inputs["height"])
157 | 
158 |         return cam, cam_orig, output
159 | 


--------------------------------------------------------------------------------
/img/grad_cam++.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexriedel1/detectron2-GradCAM/f1db821c008db4695b5aefb26d6a38aa09722f86/img/grad_cam++.png


--------------------------------------------------------------------------------
/img/grad_cam.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexriedel1/detectron2-GradCAM/f1db821c008db4695b5aefb26d6a38aa09722f86/img/grad_cam.png


--------------------------------------------------------------------------------
/img/input.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexriedel1/detectron2-GradCAM/f1db821c008db4695b5aefb26d6a38aa09722f86/img/input.jpg


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
 1 | import matplotlib
 2 | import matplotlib.pyplot as plt
 3 | from detectron2_gradcam import Detectron2GradCAM
 4 | 
 5 | from detectron2.utils.visualizer import Visualizer
 6 | from detectron2.data import MetadataCatalog
 7 | from detectron2 import model_zoo
 8 | 
 9 | from gradcam import GradCAM, GradCamPlusPlus
10 | 
11 | 
12 | plt.rcParams["figure.figsize"] = (30,10)
13 | 
14 | img_path = "img/input.jpg"
15 | config_file = model_zoo.get_config_file("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml")
16 | model_file = model_zoo.get_checkpoint_url("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml")
17 | 
18 | config_list = [
19 | "MODEL.ROI_HEADS.SCORE_THRESH_TEST", "0.5",
20 | "MODEL.ROI_HEADS.NUM_CLASSES", "80",
21 | "MODEL.WEIGHTS", model_file
22 | ]
23 | 
24 | layer_name = "backbone.bottom_up.res5.2.conv3"
25 | instance = 8 #CAM is generated per object instance, not per class!
26 | 
27 | def main():
28 |     cam_extractor = Detectron2GradCAM(config_file, config_list, img_path=img_path)
29 |     grad_cam = GradCamPlusPlus
30 | 
31 |     image_dict, cam_orig = cam_extractor.get_cam(target_instance=instance, layer_name=layer_name, grad_cam_instance=grad_cam)
32 | 
33 |     v = Visualizer(image_dict["image"], MetadataCatalog.get(cam_extractor.cfg.DATASETS.TRAIN[0]), scale=1.0)
34 |     out = v.draw_instance_predictions(image_dict["output"]["instances"][instance].to("cpu"))
35 | 
36 |     plt.imshow(out.get_image(), interpolation='none')
37 |     plt.imshow(image_dict["cam"], cmap='jet', alpha=0.5)
38 |     plt.title(f"CAM for Instance {instance} (class {image_dict['label']})")
39 |     plt.savefig(f"instance_{instance}_cam.jpg", dpi=100)
40 |     plt.show()
41 | 
42 | 
43 | if __name__ == "__main__":
44 |     main()
45 | 


--------------------------------------------------------------------------------