93 | """
94 |
95 | # autosummary_generate = True
96 |
97 | copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: "
98 | copybutton_prompt_is_regexp = True
99 | copybutton_line_continuation_character = "\\"
100 | copybutton_here_doc_delimiter = "EOT"
101 |
102 | # -- Options for HTML output -------------------------------------------------
103 |
104 | # The theme to use for HTML and HTML Help pages. See the documentation for
105 | # a list of builtin themes.
106 | #
107 | html_theme = 'sphinx_rtd_theme'
108 | # html_theme = 'alabaster'
109 |
110 | html_favicon = '_static/favicon.svg'
111 |
112 | # Add any paths that contain custom static files (such as style sheets) here,
113 | # relative to this directory. They are copied after the builtin static files,
114 | # so a file named "default.css" will overwrite the builtin "default.css".
115 | html_static_path = ['_static']
116 |
117 | bibtex_bibfiles = ['bibliography.bib']
118 | bibtex_default_style = 'author_year_style'
119 | bibtex_reference_style = 'author_year'
120 |
121 |
122 | class AuthorYearLabelStyle(BaseLabelStyle):
123 | def format_labels(self, sorted_entries):
124 | for entry in sorted_entries:
125 | yield f'[{entry.persons["author"][0].last_names[0]} et al., {entry.fields["year"]}]'
126 |
127 |
128 | class AuthorYearStyle(PlainStyle):
129 | default_label_style = AuthorYearLabelStyle
130 |
131 |
132 | register_plugin('pybtex.style.formatting', 'author_year_style', AuthorYearStyle)
133 |
134 |
135 | def getrev():
136 | try:
137 | revision = run(
138 | ['git', 'describe', '--tags', 'HEAD'],
139 | capture_output=True,
140 | check=True,
141 | text=True
142 | ).stdout[:-1]
143 | except CalledProcessError:
144 | revision = 'master'
145 |
146 | return revision
147 |
148 |
149 | REVISION = getrev()
150 |
151 | extlinks = {
152 | 'repo': (
153 | f'https://github.com/chr5tphr/zennit/blob/{REVISION}/%s',
154 | '%s'
155 | )
156 | }
157 |
158 | LINKCODE_URL = (
159 | f'https://github.com/chr5tphr/zennit/blob/{REVISION}'
160 | '/src/{filepath}#L{linestart}-L{linestop}'
161 | )
162 |
163 |
164 | # revised from https://gist.github.com/nlgranger/55ff2e7ff10c280731348a16d569cb73
165 | def linkcode_resolve(domain, info):
166 | if domain != 'py' or not info['module']:
167 | return None
168 |
169 | modname = info['module']
170 | topmodulename = modname.split('.')[0]
171 | fullname = info['fullname']
172 |
173 | submod = sys.modules.get(modname)
174 | if submod is None:
175 | return None
176 |
177 | obj = submod
178 | for part in fullname.split('.'):
179 | try:
180 | obj = getattr(obj, part)
181 | except Exception:
182 | return None
183 |
184 | try:
185 | modpath = pkg_resources.require(topmodulename)[0].location
186 | filepath = os.path.relpath(inspect.getsourcefile(obj), modpath)
187 | if filepath is None:
188 | return
189 | except Exception:
190 | return None
191 |
192 | try:
193 | source, lineno = inspect.getsourcelines(obj)
194 | except OSError:
195 | return None
196 | else:
197 | linestart, linestop = lineno, lineno + len(source) - 1
198 |
199 | return LINKCODE_URL.format(filepath=filepath, linestart=linestart, linestop=linestop)
200 |
--------------------------------------------------------------------------------
/docs/source/getting-started.rst:
--------------------------------------------------------------------------------
1 | ================
2 | Getting started
3 | ================
4 |
5 |
6 | Install
7 | -------
8 |
9 | Zennit can be installed directly from PyPI:
10 |
11 | .. code-block:: console
12 |
13 | $ pip install zennit
14 |
15 | For the current development version, or to try out examples, Zennit may be
16 | alternatively cloned and installed with
17 |
18 | .. code-block:: console
19 |
20 | $ git clone https://github.com/chr5tphr/zennit.git
21 | $ pip install ./zennit
22 |
23 | Basic Usage
24 | -----------
25 |
26 | Zennit implements propagation-based attribution methods by overwriting the
27 | gradient of PyTorch modules in PyTorch's auto-differentiation engine. This means
28 | that Zennit will only work on models which are strictly implemented using
29 | PyTorch modules, including activation functions. The following demonstrates a
30 | setup to compute Layer-wise Relevance Propagation (LRP) relevance for a simple
31 | model and random data.
32 |
33 | .. code-block:: python
34 |
35 | import torch
36 | from torch.nn import Sequential, Conv2d, ReLU, Linear, Flatten
37 |
38 |
39 | # setup the model and data
40 | model = Sequential(
41 | Conv2d(3, 10, 3, padding=1),
42 | ReLU(),
43 | Flatten(),
44 | Linear(10 * 32 * 32, 10),
45 | )
46 | input = torch.randn(1, 3, 32, 32)
47 |
48 | The most important high-level structures in Zennit are ``Composites``,
49 | ``Attributors`` and ``Canonizers``.
50 |
51 |
52 | Composites
53 | ^^^^^^^^^^
54 |
55 | Composites map ``Rules`` to modules based on their properties and context to
56 | modify their gradient. The most common composites for LRP are implemented in
57 | :py:mod:`zennit.composites`.
58 |
59 | The following computes LRP relevance using the ``EpsilonPlusFlat`` composite:
60 |
61 | .. code-block:: python
62 |
63 | from zennit.composites import EpsilonPlusFlat
64 |
65 |
66 | # create a composite instance
67 | composite = EpsilonPlusFlat()
68 |
69 | # use the following instead to ignore bias for the relevance
70 | # composite = EpsilonPlusFlat(zero_params='bias')
71 |
72 | # make sure the input requires a gradient
73 | input.requires_grad = True
74 |
75 | # compute the output and gradient within the composite's context
76 | with composite.context(model) as modified_model:
77 | output = modified_model(input)
78 | # gradient/ relevance wrt. class/output 0
79 | output.backward(gradient=torch.eye(10)[[0]])
80 | # relevance is not accumulated in .grad if using torch.autograd.grad
81 | # relevance, = torch.autograd.grad(output, input, torch.eye(10)[[0])
82 |
83 | # gradient is accumulated in input.grad
84 | print('Backward:', input.grad)
85 |
86 |
87 | The context created by :py:func:`zennit.core.Composite.context` registers the
88 | composite, which means that all rules are applied according to the composite's
89 | mapping. See :doc:`/how-to/use-rules-composites-and-canonizers` for information on
90 | using composites, :py:mod:`zennit.composites` for an API reference and
91 | :doc:`/how-to/write-custom-composites` for writing new compositors. Available
92 | ``Rules`` can be found in :py:mod:`zennit.rules`, their use is described in
93 | :doc:`/how-to/use-rules-composites-and-canonizers` and how to add new ones is described in
94 | :doc:`/how-to/write-custom-rules`.
95 |
96 | Attributors
97 | ^^^^^^^^^^^
98 |
99 | Alternatively, *attributors* may be used instead of ``composite.context``.
100 |
101 | .. code-block:: python
102 |
103 | from zennit.attribution import Gradient
104 |
105 |
106 | attributor = Gradient(model, composite)
107 |
108 | with attributor:
109 | # gradient/ relevance wrt. output/class 1
110 | output, relevance = attributor(input, torch.eye(10)[[1]])
111 |
112 | print('EpsilonPlusFlat:', relevance)
113 |
114 | Attribution methods which are not propagation-based, like
115 | :py:class:`zennit.attribution.SmoothGrad` are implemented as attributors, and
116 | may be combined with propagation-based (composite) approaches.
117 |
118 | .. code-block:: python
119 |
120 | from zennit.attribution import SmoothGrad
121 |
122 |
123 | # we do not need a composite to compute vanilla SmoothGrad
124 | with SmoothGrad(model, noise_level=0.1, n_iter=10) as attributor:
125 | # gradient/ relevance wrt. output/class 7
126 | output, relevance = attributor(input, torch.eye(10)[[7]])
127 |
128 | print('SmoothGrad:', relevance)
129 |
130 | More information on attributors can be found in :doc:`/how-to/use-attributors`
131 | and :doc:`/how-to/write-custom-attributors`.
132 |
133 | Canonizers
134 | ^^^^^^^^^^
135 |
136 | For some modules and operations, Layer-wise Relevance Propagation (LRP) is not
137 | implementation-invariant, eg. ``BatchNorm -> Dense -> ReLU`` will be attributed
138 | differently than ``Dense -> BatchNorm -> ReLU``. Therefore, LRP needs a
139 | canonical form of the model, which is implemented in ``Canonizers``. These may
140 | be simply supplied when instantiating a composite:
141 |
142 | .. code-block:: python
143 |
144 | from torchvision.models import vgg16
145 | from zennit.composites import EpsilonGammaBox
146 | from zennit.torchvision import VGGCanonizer
147 |
148 |
149 | # instantiate the model
150 | model = vgg16()
151 | # create the canonizers
152 | canonizers = [VGGCanonizer()]
153 | # EpsilonGammaBox needs keyword arguments 'low' and 'high'
154 | composite = EpsilonGammaBox(low=-3., high=3., canonizers=canonizers)
155 |
156 | with Gradient(model, composite) as attributor:
157 | # gradient/ relevance wrt. output/class 0
158 | # torchvision.vgg16 has 1000 output classes by default
159 | output, relevance = attributor(input, torch.eye(1000)[[0]])
160 |
161 | print('EpsilonGammaBox:', relevance)
162 |
163 | Some pre-defined canonizers for models from ``torchvision`` can be found in
164 | :py:mod:`zennit.torchvision`. The :py:class:`zennit.torchvision.VGGCanonizer`
165 | specifically is simply :py:class:`zennit.canonizers.SequentialMergeBatchNorm`,
166 | which may be used when ``BatchNorm`` is used in sequential models. Note that for
167 | ``SequentialMergeBatchNorm`` to work, all functions (linear layers, activations,
168 | ...) must be modules and assigned to their parent module in the order they are
169 | visited (see :py:class:`zennit.canonizers.SequentialMergeBatchNorm`). For more
170 | information on canonizers see :doc:`/how-to/use-rules-composites-and-canonizers` and
171 | :doc:`/how-to/write-custom-canonizers`.
172 |
173 |
174 | Visualizing Results
175 | ^^^^^^^^^^^^^^^^^^^
176 |
177 | While attribution approaches are not limited to the domain of images, they are
178 | predominantly used on image models and produce heat maps of relevance. For
179 | this reason, Zennit implements methods to visualize relevance heat maps.
180 |
181 | .. code-block:: python
182 |
183 | from zennit.image import imsave
184 |
185 |
186 | # sum over the color channels
187 | heatmap = relevance.sum(1)
188 | # get the absolute maximum, to center the heat map around 0
189 | amax = heatmap.abs().numpy().max((1, 2))
190 |
191 | # save heat map with color map 'coldnhot'
192 | imsave(
193 | 'heatmap.png',
194 | heatmap[0],
195 | vmin=-amax,
196 | vmax=amax,
197 | cmap='coldnhot',
198 | level=1.0,
199 | grid=False
200 | )
201 |
202 | Information on ``imsave`` can be found at :py:func:`zennit.image.imsave`.
203 | Saving an image with 3 color channels will result in the image being saved
204 | without a color map but with the channels assumed as RGB. The keyword argument
205 | ``grid`` will create a grid of multiple images over the batch dimension if
206 | ``True``. Custom color maps may be created with
207 | :py:class:`zennit.cmap.ColorMap`, eg. to save the previous image with a color
208 | map ranging from blue to yellow to red:
209 |
210 | .. code-block:: python
211 |
212 | from zennit.cmap import ColorMap
213 |
214 |
215 | # 00f is blue, ff0 is yellow, f00 is red, 0x80 is the center of the range
216 | cmap = ColorMap('00f,80:ff0,f00')
217 |
218 | imsave(
219 | 'heatmap.png',
220 | heatmap,
221 | vmin=-amax,
222 | vmax=amax,
223 | cmap=cmap,
224 | level=1.0,
225 | grid=True
226 | )
227 |
228 | More details to visualize heat maps and color maps can be found in
229 | :doc:`/how-to/visualize-results`. The ColorMap specification language is
230 | described in :py:class:`zennit.cmap.ColorMap` and built-in color maps are
231 | implemented in :py:obj:`zennit.image.CMAPS`.
232 |
233 | Example Script
234 | --------------
235 |
236 | A ready-to use example to analyze a few ImageNet models provided by torchvision
237 | can be found at :repo:`share/example/feed_forward.py`.
238 |
239 | The following setup requires bash, cURL and (magic-)file.
240 |
241 | Create a virtual environment, install Zennit and download the example scripts:
242 |
243 | .. code-block:: console
244 |
245 | $ mkdir zennit-example
246 | $ cd zennit-example
247 | $ python -m venv .venv
248 | $ .venv/bin/pip install zennit
249 | $ curl -o feed_forward.py \
250 | 'https://raw.githubusercontent.com/chr5tphr/zennit/master/share/example/feed_forward.py'
251 | $ curl -o download-lighthouses.sh \
252 | 'https://raw.githubusercontent.com/chr5tphr/zennit/master/share/scripts/download-lighthouses.sh'
253 |
254 | Prepare the data required for the example:
255 |
256 | .. code-block:: console
257 |
258 | $ mkdir params data results
259 | $ bash download-lighthouses.sh --output data/lighthouses
260 | $ curl -o params/vgg16-397923af.pth 'https://download.pytorch.org/models/vgg16-397923af.pth'
261 |
262 | This creates the needed directories and downloads the pre-trained vgg16
263 | parameters and 8 images of light houses from wikimedia commons into the
264 | required label-directory structure for the imagenet dataset in PyTorch.
265 |
266 | The ``feed_forward.py`` example can then be run using:
267 |
268 | .. code-block:: console
269 |
270 | $ .venv/bin/python feed_forward.py \
271 | data/lighthouses \
272 | 'results/vgg16_epsilon_gamma_box_{sample:02d}.png' \
273 | --inputs 'results/vgg16_input_{sample:02d}.png' \
274 | --parameters params/vgg16-397923af.pth \
275 | --model vgg16 \
276 | --composite epsilon_gamma_box \
277 | --no-bias \
278 | --relevance-norm symmetric \
279 | --cmap coldnhot
280 |
281 | which computes the lrp heatmaps according to the ``epsilon_gamma_box`` rule and
282 | stores them in results, along with the respective input images. Other possible
283 | composites that can be passed to ``--composites`` are, e.g., ``epsilon_plus``,
284 | ``epsilon_alpha2_beta1_flat``, ``guided_backprop``, ``excitation_backprop``.
285 | The bias can be ignored in the LRP-computation by passing ``--no-bias``.
286 |
287 |
288 | ..
289 | The resulting heatmaps may look like the following:
290 |
291 | .. image:: /img/beacon_vgg16_epsilon_gamma_box.png
292 | :alt: Lighthouses with Attributions
293 |
294 | Alternatively, heatmaps for SmoothGrad with absolute relevances may be computed
295 | by omitting ``--composite`` and supplying ``--attributor``:
296 |
297 | .. code-block:: console
298 |
299 | $ .venv/bin/python feed_forward.py \
300 | data/lighthouses \
301 | 'results/vgg16_smoothgrad_{sample:02d}.png' \
302 | --inputs 'results/vgg16_input_{sample:02d}.png' \
303 | --parameters params/vgg16-397923af.pth \
304 | --model vgg16 \
305 | --attributor smoothgrad \
306 | --relevance-norm absolute \
307 | --cmap hot
308 |
309 | For Integrated Gradients, ``--attributor integrads`` may be provided.
310 |
311 | Heatmaps for Occlusion Analysis with unaligned relevances may be computed by
312 | executing:
313 |
314 | .. code-block:: console
315 |
316 | $ .venv/bin/python feed_forward.py \
317 | data/lighthouses \
318 | 'results/vgg16_occlusion_{sample:02d}.png' \
319 | --inputs 'results/vgg16_input_{sample:02d}.png' \
320 | --parameters params/vgg16-397923af.pth \
321 | --model vgg16 \
322 | --attributor occlusion \
323 | --relevance-norm unaligned \
324 | --cmap hot
325 |
326 |
--------------------------------------------------------------------------------
/docs/source/how-to/compute-second-order-gradients.rst:
--------------------------------------------------------------------------------
1 | ================================
2 | Computing Second Order Gradients
3 | ================================
4 |
5 | Sometimes, it may be necessary to compute the gradient of the attribution. One
6 | example is to compute the gradient with respect to the input in order to
7 | find adversarial explanations :cite:p:`dombrowski2019explanations`,
8 | or to regularize or transform the attributions of a network
9 | :cite:p:`anders2020fairwashing`.
10 |
11 | In Zennit, the attribution is computed using the modified gradient, which means
12 | that in order to compute the gradient of the attribution, the second order
13 | gradient needs to be computed. Pytorch natively supports the computation of
14 | higher order gradients, simply by supplying ``create_graph=True`` with
15 | :py:func:`torch.autograd.grad` to declare that the backward-function needs to
16 | be backward-able itself.
17 |
18 |
19 | Vanilla Gradient and ReLU
20 | -------------------------
21 |
22 | If we simply need the second order gradient of a model, without using Zennit, we can do the following:
23 |
24 | .. code-block:: python
25 |
26 | import torch
27 | from torch.nn import Sequential, Conv2d, ReLU, Linear, Flatten
28 |
29 |
30 | # setup the model and data
31 | model = Sequential(
32 | Conv2d(3, 10, 3, padding=1),
33 | ReLU(),
34 | Flatten(),
35 | Linear(10 * 32 * 32, 10),
36 | )
37 | input = torch.randn(1, 3, 32, 32)
38 |
39 | # make sure the input requires a gradient
40 | input.requires_grad = True
41 |
42 | output = model(input)
43 | # a vector for the vector-jacobian-product, i.e. the grad_output
44 | target = torch.ones_like(output)
45 |
46 | grad, = torch.autograd.grad(output, input, target, create_graph=True)
47 |
48 | # the grad_output for grad
49 | gradtarget = torch.ones_like(grad)
50 | # compute the second order gradient
51 | gradgrad, = torch.autograd.grad(grad, input, gradtarget)
52 |
53 | Here, you might notice that ``gradgrad`` is all zeros, regardless of the input
54 | and model parameters. The culprit is ``ReLU``, which has a gradient of zero
55 | everywhere except at zero, where it is undefined. In order to get a meaningful
56 | gradient, we could instead use a *smooth* activation function in our model.
57 | However, ReLU models are quite common, and we may not like to retrain every
58 | model using only smooth activation functions.
59 |
60 | :cite:t:`dombrowski2019explanations` proposed to replace the ReLU activations
61 | with its smooth variation, the *Softplus* function:
62 |
63 | .. math::
64 |
65 | \text{Softplus}(x;\beta) = \frac{1}{\beta} \log (1 + \exp (\beta x))
66 | \,\text{.}
67 |
68 | With :math:`\beta\rightarrow\infty`, Softplus will be equivalent to ReLU, but in
69 | practice choosing :math:`\beta = 10` is most often sufficient to keep the model
70 | output unchanged but still obtain a meaningful second order gradient.
71 |
72 | To temporarily replace the ReLU gradients in-place, we can use the
73 | :py:class:`~zennit.rules.ReLUBetaSmooth` rule:
74 |
75 |
76 | .. code-block:: python
77 |
78 | from zennit.composites import BetaSmooth
79 |
80 | # LayerMapComposite which assigns the ReLUBetaSmooth hook to ReLUs
81 | composite = BetaSmooth(beta_smooth=10.)
82 |
83 | with composite.context(model):
84 | output = model(input)
85 | target = torch.ones_like(output)
86 | grad, = torch.autograd.grad(output, input, target, create_graph=True)
87 |
88 | gradtarget = torch.ones_like(grad)
89 | gradgrad, = torch.autograd.grad(grad, input, gradtarget)
90 |
91 | Notice here that we computed the second order gradient **outside** of the
92 | composite context. A property of the Pytorch gradients hooks is that they are
93 | also called when the *second* order gradient with respect to a tensor is
94 | computed.
95 | Due to this, computing the second order gradient *while rules are still
96 | registered* will lead to incorrect results.
97 |
98 | Temporarily Disabling Hooks
99 | ---------------------------
100 |
101 | In order compute the second order gradient *without* removing the hooks (i.e. to
102 | compute multiple values in a loop), we can temporarily deactivate them using
103 | :py:meth:`zennit.core.Composite.inactive`:
104 |
105 | .. code-block:: python
106 |
107 | with composite.context(model):
108 | output = model(input)
109 | target = torch.ones_like(output)
110 | grad, = torch.autograd.grad(output, input, target, create_graph=True)
111 |
112 | # temporarily disable all hooks registered by composite
113 | with composite.inactive():
114 | gradtarget = torch.ones_like(grad)
115 | gradgrad, = torch.autograd.grad(grad, input, gradtarget)
116 |
117 | All Attributors support the computation of gradients. For gradient-based
118 | attributors like :py:class:`~zennit.attribution.Gradient` or
119 | :py:class:`~zennit.attribution.SmoothGrad`, the ``create_graph=True`` parameter
120 | can be supplied to the class constructor:
121 |
122 | .. code-block:: python
123 |
124 | from zennit.attribution import Gradient
125 | from zennit.composites import EpsilonGammaBox
126 |
127 | # any composites support second order gradients
128 | composite = EpsilonGammaBox(low=-3., high=3.)
129 |
130 | with Gradient(model, composite, create_graph=True) as attributor:
131 | output, grad = attributor(input, torch.ones_like)
132 |
133 | # temporarily disable all hooks registered by the attributor's composite
134 | with attributor.inactive():
135 | gradtarget = torch.ones_like(grad)
136 | gradgrad, = torch.autograd.grad(grad, input, gradtarget)
137 |
138 | Here, we also used a different composite, which results in the gradient
139 | computation of the modified gradient. Since the ReLU gradient is ignored (using
140 | the :py:class:`~zennit.rules.Pass` rule) for Layer-wise Relevance
141 | Propagation-specific composites, we do not need to use the
142 | :py:class:`~zennit.rules.ReLUBetaSmooth` rule. However, if this behaviour
143 | should be overwritten, :ref:`cooperative-layermapcomposites` can be used.
144 |
145 | Using Hooks Only
146 | ----------------
147 |
148 | Under the hood, :py:class:`~zennit.core.Hook` has an attribute ``active``,
149 | which, when set to ``False``, will not execute the associated backward function.
150 | A minimal example without using composites would look like the following:
151 |
152 | .. code-block:: python
153 |
154 | from zennit.rules import Epsilon
155 |
156 | conv = Conv2d(3, 10, 3, padding=1)
157 |
158 | # create and register the hook
159 | epsilon = Epsilon()
160 | handles = epsilon.register(conv)
161 |
162 | output = conv(input)
163 | target = torch.ones_like(output)
164 | grad, = torch.autograd.grad(output, input, target, create_graph=True)
165 |
166 | # during this block, epsilon will be inactive
167 | epsilon.active = False
168 | grad_target = torch.ones_like(grad)
169 | gradgrad, = torch.autograd.grad(grad, input, grad_target)
170 | epsilon.active = True
171 |
172 | # after calling handles.remove, epsilon will also be inactive
173 | handles.remove()
174 |
175 | The same can here also be achieved by simply removing the handles before calling
176 | ``torch.autograd.grad`` on ``grad``, although the hooks would then need to be
177 | re-registered in order to compute the epsilon-modified gradient again.
178 |
--------------------------------------------------------------------------------
/docs/source/how-to/get-intermediate-relevance.rst:
--------------------------------------------------------------------------------
1 | ==============================
2 | Getting Intermediate Relevance
3 | ==============================
4 |
5 | In some cases, intermediate gradients or relevances of a model may be needed.
6 | Since Zennit uses Pytorch's autograd engine, intermediate relevances can be
7 | retained simply as the intermediate gradients of accessible non-leaf tensors
8 | in the tensor's ``.grad`` attribute by calling ``tensor.retain_grad()`` before
9 | the gradient computation.
10 |
11 | In most cases when using ``torch.nn.Module``-based models, the intermediate
12 | outputs are not easily accessible, which we can solve by using forward-hooks.
13 |
14 | We create following setting with some random input data and a simple, randomly
15 | initialized model, for which we want to compute the LRP EpsilonPlus relevance:
16 |
17 | .. code-block:: python
18 |
19 | import torch
20 | from torch.nn import Sequential, Conv2d, ReLU, Linear, Flatten
21 |
22 | from zennit.attribution import Gradient
23 | from zennit.composites import EpsilonPlusFlat
24 |
25 | # setup the model and data
26 | model = Sequential(
27 | Conv2d(3, 10, 3, padding=1),
28 | ReLU(),
29 | Flatten(),
30 | Linear(10 * 32 * 32, 10),
31 | )
32 | input = torch.randn(1, 3, 32, 32)
33 |
34 | # make sure the input requires a gradient
35 | input.requires_grad = True
36 |
37 | # create a composite instance
38 | composite = EpsilonPlusFlat()
39 |
40 | # create a gradient attributor
41 | attributor = Gradient(model, composite)
42 |
43 | Now we create a function ``store_hook`` which we register as a forward hook to
44 | all modules. The function sets the module's attribute ``.output`` to its output
45 | tensor, and ensures the gradient is stored in the tensor's ``.grad`` attribute
46 | even if it is not a leaf-tensor by using ``.retain_grad()``.
47 |
48 | .. code-block:: python
49 |
50 | # create a hook to keep track of intermediate outputs
51 | def store_hook(module, input, output):
52 | # set the current module's attribute 'output' to the its tensor
53 | module.output = output
54 | # keep the output tensor gradient, even if it is not a leaf-tensor
55 | output.retain_grad()
56 |
57 | # enter the attributor's context to register the rule-hooks
58 | with attributor:
59 | # register the store_hook AFTER the rule-hooks have been registered (by
60 | # entering the context) so we get the last output before the next module
61 | handles = [
62 | module.register_forward_hook(store_hook) for module in model.modules()
63 | ]
64 | # compute the relevance wrt. output/class 1
65 | output, relevance = attributor(input, torch.eye(10)[[1]])
66 |
67 | # remove the hooks using store_hook
68 | for handle in handles:
69 | handle.remove()
70 |
71 | # print the gradient tensors for demonstration
72 | for name, module in model.named_modules():
73 | print(f'{name}: {module.output.grad}')
74 |
75 | The hooks are registered within the attributor's with-context, such that they
76 | are applied after the rule hooks. Once we are finished, we can remove the
77 | store-hooks by calling ``.remove()`` on all handles returned when registering the
78 | hooks.
79 |
80 | Be aware that storing the intermediate outputs and their gradients may require
81 | significantly more memory, depending on the model. In practice, it may be better
82 | to register the store-hook only to modules for which the relevance is needed.
83 |
--------------------------------------------------------------------------------
/docs/source/how-to/index.rst:
--------------------------------------------------------------------------------
1 | ================
2 | How-Tos
3 | ================
4 |
5 |
6 | These How-Tos give more detailed information on how to use Zennit.
7 |
8 | .. toctree::
9 | :maxdepth: 1
10 |
11 | use-rules-composites-and-canonizers
12 | use-attributors
13 | visualize-results
14 | get-intermediate-relevance
15 | compute-second-order-gradients
16 | write-custom-composites
17 | write-custom-canonizers
18 | write-custom-rules
19 | write-custom-attributors
20 |
--------------------------------------------------------------------------------
/docs/source/how-to/use-attributors.rst:
--------------------------------------------------------------------------------
1 | =================
2 | Using Attributors
3 | =================
4 |
5 | **Attributors** are used to both shorten Zennit's common ``composite.context ->
6 | gradient`` approach, as well as provide model-agnostic attribution approaches.
7 | Available **Attributors** can be found in :py:mod:`zennit.attribution`, some of
8 | which are:
9 |
10 | * :py:class:`~zennit.attribution.Gradient`, which computes the gradient
11 | * :py:class:`~zennit.attribution.IntegratedGradients`, which computes the
12 | Integrated Gradients
13 | * :py:class:`~zennit.attribution.SmoothGrad`, which computes SmoothGrad
14 | * :py:class:`~zennit.attribution.Occlusion`, which computes the attribution
15 | based on the model output activation values when occluding parts of the input
16 | with a sliding window
17 |
18 | Using the basic :py:class:`~zennit.attribution.Gradient`, the unmodified
19 | gradient may be computed with:
20 |
21 | .. code-block:: python
22 |
23 | import torch
24 | from torch.nn import Sequential, Conv2d, ReLU, Linear, Flatten
25 | from zennit.attribution import Gradient
26 |
27 | # setup the model
28 | model = Sequential(
29 | Conv2d(3, 8, 3, padding=1),
30 | ReLU(),
31 | Conv2d(8, 16, 3, padding=1),
32 | ReLU(),
33 | Flatten(),
34 | Linear(16 * 32 * 32, 1024),
35 | ReLU(),
36 | Linear(1024, 10),
37 | )
38 | # some random input data
39 | input = torch.randn(1, 3, 32, 32, requires_grad=True)
40 |
41 | # compute the gradient and output using the Gradient attributor
42 | with Gradient(model) as attributor:
43 | output, relevance = attributor(input)
44 |
45 | Computing attributions using a composite can be done with:
46 |
47 | .. code-block:: python
48 |
49 | from zennit.composites import EpsilonPlusFlat
50 |
51 | # prepare the composite
52 | composite = EpsilonPlusFlat()
53 |
54 | # compute the gradient within the composite's context, i.e. the
55 | # EpsilonPlusFlat LRP relevance
56 | with Gradient(model, composite) as attributor:
57 | # torch.eye is used here to get a one-hot encoding of the
58 | # first (index 0) label
59 | output, relevance = attributor(input, torch.eye(10)[[0]])
60 |
61 | which uses the second argument ``attr_output_fn`` of the call to
62 | :py:class:`~zennit.attribution.Attributor` to specify a constant tensor used for
63 | the *output relevance* (i.e. ``grad_output``), but alternatively, a function
64 | of the output may also be used:
65 |
66 | .. code-block:: python
67 |
68 | def one_hot_max(output):
69 | '''Get the one-hot encoded max at the original indices in dim=1'''
70 | values, indices = output.max(1)
71 | return values[:, None] * torch.eye(output.shape[1])[indices]
72 |
73 | with Gradient(model) as attributor:
74 | output, relevance = attributor(input, one_hot_max)
75 |
76 | The constructor of :py:class:`~zennit.attribution.Attributor` also has a third
77 | argument ``attr_output``, which also can either be a constant
78 | :py:class:`~torch.Tensor`, or a function of the model's output and specifies
79 | which *output relevance* (i.e. ``grad_output``) should be used by default. When
80 | not supplying anything, the default will be the *identity*. If the default
81 | should be for example ones for all outputs, one could write:
82 |
83 | .. code-block:: python
84 |
85 | # compute the gradient and output using the Gradient attributor, and with
86 | # a vector of ones as grad_output
87 | with Gradient(model, attr_output=torch.ones_like) as attributor:
88 | output, relevance = attributor(input)
89 |
90 | Gradient-based **Attributors** like
91 | :py:class:`~zennit.attribution.IntegratedGradients` and
92 | :py:class:`~zennit.attribution.SmoothGrad` may also be used together with
93 | composites to produce *hybrid attributions*:
94 |
95 | .. code-block:: python
96 |
97 | from zennit.attribution import SmoothGrad
98 |
99 | # prepare the composite
100 | composite = EpsilonPlusFlat()
101 |
102 | # do a *smooth* version of EpsilonPlusFlat LRP by using the SmoothGrad
103 | # attributor in combination with the composite
104 | with SmoothGrad(model, composite, noise_level=0.1, n_iter=20) as attributor:
105 | output, relevance = attributor(input, torch.eye(10)[[0]])
106 |
107 | which in this case will sample 20 samples in an epsilon-ball (size controlled
108 | with `noise_level`) around the input. Note that for Zennit's implementation of
109 | :py:class:`~zennit.attribution.SmoothGrad`, the first sample will always be the
110 | original input, i.e. ``SmoothGrad(model, n_iter=1)`` will produce the plain
111 | gradient as ``Gradient(model)`` would.
112 |
113 | :py:class:`~zennit.attribution.Occlusion` will move a sliding window with
114 | arbitrary size and strides over an input with any dimensionality. In addition to
115 | specifying window-size and strides, a function may be specified, which will be
116 | supplied with the input and a mask. When using the default, everything within
117 | the sliding window will be set to zero. A function
118 | :py:func:`zennit.attribution.occlude_independent` is available to simplify the
119 | process of specifying how to fill the window, and to invert the window if
120 | desired. The following adds some gaussian noise to the area within the sliding
121 | window:
122 |
123 | .. code-block:: python
124 |
125 | from functools import partial
126 | from zennit.attribution import Occlusion, occlude_independent
127 |
128 | input = torch.randn((16, 3, 32, 32))
129 |
130 | attributor = Occlusion(
131 | model,
132 | window=8, # 8x8 overlapping windows
133 | stride=4, # with strides 4x4
134 | occlusion_fn=partial( # occlusion_fn gets the full input and a mask
135 | occlude_independent, # applies fill_fn at provided mask
136 | fill_fn=lambda x: x * torch.randn_like(x) * 0.2, # add some noise
137 | invert=False # do not invert, i.e. occlude *within* mask
138 | )
139 | )
140 | with attributor:
141 | # for occlusion, the score for each window-pass is the sum of the
142 | # provided *grad_output*, which we choose as the model output at index 0
143 | output, relevance = attributor(input, lambda out: torch.eye(10)[[0]] * out)
144 |
145 |
146 | Note that while the interface allows to pass a composite for any
147 | :py:class:`~zennit.attribution.Attributor`, using a composite with
148 | :py:class:`~zennit.attribution.Occlusion` does not change the outcome, as it
149 | does not utilize the gradient.
150 |
151 | An introduction on how to write custom **Attributors** can be found at
152 | :doc:`/how-to/write-custom-attributors`.
153 |
--------------------------------------------------------------------------------
/docs/source/how-to/write-custom-attributors.rst:
--------------------------------------------------------------------------------
1 | ==========================
2 | Writing Custom Attributors
3 | ==========================
4 |
5 | **Attributors** provide an additional layer of abstraction over the context of
6 | **Composites**, and are used to directly produce *attributions*, which may or
7 | may not be computed with modified gradients, if they are used, from
8 | **Composites**.
9 | More information on **Attributors**, examples and their use can be found in
10 | :doc:`/how-to/use-attributors`.
11 |
12 | **Attributors** can be used to implement non-layer-wise or only partly
13 | layer-wise attribution methods.
14 | For this, it is enough to define a subclass of
15 | :py:class:`zennit.attribution.Attributor` and implement its
16 | :py:meth:`~zennit.attribution.Attributor.forward` and optionally its
17 | :py:meth:`~zennit.attribution.Attributor.__init__` methods.
18 |
19 | :py:meth:`~zennit.attribution.Attributor.forward` takes 2 arguments, the tensor
20 | with respect to which the attribution shall be computed ``input``, and
21 | ``attr_output_fn``, which is a function that, given the output of the
22 | attributed model, computes the *gradient output* for the gradient computation,
23 | which is, for example, a one-hot encoding of the target label of the attributed
24 | input.
25 | When calling an :py:class:`~zennit.attribution.Attributor`, the ``__call__``
26 | function will ensure ``forward`` receives a valid function to transform the
27 | output of the analyzed model to a tensor which can be used for the
28 | ``grad_output`` argument of :py:func:`torch.autograd.grad`.
29 | A constant tensor or function is provided by the user either to ``__init__`` or
30 | to ``__call__``.
31 | It is expected that :py:meth:`~zennit.attribution.Attributor.forward` will
32 | return a tuple containing, in order, the model output and the attribution.
33 |
34 | As an example, we can implement *gradient times input* in the following way:
35 |
36 | .. code-block:: python
37 |
38 | import torch
39 | from torchvision.models import vgg11
40 |
41 | from zennit.attribution import Attributor
42 |
43 |
44 | class GradientTimesInput(Attributor):
45 | '''Model-agnostic gradient times input.'''
46 | def forward(self, input, attr_output_fn):
47 | '''Compute gradient times input.'''
48 | input_detached = input.detach().requires_grad_(True)
49 | output = self.model(input_detached)
50 | gradient, = torch.autograd.grad(
51 | (output,), (input_detached,), (attr_output_fn(output.detach()),)
52 | )
53 | relevance = gradient * input
54 | return output, relevance
55 |
56 | model = vgg11()
57 | data = torch.randn((1, 3, 224, 224))
58 |
59 | with GradientTimesInput(model) as attributor:
60 | output, relevance = attributor(data)
61 |
62 | :py:class:`~zennit.attribution.Attributor` accepts an optional
63 | :py:class:`~zennit.core.Composite`, which, if supplied, will always be used to
64 | create a context in ``__call__`` around ``forward``.
65 | For the ``GradientTimesInput`` class above, using a **Composite** will probably
66 | not produce anything useful, although more involved combinations of custom
67 | **Rules** and a custom **Attributor** can be used to implement complex
68 | attribution methods with both model-agnostic and layer-wise parts.
69 |
70 | The following shows an example of *sensitivity analysis*, which is the absolute
71 | value, with a custom ``__init__()`` where we can pass the argument
72 | ``sum_channels`` to specify whether the **Attributor** should sum over the
73 | channel dimension:
74 |
75 | .. code-block:: python
76 |
77 | import torch
78 | from torchvision.models import vgg11
79 |
80 | from zennit.attribution import Attributor
81 |
82 |
83 | class SensitivityAnalysis(Attributor):
84 | '''Model-agnostic sensitivity analysis which optionally sums over color
85 | channels.
86 | '''
87 | def __init__(
88 | self, model, sum_channels=False, composite=None, attr_output=None
89 | ):
90 | super().__init__(
91 | model, composite=composite, attr_output=attr_output
92 | )
93 |
94 | self.sum_channels = sum_channels
95 |
96 |
97 | def forward(self, input, attr_output_fn):
98 | '''Compute the absolute gradient (or the sensitivity) and
99 | optionally sum over the color channels.
100 | '''
101 | input_detached = input.detach().requires_grad_(True)
102 | output = self.model(input_detached)
103 | gradient, = torch.autograd.grad(
104 | (output,), (input_detached,), (attr_output_fn(output.detach()),)
105 | )
106 | relevance = gradient.abs()
107 | if self.sum_channels:
108 | relevance = relevance.sum(1)
109 | return output, relevance
110 |
111 | model = vgg11()
112 | data = torch.randn((1, 3, 224, 224))
113 |
114 | with SensitivityAnalysis(model, sum_channels=True) as attributor:
115 | output, relevance = attributor(data)
116 |
--------------------------------------------------------------------------------
/docs/source/how-to/write-custom-canonizers.rst:
--------------------------------------------------------------------------------
1 | =========================
2 | Writing Custom Canonizers
3 | =========================
4 |
5 | **Canonizers** are used to temporarily transform models into a canonical form to
6 | mitigate the lack of implementation invariance of methods Layer-wise Relevance
7 | Propagation (LRP). A general introduction to **Canonizers** can be found here:
8 | :ref:`use-canonizers`.
9 |
10 | As both **Canonizers** and **Composites** (via **Rules**) change the outcome of
11 | the attribution, it can be a little bit confusing in the beginning when
12 | challenged with the question whether a novel network architectures needs a new
13 | set of **Rules** and **Composites**, or if it should be adapted to the existing
14 | framework using **Canonizers**. While ultimately it depends on the design
15 | preference of the developer, our suggestion is to go through the following steps
16 | in order:
17 |
18 | 1. Check whether a custom **Composite** is enough to correctly attribute the
19 | model, i.e. the new layer-type is only a composition of existing layer types
20 | without any unaccounted intermediate steps or incapabilities with existing
21 | rules.
22 | 2. If some of the rules which should be used are incompatible without changes
23 | (e.g. subsequent linear layers), or some parts of a module has intermediate
24 | computations that are not implemented with sub-modules, it should be checked
25 | whether a **Canonizer** can be implemented to fix these issues. If you are in
26 | control of the module in question, check whether rewriting the module with
27 | sub-modules is easier than implementing a **Canonizer**.
28 | 3. If the module consists of computations which cannot be separated into
29 | existing modules with compatible rules, or would result in an overly complex
30 | architecture, a custom **Rule** may be the choice to go with.
31 |
32 | **Rules** and **Composites** are not designed to change the forward computation
33 | of a model. While **Canonizers** can change the outcome of the forward pass,
34 | this should be used with care, since a modified function output means that the
35 | function itself has been modified, which will therefore result in an attribution
36 | of the modified function instead.
37 |
38 | To implement a custom **Canonizer**, a class inheriting from
39 | :py:class:`zennit.canonizers.Canonizer` needs to implement the following four
40 | methods:
41 |
42 | * :py:meth:`~zennit.canonizers.Canonizer.apply`, which finds the sub-modules
43 | that should be modified by the **Canonizer** and passes their information to ...
44 | * :py:meth:`~zennit.canonizers.Canonizer.register`, which copies the current
45 | instance using :py:meth:`~zennit.canonizers.Canonizer.copy`, applies the
46 | changes that should be introduced by the **Canonizer**, and makes sure they
47 | can be reverted later, using ...
48 | * :py:meth:`~zennit.canonizers.Canonizer.remove`, which reverts the changes
49 | introduced by the **Canonizer**, by i.e. loading the original parameters which
50 | were temporarily stored, and
51 | * :py:meth:`~zennit.canonizers.Canonizer.copy`, which copies the current
52 | instance, to create an individual instance for each applicable module with the
53 | same parameters.
54 |
55 | Suppose we have a ReLU model (e.g. VGG11) for which we want to compute the
56 | second-order derivative, e.g. to find an adversarial explanation (see
57 | :cite:p:`dombrowski2019explanations`). The ReLU is not differentiable at 0, and
58 | its second order derivative is zero everywhere except at 0, where it is
59 | undefined. :cite:t:`dombrowski2019explanations` replace the ReLU activations in
60 | a model with *Softplus* activations, which when running *beta* towards infinity
61 | will be identical to the ReLU activation. For the numerical estimate, it is
62 | enough to set *beta* to a relatively large value, e.g. to 10. The following is
63 | an implementation of the **SoftplusCanonizer**, which will temporarily replace
64 | the ReLU activations in a model with Softplus activations:
65 |
66 | .. code-block:: python
67 |
68 | import torch
69 |
70 | from zennit.canonizers import Canonizer
71 |
72 |
73 | class SoftplusCanonizer(Canonizer):
74 | '''Replaces ReLUs with Softplus units.'''
75 | def __init__(self, beta=10.):
76 | self.beta = beta
77 | self.module = None
78 | self.relu_children = None
79 |
80 | def apply(self, root_module):
81 | '''Iterate all modules under root_module and register the Canonizer
82 | if they have immediate ReLU sub-modules.
83 | '''
84 | # track the SoftplusCanonizer instances to remove them later
85 | instances = []
86 | # iterate recursively over all modules
87 | for module in root_module.modules():
88 | # get all the direct sub-module instances of torch.nn.ReLU
89 | relu_children = [
90 | (name, child)
91 | for name, child in module.named_children()
92 | if isinstance(child, torch.nn.ReLU)
93 | ]
94 | # if there is at least on direct ReLU sub-module
95 | if relu_children:
96 | # create a copy (with the same beta parameter)
97 | instance = self.copy()
98 | # register the module
99 | instance.register(module, relu_children)
100 | # add the copy to the instance list
101 | instances.append(instance)
102 | return instances
103 |
104 | def register(self, module, relu_children):
105 | '''Store the module and the immediate ReLU-sub-modules, and then
106 | overwrite the attributes that corresponds to each ReLU-sub-modules
107 | with a new instance of ``torch.nn.Softplus``.
108 | '''
109 | self.module = module
110 | self.relu_children = relu_children
111 | for name, _ in relu_children:
112 | # set each of the attributes corresponding to the ReLU to a new
113 | # instance of torch.nn.Softplus
114 | setattr(module, name, torch.nn.Softplus(beta=self.beta))
115 |
116 | def remove(self):
117 | '''Undo the changes introduces by this Canonizer, by setting the
118 | appropriate attributes of the stored module back to the original
119 | ReLU sub-module instance.
120 | '''
121 | for name, child in self.relu_children:
122 | setattr(self.module, name, child)
123 |
124 | def copy(self):
125 | '''Create a copy of this instance. Each module requires its own
126 | instance to call ``.register``.
127 | '''
128 | return SoftplusCanonizer(beta=self.beta)
129 |
130 |
131 | Note that we can only replace modules by changing their immediate parent. This
132 | means that if ``root_module`` was a ``torch.nn.ReLU`` itself, it would be
133 | impossible to replace it with a ``torch.nn.Softplus`` without replacing the
134 | ``root_module`` itself.
135 |
136 | For demonstration purposes, we can compute the gradient w.r.t. a loss which uses
137 | the gradient of the modified model in the following way:
138 |
139 | .. code-block:: python
140 |
141 | import torch
142 | from torchvision.models import vgg11
143 |
144 | from zennit.core import Composite
145 | from zennit.image import imgify
146 |
147 |
148 | # create a VGG11 model with random parameters
149 | model = vgg11()
150 | # use the Canonizer with an "empty" Composite (without specifying
151 | # module_map), which will not attach rules to any sub-module, thus resulting
152 | # in a plain gradient computation, but with a Canonizer applied
153 | composite = Composite(
154 | canonizers=[SoftplusCanonizer()]
155 | )
156 |
157 | input = torch.randn(1, 3, 224, 224, requires_grad=True)
158 | target = torch.eye(1000)[[0]]
159 | with composite.context(model) as modified_model:
160 | out = modified_model(input)
161 | relevance, = torch.autograd.grad(out, input, target, create_graph=True)
162 | # find adversarial example such that input and its respective
163 | # attribution are close
164 | loss = ((relevance - input.detach()) ** 2).mean()
165 | # compute the gradient of input w.r.t. loss, using the second order
166 | # derivative w.r.t. input; note that this currently does not work when
167 | # using BasicHook, which detaches the gradient to avoid wrong values
168 | adv_grad, = torch.autograd.grad(loss, input)
169 |
170 | # visualize adv_grad
171 | imgify(adv_grad[0].abs().sum(0), cmap='hot').show()
172 |
173 |
--------------------------------------------------------------------------------
/docs/source/how-to/write-custom-composites.rst:
--------------------------------------------------------------------------------
1 | =========================
2 | Writing Custom Composites
3 | =========================
4 |
5 | Zennit provides a number of commonly used **Composites**.
6 | While these are often enough for feed-forward-type neural networks, one primary goal of Zennit is to provide the tools to easily customize the computation of rule-based attribution methods.
7 | This is especially useful to analyze novel architectures, for which no attribution-approach has been designed before.
8 |
9 | For most use-cases, using the abstract **Composites** :py:class:`~zennit.composites.LayerMapComposite`, :py:class:`~zennit.composites.SpecialFirstLayerMapComposite`, and :py:class:`~zennit.composites.NameMapComposite` already provides enough freedom to customize which Layer should receive which rule. See :ref:`use-composites` for an introduction.
10 | Depending on the setup, it may however be more convenient to either directly use or implement a new **Composite** by creating a Subclass from :py:class:`zennit.core.Composite`.
11 | In either case, the :py:class:`~zennit.core.Composite` requires an argument ``module_map``, which is a function with the signature ``(ctx: dict, name: str, module: torch.nn.Module) -> Hook or None``, which, given a context dict, the name of a single module and the module itself, either returns an instance of :py:class:`~zennit.core.Hook` which should be copied and registered to the module, or ``None`` if no ``Hook`` should be applied.
12 | The context dict ``ctx`` can be used to track subsequent calls to the ``module_map`` function, e.g. to count the number of processed modules, or to verify if some condition has been met before, e.g. a linear layer has been seen before.
13 | The ``module_map`` is used in :py:meth:`zennit.core.Composite.register`, where the context dict is initialized to an empty dict ``{}`` before iterating over all the sub-modules of the root-module to which the composite will be registered.
14 | The iteration is done using :py:meth:`torch.nn.Module.named_modules`, which will therefore dictate the order modules are visited, which is depth-first in the order sub-modules were assigned.
15 |
16 | A simple **Composite**, which only provides rules for linear layers that are leaves and bases the rule on how many leaf modules were visited before could be implemented like the following:
17 |
18 |
19 | .. code-block:: python
20 |
21 | import torch
22 | from torchvision.models import vgg16
23 | from zennit.rules import Epsilon, AlphaBeta
24 | from zennit.types import Linear
25 | from zennit.core import Composite
26 | from zennit.attribution import Gradient
27 |
28 |
29 | def module_map(ctx, name, module):
30 | # check whether there is at least one child, i.e. the module is not a leaf
31 | try:
32 | next(module.children())
33 | except StopIteration:
34 | # StopIteration is raised if the iterator has no more elements,
35 | # which means in this case there are no children and module is a leaf
36 | pass
37 | else:
38 | # if StopIteration is not raised on the first element, module is not a leaf
39 | return None
40 |
41 | # if the module is not Linear, we do not want to assign a hook
42 | if not isinstance(module, Linear):
43 | return None
44 |
45 | # count the number of the leaves processed yet in 'leafnum'
46 | if 'leafnum' not in ctx:
47 | ctx['leafnum'] = 0
48 | else:
49 | ctx['leafnum'] += 1
50 |
51 | # the first 10 leaf-modules which are of type Linear should be assigned
52 | # the Alpha2Beta1 rule
53 | if ctx['leafnum'] < 10:
54 | return AlphaBeta(alpha=2, beta=1)
55 | # all other rules should be assigned Epsilon
56 | return Epsilon(epsilon=1e-3)
57 |
58 |
59 | # we can then create a composite by passing the module_map function
60 | # canonizers may also be passed as with all composites
61 | composite = Composite(module_map=module_map)
62 |
63 | # try out the composite
64 | model = vgg16()
65 | with Gradient(model, composite) as attributor:
66 | out, grad = attributor(torch.randn(1, 3, 224, 224))
67 |
68 |
69 | A more general **Composite**, where we can specify which layer number and which type should be assigned which rule, can be implemented by creating a class:
70 |
71 | .. code-block:: python
72 |
73 | from itertools import islice
74 |
75 | import torch
76 | from torchvision.models import vgg16
77 | from zennit.rules import Epsilon, ZBox, Gamma, Pass, Norm
78 | from zennit.types import Linear, Convolution, Activation, AvgPool
79 | from zennit.core import Composite
80 | from zennit.attribution import Gradient
81 |
82 |
83 | class LeafNumberTypeComposite(Composite):
84 | def __init__(self, leafnum_map):
85 | # pass the class method self.mapping as the module_map
86 | super().__init__(module_map=self.mapping)
87 | # set the instance attribute so we can use it in self.mapping
88 | self.leafnum_map = leafnum_map
89 |
90 | def mapping(self, ctx, name, module):
91 | # check whether there is at least one child, i.e. the module is not a leaf
92 | # but this time shorter using itertools.islice to get at most one child
93 | if list(islice(module.children(), 1)):
94 | return None
95 |
96 | # count the number of the leaves processed yet in 'leafnum'
97 | # this time in a single line with get and all layers count, e.g. ReLU
98 | ctx['leafnum'] = ctx.get('leafnum', -1) + 1
99 |
100 | # loop over the leafnum_map and use the first template for which
101 | # the module type matches and the current ctx['leafnum'] falls into
102 | # the bounds
103 | for (low, high), dtype, template in self.leafnum_map:
104 | if isinstance(module, dtype) and low <= ctx['leafnum'] < high:
105 | return template
106 | # if none of the leafnum_map apply this means there is no rule
107 | # matching the current layer
108 | return None
109 |
110 |
111 | # this can be compared with int and will always be larger
112 | inf = float('inf')
113 |
114 | # we create an example leafnum-map, note that Linear is here
115 | # zennit.types.Linear and not torch.nn.Linear
116 | # the first two entries are for demonstration only and would
117 | # in practice most likely be a single "Linear" with appropriate low/high
118 | leafnum_map = [
119 | [(0, 1), Convolution, ZBox(low=-3.0, high=3.0)],
120 | [(0, 1), torch.nn.Linear, ZBox(low=0.0, high=1.0)],
121 | [(1, 17), Linear, Gamma(gamma=0.25)],
122 | [(17, 31), Linear, Epsilon(epsilon=0.5)],
123 | [(31, inf), Linear, Epsilon(epsilon=1e-9)],
124 | # catch all activations
125 | [(0, inf), Activation, Pass()],
126 | # explicit None is possible e.g. to (ab-)use precedence
127 | [(0, 17), torch.nn.MaxPool2d, None],
128 | # catch all AvgPool/MaxPool2d, isinstance also accepts tuples of types
129 | [(0, inf), (AvgPool, torch.nn.MaxPool2d), Norm()],
130 | ]
131 |
132 | # finally, create the composite using the leafnum_map
133 | composite = LeafNumberTypeComposite(leafnum_map)
134 |
135 | # try out the composite
136 | model = vgg16()
137 | with Gradient(model, composite) as attributor:
138 | out, grad = attributor(torch.randn(1, 3, 224, 224))
139 |
140 | In practice, however, we do not recommend to use the index of the layer when designing **Composites**, because most of the time, when such a configuration is chosen, it is done to shape the **Composite** for an explicit model.
141 | For these kinds of **Composites**, a :py:class:`~zennit.composites.NameMapComposite` will directly map the name of a sub-module to a Hook, which is a more explicit and transparent way to create a special **Composite** for a single neural network.
142 |
--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
1 | ====================
2 | Zennit Documentation
3 | ====================
4 |
5 | Zennit (Zennit Explains Neural Networks in Torch) is a python framework using PyTorch to compute local attributions in the sense of eXplainable AI (XAI) with a focus on Layer-wise Relevance Propagation.
6 | It works by defining *rules* which are used to overwrite the gradient of PyTorch modules in PyTorch's auto-differentiation engine.
7 | Rules are mapped to layers with *composites*, which contain directions to compute the attributions of a full model, which maps rules to modules based on their properties and context.
8 |
9 | Zennit is available on PyPI and may be installed using:
10 |
11 | .. code-block:: console
12 |
13 | $ pip install zennit
14 |
15 | Contents
16 | --------
17 |
18 | .. toctree::
19 | :maxdepth: 2
20 |
21 | getting-started
22 | how-to/index
23 | tutorial/index
24 | reference/index
25 | bibliography
26 |
27 | Indices and tables
28 | ------------------
29 |
30 | * :ref:`genindex`
31 | * :ref:`modindex`
32 | * :ref:`search`
33 |
34 |
35 | Citing
36 | ------
37 |
38 | If you find Zennit useful, why not cite our related paper :cite:p:`anders2021software`:
39 |
40 | .. code-block:: bibtex
41 |
42 | @article{anders2021software,
43 | author = {Anders, Christopher J. and
44 | Neumann, David and
45 | Samek, Wojciech and
46 | Müller, Klaus-Robert and
47 | Lapuschkin, Sebastian},
48 | title = {Software for Dataset-wide XAI: From Local Explanations to Global Insights with {Zennit}, {CoRelAy}, and {ViRelAy}},
49 | journal = {CoRR},
50 | volume = {abs/2106.13200},
51 | year = {2021},
52 | }
53 |
54 |
--------------------------------------------------------------------------------
/docs/source/reference/index.rst:
--------------------------------------------------------------------------------
1 | ================
2 | API Reference
3 | ================
4 |
5 | .. autosummary::
6 | :toctree:
7 | :nosignatures:
8 | :recursive:
9 | :template: modules.rst
10 |
11 | zennit.attribution
12 | zennit.canonizers
13 | zennit.cmap
14 | zennit.composites
15 | zennit.core
16 | zennit.image
17 | zennit.layer
18 | zennit.rules
19 | zennit.torchvision
20 | zennit.types
21 |
--------------------------------------------------------------------------------
/docs/source/tutorial/index.rst:
--------------------------------------------------------------------------------
1 | ================
2 | Tutorials
3 | ================
4 |
5 | .. toctree::
6 | :maxdepth: 1
7 |
8 | image-classification-vgg-resnet
9 | ..
10 | image-segmentation-with-unet
11 | text-classification-with-tbd
12 | audio-classification-with-tbd
13 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import re
3 | from setuptools import setup, find_packages
4 | from subprocess import run, CalledProcessError
5 |
6 |
7 | def get_long_description(project_path):
8 | '''Fetch the README contents and replace relative links with absolute ones
9 | pointing to github for correct behaviour on PyPI.
10 | '''
11 | try:
12 | revision = run(
13 | ['git', 'describe', '--tags'],
14 | capture_output=True,
15 | check=True,
16 | text=True
17 | ).stdout[:-1]
18 | except CalledProcessError:
19 | try:
20 | with open('PKG-INFO', 'r') as fd:
21 | body = fd.read().partition('\n\n')[2]
22 | if body:
23 | return body
24 | except FileNotFoundError:
25 | revision = 'master'
26 |
27 | with open('README.md', 'r', encoding='utf-8') as fd:
28 | long_description = fd.read()
29 |
30 | link_root = {
31 | '': f'https://github.com/{project_path}/blob',
32 | '!': f'https://raw.githubusercontent.com/{project_path}',
33 | }
34 |
35 | def replace(mobj):
36 | return f'{mobj[1]}[{mobj[2]}]({link_root[mobj[1]]}/{revision}/{mobj[3]})'
37 |
38 | link_rexp = re.compile(r'(!?)\[([^\]]*)\]\((?!https?://|/)([^\)]+)\)')
39 | return link_rexp.sub(replace, long_description)
40 |
41 |
42 | setup(
43 | name='zennit',
44 | use_scm_version=True,
45 | author='chrstphr',
46 | author_email='zennit@j0d.de',
47 | description='Attribution of Neural Networks using PyTorch',
48 | long_description=get_long_description('chr5tphr/zennit'),
49 | long_description_content_type='text/markdown',
50 | url='https://github.com/chr5tphr/zennit',
51 | packages=find_packages(where='src', include=['zennit*']),
52 | package_dir={'': 'src'},
53 | install_requires=[
54 | 'click',
55 | 'numpy',
56 | 'Pillow',
57 | 'torch>=1.7.0',
58 | 'torchvision',
59 | ],
60 | setup_requires=[
61 | 'setuptools_scm',
62 | ],
63 | extras_require={
64 | 'docs': [
65 | 'sphinx-copybutton>=0.4.0',
66 | 'sphinx-rtd-theme>=1.0.0',
67 | 'sphinxcontrib.datatemplates>=0.9.0',
68 | 'sphinxcontrib.bibtex>=2.4.1',
69 | 'nbsphinx>=0.8.8',
70 | 'nbconvert<7.14', # see https://github.com/jupyter/nbconvert/issues/2092
71 | 'ipykernel>=6.13.0',
72 | ],
73 | 'tests': [
74 | 'pytest',
75 | 'pytest-cov',
76 | ]
77 | },
78 | python_requires='>=3.7',
79 | classifiers=[
80 | 'Development Status :: 3 - Alpha',
81 | 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)',
82 | 'Programming Language :: Python :: 3.7',
83 | 'Programming Language :: Python :: 3.8',
84 | 'Programming Language :: Python :: 3.9',
85 | ]
86 | )
87 |
--------------------------------------------------------------------------------
/share/example/feed_forward.py:
--------------------------------------------------------------------------------
1 | '''A quick example to generate heatmaps for vgg16.'''
2 | import os
3 | from functools import partial
4 |
5 | import click
6 | import torch
7 | import numpy as np
8 | from torch.utils.data import DataLoader, Subset
9 | from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor
10 | from torchvision.datasets import ImageFolder
11 | from torchvision.models import vgg11, vgg11_bn, vgg16, vgg16_bn, resnet18, resnet50
12 |
13 | from zennit.attribution import Gradient, SmoothGrad, IntegratedGradients, Occlusion
14 | from zennit.composites import COMPOSITES
15 | from zennit.core import Hook
16 | from zennit.image import imsave, CMAPS
17 | from zennit.layer import Sum
18 | from zennit.torchvision import VGGCanonizer, ResNetCanonizer
19 |
20 |
21 | MODELS = {
22 | 'vgg16': (vgg16, VGGCanonizer),
23 | 'vgg16_bn': (vgg16_bn, VGGCanonizer),
24 | 'vgg11': (vgg11, VGGCanonizer),
25 | 'vgg11_bn': (vgg11_bn, VGGCanonizer),
26 | 'resnet18': (resnet18, ResNetCanonizer),
27 | 'resnet50': (resnet50, ResNetCanonizer),
28 | }
29 |
30 | ATTRIBUTORS = {
31 | 'gradient': Gradient,
32 | 'smoothgrad': SmoothGrad,
33 | 'integrads': IntegratedGradients,
34 | 'occlusion': Occlusion,
35 | 'inputxgrad': IntegratedGradients,
36 | }
37 |
38 |
39 | class SumSingle(Hook):
40 | def __init__(self, dim=1):
41 | super().__init__()
42 | self.dim = dim
43 |
44 | def backward(self, module, grad_input, grad_output):
45 | elems = [torch.zeros_like(grad_output[0])] * (grad_input[0].shape[-1])
46 | elems[self.dim] = grad_output[0]
47 | return (torch.stack(elems, dim=-1),)
48 |
49 |
50 | class BatchNormalize:
51 | def __init__(self, mean, std, device=None):
52 | self.mean = torch.tensor(mean, device=device)[None, :, None, None]
53 | self.std = torch.tensor(std, device=device)[None, :, None, None]
54 |
55 | def __call__(self, tensor):
56 | return (tensor - self.mean) / self.std
57 |
58 |
59 | class AllowEmptyClassImageFolder(ImageFolder):
60 | '''Subclass of ImageFolder, which only finds non-empty classes, but with their correct indices given other empty
61 | classes. This counter-acts the changes in torchvision 0.10.0, in which DatasetFolder does not allow empty classes
62 | anymore by default. Versions before 0.10.0 do not expose `find_classes`, and thus this change does not change the
63 | functionality of `ImageFolder` in earlier versions.
64 | '''
65 | def find_classes(self, directory):
66 | with os.scandir(directory) as scanit:
67 | class_info = sorted((entry.name, len(list(os.scandir(entry.path)))) for entry in scanit if entry.is_dir())
68 | class_to_idx = {class_name: index for index, (class_name, n_members) in enumerate(class_info) if n_members}
69 | if not class_to_idx:
70 | raise FileNotFoundError(f'No non-empty classes found in \'{directory}\'.')
71 | return list(class_to_idx), class_to_idx
72 |
73 |
74 | @click.command()
75 | @click.argument('dataset-root', type=click.Path(file_okay=False))
76 | @click.argument('relevance_format', type=click.Path(dir_okay=False, writable=True))
77 | @click.option('--attributor', 'attributor_name', type=click.Choice(list(ATTRIBUTORS)), default='gradient')
78 | @click.option('--composite', 'composite_name', type=click.Choice(list(COMPOSITES)))
79 | @click.option('--model', 'model_name', type=click.Choice(list(MODELS)), default='vgg16_bn')
80 | @click.option('--parameters', type=click.Path(dir_okay=False))
81 | @click.option(
82 | '--inputs',
83 | 'input_format',
84 | type=click.Path(dir_okay=False, writable=True),
85 | help='Input image format string. {sample} is replaced with the sample index.'
86 | )
87 | @click.option('--batch-size', type=int, default=16)
88 | @click.option('--max-samples', type=int)
89 | @click.option('--n-outputs', type=int, default=1000)
90 | @click.option('--cpu/--gpu', default=True)
91 | @click.option('--shuffle/--no-shuffle', default=False)
92 | @click.option('--with-bias/--no-bias', default=True)
93 | @click.option('--with-residual/--no-residual', default=True)
94 | @click.option('--relevance-norm', type=click.Choice(['symmetric', 'absolute', 'unaligned']), default='symmetric')
95 | @click.option('--cmap', type=click.Choice(list(CMAPS)), default='coldnhot')
96 | @click.option('--level', type=float, default=1.0)
97 | @click.option('--seed', type=int, default=0xDEADBEEF)
98 | def main(
99 | dataset_root,
100 | relevance_format,
101 | attributor_name,
102 | composite_name,
103 | model_name,
104 | parameters,
105 | input_format,
106 | batch_size,
107 | max_samples,
108 | n_outputs,
109 | cpu,
110 | shuffle,
111 | with_bias,
112 | with_residual,
113 | cmap,
114 | level,
115 | relevance_norm,
116 | seed
117 | ):
118 | '''Generate heatmaps of an image folder at DATASET_ROOT to files RELEVANCE_FORMAT.
119 | RELEVANCE_FORMAT is a format string, for which {sample} is replaced with the sample index.
120 | '''
121 | # set a manual seed for the RNG
122 | torch.manual_seed(seed)
123 |
124 | # use the gpu if requested and available, else use the cpu
125 | device = torch.device('cuda:0' if torch.cuda.is_available() and not cpu else 'cpu')
126 |
127 | # mean and std of ILSVRC2012 as computed for the torchvision models
128 | norm_fn = BatchNormalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225), device=device)
129 |
130 | # transforms as used for torchvision model evaluation
131 | transform = Compose([
132 | Resize(256),
133 | CenterCrop(224),
134 | ToTensor(),
135 | ])
136 |
137 | # the dataset is a folder containing folders with samples, where each folder corresponds to one label
138 | dataset = AllowEmptyClassImageFolder(dataset_root, transform=transform)
139 |
140 | # limit the number of output samples, if requested, by creating a subset
141 | if max_samples is not None:
142 | if shuffle:
143 | indices = sorted(np.random.choice(len(dataset), min(len(dataset), max_samples), replace=False))
144 | else:
145 | indices = range(min(len(dataset), max_samples))
146 | dataset = Subset(dataset, indices)
147 |
148 | loader = DataLoader(dataset, shuffle=shuffle, batch_size=batch_size)
149 |
150 | model = MODELS[model_name][0]()
151 |
152 | # load model parameters if requested; the parameter file may need to be downloaded separately
153 | if parameters is not None:
154 | state_dict = torch.load(parameters)
155 | model.load_state_dict(state_dict)
156 | model.to(device)
157 | model.eval()
158 |
159 | # disable requires_grad for all parameters, we do not need their modified gradients
160 | for param in model.parameters():
161 | param.requires_grad = False
162 |
163 | # convenience identity matrix to produce one-hot encodings
164 | eye = torch.eye(n_outputs, device=device)
165 |
166 | # function to compute output relevance given the function output and a target
167 | def attr_output_fn(output, target):
168 | # output times one-hot encoding of the target labels of size (len(target), 1000)
169 | return output * eye[target]
170 |
171 | # create a composite if composite_name was set, otherwise we do not use a composite
172 | composite = None
173 | if composite_name is not None:
174 | composite_kwargs = {}
175 | if composite_name == 'epsilon_gamma_box':
176 | # the maximal input shape, needed for the ZBox rule
177 | shape = (batch_size, 3, 224, 224)
178 |
179 | # the highest and lowest pixel values for the ZBox rule
180 | composite_kwargs['low'] = norm_fn(torch.zeros(*shape, device=device))
181 | composite_kwargs['high'] = norm_fn(torch.ones(*shape, device=device))
182 | if not with_residual and 'resnet' in model_name:
183 | # skip the residual connection through the Sum added by the ResNetCanonizer
184 | composite_kwargs['layer_map'] = [(Sum, SumSingle(1))]
185 |
186 | # provide the name 'bias' in zero_params if no bias should be used to compute the relevance
187 | if not with_bias and composite_name in [
188 | 'epsilon_gamma_box',
189 | 'epsilon_plus',
190 | 'epsilon_alpha2_beta1',
191 | 'epsilon_plus_flat',
192 | 'epsilon_alpha2_beta1_flat',
193 | 'excitation_backprop',
194 | ]:
195 | composite_kwargs['zero_params'] = ['bias']
196 |
197 | # use torchvision specific canonizers, as supplied in the MODELS dict
198 | composite_kwargs['canonizers'] = [MODELS[model_name][1]()]
199 |
200 | # create a composite specified by a name; the COMPOSITES dict includes all preset composites provided by zennit.
201 | composite = COMPOSITES[composite_name](**composite_kwargs)
202 |
203 | # specify some attributor-specific arguments
204 | attributor_kwargs = {
205 | 'smoothgrad': {'noise_level': 0.1, 'n_iter': 20},
206 | 'integrads': {'n_iter': 20},
207 | 'inputxgrad': {'n_iter': 1},
208 | 'occlusion': {'window': (56, 56), 'stride': (28, 28)},
209 | }.get(attributor_name, {})
210 |
211 | # create an attributor, given the ATTRIBUTORS dict given above. If composite is None, the gradient will not be
212 | # modified for the attribution
213 | attributor = ATTRIBUTORS[attributor_name](model, composite, **attributor_kwargs)
214 |
215 | # the current sample index for creating file names
216 | sample_index = 0
217 |
218 | # the accuracy
219 | accuracy = 0.
220 |
221 | # enter the attributor context outside the data loader loop, such that its canonizers and hooks do not need to be
222 | # registered and removed for each step. This registers the composite (and applies the canonizer) to the model
223 | # within the with-statement
224 | with attributor:
225 | for data, target in loader:
226 | # we use data without the normalization applied for visualization, and with the normalization applied as
227 | # the model input
228 | data_norm = norm_fn(data.to(device))
229 |
230 | # create output relevance function of output with fixed target
231 | output_relevance = partial(attr_output_fn, target=target)
232 |
233 | # this will compute the modified gradient of model, where the output relevance is chosen by the as the
234 | # model's output for the ground-truth label index
235 | output, relevance = attributor(data_norm, output_relevance)
236 |
237 | # sum over the color channel for visualization
238 | relevance = np.array(relevance.sum(1).detach().cpu())
239 |
240 | # normalize between 0. and 1. given the specified strategy
241 | if relevance_norm == 'symmetric':
242 | # 0-aligned symmetric relevance, negative and positive can be compared, the original 0. becomes 0.5
243 | amax = np.abs(relevance).max((1, 2), keepdims=True)
244 | relevance = (relevance + amax) / 2 / amax
245 | elif relevance_norm == 'absolute':
246 | # 0-aligned absolute relevance, only the amplitude of relevance matters, the original 0. becomes 0.
247 | relevance = np.abs(relevance)
248 | relevance /= relevance.max((1, 2), keepdims=True)
249 | elif relevance_norm == 'unaligned':
250 | # do not align, the original minimum value becomes 0., the original maximum becomes 1.
251 | rmin = relevance.min((1, 2), keepdims=True)
252 | rmax = relevance.max((1, 2), keepdims=True)
253 | relevance = (relevance - rmin) / (rmax - rmin)
254 |
255 | for n in range(len(data)):
256 | fname = relevance_format.format(sample=sample_index + n)
257 | # zennit.image.imsave will create an appropriate heatmap given a cmap specification
258 | imsave(fname, relevance[n], vmin=0., vmax=1., level=level, cmap=cmap)
259 | if input_format is not None:
260 | fname = input_format.format(sample=sample_index + n)
261 | # if there are 3 color channels, imsave will not create a heatmap, but instead save the image with
262 | # its appropriate colors
263 | imsave(fname, data[n])
264 | sample_index += len(data)
265 |
266 | # update the accuracy
267 | accuracy += (output.argmax(1) == target).sum().detach().cpu().item()
268 |
269 | accuracy /= len(dataset)
270 | print(f'Accuracy: {accuracy:.2f}')
271 |
272 |
273 | if __name__ == '__main__':
274 | main()
275 |
--------------------------------------------------------------------------------
/share/img/beacon_resnet50_various.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/chr5tphr/zennit/e5699aa7e6fb98bec67505af917d0a17cd81d3b5/share/img/beacon_resnet50_various.webp
--------------------------------------------------------------------------------
/share/img/beacon_vgg16_epsilon_gamma_box.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/chr5tphr/zennit/e5699aa7e6fb98bec67505af917d0a17cd81d3b5/share/img/beacon_vgg16_epsilon_gamma_box.png
--------------------------------------------------------------------------------
/share/img/beacon_vgg16_various.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/chr5tphr/zennit/e5699aa7e6fb98bec67505af917d0a17cd81d3b5/share/img/beacon_vgg16_various.webp
--------------------------------------------------------------------------------
/share/img/zennit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/chr5tphr/zennit/e5699aa7e6fb98bec67505af917d0a17cd81d3b5/share/img/zennit.png
--------------------------------------------------------------------------------
/share/img/zennit.svg:
--------------------------------------------------------------------------------
1 |
2 | 
85 |
86 |
87 |
88 |
89 | 
90 |
91 |
92 |