├── .github └── workflows │ └── blank.yml ├── INSTALL.md ├── LICENSE ├── README.md ├── __init__.py ├── coverages ├── __init__.py ├── idc.py ├── kmn.py ├── neuron_cov.py ├── sa.py ├── ss.py └── tkn.py ├── lrp_toolbox ├── .gitignore ├── __init__.py ├── cnn_training_test.py ├── data_io.py ├── lrp_cnn_demo.py ├── lrp_demo.py ├── mini.py ├── model_io.py ├── models │ └── MNIST │ │ ├── LeNet1.txt │ │ └── LeNet5.txt ├── modules │ ├── __init__.py │ ├── convolution.py │ ├── flatten.py │ ├── linear.py │ ├── maxpool.py │ ├── module.py │ ├── rect.py │ ├── sequential.py │ ├── softmax.py │ ├── sumpool.py │ └── tanh.py ├── render.py └── training_test.py ├── neural_networks ├── LeNet1.h5 ├── LeNet1.json ├── LeNet4.h5 ├── LeNet4.json ├── LeNet5.h5 ├── LeNet5.json ├── __init__.py ├── cifar_original.h5 ├── dave2.h5 └── dave_model.py ├── run.py └── utils.py /.github/workflows/blank.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Run a one-line script 13 | run: echo Hello, world! 14 | - name: Run a multi-line script 15 | run: | 16 | echo Add other actions to build, 17 | echo test, and deploy your project. 18 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | ## Setting up DeepImportance 2 | 3 | One can reeach the source code of our implementation via terminal command 4 | 5 | git clone https://github.com/DeepImportance/deepimportance_code_release.git 6 | 7 | Running this code requires `python2.7` to be installed. Check [this](https://www.python.org/downloads/) 8 | website for installation. 9 | 10 | Also, there are multiple libraries needed. We strongly recommend you to create 11 | a virtual environment and install packages in this environment. 12 | For creating a virtual environment check 13 | [this](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/) 14 | webpage. 15 | 16 | Once the virtual environemnt is set, you can install required libraries by using 17 | `pip package manager` as shown below. If `pip` is not installed on your 18 | computer first install it as described 19 | [here](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/) 20 | or [here](https://pip.pypa.io/en/stable/installing/). 21 | 22 | pip install --upgrade https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.10.0-py2-none-any.whl (OSX) 23 | pip install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.10.0-py2-none-any.whl (Linux) 24 | pip install Keras==2.2.2 25 | pip install numpy 26 | pip install scikit-learn 27 | pip install scikit-image 28 | pip install matplotlib 29 | pip install pillow 30 | pip install cleverhans 31 | 32 | Once all the packages are installed you are ready to run DeepImportance code. 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 DeepImportance 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Importance-Driven Deep Learning System Testing -ICSE 2020 2 | 3 | ### About 4 | This paper [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3628024.svg)](https://doi.org/10.5281/zenodo.3628024) 5 | presents DeepImportance, a systematic testing methodology accompanied by an Importance-Driven (IDC) 6 | test adequacy criterion for DL systems. Applying IDC enables to 7 | establish a layer-wise functional understanding of the importance 8 | of DL system components and use this information to guide the 9 | generation of semantically-diverse test sets. Our empirical evalua- 10 | tion on several DL systems, across multiple DL datasets and with 11 | state-of-the-art adversarial generation techniques demonstrates the 12 | usefulness and effectiveness of DeepImportance and its ability to 13 | guide the engineering of more robust DL systems. 14 | 15 | ### Repository 16 | This repository includes details about the artifact corresponding to implementation of DeepImportance. 17 | Our implementation is publicly available in 18 | [DeepImportance repository](https://github.com/DeepImportance/deepimportance_code_release). 19 | This artifact allows reproducing the experimental results presented in the paper. Below we 20 | describe how to reproduce results. Before going further, first, check 21 | installation page (i.e. INSTALL.md). 22 | 23 | 24 | ### Notes 25 | * If you use Python 3.8, Cleverhans does not yet support Tensforflow 2.x, so you should make a change at utils_tf.py 26 | ```python 27 | def kl_with_logits(p_logits, q_logits, scope=None, loss_collection=tf.compat.v1.GraphKeys.REGULARIZATION_LOSSES): 28 | ``` 29 | See [https://github.com/cleverhans-lab/cleverhans/issues/1183](https://github.com/cleverhans-lab/cleverhans/issues/1183) 30 | 31 | 32 | 33 | ### Updates 34 | 35 | * **02/03/21**: Updated with support for Python 3.8 36 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/__init__.py -------------------------------------------------------------------------------- /coverages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/coverages/__init__.py -------------------------------------------------------------------------------- /coverages/idc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn import cluster 3 | 4 | from utils import save_quantization, load_quantization, save_totalR, load_totalR 5 | from utils import save_layerwise_relevances, load_layerwise_relevances 6 | from utils import get_layer_outs_new, create_dir 7 | from lrp_toolbox.model_io import write, read 8 | 9 | from sklearn.metrics import silhouette_score 10 | 11 | experiment_folder = 'experiments' 12 | model_folder = 'neural_networks' 13 | 14 | class ImportanceDrivenCoverage: 15 | def __init__(self,model, model_name, num_relevant_neurons, selected_class, subject_layer, 16 | train_inputs, train_labels): 17 | self.covered_combinations = () 18 | 19 | self.model = model 20 | self.model_name = model_name 21 | self.num_relevant_neurons = num_relevant_neurons 22 | self.selected_class = selected_class 23 | self.subject_layer = subject_layer 24 | self.train_inputs = train_inputs 25 | self.train_labels = train_labels 26 | 27 | #create experiment directory if not exists 28 | create_dir(experiment_folder) 29 | 30 | 31 | def get_measure_state(self): 32 | return self.covered_combinations 33 | 34 | def set_measure_state(self, covered_combinations): 35 | self.covered_combinations = covered_combinations 36 | 37 | def test(self, test_inputs): 38 | ######################### 39 | #1.Find Relevant Neurons# 40 | ######################### 41 | 42 | try: 43 | print("Loading relevance scores") 44 | totalR = load_totalR('%s/%s_%s_%d' 45 | %(experiment_folder, self.model_name, 46 | 'totalR', self.selected_class), 0) 47 | 48 | relevant_neurons = np.argsort(totalR[self.subject_layer])[0][::-1][:self.num_relevant_neurons] 49 | # relevant_neurons = load_layerwise_relevances('%s/%s_%d_%d_%d' 50 | # %(experiment_folder, 51 | # self.model_name, 52 | # self.num_relevant_neurons, 53 | # self.selected_class, 54 | # self.subject_layer)) 55 | except Exception as e: 56 | print("Relevance scores must be calculated. Doing it now!") 57 | # Convert keras model into txt 58 | model_path = model_folder + '/' + self.model_name 59 | write(model_path, model_path, num_channels=test_inputs[0].shape[-1], fmt='keras_txt') 60 | 61 | lrpmodel = read(model_path + '.txt', 'txt') # 99.16% prediction accuracy 62 | lrpmodel.drop_softmax_output_layer() # drop softnax output layer for analysis 63 | 64 | relevant_neurons, least_relevant_neurons, total_R = find_relevant_neurons( 65 | self.model, lrpmodel, self.train_inputs, self.train_labels, 66 | self.subject_layer, self.num_relevant_neurons, None, 'sum') 67 | 68 | save_totalR(total_R, '%s/%s_%s_%d' 69 | %(experiment_folder, self.model_name, 70 | 'totalR', self.selected_class), 0) 71 | 72 | 73 | #################################### 74 | #2.Quantize Relevant Neuron Outputs# 75 | #################################### 76 | if 'conv' in self.model.layers[self.subject_layer].name: is_conv = True 77 | else: is_conv = False 78 | 79 | try: 80 | print("Loading unsupervised clustering results") 81 | qtized = load_quantization('%s/%s_%d_%d_%d_silhouette' 82 | %(experiment_folder, 83 | self.model_name, 84 | self.selected_class, 85 | self.subject_layer, 86 | self.num_relevant_neurons),0) 87 | except: 88 | print("Clustering results NOT FOUND; Calculating them now!") 89 | train_layer_outs = get_layer_outs_new(self.model, np.array(self.train_inputs)) 90 | 91 | qtized = quantizeSilhouette(train_layer_outs[self.subject_layer], is_conv, 92 | relevant_neurons) 93 | save_quantization(qtized, '%s/%s_%d_%d_%d_silhouette' 94 | %(experiment_folder, 95 | self.model_name, 96 | self.selected_class, 97 | self.subject_layer, 98 | self.num_relevant_neurons),0) 99 | 100 | 101 | #################### 102 | #3.Measure coverage# 103 | #################### 104 | print("Calculating IDC coverage") 105 | test_layer_outs = get_layer_outs_new(self.model, np.array(test_inputs)) 106 | 107 | coverage, covered_combinations, max_comb = measure_idc(self.model, self.model_name, 108 | test_inputs, self.subject_layer, 109 | relevant_neurons, 110 | self.selected_class, 111 | test_layer_outs, qtized, is_conv, 112 | self.covered_combinations) 113 | 114 | return coverage, covered_combinations, max_comb 115 | 116 | 117 | def quantize(out_vectors, conv, relevant_neurons, n_clusters=3): 118 | #if conv: n_clusters+=1 119 | quantized_ = [] 120 | 121 | for i in range(out_vectors.shape[-1]): 122 | out_i = [] 123 | for l in out_vectors: 124 | if conv: #conv layer 125 | out_i.append(np.mean(l[...,i])) 126 | else: 127 | out_i.append(l[i]) 128 | 129 | #If it is a convolutional layer no need for 0 output check 130 | if not conv: out_i = filter(lambda elem: elem != 0, out_i) 131 | values = [] 132 | if not len(out_i) < 10: #10 is threshold of number positives in all test input activations 133 | kmeans = cluster.KMeans(n_clusters=n_clusters) 134 | kmeans.fit(np.array(out_i).reshape(-1, 1)) 135 | values = kmeans.cluster_centers_.squeeze() 136 | values = list(values) 137 | values = limit_precision(values) 138 | 139 | #if not conv: values.append(0) #If it is convolutional layer we dont add directly since thake average of whole filter. 140 | 141 | quantized_.append(values) 142 | 143 | quantized_ = [quantized_[rn] for rn in relevant_neurons] 144 | 145 | return quantized_ 146 | 147 | 148 | def quantizeSilhouette(out_vectors, conv, relevant_neurons): 149 | quantized_ = [] 150 | 151 | for i in relevant_neurons: 152 | 153 | out_i = [] 154 | for l in out_vectors: 155 | if conv: #conv layer 156 | out_i.append(np.mean(l[...,i])) 157 | else: 158 | out_i.append(l[i]) 159 | 160 | #If it is a convolutional layer no need for 0 output check 161 | if not conv: 162 | out_i = [item for item in out_i if item != 0] # out_i = filter(lambda elem: elem != 0, out_i) 163 | 164 | values = [] 165 | if not len(out_i) < 10: #10 is threshold of number positives in all test input activations 166 | clusterSize = range(2, 5)#[2, 3, 4] 167 | clustersDict = {} 168 | for clusterNum in clusterSize: 169 | kmeans = cluster.KMeans(n_clusters=clusterNum) 170 | clusterLabels = kmeans.fit_predict(np.array(out_i).reshape(-1, 1)) 171 | silhouetteAvg = silhouette_score(np.array(out_i).reshape(-1, 1), clusterLabels) 172 | clustersDict [silhouetteAvg] = kmeans 173 | 174 | maxSilhouetteScore = max(clustersDict.keys()) 175 | bestKMean = clustersDict[maxSilhouetteScore] 176 | 177 | values = bestKMean.cluster_centers_.squeeze() 178 | values = list(values) 179 | values = limit_precision(values) 180 | 181 | # if not conv: values.append(0) #If it is convolutional layer we dont add directly since thake average of whole filter. 182 | if len(values) == 0: 183 | values.append(0) 184 | 185 | quantized_.append(values) 186 | 187 | return quantized_ 188 | 189 | 190 | def quantizeSilhouetteOld(out_vectors, conv, relevant_neurons): 191 | #if conv: n_clusters+=1 192 | quantized_ = [] 193 | 194 | for i in range(out_vectors.shape[-1]): 195 | if i not in relevant_neurons: continue 196 | 197 | out_i = [] 198 | for l in out_vectors: 199 | if conv: #conv layer 200 | out_i.append(np.mean(l[...,i])) 201 | else: 202 | out_i.append(l[i]) 203 | 204 | #If it is a convolutional layer no need for 0 output check 205 | if not conv: 206 | out_i = [item for item in out_i if item != 0] 207 | # out_i = filter(lambda elem: elem != 0, out_i) 208 | values = [] 209 | 210 | if not len(out_i) < 10: #10 is threshold of number positives in all test input activations 211 | 212 | clusterSize = range(2, 6)#[2, 3, 4, 5] 213 | clustersDict = {} 214 | for clusterNum in clusterSize: 215 | kmeans = cluster.KMeans(n_clusters=clusterNum) 216 | clusterLabels = kmeans.fit_predict(np.array(out_i).reshape(-1, 1)) 217 | silhouetteAvg = silhouette_score(np.array(out_i).reshape(-1, 1), clusterLabels) 218 | clustersDict [silhouetteAvg] = kmeans 219 | 220 | maxSilhouetteScore = max(clustersDict.keys()) 221 | bestKMean = clustersDict[maxSilhouetteScore] 222 | 223 | values = bestKMean.cluster_centers_.squeeze() 224 | values = list(values) 225 | values = limit_precision(values) 226 | 227 | #if not conv: values.append(0) #If it is convolutional layer we dont add directly since thake average of whole filter. 228 | if len(values) == 0: values.append(0) 229 | 230 | quantized_.append(values) 231 | #quantized_ = [quantized_[rn] for rn in relevant_neurons] 232 | 233 | return quantized_ 234 | 235 | 236 | def limit_precision(values, prec=2): 237 | limited_values = [] 238 | for v in values: 239 | limited_values.append(round(v,prec)) 240 | 241 | return limited_values 242 | 243 | 244 | def determine_quantized_cover(lout, quantized): 245 | covered_comb = [] 246 | for idx, l in enumerate(lout): 247 | #if l == 0: 248 | # covered_comb.append(0) 249 | #else: 250 | closest_q = min(quantized[idx], key=lambda x:abs(x-l)) 251 | covered_comb.append(closest_q) 252 | 253 | return covered_comb 254 | 255 | 256 | def measure_idc(model, model_name, test_inputs, subject_layer, 257 | relevant_neurons, sel_class, 258 | test_layer_outs, qtized, is_conv, 259 | covered_combinations=()): 260 | 261 | subject_layer = subject_layer - 1 262 | for test_idx in range(len(test_inputs)): 263 | if is_conv: 264 | lout = [] 265 | for r in relevant_neurons: 266 | lout.append(np.mean(test_layer_outs[subject_layer][test_idx][...,r])) 267 | else: 268 | lout = test_layer_outs[subject_layer][test_idx][relevant_neurons] 269 | 270 | comb_to_add = determine_quantized_cover(lout, qtized) 271 | 272 | if comb_to_add not in covered_combinations: 273 | covered_combinations += (comb_to_add,) 274 | 275 | max_comb = 1#q_granularity**len(relevant_neurons) 276 | for q in qtized: 277 | max_comb *= len(q) 278 | 279 | covered_num = len(covered_combinations) 280 | coverage = float(covered_num)/max_comb 281 | 282 | return coverage*100, covered_combinations, max_comb 283 | 284 | 285 | def find_relevant_neurons(kerasmodel, lrpmodel, inps, outs, subject_layer, \ 286 | num_rel, lrpmethod=None, final_relevance_method='sum'): 287 | 288 | #final_relevants = np.zeros([1, kerasmodel.layers[subject_layer].output_shape[-1]]) 289 | 290 | totalR = None 291 | cnt = 0 292 | for inp in inps: 293 | cnt+=1 294 | ypred = lrpmodel.forward(np.expand_dims(inp, axis=0)) 295 | 296 | #prepare initial relevance to reflect the model's dominant prediction (ie depopulate non-dominant output neurons) 297 | mask = np.zeros_like(ypred) 298 | mask[:,np.argmax(ypred)] = 1 299 | Rinit = ypred*mask 300 | 301 | if not lrpmethod: 302 | R_inp, R_all = lrpmodel.lrp(Rinit) #as Eq(56) from DOI: 10.1371/journal.pone.0130140 303 | elif lrpmethod == 'epsilon': 304 | R_inp, R_all = lrpmodel.lrp(Rinit,'epsilon',0.01) #as Eq(58) from DOI: 10.1371/journal.pone.0130140 305 | elif lrpmethod == 'alphabeta': 306 | R_inp, R_all = lrpmodel.lrp(Rinit,'alphabeta',3) #as Eq(60) from DOI: 10.1371/journal.pone.0130140 307 | else: 308 | print('Unknown LRP method!') 309 | raise Exception 310 | 311 | if totalR: 312 | for idx, elem in enumerate(totalR): 313 | totalR[idx] = elem + R_all[idx] 314 | else: 315 | totalR = R_all 316 | 317 | # THE MOST RELEVANT THE LEAST RELEVANT 318 | return np.argsort(totalR[subject_layer])[0][::-1][:num_rel], np.argsort(totalR[subject_layer])[0][:num_rel], totalR 319 | # return np.argsort(final_relevants)[0][::-1][:num_rel], np.argsort(final_relevants)[0][:num_rel], totalR 320 | 321 | 322 | 323 | 324 | -------------------------------------------------------------------------------- /coverages/kmn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | from utils import get_layer_outs_new, calc_major_func_regions, percent_str 4 | from math import floor 5 | 6 | 7 | class DeepGaugePercentCoverage: 8 | """ 9 | Implements KMN, NBC and SNAC coverage metrics from "DeepGauge: Multi-Granularity Testing Criteria for Deep Learning 10 | Systems" by Ma et al. 11 | 12 | Supports incremental measurements using which one can observe the effect of new inputs to the coverage 13 | values. 14 | """ 15 | 16 | def __init__(self, model, k, train_inputs=None, major_func_regions=None, skip_layers=None): 17 | """ 18 | :param model: Model 19 | :param k: k parameter for KNC metric 20 | :param train_inputs: Training inputs which used to compute the major function regions. Omitted if 21 | major_func_regions is provided. 22 | :param major_func_regions: Major function regions as defined in the paper. If not supplied, model will be run 23 | on the whole train_inputs once. This is intended to speed up the process when the class is used in incremental 24 | manner. 25 | :param skip_layers: Layers to be skipped (e.g. flatten layers) 26 | """ 27 | 28 | self.activation_table_by_section, self.upper_activation_table, self.lower_activation_table = {}, {}, {} 29 | self.neuron_set = set() 30 | 31 | self.model = model 32 | self.k = k 33 | self.skip_layers = skip_layers = ([] if skip_layers is None else skip_layers) 34 | 35 | if major_func_regions is None: 36 | if train_inputs is None: 37 | raise ValueError("Training inputs must be provided when major function regions are not given") 38 | 39 | self.major_func_regions = calc_major_func_regions(model, train_inputs, skip_layers) 40 | else: 41 | self.major_func_regions = major_func_regions 42 | 43 | def get_measure_state(self): 44 | """ 45 | Returns a state object that can be used to incrementally measure the coverage 46 | :return: The current measurement state 47 | """ 48 | return [self.activation_table_by_section, self.upper_activation_table, self.lower_activation_table, 49 | self.neuron_set] 50 | 51 | def set_measure_state(self, state): 52 | """ 53 | Restores the measure state to continue the measurement from that state 54 | :param state: Measurement state as returned from a call to get_measure_state 55 | :return: None 56 | """ 57 | self.activation_table_by_section = state[0] 58 | self.upper_activation_table = state[1] 59 | self.lower_activation_table = state[2] 60 | self.neuron_set = state[3] 61 | 62 | def test(self, test_inputs): 63 | """ 64 | Measures KMN, NBC and SNAC coverages 65 | :param test_inputs: Inputs 66 | :return: A tuple containing the coverage results along with the values that are used to compute them as 67 | described in the paper 68 | """ 69 | outs = get_layer_outs_new(self.model, test_inputs, self.skip_layers) 70 | 71 | kmnc_cnt = 0 72 | nbc_cnt = 0 73 | snac_cnt = 0 74 | kmnc_used_inps = [] 75 | low_used_inps = [] 76 | strong_used_inps = [] 77 | 78 | for layer_index, layer_out in enumerate(outs): # layer_out is output of a particular layer for all inputs 79 | cntr = 0 80 | 81 | for out_for_input in layer_out: # out_for_input is output of a particular layer for single input 82 | for neuron_index in range(out_for_input.shape[-1]): 83 | neuron_out = np.mean(out_for_input[..., neuron_index]) 84 | global_neuron_index = (layer_index, neuron_index) 85 | 86 | self.neuron_set.add(global_neuron_index) 87 | 88 | neuron_low = self.major_func_regions[layer_index][0][neuron_index] 89 | neuron_high = self.major_func_regions[layer_index][1][neuron_index] 90 | section_length = (neuron_high - neuron_low) / self.k 91 | section_index = floor((neuron_out - neuron_low) / section_length) if section_length > 0 else 0 92 | 93 | if not (global_neuron_index, 94 | section_index) in self.activation_table_by_section and cntr not in kmnc_used_inps: 95 | kmnc_cnt += 1 96 | kmnc_used_inps.append(cntr) 97 | 98 | self.activation_table_by_section[(global_neuron_index, section_index)] = True 99 | 100 | f1 = False 101 | f2 = False 102 | if neuron_out < neuron_low: 103 | if global_neuron_index not in self.lower_activation_table and cntr not in low_used_inps: 104 | f1 = True 105 | low_used_inps.append(cntr) 106 | self.lower_activation_table[global_neuron_index] = True 107 | elif neuron_out > neuron_high: 108 | if global_neuron_index not in self.upper_activation_table and cntr not in strong_used_inps: 109 | f2 = True 110 | snac_cnt += 1 111 | strong_used_inps.append(cntr) 112 | self.upper_activation_table[global_neuron_index] = True 113 | 114 | if f1 or f2: 115 | nbc_cnt += 1 116 | 117 | cntr += 1 118 | 119 | multisection_activated = len(self.activation_table_by_section.keys()) 120 | lower_activated = len(self.lower_activation_table.keys()) 121 | upper_activated = len(self.upper_activation_table.keys()) 122 | 123 | total = len(self.neuron_set) 124 | 125 | return (percent_str(multisection_activated, self.k * total), #  kmn 126 | multisection_activated, 127 | percent_str(upper_activated + lower_activated, 2 * total), #  nbc 128 | percent_str(upper_activated, total), # snac 129 | lower_activated, upper_activated, total, 130 | multisection_activated, upper_activated, lower_activated, total, 131 | outs, kmnc_cnt, nbc_cnt, snac_cnt) 132 | -------------------------------------------------------------------------------- /coverages/neuron_cov.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('../') 4 | 5 | import numpy as np 6 | from utils import get_layer_outs_new, percent_str 7 | from collections import defaultdict 8 | 9 | 10 | def default_scale(intermediate_layer_output, rmax=1, rmin=0): 11 | X_std = (intermediate_layer_output - intermediate_layer_output.min()) / ( 12 | intermediate_layer_output.max() - intermediate_layer_output.min()) 13 | X_scaled = X_std * (rmax - rmin) + rmin 14 | 15 | return X_scaled 16 | 17 | 18 | class NeuronCoverage: 19 | """ 20 | Implements Neuron Coverage metric from "DeepXplore: Automated Whitebox Testing of Deep Learning Systems" by Pei 21 | et al. 22 | 23 | Supports incremental measurements using which one can observe the effect of new inputs to the coverage 24 | values. 25 | """ 26 | 27 | def __init__(self, model, scaler=default_scale, threshold=0, skip_layers=None): 28 | self.activation_table = defaultdict(bool) 29 | 30 | self.model = model 31 | self.scaler = scaler 32 | self.threshold = threshold 33 | self.skip_layers = skip_layers = ([] if skip_layers is None else skip_layers) 34 | 35 | def get_measure_state(self): 36 | return [self.activation_table] 37 | 38 | def set_measure_state(self, state): 39 | self.activation_table = state[0] 40 | 41 | def test(self, test_inputs): 42 | """ 43 | :param test_inputs: Inputs 44 | :return: Tuple containing the coverage and the measurements used to compute the coverage. 0th element is the 45 | percentage neuron coverage value. 46 | """ 47 | outs = get_layer_outs_new(self.model, test_inputs, self.skip_layers) 48 | used_inps = [] 49 | nc_cnt = 0 50 | for layer_index, layer_out in enumerate(outs): # layer_out is output of layer for all inputs 51 | inp_cnt = 0 52 | for out_for_input in layer_out: # out_for_input is output of layer for single input 53 | out_for_input = self.scaler(out_for_input) 54 | for neuron_index in range(out_for_input.shape[-1]): 55 | if not self.activation_table[(layer_index, neuron_index)] and np.mean( 56 | out_for_input[..., neuron_index]) > self.threshold and inp_cnt not in used_inps: 57 | used_inps.append(inp_cnt) 58 | nc_cnt += 1 59 | self.activation_table[(layer_index, neuron_index)] = self.activation_table[ 60 | (layer_index, neuron_index)] or np.mean( 61 | out_for_input[..., neuron_index]) > self.threshold 62 | 63 | inp_cnt += 1 64 | 65 | covered = len([1 for c in self.activation_table.values() if c]) 66 | total = len(self.activation_table.keys()) 67 | 68 | return percent_str(covered, total), covered, total, outs, nc_cnt 69 | -------------------------------------------------------------------------------- /coverages/sa.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | from multiprocessing import Pool 5 | from keras.models import load_model, Model 6 | from scipy.stats import gaussian_kde 7 | 8 | from utils import * 9 | 10 | class SurpriseAdequacy: 11 | def __init__(self, surprise, model, train_inputs, layer_names, upper_bound, dataset): 12 | 13 | self.surprise = surprise 14 | self.model = model 15 | self.train_inputs = train_inputs 16 | self.layer_names = layer_names 17 | self.upper_bound = upper_bound 18 | self.n_buckets = 1000 19 | self.dataset = dataset 20 | self.save_path='./sa_data' 21 | if dataset == 'drive': self.is_classification = False 22 | else: self.is_classification = True 23 | self.num_classes = 10 24 | self.var_threshold = 1e-5 25 | 26 | def get_measure_state(self): 27 | return self.surprise 28 | 29 | def set_measure_state(self, surprise): 30 | self.surprise = surprise 31 | 32 | def test(self, test_inputs, dataset_name, instance='dsa'): 33 | 34 | if instance == 'lsa': 35 | 36 | print(len(test_inputs)) 37 | print(len(self.surprise)) 38 | 39 | target_lsa = fetch_lsa(self.model, self.train_inputs, test_inputs, 40 | dataset_name, self.layer_names, 41 | self.num_classes, self.is_classification, 42 | self.var_threshold, self.save_path, self.dataset) 43 | 44 | 45 | coverage = get_sc(np.amin(target_lsa), self.upper_bound, 46 | self.n_buckets, target_lsa) 47 | 48 | 49 | elif instance == 'dsa': 50 | 51 | target_dsa = fetch_dsa(self.model, self.train_inputs, test_inputs, 52 | dataset_name, self.layer_names, 53 | self.num_classes, self.is_classification, 54 | self.save_path, self.dataset) 55 | 56 | coverage = get_sc(np.amin(target_dsa), self.upper_bound, 57 | self.n_buckets, target_dsa) 58 | 59 | 60 | return coverage 61 | 62 | 63 | 64 | def _aggr_output(x): 65 | return [np.mean(x[..., j]) for j in range(x.shape[-1])] 66 | 67 | 68 | def _get_saved_path(base_path, dataset, dtype, layer_names): 69 | """Determine saved path of ats and pred 70 | 71 | Args: 72 | base_path (str): Base save path. 73 | dataset (str): Name of dataset. 74 | dtype (str): Name of dataset type (e.g., train, test, fgsm, ...). 75 | layer_names (list): List of layer names. 76 | 77 | Returns: 78 | ats_path: File path of ats. 79 | pred_path: File path of pred (independent of layers) 80 | """ 81 | 82 | joined_layer_names = "_".join(layer_names) 83 | return ( 84 | os.path.join( 85 | base_path, 86 | dataset + "_" + dtype + "_" + joined_layer_names + "_ats" + ".npy", 87 | ), 88 | os.path.join(base_path, dataset + "_" + dtype + "_pred" + ".npy"), 89 | ) 90 | 91 | 92 | def get_ats( 93 | model, 94 | dataset, 95 | name, 96 | layer_names, 97 | save_path=None, 98 | batch_size=128, 99 | is_classification=True, 100 | num_classes=10, 101 | num_proc=10, 102 | ): 103 | """Extract activation traces of dataset from model. 104 | 105 | Args: 106 | model (keras model): Subject model. 107 | dataset (list): Set of inputs fed into the model. 108 | name (str): Name of input set. 109 | layer_names (list): List of selected layer names. 110 | save_path (tuple): Paths of being saved ats and pred. 111 | batch_size (int): Size of batch when serving. 112 | is_classification (bool): Task type, True if classification task or False. 113 | num_classes (int): The number of classes (labels) in the dataset. 114 | num_proc (int): The number of processes for multiprocessing. 115 | 116 | Returns: 117 | ats (list): List of (layers, inputs, neuron outputs). 118 | pred (list): List of predicted classes. 119 | """ 120 | 121 | temp_model = Model( 122 | inputs=model.input, 123 | outputs=[model.get_layer(layer_name).output for layer_name in layer_names], 124 | ) 125 | 126 | prefix = "[" + name + "] " 127 | 128 | if is_classification: 129 | p = Pool(num_proc) 130 | print(prefix + "Model serving") 131 | 132 | print(len(model.layers)) 133 | 134 | pred = model.predict(dataset, batch_size=batch_size, verbose=1) 135 | if len(layer_names) == 1: 136 | layer_outputs = [ 137 | temp_model.predict(dataset, batch_size=batch_size, verbose=1) 138 | ] 139 | else: 140 | layer_outputs = temp_model.predict( 141 | dataset, batch_size=batch_size, verbose=1 142 | ) 143 | 144 | print(prefix + "Processing ATs") 145 | ats = None 146 | for layer_name, layer_output in zip(layer_names, layer_outputs): 147 | print("Layer: " + layer_name) 148 | if layer_output[0].ndim == 3: 149 | # For convolutional layers 150 | layer_matrix = np.array( 151 | p.map(_aggr_output, [layer_output[i] for i in range(len(dataset))]) 152 | ) 153 | else: 154 | layer_matrix = np.array(layer_output) 155 | 156 | if ats is None: 157 | ats = layer_matrix 158 | else: 159 | ats = np.append(ats, layer_matrix, axis=1) 160 | layer_matrix = None 161 | 162 | if save_path is not None: 163 | np.save(save_path[0], ats) 164 | np.save(save_path[1], pred) 165 | 166 | return ats, pred 167 | 168 | 169 | def find_closest_at(at, train_ats): 170 | """The closest distance between subject AT and training ATs. 171 | 172 | Args: 173 | at (list): List of activation traces of an input. 174 | train_ats (list): List of activation traces in training set (filtered) 175 | 176 | Returns: 177 | dist (int): The closest distance. 178 | at (list): Training activation trace that has the closest distance. 179 | """ 180 | 181 | dist = np.linalg.norm(at - train_ats, axis=1) 182 | return (min(dist), train_ats[np.argmin(dist)]) 183 | 184 | 185 | def _get_train_target_ats(model, x_train, x_target, target_name, layer_names, 186 | num_classes, is_classification, save_path, dataset): 187 | """Extract ats of train and target inputs. If there are saved files, then skip it. 188 | 189 | Args: 190 | model (keras model): Subject model. 191 | x_train (list): Set of training inputs. 192 | x_target (list): Set of target (test or adversarial) inputs. 193 | target_name (str): Name of target set. 194 | layer_names (list): List of selected layer names. 195 | args: keyboard args. 196 | 197 | Returns: 198 | train_ats (list): ats of train set. 199 | train_pred (list): pred of train set. 200 | target_ats (list): ats of target set. 201 | target_pred (list): pred of target set. 202 | """ 203 | 204 | saved_train_path = _get_saved_path(save_path, dataset, "train", layer_names) 205 | if os.path.exists(saved_train_path[0]): 206 | print("Found saved {} ATs, skip serving".format("train")) 207 | # In case train_ats is stored in a disk 208 | train_ats = np.load(saved_train_path[0]) 209 | train_pred = np.load(saved_train_path[1]) 210 | else: 211 | train_ats, train_pred = get_ats( 212 | model, 213 | x_train, 214 | "train", 215 | layer_names, 216 | num_classes=num_classes, 217 | is_classification=is_classification, 218 | save_path=saved_train_path, 219 | ) 220 | print("train ATs is saved at " + saved_train_path[0]) 221 | 222 | saved_target_path = _get_saved_path( 223 | save_path, dataset, target_name, layer_names 224 | ) 225 | #Team DEEPLRP 226 | if False:#os.path.exists(saved_target_path[0]): 227 | print("Found saved {} ATs, skip serving").format(target_name) 228 | # In case target_ats is stored in a disk 229 | target_ats = np.load(saved_target_path[0]) 230 | target_pred = np.load(saved_target_path[1]) 231 | else: 232 | target_ats, target_pred = get_ats( 233 | model, 234 | x_target, 235 | target_name, 236 | layer_names, 237 | num_classes=num_classes, 238 | is_classification=is_classification, 239 | save_path=saved_target_path, 240 | ) 241 | print(target_name + " ATs is saved at " + saved_target_path[0]) 242 | 243 | return train_ats, train_pred, target_ats, target_pred 244 | 245 | 246 | def fetch_dsa(model, x_train, x_target, target_name, layer_names, num_classes, 247 | is_classification, save_path, dataset): 248 | """Distance-based SA 249 | 250 | Args: 251 | model (keras model): Subject model. 252 | x_train (list): Set of training inputs. 253 | x_target (list): Set of target (test or adversarial) inputs. 254 | target_name (str): Name of target set. 255 | layer_names (list): List of selected layer names. 256 | args: keyboard args. 257 | 258 | Returns: 259 | dsa (list): List of dsa for each target input. 260 | """ 261 | 262 | #assert args.is_classification == True 263 | 264 | prefix = "[" + target_name + "] " 265 | train_ats, train_pred, target_ats, target_pred = _get_train_target_ats( 266 | model, x_train, x_target, target_name, layer_names, num_classes, 267 | is_classification, save_path, dataset 268 | ) 269 | 270 | class_matrix = {} 271 | all_idx = [] 272 | for i, label in enumerate(train_pred): 273 | if label.argmax(axis=-1) not in class_matrix: 274 | class_matrix[label.argmax(axis=-1)] = [] 275 | class_matrix[label.argmax(axis=-1)].append(i) 276 | all_idx.append(i) 277 | 278 | dsa = [] 279 | 280 | print(prefix + "Fetching DSA") 281 | for i, at in enumerate(target_ats): 282 | label = target_pred[i].argmax(axis=-1) 283 | a_dist, a_dot = find_closest_at(at, train_ats[class_matrix[label]]) 284 | b_dist, _ = find_closest_at( 285 | a_dot, train_ats[list(set(all_idx) - set(class_matrix[label]))] 286 | ) 287 | dsa.append(a_dist / b_dist) 288 | 289 | return dsa 290 | 291 | 292 | def _get_kdes(train_ats, train_pred, class_matrix, is_classification, num_classes, var_threshold): 293 | """Kernel density estimation 294 | 295 | Args: 296 | train_ats (list): List of activation traces in training set. 297 | train_pred (list): List of prediction of train set. 298 | class_matrix (list): List of index of classes. 299 | args: Keyboard args. 300 | 301 | Returns: 302 | kdes (list): List of kdes per label if classification task. 303 | removed_cols (list): List of removed columns by variance threshold. 304 | """ 305 | 306 | removed_cols = [] 307 | if is_classification: 308 | for label in range(num_classes): 309 | col_vectors = np.transpose(train_ats[class_matrix[label]]) 310 | for i in range(col_vectors.shape[0]): 311 | if ( 312 | np.var(col_vectors[i]) < var_threshold 313 | and i not in removed_cols 314 | ): 315 | removed_cols.append(i) 316 | 317 | kdes = {} 318 | for label in range(num_classes): 319 | refined_ats = np.transpose(train_ats[class_matrix[label]]) 320 | refined_ats = np.delete(refined_ats, removed_cols, axis=0) 321 | 322 | if refined_ats.shape[0] == 0: 323 | print("ats were removed by threshold {}".format(var_threshold)) 324 | break 325 | kdes[label] = gaussian_kde(refined_ats) 326 | 327 | else: 328 | col_vectors = np.transpose(train_ats) 329 | for i in range(col_vectors.shape[0]): 330 | if np.var(col_vectors[i]) < var_threshold: 331 | removed_cols.append(i) 332 | 333 | refined_ats = np.transpose(train_ats) 334 | refined_ats = np.delete(refined_ats, removed_cols, axis=0) 335 | if refined_ats.shape[0] == 0: 336 | print("ats were removed by threshold {}".format(var_threshold)) 337 | kdes = [gaussian_kde(refined_ats)] 338 | 339 | print("The number of removed columns: {}".format(len(removed_cols))) 340 | 341 | return kdes, removed_cols 342 | 343 | 344 | def _get_lsa(kde, at, removed_cols): 345 | refined_at = np.delete(at, removed_cols, axis=0) 346 | return np.asscalar(-kde.logpdf(np.transpose(refined_at))) 347 | 348 | 349 | def fetch_lsa(model, x_train, x_target, target_name, layer_names, num_classes, 350 | is_classification, var_threshold, save_path, dataset): 351 | """Likelihood-based SA 352 | 353 | Args: 354 | model (keras model): Subject model. 355 | x_train (list): Set of training inputs. 356 | x_target (list): Set of target (test or[] adversarial) inputs. 357 | target_name (str): Name of target set. 358 | layer_names (list): List of selected layer names. 359 | args: Keyboard args. 360 | 361 | Returns: 362 | lsa (list): List of lsa for each target input. 363 | """ 364 | 365 | prefix = "[" + target_name + "] " 366 | train_ats, train_pred, target_ats, target_pred = _get_train_target_ats( 367 | model, x_train, x_target, target_name, layer_names, num_classes, 368 | is_classification, save_path, dataset) 369 | 370 | class_matrix = {} 371 | if is_classification: 372 | for i, label in enumerate(train_pred): 373 | if label.argmax(axis=-1) not in class_matrix: 374 | class_matrix[label.argmax(axis=-1)] = [] 375 | class_matrix[label.argmax(axis=-1)].append(i) 376 | print('yes') 377 | print(class_matrix.keys()) 378 | 379 | kdes, removed_cols = _get_kdes(train_ats, train_pred, class_matrix, 380 | is_classification, num_classes, var_threshold) 381 | 382 | lsa = [] 383 | print(prefix + "Fetching LSA") 384 | if is_classification: 385 | for i, at in enumerate(target_ats): 386 | label = target_pred[i].argmax(axis=-1) 387 | kde = kdes[label] 388 | lsa.append(_get_lsa(kde, at, removed_cols)) 389 | else: 390 | kde = kdes[0] 391 | for at in target_ats: 392 | lsa.append(_get_lsa(kde, at, removed_cols)) 393 | 394 | return lsa 395 | 396 | 397 | def get_sc(lower, upper, k, sa): 398 | """Surprise Coverage 399 | 400 | Args: 401 | lower (int): Lower bound. 402 | upper (int): Upper bound. 403 | k (int): The number of buckets. 404 | sa (list): List of lsa or dsa. 405 | 406 | Returns: 407 | cov (int): Surprise coverage. 408 | """ 409 | 410 | buckets = np.digitize(sa, np.linspace(lower, upper, k)) 411 | return len(list(set(buckets))) / float(k) * 100 412 | -------------------------------------------------------------------------------- /coverages/ss.py: -------------------------------------------------------------------------------- 1 | from utils import get_layer_outs_new, get_layer_outs, percent_str 2 | from collections import defaultdict 3 | 4 | 5 | def default_sign_fn(x, _=None): 6 | """ 7 | Default sign function implementation 8 | :param x: 9 | :param _: Ignored (included for interface consistency) 10 | :return: Sign value (+1 or -1) 11 | """ 12 | return +1 if (x > 0).any() else -1 13 | 14 | 15 | class SSCover: 16 | """ 17 | Implements SS (Sign-Sign) coverage metric from "Testing Deep Neural Networks" by Sun et al. 18 | 19 | Class also supports incremental measurements using which one can observe the effect of new inputs to the coverage 20 | values. 21 | """ 22 | 23 | def __init__(self, model, sign_fn=default_sign_fn, skip_layers=None): 24 | """ 25 | :param model: Model 26 | :param sign_fn: Sign function that returns either +1 or -1 for a tensor 27 | :param skip_layers: Layers to be skipped (e.g. flatten layers) 28 | """ 29 | self.cover_set = set() 30 | self.sign_sets = defaultdict(set) 31 | 32 | self.model = model 33 | self.sign_fn = sign_fn 34 | self.skip_layers = skip_layers = ([] if skip_layers is None else skip_layers) 35 | self.layers = [model.layers[i] for i in range(len(model.layers)) if i not in skip_layers] 36 | 37 | def get_measure_state(self): 38 | return [self.cover_set, self.sign_sets] 39 | 40 | def set_measure_state(self, state): 41 | self.cover_set = state[0] 42 | self.sign_sets = state[1] 43 | 44 | def test(self, test_inputs): 45 | outs = get_layer_outs_new(self.model, test_inputs, self.skip_layers) 46 | 47 | total_pairs = 0 48 | 49 | for layer_index in range(len(outs) - 1): 50 | lower_layer_outs, upper_layer_outs = outs[layer_index], outs[layer_index + 1] 51 | lower_layer_fn, upper_layer_fn = self.layers[layer_index], self.layers[layer_index + 1] 52 | 53 | for lower_neuron_index in range(lower_layer_outs.shape[-1]): 54 | for upper_neuron_index in range(upper_layer_outs.shape[-1]): 55 | total_pairs += 1 56 | 57 | sign_set = self.sign_sets[(lower_neuron_index, upper_neuron_index)] 58 | for input_index in range(len(test_inputs)): 59 | rest_signs = \ 60 | tuple(self.sign_fn(lower_layer_outs[input_index, ..., i], lower_layer_fn) 61 | for i in range(lower_layer_outs.shape[-1]) if i != lower_neuron_index) 62 | 63 | all_signs = ( 64 | self.sign_fn(lower_layer_outs[input_index, ..., lower_neuron_index], lower_layer_fn), 65 | self.sign_fn(upper_layer_outs[input_index, ..., upper_neuron_index], upper_layer_fn), 66 | rest_signs) 67 | 68 | if (-1 * all_signs[0], -1 * all_signs[1], all_signs[2]) in sign_set: 69 | self.cover_set.add( 70 | ((layer_index, lower_neuron_index), (layer_index + 1, upper_neuron_index))) 71 | 72 | sign_set.add(all_signs) 73 | 74 | covered_pair_count = len(self.cover_set) 75 | 76 | return percent_str(covered_pair_count, total_pairs), covered_pair_count, total_pairs, self.cover_set, outs 77 | 78 | 79 | # Reference implementation 80 | def _measure_ss_cover_naive(model, test_inputs, sign_fn=default_sign_fn, skip_layers=None): 81 | if skip_layers is None: 82 | skip_layers = [] 83 | 84 | cover_set = set() 85 | outs = get_layer_outs(model, test_inputs, skip_layers) 86 | total_pairs = 0 87 | 88 | for layer_index in range(len(outs) - 1): 89 | lower_layer_outs, upper_layer_outs = outs[layer_index][0], outs[layer_index + 1][0] 90 | lower_layer_fn, upper_layer_fn = model.layers[layer_index], model.layers[layer_index + 1] 91 | 92 | print(layer_index) 93 | 94 | for lower_neuron_index in range(lower_layer_outs.shape[-1]): 95 | for upper_neuron_index in range(upper_layer_outs.shape[-1]): 96 | total_pairs += 1 97 | 98 | for input_index_i in range(len(test_inputs)): 99 | for input_index_j in range(input_index_i + 1, len(test_inputs)): 100 | 101 | covered = (sign_fn(lower_layer_outs[input_index_i][lower_neuron_index], lower_layer_fn) != 102 | sign_fn(lower_layer_outs[input_index_j][lower_neuron_index], lower_layer_fn)) 103 | 104 | covered = covered and \ 105 | (sign_fn(upper_layer_outs[input_index_i][upper_neuron_index], upper_layer_fn) != 106 | sign_fn(upper_layer_outs[input_index_j][upper_neuron_index], upper_layer_fn)) 107 | 108 | for other_lower_neuron_index in range(lower_layer_outs.shape[-1]): 109 | if other_lower_neuron_index == lower_neuron_index: 110 | continue 111 | 112 | covered = covered and \ 113 | (sign_fn(lower_layer_outs[input_index_i][other_lower_neuron_index], 114 | lower_layer_fn) == 115 | sign_fn(lower_layer_outs[input_index_j][other_lower_neuron_index], 116 | lower_layer_fn)) 117 | 118 | if covered: 119 | cover_set.add(((layer_index, lower_neuron_index), (layer_index + 1, upper_neuron_index))) 120 | 121 | covered_pair_count = len(cover_set) 122 | 123 | return covered_pair_count, total_pairs, cover_set 124 | -------------------------------------------------------------------------------- /coverages/tkn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | from utils import get_layer_outs, get_layer_outs_new, percent_str 4 | 5 | 6 | class DeepGaugeLayerLevelCoverage: 7 | """ 8 | Implements TKN and TKN-with-pattern coverage metrics from "DeepGauge: Multi-Granularity Testing Criteria for Deep 9 | Learning Systems" by Ma et al. 10 | 11 | Supports incremental measurements using which one can observe the effect of new inputs to the coverage 12 | values. 13 | """ 14 | 15 | def __init__(self, model, k, skip_layers=None): 16 | """ 17 | :param model: Model 18 | :param k: k parameter (see the paper) 19 | :param skip_layers: Layers to be skipped (e.g. flatten layers) 20 | """ 21 | self.activation_table = {} 22 | self.pattern_set = set() 23 | 24 | self.model = model 25 | self.k = k 26 | self.skip_layers = skip_layers = ([] if skip_layers is None else skip_layers) 27 | 28 | def get_measure_state(self): 29 | return [self.activation_table, self.pattern_set] 30 | 31 | def set_measure_state(self, state): 32 | self.activation_table = state[0] 33 | self.pattern_set = state[1] 34 | 35 | def test(self, test_inputs): 36 | """ 37 | :param test_inputs: Inputs 38 | :return: Tuple consisting of coverage results along with the measurements that are used to compute the 39 | coverages. 0th element is the TKN value and 3th element is the pattern count for TKN-with-pattern. 40 | """ 41 | outs = get_layer_outs_new(self.model, test_inputs, self.skip_layers) 42 | 43 | neuron_count_by_layer = {} 44 | 45 | layer_count = len(outs) 46 | 47 | inc_cnt_tkn = 0 48 | for input_index in range(len(test_inputs)): # out_for_input is output of layer for single input 49 | pattern = [] 50 | 51 | inc_flag = False 52 | for layer_index in range(layer_count): # layer_out is output of layer for all inputs 53 | out_for_input = outs[layer_index][input_index] 54 | 55 | neuron_outs = np.zeros((out_for_input.shape[-1],)) 56 | neuron_count_by_layer[layer_index] = len(neuron_outs) 57 | for i in range(out_for_input.shape[-1]): 58 | neuron_outs[i] = np.mean(out_for_input[..., i]) 59 | 60 | top_k_neuron_indexes = (np.argsort(neuron_outs, axis=None)[-self.k:len(neuron_outs)]) 61 | pattern.append(tuple(top_k_neuron_indexes)) 62 | 63 | for neuron_index in top_k_neuron_indexes: 64 | if not (layer_index, neuron_index) in self.activation_table: inc_flag = True 65 | self.activation_table[(layer_index, neuron_index)] = True 66 | 67 | if layer_index + 1 == layer_count: 68 | self.pattern_set.add(tuple(pattern)) 69 | 70 | if inc_flag: 71 | inc_cnt_tkn += 1 72 | 73 | neuron_count = sum(neuron_count_by_layer.values()) 74 | covered = len(self.activation_table.keys()) 75 | 76 | print(percent_str(covered, neuron_count)) 77 | # TKNC #TKNP 78 | return percent_str(covered, neuron_count), covered, neuron_count, len(self.pattern_set), outs, inc_cnt_tkn 79 | -------------------------------------------------------------------------------- /lrp_toolbox/.gitignore: -------------------------------------------------------------------------------- 1 | *.project 2 | *.pydevproject 3 | *.pyc 4 | *.bbl 5 | *.asv 6 | doc/manual/manual.aux 7 | doc/manual/manual.blg 8 | doc/manual/manual.synctex.gz 9 | doc/manual/manual.log 10 | doc/manual/manual.out 11 | doc/manual/manual.toc 12 | heatmap.npy 13 | heatmap.png 14 | xor_net_small_1000.txt 15 | xor_net_small_1000_newformat.txt 16 | xor_net_small_1000m.txt 17 | xor_net_small_1000m_newformat.txt 18 | hm_m.png 19 | python/.vscode/launch.json 20 | -------------------------------------------------------------------------------- /lrp_toolbox/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/lrp_toolbox/__init__.py -------------------------------------------------------------------------------- /lrp_toolbox/cnn_training_test.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 30.11.2016 6 | @version: 1.2+ 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | ''' 10 | 11 | import modules 12 | import model_io 13 | import data_io 14 | 15 | import numpy as np ; na = np.newaxis 16 | 17 | #load the mnist data 18 | Xtrain = data_io.read('../data/MNIST/train_images.npy') 19 | Ytrain = data_io.read('../data/MNIST/train_labels.npy') 20 | 21 | Xtest = data_io.read('../data/MNIST/test_images.npy') 22 | Ytest = data_io.read('../data/MNIST/test_labels.npy') 23 | 24 | #transfer the pixel values from [0 255] to [-1 1] 25 | Xtrain = Xtrain / 127.5 -1 26 | Xtest = Xtest / 127.5 -1 27 | 28 | #reshape the vector representations of the mnist data back to image format. extend the image vertically and horizontally by 4 pixels each. 29 | Xtrain = np.reshape(Xtrain,[Xtrain.shape[0],28,28,1]) 30 | Xtrain = np.pad(Xtrain,((0,0),(2,2),(2,2),(0,0)), 'constant', constant_values = (-1.,)) 31 | 32 | Xtest = np.reshape(Xtest,[Xtest.shape[0],28,28,1]) 33 | Xtest = np.pad(Xtest,((0,0),(2,2),(2,2),(0,0)), 'constant', constant_values = (-1.,)) 34 | 35 | #transform numeric class labels to indicator vectors. 36 | I = Ytrain[:,0].astype(int) 37 | Ytrain = np.zeros([Xtrain.shape[0],np.unique(Ytrain).size]) 38 | Ytrain[np.arange(Ytrain.shape[0]),I] = 1 39 | 40 | I = Ytest[:,0].astype(int) 41 | Ytest = np.zeros([Xtest.shape[0],np.unique(Ytest).size]) 42 | Ytest[np.arange(Ytest.shape[0]),I] = 1 43 | 44 | 45 | #model a network according to LeNet-5 architecture 46 | lenet = modules.Sequential([ 47 | modules.Convolution(filtersize=(5,5,1,10),stride = (1,1)),\ 48 | modules.Rect(),\ 49 | modules.SumPool(pool=(2,2),stride=(2,2)),\ 50 | modules.Convolution(filtersize=(5,5,10,25),stride = (1,1)),\ 51 | modules.Rect(),\ 52 | modules.SumPool(pool=(2,2),stride=(2,2)),\ 53 | modules.Convolution(filtersize=(4,4,25,100),stride = (1,1)),\ 54 | modules.Rect(),\ 55 | modules.SumPool(pool=(2,2),stride=(2,2)),\ 56 | modules.Convolution(filtersize=(1,1,100,10),stride = (1,1)),\ 57 | modules.Flatten() 58 | ]) 59 | 60 | #train the network. 61 | lenet.train( X=Xtrain,\ 62 | Y=Ytrain,\ 63 | Xval=Xtest,\ 64 | Yval=Ytest,\ 65 | iters=10**6,\ 66 | lrate=0.001,\ 67 | batchsize=25) 68 | 69 | #save the network 70 | model_io.write(lenet, '../LeNet-5.txt') 71 | 72 | 73 | 74 | 75 | #a slight variation to test max pooling layers. this model should train faster. 76 | maxnet = modules.Sequential([ 77 | modules.Convolution(filtersize=(5,5,1,10),stride = (1,1)),\ 78 | modules.Rect(),\ 79 | modules.MaxPool(pool=(2,2),stride=(2,2)),\ 80 | modules.Convolution(filtersize=(5,5,10,25),stride = (1,1)),\ 81 | modules.Rect(),\ 82 | modules.MaxPool(pool=(2,2),stride=(2,2)),\ 83 | modules.Convolution(filtersize=(4,4,25,100),stride = (1,1)),\ 84 | modules.Rect(),\ 85 | modules.MaxPool(pool=(2,2),stride=(2,2)),\ 86 | modules.Convolution(filtersize=(1,1,100,10),stride = (1,1)),\ 87 | modules.Flatten(),\ 88 | modules.SoftMax() 89 | ]) 90 | 91 | #train the network. 92 | maxnet.train( X=Xtrain,\ 93 | Y=Ytrain,\ 94 | Xval=Xtest,\ 95 | Yval=Ytest,\ 96 | iters=10**6,\ 97 | lrate=0.001,\ 98 | batchsize=25) 99 | 100 | #save the network 101 | model_io.write(maxnet, '../LeNet-5-maxpooling.txt') -------------------------------------------------------------------------------- /lrp_toolbox/data_io.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 14.08.2015 6 | @version: 1.0 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | ''' 10 | 11 | import os 12 | import numpy as np ; na = np.newaxis 13 | import scipy.io as scio 14 | 15 | 16 | 17 | #-------------------- 18 | # data reading 19 | #-------------------- 20 | 21 | def read(path, fmt = None): 22 | ''' 23 | Read [N x D]-sized block-formatted data from a given path. 24 | Supported data formats are 25 | plain text (ascii-matrices) 26 | numpy-compressed data (npy- or npz-files) 27 | matlab data files (mat-files) 28 | 29 | Parameters 30 | ---------- 31 | 32 | path : str 33 | the path to the file to read 34 | 35 | fmt : str 36 | optional. if explicitly given, the file will be interpreted as mat, txt, npy or npz. elsewise, interpretation format will be inferred from the file name 37 | 38 | 39 | Returns 40 | ------- 41 | 42 | data : numpy.ndarray 43 | 44 | ''' 45 | if not os.path.exists(path): 46 | raise IOError('data_io.read : No such file or directory: {0}'.format(path)) 47 | 48 | if fmt is None: #try to infer format 49 | fmt = os.path.splitext(path)[1].replace('.','').lower() 50 | 51 | data = _read_as[fmt](path) 52 | 53 | return data 54 | 55 | 56 | def _read_np(path): 57 | print('loading np-formatted data from',path) 58 | return np.load(path) 59 | 60 | 61 | def _read_mat(path): 62 | print('loading matlab formatted data from', path) 63 | return scio.loadmat(path)['data'] 64 | 65 | 66 | def _read_txt(path): 67 | print('loading plain text data from',path) 68 | return np.loadtxt(path) 69 | 70 | _read_as = {'npy':_read_np,\ 71 | 'npz':_read_np,\ 72 | '' :_read_np,\ 73 | 'mat':_read_mat,\ 74 | 'txt':_read_txt,\ 75 | } 76 | 77 | 78 | #-------------------- 79 | # data writing 80 | #-------------------- 81 | 82 | 83 | def write(data, path, fmt = None): 84 | ''' 85 | Write [N x D]-sized block-formatted data to a given path. 86 | Supported data formats are 87 | plain text (ascii-matrices) 88 | numpy-compressed data (npy- or npz-files) 89 | matlab data files (mat-files) 90 | 91 | Parameters 92 | ---------- 93 | 94 | data : numpy.ndarray 95 | a [N x D] - shaped, two-dimensional array of data. 96 | 97 | path : str 98 | the path to write the data to 99 | 100 | fmt : str 101 | optional. if explicitly given, the file will be written as mat, txt, npy or npz. elsewise, interpretation format will be inferred from the file name 102 | 103 | ''' 104 | 105 | if fmt is None: #try to infer format 106 | fmt = os.path.splitext(path)[1].replace('.','').lower() 107 | 108 | _write_as[fmt](data,path) 109 | 110 | 111 | def _write_np(data, path): 112 | print('writing data in npy-format to',path) 113 | np.save(path, data) 114 | 115 | def _write_mat(data, path): 116 | print('writing data in mat-format to',path) 117 | scio.savemat(path, {'data':data}, appendmat = False) 118 | 119 | def _write_txt(data, path): 120 | print('writing data as plain text to',path) 121 | np.savetxt(path, data) 122 | 123 | 124 | _write_as = {'npy':_write_np,\ 125 | 'npz':_write_np,\ 126 | '' :_write_np,\ 127 | 'mat':_write_mat,\ 128 | 'txt':_write_txt,\ 129 | } 130 | -------------------------------------------------------------------------------- /lrp_toolbox/lrp_cnn_demo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 25.10.2016 6 | @version: 1.2+ 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | 10 | The purpose of this module is to demonstrate the process of obtaining pixel-wise explanations for given data points at hand of the MNIST hand written digit data set 11 | with CNN models, using the LeNet-5 architecture. 12 | 13 | The module first loads a pre-trained neural network model and the MNIST test set with labels and transforms the data such that each pixel value is within the range of [-1 1]. 14 | The data is then randomly permuted and for the first 10 samples due to the permuted order, a prediction is computed by the network, which is then as a next step explained 15 | by attributing relevance values to each of the input pixels. 16 | 17 | finally, the resulting heatmap is rendered as an image and (over)written out to disk and displayed. 18 | ''' 19 | 20 | 21 | import matplotlib.pyplot as plt 22 | import numpy as np ; na = np.newaxis 23 | 24 | import model_io 25 | import data_io 26 | import render 27 | 28 | #load a neural network, as well as the MNIST test data and some labels 29 | nn = model_io.read('../models/MNIST/LeNet-5.nn') # 99.23% prediction accuracy 30 | nn.drop_softmax_output_layer() #drop softnax output layer for analyses 31 | 32 | X = data_io.read('../data/MNIST/test_images.npy') 33 | Y = data_io.read('../data/MNIST/test_labels.npy') 34 | 35 | 36 | # transfer pixel values from [0 255] to [-1 1] to satisfy the expected input / training paradigm of the model 37 | X = X / 127.5 - 1. 38 | 39 | #reshape the vector representations in X to match the requirements of the CNN input 40 | X = np.reshape(X,[X.shape[0],28,28,1]) 41 | X = np.pad(X,((0,0),(2,2),(2,2),(0,0)), 'constant', constant_values = (-1.,)) 42 | 43 | # transform numeric class labels to vector indicator for uniformity. assume presence of all classes within the label set 44 | I = Y[:,0].astype(int) 45 | Y = np.zeros([X.shape[0],np.unique(Y).size]) 46 | Y[np.arange(Y.shape[0]),I] = 1 47 | 48 | acc = np.mean(np.argmax(nn.forward(X), axis=1) == np.argmax(Y, axis=1)) 49 | print('model test accuracy is: {:0.4f}'.format(acc)) 50 | 51 | #permute data order for demonstration. or not. your choice. 52 | I = np.arange(X.shape[0]) 53 | #I = np.random.permutation(I) 54 | 55 | #predict and perform LRP for the 10 first samples 56 | for i in I[:10]: 57 | x = X[i:i+1,...] 58 | 59 | #forward pass and prediction 60 | ypred = nn.forward(x) 61 | print('True Class: ', np.argmax(Y[i])) 62 | print('Predicted Class:', np.argmax(ypred),'\n') 63 | 64 | #prepare initial relevance to reflect the model's dominant prediction (ie depopulate non-dominant output neurons) 65 | mask = np.zeros_like(ypred) 66 | mask[:,np.argmax(ypred)] = 1 67 | Rinit = ypred*mask 68 | 69 | 70 | #compute first layer relevance according to prediction 71 | #R = nn.lrp(Rinit) #as Eq(56) from DOI: 10.1371/journal.pone.0130140 72 | R = nn.lrp(Rinit,'epsilon',1.) #as Eq(58) from DOI: 10.1371/journal.pone.0130140 73 | #R = nn.lrp(Rinit,'alphabeta',2) #as Eq(60) from DOI: 10.1371/journal.pone.0130140 74 | 75 | #R = nn.lrp(ypred*Y[na,i],'epsilon',1.) #compute first layer relevance according to the true class label 76 | 77 | 78 | ''' 79 | #compute first layer relvance for an arbitrarily selected class 80 | for yselect in range(10): 81 | yselect = (np.arange(Y.shape[1])[na,:] == yselect)*1. 82 | R = nn.lrp(ypred*yselect,'epsilon',0.1) 83 | ''' 84 | 85 | ''' 86 | # you may also specify different decompositions for each layer, e.g. as below: 87 | # first, set all layers (by calling set_lrp_parameters on the container module 88 | # of class Sequential) to perform alpha-beta decomposition with alpha = 1. 89 | # this causes the resulting relevance map to display excitation potential for the prediction 90 | # 91 | nn.set_lrp_parameters('alpha',1.) 92 | # 93 | # set the first layer (a convolutional layer) decomposition variant to 'w^2'. This may be especially 94 | # usefill if input values are ranged [0 V], with 0 being a frequent occurrence, but one still wishes to know about 95 | # the relevance feedback propagated to the pixels below the filter 96 | # the result with display relevance in important areas despite zero input activation energy. 97 | # 98 | nn.modules[0].set_lrp_parameters('ww') # also try 'flat' 99 | # compute the relevance map 100 | R = nn.lrp(Rinit) 101 | ''' 102 | 103 | 104 | 105 | #sum over the third (color channel) axis. not necessary here, but for color images it would be. 106 | R = R.sum(axis=3) 107 | #same for input. create brightness image in [0,1]. 108 | xs = ((x+1.)/2.).sum(axis=3) 109 | 110 | #render input and heatmap as rgb images 111 | digit = render.digit_to_rgb(xs, scaling = 3) 112 | hm = render.hm_to_rgb(R, X = xs, scaling = 3, sigma = 2) 113 | digit_hm = render.save_image([digit,hm],'../heatmap.png') 114 | data_io.write(R,'../heatmap.npy') 115 | 116 | #display the image as written to file 117 | plt.imshow(digit_hm, interpolation = 'none') 118 | plt.axis('off') 119 | plt.show() 120 | 121 | 122 | #note that modules.Sequential allows for batch processing inputs 123 | ''' 124 | x = X[:10,...] 125 | y = nn.forward(x) 126 | R = nn.lrp(y) 127 | data_io.write(R,'../Rbatch.npy') 128 | ''' 129 | -------------------------------------------------------------------------------- /lrp_toolbox/lrp_demo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 14.08.2015 6 | @version: 1.0 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | 10 | The purpose of this module is to demonstrate the process of obtaining pixel-wise explanations for given data points at hand of the MNIST hand written digit data set. 11 | 12 | The module first loads a pre-trained neural network model and the MNIST test set with labels and transforms the data such that each pixel value is within the range of [-1 1]. 13 | The data is then randomly permuted and for the first 10 samples due to the permuted order, a prediction is computed by the network, which is then as a next step explained 14 | by attributing relevance values to each of the input pixels. 15 | 16 | finally, the resulting heatmap is rendered as an image and (over)written out to disk and displayed. 17 | ''' 18 | 19 | 20 | import matplotlib.pyplot as plt 21 | import numpy as np ; na = np.newaxis 22 | import model_io 23 | import data_io 24 | import render 25 | import sys, os 26 | #sys.path.append('./../models/') 27 | sys.path.append('./../') 28 | from keras.layers import Input 29 | from utils import load_MNIST 30 | 31 | 32 | #load a neural network, as well as the MNIST test data and some labels 33 | 34 | nn = model_io.read('../neural_networks/LeNet5.txt', 'txt') # 99.16% prediction accuracy 35 | #nn = model_io.read('./models/MNIST/LeNet5.txt', 'txt') # 99.16% prediction accuracy 36 | nn.drop_softmax_output_layer() #drop softnax output layer for analyses 37 | 38 | X_train, Y_train, X_test, Y_test = load_MNIST(channel_first=False) 39 | 40 | x = X_test[50] 41 | 42 | #forward pass and predictio 43 | ypred = nn.forward(np.expand_dims(x, axis=0)) 44 | print(Y_test[50]) 45 | print(ypred) 46 | print('Predicted Class:', np.argmax(ypred),'\n') 47 | 48 | #prepare initial relevance to reflect the model's dominant prediction (ie depopulate non-dominant output neurons) 49 | mask = np.zeros_like(ypred) 50 | mask[:,np.argmax(ypred)] = 1 51 | Rinit = ypred*mask 52 | print(Rinit) 53 | 54 | #compute first layer relevance according to prediction 55 | #R, _ = nn.lrp(Rinit, 'simple') 56 | #R, _ = nn.lrp(Rinit) #as Eq(56) from DOI: 10.1371/journal.pone.0130140 57 | #R, _ = nn.lrp(Rinit,'epsilon',0.01) #as Eq(58) from DOI: 10.1371/journal.pone.0130140 58 | R, _ = nn.lrp(Rinit,'alphabeta',1) #as Eq(60) from DOI: 10.1371/journal.pone.0130140 59 | 60 | #R = R[:, 2:-2, 2:-2, :] 61 | 62 | x = (x+1.)/2. 63 | 64 | #render input and heatmap as rgb images 65 | digit = render.digit_to_rgb(x, scaling = 3) 66 | hm = render.hm_to_rgb(R, X = x, scaling = 3, sigma = 2) # 67 | digit_hm = render.save_image([digit,hm],'../heatmap.png') 68 | data_io.write(R,'../heatmap.npy') 69 | 70 | #display the image as written to file 71 | #plt.imshow(digit_hm, interpolation = 'none') 72 | #plt.axis('off') 73 | #plt.show() 74 | 75 | -------------------------------------------------------------------------------- /lrp_toolbox/mini.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 21.09.2015 6 | @version: 1.0 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | ''' 10 | 11 | 12 | # imports 13 | import model_io 14 | import data_io 15 | import render 16 | 17 | import numpy as np 18 | na = np.newaxis 19 | # end of imports 20 | 21 | nn = model_io.read('../models/MNIST/long-rect.nn') # read model 22 | X = data_io.read('../data/MNIST/test_images.npy')[na,0,:] # load first MNIST test image 23 | X = X / 127.5 - 1 # normalized data to range [-1 1] 24 | 25 | Ypred = nn.forward(X) # forward pass through network 26 | R = nn.lrp(Ypred) # lrp to explain prediction of X 27 | 28 | # render rgb images and save as image 29 | digit = render.digit_to_rgb(X) 30 | hm = render.hm_to_rgb(R, X) # render heatmap R, use X as outline 31 | render.save_image([digit, hm], '../hm_py.png') 32 | -------------------------------------------------------------------------------- /lrp_toolbox/model_io.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 14.08.2015 6 | @version: 1.2+ 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | ''' 10 | 11 | import os 12 | import pickle 13 | import numpy as np 14 | from .modules import Sequential,Linear,Tanh,Rect,SoftMax,Convolution,Flatten,SumPool,MaxPool 15 | #Team DeepFault 16 | from keras.models import model_from_json, load_model 17 | from keras.layers import Input 18 | import json 19 | 20 | #-------------------- 21 | # model reading 22 | #-------------------- 23 | 24 | def read(path, fmt = None): 25 | ''' 26 | Read neural network model from given path. Supported are files written in either plain text or via python's pickle module. 27 | 28 | Parameters 29 | ---------- 30 | 31 | path : str 32 | the path to the file to read 33 | 34 | fmt : str 35 | optional. explicitly state how to interpret the target file. if not given, format is inferred from path. 36 | options are 'pickled','pickle','' and 'nn' to specify the pickle file format and 'txt' for a plain text 37 | format shared with the matlab implementation of the toolbox 38 | 39 | Returns 40 | ------- 41 | model: modules.Sequential 42 | the neural network model, realized as a sequence of network modules. 43 | 44 | Notes 45 | ----- 46 | the plain text file format is shared with the matlab implementation of the LRP Toolbox and describes 47 | the model by listing its computational layers line by line as 48 | 49 | [ ] 50 | [] 51 | 52 | since all implemented modules except for modules.Linear operate point-wise on the given data, the optional 53 | information indicated by brackets [ ] is not used and only the name of the layer is witten, e.g. 54 | 55 | Rect 56 | 57 | Tanh 58 | 59 | SoftMax 60 | 61 | Flatten 62 | 63 | The exception formed by the linear layer implementation modules.Linear and modules.Convolution incorporates in raw text form as 64 | 65 | Linear m n 66 | W.flatten() 67 | B.flatten() 68 | 69 | with m and n being integer values describing the dimensions of the weight matrix W as [m x n] , 70 | W being the human readable ascii-representation of the flattened matrix in m * n white space separated double values. 71 | After the line describing W, the bias term B is written out as a single line of n white space separated double values. 72 | 73 | Convolution h w d n s0 s1 74 | W.flatten() 75 | B.flatten() 76 | 77 | Semantics as above, with h, w, d being the filter heigth, width and depth and n being the number of filters of that layer. 78 | s0 and s1 specify the stride parameter in vertical (axis 0) and horizontal (axis 1) direction the layer operates on. 79 | 80 | Pooling layers have a parameterized one-line-description 81 | 82 | [Max|Sum]Pool h w s0 s1 83 | 84 | with h and w designating the pooling mask size and s0 and s1 the pooling stride. 85 | ''' 86 | 87 | if not os.path.exists(path): 88 | raise IOError('model_io.read : No such file or directory: {0}'.format(path)) 89 | 90 | if fmt is None: #try to infer format 91 | fmt = os.path.splitext(path)[1].replace('.','').lower() 92 | 93 | return _read_as[fmt](path) 94 | 95 | 96 | def _read_pickled(path): 97 | print('loading pickled model from',path) 98 | with open(path,'r') as f: 99 | p = pickle.load(f, encoding='latin1') 100 | return p 101 | 102 | 103 | def _read_txt(path): 104 | print('loading plain text model from',path) 105 | 106 | def _read_txt_helper(path): 107 | with open(path,'r') as f: 108 | content = f.read().split('\n') 109 | 110 | modules = [] 111 | c = 0 112 | line = content[c] 113 | 114 | while len(line) > 0: 115 | if line.startswith(Linear.__name__): # @UndefinedVariable import error suppression for PyDev users 116 | ''' 117 | Format of linear layer 118 | Linear 119 | 120 | 121 | ''' 122 | _,m,n = line.split(); m = int(m); n = int(n) 123 | layer = Linear(m,n) 124 | layer.W = np.array([float(weightstring) for weightstring in content[c+1].split() if len(weightstring) > 0]).reshape((m,n)) 125 | layer.B = np.array([float(weightstring) for weightstring in content[c+2].split() if len(weightstring) > 0]) 126 | modules.append(layer) 127 | c+=3 # the description of a linear layer spans three lines 128 | 129 | elif line.startswith(Convolution.__name__): # @UndefinedVariable import error suppression for PyDev users 130 | ''' 131 | Format of convolution layer 132 | Convolution 133 | 134 | 135 | ''' 136 | 137 | _,h,w,d,n,s0,s1 = line.split() 138 | h = int(h); w = int(w); d = int(d); n = int(n); s0 = int(s0); s1 = int(s1) 139 | layer = Convolution(filtersize=(h,w,d,n), stride=(s0,s1)) 140 | layer.W = np.array([float(weightstring) for weightstring in content[c+1].split() if len(weightstring) > 0]).reshape((h,w,d,n)) 141 | layer.B = np.array([float(weightstring) for weightstring in content[c+2].split() if len(weightstring) > 0]) 142 | modules.append(layer) 143 | c+=3 #the description of a convolution layer spans three lines 144 | 145 | elif line.startswith(SumPool.__name__): # @UndefinedVariable import error suppression for PyDev users 146 | ''' 147 | Format of sum pooling layer 148 | SumPool 149 | ''' 150 | 151 | _,h,w,s0,s1 = line.split() 152 | h = int(h); w = int(w); s0 = int(s0); s1 = int(s1) 153 | layer = SumPool(pool=(h,w),stride=(s0,s1)) 154 | modules.append(layer) 155 | c+=1 # one line of parameterized layer description 156 | 157 | elif line.startswith(MaxPool.__name__): # @UndefinedVariable import error suppression for PyDev users 158 | ''' 159 | Format of max pooling layer 160 | MaxPool 161 | ''' 162 | 163 | _,h,w,s0,s1 = line.split() 164 | h = int(h); w = int(w); s0 = int(s0); s1 = int(s1) 165 | layer = MaxPool(pool=(h,w),stride=(s0,s1)) 166 | modules.append(layer) 167 | c+=1 # one line of parameterized layer description 168 | 169 | elif line.startswith(Flatten.__name__): # @UndefinedVariable import error suppression for PyDev users 170 | modules.append(Flatten()) ; c+=1 #one line of parameterless layer description 171 | print('flatten OK') 172 | elif line.startswith(Rect.__name__): # @UndefinedVariable import error suppression for PyDev users 173 | modules.append(Rect()) ; c+= 1 #one line of parameterless layer description 174 | elif line.startswith(Tanh.__name__): # @UndefinedVariable import error suppression for PyDev users 175 | modules.append(Tanh()) ; c+= 1 #one line of parameterless layer description 176 | elif line.startswith(SoftMax.__name__): # @UndefinedVariable import error suppression for PyDev users 177 | modules.append(SoftMax()) ; c+= 1 #one line of parameterless layer description 178 | else: 179 | raise ValueError('Layer type identifier' + [s for s in line.split() if len(s) > 0][0] + ' not supported for reading from plain text file') 180 | 181 | #skip info of previous layers, read in next layer header 182 | line = content[c] 183 | 184 | 185 | 186 | return Sequential(modules) 187 | # END _read_txt_helper() 188 | 189 | try: 190 | return _read_txt_helper(path) 191 | 192 | except ValueError as e: 193 | #numpy.reshape may throw ValueErros if reshaping does not work out. 194 | #In this case: fall back to reading the old plain text format. 195 | print('probable reshaping/formatting error while reading plain text network file.') 196 | print('ValueError message:', e.message) 197 | print('Attempting fall-back to legacy plain text format interpretation...') 198 | return _read_txt_old(path) 199 | print('fall-back successfull!') 200 | 201 | 202 | def _read_txt_old(path): 203 | print('loading plain text model from', path) 204 | 205 | with open(path, 'r') as f: 206 | content = f.read().split('\n') 207 | 208 | modules = [] 209 | c = 0 210 | line = content[c] 211 | while len(line) > 0: 212 | if line.startswith(Linear.__name__): # @UndefinedVariable import error suppression for PyDev users 213 | lineparts = line.split() 214 | m = int(lineparts[1]) 215 | n = int(lineparts[2]) 216 | mod = Linear(m,n) 217 | for i in range(m): 218 | c+=1 219 | mod.W[i,:] = np.array([float(val) for val in content[c].split() if len(val) > 0]) 220 | 221 | c+=1 222 | mod.B = np.array([float(val) for val in content[c].split()]) 223 | modules.append(mod) 224 | 225 | elif line.startswith(Rect.__name__): # @UndefinedVariable import error suppression for PyDev users 226 | modules.append(Rect()) 227 | elif line.startswith(Tanh.__name__): # @UndefinedVariable import error suppression for PyDev users 228 | modules.append(Tanh()) 229 | elif line.startswith(SoftMax.__name__): # @UndefinedVariable import error suppression for PyDev users 230 | modules.append(SoftMax()) 231 | else: 232 | raise ValueError('Layer type ' + [s for s in line.split() if len(s) > 0][0] + ' not supported by legacy plain text format.') 233 | 234 | c+=1; 235 | line = content[c] 236 | 237 | return Sequential(modules) 238 | 239 | 240 | _read_as = {'pickled': _read_pickled,\ 241 | 'pickle':_read_pickled,\ 242 | 'nn':_read_pickled,\ 243 | '':_read_pickled,\ 244 | 'txt':_read_txt,\ 245 | } 246 | 247 | 248 | 249 | 250 | #-------------------- 251 | # model writing 252 | #-------------------- 253 | 254 | def write(model, path, num_channels = None, fmt = None): 255 | ''' 256 | Write neural a network model to a given path. Supported are either plain text or via python's pickle module. 257 | The model is cleaned of any temporary variables , e.g. hidden layer inputs or outputs, prior to writing 258 | 259 | Parameters 260 | ---------- 261 | 262 | model : modules.Sequential 263 | the object representing the model. 264 | 265 | path : str 266 | the path to the file to read 267 | 268 | fmt : str 269 | optional. explicitly state how to write the file. if not given, format is inferred from path. 270 | options are 'pickled','pickle','' and 'nn' to specify the pickle file format and 'txt' for a plain text 271 | format shared with the matlab implementation of the toolbox 272 | 273 | Notes 274 | ----- 275 | see the Notes - Section in the function documentation of model_io.read() for general info and a format 276 | specification of the plain text representation of neural network models 277 | ''' 278 | if not fmt == 'keras_txt': 279 | model.clean() 280 | 281 | if fmt is None: 282 | fmt = os.path.splitext(path)[1].replace('.','').lower() 283 | 284 | _write_as[fmt](model, path, num_channels) 285 | 286 | 287 | def _write_pickled(model, path, num_channels=None): 288 | print('writing model pickled to',path) 289 | with open(path, 'wb') as f: 290 | pickle.dump(model,f,pickle.HIGHEST_PROTOCOL) 291 | 292 | 293 | def _write_txt(model,path, num_channels=None): 294 | print('writing model as plain text to',path) 295 | 296 | if not isinstance(model, Sequential): 297 | raise Exception('Argument "model" must be an instance of module.Sequential, wrapping a sequence of neural network computation layers, but is {0}'.format(type(model))) 298 | 299 | with open(path, 'wb') as f: 300 | for layer in model.modules: 301 | if isinstance(layer,Linear): 302 | ''' 303 | Format of linear layer 304 | Linear 305 | 306 | 307 | ''' 308 | 309 | f.write('{0} {1} {2}\n'.format(layer.__class__.__name__,layer.m,layer.n)) 310 | f.write(' '.join([repr(w) for w in layer.W.flatten()]) + '\n') 311 | f.write(' '.join([repr(b) for b in layer.B.flatten()]) + '\n') 312 | 313 | elif isinstance(layer,Convolution): 314 | ''' 315 | Format of convolution layer 316 | Convolution 317 | 318 | 319 | ''' 320 | 321 | f.write('{0} {1} {2} {3} {4} {5} {6}\n'.format( 322 | layer.__class__.__name__,\ 323 | layer.fh,\ 324 | layer.fw,\ 325 | layer.fd,\ 326 | layer.n,\ 327 | layer.stride[0],\ 328 | layer.stride[1] 329 | )) 330 | f.write(' '.join([repr(w) for w in layer.W.flatten()]) + '\n') 331 | f.write(' '.join([repr(b) for b in layer.B.flatten()]) + '\n') 332 | 333 | elif isinstance(layer,SumPool): 334 | ''' 335 | Format of sum pooling layer 336 | SumPool 337 | ''' 338 | 339 | f.write('{0} {1} {2} {3} {4}\n'.format( 340 | layer.__class__.__name__,\ 341 | layer.pool[0],\ 342 | layer.pool[1],\ 343 | layer.stride[0],\ 344 | layer.stride[1])) 345 | 346 | elif isinstance(layer,MaxPool): 347 | ''' 348 | Format of max pooling layer 349 | MaxPool 350 | ''' 351 | 352 | f.write('{0} {1} {2} {3} {4}\n'.format( 353 | layer.__class__.__name__,\ 354 | layer.pool[0],\ 355 | layer.pool[1],\ 356 | layer.stride[0],\ 357 | layer.stride[1])) 358 | 359 | else: 360 | ''' 361 | all other layers are free from parameters. Format is thus: 362 | 363 | ''' 364 | f.write(layer.__class__.__name__ + '\n') 365 | 366 | 367 | def _write_keras_txt(source_path, destination_path, num_channels): 368 | 369 | ''' 370 | Source path should include architecture (e.g. .json) and weights (e.g. .h5) 371 | files. 372 | ''' 373 | 374 | print('writing keras model as plain text to', destination_path) 375 | 376 | model_name = source_path.split('/')[-1] 377 | 378 | from utils import load_dave_model 379 | if 'dave' in source_path.lower(): 380 | model = load_dave_model() 381 | model_json = model.to_json() 382 | json_obj = json.loads(model_json) 383 | else: 384 | try: 385 | json_file = open(source_path + '.json', 'r') 386 | file_content = json_file.read() 387 | json_file.close() 388 | 389 | model = model_from_json(file_content) 390 | model.load_weights(source_path + '.h5') 391 | 392 | json_obj = json.loads(file_content) 393 | 394 | except: 395 | model = load_model(source_path + '.h5') 396 | model_json = model.to_json() 397 | json_obj = json.loads(model_json) 398 | 399 | 400 | txtfile = open(source_path + '.txt', 'w') 401 | 402 | prev_layer = None 403 | #TODO: SHOULD BE AUTOMATICALLY DETERMINED 404 | prev_filter_num = str(num_channels) #WARNING: THIS SHOULD BE 3 FOR THE CIFAR-10 DATASET 405 | for layer, json_elem in zip(model.layers, json_obj['config']['layers']): 406 | if json_elem['class_name'] == 'Activation': 407 | if json_elem['config']['activation'] == 'softmax': 408 | continue 409 | #txtfile.write('Softmax') 410 | elif json_elem['config']['activation'] == 'relu': 411 | txtfile.write('Rect') 412 | elif json_elem['class_name'] == 'Dense': 413 | txtfile.write('Linear ' + str(prev_layer.output_shape[-1]) + ' ' \ 414 | + str(layer.output_shape[-1])) 415 | elif json_elem['class_name'] == 'Conv2D': 416 | txtfile.write('Convolution ' + str(json_elem['config']['kernel_size'][0])\ 417 | + ' ' + str(json_elem['config']['kernel_size'][1]) + ' ' \ 418 | + prev_filter_num + ' ' + str(json_elem['config']['filters']) + ' '\ 419 | + str(json_elem['config']['strides'][0]) + ' ' \ 420 | + str(json_elem['config']['strides'][1])) 421 | elif json_elem['class_name'] == 'MaxPooling2D': 422 | print(layer.output_shape) 423 | txtfile.write('MaxPool ' + str(json_elem['config']['pool_size'][0]) + ' '\ 424 | + str(json_elem['config']['pool_size'][1]) + ' ' \ 425 | + str(json_elem['config']['strides'][0]) + ' ' \ 426 | + str(json_elem['config']['strides'][1])) 427 | elif json_elem['class_name'] == 'Flatten': 428 | txtfile.write(json_elem['class_name']) 429 | else: 430 | print("Unknown layer: " + str(json_elem['class_name'])) 431 | continue 432 | 433 | txtfile.write('\n') 434 | 435 | try: 436 | for elem in layer.get_weights()[0].flatten(): 437 | txtfile.write(str(elem) + ' ') 438 | txtfile.write('\n') 439 | for elem in layer.get_weights()[1].flatten(): 440 | txtfile.write(str(elem) + ' ') 441 | txtfile.write('\n') 442 | except: 443 | pass 444 | 445 | if json_elem['class_name'] == 'Dense' and \ 446 | json_elem['config']['activation'] == 'relu': 447 | txtfile.write('Rect') 448 | txtfile.write('\n') 449 | 450 | prev_layer = layer 451 | if json_elem['class_name'] == 'Conv2D': 452 | prev_filter_num = str(json_elem['config']['filters']) 453 | 454 | txtfile.close() 455 | 456 | 457 | 458 | _write_as = {'pickled': _write_pickled,\ 459 | 'pickle':_write_pickled,\ 460 | 'nn':_write_pickled,\ 461 | '':_write_pickled,\ 462 | 'txt':_write_txt,\ 463 | 'keras_txt': _write_keras_txt,\ 464 | } 465 | 466 | -------------------------------------------------------------------------------- /lrp_toolbox/modules/__init__.py: -------------------------------------------------------------------------------- 1 | from .module import Module 2 | from .linear import Linear 3 | from .tanh import Tanh 4 | from .rect import Rect 5 | from .softmax import SoftMax 6 | from .maxpool import MaxPool 7 | from .sumpool import SumPool 8 | from .flatten import Flatten 9 | from .convolution import Convolution 10 | from .sequential import Sequential -------------------------------------------------------------------------------- /lrp_toolbox/modules/convolution.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 20.10.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | na = np.newaxis 15 | 16 | 17 | # ------------------------------- 18 | # 2D Convolution layer 19 | # ------------------------------- 20 | 21 | class Convolution(Module): 22 | 23 | def __init__(self, filtersize=(5,5,3,32), stride = (2,2)): 24 | ''' 25 | Constructor for a Convolution layer. 26 | 27 | Parameters 28 | ---------- 29 | 30 | filtersize : 4-tuple with values (h,w,d,n), where 31 | h = filter heigth 32 | w = filter width 33 | d = filter depth 34 | n = number of filters = number of outputs 35 | 36 | stride : 2-tuple (h,w), where 37 | h = step size for filter application in vertical direction 38 | w = step size in horizontal direction 39 | 40 | ''' 41 | 42 | Module.__init__(self) 43 | 44 | self.fh, self.fw, self.fd, self.n = filtersize 45 | self.stride = stride 46 | 47 | self.W = np.random.normal(0,1./(self.fh*self.fw*self.fd)**.5, filtersize) 48 | self.B = np.zeros([self.n]) 49 | 50 | 51 | def forward(self,X,padding=False,lrp_aware=False): 52 | ''' 53 | Realizes the forward pass of an input through the convolution layer. 54 | 55 | Parameters 56 | ---------- 57 | X : numpy.ndarray 58 | a network input, shaped (N,H,W,D), with 59 | N = batch size 60 | H, W, D = input size in heigth, width, depth 61 | 62 | lrp_aware : bool 63 | controls whether the forward pass is to be computed with awareness for multiple following 64 | LRP calls. this will sacrifice speed in the forward pass but will save time if multiple LRP 65 | calls will follow for the current X, e.g. wit different parameter settings or for multiple 66 | target classes. 67 | 68 | Returns 69 | ------- 70 | Y : numpy.ndarray 71 | the layer outputs. 72 | ''' 73 | 74 | self.lrp_aware = lrp_aware 75 | 76 | #apply padding - Team DeepFault 77 | #TODO: Should be automatically handled 78 | #padding=True 79 | if padding: 80 | X = np.pad(X, ((0,0), (2,2), (2,2), (0,0)), 'constant', constant_values=(0.,)) # FOR LENET MODELS 81 | 82 | self.X = X 83 | N,H,W,D = X.shape 84 | 85 | hf, wf, df, nf = self.W.shape 86 | hstride, wstride = self.stride 87 | numfilters = self.n 88 | 89 | #for ttt in range(X.shape[1]): 90 | # print(X[0,ttt,:,0]) 91 | #print(X[:, 4*hstride:4*hstride+hf: , 4*wstride:4*wstride+wf: , : ]) 92 | 93 | #assume the given pooling and stride parameters are carefully chosen. 94 | Hout = (H - hf) // hstride + 1 95 | Wout = (W - wf) // wstride + 1 96 | 97 | 98 | #initialize pooled output 99 | self.Y = np.zeros((N,Hout,Wout,numfilters)) 100 | 101 | if self.lrp_aware: 102 | self.Z = np.zeros((N, Hout, Wout, hf, wf, df, nf)) #initialize container for precomputed forward messages 103 | for i in range(Hout): 104 | for j in range(Wout): 105 | self.Z[:,i,j,...] = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] # N, hf, wf, df, nf 106 | self.Y[:,i,j,:] = self.Z[:,i,j,...].sum(axis=(1,2,3)) + self.B 107 | else: 108 | for i in range(Hout): 109 | for j in range(Wout): 110 | self.Y[:,i,j,:] = np.tensordot(X[:, i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ],self.W,axes = ([1,2,3],[0,1,2])) + self.B 111 | 112 | return self.Y 113 | 114 | 115 | def backward(self,DY): 116 | ''' 117 | Backward-passes an input error gradient DY towards the input neurons of this layer. 118 | 119 | Parameters 120 | ---------- 121 | 122 | DY : numpy.ndarray 123 | an error gradient shaped same as the output array of forward, i.e. (N,Hy,Wy,Dy) with 124 | N = number of samples in the batch 125 | Hy = heigth of the output 126 | Wy = width of the output 127 | Dy = output depth = input depth 128 | 129 | 130 | Returns 131 | ------- 132 | 133 | DX : numpy.ndarray 134 | the error gradient propagated towards the input 135 | 136 | ''' 137 | 138 | self.DY = DY 139 | N,Hy,Wy,NF = DY.shape 140 | hf,wf,df,NF = self.W.shape 141 | hstride, wstride = self.stride 142 | 143 | DX = np.zeros_like(self.X,dtype=np.float) 144 | 145 | 146 | if not (hf == wf and self.stride == (1,1)): 147 | for i in range(Hy): 148 | for j in range(Wy): 149 | DX[:,i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : ] += (self.W[na,...] * DY[:,i:i+1,j:j+1,na,:]).sum(axis=4) #sum over all the filters 150 | else: 151 | for i in range(hf): 152 | for j in range(wf): 153 | DX[:,i:i+Hy:hstride,j:j+Wy:wstride,:] += np.dot(DY,self.W[i,j,:,:].T) 154 | 155 | return DX #* (hf*wf*df)**.5 / (NF*Hy*Wy)**.5 156 | 157 | 158 | def update(self,lrate): 159 | N,Hx,Wx,Dx = self.X.shape 160 | N,Hy,Wy,NF = self.DY.shape 161 | 162 | hf,wf,df,NF = self.W.shape 163 | hstride, wstride = self.stride 164 | 165 | DW = np.zeros_like(self.W,dtype=np.float) 166 | 167 | if not (hf == wf and self.stride == (1,1)): 168 | for i in range(Hy): 169 | for j in range(Wy): 170 | DW += (self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , :, na] * self.DY[:,i:i+1,j:j+1,na,:]).sum(axis=0) 171 | else: 172 | for i in range(hf): 173 | for j in range(wf): 174 | DW[i,j,:,:] = np.tensordot(self.X[:,i:i+Hy:hstride,j:j+Wy:wstride,:],self.DY,axes=([0,1,2],[0,1,2])) 175 | 176 | DB = self.DY.sum(axis=(0,1,2)) 177 | self.W -= lrate * DW / (hf*wf*df*Hy*Wy)**.5 178 | self.B -= lrate * DB / (Hy*Wy)**.5 179 | 180 | 181 | def clean(self): 182 | self.X = None 183 | self.Y = None 184 | self.DY = None 185 | 186 | 187 | def _simple_lrp_slow(self,R): 188 | ''' 189 | LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140 190 | This function shows all necessary operations to perform LRP in one place and is therefore not optimized 191 | ''' 192 | 193 | N,Hout,Wout,NF = R.shape 194 | hf,wf,df,NF = self.W.shape 195 | hstride, wstride = self.stride 196 | 197 | Rx = np.zeros_like(self.X,dtype=np.float) 198 | 199 | for i in range(Hout): 200 | for j in range(Wout): 201 | Z = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] 202 | Zs = Z.sum(axis=(1,2,3),keepdims=True) + self.B[na,na,na,na,...] 203 | Zs += 1e-16*((Zs >= 0)*2 - 1.) # add a weak numerical stabilizer to cushion division by zero 204 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += ((Z/Zs) * R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 205 | return Rx 206 | 207 | 208 | 209 | def _simple_lrp(self,R): 210 | ''' 211 | LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140 212 | ''' 213 | 214 | N,Hout,Wout,NF = R.shape 215 | hf,wf,df,NF = self.W.shape 216 | hstride, wstride = self.stride 217 | 218 | Rx = np.zeros_like(self.X,dtype=np.float) 219 | R_norm = R / (self.Y + 1e-16*((self.Y >= 0)*2 - 1.)) 220 | 221 | for i in range(Hout): 222 | for j in range(Wout): 223 | if self.lrp_aware: 224 | Z = self.Z[:,i,j,...] 225 | else: 226 | Z = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] 227 | 228 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += (Z * (R_norm[:,i:i+1,j:j+1,na,:])).sum(axis=4) 229 | return Rx 230 | 231 | def _flat_lrp(self,R): 232 | ''' 233 | distribute relevance for each output evenly to the output neurons' receptive fields. 234 | ''' 235 | 236 | N,Hout,Wout,NF = R.shape 237 | hf,wf,df,NF = self.W.shape 238 | hstride, wstride = self.stride 239 | 240 | Rx = np.zeros_like(self.X,dtype=np.float) 241 | 242 | for i in range(Hout): 243 | for j in range(Wout): 244 | Z = np.ones((N,hf,wf,df,NF)) 245 | Zs = Z.sum(axis=(1,2,3),keepdims=True) 246 | 247 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += ((Z/Zs) * R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 248 | return Rx 249 | 250 | def _ww_lrp(self,R): 251 | ''' 252 | LRP according to Eq(12) in https://arxiv.org/pdf/1512.02479v1.pdf 253 | ''' 254 | 255 | N,Hout,Wout,NF = R.shape 256 | hf,wf,df,NF = self.W.shape 257 | hstride, wstride = self.stride 258 | 259 | Rx = np.zeros_like(self.X,dtype=np.float) 260 | 261 | for i in range(Hout): 262 | for j in range(Wout): 263 | Z = self.W[na,...]**2 264 | Zs = Z.sum(axis=(1,2,3),keepdims=True) 265 | 266 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += ((Z/Zs) * R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 267 | return Rx 268 | 269 | def _epsilon_lrp_slow(self,R,epsilon): 270 | ''' 271 | LRP according to Eq(58) in DOI: 10.1371/journal.pone.0130140 272 | This function shows all necessary operations to perform LRP in one place and is therefore not optimized 273 | ''' 274 | 275 | N,Hout,Wout,NF = R.shape 276 | hf,wf,df,NF = self.W.shape 277 | hstride, wstride = self.stride 278 | 279 | Rx = np.zeros_like(self.X,dtype=np.float) 280 | 281 | for i in range(Hout): 282 | for j in range(Wout): 283 | Z = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] 284 | Zs = Z.sum(axis=(1,2,3),keepdims=True) + self.B[na,na,na,na,...] 285 | Zs += epsilon*((Zs >= 0)*2-1) 286 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += ((Z/Zs) * R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 287 | return Rx 288 | 289 | 290 | def _epsilon_lrp(self,R,epsilon): 291 | ''' 292 | LRP according to Eq(58) in DOI: 10.1371/journal.pone.0130140 293 | ''' 294 | 295 | N,Hout,Wout,NF = R.shape 296 | hf,wf,df,NF = self.W.shape 297 | hstride, wstride = self.stride 298 | 299 | Rx = np.zeros_like(self.X,dtype=np.float) 300 | R_norm = R / (self.Y + epsilon*((self.Y >= 0)*2 - 1.)) 301 | 302 | for i in range(Hout): 303 | for j in range(Wout): 304 | if self.lrp_aware: 305 | Z = self.Z[:,i,j,...] 306 | else: 307 | Z = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] 308 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += (Z * (R_norm[:,i:i+1,j:j+1,na,:])).sum(axis=4) 309 | return Rx 310 | 311 | 312 | def _alphabeta_lrp_slow(self,R,alpha): 313 | ''' 314 | LRP according to Eq(60) in DOI: 10.1371/journal.pone.0130140 315 | ''' 316 | 317 | beta = 1 - alpha 318 | 319 | N,Hout,Wout,NF = R.shape 320 | hf,wf,df,NF = self.W.shape 321 | hstride, wstride = self.stride 322 | 323 | Rx = np.zeros_like(self.X,dtype=np.float) 324 | 325 | for i in range(Hout): 326 | for j in range(Wout): 327 | Z = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] 328 | 329 | if not alpha == 0: 330 | Zp = Z * (Z > 0) 331 | Bp = (self.B * (self.B > 0))[na,na,na,na,...] 332 | Zsp = Zp.sum(axis=(1,2,3),keepdims=True) + Bp 333 | Ralpha = alpha * ((Zp/Zsp) * R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 334 | Ralpha[np.isnan(Ralpha)] = 0 335 | else: 336 | Ralpha = 0 337 | 338 | if not beta == 0: 339 | Zn = Z * (Z < 0) 340 | Bn = (self.B * (self.B < 0))[na,na,na,na,...] 341 | Zsn = Zn.sum(axis=(1,2,3),keepdims=True) + Bn 342 | Rbeta = beta * ((Zn/Zsn) * R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 343 | Rbeta[np.isnan(Rbeta)] = 0 344 | else: 345 | Rbeta = 0 346 | 347 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += Ralpha + Rbeta 348 | 349 | return Rx 350 | 351 | 352 | def _alphabeta_lrp(self,R,alpha): 353 | ''' 354 | LRP according to Eq(60) in DOI: 10.1371/journal.pone.0130140 355 | ''' 356 | 357 | beta = 1 - alpha 358 | 359 | N,Hout,Wout,NF = R.shape 360 | hf,wf,df,NF = self.W.shape 361 | hstride, wstride = self.stride 362 | 363 | Rx = np.zeros_like(self.X,dtype=np.float) 364 | 365 | for i in range(Hout): 366 | for j in range(Wout): 367 | if self.lrp_aware: 368 | Z = self.Z[:,i,j,...] 369 | else: 370 | Z = self.W[na,...] * self.X[:, i*hstride:i*hstride+hf , j*wstride:j*wstride+wf , : , na] 371 | 372 | Zplus = Z > 0 #index mask of positive forward predictions 373 | 374 | if alpha * beta != 0 : #the general case: both parameters are not 0 375 | Zp = Z * Zplus 376 | Zsp = Zp.sum(axis=(1,2,3),keepdims=True) + (self.B * (self.B > 0))[na,na,na,na,...] + 1e-16 377 | 378 | Zn = Z - Zp 379 | Zsn = self.Y[:,i:i+1,j:j+1,na,:] - Zsp - 1e-16 380 | 381 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += ((alpha * (Zp/Zsp) + beta * (Zn/Zsn))*R[:,i:i+1,j:j+1,na,:]).sum(axis=4) 382 | 383 | elif alpha: #only alpha is not 0 -> alpha = 1, beta = 0 384 | Zp = Z * Zplus 385 | Zsp = Zp.sum(axis=(1,2,3),keepdims=True) + (self.B * (self.B > 0))[na,na,na,na,...] + 1e-16 386 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += (Zp*(R[:,i:i+1,j:j+1,na,:]/Zsp)).sum(axis=4) 387 | 388 | elif beta: # only beta is not 0 -> alpha = 0, beta = 1 389 | Zn = Z * np.invert(Zplus) 390 | Zsn = Zn.sum(axis=(1,2,3),keepdims=True) + (self.B * (self.B < 0))[na,na,na,na,...] + 1e-16 391 | Rx[:,i*hstride:i*hstride+hf: , j*wstride:j*wstride+wf: , : ] += (Zn*(R[:,i:i+1,j:j+1,na,:]/Zsn)).sum(axis=4) 392 | 393 | else: 394 | raise Exception('This case should never occur: alpha={}, beta={}.'.format(alpha, beta)) 395 | 396 | return Rx 397 | -------------------------------------------------------------------------------- /lrp_toolbox/modules/flatten.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 20.10.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | 15 | # ------------------------------- 16 | # Flattening Layer 17 | # ------------------------------- 18 | 19 | class Flatten(Module): 20 | ''' 21 | Flattening layer. 22 | ''' 23 | 24 | def __init__(self): 25 | Module.__init__(self) 26 | self.inputshape = [] 27 | 28 | def backward(self,DY): 29 | ''' 30 | Just backward-passes the input gradient DY and reshapes it to fit the input. 31 | ''' 32 | return np.reshape(DY,self.inputshape) 33 | 34 | def forward(self,X,*args,**kwargs): 35 | ''' 36 | Transforms each sample in X to a one-dimensional array. 37 | Shape change according to C-order. 38 | ''' 39 | self.inputshape = X.shape # N x H x W x D 40 | return np.reshape(X,[self.inputshape[0],np.prod(self.inputshape[1:])]) 41 | 42 | def lrp(self,R, *args, **kwargs): 43 | ''' 44 | Receives upper layer input relevance R and reshapes it to match the input neurons. 45 | ''' 46 | # just propagate R further down. 47 | # makes sure subroutines never get called. 48 | return np.reshape(R,self.inputshape) -------------------------------------------------------------------------------- /lrp_toolbox/modules/linear.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | na = np.newaxis 15 | 16 | # ------------------------------- 17 | # Linear layer 18 | # ------------------------------- 19 | class Linear(Module): 20 | ''' 21 | Linear Layer 22 | ''' 23 | 24 | def __init__(self,m,n): 25 | ''' 26 | Initiates an instance of a linear computation layer. 27 | 28 | Parameters 29 | ---------- 30 | m : int 31 | input dimensionality 32 | n : int 33 | output dimensionality 34 | 35 | Returns 36 | ------- 37 | the newly created object instance 38 | ''' 39 | Module.__init__(self) 40 | self.m = m 41 | self.n = n 42 | self.B = np.zeros([self.n]) 43 | self.W = np.random.normal(0,1.0*m**(-.5),[self.m,self.n]) 44 | 45 | 46 | def forward(self,X,lrp_aware=False): 47 | ''' 48 | Forward-transforms an input X 49 | 50 | Parameters 51 | ---------- 52 | 53 | X : numpy.ndarray 54 | the input, shaped [N,D], where N is the number of samples and D their dimensionality 55 | 56 | Returns 57 | ------- 58 | Y : numpy.ndarray 59 | the transformed data shaped [N,M], with M being the number of output neurons 60 | ''' 61 | self.lrp_aware = lrp_aware 62 | if self.lrp_aware: 63 | self.X = X 64 | self.Z = self.W[na,:,:]*self.X[:,:,na] 65 | self.Y = self.Z.sum(axis=1) + self.B 66 | else: 67 | self.X = X 68 | self.Y = np.dot(X,self.W)+self.B 69 | 70 | return self.Y 71 | 72 | 73 | def backward(self,DY): 74 | ''' 75 | Backward pass through the linear layer, computing the derivative wrt the inputs. 76 | Ensures a well-conditioned output gradient 77 | 78 | Parameters 79 | ---------- 80 | 81 | DY : numpy.ndarray 82 | the backpropagated error signal as input, shaped [N,M] 83 | 84 | Returns 85 | ------- 86 | 87 | DX : numpy.ndarray 88 | the computed output derivative of the error signal wrt X, shaped [N,D] 89 | ''' 90 | 91 | self.dW = np.dot(self.X.T,DY) 92 | self.dB = DY.sum(axis=0) 93 | return np.dot(DY,self.W.T)*self.m**.5/self.n**.5 94 | 95 | 96 | def update(self, lrate): 97 | ''' 98 | Update the model weights 99 | ''' 100 | self.W -= lrate*self.dW/self.m**.5 101 | self.B -= lrate*self.dB/self.n**.25 102 | 103 | 104 | def clean(self): 105 | ''' 106 | Removes temporarily stored variables from this layer 107 | ''' 108 | self.X = None 109 | self.Y = None 110 | self.dW = None 111 | self.dB = None 112 | 113 | 114 | def _simple_lrp_slow(self,R): 115 | ''' 116 | LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140. 117 | This function shows all necessary operations to perform LRP in one place and is therefore not optimized 118 | ''' 119 | Z = self.W[na,:,:]*self.X[:,:,na] #localized preactivations 120 | Zs = Z.sum(axis=1)[:,na,:] +self.B[na,na,:] #preactivations 121 | Zs += 1e-16*((Zs >= 0)*2 - 1.) #add weak default stabilizer to denominator 122 | return ((Z / Zs) * R[:,na,:]).sum(axis=2) 123 | 124 | 125 | def _simple_lrp(self,R): 126 | ''' 127 | LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140 128 | ''' 129 | 130 | # Has the forward pass been computed lrp-aware? 131 | # This exchanges time spent in the forward pass for lower LRP time 132 | # and is useful, if e.g. several parameter settings for LRP need to be evaluated 133 | # for the same input data. 134 | Zs = self.Y + 1e-16*((self.Y >= 0)*2 - 1.) #add weakdefault stabilizer to denominator 135 | if self.lrp_aware: 136 | return (self.Z * (R/Zs)[:,na,:]).sum(axis=2) 137 | else: 138 | Z = self.W[na,:,:]*self.X[:,:,na] #localized preactivations 139 | return (Z * (R/Zs)[:,na,:]).sum(axis=2) 140 | 141 | 142 | 143 | def _flat_lrp(self,R): 144 | ''' 145 | distribute relevance for each output evenly to the output neurons' receptive fields. 146 | note that for fully connected layers, this results in a uniform lower layer relevance map. 147 | ''' 148 | Z = np.ones_like(self.W[na,:,:]) 149 | Zs = Z.sum(axis=1)[:,na,:] 150 | return ((Z / Zs) * R[:,na,:]).sum(axis=2) 151 | 152 | 153 | def _ww_lrp(self,R): 154 | ''' 155 | LRR according to Eq(12) in https://arxiv.org/pdf/1512.02479v1.pdf 156 | ''' 157 | Z = self.W[na,:,:]**2 158 | Zs = Z.sum(axis=1)[:,na,:] 159 | return ((Z / Zs) * R[:,na,:]).sum(axis=2) 160 | 161 | 162 | def _epsilon_lrp_slow(self,R,epsilon): 163 | ''' 164 | LRP according to Eq(58) in DOI: 10.1371/journal.pone.0130140 165 | This function shows all necessary operations to perform LRP in one place and is therefore not optimized 166 | ''' 167 | 168 | Z = self.W[na,:,:]*self.X[:,:,na] # localized preactivations 169 | Zs = Z.sum(axis=1)[:,na,:] +self.B[na,na,:] # preactivations 170 | 171 | # add slack to denominator. we require sign(0) = 1. since np.sign(0) = 0 would defeat the purpose of the numeric stabilizer we do not use it. 172 | Zs += epsilon * ((Zs >= 0)*2-1) 173 | return ((Z / Zs) * R[:,na,:]).sum(axis=2) 174 | 175 | 176 | def _epsilon_lrp(self,R,epsilon): 177 | ''' 178 | LRP according to Eq(58) in DOI: 10.1371/journal.pone.0130140 179 | ''' 180 | 181 | Zs = self.Y + epsilon * ((self.Y >= 0)*2-1)#prepare stabilized denominator 182 | 183 | # Has the forward pass been computed lrp-aware? 184 | # This exchanges time spent in the forward pass for lower LRP time 185 | # and is useful, if e.g. several parameter settings for LRP need to be evaluated 186 | # for the same input data. 187 | if self.lrp_aware: 188 | return (self.Z * (R/Zs)[:,na,:]).sum(axis=2) 189 | else: 190 | Z = self.W[na,:,:]*self.X[:,:,na] #localized preactivations 191 | return (Z * (R/Zs)[:,na,:]).sum(axis=2) 192 | 193 | 194 | def _alphabeta_lrp_slow(self,R,alpha): 195 | ''' 196 | LRP according to Eq(60) in DOI: 10.1371/journal.pone.0130140 197 | This function shows all necessary operations to perform LRP in one place and is therefore not optimized 198 | ''' 199 | beta = 1 - alpha 200 | Z = self.W[na,:,:]*self.X[:,:,na] # localized preactivations 201 | 202 | if not alpha == 0: 203 | Zp = Z * (Z > 0) 204 | Zsp = Zp.sum(axis=1)[:,na,:] + (self.B * (self.B > 0))[na,na,:] 205 | Ralpha = alpha * ((Zp / Zsp) * R[:,na,:]).sum(axis=2) 206 | else: 207 | Ralpha = 0 208 | 209 | if not beta == 0: 210 | Zn = Z * (Z < 0) 211 | Zsn = Zn.sum(axis=1)[:,na,:] + (self.B * (self.B < 0))[na,na,:] 212 | Rbeta = beta * ((Zn / Zsn) * R[:,na,:]).sum(axis=2) 213 | else: 214 | Rbeta = 0 215 | 216 | return Ralpha + Rbeta 217 | 218 | 219 | def _alphabeta_lrp(self,R,alpha): 220 | ''' 221 | LRP according to Eq(60) in DOI: 10.1371/journal.pone.0130140 222 | ''' 223 | beta = 1 - alpha 224 | if self.lrp_aware: 225 | Z = self.Z 226 | else: 227 | Z = self.W[na,:,:]*self.X[:,:,na] # localized preactivations 228 | 229 | 230 | #index mask of positive forward predictions 231 | Zplus = Z > 0 232 | if alpha * beta != 0: #the general case: both parameters are not 0 233 | Zp = Z * Zplus 234 | Zsp = Zp.sum(axis=1) + (self.B * (self.B > 0))[na,:] + 1e-16 235 | 236 | Zn = Z - Zp 237 | Zsn = self.Y - Zsp - 1e-16 238 | 239 | return alpha * (Zp*(R/Zsp)[:,na,:]).sum(axis=2) + beta * (Zn * (R/Zsn)[:,na,:]).sum(axis=2) 240 | 241 | elif alpha: #only alpha is not 0 -> alpha = 1, beta = 0 242 | Zp = Z * Zplus 243 | Zsp = Zp.sum(axis=1) + (self.B * (self.B > 0))[na,:] + 1e-16 244 | return (Zp*(R/Zsp)[:,na,:]).sum(axis=2) 245 | 246 | elif beta: # only beta is not 0 -> alpha = 0, beta = 1 247 | Zn = Z * np.invert(Zplus) 248 | Zsn = Zn.sum(axis=1) + (self.B * (self.B < 0))[na,:] - 1e-16 249 | return (Zn * (R/Zsn)[:,na,:]).sum(axis=2) 250 | 251 | else: 252 | raise Exception('This case should never occur: alpha={}, beta={}.'.format(alpha, beta)) 253 | 254 | -------------------------------------------------------------------------------- /lrp_toolbox/modules/maxpool.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 20.10.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | 15 | # ------------------------------- 16 | # Max Pooling layer 17 | # ------------------------------- 18 | 19 | class MaxPool(Module): 20 | def __init__(self,pool=(2,2),stride=(2,2)): 21 | ''' 22 | Constructor for the max pooling layer object 23 | 24 | Parameters 25 | ---------- 26 | 27 | pool : tuple (h,w) 28 | the size of the pooling mask in vertical (h) and horizontal (w) direction 29 | 30 | stride : tuple (h,w) 31 | the vertical (h) and horizontal (w) step sizes between filter applications. 32 | ''' 33 | 34 | Module.__init__(self) 35 | 36 | self.pool = pool 37 | self.stride = stride 38 | 39 | 40 | def forward(self,X,*args,**kwargs): 41 | ''' 42 | Realizes the forward pass of an input through the max pooling layer. 43 | 44 | Parameters 45 | ---------- 46 | X : numpy.ndarray 47 | a network input, shaped (N,H,W,D), with 48 | N = batch size 49 | H, W, D = input size in heigth, width, depth 50 | 51 | Returns 52 | ------- 53 | Y : numpy.ndarray 54 | the max-pooled outputs, reduced in size due to given stride and pooling size 55 | ''' 56 | 57 | self.X = X 58 | N,H,W,D = X.shape 59 | 60 | hpool, wpool = self.pool 61 | hstride, wstride = self.stride 62 | 63 | #assume the given pooling and stride parameters are carefully chosen. 64 | Hout = (H - hpool) // hstride + 1 65 | Wout = (W - wpool) // wstride + 1 66 | 67 | #initialize pooled output 68 | self.Y = np.zeros((N,Hout,Wout,D)) 69 | 70 | for i in range(Hout): 71 | for j in range(Wout): 72 | self.Y[:,i,j,:] = X[:, i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ].max(axis=(1,2)) 73 | return self.Y 74 | 75 | 76 | def backward(self,DY): 77 | ''' 78 | Backward-passes an input error gradient DY towards the domintly ativating neurons of this max pooling layer. 79 | 80 | Parameters 81 | ---------- 82 | 83 | DY : numpy.ndarray 84 | an error gradient shaped same as the output array of forward, i.e. (N,Hy,Wy,Dy) with 85 | N = number of samples in the batch 86 | Hy = heigth of the output 87 | Wy = width of the output 88 | Dy = output depth = input depth 89 | 90 | 91 | Returns 92 | ------- 93 | 94 | DX : numpy.ndarray 95 | the error gradient propagated towards the input 96 | 97 | ''' 98 | 99 | N,H,W,D = self.X.shape 100 | 101 | hpool, wpool = self.pool 102 | hstride, wstride = self.stride 103 | 104 | #assume the given pooling and stride parameters are carefully chosen. 105 | Hout = (H - hpool) // hstride + 1 106 | Wout = (W - wpool) // wstride + 1 107 | 108 | #distribute the gradient towards the max activation(s) 109 | #the max activation value is already known via self.Y 110 | DX = np.zeros_like(self.X,dtype=np.float) 111 | for i in range(Hout): 112 | for j in range(Wout): 113 | DX[:,i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool,:] += DY[:,i:i+1,j:j+1,:] * (self.Y[:,i:i+1,j:j+1,:] == self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ]) 114 | return DX 115 | 116 | 117 | def clean(self): 118 | self.X = None 119 | self.Y = None 120 | 121 | 122 | 123 | 124 | def _simple_lrp_slow(self,R): 125 | N,H,W,D = self.X.shape 126 | 127 | hpool, wpool = self.pool 128 | hstride, wstride = self.stride 129 | 130 | #assume the given pooling and stride parameters are carefully chosen. 131 | Hout = (H - hpool) // hstride + 1 132 | Wout = (W - wpool) // wstride + 1 133 | 134 | Rx = np.zeros_like(self.X,dtype=np.float) 135 | 136 | for i in range(Hout): 137 | for j in range(Wout): 138 | Z = self.Y[:,i:i+1,j:j+1,:] == self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] 139 | Zs = Z.sum(axis=(1,2),keepdims=True,dtype=np.float) #thanks user wodtko for reporting this bug/fix 140 | Rx[:,i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool,:] += (Z / Zs) * R[:,i:i+1,j:j+1,:] 141 | return Rx 142 | 143 | 144 | def _simple_lrp(self,R): 145 | return self._simple_lrp_slow(R) 146 | 147 | def _flat_lrp(self,R): 148 | ''' 149 | distribute relevance for each output evenly to the output neurons' receptive fields. 150 | ''' 151 | N,H,W,D = self.X.shape 152 | 153 | hpool, wpool = self.pool 154 | hstride, wstride = self.stride 155 | 156 | #assume the given pooling and stride parameters are carefully chosen. 157 | Hout = (H - hpool) // hstride + 1 158 | Wout = (W - wpool) // wstride + 1 159 | 160 | Rx = np.zeros_like(self.X,dtype=np.float) 161 | 162 | for i in range(Hout): 163 | for j in range(Wout): 164 | Z = np.ones([N,hpool,wpool,D]) 165 | Zs = Z.sum(axis=(1,2),keepdims=True) 166 | Rx[:,i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool,:] += (Z / Zs) * R[:,i:i+1,j:j+1,:] 167 | return Rx 168 | 169 | def _ww_lrp(self,R): 170 | ''' 171 | There are no weights to use. default to _flat_lrp(R) 172 | ''' 173 | return self._flat_lrp(R) 174 | 175 | def _epsilon_lrp(self,R,epsilon): 176 | ''' 177 | Since there is only one (or several equally strong) dominant activations, default to _simple_lrp 178 | ''' 179 | return self._simple_lrp(R) 180 | 181 | def _alphabeta_lrp(self,R,alpha): 182 | ''' 183 | Since there is only one (or several equally strong) dominant activations, default to _simple_lrp 184 | ''' 185 | return self._simple_lrp(R) 186 | -------------------------------------------------------------------------------- /lrp_toolbox/modules/module.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | # ------------------------------- 13 | # Modules for the neural network 14 | # ------------------------------- 15 | class Module: 16 | ''' 17 | Superclass for all computation layer implementations 18 | ''' 19 | 20 | def __init__(self): 21 | ''' The constructor ''' 22 | 23 | #values for presetting lrp decomposition parameters per layer 24 | self.lrp_var = None 25 | self.lrp_param = 1. 26 | 27 | def backward(self,DY): 28 | ''' backward passes the error gradient DY to the input neurons ''' 29 | return DY 30 | 31 | def train(self, X, Y, *args, **kwargs): 32 | ''' implements (currently in modules.Sequential only) a simple training routine ''' 33 | 34 | def forward(self,X,lrp_aware=False): 35 | ''' forward passes the input data X to the layer's output neurons. 36 | 37 | Parameters 38 | ---------- 39 | 40 | X : numpy.ndarray 41 | the input activations for this layer, shaped [batchsize, ...] 42 | 43 | lrp_aware : bool 44 | controls whether the forward pass is to be computed with awareness for multiple following 45 | LRP calls. this will sacrifice speed in the forward pass but will save time if multiple LRP 46 | calls will follow for the current X, e.g. wit different parameter settings or for multiple 47 | target classes. 48 | 49 | ''' 50 | return X 51 | 52 | def update(self, lrate): 53 | ''' update should implement the layer parameter updating step ''' 54 | pass 55 | 56 | def clean(self): 57 | ''' clean can be used to remove any temporary variables from the layer, e.g. just before serializing the layer object''' 58 | pass 59 | 60 | 61 | 62 | def set_lrp_parameters(self,lrp_var=None,param=None): 63 | ''' pre-sets lrp parameters to use for this layer. see the documentation of Module.lrp for details ''' 64 | self.lrp_var = lrp_var 65 | self.lrp_param = param 66 | 67 | def lrp(self,R, lrp_var=None,param=None): 68 | ''' 69 | Performs LRP by calling subroutines, depending on lrp_var and param or 70 | preset values specified via Module.set_lrp_parameters(lrp_var,lrp_param) 71 | 72 | If lrp parameters have been pre-specified (per layer), the corresponding decomposition 73 | will be applied during a call of lrp(). 74 | 75 | Specifying lrp parameters explicitly when calling lrp(), e.g. net.lrp(R,lrp_var='alpha',param=2.), 76 | will override the preset values for the current call. 77 | 78 | How to use: 79 | 80 | net.forward(X) #forward feed some data you wish to explain to populat the net. 81 | 82 | then either: 83 | 84 | net.lrp() #to perform the naive approach to lrp implemented in _simple_lrp for each layer 85 | 86 | or: 87 | 88 | for m in net.modules: 89 | m.set_lrp_parameters(...) 90 | net.lrp() #to preset a lrp configuration to each layer in the net 91 | 92 | or: 93 | 94 | net.lrp(somevariantname,someparameter) # to explicitly call the specified parametrization for all layers (where applicable) and override any preset configurations. 95 | 96 | Parameters 97 | ---------- 98 | 99 | R : numpy.ndarray 100 | relevance input for LRP. 101 | should be of the same shape as the previously produced output by .forward 102 | 103 | lrp_var : str 104 | either 'none' or 'simple' or None for standard LRP , 105 | 'slow' or 'simple_slow' for the explicit implementation of LRP, 106 | 'epsilon' for an added epsilon slack in the denominator, 107 | 'epsilon_slow' for the explicit implementation of the epsilon stabilized variant 108 | 'alphabeta' or 'alpha' for weighting positive and negative contributions separately. param specifies alpha with alpha + beta = 1 109 | 'flat' projects an upper layer neuron's relevance uniformly over its receptive field. 110 | 'ww' or 'w^2' only considers the square weights w_ij^2 as qantities to distribute relevances with. 111 | 112 | param : double 113 | the respective parameter for the lrp method of choice 114 | 115 | Returns 116 | ------- 117 | R : the backward-propagated relevance scores. 118 | shaped identically to the previously processed inputs in .forward 119 | ''' 120 | 121 | if lrp_var is None and param is None: 122 | # module.lrp(R) has been called without further parameters. 123 | # set default values / preset values 124 | lrp_var = self.lrp_var 125 | param = self.lrp_param 126 | 127 | if lrp_var is None or lrp_var.lower() == 'none' or lrp_var.lower() == 'simple': 128 | return self._simple_lrp(R) 129 | elif lrp_var.lower() == 'slow' or lrp_var.lower() == 'simple_slow': 130 | return self._simple_lrp_slow(R) 131 | 132 | elif lrp_var.lower() == 'flat': 133 | return self._flat_lrp(R) 134 | elif lrp_var.lower() == 'ww' or lrp_var.lower() == 'w^2': 135 | return self._ww_lrp(R) 136 | 137 | elif lrp_var.lower() == 'epsilon': 138 | return self._epsilon_lrp(R,param) 139 | elif lrp_var.lower() == 'epsilon_slow': 140 | return self._epsilon_lrp_slow(R,param) 141 | 142 | elif lrp_var.lower() == 'alphabeta' or lrp_var.lower() == 'alpha': 143 | return self._alphabeta_lrp(R,param) 144 | elif lrp_var.lower() == 'alphabeta_slow' or lrp_var.lower() == 'alpha_slow': 145 | return self._alphabeta_lrp_slow(R,param) 146 | 147 | else: 148 | raise Exception('Unknown lrp variant {}'.format(lrp_var)) 149 | 150 | 151 | # --------------------------------------------------------- 152 | # Methods below should be implemented by inheriting classes 153 | # --------------------------------------------------------- 154 | 155 | def _simple_lrp(self,R): 156 | raise NotImplementedError() 157 | 158 | def _simple_lrp_slow(self,R): 159 | raise NotImplementedError() 160 | 161 | def _flat_lrp(self,R): 162 | raise NotImplementedError() 163 | 164 | def _ww_lrp(self,R): 165 | raise NotImplementedError() 166 | 167 | def _epsilon_lrp(self,R,param): 168 | raise NotImplementedError() 169 | 170 | def _epsilon_lrp_slow(self,R,param): 171 | raise NotImplementedError() 172 | 173 | def _alphabeta_lrp(self,R,param): 174 | raise NotImplementedError() 175 | 176 | def _alphabeta_lrp_slow(self,R,param): 177 | raise NotImplementedError() -------------------------------------------------------------------------------- /lrp_toolbox/modules/rect.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | 15 | # ------------------------------- 16 | # Rectification layer 17 | # ------------------------------- 18 | class Rect(Module): 19 | ''' 20 | Rectification Layer 21 | ''' 22 | def __init__(self): 23 | Module.__init__(self) 24 | 25 | def forward(self,X,*args,**kwargs ): 26 | self.Y = np.maximum(0,X) 27 | return self.Y 28 | 29 | def backward(self,DY): 30 | return DY*(self.Y!=0) 31 | 32 | def clean(self): 33 | self.Y = None 34 | 35 | def lrp(self,R,*args,**kwargs): 36 | # component-wise operations within this layer 37 | # -> 38 | # just propagate R further down. 39 | # makes sure subroutines never get called. 40 | return R -------------------------------------------------------------------------------- /lrp_toolbox/modules/sequential.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import copy 13 | import sys 14 | import numpy as np 15 | import time 16 | from .module import Module 17 | na = np.newaxis 18 | 19 | # ------------------------------- 20 | # Sequential layer 21 | # ------------------------------- 22 | class Sequential(Module): 23 | ''' 24 | Top level access point and incorporation of the neural network implementation. 25 | Sequential manages a sequence of computational neural network modules and passes 26 | along in- and outputs. 27 | ''' 28 | 29 | def __init__(self,modules): 30 | ''' 31 | Constructor 32 | 33 | Parameters 34 | ---------- 35 | modules : list, tuple, etc. enumerable. 36 | an enumerable collection of instances of class Module 37 | ''' 38 | Module.__init__(self) 39 | self.modules = modules 40 | 41 | 42 | def drop_softmax_output_layer(self): 43 | ''' 44 | This function removes the softmax output layer from the model, if there is any. 45 | ''' 46 | from .softmax import SoftMax 47 | if isinstance(self.modules[-1], SoftMax): 48 | print('removing softmax output mapping') 49 | del self.modules[-1] 50 | else: 51 | print('output layer is not softmax. nothing to do') 52 | 53 | 54 | 55 | def forward(self,X,lrp_aware=False): 56 | ''' 57 | Realizes the forward pass of an input through the net 58 | 59 | Parameters 60 | ---------- 61 | X : numpy.ndarray 62 | a network input. 63 | 64 | lrp_aware : bool 65 | controls whether the forward pass is to be computed with awareness for multiple following 66 | LRP calls. this will sacrifice speed in the forward pass but will save time if multiple LRP 67 | calls will follow for the current X, e.g. wit different parameter settings or for multiple 68 | target classes. 69 | 70 | Returns 71 | ------- 72 | X : numpy.ndarray 73 | the output of the network's final layer 74 | ''' 75 | 76 | for m in self.modules: 77 | X = m.forward(X,lrp_aware=lrp_aware) 78 | return X 79 | 80 | 81 | def backward(self,DY): 82 | for m in self.modules[::-1]: 83 | DY = m.backward(DY) 84 | return DY 85 | 86 | 87 | def update(self,lrate): 88 | for m in self.modules: 89 | m.update(lrate) 90 | 91 | 92 | def clean(self): 93 | ''' 94 | Removes temporary variables from all network layers. 95 | ''' 96 | for m in self.modules: 97 | m.clean() 98 | 99 | 100 | def train(self, X, Y, Xval = [], Yval = [], batchsize = 25, iters = 10000, lrate = 0.005, lrate_decay = None, lfactor_initial=1.0 , status = 250, convergence = -1, transform = None): 101 | ''' 102 | Provides a method for training the neural net (self) based on given data. 103 | 104 | Parameters 105 | ---------- 106 | 107 | X : numpy.ndarray 108 | the training data, formatted to (N,D) shape, with N being the number of samples and D their dimensionality 109 | 110 | Y : numpy.ndarray 111 | the training labels, formatted to (N,C) shape, with N being the number of samples and C the number of output classes. 112 | 113 | Xval : numpy.ndarray 114 | some optional validation data. used to measure network performance during training. 115 | shaped (M,D) 116 | 117 | Yval : numpy.ndarray 118 | the validation labels. shaped (M,C) 119 | 120 | batchsize : int 121 | the batch size to use for training 122 | 123 | iters : int 124 | max number of training iterations 125 | 126 | lrate : float 127 | the initial learning rate. the learning rate is adjusted during training with increased model performance. See lrate_decay 128 | 129 | lrate_decay : string 130 | controls if and how the learning rate is adjusted throughout training: 131 | 'none' or None disables learning rate adaption. This is the DEFAULT behaviour. 132 | 'sublinear' adjusts the learning rate to lrate*(1-Accuracy**2) during an evaluation step, often resulting in a better performing model. 133 | 'linear' adjusts the learning rate to lrate*(1-Accuracy) during an evaluation step, often resulting in a better performing model. 134 | 135 | lfactor_initial : float 136 | specifies an initial discount on the given learning rate, e.g. when retraining an established network in combination with a learning rate decay, 137 | it might be undesirable to use the given learning rate in the beginning. this could have been done better. TODO: do better. 138 | Default value is 1.0 139 | 140 | status : int 141 | number of iterations (i.e. number of rounds of batch forward pass, gradient backward pass, parameter update) of silent training 142 | until status print and evaluation on validation data. 143 | 144 | convergence : int 145 | number of consecutive allowed status evaluations with no more model improvements until we accept the model has converged. 146 | Set <=0 to disable. Disabled by DEFAULT. 147 | Set to any value > 0 to control the maximal consecutive number (status * convergence) iterations allowed without model improvement, until convergence is accepted. 148 | 149 | transform : function handle 150 | a function taking as an input a batch of training data sized [N,D] and returning a batch sized [N,D] with added noise or other various data transformations. It's up to you! 151 | default value is None for no transformation. 152 | expected syntax is, with X.shape == Xt.shape == (N,D) 153 | def yourFunction(X): 154 | Xt = someStuff(X) 155 | return Xt 156 | ''' 157 | 158 | def randperm(N,b): 159 | ''' 160 | helper method for picking b unique random indices from a range [0,N[. 161 | we do not use numpy.random.permutation or numpy.random.choice 162 | due to known severe performance issues with drawing without replacement. 163 | if the ratio of N/b is high enough, we should see a huge performance gain. 164 | 165 | N : int 166 | range of indices [0,N[ to choose from.m, s = divmod(seconds, 60) 167 | 168 | 169 | b : the number of unique indices to pick. 170 | ''' 171 | assert(b <= N) # if this fails no valid solution can be found. 172 | I = np.arange(0) 173 | while I.size < b: 174 | I = np.unique(np.append(I,np.random.randint(0,N,[b-I.size,]))) 175 | 176 | return I 177 | 178 | t_start = time.time() 179 | untilConvergence = convergence; learningFactor = lfactor_initial 180 | bestAccuracy = 0.0; bestLayers = copy.deepcopy(self.modules) 181 | bestLoss = np.Inf; bestIter = 0 182 | 183 | N = X.shape[0] 184 | for d in range(iters): 185 | 186 | #the actual training: 187 | #first, pick samples at random 188 | samples = randperm(N,batchsize) 189 | 190 | #transform batch data (maybe) 191 | if transform == None: 192 | batch = X[samples,:] 193 | else: 194 | batch = transform(X[samples,:]) 195 | 196 | #forward and backward propagation steps with parameter update 197 | Ypred = self.forward(batch) 198 | self.backward(Ypred - Y[samples,:]) #l1-loss 199 | self.update(lrate*learningFactor) 200 | 201 | #periodically evaluate network and optionally adjust learning rate or check for convergence. 202 | if (d+1) % status == 0: 203 | if not Xval == [] and not Yval == []: #if given, evaluate on validation data 204 | Ypred = self.forward(Xval) 205 | acc = np.mean(np.argmax(Ypred, axis=1) == np.argmax(Yval, axis=1)) 206 | l1loss = np.abs(Ypred - Yval).sum()/Yval.shape[0] 207 | print('Accuracy after {0} iterations on validation set: {1}% (l1-loss: {2:.4})'.format(d+1, acc*100,l1loss)) 208 | 209 | else: #evaluate on the training data only 210 | Ypred = self.forward(X) 211 | acc = np.mean(np.argmax(Ypred, axis=1) == np.argmax(Y, axis=1)) 212 | l1loss = np.abs(Ypred - Y).sum()/Y.shape[0] 213 | print('Accuracy after {0} iterations on training data: {1}% (l1-loss: {2:.4})'.format(d+1,acc*100,l1loss)) 214 | 215 | 216 | #save current network parameters if we have improved 217 | #if acc >= bestAccuracy and l1loss <= bestLoss: 218 | # only go by loss 219 | if l1loss <= bestLoss: 220 | print(' New loss-optimal parameter set encountered. saving....') 221 | bestAccuracy = acc 222 | bestLoss = l1loss 223 | bestLayers = copy.deepcopy(self.modules) 224 | bestIter = d 225 | 226 | #adjust learning rate 227 | if lrate_decay == None or lrate_decay == 'none': 228 | pass # no adjustment 229 | elif lrate_decay == 'sublinear': 230 | #slow down learning to better converge towards an optimum with increased network performance. 231 | learningFactor = 1.-(acc*acc) 232 | print(' Adjusting learning rate to {0} ~ {1}% of its initial value'.format(learningFactor*lrate, np.round(learningFactor*100,2))) 233 | elif lrate_decay == 'linear': 234 | #slow down learning to better converge towards an optimum with increased network performance. 235 | learningFactor = 1.-acc 236 | print(' Adjusting learning rate to {0} ~ {1}% of its initial value'.format(learningFactor*lrate, np.round(learningFactor*100,2))) 237 | 238 | #refresh number of allowed search steps until convergence 239 | untilConvergence = convergence 240 | else: 241 | untilConvergence-=1 242 | if untilConvergence == 0 and convergence > 0: 243 | print(' No more recorded model improvements for {0} evaluations. Accepting model convergence.'.format(convergence)) 244 | break 245 | 246 | t_elapsed = time.time() - t_start 247 | percent_done = float(d+1)/iters #d+1 because we are after the iteration's heavy lifting 248 | t_remaining_estimated = t_elapsed/percent_done - t_elapsed 249 | 250 | m, s = divmod(t_remaining_estimated, 60) 251 | h, m = divmod(m, 60) 252 | d, h = divmod(h, 24) 253 | 254 | timestring = '{}d {}h {}m {}s'.format(int(d), int(h), int(m), int(s)) 255 | print(' Estimate time until current training ends : {} ({:.2f}% done)'.format(timestring, percent_done*100)) 256 | 257 | elif (d+1) % (status/10) == 0: 258 | # print 'alive' signal 259 | #sys.stdout.write('.') 260 | l1loss = np.abs(Ypred - Y[samples,:]).sum()/Ypred.shape[0] 261 | sys.stdout.write('batch# {}, lrate {}, l1-loss {:.4}\n'.format(d+1,lrate*learningFactor,l1loss)) 262 | sys.stdout.flush() 263 | 264 | #after training, either due to convergence or iteration limit 265 | print('Setting network parameters to best encountered network state with {}% accuracy and a loss of {} from iteration {}.'.format(bestAccuracy*100, bestLoss, bestIter)) 266 | self.modules = bestLayers 267 | 268 | 269 | def set_lrp_parameters(self,lrp_var=None,param=None): 270 | for m in self.modules: 271 | m.set_lrp_parameters(lrp_var=lrp_var,param=param) 272 | 273 | def lrp(self,R,lrp_var=None,param=None): 274 | ''' 275 | Performs LRP by calling subroutines, depending on lrp_var and param or 276 | preset values specified via Module.set_lrp_parameters(lrp_var,lrp_param) 277 | 278 | If lrp parameters have been pre-specified (per layer), the corresponding decomposition 279 | will be applied during a call of lrp(). 280 | 281 | Specifying lrp parameters explicitly when calling lrp(), e.g. net.lrp(R,lrp_var='alpha',param=2.), 282 | will override the preset values for the current call. 283 | 284 | How to use: 285 | 286 | net.forward(X) #forward feed some data you wish to explain to populat the net. 287 | 288 | then either: 289 | 290 | net.lrp() #to perform the naive approach to lrp implemented in _simple_lrp for each layer 291 | 292 | or: 293 | 294 | for m in net.modules: 295 | m.set_lrp_parameters(...) 296 | net.lrp() #to preset a lrp configuration to each layer in the net 297 | 298 | or: 299 | 300 | net.lrp(somevariantname,someparameter) # to explicitly call the specified parametrization for all layers (where applicable) and override any preset configurations. 301 | 302 | Parameters 303 | ---------- 304 | R : numpy.ndarray 305 | final layer relevance values. usually the network's prediction of some data points 306 | for which the output relevance is to be computed 307 | dimensionality should be equal to the previously computed predictions 308 | 309 | lrp_var : str 310 | either 'none' or 'simple' or None for standard Lrp , 311 | 'epsilon' for an added epsilon slack in the denominator 312 | 'alphabeta' or 'alpha' for weighting positive and negative contributions separately. param specifies alpha with alpha + beta = 1 313 | 'flat' projects an upper layer neuron's relevance uniformly over its receptive field. 314 | 'ww' or 'w^2' only considers the square weights w_ij^2 as qantities to distribute relevances with. 315 | 316 | param : double 317 | the respective parameter for the lrp method of choice 318 | 319 | Returns 320 | ------- 321 | 322 | R : numpy.ndarray 323 | the first layer relevances as produced by the neural net wrt to the previously forward 324 | passed input data. dimensionality is equal to the previously into forward entered input data 325 | 326 | Note 327 | ---- 328 | 329 | Requires the net to be populated with temporary variables, i.e. forward needed to be called with the input 330 | for which the explanation is to be computed. calling clean in between forward and lrp invalidates the 331 | temporary data 332 | ''' 333 | 334 | all_layer_relevances = [] 335 | for m in self.modules[::-1]: 336 | R = m.lrp(R,lrp_var,param) 337 | all_layer_relevances.append(R) 338 | 339 | return R, all_layer_relevances[:-1][::-1] 340 | -------------------------------------------------------------------------------- /lrp_toolbox/modules/softmax.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | 15 | # ------------------------------- 16 | # Softmax layer 17 | # ------------------------------- 18 | class SoftMax(Module): 19 | ''' 20 | Softmax Layer 21 | ''' 22 | 23 | def __init__(self): 24 | Module.__init__(self) 25 | 26 | def forward(self,X,*args,**kwargs): 27 | self.X = X 28 | self.Y = np.exp(X) / np.exp(X).sum(axis=1,keepdims=True) 29 | return self.Y 30 | 31 | 32 | def lrp(self,R,*args,**kwargs): 33 | # just propagate R further down. 34 | # makes sure subroutines never get called. 35 | #return R*self.X 36 | return R 37 | 38 | def clean(self): 39 | self.X = None 40 | self.Y = None -------------------------------------------------------------------------------- /lrp_toolbox/modules/sumpool.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | 15 | # ------------------------------- 16 | # Sum Pooling layer 17 | # ------------------------------- 18 | 19 | class SumPool(Module): 20 | 21 | def __init__(self,pool=(2,2),stride=(2,2)): 22 | ''' 23 | Constructor for the sum pooling layer object 24 | 25 | Parameters 26 | ---------- 27 | 28 | pool : tuple (h,w) 29 | the size of the pooling mask in vertical (h) and horizontal (w) direction 30 | 31 | stride : tuple (h,w) 32 | the vertical (h) and horizontal (w) step sizes between filter applications. 33 | ''' 34 | 35 | Module.__init__(self) 36 | 37 | self.pool = pool 38 | self.stride = stride 39 | 40 | def forward(self,X,*args,**kwargs): 41 | ''' 42 | Realizes the forward pass of an input through the sum pooling layer. 43 | 44 | Parameters 45 | ---------- 46 | X : numpy.ndarray 47 | a network input, shaped (N,H,W,D), with 48 | N = batch size 49 | H, W, D = input size in heigth, width, depth 50 | 51 | Returns 52 | ------- 53 | Y : numpy.ndarray 54 | the sum-pooled outputs, reduced in size due to given stride and pooling size 55 | ''' 56 | 57 | self.X = X 58 | N,H,W,D = X.shape 59 | 60 | hpool, wpool = self.pool 61 | hstride, wstride = self.stride 62 | 63 | #assume the given pooling and stride parameters are carefully chosen. 64 | Hout = int((H - hpool) / hstride + 1) 65 | Wout = int((W - wpool) / wstride + 1) 66 | 67 | normalizer = 1./np.sqrt(hpool*wpool) 68 | 69 | #initialize pooled output 70 | self.Y = np.zeros((N,Hout,Wout,D)) 71 | 72 | for i in range(Hout): 73 | for j in range(Wout): 74 | self.Y[:,i,j,:] = X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ].sum(axis=(1,2)) * normalizer #normalizer to keep the output well conditioned 75 | return self.Y 76 | 77 | 78 | def backward(self,DY): 79 | ''' 80 | Backward-passes an input error gradient DY towards the input neurons of this sum pooling layer. 81 | 82 | Parameters 83 | ---------- 84 | 85 | DY : numpy.ndarray 86 | an error gradient shaped same as the output array of forward, i.e. (N,Hy,Wy,Dy) with 87 | N = number of samples in the batch 88 | Hy = heigth of the output 89 | Wy = width of the output 90 | Dy = output depth = input depth 91 | 92 | 93 | Returns 94 | ------- 95 | 96 | DX : numpy.ndarray 97 | the error gradient propagated towards the input 98 | 99 | ''' 100 | 101 | # DY is of shape N, Hout, Wout, nfilters 102 | N,H,W,D = self.X.shape 103 | 104 | hpool, wpool = self.pool 105 | hstride, wstride = self.stride 106 | 107 | #assume the given pooling and stride parameters are carefully chosen. 108 | Hout = int((H - hpool) / hstride + 1) 109 | Wout = int((W - wpool) / wstride + 1) 110 | 111 | normalizer = 1./np.sqrt(hpool * wpool) 112 | 113 | #distribute the gradient (1 * DY) towards across all contributing inputs evenly 114 | DX = np.zeros_like(self.X) 115 | for i in range(Hout): 116 | for j in range(Wout): 117 | DX[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += DY[:,i:i+1,j:j+1,:] * normalizer # normalizer to produce well-conditioned gradients 118 | return DX 119 | 120 | 121 | def clean(self): 122 | self.X = None 123 | self.Y = None 124 | 125 | def _simple_lrp_slow(self,R): 126 | ''' 127 | LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140 128 | ''' 129 | N,H,W,D = self.X.shape 130 | 131 | hpool, wpool = self.pool 132 | hstride, wstride = self.stride 133 | 134 | #assume the given pooling and stride parameters are carefully chosen. 135 | Hout = int((H - hpool) / hstride + 1) 136 | Wout = int((W - wpool) / wstride + 1) 137 | 138 | Rx = np.zeros(self.X.shape) 139 | for i in range(Hout): 140 | for j in range(Wout): 141 | Z = self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] #input activations. 142 | Zs = Z.sum(axis=(1,2),keepdims=True) 143 | Zs += 1e-12*((Zs >= 0)*2-1) # add a weak numerical stabilizer to cushion an all-zero input 144 | 145 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += (Z/Zs) * R[:,i:i+1,j:j+1,:] #distribute relevance propoprtional to input activations per layer 146 | 147 | return Rx 148 | 149 | 150 | def _simple_lrp(self,R): 151 | ''' 152 | LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140 153 | ''' 154 | N,H,W,D = self.X.shape 155 | 156 | hpool, wpool = self.pool 157 | hstride, wstride = self.stride 158 | 159 | #assume the given pooling and stride parameters are carefully chosen. 160 | Hout = int((H - hpool) / hstride + 1) 161 | Wout = int((W - wpool) / wstride + 1) 162 | 163 | Rx = np.zeros(self.X.shape) 164 | normalizer = 1./np.sqrt(hpool*wpool) 165 | R_norm = R / (self.Y/normalizer + 1e-12*((self.Y/normalizer >= 0)*2 - 1.)) #factor in normalizer applied to Y in the forward pass 166 | 167 | 168 | for i in range(Hout): 169 | for j in range(Wout): 170 | Z = self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] #input activations. 171 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += Z * (R_norm[:,i:i+1,j:j+1,:]) 172 | 173 | return Rx 174 | 175 | 176 | def _flat_lrp(self,R): 177 | ''' 178 | distribute relevance for each output evenly to the output neurons' receptive fields. 179 | ''' 180 | N,H,W,D = self.X.shape 181 | 182 | hpool, wpool = self.pool 183 | hstride, wstride = self.stride 184 | 185 | #assume the given pooling and stride parameters are carefully chosen. 186 | Hout = int((H - hpool) / hstride + 1) 187 | Wout = int((W - wpool) / wstride + 1) 188 | 189 | Rx = np.zeros_like(self.X,dtype=np.float) 190 | 191 | for i in range(Hout): 192 | for j in range(Wout): 193 | Z = np.ones([N,hpool,wpool,D]) 194 | Zs = Z.sum(axis=(1,2),keepdims=True) 195 | Rx[:,i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool,:] += (Z / Zs) * R[:,i:i+1,j:j+1,:] 196 | return Rx 197 | 198 | def _ww_lrp(self,R): 199 | ''' 200 | due to uniform weights used for sum pooling (1), this method defaults to _flat_lrp(R) 201 | ''' 202 | return self._flat_lrp(R) 203 | 204 | 205 | def _epsilon_lrp_slow(self,R,epsilon): 206 | ''' 207 | LRP according to Eq(58) in DOI: 10.1371/journal.pone.0130140 208 | ''' 209 | N,H,W,D = self.X.shape 210 | 211 | hpool, wpool = self.pool 212 | hstride, wstride = self.stride 213 | 214 | #assume the given pooling and stride parameters are carefully chosen. 215 | Hout = int((H - hpool) / hstride + 1) 216 | Wout = int((W - wpool) / wstride + 1) 217 | 218 | Rx = np.zeros(self.X.shape) 219 | for i in range(Hout): 220 | for j in range(Wout): 221 | Z = self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] #input activations. 222 | Zs = Z.sum(axis=(1,2),keepdims=True) 223 | Zs += epsilon*((Zs >= 0)*2-1) # add a epsilon stabilizer to cushion an all-zero input 224 | 225 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += (Z/Zs) * R[:,i:i+1,j:j+1,:] #distribute relevance propoprtional to input activations per layer 226 | 227 | return Rx 228 | 229 | 230 | def _epsilon_lrp(self,R,epsilon): 231 | ''' 232 | LRP according to Eq(58) in DOI: 10.1371/journal.pone.0130140 233 | ''' 234 | N,H,W,D = self.X.shape 235 | 236 | hpool, wpool = self.pool 237 | hstride, wstride = self.stride 238 | 239 | #assume the given pooling and stride parameters are carefully chosen. 240 | Hout = int((H - hpool) / hstride + 1) 241 | Wout = int((W - wpool) / wstride + 1) 242 | 243 | Rx = np.zeros(self.X.shape) 244 | normalizer = 1./np.sqrt(hpool*wpool) #factor in normalizer applied to Y in the forward pass 245 | R_norm = R / (self.Y/normalizer + epsilon*((self.Y >= 0)*2 - 1.)) 246 | 247 | for i in range(Hout): 248 | for j in range(Wout): 249 | Z = self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] #input activations. 250 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += Z * (R_norm[:,i:i+1,j:j+1,:]) 251 | 252 | return Rx 253 | 254 | 255 | 256 | #def _epsilon_lrp(self,R,epsilon): 257 | # return self._epsilon_lrp_slow(R,epsilon) 258 | 259 | 260 | 261 | 262 | 263 | # yes, we can do this. no, it will not make sense most of the time. by default, _lrp_simple will be called. see line 152 264 | def _alphabeta_lrp_slow(self,R,alpha): 265 | ''' 266 | LRP according to Eq(60) in DOI: 10.1371/journal.pone.0130140 267 | ''' 268 | 269 | beta = 1-alpha 270 | 271 | N,H,W,D = self.X.shape 272 | 273 | hpool, wpool = self.pool 274 | hstride, wstride = self.stride 275 | 276 | #assume the given pooling and stride parameters are carefully chosen. 277 | Hout = int((H - hpool) / hstride + 1) 278 | Wout = int((W - wpool) / wstride + 1) 279 | 280 | #distribute the gradient towards across all inputs evenly 281 | Rx = np.zeros(self.X.shape) 282 | for i in range(Hout): 283 | for j in range(Wout): 284 | Z = self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] #input activations. 285 | 286 | if not alpha == 0: 287 | Zp = Z * (Z > 0) 288 | Zsp = Zp.sum(axis=(1,2),keepdims=True) + 1e-16 #zero division is quite likely in sum pooling layers when using the alpha-variant 289 | Ralpha = alpha * (Zp/Zsp) * R[:,i:i+1,j:j+1,:] 290 | else: 291 | Ralpha = 0 292 | 293 | if not beta == 0: 294 | Zn = Z * (Z < 0) 295 | Zsn = Zn.sum(axis=(1,2),keepdims=True) - 1e-16 #zero division is quite likely in sum pooling layers when using the alpha-variant 296 | Rbeta = beta * (Zn/Zsn) * R[:,i:i+1,j:j+1,:] 297 | else: 298 | Rbeta = 0 299 | 300 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += Ralpha + Rbeta 301 | 302 | return Rx 303 | 304 | def _alphabeta_lrp(self, R, alpha): 305 | ''' 306 | LRP according to Eq(60) in DOI: 10.1371/journal.pone.0130140 307 | ''' 308 | 309 | beta = 1-alpha 310 | 311 | N,H,W,D = self.X.shape 312 | 313 | hpool, wpool = self.pool 314 | hstride, wstride = self.stride 315 | 316 | #assume the given pooling and stride parameters are carefully chosen. 317 | Hout = int((H - hpool) / hstride + 1) 318 | Wout = int((W - wpool) / wstride + 1) 319 | normalizer = 1./np.sqrt(hpool*wpool) #factor in normalizer applied to Y in the forward pass 320 | 321 | #distribute the gradient towards across all inputs evenly 322 | Rx = np.zeros(self.X.shape) 323 | for i in range(Hout): 324 | for j in range(Wout): 325 | Z = self.X[:, i*hstride:i*hstride+hpool , j*wstride:j*wstride+wpool , : ] #input activations. 326 | Zplus = Z > 0 #index mask of positive forward predictions 327 | 328 | if alpha * beta != 0 : #the general case: both parameters are not 0 329 | Zp = Z * Zplus 330 | Zsp = Zp.sum(axis=(1,2),keepdims=True)+ 1e-16 331 | 332 | Zn = Z - Zp 333 | Zsn = self.Y[:,i:i+1,j:j+1,:]/normalizer - Zsp - 1e-16 334 | 335 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += (alpha*(Zp/Zsp) + beta*Zn*(R[:,i:i+1,j:j+1,:]/Zsn))*R[:,i:i+1,j:j+1,:] 336 | 337 | elif alpha: #only alpha is not 0 -> alpha = 1, beta = 0 338 | Zp = Z * Zplus 339 | Zsp = Zp.sum(axis=(1,2),keepdims=True) + 1e-16 340 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += Zp*(R[:,i:i+1,j:j+1,:]/Zsp) 341 | 342 | elif beta: # only beta is not 0 -> alpha = 0, beta = 1 343 | Zn = Z * np.invert(Zplus) 344 | Zsn = Zn.sum(axis=(1,2),keepdims=True) - 1e-16 345 | Rx[:,i*hstride:i*hstride+hpool: , j*wstride:j*wstride+wpool: , : ] += Zn*(R[:,i:i+1,j:j+1,:]/Zsn) 346 | 347 | else: 348 | raise Exception('This case should never occur: alpha={}, beta={}.'.format(alpha, beta)) 349 | 350 | return Rx 351 | -------------------------------------------------------------------------------- /lrp_toolbox/modules/tanh.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @author: Gregoire Montavon 4 | @maintainer: Sebastian Lapuschkin 5 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 6 | @date: 14.08.2015 7 | @version: 1.2+ 8 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 9 | @license : BSD-2-Clause 10 | ''' 11 | 12 | import numpy as np 13 | from .module import Module 14 | 15 | # ------------------------------- 16 | # Tanh layer 17 | # ------------------------------- 18 | class Tanh(Module): 19 | ''' 20 | Tanh Layer 21 | ''' 22 | 23 | def __init__(self): 24 | Module.__init__(self) 25 | 26 | def forward(self,X,*args,**kwargs): 27 | self.Y = np.tanh(X) 28 | return self.Y 29 | 30 | 31 | def backward(self,DY): 32 | return DY*(1.0-self.Y**2) 33 | 34 | 35 | def clean(self): 36 | self.Y = None 37 | 38 | def lrp(self,R,*args,**kwargs): 39 | # component-wise operations within this layer 40 | # -> 41 | # just propagate R further down. 42 | # makes sure subroutines never get called. 43 | return R -------------------------------------------------------------------------------- /lrp_toolbox/render.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 14.08.2015 6 | @version: 1.2+ 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | 10 | provides methods to draw heatmaps beautifully. 11 | ''' 12 | 13 | import numpy as np 14 | import matplotlib.cm 15 | from matplotlib.cm import ScalarMappable 16 | import skimage.io 17 | try: 18 | from skimage.feature import canny 19 | except: 20 | from skimage.filter import canny 21 | 22 | 23 | def vec2im(V, shape = () ): 24 | ''' 25 | Transform an array V into a specified shape - or if no shape is given assume a square output format. 26 | 27 | Parameters 28 | ---------- 29 | 30 | V : numpy.ndarray 31 | an array either representing a matrix or vector to be reshaped into an two-dimensional image 32 | 33 | shape : tuple or list 34 | optional. containing the shape information for the output array if not given, the output is assumed to be square 35 | 36 | Returns 37 | ------- 38 | 39 | W : numpy.ndarray 40 | with W.shape = shape or W.shape = [np.sqrt(V.size)]*2 41 | 42 | ''' 43 | 44 | if len(shape) < 2: 45 | shape = [int(np.sqrt(V.size))]*2 46 | 47 | return np.reshape(V, shape) 48 | 49 | 50 | def enlarge_image(img, scaling = 3): 51 | ''' 52 | Enlarges a given input matrix by replicating each pixel value scaling times in horizontal and vertical direction. 53 | 54 | Parameters 55 | ---------- 56 | 57 | img : numpy.ndarray 58 | array of shape [H x W] OR [H x W x D] 59 | 60 | scaling : int 61 | positive integer value > 0 62 | 63 | Returns 64 | ------- 65 | 66 | out : numpy.ndarray 67 | two-dimensional array of shape [scaling*H x scaling*W] 68 | OR 69 | three-dimensional array of shape [scaling*H x scaling*W x D] 70 | depending on the dimensionality of the input 71 | ''' 72 | 73 | if scaling < 1 or not isinstance(scaling,int): 74 | print('scaling factor needs to be an int >= 1') 75 | 76 | if len(img.shape) == 2: 77 | H,W = img.shape 78 | 79 | out = np.zeros((scaling*H, scaling*W)) 80 | for h in range(H): 81 | fh = scaling*h 82 | for w in range(W): 83 | fw = scaling*w 84 | out[fh:fh+scaling, fw:fw+scaling] = img[h,w] 85 | 86 | elif len(img.shape) == 3: 87 | H,W,D = img.shape 88 | 89 | out = np.zeros((scaling*H, scaling*W,D)) 90 | for h in range(H): 91 | fh = scaling*h 92 | for w in range(W): 93 | fw = scaling*w 94 | out[fh:fh+scaling, fw:fw+scaling,:] = img[h,w,:] 95 | 96 | return out 97 | 98 | 99 | def repaint_corner_pixels(rgbimg, scaling = 3): 100 | ''' 101 | DEPRECATED/OBSOLETE. 102 | 103 | Recolors the top left and bottom right pixel (groups) with the average rgb value of its three neighboring pixel (groups). 104 | The recoloring visually masks the opposing pixel values which are a product of stabilizing the scaling. 105 | Assumes those image ares will pretty much never show evidence. 106 | 107 | Parameters 108 | ---------- 109 | 110 | rgbimg : numpy.ndarray 111 | array of shape [H x W x 3] 112 | 113 | scaling : int 114 | positive integer value > 0 115 | 116 | Returns 117 | ------- 118 | 119 | rgbimg : numpy.ndarray 120 | three-dimensional array of shape [scaling*H x scaling*W x 3] 121 | ''' 122 | 123 | 124 | #top left corner. 125 | rgbimg[0:scaling,0:scaling,:] = (rgbimg[0,scaling,:] + rgbimg[scaling,0,:] + rgbimg[scaling, scaling,:])/3.0 126 | #bottom right corner 127 | rgbimg[-scaling:,-scaling:,:] = (rgbimg[-1,-1-scaling, :] + rgbimg[-1-scaling, -1, :] + rgbimg[-1-scaling,-1-scaling,:])/3.0 128 | return rgbimg 129 | 130 | 131 | def digit_to_rgb(X, scaling=3, shape = (), cmap = 'binary'): 132 | ''' 133 | Takes as input an intensity array and produces a rgb image due to some color map 134 | 135 | Parameters 136 | ---------- 137 | 138 | X : numpy.ndarray 139 | intensity matrix as array of shape [M x N] 140 | 141 | scaling : int 142 | optional. positive integer value > 0 143 | 144 | shape: tuple or list of its , length = 2 145 | optional. if not given, X is reshaped to be square. 146 | 147 | cmap : str 148 | name of color map of choice. default is 'binary' 149 | 150 | Returns 151 | ------- 152 | 153 | image : numpy.ndarray 154 | three-dimensional array of shape [scaling*H x scaling*W x 3] , where H*W == M*N 155 | ''' 156 | 157 | #create color map object from name string 158 | cmap = eval('matplotlib.cm.{}'.format(cmap)) 159 | 160 | image = enlarge_image(vec2im(X,shape), scaling) #enlarge 161 | image = cmap(image.flatten())[...,0:3].reshape([image.shape[0],image.shape[1],3]) #colorize, reshape 162 | 163 | return image 164 | 165 | 166 | 167 | def hm_to_rgb(R, X = None, scaling = 3, shape = (), sigma = 2, cmap = 'jet', normalize = True): 168 | ''' 169 | Takes as input an intensity array and produces a rgb image for the represented heatmap. 170 | optionally draws the outline of another input on top of it. 171 | 172 | Parameters 173 | ---------- 174 | 175 | R : numpy.ndarray 176 | the heatmap to be visualized, shaped [M x N] 177 | 178 | X : numpy.ndarray 179 | optional. some input, usually the data point for which the heatmap R is for, which shall serve 180 | as a template for a black outline to be drawn on top of the image 181 | shaped [M x N] 182 | 183 | scaling: int 184 | factor, on how to enlarge the heatmap (to control resolution and as a inverse way to control outline thickness) 185 | after reshaping it using shape. 186 | 187 | shape: tuple or list, length = 2 188 | optional. if not given, X is reshaped to be square. 189 | 190 | sigma : double 191 | optional. sigma-parameter for the canny algorithm used for edge detection. the found edges are drawn as outlines. 192 | 193 | cmap : str 194 | optional. color map of choice 195 | 196 | normalize : bool 197 | optional. whether to normalize the heatmap to [-1 1] prior to colorization or not. 198 | 199 | Returns 200 | ------- 201 | 202 | rgbimg : numpy.ndarray 203 | three-dimensional array of shape [scaling*H x scaling*W x 3] , where H*W == M*N 204 | ''' 205 | 206 | R = enlarge_image(vec2im(R,shape), scaling) 207 | 208 | if cmap in custom_maps: 209 | rgb = custom_maps[cmap](R) 210 | else: 211 | if normalize: 212 | R = R / np.max(np.abs(R)) # normalize to [-1,1] wrt to max relevance magnitude 213 | R = (R + 1.)/2. # shift/normalize to [0,1] for color mapping 214 | 215 | #create color map object from name string 216 | cmap = eval('matplotlib.cm.{}'.format(cmap)) 217 | 218 | # apply colormap 219 | rgb = cmap(R.flatten())[...,0:3].reshape([R.shape[0],R.shape[1],3]) 220 | #rgb = repaint_corner_pixels(rgb, scaling) #obsolete due to directly calling the color map with [0,1]-normalized inputs 221 | 222 | if not X is None: #compute the outline of the input 223 | X = enlarge_image(vec2im(X,shape), scaling) 224 | xdims = X.shape 225 | Rdims = R.shape 226 | 227 | if not np.all(xdims == Rdims): 228 | print('transformed heatmap and data dimension mismatch. data dimensions differ?') 229 | print('R.shape = ',Rdims, 'X.shape = ', xdims) 230 | print('skipping drawing of outline\n') 231 | else: 232 | edges = canny(X, sigma=sigma) 233 | edges = np.invert(np.dstack([edges]*3))*1.0 234 | rgb *= edges # set outline pixels to black color 235 | 236 | return rgb 237 | 238 | 239 | def save_image(rgb_images, path, gap = 2): 240 | ''' 241 | Takes as input a list of rgb images, places them next to each other with a gap and writes out the result. 242 | 243 | Parameters 244 | ---------- 245 | 246 | rgb_images : list , tuple, collection. such stuff 247 | each item in the collection is expected to be an rgb image of dimensions [H x _ x 3] 248 | where the width is variable 249 | 250 | path : str 251 | the output path of the assembled image 252 | 253 | gap : int 254 | optional. sets the width of a black area of pixels realized as an image shaped [H x gap x 3] in between the input images 255 | 256 | Returns 257 | ------- 258 | 259 | image : numpy.ndarray 260 | the assembled image as written out to path 261 | ''' 262 | 263 | sz = [] 264 | image = [] 265 | for i in range(len(rgb_images)): 266 | if not sz: 267 | sz = rgb_images[i].shape 268 | image = rgb_images[i] 269 | gap = np.zeros((sz[0],gap,sz[2])) 270 | continue 271 | if not sz[0] == rgb_images[i].shape[0] and sz[1] == rgb_images[i].shape[2]: 272 | print('image',i, 'differs in size. unable to perform horizontal alignment') 273 | print('expected: Hx_xD = {0}x_x{1}'.format(sz[0],sz[1])) 274 | print('got : Hx_xD = {0}x_x{1}'.format(rgb_images[i].shape[0],rgb_images[i].shape[1])) 275 | print('skipping image\n') 276 | else: 277 | image = np.hstack((image,gap,rgb_images[i])) 278 | 279 | image *= 255 280 | image = image.astype(np.uint8) 281 | 282 | print('saving image to ', path) 283 | skimage.io.imsave(path,image) 284 | return image 285 | 286 | 287 | # ################## # 288 | # custom color maps: # 289 | # ################## # 290 | 291 | def gregoire_gray_red(R): 292 | basegray = 0.8 #floating point gray 293 | 294 | maxabs = np.max(R) 295 | RGB = np.ones([R.shape[0], R.shape[1],3]) * basegray #uniform gray image. 296 | 297 | tvals = np.maximum(np.minimum(R/maxabs,1.0),-1.0) 298 | negatives = R < 0 299 | 300 | RGB[negatives,0] += tvals[negatives]*basegray 301 | RGB[negatives,1] += tvals[negatives]*basegray 302 | RGB[negatives,2] += -tvals[negatives]*(1-basegray) 303 | 304 | positives = R>=0 305 | RGB[positives,0] += tvals[positives]*(1-basegray) 306 | RGB[positives,1] += -tvals[positives]*basegray 307 | RGB[positives,2] += -tvals[positives]*basegray 308 | 309 | return RGB 310 | 311 | 312 | def gregoire_black_green(R): 313 | maxabs = np.max(R) 314 | RGB = np.zeros([R.shape[0], R.shape[1],3]) 315 | 316 | negatives = R<0 317 | RGB[negatives,2] = -R[negatives]/maxabs 318 | 319 | positives = R>=0 320 | RGB[positives,1] = R[positives]/maxabs 321 | 322 | return RGB 323 | 324 | 325 | def gregoire_black_firered(R): 326 | R = R / np.max(np.abs(R)) 327 | x = R 328 | 329 | hrp = np.clip(x-0.00,0,0.25)/0.25 330 | hgp = np.clip(x-0.25,0,0.25)/0.25 331 | hbp = np.clip(x-0.50,0,0.50)/0.50 332 | 333 | hbn = np.clip(-x-0.00,0,0.25)/0.25 334 | hgn = np.clip(-x-0.25,0,0.25)/0.25 335 | hrn = np.clip(-x-0.50,0,0.50)/0.50 336 | 337 | return np.concatenate([(hrp+hrn)[...,None],(hgp+hgn)[...,None],(hbp+hbn)[...,None]],axis = 2) 338 | 339 | 340 | def gregoire_gray_red2(R): 341 | v = np.var(R) 342 | R[R > 10*v] = 0 343 | R[R<0] = 0 344 | R = R / np.max(R) 345 | #(this is copypasta) 346 | x=R 347 | 348 | # positive relevance 349 | hrp = 0.9 - np.clip(x-0.3,0,0.7)/0.7*0.5 350 | hgp = 0.9 - np.clip(x-0.0,0,0.3)/0.3*0.5 - np.clip(x-0.3,0,0.7)/0.7*0.4 351 | hbp = 0.9 - np.clip(x-0.0,0,0.3)/0.3*0.5 - np.clip(x-0.3,0,0.7)/0.7*0.4 352 | 353 | # negative relevance 354 | hrn = 0.9 - np.clip(-x-0.0,0,0.3)/0.3*0.5 - np.clip(-x-0.3,0,0.7)/0.7*0.4 355 | hgn = 0.9 - np.clip(-x-0.0,0,0.3)/0.3*0.5 - np.clip(-x-0.3,0,0.7)/0.7*0.4 356 | hbn = 0.9 - np.clip(-x-0.3,0,0.7)/0.7*0.5 357 | 358 | hr = hrp*(x>=0)+hrn*(x<0) 359 | hg = hgp*(x>=0)+hgn*(x<0) 360 | hb = hbp*(x>=0)+hbn*(x<0) 361 | 362 | 363 | return np.concatenate([hr[...,None],hg[...,None],hb[...,None]],axis=2) 364 | 365 | 366 | 367 | def alex_black_yellow(R): 368 | 369 | maxabs = np.max(R) 370 | RGB = np.zeros([R.shape[0], R.shape[1],3]) 371 | 372 | negatives = R<0 373 | RGB[negatives,2] = -R[negatives]/maxabs 374 | 375 | positives = R>=0 376 | RGB[positives,0] = R[positives]/maxabs 377 | RGB[positives,1] = R[positives]/maxabs 378 | 379 | return RGB 380 | 381 | 382 | #list of supported color map names. the maps need to be implemented ABOVE this line because of PYTHON 383 | custom_maps = {'gray-red':gregoire_gray_red,\ 384 | 'gray-red2':gregoire_gray_red2,\ 385 | 'black-green':gregoire_black_green,\ 386 | 'black-firered':gregoire_black_firered,\ 387 | 'blue-black-yellow':alex_black_yellow} 388 | -------------------------------------------------------------------------------- /lrp_toolbox/training_test.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @author: Sebastian Lapuschkin 3 | @maintainer: Sebastian Lapuschkin 4 | @contact: sebastian.lapuschkin@hhi.fraunhofer.de, wojciech.samek@hhi.fraunhofer.de 5 | @date: 30.09.2015 6 | @version: 1.0 7 | @copyright: Copyright (c) 2015-2017, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller, Wojciech Samek 8 | @license : BSD-2-Clause 9 | ''' 10 | 11 | import modules 12 | import model_io 13 | 14 | import numpy as np ; na = np.newaxis 15 | 16 | D,N = 2,200000 17 | 18 | #this is the XOR problem. 19 | X = np.random.rand(N,D) #we want [NxD] data 20 | X = (X > 0.5)*1.0 21 | Y = X[:,0] == X[:,1] 22 | Y = (np.vstack((Y, np.invert(Y)))*1.0).T # and [NxC] labels 23 | 24 | X += np.random.randn(N,D)*0.1 # add some noise to the data. 25 | 26 | #build a network 27 | nn = modules.Sequential([modules.Linear(2,3), modules.Tanh(),modules.Linear(3,15), modules.Tanh(), modules.Linear(15,15), modules.Tanh(), modules.Linear(15,3), modules.Tanh() ,modules.Linear(3,2), modules.SoftMax()]) 28 | #train the network. 29 | nn.train(X,Y,Xval=X,Yval=Y, batchsize = 5) 30 | 31 | #save the network 32 | model_io.write(nn, '../xor_net_small_1000.txt') 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /neural_networks/LeNet1.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/neural_networks/LeNet1.h5 -------------------------------------------------------------------------------- /neural_networks/LeNet1.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "keras_version": "2.2.4", "config": {"layers": [{"class_name": "InputLayer", "config": {"dtype": "float32", "batch_input_shape": [null, 28, 28, 1], "name": "input_1", "sparse": false}, "inbound_nodes": [], "name": "input_1"}, {"class_name": "Conv2D", "config": {"kernel_constraint": null, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "block1_conv1", "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "data_format": "channels_last", "padding": "valid", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 4, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [5, 5]}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "name": "block1_conv1"}, {"class_name": "MaxPooling2D", "config": {"name": "block1_pool1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}, "inbound_nodes": [[["block1_conv1", 0, 0, {}]]], "name": "block1_pool1"}, {"class_name": "Conv2D", "config": {"kernel_constraint": null, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "block2_conv1", "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "data_format": "channels_last", "padding": "valid", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 12, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [5, 5]}, "inbound_nodes": [[["block1_pool1", 0, 0, {}]]], "name": "block2_conv1"}, {"class_name": "MaxPooling2D", "config": {"name": "block2_pool1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}, "inbound_nodes": [[["block2_conv1", 0, 0, {}]]], "name": "block2_pool1"}, {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten", "data_format": "channels_last"}, "inbound_nodes": [[["block2_pool1", 0, 0, {}]]], "name": "flatten"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "softmax", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["flatten", 0, 0, {}]]], "name": "softmax"}], "input_layers": [["input_1", 0, 0]], "output_layers": [["softmax", 0, 0]], "name": "model_1"}, "backend": "tensorflow"} -------------------------------------------------------------------------------- /neural_networks/LeNet4.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/neural_networks/LeNet4.h5 -------------------------------------------------------------------------------- /neural_networks/LeNet4.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "keras_version": "2.2.4", "config": {"layers": [{"class_name": "InputLayer", "config": {"dtype": "float32", "batch_input_shape": [null, 28, 28, 1], "name": "input_1", "sparse": false}, "inbound_nodes": [], "name": "input_1"}, {"class_name": "Conv2D", "config": {"kernel_constraint": null, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "block1_conv1", "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "data_format": "channels_last", "padding": "valid", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 6, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [5, 5]}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "name": "block1_conv1"}, {"class_name": "MaxPooling2D", "config": {"name": "block1_pool1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}, "inbound_nodes": [[["block1_conv1", 0, 0, {}]]], "name": "block1_pool1"}, {"class_name": "Conv2D", "config": {"kernel_constraint": null, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "block2_conv1", "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "data_format": "channels_last", "padding": "valid", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 16, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [5, 5]}, "inbound_nodes": [[["block1_pool1", 0, 0, {}]]], "name": "block2_conv1"}, {"class_name": "MaxPooling2D", "config": {"name": "block2_pool1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}, "inbound_nodes": [[["block2_conv1", 0, 0, {}]]], "name": "block2_pool1"}, {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten", "data_format": "channels_last"}, "inbound_nodes": [[["block2_pool1", 0, 0, {}]]], "name": "flatten"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "fc1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 84, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["flatten", 0, 0, {}]]], "name": "fc1"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "softmax", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["fc1", 0, 0, {}]]], "name": "softmax"}], "input_layers": [["input_1", 0, 0]], "output_layers": [["softmax", 0, 0]], "name": "model_1"}, "backend": "tensorflow"} -------------------------------------------------------------------------------- /neural_networks/LeNet5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/neural_networks/LeNet5.h5 -------------------------------------------------------------------------------- /neural_networks/LeNet5.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "keras_version": "2.2.4", "config": {"layers": [{"class_name": "InputLayer", "config": {"dtype": "float32", "batch_input_shape": [null, 28, 28, 1], "name": "input_2", "sparse": false}, "inbound_nodes": [], "name": "input_2"}, {"class_name": "Conv2D", "config": {"kernel_constraint": null, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "block1_conv1", "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "data_format": "channels_last", "padding": "valid", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 6, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [5, 5]}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "name": "block1_conv1"}, {"class_name": "MaxPooling2D", "config": {"name": "block1_pool1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}, "inbound_nodes": [[["block1_conv1", 0, 0, {}]]], "name": "block1_pool1"}, {"class_name": "Conv2D", "config": {"kernel_constraint": null, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "block2_conv1", "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "data_format": "channels_last", "padding": "valid", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 16, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [5, 5]}, "inbound_nodes": [[["block1_pool1", 0, 0, {}]]], "name": "block2_conv1"}, {"class_name": "MaxPooling2D", "config": {"name": "block2_pool1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}, "inbound_nodes": [[["block2_conv1", 0, 0, {}]]], "name": "block2_pool1"}, {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten", "data_format": "channels_last"}, "inbound_nodes": [[["block2_pool1", 0, 0, {}]]], "name": "flatten"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "fc1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 120, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["flatten", 0, 0, {}]]], "name": "fc1"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "fc2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 84, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["fc1", 0, 0, {}]]], "name": "fc2"}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "before_softmax", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}, "inbound_nodes": [[["fc2", 0, 0, {}]]], "name": "before_softmax"}, {"class_name": "Activation", "config": {"activation": "softmax", "trainable": true, "name": "predictions"}, "inbound_nodes": [[["before_softmax", 0, 0, {}]]], "name": "predictions"}], "input_layers": [["input_2", 0, 0]], "output_layers": [["predictions", 0, 0]], "name": "model_2"}, "backend": "tensorflow"} -------------------------------------------------------------------------------- /neural_networks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/neural_networks/__init__.py -------------------------------------------------------------------------------- /neural_networks/cifar_original.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/neural_networks/cifar_original.h5 -------------------------------------------------------------------------------- /neural_networks/dave2.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepImportance/deepimportance_code_release/62bea4004860cdf513e7f7036ddfcff4a7d89064/neural_networks/dave2.h5 -------------------------------------------------------------------------------- /neural_networks/dave_model.py: -------------------------------------------------------------------------------- 1 | # usage: python driving_models.py 1 - train the dave-orig model 2 | 3 | from __future__ import print_function 4 | 5 | import sys 6 | 7 | from keras.layers import Convolution2D, Input, Dense, Flatten, Lambda, MaxPooling2D, Dropout 8 | import tensorflow as tf 9 | from keras.models import Model 10 | 11 | #from data_utils import load_train_data, load_test_data 12 | from utils import * 13 | 14 | def atan_layer(x): 15 | return tf.multiply(tf.atan(x), 2) 16 | 17 | def atan_layer_shape(input_shape): 18 | return input_shape 19 | 20 | def Dave_orig(input_tensor=None, load_weights=False): # original dave 21 | if input_tensor is None: 22 | input_tensor = Input(shape=(100, 100, 3)) 23 | x = Convolution2D(24, (5, 5), padding='valid', activation='relu', strides=(2, 2), name='block1_conv1')(input_tensor) 24 | x = Convolution2D(36, (5, 5), padding='valid', activation='relu', strides=(2, 2), name='block1_conv2')(x) 25 | x = Convolution2D(48, (5, 5), padding='valid', activation='relu', strides=(2, 2), name='block1_conv3')(x) 26 | x = Convolution2D(64, (3, 3), padding='valid', activation='relu', strides=(1, 1), name='block1_conv4')(x) 27 | x = Convolution2D(64, (3, 3), padding='valid', activation='relu', strides=(1, 1), name='block1_conv5')(x) 28 | x = Flatten(name='flatten')(x) 29 | x = Dense(1164, activation='relu', name='fc1')(x) 30 | x = Dense(100, activation='relu', name='fc2')(x) 31 | x = Dense(50, activation='relu', name='fc3')(x) 32 | x = Dense(10, activation='relu', name='fc4')(x) 33 | x = Dense(1, name='before_prediction')(x) 34 | x = Lambda(atan_layer, output_shape=atan_layer_shape, name='prediction')(x) 35 | 36 | m = Model(input_tensor, x) 37 | if load_weights: 38 | m.load_weights('./neural_networks/dave2.h5') 39 | 40 | # compiling 41 | m.compile(loss='mse', optimizer='adadelta') 42 | return m 43 | 44 | ''' 45 | def Dave_norminit(input_tensor=None, load_weights=False): # original dave with normal initialization 46 | if input_tensor is None: 47 | input_tensor = Input(shape=(100, 100, 3)) 48 | x = Convolution2D(24, (5, 5), padding='valid', activation='relu', strides=(2, 2), 49 | name='block1_conv1')(input_tensor) 50 | x = Convolution2D(36, (5, 5), padding='valid', activation='relu', strides=(2, 2), 51 | name='block1_conv2')(x) 52 | x = Convolution2D(48, (5, 5), padding='valid', activation='relu', strides=(2, 2), 53 | name='block1_conv3')(x) 54 | x = Convolution2D(64, (3, 3), padding='valid', activation='relu', strides=(1, 1), 55 | name='block1_conv4')(x) 56 | x = Convolution2D(64, (3, 3), padding='valid', activation='relu', strides=(1, 1), 57 | name='block1_conv5')(x) 58 | x = Flatten(name='flatten')(x) 59 | x = Dense(1164, kernel_initializer=normal_init, activation='relu', name='fc1')(x) 60 | x = Dense(100, kernel_initializer=normal_init, activation='relu', name='fc2')(x) 61 | x = Dense(50, kernel_initializer=normal_init, activation='relu', name='fc3')(x) 62 | x = Dense(10, kernel_initializer=normal_init, activation='relu', name='fc4')(x) 63 | x = Dense(1, name='before_prediction')(x) 64 | x = Lambda(atan_layer, output_shape=atan_layer_shape, name='prediction')(x) 65 | 66 | m = Model(input_tensor, x) 67 | if load_weights: 68 | m.load_weights('./Model2.h5') 69 | 70 | # compiling 71 | m.compile(loss='mse', optimizer='adadelta') 72 | return m 73 | 74 | 75 | def Dave_dropout(input_tensor=None, load_weights=False): # simplified dave 76 | if input_tensor is None: 77 | input_tensor = Input(shape=(100, 100, 3)) 78 | x = Convolution2D(16, (3, 3), padding='valid', activation='relu', name='block1_conv1')(input_tensor) 79 | x = MaxPooling2D(pool_size=(2, 2), name='block1_pool1')(x) 80 | x = Convolution2D(32, (3, 3), padding='valid', activation='relu', name='block1_conv2')(x) 81 | x = MaxPooling2D(pool_size=(2, 2), name='block1_pool2')(x) 82 | x = Convolution2D(64, (3, 3), padding='valid', activation='relu', name='block1_conv3')(x) 83 | x = MaxPooling2D(pool_size=(2, 2), name='block1_pool3')(x) 84 | x = Flatten(name='flatten')(x) 85 | x = Dense(500, activation='relu', name='fc1')(x) 86 | x = Dropout(.5)(x) 87 | x = Dense(100, activation='relu', name='fc2')(x) 88 | x = Dropout(.25)(x) 89 | x = Dense(20, activation='relu', name='fc3')(x) 90 | x = Dense(1, name='before_prediction')(x) 91 | x = Lambda(atan_layer, output_shape=atan_layer_shape, name="prediction")(x) 92 | 93 | m = Model(input_tensor, x) 94 | if load_weights: 95 | m.load_weights('./Model3.h5') 96 | 97 | # compiling 98 | m.compile(loss='mse', optimizer='adadelta') 99 | return m 100 | 101 | 102 | if __name__ == '__main__': 103 | # train the model 104 | batch_size = 256 105 | nb_epoch = 10 106 | model_name = sys.argv[1] 107 | 108 | if model_name == '1': 109 | model = Dave_orig() 110 | save_model_name = './Model1.h5' 111 | elif model_name == '2': 112 | # K.set_learning_phase(1) 113 | model = Dave_norminit() 114 | save_model_name = './Model2.h5' 115 | elif model_name == '3': 116 | # K.set_learning_phase(1) 117 | model = Dave_dropout() 118 | save_model_name = './Model3.h5' 119 | else: 120 | print('invalid model name, must one of 1, 2 or 3') 121 | 122 | # the data, shuffled and split between train and test sets 123 | train_generator, samples_per_epoch = load_train_data(batch_size=batch_size, shape=(100, 100)) 124 | 125 | # trainig 126 | model.fit_generator(train_generator, 127 | steps_per_epoch=math.ceil(samples_per_epoch * 1. / batch_size), 128 | epochs=nb_epoch, 129 | workers=8, 130 | use_multiprocessing=True) 131 | print( 'Model trained' ) 132 | 133 | # evaluation 134 | K.set_learning_phase(0) 135 | test_generator, samples_per_epoch = load_test_data(batch_size=batch_size, shape=(100, 100)) 136 | model.evaluate_generator(test_generator, 137 | steps=math.ceil(samples_per_epoch * 1. / batch_size)) 138 | # save model 139 | model.save_weights(save_model_name) 140 | ''' 141 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from datetime import datetime 4 | from keras.models import model_from_json, load_model, save_model 5 | 6 | from utils import load_MNIST, load_CIFAR 7 | from utils import filter_val_set, get_trainable_layers 8 | from utils import generate_adversarial, filter_correct_classifications 9 | from coverages.idc import ImportanceDrivenCoverage 10 | from coverages.neuron_cov import NeuronCoverage 11 | from coverages.tkn import DeepGaugeLayerLevelCoverage 12 | from coverages.kmn import DeepGaugePercentCoverage 13 | from coverages.ss import SSCover 14 | from coverages.sa import SurpriseAdequacy 15 | 16 | 17 | __version__ = 0.9 18 | 19 | 20 | def scale(intermediate_layer_output, rmax=1, rmin=0): 21 | X_std = (intermediate_layer_output - intermediate_layer_output.min()) / ( 22 | intermediate_layer_output.max() - intermediate_layer_output.min()) 23 | X_scaled = X_std * (rmax - rmin) + rmin 24 | return X_scaled 25 | 26 | def by_indices(outs, indices): 27 | return [[outs[i][0][indices]] for i in range(len(outs))] 28 | 29 | 30 | def parse_arguments(): 31 | """ 32 | Parse command line argument and construct the DNN 33 | :return: a dictionary comprising the command-line arguments 34 | """ 35 | 36 | # define the program description 37 | text = 'Coverage Analyzer for DNNs' 38 | 39 | # initiate the parser 40 | parser = argparse.ArgumentParser(description=text) 41 | 42 | # add new command-line arguments 43 | parser.add_argument("-V", "--version", help="show program version", 44 | action="version", version="DeepFault %f" % __version__) 45 | parser.add_argument("-M", "--model", help="Path to the model to be loaded.\ 46 | The specified model will be used.")#, required=True) 47 | # choices=['lenet1','lenet4', 'lenet5'], required=True) 48 | parser.add_argument("-DS", "--dataset", help="The dataset to be used (mnist\ 49 | or cifar10).", choices=["mnist","cifar10"])#, required=True) 50 | parser.add_argument("-A", "--approach", help="the approach to be employed \ 51 | to measure coverage", choices=['idc','nc','kmnc', 52 | 'nbc','snac','tknc','ssc', 'lsa', 'dsa']) 53 | parser.add_argument("-C", "--class", help="the selected class", type=int) 54 | parser.add_argument("-Q", "--quantize", help="quantization granularity for \ 55 | combinatorial other_coverage_metrics.", type= int) 56 | parser.add_argument("-L", "--layer", help="the subject layer's index for \ 57 | combinatorial cov. NOTE THAT ONLY TRAINABLE LAYERS CAN \ 58 | BE SELECTED", type= int) 59 | parser.add_argument("-KS", "--k_sections", help="number of sections used in \ 60 | k multisection other_coverage_metrics", type=int) 61 | parser.add_argument("-KN", "--k_neurons", help="number of neurons used in \ 62 | top k neuron other_coverage_metrics", type=int) 63 | parser.add_argument("-RN", "--rel_neurons", help="number of neurons considered\ 64 | as relevant in combinatorial other_coverage_metrics", type=int) 65 | parser.add_argument("-AT", "--act_threshold", help="a threshold value used\ 66 | to consider if a neuron is activated or not.", type=float) 67 | parser.add_argument("-R", "--repeat", help="index of the repeating. (for\ 68 | the cases where you need to run the same experiments \ 69 | multiple times)", type=int) 70 | parser.add_argument("-LOG", "--logfile", help="path to log file") 71 | parser.add_argument("-ADV", "--advtype", help="path to log file") 72 | 73 | 74 | # parse command-line arguments 75 | 76 | 77 | # parse command-line arguments 78 | args = parser.parse_args() 79 | 80 | return vars(args) 81 | 82 | 83 | if __name__ == "__main__": 84 | args = parse_arguments() 85 | model_path = args['model'] if args['model'] else 'neural_networks/LeNet5' 86 | dataset = args['dataset'] if args['dataset'] else 'mnist' 87 | approach = args['approach'] if args['approach'] else 'idc' 88 | num_rel_neurons= args['rel_neurons'] if args['rel_neurons'] else 2 89 | act_threshold = args['act_threshold'] if args['act_threshold'] else 0 90 | top_k = args['k_neurons'] if args['k_neurons'] else 3 91 | k_sect = args['k_sections'] if args['k_sections'] else 1000 92 | selected_class = args['class'] if not args['class']==None else -1 #ALL CLASSES 93 | repeat = args['repeat'] if args['repeat'] else 1 94 | logfile_name = args['logfile'] if args['logfile'] else 'result.log' 95 | quantization_granularity = args['quantize'] if args['quantize'] else 3 96 | adv_type = args['advtype'] if args['advtype'] else 'fgsm' 97 | 98 | logfile = open(logfile_name, 'a') 99 | 100 | #################### 101 | # 0) Load data 102 | if dataset == 'mnist': 103 | X_train, Y_train, X_test, Y_test = load_MNIST(channel_first=False) 104 | img_rows, img_cols = 28, 28 105 | else: 106 | X_train, Y_train, X_test, Y_test = load_CIFAR() 107 | img_rows, img_cols = 32, 32 108 | 109 | if not selected_class == -1: 110 | X_train, Y_train = filter_val_set(selected_class, X_train, Y_train) #Get training input for selected_class 111 | X_test, Y_test = filter_val_set(selected_class, X_test, Y_test) #Get testing input for selected_class 112 | 113 | 114 | 115 | #################### 116 | # 1) Setup the model 117 | model_name = model_path.split('/')[-1] 118 | 119 | try: 120 | json_file = open(model_path + '.json', 'r') #Read Keras model parameters (stored in JSON file) 121 | file_content = json_file.read() 122 | json_file.close() 123 | 124 | model = model_from_json(file_content) 125 | model.load_weights(model_path + '.h5') 126 | 127 | # Compile the model before using 128 | model.compile(loss='categorical_crossentropy', 129 | optimizer='adam', 130 | metrics=['accuracy']) 131 | except: 132 | model = load_model(model_path + '.h5') 133 | 134 | # 2) Load necessary information 135 | trainable_layers = get_trainable_layers(model) 136 | non_trainable_layers = list(set(range(len(model.layers))) - set(trainable_layers)) 137 | print('Trainable layers: ' + str(trainable_layers)) 138 | print('Non trainable layers: ' + str(non_trainable_layers)) 139 | 140 | experiment_folder = 'experiments' 141 | 142 | #Investigate the penultimate layer 143 | subject_layer = args['layer'] if not args['layer'] == None else -1 144 | subject_layer = trainable_layers[subject_layer] 145 | 146 | skip_layers = [0] #SKIP LAYERS FOR NC, KMNC, NBC etc. 147 | for idx, lyr in enumerate(model.layers): 148 | if 'flatten' in lyr.__class__.__name__.lower(): skip_layers.append(idx) 149 | 150 | print("Skipping layers:", skip_layers) 151 | 152 | #################### 153 | # 3) Analyze Coverages 154 | if approach == 'nc': 155 | 156 | nc = NeuronCoverage(model, threshold=.75, skip_layers = skip_layers) #SKIP ONLY INPUT AND FLATTEN LAYERS 157 | coverage, _, _, _, _ = nc.test(X_test) 158 | print("Your test set's coverage is: ", coverage) 159 | 160 | nc.set_measure_state(nc.get_measure_state()) 161 | 162 | elif approach == 'idc': 163 | print("\nRunning IDC for %d relevant neurons" % (num_rel_neurons)) 164 | 165 | X_train_corr, Y_train_corr, _, _, = filter_correct_classifications(model, 166 | X_train, 167 | Y_train) 168 | 169 | idc = ImportanceDrivenCoverage(model, model_name, num_rel_neurons, selected_class, 170 | subject_layer, X_train_corr, Y_train_corr) 171 | 172 | coverage, covered_combinations, max_comb = idc.test(X_test) 173 | print("Analysed %d test inputs" % len(Y_train_corr)) 174 | print("IDC test set coverage: %.2f%% " % (coverage)) 175 | print("Covered combinations: ", len(covered_combinations)) 176 | print("Total combinations: ", max_comb) 177 | 178 | idc.set_measure_state(covered_combinations) 179 | 180 | elif approach == 'kmnc' or approach == 'nbc' or approach == 'snac': 181 | 182 | res = { 183 | 'model_name': model_name, 184 | 'num_section': k_sect, 185 | } 186 | 187 | dg = DeepGaugePercentCoverage(model, k_sect, X_train, None, skip_layers) 188 | score = dg.test(X_test) 189 | 190 | elif approach == 'tknc': 191 | 192 | dg = DeepGaugeLayerLevelCoverage(model, top_k, skip_layers=skip_layers) 193 | orig_coverage, _, _, _, _, orig_incrs = dg.test(X_test) 194 | _, orig_acc = model.evaluate(X_test, Y_test) 195 | 196 | elif approach == 'ssc': 197 | 198 | ss = SSCover(model, skip_layers=non_trainable_layers) 199 | score = ss.test(X_test) 200 | 201 | print("Your test set's coverage is: ", score) 202 | 203 | elif approach == 'lsa' or approach == 'dsa': 204 | upper_bound = 2000 205 | 206 | layer_names = [model.layers[-3].name] 207 | 208 | #for lyr in model.layers: 209 | # layer_names.append(lyr.name) 210 | 211 | sa = SurpriseAdequacy(model, X_train, layer_names, upper_bound, dataset) 212 | sa.test(X_test, approach) 213 | 214 | logfile.close() 215 | 216 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | import os 3 | import h5py 4 | import sys 5 | import datetime 6 | import time 7 | import random 8 | import numpy as np 9 | # import matplotlib.pyplot as plt 10 | from cleverhans.attacks import SaliencyMapMethod, FastGradientMethod, CarliniWagnerL2, BasicIterativeMethod 11 | from cleverhans.utils_keras import KerasModelWrapper 12 | from keras import backend as K 13 | from keras.applications.imagenet_utils import preprocess_input 14 | from keras.datasets import mnist, cifar10 15 | from keras.preprocessing import image 16 | from keras.models import model_from_json 17 | from keras.layers import Input 18 | from keras.utils import np_utils 19 | from keras import models 20 | from lrp_toolbox.model_io import read 21 | from neural_networks.dave_model import Dave_orig 22 | 23 | random.seed(123) 24 | np.random.seed(123) 25 | 26 | 27 | def load_CIFAR(one_hot=True): 28 | (X_train, y_train), (X_test, y_test) = cifar10.load_data() 29 | 30 | if one_hot: 31 | y_train = np_utils.to_categorical(y_train, num_classes=10) 32 | y_test = np_utils.to_categorical(y_test, num_classes=10) 33 | 34 | return X_train, y_train, X_test, y_test 35 | 36 | 37 | def load_MNIST(one_hot=True, channel_first=True): 38 | """ 39 | Load MNIST data 40 | :param channel_first: 41 | :param one_hot: 42 | :return: 43 | """ 44 | # Load data 45 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 46 | 47 | # Preprocess dataset 48 | # Normalization and reshaping of input. 49 | if channel_first: 50 | X_train = X_train.reshape(X_train.shape[0], 1, 28, 28) 51 | X_test = X_test.reshape(X_test.shape[0], 1, 28, 28) 52 | else: 53 | X_train = X_train.reshape(X_train.shape[0], 28, 28, 1) 54 | X_test = X_test.reshape(X_test.shape[0], 28, 28, 1) 55 | 56 | X_train = X_train.astype('float32') 57 | X_test = X_test.astype('float32') 58 | X_train /= 255 59 | X_test /= 255 60 | 61 | if one_hot: 62 | # For output, it is important to change number to one-hot vector. 63 | y_train = np_utils.to_categorical(y_train, num_classes=10) 64 | y_test = np_utils.to_categorical(y_test, num_classes=10) 65 | 66 | return X_train, y_train, X_test, y_test 67 | 68 | 69 | def load_driving_data(path='driving_data/', batch_size=64, shape=(100, 100)): 70 | xs = [] 71 | ys = [] 72 | start_load_time = time.time() 73 | with open(path + 'final_example.csv', 'r') as f: 74 | for i, line in enumerate(f): 75 | if i == 0: 76 | continue 77 | xs.append(path + 'center/' + line.split(',')[0] + '.jpg') 78 | ys.append(float(line.split(',')[1])) 79 | # shuffle list of images 80 | c = list(zip(xs, ys)) 81 | random.shuffle(c) 82 | xs, ys = zip(*c) 83 | 84 | train_xs = xs 85 | train_ys = ys 86 | 87 | return train_xs, train_ys 88 | 89 | 90 | def data_generator(xs, ys, target_size, batch_size=64): 91 | gen_state = 0 92 | while 1: 93 | if gen_state + batch_size > len(xs): 94 | paths = xs[gen_state: len(xs)] 95 | y = ys[gen_state: len(xs)] 96 | X = [preprocess_image(x, target_size)[0] for x in paths] 97 | gen_state = 0 98 | else: 99 | paths = xs[gen_state: gen_state + batch_size] 100 | y = ys[gen_state: gen_state + batch_size] 101 | X = [preprocess_image(x, target_size)[0] for x in paths] 102 | gen_state += batch_size 103 | yield np.array(X), np.array(y) 104 | 105 | 106 | def preprocess_image(img_path, target_size=(100, 100)): 107 | img = image.load_img(img_path, target_size=target_size) 108 | input_img_data = image.img_to_array(img) 109 | input_img_data = np.expand_dims(input_img_data, axis=0) 110 | input_img_data = preprocess_input(input_img_data) 111 | return input_img_data 112 | 113 | 114 | def deprocess_image(x): 115 | x = x.reshape((100, 100, 3)) 116 | # Remove zero-center by mean pixel 117 | x[:, :, 0] += 103.939 118 | x[:, :, 1] += 116.779 119 | x[:, :, 2] += 123.68 120 | # 'BGR'->'RGB' 121 | x = x[:, :, ::-1] 122 | x = np.clip(x, 0, 255).astype('uint8') 123 | return x 124 | 125 | 126 | def load_dave_model(): 127 | # input image dimensions 128 | img_rows, img_cols = 100, 100 129 | input_shape = (img_rows, img_cols, 3) 130 | 131 | # define input tensor as a placeholder 132 | input_tensor = Input(shape=input_shape) 133 | 134 | # load multiple models sharing same input tensor 135 | model = Dave_orig(input_tensor=input_tensor, load_weights=True) 136 | 137 | return model 138 | 139 | 140 | def load_model(model_name): 141 | json_file = open(model_name + '.json', 'r') 142 | loaded_model_json = json_file.read() 143 | json_file.close() 144 | model = model_from_json(loaded_model_json) 145 | # load weights into model 146 | model.load_weights(model_name + '.h5') 147 | 148 | model.compile(loss='categorical_crossentropy', 149 | optimizer='adam', 150 | metrics=['accuracy']) 151 | 152 | print("Model structure loaded from ", model_name) 153 | return model 154 | 155 | 156 | def get_layer_outs_old(model, class_specific_test_set): 157 | inp = model.input # input placeholder 158 | outputs = [layer.output for layer in model.layers] # all layer outputs 159 | functors = [K.function([inp] + [K.learning_phase()], [out]) for out in outputs] # evaluation functions 160 | # Testing 161 | layer_outs = [func([class_specific_test_set, 1.]) for func in functors] 162 | 163 | return layer_outs 164 | 165 | 166 | def get_layer_outs(model, test_input, skip=[]): 167 | inp = model.input # input placeholder 168 | outputs = [layer.output for index, layer in enumerate(model.layers) \ 169 | if index not in skip] 170 | 171 | functors = [K.function([inp], [out]) for out in outputs] # evaluation functions 172 | 173 | layer_outs = [func([test_input]) for func in functors] 174 | 175 | return layer_outs 176 | 177 | 178 | def get_layer_outs_new(model, inputs, skip=[]): 179 | # TODO: FIX LATER. This is done for solving incompatibility in Simos' computer 180 | # It is a shortcut. 181 | # skip.append(0) 182 | evaluater = models.Model(inputs=model.input, 183 | outputs=[layer.output for index, layer in enumerate(model.layers) \ 184 | if index not in skip]) 185 | 186 | # Insert some dummy value in the beginning to avoid messing with layer index 187 | # arrangements in the main flow 188 | # outs = evaluater.predict(inputs) 189 | # outs.insert(0, inputs) 190 | 191 | # return outs 192 | 193 | return evaluater.predict(inputs) 194 | 195 | 196 | def calc_major_func_regions(model, train_inputs, skip=None): 197 | if skip is None: 198 | skip = [] 199 | 200 | outs = get_layer_outs_new(model, train_inputs, skip=skip) 201 | 202 | major_regions = [] 203 | 204 | for layer_index, layer_out in enumerate(outs): # layer_out is output of layer for all inputs 205 | layer_out = layer_out.mean(axis=tuple(i for i in range(1, layer_out.ndim - 1))) 206 | 207 | major_regions.append((layer_out.min(axis=0), layer_out.max(axis=0))) 208 | 209 | return major_regions 210 | 211 | 212 | def get_layer_outputs_by_layer_name(model, test_input, skip=None): 213 | if skip is None: 214 | skip = [] 215 | 216 | inp = model.input # input placeholder 217 | outputs = {layer.name: layer.output for index, layer in enumerate(model.layers) 218 | if (index not in skip and 'input' not in layer.name)} # all layer outputs (except input for functionals) 219 | functors = {name: K.function([inp], [out]) for name, out in outputs.items()} # evaluation functions 220 | 221 | layer_outs = {name: func([test_input]) for name, func in functors.items()} 222 | return layer_outs 223 | 224 | 225 | def get_layer_inputs(model, test_input, skip=None, outs=None): 226 | if skip is None: 227 | skip = [] 228 | 229 | if outs is None: 230 | outs = get_layer_outs(model, test_input) 231 | 232 | inputs = [] 233 | 234 | for i in range(len(outs)): 235 | weights, biases = model.layers[i].get_weights() 236 | 237 | inputs_for_layer = [] 238 | 239 | for input_index in range(len(test_input)): 240 | inputs_for_layer.append( 241 | np.add(np.dot(outs[i - 1][0][input_index] if i > 0 else test_input[input_index], weights), biases)) 242 | 243 | inputs.append(inputs_for_layer) 244 | 245 | return [inputs[i] for i in range(len(inputs)) if i not in skip] 246 | 247 | 248 | def get_python_version(): 249 | if (sys.version_info > (3, 0)): 250 | # Python 3 code in this block 251 | return 3 252 | else: 253 | # Python 2 code in this block 254 | return 2 255 | 256 | 257 | # def show_image(vector): 258 | # img = vector 259 | # plt.imshow(img) 260 | # plt.show() 261 | 262 | 263 | def save_quantization(qtized, filename, group_index): 264 | with h5py.File(filename + '_quantization.h5', 'w') as hf: 265 | group = hf.create_group('group' + str(group_index)) 266 | for i in range(len(qtized)): 267 | group.create_dataset("q" + str(i), data=qtized[i]) 268 | 269 | print("Quantization results saved to %s" % (filename)) 270 | return 271 | 272 | 273 | def load_quantization(filename, group_index): 274 | try: 275 | with h5py.File(filename + '_quantization.h5', 'r') as hf: 276 | group = hf.get('group' + str(group_index)) 277 | i = 0 278 | qtized = [] 279 | while True: 280 | # qtized.append(group.get('q' + str(i)).value) 281 | qtized.append(group.get('q' + str(i))[()]) 282 | i += 1 283 | 284 | except (IOError) as error: 285 | print("Could not open file: ", filename) 286 | sys.exit(-1) 287 | except (AttributeError, TypeError) as error: 288 | print("Quantization results loaded from %s" % (filename)) 289 | return qtized 290 | 291 | 292 | def save_data(data, filename): 293 | with h5py.File(filename + '_dataset.h5', 'w') as hf: 294 | hf.create_dataset("dataset", data=data) 295 | 296 | print("Data saved to %s" % (filename)) 297 | return 298 | 299 | 300 | def load_data(filename): 301 | with h5py.File(filename + '_dataset.h5', 'r') as hf: 302 | dataset = hf["dataset"][:] 303 | 304 | print("Data loaded from %s" % (filename)) 305 | return dataset 306 | 307 | 308 | def save_layerwise_relevances(relevant_neurons, filename): 309 | with h5py.File(filename + '_relevant_neurons.h5', 'w') as hf: 310 | hf.create_dataset("relevant_neurons", 311 | data=relevant_neurons) 312 | print("Relevant neurons saved to %s" % (filename)) 313 | return 314 | 315 | 316 | def load_layerwise_relevances(filename): 317 | with h5py.File(filename + '_relevant_neurons.h5', 318 | 'r') as hf: 319 | relevant_neurons = hf["relevant_neurons"][:] 320 | 321 | print("Layerwise relevances loaded from %s" % (filename)) 322 | 323 | return relevant_neurons 324 | 325 | 326 | def save_perturbed_test(x_perturbed, y_perturbed, filename): 327 | # save X 328 | with h5py.File(filename + '_perturbations_x.h5', 'w') as hf: 329 | hf.create_dataset("x_perturbed", data=x_perturbed) 330 | 331 | # save Y 332 | with h5py.File(filename + '_perturbations_y.h5', 'w') as hf: 333 | hf.create_dataset("y_perturbed", data=y_perturbed) 334 | 335 | print("Layerwise relevances saved to %s" % (filename)) 336 | return 337 | 338 | 339 | def load_perturbed_test(filename): 340 | # read X 341 | with h5py.File(filename + '_perturbations_x.h5', 'r') as hf: 342 | x_perturbed = hf["x_perturbed"][:] 343 | 344 | # read Y 345 | with h5py.File(filename + '_perturbations_y.h5', 'r') as hf: 346 | y_perturbed = hf["y_perturbed"][:] 347 | 348 | return x_perturbed, y_perturbed 349 | 350 | 351 | def save_perturbed_test_groups(x_perturbed, y_perturbed, filename, group_index): 352 | # save X 353 | filename = filename + '_perturbations.h5' 354 | with h5py.File(filename, 'a') as hf: 355 | group = hf.create_group('group' + str(group_index)) 356 | group.create_dataset("x_perturbed", data=x_perturbed) 357 | group.create_dataset("y_perturbed", data=y_perturbed) 358 | 359 | print("Classifications saved in ", filename) 360 | 361 | return 362 | 363 | 364 | def load_perturbed_test_groups(filename, group_index): 365 | with h5py.File(filename + '_perturbations.h5', 'r') as hf: 366 | group = hf.get('group' + str(group_index)) 367 | x_perturbed = group.get('x_perturbed').value 368 | y_perturbed = group.get('y_perturbed').value 369 | 370 | return x_perturbed, y_perturbed 371 | 372 | 373 | def create_experiment_dir(experiment_path, model_name, 374 | selected_class, step_size, 375 | approach, susp_num, repeat): 376 | # define experiments name, create directory experiments directory if it 377 | # doesnt exist 378 | experiment_name = model_name + '_C' + str(selected_class) + '_SS' + \ 379 | str(step_size) + '_' + approach + '_SN' + str(susp_num) + '_R' + str(repeat) 380 | 381 | if not os.path.exists(experiment_path): 382 | os.makedirs(experiment_path) 383 | 384 | return experiment_name 385 | 386 | 387 | def save_classifications(correct_classifications, misclassifications, filename, group_index): 388 | filename = filename + '_classifications.h5' 389 | with h5py.File(filename, 'a') as hf: 390 | group = hf.create_group('group' + str(group_index)) 391 | group.create_dataset("correct_classifications", data=correct_classifications) 392 | group.create_dataset("misclassifications", data=misclassifications) 393 | 394 | print("Classifications saved in ", filename) 395 | return 396 | 397 | 398 | def load_classifications(filename, group_index): 399 | filename = filename + '_classifications.h5' 400 | print 401 | filename 402 | try: 403 | with h5py.File(filename, 'r') as hf: 404 | group = hf.get('group' + str(group_index)) 405 | correct_classifications = group.get('correct_classifications').value 406 | misclassifications = group.get('misclassifications').value 407 | 408 | print("Classifications loaded from ", filename) 409 | return correct_classifications, misclassifications 410 | except (IOError) as error: 411 | print("Could not open file: ", filename) 412 | sys.exit(-1) 413 | 414 | 415 | def save_totalR(totalR, filename, group_index): 416 | filename = filename + '_relevances.h5' 417 | with h5py.File(filename, 'a') as hf: 418 | group = hf.create_group('group' + str(group_index)) 419 | for i in range(len(totalR)): 420 | group.create_dataset("totalR_" + str(i), data=totalR[i]) 421 | 422 | print("total relevance data saved in ", filename) 423 | return 424 | 425 | 426 | def load_totalR(filename, group_index): 427 | filename = filename + '_relevances.h5' 428 | try: 429 | with h5py.File(filename, 'r') as hf: 430 | group = hf.get('group' + str(group_index)) 431 | i = 0 432 | totalR = [] 433 | while True: 434 | # totalR.append(group.get('totalR_' + str(i)).value) 435 | totalR.append(group.get('totalR_' + str(i))[()]) 436 | i += 1 437 | 438 | except (IOError) as error: 439 | print("File %s does not exist" % (filename)) 440 | # print("Could not open file: ", filename) 441 | # traceback.print_exc() 442 | return None 443 | except (AttributeError, TypeError) as error: 444 | # because we don't know the exact dimensions (number of layers of our network) 445 | # we leave it to iterate until it throws an attribute error, and then return 446 | # layer outs to the caller function 447 | print("totalR loaded from ", filename) 448 | return totalR 449 | 450 | 451 | def save_layer_outs(layer_outs, filename, group_index): 452 | filename = filename + '_layer_outs.h5' 453 | with h5py.File(filename, 'a') as hf: 454 | group = hf.create_group('group' + str(group_index)) 455 | for i in range(len(layer_outs)): 456 | group.create_dataset("layer_outs_" + str(i), data=layer_outs[i]) 457 | 458 | print("Layer outs saved in ", filename) 459 | return 460 | 461 | 462 | def load_layer_outs(filename, group_index): 463 | filename = filename + '_layer_outs.h5' 464 | try: 465 | with h5py.File(filename, 'r') as hf: 466 | group = hf.get('group' + str(group_index)) 467 | i = 0 468 | layer_outs = [] 469 | while True: 470 | # layer_outs.append(group.get('layer_outs_' + str(i)).value) 471 | layer_outs.append(group.get('layer_outs_' + str(i))[()]) 472 | i += 1 473 | 474 | except (IOError) as error: 475 | print("Could not open file: ", filename) 476 | traceback.print_exc() 477 | sys.exit(-1) 478 | except (AttributeError, TypeError) as error: 479 | # because we don't know the exact dimensions (number of layers of our network) 480 | # we leave it to iterate until it throws an attribute error, and then return 481 | # layer outs to the caller function 482 | print("Layer outs loaded from ", filename) 483 | return layer_outs 484 | 485 | 486 | def filter_correct_classifications(model, X, Y): 487 | X_corr = [] 488 | Y_corr = [] 489 | X_misc = [] 490 | Y_misc = [] 491 | preds = model.predict(X) # np.expand_dims(x,axis=0)) 492 | 493 | for idx, pred in enumerate(preds): 494 | if np.argmax(pred) == np.argmax(Y[idx]): 495 | X_corr.append(X[idx]) 496 | Y_corr.append(Y[idx]) 497 | else: 498 | X_misc.append(X[idx]) 499 | Y_misc.append(Y[idx]) 500 | 501 | ''' 502 | for x, y in zip(X, Y): 503 | if np.argmax(p) == np.argmax(y): 504 | X_corr.append(x) 505 | Y_corr.append(y) 506 | else: 507 | X_misc.append(x) 508 | Y_misc.append(y) 509 | ''' 510 | 511 | return np.array(X_corr), np.array(Y_corr), np.array(X_misc), np.array(Y_misc) 512 | 513 | 514 | def filter_val_set(desired_class, X, Y): 515 | """ 516 | Filter the given sets and return only those that match the desired_class value 517 | :param desired_class: 518 | :param X: 519 | :param Y: 520 | :return: 521 | """ 522 | X_class = [] 523 | Y_class = [] 524 | for x, y in zip(X, Y): 525 | if y[desired_class] == 1: 526 | X_class.append(x) 527 | Y_class.append(y) 528 | print("Validation set filtered for desired class: " + str(desired_class)) 529 | return np.array(X_class), np.array(Y_class) 530 | 531 | 532 | def normalize(x): 533 | # utility function to normalize a tensor by its L2 norm 534 | return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 535 | 536 | 537 | def get_trainable_layers(model): 538 | trainable_layers = [] 539 | for idx, layer in enumerate(model.layers): 540 | try: 541 | if 'input' not in layer.name and 'softmax' not in layer.name and \ 542 | 'pred' not in layer.name and 'drop' not in layer.name: 543 | weights = layer.get_weights()[0] 544 | trainable_layers.append(model.layers.index(layer)) 545 | except: 546 | pass 547 | 548 | # trainable_layers = trainable_layers[:-1] # ignore the output layer 549 | 550 | return trainable_layers 551 | 552 | 553 | def weight_analysis(model, target_layer): 554 | threshold_weight = 0.1 555 | deactivatables = [] 556 | for i in range(2, target_layer + 1): 557 | for k in range(model.layers[i - 1].output_shape[1]): 558 | neuron_weights = model.layers[i].get_weights()[0][k] 559 | deactivate = True 560 | for j in range(len(neuron_weights)): 561 | if neuron_weights[j] > threshold_weight: 562 | deactivate = False 563 | 564 | if deactivate: 565 | deactivatables.append((i, k)) 566 | 567 | return deactivatables 568 | 569 | 570 | def percent_str(part, whole): 571 | return "{0}%".format(float(part) / whole * 100) 572 | 573 | 574 | def generate_adversarial(original_input, method, model, 575 | target=None, target_class=None, sess=None, **kwargs): 576 | if not hasattr(generate_adversarial, "attack_types"): 577 | generate_adversarial.attack_types = { 578 | 'fgsm': FastGradientMethod, 579 | 'jsma': SaliencyMapMethod, 580 | 'cw': CarliniWagnerL2, 581 | 'bim': BasicIterativeMethod 582 | } 583 | 584 | if sess is None: 585 | sess = K.get_session() 586 | 587 | if method in generate_adversarial.attack_types: 588 | attacker = generate_adversarial.attack_types[method](KerasModelWrapper(model), sess) 589 | else: 590 | raise Exception("Method not supported") 591 | 592 | if type(original_input) is list: 593 | original_input = np.asarray(original_input) 594 | else: 595 | original_input = np.asarray([original_input]) 596 | 597 | if target_class is not None: 598 | target_class = [target_class] 599 | 600 | if target is None and target_class is not None: 601 | target = np.zeros((len(target_class), model.output_shape[1])) 602 | target[np.arange(len(target_class)), target_class] = 1 603 | 604 | if target is not None: 605 | kwargs['y_target'] = target 606 | 607 | return attacker.generate_np(original_input, **kwargs) 608 | 609 | 610 | def find_relevant_pixels(inputs, model_path, lrpmethod, relevance_percentile): 611 | lrpmodel = read(model_path + '.txt', 'txt') # 99.16% prediction accuracy 612 | lrpmodel.drop_softmax_output_layer() # drop softnax output layer for analysis 613 | 614 | all_relevant_pixels = [] 615 | 616 | for inp in inputs: 617 | ypred = lrpmodel.forward(np.expand_dims(inp, axis=0)) 618 | 619 | mask = np.zeros_like(ypred) 620 | mask[:, np.argmax(ypred)] = 1 621 | Rinit = ypred * mask 622 | 623 | if lrpmethod == 'simple': 624 | R_inp, R_all = lrpmodel.lrp(Rinit) # as Eq(56) from DOI: 10.1371/journal.pone.0130140 625 | elif lrpmethod == 'epsilon': 626 | R_inp, R_all = lrpmodel.lrp(Rinit, 'epsilon', 0.01) # as Eq(58) from DOI: 10.1371/journal.pone.0130140 627 | elif lrpmethod == 'alphabeta': 628 | R_inp, R_all = lrpmodel.lrp(Rinit, 'alphabeta', 3) # as Eq(60) from DOI: 10.1371/journal.pone.0130140 629 | 630 | if 'lenet' in model_path.lower(): 631 | R_inp_flat = R_inp.reshape(28 * 28) 632 | elif 'cifar' in model_path.lower(): 633 | R_inp_flat = R_inp.reshape(32 * 32 * 3) 634 | else: 635 | R_inp_flat = R_inp.reshape(100 * 100 * 3) 636 | 637 | abs_R_inp_flat = np.absolute(R_inp_flat) 638 | 639 | relevance_threshold = np.percentile(abs_R_inp_flat, relevance_percentile) 640 | # if relevance_threshold < 0: relevance_threshold = 0 641 | 642 | s = datetime.datetime.now() 643 | if 'lenet' in model_path.lower(): 644 | R_inp = np.absolute(R_inp.reshape(28, 28)) 645 | elif 'cifar' in model_path.lower(): 646 | R_inp = np.absolute(R_inp.reshape(32, 32, 3)) 647 | else: 648 | R_inp = np.absolute(R_inp.reshape(100, 100, 3)) 649 | 650 | relevant_pixels = np.where(R_inp > relevance_threshold) 651 | all_relevant_pixels.append(relevant_pixels) 652 | return all_relevant_pixels 653 | 654 | 655 | def save_relevant_pixels(filename, relevant_pixels): 656 | with h5py.File(filename + '_relevant_pixels.h5', 'a') as hf: 657 | group = hf.create_group('gg') 658 | for i in range(len(relevant_pixels)): 659 | group.create_dataset("relevant_pixels_" + str(i), data=relevant_pixels[i]) 660 | 661 | print("Relevant pixels saved to %s" % (filename)) 662 | return 663 | 664 | 665 | def load_relevant_pixels(filename): 666 | try: 667 | with h5py.File(filename + '_relevant_pixels.h5', 'r') as hf: 668 | group = hf.get('gg') 669 | i = 0 670 | relevant_pixels = [] 671 | while True: 672 | # relevant_pixels.append(group.get('relevant_pixels_' + str(i)).value) 673 | relevant_pixels.append(group.get('relevant_pixels_' + str(i))[()]) 674 | i += 1 675 | except (AttributeError, TypeError) as error: 676 | # because we don't know the exact number of inputs in each class 677 | # we leave it to iterate until it throws an attribute error, and then return 678 | # return relevant pixels to the caller function 679 | 680 | print("Relevant pixels loaded from %s" % (filename)) 681 | 682 | return relevant_pixels 683 | 684 | 685 | def create_dir(dir_name): 686 | if not os.path.exists(dir_name): 687 | os.makedirs(dir_name) --------------------------------------------------------------------------------