├── CompetitiveAlphaBeta.py ├── CompetitiveMCTS.py ├── CooperativeAStar.py ├── CooperativeMCTS.py ├── DataCollection.py ├── DataSet.py ├── FeatureExtraction.py ├── GameMoves.py ├── LICENSE ├── NeuralNetwork.py ├── Oxford-AIMS-CDT-SystemsVerification-HT2020.pdf ├── Oxford-AIMS-CDT-SystemsVerification.md ├── README.md ├── basics.py ├── commands.sh ├── figures ├── Adv_CIFAR10.png ├── Adv_GTSRB.png ├── Adv_MNIST.png ├── Adversary.png ├── Architecture.png ├── Competitive_CIFAR10.png ├── Competitive_GTSRB.png ├── Cooperative_GTSRB.png ├── Cooperative_MNIST.png ├── FR.png ├── Feature.png ├── Game.png ├── Lipschitz.png └── MSR.png ├── lowerbound.py ├── main.py ├── models ├── cifar10.h5 └── mnist.h5 └── upperbound.py /CompetitiveAlphaBeta.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Construct a CompetitiveAlphaBeta class to compute 5 | the lower bound of Player I’s maximum adversary distance 6 | while Player II being competitive. 7 | 8 | Author: Min Wu 9 | Email: min.wu@cs.ox.ac.uk 10 | """ 11 | from numpy import inf 12 | 13 | from FeatureExtraction import * 14 | from basics import * 15 | 16 | 17 | class CompetitiveAlphaBeta: 18 | def __init__(self, image, model, eta, tau, bounds=(0, 1)): 19 | self.IMAGE = image 20 | self.IMAGE_BOUNDS = bounds 21 | self.MODEL = model 22 | self.DIST_METRIC = eta[0] 23 | self.DIST_VAL = eta[1] 24 | self.TAU = tau 25 | self.LABEL, _ = self.MODEL.predict(self.IMAGE) 26 | 27 | feature_extraction = FeatureExtraction(pattern='grey-box') 28 | self.PARTITIONS = feature_extraction.get_partitions(self.IMAGE, self.MODEL, num_partition=10) 29 | 30 | self.ALPHA = {} 31 | self.BETA = {} 32 | self.MANI_BETA = {} 33 | self.MANI_DIST = {} 34 | self.CURRENT_MANI = () 35 | 36 | self.ROBUST_FEATURE_FOUND = False 37 | self.ROBUST_FEATURE = [] 38 | self.FRAGILE_FEATURE_FOUND = False 39 | self.LEAST_FRAGILE_FEATURE = [] 40 | 41 | print("Distance metric %s, with bound value %s." % (self.DIST_METRIC, self.DIST_VAL)) 42 | 43 | def target_pixels(self, image, pixels): 44 | (row, col, chl) = image.shape 45 | 46 | atomic_manipulations = [] 47 | manipulated_images = [] 48 | for (x, y) in pixels: 49 | for z in range(chl): 50 | atomic = (x, y, z, 1 * self.TAU) 51 | valid, atomic_image = self.apply_atomic_manipulation(image, atomic) 52 | if valid is True: 53 | manipulated_images.append(atomic_image) 54 | atomic_manipulations.append(atomic) 55 | atomic = (x, y, z, -1 * self.TAU) 56 | valid, atomic_image = self.apply_atomic_manipulation(image, atomic) 57 | if valid is True: 58 | manipulated_images.append(atomic_image) 59 | atomic_manipulations.append(atomic) 60 | manipulated_images = np.asarray(manipulated_images) 61 | 62 | probabilities = self.MODEL.model.predict(manipulated_images) 63 | labels = probabilities.argmax(axis=1) 64 | 65 | for idx in range(len(manipulated_images)): 66 | if not diffImage(manipulated_images[idx], self.IMAGE): 67 | continue 68 | dist = self.cal_distance(manipulated_images[idx], self.IMAGE) 69 | if labels[idx] != self.LABEL: 70 | self.MANI_BETA.update({self.CURRENT_MANI + atomic_manipulations[idx]: dist}) 71 | self.MANI_DIST.update({self.CURRENT_MANI + atomic_manipulations[idx]: dist}) 72 | else: 73 | self.MANI_BETA.update({self.CURRENT_MANI + atomic_manipulations[idx]: inf}) 74 | self.MANI_DIST.update({self.CURRENT_MANI + atomic_manipulations[idx]: dist}) 75 | 76 | def apply_atomic_manipulation(self, image, atomic): 77 | atomic_image = image.copy() 78 | chl = atomic[0:3] 79 | manipulate = atomic[3] 80 | 81 | if (atomic_image[chl] >= max(self.IMAGE_BOUNDS) and manipulate >= 0) or ( 82 | atomic_image[chl] <= min(self.IMAGE_BOUNDS) and manipulate <= 0): 83 | valid = False 84 | return valid, atomic_image 85 | else: 86 | if atomic_image[chl] + manipulate > max(self.IMAGE_BOUNDS): 87 | atomic_image[chl] = max(self.IMAGE_BOUNDS) 88 | elif atomic_image[chl] + manipulate < min(self.IMAGE_BOUNDS): 89 | atomic_image[chl] = min(self.IMAGE_BOUNDS) 90 | else: 91 | atomic_image[chl] += manipulate 92 | valid = True 93 | return valid, atomic_image 94 | 95 | def cal_distance(self, image1, image2): 96 | if self.DIST_METRIC == 'L0': 97 | return l0Distance(image1, image2) 98 | elif self.DIST_METRIC == 'L1': 99 | return l1Distance(image1, image2) 100 | elif self.DIST_METRIC == 'L2': 101 | return l2Distance(image1, image2) 102 | else: 103 | print("Unrecognised distance metric. " 104 | "Try 'L0', 'L1', or 'L2'.") 105 | 106 | def play_game(self, image): 107 | 108 | for partitionID, pixels in self.PARTITIONS.items(): 109 | self.MANI_BETA = {} 110 | self.MANI_DIST = {} 111 | self.CURRENT_MANI = () 112 | print("partition ID:", partitionID) 113 | self.target_pixels(image, pixels) 114 | 115 | while min(self.MANI_BETA.values()) is inf and min(self.MANI_DIST.values()) <= self.DIST_VAL: 116 | min_dist = min(self.MANI_BETA.values()) 117 | print("Current min distance:", min_dist) 118 | 119 | print("Adversary not found.") 120 | mani_distance = copy.deepcopy(self.MANI_BETA) 121 | for atom, _ in mani_distance.items(): 122 | self.MANI_BETA.pop(atom) 123 | self.CURRENT_MANI = atom 124 | self.MANI_DIST.pop(atom) 125 | 126 | new_image = copy.deepcopy(self.IMAGE) 127 | atomic_list = [atom[i:i + 4] for i in range(0, len(atom), 4)] 128 | for atomic in atomic_list: 129 | valid, new_image = self.apply_atomic_manipulation(new_image, atomic) 130 | 131 | self.target_pixels(new_image, pixels) 132 | 133 | if min(self.MANI_BETA.values()) > self.DIST_VAL or min(self.MANI_DIST.values()) > self.DIST_VAL: 134 | print("Distance:", ) 135 | print("Adversarial distance exceeds distance bound.") 136 | self.BETA.update({partitionID: None}) 137 | elif min(self.MANI_BETA.values()) is not inf: 138 | print("Adversary found.") 139 | adv_mani = min(self.MANI_BETA, key=self.MANI_BETA.get) 140 | print("Manipulations:", adv_mani) 141 | adv_dist = self.MANI_BETA[adv_mani] 142 | print("Distance:", adv_dist) 143 | self.BETA.update({partitionID: [adv_mani, adv_dist]}) 144 | 145 | for partitionID, beta in self.MANI_BETA: 146 | print(partitionID, beta) 147 | if beta is None: 148 | print("Feature %s is robust." % partitionID) 149 | self.MANI_BETA.pop(partitionID) 150 | self.ROBUST_FEATURE_FOUND = True 151 | self.ROBUST_FEATURE.append(partitionID) 152 | if self.MANI_BETA: 153 | self.FRAGILE_FEATURE_FOUND = True 154 | self.ALPHA = max(self.BETA, key=self.BETA.get) 155 | self.LEAST_FRAGILE_FEATURE = self.ALPHA 156 | print("Among fragile features, the least fragile feature is:\n" 157 | % self.LEAST_FRAGILE_FEATURE) 158 | 159 | 160 | """ 161 | def play_game(self, image): 162 | self.player1(image) 163 | for partitionID, beta in self.MANI_BETA: 164 | print(partitionID, beta) 165 | if beta is None: 166 | print("Feature %s is robust." % partitionID) 167 | self.MANI_BETA.pop(partitionID) 168 | self.ROBUST_FEATURE_FOUND = True 169 | self.ROBUST_FEATURE.append(partitionID) 170 | if self.MANI_BETA: 171 | self.FRAGILE_FEATURE_FOUND = True 172 | self.ALPHA = max(self.BETA, key=self.BETA.get) 173 | self.LEAST_FRAGILE_FEATURE = self.ALPHA 174 | print("Among fragile features, the least fragile feature is:\n" 175 | % self.LEAST_FRAGILE_FEATURE) 176 | 177 | def player1(self, image, partition_idx=None): 178 | # Alpha 179 | if partition_idx is None: 180 | for partitionID in self.PARTITIONS.keys(): 181 | self.MANI_BETA = {} 182 | self.MANI_DIST = {} 183 | self.CURRENT_MANI = () 184 | print("partition ID:", partitionID) 185 | self.player2(image, partitionID) 186 | else: 187 | self.player2(image, partition_idx) 188 | 189 | def player2(self, image, partition_idx): 190 | # Beta 191 | pixels = self.PARTITIONS[partition_idx] 192 | if not self.MANI_DIST: 193 | self.target_pixels(image, pixels) 194 | self.player1(image, partition_idx=partition_idx) 195 | else: 196 | min_dist = min(self.MANI_BETA.values()) 197 | print("Current min distance:", min_dist) 198 | if min_dist is not inf: 199 | print("Adversary found.") 200 | adv_mani = min(self.MANI_BETA, key=self.MANI_BETA.get) 201 | print("Manipulations:", adv_mani) 202 | adv_dist = self.MANI_BETA[adv_mani] 203 | self.BETA.update({partition_idx: [adv_mani, adv_dist]}) 204 | elif min(self.MANI_DIST.values()) >= self.DIST_VAL: 205 | print("Adversarial distance exceeds distance bound.") 206 | self.BETA.update({partition_idx: None}) 207 | else: 208 | print("Adversary not found.") 209 | mani_distance = copy.deepcopy(self.MANI_BETA) 210 | for atom, _ in mani_distance.items(): 211 | self.MANI_BETA.pop(atom) 212 | self.CURRENT_MANI = atom 213 | self.MANI_DIST.pop(atom) 214 | 215 | new_image = copy.deepcopy(self.IMAGE) 216 | atomic_list = [atom[i:i + 4] for i in range(0, len(atom), 4)] 217 | for atomic in atomic_list: 218 | valid, new_image = self.apply_atomic_manipulation(new_image, atomic) 219 | 220 | self.target_pixels(new_image, pixels) 221 | 222 | self.player1(image, partition_idx) 223 | """ 224 | -------------------------------------------------------------------------------- /CompetitiveMCTS.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | A data structure for organising search 5 | 6 | author: Xiaowei Huang 7 | """ 8 | 9 | import numpy as np 10 | import time 11 | import os 12 | import copy 13 | import sys 14 | import operator 15 | import random 16 | import math 17 | 18 | from basics import * 19 | from GameMoves import * 20 | 21 | MCTS_multi_samples = 1 22 | effectiveConfidenceWhenChanging = 0.0 23 | explorationRate = math.sqrt(2) 24 | 25 | 26 | class MCTSCompetitive: 27 | 28 | def __init__(self, data_set, model, image_index, image, tau, eta): 29 | self.data_set = data_set 30 | self.image_index = image_index 31 | self.image = image 32 | self.model = model 33 | self.tau = tau 34 | self.eta = eta 35 | 36 | (self.originalClass, self.originalConfident) = self.model.predict(self.image) 37 | 38 | self.moves = GameMoves(self.data_set, self.model, self.image, self.tau, self.image_index) 39 | 40 | self.cost = {} 41 | self.numberOfVisited = {} 42 | self.parent = {} 43 | self.children = {} 44 | self.children[-1] = {0} 45 | self.fullyExpanded = {} 46 | 47 | self.indexToNow = 0 48 | # current root node 49 | self.rootIndex = 0 50 | 51 | # maintain for every node on the tree the current best 52 | self.bestCaseList = {} 53 | # best case for the root node 54 | # please note the difference with the cooperative game 55 | self.bestCase = (2 ^ 20, {}) 56 | 57 | self.manipulation = {} 58 | # initialise root node 59 | self.manipulation[-1] = {} 60 | self.initialiseLeafNode(0, -1, {}) 61 | self.bestCaseList[0] = (0, []) 62 | self.bestCaseList[-1] = (0, []) 63 | 64 | # record all the keypoints: index -> kp 65 | self.keypoints = {} 66 | # mapping nodes to keypoints 67 | self.keypoint = {} 68 | self.keypoint[-1] = 0 69 | self.keypoint[0] = 0 70 | 71 | # local actions 72 | self.actions = {} 73 | self.usedActionsID = {} 74 | self.indexToActionID = {} 75 | 76 | self.numConverge = 0 77 | 78 | # how many sampling is conducted 79 | self.numOfSampling = 0 80 | 81 | # number of adversarial examples 82 | self.numAdv = 0 83 | 84 | # temporary variables for sampling 85 | self.atomicManipulationPath = [] 86 | self.depth = 0 87 | self.availableActionIDs = [] 88 | self.usedActionIDs = [] 89 | 90 | def initialiseMoves(self): 91 | # initialise actions according to the type of manipulations 92 | actions = self.moves.moves 93 | print((actions.keys())) 94 | self.keypoints[0] = 0 95 | i = 1 96 | for k in actions[0]: 97 | self.keypoints[i] = k 98 | i += 1 99 | 100 | for i in range(len(actions)): 101 | ast = {} 102 | for j in range(len(actions[i])): 103 | ast[j] = actions[i][j] 104 | self.actions[i] = ast 105 | nprint("%s actions have been initialised. " % (len(self.actions))) 106 | 107 | def initialiseLeafNode(self, index, parentIndex, newAtomicManipulation): 108 | nprint("initialising a leaf node %s from the node %s" % (index, parentIndex)) 109 | self.manipulation[index] = mergeTwoDicts(self.manipulation[parentIndex], newAtomicManipulation) 110 | self.cost[index] = 0 111 | self.parent[index] = parentIndex 112 | self.children[index] = [] 113 | self.fullyExpanded[index] = False 114 | self.numberOfVisited[index] = 0 115 | 116 | # activations1 = self.moves.applyManipulation(self.image,self.manipulation[index]) 117 | 118 | def destructor(self): 119 | self.image = 0 120 | self.image = 0 121 | self.model = 0 122 | self.model = 0 123 | self.manipulatedDimensions = {} 124 | self.manipulation = {} 125 | self.cost = {} 126 | self.parent = {} 127 | self.children = {} 128 | self.fullyExpanded = {} 129 | self.numberOfVisited = {} 130 | 131 | self.actions = {} 132 | self.usedActionsID = {} 133 | self.indexToActionID = {} 134 | 135 | # move one step forward 136 | # it means that we need to remove children other than the new root 137 | def makeOneMove(self, newRootIndex): 138 | if self.keypoint[newRootIndex] != 0: 139 | player = "the first player" 140 | else: 141 | player = "the second player" 142 | print("%s making a move into the new root %s, whose value is %s and visited number is %s" 143 | % (player, newRootIndex, self.cost[newRootIndex], self.numberOfVisited[newRootIndex])) 144 | self.removeChildren(self.rootIndex, [newRootIndex]) 145 | self.rootIndex = newRootIndex 146 | 147 | def removeChildren(self, index, indicesToAvoid): 148 | if self.fullyExpanded[index] is True: 149 | for childIndex in self.children[index]: 150 | if childIndex not in indicesToAvoid: self.removeChildren(childIndex, []) 151 | self.manipulation.pop(index, None) 152 | self.cost.pop(index, None) 153 | self.parent.pop(index, None) 154 | self.keypoint.pop(index, None) 155 | self.children.pop(index, None) 156 | self.fullyExpanded.pop(index, None) 157 | self.numberOfVisited.pop(index, None) 158 | 159 | def bestChild(self, index): 160 | allValues = {} 161 | for childIndex in self.children[index]: 162 | allValues[childIndex] = float(self.numberOfVisited[childIndex]) / self.cost[childIndex] 163 | nprint("finding best children from %s" % allValues) 164 | # for competitive 165 | return max(allValues.items(), key=operator.itemgetter(1))[0] 166 | 167 | def treeTraversal(self, index): 168 | if self.fullyExpanded[index] is True: 169 | nprint("tree traversal on node %s with childrens %s" % (index, self.children[index])) 170 | allValues = {} 171 | for childIndex in self.children[index]: 172 | # UCB values 173 | allValues[childIndex] = ((float(self.numberOfVisited[childIndex]) / self.cost[childIndex]) * self.eta[1] 174 | + explorationRate * math.sqrt( 175 | math.log(self.numberOfVisited[index]) / float(self.numberOfVisited[childIndex]))) 176 | 177 | if self.keypoint[index] == 0: 178 | allValues2 = {} 179 | for k, v in allValues.items(): 180 | allValues2[k] = 1 / float(allValues[k]) 181 | probdist = [x / sum(allValues2.values()) for x in allValues2.values()] 182 | # nextIndex = np.random.choice(list(allValues.keys()), 1, p=probdist)[0] 183 | nextIndex = list(allValues.keys())[probdist.index(max(probdist))] 184 | else: 185 | probdist = [x / sum(allValues.values()) for x in allValues.values()] 186 | # nextIndex = np.random.choice(list(allValues.keys()), 1, p=probdist)[0] 187 | nextIndex = list(allValues.keys())[probdist.index(max(probdist))] 188 | 189 | if self.keypoint[index] in self.usedActionsID.keys() and self.keypoint[index] != 0: 190 | self.usedActionsID[self.keypoint[index]].append(self.indexToActionID[index]) 191 | elif self.keypoint[index] != 0: 192 | self.usedActionsID[self.keypoint[index]] = [self.indexToActionID[index]] 193 | 194 | return self.treeTraversal(nextIndex) 195 | 196 | else: 197 | nprint("tree traversal terminated on node %s" % index) 198 | availableActions = copy.deepcopy(self.actions) 199 | # for k in self.usedActionsID.keys(): 200 | # for i in self.usedActionsID[k]: 201 | # availableActions[k].pop(i, None) 202 | return index, availableActions 203 | 204 | def initialiseExplorationNode(self, index, availableActions): 205 | nprint("expanding %s" % index) 206 | if self.keypoint[index] != 0: 207 | for (actionId, am) in availableActions[self.keypoint[index]].items(): 208 | self.indexToNow += 1 209 | self.keypoint[self.indexToNow] = 0 210 | self.indexToActionID[self.indexToNow] = actionId 211 | self.initialiseLeafNode(self.indexToNow, index, am) 212 | self.children[index].append(self.indexToNow) 213 | self.bestCaseList[self.indexToNow] = (0, {}) 214 | else: 215 | for kp in list(set(self.keypoints.keys()) - set([0])): 216 | self.indexToNow += 1 217 | self.keypoint[self.indexToNow] = kp 218 | self.indexToActionID[self.indexToNow] = 0 219 | self.initialiseLeafNode(self.indexToNow, index, {}) 220 | self.children[index].append(self.indexToNow) 221 | self.bestCaseList[self.indexToNow] = (self.eta[1], {}) 222 | 223 | self.fullyExpanded[index] = True 224 | self.usedActionsID = {} 225 | return self.children[index] 226 | 227 | def backPropagation(self, index, value): 228 | self.cost[index] += value 229 | self.numberOfVisited[index] += 1 230 | if self.parent[index] in self.parent: 231 | nprint("start backPropagating the value %s from node %s, whose parent node is %s" % ( 232 | value, index, self.parent[index])) 233 | self.backPropagation(self.parent[index], value) 234 | else: 235 | nprint("backPropagating ends on node %s" % index) 236 | 237 | # start random sampling and return the Euclidean value as the value 238 | def sampling(self, index, availableActions): 239 | nprint("start sampling node %s" % index) 240 | availableActions2 = copy.deepcopy(availableActions) 241 | # availableActions2[self.keypoint[index]].pop(self.indexToActionID[index], None) 242 | sampleValues = [] 243 | samplePaths = [] 244 | i = 0 245 | for i in range(MCTS_multi_samples): 246 | self.atomicManipulationPath = self.manipulation[index] 247 | self.depth = 0 248 | self.availableActionIDs = {} 249 | for k in self.keypoints.keys(): 250 | self.availableActionIDs[k] = list(availableActions2[k].keys()) 251 | self.usedActionIDs = {} 252 | for k in self.keypoints.keys(): 253 | self.usedActionIDs[k] = [] 254 | (childTerminated, val) = self.sampleNext(self.keypoint[index]) 255 | self.numOfSampling += 1 256 | sampleValues.append(val) 257 | samplePaths.append(self.atomicManipulationPath) 258 | i += 1 259 | 260 | if self.keypoint[index] == 0: 261 | return childTerminated, min(sampleValues) 262 | else: 263 | minIndex = sampleValues.index(min(sampleValues)) 264 | # print(index, self.bestCaseList[index][0], min(sampleValues), self.eta) 265 | if self.bestCaseList[index][0] > sampleValues[minIndex]: 266 | nprint("on node %s, update best case from %s to %s, start updating ancestor nodes" % ( 267 | index, self.bestCaseList[index][0], sampleValues[minIndex])) 268 | self.numConverge += 1 269 | self.bestCaseList[index] = (sampleValues[minIndex], samplePaths[minIndex]) 270 | 271 | # update best case 272 | self.updateBestCase(index) 273 | return childTerminated, min(sampleValues) 274 | 275 | def computeDistance(self, newImage): 276 | (distMethod, _) = self.eta 277 | if distMethod == "L2": 278 | dist = l2Distance(newImage, self.image) 279 | elif distMethod == "L1": 280 | dist = l1Distance(newImage, self.image) 281 | elif distMethod == "Percentage": 282 | dist = diffPercent(newImage, self.image) 283 | elif distMethod == "NumDiffs": 284 | dist = diffPercent(newImage, self.image) * self.image.size 285 | return dist 286 | 287 | def sampleNext(self, k): 288 | activations1 = self.moves.applyManipulation(self.image, self.atomicManipulationPath) 289 | (newClass, newConfident) = self.model.predict(activations1) 290 | (distMethod, distVal) = self.eta 291 | dist = self.computeDistance(activations1) 292 | 293 | # need not only class change, but also high confidence adversary examples 294 | if newClass != self.originalClass and newConfident > effectiveConfidenceWhenChanging: 295 | nprint("sampling a path ends in a terminal node with depth %s... " % self.depth) 296 | self.atomicManipulationPath = self.scrutinizePath(self.atomicManipulationPath) 297 | activations2 = self.moves.applyManipulation(self.image, self.atomicManipulationPath) 298 | dist2 = self.computeDistance(activations2) 299 | self.numAdv += 1 300 | 301 | return (self.depth == 0, dist2) 302 | 303 | elif dist > distVal: 304 | nprint("sampling a path ends by eta with depth %s ... " % self.depth) 305 | return (self.depth == 0, distVal) 306 | 307 | elif not list(set(self.availableActionIDs[k]) - set(self.usedActionIDs[k])): 308 | nprint("sampling a path ends with depth %s because no more actions can be taken ... " % self.depth) 309 | return (self.depth == 0, distVal) 310 | 311 | elif self.depth > (self.eta[1] / self.tau): 312 | nprint( 313 | "sampling a path ends with depth %s more than the prespecifided maximum sampling depth ... " % self.depth) 314 | return (self.depth == 0, distVal) 315 | 316 | else: 317 | # print("continue sampling node ... ") 318 | # randomActionIndex = random.choice(list(set(self.availableActionIDs[k])-set(self.usedActionIDs[k]))) 319 | randomActionIndex = random.choice(self.availableActionIDs[k]) 320 | if k == 0: 321 | nextAtomicManipulation = {} 322 | else: 323 | nextAtomicManipulation = self.actions[k][randomActionIndex] 324 | # self.availableActionIDs[k].remove(randomActionIndex) 325 | # self.usedActionIDs[k].append(randomActionIndex) 326 | newManipulationPath = mergeTwoDicts(self.atomicManipulationPath, nextAtomicManipulation) 327 | activations2 = self.moves.applyManipulation(self.image, newManipulationPath) 328 | (newClass2, newConfident2) = self.model.predict(activations2) 329 | 330 | self.atomicManipulationPath = newManipulationPath 331 | self.depth = self.depth + 1 332 | if k == 0: 333 | return self.sampleNext(randomActionIndex) 334 | else: 335 | return self.sampleNext(0) 336 | 337 | def scrutinizePath(self, manipulations): 338 | flag = False 339 | tempManipulations = copy.deepcopy(manipulations) 340 | for k, v in manipulations.items(): 341 | tempManipulations[k] = 0 342 | activations1 = self.moves.applyManipulation(self.image, tempManipulations) 343 | (newClass, newConfident) = self.model.predict(activations1) 344 | if newClass != self.originalClass: 345 | manipulations.pop(k) 346 | flag = True 347 | break 348 | 349 | if flag is True: 350 | return self.scrutinizePath(manipulations) 351 | else: 352 | return manipulations 353 | 354 | def terminalNode(self, index): 355 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 356 | (newClass, _) = self.model.predict(activations1) 357 | return newClass != self.originalClass 358 | 359 | def terminatedByEta(self, index): 360 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 361 | dist = self.computeDistance(activations1) 362 | nprint("terminated by controlled search: distance = %s" % dist) 363 | return dist > self.eta[1] 364 | 365 | def applyManipulation(self, manipulation): 366 | activations1 = self.moves.applyManipulation(self.image, manipulation) 367 | return activations1 368 | 369 | def l2Dist(self, index): 370 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 371 | return l2Distance(self.image, activations1) 372 | 373 | def l1Dist(self, index): 374 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 375 | return l1Distance(self.image, activations1) 376 | 377 | def l0Dist(self, index): 378 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 379 | return l0Distance(self.image, activations1) 380 | 381 | def diffImage(self, index): 382 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 383 | return diffImage(self.image, activations1) 384 | 385 | def diffPercent(self, index): 386 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 387 | return diffPercent(self.image, activations1) 388 | 389 | def updateBestCase(self, index): 390 | if index > 0: 391 | parentIndex = self.parent[index] 392 | if self.keypoint[parentIndex] == 0: 393 | tempVal = 0 394 | tempPath = [] 395 | for childIndex in self.children[parentIndex]: 396 | if self.bestCaseList[childIndex][0] > tempVal: 397 | tempVal = self.bestCaseList[childIndex][0] 398 | tempPath = self.bestCaseList[childIndex][1] 399 | self.bestCaseList[parentIndex] = (tempVal, tempPath) 400 | else: 401 | tempVal = self.eta[1] 402 | tempPath = [] 403 | for childIndex in self.children[parentIndex]: 404 | if self.bestCaseList[childIndex][0] < tempVal: 405 | tempVal = self.bestCaseList[childIndex][0] 406 | tempPath = self.bestCaseList[childIndex][1] 407 | self.bestCaseList[parentIndex] = (tempVal, tempPath) 408 | self.updateBestCase(parentIndex) 409 | else: 410 | if self.bestCase[0] != self.bestCaseList[0][0]: 411 | self.bestCase = self.bestCaseList[0] 412 | nprint("the best case is updated into distance %s and manipulation %s" % ( 413 | self.bestCase[0], self.bestCase[1])) 414 | 415 | def bestFeatures(self): 416 | bestManipulation = self.bestCase[1] 417 | maxdims = [] 418 | nf = 0 419 | for i in range(1, len(self.actions)): 420 | md = [] 421 | flag = False 422 | for k, v in bestManipulation.items(): 423 | for k1, v1 in self.actions[i].items(): 424 | md += list(v1.keys()) 425 | if k in v1.keys(): 426 | flag = True 427 | if flag is True: 428 | nf += 1 429 | maxdims += md 430 | if len(maxdims) == 0: 431 | return (0, 0) 432 | elif len(maxdims[0]) == 3: 433 | maxdims = [(x, y) for (x, y, z) in maxdims] 434 | return (nf, maxdims) 435 | -------------------------------------------------------------------------------- /CooperativeAStar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Construct a CooperativeAStar class to compute 5 | the lower bound of Player I’s minimum adversary distance 6 | while Player II being cooperative. 7 | 8 | Author: Min Wu 9 | Email: min.wu@cs.ox.ac.uk 10 | """ 11 | 12 | import heapq 13 | 14 | from FeatureExtraction import * 15 | from basics import * 16 | 17 | 18 | class CooperativeAStar: 19 | def __init__(self, dataset, idx, image, model, eta, tau, bounds=(0, 1)): 20 | self.DATASET = dataset 21 | self.IDX = idx 22 | self.IMAGE = image 23 | self.IMAGE_BOUNDS = bounds 24 | self.MODEL = model 25 | self.DIST_METRIC = eta[0] 26 | self.DIST_VAL = eta[1] 27 | self.TAU = tau 28 | self.LABEL, _ = self.MODEL.predict(self.IMAGE) 29 | 30 | feature_extraction = FeatureExtraction(pattern='grey-box') 31 | self.PARTITIONS = feature_extraction.get_partitions(self.IMAGE, self.MODEL, num_partition=10) 32 | 33 | self.DIST_EVALUATION = {} 34 | self.ADV_MANIPULATION = () 35 | self.ADVERSARY_FOUND = None 36 | self.ADVERSARY = None 37 | 38 | self.CURRENT_SAFE = [0] 39 | 40 | print("Distance metric %s, with bound value %s." % (self.DIST_METRIC, self.DIST_VAL)) 41 | 42 | def target_pixels(self, image, pixels): 43 | # tau = self.TAU 44 | # model = self.MODEL 45 | (row, col, chl) = image.shape 46 | 47 | # img_batch = np.kron(np.ones((chl * 2, 1, 1, 1)), image) 48 | # atomic_manipulations = {} 49 | # manipulated_images = [] 50 | # idx = 0 51 | # for (x, y) in pixels: 52 | # changed_img_batch = img_batch.copy() 53 | # for z in range(chl): 54 | # atomic = (x, y, z, 1 * tau) 55 | # changed_img_batch[z * 2] = self.atomic_manipulation(image, atomic) 56 | # # changed_img_batch[z * 2, x, y, z] += tau 57 | # atomic_manipulations.update({idx: atomic}) 58 | # idx += 1 59 | # atomic = (x, y, z, -1 * tau) 60 | # changed_img_batch[z * 2 + 1] = self.atomic_manipulation(image, atomic) 61 | # # changed_img_batch[z * 2 + 1, x, y, z] -= tau 62 | # atomic_manipulations.update({idx: atomic}) 63 | # idx += 1 64 | # manipulated_images.append(changed_img_batch) # each loop append [chl*2, row, col, chl] 65 | # 66 | # manipulated_images = np.asarray(manipulated_images) # [len(pixels), chl*2, row, col, chl] 67 | # manipulated_images = manipulated_images.reshape(len(pixels) * chl * 2, row, col, chl) 68 | 69 | atomic_manipulations = [] 70 | manipulated_images = [] 71 | for (x, y) in pixels: 72 | for z in range(chl): 73 | atomic = (x, y, z, 1 * self.TAU) 74 | valid, atomic_image = self.apply_atomic_manipulation(image, atomic) 75 | if valid is True: 76 | manipulated_images.append(atomic_image) 77 | atomic_manipulations.append(atomic) 78 | atomic = (x, y, z, -1 * self.TAU) 79 | valid, atomic_image = self.apply_atomic_manipulation(image, atomic) 80 | if valid is True: 81 | manipulated_images.append(atomic_image) 82 | atomic_manipulations.append(atomic) 83 | manipulated_images = np.asarray(manipulated_images) 84 | 85 | probabilities = self.MODEL.model.predict(manipulated_images) 86 | # softmax_logits = self.MODEL.softmax_logits(manipulated_images) 87 | 88 | if self.ADV_MANIPULATION: 89 | atomic_list = [self.ADV_MANIPULATION[i:i + 4] for i in range(0, len(self.ADV_MANIPULATION), 4)] 90 | 91 | for idx in range(len(manipulated_images)): 92 | if not diffImage(manipulated_images[idx], self.IMAGE) or not diffImage(manipulated_images[idx], image): 93 | continue 94 | cost = self.cal_distance(manipulated_images[idx], self.IMAGE) 95 | [p_max, p_2dn_max] = heapq.nlargest(2, probabilities[idx]) 96 | heuristic = (p_max - p_2dn_max) * 2 * self.TAU # heuristic value determines Admissible (lb) or not (ub) 97 | estimation = cost + heuristic 98 | 99 | valid = True 100 | if self.ADV_MANIPULATION: 101 | for atomic in atomic_list: # atomic: [x, y, z, +/-tau] 102 | if atomic_manipulations[idx][0:3] == atomic[0:3] and atomic_manipulations[idx][3] == -atomic[3]: 103 | valid = False 104 | if valid is True: 105 | self.DIST_EVALUATION.update({self.ADV_MANIPULATION + atomic_manipulations[idx]: estimation}) 106 | 107 | # self.DIST_EVALUATION.update({self.ADV_MANIPULATION + atomic_manipulations[idx]: estimation}) 108 | # print("Atomic manipulations of target pixels done.") 109 | 110 | def apply_atomic_manipulation(self, image, atomic): 111 | atomic_image = image.copy() 112 | chl = atomic[0:3] 113 | manipulate = atomic[3] 114 | 115 | if (atomic_image[chl] >= max(self.IMAGE_BOUNDS) and manipulate >= 0) or ( 116 | atomic_image[chl] <= min(self.IMAGE_BOUNDS) and manipulate <= 0): 117 | valid = False 118 | return valid, atomic_image 119 | else: 120 | if atomic_image[chl] + manipulate > max(self.IMAGE_BOUNDS): 121 | atomic_image[chl] = max(self.IMAGE_BOUNDS) 122 | elif atomic_image[chl] + manipulate < min(self.IMAGE_BOUNDS): 123 | atomic_image[chl] = min(self.IMAGE_BOUNDS) 124 | else: 125 | atomic_image[chl] += manipulate 126 | valid = True 127 | return valid, atomic_image 128 | 129 | def cal_distance(self, image1, image2): 130 | if self.DIST_METRIC == 'L0': 131 | return l0Distance(image1, image2) 132 | elif self.DIST_METRIC == 'L1': 133 | return l1Distance(image1, image2) 134 | elif self.DIST_METRIC == 'L2': 135 | return l2Distance(image1, image2) 136 | else: 137 | print("Unrecognised distance metric. " 138 | "Try 'L0', 'L1', or 'L2'.") 139 | 140 | def play_game(self, image): 141 | new_image = copy.deepcopy(self.IMAGE) 142 | new_label, new_confidence = self.MODEL.predict(new_image) 143 | 144 | while self.cal_distance(self.IMAGE, new_image) <= self.DIST_VAL and new_label == self.LABEL: 145 | # for partitionID in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]: 146 | for partitionID in self.PARTITIONS.keys(): 147 | pixels = self.PARTITIONS[partitionID] 148 | self.target_pixels(new_image, pixels) 149 | 150 | self.ADV_MANIPULATION = min(self.DIST_EVALUATION, key=self.DIST_EVALUATION.get) 151 | print("Current best manipulations:", self.ADV_MANIPULATION) 152 | # print("%s distance (estimated): %s" % (self.DIST_METRIC, self.DIST_EVALUATION[self.ADV_MANIPULATION])) 153 | self.DIST_EVALUATION.pop(self.ADV_MANIPULATION) 154 | 155 | new_image = copy.deepcopy(self.IMAGE) 156 | atomic_list = [self.ADV_MANIPULATION[i:i + 4] for i in range(0, len(self.ADV_MANIPULATION), 4)] 157 | for atomic in atomic_list: 158 | valid, new_image = self.apply_atomic_manipulation(new_image, atomic) 159 | dist = self.cal_distance(self.IMAGE, new_image) 160 | print("%s distance (actual): %s" % (self.DIST_METRIC, dist)) 161 | 162 | new_label, new_confidence = self.MODEL.predict(new_image) 163 | if self.cal_distance(self.IMAGE, new_image) > self.DIST_VAL: 164 | # print("Adversarial distance exceeds distance budget.") 165 | self.ADVERSARY_FOUND = False 166 | break 167 | elif new_label != self.LABEL: 168 | # print("Adversarial image is found.") 169 | self.ADVERSARY_FOUND = True 170 | self.ADVERSARY = new_image 171 | break 172 | 173 | if self.CURRENT_SAFE[-1] != dist: 174 | self.CURRENT_SAFE.append(dist) 175 | path = "%s_pic/idx_%s_Safe_currentBest_%s.png" % (self.DATASET, self.IDX, len(self.CURRENT_SAFE) - 1) 176 | self.MODEL.save_input(new_image, path) 177 | 178 | 179 | 180 | """ 181 | def play_game(self, image): 182 | self.player1(image) 183 | 184 | self.ADV_MANIPULATION = min(self.DIST_EVALUATION, key=self.DIST_EVALUATION.get) 185 | self.DIST_EVALUATION.pop(self.ADV_MANIPULATION) 186 | print("Current best manipulations:", self.ADV_MANIPULATION) 187 | 188 | new_image = copy.deepcopy(self.IMAGE) 189 | atomic_list = [self.ADV_MANIPULATION[i:i + 4] for i in range(0, len(self.ADV_MANIPULATION), 4)] 190 | for atomic in atomic_list: 191 | valid, new_image = self.apply_atomic_manipulation(new_image, atomic) 192 | print("%s distance: %s" % (self.DIST_METRIC, self.cal_distance(self.IMAGE, new_image))) 193 | 194 | new_label, new_confidence = self.MODEL.predict(new_image) 195 | if self.cal_distance(self.IMAGE, new_image) > self.DIST_VAL: 196 | # print("Adversarial distance exceeds distance bound.") 197 | self.ADVERSARY_FOUND = False 198 | elif new_label != self.LABEL: 199 | # print("Adversarial image is found.") 200 | self.ADVERSARY_FOUND = True 201 | self.ADVERSARY = new_image 202 | else: 203 | self.play_game(new_image) 204 | 205 | def player1(self, image): 206 | # print("Player I is acting on features.") 207 | 208 | for partitionID in self.PARTITIONS.keys(): 209 | self.player2(image, partitionID) 210 | 211 | def player2(self, image, partition_idx): 212 | # print("Player II is acting on pixels in each partition.") 213 | 214 | pixels = self.PARTITIONS[partition_idx] 215 | self.target_pixels(image, pixels) 216 | """ 217 | -------------------------------------------------------------------------------- /CooperativeMCTS.py: -------------------------------------------------------------------------------- 1 | 5 # !/usr/bin/env python 2 | 3 | """ 4 | A data structure for organising search 5 | 6 | author: Xiaowei Huang 7 | """ 8 | 9 | import numpy as np 10 | import time 11 | import os 12 | import copy 13 | import sys 14 | import operator 15 | import random 16 | import math 17 | 18 | from basics import * 19 | from GameMoves import * 20 | 21 | MCTS_multi_samples = 3 22 | effectiveConfidenceWhenChanging = 0.0 23 | explorationRate = math.sqrt(2) 24 | 25 | 26 | class MCTSCooperative: 27 | 28 | def __init__(self, data_set, model, image_index, image, tau, eta): 29 | self.data_set = data_set 30 | self.image_index = image_index 31 | self.image = image 32 | self.model = model 33 | self.tau = tau 34 | self.eta = eta 35 | 36 | (self.originalClass, self.originalConfident) = self.model.predict(self.image) 37 | 38 | self.moves = GameMoves(self.data_set, self.model, self.image, self.tau, self.image_index) 39 | 40 | self.cost = {} 41 | self.numberOfVisited = {} 42 | self.parent = {} 43 | self.children = {} 44 | self.fullyExpanded = {} 45 | 46 | self.indexToNow = 0 47 | # current root node 48 | self.rootIndex = 0 49 | 50 | self.manipulation = {} 51 | # initialise root node 52 | self.manipulation[-1] = {} 53 | self.initialiseLeafNode(0, -1, {}) 54 | 55 | # record all the keypoints: index -> kp 56 | self.keypoints = {} 57 | # mapping nodes to keypoints 58 | self.keypoint = {} 59 | self.keypoint[0] = 0 60 | 61 | # local actions 62 | self.actions = {} 63 | self.usedActionsID = {} 64 | self.indexToActionID = {} 65 | 66 | # best case 67 | self.bestCase = (2 ^ 20, {}) 68 | self.numConverge = 0 69 | 70 | # number of adversarial examples 71 | self.numAdv = 0 72 | 73 | # how many sampling is conducted 74 | self.numOfSampling = 0 75 | 76 | # temporary variables for sampling 77 | self.atomicManipulationPath = [] 78 | self.depth = 0 79 | self.availableActionIDs = [] 80 | self.usedActionIDs = [] 81 | 82 | def initialiseMoves(self): 83 | # initialise actions according to the type of manipulations 84 | actions = self.moves.moves 85 | self.keypoints[0] = 0 86 | i = 1 87 | for k in actions[0]: 88 | self.keypoints[i] = k 89 | i += 1 90 | 91 | for i in range(len(actions)): 92 | ast = {} 93 | for j in range(len(actions[i])): 94 | ast[j] = actions[i][j] 95 | self.actions[i] = ast 96 | nprint("%s actions have been initialised." % (len(self.actions))) 97 | 98 | def initialiseLeafNode(self, index, parentIndex, newAtomicManipulation): 99 | nprint("initialising a leaf node %s from the node %s" % (index, parentIndex)) 100 | self.manipulation[index] = mergeTwoDicts(self.manipulation[parentIndex], newAtomicManipulation) 101 | self.cost[index] = 0 102 | self.parent[index] = parentIndex 103 | self.children[index] = [] 104 | self.fullyExpanded[index] = False 105 | self.numberOfVisited[index] = 0 106 | 107 | def destructor(self): 108 | self.image = 0 109 | self.image = 0 110 | self.model = 0 111 | self.model = 0 112 | self.manipulatedDimensions = {} 113 | self.manipulation = {} 114 | self.cost = {} 115 | self.parent = {} 116 | self.children = {} 117 | self.fullyExpanded = {} 118 | self.numberOfVisited = {} 119 | 120 | self.actions = {} 121 | self.usedActionsID = {} 122 | self.indexToActionID = {} 123 | 124 | # move one step forward 125 | # it means that we need to remove children other than the new root 126 | def makeOneMove(self, newRootIndex): 127 | if self.keypoint[newRootIndex] != 0: 128 | player = "the first player" 129 | else: 130 | player = "the second player" 131 | print("%s making a move into the new root %s, whose value is %s and visited number is %s" % ( 132 | player, newRootIndex, self.cost[newRootIndex], self.numberOfVisited[newRootIndex])) 133 | self.removeChildren(self.rootIndex, [newRootIndex]) 134 | self.rootIndex = newRootIndex 135 | 136 | def removeChildren(self, index, indicesToAvoid): 137 | if self.fullyExpanded[index] is True: 138 | for childIndex in self.children[index]: 139 | if childIndex not in indicesToAvoid: self.removeChildren(childIndex, []) 140 | self.manipulation.pop(index, None) 141 | self.cost.pop(index, None) 142 | self.parent.pop(index, None) 143 | self.keypoint.pop(index, None) 144 | self.children.pop(index, None) 145 | self.fullyExpanded.pop(index, None) 146 | self.numberOfVisited.pop(index, None) 147 | 148 | def bestChild(self, index): 149 | allValues = {} 150 | for childIndex in self.children[index]: 151 | allValues[childIndex] = float(self.numberOfVisited[childIndex]) / self.cost[childIndex] 152 | nprint("finding best children from %s" % allValues) 153 | # for cooperative 154 | return max(allValues.items(), key=operator.itemgetter(1))[0] 155 | 156 | def treeTraversal(self, index): 157 | if self.fullyExpanded[index] is True: 158 | nprint("tree traversal on node %s with children %s" % (index, self.children[index])) 159 | allValues = {} 160 | for childIndex in self.children[index]: 161 | # UCB values 162 | allValues[childIndex] = ((float(self.numberOfVisited[childIndex]) / self.cost[childIndex]) * self.eta[1] 163 | + explorationRate * math.sqrt( 164 | math.log(self.numberOfVisited[index]) / float(self.numberOfVisited[childIndex]))) 165 | 166 | # for cooperative 167 | nextIndex = np.random.choice(list(allValues.keys()), 1, 168 | p=[x / sum(allValues.values()) for x in allValues.values()])[0] 169 | 170 | if self.keypoint[index] in self.usedActionsID.keys() and self.keypoint[index] != 0: 171 | self.usedActionsID[self.keypoint[index]].append(self.indexToActionID[index]) 172 | elif self.keypoint[index] != 0: 173 | self.usedActionsID[self.keypoint[index]] = [self.indexToActionID[index]] 174 | 175 | return self.treeTraversal(nextIndex) 176 | 177 | else: 178 | nprint("tree traversal terminated on node %s" % index) 179 | availableActions = copy.deepcopy(self.actions) 180 | # for k in self.usedActionsID.keys(): 181 | # for i in self.usedActionsID[k]: 182 | # availableActions[k].pop(i, None) 183 | return index, availableActions 184 | 185 | def usefulAction(self, ampath, am): 186 | newAtomicManipulation = mergeTwoDicts(ampath, am) 187 | activations0 = self.moves.applyManipulation(self.image, ampath) 188 | (newClass0, newConfident0) = self.model.predict(activations0) 189 | activations1 = self.moves.applyManipulation(self.image, newAtomicManipulation) 190 | (newClass1, newConfident1) = self.model.predict(activations1) 191 | if abs(newConfident0 - newConfident1) < 10 ^ -6: 192 | return False 193 | else: 194 | return True 195 | 196 | def usefulActionNew(self, ampath, am, oldConfident): 197 | newAtomicManipulation = mergeTwoDicts(ampath, am) 198 | activations1 = self.moves.applyManipulation(self.image, newAtomicManipulation) 199 | dist = self.computeDistance(activations1) 200 | (newClass1, newConfident1) = self.model.predict(activations1) 201 | if abs(oldConfident - newConfident1) < 10 ^ -6: 202 | return (False, (newClass1, newConfident1), dist) 203 | else: 204 | return (True, (newClass1, newConfident1), dist) 205 | 206 | def initialiseExplorationNode(self, index, availableActions): 207 | nprint("expanding %s" % index) 208 | if self.keypoint[index] != 0: 209 | for (actionId, am) in availableActions[self.keypoint[index]].items(): 210 | if self.usefulAction(self.manipulation[index], am) == True: 211 | self.indexToNow += 1 212 | self.keypoint[self.indexToNow] = 0 213 | self.indexToActionID[self.indexToNow] = actionId 214 | self.initialiseLeafNode(self.indexToNow, index, am) 215 | self.children[index].append(self.indexToNow) 216 | else: 217 | for kp in list(set(self.keypoints.keys()) - set([0])): 218 | self.indexToNow += 1 219 | self.keypoint[self.indexToNow] = kp 220 | self.indexToActionID[self.indexToNow] = 0 221 | self.initialiseLeafNode(self.indexToNow, index, {}) 222 | self.children[index].append(self.indexToNow) 223 | 224 | self.fullyExpanded[index] = True 225 | self.usedActionsID = {} 226 | return self.children[index] 227 | 228 | def backPropagation(self, index, value): 229 | self.cost[index] += value 230 | self.numberOfVisited[index] += 1 231 | if self.parent[index] in self.parent: 232 | nprint("start backPropagating the value %s from node %s, whose parent node is %s" % ( 233 | value, index, self.parent[index])) 234 | self.backPropagation(self.parent[index], value) 235 | else: 236 | nprint("backPropagating ends on node %s" % index) 237 | 238 | # start random sampling and return the Euclidean value as the value 239 | def sampling(self, index, availableActions): 240 | nprint("start sampling node %s" % index) 241 | availableActions2 = copy.deepcopy(availableActions) 242 | sampleValues = [] 243 | i = 0 244 | for i in range(MCTS_multi_samples): 245 | self.atomicManipulationPath = self.manipulation[index] 246 | self.depth = 0 247 | self.availableActionIDs = {} 248 | for k in self.keypoints.keys(): 249 | self.availableActionIDs[k] = list(availableActions2[k].keys()) 250 | self.usedActionIDs = {} 251 | for k in self.keypoints.keys(): 252 | self.usedActionIDs[k] = [] 253 | activations1 = self.moves.applyManipulation(self.image, self.atomicManipulationPath) 254 | result = self.model.predict(activations1) 255 | dist = self.computeDistance(activations1) 256 | (childTerminated, val) = self.sampleNext(self.keypoint[index],result,dist) 257 | self.numOfSampling += 1 258 | sampleValues.append(val) 259 | i += 1 260 | return childTerminated, min(sampleValues) 261 | 262 | def computeDistance(self, newImage): 263 | (distMethod, _) = self.eta 264 | if distMethod == "L2": 265 | dist = l2Distance(newImage, self.image) 266 | elif distMethod == "L1": 267 | dist = l1Distance(newImage, self.image) 268 | elif distMethod == "Percentage": 269 | dist = diffPercent(newImage, self.image) 270 | elif distMethod == "NumDiffs": 271 | dist = diffPercent(newImage, self.image) * self.image.size 272 | return dist 273 | 274 | def sampleNext(self, k, newResult, dist): 275 | (newClass, newConfident) = newResult 276 | ''' 277 | if newClass != self.originalClass and newConfident > effectiveConfidenceWhenChanging: 278 | nprint("sampling a path ends in a terminal node with depth %s... " % self.depth) 279 | self.atomicManipulationPath = self.scrutinizePath(self.atomicManipulationPath) 280 | self.numAdv += 1 281 | nprint("current best %s, considered to be replaced by %s" % (self.bestCase[0], dist)) 282 | if self.bestCase[0] > dist: 283 | print("update best case from %s to %s" % (self.bestCase[0], dist)) 284 | self.numConverge += 1 285 | self.bestCase = (dist, self.atomicManipulationPath) 286 | path0 = "%s_pic/%s_currentBest_%s.png" % (self.data_set, self.image_index, self.numConverge) 287 | self.model.save_input(activations1, path0) 288 | return (True, newConfident) 289 | else: 290 | return (False, newConfident) 291 | ''' 292 | 293 | (distMethod, distVal) = self.eta 294 | 295 | # need not only class change, but also high confidence adversary examples 296 | if newClass != self.originalClass and newConfident > effectiveConfidenceWhenChanging: 297 | nprint("sampling a path ends in a terminal node with depth %s... " % self.depth) 298 | self.atomicManipulationPath = self.scrutinizePath(self.atomicManipulationPath) 299 | self.numAdv += 1 300 | nprint("current best %s, considered to be replaced by %s" % (self.bestCase[0], dist)) 301 | if self.bestCase[0] > dist: 302 | print("update best case from %s to %s" % (self.bestCase[0], dist)) 303 | self.numConverge += 1 304 | self.bestCase = (dist, self.atomicManipulationPath) 305 | path0 = "%s_pic/%s_Unsafe_currentBest_%s.png" % (self.data_set, self.image_index, self.numConverge) 306 | 307 | activations1 = self.moves.applyManipulation(self.image, self.atomicManipulationPath) 308 | self.model.save_input(activations1, path0) 309 | return (self.depth == 0, dist) 310 | 311 | elif dist > distVal: ########################## 312 | nprint("sampling a path ends by eta with depth %s ... " % self.depth) 313 | return (self.depth == 0, distVal) 314 | 315 | elif (not list(set(self.availableActionIDs[k]) - set(self.usedActionIDs[k]))) or len(self.availableActionIDs[k])==0: #################### 316 | print("sampling a path ends with depth %s because no more actions can be taken ... " % self.depth) 317 | return (self.depth == 0, distVal) 318 | 319 | # elif self.depth > (self.eta[1] / self.tau) * 2: 320 | # print( 321 | # "sampling a path ends with depth %s more than the prespecifided maximum sampling depth ... the largest distance is %s " % (self.depth,dist) ) 322 | # return (self.depth == 0, distVal) 323 | 324 | else: 325 | #print("continue sampling node ... ") 326 | # randomActionIndex = random.choice(list(set(self.availableActionIDs[k])-set(self.usedActionIDs[k]))) 327 | 328 | i = 0 329 | while True: 330 | 331 | randomActionIndex = random.choice(self.availableActionIDs[k]) 332 | 333 | if k == 0: 334 | nextAtomicManipulation = {} 335 | else: 336 | nextAtomicManipulation = self.actions[k][randomActionIndex] 337 | newResult = self.usefulActionNew(self.atomicManipulationPath,nextAtomicManipulation,newConfident) 338 | if nextAtomicManipulation == {} or i > 10 or newResult[0] or len(self.availableActionIDs[k])==0: 339 | #if(k!=0): 340 | #self.availableActionIDs[k].remove(randomActionIndex) 341 | #self.usedActionIDs[k].append(randomActionIndex) 342 | break 343 | 344 | i += 1 345 | 346 | 347 | newManipulationPath = mergeTwoDicts(self.atomicManipulationPath, nextAtomicManipulation) 348 | #activations2 = self.moves.applyManipulation(self.image, newManipulationPath) 349 | #(newClass2, newConfident2) = self.model.predict(activations2) 350 | 351 | self.atomicManipulationPath = newManipulationPath 352 | self.depth = self.depth + 1 353 | if k == 0: 354 | return self.sampleNext(randomActionIndex,newResult[1],newResult[2]) 355 | else: 356 | return self.sampleNext(0,newResult[1],newResult[2]) 357 | 358 | def scrutinizePath(self, manipulations): 359 | flag = False 360 | tempManipulations = copy.deepcopy(manipulations) 361 | for k, v in manipulations.items(): 362 | tempManipulations[k] = 0 363 | activations1 = self.moves.applyManipulation(self.image, tempManipulations) 364 | (newClass, newConfident) = self.model.predict(activations1) 365 | if newClass != self.originalClass: 366 | manipulations.pop(k) 367 | flag = True 368 | break 369 | 370 | if flag is True: 371 | return self.scrutinizePath(manipulations) 372 | else: 373 | return manipulations 374 | 375 | def terminalNode(self, index): 376 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 377 | (newClass, _) = self.model.predict(activations1) 378 | return newClass != self.originalClass 379 | 380 | def terminatedByEta(self, index): 381 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 382 | dist = self.computeDistance(activations1) 383 | nprint("terminated by controlled search: distance = %s" % dist) 384 | return dist > self.eta[1] 385 | 386 | def applyManipulation(self, manipulation): 387 | activations1 = self.moves.applyManipulation(self.image, manipulation) 388 | return activations1 389 | 390 | def l2Dist(self, index): 391 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 392 | return l2Distance(self.image, activations1) 393 | 394 | def l1Dist(self, index): 395 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 396 | return l1Distance(self.image, activations1) 397 | 398 | def l0Dist(self, index): 399 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 400 | return l0Distance(self.image, activations1) 401 | 402 | def diffImage(self, index): 403 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 404 | return diffImage(self.image, activations1) 405 | 406 | def diffPercent(self, index): 407 | activations1 = self.moves.applyManipulation(self.image, self.manipulation[index]) 408 | return diffPercent(self.image, activations1) 409 | -------------------------------------------------------------------------------- /DataCollection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | author: Xiaowei Huang 5 | """ 6 | 7 | import numpy as np 8 | import time 9 | import os 10 | import copy 11 | 12 | from basics import assure_path_exists 13 | 14 | 15 | class DataCollection: 16 | index = 0 17 | layer = 0 18 | fileHandler = 0 19 | 20 | def __init__(self, filenamePostfix): 21 | self.runningTime = {} 22 | self.manipulationPercentage = {} 23 | self.l2Distance = {} 24 | self.l1Distance = {} 25 | self.l0Distance = {} 26 | self.confidence = {} 27 | assure_path_exists("dataCollection/") 28 | self.fileName = "dataCollection/%s.txt" % filenamePostfix 29 | self.fileHandler = open(self.fileName, 'a') 30 | self.maxFeatures = {} 31 | 32 | def initialiseIndex(self, index): 33 | self.index = index 34 | 35 | def addMaxFeatures(self, max_feature): 36 | self.maxFeatures[self.index] = max_feature 37 | 38 | def addRunningTime(self, running_time): 39 | self.runningTime[self.index] = running_time 40 | 41 | def addConfidence(self, confidence): 42 | self.confidence[self.index] = confidence 43 | 44 | def addManipulationPercentage(self, mani_percentage): 45 | self.manipulationPercentage[self.index] = mani_percentage 46 | 47 | def addl2Distance(self, l2dist): 48 | self.l2Distance[self.index] = l2dist 49 | 50 | def addl1Distance(self, l1dist): 51 | self.l1Distance[self.index] = l1dist 52 | 53 | def addl0Distance(self, l0dist): 54 | self.l0Distance[self.index] = l0dist 55 | 56 | def addComment(self, comment): 57 | self.fileHandler.write(comment) 58 | 59 | def provideDetails(self): 60 | if not bool(self.maxFeatures): 61 | self.fileHandler.write("running time: \n") 62 | for i, r in self.runningTime.items(): 63 | self.fileHandler.write("%s:%s\n" % (i, r)) 64 | 65 | self.fileHandler.write("manipulation percentage: \n") 66 | for i, r in self.manipulationPercentage.items(): 67 | self.fileHandler.write("%s:%s\n" % (i, r)) 68 | 69 | self.fileHandler.write("L2 distance: \n") 70 | for i, r in self.l2Distance.items(): 71 | self.fileHandler.write("%s:%s\n" % (i, r)) 72 | 73 | self.fileHandler.write("L1 distance: \n") 74 | for i, r in self.l1Distance.items(): 75 | self.fileHandler.write("%s:%s\n" % (i, r)) 76 | 77 | self.fileHandler.write("L0 distance: \n") 78 | for i, r in self.l0Distance.items(): 79 | self.fileHandler.write("%s:%s\n" % (i, r)) 80 | 81 | self.fileHandler.write("confidence: \n") 82 | for i, r in self.confidence.items(): 83 | self.fileHandler.write("%s:%s\n" % (i, r)) 84 | self.fileHandler.write("\n") 85 | 86 | self.fileHandler.write("max features: \n") 87 | for i, r in self.maxFeatures.items(): 88 | self.fileHandler.write("%s:%s\n" % (i, r)) 89 | self.fileHandler.write("\n") 90 | else: 91 | self.fileHandler.write("none of the inputs were successfully manipulated.\n") 92 | 93 | def summarise(self): 94 | if len(self.manipulationPercentage) is 0: 95 | self.fileHandler.write("none of the images were successfully manipulated.\n") 96 | return 97 | else: 98 | # art = sum(self.runningTime.values()) / len(self.runningTime.values()) 99 | art = np.mean(list(self.runningTime.values())) 100 | self.fileHandler.write("average running time: %s\n" % art) 101 | # amp = sum(self.manipulationPercentage.values()) / len(self.manipulationPercentage.values()) 102 | amp = np.mean(list(self.manipulationPercentage.values())) 103 | self.fileHandler.write("average manipulation percentage: %s\n" % amp) 104 | # l2dist = sum(self.l2Distance.values()) / len(self.l2Distance.values()) 105 | l2dist = np.mean(list(self.l2Distance.values())) 106 | self.fileHandler.write("average Euclidean (L2) distance: %s\n" % l2dist) 107 | # l1dist = sum(self.l1Distance.values()) / len(self.l1Distance.values()) 108 | l1dist = np.mean(list(self.l1Distance.values())) 109 | self.fileHandler.write("average Manhattan (L1) distance: %s\n" % l1dist) 110 | # l0dist = sum(self.l0Distance.values()) / len(self.l0Distance.values()) 111 | l0dist = np.mean(list(self.l0Distance.values())) 112 | self.fileHandler.write("average Hamming (L0) distance: %s\n\n\n\n\n" % l0dist) 113 | 114 | def close(self): 115 | self.fileHandler.close() 116 | -------------------------------------------------------------------------------- /DataSet.py: -------------------------------------------------------------------------------- 1 | """ 2 | Construct a NeuralNetwork class to include operations 3 | related to various datasets and corresponding models. 4 | 5 | Author: Min Wu 6 | Email: min.wu@cs.ox.ac.uk 7 | """ 8 | 9 | import keras 10 | from keras.datasets import mnist, cifar10 11 | from skimage import io, color, exposure, transform 12 | import pandas as pd 13 | import numpy as np 14 | import h5py 15 | import os 16 | import glob 17 | 18 | 19 | # Define a Neural Network class. 20 | class DataSet: 21 | # Specify which dataset at initialisation. 22 | def __init__(self, data_set, trainOrTest): 23 | self.data_set = data_set 24 | 25 | # for a mnist model. 26 | if self.data_set == 'mnist': 27 | num_classes = 10 28 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 29 | img_rows, img_cols, img_chls = 28, 28, 1 30 | if trainOrTest == "training": 31 | x = x_train.reshape(x_train.shape[0], img_rows, img_cols, img_chls) 32 | y = keras.utils.to_categorical(y_train, num_classes) 33 | else: 34 | x = x_test.reshape(x_test.shape[0], img_rows, img_cols, img_chls) 35 | y = keras.utils.to_categorical(y_test, num_classes) 36 | 37 | x = x.astype('float32') 38 | x /= 255 39 | 40 | # for a cifar10 model. 41 | elif self.data_set == 'cifar10': 42 | num_classes = 10 43 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 44 | img_rows, img_cols, img_chls = 32, 32, 3 45 | if trainOrTest == "training": 46 | x = x_train.reshape(x_train.shape[0], img_rows, img_cols, img_chls) 47 | y = keras.utils.to_categorical(y_train, num_classes) 48 | else: 49 | x = x_test.reshape(x_test.shape[0], img_rows, img_cols, img_chls) 50 | y = keras.utils.to_categorical(y_test, num_classes) 51 | 52 | x = x.astype('float32') 53 | x /= 255 54 | 55 | # for a gtsrb model. 56 | elif self.data_set == 'gtsrb': 57 | num_classes = 43 58 | img_rows, img_cols, img_chls = 48, 48, 3 59 | if trainOrTest == "training": 60 | directory = 'models/GTSRB/Final_Training/' 61 | try: 62 | with h5py.File(directory + 'gtsrb_training.h5') as hf: 63 | x_train, y_train = hf['imgs'][:], hf['labels'][:] 64 | x = x_train.reshape(x_train.shape[0], img_rows, img_cols, img_chls) 65 | y = keras.utils.to_categorical(y_train, num_classes) 66 | 67 | except (IOError, OSError, KeyError): 68 | imgs = [] 69 | labels = [] 70 | all_img_paths = glob.glob(os.path.join(directory + 'Images/', '*/*.ppm')) 71 | np.random.shuffle(all_img_paths) 72 | for img_path in all_img_paths: 73 | try: 74 | img = self.preprocess_img(io.imread(img_path), img_rows, img_cols) 75 | label = self.get_class(img_path) 76 | imgs.append(img) 77 | labels.append(label) 78 | 79 | if len(imgs) % 1000 == 0: print("Processed {}/{}".format(len(imgs), len(all_img_paths))) 80 | except (IOError, OSError): 81 | print('missed', img_path) 82 | pass 83 | 84 | x_train = np.array(imgs, dtype='float32') 85 | y_train = np.array(labels, dtype='uint8') 86 | 87 | with h5py.File(directory + 'gtsrb_training.h5', 'w') as hf: 88 | hf.create_dataset('imgs', data=x_train) 89 | hf.create_dataset('labels', data=y_train) 90 | x = x_train.reshape(x_train.shape[0], img_rows, img_cols, img_chls) 91 | y = keras.utils.to_categorical(y_train, num_classes) 92 | 93 | else: 94 | directory = 'models/GTSRB/Final_Test/' 95 | try: 96 | with h5py.File(directory + 'gtsrb_test.h5') as hf: 97 | x_test, y_test = hf['imgs'][:], hf['labels'][:] 98 | x = x_test.reshape(x_test.shape[0], img_rows, img_cols, img_chls) 99 | y = keras.utils.to_categorical(y_test, num_classes) 100 | 101 | except (IOError, OSError, KeyError): 102 | test = pd.read_csv(directory + 'GT-final_test.csv', sep=';') 103 | x_test = [] 104 | y_test = [] 105 | for file_name, class_id in zip(list(test['Filename']), list(test['ClassId'])): 106 | img_path = os.path.join(directory + 'Images/', file_name) 107 | x_test.append(self.preprocess_img(io.imread(img_path), img_rows, img_cols)) 108 | y_test.append(class_id) 109 | 110 | x_test = np.array(x_test, dtype='float32') 111 | y_test = np.array(y_test, dtype='uint8') 112 | 113 | with h5py.File(directory + 'gtsrb_test.h5', 'w') as hf: 114 | hf.create_dataset('imgs', data=x_test) 115 | hf.create_dataset('labels', data=y_test) 116 | x = x_test.reshape(x_test.shape[0], img_rows, img_cols, img_chls) 117 | y = keras.utils.to_categorical(y_test, num_classes) 118 | 119 | else: 120 | print("Unsupported dataset %s. Try 'mnist' or 'cifar10'." % data_set) 121 | exit() 122 | 123 | self.x = x 124 | self.y = y 125 | 126 | # get dataset 127 | def get_dataset(self): 128 | return self.x, self.y 129 | 130 | def get_input(self, index): 131 | return self.x[index] 132 | 133 | def preprocess_img(self, img, img_rows, img_cols): 134 | # Histogram normalization in y 135 | hsv = color.rgb2hsv(img) 136 | hsv[:, :, 2] = exposure.equalize_hist(hsv[:, :, 2]) 137 | img = color.hsv2rgb(hsv) 138 | 139 | # central scrop 140 | min_side = min(img.shape[:-1]) 141 | centre = img.shape[0] // 2, img.shape[1] // 2 142 | img = img[centre[0] - min_side // 2:centre[0] + min_side // 2, 143 | centre[1] - min_side // 2:centre[1] + min_side // 2, :] 144 | 145 | # rescale to standard size 146 | img = transform.resize(img, (img_rows, img_cols)) 147 | 148 | # roll color axis to axis 0 149 | # img = np.rollaxis(img, -1) 150 | 151 | return img 152 | 153 | def get_class(self, img_path): 154 | return int(img_path.split('/')[-2]) 155 | -------------------------------------------------------------------------------- /FeatureExtraction.py: -------------------------------------------------------------------------------- 1 | """ 2 | Construct a FeatureExtraction class to retrieve 3 | 'key points', 'partitions', `saliency map' of an image 4 | in a black-box or grey-box pattern. 5 | 6 | Author: Min Wu 7 | Email: min.wu@cs.ox.ac.uk 8 | """ 9 | 10 | import copy 11 | import numpy as np 12 | import cv2 13 | import random 14 | from scipy.stats import norm 15 | from keras import backend as K 16 | from matplotlib import pyplot as plt 17 | 18 | 19 | # Define a Feature Extraction class. 20 | class FeatureExtraction: 21 | def __init__(self, pattern='grey-box'): 22 | self.PATTERN = pattern 23 | 24 | # black-box parameters 25 | self.IMG_ENLARGE_RATIO = 1 26 | self.IMAGE_SIZE_BOUND = 100 27 | self.MAX_NUM_OF_PIXELS_PER_KEY_POINT = 1000000 28 | 29 | # grey-box parameters 30 | self.NUM_PARTITION = 10 31 | self.PIXEL_BOUNDS = (0, 1) 32 | self.NUM_OF_PIXEL_MANIPULATION = 2 33 | 34 | # Get key points of an image. 35 | def get_key_points(self, image, num_partition=10): 36 | self.NUM_PARTITION = num_partition 37 | 38 | # Black-box pattern: get key points from SIFT, 39 | # enlarge the image if it is small. 40 | if self.PATTERN == 'black-box': 41 | image = copy.deepcopy(image) 42 | 43 | sift = cv2.xfeatures2d.SIFT_create() # cv2.SIFT() # cv2.SURF(400) 44 | 45 | if np.max(image) <= 1: 46 | image = (image * 255).astype(np.uint8) 47 | else: 48 | image = image.astype(np.uint8) 49 | 50 | if max(image.shape) < self.IMAGE_SIZE_BOUND: 51 | # For a small image, SIFT works by enlarging the image. 52 | image = cv2.resize(image, (0, 0), fx=self.IMG_ENLARGE_RATIO, fy=self.IMG_ENLARGE_RATIO) 53 | key_points, _ = sift.detectAndCompute(image, None) 54 | for i in range(len(key_points)): 55 | old_pt = (key_points[i].pt[0], key_points[i].pt[1]) 56 | key_points[i].pt = (int(old_pt[0] / self.IMG_ENLARGE_RATIO), 57 | int(old_pt[1] / self.IMG_ENLARGE_RATIO)) 58 | else: 59 | key_points, _ = sift.detectAndCompute(image, None) 60 | 61 | # Grey-box pattern: get key points from partition ID. 62 | elif self.PATTERN == 'grey-box': 63 | key_points = [key for key in range(self.NUM_PARTITION)] 64 | 65 | else: 66 | print("Unrecognised feature extraction pattern. " 67 | "Try 'black-box' or 'grey-box'.") 68 | 69 | return key_points 70 | 71 | # Get partitions of an image. 72 | def get_partitions(self, image, model=None, num_partition=10, pixel_bounds=(0, 1)): 73 | self.NUM_PARTITION = num_partition 74 | self.PIXEL_BOUNDS = pixel_bounds 75 | 76 | # Grey-box pattern: must specify a neural network. 77 | if self.PATTERN == 'grey-box' and model is None: 78 | print("For 'grey-box' feature extraction, please specify a neural network.") 79 | exit 80 | 81 | # Grey-box pattern: get partitions from saliency map. 82 | if self.PATTERN == 'grey-box': 83 | print("Extracting image features using '%s' pattern." % self.PATTERN) 84 | 85 | saliency_map = self.get_saliency_map(image, model) 86 | 87 | partitions = {} 88 | quotient, remainder = divmod(len(saliency_map), self.NUM_PARTITION) 89 | for key in range(self.NUM_PARTITION): 90 | partitions[key] = [(int(saliency_map[idx, 0]), int(saliency_map[idx, 1])) for idx in 91 | range(key * quotient, (key + 1) * quotient)] 92 | if key == self.NUM_PARTITION - 1: 93 | partitions[key].extend((int(saliency_map[idx, 0]), int(saliency_map[idx, 1])) for idx in 94 | range((key + 1) * quotient, len(saliency_map))) 95 | return partitions 96 | 97 | # Black-box pattern: get partitions from key points. 98 | elif self.PATTERN == 'black-box': 99 | print("Extracting image features using '%s' pattern." % self.PATTERN) 100 | 101 | key_points = self.get_key_points(image) 102 | print("%s keypoints are found. " % (len(key_points))) 103 | 104 | partitions = {} 105 | # For small images, such as MNIST, CIFAR10. 106 | if max(image.shape) < self.IMAGE_SIZE_BOUND: 107 | for x in range(max(image.shape)): 108 | for y in range(max(image.shape)): 109 | ps = 0 110 | maxk = -1 111 | for i in range(len(key_points)): 112 | k = key_points[i - 1] 113 | dist2 = np.linalg.norm(np.array([x, y]) - np.array([k.pt[0], k.pt[1]])) 114 | ps2 = norm.pdf(dist2, loc=0.0, scale=k.size) 115 | if ps2 > ps: 116 | ps = ps2 117 | maxk = i 118 | if maxk in partitions.keys(): 119 | partitions[maxk].append((x, y)) 120 | else: 121 | partitions[maxk] = [(x, y)] 122 | # If a partition gets too many pixels, randomly remove some pixels. 123 | if self.MAX_NUM_OF_PIXELS_PER_KEY_POINT > 0: 124 | for mk in partitions.keys(): 125 | begining_num = len(partitions[mk]) 126 | for i in range(begining_num - self.MAX_NUM_OF_PIXELS_PER_KEY_POINT): 127 | partitions[mk].remove(random.choice(partitions[mk])) 128 | return partitions 129 | # For large images, such as ImageNet. 130 | else: 131 | key_points = key_points[:200] 132 | each_num = max(image.shape) ** 2 / len(key_points) 133 | maxk = 1 134 | partitions[maxk] = [] 135 | for x in range(max(image.shape)): 136 | for y in range(max(image.shape)): 137 | if len(partitions[maxk]) <= each_num: 138 | partitions[maxk].append((x, y)) 139 | else: 140 | maxk += 1 141 | partitions[maxk] = [(x, y)] 142 | return partitions 143 | 144 | else: 145 | print("Unrecognised feature extraction pattern." 146 | "Try 'black-box' or 'grey-box'.") 147 | 148 | # Get saliency map of an image. 149 | def get_saliency_map(self, image, model, pixel_bounds=(0, 1)): 150 | self.PIXEL_BOUNDS = pixel_bounds 151 | 152 | image_class, _ = model.predict(image) 153 | 154 | new_pixel_list = np.linspace(self.PIXEL_BOUNDS[0], self.PIXEL_BOUNDS[1], self.NUM_OF_PIXEL_MANIPULATION) 155 | image_batch = np.kron(np.ones((self.NUM_OF_PIXEL_MANIPULATION, 1, 1, 1)), image) 156 | 157 | manipulated_images = [] 158 | (row, col, chl) = image.shape 159 | for i in range(0, row): 160 | for j in range(0, col): 161 | # need to be very careful about image.copy() 162 | changed_image_batch = image_batch.copy() 163 | for p in range(0, self.NUM_OF_PIXEL_MANIPULATION): 164 | changed_image_batch[p, i, j, :] = new_pixel_list[p] 165 | manipulated_images.append(changed_image_batch) # each loop append [pixel_num, row, col, chl] 166 | 167 | manipulated_images = np.asarray(manipulated_images) # [row*col, pixel_num, row, col, chl] 168 | manipulated_images = manipulated_images.reshape(row * col * self.NUM_OF_PIXEL_MANIPULATION, row, col, chl) 169 | 170 | # Use softmax logits instead of probabilities, 171 | # as probabilities may not reflect precise influence of one single pixel change. 172 | features_list = model.softmax_logits(manipulated_images) 173 | feature_change = features_list[:, image_class].reshape(-1, self.NUM_OF_PIXEL_MANIPULATION).transpose() 174 | 175 | min_indices = np.argmin(feature_change, axis=0) 176 | min_values = np.amin(feature_change, axis=0) 177 | min_idx_values = min_indices.astype('float32') / (self.NUM_OF_PIXEL_MANIPULATION - 1) 178 | 179 | [x, y] = np.meshgrid(np.arange(row), np.arange(col)) 180 | x = x.flatten('F') # to flatten in column-major order 181 | y = y.flatten('F') # to flatten in column-major order 182 | 183 | target_feature_list = np.hstack((np.split(x, len(x)), 184 | np.split(y, len(y)), 185 | np.split(min_values, len(min_values)), 186 | np.split(min_idx_values, len(min_idx_values)))) 187 | 188 | saliency_map = target_feature_list[target_feature_list[:, 2].argsort()] 189 | 190 | return saliency_map 191 | 192 | def plot_saliency_map(self, image, partitions, path): 193 | heatmap = np.zeros(image.shape[0:2]) 194 | for partitionID in partitions.keys(): 195 | pixels = partitions[partitionID] 196 | for pixel in pixels: 197 | heatmap[pixel] = partitionID + 1 198 | plt.imsave(path, heatmap) 199 | 200 | -------------------------------------------------------------------------------- /GameMoves.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | author: Xiaowei Huang 5 | revised by: Min Wu (min.wu@cs.ox.ac.uk) 6 | 7 | """ 8 | 9 | import sys 10 | import matplotlib.pyplot as plt 11 | import matplotlib.image as mpimg 12 | import cv2 13 | import numpy as np 14 | from keras import backend as K 15 | from scipy.stats import truncnorm, norm 16 | 17 | from basics import * 18 | from FeatureExtraction import * 19 | import collections 20 | 21 | 22 | ############################################################ 23 | # 24 | # initialise possible moves for a two-player game 25 | # 26 | ################################################################ 27 | 28 | 29 | class GameMoves: 30 | 31 | def __init__(self, data_set, model, image, tau, image_index): 32 | self.data_set = data_set 33 | self.model = model 34 | self.image = image 35 | self.tau = tau 36 | self.image_index = image_index 37 | self.maxVal = np.max(image) 38 | self.minVal = np.min(image) 39 | 40 | feature_extraction = FeatureExtraction(pattern='grey-box') 41 | kps = feature_extraction.get_key_points(self.image, num_partition=10) 42 | partitions = feature_extraction.get_partitions(self.image, self.model, num_partition=10) 43 | 44 | # path = "%s_pic/%s_Saliency_(%s).png" % (self.data_set, self.image_index, feature_extraction.PATTERN) 45 | # feature_extraction.plot_saliency_map(self.image, partitions=partitions, path=path) 46 | 47 | 48 | img_enlarge_ratio = 1 49 | image1 = copy.deepcopy(self.image) 50 | # if np.max(image1) <= 1: 51 | # image1 = (image1 * 255).astype(np.uint8) 52 | # else: 53 | # image1 = image1.astype(np.uint8) 54 | # 55 | # if max(image1.shape) < 100: 56 | # # for small images, sift works by enlarging the images 57 | # image1 = cv2.resize(image1, (0, 0), fx=img_enlarge_ratio, fy=img_enlarge_ratio) 58 | # kps = self.SIFT_Filtered_twoPlayer(image1) 59 | # for i in range(len(kps)): 60 | # oldpt = (kps[i].pt[0], kps[i].pt[1]) 61 | # kps[i].pt = (int(oldpt[0] / img_enlarge_ratio), int(oldpt[1] / img_enlarge_ratio)) 62 | # else: 63 | # kps = self.SIFT_Filtered_twoPlayer(image1) 64 | # 65 | # print("%s keypoints are found. " % (len(kps))) 66 | 67 | actions = dict() 68 | actions[0] = kps 69 | s = 1 70 | kp2 = [] 71 | 72 | if len(image1.shape) == 2: 73 | image0 = np.zeros(image1.shape) 74 | else: 75 | image0 = np.zeros(image1.shape[:2]) 76 | 77 | # to compute a partition of the pixels, for an image classification task 78 | # partitions = self.getPartition(image1, kps) 79 | print("The pixels are partitioned with respect to keypoints.") 80 | 81 | # construct moves according to the obtained the partitions 82 | num_of_manipulations = 0 83 | for k, blocks in partitions.items(): 84 | all_atomic_manipulations = [] 85 | 86 | for i in range(len(blocks)): 87 | x = blocks[i][0] 88 | y = blocks[i][1] 89 | 90 | (_, _, chl) = image1.shape 91 | 92 | # + tau 93 | if image0[x][y] == 0: 94 | 95 | atomic_manipulation = dict() 96 | for j in range(chl): 97 | atomic_manipulation[(x, y, j)] = self.tau 98 | all_atomic_manipulations.append(atomic_manipulation) 99 | 100 | atomic_manipulation = dict() 101 | for j in range(chl): 102 | atomic_manipulation[(x, y, j)] = -1 * self.tau 103 | all_atomic_manipulations.append(atomic_manipulation) 104 | 105 | image0[x][y] = 1 106 | 107 | # actions[k] = all_atomic_manipulations 108 | actions[s] = all_atomic_manipulations 109 | kp2.append(kps[s - 1]) 110 | 111 | s += 1 112 | # print("%s manipulations have been initialised for keypoint (%s,%s), whose response is %s." 113 | # % (len(all_atomic_manipulations), int(kps[k - 1].pt[0] / img_enlarge_ratio), 114 | # int(kps[k - 1].pt[1] / img_enlarge_ratio), kps[k - 1].response)) 115 | num_of_manipulations += len(all_atomic_manipulations) 116 | 117 | # index-0 keeps the keypoints, actual actions start from 1 118 | actions[0] = kp2 119 | print("the number of all manipulations initialised: %s\n" % num_of_manipulations) 120 | self.moves = actions 121 | 122 | def applyManipulation(self, image, manipulation): 123 | # apply a specific manipulation to have a manipulated input 124 | #image1 = copy.deepcopy(image) 125 | #maxVal = np.max(image1) 126 | #minVal = np.min(image1) 127 | #for elt in list(manipulation.keys()): 128 | # (fst, snd, thd) = elt 129 | # image1[fst][snd][thd] += manipulation[elt] 130 | # if image1[fst][snd][thd] < minVal: 131 | # image1[fst][snd][thd] = minVal 132 | # elif image1[fst][snd][thd] > maxVal: 133 | # image1[fst][snd][thd] = maxVal 134 | #return image1 135 | image1 = copy.deepcopy(image) 136 | if len(manipulation.keys())>0: 137 | a = np.array(list(manipulation.keys())) 138 | image1[a[:,0],a[:,1],a[:,2]]=np.clip(image1[a[:,0],a[:,1],a[:,2]] 139 | +np.array(list(manipulation.values())) 140 | ,self.maxVal,self.minVal) 141 | return image1 142 | else: 143 | return image1 144 | 145 | 146 | """ 147 | def SIFT_Filtered_twoPlayer(self, image): # threshold=0.0): 148 | sift = cv2.xfeatures2d.SIFT_create() # cv2.SIFT() # cv2.SURF(400) # 149 | kp, des = sift.detectAndCompute(image, None) 150 | return kp 151 | 152 | def getPartition(self, image, kps): 153 | # get partition by keypoints 154 | import operator 155 | import random 156 | max_num_of_pixels_per_key_point = 1000000 157 | blocks = {} 158 | if self.data_set != "imageNet": 159 | for x in range(max(image.shape)): 160 | for y in range(max(image.shape)): 161 | ps = 0 162 | maxk = -1 163 | for i in range(1, len(kps) + 1): 164 | k = kps[i - 1] 165 | dist2 = np.linalg.norm(np.array([x, y]) - np.array([k.pt[0], k.pt[1]])) 166 | ps2 = norm.pdf(dist2, loc=0.0, scale=k.size) 167 | if ps2 > ps: 168 | ps = ps2 169 | maxk = i 170 | if maxk in blocks.keys(): 171 | blocks[maxk].append((x, y)) 172 | else: 173 | blocks[maxk] = [(x, y)] 174 | if max_num_of_pixels_per_key_point > 0: 175 | for mk in blocks.keys(): 176 | begining_num = len(blocks[mk]) 177 | for i in range(begining_num - max_num_of_pixels_per_key_point): 178 | blocks[mk].remove(random.choice(blocks[mk])) 179 | return blocks 180 | else: 181 | kps = kps[:200] 182 | eachNum = max(image.shape) ** 2 / len(kps) 183 | maxk = 1 184 | blocks[maxk] = [] 185 | for x in range(max(image.shape)): 186 | for y in range(max(image.shape)): 187 | if len(blocks[maxk]) <= eachNum: 188 | blocks[maxk].append((x, y)) 189 | else: 190 | maxk += 1 191 | blocks[maxk] = [(x, y)] 192 | return blocks 193 | """ 194 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, TrustAI 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /NeuralNetwork.py: -------------------------------------------------------------------------------- 1 | """ 2 | Construct a NeuralNetwork class to include operations 3 | related to various datasets and corresponding models. 4 | 5 | Author: Min Wu 6 | Email: min.wu@cs.ox.ac.uk 7 | """ 8 | 9 | import cv2 10 | import copy 11 | import keras 12 | from keras.datasets import mnist, cifar10 13 | from keras.models import Sequential, load_model 14 | from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D 15 | from keras.preprocessing.image import ImageDataGenerator 16 | from keras.preprocessing import image as Image 17 | from keras import backend as K 18 | from matplotlib import pyplot as plt 19 | from tensorflow.python.keras.backend import eager_learning_phase_scope 20 | 21 | from basics import assure_path_exists 22 | from DataSet import * 23 | import numpy as np 24 | 25 | # Define a Neural Network class. 26 | class NeuralNetwork: 27 | # Specify which dataset at initialisation. 28 | def __init__(self, data_set): 29 | self.data_set = data_set 30 | self.model = Sequential() 31 | assure_path_exists("%s_pic/" % self.data_set) 32 | 33 | def predict(self, image): 34 | image = np.expand_dims(image, axis=0) 35 | predict_value = self.model.predict(image) 36 | new_class = np.argmax(np.ravel(predict_value)) 37 | confident = np.amax(np.ravel(predict_value)) 38 | return new_class, confident 39 | 40 | def predict_with_margin(self, image): 41 | image = np.expand_dims(image, axis=0) 42 | predict_value = self.model.predict(image) 43 | new_class = np.argmax(np.ravel(predict_value)) 44 | confident = np.amax(np.ravel(predict_value)) 45 | margin = confident - np.sort(np.ravel(predict_value))[-2] 46 | return new_class, confident, margin 47 | 48 | # To train a neural network. 49 | def train_network(self): 50 | # Train an mnist model. 51 | if self.data_set == 'mnist': 52 | batch_size = 128 53 | num_classes = 10 54 | epochs = 50 55 | img_rows, img_cols = 28, 28 56 | 57 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 58 | 59 | x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) 60 | x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) 61 | input_shape = (img_rows, img_cols, 1) 62 | 63 | x_train = x_train.astype('float32') 64 | x_test = x_test.astype('float32') 65 | x_train /= 255 66 | x_test /= 255 67 | 68 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 69 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 70 | 71 | model = Sequential() 72 | model.add(Conv2D(32, (3, 3), activation='relu', input_shape=input_shape)) 73 | model.add(Conv2D(32, (3, 3), activation='relu')) 74 | model.add(MaxPooling2D(pool_size=(2, 2))) 75 | model.add(Conv2D(64, (3, 3), activation='relu')) 76 | model.add(Conv2D(64, (3, 3), activation='relu')) 77 | model.add(MaxPooling2D(pool_size=(2, 2))) 78 | model.add(Flatten()) 79 | model.add(Dense(200, activation='relu')) 80 | model.add(Dropout(0.5)) 81 | model.add(Dense(200, activation='relu')) 82 | model.add(Dense(num_classes, activation='softmax')) 83 | 84 | model.compile(loss='categorical_crossentropy', 85 | optimizer=keras.optimizers.Adadelta(), 86 | metrics=['accuracy']) 87 | 88 | model.fit(x_train, y_train, 89 | batch_size=batch_size, 90 | epochs=epochs, 91 | verbose=1, 92 | validation_data=(x_test, y_test)) 93 | 94 | score = model.evaluate(x_test, y_test, verbose=0) 95 | print("Test loss:", score[0]) 96 | print("Test accuracy:", score[1]) 97 | 98 | self.model = model 99 | 100 | # Train a cifar10 model. 101 | elif self.data_set == 'cifar10': 102 | batch_size = 128 103 | num_classes = 10 104 | epochs = 50 105 | img_rows, img_cols, img_chls = 32, 32, 3 106 | data_augmentation = True 107 | 108 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 109 | 110 | x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, img_chls) 111 | x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, img_chls) 112 | input_shape = (img_rows, img_cols, img_chls) 113 | 114 | x_train = x_train.astype('float32') 115 | x_test = x_test.astype('float32') 116 | x_train /= 255 117 | x_test /= 255 118 | 119 | y_train = keras.utils.np_utils.to_categorical(y_train, num_classes) 120 | y_test = keras.utils.np_utils.to_categorical(y_test, num_classes) 121 | 122 | model = Sequential() 123 | model.add(Conv2D(64, (3, 3), activation='relu', input_shape=input_shape)) 124 | model.add(Conv2D(64, (3, 3), activation='relu')) 125 | model.add(MaxPooling2D(pool_size=(2, 2))) 126 | model.add(Conv2D(128, (3, 3), activation='relu')) 127 | model.add(Conv2D(128, (3, 3), activation='relu')) 128 | model.add(MaxPooling2D(pool_size=(2, 2))) 129 | model.add(Flatten()) 130 | model.add(Dense(256, activation='relu')) 131 | model.add(Dropout(0.5)) 132 | model.add(Dense(256, activation='relu')) 133 | model.add(Dense(num_classes, activation='softmax')) 134 | 135 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 136 | model.compile(loss='categorical_crossentropy', 137 | optimizer=opt, 138 | metrics=['accuracy']) 139 | 140 | if not data_augmentation: 141 | print("Not using data augmentation.") 142 | model.fit(x_train, y_train, 143 | batch_size=batch_size, 144 | epochs=epochs, 145 | validation_data=(x_test, y_test), 146 | shuffle=True) 147 | else: 148 | print("Using real-time data augmentation.") 149 | datagen = ImageDataGenerator( 150 | featurewise_center=False, 151 | samplewise_center=False, 152 | featurewise_std_normalization=False, 153 | samplewise_std_normalization=False, 154 | zca_whitening=False, 155 | rotation_range=0, 156 | width_shift_range=0.1, 157 | height_shift_range=0.1, 158 | horizontal_flip=True, 159 | vertical_flip=False) 160 | 161 | datagen.fit(x_train) 162 | model.fit_generator(datagen.flow(x_train, y_train, 163 | batch_size=batch_size), 164 | epochs=epochs, 165 | validation_data=(x_test, y_test), 166 | workers=4) 167 | 168 | scores = model.evaluate(x_test, y_test, verbose=0) 169 | print("Test loss:", scores[0]) 170 | print("Test accuracy:", scores[1]) 171 | 172 | self.model = model 173 | 174 | # Train a gtsrb model. 175 | elif self.data_set == 'gtsrb': 176 | batch_size = 128 177 | num_classes = 43 178 | epochs = 50 179 | img_rows, img_cols, img_chls = 48, 48, 3 180 | data_augmentation = True 181 | 182 | train = DataSet('gtsrb', 'training') 183 | x_train, y_train = train.x, train.y 184 | test = DataSet('gtsrb', 'test') 185 | x_test, y_test = test.x, test.y 186 | input_shape = (img_rows, img_cols, img_chls) 187 | 188 | model = Sequential() 189 | model.add(Conv2D(64, (3, 3), activation='relu', input_shape=input_shape)) 190 | model.add(Conv2D(64, (3, 3), activation='relu')) 191 | model.add(MaxPooling2D(pool_size=(2, 2))) 192 | model.add(Conv2D(128, (3, 3), activation='relu')) 193 | model.add(Conv2D(128, (3, 3), activation='relu')) 194 | model.add(MaxPooling2D(pool_size=(2, 2))) 195 | model.add(Flatten()) 196 | model.add(Dense(256, activation='relu')) 197 | model.add(Dropout(0.5)) 198 | model.add(Dense(256, activation='relu')) 199 | model.add(Dense(num_classes, activation='softmax')) 200 | 201 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 202 | model.compile(loss='categorical_crossentropy', 203 | optimizer=opt, 204 | metrics=['accuracy']) 205 | 206 | if not data_augmentation: 207 | print("Not using data augmentation.") 208 | model.fit(x_train, y_train, 209 | batch_size=batch_size, 210 | epochs=epochs, 211 | validation_data=(x_test, y_test), 212 | shuffle=True) 213 | else: 214 | print("Using real-time data augmentation.") 215 | datagen = ImageDataGenerator( 216 | featurewise_center=False, 217 | samplewise_center=False, 218 | featurewise_std_normalization=False, 219 | samplewise_std_normalization=False, 220 | zca_whitening=False, 221 | rotation_range=0, 222 | width_shift_range=0.1, 223 | height_shift_range=0.1, 224 | horizontal_flip=True, 225 | vertical_flip=False) 226 | 227 | datagen.fit(x_train) 228 | model.fit_generator(datagen.flow(x_train, y_train, 229 | batch_size=batch_size), 230 | epochs=epochs, 231 | validation_data=(x_test, y_test), 232 | workers=4) 233 | 234 | scores = model.evaluate(x_test, y_test, verbose=0) 235 | print("Test loss:", scores[0]) 236 | print("Test accuracy:", scores[1]) 237 | 238 | self.model = model 239 | 240 | else: 241 | print("Unsupported dataset %s. Try 'mnist' or 'cifar10' or 'gtsrb'." % self.data_set) 242 | self.save_network() 243 | 244 | # To save the neural network to disk. 245 | def save_network(self): 246 | if self.data_set == 'mnist': 247 | self.model.save('models/mnist.h5') 248 | print("Neural network saved to disk.") 249 | 250 | elif self.data_set == 'cifar10': 251 | self.model.save('models/cifar10.h5') 252 | print("Neural network saved to disk.") 253 | 254 | elif self.data_set == 'gtsrb': 255 | self.model.save('models/gtsrb.h5') 256 | print("Neural network saved to disk.") 257 | 258 | else: 259 | print("save_network: Unsupported dataset.") 260 | 261 | # To load a neural network from disk. 262 | def load_network(self): 263 | if self.data_set == 'mnist': 264 | self.model = load_model('models/mnist.h5') 265 | print("Neural network loaded from disk.") 266 | 267 | elif self.data_set == 'cifar10': 268 | self.model = load_model('models/cifar10.h5') 269 | print("Neural network loaded from disk.") 270 | 271 | elif self.data_set == 'gtsrb': 272 | try: 273 | self.model = load_model('models/gtsrb.h5') 274 | print("Neural network loaded from disk.") 275 | except (IOError, OSError): 276 | self.train_network() 277 | 278 | else: 279 | print("load_network: Unsupported dataset.") 280 | 281 | def save_input(self, image, filename): 282 | image = Image.array_to_img(image.copy()) 283 | plt.imsave(filename, image) 284 | # causes discrepancy 285 | # image_cv = copy.deepcopy(image) 286 | # cv2.imwrite(filename, image_cv * 255.0, [cv2.IMWRITE_PNG_COMPRESSION, 9]) 287 | 288 | def get_label(self, index): 289 | if self.data_set == 'mnist': 290 | labels = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] 291 | elif self.data_set == 'cifar10': 292 | labels = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] 293 | elif self.data_set == 'gtsrb': 294 | labels = ['speed limit 20 (prohibitory)', 'speed limit 30 (prohibitory)', 295 | 'speed limit 50 (prohibitory)', 'speed limit 60 (prohibitory)', 296 | 'speed limit 70 (prohibitory)', 'speed limit 80 (prohibitory)', 297 | 'restriction ends 80 (other)', 'speed limit 100 (prohibitory)', 298 | 'speed limit 120 (prohibitory)', 'no overtaking (prohibitory)', 299 | 'no overtaking (trucks) (prohibitory)', 'priority at next intersection (danger)', 300 | 'priority road (other)', 'give way (other)', 'stop (other)', 301 | 'no traffic both ways (prohibitory)', 'no trucks (prohibitory)', 302 | 'no entry (other)', 'danger (danger)', 'bend left (danger)', 303 | 'bend right (danger)', 'bend (danger)', 'uneven road (danger)', 304 | 'slippery road (danger)', 'road narrows (danger)', 'construction (danger)', 305 | 'traffic signal (danger)', 'pedestrian crossing (danger)', 'school crossing (danger)', 306 | 'cycles crossing (danger)', 'snow (danger)', 'animals (danger)', 307 | 'restriction ends (other)', 'go right (mandatory)', 'go left (mandatory)', 308 | 'go straight (mandatory)', 'go right or straight (mandatory)', 309 | 'go left or straight (mandatory)', 'keep right (mandatory)', 310 | 'keep left (mandatory)', 'roundabout (mandatory)', 311 | 'restriction ends (overtaking) (other)', 'restriction ends (overtaking (trucks)) (other)'] 312 | else: 313 | print("LABELS: Unsupported dataset.") 314 | return labels[index] 315 | 316 | # Get softmax logits, i.e., the inputs to the softmax function of the classification layer, 317 | # as softmax probabilities may be too close to each other after just one pixel manipulation. 318 | def softmax_logits(self, manipulated_images, batch_size=512): 319 | model = self.model 320 | 321 | func = K.function([model.layers[0].input], 322 | [model.layers[model.layers.__len__() - 1].output.op.inputs[0]]) 323 | 324 | # func = K.function([model.layers[0].input] + [K.learning_phase()], 325 | # [model.layers[model.layers.__len__() - 1].output]) 326 | 327 | if len(manipulated_images) >= batch_size: 328 | softmax_logits = [] 329 | batch, remainder = divmod(len(manipulated_images), batch_size) 330 | for b in range(batch): 331 | logits = func([manipulated_images[b * batch_size:(b + 1) * batch_size], 0])[0] 332 | softmax_logits.append(logits) 333 | softmax_logits = np.asarray(softmax_logits) 334 | softmax_logits = softmax_logits.reshape(batch * batch_size, model.output_shape[1]) 335 | # note that here if logits is empty, it is fine, as it won't be concatenated. 336 | logits = func([manipulated_images[batch * batch_size:len(manipulated_images)], 0])[0] 337 | softmax_logits = np.concatenate((softmax_logits, logits), axis=0) 338 | else: 339 | softmax_logits = func([manipulated_images, 0])[0] 340 | 341 | # softmax_logits = func([manipulated_images, 0])[0] 342 | # print(softmax_logits.shape) 343 | return softmax_logits 344 | -------------------------------------------------------------------------------- /Oxford-AIMS-CDT-SystemsVerification-HT2020.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/Oxford-AIMS-CDT-SystemsVerification-HT2020.pdf -------------------------------------------------------------------------------- /Oxford-AIMS-CDT-SystemsVerification.md: -------------------------------------------------------------------------------- 1 | # Oxford AIMS CDT Systems Verification (HT, 2020) 2 | 3 | ## Lab Session: DeepGame 4 | 5 | This is the webpage for the lab session of the __[AIMS CDT](https://eng.ox.ac.uk/aims-cdt/) Systems Verification__ course. The lecture slides of this course can be found [here](https://github.com/TrustAI/DeepGame/blob/master/Oxford-AIMS-CDT-SystemsVerification-HT2020.pdf). 6 | 7 | To better understand the tool __DeepGame__, please feel free to look into the accompanying published paper: 8 | [A game-based approximate verification of deep neural networks with provable guarantees](https://www.sciencedirect.com/science/article/pii/S0304397519304426). 9 | 10 | In general, DeepGame _verifies_ the robustness of deep neural networks via a two-player turn-based _game_. It solves two problems — the _maximum safe radius_ problem in a _cooperative_ game and the _feature robustness_ problem in a _competitive_ game. 11 | 12 | In this lab session, we primarily focus on the _maximum safe radius_ problem in a _cooperative_ game. Specifically, we look into three aspects: (1) search for adversarial examples, (2) generation of saliency maps, and (3) robustness guarantees of deep neural networks. 13 | 14 | We start with the installation of the DeepGame tool. 15 | 16 | 17 | 18 | ### 0. Installation of DeepGame 19 | 20 | Download the __DeepGame__ tool from https://github.com/minwu-cs/DeepGame. In order to run the tool, _Python_ and some other packages such as _keras_ and _numpy_ need to be installed. 21 | 22 | Below is a list of the developer's platform for reference in case there is package inconsistency. 23 | 24 | ###### Developer's Platform 25 | ``` 26 | python 3.5.5 27 | keras 2.1.5 28 | tensorflow 1.1.0 29 | numpy 1.14.2 30 | matplotlib 2.2.2 31 | scipy 1.0.1 32 | opencv-contrib-python 3.4.1.15 33 | scikit-image 0.14.0 34 | ``` 35 | 36 | Use the following command line to run DeepGame. 37 | 38 | ###### Run 39 | ``` 40 | python main.py mnist ub cooperative 67 L2 10 1 41 | ``` 42 | Here, `main.py` is the main Python file, in which you may find each parameter correspondingly. 43 | 44 | ```javascript 45 | dataSetName = 'mnist' 46 | bound = 'ub' 47 | gameType = 'cooperative' 48 | image_index = 67 49 | eta = ('L2', 10) 50 | tau = 1 51 | ``` 52 | 53 | Detailed explanation of each parameter is as follows. 54 | - `dataSetName` refers to the name of the dataset. Currently DeepGame supports three image datasets `mnist`, `cifar10`, and `gtsrb`. 55 | - `bound` denotes whether it is an upper bound `ub` or a lower bound `lb`. 56 | - `gameType` indicates the type of the game. Specifically, a `cooperative` game is to compute the _maximum safe radius_ whereas a `competitive` game is to compute the _feature robustness_. 57 | - `image_index` is the index of the image in the dataset. 58 | - `eta` determines the distance metric and the distance budget. For the metric, `L2` is short for the _L2 norm_, i.e., the Euclidean distance. Use `L1` for the _L1 norm_ (Manhattan distance), or `L0` for the Hamming distance. In this case, `10` is a possible distance budget. 59 | - `tau` denotes the value of _atomic manipulation_ imposed on each dimension of the input, i.e., each pixel/channel of an image. 60 | 61 | 62 | Alternatively, you may run DeepGame via the following command line, which allows a sequential execution of the above command line. 63 | ``` 64 | ./commands.sh 65 | ``` 66 | Within the `commands.sh` file, you may set the parameter values as needed. 67 | ```javascript 68 | for i in {0..10} 69 | do 70 | python main.py mnist ub cooperative $i L2 10 1 71 | python main.py mnist lb cooperative $i L2 0.01 1 72 | done 73 | exit 0 74 | ``` 75 | 76 | ------------------- 77 | 78 | 79 | 80 | 81 | ### 1. Search for Adversarial Examples 82 | 83 | An _adversarial example_ is an input which, though initially classified correctly by a neural network, is misclassified after a minor, perhaps imperceptible, perturbation. 84 | 85 | In DeepGame, this minor perturbation is set by the parameter `tau`, which imposes an _atomic manipulation_ on each pixel/channel of an input image. After pre-processing of the image datasets, all pixel values are normalised from [0,255] to [0,1], therefore we set the `tau` value from `(0,1]`. 86 | 87 | To search for adversarial examples, we let the two players work in a _cooperative_ game and utilise the _Monte Carlo tree search_ algorithm. From the original image as the root, the game tree expands when the two players proceed in a turn-based way, where Player I chooses a feature of an input to perturb and then Player II determines the atomic manipulations within this chosen feature. 88 | 89 | The _termination condition_ for the game tree is that either an adversarial example is found or a distance budget based on a certain metric `eta` is reached. Note that the _distance budget_ should be a reasonable value because if perturbations imposed on the input are too much to the extent that even humans are not able to distinguish, then it is no longer sensible to require a neural network to classify correctly. 90 | 91 | When the execution of DeepGame proceeds, improved adversarial examples in the sense of with fewer and fewer modifications are generated. 92 | 93 | #### Questions: 94 | 95 | > 1. Produce some adversarial examples on the MNIST dataset via utilising the _Monte Carlo tree search_ algorithm. 96 | > _Requirements: (1) try image index 67 of the MNIST dataset; (2) based on the L1 norm and set the distance budget as 10; (3) let the atomic manipulation value be 1._ 97 | 98 | 99 | Below are some adversarial examples of the MNIST, CIFAR-10, and GTSRB datasets when the metric is the L2 norm. 100 | 101 | ![alt text](figures/Adversary.png) 102 | 103 | 104 | 105 | 106 | 123 | 124 | ------------------- 125 | 126 | 127 | 128 | 129 | 130 | ### 2. Generation of Saliency Maps 131 | 132 | To facilitate the explainability and the interpretability of the deep neural networks, DeepGame can generate the _saliency map_ of an input point, to better demonstrate how a network model actually 'sees' or 'understands' an image. 133 | 134 | To generate the saliency map, DeepGame utilises two _feature extraction_ methods to partition the input into a disjoint set of dimensions with varying saliency levels. One method is called the _grey-box_ feature extraction, which exploits the output logits/probabilities of a classifier to discover the more sensitive input dimensions, whereas the other approach is _black-box_ as it does not reply on networks but instead directly applies the scale-invariant feature transform (SIFT) on the input locally. 135 | 136 | In DeepGame, these feature extraction methods are defined in the `FeatureExtraction.py` file. You may set a specific `grey-box` or `black-box` pattern via the following code line in the `GameMoves.py` file. 137 | 138 | ```javascript 139 | feature_extraction = FeatureExtraction(pattern='grey-box') 140 | ``` 141 | In order to output the heatmap of the saliency map, you need to uncomment these two lines in `GameMoves.py`. 142 | 143 | ```javascript 144 | # path = "%s_pic/%s_Saliency_(%s).png" % (self.data_set, self.image_index, feature_extraction.PATTERN) 145 | # feature_extraction.plot_saliency_map(self.image, partitions=partitions, path=path) 146 | ``` 147 | 148 | 149 | #### Questions: 150 | 151 | > 2. Generate a saliency map of an image with the _grey-box_ feature extraction method. 152 | > _Requirements: (1) try the same MNIST image with index 67; (2) compare the saliency map with the original image._ 153 | 154 | > 3. Change the feature extraction method from grey-box to _black-box_, and generate another saliency map of the image. 155 | > _Requirements: (1) use the same MNIST image with index 67 and keep the other parameters unchanged; (2) compare these two saliency maps with the original image._ 156 | 157 | Below exhibits two saliency maps of a CIFAR-10 image and a GTSRB image from the grey-box extraction method. 158 | 159 | ![alt text](figures/Feature.png) 160 | 161 | ------------------- 162 | 163 | 164 | 165 | 166 | 167 | ### 3. Robustness Guarantees of Deep Neural Networks 168 | 169 | Apart from searching for adversarial examples and generating saliency maps, our tool DeepGame is more importantly a verification tool that can _verify deep neural networks with provable guarantees_. 170 | 171 | Regarding this, we compute the _maximum safe radius of a neural network with respect to an input_, which is a distance such that, with imposed perturbations below the distance, all the input points are safe, whereas if above the distance, there definitely exists an adversarial example. 172 | 173 | While the number of possible input points is infinite, the maximum safe radius can be approximated by descretising the input space under the assumption of _Lipschitz continuity_. Intuitively, it is sufficient to consider a finite number of uniformly sampled inputs on the grid when the distance between the grid points is small. Here, the distance between grid points is reflected in the parameter `tau`, which sets the atomic manipulation. 174 | 175 | As computing the maximum safe radius directly is NP-hard, we compute the _lower and upper bounds_ of it and show the convergence trend. In other words, every _lower bound_ guarantees that all the possible perturbations below it are safe, while every _upper bound_ indicates the existence of an adversarial example. That said, every adversarial example generated from the _Monte Carlo tree search_ in Question 1 contributes to an upper bound. As for the lower bound, DeepGame utilises the _Admissible A*_ algorithm in a cooperative game and the _Alpha-Beta Pruning_ algorithm in a competitive game. 176 | 177 | 178 | #### Questions: 179 | > 4. Plot a figure to illustrate the convergence of the lower and upper bounds of the maximum safe radius. 180 | > _Requirements: (1) try the same MNIST image with index 67; (2) based on the Euclidean distance; (3) for the upper bound, set the distance budget as 10; (4) for the lower bound, set the distance budget as 0.01; (5) set the atomic manipulation as 1; (6) use grey-box feature extraction method._ 181 | 182 | > 5. Exhibit some safe perturbations imposed on the original image corresponding to the lower bounds, and also some adversarial examples generated as a by-product when computing the upper bounds. 183 | 184 | > 6. Change the value of _atomic manipulation_ in the range of `(0,1]`, for example, `0.5`, `0.05`, or `0.005`, and observe its influence on the lower bound. 185 | 186 | > 7. Evaluate the robustness of a neural network trained on another dataset. Plot the convergence of the bounds and display the safe and unsafe adversarial perturbations. 187 | > _Requirements: (1) try an image from the CIFAR-10 dataset with index from 0 to 99._ 188 | 189 | 190 | Below suggests a possible solution to the above Questions 4 and 5. 191 | 192 | ![alt text](figures/Cooperative_MNIST.png) 193 | 194 | ------------------- 195 | 196 | 197 | 198 | 199 | 200 | ### Citation of DeepGame: 201 | 202 | Below is the citation of the accompanying paper: 203 | 204 | [A game-based approximate verification of deep neural networks with provable guarantees](https://www.sciencedirect.com/science/article/pii/S0304397519304426). 205 | 206 | ``` 207 | @article{wu2020game, 208 | title = "A Game-Based Approximate Verification of Deep Neural Networks with Provable Guarantees", 209 | author = "Wu, Min and Wicker, Matthew and Ruan, Wenjie and Huang, Xiaowei and Kwiatkowska, Marta", 210 | journal = "Theoretical Computer Science", 211 | volume = "807", 212 | pages = "298 - 329", 213 | year = "2020", 214 | note = "In memory of Maurice Nivat, a founding father of Theoretical Computer Science - Part II", 215 | issn = "0304-3975", 216 | doi = "https://doi.org/10.1016/j.tcs.2019.05.046", 217 | url = "http://www.sciencedirect.com/science/article/pii/S0304397519304426" 218 | } 219 | ``` 220 | 221 | 222 | 223 | 224 | 225 | ### Remark 226 | 227 | This webpage is for the lab session of the AIMS CDT Systems Verification course. Should you have any questions, please feel free to contact the teaching assistant __Min Wu__ via min.wu@cs.ox.ac.uk. 228 | 229 | Best wishes, 230 | 231 | Min 232 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeepGame (A Game-Based Approximate Verification of Deep Neural Networks with Provable Guarantees) 2 | 3 | **Min Wu, Matthew Wicker, Wenjie Ruan, Xiaowei Huang, Marta Kwiatkowska.** 4 | 5 | The accompanying paper [A game-based approximate verification of deep neural networks with provable guarantees](https://www.sciencedirect.com/science/article/pii/S0304397519304426) is accepted by Theoretical Computer Science and available online since 2019. 6 | 7 | #### Citation 8 | ``` 9 | @article{wu2020game, 10 | title = "A Game-Based Approximate Verification of Deep Neural Networks with Provable Guarantees", 11 | author = "Wu, Min and Wicker, Matthew and Ruan, Wenjie and Huang, Xiaowei and Kwiatkowska, Marta", 12 | journal = "Theoretical Computer Science", 13 | volume = "807", 14 | pages = "298 - 329", 15 | year = "2020", 16 | note = "In memory of Maurice Nivat, a founding father of Theoretical Computer Science - Part II", 17 | issn = "0304-3975", 18 | doi = "https://doi.org/10.1016/j.tcs.2019.05.046", 19 | url = "http://www.sciencedirect.com/science/article/pii/S0304397519304426" 20 | } 21 | ``` 22 | 23 | # Abstract 24 | Despite the improved accuracy of deep neural networks, the discovery of adversarial examples has raised serious safety concerns. In this paper, we study two variants of pointwise robustness, the maximum safe radius problem, which for a given input sample computes the minimum distance to an adversarial example, and the feature robustness problem, which aims to quantify the robustness of individual features to adversarial perturbations. We demonstrate that, under the assumption of Lipschitz continuity, both problems can be approximated using finite optimisation by discretising the input space, and the approximation has provable guarantees, i.e., the error is bounded. We then show that the resulting optimisation problems can be reduced to the solution of two-player turn-based games, where the first player selects features and the second perturbs the image within the feature. While the second player aims to minimise the distance to an adversarial example, depending on the optimisation objective the first player can be cooperative or competitive. We employ an anytime approach to solve the games, in the sense of approximating the value of a game by monotonically improving its upper and lower bounds. The Monte Carlo tree search algorithm is applied to compute upper bounds for both games, and the Admissible A* and the Alpha-Beta Pruning algorithms are, respectively, used to compute lower bounds for the maximum safety radius and feature robustness games. When working on the upper bound of the maximum safe radius problem, our tool demonstrates competitive performance against existing adversarial example crafting algorithms. Furthermore, we show how our framework can be deployed to evaluate pointwise robustness of neural networks in safety-critical applications such as traffic sign recognition in self-driving cars. 25 | 26 | # Problem Statement 27 | ![alt text](figures/MSR.png) 28 | ![alt text](figures/FR.png) 29 | 30 | # Approach Architecture 31 | ![alt text](figures/Architecture.png) 32 | ![alt text](figures/Lipschitz.png) 33 | ![alt text](figures/Game.png) 34 | 35 | # Convergence Results 36 | ![alt text](figures/Cooperative_MNIST.png) 37 | ![alt text](figures/Cooperative_GTSRB.png) 38 | ![alt text](figures/Feature.png) 39 | ![alt text](figures/Competitive_CIFAR10.png) 40 | ![alt text](figures/Competitive_GTSRB.png) 41 | 42 | # Adversarial Examples 43 | ![alt text](figures/Adversary.png) 44 | ![alt text](figures/Adv_MNIST.png) 45 | ![alt text](figures/Adv_CIFAR10.png) 46 | ![alt text](figures/Adv_GTSRB.png) 47 | 48 | # Developer's Platform 49 | ``` 50 | python 3.5.5 51 | keras 2.1.3 52 | tensorflow-gpu 1.4.0 53 | numpy 1.14.3 54 | matplotlib 2.2.2 55 | scipy 1.1.0 56 | ``` 57 | 58 | # Run 59 | ``` 60 | python main.py mnist ub cooperative 67 L2 10 1 61 | ``` 62 | or 63 | ``` 64 | ./commands.sh 65 | ``` 66 | 67 | # Remark 68 | This tool is under active development and maintenance, please feel free to contact us about any problem encountered. 69 | 70 | Best wishes, 71 | 72 | xiaowei.huang@cs.ox.ac.uk 73 | 74 | min.wu@cs.ox.ac.uk 75 | -------------------------------------------------------------------------------- /basics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | import numpy as np 5 | import math 6 | import time 7 | import os 8 | import copy 9 | 10 | 11 | def assure_path_exists(path): 12 | directory = os.path.dirname(path) 13 | if not os.path.exists(directory): 14 | os.makedirs(directory) 15 | 16 | 17 | def current_milli_time(): 18 | return int(round(time.time() * 1000) % 4294967296) 19 | 20 | 21 | def diffImage(image1, image2): 22 | return list(zip(*np.nonzero(np.subtract(image1, image2)))) 23 | 24 | 25 | def diffPercent(image1, image2): 26 | return len(diffImage(image1, image2)) / float(image1.size) 27 | 28 | 29 | def numDiffs(image1, image2): 30 | return len(diffImage(image1, image2)) 31 | 32 | 33 | def l2Distance(image1, image2): 34 | return math.sqrt(np.sum(np.square(np.subtract(image1, image2)))) 35 | 36 | 37 | def l1Distance(image1, image2): 38 | return np.sum(np.absolute(np.subtract(image1, image2))) 39 | 40 | 41 | def l0Distance(image1, image2): 42 | return np.count_nonzero(np.absolute(np.subtract(image1, image2))) 43 | 44 | 45 | def mergeTwoDicts(x, y): 46 | z = x.copy() 47 | for key, value in y.items(): 48 | if key in z.keys(): 49 | z[key] += y[key] 50 | else: 51 | z[key] = y[key] 52 | # z.update(y) 53 | return z 54 | 55 | 56 | def nprint(str): 57 | return 0 58 | 59 | 60 | def printDict(dictionary): 61 | for key, value in dictionary.items(): 62 | print("%s : %s" % (key, value)) 63 | print("\n") 64 | -------------------------------------------------------------------------------- /commands.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for i in {0..10} 3 | do 4 | python main.py mnist ub cooperative $i L2 10 1 5 | python main.py mnist lb cooperative $i L2 0.01 1 6 | done 7 | exit 0 8 | -------------------------------------------------------------------------------- /figures/Adv_CIFAR10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Adv_CIFAR10.png -------------------------------------------------------------------------------- /figures/Adv_GTSRB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Adv_GTSRB.png -------------------------------------------------------------------------------- /figures/Adv_MNIST.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Adv_MNIST.png -------------------------------------------------------------------------------- /figures/Adversary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Adversary.png -------------------------------------------------------------------------------- /figures/Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Architecture.png -------------------------------------------------------------------------------- /figures/Competitive_CIFAR10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Competitive_CIFAR10.png -------------------------------------------------------------------------------- /figures/Competitive_GTSRB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Competitive_GTSRB.png -------------------------------------------------------------------------------- /figures/Cooperative_GTSRB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Cooperative_GTSRB.png -------------------------------------------------------------------------------- /figures/Cooperative_MNIST.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Cooperative_MNIST.png -------------------------------------------------------------------------------- /figures/FR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/FR.png -------------------------------------------------------------------------------- /figures/Feature.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Feature.png -------------------------------------------------------------------------------- /figures/Game.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Game.png -------------------------------------------------------------------------------- /figures/Lipschitz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/Lipschitz.png -------------------------------------------------------------------------------- /figures/MSR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/figures/MSR.png -------------------------------------------------------------------------------- /lowerbound.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Construct a 'lowerbound' function to compute 5 | the lower bound of Player I’s minimum adversary distance 6 | while Player II being cooperative, or Player I's maximum 7 | adversary distance whilst Player II being competitive. 8 | 9 | Author: Min Wu 10 | Email: min.wu@cs.ox.ac.uk 11 | """ 12 | 13 | from CooperativeAStar import * 14 | from CompetitiveAlphaBeta import * 15 | from NeuralNetwork import * 16 | from DataSet import * 17 | 18 | 19 | def lowerbound(dataset_name, image_index, game_type, eta, tau): 20 | NN = NeuralNetwork(dataset_name) 21 | NN.load_network() 22 | print("Dataset is %s." % NN.data_set) 23 | NN.model.summary() 24 | 25 | dataset = DataSet(dataset_name, 'testing') 26 | image = dataset.get_input(image_index) 27 | (label, confidence) = NN.predict(image) 28 | label_str = NN.get_label(int(label)) 29 | print("Working on input with index %s, whose class is '%s' and the confidence is %s." 30 | % (image_index, label_str, confidence)) 31 | print("The second player is being %s." % game_type) 32 | 33 | path = "%s_pic/idx_%s_label_[%s]_with_confidence_%s.png" % ( 34 | dataset_name, image_index, label_str, confidence) 35 | NN.save_input(image, path) 36 | 37 | if game_type == 'cooperative': 38 | tic = time.time() 39 | cooperative = CooperativeAStar(dataset_name, image_index, image, NN, eta, tau) 40 | cooperative.play_game(image) 41 | if cooperative.ADVERSARY_FOUND is True: 42 | elapsed = time.time() - tic 43 | adversary = cooperative.ADVERSARY 44 | adv_label, adv_confidence = NN.predict(adversary) 45 | adv_label_str = NN.get_label(int(adv_label)) 46 | 47 | print("\nFound an adversary within pre-specified bounded computational resource. " 48 | "\nThe following is its information: ") 49 | print("difference between images: %s" % (diffImage(image, adversary))) 50 | l2dist = l2Distance(image, adversary) 51 | l1dist = l1Distance(image, adversary) 52 | l0dist = l0Distance(image, adversary) 53 | percent = diffPercent(image, adversary) 54 | print("L2 distance %s" % l2dist) 55 | print("L1 distance %s" % l1dist) 56 | print("L0 distance %s" % l0dist) 57 | print("manipulated percentage distance %s" % percent) 58 | print("class is changed into '%s' with confidence %s\n" % (adv_label_str, adv_confidence)) 59 | 60 | path = "%s_pic/idx_%s_modified_into_[%s]_with_confidence_%s.png" % ( 61 | dataset_name, image_index, adv_label_str, adv_confidence) 62 | NN.save_input(adversary, path) 63 | if eta[0] == 'L0': 64 | dist = l0dist 65 | elif eta[0] == 'L1': 66 | dist = l1dist 67 | elif eta[0] == 'L2': 68 | dist = l2dist 69 | else: 70 | print("Unrecognised distance metric.") 71 | path = "%s_pic/idx_%s_modified_diff_%s=%s_time=%s.png" % ( 72 | dataset_name, image_index, eta[0], dist, elapsed) 73 | NN.save_input(np.absolute(image - adversary), path) 74 | else: 75 | print("Adversarial distance exceeds distance budget.") 76 | 77 | elif game_type == 'competitive': 78 | competitive = CompetitiveAlphaBeta(image, NN, eta, tau) 79 | competitive.play_game(image) 80 | 81 | else: 82 | print("Unrecognised game type. Try 'cooperative' or 'competitive'.") 83 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras import backend as K 3 | import sys 4 | import os 5 | sys.setrecursionlimit(10000) 6 | from NeuralNetwork import * 7 | from DataSet import * 8 | from DataCollection import * 9 | from upperbound import upperbound 10 | from lowerbound import lowerbound 11 | 12 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 13 | 14 | # the first way of defining parameters 15 | if len(sys.argv) == 8: 16 | 17 | if sys.argv[1] == 'mnist' or sys.argv[1] == 'cifar10' or sys.argv[1] == 'gtsrb': 18 | dataSetName = sys.argv[1] 19 | else: 20 | print("please specify as the 1st argument the dataset: mnist or cifar10 or gtsrb") 21 | exit 22 | 23 | if sys.argv[2] == 'ub' or sys.argv[2] == 'lb': 24 | bound = sys.argv[2] 25 | else: 26 | print("please specify as the 2nd argument the bound: ub or lb") 27 | exit 28 | 29 | if sys.argv[3] == 'cooperative' or sys.argv[3] == 'competitive': 30 | gameType = sys.argv[3] 31 | else: 32 | print("please specify as the 3nd argument the game mode: cooperative or competitive") 33 | exit 34 | 35 | if isinstance(int(sys.argv[4]), int): 36 | image_index = int(sys.argv[4]) 37 | else: 38 | print("please specify as the 4th argument the index of the image: [int]") 39 | exit 40 | 41 | if sys.argv[5] == 'L0' or sys.argv[5] == 'L1' or sys.argv[5] == 'L2': 42 | distanceMeasure = sys.argv[5] 43 | else: 44 | print("please specify as the 5th argument the distance measure: L0, L1, or L2") 45 | exit 46 | 47 | if isinstance(float(sys.argv[6]), float): 48 | distance = float(sys.argv[6]) 49 | else: 50 | print("please specify as the 6th argument the distance: [int/float]") 51 | exit 52 | eta = (distanceMeasure, distance) 53 | 54 | if isinstance(float(sys.argv[7]), float): 55 | tau = float(sys.argv[7]) 56 | else: 57 | print("please specify as the 7th argument the tau: [int/float]") 58 | exit 59 | 60 | elif len(sys.argv) == 1: 61 | # the second way of defining parameters 62 | dataSetName = 'cifar10' 63 | bound = 'lb' 64 | gameType = 'cooperative' 65 | image_index = 213 66 | eta = ('L2', 10) 67 | tau = 1 68 | 69 | # calling algorithms 70 | # dc = DataCollection("%s_%s_%s_%s_%s_%s_%s" % (dataSetName, bound, tau, gameType, image_index, eta[0], eta[1])) 71 | # dc.initialiseIndex(image_index) 72 | 73 | if bound == 'ub': 74 | (elapsedTime, newConfident, percent, l2dist, l1dist, l0dist, maxFeatures) = upperbound(dataSetName, bound, tau, 75 | gameType, image_index, eta) 76 | 77 | # dc.addRunningTime(elapsedTime) 78 | # dc.addConfidence(newConfident) 79 | # dc.addManipulationPercentage(percent) 80 | # dc.addl2Distance(l2dist) 81 | # dc.addl1Distance(l1dist) 82 | # dc.addl0Distance(l0dist) 83 | # dc.addMaxFeatures(maxFeatures) 84 | 85 | elif bound == 'lb': 86 | lowerbound(dataSetName, image_index, gameType, eta, tau) 87 | 88 | else: 89 | print("Unrecognised bound setting.\n" 90 | "Try 'ub' for upper bound or 'lb' for lower bound.\n") 91 | exit 92 | 93 | # dc.provideDetails() 94 | # dc.summarise() 95 | # dc.close() 96 | 97 | K.clear_session() 98 | -------------------------------------------------------------------------------- /models/cifar10.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/models/cifar10.h5 -------------------------------------------------------------------------------- /models/mnist.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TrustAI/DeepGame/b53f5fefddcef80197be4706015d6e7a40f8bc60/models/mnist.h5 -------------------------------------------------------------------------------- /upperbound.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from NeuralNetwork import * 3 | from DataSet import * 4 | from CompetitiveMCTS import * 5 | from CooperativeMCTS import * 6 | 7 | 8 | def upperbound(dataSetName, bound, tau, gameType, image_index, eta): 9 | start_time = time.time() 10 | 11 | MCTS_all_maximal_time = 300 12 | MCTS_level_maximal_time = 60 13 | 14 | NN = NeuralNetwork(dataSetName) 15 | NN.load_network() 16 | print("Dataset is %s." % NN.data_set) 17 | NN.model.summary() 18 | 19 | dataset = DataSet(dataSetName, 'testing') 20 | image = dataset.get_input(image_index) 21 | (label, confident) = NN.predict(image) 22 | origClassStr = NN.get_label(int(label)) 23 | print("Working on input with index %s, whose class is '%s' and the confidence is %s." 24 | % (image_index, origClassStr, confident)) 25 | print("the second player is %s." % gameType) 26 | 27 | # tau = 1 28 | 29 | # choose between "cooperative" and "competitive" 30 | if gameType == 'cooperative': 31 | mctsInstance = MCTSCooperative(dataSetName, NN, image_index, image, tau, eta) 32 | mctsInstance.initialiseMoves() 33 | 34 | start_time_all = time.time() 35 | runningTime_all = 0 36 | start_time_level = time.time() 37 | runningTime_level = 0 38 | currentBest = eta[1] 39 | while runningTime_all <= MCTS_all_maximal_time: 40 | 41 | ''' 42 | if runningTime_level > MCTS_level_maximal_time: 43 | bestChild = mctsInstance.bestChild(mctsInstance.rootIndex) 44 | # pick the current best move to take 45 | mctsInstance.makeOneMove(bestChild) 46 | start_time_level = time.time() 47 | ''' 48 | 49 | 50 | # Here are three steps for MCTS 51 | (leafNode, availableActions) = mctsInstance.treeTraversal(mctsInstance.rootIndex) 52 | newNodes = mctsInstance.initialiseExplorationNode(leafNode, availableActions) 53 | for node in newNodes: 54 | (_, value) = mctsInstance.sampling(node, availableActions) 55 | mctsInstance.backPropagation(node, value) 56 | if currentBest > mctsInstance.bestCase[0]: 57 | print("best distance up to now is %s" % (str(mctsInstance.bestCase[0]))) 58 | currentBest = mctsInstance.bestCase[0] 59 | bestChild = mctsInstance.bestChild(mctsInstance.rootIndex) 60 | 61 | # store the current best 62 | (_, bestManipulation) = mctsInstance.bestCase 63 | image1 = mctsInstance.applyManipulation(bestManipulation) 64 | path0 = "%s_pic/%s_Unsafe_currentBest.png" % (dataSetName, image_index) 65 | NN.save_input(image1, path0) 66 | 67 | runningTime_all = time.time() - start_time_all 68 | runningTime_level = time.time() - start_time_level 69 | 70 | (_, bestManipulation) = mctsInstance.bestCase 71 | 72 | print("the number of sampling: %s" % mctsInstance.numOfSampling) 73 | print("the number of adversarial examples: %s\n" % mctsInstance.numAdv) 74 | 75 | image1 = mctsInstance.applyManipulation(bestManipulation) 76 | (newClass, newConfident) = NN.predict(image1) 77 | newClassStr = NN.get_label(int(newClass)) 78 | 79 | if newClass != label: 80 | path0 = "%s_pic/%s_%s_modified_into_%s_with_confidence_%s.png" % ( 81 | dataSetName, image_index, origClassStr, newClassStr, newConfident) 82 | NN.save_input(image1, path0) 83 | path0 = "%s_pic/%s_diff.png" % (dataSetName, image_index) 84 | NN.save_input(np.absolute(image - image1), path0) 85 | print("\nfound an adversary image within pre-specified bounded computational resource. " 86 | "The following is its information: ") 87 | print("difference between images: %s" % (diffImage(image, image1))) 88 | 89 | print("number of adversarial examples found: %s" % mctsInstance.numAdv) 90 | 91 | l2dist = l2Distance(mctsInstance.image, image1) 92 | l1dist = l1Distance(mctsInstance.image, image1) 93 | l0dist = l0Distance(mctsInstance.image, image1) 94 | percent = diffPercent(mctsInstance.image, image1) 95 | print("L2 distance %s" % l2dist) 96 | print("L1 distance %s" % l1dist) 97 | print("L0 distance %s" % l0dist) 98 | print("manipulated percentage distance %s" % percent) 99 | print("class is changed into '%s' with confidence %s\n" % (newClassStr, newConfident)) 100 | 101 | return time.time() - start_time_all, newConfident, percent, l2dist, l1dist, l0dist, 0 102 | 103 | else: 104 | print("\nfailed to find an adversary image within pre-specified bounded computational resource. ") 105 | return 0, 0, 0, 0, 0, 0, 0 106 | 107 | elif gameType == 'competitive': 108 | 109 | mctsInstance = MCTSCompetitive(dataSetName, NN, image_index, image, tau, eta) 110 | mctsInstance.initialiseMoves() 111 | 112 | start_time_all = time.time() 113 | runningTime_all = 0 114 | currentBest = eta[1] 115 | currentBestIndex = 0 116 | while runningTime_all <= MCTS_all_maximal_time: 117 | 118 | (leafNode, availableActions) = mctsInstance.treeTraversal(mctsInstance.rootIndex) 119 | newNodes = mctsInstance.initialiseExplorationNode(leafNode, availableActions) 120 | for node in newNodes: 121 | (_, value) = mctsInstance.sampling(node, availableActions) 122 | mctsInstance.backPropagation(node, value) 123 | if currentBest > mctsInstance.bestCase[0]: 124 | print("best distance up to now is %s" % (str(mctsInstance.bestCase[0]))) 125 | currentBest = mctsInstance.bestCase[0] 126 | currentBestIndex += 1 127 | 128 | # store the current best 129 | (_, bestManipulation) = mctsInstance.bestCase 130 | image1 = mctsInstance.applyManipulation(bestManipulation) 131 | path0 = "%s_pic/%s_Unsafe_currentBest_%s.png" % (dataSetName, image_index, currentBestIndex) 132 | NN.save_input(image1, path0) 133 | 134 | runningTime_all = time.time() - start_time_all 135 | 136 | (bestValue, bestManipulation) = mctsInstance.bestCase 137 | 138 | print("the number of sampling: %s" % mctsInstance.numOfSampling) 139 | print("the number of adversarial examples: %s\n" % mctsInstance.numAdv) 140 | 141 | print("the number of max features is %s" % mctsInstance.bestFeatures()[0]) 142 | maxfeatures = mctsInstance.bestFeatures()[0] 143 | 144 | if bestValue < eta[1]: 145 | 146 | image1 = mctsInstance.applyManipulation(bestManipulation) 147 | (newClass, newConfident) = NN.predict(image1) 148 | newClassStr = NN.get_label(int(newClass)) 149 | 150 | if newClass != label: 151 | path0 = "%s_pic/%s_%s_modified_into_%s_with_confidence_%s.png" % ( 152 | dataSetName, image_index, origClassStr, newClassStr, newConfident) 153 | NN.save_input(image1, path0) 154 | path0 = "%s_pic/%s_diff.png" % (dataSetName, image_index) 155 | NN.save_input(np.absolute(image - image1), path0) 156 | print("\nfound an adversary image within pre-specified bounded computational resource. " 157 | "The following is its information: ") 158 | print("difference between images: %s" % (diffImage(image, image1))) 159 | 160 | print("number of adversarial examples found: %s" % mctsInstance.numAdv) 161 | 162 | l2dist = l2Distance(mctsInstance.image, image1) 163 | l1dist = l1Distance(mctsInstance.image, image1) 164 | l0dist = l0Distance(mctsInstance.image, image1) 165 | percent = diffPercent(mctsInstance.image, image1) 166 | print("L2 distance %s" % l2dist) 167 | print("L1 distance %s" % l1dist) 168 | print("L0 distance %s" % l0dist) 169 | print("manipulated percentage distance %s" % percent) 170 | print("class is changed into '%s' with confidence %s\n" % (newClassStr, newConfident)) 171 | 172 | return time.time() - start_time_all, newConfident, percent, l2dist, l1dist, l0dist, maxfeatures 173 | 174 | else: 175 | print("\nthe robustness of the (input, model) is under control, " 176 | "with the first player is able to defeat the second player " 177 | "who aims to find adversarial example by " 178 | "playing suitable strategies on selecting features. ") 179 | return 0, 0, 0, 0, 0, 0, 0 180 | 181 | else: 182 | 183 | print("\nthe robustness of the (input, model) is under control, " 184 | "with the first player is able to defeat the second player " 185 | "who aims to find adversarial example by " 186 | "playing suitable strategies on selecting features. ") 187 | return 0, 0, 0, 0, 0, 0, 0 188 | 189 | else: 190 | print("Unrecognised game type. Try 'cooperative' or 'competitive'.") 191 | 192 | runningTime = time.time() - start_time 193 | 194 | 195 | ''' 196 | 197 | if gameType == 'cooperative': 198 | mctsInstance = MCTSCooperative(dataSetName, NN, image_index, image, tau, eta) 199 | mctsInstance.initialiseMoves() 200 | 201 | start_time_all = time.time() 202 | runningTime_all = 0 203 | numberOfMoves = 0 204 | while (not mctsInstance.terminalNode(mctsInstance.rootIndex) and 205 | not mctsInstance.terminatedByEta(mctsInstance.rootIndex) and 206 | runningTime_all <= MCTS_all_maximal_time): 207 | print("the number of moves we have made up to now: %s" % numberOfMoves) 208 | l2dist = mctsInstance.l2Dist(mctsInstance.rootIndex) 209 | l1dist = mctsInstance.l1Dist(mctsInstance.rootIndex) 210 | l0dist = mctsInstance.l0Dist(mctsInstance.rootIndex) 211 | percent = mctsInstance.diffPercent(mctsInstance.rootIndex) 212 | diffs = mctsInstance.diffImage(mctsInstance.rootIndex) 213 | print("L2 distance %s" % l2dist) 214 | print("L1 distance %s" % l1dist) 215 | print("L0 distance %s" % l0dist) 216 | print("manipulated percentage distance %s" % percent) 217 | print("manipulated dimensions %s" % diffs) 218 | 219 | start_time_level = time.time() 220 | runningTime_level = 0 221 | childTerminated = False 222 | currentBest = eta[1] 223 | while runningTime_level <= MCTS_level_maximal_time: 224 | # Here are three steps for MCTS 225 | (leafNode, availableActions) = mctsInstance.treeTraversal(mctsInstance.rootIndex) 226 | newNodes = mctsInstance.initialiseExplorationNode(leafNode, availableActions) 227 | for node in newNodes: 228 | (childTerminated, value) = mctsInstance.sampling(node, availableActions) 229 | mctsInstance.backPropagation(node, value) 230 | runningTime_level = time.time() - start_time_level 231 | if currentBest > mctsInstance.bestCase[0]: 232 | print("best possible distance up to now is %s" % (str(mctsInstance.bestCase[0]))) 233 | currentBest = mctsInstance.bestCase[0] 234 | bestChild = mctsInstance.bestChild(mctsInstance.rootIndex) 235 | # pick the current best move to take 236 | mctsInstance.makeOneMove(bestChild) 237 | 238 | image1 = mctsInstance.applyManipulation(mctsInstance.manipulation[mctsInstance.rootIndex]) 239 | diffs = mctsInstance.diffImage(mctsInstance.rootIndex) 240 | path0 = "%s_pic/%s_temp_%s.png" % (dataSetName, image_index, len(diffs)) 241 | NN.save_input(image1, path0) 242 | (newClass, newConfident) = NN.predict(image1) 243 | print("confidence: %s" % newConfident) 244 | 245 | # break if we found that one of the children is a misclassification 246 | if childTerminated is True: 247 | break 248 | 249 | # store the current best 250 | (_, bestManipulation) = mctsInstance.bestCase 251 | image1 = mctsInstance.applyManipulation(bestManipulation) 252 | path0 = "%s_pic/%s_currentBest.png" % (dataSetName, image_index) 253 | NN.save_input(image1, path0) 254 | 255 | numberOfMoves += 1 256 | runningTime_all = time.time() - start_time_all 257 | 258 | (_, bestManipulation) = mctsInstance.bestCase 259 | 260 | image1 = mctsInstance.applyManipulation(bestManipulation) 261 | (newClass, newConfident) = NN.predict(image1) 262 | newClassStr = NN.get_label(int(newClass)) 263 | 264 | if newClass != label: 265 | path0 = "%s_pic/%s_%s_modified_into_%s_with_confidence_%s.png" % ( 266 | dataSetName, image_index, origClassStr, newClassStr, newConfident) 267 | NN.save_input(image1, path0) 268 | path0 = "%s_pic/%s_diff.png" % (dataSetName, image_index) 269 | NN.save_input(np.subtract(image, image1), path0) 270 | print("\nfound an adversary image within pre-specified bounded computational resource. " 271 | "The following is its information: ") 272 | print("difference between images: %s" % (diffImage(image, image1))) 273 | 274 | print("number of adversarial examples found: %s" % mctsInstance.numAdv) 275 | 276 | l2dist = l2Distance(mctsInstance.image, image1) 277 | l1dist = l1Distance(mctsInstance.image, image1) 278 | l0dist = l0Distance(mctsInstance.image, image1) 279 | percent = diffPercent(mctsInstance.image, image1) 280 | print("L2 distance %s" % l2dist) 281 | print("L1 distance %s" % l1dist) 282 | print("L0 distance %s" % l0dist) 283 | print("manipulated percentage distance %s" % percent) 284 | print("class is changed into '%s' with confidence %s\n" % (newClassStr, newConfident)) 285 | 286 | return time.time() - start_time_all, newConfident, percent, l2dist, l1dist, l0dist, 0 287 | 288 | else: 289 | print("\nfailed to find an adversary image within pre-specified bounded computational resource. ") 290 | return 0, 0, 0, 0, 0, 0, 0 291 | 292 | 293 | ''' 294 | --------------------------------------------------------------------------------