├── clip ├── __init__.py ├── bpe_simple_vocab_16e6.txt.gz ├── simple_tokenizer.py ├── clip.py └── model.py ├── CLIP-ES.png ├── pytorch_grad_cam ├── utils │ ├── __init__.py │ ├── svd_on_activations.py │ ├── reshape_transforms.py │ ├── find_layers.py │ ├── model_targets.py │ └── image.py ├── grad_cam.py ├── __init__.py ├── eigen_grad_cam.py ├── eigen_cam.py ├── xgrad_cam.py ├── layer_cam.py ├── grad_cam_plusplus.py ├── activations_and_gradients.py ├── score_cam.py ├── guided_backprop.py ├── fullgrad_cam.py ├── ablation_cam_multilayer.py ├── ablation_layer.py ├── ablation_cam.py └── base_cam.py ├── LICENSE ├── .gitignore ├── clip_text.py ├── eval_cam.py ├── utils.py ├── eval_cam_with_crf.py ├── README.md ├── generate_cams_coco14.py ├── generate_cams_voc12.py └── voc12 ├── val.txt └── train.txt /clip/__init__.py: -------------------------------------------------------------------------------- 1 | from .clip import * 2 | -------------------------------------------------------------------------------- /CLIP-ES.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linyq2117/CLIP-ES/HEAD/CLIP-ES.png -------------------------------------------------------------------------------- /clip/bpe_simple_vocab_16e6.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linyq2117/CLIP-ES/HEAD/clip/bpe_simple_vocab_16e6.txt.gz -------------------------------------------------------------------------------- /pytorch_grad_cam/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from pytorch_grad_cam.utils.image import deprocess_image 2 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 3 | from pytorch_grad_cam.utils import model_targets 4 | from pytorch_grad_cam.utils import reshape_transforms -------------------------------------------------------------------------------- /pytorch_grad_cam/grad_cam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pytorch_grad_cam.base_cam import BaseCAM 3 | 4 | 5 | class GradCAM(BaseCAM): 6 | def __init__(self, model, target_layers, use_cuda=False, 7 | reshape_transform=None): 8 | super( 9 | GradCAM, 10 | self).__init__( 11 | model, 12 | target_layers, 13 | use_cuda, 14 | reshape_transform) 15 | 16 | def get_cam_weights(self, 17 | input_tensor, 18 | target_layer, 19 | target_category, 20 | activations, 21 | grads): 22 | 23 | return np.mean(grads, axis=(2, 3)) 24 | -------------------------------------------------------------------------------- /pytorch_grad_cam/__init__.py: -------------------------------------------------------------------------------- 1 | from pytorch_grad_cam.grad_cam import GradCAM 2 | from pytorch_grad_cam.ablation_layer import AblationLayer, AblationLayerVit, AblationLayerFasterRCNN 3 | from pytorch_grad_cam.ablation_cam import AblationCAM 4 | from pytorch_grad_cam.xgrad_cam import XGradCAM 5 | from pytorch_grad_cam.grad_cam_plusplus import GradCAMPlusPlus 6 | from pytorch_grad_cam.score_cam import ScoreCAM 7 | from pytorch_grad_cam.layer_cam import LayerCAM 8 | from pytorch_grad_cam.eigen_cam import EigenCAM 9 | from pytorch_grad_cam.eigen_grad_cam import EigenGradCAM 10 | from pytorch_grad_cam.fullgrad_cam import FullGrad 11 | from pytorch_grad_cam.guided_backprop import GuidedBackpropReLUModel 12 | from pytorch_grad_cam.activations_and_gradients import ActivationsAndGradients 13 | import pytorch_grad_cam.utils.model_targets 14 | import pytorch_grad_cam.utils.reshape_transforms -------------------------------------------------------------------------------- /pytorch_grad_cam/eigen_grad_cam.py: -------------------------------------------------------------------------------- 1 | from pytorch_grad_cam.base_cam import BaseCAM 2 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 3 | 4 | # Like Eigen CAM: https://arxiv.org/abs/2008.00299 5 | # But multiply the activations x gradients 6 | 7 | 8 | class EigenGradCAM(BaseCAM): 9 | def __init__(self, model, target_layers, use_cuda=False, 10 | reshape_transform=None): 11 | super(EigenGradCAM, self).__init__(model, target_layers, use_cuda, 12 | reshape_transform) 13 | 14 | def get_cam_image(self, 15 | input_tensor, 16 | target_layer, 17 | target_category, 18 | activations, 19 | grads, 20 | eigen_smooth): 21 | return get_2d_projection(grads * activations) 22 | -------------------------------------------------------------------------------- /pytorch_grad_cam/utils/svd_on_activations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def get_2d_projection(activation_batch): 5 | # TBD: use pytorch batch svd implementation 6 | activation_batch[np.isnan(activation_batch)] = 0 7 | projections = [] 8 | for activations in activation_batch: 9 | reshaped_activations = (activations).reshape( 10 | activations.shape[0], -1).transpose() 11 | # Centering before the SVD seems to be important here, 12 | # Otherwise the image returned is negative 13 | reshaped_activations = reshaped_activations - \ 14 | reshaped_activations.mean(axis=0) 15 | U, S, VT = np.linalg.svd(reshaped_activations, full_matrices=True) 16 | projection = reshaped_activations @ VT[0, :] 17 | projection = projection.reshape(activations.shape[1:]) 18 | projections.append(projection) 19 | return np.float32(projections) 20 | -------------------------------------------------------------------------------- /pytorch_grad_cam/eigen_cam.py: -------------------------------------------------------------------------------- 1 | from pytorch_grad_cam.base_cam import BaseCAM 2 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 3 | 4 | # https://arxiv.org/abs/2008.00299 5 | 6 | 7 | class EigenCAM(BaseCAM): 8 | def __init__(self, model, target_layers, use_cuda=False, 9 | reshape_transform=None): 10 | super(EigenCAM, self).__init__(model, 11 | target_layers, 12 | use_cuda, 13 | reshape_transform, 14 | uses_gradients=False) 15 | 16 | def get_cam_image(self, 17 | input_tensor, 18 | target_layer, 19 | target_category, 20 | activations, 21 | grads, 22 | eigen_smooth): 23 | return get_2d_projection(activations) 24 | -------------------------------------------------------------------------------- /pytorch_grad_cam/xgrad_cam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pytorch_grad_cam.base_cam import BaseCAM 3 | 4 | 5 | class XGradCAM(BaseCAM): 6 | def __init__( 7 | self, 8 | model, 9 | target_layers, 10 | use_cuda=False, 11 | reshape_transform=None): 12 | super( 13 | XGradCAM, 14 | self).__init__( 15 | model, 16 | target_layers, 17 | use_cuda, 18 | reshape_transform) 19 | 20 | def get_cam_weights(self, 21 | input_tensor, 22 | target_layer, 23 | target_category, 24 | activations, 25 | grads): 26 | sum_activations = np.sum(activations, axis=(2, 3)) 27 | eps = 1e-7 28 | weights = grads * activations / \ 29 | (sum_activations[:, :, None, None] + eps) 30 | weights = weights.sum(axis=(2, 3)) 31 | return weights 32 | -------------------------------------------------------------------------------- /pytorch_grad_cam/utils/reshape_transforms.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def fasterrcnn_reshape_transform(x): 4 | target_size = x['pool'].size()[-2 : ] 5 | activations = [] 6 | for key, value in x.items(): 7 | activations.append(torch.nn.functional.interpolate(torch.abs(value), target_size, mode='bilinear')) 8 | activations = torch.cat(activations, axis=1) 9 | return activations 10 | 11 | def swinT_reshape_transform(tensor, height=7, width=7): 12 | result = tensor.reshape(tensor.size(0), 13 | height, width, tensor.size(2)) 14 | 15 | # Bring the channels to the first dimension, 16 | # like in CNNs. 17 | result = result.transpose(2, 3).transpose(1, 2) 18 | return result 19 | 20 | def vit_reshape_transform(tensor, height=14, width=14): 21 | result = tensor[:, 1:, :].reshape(tensor.size(0), 22 | height, width, tensor.size(2)) 23 | 24 | # Bring the channels to the first dimension, 25 | # like in CNNs. 26 | result = result.transpose(2, 3).transpose(1, 2) 27 | return result 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 linyq2117 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pytorch_grad_cam/layer_cam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pytorch_grad_cam.base_cam import BaseCAM 3 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 4 | 5 | # https://ieeexplore.ieee.org/document/9462463 6 | 7 | 8 | class LayerCAM(BaseCAM): 9 | def __init__( 10 | self, 11 | model, 12 | target_layers, 13 | use_cuda=False, 14 | reshape_transform=None): 15 | super( 16 | LayerCAM, 17 | self).__init__( 18 | model, 19 | target_layers, 20 | use_cuda, 21 | reshape_transform) 22 | 23 | def get_cam_image(self, 24 | input_tensor, 25 | target_layer, 26 | target_category, 27 | activations, 28 | grads, 29 | eigen_smooth): 30 | spatial_weighted_activations = np.maximum(grads, 0) * activations 31 | 32 | if eigen_smooth: 33 | cam = get_2d_projection(spatial_weighted_activations) 34 | else: 35 | cam = spatial_weighted_activations.sum(axis=1) 36 | return cam 37 | -------------------------------------------------------------------------------- /pytorch_grad_cam/utils/find_layers.py: -------------------------------------------------------------------------------- 1 | def replace_layer_recursive(model, old_layer, new_layer): 2 | for name, layer in model._modules.items(): 3 | if layer == old_layer: 4 | model._modules[name] = new_layer 5 | return True 6 | elif replace_layer_recursive(layer, old_layer, new_layer): 7 | return True 8 | return False 9 | 10 | 11 | def replace_all_layer_type_recursive(model, old_layer_type, new_layer): 12 | for name, layer in model._modules.items(): 13 | if isinstance(layer, old_layer_type): 14 | model._modules[name] = new_layer 15 | replace_all_layer_type_recursive(layer, old_layer_type, new_layer) 16 | 17 | 18 | def find_layer_types_recursive(model, layer_types): 19 | def predicate(layer): 20 | return type(layer) in layer_types 21 | return find_layer_predicate_recursive(model, predicate) 22 | 23 | 24 | def find_layer_predicate_recursive(model, predicate): 25 | result = [] 26 | for name, layer in model._modules.items(): 27 | if predicate(layer): 28 | result.append(layer) 29 | result.extend(find_layer_predicate_recursive(layer, predicate)) 30 | return result -------------------------------------------------------------------------------- /pytorch_grad_cam/grad_cam_plusplus.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pytorch_grad_cam.base_cam import BaseCAM 3 | 4 | # https://arxiv.org/abs/1710.11063 5 | 6 | 7 | class GradCAMPlusPlus(BaseCAM): 8 | def __init__(self, model, target_layers, use_cuda=False, 9 | reshape_transform=None): 10 | super(GradCAMPlusPlus, self).__init__(model, target_layers, use_cuda, 11 | reshape_transform) 12 | 13 | def get_cam_weights(self, 14 | input_tensor, 15 | target_layers, 16 | target_category, 17 | activations, 18 | grads): 19 | grads_power_2 = grads**2 20 | grads_power_3 = grads_power_2 * grads 21 | # Equation 19 in https://arxiv.org/abs/1710.11063 22 | sum_activations = np.sum(activations, axis=(2, 3)) 23 | eps = 0.000001 24 | aij = grads_power_2 / (2 * grads_power_2 + 25 | sum_activations[:, :, None, None] * grads_power_3 + eps) 26 | # Now bring back the ReLU from eq.7 in the paper, 27 | # And zero out aijs where the activations are 0 28 | aij = np.where(grads != 0, aij, 0) 29 | 30 | weights = np.maximum(grads, 0) * aij 31 | weights = np.sum(weights, axis=(2, 3)) 32 | return weights 33 | -------------------------------------------------------------------------------- /pytorch_grad_cam/activations_and_gradients.py: -------------------------------------------------------------------------------- 1 | class ActivationsAndGradients: 2 | """ Class for extracting activations and 3 | registering gradients from targetted intermediate layers """ 4 | 5 | def __init__(self, model, target_layers, reshape_transform): 6 | self.model = model 7 | self.gradients = [] 8 | self.activations = [] 9 | self.reshape_transform = reshape_transform 10 | self.handles = [] 11 | for target_layer in target_layers: 12 | self.handles.append( 13 | target_layer.register_forward_hook(self.save_activation)) 14 | # Because of https://github.com/pytorch/pytorch/issues/61519, 15 | # we don't use backward hook to record gradients. 16 | self.handles.append( 17 | target_layer.register_forward_hook(self.save_gradient)) 18 | 19 | def save_activation(self, module, input, output): 20 | activation = output 21 | 22 | if self.reshape_transform is not None: 23 | activation = self.reshape_transform(activation, self.height, self.width) 24 | self.activations.append(activation.cpu().detach()) 25 | 26 | def save_gradient(self, module, input, output): 27 | if not hasattr(output, "requires_grad") or not output.requires_grad: 28 | # You can only register hooks on tensor requires grad. 29 | return 30 | 31 | # Gradients are computed in reverse order 32 | def _store_grad(grad): 33 | if self.reshape_transform is not None: 34 | grad = self.reshape_transform(grad, self.height, self.width) 35 | self.gradients = [grad.cpu().detach()] + self.gradients 36 | 37 | output.register_hook(_store_grad) 38 | 39 | def __call__(self, x, H, W): 40 | self.height = H // 16 41 | self.width = W // 16 42 | self.gradients = [] 43 | self.activations = [] 44 | if isinstance(x, list): 45 | return self.model.forward_last_layer(x[0], x[1]) 46 | else: 47 | return self.model(x) 48 | 49 | def release(self): 50 | for handle in self.handles: 51 | handle.remove() 52 | -------------------------------------------------------------------------------- /pytorch_grad_cam/utils/model_targets.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torchvision 4 | 5 | class ClassifierOutputTarget: 6 | def __init__(self, category): 7 | self.category = category 8 | def __call__(self, model_output): 9 | if len(model_output.shape) == 1: 10 | return model_output[self.category] 11 | return model_output[:, self.category] 12 | 13 | class SemanticSegmentationTarget: 14 | """ Gets a binary spatial mask and a category, 15 | And return the sum of the category scores, 16 | of the pixels in the mask. """ 17 | def __init__(self, category, mask): 18 | self.category = category 19 | self.mask = torch.from_numpy(mask) 20 | if torch.cuda.is_available(): 21 | self.mask = self.mask.cuda() 22 | 23 | def __call__(self, model_output): 24 | return (model_output[self.category, :, : ] * self.mask).sum() 25 | 26 | 27 | class FasterRCNNBoxScoreTarget: 28 | """ For every original detected bounding box specified in "bounding boxes", 29 | assign a score on how the current bounding boxes match it, 30 | 1. In IOU 31 | 2. In the classification score. 32 | If there is not a large enough overlap, or the category changed, 33 | assign a score of 0. 34 | 35 | The total score is the sum of all the box scores. 36 | """ 37 | 38 | def __init__(self, labels, bounding_boxes, iou_threshold=0.5): 39 | self.labels = labels 40 | self.bounding_boxes = bounding_boxes 41 | self.iou_threshold = iou_threshold 42 | 43 | def __call__(self, model_outputs): 44 | output = torch.Tensor([0]) 45 | if torch.cuda.is_available(): 46 | output = output.cuda() 47 | 48 | if len(model_outputs["boxes"]) == 0: 49 | return output 50 | 51 | for box, label in zip(self.bounding_boxes, self.labels): 52 | box = torch.Tensor(box[None, :]) 53 | if torch.cuda.is_available(): 54 | box = box.cuda() 55 | 56 | ious = torchvision.ops.box_iou(box, model_outputs["boxes"]) 57 | index = ious.argmax() 58 | if ious[0, index] > self.iou_threshold and model_outputs["labels"][index] == label: 59 | score = ious[0, index] + model_outputs["scores"][index] 60 | output = output + score 61 | return output -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /pytorch_grad_cam/utils/image.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import torch 4 | from torchvision.transforms import Compose, Normalize, ToTensor 5 | 6 | 7 | def preprocess_image(img: np.ndarray, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) -> torch.Tensor: 8 | preprocessing = Compose([ 9 | ToTensor(), 10 | Normalize(mean=mean, std=std) 11 | ]) 12 | return preprocessing(img.copy()).unsqueeze(0) 13 | 14 | 15 | def deprocess_image(img): 16 | """ see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 """ 17 | img = img - np.mean(img) 18 | img = img / (np.std(img) + 1e-5) 19 | img = img * 0.1 20 | img = img + 0.5 21 | img = np.clip(img, 0, 1) 22 | return np.uint8(img * 255) 23 | 24 | 25 | def show_cam_on_image(img: np.ndarray, 26 | mask: np.ndarray, 27 | use_rgb: bool = False, 28 | colormap: int = cv2.COLORMAP_JET) -> np.ndarray: 29 | """ This function overlays the cam mask on the image as an heatmap. 30 | By default the heatmap is in BGR format. 31 | 32 | :param img: The base image in RGB or BGR format. 33 | :param mask: The cam mask. 34 | :param use_rgb: Whether to use an RGB or BGR heatmap, this should be set to True if 'img' is in RGB format. 35 | :param colormap: The OpenCV colormap to be used. 36 | :returns: The default image with the cam overlay. 37 | """ 38 | heatmap = cv2.applyColorMap(np.uint8(255 * mask), colormap) 39 | if use_rgb: 40 | heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB) 41 | heatmap = np.float32(heatmap) / 255 42 | 43 | if np.max(img) > 1: 44 | raise Exception( 45 | "The input image should np.float32 in the range [0, 1]") 46 | 47 | cam = heatmap + img 48 | cam = cam / np.max(cam) 49 | return np.uint8(255 * cam) 50 | 51 | def scale_cam_image(cam, target_size=None): 52 | result = [] 53 | for img in cam: 54 | img = img - np.min(img) 55 | img = img / (1e-7 + np.max(img)) 56 | if target_size is not None: 57 | img = cv2.resize(img, target_size) 58 | result.append(img) 59 | result = np.float32(result) 60 | 61 | return result 62 | 63 | def scale_accross_batch_and_channels(tensor, target_size): 64 | batch_size, channel_size = tensor.shape[:2] 65 | reshaped_tensor = tensor.reshape( 66 | batch_size * channel_size, *tensor.shape[2:]) 67 | result = scale_cam_image(reshaped_tensor, target_size) 68 | result = result.reshape( 69 | batch_size, 70 | channel_size, 71 | target_size[1], 72 | target_size[0]) 73 | return result 74 | -------------------------------------------------------------------------------- /pytorch_grad_cam/score_cam.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import tqdm 3 | from pytorch_grad_cam.base_cam import BaseCAM 4 | 5 | 6 | class ScoreCAM(BaseCAM): 7 | def __init__( 8 | self, 9 | model, 10 | target_layers, 11 | use_cuda=False, 12 | reshape_transform=None): 13 | super(ScoreCAM, self).__init__(model, 14 | target_layers, 15 | use_cuda, 16 | reshape_transform=reshape_transform, 17 | uses_gradients=False) 18 | 19 | if len(target_layers) > 0: 20 | print("Warning: You are using ScoreCAM with target layers, " 21 | "however ScoreCAM will ignore them.") 22 | 23 | def get_cam_weights(self, 24 | input_tensor, 25 | target_layer, 26 | targets, 27 | activations, 28 | grads): 29 | with torch.no_grad(): 30 | upsample = torch.nn.UpsamplingBilinear2d( 31 | size=input_tensor.shape[-2:]) 32 | activation_tensor = torch.from_numpy(activations) 33 | if self.cuda: 34 | activation_tensor = activation_tensor.cuda() 35 | 36 | upsampled = upsample(activation_tensor) 37 | 38 | maxs = upsampled.view(upsampled.size(0), 39 | upsampled.size(1), -1).max(dim=-1)[0] 40 | mins = upsampled.view(upsampled.size(0), 41 | upsampled.size(1), -1).min(dim=-1)[0] 42 | 43 | maxs, mins = maxs[:, :, None, None], mins[:, :, None, None] 44 | upsampled = (upsampled - mins) / (maxs - mins) 45 | 46 | input_tensors = input_tensor[:, None, 47 | :, :] * upsampled[:, :, None, :, :] 48 | 49 | if hasattr(self, "batch_size"): 50 | BATCH_SIZE = self.batch_size 51 | else: 52 | BATCH_SIZE = 16 53 | 54 | scores = [] 55 | for target, tensor in zip(targets, input_tensors): 56 | for i in tqdm.tqdm(range(0, tensor.size(0), BATCH_SIZE)): 57 | batch = tensor[i: i + BATCH_SIZE, :] 58 | outputs = [target(o).cpu().item() for o in self.model(batch)] 59 | scores.extend(outputs) 60 | scores = torch.Tensor(scores) 61 | scores = scores.view(activations.shape[0], activations.shape[1]) 62 | weights = torch.nn.Softmax(dim=-1)(scores).numpy() 63 | return weights 64 | -------------------------------------------------------------------------------- /clip_text.py: -------------------------------------------------------------------------------- 1 | 2 | BACKGROUND_CATEGORY = ['ground','land','grass','tree','building','wall','sky','lake','water','river','sea','railway','railroad','keyboard','helmet', 3 | 'cloud','house','mountain','ocean','road','rock','street','valley','bridge','sign', 4 | ] 5 | 6 | class_names = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 7 | 'bus', 'car', 'cat', 'chair', 'cow', 8 | 'diningtable', 'dog', 'horse', 'motorbike', 'person', 9 | 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 10 | ] 11 | 12 | new_class_names = ['aeroplane', 'bicycle', 'bird avian', 'boat', 'bottle', 13 | 'bus', 'car', 'cat', 'chair seat', 'cow', 14 | 'diningtable', 'dog', 'horse', 'motorbike', 'person with clothes,people,human', 15 | 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor screen', 16 | ] 17 | 18 | 19 | class_names_coco = ['person','bicycle','car','motorbike','aeroplane', 20 | 'bus','train','truck','boat','traffic light', 21 | 'fire hydrant','stop sign','parking meter','bench','bird', 22 | 'cat','dog','horse','sheep','cow', 23 | 'elephant','bear','zebra','giraffe','backpack', 24 | 'umbrella','handbag','tie','suitcase','frisbee', 25 | 'skis','snowboard','sports ball','kite','baseball bat', 26 | 'baseball glove','skateboard','surfboard','tennis racket','bottle', 27 | 'wine glass','cup','fork','knife','spoon', 28 | 'bowl','banana','apple','sandwich','orange', 29 | 'broccoli','carrot','hot dog','pizza','donut', 30 | 'cake','chair','sofa','pottedplant','bed', 31 | 'diningtable','toilet','tvmonitor','laptop','mouse', 32 | 'remote','keyboard','cell phone','microwave','oven', 33 | 'toaster','sink','refrigerator','book','clock', 34 | 'vase','scissors','teddy bear','hair drier','toothbrush', 35 | ] 36 | 37 | new_class_names_coco = ['person with clothes,people,human','bicycle','car','motorbike','aeroplane', 38 | 'bus','train','truck','boat','traffic light', 39 | 'fire hydrant','stop sign','parking meter','bench','bird avian', 40 | 'cat','dog','horse','sheep','cow', 41 | 'elephant','bear','zebra','giraffe','backpack,bag', 42 | 'umbrella,parasol','handbag,purse','necktie','suitcase','frisbee', 43 | 'skis','sknowboard','sports ball','kite','baseball bat', 44 | 'glove','skateboard','surfboard','tennis racket','bottle', 45 | 'wine glass','cup','fork','knife','dessertspoon', 46 | 'bowl','banana','apple','sandwich','orange', 47 | 'broccoli','carrot','hot dog','pizza','donut', 48 | 'cake','chair seat','sofa','pottedplant','bed', 49 | 'diningtable','toilet','tvmonitor screen','laptop','mouse', 50 | 'remote control','keyboard','cell phone','microwave','oven', 51 | 'toaster','sink','refrigerator','book','clock', 52 | 'vase','scissors','teddy bear','hairdrier,blowdrier','toothbrush', 53 | ] 54 | 55 | 56 | BACKGROUND_CATEGORY_COCO = ['ground','land','grass','tree','building','wall','sky','lake','water','river','sea','railway','railroad','helmet', 57 | 'cloud','house','mountain','ocean','road','rock','street','valley','bridge', 58 | ] 59 | -------------------------------------------------------------------------------- /pytorch_grad_cam/guided_backprop.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch.autograd import Function 4 | from pytorch_grad_cam.utils.find_layers import replace_all_layer_type_recursive 5 | 6 | 7 | class GuidedBackpropReLU(Function): 8 | @staticmethod 9 | def forward(self, input_img): 10 | positive_mask = (input_img > 0).type_as(input_img) 11 | output = torch.addcmul( 12 | torch.zeros( 13 | input_img.size()).type_as(input_img), 14 | input_img, 15 | positive_mask) 16 | self.save_for_backward(input_img, output) 17 | return output 18 | 19 | @staticmethod 20 | def backward(self, grad_output): 21 | input_img, output = self.saved_tensors 22 | grad_input = None 23 | 24 | positive_mask_1 = (input_img > 0).type_as(grad_output) 25 | positive_mask_2 = (grad_output > 0).type_as(grad_output) 26 | grad_input = torch.addcmul( 27 | torch.zeros( 28 | input_img.size()).type_as(input_img), 29 | torch.addcmul( 30 | torch.zeros( 31 | input_img.size()).type_as(input_img), 32 | grad_output, 33 | positive_mask_1), 34 | positive_mask_2) 35 | return grad_input 36 | 37 | 38 | class GuidedBackpropReLUasModule(torch.nn.Module): 39 | def __init__(self): 40 | super(GuidedBackpropReLUasModule, self).__init__() 41 | 42 | def forward(self, input_img): 43 | return GuidedBackpropReLU.apply(input_img) 44 | 45 | 46 | class GuidedBackpropReLUModel: 47 | def __init__(self, model, use_cuda): 48 | self.model = model 49 | self.model.eval() 50 | self.cuda = use_cuda 51 | if self.cuda: 52 | self.model = self.model.cuda() 53 | 54 | def forward(self, input_img): 55 | return self.model(input_img) 56 | 57 | def recursive_replace_relu_with_guidedrelu(self, module_top): 58 | 59 | for idx, module in module_top._modules.items(): 60 | self.recursive_replace_relu_with_guidedrelu(module) 61 | if module.__class__.__name__ == 'ReLU': 62 | module_top._modules[idx] = GuidedBackpropReLU.apply 63 | print("b") 64 | 65 | def recursive_replace_guidedrelu_with_relu(self, module_top): 66 | try: 67 | for idx, module in module_top._modules.items(): 68 | self.recursive_replace_guidedrelu_with_relu(module) 69 | if module == GuidedBackpropReLU.apply: 70 | module_top._modules[idx] = torch.nn.ReLU() 71 | except BaseException: 72 | pass 73 | 74 | def __call__(self, input_img, target_category=None): 75 | replace_all_layer_type_recursive(self.model, 76 | torch.nn.ReLU, 77 | GuidedBackpropReLUasModule()) 78 | 79 | if self.cuda: 80 | input_img = input_img.cuda() 81 | 82 | input_img = input_img.requires_grad_(True) 83 | 84 | output = self.forward(input_img) 85 | 86 | if target_category is None: 87 | target_category = np.argmax(output.cpu().data.numpy()) 88 | 89 | loss = output[0, target_category] 90 | loss.backward(retain_graph=True) 91 | 92 | output = input_img.grad.cpu().data.numpy() 93 | output = output[0, :, :, :] 94 | output = output.transpose((1, 2, 0)) 95 | 96 | replace_all_layer_type_recursive(self.model, 97 | GuidedBackpropReLUasModule, 98 | torch.nn.ReLU()) 99 | 100 | return output 101 | -------------------------------------------------------------------------------- /pytorch_grad_cam/fullgrad_cam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from pytorch_grad_cam.base_cam import BaseCAM 4 | from pytorch_grad_cam.utils.find_layers import find_layer_predicate_recursive 5 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 6 | from pytorch_grad_cam.utils.image import scale_accross_batch_and_channels, scale_cam_image 7 | 8 | # https://arxiv.org/abs/1905.00780 9 | 10 | 11 | class FullGrad(BaseCAM): 12 | def __init__(self, model, target_layers, use_cuda=False, 13 | reshape_transform=None): 14 | if len(target_layers) > 0: 15 | print( 16 | "Warning: target_layers is ignored in FullGrad. All bias layers will be used instead") 17 | 18 | def layer_with_2D_bias(layer): 19 | bias_target_layers = [torch.nn.Conv2d, torch.nn.BatchNorm2d] 20 | if type(layer) in bias_target_layers and layer.bias is not None: 21 | return True 22 | return False 23 | target_layers = find_layer_predicate_recursive( 24 | model, layer_with_2D_bias) 25 | super( 26 | FullGrad, 27 | self).__init__( 28 | model, 29 | target_layers, 30 | use_cuda, 31 | reshape_transform, 32 | compute_input_gradient=True) 33 | self.bias_data = [self.get_bias_data( 34 | layer).cpu().numpy() for layer in target_layers] 35 | 36 | def get_bias_data(self, layer): 37 | # Borrowed from official paper impl: 38 | # https://github.com/idiap/fullgrad-saliency/blob/master/saliency/tensor_extractor.py#L47 39 | if isinstance(layer, torch.nn.BatchNorm2d): 40 | bias = - (layer.running_mean * layer.weight 41 | / torch.sqrt(layer.running_var + layer.eps)) + layer.bias 42 | return bias.data 43 | else: 44 | return layer.bias.data 45 | 46 | def compute_cam_per_layer( 47 | self, 48 | input_tensor, 49 | target_category, 50 | eigen_smooth): 51 | input_grad = input_tensor.grad.data.cpu().numpy() 52 | grads_list = [g.cpu().data.numpy() for g in 53 | self.activations_and_grads.gradients] 54 | cam_per_target_layer = [] 55 | target_size = self.get_target_width_height(input_tensor) 56 | 57 | gradient_multiplied_input = input_grad * input_tensor.data.cpu().numpy() 58 | gradient_multiplied_input = np.abs(gradient_multiplied_input) 59 | gradient_multiplied_input = scale_accross_batch_and_channels( 60 | gradient_multiplied_input, 61 | target_size) 62 | cam_per_target_layer.append(gradient_multiplied_input) 63 | 64 | # Loop over the saliency image from every layer 65 | assert(len(self.bias_data) == len(grads_list)) 66 | for bias, grads in zip(self.bias_data, grads_list): 67 | bias = bias[None, :, None, None] 68 | # In the paper they take the absolute value, 69 | # but possibily taking only the positive gradients will work 70 | # better. 71 | bias_grad = np.abs(bias * grads) 72 | result = scale_accross_batch_and_channels( 73 | bias_grad, target_size) 74 | result = np.sum(result, axis=1) 75 | cam_per_target_layer.append(result[:, None, :]) 76 | cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1) 77 | if eigen_smooth: 78 | # Resize to a smaller image, since this method typically has a very large number of channels, 79 | # and then consumes a lot of memory 80 | cam_per_target_layer = scale_accross_batch_and_channels( 81 | cam_per_target_layer, (target_size[0] // 8, target_size[1] // 8)) 82 | cam_per_target_layer = get_2d_projection(cam_per_target_layer) 83 | cam_per_target_layer = cam_per_target_layer[:, None, :, :] 84 | cam_per_target_layer = scale_accross_batch_and_channels( 85 | cam_per_target_layer, 86 | target_size) 87 | else: 88 | cam_per_target_layer = np.sum( 89 | cam_per_target_layer, axis=1)[:, None, :] 90 | 91 | return cam_per_target_layer 92 | 93 | def aggregate_multi_layers(self, cam_per_target_layer): 94 | result = np.sum(cam_per_target_layer, axis=1) 95 | return scale_cam_image(result) 96 | -------------------------------------------------------------------------------- /eval_cam.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import os 4 | from PIL import Image 5 | import argparse 6 | 7 | def print_iou(iou, dname='voc'): 8 | iou_dict = {} 9 | for i in range(len(iou)-1): 10 | iou_dict[i] = iou[i+1] 11 | print(iou_dict) 12 | 13 | return iou_dict 14 | 15 | def _fast_hist(label_true, label_pred, n_class): 16 | mask = (label_true >= 0) & (label_true < n_class) 17 | hist = np.bincount( 18 | n_class * label_true[mask].astype(int) + label_pred[mask], 19 | minlength=n_class ** 2, 20 | ).reshape(n_class, n_class) 21 | return hist 22 | 23 | def scores(label_trues, label_preds, n_class): 24 | hist = np.zeros((n_class, n_class)) 25 | for lt, lp in zip(label_trues, label_preds): 26 | hist += _fast_hist(lt.flatten(), lp.flatten(), n_class) 27 | acc = np.diag(hist).sum() / hist.sum() 28 | acc_cls = np.diag(hist) / hist.sum(axis=1) 29 | acc_cls = np.nanmean(acc_cls) 30 | iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist)) 31 | valid = hist.sum(axis=1) > 0 # added 32 | mean_iu = np.nanmean(iu[valid]) 33 | freq = hist.sum(axis=1) / hist.sum() 34 | fwavacc = (freq[freq > 0] * iu[freq > 0]).sum() 35 | cls_iu = dict(zip(range(n_class), iu)) 36 | 37 | return { 38 | "Pixel Accuracy": acc, 39 | "Mean Accuracy": acc_cls, 40 | "Frequency Weighted IoU": fwavacc, 41 | "Mean IoU": mean_iu, 42 | "Class IoU": cls_iu, 43 | } 44 | 45 | def run_eval_cam(args, print_log=False, is_coco=False): 46 | preds = [] 47 | labels = [] 48 | n_images = 0 49 | for i, id in enumerate(eval_list): 50 | n_images += 1 51 | if args.cam_type == 'png': 52 | label_path = os.path.join(args.cam_out_dir, id + '.png') 53 | cls_labels = np.asarray(Image.open(label_path), dtype=np.uint8) 54 | else: 55 | cam_dict = np.load(os.path.join(args.cam_out_dir, id + '.npy'), allow_pickle=True).item() 56 | cams = cam_dict[args.cam_type] 57 | if 'bg' not in args.cam_type: 58 | if args.cam_eval_thres < 1: 59 | cams = np.pad(cams, ((1, 0), (0, 0), (0, 0)), mode='constant', constant_values=args.cam_eval_thres) 60 | else: 61 | bg_score = np.power(1 - np.max(cams, axis=0, keepdims=True), args.cam_eval_thres) 62 | cams = np.concatenate((bg_score, cams), axis=0) 63 | keys = np.pad(cam_dict['keys'] + 1, (1, 0), mode='constant') 64 | cls_labels = np.argmax(cams, axis=0) 65 | cls_labels = keys[cls_labels].astype(np.uint8) 66 | preds.append(cls_labels) 67 | gt_file = os.path.join(args.gt_root, '%s.png' % id) 68 | gt = np.array(Image.open(gt_file)).astype(np.uint8) 69 | labels.append(gt) 70 | 71 | iou = scores(labels, preds, n_class=21 if not is_coco else 81) 72 | 73 | if print_log: 74 | print(iou) 75 | 76 | return iou["Mean IoU"] 77 | 78 | if __name__ == "__main__": 79 | parser = argparse.ArgumentParser() 80 | parser.add_argument("--cam_out_dir", default="./cam_out", type=str) 81 | parser.add_argument("--cam_type", default="attn_highres", type=str) 82 | parser.add_argument("--split_file", default="/home/xxx/datasets/VOC2012/ImageSets/Segmentation/train.txt", type=str) 83 | parser.add_argument("--cam_eval_thres", default=2, type=float) 84 | parser.add_argument("--gt_root", default="/home/xxx/datasets/VOC2012/SegmentationClassAug", type=str) 85 | args = parser.parse_args() 86 | 87 | is_coco = 'coco' in args.cam_out_dir 88 | if 'voc' in args.cam_out_dir: 89 | eval_list = list(np.loadtxt(args.split_file, dtype=str)) 90 | elif 'coco' in args.cam_out_dir: 91 | file_list = tuple(open(args.split_file, "r")) 92 | file_list = [id_.rstrip().split(" ") for id_ in file_list] 93 | eval_list = [x[0] for x in file_list]#[:2000] 94 | print('{} images to eval'.format(len(eval_list))) 95 | 96 | if 'bg' in args.cam_type or 'png' in args.cam_type: 97 | iou = run_eval_cam(args, True) 98 | else: 99 | if args.cam_eval_thres < 1: 100 | thres_list = [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6] 101 | else: 102 | if 'attn' in args.cam_type: 103 | thres_list = [1, 2] 104 | else: 105 | thres_list =[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 106 | max_iou = 0 107 | max_thres = 0 108 | for thres in thres_list: 109 | args.cam_eval_thres = thres 110 | iou = run_eval_cam(args, print_log=False, is_coco=is_coco) 111 | print(thres, iou) 112 | if iou > max_iou: 113 | max_iou = iou 114 | max_thres = thres 115 | 116 | args.cam_eval_thres = max_thres 117 | iou = run_eval_cam(args, print_log=True, is_coco=is_coco) 118 | -------------------------------------------------------------------------------- /clip/simple_tokenizer.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import html 3 | import os 4 | from functools import lru_cache 5 | 6 | import ftfy 7 | import regex as re 8 | 9 | 10 | @lru_cache() 11 | def default_bpe(): 12 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz") 13 | 14 | 15 | @lru_cache() 16 | def bytes_to_unicode(): 17 | """ 18 | Returns list of utf-8 byte and a corresponding list of unicode strings. 19 | The reversible bpe codes work on unicode strings. 20 | This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. 21 | When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. 22 | This is a signficant percentage of your normal, say, 32K bpe vocab. 23 | To avoid that, we want lookup tables between utf-8 bytes and unicode strings. 24 | And avoids mapping to whitespace/control characters the bpe code barfs on. 25 | """ 26 | bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) 27 | cs = bs[:] 28 | n = 0 29 | for b in range(2**8): 30 | if b not in bs: 31 | bs.append(b) 32 | cs.append(2**8+n) 33 | n += 1 34 | cs = [chr(n) for n in cs] 35 | return dict(zip(bs, cs)) 36 | 37 | 38 | def get_pairs(word): 39 | """Return set of symbol pairs in a word. 40 | Word is represented as tuple of symbols (symbols being variable-length strings). 41 | """ 42 | pairs = set() 43 | prev_char = word[0] 44 | for char in word[1:]: 45 | pairs.add((prev_char, char)) 46 | prev_char = char 47 | return pairs 48 | 49 | 50 | def basic_clean(text): 51 | text = ftfy.fix_text(text) 52 | text = html.unescape(html.unescape(text)) 53 | return text.strip() 54 | 55 | 56 | def whitespace_clean(text): 57 | text = re.sub(r'\s+', ' ', text) 58 | text = text.strip() 59 | return text 60 | 61 | 62 | class SimpleTokenizer(object): 63 | def __init__(self, bpe_path: str = default_bpe()): 64 | self.byte_encoder = bytes_to_unicode() 65 | self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} 66 | merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') 67 | merges = merges[1:49152-256-2+1] 68 | merges = [tuple(merge.split()) for merge in merges] 69 | vocab = list(bytes_to_unicode().values()) 70 | vocab = vocab + [v+'' for v in vocab] 71 | for merge in merges: 72 | vocab.append(''.join(merge)) 73 | vocab.extend(['<|startoftext|>', '<|endoftext|>']) 74 | self.encoder = dict(zip(vocab, range(len(vocab)))) 75 | self.decoder = {v: k for k, v in self.encoder.items()} 76 | self.bpe_ranks = dict(zip(merges, range(len(merges)))) 77 | self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} 78 | self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) 79 | 80 | def bpe(self, token): 81 | if token in self.cache: 82 | return self.cache[token] 83 | word = tuple(token[:-1]) + ( token[-1] + '',) 84 | pairs = get_pairs(word) 85 | 86 | if not pairs: 87 | return token+'' 88 | 89 | while True: 90 | bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) 91 | if bigram not in self.bpe_ranks: 92 | break 93 | first, second = bigram 94 | new_word = [] 95 | i = 0 96 | while i < len(word): 97 | try: 98 | j = word.index(first, i) 99 | new_word.extend(word[i:j]) 100 | i = j 101 | except: 102 | new_word.extend(word[i:]) 103 | break 104 | 105 | if word[i] == first and i < len(word)-1 and word[i+1] == second: 106 | new_word.append(first+second) 107 | i += 2 108 | else: 109 | new_word.append(word[i]) 110 | i += 1 111 | new_word = tuple(new_word) 112 | word = new_word 113 | if len(word) == 1: 114 | break 115 | else: 116 | pairs = get_pairs(word) 117 | word = ' '.join(word) 118 | self.cache[token] = word 119 | return word 120 | 121 | def encode(self, text): 122 | bpe_tokens = [] 123 | text = whitespace_clean(basic_clean(text)).lower() 124 | for token in re.findall(self.pat, text): 125 | token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) 126 | bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) 127 | return bpe_tokens 128 | 129 | def decode(self, tokens): 130 | text = ''.join([self.decoder[token] for token in tokens]) 131 | text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') 132 | return text 133 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | _CONTOUR_INDEX = 1 if cv2.__version__.split('.')[0] == '3' else 0 4 | 5 | 6 | def check_box_convention(boxes, convention): 7 | """ 8 | Args: 9 | boxes: numpy.ndarray(dtype=np.int or np.float, shape=(num_boxes, 4)) 10 | convention: string. One of ['x0y0x1y1', 'xywh']. 11 | Raises: 12 | RuntimeError if box does not meet the convention. 13 | """ 14 | if (boxes < 0).any(): 15 | raise RuntimeError("Box coordinates must be non-negative.") 16 | 17 | if len(boxes.shape) == 1: 18 | boxes = np.expand_dims(boxes, 0) 19 | elif len(boxes.shape) != 2: 20 | raise RuntimeError("Box array must have dimension (4) or " 21 | "(num_boxes, 4).") 22 | 23 | if boxes.shape[1] != 4: 24 | raise RuntimeError("Box array must have dimension (4) or " 25 | "(num_boxes, 4).") 26 | 27 | if convention == 'x0y0x1y1': 28 | widths = boxes[:, 2] - boxes[:, 0] 29 | heights = boxes[:, 3] - boxes[:, 1] 30 | elif convention == 'xywh': 31 | widths = boxes[:, 2] 32 | heights = boxes[:, 3] 33 | else: 34 | raise ValueError("Unknown convention {}.".format(convention)) 35 | 36 | if (widths < 0).any() or (heights < 0).any(): 37 | raise RuntimeError("Boxes do not follow the {} convention." 38 | .format(convention)) 39 | 40 | def calculate_multiple_iou(box_a, box_b): 41 | """ 42 | Args: 43 | box_a: numpy.ndarray(dtype=np.int, shape=(num_a, 4)) 44 | x0y0x1y1 convention. 45 | box_b: numpy.ndarray(dtype=np.int, shape=(num_b, 4)) 46 | x0y0x1y1 convention. 47 | Returns: 48 | ious: numpy.ndarray(dtype=np.int, shape(num_a, num_b)) 49 | """ 50 | num_a = box_a.shape[0] 51 | num_b = box_b.shape[0] 52 | 53 | check_box_convention(box_a, 'x0y0x1y1') 54 | check_box_convention(box_b, 'x0y0x1y1') 55 | 56 | # num_a x 4 -> num_a x num_b x 4 57 | box_a = np.tile(box_a, num_b) 58 | box_a = np.expand_dims(box_a, axis=1).reshape((num_a, num_b, -1)) 59 | 60 | # num_b x 4 -> num_b x num_a x 4 61 | box_b = np.tile(box_b, num_a) 62 | box_b = np.expand_dims(box_b, axis=1).reshape((num_b, num_a, -1)) 63 | 64 | # num_b x num_a x 4 -> num_a x num_b x 4 65 | box_b = np.transpose(box_b, (1, 0, 2)) 66 | 67 | # num_a x num_b 68 | min_x = np.maximum(box_a[:, :, 0], box_b[:, :, 0]) 69 | min_y = np.maximum(box_a[:, :, 1], box_b[:, :, 1]) 70 | max_x = np.minimum(box_a[:, :, 2], box_b[:, :, 2]) 71 | max_y = np.minimum(box_a[:, :, 3], box_b[:, :, 3]) 72 | 73 | # num_a x num_b 74 | area_intersect = (np.maximum(0, max_x - min_x + 1) 75 | * np.maximum(0, max_y - min_y + 1)) 76 | area_a = ((box_a[:, :, 2] - box_a[:, :, 0] + 1) * 77 | (box_a[:, :, 3] - box_a[:, :, 1] + 1)) 78 | area_b = ((box_b[:, :, 2] - box_b[:, :, 0] + 1) * 79 | (box_b[:, :, 3] - box_b[:, :, 1] + 1)) 80 | 81 | denominator = area_a + area_b - area_intersect 82 | degenerate_indices = np.where(denominator <= 0) 83 | denominator[degenerate_indices] = 1 84 | 85 | ious = area_intersect / denominator 86 | ious[degenerate_indices] = 0 87 | return ious 88 | 89 | def parse_xml_to_dict(xml): 90 | """ 91 | 将xml文件解析成字典形式，参考tensorflow的recursive_parse_xml_to_dict 92 | Args: 93 | xml: xml tree obtained by parsing XML file contents using lxml.etree 94 | 95 | Returns: 96 | Python dictionary holding XML contents. 97 | """ 98 | 99 | if len(xml) == 0: # 遍历到底层，直接返回tag对应的信息 100 | return {xml.tag: xml.text} 101 | 102 | result = {} 103 | for child in xml: 104 | child_result = parse_xml_to_dict(child) # 递归遍历标签信息 105 | if child.tag != 'object': 106 | result[child.tag] = child_result[child.tag] 107 | else: 108 | if child.tag not in result: # 因为object可能有多个，所以需要放入列表里 109 | result[child.tag] = [] 110 | result[child.tag].append(child_result[child.tag]) 111 | return {xml.tag: result} 112 | 113 | 114 | 115 | def scoremap2bbox(scoremap, threshold, multi_contour_eval=False): 116 | height, width = scoremap.shape 117 | scoremap_image = np.expand_dims((scoremap * 255).astype(np.uint8), 2) 118 | _, thr_gray_heatmap = cv2.threshold( 119 | src=scoremap_image, 120 | thresh=int(threshold * np.max(scoremap_image)), 121 | maxval=255, 122 | type=cv2.THRESH_BINARY) 123 | contours = cv2.findContours( 124 | image=thr_gray_heatmap, 125 | mode=cv2.RETR_TREE, 126 | method=cv2.CHAIN_APPROX_SIMPLE)[_CONTOUR_INDEX] 127 | 128 | if len(contours) == 0: 129 | return np.asarray([[0, 0, 0, 0]]), 1 130 | 131 | if not multi_contour_eval: 132 | contours = [max(contours, key=cv2.contourArea)] 133 | 134 | estimated_boxes = [] 135 | for contour in contours: 136 | x, y, w, h = cv2.boundingRect(contour) 137 | x0, y0, x1, y1 = x, y, x + w, y + h 138 | x1 = min(x1, width - 1) 139 | y1 = min(y1, height - 1) 140 | estimated_boxes.append([x0, y0, x1, y1]) 141 | 142 | return np.asarray(estimated_boxes), len(contours) 143 | -------------------------------------------------------------------------------- /pytorch_grad_cam/ablation_cam_multilayer.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import torch 4 | import tqdm 5 | from pytorch_grad_cam.base_cam import BaseCAM 6 | 7 | 8 | class AblationLayer(torch.nn.Module): 9 | def __init__(self, layer, reshape_transform, indices): 10 | super(AblationLayer, self).__init__() 11 | 12 | self.layer = layer 13 | self.reshape_transform = reshape_transform 14 | # The channels to zero out: 15 | self.indices = indices 16 | 17 | def forward(self, x): 18 | self.__call__(x) 19 | 20 | def __call__(self, x): 21 | output = self.layer(x) 22 | 23 | # Hack to work with ViT, 24 | # Since the activation channels are last and not first like in CNNs 25 | # Probably should remove it? 26 | if self.reshape_transform is not None: 27 | output = output.transpose(1, 2) 28 | 29 | for i in range(output.size(0)): 30 | 31 | # Commonly the minimum activation will be 0, 32 | # And then it makes sense to zero it out. 33 | # However depending on the architecture, 34 | # If the values can be negative, we use very negative values 35 | # to perform the ablation, deviating from the paper. 36 | if torch.min(output) == 0: 37 | output[i, self.indices[i], :] = 0 38 | else: 39 | ABLATION_VALUE = 1e5 40 | output[i, self.indices[i], :] = torch.min( 41 | output) - ABLATION_VALUE 42 | 43 | if self.reshape_transform is not None: 44 | output = output.transpose(2, 1) 45 | 46 | return output 47 | 48 | 49 | def replace_layer_recursive(model, old_layer, new_layer): 50 | for name, layer in model._modules.items(): 51 | if layer == old_layer: 52 | model._modules[name] = new_layer 53 | return True 54 | elif replace_layer_recursive(layer, old_layer, new_layer): 55 | return True 56 | return False 57 | 58 | 59 | class AblationCAM(BaseCAM): 60 | def __init__(self, model, target_layers, use_cuda=False, 61 | reshape_transform=None): 62 | super(AblationCAM, self).__init__(model, target_layers, use_cuda, 63 | reshape_transform) 64 | 65 | if len(target_layers) > 1: 66 | print( 67 | "Warning. You are usign Ablation CAM with more than 1 layers. " 68 | "This is supported only if all layers have the same output shape") 69 | 70 | def set_ablation_layers(self): 71 | self.ablation_layers = [] 72 | for target_layer in self.target_layers: 73 | ablation_layer = AblationLayer(target_layer, 74 | self.reshape_transform, indices=[]) 75 | self.ablation_layers.append(ablation_layer) 76 | replace_layer_recursive(self.model, target_layer, ablation_layer) 77 | 78 | def unset_ablation_layers(self): 79 | # replace the model back to the original state 80 | for ablation_layer, target_layer in zip( 81 | self.ablation_layers, self.target_layers): 82 | replace_layer_recursive(self.model, ablation_layer, target_layer) 83 | 84 | def set_ablation_layer_batch_indices(self, indices): 85 | for ablation_layer in self.ablation_layers: 86 | ablation_layer.indices = indices 87 | 88 | def trim_ablation_layer_batch_indices(self, keep): 89 | for ablation_layer in self.ablation_layers: 90 | ablation_layer.indices = ablation_layer.indices[:keep] 91 | 92 | def get_cam_weights(self, 93 | input_tensor, 94 | target_category, 95 | activations, 96 | grads): 97 | with torch.no_grad(): 98 | outputs = self.model(input_tensor).cpu().numpy() 99 | original_scores = [] 100 | for i in range(input_tensor.size(0)): 101 | original_scores.append(outputs[i, target_category[i]]) 102 | original_scores = np.float32(original_scores) 103 | 104 | self.set_ablation_layers() 105 | 106 | if hasattr(self, "batch_size"): 107 | BATCH_SIZE = self.batch_size 108 | else: 109 | BATCH_SIZE = 32 110 | 111 | number_of_channels = activations.shape[1] 112 | weights = [] 113 | 114 | with torch.no_grad(): 115 | # Iterate over the input batch 116 | for tensor, category in zip(input_tensor, target_category): 117 | batch_tensor = tensor.repeat(BATCH_SIZE, 1, 1, 1) 118 | for i in tqdm.tqdm(range(0, number_of_channels, BATCH_SIZE)): 119 | self.set_ablation_layer_batch_indices( 120 | list(range(i, i + BATCH_SIZE))) 121 | 122 | if i + BATCH_SIZE > number_of_channels: 123 | keep = number_of_channels - i 124 | batch_tensor = batch_tensor[:keep] 125 | self.trim_ablation_layer_batch_indices(self, keep) 126 | score = self.model(batch_tensor)[:, category].cpu().numpy() 127 | weights.extend(score) 128 | 129 | weights = np.float32(weights) 130 | weights = weights.reshape(activations.shape[:2]) 131 | original_scores = original_scores[:, None] 132 | weights = (original_scores - weights) / original_scores 133 | 134 | # replace the model back to the original state 135 | self.unset_ablation_layers() 136 | return weights 137 | -------------------------------------------------------------------------------- /pytorch_grad_cam/ablation_layer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from collections import OrderedDict 3 | import numpy as np 4 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 5 | 6 | 7 | class AblationLayer(torch.nn.Module): 8 | def __init__(self): 9 | super(AblationLayer, self).__init__() 10 | 11 | def objectiveness_mask_from_svd(self, activations, threshold=0.01): 12 | """ Experimental method to get a binary mask to compare if the activation is worth ablating. 13 | The idea is to apply the EigenCAM method by doing PCA on the activations. 14 | Then we create a binary mask by comparing to a low threshold. 15 | Areas that are masked out, are probably not interesting anyway. 16 | """ 17 | 18 | projection = get_2d_projection(activations[None, :])[0, :] 19 | projection = np.abs(projection) 20 | projection = projection - projection.min() 21 | projection = projection / projection.max() 22 | projection = projection > threshold 23 | return projection 24 | 25 | def activations_to_be_ablated(self, activations, ratio_channels_to_ablate=1.0): 26 | """ Experimental method to get a binary mask to compare if the activation is worth ablating. 27 | Create a binary CAM mask with objectiveness_mask_from_svd. 28 | Score each Activation channel, by seeing how much of its values are inside the mask. 29 | Then keep the top channels. 30 | 31 | """ 32 | if ratio_channels_to_ablate == 1.0: 33 | self.indices = np.int32(range(activations.shape[0])) 34 | return self.indices 35 | 36 | projection = self.objectiveness_mask_from_svd(activations) 37 | 38 | scores = [] 39 | for channel in activations: 40 | normalized = np.abs(channel) 41 | normalized = normalized - normalized.min() 42 | normalized = normalized / np.max(normalized) 43 | score = (projection*normalized).sum() / normalized.sum() 44 | scores.append(score) 45 | scores = np.float32(scores) 46 | 47 | indices = list(np.argsort(scores)) 48 | high_score_indices = indices[::-1][: int(len(indices) * ratio_channels_to_ablate)] 49 | low_score_indices = indices[: int(len(indices) * ratio_channels_to_ablate)] 50 | self.indices = np.int32(high_score_indices + low_score_indices) 51 | return self.indices 52 | 53 | def set_next_batch(self, input_batch_index, activations, num_channels_to_ablate): 54 | """ This creates the next batch of activations from the layer. 55 | Just take corresponding batch member from activations, and repeat it num_channels_to_ablate times. 56 | """ 57 | self.activations = activations[input_batch_index, :, :, :].clone().unsqueeze(0).repeat(num_channels_to_ablate, 1, 1, 1) 58 | 59 | def __call__(self, x): 60 | output = self.activations 61 | for i in range(output.size(0)): 62 | # Commonly the minimum activation will be 0, 63 | # And then it makes sense to zero it out. 64 | # However depending on the architecture, 65 | # If the values can be negative, we use very negative values 66 | # to perform the ablation, deviating from the paper. 67 | if torch.min(output) == 0: 68 | output[i, self.indices[i], :] = 0 69 | else: 70 | ABLATION_VALUE = 1e7 71 | output[i, self.indices[i], :] = torch.min( 72 | output) - ABLATION_VALUE 73 | 74 | return output 75 | 76 | 77 | class AblationLayerVit(AblationLayer): 78 | def __init__(self): 79 | super(AblationLayerVit, self).__init__() 80 | 81 | def __call__(self, x): 82 | output = self.activations 83 | output = output.transpose(1, 2) 84 | for i in range(output.size(0)): 85 | 86 | # Commonly the minimum activation will be 0, 87 | # And then it makes sense to zero it out. 88 | # However depending on the architecture, 89 | # If the values can be negative, we use very negative values 90 | # to perform the ablation, deviating from the paper. 91 | if torch.min(output) == 0: 92 | output[i, self.indices[i], :] = 0 93 | else: 94 | ABLATION_VALUE = 1e7 95 | output[i, self.indices[i], :] = torch.min( 96 | output) - ABLATION_VALUE 97 | 98 | output = output.transpose(2, 1) 99 | 100 | return output 101 | 102 | def set_next_batch(self, input_batch_index, activations, num_channels_to_ablate): 103 | """ This creates the next batch of activations from the layer. 104 | Just take corresponding batch member from activations, and repeat it num_channels_to_ablate times. 105 | """ 106 | self.activations = activations[input_batch_index, :, :].clone().unsqueeze(0).repeat(num_channels_to_ablate, 1, 1) 107 | 108 | 109 | 110 | class AblationLayerFasterRCNN(AblationLayer): 111 | def __init__(self): 112 | super(AblationLayerFasterRCNN, self).__init__() 113 | 114 | def set_next_batch(self, input_batch_index, activations, num_channels_to_ablate): 115 | """ Extract the next batch member from activations, 116 | and repeat it num_channels_to_ablate times. 117 | """ 118 | self.activations = OrderedDict() 119 | for key, value in activations.items(): 120 | fpn_activation = value[input_batch_index, :, :, :].clone().unsqueeze(0) 121 | self.activations[key] = fpn_activation.repeat(num_channels_to_ablate, 1, 1, 1) 122 | 123 | def __call__(self, x): 124 | result = self.activations 125 | layers = {0: '0', 1: '1', 2: '2', 3: '3', 4: 'pool'} 126 | num_channels_to_ablate = result['pool'].size(0) 127 | for i in range(num_channels_to_ablate): 128 | pyramid_layer = int(self.indices[i]/256) 129 | index_in_pyramid_layer = int(self.indices[i] % 256) 130 | result[layers[pyramid_layer]][i, index_in_pyramid_layer, :, :] = -1000 131 | return result 132 | -------------------------------------------------------------------------------- /eval_cam_with_crf.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | import numpy as np 4 | import torch.nn.functional as F 5 | import joblib 6 | import multiprocessing 7 | import pydensecrf.densecrf as dcrf 8 | import pydensecrf.utils as utils 9 | import cv2 10 | from PIL import Image 11 | import argparse 12 | 13 | 14 | class DenseCRF(object): 15 | def __init__(self, iter_max, pos_w, pos_xy_std, bi_w, bi_xy_std, bi_rgb_std): 16 | self.iter_max = iter_max 17 | self.pos_w = pos_w 18 | self.pos_xy_std = pos_xy_std 19 | self.bi_w = bi_w 20 | self.bi_xy_std = bi_xy_std 21 | self.bi_rgb_std = bi_rgb_std 22 | 23 | def __call__(self, image, probmap): 24 | C, H, W = probmap.shape 25 | 26 | U = utils.unary_from_softmax(probmap) 27 | U = np.ascontiguousarray(U) 28 | 29 | image = np.ascontiguousarray(image) 30 | 31 | d = dcrf.DenseCRF2D(W, H, C) 32 | d.setUnaryEnergy(U) 33 | d.addPairwiseGaussian(sxy=self.pos_xy_std, compat=self.pos_w) 34 | d.addPairwiseBilateral( 35 | sxy=self.bi_xy_std, srgb=self.bi_rgb_std, rgbim=image, compat=self.bi_w 36 | ) 37 | 38 | Q = d.inference(self.iter_max) 39 | Q = np.array(Q).reshape((C, H, W)) 40 | 41 | return Q 42 | 43 | def makedirs(dirs): 44 | if not os.path.exists(dirs): 45 | os.makedirs(dirs) 46 | 47 | def _fast_hist(label_true, label_pred, n_class): 48 | mask = (label_true >= 0) & (label_true < n_class) 49 | hist = np.bincount( 50 | n_class * label_true[mask].astype(int) + label_pred[mask], 51 | minlength=n_class ** 2, 52 | ).reshape(n_class, n_class) 53 | return hist 54 | 55 | def scores(label_trues, label_preds, n_class): 56 | hist = np.zeros((n_class, n_class)) 57 | for lt, lp in zip(label_trues, label_preds): 58 | hist += _fast_hist(lt.flatten(), lp.flatten(), n_class) 59 | acc = np.diag(hist).sum() / hist.sum() 60 | acc_cls = np.diag(hist) / hist.sum(axis=1) 61 | acc_cls = np.nanmean(acc_cls) 62 | iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist)) 63 | valid = hist.sum(axis=1) > 0 # added 64 | mean_iu = np.nanmean(iu[valid]) 65 | freq = hist.sum(axis=1) / hist.sum() 66 | fwavacc = (freq[freq > 0] * iu[freq > 0]).sum() 67 | cls_iu = dict(zip(range(n_class), iu)) 68 | 69 | return { 70 | "Pixel Accuracy": acc, 71 | "Mean Accuracy": acc_cls, 72 | "Frequency Weighted IoU": fwavacc, 73 | "Mean IoU": mean_iu, 74 | "Class IoU": cls_iu, 75 | } 76 | 77 | def crf(n_jobs, is_coco=False): 78 | """ 79 | CRF post-processing on pre-computed logits 80 | """ 81 | 82 | # Configuration 83 | torch.set_grad_enabled(False) 84 | print("# jobs:", n_jobs) 85 | 86 | # CRF post-processor 87 | postprocessor = DenseCRF( 88 | iter_max=10, 89 | pos_xy_std=1, 90 | pos_w=3, 91 | bi_xy_std=67, 92 | bi_rgb_std=3, 93 | bi_w=4, 94 | ) 95 | 96 | # Process per sample 97 | def process(i): 98 | image_id = eval_list[i] 99 | image_path = os.path.join(args.image_root, image_id + '.jpg') 100 | image = cv2.imread(image_path, cv2.IMREAD_COLOR).astype(np.float32) 101 | label_path = os.path.join(args.gt_root, image_id + '.png') 102 | gt_label = np.asarray(Image.open(label_path), dtype=np.int32) 103 | # Mean subtraction 104 | image -= mean_bgr 105 | # HWC -> CHW 106 | image = image.transpose(2, 0, 1) 107 | 108 | filename = os.path.join(args.cam_out_dir, image_id + ".npy") 109 | cam_dict = np.load(filename, allow_pickle=True).item() 110 | cams = cam_dict['attn_highres'] 111 | bg_score = np.power(1 - np.max(cams, axis=0, keepdims=True), 1) 112 | cams = np.concatenate((bg_score, cams), axis=0) 113 | prob = cams 114 | 115 | image = image.astype(np.uint8).transpose(1, 2, 0) 116 | prob = postprocessor(image, prob) 117 | 118 | label = np.argmax(prob, axis=0) 119 | keys = np.pad(cam_dict['keys'] + 1, (1, 0), mode='constant') 120 | label = keys[label] 121 | if not args.eval_only: 122 | confidence = np.max(prob, axis=0) 123 | label[confidence < 0.95] = 255 124 | cv2.imwrite(os.path.join(args.pseudo_mask_save_path, image_id + '.png'), label.astype(np.uint8)) 125 | 126 | return label.astype(np.uint8), gt_label.astype(np.uint8) 127 | 128 | # CRF in multi-process 129 | results = joblib.Parallel(n_jobs=n_jobs, verbose=10, pre_dispatch="all")( 130 | [joblib.delayed(process)(i) for i in range(len(eval_list))] 131 | ) 132 | if args.eval_only: 133 | preds, gts = zip(*results) 134 | 135 | # Pixel Accuracy, Mean Accuracy, Class IoU, Mean IoU, Freq Weighted IoU 136 | score = scores(gts, preds, n_class=21 if not is_coco else 81) 137 | print(score) 138 | 139 | if __name__ == "__main__": 140 | parser = argparse.ArgumentParser() 141 | parser.add_argument("--cam_out_dir", default="./cam_out", type=str) 142 | parser.add_argument("--pseudo_mask_save_path", default="/home/xxx/code/code48/ablation/usss/voc/val_attn07_crf", type=str) 143 | parser.add_argument("--split_file", default="/home/xxx/datasets/VOC2012/ImageSets/Segmentation/train.txt", 144 | type=str) 145 | parser.add_argument("--cam_eval_thres", default=2, type=float) 146 | parser.add_argument("--gt_root", default="/home/xxx/datasets/VOC2012/SegmentationClassAug", type=str) 147 | parser.add_argument("--image_root", default="/home/xxx/datasets/VOC2012/JPEGImages", type=str) 148 | parser.add_argument("--eval_only", action="store_true") 149 | args = parser.parse_args() 150 | 151 | is_coco = 'coco' in args.cam_out_dir 152 | if 'voc' in args.cam_out_dir: 153 | eval_list = list(np.loadtxt(args.split_file, dtype=str)) 154 | elif 'coco' in args.cam_out_dir: 155 | file_list = tuple(open(args.split_file, "r")) 156 | file_list = [id_.rstrip().split(" ") for id_ in file_list] 157 | eval_list = [x[0] for x in file_list]#[:2000] 158 | print('{} images to eval'.format(len(eval_list))) 159 | 160 | if not args.eval_only and not os.path.exists(args.pseudo_mask_save_path): 161 | os.makedirs(args.pseudo_mask_save_path) 162 | 163 | mean_bgr = (104.008, 116.669, 122.675) 164 | n_jobs =multiprocessing.cpu_count() 165 | crf(n_jobs, is_coco) 166 | -------------------------------------------------------------------------------- /pytorch_grad_cam/ablation_cam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import tqdm 4 | from typing import Callable, List 5 | from pytorch_grad_cam.base_cam import BaseCAM 6 | from pytorch_grad_cam.utils.find_layers import replace_layer_recursive 7 | from pytorch_grad_cam.ablation_layer import AblationLayer 8 | 9 | 10 | """ Implementation of AblationCAM 11 | https://openaccess.thecvf.com/content_WACV_2020/papers/Desai_Ablation-CAM_Visual_Explanations_for_Deep_Convolutional_Network_via_Gradient-free_Localization_WACV_2020_paper.pdf 12 | 13 | Ablate individual activations, and then measure the drop in the target score. 14 | 15 | In the current implementation, the target layer activations is cached, so it won't be re-computed. 16 | However layers before it, if any, will not be cached. 17 | This means that if the target layer is a large block, for example model.featuers (in vgg), there will 18 | be a large save in run time. 19 | 20 | Since we have to go over many channels and ablate them, and every channel ablation requires a forward pass, 21 | it would be nice if we could avoid doing that for channels that won't contribute anwyay, making it much faster. 22 | The parameter ratio_channels_to_ablate controls how many channels should be ablated, using an experimental method 23 | (to be improved). The default 1.0 value means that all channels will be ablated. 24 | """ 25 | 26 | 27 | class AblationCAM(BaseCAM): 28 | def __init__(self, 29 | model: torch.nn.Module, 30 | target_layers: List[torch.nn.Module], 31 | use_cuda: bool = False, 32 | reshape_transform: Callable = None, 33 | ablation_layer: torch.nn.Module = AblationLayer(), 34 | batch_size: int = 32, 35 | ratio_channels_to_ablate: float = 1.0) -> None: 36 | 37 | super(AblationCAM, self).__init__(model, 38 | target_layers, 39 | use_cuda, 40 | reshape_transform, 41 | uses_gradients=False) 42 | self.batch_size = batch_size 43 | self.ablation_layer = ablation_layer 44 | self.ratio_channels_to_ablate = ratio_channels_to_ablate 45 | 46 | def save_activation(self, module, input, output) -> None: 47 | """ Helper function to save the raw activations from the target layer """ 48 | self.activations = output 49 | 50 | def assemble_ablation_scores(self, 51 | new_scores: list, 52 | original_score: float , 53 | ablated_channels: np.ndarray, 54 | number_of_channels: int) -> np.ndarray: 55 | """ Take the value from the channels that were ablated, 56 | and just set the original score for the channels that were skipped """ 57 | 58 | index = 0 59 | result = [] 60 | sorted_indices = np.argsort(ablated_channels) 61 | ablated_channels = ablated_channels[sorted_indices] 62 | new_scores = np.float32(new_scores)[sorted_indices] 63 | 64 | for i in range(number_of_channels): 65 | if index < len(ablated_channels) and ablated_channels[index] == i: 66 | weight = new_scores[index] 67 | index = index + 1 68 | else: 69 | weight = original_score 70 | result.append(weight) 71 | 72 | return result 73 | 74 | def get_cam_weights(self, 75 | input_tensor: torch.Tensor, 76 | target_layer: torch.nn.Module, 77 | targets: List[Callable], 78 | activations: torch.Tensor, 79 | grads: torch.Tensor) -> np.ndarray: 80 | 81 | # Do a forward pass, compute the target scores, and cache the activations 82 | handle = target_layer.register_forward_hook(self.save_activation) 83 | with torch.no_grad(): 84 | outputs = self.model(input_tensor) 85 | handle.remove() 86 | original_scores = np.float32([target(output).cpu().item() for target, output in zip(targets, outputs)]) 87 | 88 | # Replace the layer with the ablation layer. 89 | # When we finish, we will replace it back, so the original model is unchanged. 90 | ablation_layer = self.ablation_layer 91 | replace_layer_recursive(self.model, target_layer, ablation_layer) 92 | 93 | number_of_channels = activations.shape[1] 94 | weights = [] 95 | # This is a "gradient free" method, so we don't need gradients here. 96 | with torch.no_grad(): 97 | # Loop over each of the batch images and ablate activations for it. 98 | for batch_index, (target, tensor) in enumerate(zip(targets, input_tensor)): 99 | new_scores = [] 100 | batch_tensor = tensor.repeat(self.batch_size, 1, 1, 1) 101 | 102 | # Check which channels should be ablated. Normally this will be all channels, 103 | # But we can also try to speed this up by using a low ratio_channels_to_ablate. 104 | channels_to_ablate = ablation_layer.activations_to_be_ablated(activations[batch_index, :], 105 | self.ratio_channels_to_ablate) 106 | number_channels_to_ablate = len(channels_to_ablate) 107 | 108 | for i in tqdm.tqdm(range(0, number_channels_to_ablate, self.batch_size)): 109 | if i + self.batch_size > number_channels_to_ablate: 110 | batch_tensor = batch_tensor[:(number_channels_to_ablate - i)] 111 | 112 | # Change the state of the ablation layer so it ablates the next channels. 113 | # TBD: Move this into the ablation layer forward pass. 114 | ablation_layer.set_next_batch(input_batch_index=batch_index, 115 | activations=self.activations, 116 | num_channels_to_ablate=batch_tensor.size(0)) 117 | score = [target(o).cpu().item() for o in self.model(batch_tensor)] 118 | new_scores.extend(score) 119 | ablation_layer.indices = ablation_layer.indices[batch_tensor.size(0):] 120 | 121 | new_scores = self.assemble_ablation_scores(new_scores, 122 | original_scores[batch_index], 123 | channels_to_ablate, 124 | number_of_channels) 125 | weights.extend(new_scores) 126 | 127 | weights = np.float32(weights) 128 | weights = weights.reshape(activations.shape[:2]) 129 | original_scores = original_scores[:, None] 130 | weights = (original_scores - weights) / original_scores 131 | 132 | # Replace the model back to the original state 133 | replace_layer_recursive(self.model, ablation_layer, target_layer) 134 | return weights 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/clip-is-also-an-efficient-segmenter-a-text/weakly-supervised-semantic-segmentation-on)](https://paperswithcode.com/sota/weakly-supervised-semantic-segmentation-on?p=clip-is-also-an-efficient-segmenter-a-text) 2 | [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/clip-is-also-an-efficient-segmenter-a-text/weakly-supervised-semantic-segmentation-on-1)](https://paperswithcode.com/sota/weakly-supervised-semantic-segmentation-on-1?p=clip-is-also-an-efficient-segmenter-a-text) 3 | [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/clip-is-also-an-efficient-segmenter-a-text/weakly-supervised-semantic-segmentation-on-4)](https://paperswithcode.com/sota/weakly-supervised-semantic-segmentation-on-4?p=clip-is-also-an-efficient-segmenter-a-text) 4 | # CLIP is Also an Efficient Segmenter: A Text-Driven Approach for Weakly Supervised Semantic Segmentation (CVPR 2023) 5 | :closed_book: [[arxiv]](https://arxiv.org/abs/2212.09506) [[CVPR]](https://openaccess.thecvf.com/content/CVPR2023/html/Lin_CLIP_Is_Also_an_Efficient_Segmenter_A_Text-Driven_Approach_for_CVPR_2023_paper.html) 6 | 7 | ![images](CLIP-ES.png) 8 | 9 | ## :tada: :tada: :tada: News 10 | 11 | - **`2023/12/09` Our new paper [TagCLIP: A Local-to-Global Framework to Enhance Open-Vocabulary Multi-Label Classification of CLIP Without Training](https://github.com/linyq2117/TagCLIP) is accepted by AAAI 2024. It can generate image-level labels based on frozen CLIP and can realize annotation-free semantic segmentation without any training when combining with CLIP-ES.** 12 | 13 | - **`2023/2/28` Our paper is accepted by CVPR 2023.** 14 | 15 | ## Reqirements 16 | 17 | ``` 18 | # create conda env 19 | conda create -n clip-es python=3.9 20 | conda activate clip-es 21 | 22 | # install packages 23 | pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html 24 | pip install opencv-python ftfy regex tqdm ttach tensorboard lxml cython 25 | 26 | # install pydensecrf from source 27 | git clone https://github.com/lucasb-eyer/pydensecrf 28 | cd pydensecrf 29 | python setup.py install 30 | ``` 31 | 32 | ## Preparing Datasets 33 | ### PASCAL VOC2012 34 | Download images in PASCAL VOC2012 dataset at [here](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/) and the train_aug groundtruth at [here](https://www.dropbox.com/s/oeu149j8qtbs1x0/SegmentationClassAug.zip?dl=0). 35 | The structure of `/your_home_dir/datasets/VOC2012`should be organized as follows: 36 | 37 | ``` 38 | ---VOC2012/ 39 | --Annotations 40 | --ImageSets 41 | --JPEGImages 42 | --SegmentationClass 43 | --SegmentationClassAug 44 | ``` 45 | 46 | ### MS COCO2014 47 | Download MS COCO images from the [official website](https://cocodataset.org/#download). 48 | Download semantic segmentation annotations for the MS COCO dataset at [here](https://drive.google.com/file/d/1pRE9SEYkZKVg0Rgz2pi9tg48j7GlinPV/view?usp=sharing). 49 | The structure of `/your_home_dir/datasets/COCO2014`are suggested to be organized as follows: 50 | ``` 51 | ---COCO2014/ 52 | --Annotations 53 | --JPEGImages 54 | -train2014 55 | -val2014 56 | --SegmentationClass 57 | ``` 58 | 59 | ### Preparing pre-trained model 60 | Download CLIP pre-trained [ViT-B/16] at [here](https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt) and put it to `/your_home_dir/pretrained_models/clip`. 61 | 62 | ## Usage 63 | ### Step 1. Generate CAMs for train (train_aug) set. 64 | ``` 65 | # For VOC12 66 | CUDA_VISIBLE_DEVICES=0 python generate_cams_voc12.py --img_root /your_home_dir/datasets/VOC2012/JPEGImages --split_file ./voc12/train_aug.txt --model /your_home_dir/pretrained_models/clip/ViT-B-16.pt --num_workers 1 --cam_out_dir ./output/voc12/cams 67 | 68 | # For COCO14 69 | CUDA_VISIBLE_DEVICES=0 python generate_cams_coco14.py --img_root /your_home_dir/datasets/COCO2014/JPEGImages/train2014 --split_file ./coco14/train.txt --model /your_home_dir/pretrained_models/clip/ViT-B-16.pt --num_workers 1 --cam_out_dir ./output/coco14/cams 70 | ``` 71 | 72 | ### Step 2. Evaluate generated CAMs and use CRF to postprocess 73 | ``` 74 | # (optional) evaluate generated CAMs 75 | ## for VOC12 76 | python eval_cam.py --cam_out_dir ./output/voc12/cams --cam_type attn_highres --gt_root /your_home_dir/datasets/VOC2012/SegmentationClassAug --split_file ./voc12/train.txt 77 | ## for COCO14 78 | python eval_cam.py --cam_out_dir ./output/coco14/cams --cam_type attn_highres --gt_root /your_home_dir/datasets/COCO2014/SegmentationClass --split_file ./coco14/train.txt 79 | 80 | # use CRF process to generate pseudo masks 81 | (realize confidence-guided loss by setting pixels with low confidence to 255) 82 | ## for VOC12 83 | python eval_cam_with_crf.py --cam_out_dir ./output/voc12/cams --gt_root /your_home_dir/datasets/VOC2012/SegmentationClassAug --image_root /your_home_dir/datasets/VOC2012/JPEGImages --split_file ./voc12/train_aug.txt --pseudo_mask_save_path ./output/voc12/pseudo_masks 84 | ## for COCO14 85 | python eval_cam_with_crf.py --cam_out_dir ./output/coco14/cams --gt_root /your_home_dir/datasets/COCO2014/SegmentationClass --image_root /your_home_dir/datasets/COCO2014/JPEGImages/train2014 --split_file ./coco14/train.txt --pseudo_mask_save_path ./output/coco2014/pseudo_masks 86 | 87 | # eval CRF processed pseudo masks 88 | ## for VOC12 89 | python eval_cam_with_crf.py --cam_out_dir ./output/voc12/cams --gt_root /your_home_dir/datasets/VOC2012/SegmentationClassAug --image_root /your_home_dir/datasets/VOC2012/JPEGImages --split_file ./voc12/train_aug.txt --eval_only 90 | ## for COCO14 91 | python eval_cam_with_crf.py --cam_out_dir ./output/coco14/cams --gt_root /your_home_dir/datasets/COCO2014/SegmentationClass --image_root /your_home_dir/datasets/COCO2014/JPEGImages/train2014 --split_file ./coco14/train.txt --eval_only 92 | 93 | ``` 94 | The generated pseudo masks of VOC12 and COCO14 can be found at [Google Drive](https://drive.google.com/drive/folders/1i9fp2c87s5ungxFpLFgUm_zUACpLuKtJ?usp=sharing). 95 | ### Step 3. Train Segmentation Model 96 | To train DeepLab-v2, we refer to [deeplab-pytorch](https://github.com/kazuto1011/deeplab-pytorch). The ImageNet pre-trained model can be found in [AdvCAM](https://github.com/jbeomlee93/AdvCAM). 97 | 98 | ## Results 99 | ### The quality of generated pseudo masks on PASCAL VOC2012 train set. 100 | | Method | CAMs | +CRF | 101 | | --- | --- | --- | 102 | | CLIP-ES | 70.8 | 75.0 | 103 | ### Segmentation results on PASCAL VOC2012 val and test sets. 104 | | Method | Network | Pretrained | val | test | 105 | | --- | --- | --- | --- | --- | 106 | | CLIP-ES | DeepLabV2 | ImageNet | 71.1 | 71.4 | 107 | | CLIP-ES | DeepLabV2 | COCO | 73.8 | 73.9 | 108 | ### Segmentation results on MS COCO2014 val set. 109 | | Method | Network | Pretrained | val | 110 | | --- | --- | --- | --- | 111 | | CLIP-ES | DeepLabV2 | ImageNet | 45.4 | 112 | ## Acknowledgement 113 | We borrowed the code from [CLIP](https://github.com/openai/CLIP) and [pytorch_grad_cam](https://github.com/jacobgil/pytorch-grad-cam/tree/61e9babae8600351b02b6e90864e4807f44f2d4a). Thanks for their wonderful works. 114 | 115 | ## Citation 116 | If you find this project helpful for your research, please consider citing the following BibTeX entry. 117 | ``` 118 | @InProceedings{Lin_2023_CVPR, 119 | author = {Lin, Yuqi and Chen, Minghao and Wang, Wenxiao and Wu, Boxi and Li, Ke and Lin, Binbin and Liu, Haifeng and He, Xiaofei}, 120 | title = {CLIP Is Also an Efficient Segmenter: A Text-Driven Approach for Weakly Supervised Semantic Segmentation}, 121 | booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 122 | month = {June}, 123 | year = {2023}, 124 | pages = {15305-15314} 125 | } 126 | ``` -------------------------------------------------------------------------------- /pytorch_grad_cam/base_cam.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import ttach as tta 4 | from typing import Callable, List, Tuple 5 | from pytorch_grad_cam.activations_and_gradients import ActivationsAndGradients 6 | from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection 7 | from pytorch_grad_cam.utils.image import scale_cam_image 8 | from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget 9 | 10 | 11 | class BaseCAM: 12 | def __init__(self, 13 | model: torch.nn.Module, 14 | target_layers: List[torch.nn.Module], 15 | use_cuda: bool = False, 16 | reshape_transform: Callable = None, 17 | compute_input_gradient: bool = False, 18 | uses_gradients: bool = True) -> None: 19 | self.model = model.eval() 20 | self.target_layers = target_layers 21 | self.cuda = use_cuda 22 | if self.cuda: 23 | self.model = model.cuda() 24 | self.reshape_transform = reshape_transform 25 | self.compute_input_gradient = compute_input_gradient 26 | self.uses_gradients = uses_gradients 27 | self.activations_and_grads = ActivationsAndGradients( 28 | self.model, target_layers, reshape_transform) 29 | 30 | """ Get a vector of weights for every channel in the target layer. 31 | Methods that return weights channels, 32 | will typically need to only implement this function. """ 33 | 34 | def get_cam_weights(self, 35 | input_tensor: torch.Tensor, 36 | target_layers: List[torch.nn.Module], 37 | targets: List[torch.nn.Module], 38 | activations: torch.Tensor, 39 | grads: torch.Tensor) -> np.ndarray: 40 | raise Exception("Not Implemented") 41 | 42 | def get_cam_image(self, 43 | input_tensor: torch.Tensor, 44 | target_layer: torch.nn.Module, 45 | targets: List[torch.nn.Module], 46 | activations: torch.Tensor, 47 | grads: torch.Tensor, 48 | eigen_smooth: bool = False) -> np.ndarray: 49 | 50 | weights = self.get_cam_weights(input_tensor, 51 | target_layer, 52 | targets, 53 | activations, 54 | grads) 55 | weighted_activations = weights[:, :, None, None] * activations 56 | if eigen_smooth: 57 | cam = get_2d_projection(weighted_activations) 58 | else: 59 | cam = weighted_activations.sum(axis=1) 60 | return cam 61 | 62 | def forward(self, 63 | input_tensor: torch.Tensor, 64 | targets: List[torch.nn.Module], 65 | target_size, 66 | eigen_smooth: bool = False) -> np.ndarray: 67 | 68 | if self.cuda: 69 | input_tensor = input_tensor.cuda() 70 | 71 | if self.compute_input_gradient: 72 | input_tensor = torch.autograd.Variable(input_tensor, 73 | requires_grad=True) 74 | 75 | W,H = self.get_target_width_height(input_tensor) 76 | outputs = self.activations_and_grads(input_tensor,H,W) 77 | if targets is None: 78 | if isinstance(input_tensor, list): 79 | target_categories = np.argmax(outputs[0].cpu().data.numpy(), axis=-1) 80 | else: 81 | target_categories = np.argmax(outputs.cpu().data.numpy(), axis=-1) 82 | targets = [ClassifierOutputTarget(category) for category in target_categories] 83 | 84 | if self.uses_gradients: 85 | self.model.zero_grad() 86 | if isinstance(input_tensor, list): 87 | loss = sum([target(output[0]) for target, output in zip(targets, outputs)]) 88 | else: 89 | loss = sum([target(output) for target, output in zip(targets, outputs)]) 90 | loss.backward(retain_graph=True) 91 | 92 | # In most of the saliency attribution papers, the saliency is 93 | # computed with a single target layer. 94 | # Commonly it is the last convolutional layer. 95 | # Here we support passing a list with multiple target layers. 96 | # It will compute the saliency image for every image, 97 | # and then aggregate them (with a default mean aggregation). 98 | # This gives you more flexibility in case you just want to 99 | # use all conv layers for example, all Batchnorm layers, 100 | # or something else. 101 | cam_per_layer = self.compute_cam_per_layer(input_tensor, 102 | targets, 103 | target_size, 104 | eigen_smooth) 105 | if isinstance(input_tensor, list): 106 | return self.aggregate_multi_layers(cam_per_layer), outputs[0], outputs[1] 107 | else: 108 | return self.aggregate_multi_layers(cam_per_layer), outputs 109 | 110 | def get_target_width_height(self, 111 | input_tensor: torch.Tensor) -> Tuple[int, int]: 112 | if isinstance(input_tensor, list): 113 | width, height = input_tensor[-1], input_tensor[-2] 114 | return width, height 115 | 116 | def compute_cam_per_layer( 117 | self, 118 | input_tensor: torch.Tensor, 119 | targets: List[torch.nn.Module], 120 | target_size, 121 | eigen_smooth: bool) -> np.ndarray: 122 | activations_list = [a.cpu().data.numpy() 123 | for a in self.activations_and_grads.activations] 124 | grads_list = [g.cpu().data.numpy() 125 | for g in self.activations_and_grads.gradients] 126 | 127 | cam_per_target_layer = [] 128 | # Loop over the saliency image from every layer 129 | for i in range(len(self.target_layers)): 130 | target_layer = self.target_layers[i] 131 | layer_activations = None 132 | layer_grads = None 133 | if i < len(activations_list): 134 | layer_activations = activations_list[i] 135 | if i < len(grads_list): 136 | layer_grads = grads_list[i] 137 | 138 | cam = self.get_cam_image(input_tensor, 139 | target_layer, 140 | targets, 141 | layer_activations, 142 | layer_grads, 143 | eigen_smooth) 144 | cam = np.maximum(cam, 0).astype(np.float32)#float16->32 145 | scaled = scale_cam_image(cam, target_size) 146 | cam_per_target_layer.append(scaled[:, None, :]) 147 | 148 | return cam_per_target_layer 149 | 150 | def aggregate_multi_layers(self, cam_per_target_layer: np.ndarray) -> np.ndarray: 151 | cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1) 152 | cam_per_target_layer = np.maximum(cam_per_target_layer, 0) 153 | result = np.mean(cam_per_target_layer, axis=1) 154 | return scale_cam_image(result) 155 | 156 | def forward_augmentation_smoothing(self, 157 | input_tensor: torch.Tensor, 158 | targets: List[torch.nn.Module], 159 | eigen_smooth: bool = False) -> np.ndarray: 160 | transforms = tta.Compose( 161 | [ 162 | tta.HorizontalFlip(), 163 | tta.Multiply(factors=[0.9, 1, 1.1]), 164 | ] 165 | ) 166 | cams = [] 167 | for transform in transforms: 168 | augmented_tensor = transform.augment_image(input_tensor) 169 | cam = self.forward(augmented_tensor, 170 | targets, 171 | eigen_smooth) 172 | 173 | # The ttach library expects a tensor of size BxCxHxW 174 | cam = cam[:, None, :, :] 175 | cam = torch.from_numpy(cam) 176 | cam = transform.deaugment_mask(cam) 177 | 178 | # Back to numpy float32, HxW 179 | cam = cam.numpy() 180 | cam = cam[:, 0, :, :] 181 | cams.append(cam) 182 | 183 | cam = np.mean(np.float32(cams), axis=0) 184 | return cam 185 | 186 | def __call__(self, 187 | input_tensor: torch.Tensor, 188 | targets: List[torch.nn.Module] = None, 189 | target_size=None, 190 | aug_smooth: bool = False, 191 | eigen_smooth: bool = False) -> np.ndarray: 192 | 193 | # Smooth the CAM result with test time augmentation 194 | if aug_smooth is True: 195 | return self.forward_augmentation_smoothing( 196 | input_tensor, targets, eigen_smooth) 197 | 198 | return self.forward(input_tensor, 199 | targets, target_size,eigen_smooth) 200 | 201 | def __del__(self): 202 | self.activations_and_grads.release() 203 | 204 | def __enter__(self): 205 | return self 206 | 207 | def __exit__(self, exc_type, exc_value, exc_tb): 208 | self.activations_and_grads.release() 209 | if isinstance(exc_value, IndexError): 210 | # Handle IndexError here... 211 | print( 212 | f"An exception occurred in CAM with block: {exc_type}. Message: {exc_value}") 213 | return True 214 | -------------------------------------------------------------------------------- /clip/clip.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import urllib 4 | import warnings 5 | from typing import Any, Union, List 6 | from pkg_resources import packaging 7 | 8 | import torch 9 | from PIL import Image 10 | from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize 11 | from tqdm import tqdm 12 | 13 | from .model import build_model 14 | from .simple_tokenizer import SimpleTokenizer as _Tokenizer 15 | from collections import OrderedDict 16 | 17 | try: 18 | from torchvision.transforms import InterpolationMode 19 | BICUBIC = InterpolationMode.BICUBIC 20 | except ImportError: 21 | BICUBIC = Image.BICUBIC 22 | 23 | 24 | if packaging.version.parse(torch.__version__) < packaging.version.parse("1.7.1"): 25 | warnings.warn("PyTorch version 1.7.1 or higher is recommended") 26 | 27 | 28 | __all__ = ["available_models", "load", "tokenize"] 29 | _tokenizer = _Tokenizer() 30 | 31 | _MODELS = { 32 | "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt", 33 | "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt", 34 | "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt", 35 | "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt", 36 | "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt", 37 | "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt", 38 | "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt", 39 | "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt", 40 | "ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt", 41 | } 42 | 43 | 44 | def _download(url: str, root: str): 45 | os.makedirs(root, exist_ok=True) 46 | filename = os.path.basename(url) 47 | 48 | expected_sha256 = url.split("/")[-2] 49 | download_target = os.path.join(root, filename) 50 | 51 | if os.path.exists(download_target) and not os.path.isfile(download_target): 52 | raise RuntimeError(f"{download_target} exists and is not a regular file") 53 | 54 | if os.path.isfile(download_target): 55 | if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256: 56 | return download_target 57 | else: 58 | warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file") 59 | 60 | with urllib.request.urlopen(url) as source, open(download_target, "wb") as output: 61 | with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop: 62 | while True: 63 | buffer = source.read(8192) 64 | if not buffer: 65 | break 66 | 67 | output.write(buffer) 68 | loop.update(len(buffer)) 69 | 70 | if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256: 71 | raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match") 72 | 73 | return download_target 74 | 75 | 76 | def _convert_image_to_rgb(image): 77 | return image.convert("RGB") 78 | 79 | 80 | def _transform(n_px): 81 | return Compose([ 82 | Resize(n_px, interpolation=BICUBIC), 83 | CenterCrop(n_px), 84 | _convert_image_to_rgb, 85 | ToTensor(), 86 | Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), 87 | ]) 88 | 89 | 90 | def available_models() -> List[str]: 91 | """Returns the names of available CLIP models""" 92 | return list(_MODELS.keys()) 93 | 94 | 95 | def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None): 96 | """Load a CLIP model 97 | 98 | Parameters 99 | ---------- 100 | name : str 101 | A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict 102 | 103 | device : Union[str, torch.device] 104 | The device to put the loaded model 105 | 106 | jit : bool 107 | Whether to load the optimized JIT model or more hackable non-JIT model (default). 108 | 109 | download_root: str 110 | path to download the model files; by default, it uses "~/.cache/clip" 111 | 112 | Returns 113 | ------- 114 | model : torch.nn.Module 115 | The CLIP model 116 | 117 | preprocess : Callable[[PIL.Image], torch.Tensor] 118 | A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input 119 | """ 120 | if name in _MODELS: 121 | model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip")) 122 | elif os.path.isfile(name): 123 | model_path = name 124 | else: 125 | raise RuntimeError(f"Model {name} not found; available models = {available_models()}") 126 | 127 | with open(model_path, 'rb') as opened_file: 128 | try: 129 | # loading JIT archive 130 | model = torch.jit.load(opened_file, map_location=device if jit else "cpu").eval() 131 | state_dict = None 132 | except RuntimeError: 133 | # loading saved state dict 134 | if jit: 135 | warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead") 136 | jit = False 137 | if 'RN50' in model_path: 138 | state_dict = torch.load(opened_file, map_location="cpu") 139 | else: 140 | state_dict0 = torch.load(model_path, map_location="cpu") 141 | state_dict = OrderedDict() 142 | for k in state_dict0.keys(): 143 | state_dict[k.replace('module.', '')] = state_dict0[k] 144 | 145 | 146 | if not jit: 147 | model = build_model(state_dict or model.state_dict()).to(device) 148 | if str(device) == "cpu": 149 | model.float() 150 | return model, _transform(model.visual.input_resolution) 151 | 152 | # patch the device names 153 | device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[]) 154 | device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1] 155 | 156 | def patch_device(module): 157 | try: 158 | graphs = [module.graph] if hasattr(module, "graph") else [] 159 | except RuntimeError: 160 | graphs = [] 161 | 162 | if hasattr(module, "forward1"): 163 | graphs.append(module.forward1.graph) 164 | 165 | for graph in graphs: 166 | for node in graph.findAllNodes("prim::Constant"): 167 | if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"): 168 | node.copyAttributes(device_node) 169 | 170 | model.apply(patch_device) 171 | patch_device(model.encode_image) 172 | patch_device(model.encode_text) 173 | 174 | # patch dtype to float32 on CPU 175 | if str(device) == "cpu": 176 | float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[]) 177 | float_input = list(float_holder.graph.findNode("aten::to").inputs())[1] 178 | float_node = float_input.node() 179 | 180 | def patch_float(module): 181 | try: 182 | graphs = [module.graph] if hasattr(module, "graph") else [] 183 | except RuntimeError: 184 | graphs = [] 185 | 186 | if hasattr(module, "forward1"): 187 | graphs.append(module.forward1.graph) 188 | 189 | for graph in graphs: 190 | for node in graph.findAllNodes("aten::to"): 191 | inputs = list(node.inputs()) 192 | for i in [1, 2]: # dtype can be the second or third argument to aten::to() 193 | if inputs[i].node()["value"] == 5: 194 | inputs[i].node().copyAttributes(float_node) 195 | 196 | model.apply(patch_float) 197 | patch_float(model.encode_image) 198 | patch_float(model.encode_text) 199 | 200 | model.float() 201 | 202 | return model, _transform(model.input_resolution.item()) 203 | 204 | 205 | def tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]: 206 | """ 207 | Returns the tokenized representation of given input string(s) 208 | 209 | Parameters 210 | ---------- 211 | texts : Union[str, List[str]] 212 | An input string or a list of input strings to tokenize 213 | 214 | context_length : int 215 | The context length to use; all CLIP models use 77 as the context length 216 | 217 | truncate: bool 218 | Whether to truncate the text in case its encoding is longer than the context length 219 | 220 | Returns 221 | ------- 222 | A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]. 223 | We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long. 224 | """ 225 | if isinstance(texts, str): 226 | texts = [texts] 227 | 228 | sot_token = _tokenizer.encoder["<|startoftext|>"] 229 | eot_token = _tokenizer.encoder["<|endoftext|>"] 230 | all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts] 231 | if packaging.version.parse(torch.__version__) < packaging.version.parse("1.8.0"): 232 | result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) 233 | else: 234 | result = torch.zeros(len(all_tokens), context_length, dtype=torch.int) 235 | 236 | for i, tokens in enumerate(all_tokens): 237 | if len(tokens) > context_length: 238 | if truncate: 239 | tokens = tokens[:context_length] 240 | tokens[-1] = eot_token 241 | else: 242 | raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}") 243 | result[i, :len(tokens)] = torch.tensor(tokens) 244 | 245 | return result 246 | -------------------------------------------------------------------------------- /generate_cams_coco14.py: -------------------------------------------------------------------------------- 1 | # -*- coding:UTF-8 -*- 2 | from pytorch_grad_cam import GradCAM 3 | import torch 4 | import clip 5 | from PIL import Image 6 | import numpy as np 7 | import cv2 8 | import os 9 | 10 | from tqdm import tqdm 11 | from pytorch_grad_cam.utils.image import scale_cam_image 12 | from utils import scoremap2bbox 13 | from clip_text import class_names, new_class_names_coco, BACKGROUND_CATEGORY_COCO 14 | import argparse 15 | from torch import multiprocessing 16 | from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize, RandomHorizontalFlip 17 | try: 18 | from torchvision.transforms import InterpolationMode 19 | BICUBIC = InterpolationMode.BICUBIC 20 | except ImportError: 21 | BICUBIC = Image.BICUBIC 22 | import warnings 23 | warnings.filterwarnings("ignore") 24 | _CONTOUR_INDEX = 1 if cv2.__version__.split('.')[0] == '3' else 0 25 | 26 | def reshape_transform(tensor, height=28, width=28): 27 | tensor = tensor.permute(1, 0, 2) 28 | result = tensor[:, 1:, :].reshape(tensor.size(0), height, width, tensor.size(2)) 29 | 30 | # Bring the channels to the first dimension, 31 | # like in CNNs. 32 | result = result.transpose(2, 3).transpose(1, 2) 33 | return result 34 | 35 | def split_dataset(dataset, all_label_list, n_splits): 36 | if n_splits == 1: 37 | return [dataset], [all_label_list] 38 | part = len(dataset) // n_splits 39 | dataset_list = [] 40 | split_label_list = [] 41 | for i in range(n_splits - 1): 42 | dataset_list.append(dataset[i*part:(i+1)*part]) 43 | split_label_list.append(all_label_list[i*part:(i+1)*part]) 44 | dataset_list.append(dataset[(i+1)*part:]) 45 | split_label_list.append(all_label_list[(i+1)*part:]) 46 | 47 | return dataset_list, split_label_list 48 | 49 | def zeroshot_classifier(classnames, templates, model): 50 | with torch.no_grad(): 51 | zeroshot_weights = [] 52 | for classname in classnames: 53 | texts = [template.format(classname) for template in templates] #format with class 54 | texts = clip.tokenize(texts).to(device) #tokenize 55 | class_embeddings = model.encode_text(texts) #embed with text encoder 56 | class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True) 57 | class_embedding = class_embeddings.mean(dim=0) 58 | class_embedding /= class_embedding.norm() 59 | zeroshot_weights.append(class_embedding) 60 | zeroshot_weights = torch.stack(zeroshot_weights, dim=1).to(device) 61 | return zeroshot_weights.t() 62 | 63 | 64 | class ClipOutputTarget: 65 | def __init__(self, category): 66 | self.category = category 67 | def __call__(self, model_output): 68 | if len(model_output.shape) == 1: 69 | return model_output[self.category] 70 | return model_output[:, self.category] 71 | 72 | 73 | def _convert_image_to_rgb(image): 74 | return image.convert("RGB") 75 | 76 | def _transform_resize(h, w): 77 | return Compose([ 78 | Resize((h,w), interpolation=BICUBIC), 79 | _convert_image_to_rgb, 80 | ToTensor(), 81 | Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), 82 | ]) 83 | 84 | 85 | def img_ms_and_flip(img_path, ori_height, ori_width, scales=[1.0], patch_size=16): 86 | all_imgs = [] 87 | for scale in scales: 88 | preprocess = _transform_resize(int(np.ceil(scale * int(ori_height) / patch_size) * patch_size), int(np.ceil(scale * int(ori_width) / patch_size) * patch_size)) 89 | image = preprocess(Image.open(img_path)) 90 | image_ori = image 91 | image_flip = torch.flip(image, [-1]) 92 | all_imgs.append(image_ori) 93 | all_imgs.append(image_flip) 94 | return all_imgs 95 | 96 | 97 | def perform(process_id, dataset_list, args, model, bg_text_features, fg_text_features, cam, split_label_list): 98 | n_gpus = torch.cuda.device_count() 99 | device_id = "cuda:{}".format(process_id % n_gpus) 100 | databin = dataset_list[process_id] 101 | all_label_list = split_label_list[process_id] 102 | model = model.to(device_id) 103 | bg_text_features = bg_text_features.to(device_id) 104 | fg_text_features = fg_text_features.to(device_id) 105 | for im_idx, im in enumerate(tqdm(databin)): 106 | img_path = os.path.join(args.img_root, im) 107 | ori_image = Image.open(img_path) 108 | ori_height, ori_width = np.asarray(ori_image).shape[:2] 109 | label_id_list = all_label_list[im_idx] 110 | label_list = [] 111 | for lid in label_id_list: 112 | label_list.append(new_class_names_coco[int(lid)]) 113 | label_id_list = [int(lid) for lid in label_id_list] 114 | 115 | ms_imgs = img_ms_and_flip(img_path, ori_height, ori_width, scales=[1.0]) 116 | ms_imgs = [ms_imgs[0]] 117 | cam_all_scales = [] 118 | highres_cam_all_scales = [] 119 | refined_cam_all_scales = [] 120 | for image in ms_imgs: 121 | image = image.unsqueeze(0) 122 | h, w = image.shape[-2], image.shape[-1] 123 | image = image.to(device_id) 124 | image_features, attn_weight_list = model.encode_image(image, h, w) 125 | 126 | cam_to_save = [] 127 | highres_cam_to_save = [] 128 | refined_cam_to_save = [] 129 | keys = [] 130 | 131 | bg_features_temp = bg_text_features.to(device_id) # [bg_id_for_each_image[im_idx]].to(device_id) 132 | fg_features_temp = fg_text_features[label_id_list].to(device_id) 133 | text_features_temp = torch.cat([fg_features_temp, bg_features_temp], dim=0) 134 | input_tensor = [image_features, text_features_temp.to(device_id), h, w] 135 | 136 | for idx, label in enumerate(label_list): 137 | keys.append(new_class_names_coco.index(label)) 138 | targets = [ClipOutputTarget(label_list.index(label))] 139 | 140 | #torch.cuda.empty_cache() 141 | grayscale_cam, logits_per_image, attn_weight_last = cam(input_tensor=input_tensor, 142 | targets=targets, 143 | target_size=None) # (ori_width, ori_height)) 144 | 145 | grayscale_cam = grayscale_cam[0, :] 146 | 147 | grayscale_cam_highres = cv2.resize(grayscale_cam, (ori_width, ori_height)) 148 | highres_cam_to_save.append(torch.tensor(grayscale_cam_highres)) 149 | 150 | if idx == 0: 151 | attn_weight_list.append(attn_weight_last) 152 | attn_weight = [aw[:, 1:, 1:] for aw in attn_weight_list] # (b, hxw, hxw) 153 | attn_weight = torch.stack(attn_weight, dim=0)[-8:] 154 | attn_weight = torch.mean(attn_weight, dim=0) 155 | attn_weight = attn_weight[0].cpu().detach() 156 | attn_weight = attn_weight.float() 157 | 158 | box, cnt = scoremap2bbox(scoremap=grayscale_cam, threshold=0.7, multi_contour_eval=True) 159 | aff_mask = torch.zeros((grayscale_cam.shape[0],grayscale_cam.shape[1])) 160 | for i_ in range(cnt): 161 | x0_, y0_, x1_, y1_ = box[i_] 162 | aff_mask[y0_:y1_, x0_:x1_] = 1 163 | 164 | aff_mask = aff_mask.view(1,grayscale_cam.shape[0] * grayscale_cam.shape[1]) 165 | aff_mat = attn_weight 166 | 167 | trans_mat = aff_mat / torch.sum(aff_mat, dim=0, keepdim=True) 168 | trans_mat = trans_mat / torch.sum(trans_mat, dim=1, keepdim=True) 169 | 170 | for _ in range(2): 171 | trans_mat = trans_mat / torch.sum(trans_mat, dim=0, keepdim=True) 172 | trans_mat = trans_mat / torch.sum(trans_mat, dim=1, keepdim=True) 173 | trans_mat = (trans_mat + trans_mat.transpose(1, 0)) / 2 174 | 175 | for _ in range(1): 176 | trans_mat = torch.matmul(trans_mat, trans_mat) 177 | 178 | trans_mat = trans_mat * aff_mask 179 | 180 | cam_to_refine = torch.FloatTensor(grayscale_cam) 181 | cam_to_refine = cam_to_refine.view(-1,1) 182 | 183 | # (n,n) * (n,1)->(n,1) 184 | cam_refined = torch.matmul(trans_mat, cam_to_refine).reshape(h //16, w // 16) 185 | cam_refined = cam_refined.cpu().numpy().astype(np.float32) 186 | cam_refined_highres = scale_cam_image([cam_refined], (ori_width, ori_height))[0] 187 | refined_cam_to_save.append(torch.tensor(cam_refined_highres)) 188 | 189 | keys = torch.tensor(keys) 190 | #cam_all_scales.append(torch.stack(cam_to_save,dim=0)) 191 | highres_cam_all_scales.append(torch.stack(highres_cam_to_save,dim=0)) 192 | refined_cam_all_scales.append(torch.stack(refined_cam_to_save,dim=0)) 193 | 194 | 195 | #cam_all_scales = cam_all_scales[0] 196 | highres_cam_all_scales = highres_cam_all_scales[0] 197 | refined_cam_all_scales = refined_cam_all_scales[0] 198 | np.save(os.path.join(args.cam_out_dir, im.replace('jpg', 'npy')), 199 | {"keys": keys.numpy(), 200 | # "strided_cam": cam_per_scales.cpu().numpy(), 201 | #"highres": highres_cam_all_scales.cpu().numpy().astype(np.float16), 202 | "attn_highres": refined_cam_all_scales.cpu().numpy().astype(np.float16), 203 | }) 204 | return 0 205 | 206 | if __name__ == "__main__": 207 | parser = argparse.ArgumentParser(description='') 208 | parser.add_argument('--img_root', type=str, default='/home/xxx/datasets/COCO2014/JPEGImages/train2014') 209 | parser.add_argument('--split_file', type=str, default='./coco14/train.txt') 210 | parser.add_argument('--cam_out_dir', type=str, default='./final/ablation/coco_test') 211 | parser.add_argument('--model', type=str, default='/home/xxx/pretrained_models/clip/ViT-B-16.pt') 212 | parser.add_argument('--num_workers', type=int, default=1) 213 | args = parser.parse_args() 214 | 215 | device = "cuda" if torch.cuda.is_available() else "cpu" 216 | print(device) 217 | 218 | file_list = tuple(open(args.split_file, "r")) 219 | file_list = [id_.rstrip().split(" ") for id_ in file_list] 220 | train_list = [x[0] + '.jpg' for x in file_list] # [2000:] 221 | all_label_list = [x[1:] for x in file_list] # [2000:] 222 | print(len(train_list)) 223 | 224 | if not os.path.exists(args.cam_out_dir): 225 | os.makedirs(args.cam_out_dir) 226 | 227 | model, _ = clip.load(args.model, device=device) 228 | bg_text_features = zeroshot_classifier(BACKGROUND_CATEGORY_COCO, ['a clean origami {}.'], model)#['a rendering of a weird {}.'], model) 229 | fg_text_features = zeroshot_classifier(new_class_names_coco, ['a clean origami {}.'], model)#['a rendering of a weird {}.'], model) 230 | 231 | target_layers = [model.visual.transformer.resblocks[-1].ln_1] 232 | cam = GradCAM(model=model, target_layers=target_layers, reshape_transform=reshape_transform) 233 | 234 | dataset_list, split_label_list = split_dataset(train_list, all_label_list, n_splits=args.num_workers) 235 | if args.num_workers == 1: 236 | perform(0, dataset_list, args, model, bg_text_features, fg_text_features, cam, split_label_list) 237 | else: 238 | multiprocessing.spawn(perform, nprocs=args.num_workers, 239 | args=(dataset_list, args, model, bg_text_features, fg_text_features, cam, split_label_list)) 240 | -------------------------------------------------------------------------------- /generate_cams_voc12.py: -------------------------------------------------------------------------------- 1 | # -*- coding:UTF-8 -*- 2 | from pytorch_grad_cam import GradCAM 3 | import torch 4 | import clip 5 | from PIL import Image 6 | import numpy as np 7 | import cv2 8 | import os 9 | 10 | from tqdm import tqdm 11 | from pytorch_grad_cam.utils.image import scale_cam_image 12 | from utils import parse_xml_to_dict, scoremap2bbox 13 | from clip_text import class_names, new_class_names, BACKGROUND_CATEGORY#, imagenet_templates 14 | import argparse 15 | from lxml import etree 16 | import time 17 | from torch import multiprocessing 18 | from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize, RandomHorizontalFlip 19 | try: 20 | from torchvision.transforms import InterpolationMode 21 | BICUBIC = InterpolationMode.BICUBIC 22 | except ImportError: 23 | BICUBIC = Image.BICUBIC 24 | import warnings 25 | warnings.filterwarnings("ignore") 26 | _CONTOUR_INDEX = 1 if cv2.__version__.split('.')[0] == '3' else 0 27 | 28 | def reshape_transform(tensor, height=28, width=28): 29 | tensor = tensor.permute(1, 0, 2) 30 | result = tensor[:, 1:, :].reshape(tensor.size(0), height, width, tensor.size(2)) 31 | 32 | # Bring the channels to the first dimension, 33 | # like in CNNs. 34 | result = result.transpose(2, 3).transpose(1, 2) 35 | return result 36 | 37 | def split_dataset(dataset, n_splits): 38 | if n_splits == 1: 39 | return [dataset] 40 | part = len(dataset) // n_splits 41 | dataset_list = [] 42 | for i in range(n_splits - 1): 43 | dataset_list.append(dataset[i*part:(i+1)*part]) 44 | dataset_list.append(dataset[(i+1)*part:]) 45 | 46 | return dataset_list 47 | 48 | def zeroshot_classifier(classnames, templates, model): 49 | with torch.no_grad(): 50 | zeroshot_weights = [] 51 | for classname in classnames: 52 | texts = [template.format(classname) for template in templates] #format with class 53 | texts = clip.tokenize(texts).to(device) #tokenize 54 | class_embeddings = model.encode_text(texts) #embed with text encoder 55 | class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True) 56 | class_embedding = class_embeddings.mean(dim=0) 57 | class_embedding /= class_embedding.norm() 58 | zeroshot_weights.append(class_embedding) 59 | zeroshot_weights = torch.stack(zeroshot_weights, dim=1).to(device) 60 | return zeroshot_weights.t() 61 | 62 | class ClipOutputTarget: 63 | def __init__(self, category): 64 | self.category = category 65 | def __call__(self, model_output): 66 | if len(model_output.shape) == 1: 67 | return model_output[self.category] 68 | return model_output[:, self.category] 69 | 70 | 71 | def _convert_image_to_rgb(image): 72 | return image.convert("RGB") 73 | 74 | def _transform_resize(h, w): 75 | return Compose([ 76 | Resize((h,w), interpolation=BICUBIC), 77 | _convert_image_to_rgb, 78 | ToTensor(), 79 | Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), 80 | ]) 81 | 82 | def img_ms_and_flip(img_path, ori_height, ori_width, scales=[1.0], patch_size=16): 83 | all_imgs = [] 84 | for scale in scales: 85 | preprocess = _transform_resize(int(np.ceil(scale * int(ori_height) / patch_size) * patch_size), int(np.ceil(scale * int(ori_width) / patch_size) * patch_size)) 86 | image = preprocess(Image.open(img_path)) 87 | image_ori = image 88 | image_flip = torch.flip(image, [-1]) 89 | all_imgs.append(image_ori) 90 | all_imgs.append(image_flip) 91 | return all_imgs 92 | 93 | 94 | def perform(process_id, dataset_list, args, model, bg_text_features, fg_text_features, cam): 95 | n_gpus = torch.cuda.device_count() 96 | device_id = "cuda:{}".format(process_id % n_gpus) 97 | databin = dataset_list[process_id] 98 | model = model.to(device_id) 99 | bg_text_features = bg_text_features.to(device_id) 100 | fg_text_features = fg_text_features.to(device_id) 101 | for im_idx, im in enumerate(tqdm(databin)): 102 | img_path = os.path.join(args.img_root, im) 103 | xmlfile = img_path.replace('/JPEGImages', '/Annotations') 104 | xmlfile = xmlfile.replace('.jpg', '.xml') 105 | with open(xmlfile) as fid: 106 | xml_str = fid.read() 107 | xml = etree.fromstring(xml_str) # etree包 读取xml文件 108 | data = parse_xml_to_dict(xml)["annotation"] 109 | 110 | ori_width = int(data['size']['width']) 111 | ori_height = int(data['size']['height']) 112 | 113 | label_list = [] 114 | label_id_list = [] 115 | for obj in data["object"]: 116 | obj["name"] = new_class_names[class_names.index(obj["name"])] 117 | if obj["name"] not in label_list: 118 | label_list.append(obj["name"]) 119 | label_id_list.append(new_class_names.index(obj["name"])) 120 | 121 | if len(label_list) == 0: 122 | print("{} not have valid object".format(im)) 123 | return 124 | 125 | ms_imgs = img_ms_and_flip(img_path, ori_height, ori_width, scales=[1.0]) 126 | ms_imgs = [ms_imgs[0]] 127 | cam_all_scales = [] 128 | highres_cam_all_scales = [] 129 | refined_cam_all_scales = [] 130 | for image in ms_imgs: 131 | image = image.unsqueeze(0) 132 | h, w = image.shape[-2], image.shape[-1] 133 | image = image.to(device_id) 134 | image_features, attn_weight_list = model.encode_image(image, h, w) 135 | 136 | cam_to_save = [] 137 | highres_cam_to_save = [] 138 | refined_cam_to_save = [] 139 | keys = [] 140 | 141 | bg_features_temp = bg_text_features.to(device_id) # [bg_id_for_each_image[im_idx]].to(device_id) 142 | fg_features_temp = fg_text_features[label_id_list].to(device_id) 143 | text_features_temp = torch.cat([fg_features_temp, bg_features_temp], dim=0) 144 | input_tensor = [image_features, text_features_temp.to(device_id), h, w] 145 | 146 | for idx, label in enumerate(label_list): 147 | keys.append(new_class_names.index(label)) 148 | targets = [ClipOutputTarget(label_list.index(label))] 149 | 150 | #torch.cuda.empty_cache() 151 | grayscale_cam, logits_per_image, attn_weight_last = cam(input_tensor=input_tensor, 152 | targets=targets, 153 | target_size=None) # (ori_width, ori_height)) 154 | 155 | grayscale_cam = grayscale_cam[0, :] 156 | 157 | grayscale_cam_highres = cv2.resize(grayscale_cam, (ori_width, ori_height)) 158 | highres_cam_to_save.append(torch.tensor(grayscale_cam_highres)) 159 | 160 | if idx == 0: 161 | attn_weight_list.append(attn_weight_last) 162 | attn_weight = [aw[:, 1:, 1:] for aw in attn_weight_list] # (b, hxw, hxw) 163 | attn_weight = torch.stack(attn_weight, dim=0)[-8:] 164 | attn_weight = torch.mean(attn_weight, dim=0) 165 | attn_weight = attn_weight[0].cpu().detach() 166 | attn_weight = attn_weight.float() 167 | 168 | box, cnt = scoremap2bbox(scoremap=grayscale_cam, threshold=0.4, multi_contour_eval=True) 169 | aff_mask = torch.zeros((grayscale_cam.shape[0],grayscale_cam.shape[1])) 170 | for i_ in range(cnt): 171 | x0_, y0_, x1_, y1_ = box[i_] 172 | aff_mask[y0_:y1_, x0_:x1_] = 1 173 | 174 | aff_mask = aff_mask.view(1,grayscale_cam.shape[0] * grayscale_cam.shape[1]) 175 | aff_mat = attn_weight 176 | 177 | trans_mat = aff_mat / torch.sum(aff_mat, dim=0, keepdim=True) 178 | trans_mat = trans_mat / torch.sum(trans_mat, dim=1, keepdim=True) 179 | 180 | for _ in range(2): 181 | trans_mat = trans_mat / torch.sum(trans_mat, dim=0, keepdim=True) 182 | trans_mat = trans_mat / torch.sum(trans_mat, dim=1, keepdim=True) 183 | trans_mat = (trans_mat + trans_mat.transpose(1, 0)) / 2 184 | 185 | for _ in range(1): 186 | trans_mat = torch.matmul(trans_mat, trans_mat) 187 | 188 | trans_mat = trans_mat * aff_mask 189 | 190 | cam_to_refine = torch.FloatTensor(grayscale_cam) 191 | cam_to_refine = cam_to_refine.view(-1,1) 192 | 193 | # (n,n) * (n,1)->(n,1) 194 | cam_refined = torch.matmul(trans_mat, cam_to_refine).reshape(h //16, w // 16) 195 | cam_refined = cam_refined.cpu().numpy().astype(np.float32) 196 | cam_refined_highres = scale_cam_image([cam_refined], (ori_width, ori_height))[0] 197 | refined_cam_to_save.append(torch.tensor(cam_refined_highres)) 198 | 199 | keys = torch.tensor(keys) 200 | #cam_all_scales.append(torch.stack(cam_to_save,dim=0)) 201 | highres_cam_all_scales.append(torch.stack(highres_cam_to_save,dim=0)) 202 | refined_cam_all_scales.append(torch.stack(refined_cam_to_save,dim=0)) 203 | 204 | 205 | #cam_all_scales = cam_all_scales[0] 206 | highres_cam_all_scales = highres_cam_all_scales[0] 207 | refined_cam_all_scales = refined_cam_all_scales[0] 208 | 209 | np.save(os.path.join(args.cam_out_dir, im.replace('jpg', 'npy')), 210 | {"keys": keys.numpy(), 211 | # "strided_cam": cam_per_scales.cpu().numpy(), 212 | #"highres": highres_cam_all_scales.cpu().numpy().astype(np.float16), 213 | "attn_highres": refined_cam_all_scales.cpu().numpy().astype(np.float16), 214 | }) 215 | return 0 216 | 217 | if __name__ == "__main__": 218 | parser = argparse.ArgumentParser(description='') 219 | parser.add_argument('--img_root', type=str, default='/home/xxx/datasets/VOC2012/JPEGImages') 220 | parser.add_argument('--split_file', type=str, default='./voc12/train.txt') 221 | parser.add_argument('--cam_out_dir', type=str, default='./final/ablation/baseline') 222 | parser.add_argument('--model', type=str, default='/home/xxx/pretrained_models/clip/ViT-B-16.pt') 223 | parser.add_argument('--num_workers', type=int, default=1) 224 | args = parser.parse_args() 225 | 226 | device = "cuda" if torch.cuda.is_available() else "cpu" 227 | print(device) 228 | 229 | train_list = np.loadtxt(args.split_file, dtype=str) 230 | train_list = [x + '.jpg' for x in train_list] 231 | 232 | if not os.path.exists(args.cam_out_dir): 233 | os.makedirs(args.cam_out_dir) 234 | 235 | model, _ = clip.load(args.model, device=device) 236 | bg_text_features = zeroshot_classifier(BACKGROUND_CATEGORY, ['a clean origami {}.'], model)#['a rendering of a weird {}.'], model) 237 | fg_text_features = zeroshot_classifier(new_class_names, ['a clean origami {}.'], model)#['a rendering of a weird {}.'], model) 238 | 239 | target_layers = [model.visual.transformer.resblocks[-1].ln_1] 240 | cam = GradCAM(model=model, target_layers=target_layers, reshape_transform=reshape_transform) 241 | 242 | dataset_list = split_dataset(train_list, n_splits=args.num_workers) 243 | if args.num_workers == 1: 244 | perform(0, dataset_list, args, model, bg_text_features, fg_text_features, cam) 245 | else: 246 | multiprocessing.spawn(perform, nprocs=args.num_workers, 247 | args=(dataset_list, args, model, bg_text_features, fg_text_features, cam)) 248 | 249 | -------------------------------------------------------------------------------- /clip/model.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from typing import Tuple, Union 3 | 4 | import numpy as np 5 | import torch 6 | import torch.nn.functional as F 7 | from torch import nn 8 | 9 | def upsample_pos_emb(emb, new_size): 10 | # upsample the pretrained embedding for higher resolution 11 | # emb size NxD 12 | first = emb[:1, :] 13 | emb = emb[1:, :] 14 | N, D = emb.size(0), emb.size(1) 15 | size = int(np.sqrt(N)) 16 | assert size * size == N 17 | #new_size = size * self.upsample 18 | emb = emb.permute(1, 0) 19 | emb = emb.view(1, D, size, size).contiguous() 20 | emb = F.upsample(emb, size=new_size, mode='bilinear',) 21 | emb = emb.view(D, -1).contiguous() 22 | emb = emb.permute(1, 0) 23 | emb = torch.cat([first, emb], 0) 24 | emb = nn.parameter.Parameter(emb.half()) 25 | return emb 26 | 27 | class Bottleneck(nn.Module): 28 | expansion = 4 29 | 30 | def __init__(self, inplanes, planes, stride=1): 31 | super().__init__() 32 | 33 | # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 34 | self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) 35 | self.bn1 = nn.BatchNorm2d(planes) 36 | self.relu1 = nn.ReLU(inplace=True) 37 | 38 | self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) 39 | self.bn2 = nn.BatchNorm2d(planes) 40 | self.relu2 = nn.ReLU(inplace=True) 41 | 42 | self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() 43 | 44 | self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) 45 | self.bn3 = nn.BatchNorm2d(planes * self.expansion) 46 | self.relu3 = nn.ReLU(inplace=True) 47 | 48 | self.downsample = None 49 | self.stride = stride 50 | 51 | if stride > 1 or inplanes != planes * Bottleneck.expansion: 52 | # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 53 | self.downsample = nn.Sequential(OrderedDict([ 54 | ("-1", nn.AvgPool2d(stride)), 55 | ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)), 56 | ("1", nn.BatchNorm2d(planes * self.expansion)) 57 | ])) 58 | 59 | def forward(self, x: torch.Tensor): 60 | identity = x 61 | 62 | out = self.relu1(self.bn1(self.conv1(x))) 63 | out = self.relu2(self.bn2(self.conv2(out))) 64 | out = self.avgpool(out) 65 | out = self.bn3(self.conv3(out)) 66 | 67 | if self.downsample is not None: 68 | identity = self.downsample(x) 69 | 70 | out += identity 71 | out = self.relu3(out) 72 | return out 73 | 74 | 75 | class AttentionPool2d(nn.Module): 76 | def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): 77 | super().__init__() 78 | self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5) 79 | self.k_proj = nn.Linear(embed_dim, embed_dim) 80 | self.q_proj = nn.Linear(embed_dim, embed_dim) 81 | self.v_proj = nn.Linear(embed_dim, embed_dim) 82 | self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) 83 | self.num_heads = num_heads 84 | 85 | def forward(self, x, H, W): 86 | x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1) # NCHW -> (HW)NC 87 | x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC 88 | self.positional_embedding_new = upsample_pos_emb(self.positional_embedding, (H//32,W//32)) 89 | x = x + self.positional_embedding_new[:, None, :].to(x.dtype) # (HW+1)NC 90 | x, attn_weight = F.multi_head_attention_forward( 91 | query=x, key=x, value=x, 92 | embed_dim_to_check=x.shape[-1], 93 | num_heads=self.num_heads, 94 | q_proj_weight=self.q_proj.weight, 95 | k_proj_weight=self.k_proj.weight, 96 | v_proj_weight=self.v_proj.weight, 97 | in_proj_weight=None, 98 | in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), 99 | bias_k=None, 100 | bias_v=None, 101 | add_zero_attn=False, 102 | dropout_p=0, 103 | out_proj_weight=self.c_proj.weight, 104 | out_proj_bias=self.c_proj.bias, 105 | use_separate_proj_weight=True, 106 | training=self.training, 107 | need_weights=False 108 | ) 109 | return x[0] 110 | 111 | 112 | class ModifiedResNet(nn.Module): 113 | """ 114 | A ResNet class that is similar to torchvision's but contains the following changes: 115 | - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. 116 | - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 117 | - The final pooling layer is a QKV attention instead of an average pool 118 | """ 119 | 120 | def __init__(self, layers, output_dim, heads, input_resolution=224, width=64): 121 | super().__init__() 122 | self.output_dim = output_dim 123 | self.input_resolution = input_resolution 124 | 125 | # the 3-layer stem 126 | self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) 127 | self.bn1 = nn.BatchNorm2d(width // 2) 128 | self.relu1 = nn.ReLU(inplace=True) 129 | self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False) 130 | self.bn2 = nn.BatchNorm2d(width // 2) 131 | self.relu2 = nn.ReLU(inplace=True) 132 | self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False) 133 | self.bn3 = nn.BatchNorm2d(width) 134 | self.relu3 = nn.ReLU(inplace=True) 135 | self.avgpool = nn.AvgPool2d(2) 136 | 137 | # residual layers 138 | self._inplanes = width # this is a *mutable* variable used during construction 139 | self.layer1 = self._make_layer(width, layers[0]) 140 | self.layer2 = self._make_layer(width * 2, layers[1], stride=2) 141 | self.layer3 = self._make_layer(width * 4, layers[2], stride=2) 142 | self.layer4 = self._make_layer(width * 8, layers[3], stride=2) 143 | 144 | embed_dim = width * 32 # the ResNet feature dimension 145 | self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim) 146 | 147 | def _make_layer(self, planes, blocks, stride=1): 148 | layers = [Bottleneck(self._inplanes, planes, stride)] 149 | 150 | self._inplanes = planes * Bottleneck.expansion 151 | for _ in range(1, blocks): 152 | layers.append(Bottleneck(self._inplanes, planes)) 153 | 154 | return nn.Sequential(*layers) 155 | 156 | def forward(self, x, H, W): 157 | def stem(x): 158 | x = self.relu1(self.bn1(self.conv1(x))) 159 | x = self.relu2(self.bn2(self.conv2(x))) 160 | x = self.relu3(self.bn3(self.conv3(x))) 161 | x = self.avgpool(x) 162 | return x 163 | 164 | x = x.type(self.conv1.weight.dtype) 165 | x = stem(x) 166 | x = self.layer1(x) 167 | x = self.layer2(x) 168 | x = self.layer3(x) 169 | x = self.layer4(x)#(1,,2048, 7, 7) 170 | x_pooled = self.attnpool(x, H, W) 171 | 172 | return x_pooled 173 | 174 | 175 | class LayerNorm(nn.LayerNorm): 176 | """Subclass torch's LayerNorm to handle fp16.""" 177 | 178 | def forward(self, x: torch.Tensor): 179 | orig_type = x.dtype 180 | ret = super().forward(x.type(torch.float32)) 181 | return ret.type(orig_type) 182 | 183 | 184 | class QuickGELU(nn.Module): 185 | def forward(self, x: torch.Tensor): 186 | return x * torch.sigmoid(1.702 * x) 187 | 188 | 189 | class ResidualAttentionBlock(nn.Module): 190 | def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): 191 | super().__init__() 192 | 193 | self.attn = nn.MultiheadAttention(d_model, n_head) 194 | self.ln_1 = LayerNorm(d_model) 195 | self.mlp = nn.Sequential(OrderedDict([ 196 | ("c_fc", nn.Linear(d_model, d_model * 4)), 197 | ("gelu", QuickGELU()), 198 | ("c_proj", nn.Linear(d_model * 4, d_model)) 199 | ])) 200 | self.ln_2 = LayerNorm(d_model) 201 | self.attn_mask = attn_mask 202 | 203 | def attention(self, x: torch.Tensor): 204 | self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None 205 | return self.attn(x, x, x, need_weights=True, attn_mask=self.attn_mask)#[0] 206 | 207 | def forward(self, x: torch.Tensor): 208 | attn_output, attn_weight = self.attention(self.ln_1(x))#(L,N,E) (N,L,L) 209 | x = x + attn_output 210 | x = x + self.mlp(self.ln_2(x)) 211 | return x, attn_weight 212 | 213 | 214 | 215 | class Transformer(nn.Module): 216 | def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None): 217 | super().__init__() 218 | self.width = width 219 | self.layers = layers 220 | self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]) 221 | 222 | def forward(self, x: torch.Tensor): 223 | attn_weights = [] 224 | with torch.no_grad(): 225 | layers = self.layers if x.shape[0] == 77 else self.layers-1 226 | for i in range(layers): 227 | x, attn_weight = self.resblocks[i](x) 228 | attn_weights.append(attn_weight) 229 | ''' 230 | for i in range(self.layers-1, self.layers): 231 | x, attn_weight = self.resblocks[i](x) 232 | attn_weights.append(attn_weight) 233 | #feature_map_list.append(x) 234 | ''' 235 | return x, attn_weights 236 | 237 | 238 | class VisionTransformer(nn.Module): 239 | def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int): 240 | super().__init__() 241 | self.input_resolution = input_resolution 242 | self.output_dim = output_dim 243 | self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False) 244 | 245 | scale = width ** -0.5 246 | self.class_embedding = nn.Parameter(scale * torch.randn(width)) 247 | self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)) 248 | self.ln_pre = LayerNorm(width) 249 | 250 | self.transformer = Transformer(width, layers, heads) 251 | 252 | self.ln_post = LayerNorm(width) 253 | self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) 254 | self.patch_size = patch_size 255 | 256 | def forward(self, x: torch.Tensor, H, W): 257 | 258 | 259 | self.positional_embedding_new = upsample_pos_emb(self.positional_embedding, (H//16,W//16)) 260 | x = self.conv1(x) # shape = [*, width, grid, grid] 261 | x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2] 262 | x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] 263 | x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width] 264 | x = x + self.positional_embedding_new.to(x.dtype) 265 | x = self.ln_pre(x) 266 | 267 | x = x.permute(1, 0, 2) # NLD -> LND 268 | x, attn_weight = self.transformer(x) 269 | ''' 270 | x = x.permute(1, 0, 2) # LND -> NLD 271 | 272 | x = self.ln_post(x) 273 | #x = x[:, 0, :] 274 | #x = x[:,1:,:] 275 | x = torch.mean(x[:,1:,:],dim=1) 276 | #feature_map_list.append(x) 277 | 278 | if self.proj is not None: 279 | x = x @ self.proj 280 | ''' 281 | 282 | return x, attn_weight#cls_attn 283 | 284 | 285 | class CLIP(nn.Module): 286 | def __init__(self, 287 | embed_dim: int, 288 | # vision 289 | image_resolution: int, 290 | vision_layers: Union[Tuple[int, int, int, int], int], 291 | vision_width: int, 292 | vision_patch_size: int, 293 | # text 294 | context_length: int, 295 | vocab_size: int, 296 | transformer_width: int, 297 | transformer_heads: int, 298 | transformer_layers: int 299 | ): 300 | super().__init__() 301 | 302 | self.context_length = context_length 303 | 304 | if isinstance(vision_layers, (tuple, list)): 305 | vision_heads = vision_width * 32 // 64 306 | self.visual = ModifiedResNet( 307 | layers=vision_layers, 308 | output_dim=embed_dim, 309 | heads=vision_heads, 310 | input_resolution=image_resolution, 311 | width=vision_width 312 | ) 313 | else: 314 | vision_heads = vision_width // 64 315 | self.visual = VisionTransformer( 316 | input_resolution=image_resolution, 317 | patch_size=vision_patch_size, 318 | width=vision_width, 319 | layers=vision_layers, 320 | heads=vision_heads, 321 | output_dim=embed_dim 322 | ) 323 | 324 | self.transformer = Transformer( 325 | width=transformer_width, 326 | layers=transformer_layers, 327 | heads=transformer_heads, 328 | attn_mask=self.build_attention_mask() 329 | ) 330 | 331 | self.vocab_size = vocab_size 332 | self.token_embedding = nn.Embedding(vocab_size, transformer_width) 333 | self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width)) 334 | self.ln_final = LayerNorm(transformer_width) 335 | 336 | self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim)) 337 | self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) 338 | 339 | self.initialize_parameters() 340 | 341 | def initialize_parameters(self): 342 | nn.init.normal_(self.token_embedding.weight, std=0.02) 343 | nn.init.normal_(self.positional_embedding, std=0.01) 344 | 345 | if isinstance(self.visual, ModifiedResNet): 346 | if self.visual.attnpool is not None: 347 | std = self.visual.attnpool.c_proj.in_features ** -0.5 348 | nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) 349 | nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) 350 | nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) 351 | nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) 352 | 353 | for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]: 354 | for name, param in resnet_block.named_parameters(): 355 | if name.endswith("bn3.weight"): 356 | nn.init.zeros_(param) 357 | 358 | proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) 359 | attn_std = self.transformer.width ** -0.5 360 | fc_std = (2 * self.transformer.width) ** -0.5 361 | for block in self.transformer.resblocks: 362 | nn.init.normal_(block.attn.in_proj_weight, std=attn_std) 363 | nn.init.normal_(block.attn.out_proj.weight, std=proj_std) 364 | nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) 365 | nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) 366 | 367 | if self.text_projection is not None: 368 | nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) 369 | 370 | def build_attention_mask(self): 371 | # lazily create causal attention mask, with full attention between the vision tokens 372 | # pytorch uses additive attention mask; fill with -inf 373 | mask = torch.empty(self.context_length, self.context_length) 374 | mask.fill_(float("-inf")) 375 | mask.triu_(1) # zero out the lower diagonal 376 | return mask 377 | 378 | @property 379 | def dtype(self): 380 | return self.visual.conv1.weight.dtype 381 | 382 | def encode_image(self, image, H, W): 383 | return self.visual(image.type(self.dtype), H, W) 384 | 385 | def encode_text(self, text): 386 | x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model] 387 | 388 | x = x + self.positional_embedding.type(self.dtype) 389 | x = x.permute(1, 0, 2) # NLD -> LND 390 | x, attn_weight = self.transformer(x) 391 | x = x.permute(1, 0, 2) # LND -> NLD 392 | x = self.ln_final(x).type(self.dtype) 393 | 394 | # x.shape = [batch_size, n_ctx, transformer.width] 395 | # take features from the eot embedding (eot_token is the highest number in each sequence) 396 | x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection 397 | 398 | return x 399 | 400 | def forward_last_layer(self, image_features, text_features): 401 | x, attn_weight = self.visual.transformer.resblocks[self.visual.transformer.layers-1](image_features) 402 | x = x.permute(1, 0, 2) # LND -> NLD 403 | 404 | x = self.visual.ln_post(x) 405 | x = torch.mean(x[:, 1:, :], dim=1) 406 | 407 | if self.visual.proj is not None: 408 | x = x @ self.visual.proj 409 | 410 | image_features = x 411 | 412 | # normalized features 413 | image_features = image_features / image_features.norm(dim=1, keepdim=True) 414 | text_features = text_features / text_features.norm(dim=1, keepdim=True) 415 | # cosine similarity as logits 416 | logit_scale = self.logit_scale.exp() 417 | logits_per_image = logit_scale * image_features @ text_features.t() 418 | 419 | # shape = [global_batch_size, global_batch_size] 420 | logits_per_image = logits_per_image.softmax(dim=-1) 421 | 422 | return logits_per_image, attn_weight 423 | 424 | 425 | 426 | 427 | def forward(self, image, text): 428 | image_features, feature_map, cls_attn = self.encode_image(image) 429 | with torch.no_grad(): 430 | text_features = self.encode_text(text) 431 | 432 | # normalized features 433 | image_features = image_features / image_features.norm(dim=1, keepdim=True) 434 | text_features = text_features / text_features.norm(dim=1, keepdim=True) 435 | 436 | # cosine similarity as logits 437 | logit_scale = self.logit_scale.exp() 438 | logits_per_image = logit_scale * image_features @ text_features.t() 439 | #logits_per_text = logits_per_image.t() 440 | 441 | # shape = [global_batch_size, global_batch_size] 442 | return logits_per_image, logits_per_text 443 | 444 | 445 | def convert_weights(model: nn.Module): 446 | """Convert applicable model parameters to fp16""" 447 | 448 | def _convert_weights_to_fp16(l): 449 | if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)): 450 | l.weight.data = l.weight.data.half() 451 | if l.bias is not None: 452 | l.bias.data = l.bias.data.half() 453 | 454 | if isinstance(l, nn.MultiheadAttention): 455 | for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]: 456 | tensor = getattr(l, attr) 457 | if tensor is not None: 458 | tensor.data = tensor.data.half() 459 | 460 | for name in ["text_projection", "proj"]: 461 | if hasattr(l, name): 462 | attr = getattr(l, name) 463 | if attr is not None: 464 | attr.data = attr.data.half() 465 | 466 | model.apply(_convert_weights_to_fp16) 467 | 468 | 469 | def build_model(state_dict: dict): 470 | vit = "visual.proj" in state_dict 471 | ''' 472 | inv_freq = 1. / (10000 ** (torch.arange(0, 2048, 2, dtype=torch.float) / 2048)) 473 | position = torch.arange(50, dtype=torch.float) 474 | sinusoid_inp = torch.einsum('i,j -> ij', position, inv_freq) 475 | embeddings = torch.cat((sinusoid_inp.sin(), sinusoid_inp.cos()), dim=-1) / 2048 ** 0.5 476 | state_dict["visual.attnpool.positional_embedding"] = embeddings 477 | ''' 478 | #state_dict["visual.positional_embedding"] = upsample_pos_emb(state_dict["visual.positional_embedding"], 28) 479 | 480 | 481 | if vit: 482 | vision_width = state_dict["visual.conv1.weight"].shape[0] 483 | vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")]) 484 | vision_patch_size = state_dict["visual.conv1.weight"].shape[-1] 485 | grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5) 486 | image_resolution = vision_patch_size * grid_size 487 | else: 488 | counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]] 489 | vision_layers = tuple(counts) 490 | vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0] 491 | output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5) 492 | vision_patch_size = None 493 | assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0] 494 | image_resolution = output_width * 32 495 | 496 | embed_dim = state_dict["text_projection"].shape[1] 497 | context_length = state_dict["positional_embedding"].shape[0] 498 | vocab_size = state_dict["token_embedding.weight"].shape[0] 499 | transformer_width = state_dict["ln_final.weight"].shape[0] 500 | transformer_heads = transformer_width // 64 501 | transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks"))) 502 | 503 | model = CLIP( 504 | embed_dim, 505 | image_resolution, vision_layers, vision_width, vision_patch_size, 506 | context_length, vocab_size, transformer_width, transformer_heads, transformer_layers 507 | ) 508 | 509 | for key in ["input_resolution", "context_length", "vocab_size"]: 510 | if key in state_dict: 511 | del state_dict[key] 512 | 513 | 514 | 515 | convert_weights(model) 516 | model.load_state_dict(state_dict) 517 | return model.eval() 518 | -------------------------------------------------------------------------------- /voc12/val.txt: -------------------------------------------------------------------------------- 1 | 2007_000033 2 | 2007_000042 3 | 2007_000061 4 | 2007_000123 5 | 2007_000129 6 | 2007_000175 7 | 2007_000187 8 | 2007_000323 9 | 2007_000332 10 | 2007_000346 11 | 2007_000452 12 | 2007_000464 13 | 2007_000491 14 | 2007_000529 15 | 2007_000559 16 | 2007_000572 17 | 2007_000629 18 | 2007_000636 19 | 2007_000661 20 | 2007_000663 21 | 2007_000676 22 | 2007_000727 23 | 2007_000762 24 | 2007_000783 25 | 2007_000799 26 | 2007_000804 27 | 2007_000830 28 | 2007_000837 29 | 2007_000847 30 | 2007_000862 31 | 2007_000925 32 | 2007_000999 33 | 2007_001154 34 | 2007_001175 35 | 2007_001239 36 | 2007_001284 37 | 2007_001288 38 | 2007_001289 39 | 2007_001299 40 | 2007_001311 41 | 2007_001321 42 | 2007_001377 43 | 2007_001408 44 | 2007_001423 45 | 2007_001430 46 | 2007_001457 47 | 2007_001458 48 | 2007_001526 49 | 2007_001568 50 | 2007_001585 51 | 2007_001586 52 | 2007_001587 53 | 2007_001594 54 | 2007_001630 55 | 2007_001677 56 | 2007_001678 57 | 2007_001717 58 | 2007_001733 59 | 2007_001761 60 | 2007_001763 61 | 2007_001774 62 | 2007_001884 63 | 2007_001955 64 | 2007_002046 65 | 2007_002094 66 | 2007_002119 67 | 2007_002132 68 | 2007_002260 69 | 2007_002266 70 | 2007_002268 71 | 2007_002284 72 | 2007_002376 73 | 2007_002378 74 | 2007_002387 75 | 2007_002400 76 | 2007_002412 77 | 2007_002426 78 | 2007_002427 79 | 2007_002445 80 | 2007_002470 81 | 2007_002539 82 | 2007_002565 83 | 2007_002597 84 | 2007_002618 85 | 2007_002619 86 | 2007_002624 87 | 2007_002643 88 | 2007_002648 89 | 2007_002719 90 | 2007_002728 91 | 2007_002823 92 | 2007_002824 93 | 2007_002852 94 | 2007_002903 95 | 2007_003011 96 | 2007_003020 97 | 2007_003022 98 | 2007_003051 99 | 2007_003088 100 | 2007_003101 101 | 2007_003106 102 | 2007_003110 103 | 2007_003131 104 | 2007_003134 105 | 2007_003137 106 | 2007_003143 107 | 2007_003169 108 | 2007_003188 109 | 2007_003194 110 | 2007_003195 111 | 2007_003201 112 | 2007_003349 113 | 2007_003367 114 | 2007_003373 115 | 2007_003499 116 | 2007_003503 117 | 2007_003506 118 | 2007_003530 119 | 2007_003571 120 | 2007_003587 121 | 2007_003611 122 | 2007_003621 123 | 2007_003682 124 | 2007_003711 125 | 2007_003714 126 | 2007_003742 127 | 2007_003786 128 | 2007_003841 129 | 2007_003848 130 | 2007_003861 131 | 2007_003872 132 | 2007_003917 133 | 2007_003957 134 | 2007_003991 135 | 2007_004033 136 | 2007_004052 137 | 2007_004112 138 | 2007_004121 139 | 2007_004143 140 | 2007_004189 141 | 2007_004190 142 | 2007_004193 143 | 2007_004241 144 | 2007_004275 145 | 2007_004281 146 | 2007_004380 147 | 2007_004392 148 | 2007_004405 149 | 2007_004468 150 | 2007_004483 151 | 2007_004510 152 | 2007_004538 153 | 2007_004558 154 | 2007_004644 155 | 2007_004649 156 | 2007_004712 157 | 2007_004722 158 | 2007_004856 159 | 2007_004866 160 | 2007_004902 161 | 2007_004969 162 | 2007_005058 163 | 2007_005074 164 | 2007_005107 165 | 2007_005114 166 | 2007_005149 167 | 2007_005173 168 | 2007_005281 169 | 2007_005294 170 | 2007_005296 171 | 2007_005304 172 | 2007_005331 173 | 2007_005354 174 | 2007_005358 175 | 2007_005428 176 | 2007_005460 177 | 2007_005469 178 | 2007_005509 179 | 2007_005547 180 | 2007_005600 181 | 2007_005608 182 | 2007_005626 183 | 2007_005689 184 | 2007_005696 185 | 2007_005705 186 | 2007_005759 187 | 2007_005803 188 | 2007_005813 189 | 2007_005828 190 | 2007_005844 191 | 2007_005845 192 | 2007_005857 193 | 2007_005911 194 | 2007_005915 195 | 2007_005978 196 | 2007_006028 197 | 2007_006035 198 | 2007_006046 199 | 2007_006076 200 | 2007_006086 201 | 2007_006117 202 | 2007_006171 203 | 2007_006241 204 | 2007_006260 205 | 2007_006277 206 | 2007_006348 207 | 2007_006364 208 | 2007_006373 209 | 2007_006444 210 | 2007_006449 211 | 2007_006549 212 | 2007_006553 213 | 2007_006560 214 | 2007_006647 215 | 2007_006678 216 | 2007_006680 217 | 2007_006698 218 | 2007_006761 219 | 2007_006802 220 | 2007_006837 221 | 2007_006841 222 | 2007_006864 223 | 2007_006866 224 | 2007_006946 225 | 2007_007007 226 | 2007_007084 227 | 2007_007109 228 | 2007_007130 229 | 2007_007165 230 | 2007_007168 231 | 2007_007195 232 | 2007_007196 233 | 2007_007203 234 | 2007_007211 235 | 2007_007235 236 | 2007_007341 237 | 2007_007414 238 | 2007_007417 239 | 2007_007470 240 | 2007_007477 241 | 2007_007493 242 | 2007_007498 243 | 2007_007524 244 | 2007_007534 245 | 2007_007624 246 | 2007_007651 247 | 2007_007688 248 | 2007_007748 249 | 2007_007795 250 | 2007_007810 251 | 2007_007815 252 | 2007_007818 253 | 2007_007836 254 | 2007_007849 255 | 2007_007881 256 | 2007_007996 257 | 2007_008051 258 | 2007_008084 259 | 2007_008106 260 | 2007_008110 261 | 2007_008204 262 | 2007_008222 263 | 2007_008256 264 | 2007_008260 265 | 2007_008339 266 | 2007_008374 267 | 2007_008415 268 | 2007_008430 269 | 2007_008543 270 | 2007_008547 271 | 2007_008596 272 | 2007_008645 273 | 2007_008670 274 | 2007_008708 275 | 2007_008722 276 | 2007_008747 277 | 2007_008802 278 | 2007_008815 279 | 2007_008897 280 | 2007_008944 281 | 2007_008964 282 | 2007_008973 283 | 2007_008980 284 | 2007_009015 285 | 2007_009068 286 | 2007_009084 287 | 2007_009088 288 | 2007_009096 289 | 2007_009221 290 | 2007_009245 291 | 2007_009251 292 | 2007_009252 293 | 2007_009258 294 | 2007_009320 295 | 2007_009323 296 | 2007_009331 297 | 2007_009346 298 | 2007_009392 299 | 2007_009413 300 | 2007_009419 301 | 2007_009446 302 | 2007_009458 303 | 2007_009521 304 | 2007_009562 305 | 2007_009592 306 | 2007_009654 307 | 2007_009655 308 | 2007_009684 309 | 2007_009687 310 | 2007_009691 311 | 2007_009706 312 | 2007_009750 313 | 2007_009756 314 | 2007_009764 315 | 2007_009794 316 | 2007_009817 317 | 2007_009841 318 | 2007_009897 319 | 2007_009911 320 | 2007_009923 321 | 2007_009938 322 | 2008_000009 323 | 2008_000016 324 | 2008_000073 325 | 2008_000075 326 | 2008_000080 327 | 2008_000107 328 | 2008_000120 329 | 2008_000123 330 | 2008_000149 331 | 2008_000182 332 | 2008_000213 333 | 2008_000215 334 | 2008_000223 335 | 2008_000233 336 | 2008_000234 337 | 2008_000239 338 | 2008_000254 339 | 2008_000270 340 | 2008_000271 341 | 2008_000345 342 | 2008_000359 343 | 2008_000391 344 | 2008_000401 345 | 2008_000464 346 | 2008_000469 347 | 2008_000474 348 | 2008_000501 349 | 2008_000510 350 | 2008_000533 351 | 2008_000573 352 | 2008_000589 353 | 2008_000602 354 | 2008_000630 355 | 2008_000657 356 | 2008_000661 357 | 2008_000662 358 | 2008_000666 359 | 2008_000673 360 | 2008_000700 361 | 2008_000725 362 | 2008_000731 363 | 2008_000763 364 | 2008_000765 365 | 2008_000782 366 | 2008_000795 367 | 2008_000811 368 | 2008_000848 369 | 2008_000853 370 | 2008_000863 371 | 2008_000911 372 | 2008_000919 373 | 2008_000943 374 | 2008_000992 375 | 2008_001013 376 | 2008_001028 377 | 2008_001040 378 | 2008_001070 379 | 2008_001074 380 | 2008_001076 381 | 2008_001078 382 | 2008_001135 383 | 2008_001150 384 | 2008_001170 385 | 2008_001231 386 | 2008_001249 387 | 2008_001260 388 | 2008_001283 389 | 2008_001308 390 | 2008_001379 391 | 2008_001404 392 | 2008_001433 393 | 2008_001439 394 | 2008_001478 395 | 2008_001491 396 | 2008_001504 397 | 2008_001513 398 | 2008_001514 399 | 2008_001531 400 | 2008_001546 401 | 2008_001547 402 | 2008_001580 403 | 2008_001629 404 | 2008_001640 405 | 2008_001682 406 | 2008_001688 407 | 2008_001715 408 | 2008_001821 409 | 2008_001874 410 | 2008_001885 411 | 2008_001895 412 | 2008_001966 413 | 2008_001971 414 | 2008_001992 415 | 2008_002043 416 | 2008_002152 417 | 2008_002205 418 | 2008_002212 419 | 2008_002239 420 | 2008_002240 421 | 2008_002241 422 | 2008_002269 423 | 2008_002273 424 | 2008_002358 425 | 2008_002379 426 | 2008_002383 427 | 2008_002429 428 | 2008_002464 429 | 2008_002467 430 | 2008_002492 431 | 2008_002495 432 | 2008_002504 433 | 2008_002521 434 | 2008_002536 435 | 2008_002588 436 | 2008_002623 437 | 2008_002680 438 | 2008_002681 439 | 2008_002775 440 | 2008_002778 441 | 2008_002835 442 | 2008_002859 443 | 2008_002864 444 | 2008_002900 445 | 2008_002904 446 | 2008_002929 447 | 2008_002936 448 | 2008_002942 449 | 2008_002958 450 | 2008_003003 451 | 2008_003026 452 | 2008_003034 453 | 2008_003076 454 | 2008_003105 455 | 2008_003108 456 | 2008_003110 457 | 2008_003135 458 | 2008_003141 459 | 2008_003155 460 | 2008_003210 461 | 2008_003238 462 | 2008_003270 463 | 2008_003330 464 | 2008_003333 465 | 2008_003369 466 | 2008_003379 467 | 2008_003451 468 | 2008_003461 469 | 2008_003477 470 | 2008_003492 471 | 2008_003499 472 | 2008_003511 473 | 2008_003546 474 | 2008_003576 475 | 2008_003577 476 | 2008_003676 477 | 2008_003709 478 | 2008_003733 479 | 2008_003777 480 | 2008_003782 481 | 2008_003821 482 | 2008_003846 483 | 2008_003856 484 | 2008_003858 485 | 2008_003874 486 | 2008_003876 487 | 2008_003885 488 | 2008_003886 489 | 2008_003926 490 | 2008_003976 491 | 2008_004069 492 | 2008_004101 493 | 2008_004140 494 | 2008_004172 495 | 2008_004175 496 | 2008_004212 497 | 2008_004279 498 | 2008_004339 499 | 2008_004345 500 | 2008_004363 501 | 2008_004367 502 | 2008_004396 503 | 2008_004399 504 | 2008_004453 505 | 2008_004477 506 | 2008_004552 507 | 2008_004562 508 | 2008_004575 509 | 2008_004610 510 | 2008_004612 511 | 2008_004621 512 | 2008_004624 513 | 2008_004654 514 | 2008_004659 515 | 2008_004687 516 | 2008_004701 517 | 2008_004704 518 | 2008_004705 519 | 2008_004754 520 | 2008_004758 521 | 2008_004854 522 | 2008_004910 523 | 2008_004995 524 | 2008_005049 525 | 2008_005089 526 | 2008_005097 527 | 2008_005105 528 | 2008_005145 529 | 2008_005197 530 | 2008_005217 531 | 2008_005242 532 | 2008_005245 533 | 2008_005254 534 | 2008_005262 535 | 2008_005338 536 | 2008_005398 537 | 2008_005399 538 | 2008_005422 539 | 2008_005439 540 | 2008_005445 541 | 2008_005525 542 | 2008_005544 543 | 2008_005628 544 | 2008_005633 545 | 2008_005637 546 | 2008_005642 547 | 2008_005676 548 | 2008_005680 549 | 2008_005691 550 | 2008_005727 551 | 2008_005738 552 | 2008_005812 553 | 2008_005904 554 | 2008_005915 555 | 2008_006008 556 | 2008_006036 557 | 2008_006055 558 | 2008_006063 559 | 2008_006108 560 | 2008_006130 561 | 2008_006143 562 | 2008_006159 563 | 2008_006216 564 | 2008_006219 565 | 2008_006229 566 | 2008_006254 567 | 2008_006275 568 | 2008_006325 569 | 2008_006327 570 | 2008_006341 571 | 2008_006408 572 | 2008_006480 573 | 2008_006523 574 | 2008_006526 575 | 2008_006528 576 | 2008_006553 577 | 2008_006554 578 | 2008_006703 579 | 2008_006722 580 | 2008_006752 581 | 2008_006784 582 | 2008_006835 583 | 2008_006874 584 | 2008_006981 585 | 2008_006986 586 | 2008_007025 587 | 2008_007031 588 | 2008_007048 589 | 2008_007120 590 | 2008_007123 591 | 2008_007143 592 | 2008_007194 593 | 2008_007219 594 | 2008_007273 595 | 2008_007350 596 | 2008_007378 597 | 2008_007392 598 | 2008_007402 599 | 2008_007497 600 | 2008_007498 601 | 2008_007507 602 | 2008_007513 603 | 2008_007527 604 | 2008_007548 605 | 2008_007596 606 | 2008_007677 607 | 2008_007737 608 | 2008_007797 609 | 2008_007804 610 | 2008_007811 611 | 2008_007814 612 | 2008_007828 613 | 2008_007836 614 | 2008_007945 615 | 2008_007994 616 | 2008_008051 617 | 2008_008103 618 | 2008_008127 619 | 2008_008221 620 | 2008_008252 621 | 2008_008268 622 | 2008_008296 623 | 2008_008301 624 | 2008_008335 625 | 2008_008362 626 | 2008_008392 627 | 2008_008393 628 | 2008_008421 629 | 2008_008434 630 | 2008_008469 631 | 2008_008629 632 | 2008_008682 633 | 2008_008711 634 | 2008_008746 635 | 2009_000012 636 | 2009_000013 637 | 2009_000022 638 | 2009_000032 639 | 2009_000037 640 | 2009_000039 641 | 2009_000074 642 | 2009_000080 643 | 2009_000087 644 | 2009_000096 645 | 2009_000121 646 | 2009_000136 647 | 2009_000149 648 | 2009_000156 649 | 2009_000201 650 | 2009_000205 651 | 2009_000219 652 | 2009_000242 653 | 2009_000309 654 | 2009_000318 655 | 2009_000335 656 | 2009_000351 657 | 2009_000354 658 | 2009_000387 659 | 2009_000391 660 | 2009_000412 661 | 2009_000418 662 | 2009_000421 663 | 2009_000426 664 | 2009_000440 665 | 2009_000446 666 | 2009_000455 667 | 2009_000457 668 | 2009_000469 669 | 2009_000487 670 | 2009_000488 671 | 2009_000523 672 | 2009_000573 673 | 2009_000619 674 | 2009_000628 675 | 2009_000641 676 | 2009_000664 677 | 2009_000675 678 | 2009_000704 679 | 2009_000705 680 | 2009_000712 681 | 2009_000716 682 | 2009_000723 683 | 2009_000727 684 | 2009_000730 685 | 2009_000731 686 | 2009_000732 687 | 2009_000771 688 | 2009_000825 689 | 2009_000828 690 | 2009_000839 691 | 2009_000840 692 | 2009_000845 693 | 2009_000879 694 | 2009_000892 695 | 2009_000919 696 | 2009_000924 697 | 2009_000931 698 | 2009_000935 699 | 2009_000964 700 | 2009_000989 701 | 2009_000991 702 | 2009_000998 703 | 2009_001008 704 | 2009_001082 705 | 2009_001108 706 | 2009_001160 707 | 2009_001215 708 | 2009_001240 709 | 2009_001255 710 | 2009_001278 711 | 2009_001299 712 | 2009_001300 713 | 2009_001314 714 | 2009_001332 715 | 2009_001333 716 | 2009_001363 717 | 2009_001391 718 | 2009_001411 719 | 2009_001433 720 | 2009_001505 721 | 2009_001535 722 | 2009_001536 723 | 2009_001565 724 | 2009_001607 725 | 2009_001644 726 | 2009_001663 727 | 2009_001683 728 | 2009_001684 729 | 2009_001687 730 | 2009_001718 731 | 2009_001731 732 | 2009_001765 733 | 2009_001768 734 | 2009_001775 735 | 2009_001804 736 | 2009_001816 737 | 2009_001818 738 | 2009_001850 739 | 2009_001851 740 | 2009_001854 741 | 2009_001941 742 | 2009_001991 743 | 2009_002012 744 | 2009_002035 745 | 2009_002042 746 | 2009_002082 747 | 2009_002094 748 | 2009_002097 749 | 2009_002122 750 | 2009_002150 751 | 2009_002155 752 | 2009_002164 753 | 2009_002165 754 | 2009_002171 755 | 2009_002185 756 | 2009_002202 757 | 2009_002221 758 | 2009_002238 759 | 2009_002239 760 | 2009_002265 761 | 2009_002268 762 | 2009_002291 763 | 2009_002295 764 | 2009_002317 765 | 2009_002320 766 | 2009_002346 767 | 2009_002366 768 | 2009_002372 769 | 2009_002382 770 | 2009_002390 771 | 2009_002415 772 | 2009_002445 773 | 2009_002487 774 | 2009_002521 775 | 2009_002527 776 | 2009_002535 777 | 2009_002539 778 | 2009_002549 779 | 2009_002562 780 | 2009_002568 781 | 2009_002571 782 | 2009_002573 783 | 2009_002584 784 | 2009_002591 785 | 2009_002594 786 | 2009_002604 787 | 2009_002618 788 | 2009_002635 789 | 2009_002638 790 | 2009_002649 791 | 2009_002651 792 | 2009_002727 793 | 2009_002732 794 | 2009_002749 795 | 2009_002753 796 | 2009_002771 797 | 2009_002808 798 | 2009_002856 799 | 2009_002887 800 | 2009_002888 801 | 2009_002928 802 | 2009_002936 803 | 2009_002975 804 | 2009_002982 805 | 2009_002990 806 | 2009_003003 807 | 2009_003005 808 | 2009_003043 809 | 2009_003059 810 | 2009_003063 811 | 2009_003065 812 | 2009_003071 813 | 2009_003080 814 | 2009_003105 815 | 2009_003123 816 | 2009_003193 817 | 2009_003196 818 | 2009_003217 819 | 2009_003224 820 | 2009_003241 821 | 2009_003269 822 | 2009_003273 823 | 2009_003299 824 | 2009_003304 825 | 2009_003311 826 | 2009_003323 827 | 2009_003343 828 | 2009_003378 829 | 2009_003387 830 | 2009_003406 831 | 2009_003433 832 | 2009_003450 833 | 2009_003466 834 | 2009_003481 835 | 2009_003494 836 | 2009_003498 837 | 2009_003504 838 | 2009_003507 839 | 2009_003517 840 | 2009_003523 841 | 2009_003542 842 | 2009_003549 843 | 2009_003551 844 | 2009_003564 845 | 2009_003569 846 | 2009_003576 847 | 2009_003589 848 | 2009_003607 849 | 2009_003640 850 | 2009_003666 851 | 2009_003696 852 | 2009_003703 853 | 2009_003707 854 | 2009_003756 855 | 2009_003771 856 | 2009_003773 857 | 2009_003804 858 | 2009_003806 859 | 2009_003810 860 | 2009_003849 861 | 2009_003857 862 | 2009_003858 863 | 2009_003895 864 | 2009_003903 865 | 2009_003904 866 | 2009_003928 867 | 2009_003938 868 | 2009_003971 869 | 2009_003991 870 | 2009_004021 871 | 2009_004033 872 | 2009_004043 873 | 2009_004070 874 | 2009_004072 875 | 2009_004084 876 | 2009_004099 877 | 2009_004125 878 | 2009_004140 879 | 2009_004217 880 | 2009_004221 881 | 2009_004247 882 | 2009_004248 883 | 2009_004255 884 | 2009_004298 885 | 2009_004324 886 | 2009_004455 887 | 2009_004494 888 | 2009_004497 889 | 2009_004504 890 | 2009_004507 891 | 2009_004509 892 | 2009_004540 893 | 2009_004568 894 | 2009_004579 895 | 2009_004581 896 | 2009_004590 897 | 2009_004592 898 | 2009_004594 899 | 2009_004635 900 | 2009_004653 901 | 2009_004687 902 | 2009_004721 903 | 2009_004730 904 | 2009_004732 905 | 2009_004738 906 | 2009_004748 907 | 2009_004789 908 | 2009_004799 909 | 2009_004801 910 | 2009_004848 911 | 2009_004859 912 | 2009_004867 913 | 2009_004882 914 | 2009_004886 915 | 2009_004895 916 | 2009_004942 917 | 2009_004969 918 | 2009_004987 919 | 2009_004993 920 | 2009_004994 921 | 2009_005038 922 | 2009_005078 923 | 2009_005087 924 | 2009_005089 925 | 2009_005137 926 | 2009_005148 927 | 2009_005156 928 | 2009_005158 929 | 2009_005189 930 | 2009_005190 931 | 2009_005217 932 | 2009_005219 933 | 2009_005220 934 | 2009_005231 935 | 2009_005260 936 | 2009_005262 937 | 2009_005302 938 | 2010_000003 939 | 2010_000038 940 | 2010_000065 941 | 2010_000083 942 | 2010_000084 943 | 2010_000087 944 | 2010_000110 945 | 2010_000159 946 | 2010_000160 947 | 2010_000163 948 | 2010_000174 949 | 2010_000216 950 | 2010_000238 951 | 2010_000241 952 | 2010_000256 953 | 2010_000272 954 | 2010_000284 955 | 2010_000309 956 | 2010_000318 957 | 2010_000330 958 | 2010_000335 959 | 2010_000342 960 | 2010_000372 961 | 2010_000422 962 | 2010_000426 963 | 2010_000427 964 | 2010_000502 965 | 2010_000530 966 | 2010_000552 967 | 2010_000559 968 | 2010_000572 969 | 2010_000573 970 | 2010_000622 971 | 2010_000628 972 | 2010_000639 973 | 2010_000666 974 | 2010_000679 975 | 2010_000682 976 | 2010_000683 977 | 2010_000724 978 | 2010_000738 979 | 2010_000764 980 | 2010_000788 981 | 2010_000814 982 | 2010_000836 983 | 2010_000874 984 | 2010_000904 985 | 2010_000906 986 | 2010_000907 987 | 2010_000918 988 | 2010_000929 989 | 2010_000941 990 | 2010_000952 991 | 2010_000961 992 | 2010_001000 993 | 2010_001010 994 | 2010_001011 995 | 2010_001016 996 | 2010_001017 997 | 2010_001024 998 | 2010_001036 999 | 2010_001061 1000 | 2010_001069 1001 | 2010_001070 1002 | 2010_001079 1003 | 2010_001104 1004 | 2010_001124 1005 | 2010_001149 1006 | 2010_001151 1007 | 2010_001174 1008 | 2010_001206 1009 | 2010_001246 1010 | 2010_001251 1011 | 2010_001256 1012 | 2010_001264 1013 | 2010_001292 1014 | 2010_001313 1015 | 2010_001327 1016 | 2010_001331 1017 | 2010_001351 1018 | 2010_001367 1019 | 2010_001376 1020 | 2010_001403 1021 | 2010_001448 1022 | 2010_001451 1023 | 2010_001522 1024 | 2010_001534 1025 | 2010_001553 1026 | 2010_001557 1027 | 2010_001563 1028 | 2010_001577 1029 | 2010_001579 1030 | 2010_001646 1031 | 2010_001656 1032 | 2010_001692 1033 | 2010_001699 1034 | 2010_001734 1035 | 2010_001752 1036 | 2010_001767 1037 | 2010_001768 1038 | 2010_001773 1039 | 2010_001820 1040 | 2010_001830 1041 | 2010_001851 1042 | 2010_001908 1043 | 2010_001913 1044 | 2010_001951 1045 | 2010_001956 1046 | 2010_001962 1047 | 2010_001966 1048 | 2010_001995 1049 | 2010_002017 1050 | 2010_002025 1051 | 2010_002030 1052 | 2010_002106 1053 | 2010_002137 1054 | 2010_002142 1055 | 2010_002146 1056 | 2010_002147 1057 | 2010_002150 1058 | 2010_002161 1059 | 2010_002200 1060 | 2010_002228 1061 | 2010_002232 1062 | 2010_002251 1063 | 2010_002271 1064 | 2010_002305 1065 | 2010_002310 1066 | 2010_002336 1067 | 2010_002348 1068 | 2010_002361 1069 | 2010_002390 1070 | 2010_002396 1071 | 2010_002422 1072 | 2010_002450 1073 | 2010_002480 1074 | 2010_002512 1075 | 2010_002531 1076 | 2010_002536 1077 | 2010_002538 1078 | 2010_002546 1079 | 2010_002623 1080 | 2010_002682 1081 | 2010_002691 1082 | 2010_002693 1083 | 2010_002701 1084 | 2010_002763 1085 | 2010_002792 1086 | 2010_002868 1087 | 2010_002900 1088 | 2010_002902 1089 | 2010_002921 1090 | 2010_002929 1091 | 2010_002939 1092 | 2010_002988 1093 | 2010_003014 1094 | 2010_003060 1095 | 2010_003123 1096 | 2010_003127 1097 | 2010_003132 1098 | 2010_003168 1099 | 2010_003183 1100 | 2010_003187 1101 | 2010_003207 1102 | 2010_003231 1103 | 2010_003239 1104 | 2010_003275 1105 | 2010_003276 1106 | 2010_003293 1107 | 2010_003302 1108 | 2010_003325 1109 | 2010_003362 1110 | 2010_003365 1111 | 2010_003381 1112 | 2010_003402 1113 | 2010_003409 1114 | 2010_003418 1115 | 2010_003446 1116 | 2010_003453 1117 | 2010_003468 1118 | 2010_003473 1119 | 2010_003495 1120 | 2010_003506 1121 | 2010_003514 1122 | 2010_003531 1123 | 2010_003532 1124 | 2010_003541 1125 | 2010_003547 1126 | 2010_003597 1127 | 2010_003675 1128 | 2010_003708 1129 | 2010_003716 1130 | 2010_003746 1131 | 2010_003758 1132 | 2010_003764 1133 | 2010_003768 1134 | 2010_003771 1135 | 2010_003772 1136 | 2010_003781 1137 | 2010_003813 1138 | 2010_003820 1139 | 2010_003854 1140 | 2010_003912 1141 | 2010_003915 1142 | 2010_003947 1143 | 2010_003956 1144 | 2010_003971 1145 | 2010_004041 1146 | 2010_004042 1147 | 2010_004056 1148 | 2010_004063 1149 | 2010_004104 1150 | 2010_004120 1151 | 2010_004149 1152 | 2010_004165 1153 | 2010_004208 1154 | 2010_004219 1155 | 2010_004226 1156 | 2010_004314 1157 | 2010_004320 1158 | 2010_004322 1159 | 2010_004337 1160 | 2010_004348 1161 | 2010_004355 1162 | 2010_004369 1163 | 2010_004382 1164 | 2010_004419 1165 | 2010_004432 1166 | 2010_004472 1167 | 2010_004479 1168 | 2010_004519 1169 | 2010_004520 1170 | 2010_004529 1171 | 2010_004543 1172 | 2010_004550 1173 | 2010_004551 1174 | 2010_004556 1175 | 2010_004559 1176 | 2010_004628 1177 | 2010_004635 1178 | 2010_004662 1179 | 2010_004697 1180 | 2010_004757 1181 | 2010_004763 1182 | 2010_004772 1183 | 2010_004783 1184 | 2010_004789 1185 | 2010_004795 1186 | 2010_004815 1187 | 2010_004825 1188 | 2010_004828 1189 | 2010_004856 1190 | 2010_004857 1191 | 2010_004861 1192 | 2010_004941 1193 | 2010_004946 1194 | 2010_004951 1195 | 2010_004980 1196 | 2010_004994 1197 | 2010_005013 1198 | 2010_005021 1199 | 2010_005046 1200 | 2010_005063 1201 | 2010_005108 1202 | 2010_005118 1203 | 2010_005159 1204 | 2010_005160 1205 | 2010_005166 1206 | 2010_005174 1207 | 2010_005180 1208 | 2010_005187 1209 | 2010_005206 1210 | 2010_005245 1211 | 2010_005252 1212 | 2010_005284 1213 | 2010_005305 1214 | 2010_005344 1215 | 2010_005353 1216 | 2010_005366 1217 | 2010_005401 1218 | 2010_005421 1219 | 2010_005428 1220 | 2010_005432 1221 | 2010_005433 1222 | 2010_005496 1223 | 2010_005501 1224 | 2010_005508 1225 | 2010_005531 1226 | 2010_005534 1227 | 2010_005575 1228 | 2010_005582 1229 | 2010_005606 1230 | 2010_005626 1231 | 2010_005644 1232 | 2010_005664 1233 | 2010_005705 1234 | 2010_005706 1235 | 2010_005709 1236 | 2010_005718 1237 | 2010_005719 1238 | 2010_005727 1239 | 2010_005762 1240 | 2010_005788 1241 | 2010_005860 1242 | 2010_005871 1243 | 2010_005877 1244 | 2010_005888 1245 | 2010_005899 1246 | 2010_005922 1247 | 2010_005991 1248 | 2010_005992 1249 | 2010_006026 1250 | 2010_006034 1251 | 2010_006054 1252 | 2010_006070 1253 | 2011_000045 1254 | 2011_000051 1255 | 2011_000054 1256 | 2011_000066 1257 | 2011_000070 1258 | 2011_000112 1259 | 2011_000173 1260 | 2011_000178 1261 | 2011_000185 1262 | 2011_000226 1263 | 2011_000234 1264 | 2011_000238 1265 | 2011_000239 1266 | 2011_000248 1267 | 2011_000283 1268 | 2011_000291 1269 | 2011_000310 1270 | 2011_000312 1271 | 2011_000338 1272 | 2011_000396 1273 | 2011_000412 1274 | 2011_000419 1275 | 2011_000435 1276 | 2011_000436 1277 | 2011_000438 1278 | 2011_000455 1279 | 2011_000456 1280 | 2011_000479 1281 | 2011_000481 1282 | 2011_000482 1283 | 2011_000503 1284 | 2011_000512 1285 | 2011_000521 1286 | 2011_000526 1287 | 2011_000536 1288 | 2011_000548 1289 | 2011_000566 1290 | 2011_000585 1291 | 2011_000598 1292 | 2011_000607 1293 | 2011_000618 1294 | 2011_000638 1295 | 2011_000658 1296 | 2011_000661 1297 | 2011_000669 1298 | 2011_000747 1299 | 2011_000780 1300 | 2011_000789 1301 | 2011_000807 1302 | 2011_000809 1303 | 2011_000813 1304 | 2011_000830 1305 | 2011_000843 1306 | 2011_000874 1307 | 2011_000888 1308 | 2011_000900 1309 | 2011_000912 1310 | 2011_000953 1311 | 2011_000969 1312 | 2011_001005 1313 | 2011_001014 1314 | 2011_001020 1315 | 2011_001047 1316 | 2011_001060 1317 | 2011_001064 1318 | 2011_001069 1319 | 2011_001071 1320 | 2011_001082 1321 | 2011_001110 1322 | 2011_001114 1323 | 2011_001159 1324 | 2011_001161 1325 | 2011_001190 1326 | 2011_001232 1327 | 2011_001263 1328 | 2011_001276 1329 | 2011_001281 1330 | 2011_001287 1331 | 2011_001292 1332 | 2011_001313 1333 | 2011_001341 1334 | 2011_001346 1335 | 2011_001350 1336 | 2011_001407 1337 | 2011_001416 1338 | 2011_001421 1339 | 2011_001434 1340 | 2011_001447 1341 | 2011_001489 1342 | 2011_001529 1343 | 2011_001530 1344 | 2011_001534 1345 | 2011_001546 1346 | 2011_001567 1347 | 2011_001589 1348 | 2011_001597 1349 | 2011_001601 1350 | 2011_001607 1351 | 2011_001613 1352 | 2011_001614 1353 | 2011_001619 1354 | 2011_001624 1355 | 2011_001642 1356 | 2011_001665 1357 | 2011_001669 1358 | 2011_001674 1359 | 2011_001708 1360 | 2011_001713 1361 | 2011_001714 1362 | 2011_001722 1363 | 2011_001726 1364 | 2011_001745 1365 | 2011_001748 1366 | 2011_001775 1367 | 2011_001782 1368 | 2011_001793 1369 | 2011_001794 1370 | 2011_001812 1371 | 2011_001862 1372 | 2011_001863 1373 | 2011_001868 1374 | 2011_001880 1375 | 2011_001910 1376 | 2011_001984 1377 | 2011_001988 1378 | 2011_002002 1379 | 2011_002040 1380 | 2011_002041 1381 | 2011_002064 1382 | 2011_002075 1383 | 2011_002098 1384 | 2011_002110 1385 | 2011_002121 1386 | 2011_002124 1387 | 2011_002150 1388 | 2011_002156 1389 | 2011_002178 1390 | 2011_002200 1391 | 2011_002223 1392 | 2011_002244 1393 | 2011_002247 1394 | 2011_002279 1395 | 2011_002295 1396 | 2011_002298 1397 | 2011_002308 1398 | 2011_002317 1399 | 2011_002322 1400 | 2011_002327 1401 | 2011_002343 1402 | 2011_002358 1403 | 2011_002371 1404 | 2011_002379 1405 | 2011_002391 1406 | 2011_002498 1407 | 2011_002509 1408 | 2011_002515 1409 | 2011_002532 1410 | 2011_002535 1411 | 2011_002548 1412 | 2011_002575 1413 | 2011_002578 1414 | 2011_002589 1415 | 2011_002592 1416 | 2011_002623 1417 | 2011_002641 1418 | 2011_002644 1419 | 2011_002662 1420 | 2011_002675 1421 | 2011_002685 1422 | 2011_002713 1423 | 2011_002730 1424 | 2011_002754 1425 | 2011_002812 1426 | 2011_002863 1427 | 2011_002879 1428 | 2011_002885 1429 | 2011_002929 1430 | 2011_002951 1431 | 2011_002975 1432 | 2011_002993 1433 | 2011_002997 1434 | 2011_003003 1435 | 2011_003011 1436 | 2011_003019 1437 | 2011_003030 1438 | 2011_003055 1439 | 2011_003085 1440 | 2011_003103 1441 | 2011_003114 1442 | 2011_003145 1443 | 2011_003146 1444 | 2011_003182 1445 | 2011_003197 1446 | 2011_003205 1447 | 2011_003240 1448 | 2011_003256 1449 | 2011_003271 1450 | -------------------------------------------------------------------------------- /voc12/train.txt: -------------------------------------------------------------------------------- 1 | 2007_000032 2 | 2007_000039 3 | 2007_000063 4 | 2007_000068 5 | 2007_000121 6 | 2007_000170 7 | 2007_000241 8 | 2007_000243 9 | 2007_000250 10 | 2007_000256 11 | 2007_000333 12 | 2007_000363 13 | 2007_000364 14 | 2007_000392 15 | 2007_000480 16 | 2007_000504 17 | 2007_000515 18 | 2007_000528 19 | 2007_000549 20 | 2007_000584 21 | 2007_000645 22 | 2007_000648 23 | 2007_000713 24 | 2007_000720 25 | 2007_000733 26 | 2007_000738 27 | 2007_000768 28 | 2007_000793 29 | 2007_000822 30 | 2007_000836 31 | 2007_000876 32 | 2007_000904 33 | 2007_001027 34 | 2007_001073 35 | 2007_001149 36 | 2007_001185 37 | 2007_001225 38 | 2007_001340 39 | 2007_001397 40 | 2007_001416 41 | 2007_001420 42 | 2007_001439 43 | 2007_001487 44 | 2007_001595 45 | 2007_001602 46 | 2007_001609 47 | 2007_001698 48 | 2007_001704 49 | 2007_001709 50 | 2007_001724 51 | 2007_001764 52 | 2007_001825 53 | 2007_001834 54 | 2007_001857 55 | 2007_001872 56 | 2007_001901 57 | 2007_001917 58 | 2007_001960 59 | 2007_002024 60 | 2007_002055 61 | 2007_002088 62 | 2007_002099 63 | 2007_002105 64 | 2007_002107 65 | 2007_002120 66 | 2007_002142 67 | 2007_002198 68 | 2007_002212 69 | 2007_002216 70 | 2007_002227 71 | 2007_002234 72 | 2007_002273 73 | 2007_002281 74 | 2007_002293 75 | 2007_002361 76 | 2007_002368 77 | 2007_002370 78 | 2007_002403 79 | 2007_002462 80 | 2007_002488 81 | 2007_002545 82 | 2007_002611 83 | 2007_002639 84 | 2007_002668 85 | 2007_002669 86 | 2007_002760 87 | 2007_002789 88 | 2007_002845 89 | 2007_002895 90 | 2007_002896 91 | 2007_002914 92 | 2007_002953 93 | 2007_002954 94 | 2007_002967 95 | 2007_003000 96 | 2007_003118 97 | 2007_003178 98 | 2007_003189 99 | 2007_003190 100 | 2007_003191 101 | 2007_003205 102 | 2007_003207 103 | 2007_003251 104 | 2007_003267 105 | 2007_003286 106 | 2007_003330 107 | 2007_003431 108 | 2007_003451 109 | 2007_003525 110 | 2007_003529 111 | 2007_003541 112 | 2007_003565 113 | 2007_003580 114 | 2007_003593 115 | 2007_003604 116 | 2007_003668 117 | 2007_003715 118 | 2007_003778 119 | 2007_003788 120 | 2007_003815 121 | 2007_003876 122 | 2007_003889 123 | 2007_003910 124 | 2007_004003 125 | 2007_004009 126 | 2007_004065 127 | 2007_004081 128 | 2007_004166 129 | 2007_004289 130 | 2007_004291 131 | 2007_004328 132 | 2007_004423 133 | 2007_004459 134 | 2007_004476 135 | 2007_004481 136 | 2007_004500 137 | 2007_004537 138 | 2007_004627 139 | 2007_004663 140 | 2007_004705 141 | 2007_004707 142 | 2007_004768 143 | 2007_004769 144 | 2007_004810 145 | 2007_004830 146 | 2007_004841 147 | 2007_004948 148 | 2007_004951 149 | 2007_004988 150 | 2007_004998 151 | 2007_005043 152 | 2007_005064 153 | 2007_005086 154 | 2007_005124 155 | 2007_005130 156 | 2007_005144 157 | 2007_005210 158 | 2007_005212 159 | 2007_005227 160 | 2007_005248 161 | 2007_005262 162 | 2007_005264 163 | 2007_005266 164 | 2007_005273 165 | 2007_005314 166 | 2007_005360 167 | 2007_005368 168 | 2007_005430 169 | 2007_005647 170 | 2007_005688 171 | 2007_005702 172 | 2007_005790 173 | 2007_005797 174 | 2007_005859 175 | 2007_005878 176 | 2007_005902 177 | 2007_005951 178 | 2007_005988 179 | 2007_005989 180 | 2007_006004 181 | 2007_006066 182 | 2007_006134 183 | 2007_006136 184 | 2007_006151 185 | 2007_006212 186 | 2007_006232 187 | 2007_006254 188 | 2007_006281 189 | 2007_006303 190 | 2007_006317 191 | 2007_006400 192 | 2007_006409 193 | 2007_006445 194 | 2007_006477 195 | 2007_006483 196 | 2007_006490 197 | 2007_006530 198 | 2007_006581 199 | 2007_006585 200 | 2007_006605 201 | 2007_006615 202 | 2007_006641 203 | 2007_006660 204 | 2007_006661 205 | 2007_006673 206 | 2007_006699 207 | 2007_006704 208 | 2007_006803 209 | 2007_006832 210 | 2007_006865 211 | 2007_006899 212 | 2007_006900 213 | 2007_006944 214 | 2007_007003 215 | 2007_007021 216 | 2007_007048 217 | 2007_007098 218 | 2007_007154 219 | 2007_007230 220 | 2007_007250 221 | 2007_007355 222 | 2007_007387 223 | 2007_007398 224 | 2007_007415 225 | 2007_007432 226 | 2007_007447 227 | 2007_007480 228 | 2007_007481 229 | 2007_007523 230 | 2007_007530 231 | 2007_007585 232 | 2007_007591 233 | 2007_007621 234 | 2007_007649 235 | 2007_007698 236 | 2007_007726 237 | 2007_007772 238 | 2007_007773 239 | 2007_007783 240 | 2007_007878 241 | 2007_007890 242 | 2007_007891 243 | 2007_007902 244 | 2007_007908 245 | 2007_007930 246 | 2007_007947 247 | 2007_007948 248 | 2007_008043 249 | 2007_008072 250 | 2007_008085 251 | 2007_008140 252 | 2007_008142 253 | 2007_008203 254 | 2007_008218 255 | 2007_008219 256 | 2007_008307 257 | 2007_008403 258 | 2007_008407 259 | 2007_008468 260 | 2007_008526 261 | 2007_008571 262 | 2007_008575 263 | 2007_008714 264 | 2007_008764 265 | 2007_008778 266 | 2007_008801 267 | 2007_008821 268 | 2007_008927 269 | 2007_008932 270 | 2007_008945 271 | 2007_008948 272 | 2007_008994 273 | 2007_009030 274 | 2007_009052 275 | 2007_009082 276 | 2007_009139 277 | 2007_009209 278 | 2007_009216 279 | 2007_009295 280 | 2007_009322 281 | 2007_009327 282 | 2007_009348 283 | 2007_009422 284 | 2007_009435 285 | 2007_009436 286 | 2007_009464 287 | 2007_009527 288 | 2007_009533 289 | 2007_009550 290 | 2007_009554 291 | 2007_009580 292 | 2007_009594 293 | 2007_009597 294 | 2007_009605 295 | 2007_009607 296 | 2007_009618 297 | 2007_009630 298 | 2007_009649 299 | 2007_009665 300 | 2007_009709 301 | 2007_009724 302 | 2007_009759 303 | 2007_009779 304 | 2007_009788 305 | 2007_009807 306 | 2007_009832 307 | 2007_009889 308 | 2007_009899 309 | 2007_009901 310 | 2007_009947 311 | 2007_009950 312 | 2008_000015 313 | 2008_000019 314 | 2008_000028 315 | 2008_000033 316 | 2008_000074 317 | 2008_000089 318 | 2008_000103 319 | 2008_000105 320 | 2008_000131 321 | 2008_000144 322 | 2008_000162 323 | 2008_000187 324 | 2008_000188 325 | 2008_000197 326 | 2008_000207 327 | 2008_000217 328 | 2008_000226 329 | 2008_000235 330 | 2008_000238 331 | 2008_000259 332 | 2008_000273 333 | 2008_000284 334 | 2008_000287 335 | 2008_000289 336 | 2008_000290 337 | 2008_000309 338 | 2008_000316 339 | 2008_000336 340 | 2008_000348 341 | 2008_000361 342 | 2008_000365 343 | 2008_000399 344 | 2008_000400 345 | 2008_000415 346 | 2008_000422 347 | 2008_000436 348 | 2008_000470 349 | 2008_000491 350 | 2008_000495 351 | 2008_000505 352 | 2008_000515 353 | 2008_000540 354 | 2008_000544 355 | 2008_000567 356 | 2008_000578 357 | 2008_000584 358 | 2008_000588 359 | 2008_000595 360 | 2008_000626 361 | 2008_000645 362 | 2008_000676 363 | 2008_000696 364 | 2008_000711 365 | 2008_000716 366 | 2008_000733 367 | 2008_000760 368 | 2008_000764 369 | 2008_000778 370 | 2008_000785 371 | 2008_000832 372 | 2008_000841 373 | 2008_000860 374 | 2008_000861 375 | 2008_000870 376 | 2008_000923 377 | 2008_001030 378 | 2008_001056 379 | 2008_001106 380 | 2008_001112 381 | 2008_001118 382 | 2008_001119 383 | 2008_001137 384 | 2008_001159 385 | 2008_001169 386 | 2008_001188 387 | 2008_001203 388 | 2008_001208 389 | 2008_001215 390 | 2008_001235 391 | 2008_001245 392 | 2008_001263 393 | 2008_001274 394 | 2008_001358 395 | 2008_001375 396 | 2008_001387 397 | 2008_001399 398 | 2008_001402 399 | 2008_001408 400 | 2008_001413 401 | 2008_001462 402 | 2008_001467 403 | 2008_001479 404 | 2008_001498 405 | 2008_001510 406 | 2008_001523 407 | 2008_001566 408 | 2008_001592 409 | 2008_001601 410 | 2008_001610 411 | 2008_001632 412 | 2008_001643 413 | 2008_001691 414 | 2008_001716 415 | 2008_001719 416 | 2008_001741 417 | 2008_001761 418 | 2008_001787 419 | 2008_001829 420 | 2008_001876 421 | 2008_001882 422 | 2008_001896 423 | 2008_001926 424 | 2008_001997 425 | 2008_002032 426 | 2008_002064 427 | 2008_002066 428 | 2008_002067 429 | 2008_002073 430 | 2008_002079 431 | 2008_002080 432 | 2008_002123 433 | 2008_002160 434 | 2008_002175 435 | 2008_002177 436 | 2008_002182 437 | 2008_002200 438 | 2008_002210 439 | 2008_002215 440 | 2008_002218 441 | 2008_002221 442 | 2008_002247 443 | 2008_002248 444 | 2008_002255 445 | 2008_002258 446 | 2008_002288 447 | 2008_002338 448 | 2008_002411 449 | 2008_002425 450 | 2008_002471 451 | 2008_002473 452 | 2008_002551 453 | 2008_002641 454 | 2008_002650 455 | 2008_002697 456 | 2008_002704 457 | 2008_002710 458 | 2008_002719 459 | 2008_002749 460 | 2008_002762 461 | 2008_002772 462 | 2008_002834 463 | 2008_002868 464 | 2008_002885 465 | 2008_002894 466 | 2008_002960 467 | 2008_002970 468 | 2008_002972 469 | 2008_002993 470 | 2008_003060 471 | 2008_003065 472 | 2008_003068 473 | 2008_003083 474 | 2008_003087 475 | 2008_003094 476 | 2008_003101 477 | 2008_003168 478 | 2008_003180 479 | 2008_003196 480 | 2008_003200 481 | 2008_003208 482 | 2008_003252 483 | 2008_003329 484 | 2008_003362 485 | 2008_003373 486 | 2008_003381 487 | 2008_003415 488 | 2008_003429 489 | 2008_003480 490 | 2008_003500 491 | 2008_003523 492 | 2008_003562 493 | 2008_003585 494 | 2008_003665 495 | 2008_003691 496 | 2008_003701 497 | 2008_003703 498 | 2008_003729 499 | 2008_003769 500 | 2008_003774 501 | 2008_003779 502 | 2008_003814 503 | 2008_003913 504 | 2008_003939 505 | 2008_003947 506 | 2008_003986 507 | 2008_003998 508 | 2008_004014 509 | 2008_004026 510 | 2008_004055 511 | 2008_004080 512 | 2008_004097 513 | 2008_004112 514 | 2008_004259 515 | 2008_004321 516 | 2008_004358 517 | 2008_004365 518 | 2008_004416 519 | 2008_004430 520 | 2008_004441 521 | 2008_004547 522 | 2008_004551 523 | 2008_004583 524 | 2008_004588 525 | 2008_004607 526 | 2008_004663 527 | 2008_004750 528 | 2008_004776 529 | 2008_004822 530 | 2008_004838 531 | 2008_004841 532 | 2008_004869 533 | 2008_004892 534 | 2008_004911 535 | 2008_004914 536 | 2008_004946 537 | 2008_004983 538 | 2008_005006 539 | 2008_005074 540 | 2008_005196 541 | 2008_005214 542 | 2008_005231 543 | 2008_005266 544 | 2008_005294 545 | 2008_005300 546 | 2008_005321 547 | 2008_005342 548 | 2008_005345 549 | 2008_005367 550 | 2008_005375 551 | 2008_005512 552 | 2008_005541 553 | 2008_005600 554 | 2008_005650 555 | 2008_005668 556 | 2008_005678 557 | 2008_005679 558 | 2008_005698 559 | 2008_005706 560 | 2008_005713 561 | 2008_005714 562 | 2008_005716 563 | 2008_005747 564 | 2008_005770 565 | 2008_005839 566 | 2008_005843 567 | 2008_005845 568 | 2008_005874 569 | 2008_005926 570 | 2008_005938 571 | 2008_005945 572 | 2008_005953 573 | 2008_006032 574 | 2008_006065 575 | 2008_006070 576 | 2008_006140 577 | 2008_006182 578 | 2008_006213 579 | 2008_006215 580 | 2008_006221 581 | 2008_006289 582 | 2008_006339 583 | 2008_006345 584 | 2008_006349 585 | 2008_006353 586 | 2008_006389 587 | 2008_006434 588 | 2008_006481 589 | 2008_006482 590 | 2008_006490 591 | 2008_006509 592 | 2008_006558 593 | 2008_006655 594 | 2008_006748 595 | 2008_006751 596 | 2008_006843 597 | 2008_006873 598 | 2008_006877 599 | 2008_006908 600 | 2008_006920 601 | 2008_007011 602 | 2008_007012 603 | 2008_007090 604 | 2008_007142 605 | 2008_007165 606 | 2008_007201 607 | 2008_007239 608 | 2008_007242 609 | 2008_007245 610 | 2008_007313 611 | 2008_007355 612 | 2008_007357 613 | 2008_007375 614 | 2008_007428 615 | 2008_007433 616 | 2008_007472 617 | 2008_007581 618 | 2008_007691 619 | 2008_007759 620 | 2008_007858 621 | 2008_007998 622 | 2008_008106 623 | 2008_008193 624 | 2008_008263 625 | 2008_008323 626 | 2008_008324 627 | 2008_008343 628 | 2008_008462 629 | 2008_008476 630 | 2008_008511 631 | 2008_008521 632 | 2008_008525 633 | 2008_008541 634 | 2008_008545 635 | 2008_008550 636 | 2008_008770 637 | 2008_008773 638 | 2009_000006 639 | 2009_000015 640 | 2009_000028 641 | 2009_000029 642 | 2009_000073 643 | 2009_000100 644 | 2009_000103 645 | 2009_000133 646 | 2009_000161 647 | 2009_000176 648 | 2009_000177 649 | 2009_000250 650 | 2009_000285 651 | 2009_000347 652 | 2009_000385 653 | 2009_000400 654 | 2009_000405 655 | 2009_000408 656 | 2009_000409 657 | 2009_000420 658 | 2009_000444 659 | 2009_000454 660 | 2009_000503 661 | 2009_000505 662 | 2009_000532 663 | 2009_000535 664 | 2009_000544 665 | 2009_000553 666 | 2009_000562 667 | 2009_000603 668 | 2009_000626 669 | 2009_000635 670 | 2009_000655 671 | 2009_000662 672 | 2009_000684 673 | 2009_000690 674 | 2009_000709 675 | 2009_000720 676 | 2009_000744 677 | 2009_000746 678 | 2009_000774 679 | 2009_000801 680 | 2009_000887 681 | 2009_000894 682 | 2009_000895 683 | 2009_000906 684 | 2009_000938 685 | 2009_000987 686 | 2009_000996 687 | 2009_001002 688 | 2009_001019 689 | 2009_001027 690 | 2009_001036 691 | 2009_001070 692 | 2009_001085 693 | 2009_001095 694 | 2009_001096 695 | 2009_001100 696 | 2009_001104 697 | 2009_001117 698 | 2009_001124 699 | 2009_001137 700 | 2009_001140 701 | 2009_001145 702 | 2009_001146 703 | 2009_001163 704 | 2009_001177 705 | 2009_001197 706 | 2009_001203 707 | 2009_001205 708 | 2009_001251 709 | 2009_001253 710 | 2009_001264 711 | 2009_001268 712 | 2009_001270 713 | 2009_001283 714 | 2009_001306 715 | 2009_001311 716 | 2009_001339 717 | 2009_001359 718 | 2009_001385 719 | 2009_001388 720 | 2009_001390 721 | 2009_001403 722 | 2009_001422 723 | 2009_001443 724 | 2009_001444 725 | 2009_001481 726 | 2009_001502 727 | 2009_001514 728 | 2009_001516 729 | 2009_001544 730 | 2009_001615 731 | 2009_001625 732 | 2009_001636 733 | 2009_001640 734 | 2009_001651 735 | 2009_001664 736 | 2009_001690 737 | 2009_001693 738 | 2009_001724 739 | 2009_001735 740 | 2009_001744 741 | 2009_001755 742 | 2009_001782 743 | 2009_001783 744 | 2009_001802 745 | 2009_001828 746 | 2009_001868 747 | 2009_001871 748 | 2009_001885 749 | 2009_001888 750 | 2009_001894 751 | 2009_001898 752 | 2009_001922 753 | 2009_001937 754 | 2009_001961 755 | 2009_001964 756 | 2009_001972 757 | 2009_002010 758 | 2009_002019 759 | 2009_002052 760 | 2009_002060 761 | 2009_002072 762 | 2009_002083 763 | 2009_002117 764 | 2009_002153 765 | 2009_002204 766 | 2009_002216 767 | 2009_002229 768 | 2009_002245 769 | 2009_002262 770 | 2009_002264 771 | 2009_002281 772 | 2009_002285 773 | 2009_002314 774 | 2009_002343 775 | 2009_002362 776 | 2009_002387 777 | 2009_002409 778 | 2009_002416 779 | 2009_002419 780 | 2009_002422 781 | 2009_002423 782 | 2009_002425 783 | 2009_002448 784 | 2009_002460 785 | 2009_002472 786 | 2009_002519 787 | 2009_002530 788 | 2009_002543 789 | 2009_002567 790 | 2009_002586 791 | 2009_002588 792 | 2009_002599 793 | 2009_002613 794 | 2009_002626 795 | 2009_002628 796 | 2009_002662 797 | 2009_002674 798 | 2009_002713 799 | 2009_002715 800 | 2009_002734 801 | 2009_002763 802 | 2009_002789 803 | 2009_002820 804 | 2009_002844 805 | 2009_002845 806 | 2009_002849 807 | 2009_002862 808 | 2009_002872 809 | 2009_002885 810 | 2009_002897 811 | 2009_002912 812 | 2009_002914 813 | 2009_002917 814 | 2009_002932 815 | 2009_002972 816 | 2009_002984 817 | 2009_002988 818 | 2009_002993 819 | 2009_003006 820 | 2009_003007 821 | 2009_003012 822 | 2009_003034 823 | 2009_003035 824 | 2009_003039 825 | 2009_003053 826 | 2009_003054 827 | 2009_003075 828 | 2009_003087 829 | 2009_003088 830 | 2009_003090 831 | 2009_003142 832 | 2009_003146 833 | 2009_003147 834 | 2009_003164 835 | 2009_003172 836 | 2009_003200 837 | 2009_003249 838 | 2009_003317 839 | 2009_003340 840 | 2009_003345 841 | 2009_003353 842 | 2009_003361 843 | 2009_003369 844 | 2009_003455 845 | 2009_003461 846 | 2009_003468 847 | 2009_003497 848 | 2009_003519 849 | 2009_003522 850 | 2009_003539 851 | 2009_003555 852 | 2009_003613 853 | 2009_003636 854 | 2009_003646 855 | 2009_003660 856 | 2009_003690 857 | 2009_003697 858 | 2009_003711 859 | 2009_003734 860 | 2009_003736 861 | 2009_003757 862 | 2009_003768 863 | 2009_003783 864 | 2009_003799 865 | 2009_003815 866 | 2009_003820 867 | 2009_003825 868 | 2009_003860 869 | 2009_003865 870 | 2009_003921 871 | 2009_003922 872 | 2009_003933 873 | 2009_003961 874 | 2009_003975 875 | 2009_004091 876 | 2009_004095 877 | 2009_004105 878 | 2009_004117 879 | 2009_004171 880 | 2009_004178 881 | 2009_004180 882 | 2009_004186 883 | 2009_004191 884 | 2009_004212 885 | 2009_004213 886 | 2009_004228 887 | 2009_004249 888 | 2009_004264 889 | 2009_004278 890 | 2009_004301 891 | 2009_004316 892 | 2009_004317 893 | 2009_004327 894 | 2009_004328 895 | 2009_004334 896 | 2009_004336 897 | 2009_004368 898 | 2009_004374 899 | 2009_004409 900 | 2009_004417 901 | 2009_004425 902 | 2009_004426 903 | 2009_004434 904 | 2009_004446 905 | 2009_004464 906 | 2009_004479 907 | 2009_004519 908 | 2009_004539 909 | 2009_004561 910 | 2009_004620 911 | 2009_004626 912 | 2009_004643 913 | 2009_004656 914 | 2009_004661 915 | 2009_004674 916 | 2009_004705 917 | 2009_004790 918 | 2009_004805 919 | 2009_004829 920 | 2009_004887 921 | 2009_004888 922 | 2009_004890 923 | 2009_004901 924 | 2009_004904 925 | 2009_004919 926 | 2009_004939 927 | 2009_004980 928 | 2009_004990 929 | 2009_005000 930 | 2009_005016 931 | 2009_005031 932 | 2009_005037 933 | 2009_005055 934 | 2009_005056 935 | 2009_005069 936 | 2009_005084 937 | 2009_005085 938 | 2009_005107 939 | 2009_005118 940 | 2009_005120 941 | 2009_005128 942 | 2009_005130 943 | 2009_005141 944 | 2009_005145 945 | 2009_005160 946 | 2009_005177 947 | 2009_005194 948 | 2009_005234 949 | 2009_005236 950 | 2009_005247 951 | 2009_005269 952 | 2009_005287 953 | 2010_000002 954 | 2010_000043 955 | 2010_000063 956 | 2010_000075 957 | 2010_000076 958 | 2010_000114 959 | 2010_000117 960 | 2010_000131 961 | 2010_000132 962 | 2010_000148 963 | 2010_000187 964 | 2010_000189 965 | 2010_000195 966 | 2010_000269 967 | 2010_000285 968 | 2010_000371 969 | 2010_000392 970 | 2010_000404 971 | 2010_000436 972 | 2010_000437 973 | 2010_000466 974 | 2010_000469 975 | 2010_000492 976 | 2010_000498 977 | 2010_000503 978 | 2010_000519 979 | 2010_000567 980 | 2010_000588 981 | 2010_000632 982 | 2010_000661 983 | 2010_000675 984 | 2010_000685 985 | 2010_000746 986 | 2010_000748 987 | 2010_000772 988 | 2010_000787 989 | 2010_000810 990 | 2010_000815 991 | 2010_000847 992 | 2010_000855 993 | 2010_000885 994 | 2010_000887 995 | 2010_000978 996 | 2010_000986 997 | 2010_001043 998 | 2010_001120 999 | 2010_001131 1000 | 2010_001154 1001 | 2010_001160 1002 | 2010_001177 1003 | 2010_001183 1004 | 2010_001184 1005 | 2010_001195 1006 | 2010_001245 1007 | 2010_001247 1008 | 2010_001261 1009 | 2010_001273 1010 | 2010_001279 1011 | 2010_001282 1012 | 2010_001329 1013 | 2010_001347 1014 | 2010_001374 1015 | 2010_001386 1016 | 2010_001399 1017 | 2010_001413 1018 | 2010_001418 1019 | 2010_001422 1020 | 2010_001457 1021 | 2010_001514 1022 | 2010_001515 1023 | 2010_001561 1024 | 2010_001562 1025 | 2010_001576 1026 | 2010_001590 1027 | 2010_001595 1028 | 2010_001618 1029 | 2010_001619 1030 | 2010_001630 1031 | 2010_001660 1032 | 2010_001676 1033 | 2010_001706 1034 | 2010_001732 1035 | 2010_001748 1036 | 2010_001807 1037 | 2010_001842 1038 | 2010_001849 1039 | 2010_001850 1040 | 2010_001852 1041 | 2010_001860 1042 | 2010_001922 1043 | 2010_001923 1044 | 2010_001933 1045 | 2010_001939 1046 | 2010_001944 1047 | 2010_002018 1048 | 2010_002020 1049 | 2010_002032 1050 | 2010_002039 1051 | 2010_002047 1052 | 2010_002054 1053 | 2010_002055 1054 | 2010_002070 1055 | 2010_002097 1056 | 2010_002107 1057 | 2010_002139 1058 | 2010_002154 1059 | 2010_002166 1060 | 2010_002203 1061 | 2010_002218 1062 | 2010_002236 1063 | 2010_002254 1064 | 2010_002286 1065 | 2010_002338 1066 | 2010_002363 1067 | 2010_002379 1068 | 2010_002382 1069 | 2010_002387 1070 | 2010_002413 1071 | 2010_002418 1072 | 2010_002440 1073 | 2010_002455 1074 | 2010_002457 1075 | 2010_002499 1076 | 2010_002527 1077 | 2010_002532 1078 | 2010_002551 1079 | 2010_002556 1080 | 2010_002570 1081 | 2010_002573 1082 | 2010_002625 1083 | 2010_002659 1084 | 2010_002697 1085 | 2010_002720 1086 | 2010_002733 1087 | 2010_002750 1088 | 2010_002778 1089 | 2010_002786 1090 | 2010_002794 1091 | 2010_002811 1092 | 2010_002815 1093 | 2010_002838 1094 | 2010_002856 1095 | 2010_002870 1096 | 2010_002892 1097 | 2010_002907 1098 | 2010_002935 1099 | 2010_002937 1100 | 2010_002938 1101 | 2010_002962 1102 | 2010_002973 1103 | 2010_003010 1104 | 2010_003017 1105 | 2010_003062 1106 | 2010_003088 1107 | 2010_003093 1108 | 2010_003097 1109 | 2010_003114 1110 | 2010_003119 1111 | 2010_003153 1112 | 2010_003157 1113 | 2010_003170 1114 | 2010_003174 1115 | 2010_003203 1116 | 2010_003230 1117 | 2010_003250 1118 | 2010_003252 1119 | 2010_003269 1120 | 2010_003274 1121 | 2010_003342 1122 | 2010_003345 1123 | 2010_003380 1124 | 2010_003383 1125 | 2010_003384 1126 | 2010_003529 1127 | 2010_003534 1128 | 2010_003599 1129 | 2010_003634 1130 | 2010_003651 1131 | 2010_003665 1132 | 2010_003670 1133 | 2010_003680 1134 | 2010_003696 1135 | 2010_003717 1136 | 2010_003737 1137 | 2010_003798 1138 | 2010_003799 1139 | 2010_003884 1140 | 2010_003887 1141 | 2010_003894 1142 | 2010_003899 1143 | 2010_003911 1144 | 2010_003925 1145 | 2010_003950 1146 | 2010_003954 1147 | 2010_003958 1148 | 2010_003974 1149 | 2010_004005 1150 | 2010_004025 1151 | 2010_004060 1152 | 2010_004069 1153 | 2010_004071 1154 | 2010_004072 1155 | 2010_004074 1156 | 2010_004109 1157 | 2010_004119 1158 | 2010_004144 1159 | 2010_004154 1160 | 2010_004171 1161 | 2010_004180 1162 | 2010_004186 1163 | 2010_004210 1164 | 2010_004222 1165 | 2010_004258 1166 | 2010_004283 1167 | 2010_004288 1168 | 2010_004289 1169 | 2010_004306 1170 | 2010_004361 1171 | 2010_004363 1172 | 2010_004365 1173 | 2010_004370 1174 | 2010_004429 1175 | 2010_004450 1176 | 2010_004478 1177 | 2010_004481 1178 | 2010_004493 1179 | 2010_004499 1180 | 2010_004540 1181 | 2010_004560 1182 | 2010_004577 1183 | 2010_004598 1184 | 2010_004616 1185 | 2010_004620 1186 | 2010_004625 1187 | 2010_004669 1188 | 2010_004683 1189 | 2010_004694 1190 | 2010_004704 1191 | 2010_004721 1192 | 2010_004760 1193 | 2010_004766 1194 | 2010_004773 1195 | 2010_004805 1196 | 2010_004808 1197 | 2010_004900 1198 | 2010_004916 1199 | 2010_004933 1200 | 2010_004938 1201 | 2010_004948 1202 | 2010_004960 1203 | 2010_004963 1204 | 2010_005016 1205 | 2010_005028 1206 | 2010_005055 1207 | 2010_005064 1208 | 2010_005098 1209 | 2010_005106 1210 | 2010_005111 1211 | 2010_005119 1212 | 2010_005128 1213 | 2010_005129 1214 | 2010_005198 1215 | 2010_005202 1216 | 2010_005217 1217 | 2010_005223 1218 | 2010_005232 1219 | 2010_005277 1220 | 2010_005317 1221 | 2010_005318 1222 | 2010_005419 1223 | 2010_005429 1224 | 2010_005450 1225 | 2010_005457 1226 | 2010_005468 1227 | 2010_005471 1228 | 2010_005494 1229 | 2010_005500 1230 | 2010_005505 1231 | 2010_005506 1232 | 2010_005513 1233 | 2010_005519 1234 | 2010_005522 1235 | 2010_005596 1236 | 2010_005627 1237 | 2010_005643 1238 | 2010_005652 1239 | 2010_005663 1240 | 2010_005669 1241 | 2010_005678 1242 | 2010_005700 1243 | 2010_005721 1244 | 2010_005723 1245 | 2010_005725 1246 | 2010_005734 1247 | 2010_005744 1248 | 2010_005746 1249 | 2010_005755 1250 | 2010_005758 1251 | 2010_005775 1252 | 2010_005791 1253 | 2010_005796 1254 | 2010_005800 1255 | 2010_005805 1256 | 2010_005810 1257 | 2010_005820 1258 | 2010_005830 1259 | 2010_005835 1260 | 2010_005836 1261 | 2010_005876 1262 | 2010_005891 1263 | 2010_005898 1264 | 2010_005919 1265 | 2010_005927 1266 | 2010_005932 1267 | 2010_005951 1268 | 2010_005952 1269 | 2010_005978 1270 | 2010_005982 1271 | 2010_006009 1272 | 2011_000003 1273 | 2011_000006 1274 | 2011_000025 1275 | 2011_000027 1276 | 2011_000068 1277 | 2011_000069 1278 | 2011_000105 1279 | 2011_000108 1280 | 2011_000116 1281 | 2011_000122 1282 | 2011_000145 1283 | 2011_000149 1284 | 2011_000152 1285 | 2011_000182 1286 | 2011_000197 1287 | 2011_000208 1288 | 2011_000216 1289 | 2011_000219 1290 | 2011_000221 1291 | 2011_000222 1292 | 2011_000228 1293 | 2011_000243 1294 | 2011_000252 1295 | 2011_000258 1296 | 2011_000268 1297 | 2011_000277 1298 | 2011_000278 1299 | 2011_000293 1300 | 2011_000345 1301 | 2011_000359 1302 | 2011_000379 1303 | 2011_000382 1304 | 2011_000400 1305 | 2011_000428 1306 | 2011_000449 1307 | 2011_000453 1308 | 2011_000457 1309 | 2011_000468 1310 | 2011_000469 1311 | 2011_000513 1312 | 2011_000542 1313 | 2011_000550 1314 | 2011_000551 1315 | 2011_000556 1316 | 2011_000573 1317 | 2011_000577 1318 | 2011_000589 1319 | 2011_000594 1320 | 2011_000637 1321 | 2011_000641 1322 | 2011_000642 1323 | 2011_000646 1324 | 2011_000651 1325 | 2011_000652 1326 | 2011_000713 1327 | 2011_000758 1328 | 2011_000768 1329 | 2011_000771 1330 | 2011_000790 1331 | 2011_000793 1332 | 2011_000834 1333 | 2011_000840 1334 | 2011_000882 1335 | 2011_000893 1336 | 2011_000895 1337 | 2011_000920 1338 | 2011_000934 1339 | 2011_000944 1340 | 2011_000973 1341 | 2011_000982 1342 | 2011_000997 1343 | 2011_000999 1344 | 2011_001004 1345 | 2011_001015 1346 | 2011_001027 1347 | 2011_001133 1348 | 2011_001135 1349 | 2011_001139 1350 | 2011_001166 1351 | 2011_001175 1352 | 2011_001198 1353 | 2011_001211 1354 | 2011_001259 1355 | 2011_001270 1356 | 2011_001336 1357 | 2011_001400 1358 | 2011_001402 1359 | 2011_001411 1360 | 2011_001412 1361 | 2011_001432 1362 | 2011_001463 1363 | 2011_001475 1364 | 2011_001479 1365 | 2011_001519 1366 | 2011_001536 1367 | 2011_001542 1368 | 2011_001571 1369 | 2011_001621 1370 | 2011_001622 1371 | 2011_001632 1372 | 2011_001652 1373 | 2011_001653 1374 | 2011_001695 1375 | 2011_001710 1376 | 2011_001730 1377 | 2011_001753 1378 | 2011_001754 1379 | 2011_001764 1380 | 2011_001765 1381 | 2011_001790 1382 | 2011_001810 1383 | 2011_001855 1384 | 2011_001866 1385 | 2011_001875 1386 | 2011_001895 1387 | 2011_001902 1388 | 2011_001904 1389 | 2011_001922 1390 | 2011_001924 1391 | 2011_001928 1392 | 2011_001959 1393 | 2011_001967 1394 | 2011_001972 1395 | 2011_001974 1396 | 2011_001991 1397 | 2011_002027 1398 | 2011_002050 1399 | 2011_002107 1400 | 2011_002111 1401 | 2011_002114 1402 | 2011_002119 1403 | 2011_002134 1404 | 2011_002135 1405 | 2011_002149 1406 | 2011_002222 1407 | 2011_002224 1408 | 2011_002227 1409 | 2011_002246 1410 | 2011_002291 1411 | 2011_002300 1412 | 2011_002303 1413 | 2011_002335 1414 | 2011_002341 1415 | 2011_002350 1416 | 2011_002381 1417 | 2011_002385 1418 | 2011_002389 1419 | 2011_002398 1420 | 2011_002410 1421 | 2011_002447 1422 | 2011_002457 1423 | 2011_002464 1424 | 2011_002488 1425 | 2011_002503 1426 | 2011_002504 1427 | 2011_002511 1428 | 2011_002528 1429 | 2011_002553 1430 | 2011_002559 1431 | 2011_002561 1432 | 2011_002585 1433 | 2011_002590 1434 | 2011_002652 1435 | 2011_002656 1436 | 2011_002709 1437 | 2011_002715 1438 | 2011_002717 1439 | 2011_002752 1440 | 2011_002767 1441 | 2011_002770 1442 | 2011_002834 1443 | 2011_002851 1444 | 2011_002872 1445 | 2011_002873 1446 | 2011_002920 1447 | 2011_002932 1448 | 2011_002935 1449 | 2011_002947 1450 | 2011_002953 1451 | 2011_002956 1452 | 2011_003025 1453 | 2011_003038 1454 | 2011_003057 1455 | 2011_003066 1456 | 2011_003078 1457 | 2011_003121 1458 | 2011_003141 1459 | 2011_003151 1460 | 2011_003184 1461 | 2011_003216 1462 | 2011_003238 1463 | 2011_003246 1464 | 2011_003255 1465 | --------------------------------------------------------------------------------