├── pictures └── cover.jpg ├── README.md ├── env.yml └── experiments ├── vgg16_part_1.ipynb ├── resnet34_part_2.ipynb └── resnet34_part_1.ipynb /pictures/cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fg91/visualizing-cnn-feature-maps/HEAD/pictures/cover.jpg -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](pictures/cover.jpg) 2 | # Visualizing convolutional features using PyTorch 3 | 4 | ### Take a look at my [blog post](https://medium.com/p/70b7d87b0030) for detailed explanations. 5 | 6 | ## Getting Started 7 | 8 | Create a conda environment with the required dependencies in order to run the notebooks on your computer. 9 | 10 | Install anaconda and then run: 11 | ``` 12 | conda install nb_conda_kernels 13 | conda env create -f env.yml 14 | ipython kernel install --user --name=filter_visualizer_env 15 | ``` 16 | -------------------------------------------------------------------------------- /env.yml: -------------------------------------------------------------------------------- 1 | name: filter_visualizer_env 2 | channels: 3 | - pytorch 4 | - soumith 5 | - defaults 6 | dependencies: 7 | - ipykernel 8 | - backcall=0.1.0 9 | - blas=1.0 10 | - bleach=3.0.2 11 | - ca-certificates=2018.03.07 12 | - certifi=2018.10.15 13 | - cffi=1.11.5 14 | - click=7.0 15 | - cudatoolkit=8.0 16 | - decorator=4.3.0 17 | - entrypoints=0.2.3 18 | - flask=1.0.2 19 | - gmp=6.1.2 20 | - intel-openmp=2019.1 21 | - ipykernel=5.1.0 22 | - ipython=7.2.0 23 | - ipython_genutils=0.2.0 24 | - itsdangerous=1.1.0 25 | - jedi=0.13.1 26 | - jinja2=2.10 27 | - jsonschema=2.6.0 28 | - jupyter_client=5.2.3 29 | - jupyter_core=4.4.0 30 | - libedit=3.1.20170329 31 | - libffi=3.2.1 32 | - libgcc-ng=8.2.0 33 | - libgfortran-ng=7.3.0 34 | - libsodium=1.0.16 35 | - libstdcxx-ng=8.2.0 36 | - markupsafe=1.1.0 37 | - mistune=0.8.4 38 | - mkl=2018.0.3 39 | - mkl_fft=1.0.6 40 | - mkl_random=1.0.1 41 | - nb_conda=2.2.1 42 | - nb_conda_kernels=2.2.0 43 | - nbconvert=5.3.1 44 | - nbformat=4.4.0 45 | - ncurses=6.1 46 | - ninja=1.8.2 47 | - notebook=5.7.2 48 | - numpy=1.15.4 49 | - numpy-base=1.15.4 50 | - openssl=1.1.1a 51 | - pandoc=2.2.3.2 52 | - pandocfilters=1.4.2 53 | - parso=0.3.1 54 | - pexpect=4.6.0 55 | - pickleshare=0.7.5 56 | - pip=18.1 57 | - prometheus_client=0.4.2 58 | - prompt_toolkit=2.0.7 59 | - ptyprocess=0.6.0 60 | - pycparser=2.19 61 | - pygments=2.2.0 62 | - python=3.6.7 63 | - python-dateutil=2.7.5 64 | - pyzmq=17.1.2 65 | - readline=7.0 66 | - send2trash=1.5.0 67 | - setuptools=40.6.2 68 | - six=1.11.0 69 | - sqlite=3.25.3 70 | - terminado=0.8.1 71 | - testpath=0.4.2 72 | - tk=8.6.8 73 | - tornado=5.1.1 74 | - traitlets=4.3.2 75 | - wcwidth=0.1.7 76 | - webencodings=0.5.1 77 | - werkzeug=0.14.1 78 | - wheel=0.32.3 79 | - xz=5.2.4 80 | - zeromq=4.2.5 81 | - zlib=1.2.11 82 | - cuda80=1.0 83 | - pytorch=0.4.1 84 | - pip: 85 | - aniso8601==6.0.0 86 | - annoy==1.14.0 87 | - awscli==1.16.112 88 | - bcolz==1.2.1 89 | - chardet==3.0.4 90 | - cycler==0.10.0 91 | - descartes==1.1.0 92 | - fastai==0.7.0 93 | - feather-format==0.4.0 94 | - flasgger==0.9.2 95 | - flask-restplus==0.12.1 96 | - graphviz==0.10.1 97 | - html5lib==1.0.1 98 | - idna==2.7 99 | - ipywidgets==7.4.2 100 | - isoweek==1.3.3 101 | - jupyter==1.0.0 102 | - jupyter-console==6.0.0 103 | - kiwisolver==1.0.1 104 | - matplotlib==3.0.2 105 | - mizani==0.5.2 106 | - nmslib==1.7.3.6 107 | - opencv-python==3.4.4.19 108 | - palettable==3.1.1 109 | - pandas==0.23.4 110 | - pandas-summary==0.0.5 111 | - patsy==0.5.1 112 | - pillow==5.3.0 113 | - plotnine==0.5.1 114 | - pyarrow==0.11.1 115 | - pyasn1==0.4.5 116 | - pybind11==2.2.4 117 | - pyparsing==2.3.0 118 | - pytz==2018.7 119 | - pyyaml==3.13 120 | - qtconsole==4.4.3 121 | - requests==2.20.1 122 | - rsa==3.4.2 123 | - scikit-learn==0.20.1 124 | - scipy==1.1.0 125 | - seaborn==0.9.0 126 | - simplegeneric==0.8.1 127 | - sklearn-pandas==1.7.0 128 | - statsmodels==0.9.0 129 | - torch==0.4.1 130 | - torchtext==0.2.3 131 | - torchvision==0.2.1 132 | - tqdm==4.28.1 133 | - urllib3==1.24.1 134 | - widgetsnbextension==3.4.2 135 | 136 | -------------------------------------------------------------------------------- /experiments/vgg16_part_1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "feat_viz_vgg16.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [] 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "accelerator": "GPU" 16 | }, 17 | "cells": [ 18 | { 19 | "metadata": { 20 | "id": "I9VDrPCrR1__", 21 | "colab_type": "text" 22 | }, 23 | "cell_type": "markdown", 24 | "source": [ 25 | "# Visualizing inputs that maximally activate feature maps of a convnet\n", 26 | "\n", 27 | "## Network used: VGG16" 28 | ] 29 | }, 30 | { 31 | "metadata": { 32 | "id": "39Ch5JJ-lJSI", 33 | "colab_type": "code", 34 | "colab": {} 35 | }, 36 | "cell_type": "code", 37 | "source": [ 38 | "%matplotlib inline\n", 39 | "%reload_ext autoreload\n", 40 | "%autoreload 2" 41 | ], 42 | "execution_count": 0, 43 | "outputs": [] 44 | }, 45 | { 46 | "metadata": { 47 | "id": "M8ylYU0BO9QP", 48 | "colab_type": "code", 49 | "colab": {} 50 | }, 51 | "cell_type": "code", 52 | "source": [ 53 | "!pip install -q fastai==0.7.0 torchtext==0.2.3" 54 | ], 55 | "execution_count": 0, 56 | "outputs": [] 57 | }, 58 | { 59 | "metadata": { 60 | "id": "Q1w2HnvvlP2z", 61 | "colab_type": "code", 62 | "colab": {} 63 | }, 64 | "cell_type": "code", 65 | "source": [ 66 | "from fastai.conv_learner import *\n", 67 | "from cv2 import resize\n", 68 | "import matplotlib.gridspec as gridspec\n", 69 | "from math import ceil\n", 70 | "from IPython.display import HTML\n", 71 | "from pdb import set_trace\n", 72 | "# from scipy import ndimage" 73 | ], 74 | "execution_count": 0, 75 | "outputs": [] 76 | }, 77 | { 78 | "metadata": { 79 | "colab_type": "code", 80 | "id": "UQ2d_EVYBODD", 81 | "colab": {} 82 | }, 83 | "cell_type": "code", 84 | "source": [ 85 | "class SaveFeatures():\n", 86 | " def __init__(self, module):\n", 87 | " self.hook = module.register_forward_hook(self.hook_fn)\n", 88 | " def hook_fn(self, module, input, output):\n", 89 | " self.features = output\n", 90 | " def close(self):\n", 91 | " self.hook.remove()" 92 | ], 93 | "execution_count": 0, 94 | "outputs": [] 95 | }, 96 | { 97 | "metadata": { 98 | "id": "OFxPmIboP5c_", 99 | "colab_type": "code", 100 | "colab": {} 101 | }, 102 | "cell_type": "code", 103 | "source": [ 104 | "class FilterVisualizer():\n", 105 | " def __init__(self):\n", 106 | " self.model = vgg16(True).cuda().eval()\n", 107 | " set_trainable(self.model, False)\n", 108 | "\n", 109 | " def visualize(self, sz, layer, filter, upscaling_steps=12, upscaling_factor=1.2, lr=0.1, opt_steps=20, blur=None, save=False, print_losses=False):\n", 110 | "\n", 111 | " img = (np.random.random((sz, sz, 3)) * 20 + 128.)/255.\n", 112 | "# img = np.random.uniform(0, 1, size=(sz, sz, 3)).astype(np.float32)\n", 113 | "# median_filter_size = 4 if sz < 100 else 8\n", 114 | "# img = scipy.ndimage.filters.median_filter(img, [median_filter_size,median_filter_size,1])\n", 115 | " \n", 116 | " layer = children(self.model)[layer]\n", 117 | " activations = SaveFeatures(layer) # register hook\n", 118 | "\n", 119 | " for i in range(upscaling_steps): # scale the image up upscaling_steps times\n", 120 | " train_tfms, val_tfms = tfms_from_model(vgg16, sz)\n", 121 | " img_var = V(val_tfms(img)[None], requires_grad=True) # convert image to Variable that requires grad\n", 122 | " optimizer = torch.optim.Adam([img_var], lr=lr, weight_decay=1e-6)\n", 123 | " if i > upscaling_steps/2:\n", 124 | " opt_steps_ = int(opt_steps*1.3)\n", 125 | " else:\n", 126 | " opt_steps_ = opt_steps\n", 127 | " for n in range(opt_steps_): # optimize pixel values for opt_steps times\n", 128 | " optimizer.zero_grad()\n", 129 | " self.model(img_var)\n", 130 | " loss = -activations.features[0, filter].mean()\n", 131 | " if print_losses:\n", 132 | " if i%3==0 and n%5==0:\n", 133 | " print(f'{i} - {n} - {float(loss)}')\n", 134 | " loss.backward()\n", 135 | " optimizer.step()\n", 136 | " img = val_tfms.denorm(np.rollaxis(to_np(img_var.data),1,4))[0]\n", 137 | " self.output = img\n", 138 | " sz = int(upscaling_factor * sz) # calculate new image size\n", 139 | " img = cv2.resize(img, (sz, sz), interpolation = cv2.INTER_CUBIC) # scale image up\n", 140 | " if blur is not None: img = cv2.blur(img,(blur,blur)) # blur image to reduce high frequency patterns\n", 141 | " if save:\n", 142 | " self.save(layer, filter)\n", 143 | " activations.close()\n", 144 | " return np.clip(self.output, 0, 1)\n", 145 | " \n", 146 | " def most_activated(self, image, layer, limit_top=None):\n", 147 | "\n", 148 | " train_tfms, val_tfms = tfms_from_model(vgg16, 224)\n", 149 | " transformed = val_tfms(image)\n", 150 | "# set_trace()\n", 151 | " layer = children(self.model)[layer]\n", 152 | " activations = SaveFeatures(layer) # register hook\n", 153 | " self.model(V(transformed)[None]);\n", 154 | " # set_trace()\n", 155 | " print(activations.features.shape)\n", 156 | " # set_trace()\n", 157 | " mean_act = [activations.features[0,i].mean().data.cpu().numpy()[0] for i in range(activations.features.shape[1])]\n", 158 | " activations.close()\n", 159 | " return mean_act\n", 160 | " \n", 161 | " def get_transformed_img(self,img,sz):\n", 162 | " train_tfms, val_tfms = tfms_from_model(resnet34, sz)\n", 163 | " return val_tfms.denorm(np.rollaxis(to_np(val_tfms(img)[None]),1,4))[0]" 164 | ], 165 | "execution_count": 0, 166 | "outputs": [] 167 | }, 168 | { 169 | "metadata": { 170 | "id": "iuvrfyNyCbAZ", 171 | "colab_type": "code", 172 | "colab": { 173 | "base_uri": "https://localhost:8080/", 174 | "height": 50 175 | }, 176 | "outputId": "d2b15bb7-512b-4fe7-cd00-e32b28a728b6" 177 | }, 178 | "cell_type": "code", 179 | "source": [ 180 | "FV = FilterVisualizer()" 181 | ], 182 | "execution_count": 21, 183 | "outputs": [ 184 | { 185 | "output_type": "stream", 186 | "text": [ 187 | "Downloading: \"https://download.pytorch.org/models/vgg16_bn-6c64b313.pth\" to /root/.torch/models/vgg16_bn-6c64b313.pth\n", 188 | "100%|██████████| 553507836/553507836 [00:05<00:00, 101382015.63it/s]\n" 189 | ], 190 | "name": "stderr" 191 | } 192 | ] 193 | }, 194 | { 195 | "metadata": { 196 | "id": "p2y6QAAHTbZQ", 197 | "colab_type": "code", 198 | "outputId": "21c31455-b987-461c-9f39-28cf589da8f1", 199 | "colab": { 200 | "base_uri": "https://localhost:8080/", 201 | "height": 790 202 | } 203 | }, 204 | "cell_type": "code", 205 | "source": [ 206 | "FV.model" 207 | ], 208 | "execution_count": 0, 209 | "outputs": [ 210 | { 211 | "output_type": "execute_result", 212 | "data": { 213 | "text/plain": [ 214 | "Sequential(\n", 215 | " (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 216 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 217 | " (2): ReLU(inplace)\n", 218 | " (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 219 | " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 220 | " (5): ReLU(inplace)\n", 221 | " (6): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n", 222 | " (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 223 | " (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 224 | " (9): ReLU(inplace)\n", 225 | " (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 226 | " (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 227 | " (12): ReLU(inplace)\n", 228 | " (13): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n", 229 | " (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 230 | " (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 231 | " (16): ReLU(inplace)\n", 232 | " (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 233 | " (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 234 | " (19): ReLU(inplace)\n", 235 | " (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 236 | " (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 237 | " (22): ReLU(inplace)\n", 238 | " (23): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n", 239 | " (24): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 240 | " (25): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 241 | " (26): ReLU(inplace)\n", 242 | " (27): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 243 | " (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 244 | " (29): ReLU(inplace)\n", 245 | " (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 246 | " (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 247 | " (32): ReLU(inplace)\n", 248 | " (33): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n", 249 | " (34): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 250 | " (35): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 251 | " (36): ReLU(inplace)\n", 252 | " (37): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 253 | " (38): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 254 | " (39): ReLU(inplace)\n", 255 | " (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", 256 | " (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 257 | " (42): ReLU(inplace)\n", 258 | " (43): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n", 259 | ")" 260 | ] 261 | }, 262 | "metadata": { 263 | "tags": [] 264 | }, 265 | "execution_count": 89 266 | } 267 | ] 268 | }, 269 | { 270 | "metadata": { 271 | "id": "3ySC7IQnhbIh", 272 | "colab_type": "code", 273 | "colab": {} 274 | }, 275 | "cell_type": "code", 276 | "source": [ 277 | "import requests\n", 278 | "import base64\n", 279 | "import pprint" 280 | ], 281 | "execution_count": 0, 282 | "outputs": [] 283 | }, 284 | { 285 | "metadata": { 286 | "id": "uxsI1NBhDTg-", 287 | "colab_type": "code", 288 | "colab": {} 289 | }, 290 | "cell_type": "code", 291 | "source": [ 292 | "with open('img_at.txt') as f:\n", 293 | " ACCESS_TOKEN = f.read().strip()" 294 | ], 295 | "execution_count": 0, 296 | "outputs": [] 297 | }, 298 | { 299 | "metadata": { 300 | "id": "f25YxvxEhfn2", 301 | "colab_type": "code", 302 | "colab": {} 303 | }, 304 | "cell_type": "code", 305 | "source": [ 306 | "def upload_to_imgur(file_name,post_title,album_hash):\n", 307 | " url = 'https://api.imgur.com/3/image'\n", 308 | " fh = open(file_name, 'rb');\n", 309 | " payload = {'image': base64.b64encode(fh.read()),\n", 310 | " 'album':album_hash,\n", 311 | " 'type':'base64',\n", 312 | " 'title':post_title,\n", 313 | " 'looping':False\n", 314 | " }\n", 315 | " files = {}\n", 316 | " headers = {\n", 317 | " 'Authorization': f'Bearer {ACCESS_TOKEN}'\n", 318 | " }\n", 319 | " response = requests.request('POST', url, headers = headers, data = payload, files = files, allow_redirects=False)\n", 320 | "# print(response.json()['data']['link'])\n", 321 | " return response.json()['data']['link']" 322 | ], 323 | "execution_count": 0, 324 | "outputs": [] 325 | }, 326 | { 327 | "metadata": { 328 | "id": "IuIGDfKgQS1W", 329 | "colab_type": "code", 330 | "colab": {} 331 | }, 332 | "cell_type": "code", 333 | "source": [ 334 | "def plot_reconstructions(imgs,layer_idx,filters,n_cols=3, cell_size=4, save_fig=False,album_hash=None):\n", 335 | " n_rows = ceil((len(imgs))/n_cols)\n", 336 | "\n", 337 | " fig,axes = plt.subplots(n_rows,n_cols, figsize=(cell_size*n_cols,cell_size*n_rows))\n", 338 | " \n", 339 | " for i,ax in enumerate(axes.flat):\n", 340 | " ax.grid(False)\n", 341 | " ax.get_xaxis().set_visible(False)\n", 342 | " ax.get_yaxis().set_visible(False)\n", 343 | "\n", 344 | " if i>=len(filters):\n", 345 | " pass\n", 346 | "\n", 347 | " ax.set_title(f'fmap {filters[i]}')\n", 348 | "\n", 349 | " ax.imshow(imgs[i])\n", 350 | " fig.suptitle(f'VGG16 Layer: {layer_idx} : {children(FV.model)[layer_idx]}', fontsize=\"x-large\",y=1.0)\n", 351 | " plt.tight_layout()\n", 352 | " plt.subplots_adjust(top=0.9)\n", 353 | " if save_fig:\n", 354 | " plt.savefig(f'vgg16_layer_{layer_idx}_fmaps_{\"_\".join([str(f) for f in filters])}.png')\n", 355 | " link = upload_to_imgur(f'vgg16_layer_{layer_idx}_fmaps_{\"_\".join([str(f) for f in filters])}.png',\n", 356 | " f'vgg16_layer_{layer_idx}_fmaps_{\"_\".join([str(f) for f in filters])}',album_hash)\n", 357 | " plt.close()\n", 358 | " return link\n", 359 | " else:\n", 360 | " plt.show()\n", 361 | " return None\n", 362 | " " 363 | ], 364 | "execution_count": 0, 365 | "outputs": [] 366 | }, 367 | { 368 | "metadata": { 369 | "id": "Zf0UXoS6QoJZ", 370 | "colab_type": "code", 371 | "colab": {} 372 | }, 373 | "cell_type": "code", 374 | "source": [ 375 | "def reconstructions(layer_idx, filters,\n", 376 | " init_size=56, upscaling_steps=12, \n", 377 | " upscaling_factor=1.2, \n", 378 | " opt_steps=20, blur=5,\n", 379 | " lr=1e-1,print_losses=False,\n", 380 | " n_cols=3, cell_size=4,\n", 381 | " save_fig=False,\n", 382 | " album_hash=None):\n", 383 | " \n", 384 | " if save_fig:\n", 385 | " assert not album_hash is None\n", 386 | " imgs = []\n", 387 | " for filter in filters:\n", 388 | " imgs.append(FV.visualize(init_size,layer_idx, filter, \n", 389 | " upscaling_steps=upscaling_steps, \n", 390 | " upscaling_factor=upscaling_factor, \n", 391 | " opt_steps=opt_steps, blur=blur,\n", 392 | " lr=lr,print_losses=print_losses))\n", 393 | " \n", 394 | " return plot_reconstructions(imgs,layer_idx,filters,\n", 395 | " n_cols=n_cols,cell_size=cell_size,\n", 396 | " save_fig=save_fig,album_hash=album_hash)" 397 | ], 398 | "execution_count": 0, 399 | "outputs": [] 400 | }, 401 | { 402 | "metadata": { 403 | "id": "wP83e_YlkNlH", 404 | "colab_type": "code", 405 | "colab": {} 406 | }, 407 | "cell_type": "code", 408 | "source": [ 409 | "# l_idxs = [40]\n", 410 | "# dd = 0\n", 411 | "# for l in l_idxs:\n", 412 | "# start = 198\n", 413 | "# n_items_per_plot = 6\n", 414 | "# end =400\n", 415 | "# count = start\n", 416 | "# while count+n_items_per_plot\"))" 441 | ], 442 | "execution_count": 58, 443 | "outputs": [ 444 | { 445 | "output_type": "display_data", 446 | "data": { 447 | "text/html": [ 448 | "" 449 | ], 450 | "text/plain": [ 451 | "" 452 | ] 453 | }, 454 | "metadata": { 455 | "tags": [] 456 | } 457 | } 458 | ] 459 | }, 460 | { 461 | "metadata": { 462 | "id": "5sOi0cVKBKBX", 463 | "colab_type": "code", 464 | "colab": { 465 | "base_uri": "https://localhost:8080/", 466 | "height": 597 467 | }, 468 | "outputId": "1150f526-e79b-4f86-9e41-3f52d7bfc2f9" 469 | }, 470 | "cell_type": "code", 471 | "source": [ 472 | "url = reconstructions(layer_idx=10,filters=list(range(24,30)),\n", 473 | " save_fig=True,album_hash=\"A5KpYOz\")\n", 474 | "display(HTML(f\"\"))" 475 | ], 476 | "execution_count": 60, 477 | "outputs": [ 478 | { 479 | "output_type": "display_data", 480 | "data": { 481 | "text/html": [ 482 | "" 483 | ], 484 | "text/plain": [ 485 | "" 486 | ] 487 | }, 488 | "metadata": { 489 | "tags": [] 490 | } 491 | } 492 | ] 493 | }, 494 | { 495 | "metadata": { 496 | "id": "hmLeKVy1BhQO", 497 | "colab_type": "code", 498 | "colab": { 499 | "base_uri": "https://localhost:8080/", 500 | "height": 597 501 | }, 502 | "outputId": "2aa38db8-1647-409e-c878-6553b8b42b3d" 503 | }, 504 | "cell_type": "code", 505 | "source": [ 506 | "url = reconstructions(layer_idx=14,filters=list(range(0,6)),\n", 507 | " save_fig=True,album_hash=\"A5KpYOz\")\n", 508 | "display(HTML(f\"\"))" 509 | ], 510 | "execution_count": 62, 511 | "outputs": [ 512 | { 513 | "output_type": "display_data", 514 | "data": { 515 | "text/html": [ 516 | "" 517 | ], 518 | "text/plain": [ 519 | "" 520 | ] 521 | }, 522 | "metadata": { 523 | "tags": [] 524 | } 525 | } 526 | ] 527 | }, 528 | { 529 | "metadata": { 530 | "id": "acRza-m7Btrn", 531 | "colab_type": "code", 532 | "colab": { 533 | "base_uri": "https://localhost:8080/", 534 | "height": 597 535 | }, 536 | "outputId": "1d6142f3-5140-4a55-be86-993d01b5347c" 537 | }, 538 | "cell_type": "code", 539 | "source": [ 540 | "url = reconstructions(layer_idx=14,filters=list(range(30,36)),\n", 541 | " save_fig=True,album_hash=\"A5KpYOz\")\n", 542 | "display(HTML(f\"\"))" 543 | ], 544 | "execution_count": 64, 545 | "outputs": [ 546 | { 547 | "output_type": "display_data", 548 | "data": { 549 | "text/html": [ 550 | "" 551 | ], 552 | "text/plain": [ 553 | "" 554 | ] 555 | }, 556 | "metadata": { 557 | "tags": [] 558 | } 559 | } 560 | ] 561 | }, 562 | { 563 | "metadata": { 564 | "id": "0gmUCPFhB7gg", 565 | "colab_type": "code", 566 | "colab": { 567 | "base_uri": "https://localhost:8080/", 568 | "height": 597 569 | }, 570 | "outputId": "bdb4f4ca-8c0c-4a2b-e884-524bda2c8f7d" 571 | }, 572 | "cell_type": "code", 573 | "source": [ 574 | "url = reconstructions(layer_idx=24,filters=list(range(66,72)),\n", 575 | " save_fig=True,album_hash=\"A5KpYOz\")\n", 576 | "display(HTML(f\"\"))" 577 | ], 578 | "execution_count": 66, 579 | "outputs": [ 580 | { 581 | "output_type": "display_data", 582 | "data": { 583 | "text/html": [ 584 | "" 585 | ], 586 | "text/plain": [ 587 | "" 588 | ] 589 | }, 590 | "metadata": { 591 | "tags": [] 592 | } 593 | } 594 | ] 595 | }, 596 | { 597 | "metadata": { 598 | "id": "UQ8RrZ1rCE2k", 599 | "colab_type": "code", 600 | "colab": { 601 | "base_uri": "https://localhost:8080/", 602 | "height": 597 603 | }, 604 | "outputId": "c0e15a10-1568-46de-8508-e196150007c7" 605 | }, 606 | "cell_type": "code", 607 | "source": [ 608 | "url = reconstructions(layer_idx=24,filters=list(range(84,90)),\n", 609 | " save_fig=True,album_hash=\"A5KpYOz\")\n", 610 | "display(HTML(f\"\"))" 611 | ], 612 | "execution_count": 68, 613 | "outputs": [ 614 | { 615 | "output_type": "display_data", 616 | "data": { 617 | "text/html": [ 618 | "" 619 | ], 620 | "text/plain": [ 621 | "" 622 | ] 623 | }, 624 | "metadata": { 625 | "tags": [] 626 | } 627 | } 628 | ] 629 | }, 630 | { 631 | "metadata": { 632 | "id": "0oRrqZXzCYHq", 633 | "colab_type": "code", 634 | "colab": { 635 | "base_uri": "https://localhost:8080/", 636 | "height": 597 637 | }, 638 | "outputId": "aba168c6-f1ae-4615-d233-dcb3a9a4ae6d" 639 | }, 640 | "cell_type": "code", 641 | "source": [ 642 | "url = reconstructions(layer_idx=24,filters=list(range(204,210)),\n", 643 | " save_fig=True,album_hash=\"SsbtjRZ\")\n", 644 | "display(HTML(f\"\"))" 645 | ], 646 | "execution_count": 70, 647 | "outputs": [ 648 | { 649 | "output_type": "display_data", 650 | "data": { 651 | "text/html": [ 652 | "" 653 | ], 654 | "text/plain": [ 655 | "" 656 | ] 657 | }, 658 | "metadata": { 659 | "tags": [] 660 | } 661 | } 662 | ] 663 | }, 664 | { 665 | "metadata": { 666 | "id": "Qmo-Ieq-CeOP", 667 | "colab_type": "text" 668 | }, 669 | "cell_type": "markdown", 670 | "source": [ 671 | "More images from earlier layers can be found [here](https://imgur.com/a/A5KpYOz)." 672 | ] 673 | }, 674 | { 675 | "metadata": { 676 | "id": "SszQ6oD9_eVy", 677 | "colab_type": "code", 678 | "colab": { 679 | "base_uri": "https://localhost:8080/", 680 | "height": 597 681 | }, 682 | "outputId": "e58db3ad-7901-40a8-9ba9-29da876c7ae1" 683 | }, 684 | "cell_type": "code", 685 | "source": [ 686 | "url = reconstructions(layer_idx=40,filters=list(range(0,6)),\n", 687 | " save_fig=True,album_hash=\"SsbtjRZ\")\n", 688 | "display(HTML(f\"\"))" 689 | ], 690 | "execution_count": 49, 691 | "outputs": [ 692 | { 693 | "output_type": "display_data", 694 | "data": { 695 | "text/html": [ 696 | "" 697 | ], 698 | "text/plain": [ 699 | "" 700 | ] 701 | }, 702 | "metadata": { 703 | "tags": [] 704 | } 705 | } 706 | ] 707 | }, 708 | { 709 | "metadata": { 710 | "id": "HQQnSD3a_uBx", 711 | "colab_type": "code", 712 | "colab": { 713 | "base_uri": "https://localhost:8080/", 714 | "height": 597 715 | }, 716 | "outputId": "d14ceb76-ef3e-4754-a481-7d98608ce088" 717 | }, 718 | "cell_type": "code", 719 | "source": [ 720 | "url = reconstructions(layer_idx=40,filters=list(range(6,12)),\n", 721 | " save_fig=True,album_hash=\"SsbtjRZ\")\n", 722 | "display(HTML(f\"\"))" 723 | ], 724 | "execution_count": 51, 725 | "outputs": [ 726 | { 727 | "output_type": "display_data", 728 | "data": { 729 | "text/html": [ 730 | "" 731 | ], 732 | "text/plain": [ 733 | "" 734 | ] 735 | }, 736 | "metadata": { 737 | "tags": [] 738 | } 739 | } 740 | ] 741 | }, 742 | { 743 | "metadata": { 744 | "id": "AP25aT2pAAJl", 745 | "colab_type": "code", 746 | "colab": { 747 | "base_uri": "https://localhost:8080/", 748 | "height": 597 749 | }, 750 | "outputId": "945bcae4-79bc-4d45-e7cc-8cfb28a34991" 751 | }, 752 | "cell_type": "code", 753 | "source": [ 754 | "url = reconstructions(layer_idx=40,filters=list(range(12,18)),\n", 755 | " save_fig=True,album_hash=\"SsbtjRZ\")\n", 756 | "display(HTML(f\"\"))" 757 | ], 758 | "execution_count": 55, 759 | "outputs": [ 760 | { 761 | "output_type": "display_data", 762 | "data": { 763 | "text/html": [ 764 | "" 765 | ], 766 | "text/plain": [ 767 | "" 768 | ] 769 | }, 770 | "metadata": { 771 | "tags": [] 772 | } 773 | } 774 | ] 775 | }, 776 | { 777 | "metadata": { 778 | "id": "FqrAw5YvAPEr", 779 | "colab_type": "text" 780 | }, 781 | "cell_type": "markdown", 782 | "source": [ 783 | "Results for all feature maps from layer `40` can be found [here](https://imgur.com/a/SsbtjRZ)." 784 | ] 785 | } 786 | ] 787 | } -------------------------------------------------------------------------------- /experiments/resnet34_part_2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "feat_viz_resnet34_part_2.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [] 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "accelerator": "GPU" 16 | }, 17 | "cells": [ 18 | { 19 | "metadata": { 20 | "id": "I9VDrPCrR1__", 21 | "colab_type": "text" 22 | }, 23 | "cell_type": "markdown", 24 | "source": [ 25 | "# Finding feature maps that are maximally activated by a specific image\n", 26 | "## Network used: ResNet-34" 27 | ] 28 | }, 29 | { 30 | "metadata": { 31 | "id": "39Ch5JJ-lJSI", 32 | "colab_type": "code", 33 | "colab": {} 34 | }, 35 | "cell_type": "code", 36 | "source": [ 37 | "%matplotlib inline\n", 38 | "%reload_ext autoreload\n", 39 | "%autoreload 2" 40 | ], 41 | "execution_count": 0, 42 | "outputs": [] 43 | }, 44 | { 45 | "metadata": { 46 | "id": "M8ylYU0BO9QP", 47 | "colab_type": "code", 48 | "colab": {} 49 | }, 50 | "cell_type": "code", 51 | "source": [ 52 | "!pip install -q fastai==0.7.0 torchtext==0.2.3" 53 | ], 54 | "execution_count": 0, 55 | "outputs": [] 56 | }, 57 | { 58 | "metadata": { 59 | "id": "Q1w2HnvvlP2z", 60 | "colab_type": "code", 61 | "colab": {} 62 | }, 63 | "cell_type": "code", 64 | "source": [ 65 | "from fastai.conv_learner import *\n", 66 | "from cv2 import resize\n", 67 | "import matplotlib.gridspec as gridspec\n", 68 | "from math import ceil" 69 | ], 70 | "execution_count": 0, 71 | "outputs": [] 72 | }, 73 | { 74 | "metadata": { 75 | "id": "7bBwWLBhha2Z", 76 | "colab_type": "code", 77 | "colab": {} 78 | }, 79 | "cell_type": "code", 80 | "source": [ 81 | "from pdb import set_trace" 82 | ], 83 | "execution_count": 0, 84 | "outputs": [] 85 | }, 86 | { 87 | "metadata": { 88 | "id": "5YpDdNpkVTCS", 89 | "colab_type": "code", 90 | "colab": {} 91 | }, 92 | "cell_type": "code", 93 | "source": [ 94 | "# from scipy import ndimage" 95 | ], 96 | "execution_count": 0, 97 | "outputs": [] 98 | }, 99 | { 100 | "metadata": { 101 | "colab_type": "code", 102 | "id": "UQ2d_EVYBODD", 103 | "colab": {} 104 | }, 105 | "cell_type": "code", 106 | "source": [ 107 | "class SaveFeatures():\n", 108 | " def __init__(self, module):\n", 109 | " self.hook = module.register_forward_hook(self.hook_fn)\n", 110 | " def hook_fn(self, module, input, output):\n", 111 | " self.features = output\n", 112 | " def close(self):\n", 113 | " self.hook.remove()" 114 | ], 115 | "execution_count": 0, 116 | "outputs": [] 117 | }, 118 | { 119 | "metadata": { 120 | "id": "OFxPmIboP5c_", 121 | "colab_type": "code", 122 | "colab": {} 123 | }, 124 | "cell_type": "code", 125 | "source": [ 126 | "class FilterVisualizer():\n", 127 | " def __init__(self):\n", 128 | " self.model = nn.Sequential(*list(resnet34(True).children())[:-2]).cuda().eval()\n", 129 | " set_trainable(self.model, False)\n", 130 | "\n", 131 | " def visualize(self, sz, layer, filter, upscaling_steps=12, upscaling_factor=1.2, lr=0.1, opt_steps=20, blur=None, save=False, print_losses=False):\n", 132 | "\n", 133 | " img = (np.random.random((sz, sz, 3)) * 20 + 128.)/255.\n", 134 | "# img = np.random.uniform(0, 1, size=(sz, sz, 3)).astype(np.float32)\n", 135 | "# median_filter_size = 4 if sz < 100 else 8\n", 136 | "# img = scipy.ndimage.filters.median_filter(img, [median_filter_size,median_filter_size,1])\n", 137 | "\n", 138 | " activations = SaveFeatures(layer) # register hook\n", 139 | "\n", 140 | " for i in range(upscaling_steps): # scale the image up upscaling_steps times\n", 141 | " train_tfms, val_tfms = tfms_from_model(resnet34, sz)\n", 142 | " img_var = V(val_tfms(img)[None], requires_grad=True) # convert image to Variable that requires grad\n", 143 | " optimizer = torch.optim.Adam([img_var], lr=lr, weight_decay=1e-6)\n", 144 | " if i > upscaling_steps/2:\n", 145 | " opt_steps_ = int(opt_steps*1.3)\n", 146 | " else:\n", 147 | " opt_steps_ = opt_steps\n", 148 | " for n in range(opt_steps_): # optimize pixel values for opt_steps times\n", 149 | " optimizer.zero_grad()\n", 150 | " self.model(img_var)\n", 151 | " loss = -1*activations.features[0, filter].mean()\n", 152 | " if print_losses:\n", 153 | " if i%3==0 and n%5==0:\n", 154 | " print(f'{i} - {n} - {float(loss)}')\n", 155 | " loss.backward()\n", 156 | " optimizer.step()\n", 157 | " img = val_tfms.denorm(np.rollaxis(to_np(img_var.data),1,4))[0]\n", 158 | " self.output = img\n", 159 | " sz = int(upscaling_factor * sz) # calculate new image size\n", 160 | " img = cv2.resize(img, (sz, sz), interpolation = cv2.INTER_CUBIC) # scale image up\n", 161 | " if blur is not None: img = cv2.blur(img,(blur,blur)) # blur image to reduce high frequency patterns\n", 162 | " activations.close()\n", 163 | " return np.clip(self.output, 0, 1)\n", 164 | " \n", 165 | " def get_transformed_img(self,img,sz):\n", 166 | " train_tfms, val_tfms = tfms_from_model(resnet34, sz)\n", 167 | " return val_tfms.denorm(np.rollaxis(to_np(val_tfms(img)[None]),1,4))[0]\n", 168 | " \n", 169 | " def most_activated(self, image, layer, limit_top=None):\n", 170 | "\n", 171 | " train_tfms, val_tfms = tfms_from_model(resnet34, 224)\n", 172 | " transformed = val_tfms(image)\n", 173 | "\n", 174 | " activations = SaveFeatures(layer) # register hook\n", 175 | " self.model(V(transformed)[None]);\n", 176 | " \n", 177 | " mean_act = [activations.features[0,i].mean().data.cpu().numpy()[0] for i in range(activations.features.shape[1])]\n", 178 | " activations.close()\n", 179 | " return mean_act" 180 | ], 181 | "execution_count": 0, 182 | "outputs": [] 183 | }, 184 | { 185 | "metadata": { 186 | "id": "KGm5tqr2VinJ", 187 | "colab_type": "code", 188 | "colab": { 189 | "base_uri": "https://localhost:8080/", 190 | "height": 50 191 | }, 192 | "outputId": "b1dd5557-cdba-4458-f0a2-292a551e9316" 193 | }, 194 | "cell_type": "code", 195 | "source": [ 196 | "FV = FilterVisualizer()" 197 | ], 198 | "execution_count": 9, 199 | "outputs": [ 200 | { 201 | "output_type": "stream", 202 | "text": [ 203 | "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.torch/models/resnet34-333f7ec4.pth\n", 204 | "100%|██████████| 87306240/87306240 [00:00<00:00, 114877173.59it/s]\n" 205 | ], 206 | "name": "stderr" 207 | } 208 | ] 209 | }, 210 | { 211 | "metadata": { 212 | "id": "DBT9Yd4bYIJV", 213 | "colab_type": "code", 214 | "colab": { 215 | "base_uri": "https://localhost:8080/", 216 | "height": 2335 217 | }, 218 | "outputId": "2ffd4c58-ee55-43b6-d230-d3b56f738d35" 219 | }, 220 | "cell_type": "code", 221 | "source": [ 222 | "FV.model" 223 | ], 224 | "execution_count": 10, 225 | "outputs": [ 226 | { 227 | "output_type": "execute_result", 228 | "data": { 229 | "text/plain": [ 230 | "Sequential(\n", 231 | " (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n", 232 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 233 | " (2): ReLU(inplace)\n", 234 | " (3): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1), ceil_mode=False)\n", 235 | " (4): Sequential(\n", 236 | " (0): BasicBlock(\n", 237 | " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 238 | " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 239 | " (relu): ReLU(inplace)\n", 240 | " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 241 | " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 242 | " )\n", 243 | " (1): BasicBlock(\n", 244 | " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 245 | " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 246 | " (relu): ReLU(inplace)\n", 247 | " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 248 | " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 249 | " )\n", 250 | " (2): BasicBlock(\n", 251 | " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 252 | " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 253 | " (relu): ReLU(inplace)\n", 254 | " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 255 | " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 256 | " )\n", 257 | " )\n", 258 | " (5): Sequential(\n", 259 | " (0): BasicBlock(\n", 260 | " (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", 261 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 262 | " (relu): ReLU(inplace)\n", 263 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 264 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 265 | " (downsample): Sequential(\n", 266 | " (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", 267 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 268 | " )\n", 269 | " )\n", 270 | " (1): BasicBlock(\n", 271 | " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 272 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 273 | " (relu): ReLU(inplace)\n", 274 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 275 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 276 | " )\n", 277 | " (2): BasicBlock(\n", 278 | " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 279 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 280 | " (relu): ReLU(inplace)\n", 281 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 282 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 283 | " )\n", 284 | " (3): BasicBlock(\n", 285 | " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 286 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 287 | " (relu): ReLU(inplace)\n", 288 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 289 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 290 | " )\n", 291 | " )\n", 292 | " (6): Sequential(\n", 293 | " (0): BasicBlock(\n", 294 | " (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", 295 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 296 | " (relu): ReLU(inplace)\n", 297 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 298 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 299 | " (downsample): Sequential(\n", 300 | " (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", 301 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 302 | " )\n", 303 | " )\n", 304 | " (1): BasicBlock(\n", 305 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 306 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 307 | " (relu): ReLU(inplace)\n", 308 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 309 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 310 | " )\n", 311 | " (2): BasicBlock(\n", 312 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 313 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 314 | " (relu): ReLU(inplace)\n", 315 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 316 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 317 | " )\n", 318 | " (3): BasicBlock(\n", 319 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 320 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 321 | " (relu): ReLU(inplace)\n", 322 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 323 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 324 | " )\n", 325 | " (4): BasicBlock(\n", 326 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 327 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 328 | " (relu): ReLU(inplace)\n", 329 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 330 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 331 | " )\n", 332 | " (5): BasicBlock(\n", 333 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 334 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 335 | " (relu): ReLU(inplace)\n", 336 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 337 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 338 | " )\n", 339 | " )\n", 340 | " (7): Sequential(\n", 341 | " (0): BasicBlock(\n", 342 | " (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", 343 | " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 344 | " (relu): ReLU(inplace)\n", 345 | " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 346 | " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 347 | " (downsample): Sequential(\n", 348 | " (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", 349 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 350 | " )\n", 351 | " )\n", 352 | " (1): BasicBlock(\n", 353 | " (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 354 | " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 355 | " (relu): ReLU(inplace)\n", 356 | " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 357 | " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 358 | " )\n", 359 | " (2): BasicBlock(\n", 360 | " (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 361 | " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 362 | " (relu): ReLU(inplace)\n", 363 | " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 364 | " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 365 | " )\n", 366 | " )\n", 367 | ")" 368 | ] 369 | }, 370 | "metadata": { 371 | "tags": [] 372 | }, 373 | "execution_count": 10 374 | } 375 | ] 376 | }, 377 | { 378 | "metadata": { 379 | "id": "bjxLkmZk0nqb", 380 | "colab_type": "code", 381 | "colab": {} 382 | }, 383 | "cell_type": "code", 384 | "source": [ 385 | "import requests\n", 386 | "import base64\n", 387 | "import pprint" 388 | ], 389 | "execution_count": 0, 390 | "outputs": [] 391 | }, 392 | { 393 | "metadata": { 394 | "id": "2BbTkEgE0pDi", 395 | "colab_type": "code", 396 | "colab": {} 397 | }, 398 | "cell_type": "code", 399 | "source": [ 400 | "with open('img_at.txt') as f:\n", 401 | " ACCESS_TOKEN = f.read().strip()" 402 | ], 403 | "execution_count": 0, 404 | "outputs": [] 405 | }, 406 | { 407 | "metadata": { 408 | "id": "Lokp5dZ43hjx", 409 | "colab_type": "code", 410 | "colab": {} 411 | }, 412 | "cell_type": "code", 413 | "source": [ 414 | "def image_from_url(url,file_name):\n", 415 | " !wget -qq \"{url}\" -O {file_name}\n", 416 | " return open_image(file_name)" 417 | ], 418 | "execution_count": 0, 419 | "outputs": [] 420 | }, 421 | { 422 | "metadata": { 423 | "id": "YDRV73IZ0qVi", 424 | "colab_type": "code", 425 | "colab": {} 426 | }, 427 | "cell_type": "code", 428 | "source": [ 429 | "def upload_to_imgur(file_name,post_title,album_hash):\n", 430 | " url = 'https://api.imgur.com/3/image'\n", 431 | " fh = open(file_name, 'rb');\n", 432 | " payload = {'image': base64.b64encode(fh.read()),\n", 433 | " 'album':album_hash,\n", 434 | " 'type':'base64',\n", 435 | " 'title':post_title,\n", 436 | " 'looping':False\n", 437 | " }\n", 438 | " files = {}\n", 439 | " headers = {\n", 440 | " 'Authorization': f'Bearer {ACCESS_TOKEN}'\n", 441 | " }\n", 442 | " response = requests.request('POST', url, headers = headers, data = payload, files = files, allow_redirects=False)\n", 443 | " return response.json()['data']['link']" 444 | ], 445 | "execution_count": 0, 446 | "outputs": [] 447 | }, 448 | { 449 | "metadata": { 450 | "id": "uByz2aDB1i3d", 451 | "colab_type": "code", 452 | "colab": {} 453 | }, 454 | "cell_type": "code", 455 | "source": [ 456 | "def create_imgur_album(title,description='',privacy='hidden'):\n", 457 | " url = 'https://api.imgur.com/3/album'\n", 458 | " payload = {\n", 459 | " 'title': title,\n", 460 | " 'description': description,\n", 461 | " 'privacy':privacy}\n", 462 | " files = {}\n", 463 | " headers = {\n", 464 | " 'Authorization': f'Bearer {ACCESS_TOKEN}'\n", 465 | " }\n", 466 | " response = requests.request('POST', url, headers = headers, data = payload, files = files, allow_redirects=False)\n", 467 | " print(f\"https://imgur.com/a/{response.json()['data']['id']}\")" 468 | ], 469 | "execution_count": 0, 470 | "outputs": [] 471 | }, 472 | { 473 | "metadata": { 474 | "id": "vkaJSex927kd", 475 | "colab_type": "text" 476 | }, 477 | "cell_type": "markdown", 478 | "source": [ 479 | "## Plotting mean activations per feature map and corresponding input space" 480 | ] 481 | }, 482 | { 483 | "metadata": { 484 | "id": "hBP4p6k5gf6-", 485 | "colab_type": "code", 486 | "colab": {} 487 | }, 488 | "cell_type": "code", 489 | "source": [ 490 | "def plot_activations_and_reconstructions(imgs,activations,filters,\n", 491 | " transformed_img,n_cols=3,\n", 492 | " cell_size=4,layer_name='',\n", 493 | " save_fig=False,album_hash=None):\n", 494 | " n_rows = ceil((len(imgs)+1)/n_cols)\n", 495 | "\n", 496 | " fig = plt.figure(figsize=(cell_size*n_cols,cell_size*n_rows))\n", 497 | " gs = gridspec.GridSpec(n_rows, n_cols)\n", 498 | " tr_im_ax = plt.subplot(gs[0,0])\n", 499 | " tr_im_ax.grid(False)\n", 500 | " tr_im_ax.get_xaxis().set_visible(False)\n", 501 | " tr_im_ax.get_yaxis().set_visible(False)\n", 502 | " tr_im_ax.imshow(transformed_img)\n", 503 | " tr_im_ax.set_title('Image')\n", 504 | " \n", 505 | " act_ax = plt.subplot(gs[0, 1:])\n", 506 | " \n", 507 | " \n", 508 | " act = act_ax.plot(np.clip(activations,0.,None),linewidth=2.)\n", 509 | " for el in filters:\n", 510 | " act_ax.axvline(x=el, color='red', linestyle='--',alpha=0.4)\n", 511 | " act_ax.set_xlim(0,len(activations));\n", 512 | " act_ax.set_ylabel(f\"mean activation\");\n", 513 | " if layer_name == '':\n", 514 | " act_ax.set_title('Mean Activations')\n", 515 | " else:\n", 516 | " act_ax.set_title(f'{layer_name}')\n", 517 | " act_ax.set_facecolor('white')\n", 518 | " \n", 519 | " fmap_axes = []\n", 520 | " for r in range(1,n_rows):\n", 521 | " for c in range(n_cols):\n", 522 | " fmap_axes.append(plt.subplot(gs[r, c]))\n", 523 | " \n", 524 | " for i,ax in enumerate(fmap_axes):\n", 525 | " ax.grid(False)\n", 526 | " ax.get_xaxis().set_visible(False)\n", 527 | " ax.get_yaxis().set_visible(False)\n", 528 | "\n", 529 | " if i>=len(filters):\n", 530 | " pass\n", 531 | "\n", 532 | " ax.set_title(f'fmap {filters[i]}')\n", 533 | "\n", 534 | " ax.imshow(imgs[i])\n", 535 | " plt.tight_layout()\n", 536 | " save_name = layer_name.lower().replace(' ','_')\n", 537 | " if save_fig:\n", 538 | " plt.savefig(f'{save_name}.png')\n", 539 | " upload_to_imgur(f'{save_name}.png',\n", 540 | " f'{save_name}',album_hash)\n", 541 | " plt.close()\n", 542 | " else:\n", 543 | " plt.show()" 544 | ], 545 | "execution_count": 0, 546 | "outputs": [] 547 | }, 548 | { 549 | "metadata": { 550 | "id": "3d53GaBtRORh", 551 | "colab_type": "code", 552 | "colab": {} 553 | }, 554 | "cell_type": "code", 555 | "source": [ 556 | "def activations_and_reconstructions(img,activation_layer,fmap_layer,\n", 557 | " top_num=4,init_size=56,\n", 558 | " upscaling_steps=12, upscaling_factor=1.2,\n", 559 | " opt_steps=20, blur=5,lr=1e-1,\n", 560 | " print_losses=False,\n", 561 | " n_cols=3, cell_size=4,\n", 562 | " layer_name='',\n", 563 | " save_fig=False,\n", 564 | " album_hash=None):\n", 565 | " \n", 566 | " mean_acts = FV.most_activated(img,layer = activation_layer)\n", 567 | "\n", 568 | " most_act_fmaps = sorted(range(len(mean_acts)), key=lambda i: mean_acts[i])[-top_num:][::-1]\n", 569 | "\n", 570 | " imgs = []\n", 571 | " for filter in most_act_fmaps:\n", 572 | " imgs.append(FV.visualize(init_size,fmap_layer, filter, upscaling_steps=upscaling_steps, \n", 573 | " upscaling_factor=upscaling_factor, \n", 574 | " opt_steps=opt_steps, blur=blur,\n", 575 | " lr=lr,print_losses=False))\n", 576 | " transformed_img = FV.get_transformed_img(img,224)\n", 577 | " \n", 578 | " plot_activations_and_reconstructions(imgs,mean_acts,\n", 579 | " most_act_fmaps,transformed_img,\n", 580 | " n_cols=n_cols,cell_size=cell_size,\n", 581 | " layer_name=layer_name,\n", 582 | " save_fig=save_fig,\n", 583 | " album_hash=album_hash)" 584 | ], 585 | "execution_count": 0, 586 | "outputs": [] 587 | }, 588 | { 589 | "metadata": { 590 | "id": "LlfVXhEF1Myx", 591 | "colab_type": "code", 592 | "colab": {} 593 | }, 594 | "cell_type": "code", 595 | "source": [ 596 | "image_urls_1 = [\n", 597 | " 'https://pbs.twimg.com/profile_images/871470043200094209/1UAyRdei_400x400.jpg',\n", 598 | " 'http://farm3.static.flickr.com/2423/3957827131_90978df60b.jpg',\n", 599 | " 'http://farm1.static.flickr.com/97/209577710_344cf9b77d.jpg',\n", 600 | " 'http://farm1.static.flickr.com/232/500314013_56e18dd72e.jpg',\n", 601 | " 'http://farm4.static.flickr.com/3283/2536420364_2cbb96fb64.jpg',\n", 602 | " 'http://farm3.static.flickr.com/2418/1988187373_548be0682c.jpg',\n", 603 | " 'http://farm4.static.flickr.com/3195/2703096506_8963dac892.jpg',\n", 604 | " 'http://farm2.static.flickr.com/1093/883043423_8503b98702.jpg',\n", 605 | " 'http://farm4.static.flickr.com/3220/2659962123_a5d9d9f080.jpg',\n", 606 | " 'http://farm3.static.flickr.com/2420/2252055675_1ec3de9c46.jpg',\n", 607 | " 'http://farm4.static.flickr.com/3216/2416297192_b4ff2a87d8.jpg',\n", 608 | " 'http://farm4.static.flickr.com/3195/2690113895_507f0fb71a.jpg',\n", 609 | " 'http://farm4.static.flickr.com/3097/2476598462_f5a95c12e4.jpg',\n", 610 | " 'http://static.flickr.com/96/279601307_427e163e90_o.jpg',\n", 611 | " 'http://farm3.static.flickr.com/2155/2485884251_6e28c59fdd.jpg',\n", 612 | " 'http://farm4.static.flickr.com/3073/2820931609_3e5c8ceae2.jpg',\n", 613 | " 'http://farm1.static.flickr.com/134/325737039_2cb3de029e.jpg',\n", 614 | " 'http://farm3.static.flickr.com/2003/2047290079_c962beeb85.jpg',\n", 615 | "]\n", 616 | " " 617 | ], 618 | "execution_count": 0, 619 | "outputs": [] 620 | }, 621 | { 622 | "metadata": { 623 | "id": "tr3pwaB81IU3", 624 | "colab_type": "code", 625 | "colab": {} 626 | }, 627 | "cell_type": "code", 628 | "source": [ 629 | "image_urls_2 =[\n", 630 | " 'http://farm3.static.flickr.com/2672/3729197896_e2fcb07887.jpg',\n", 631 | " 'http://farm3.static.flickr.com/2419/2130941151_b100201751.jpg',\n", 632 | " 'http://farm3.static.flickr.com/2446/3570779025_4748186d3f.jpg',\n", 633 | " 'http://farm4.static.flickr.com/3504/3757435933_686e12d502.jpg',\n", 634 | " 'http://farm4.static.flickr.com/3198/2408009683_9eb72cd5d1.jpg',\n", 635 | " 'http://farm2.static.flickr.com/1389/1449509194_6b9a7f7003.jpg',\n", 636 | " 'http://farm4.static.flickr.com/3147/2905958970_cf47a19ece.jpg',\n", 637 | " 'http://farm4.static.flickr.com/3213/2845504219_4cd956bcf2.jpg',\n", 638 | " 'http://farm3.static.flickr.com/2099/2233817436_3f20abe7df.jpg',\n", 639 | " 'http://farm4.static.flickr.com/3176/2344377903_f5be963a37.jpg',\n", 640 | "]" 641 | ], 642 | "execution_count": 0, 643 | "outputs": [] 644 | }, 645 | { 646 | "metadata": { 647 | "id": "KpLGcUfg4uGj", 648 | "colab_type": "text" 649 | }, 650 | "cell_type": "markdown", 651 | "source": [ 652 | "Testing for `ReLU` in blocks `5` and `6` of layer `3`, and blocks `1`, `2`, and `3` in layer `4`." 653 | ] 654 | }, 655 | { 656 | "metadata": { 657 | "id": "WmhJY1gg1Urg", 658 | "colab_type": "code", 659 | "colab": {} 660 | }, 661 | "cell_type": "code", 662 | "source": [ 663 | "urls = [image_urls_1,image_urls_2]\n", 664 | "hashes = ['N0dUcS0','i1IMVM6']\n", 665 | "\n", 666 | "for i in range(2):\n", 667 | " album_hash = hashes[i]\n", 668 | " for url in urls[i]:\n", 669 | " try:\n", 670 | " image = image_from_url(url,'dummy.jpg')\n", 671 | "\n", 672 | " activations_and_reconstructions(image,children(FV.model)[6][4].relu,\n", 673 | " children(FV.model)[6][4].relu,\n", 674 | " top_num=6,layer_name='Resnet34 Layer3 Block5 Relu',\n", 675 | " save_fig=True,album_hash=album_hash)\n", 676 | "\n", 677 | " activations_and_reconstructions(image,children(FV.model)[6][5].relu,\n", 678 | " children(FV.model)[6][5].relu,\n", 679 | " top_num=6,layer_name='Resnet34 Layer3 Block6 Relu',\n", 680 | " save_fig=True,album_hash=album_hash)\n", 681 | "\n", 682 | " activations_and_reconstructions(image,children(FV.model)[7][0].relu,\n", 683 | " children(FV.model)[7][0].relu,\n", 684 | " top_num=6,layer_name='Resnet34 Layer4 Block1 Relu',\n", 685 | " save_fig=True,album_hash=album_hash)\n", 686 | "\n", 687 | " activations_and_reconstructions(image,children(FV.model)[7][1].relu,\n", 688 | " children(FV.model)[7][1].relu,\n", 689 | " top_num=6,layer_name='Resnet34 Layer4 Block2 Relu',\n", 690 | " save_fig=True,album_hash=album_hash)\n", 691 | "\n", 692 | " activations_and_reconstructions(image,children(FV.model)[7][2].relu,\n", 693 | " children(FV.model)[7][2].relu,\n", 694 | " top_num=6,layer_name='Resnet34 Layer4 Block3 Relu',\n", 695 | " save_fig=True,album_hash=album_hash)\n", 696 | " except Exception as e:\n", 697 | " print(str(e))" 698 | ], 699 | "execution_count": 0, 700 | "outputs": [] 701 | }, 702 | { 703 | "metadata": { 704 | "id": "CU7ooc0T1ZwX", 705 | "colab_type": "text" 706 | }, 707 | "cell_type": "markdown", 708 | "source": [ 709 | "## Results\n", 710 | "Plotting some of the results for feature maps in different layers." 711 | ] 712 | }, 713 | { 714 | "metadata": { 715 | "id": "NZK7GSXI5gxQ", 716 | "colab_type": "text" 717 | }, 718 | "cell_type": "markdown", 719 | "source": [ 720 | "![](https://i.imgur.com/taKzea2.png)\n", 721 | "![](https://i.imgur.com/XtIcWcv.png)\n", 722 | "\n", 723 | "---\n", 724 | "\n", 725 | "![](https://i.imgur.com/yys2bSK.png)\n", 726 | "![](https://i.imgur.com/aLNMy3W.png)\n", 727 | "\n", 728 | "---\n" 729 | ] 730 | }, 731 | { 732 | "metadata": { 733 | "id": "VJ0xkRGt55JI", 734 | "colab_type": "text" 735 | }, 736 | "cell_type": "markdown", 737 | "source": [ 738 | "![](https://i.imgur.com/o3D03T7.png)\n", 739 | "\n", 740 | "---\n", 741 | "\n", 742 | "![](https://i.imgur.com/E31wkk9.png)\n", 743 | "\n", 744 | "---\n", 745 | "\n", 746 | "![](https://i.imgur.com/7rvSiyb.png)\n", 747 | "\n", 748 | "---\n", 749 | "\n", 750 | "![](https://i.imgur.com/ZYqq8GK.png)\n", 751 | "\n", 752 | "---\n" 753 | ] 754 | }, 755 | { 756 | "metadata": { 757 | "id": "W4E-WEBa5_6k", 758 | "colab_type": "text" 759 | }, 760 | "cell_type": "markdown", 761 | "source": [ 762 | "![](https://i.imgur.com/hrffqiK.png)\n", 763 | "![](https://i.imgur.com/rY00qbs.png)\n", 764 | "\n", 765 | "---\n", 766 | "\n", 767 | "![](https://i.imgur.com/CaoW2mG.png)\n", 768 | "![](https://i.imgur.com/c4hibxG.png)\n", 769 | "\n", 770 | "---\n", 771 | "\n", 772 | "![](https://i.imgur.com/vkzrHYh.png)\n", 773 | "![](https://i.imgur.com/DibmP8m.png)" 774 | ] 775 | }, 776 | { 777 | "metadata": { 778 | "id": "IaIQaBOJ6I2C", 779 | "colab_type": "text" 780 | }, 781 | "cell_type": "markdown", 782 | "source": [ 783 | "All results can be seen [here](https://imgur.com/a/N0dUcS0) and [here](https://imgur.com/a/i1IMVM6)." 784 | ] 785 | } 786 | ] 787 | } -------------------------------------------------------------------------------- /experiments/resnet34_part_1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "feat_viz_resnet34.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [] 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "accelerator": "GPU" 16 | }, 17 | "cells": [ 18 | { 19 | "metadata": { 20 | "id": "I9VDrPCrR1__", 21 | "colab_type": "text" 22 | }, 23 | "cell_type": "markdown", 24 | "source": [ 25 | "# Visualizing inputs that maximally activate feature maps of a convnet\n", 26 | "\n", 27 | "## Network used: ResNet-34" 28 | ] 29 | }, 30 | { 31 | "metadata": { 32 | "id": "39Ch5JJ-lJSI", 33 | "colab_type": "code", 34 | "colab": {} 35 | }, 36 | "cell_type": "code", 37 | "source": [ 38 | "%matplotlib inline\n", 39 | "%reload_ext autoreload\n", 40 | "%autoreload 2" 41 | ], 42 | "execution_count": 0, 43 | "outputs": [] 44 | }, 45 | { 46 | "metadata": { 47 | "id": "M8ylYU0BO9QP", 48 | "colab_type": "code", 49 | "colab": {} 50 | }, 51 | "cell_type": "code", 52 | "source": [ 53 | "!pip install -q fastai==0.7.0 torchtext==0.2.3" 54 | ], 55 | "execution_count": 0, 56 | "outputs": [] 57 | }, 58 | { 59 | "metadata": { 60 | "id": "Q1w2HnvvlP2z", 61 | "colab_type": "code", 62 | "colab": {} 63 | }, 64 | "cell_type": "code", 65 | "source": [ 66 | "from fastai.conv_learner import *\n", 67 | "from cv2 import resize\n", 68 | "import matplotlib.gridspec as gridspec\n", 69 | "from math import ceil" 70 | ], 71 | "execution_count": 0, 72 | "outputs": [] 73 | }, 74 | { 75 | "metadata": { 76 | "id": "fhjHTcM4Q9Ey", 77 | "colab_type": "code", 78 | "colab": {} 79 | }, 80 | "cell_type": "code", 81 | "source": [ 82 | "from IPython.display import HTML" 83 | ], 84 | "execution_count": 0, 85 | "outputs": [] 86 | }, 87 | { 88 | "metadata": { 89 | "id": "7bBwWLBhha2Z", 90 | "colab_type": "code", 91 | "colab": {} 92 | }, 93 | "cell_type": "code", 94 | "source": [ 95 | "from pdb import set_trace" 96 | ], 97 | "execution_count": 0, 98 | "outputs": [] 99 | }, 100 | { 101 | "metadata": { 102 | "id": "5YpDdNpkVTCS", 103 | "colab_type": "code", 104 | "colab": {} 105 | }, 106 | "cell_type": "code", 107 | "source": [ 108 | "from scipy import ndimage" 109 | ], 110 | "execution_count": 0, 111 | "outputs": [] 112 | }, 113 | { 114 | "metadata": { 115 | "colab_type": "code", 116 | "id": "UQ2d_EVYBODD", 117 | "colab": {} 118 | }, 119 | "cell_type": "code", 120 | "source": [ 121 | "class SaveFeatures():\n", 122 | " def __init__(self, module):\n", 123 | " self.hook = module.register_forward_hook(self.hook_fn)\n", 124 | " def hook_fn(self, module, input, output):\n", 125 | " self.features = output\n", 126 | " def close(self):\n", 127 | " self.hook.remove()" 128 | ], 129 | "execution_count": 0, 130 | "outputs": [] 131 | }, 132 | { 133 | "metadata": { 134 | "id": "OFxPmIboP5c_", 135 | "colab_type": "code", 136 | "colab": {} 137 | }, 138 | "cell_type": "code", 139 | "source": [ 140 | "class FilterVisualizer():\n", 141 | " def __init__(self):\n", 142 | " self.model = nn.Sequential(*list(resnet34(True).children())[:-2]).cuda().eval()\n", 143 | " set_trainable(self.model, False)\n", 144 | "\n", 145 | " def visualize(self, sz, layer, filter, upscaling_steps=12, upscaling_factor=1.2, lr=0.1, opt_steps=20, blur=None, save=False, print_losses=False):\n", 146 | "\n", 147 | " img = (np.random.random((sz, sz, 3)) * 20 + 128.)/255.\n", 148 | "# img = np.random.uniform(0, 1, size=(sz, sz, 3)).astype(np.float32)\n", 149 | "# median_filter_size = 4 if sz < 100 else 8\n", 150 | "# img = scipy.ndimage.filters.median_filter(img, [median_filter_size,median_filter_size,1])\n", 151 | "\n", 152 | " activations = SaveFeatures(layer) # register hook\n", 153 | "\n", 154 | " for i in range(upscaling_steps): # scale the image up upscaling_steps times\n", 155 | " train_tfms, val_tfms = tfms_from_model(resnet34, sz)\n", 156 | " img_var = V(val_tfms(img)[None], requires_grad=True) # convert image to Variable that requires grad\n", 157 | " optimizer = torch.optim.Adam([img_var], lr=lr, weight_decay=1e-6)\n", 158 | " if i > upscaling_steps/2:\n", 159 | " opt_steps_ = int(opt_steps*1.3)\n", 160 | " else:\n", 161 | " opt_steps_ = opt_steps\n", 162 | " for n in range(opt_steps_): # optimize pixel values for opt_steps times\n", 163 | " optimizer.zero_grad()\n", 164 | " self.model(img_var)\n", 165 | " loss = -1*activations.features[0, filter].mean()\n", 166 | " if print_losses:\n", 167 | " if i%3==0 and n%5==0:\n", 168 | " print(f'{i} - {n} - {float(loss)}')\n", 169 | " loss.backward()\n", 170 | " optimizer.step()\n", 171 | " img = val_tfms.denorm(np.rollaxis(to_np(img_var.data),1,4))[0]\n", 172 | " self.output = img\n", 173 | " sz = int(upscaling_factor * sz) # calculate new image size\n", 174 | " img = cv2.resize(img, (sz, sz), interpolation = cv2.INTER_CUBIC) # scale image up\n", 175 | " if blur is not None: img = cv2.blur(img,(blur,blur)) # blur image to reduce high frequency patterns\n", 176 | " activations.close()\n", 177 | " return np.clip(self.output, 0, 1)\n", 178 | " \n", 179 | " def get_transformed_img(self,img,sz):\n", 180 | " train_tfms, val_tfms = tfms_from_model(resnet34, sz)\n", 181 | " return val_tfms.denorm(np.rollaxis(to_np(val_tfms(img)[None]),1,4))[0]\n", 182 | " \n", 183 | " def most_activated(self, image, layer, limit_top=None):\n", 184 | "\n", 185 | " train_tfms, val_tfms = tfms_from_model(resnet34, 224)\n", 186 | " transformed = val_tfms(image)\n", 187 | "\n", 188 | " activations = SaveFeatures(layer) # register hook\n", 189 | " self.model(V(transformed)[None]);\n", 190 | " \n", 191 | " mean_act = [activations.features[0,i].mean().data.cpu().numpy()[0] for i in range(activations.features.shape[1])]\n", 192 | " activations.close()\n", 193 | " return mean_act" 194 | ], 195 | "execution_count": 0, 196 | "outputs": [] 197 | }, 198 | { 199 | "metadata": { 200 | "id": "KGm5tqr2VinJ", 201 | "colab_type": "code", 202 | "outputId": "f6f1f85f-2644-4d4d-d437-0f11d3751d55", 203 | "colab": { 204 | "base_uri": "https://localhost:8080/", 205 | "height": 50 206 | } 207 | }, 208 | "cell_type": "code", 209 | "source": [ 210 | "FV = FilterVisualizer()" 211 | ], 212 | "execution_count": 5, 213 | "outputs": [ 214 | { 215 | "output_type": "stream", 216 | "text": [ 217 | "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.torch/models/resnet34-333f7ec4.pth\n", 218 | "100%|██████████| 87306240/87306240 [00:03<00:00, 23733638.02it/s]\n" 219 | ], 220 | "name": "stderr" 221 | } 222 | ] 223 | }, 224 | { 225 | "metadata": { 226 | "id": "DBT9Yd4bYIJV", 227 | "colab_type": "code", 228 | "outputId": "9a26f061-49b7-4210-954a-e75232fd5afe", 229 | "colab": { 230 | "base_uri": "https://localhost:8080/", 231 | "height": 2335 232 | } 233 | }, 234 | "cell_type": "code", 235 | "source": [ 236 | "FV.model" 237 | ], 238 | "execution_count": 0, 239 | "outputs": [ 240 | { 241 | "output_type": "execute_result", 242 | "data": { 243 | "text/plain": [ 244 | "Sequential(\n", 245 | " (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n", 246 | " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 247 | " (2): ReLU(inplace)\n", 248 | " (3): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1), ceil_mode=False)\n", 249 | " (4): Sequential(\n", 250 | " (0): BasicBlock(\n", 251 | " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 252 | " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 253 | " (relu): ReLU(inplace)\n", 254 | " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 255 | " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 256 | " )\n", 257 | " (1): BasicBlock(\n", 258 | " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 259 | " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 260 | " (relu): ReLU(inplace)\n", 261 | " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 262 | " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 263 | " )\n", 264 | " (2): BasicBlock(\n", 265 | " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 266 | " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 267 | " (relu): ReLU(inplace)\n", 268 | " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 269 | " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n", 270 | " )\n", 271 | " )\n", 272 | " (5): Sequential(\n", 273 | " (0): BasicBlock(\n", 274 | " (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", 275 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 276 | " (relu): ReLU(inplace)\n", 277 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 278 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 279 | " (downsample): Sequential(\n", 280 | " (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", 281 | " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 282 | " )\n", 283 | " )\n", 284 | " (1): BasicBlock(\n", 285 | " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 286 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 287 | " (relu): ReLU(inplace)\n", 288 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 289 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 290 | " )\n", 291 | " (2): BasicBlock(\n", 292 | " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 293 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 294 | " (relu): ReLU(inplace)\n", 295 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 296 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 297 | " )\n", 298 | " (3): BasicBlock(\n", 299 | " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 300 | " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 301 | " (relu): ReLU(inplace)\n", 302 | " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 303 | " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n", 304 | " )\n", 305 | " )\n", 306 | " (6): Sequential(\n", 307 | " (0): BasicBlock(\n", 308 | " (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", 309 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 310 | " (relu): ReLU(inplace)\n", 311 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 312 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 313 | " (downsample): Sequential(\n", 314 | " (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", 315 | " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 316 | " )\n", 317 | " )\n", 318 | " (1): BasicBlock(\n", 319 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 320 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 321 | " (relu): ReLU(inplace)\n", 322 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 323 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 324 | " )\n", 325 | " (2): BasicBlock(\n", 326 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 327 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 328 | " (relu): ReLU(inplace)\n", 329 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 330 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 331 | " )\n", 332 | " (3): BasicBlock(\n", 333 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 334 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 335 | " (relu): ReLU(inplace)\n", 336 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 337 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 338 | " )\n", 339 | " (4): BasicBlock(\n", 340 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 341 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 342 | " (relu): ReLU(inplace)\n", 343 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 344 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 345 | " )\n", 346 | " (5): BasicBlock(\n", 347 | " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 348 | " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 349 | " (relu): ReLU(inplace)\n", 350 | " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 351 | " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n", 352 | " )\n", 353 | " )\n", 354 | " (7): Sequential(\n", 355 | " (0): BasicBlock(\n", 356 | " (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", 357 | " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 358 | " (relu): ReLU(inplace)\n", 359 | " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 360 | " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 361 | " (downsample): Sequential(\n", 362 | " (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", 363 | " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 364 | " )\n", 365 | " )\n", 366 | " (1): BasicBlock(\n", 367 | " (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 368 | " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 369 | " (relu): ReLU(inplace)\n", 370 | " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 371 | " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 372 | " )\n", 373 | " (2): BasicBlock(\n", 374 | " (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 375 | " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 376 | " (relu): ReLU(inplace)\n", 377 | " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", 378 | " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n", 379 | " )\n", 380 | " )\n", 381 | ")" 382 | ] 383 | }, 384 | "metadata": { 385 | "tags": [] 386 | }, 387 | "execution_count": 46 388 | } 389 | ] 390 | }, 391 | { 392 | "metadata": { 393 | "id": "-FFFsEmcP6F3", 394 | "colab_type": "code", 395 | "colab": {} 396 | }, 397 | "cell_type": "code", 398 | "source": [ 399 | "import requests\n", 400 | "import base64\n", 401 | "import pprint" 402 | ], 403 | "execution_count": 0, 404 | "outputs": [] 405 | }, 406 | { 407 | "metadata": { 408 | "id": "gr9aNVwQP7o_", 409 | "colab_type": "code", 410 | "colab": {} 411 | }, 412 | "cell_type": "code", 413 | "source": [ 414 | "with open('img_at.txt') as f:\n", 415 | " ACCESS_TOKEN = f.read().strip()" 416 | ], 417 | "execution_count": 0, 418 | "outputs": [] 419 | }, 420 | { 421 | "metadata": { 422 | "id": "ssOE-MdeP0PZ", 423 | "colab_type": "code", 424 | "colab": {} 425 | }, 426 | "cell_type": "code", 427 | "source": [ 428 | "def upload_to_imgur(file_name,post_title,album_hash):\n", 429 | " url = 'https://api.imgur.com/3/image'\n", 430 | " fh = open(file_name, 'rb');\n", 431 | " payload = {'image': base64.b64encode(fh.read()),\n", 432 | " 'album':album_hash,\n", 433 | " 'type':'base64',\n", 434 | " 'title':post_title,\n", 435 | " 'looping':False\n", 436 | " }\n", 437 | " files = {}\n", 438 | " headers = {\n", 439 | " 'Authorization': f'Bearer {ACCESS_TOKEN}'\n", 440 | " }\n", 441 | " response = requests.request('POST', url, headers = headers, data = payload, files = files, allow_redirects=False)\n", 442 | "# print(response.json()['data']['link'])\n", 443 | " return response.json()['data']['link']" 444 | ], 445 | "execution_count": 0, 446 | "outputs": [] 447 | }, 448 | { 449 | "metadata": { 450 | "id": "IuIGDfKgQS1W", 451 | "colab_type": "code", 452 | "colab": {} 453 | }, 454 | "cell_type": "code", 455 | "source": [ 456 | "def plot_reconstructions_single_layer(imgs,layer_name,filters,\n", 457 | " n_cols=3,\n", 458 | " cell_size=4,save_fig=False,\n", 459 | " album_hash=None):\n", 460 | " n_rows = ceil((len(imgs))/n_cols)\n", 461 | "\n", 462 | " fig,axes = plt.subplots(n_rows,n_cols, figsize=(cell_size*n_cols,cell_size*n_rows))\n", 463 | " \n", 464 | " for i,ax in enumerate(axes.flat):\n", 465 | " ax.grid(False)\n", 466 | " ax.get_xaxis().set_visible(False)\n", 467 | " ax.get_yaxis().set_visible(False)\n", 468 | "\n", 469 | " if i>=len(filters):\n", 470 | " pass\n", 471 | "\n", 472 | " ax.set_title(f'fmap {filters[i]}')\n", 473 | "\n", 474 | " ax.imshow(imgs[i])\n", 475 | " fig.suptitle(f'ResNet34 {layer_name}', fontsize=\"x-large\",y=1.0)\n", 476 | " plt.tight_layout()\n", 477 | " plt.subplots_adjust(top=0.88)\n", 478 | " save_name = layer_name.lower().replace(' ','_')\n", 479 | " if save_fig:\n", 480 | " plt.savefig(f'resnet34_{save_name}_fmaps_{\"_\".join([str(f) for f in filters])}.png')\n", 481 | " link = upload_to_imgur(f'resnet34_{save_name}_fmaps_{\"_\".join([str(f) for f in filters])}.png',\n", 482 | " f'resnet34_{save_name}_fmaps_{\"_\".join([str(f) for f in filters])}',album_hash)\n", 483 | " plt.close()\n", 484 | " return link\n", 485 | " else:\n", 486 | " plt.show()\n", 487 | " return None" 488 | ], 489 | "execution_count": 0, 490 | "outputs": [] 491 | }, 492 | { 493 | "metadata": { 494 | "id": "Zf0UXoS6QoJZ", 495 | "colab_type": "code", 496 | "colab": {} 497 | }, 498 | "cell_type": "code", 499 | "source": [ 500 | "def reconstructions_single_layer(layer,layer_name,filters,\n", 501 | " init_size=56, upscaling_steps=12, \n", 502 | " upscaling_factor=1.2, \n", 503 | " opt_steps=20, blur=5,\n", 504 | " lr=1e-1,print_losses=False,\n", 505 | " n_cols=3, cell_size=4,\n", 506 | " save_fig=False,album_hash=None):\n", 507 | " \n", 508 | " imgs = []\n", 509 | " for i in range(len(filters)):\n", 510 | " imgs.append(FV.visualize(init_size,layer, filters[i], \n", 511 | " upscaling_steps=upscaling_steps, \n", 512 | " upscaling_factor=upscaling_factor, \n", 513 | " opt_steps=opt_steps, blur=blur,\n", 514 | " lr=lr,print_losses=print_losses))\n", 515 | " \n", 516 | " return plot_reconstructions_single_layer(imgs,layer_name,filters,\n", 517 | " n_cols=n_cols,cell_size=cell_size,\n", 518 | " save_fig=save_fig,album_hash=album_hash)" 519 | ], 520 | "execution_count": 0, 521 | "outputs": [] 522 | }, 523 | { 524 | "metadata": { 525 | "id": "dIdqrmpKvT2l", 526 | "colab_type": "code", 527 | "outputId": "e9a7cb3f-fb1e-48aa-8790-e610613fa06d", 528 | "colab": { 529 | "base_uri": "https://localhost:8080/", 530 | "height": 597 531 | } 532 | }, 533 | "cell_type": "code", 534 | "source": [ 535 | "url = reconstructions_single_layer(children(FV.model)[0],'Initial Conv',\n", 536 | " list(range(6,12)),n_cols=3,save_fig=True,\n", 537 | " album_hash='OhQ94rP')\n", 538 | "display(HTML(f\"\"))" 539 | ], 540 | "execution_count": 18, 541 | "outputs": [ 542 | { 543 | "output_type": "display_data", 544 | "data": { 545 | "text/html": [ 546 | "" 547 | ], 548 | "text/plain": [ 549 | "" 550 | ] 551 | }, 552 | "metadata": { 553 | "tags": [] 554 | } 555 | } 556 | ] 557 | }, 558 | { 559 | "metadata": { 560 | "id": "bGJnuWR8RT2Q", 561 | "colab_type": "code", 562 | "colab": { 563 | "base_uri": "https://localhost:8080/", 564 | "height": 597 565 | }, 566 | "outputId": "17b5e81b-b38c-494b-c169-b76140d4e302" 567 | }, 568 | "cell_type": "code", 569 | "source": [ 570 | "url = reconstructions_single_layer(children(FV.model)[4][0].conv1,'Layer 1 Block 1 Conv1',\n", 571 | " list(range(6,12)),n_cols=3,\n", 572 | " save_fig=True,album_hash='OhQ94rP')\n", 573 | "display(HTML(f\"\"))" 574 | ], 575 | "execution_count": 19, 576 | "outputs": [ 577 | { 578 | "output_type": "display_data", 579 | "data": { 580 | "text/html": [ 581 | "" 582 | ], 583 | "text/plain": [ 584 | "" 585 | ] 586 | }, 587 | "metadata": { 588 | "tags": [] 589 | } 590 | } 591 | ] 592 | }, 593 | { 594 | "metadata": { 595 | "id": "Cpsq5258Rc7c", 596 | "colab_type": "code", 597 | "colab": { 598 | "base_uri": "https://localhost:8080/", 599 | "height": 597 600 | }, 601 | "outputId": "ad40f123-c14a-4018-f5dd-46a907820364" 602 | }, 603 | "cell_type": "code", 604 | "source": [ 605 | "url = reconstructions_single_layer(children(FV.model)[4][0].conv2,\n", 606 | " 'Layer 1 Block 1 Conv2',list(range(6,12)),\n", 607 | " n_cols=3,save_fig=True,album_hash='OhQ94rP')\n", 608 | "display(HTML(f\"\"))" 609 | ], 610 | "execution_count": 20, 611 | "outputs": [ 612 | { 613 | "output_type": "display_data", 614 | "data": { 615 | "text/html": [ 616 | "" 617 | ], 618 | "text/plain": [ 619 | "" 620 | ] 621 | }, 622 | "metadata": { 623 | "tags": [] 624 | } 625 | } 626 | ] 627 | }, 628 | { 629 | "metadata": { 630 | "id": "NjlDhBM2RdQE", 631 | "colab_type": "code", 632 | "colab": { 633 | "base_uri": "https://localhost:8080/", 634 | "height": 597 635 | }, 636 | "outputId": "746bac3f-b807-4fb4-aa3e-132651654b92" 637 | }, 638 | "cell_type": "code", 639 | "source": [ 640 | "url = reconstructions_single_layer(children(FV.model)[4][1].conv1,\n", 641 | " 'Layer 1 Block 2 Conv1',\n", 642 | " list(range(6,12)),n_cols=3,\n", 643 | " save_fig=True,album_hash='OhQ94rP')\n", 644 | "display(HTML(f\"\"))" 645 | ], 646 | "execution_count": 21, 647 | "outputs": [ 648 | { 649 | "output_type": "display_data", 650 | "data": { 651 | "text/html": [ 652 | "" 653 | ], 654 | "text/plain": [ 655 | "" 656 | ] 657 | }, 658 | "metadata": { 659 | "tags": [] 660 | } 661 | } 662 | ] 663 | }, 664 | { 665 | "metadata": { 666 | "id": "2fSvUAsCRg40", 667 | "colab_type": "code", 668 | "colab": { 669 | "base_uri": "https://localhost:8080/", 670 | "height": 597 671 | }, 672 | "outputId": "e92fddde-616f-49e3-b81b-8a1dc86bce43" 673 | }, 674 | "cell_type": "code", 675 | "source": [ 676 | "url = reconstructions_single_layer(children(FV.model)[5][0].conv2,\n", 677 | " 'Layer 2 Block 1 Conv2',list(range(6,12)),\n", 678 | " n_cols=3,save_fig=True,album_hash='OhQ94rP')\n", 679 | "display(HTML(f\"\"))" 680 | ], 681 | "execution_count": 23, 682 | "outputs": [ 683 | { 684 | "output_type": "display_data", 685 | "data": { 686 | "text/html": [ 687 | "" 688 | ], 689 | "text/plain": [ 690 | "" 691 | ] 692 | }, 693 | "metadata": { 694 | "tags": [] 695 | } 696 | } 697 | ] 698 | }, 699 | { 700 | "metadata": { 701 | "id": "Ew1G9SHIRg-y", 702 | "colab_type": "code", 703 | "colab": { 704 | "base_uri": "https://localhost:8080/", 705 | "height": 597 706 | }, 707 | "outputId": "9b491fe0-4125-49d2-c118-a50a35710fd7" 708 | }, 709 | "cell_type": "code", 710 | "source": [ 711 | "url = reconstructions_single_layer(children(FV.model)[6][0].conv1,\n", 712 | " 'Layer 3 Block 1 Conv1',list(range(6,12)),\n", 713 | " n_cols=3,save_fig=True,album_hash='OhQ94rP')\n", 714 | "display(HTML(f\"\"))" 715 | ], 716 | "execution_count": 24, 717 | "outputs": [ 718 | { 719 | "output_type": "display_data", 720 | "data": { 721 | "text/html": [ 722 | "" 723 | ], 724 | "text/plain": [ 725 | "" 726 | ] 727 | }, 728 | "metadata": { 729 | "tags": [] 730 | } 731 | } 732 | ] 733 | }, 734 | { 735 | "metadata": { 736 | "id": "FVYVlmsHRhDF", 737 | "colab_type": "code", 738 | "colab": { 739 | "base_uri": "https://localhost:8080/", 740 | "height": 597 741 | }, 742 | "outputId": "3b69e9c0-0427-4768-f832-7c1e8c40f61b" 743 | }, 744 | "cell_type": "code", 745 | "source": [ 746 | "url = reconstructions_single_layer(children(FV.model)[6][1].conv1,'Layer 3 Block 2 Conv1',\n", 747 | " list(range(6,12)),n_cols=3,\n", 748 | " save_fig=True,album_hash='OhQ94rP')\n", 749 | "display(HTML(f\"\"))" 750 | ], 751 | "execution_count": 25, 752 | "outputs": [ 753 | { 754 | "output_type": "display_data", 755 | "data": { 756 | "text/html": [ 757 | "" 758 | ], 759 | "text/plain": [ 760 | "" 761 | ] 762 | }, 763 | "metadata": { 764 | "tags": [] 765 | } 766 | } 767 | ] 768 | }, 769 | { 770 | "metadata": { 771 | "id": "p-0L5aosRhG6", 772 | "colab_type": "code", 773 | "colab": { 774 | "base_uri": "https://localhost:8080/", 775 | "height": 597 776 | }, 777 | "outputId": "1d0bd4d9-6eb4-429a-bf25-b28315e6fa57" 778 | }, 779 | "cell_type": "code", 780 | "source": [ 781 | "url = reconstructions_single_layer(children(FV.model)[6][2].conv1,'Layer 3 Block 3 Conv1',\n", 782 | " list(range(6,12)),n_cols=3,\n", 783 | " save_fig=True,album_hash='OhQ94rP')\n", 784 | "display(HTML(f\"\"))" 785 | ], 786 | "execution_count": 26, 787 | "outputs": [ 788 | { 789 | "output_type": "display_data", 790 | "data": { 791 | "text/html": [ 792 | "" 793 | ], 794 | "text/plain": [ 795 | "" 796 | ] 797 | }, 798 | "metadata": { 799 | "tags": [] 800 | } 801 | } 802 | ] 803 | }, 804 | { 805 | "metadata": { 806 | "id": "taTkl_kZRhK_", 807 | "colab_type": "code", 808 | "colab": { 809 | "base_uri": "https://localhost:8080/", 810 | "height": 597 811 | }, 812 | "outputId": "ae990783-da02-4133-cb1c-406e311e647d" 813 | }, 814 | "cell_type": "code", 815 | "source": [ 816 | "url = reconstructions_single_layer(children(FV.model)[6][3].conv1,'Layer 3 Block 4 Conv1',\n", 817 | " list(range(6,12)),n_cols=3,\n", 818 | " save_fig=True,album_hash='OhQ94rP')\n", 819 | "display(HTML(f\"\"))" 820 | ], 821 | "execution_count": 27, 822 | "outputs": [ 823 | { 824 | "output_type": "display_data", 825 | "data": { 826 | "text/html": [ 827 | "" 828 | ], 829 | "text/plain": [ 830 | "" 831 | ] 832 | }, 833 | "metadata": { 834 | "tags": [] 835 | } 836 | } 837 | ] 838 | }, 839 | { 840 | "metadata": { 841 | "id": "MLWtXSdeRhaH", 842 | "colab_type": "code", 843 | "colab": { 844 | "base_uri": "https://localhost:8080/", 845 | "height": 597 846 | }, 847 | "outputId": "9075d228-a1b5-4103-a233-f131f6e037d1" 848 | }, 849 | "cell_type": "code", 850 | "source": [ 851 | "url = reconstructions_single_layer(children(FV.model)[6][4].conv1,'Layer 3 Block 5 Conv1',\n", 852 | " list(range(6,12)),n_cols=3,\n", 853 | " save_fig=True,album_hash='OhQ94rP')\n", 854 | "display(HTML(f\"\"))" 855 | ], 856 | "execution_count": 28, 857 | "outputs": [ 858 | { 859 | "output_type": "display_data", 860 | "data": { 861 | "text/html": [ 862 | "" 863 | ], 864 | "text/plain": [ 865 | "" 866 | ] 867 | }, 868 | "metadata": { 869 | "tags": [] 870 | } 871 | } 872 | ] 873 | }, 874 | { 875 | "metadata": { 876 | "id": "u5EclQ3eRhXi", 877 | "colab_type": "code", 878 | "colab": { 879 | "base_uri": "https://localhost:8080/", 880 | "height": 597 881 | }, 882 | "outputId": "61f1a33b-3797-4d51-a763-fd93018091f5" 883 | }, 884 | "cell_type": "code", 885 | "source": [ 886 | "url = reconstructions_single_layer(children(FV.model)[6][5].conv1,'Layer 3 Block 6 Conv1',\n", 887 | " list(range(6,12)),n_cols=3,\n", 888 | " save_fig=True,album_hash='OhQ94rP')\n", 889 | "display(HTML(f\"\"))" 890 | ], 891 | "execution_count": 29, 892 | "outputs": [ 893 | { 894 | "output_type": "display_data", 895 | "data": { 896 | "text/html": [ 897 | "" 898 | ], 899 | "text/plain": [ 900 | "" 901 | ] 902 | }, 903 | "metadata": { 904 | "tags": [] 905 | } 906 | } 907 | ] 908 | }, 909 | { 910 | "metadata": { 911 | "id": "UAO7e1llRhUu", 912 | "colab_type": "code", 913 | "colab": { 914 | "base_uri": "https://localhost:8080/", 915 | "height": 597 916 | }, 917 | "outputId": "06644c70-b734-4373-fe68-50fa1052f5d2" 918 | }, 919 | "cell_type": "code", 920 | "source": [ 921 | "url = reconstructions_single_layer(children(FV.model)[7][0].conv1,'Layer 4 Block 1 Conv1',\n", 922 | " list(range(6,12)),n_cols=3,\n", 923 | " save_fig=True,album_hash='OhQ94rP')\n", 924 | "display(HTML(f\"\"))" 925 | ], 926 | "execution_count": 30, 927 | "outputs": [ 928 | { 929 | "output_type": "display_data", 930 | "data": { 931 | "text/html": [ 932 | "" 933 | ], 934 | "text/plain": [ 935 | "" 936 | ] 937 | }, 938 | "metadata": { 939 | "tags": [] 940 | } 941 | } 942 | ] 943 | }, 944 | { 945 | "metadata": { 946 | "id": "GvQF0E9zRhRZ", 947 | "colab_type": "code", 948 | "colab": { 949 | "base_uri": "https://localhost:8080/", 950 | "height": 597 951 | }, 952 | "outputId": "5c77c133-acaf-46a4-8696-d67e3d9b03d5" 953 | }, 954 | "cell_type": "code", 955 | "source": [ 956 | "url = reconstructions_single_layer(children(FV.model)[7][1].conv1,'Layer 4 Block 2 Conv1',\n", 957 | " list(range(6,12)),n_cols=3,\n", 958 | " save_fig=True,album_hash='OhQ94rP')\n", 959 | "display(HTML(f\"\"))" 960 | ], 961 | "execution_count": 31, 962 | "outputs": [ 963 | { 964 | "output_type": "display_data", 965 | "data": { 966 | "text/html": [ 967 | "" 968 | ], 969 | "text/plain": [ 970 | "" 971 | ] 972 | }, 973 | "metadata": { 974 | "tags": [] 975 | } 976 | } 977 | ] 978 | }, 979 | { 980 | "metadata": { 981 | "id": "6nQAU6_PRhOy", 982 | "colab_type": "code", 983 | "colab": { 984 | "base_uri": "https://localhost:8080/", 985 | "height": 597 986 | }, 987 | "outputId": "7c4ea959-4313-417a-b5a6-88d621668d57" 988 | }, 989 | "cell_type": "code", 990 | "source": [ 991 | "url = reconstructions_single_layer(children(FV.model)[7][2].conv1,'Layer 4 Block 3 Conv1',\n", 992 | " list(range(6,12)),n_cols=3,\n", 993 | " save_fig=True,album_hash='OhQ94rP')\n", 994 | "display(HTML(f\"\"))" 995 | ], 996 | "execution_count": 32, 997 | "outputs": [ 998 | { 999 | "output_type": "display_data", 1000 | "data": { 1001 | "text/html": [ 1002 | "" 1003 | ], 1004 | "text/plain": [ 1005 | "" 1006 | ] 1007 | }, 1008 | "metadata": { 1009 | "tags": [] 1010 | } 1011 | } 1012 | ] 1013 | }, 1014 | { 1015 | "metadata": { 1016 | "id": "btjfCyaZSxUJ", 1017 | "colab_type": "text" 1018 | }, 1019 | "cell_type": "markdown", 1020 | "source": [ 1021 | "More images can be seen [here](https://imgur.com/a/OhQ94rP)." 1022 | ] 1023 | } 1024 | ] 1025 | } --------------------------------------------------------------------------------