├── Gated_Fusion_Transformer ├── .ipynb_checkpoints │ ├── test - 副本-checkpoint.ipynb │ └── test-checkpoint.ipynb ├── model_zoo │ └── super_resolution │ │ └── pretrain.pth ├── models │ ├── SR.c │ ├── SR.cp37-win_amd64.pyd │ ├── __pycache__ │ │ └── network_swinir.cpython-37.pyc │ └── build │ │ ├── lib.win-amd64-cpython-37 │ │ └── SR.cp37-win_amd64.pyd │ │ └── temp.win-amd64-cpython-37 │ │ └── Release │ │ ├── SR.cp37-win_amd64.exp │ │ ├── SR.cp37-win_amd64.lib │ │ └── SR.obj ├── test.ipynb ├── testdata │ ├── input │ │ ├── 1.tiff │ │ ├── 10.tiff │ │ ├── 11.tiff │ │ ├── 12.tiff │ │ ├── 13.tiff │ │ ├── 14.tiff │ │ ├── 15.tiff │ │ ├── 16.tiff │ │ ├── 17.tiff │ │ ├── 18.tiff │ │ ├── 19.tiff │ │ ├── 2.tiff │ │ ├── 20.tiff │ │ ├── 21.tiff │ │ ├── 22.tiff │ │ ├── 23.tiff │ │ ├── 3.tiff │ │ ├── 4.tiff │ │ ├── 5.tiff │ │ ├── 6.tiff │ │ ├── 7.tiff │ │ ├── 8.tiff │ │ └── 9.tiff │ └── output │ │ ├── 1.tiff.jpg │ │ ├── 10.tiff.jpg │ │ ├── 11.tiff.jpg │ │ ├── 12.tiff.jpg │ │ ├── 13.tiff.jpg │ │ ├── 14.tiff.jpg │ │ ├── 15.tiff.jpg │ │ ├── 16.tiff.jpg │ │ ├── 17.tiff.jpg │ │ ├── 18.tiff.jpg │ │ ├── 19.tiff.jpg │ │ ├── 2.tiff.jpg │ │ ├── 20.tiff.jpg │ │ ├── 21.tiff.jpg │ │ ├── 22.tiff.jpg │ │ ├── 23.tiff.jpg │ │ ├── 3.tiff.jpg │ │ ├── 4.tiff.jpg │ │ ├── 5.tiff.jpg │ │ ├── 6.tiff.jpg │ │ ├── 7.tiff.jpg │ │ ├── 8.tiff.jpg │ │ └── 9.tiff.jpg └── utils │ ├── __pycache__ │ └── util_calculate_psnr_ssim.cpython-37.pyc │ └── util_calculate_psnr_ssim.py ├── README.md ├── images ├── 20mW_256frames_1.jpg ├── 20mW_32frames_1.jpg ├── figure5.png └── target.jpg ├── requirements.txt └── synthesis_images ├── count_dark.mat ├── frame_si.m ├── input └── target.jpg ├── loadImages.m ├── main.m ├── output └── test │ ├── 10mW_1000frames_1.tiff │ ├── 10mW_128frames_1.tiff │ ├── 10mW_16frames_1.tiff │ ├── 10mW_1frames_1.tiff │ ├── 10mW_256frames_1.tiff │ ├── 10mW_2frames_1.tiff │ ├── 10mW_32frames_1.tiff │ ├── 10mW_4frames_1.tiff │ ├── 10mW_512frames_1.tiff │ ├── 10mW_64frames_1.tiff │ ├── 10mW_8frames_1.tiff │ ├── 20mW_1000frames_1.tiff │ ├── 20mW_128frames_1.tiff │ ├── 20mW_16frames_1.tiff │ ├── 20mW_1frames_1.tiff │ ├── 20mW_256frames_1.tiff │ ├── 20mW_2frames_1.tiff │ ├── 20mW_32frames_1.tiff │ ├── 20mW_4frames_1.tiff │ ├── 20mW_512frames_1.tiff │ ├── 20mW_64frames_1.tiff │ ├── 20mW_8frames_1.tiff │ ├── 40mW_1000frames_1.tiff │ ├── 40mW_128frames_1.tiff │ ├── 40mW_16frames_1.tiff │ ├── 40mW_1frames_1.tiff │ ├── 40mW_256frames_1.tiff │ ├── 40mW_2frames_1.tiff │ ├── 40mW_32frames_1.tiff │ ├── 40mW_4frames_1.tiff │ ├── 40mW_512frames_1.tiff │ ├── 40mW_64frames_1.tiff │ └── 40mW_8frames_1.tiff ├── processImage.m └── processScaleAndSubframes.m /Gated_Fusion_Transformer/.ipynb_checkpoints/test - 副本-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "70da4955", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import argparse\n", 11 | "import cv2\n", 12 | "import glob\n", 13 | "import numpy as np\n", 14 | "from collections import OrderedDict\n", 15 | "import os\n", 16 | "import torch\n", 17 | "import requests\n", 18 | "\n", 19 | "from models.network_swinir import SwinIR as net\n", 20 | "from utils import util_calculate_psnr_ssim as util" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "id": "176f49b9", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "task=\"real_sr\"\n", 31 | "scale=4\n", 32 | "noise=15\n", 33 | "jpeg=40\n", 34 | "training_patch_size=128\n", 35 | "large_model=False\n", 36 | "model_path='./model_zoo/swinir/003_realSR_BSRGAN_DFO_s64w8_SwinIR-M_x4_GAN.pth'\n", 37 | "folder_lq='./test/'\n", 38 | "tile=400\n", 39 | "tile_overlap=32\n", 40 | "save_dir='./results/'\n", 41 | "folder=folder_lq\n", 42 | "border = 0\n", 43 | "window_size = 8" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 3, 49 | "id": "3630b296", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "id": "59f8075c", 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "def define_model(task,scale,large_model,model_path):\n", 64 | " # 001 classical image sr\n", 65 | " if task == 'classical_sr':\n", 66 | " model = net(upscale=scale, in_chans=3, img_size=training_patch_size, window_size=8,\n", 67 | " img_range=1., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 68 | " mlp_ratio=2, upsampler='pixelshuffle', resi_connection='1conv')\n", 69 | " param_key_g = 'params'\n", 70 | "\n", 71 | " # 002 lightweight image sr\n", 72 | " # use 'pixelshuffledirect' to save parameters\n", 73 | " elif task == 'lightweight_sr':\n", 74 | " model = net(upscale=scale, in_chans=3, img_size=64, window_size=8,\n", 75 | " img_range=1., depths=[6, 6, 6, 6], embed_dim=60, num_heads=[6, 6, 6, 6],\n", 76 | " mlp_ratio=2, upsampler='pixelshuffledirect', resi_connection='1conv')\n", 77 | " param_key_g = 'params'\n", 78 | "\n", 79 | " # 003 real-world image sr\n", 80 | " elif task == 'real_sr':\n", 81 | " if not large_model:\n", 82 | " # use 'nearest+conv' to avoid block artifacts\n", 83 | " model = net(upscale=4, in_chans=3, img_size=64, window_size=8,\n", 84 | " img_range=1., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 85 | " mlp_ratio=2, upsampler='nearest+conv', resi_connection='1conv')\n", 86 | " else:\n", 87 | " # larger model size; use '3conv' to save parameters and memory; use ema for GAN training\n", 88 | " model = net(upscale=4, in_chans=3, img_size=64, window_size=8,\n", 89 | " img_range=1., depths=[6, 6, 6, 6, 6, 6, 6, 6, 6], embed_dim=240,\n", 90 | " num_heads=[8, 8, 8, 8, 8, 8, 8, 8, 8],\n", 91 | " mlp_ratio=2, upsampler='nearest+conv', resi_connection='3conv')\n", 92 | " param_key_g = 'params_ema'\n", 93 | "\n", 94 | " # 004 grayscale image denoising\n", 95 | " elif task == 'gray_dn':\n", 96 | " model = net(upscale=1, in_chans=1, img_size=128, window_size=8,\n", 97 | " img_range=1., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 98 | " mlp_ratio=2, upsampler='', resi_connection='1conv')\n", 99 | " param_key_g = 'params'\n", 100 | "\n", 101 | " # 005 color image denoising\n", 102 | " elif task == 'color_dn':\n", 103 | " model = net(upscale=1, in_chans=3, img_size=128, window_size=8,\n", 104 | " img_range=1., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 105 | " mlp_ratio=2, upsampler='', resi_connection='1conv')\n", 106 | " param_key_g = 'params'\n", 107 | "\n", 108 | " # 006 JPEG compression artifact reduction\n", 109 | " # use window_size=7 because JPEG encoding uses 8x8; use img_range=255 because it's sligtly better than 1\n", 110 | " elif task == 'jpeg_car':\n", 111 | " model = net(upscale=1, in_chans=1, img_size=126, window_size=7,\n", 112 | " img_range=255., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 113 | " mlp_ratio=2, upsampler='', resi_connection='1conv')\n", 114 | " param_key_g = 'params'\n", 115 | "\n", 116 | " pretrained_model = torch.load(model_path)\n", 117 | " model.load_state_dict(pretrained_model[param_key_g] if param_key_g in pretrained_model.keys() else pretrained_model, strict=True)\n", 118 | "\n", 119 | " return model" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 5, 125 | "id": "98cb1177", 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "def test(img_lq,tile,tile_overlap, model, window_size, scale):\n", 130 | " if tile is None:\n", 131 | " # test the image as a whole\n", 132 | " output = model(img_lq)\n", 133 | " else:\n", 134 | " # test the image tile by tile\n", 135 | " b, c, h, w = img_lq.size()\n", 136 | " tile = min(tile, h, w)\n", 137 | " assert tile % window_size == 0, \"tile size should be a multiple of window_size\"\n", 138 | " tile_overlap = tile_overlap\n", 139 | " sf = scale\n", 140 | "\n", 141 | " stride = tile - tile_overlap\n", 142 | " h_idx_list = list(range(0, h-tile, stride)) + [h-tile]\n", 143 | " w_idx_list = list(range(0, w-tile, stride)) + [w-tile]\n", 144 | " E = torch.zeros(b, c, h*sf, w*sf).type_as(img_lq)\n", 145 | " W = torch.zeros_like(E)\n", 146 | "\n", 147 | " for h_idx in h_idx_list:\n", 148 | " for w_idx in w_idx_list:\n", 149 | " in_patch = img_lq[..., h_idx:h_idx+tile, w_idx:w_idx+tile]\n", 150 | " out_patch = model(in_patch)\n", 151 | " out_patch_mask = torch.ones_like(out_patch)\n", 152 | "\n", 153 | " E[..., h_idx*sf:(h_idx+tile)*sf, w_idx*sf:(w_idx+tile)*sf].add_(out_patch)\n", 154 | " W[..., h_idx*sf:(h_idx+tile)*sf, w_idx*sf:(w_idx+tile)*sf].add_(out_patch_mask)\n", 155 | " output = E.div_(W)\n", 156 | "\n", 157 | " return output" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 6, 163 | "id": "cbcc8c63", 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "model = define_model(task,scale,large_model,model_path)\n", 168 | "model.eval()\n", 169 | "model = model.to(device)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 7, 175 | "id": "20e62637", 176 | "metadata": {}, 177 | "outputs": [ 178 | { 179 | "name": "stdout", 180 | "output_type": "stream", 181 | "text": [ 182 | "0\n", 183 | "./testsets/GT\\1.tiff\n", 184 | "1\n", 185 | "./testsets/GT\\10.tiff\n", 186 | "2\n", 187 | "./testsets/GT\\11.tiff\n", 188 | "3\n", 189 | "./testsets/GT\\12.tiff\n", 190 | "4\n", 191 | "./testsets/GT\\13.tiff\n", 192 | "5\n", 193 | "./testsets/GT\\14.tiff\n", 194 | "6\n", 195 | "./testsets/GT\\15.tiff\n", 196 | "7\n", 197 | "./testsets/GT\\16.tiff\n", 198 | "8\n", 199 | "./testsets/GT\\17.tiff\n", 200 | "9\n", 201 | "./testsets/GT\\18.tiff\n", 202 | "10\n", 203 | "./testsets/GT\\2.tiff\n", 204 | "11\n", 205 | "./testsets/GT\\3.tiff\n", 206 | "12\n", 207 | "./testsets/GT\\4.tiff\n", 208 | "13\n", 209 | "./testsets/GT\\5.tiff\n", 210 | "14\n", 211 | "./testsets/GT\\6.tiff\n", 212 | "15\n", 213 | "./testsets/GT\\7.tiff\n", 214 | "16\n", 215 | "./testsets/GT\\8.tiff\n", 216 | "17\n", 217 | "./testsets/GT\\9.tiff\n" 218 | ] 219 | } 220 | ], 221 | "source": [ 222 | "i=1\n", 223 | "for idx, path in enumerate(sorted(glob.glob(os.path.join(folder, '*')))):\n", 224 | " # read image\n", 225 | " print(idx)\n", 226 | " img_lq = cv2.imread(path, cv2.IMREAD_COLOR).astype(np.float32) / 255.\n", 227 | " print(path)\n", 228 | " img_lq = np.transpose(img_lq if img_lq.shape[2] == 1 else img_lq[:, :, [2, 1, 0]], (2, 0, 1)) # HCW-BGR to CHW-RGB\n", 229 | " img_lq = torch.from_numpy(img_lq).float().unsqueeze(0).to(device) # CHW-RGB to NCHW-RGB\n", 230 | " # inference\n", 231 | " with torch.no_grad():\n", 232 | " # pad input image to be a multiple of window_size\n", 233 | " _, _, h_old, w_old = img_lq.size()\n", 234 | " h_pad = (h_old // window_size + 1) * window_size - h_old\n", 235 | " w_pad = (w_old // window_size + 1) * window_size - w_old\n", 236 | " img_lq = torch.cat([img_lq, torch.flip(img_lq, [2])], 2)[:, :, :h_old + h_pad, :]\n", 237 | " img_lq = torch.cat([img_lq, torch.flip(img_lq, [3])], 3)[:, :, :, :w_old + w_pad]\n", 238 | " output = test(img_lq,tile,tile_overlap, model, window_size, scale)\n", 239 | " output = output[..., :h_old * scale, :w_old * scale]\n", 240 | " # save image\n", 241 | " output = output.data.squeeze().float().cpu().clamp_(0, 1).numpy()\n", 242 | " if output.ndim == 3:\n", 243 | " output = np.transpose(output[[2, 1, 0], :, :], (1, 2, 0)) # CHW-RGB to HCW-BGR\n", 244 | " output = (output * 255.0).round().astype(np.uint8) # float32 to uint8\n", 245 | " cv2.imwrite(f'{save_dir}/%d.jpg'%i, output)\n", 246 | " i=i+1" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "id": "0b1f86d5", 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": null, 260 | "id": "f44f2e17", 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "id": "f6537d39", 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "id": "ef13686d", 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": null, 284 | "id": "ef921ffb", 285 | "metadata": {}, 286 | "outputs": [], 287 | "source": [] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "id": "ebdd017e", 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": null, 300 | "id": "06740d75", 301 | "metadata": {}, 302 | "outputs": [], 303 | "source": [] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "53d8a60f", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": null, 316 | "id": "4dd2507a", 317 | "metadata": {}, 318 | "outputs": [], 319 | "source": [] 320 | }, 321 | { 322 | "cell_type": "code", 323 | "execution_count": null, 324 | "id": "6161f23c", 325 | "metadata": {}, 326 | "outputs": [], 327 | "source": [] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "id": "6f92456a", 333 | "metadata": {}, 334 | "outputs": [], 335 | "source": [] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "id": "813ac7ad", 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": null, 348 | "id": "ce3de71b", 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": null, 356 | "id": "89cca74c", 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": null, 364 | "id": "bce91e67", 365 | "metadata": {}, 366 | "outputs": [], 367 | "source": [] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": null, 372 | "id": "49ebadc9", 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": null, 380 | "id": "032d2057", 381 | "metadata": {}, 382 | "outputs": [], 383 | "source": [] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": null, 388 | "id": "072e79db", 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": null, 396 | "id": "d1f0d1db", 397 | "metadata": {}, 398 | "outputs": [], 399 | "source": [] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": null, 404 | "id": "6176c0c7", 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": null, 412 | "id": "08fedae0", 413 | "metadata": {}, 414 | "outputs": [], 415 | "source": [] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "id": "8f888986", 421 | "metadata": {}, 422 | "outputs": [], 423 | "source": [] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": null, 428 | "id": "db2403bf", 429 | "metadata": {}, 430 | "outputs": [], 431 | "source": [] 432 | }, 433 | { 434 | "cell_type": "code", 435 | "execution_count": null, 436 | "id": "30997470", 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "id": "531d23a0", 445 | "metadata": {}, 446 | "outputs": [], 447 | "source": [] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": null, 452 | "id": "63e3db54", 453 | "metadata": {}, 454 | "outputs": [], 455 | "source": [] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": null, 460 | "id": "8cfb1f96", 461 | "metadata": {}, 462 | "outputs": [], 463 | "source": [] 464 | }, 465 | { 466 | "cell_type": "code", 467 | "execution_count": null, 468 | "id": "5ec64270", 469 | "metadata": {}, 470 | "outputs": [], 471 | "source": [] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": null, 476 | "id": "8d6c692a", 477 | "metadata": {}, 478 | "outputs": [], 479 | "source": [] 480 | }, 481 | { 482 | "cell_type": "code", 483 | "execution_count": null, 484 | "id": "ccb365d5", 485 | "metadata": {}, 486 | "outputs": [], 487 | "source": [] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": null, 492 | "id": "d58fbd3d", 493 | "metadata": {}, 494 | "outputs": [], 495 | "source": [] 496 | }, 497 | { 498 | "cell_type": "code", 499 | "execution_count": null, 500 | "id": "7716ecc9", 501 | "metadata": {}, 502 | "outputs": [], 503 | "source": [] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": null, 508 | "id": "28b426fb", 509 | "metadata": {}, 510 | "outputs": [], 511 | "source": [] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": null, 516 | "id": "51003cd1", 517 | "metadata": {}, 518 | "outputs": [], 519 | "source": [] 520 | }, 521 | { 522 | "cell_type": "code", 523 | "execution_count": null, 524 | "id": "f572fc2f", 525 | "metadata": {}, 526 | "outputs": [], 527 | "source": [] 528 | }, 529 | { 530 | "cell_type": "code", 531 | "execution_count": null, 532 | "id": "be343378", 533 | "metadata": {}, 534 | "outputs": [], 535 | "source": [] 536 | }, 537 | { 538 | "cell_type": "code", 539 | "execution_count": null, 540 | "id": "da69c9d4", 541 | "metadata": {}, 542 | "outputs": [], 543 | "source": [] 544 | }, 545 | { 546 | "cell_type": "code", 547 | "execution_count": null, 548 | "id": "8f6aada8", 549 | "metadata": {}, 550 | "outputs": [], 551 | "source": [] 552 | }, 553 | { 554 | "cell_type": "code", 555 | "execution_count": null, 556 | "id": "5e13b74b", 557 | "metadata": {}, 558 | "outputs": [], 559 | "source": [] 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": null, 564 | "id": "742308e1", 565 | "metadata": {}, 566 | "outputs": [], 567 | "source": [] 568 | }, 569 | { 570 | "cell_type": "code", 571 | "execution_count": null, 572 | "id": "b16ea156", 573 | "metadata": {}, 574 | "outputs": [], 575 | "source": [] 576 | }, 577 | { 578 | "cell_type": "code", 579 | "execution_count": null, 580 | "id": "02108bd2", 581 | "metadata": {}, 582 | "outputs": [], 583 | "source": [] 584 | }, 585 | { 586 | "cell_type": "code", 587 | "execution_count": null, 588 | "id": "081914c7", 589 | "metadata": {}, 590 | "outputs": [], 591 | "source": [] 592 | }, 593 | { 594 | "cell_type": "code", 595 | "execution_count": null, 596 | "id": "ea07be57", 597 | "metadata": {}, 598 | "outputs": [], 599 | "source": [] 600 | }, 601 | { 602 | "cell_type": "code", 603 | "execution_count": null, 604 | "id": "df011057", 605 | "metadata": {}, 606 | "outputs": [], 607 | "source": [] 608 | }, 609 | { 610 | "cell_type": "code", 611 | "execution_count": null, 612 | "id": "4177e42f", 613 | "metadata": {}, 614 | "outputs": [], 615 | "source": [] 616 | } 617 | ], 618 | "metadata": { 619 | "kernelspec": { 620 | "display_name": "Python 3 (ipykernel)", 621 | "language": "python", 622 | "name": "python3" 623 | }, 624 | "language_info": { 625 | "codemirror_mode": { 626 | "name": "ipython", 627 | "version": 3 628 | }, 629 | "file_extension": ".py", 630 | "mimetype": "text/x-python", 631 | "name": "python", 632 | "nbconvert_exporter": "python", 633 | "pygments_lexer": "ipython3", 634 | "version": "3.7.13" 635 | } 636 | }, 637 | "nbformat": 4, 638 | "nbformat_minor": 5 639 | } 640 | -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/.ipynb_checkpoints/test-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 23, 6 | "id": "70da4955", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import argparse\n", 11 | "import cv2\n", 12 | "import glob\n", 13 | "import numpy as np\n", 14 | "from collections import OrderedDict\n", 15 | "import os\n", 16 | "import torch\n", 17 | "import requests\n", 18 | "\n", 19 | "from models.SR import SR as net\n", 20 | "from utils import util_calculate_psnr_ssim as util" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 24, 26 | "id": "176f49b9", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "task=\"real_sr\"\n", 31 | "scale=4\n", 32 | "noise=15\n", 33 | "jpeg=40\n", 34 | "training_patch_size=128\n", 35 | "large_model=False\n", 36 | "model_path='./model_zoo/super_resolution/pretrain.pth'\n", 37 | "folder_lq='./input/'\n", 38 | "tile=400\n", 39 | "tile_overlap=32\n", 40 | "save_dir='./output/'\n", 41 | "folder=folder_lq\n", 42 | "border = 0\n", 43 | "window_size = 8" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 25, 49 | "id": "3630b296", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "torch.cuda.set_device(0)\n", 54 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 26, 60 | "id": "59f8075c", 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "def define_model(task,scale,large_model,model_path):\n", 65 | " # 003 real-world image sr\n", 66 | " if task == 'real_sr':\n", 67 | " if not large_model:\n", 68 | " # use 'nearest+conv' to avoid block artifacts\n", 69 | " model = net(upscale=4, in_chans=3, img_size=64, window_size=8,\n", 70 | " img_range=1., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 71 | " mlp_ratio=2, upsampler='nearest+conv', resi_connection='1conv')\n", 72 | " else:\n", 73 | " # larger model size; use '3conv' to save parameters and memory; use ema for GAN training\n", 74 | " model = net(upscale=4, in_chans=3, img_size=64, window_size=8,\n", 75 | " img_range=1., depths=[6, 6, 6, 6, 6, 6, 6, 6, 6], embed_dim=240,\n", 76 | " num_heads=[8, 8, 8, 8, 8, 8, 8, 8, 8],\n", 77 | " mlp_ratio=2, upsampler='nearest+conv', resi_connection='3conv')\n", 78 | " param_key_g = 'params_ema'\n", 79 | "\n", 80 | "\n", 81 | " pretrained_model = torch.load(model_path)\n", 82 | " model.load_state_dict(pretrained_model[param_key_g] if param_key_g in pretrained_model.keys() else pretrained_model, strict=True)\n", 83 | "\n", 84 | " return model" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 27, 90 | "id": "98cb1177", 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "def test(img_lq,tile,tile_overlap, model, window_size, scale):\n", 95 | " if tile is None:\n", 96 | " # test the image as a whole\n", 97 | " output = model(img_lq)\n", 98 | " else:\n", 99 | " # test the image tile by tile\n", 100 | " b, c, h, w = img_lq.size()\n", 101 | " tile = min(tile, h, w)\n", 102 | " assert tile % window_size == 0, \"tile size should be a multiple of window_size\"\n", 103 | " tile_overlap = tile_overlap\n", 104 | " sf = scale\n", 105 | "\n", 106 | " stride = tile - tile_overlap\n", 107 | " h_idx_list = list(range(0, h-tile, stride)) + [h-tile]\n", 108 | " w_idx_list = list(range(0, w-tile, stride)) + [w-tile]\n", 109 | " E = torch.zeros(b, c, h*sf, w*sf).type_as(img_lq)\n", 110 | " W = torch.zeros_like(E)\n", 111 | "\n", 112 | " for h_idx in h_idx_list:\n", 113 | " for w_idx in w_idx_list:\n", 114 | " in_patch = img_lq[..., h_idx:h_idx+tile, w_idx:w_idx+tile]\n", 115 | " out_patch = model(in_patch)\n", 116 | " out_patch_mask = torch.ones_like(out_patch)\n", 117 | "\n", 118 | " E[..., h_idx*sf:(h_idx+tile)*sf, w_idx*sf:(w_idx+tile)*sf].add_(out_patch)\n", 119 | " W[..., h_idx*sf:(h_idx+tile)*sf, w_idx*sf:(w_idx+tile)*sf].add_(out_patch_mask)\n", 120 | " output = E.div_(W)\n", 121 | "\n", 122 | " return output" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 28, 128 | "id": "cbcc8c63", 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "model = define_model(task,scale,large_model,model_path)\n", 133 | "model.eval()\n", 134 | "model = model.to(device)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 29, 140 | "id": "20e62637", 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "1.jpg\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "path_list = os.listdir(folder)\n", 153 | "path_list.sort(key=lambda x:int(x.split('.')[0]))\n", 154 | "for item in path_list:\n", 155 | " impath=folder+item\n", 156 | " # read image\n", 157 | " print(item)\n", 158 | " img_lq = cv2.imread(impath, cv2.IMREAD_COLOR).astype(np.float32) / 255.\n", 159 | " img_lq = np.transpose(img_lq if img_lq.shape[2] == 1 else img_lq[:, :, [2, 1, 0]], (2, 0, 1)) # HCW-BGR to CHW-RGB\n", 160 | " img_lq = torch.from_numpy(img_lq).float().unsqueeze(0).to(device) # CHW-RGB to NCHW-RGB\n", 161 | " # inference\n", 162 | " with torch.no_grad():\n", 163 | " # pad input image to be a multiple of window_size\n", 164 | " _, _, h_old, w_old = img_lq.size()\n", 165 | " h_pad = (h_old // window_size + 1) * window_size - h_old\n", 166 | " w_pad = (w_old // window_size + 1) * window_size - w_old\n", 167 | " img_lq = torch.cat([img_lq, torch.flip(img_lq, [2])], 2)[:, :, :h_old + h_pad, :]\n", 168 | " img_lq = torch.cat([img_lq, torch.flip(img_lq, [3])], 3)[:, :, :, :w_old + w_pad]\n", 169 | " output = test(img_lq,tile,tile_overlap, model, window_size, scale)\n", 170 | " output = output[..., :h_old * scale, :w_old * scale]\n", 171 | " # save image\n", 172 | " output = output.data.squeeze().float().cpu().clamp_(0, 1).numpy()\n", 173 | " if output.ndim == 3:\n", 174 | " output = np.transpose(output[[2, 1, 0], :, :], (1, 2, 0)) # CHW-RGB to HCW-BGR\n", 175 | " output = (output * 255.0).round().astype(np.uint8) # float32 to uint8\n", 176 | " cv2.imwrite(f'{save_dir}/%s.jpg'%item, output)\n" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "id": "0b1f86d5", 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "id": "f44f2e17", 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "id": "f6537d39", 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "id": "ef13686d", 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "id": "ef921ffb", 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "ebdd017e", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "id": "06740d75", 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "id": "53d8a60f", 239 | "metadata": {}, 240 | "outputs": [], 241 | "source": [] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "id": "4dd2507a", 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": null, 254 | "id": "6161f23c", 255 | "metadata": {}, 256 | "outputs": [], 257 | "source": [] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "id": "6f92456a", 263 | "metadata": {}, 264 | "outputs": [], 265 | "source": [] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "id": "813ac7ad", 271 | "metadata": {}, 272 | "outputs": [], 273 | "source": [] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "id": "ce3de71b", 279 | "metadata": {}, 280 | "outputs": [], 281 | "source": [] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "id": "89cca74c", 287 | "metadata": {}, 288 | "outputs": [], 289 | "source": [] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": null, 294 | "id": "bce91e67", 295 | "metadata": {}, 296 | "outputs": [], 297 | "source": [] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": null, 302 | "id": "49ebadc9", 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": null, 310 | "id": "032d2057", 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "id": "072e79db", 319 | "metadata": {}, 320 | "outputs": [], 321 | "source": [] 322 | }, 323 | { 324 | "cell_type": "code", 325 | "execution_count": null, 326 | "id": "d1f0d1db", 327 | "metadata": {}, 328 | "outputs": [], 329 | "source": [] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": null, 334 | "id": "6176c0c7", 335 | "metadata": {}, 336 | "outputs": [], 337 | "source": [] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "id": "08fedae0", 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": null, 350 | "id": "8f888986", 351 | "metadata": {}, 352 | "outputs": [], 353 | "source": [] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "id": "db2403bf", 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": null, 366 | "id": "30997470", 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "execution_count": null, 374 | "id": "531d23a0", 375 | "metadata": {}, 376 | "outputs": [], 377 | "source": [] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "id": "63e3db54", 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": null, 390 | "id": "8cfb1f96", 391 | "metadata": {}, 392 | "outputs": [], 393 | "source": [] 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": null, 398 | "id": "5ec64270", 399 | "metadata": {}, 400 | "outputs": [], 401 | "source": [] 402 | }, 403 | { 404 | "cell_type": "code", 405 | "execution_count": null, 406 | "id": "8d6c692a", 407 | "metadata": {}, 408 | "outputs": [], 409 | "source": [] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": null, 414 | "id": "ccb365d5", 415 | "metadata": {}, 416 | "outputs": [], 417 | "source": [] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": null, 422 | "id": "d58fbd3d", 423 | "metadata": {}, 424 | "outputs": [], 425 | "source": [] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": null, 430 | "id": "7716ecc9", 431 | "metadata": {}, 432 | "outputs": [], 433 | "source": [] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": null, 438 | "id": "28b426fb", 439 | "metadata": {}, 440 | "outputs": [], 441 | "source": [] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": null, 446 | "id": "51003cd1", 447 | "metadata": {}, 448 | "outputs": [], 449 | "source": [] 450 | }, 451 | { 452 | "cell_type": "code", 453 | "execution_count": null, 454 | "id": "f572fc2f", 455 | "metadata": {}, 456 | "outputs": [], 457 | "source": [] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "id": "be343378", 463 | "metadata": {}, 464 | "outputs": [], 465 | "source": [] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": null, 470 | "id": "da69c9d4", 471 | "metadata": {}, 472 | "outputs": [], 473 | "source": [] 474 | }, 475 | { 476 | "cell_type": "code", 477 | "execution_count": null, 478 | "id": "8f6aada8", 479 | "metadata": {}, 480 | "outputs": [], 481 | "source": [] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": null, 486 | "id": "5e13b74b", 487 | "metadata": {}, 488 | "outputs": [], 489 | "source": [] 490 | }, 491 | { 492 | "cell_type": "code", 493 | "execution_count": null, 494 | "id": "742308e1", 495 | "metadata": {}, 496 | "outputs": [], 497 | "source": [] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": null, 502 | "id": "b16ea156", 503 | "metadata": {}, 504 | "outputs": [], 505 | "source": [] 506 | }, 507 | { 508 | "cell_type": "code", 509 | "execution_count": null, 510 | "id": "02108bd2", 511 | "metadata": {}, 512 | "outputs": [], 513 | "source": [] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "execution_count": null, 518 | "id": "081914c7", 519 | "metadata": {}, 520 | "outputs": [], 521 | "source": [] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": null, 526 | "id": "ea07be57", 527 | "metadata": {}, 528 | "outputs": [], 529 | "source": [] 530 | }, 531 | { 532 | "cell_type": "code", 533 | "execution_count": null, 534 | "id": "df011057", 535 | "metadata": {}, 536 | "outputs": [], 537 | "source": [] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": null, 542 | "id": "4177e42f", 543 | "metadata": {}, 544 | "outputs": [], 545 | "source": [] 546 | } 547 | ], 548 | "metadata": { 549 | "kernelspec": { 550 | "display_name": "Python 3 (ipykernel)", 551 | "language": "python", 552 | "name": "python3" 553 | }, 554 | "language_info": { 555 | "codemirror_mode": { 556 | "name": "ipython", 557 | "version": 3 558 | }, 559 | "file_extension": ".py", 560 | "mimetype": "text/x-python", 561 | "name": "python", 562 | "nbconvert_exporter": "python", 563 | "pygments_lexer": "ipython3", 564 | "version": "3.7.11" 565 | } 566 | }, 567 | "nbformat": 4, 568 | "nbformat_minor": 5 569 | } 570 | -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/model_zoo/super_resolution/pretrain.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/model_zoo/super_resolution/pretrain.pth -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/models/SR.cp37-win_amd64.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/models/SR.cp37-win_amd64.pyd -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/models/__pycache__/network_swinir.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/models/__pycache__/network_swinir.cpython-37.pyc -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/models/build/lib.win-amd64-cpython-37/SR.cp37-win_amd64.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/models/build/lib.win-amd64-cpython-37/SR.cp37-win_amd64.pyd -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/models/build/temp.win-amd64-cpython-37/Release/SR.cp37-win_amd64.exp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/models/build/temp.win-amd64-cpython-37/Release/SR.cp37-win_amd64.exp -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/models/build/temp.win-amd64-cpython-37/Release/SR.cp37-win_amd64.lib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/models/build/temp.win-amd64-cpython-37/Release/SR.cp37-win_amd64.lib -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/models/build/temp.win-amd64-cpython-37/Release/SR.obj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/models/build/temp.win-amd64-cpython-37/Release/SR.obj -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 8, 6 | "id": "70da4955", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import argparse\n", 11 | "import cv2\n", 12 | "import glob\n", 13 | "import numpy as np\n", 14 | "from collections import OrderedDict\n", 15 | "import os\n", 16 | "import torch\n", 17 | "import requests\n", 18 | "\n", 19 | "from models.SR import SR as net\n", 20 | "from utils import util_calculate_psnr_ssim as util" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 9, 26 | "id": "176f49b9", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "task=\"real_sr\"\n", 31 | "scale=4\n", 32 | "noise=15\n", 33 | "jpeg=40\n", 34 | "training_patch_size=128\n", 35 | "large_model=False\n", 36 | "model_path='./model_zoo/super_resolution/pretrain.pth'\n", 37 | "folder_lq='./testdata/input/'\n", 38 | "tile=400\n", 39 | "tile_overlap=32\n", 40 | "save_dir='./testdata/output/'\n", 41 | "folder=folder_lq\n", 42 | "border = 0\n", 43 | "window_size = 8" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 10, 49 | "id": "3630b296", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "torch.cuda.set_device(0)\n", 54 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 11, 60 | "id": "59f8075c", 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "def define_model(task,scale,large_model,model_path):\n", 65 | " # 003 real-world image sr\n", 66 | " if task == 'real_sr':\n", 67 | " if not large_model:\n", 68 | " # use 'nearest+conv' to avoid block artifacts\n", 69 | " model = net(upscale=4, in_chans=3, img_size=64, window_size=8,\n", 70 | " img_range=1., depths=[6, 6, 6, 6, 6, 6], embed_dim=180, num_heads=[6, 6, 6, 6, 6, 6],\n", 71 | " mlp_ratio=2, upsampler='nearest+conv', resi_connection='1conv')\n", 72 | " else:\n", 73 | " # larger model size; use '3conv' to save parameters and memory; use ema for GAN training\n", 74 | " model = net(upscale=4, in_chans=3, img_size=64, window_size=8,\n", 75 | " img_range=1., depths=[6, 6, 6, 6, 6, 6, 6, 6, 6], embed_dim=240,\n", 76 | " num_heads=[8, 8, 8, 8, 8, 8, 8, 8, 8],\n", 77 | " mlp_ratio=2, upsampler='nearest+conv', resi_connection='3conv')\n", 78 | " param_key_g = 'params_ema'\n", 79 | "\n", 80 | "\n", 81 | " pretrained_model = torch.load(model_path)\n", 82 | " model.load_state_dict(pretrained_model[param_key_g] if param_key_g in pretrained_model.keys() else pretrained_model, strict=True)\n", 83 | "\n", 84 | " return model" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 12, 90 | "id": "98cb1177", 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "def test(img_lq,tile,tile_overlap, model, window_size, scale):\n", 95 | " if tile is None:\n", 96 | " # test the image as a whole\n", 97 | " output = model(img_lq)\n", 98 | " else:\n", 99 | " # test the image tile by tile\n", 100 | " b, c, h, w = img_lq.size()\n", 101 | " tile = min(tile, h, w)\n", 102 | " assert tile % window_size == 0, \"tile size should be a multiple of window_size\"\n", 103 | " tile_overlap = tile_overlap\n", 104 | " sf = scale\n", 105 | "\n", 106 | " stride = tile - tile_overlap\n", 107 | " h_idx_list = list(range(0, h-tile, stride)) + [h-tile]\n", 108 | " w_idx_list = list(range(0, w-tile, stride)) + [w-tile]\n", 109 | " E = torch.zeros(b, c, h*sf, w*sf).type_as(img_lq)\n", 110 | " W = torch.zeros_like(E)\n", 111 | "\n", 112 | " for h_idx in h_idx_list:\n", 113 | " for w_idx in w_idx_list:\n", 114 | " in_patch = img_lq[..., h_idx:h_idx+tile, w_idx:w_idx+tile]\n", 115 | " out_patch = model(in_patch)\n", 116 | " out_patch_mask = torch.ones_like(out_patch)\n", 117 | "\n", 118 | " E[..., h_idx*sf:(h_idx+tile)*sf, w_idx*sf:(w_idx+tile)*sf].add_(out_patch)\n", 119 | " W[..., h_idx*sf:(h_idx+tile)*sf, w_idx*sf:(w_idx+tile)*sf].add_(out_patch_mask)\n", 120 | " output = E.div_(W)\n", 121 | "\n", 122 | " return output" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 13, 128 | "id": "cbcc8c63", 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "model = define_model(task,scale,large_model,model_path)\n", 133 | "model.eval()\n", 134 | "model = model.to(device)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 14, 140 | "id": "20e62637", 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "1.tiff\n", 148 | "2.tiff\n", 149 | "3.tiff\n", 150 | "4.tiff\n", 151 | "5.tiff\n", 152 | "6.tiff\n", 153 | "7.tiff\n", 154 | "8.tiff\n", 155 | "9.tiff\n", 156 | "10.tiff\n", 157 | "11.tiff\n", 158 | "12.tiff\n", 159 | "13.tiff\n", 160 | "14.tiff\n", 161 | "15.tiff\n", 162 | "16.tiff\n", 163 | "17.tiff\n", 164 | "18.tiff\n", 165 | "19.tiff\n", 166 | "20.tiff\n", 167 | "21.tiff\n", 168 | "22.tiff\n", 169 | "23.tiff\n" 170 | ] 171 | } 172 | ], 173 | "source": [ 174 | "path_list = os.listdir(folder)\n", 175 | "path_list.sort(key=lambda x:int(x.split('.')[0]))\n", 176 | "for item in path_list:\n", 177 | " impath=folder+item\n", 178 | " # read image\n", 179 | " print(item)\n", 180 | " img_lq = cv2.imread(impath, cv2.IMREAD_COLOR).astype(np.float32) / 255.\n", 181 | " img_lq = np.transpose(img_lq if img_lq.shape[2] == 1 else img_lq[:, :, [2, 1, 0]], (2, 0, 1)) # HCW-BGR to CHW-RGB\n", 182 | " img_lq = torch.from_numpy(img_lq).float().unsqueeze(0).to(device) # CHW-RGB to NCHW-RGB\n", 183 | " # inference\n", 184 | " with torch.no_grad():\n", 185 | " # pad input image to be a multiple of window_size\n", 186 | " _, _, h_old, w_old = img_lq.size()\n", 187 | " h_pad = (h_old // window_size + 1) * window_size - h_old\n", 188 | " w_pad = (w_old // window_size + 1) * window_size - w_old\n", 189 | " img_lq = torch.cat([img_lq, torch.flip(img_lq, [2])], 2)[:, :, :h_old + h_pad, :]\n", 190 | " img_lq = torch.cat([img_lq, torch.flip(img_lq, [3])], 3)[:, :, :, :w_old + w_pad]\n", 191 | " output = test(img_lq,tile,tile_overlap, model, window_size, scale)\n", 192 | " output = output[..., :h_old * scale, :w_old * scale]\n", 193 | " # save image\n", 194 | " output = output.data.squeeze().float().cpu().clamp_(0, 1).numpy()\n", 195 | " if output.ndim == 3:\n", 196 | " output = np.transpose(output[[2, 1, 0], :, :], (1, 2, 0)) # CHW-RGB to HCW-BGR\n", 197 | " output = (output * 255.0).round().astype(np.uint8) # float32 to uint8\n", 198 | " cv2.imwrite(f'{save_dir}/%s.jpg'%item, output)\n" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "id": "0b1f86d5", 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": null, 212 | "id": "f44f2e17", 213 | "metadata": {}, 214 | "outputs": [], 215 | "source": [] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "id": "f6537d39", 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "id": "ef13686d", 229 | "metadata": {}, 230 | "outputs": [], 231 | "source": [] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "id": "ef921ffb", 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "id": "ebdd017e", 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "id": "06740d75", 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": null, 260 | "id": "53d8a60f", 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "id": "4dd2507a", 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "id": "6161f23c", 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": null, 284 | "id": "6f92456a", 285 | "metadata": {}, 286 | "outputs": [], 287 | "source": [] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "id": "813ac7ad", 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": null, 300 | "id": "ce3de71b", 301 | "metadata": {}, 302 | "outputs": [], 303 | "source": [] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "89cca74c", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": null, 316 | "id": "bce91e67", 317 | "metadata": {}, 318 | "outputs": [], 319 | "source": [] 320 | }, 321 | { 322 | "cell_type": "code", 323 | "execution_count": null, 324 | "id": "49ebadc9", 325 | "metadata": {}, 326 | "outputs": [], 327 | "source": [] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "id": "032d2057", 333 | "metadata": {}, 334 | "outputs": [], 335 | "source": [] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "id": "072e79db", 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": null, 348 | "id": "d1f0d1db", 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": null, 356 | "id": "6176c0c7", 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": null, 364 | "id": "08fedae0", 365 | "metadata": {}, 366 | "outputs": [], 367 | "source": [] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": null, 372 | "id": "8f888986", 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": null, 380 | "id": "db2403bf", 381 | "metadata": {}, 382 | "outputs": [], 383 | "source": [] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": null, 388 | "id": "30997470", 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": null, 396 | "id": "531d23a0", 397 | "metadata": {}, 398 | "outputs": [], 399 | "source": [] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": null, 404 | "id": "63e3db54", 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": null, 412 | "id": "8cfb1f96", 413 | "metadata": {}, 414 | "outputs": [], 415 | "source": [] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "id": "5ec64270", 421 | "metadata": {}, 422 | "outputs": [], 423 | "source": [] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": null, 428 | "id": "8d6c692a", 429 | "metadata": {}, 430 | "outputs": [], 431 | "source": [] 432 | }, 433 | { 434 | "cell_type": "code", 435 | "execution_count": null, 436 | "id": "ccb365d5", 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "id": "d58fbd3d", 445 | "metadata": {}, 446 | "outputs": [], 447 | "source": [] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": null, 452 | "id": "7716ecc9", 453 | "metadata": {}, 454 | "outputs": [], 455 | "source": [] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": null, 460 | "id": "28b426fb", 461 | "metadata": {}, 462 | "outputs": [], 463 | "source": [] 464 | }, 465 | { 466 | "cell_type": "code", 467 | "execution_count": null, 468 | "id": "51003cd1", 469 | "metadata": {}, 470 | "outputs": [], 471 | "source": [] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": null, 476 | "id": "f572fc2f", 477 | "metadata": {}, 478 | "outputs": [], 479 | "source": [] 480 | }, 481 | { 482 | "cell_type": "code", 483 | "execution_count": null, 484 | "id": "be343378", 485 | "metadata": {}, 486 | "outputs": [], 487 | "source": [] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": null, 492 | "id": "da69c9d4", 493 | "metadata": {}, 494 | "outputs": [], 495 | "source": [] 496 | }, 497 | { 498 | "cell_type": "code", 499 | "execution_count": null, 500 | "id": "8f6aada8", 501 | "metadata": {}, 502 | "outputs": [], 503 | "source": [] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": null, 508 | "id": "5e13b74b", 509 | "metadata": {}, 510 | "outputs": [], 511 | "source": [] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": null, 516 | "id": "742308e1", 517 | "metadata": {}, 518 | "outputs": [], 519 | "source": [] 520 | }, 521 | { 522 | "cell_type": "code", 523 | "execution_count": null, 524 | "id": "b16ea156", 525 | "metadata": {}, 526 | "outputs": [], 527 | "source": [] 528 | }, 529 | { 530 | "cell_type": "code", 531 | "execution_count": null, 532 | "id": "02108bd2", 533 | "metadata": {}, 534 | "outputs": [], 535 | "source": [] 536 | }, 537 | { 538 | "cell_type": "code", 539 | "execution_count": null, 540 | "id": "081914c7", 541 | "metadata": {}, 542 | "outputs": [], 543 | "source": [] 544 | }, 545 | { 546 | "cell_type": "code", 547 | "execution_count": null, 548 | "id": "ea07be57", 549 | "metadata": {}, 550 | "outputs": [], 551 | "source": [] 552 | }, 553 | { 554 | "cell_type": "code", 555 | "execution_count": null, 556 | "id": "df011057", 557 | "metadata": {}, 558 | "outputs": [], 559 | "source": [] 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": null, 564 | "id": "4177e42f", 565 | "metadata": {}, 566 | "outputs": [], 567 | "source": [] 568 | } 569 | ], 570 | "metadata": { 571 | "kernelspec": { 572 | "display_name": "Python 3 (ipykernel)", 573 | "language": "python", 574 | "name": "python3" 575 | }, 576 | "language_info": { 577 | "codemirror_mode": { 578 | "name": "ipython", 579 | "version": 3 580 | }, 581 | "file_extension": ".py", 582 | "mimetype": "text/x-python", 583 | "name": "python", 584 | "nbconvert_exporter": "python", 585 | "pygments_lexer": "ipython3", 586 | "version": "3.7.11" 587 | } 588 | }, 589 | "nbformat": 4, 590 | "nbformat_minor": 5 591 | } 592 | -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/1.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/10.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/10.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/11.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/11.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/12.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/12.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/13.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/13.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/14.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/14.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/15.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/15.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/16.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/16.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/17.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/17.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/18.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/18.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/19.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/19.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/2.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/2.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/20.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/20.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/21.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/21.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/22.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/22.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/23.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/23.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/3.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/3.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/4.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/4.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/5.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/5.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/6.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/6.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/7.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/7.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/8.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/8.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/input/9.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/input/9.tiff -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/1.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/1.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/10.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/10.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/11.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/11.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/12.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/12.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/13.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/13.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/14.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/14.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/15.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/15.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/16.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/16.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/17.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/17.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/18.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/18.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/19.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/19.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/2.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/2.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/20.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/20.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/21.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/21.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/22.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/22.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/23.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/23.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/3.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/3.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/4.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/4.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/5.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/5.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/6.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/6.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/7.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/7.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/8.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/8.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/testdata/output/9.tiff.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/testdata/output/9.tiff.jpg -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/utils/__pycache__/util_calculate_psnr_ssim.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/Gated_Fusion_Transformer/utils/__pycache__/util_calculate_psnr_ssim.cpython-37.pyc -------------------------------------------------------------------------------- /Gated_Fusion_Transformer/utils/util_calculate_psnr_ssim.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import torch 4 | 5 | 6 | def calculate_psnr(img1, img2, crop_border, input_order='HWC', test_y_channel=False): 7 | """Calculate PSNR (Peak Signal-to-Noise Ratio). 8 | 9 | Ref: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio 10 | 11 | Args: 12 | img1 (ndarray): Images with range [0, 255]. 13 | img2 (ndarray): Images with range [0, 255]. 14 | crop_border (int): Cropped pixels in each edge of an image. These 15 | pixels are not involved in the PSNR calculation. 16 | input_order (str): Whether the input order is 'HWC' or 'CHW'. 17 | Default: 'HWC'. 18 | test_y_channel (bool): Test on Y channel of YCbCr. Default: False. 19 | 20 | Returns: 21 | float: psnr result. 22 | """ 23 | 24 | assert img1.shape == img2.shape, (f'Image shapes are differnet: {img1.shape}, {img2.shape}.') 25 | if input_order not in ['HWC', 'CHW']: 26 | raise ValueError(f'Wrong input_order {input_order}. Supported input_orders are ' '"HWC" and "CHW"') 27 | img1 = reorder_image(img1, input_order=input_order) 28 | img2 = reorder_image(img2, input_order=input_order) 29 | img1 = img1.astype(np.float64) 30 | img2 = img2.astype(np.float64) 31 | 32 | if crop_border != 0: 33 | img1 = img1[crop_border:-crop_border, crop_border:-crop_border, ...] 34 | img2 = img2[crop_border:-crop_border, crop_border:-crop_border, ...] 35 | 36 | if test_y_channel: 37 | img1 = to_y_channel(img1) 38 | img2 = to_y_channel(img2) 39 | 40 | mse = np.mean((img1 - img2) ** 2) 41 | if mse == 0: 42 | return float('inf') 43 | return 20. * np.log10(255. / np.sqrt(mse)) 44 | 45 | 46 | def _ssim(img1, img2): 47 | """Calculate SSIM (structural similarity) for one channel images. 48 | 49 | It is called by func:`calculate_ssim`. 50 | 51 | Args: 52 | img1 (ndarray): Images with range [0, 255] with order 'HWC'. 53 | img2 (ndarray): Images with range [0, 255] with order 'HWC'. 54 | 55 | Returns: 56 | float: ssim result. 57 | """ 58 | 59 | C1 = (0.01 * 255) ** 2 60 | C2 = (0.03 * 255) ** 2 61 | 62 | img1 = img1.astype(np.float64) 63 | img2 = img2.astype(np.float64) 64 | kernel = cv2.getGaussianKernel(11, 1.5) 65 | window = np.outer(kernel, kernel.transpose()) 66 | 67 | mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5] 68 | mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5] 69 | mu1_sq = mu1 ** 2 70 | mu2_sq = mu2 ** 2 71 | mu1_mu2 = mu1 * mu2 72 | sigma1_sq = cv2.filter2D(img1 ** 2, -1, window)[5:-5, 5:-5] - mu1_sq 73 | sigma2_sq = cv2.filter2D(img2 ** 2, -1, window)[5:-5, 5:-5] - mu2_sq 74 | sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2 75 | 76 | ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)) 77 | return ssim_map.mean() 78 | 79 | 80 | def calculate_ssim(img1, img2, crop_border, input_order='HWC', test_y_channel=False): 81 | """Calculate SSIM (structural similarity). 82 | 83 | Ref: 84 | Image quality assessment: From error visibility to structural similarity 85 | 86 | The results are the same as that of the official released MATLAB code in 87 | https://ece.uwaterloo.ca/~z70wang/research/ssim/. 88 | 89 | For three-channel images, SSIM is calculated for each channel and then 90 | averaged. 91 | 92 | Args: 93 | img1 (ndarray): Images with range [0, 255]. 94 | img2 (ndarray): Images with range [0, 255]. 95 | crop_border (int): Cropped pixels in each edge of an image. These 96 | pixels are not involved in the SSIM calculation. 97 | input_order (str): Whether the input order is 'HWC' or 'CHW'. 98 | Default: 'HWC'. 99 | test_y_channel (bool): Test on Y channel of YCbCr. Default: False. 100 | 101 | Returns: 102 | float: ssim result. 103 | """ 104 | 105 | assert img1.shape == img2.shape, (f'Image shapes are differnet: {img1.shape}, {img2.shape}.') 106 | if input_order not in ['HWC', 'CHW']: 107 | raise ValueError(f'Wrong input_order {input_order}. Supported input_orders are ' '"HWC" and "CHW"') 108 | img1 = reorder_image(img1, input_order=input_order) 109 | img2 = reorder_image(img2, input_order=input_order) 110 | img1 = img1.astype(np.float64) 111 | img2 = img2.astype(np.float64) 112 | 113 | if crop_border != 0: 114 | img1 = img1[crop_border:-crop_border, crop_border:-crop_border, ...] 115 | img2 = img2[crop_border:-crop_border, crop_border:-crop_border, ...] 116 | 117 | if test_y_channel: 118 | img1 = to_y_channel(img1) 119 | img2 = to_y_channel(img2) 120 | 121 | ssims = [] 122 | for i in range(img1.shape[2]): 123 | ssims.append(_ssim(img1[..., i], img2[..., i])) 124 | return np.array(ssims).mean() 125 | 126 | 127 | def _blocking_effect_factor(im): 128 | block_size = 8 129 | 130 | block_horizontal_positions = torch.arange(7, im.shape[3] - 1, 8) 131 | block_vertical_positions = torch.arange(7, im.shape[2] - 1, 8) 132 | 133 | horizontal_block_difference = ( 134 | (im[:, :, :, block_horizontal_positions] - im[:, :, :, block_horizontal_positions + 1]) ** 2).sum( 135 | 3).sum(2).sum(1) 136 | vertical_block_difference = ( 137 | (im[:, :, block_vertical_positions, :] - im[:, :, block_vertical_positions + 1, :]) ** 2).sum(3).sum( 138 | 2).sum(1) 139 | 140 | nonblock_horizontal_positions = np.setdiff1d(torch.arange(0, im.shape[3] - 1), block_horizontal_positions) 141 | nonblock_vertical_positions = np.setdiff1d(torch.arange(0, im.shape[2] - 1), block_vertical_positions) 142 | 143 | horizontal_nonblock_difference = ( 144 | (im[:, :, :, nonblock_horizontal_positions] - im[:, :, :, nonblock_horizontal_positions + 1]) ** 2).sum( 145 | 3).sum(2).sum(1) 146 | vertical_nonblock_difference = ( 147 | (im[:, :, nonblock_vertical_positions, :] - im[:, :, nonblock_vertical_positions + 1, :]) ** 2).sum( 148 | 3).sum(2).sum(1) 149 | 150 | n_boundary_horiz = im.shape[2] * (im.shape[3] // block_size - 1) 151 | n_boundary_vert = im.shape[3] * (im.shape[2] // block_size - 1) 152 | boundary_difference = (horizontal_block_difference + vertical_block_difference) / ( 153 | n_boundary_horiz + n_boundary_vert) 154 | 155 | n_nonboundary_horiz = im.shape[2] * (im.shape[3] - 1) - n_boundary_horiz 156 | n_nonboundary_vert = im.shape[3] * (im.shape[2] - 1) - n_boundary_vert 157 | nonboundary_difference = (horizontal_nonblock_difference + vertical_nonblock_difference) / ( 158 | n_nonboundary_horiz + n_nonboundary_vert) 159 | 160 | scaler = np.log2(block_size) / np.log2(min([im.shape[2], im.shape[3]])) 161 | bef = scaler * (boundary_difference - nonboundary_difference) 162 | 163 | bef[boundary_difference <= nonboundary_difference] = 0 164 | return bef 165 | 166 | 167 | def calculate_psnrb(img1, img2, crop_border, input_order='HWC', test_y_channel=False): 168 | """Calculate PSNR-B (Peak Signal-to-Noise Ratio). 169 | 170 | Ref: Quality assessment of deblocked images, for JPEG image deblocking evaluation 171 | # https://gitlab.com/Queuecumber/quantization-guided-ac/-/blob/master/metrics/psnrb.py 172 | 173 | Args: 174 | img1 (ndarray): Images with range [0, 255]. 175 | img2 (ndarray): Images with range [0, 255]. 176 | crop_border (int): Cropped pixels in each edge of an image. These 177 | pixels are not involved in the PSNR calculation. 178 | input_order (str): Whether the input order is 'HWC' or 'CHW'. 179 | Default: 'HWC'. 180 | test_y_channel (bool): Test on Y channel of YCbCr. Default: False. 181 | 182 | Returns: 183 | float: psnr result. 184 | """ 185 | 186 | assert img1.shape == img2.shape, (f'Image shapes are differnet: {img1.shape}, {img2.shape}.') 187 | if input_order not in ['HWC', 'CHW']: 188 | raise ValueError(f'Wrong input_order {input_order}. Supported input_orders are ' '"HWC" and "CHW"') 189 | img1 = reorder_image(img1, input_order=input_order) 190 | img2 = reorder_image(img2, input_order=input_order) 191 | img1 = img1.astype(np.float64) 192 | img2 = img2.astype(np.float64) 193 | 194 | if crop_border != 0: 195 | img1 = img1[crop_border:-crop_border, crop_border:-crop_border, ...] 196 | img2 = img2[crop_border:-crop_border, crop_border:-crop_border, ...] 197 | 198 | if test_y_channel: 199 | img1 = to_y_channel(img1) 200 | img2 = to_y_channel(img2) 201 | 202 | # follow https://gitlab.com/Queuecumber/quantization-guided-ac/-/blob/master/metrics/psnrb.py 203 | img1 = torch.from_numpy(img1).permute(2, 0, 1).unsqueeze(0) / 255. 204 | img2 = torch.from_numpy(img2).permute(2, 0, 1).unsqueeze(0) / 255. 205 | 206 | total = 0 207 | for c in range(img1.shape[1]): 208 | mse = torch.nn.functional.mse_loss(img1[:, c:c + 1, :, :], img2[:, c:c + 1, :, :], reduction='none') 209 | bef = _blocking_effect_factor(img1[:, c:c + 1, :, :]) 210 | 211 | mse = mse.view(mse.shape[0], -1).mean(1) 212 | total += 10 * torch.log10(1 / (mse + bef)) 213 | 214 | return float(total) / img1.shape[1] 215 | 216 | 217 | def reorder_image(img, input_order='HWC'): 218 | """Reorder images to 'HWC' order. 219 | 220 | If the input_order is (h, w), return (h, w, 1); 221 | If the input_order is (c, h, w), return (h, w, c); 222 | If the input_order is (h, w, c), return as it is. 223 | 224 | Args: 225 | img (ndarray): Input image. 226 | input_order (str): Whether the input order is 'HWC' or 'CHW'. 227 | If the input image shape is (h, w), input_order will not have 228 | effects. Default: 'HWC'. 229 | 230 | Returns: 231 | ndarray: reordered image. 232 | """ 233 | 234 | if input_order not in ['HWC', 'CHW']: 235 | raise ValueError(f'Wrong input_order {input_order}. Supported input_orders are ' "'HWC' and 'CHW'") 236 | if len(img.shape) == 2: 237 | img = img[..., None] 238 | if input_order == 'CHW': 239 | img = img.transpose(1, 2, 0) 240 | return img 241 | 242 | 243 | def to_y_channel(img): 244 | """Change to Y channel of YCbCr. 245 | 246 | Args: 247 | img (ndarray): Images with range [0, 255]. 248 | 249 | Returns: 250 | (ndarray): Images with range [0, 255] (float type) without round. 251 | """ 252 | img = img.astype(np.float32) / 255. 253 | if img.ndim == 3 and img.shape[2] == 3: 254 | img = bgr2ycbcr(img, y_only=True) 255 | img = img[..., None] 256 | return img * 255. 257 | 258 | 259 | def _convert_input_type_range(img): 260 | """Convert the type and range of the input image. 261 | 262 | It converts the input image to np.float32 type and range of [0, 1]. 263 | It is mainly used for pre-processing the input image in colorspace 264 | convertion functions such as rgb2ycbcr and ycbcr2rgb. 265 | 266 | Args: 267 | img (ndarray): The input image. It accepts: 268 | 1. np.uint8 type with range [0, 255]; 269 | 2. np.float32 type with range [0, 1]. 270 | 271 | Returns: 272 | (ndarray): The converted image with type of np.float32 and range of 273 | [0, 1]. 274 | """ 275 | img_type = img.dtype 276 | img = img.astype(np.float32) 277 | if img_type == np.float32: 278 | pass 279 | elif img_type == np.uint8: 280 | img /= 255. 281 | else: 282 | raise TypeError('The img type should be np.float32 or np.uint8, ' f'but got {img_type}') 283 | return img 284 | 285 | 286 | def _convert_output_type_range(img, dst_type): 287 | """Convert the type and range of the image according to dst_type. 288 | 289 | It converts the image to desired type and range. If `dst_type` is np.uint8, 290 | images will be converted to np.uint8 type with range [0, 255]. If 291 | `dst_type` is np.float32, it converts the image to np.float32 type with 292 | range [0, 1]. 293 | It is mainly used for post-processing images in colorspace convertion 294 | functions such as rgb2ycbcr and ycbcr2rgb. 295 | 296 | Args: 297 | img (ndarray): The image to be converted with np.float32 type and 298 | range [0, 255]. 299 | dst_type (np.uint8 | np.float32): If dst_type is np.uint8, it 300 | converts the image to np.uint8 type with range [0, 255]. If 301 | dst_type is np.float32, it converts the image to np.float32 type 302 | with range [0, 1]. 303 | 304 | Returns: 305 | (ndarray): The converted image with desired type and range. 306 | """ 307 | if dst_type not in (np.uint8, np.float32): 308 | raise TypeError('The dst_type should be np.float32 or np.uint8, ' f'but got {dst_type}') 309 | if dst_type == np.uint8: 310 | img = img.round() 311 | else: 312 | img /= 255. 313 | return img.astype(dst_type) 314 | 315 | 316 | def bgr2ycbcr(img, y_only=False): 317 | """Convert a BGR image to YCbCr image. 318 | 319 | The bgr version of rgb2ycbcr. 320 | It implements the ITU-R BT.601 conversion for standard-definition 321 | television. See more details in 322 | https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. 323 | 324 | It differs from a similar function in cv2.cvtColor: `BGR <-> YCrCb`. 325 | In OpenCV, it implements a JPEG conversion. See more details in 326 | https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. 327 | 328 | Args: 329 | img (ndarray): The input image. It accepts: 330 | 1. np.uint8 type with range [0, 255]; 331 | 2. np.float32 type with range [0, 1]. 332 | y_only (bool): Whether to only return Y channel. Default: False. 333 | 334 | Returns: 335 | ndarray: The converted YCbCr image. The output image has the same type 336 | and range as input image. 337 | """ 338 | img_type = img.dtype 339 | img = _convert_input_type_range(img) 340 | if y_only: 341 | out_img = np.dot(img, [24.966, 128.553, 65.481]) + 16.0 342 | else: 343 | out_img = np.matmul( 344 | img, [[24.966, 112.0, -18.214], [128.553, -74.203, -93.786], [65.481, -37.797, 112.0]]) + [16, 128, 128] 345 | out_img = _convert_output_type_range(out_img, img_type) 346 | return out_img 347 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # High-resolution single-photon imaging with physics-informed deep learning 2 | 3 | 4 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.8260069.svg)](https://doi.org/10.5281/zenodo.8260069) 5 | 6 | ## 1. System requirements 7 | ### 1.1 All software dependencies and operating systems 8 | The project has been tested on Windows 10 and Ubuntu 18.04.1 LTS. 9 | ### 1.2 Versions the software has been tested on 10 | The project has been tested on Python 3.7 and matlab 2022a. 11 | ### 1.3 Any required non standard hardware 12 | There is no non-standard hardware required for this project. 13 | 14 | 15 | 16 | ## 2. Installation guide 17 | ### 2.1 Instructions 18 | 19 | To install the software, clone the repository and run the following command in the terminal: 20 | ``` 21 | git clone https://github.com/bianlab/single-photon.git 22 | ``` 23 | 24 | ### 2.2 Typical install time on a "normal" desk top computer 25 | The installation time is approximately 1 minute and fluctuates depending on network conditions. 26 | 27 | ## 3. Demo 28 | ### 3.1 Instructions to run on data 29 | #### 3.1.1 How to run the software to synthesize images 30 | To run the software on the data, run the following command in the matlab terminal: 31 | ```matlab 32 | cd ./synthesis_images 33 | main.m 34 | ``` 35 | 36 | #### 3.1.2 How to run the software to enhance the SPAD array images 37 | First, follow the requirements in the `requirements.txt` file to build the running environment, and only after the environment is built can you run the SPAD array image enhancement code. 38 | We have provided some test cases in the `testdata` folder, to enhance them, first go to the Gated_Fusion_Transformer directory, 39 | ``` 40 | cd ./Gated_Fusion_Transformer 41 | ``` 42 | Next, open the ` test.ipynb` file using jupyter notebook and run the code line by line. If your environment is installed correctly, the enhanced results will be output in the ` . /testdata/` output folder. 43 | ``` 44 | source activate 45 | jupyter notebook 46 | ``` 47 | 48 | 49 | ### 3.2 Expected output 50 | 51 | #### 3.2.1 Expected output for the synthesis images 52 | The synthesis images are saved in the folder 'output/test' as 'xmW_yframes_z.tiff' where x is the power of the laser, y is the number of frames, and z is the number of the image in the sequence. Here are example synthesis images at 20mW, composed of 32 frames and 256 frames respectively: 53 | 54 | 55 | 56 | 57 | | Ground truth | Composed of 32 frames at 20mW | Composed of 256 frames at 20mW | 58 | |:---:|:---:|:---:| 59 | | ![图片1](./images/target.jpg) | ![图片2](./images/20mW_32frames_1.jpg) | ![图片3](./images/20mW_256frames_1.jpg) | 60 | 61 | 62 | The output is displayed in the terminal: 63 | ```matlab 64 | 256 frames, psnr = 15.8988 65 | 256 frames, psnr = 18.4644 66 | 256 frames, psnr = 20.8186 67 | 512 frames, psnr = 16.9459 68 | 512 frames, psnr = 20.6258 69 | 512 frames, psnr = 22.0559 70 | 1000 frames, psnr = 20.0973 71 | 1000 frames, psnr = 23.3283 72 | 1000 frames, psnr = 24.4419 73 | ...... 74 | ``` 75 | where the first number is the number of frames, and the second number is the PSNR value compared to the ground truth. 76 | 77 | #### 3.2.2 Expected output for the enhanced SPAD array images 78 | Here, we default you to use anaconda to manage your deep learning environment. 79 | We provide a pre-trained model in the `. /model_zoo/super_resolution/` folder , which is automatically called by the algorithm. After running the code, you will get the enhanced result as shown in the figure below. 80 | 81 | ![图片1](./images/figure5.png) 82 | 83 | ### 3.3 Expected run time for demo on a "normal" desktop computer 84 | The estimated time it takes to synthesize an image is typically around 0.5 seconds and can vary depending on the device. The estimated time it takes to enhance an image is typically around 0.2 second and can also vary depending on the device. 85 | 86 | ## 4. Instructions for use 87 | ### 4.1 How to run the software on your data to synthesize images 88 | To synthesize images using your data, please place your respective images inside the 'input' folder. 89 | 90 | If you wish to adjust the frames and power settings of the laser, you can modify the code snippet as shown below: 91 | ```python 92 | frames_list=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1000]; 93 | power_list=[10, 20, 40]; 94 | processImage(I[image_index], frames_list, power_list, image_index); 95 | ``` 96 | In this code: 97 | - `frames_list` represents the different frame options you can choose from. 98 | - `power_list` outlines the available power settings for the laser. 99 | - `processImage` function will then take these parameters and process the image. 100 | 101 | Please note that the array `I` stores the images, and `image_index` is used to select the specific image from this array. 102 | -------------------------------------------------------------------------------- /images/20mW_256frames_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/images/20mW_256frames_1.jpg -------------------------------------------------------------------------------- /images/20mW_32frames_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/images/20mW_32frames_1.jpg -------------------------------------------------------------------------------- /images/figure5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/images/figure5.png -------------------------------------------------------------------------------- /images/target.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/images/target.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytorch 1.8 2 | cuda 10.2 3 | opencv 4 | glob 5 | natsort 6 | pillow 7 | numpy 8 | scikit-image 9 | torchvision 10 | -------------------------------------------------------------------------------- /synthesis_images/count_dark.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/count_dark.mat -------------------------------------------------------------------------------- /synthesis_images/frame_si.m: -------------------------------------------------------------------------------- 1 | function counter=frame_si(u) 2 | 3 | % used for sub-frame construction 4 | % there are 3 parts in all 5 | %% first part : parameter initializtion 6 | counter=0; 7 | rest=0.02;% frame time ,unit: us 8 | pde=0.5;%photon detection effe 9 | t=1;% snap time: 15ms 10 | lam=u/t;% the mean of photons in 1 ms 11 | % dead=1e-4;%set dead time to 1 12 | % ilam=t/u; 13 | % disp(lam*rest) 14 | % while(rest>0) 15 | % a=exprnd(ilam); 16 | % rest=rest-a; 17 | % % disp(rest) 18 | % if rest>0 19 | % if pde>rand(1) 20 | % counter=1+counter;%count how many photons come 21 | % end 22 | % % disp(rest) 23 | % end 24 | % end 25 | % 26 | % rest=rest+a+dead; 27 | p=1-exp(-(lam*(rest))); 28 | if p>rand(1) 29 | if pde>rand(1) 30 | counter=1+counter; 31 | end 32 | end 33 | 34 | 35 | end -------------------------------------------------------------------------------- /synthesis_images/input/target.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/input/target.jpg -------------------------------------------------------------------------------- /synthesis_images/loadImages.m: -------------------------------------------------------------------------------- 1 | function I = loadImages(file_path) 2 | % Get a list of all .jpg image files in the directory 3 | img_path_list = dir(strcat(file_path,'*.jpg')); 4 | 5 | % Get the number of image files 6 | img_num = length(img_path_list); 7 | 8 | % Create a cell array to store the images 9 | I = cell(1, img_num); 10 | 11 | % Check if there are any image files 12 | if img_num > 0 13 | % Read each image file 14 | for j = 1:img_num 15 | % Get the name of the image file 16 | image_name = img_path_list(j).name; 17 | 18 | % Read the image file 19 | image = imread(strcat(file_path, image_name)); 20 | 21 | % Store the image in the cell array 22 | I{j} = image; 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /synthesis_images/main.m: -------------------------------------------------------------------------------- 1 | %% Close all figures 2 | close all; 3 | clear; 4 | %% Specify the path to the image files 5 | file_path = '.\input\'; 6 | 7 | %% Load Images 8 | I = loadImages(file_path); 9 | 10 | %% Display the first image 11 | figure; imshow(I{1}) 12 | 13 | start_index=1; 14 | end_index=length(I); % Adjust based on memory capacity 15 | frames_list=[1 2 4 8 16 32 64 128 256 512 1000]; 16 | power_list=[10 20 40]; 17 | 18 | for image_index=start_index:end_index 19 | processImage(I{image_index}, frames_list, power_list, image_index); 20 | end 21 | -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_1000frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_1000frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_128frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_128frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_16frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_16frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_1frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_1frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_256frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_256frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_2frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_2frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_32frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_32frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_4frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_4frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_512frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_512frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_64frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_64frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/10mW_8frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/10mW_8frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_1000frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_1000frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_128frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_128frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_16frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_16frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_1frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_1frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_256frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_256frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_2frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_2frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_32frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_32frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_4frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_4frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_512frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_512frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_64frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_64frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/20mW_8frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/20mW_8frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_1000frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_1000frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_128frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_128frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_16frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_16frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_1frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_1frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_256frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_256frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_2frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_2frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_32frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_32frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_4frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_4frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_512frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_512frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_64frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_64frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/output/test/40mW_8frames_1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bianlab/single-photon/8646f5243bd44601f2f5f51b26cf177a7b473d80/synthesis_images/output/test/40mW_8frames_1.tiff -------------------------------------------------------------------------------- /synthesis_images/processImage.m: -------------------------------------------------------------------------------- 1 | function processImage(image, num_sub_values, scale_values, image_index) 2 | a = image; 3 | b = rgb2gray(a); %% Convert the image to grayscale 4 | c = im2double(b); %% Convert the image to double precision 5 | 6 | num_pixel = 32; 7 | 8 | for num_sub = num_sub_values 9 | for scale = scale_values 10 | processScaleAndSubframes(c, num_pixel, num_sub, scale, image_index); 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /synthesis_images/processScaleAndSubframes.m: -------------------------------------------------------------------------------- 1 | function processScaleAndSubframes(c, num_pixel, num_sub, scale, image_index) 2 | c = imresize(c,[num_pixel 1*num_pixel]); %% Resize the image 3 | d = scale*c; 4 | subframes = zeros(num_pixel,1*num_pixel,num_sub); 5 | 6 | %% Subframes generation 7 | for i = 1:num_pixel 8 | for j = 1:num_pixel*1 9 | for k = 1:num_sub 10 | m = frame_si(d(i,j)); 11 | subframes(i,j,k) = m; 12 | end 13 | end 14 | end 15 | 16 | %% Afterpulsing 17 | % The code here adds afterpulsing effects 18 | for k=1:num_sub-1 19 | for i=1:num_pixel 20 | for j=1:num_pixel*1 21 | if subframes(i,j,k)>0 22 | for t=1:subframes(i,j,k) 23 | if rand(1)<0.01 24 | subframes(i,j,k)=subframes(i,j,k)+1; 25 | end 26 | end 27 | end 28 | 29 | end 30 | end 31 | end 32 | %% Crosstalking 33 | % The code here adds crosstalking effects 34 | 35 | for k=1:num_sub 36 | for i=1:num_pixel 37 | for j=1:num_pixel 38 | if rand(1)<0.0001 39 | if i==1 && j==1 40 | if (subframes(i+1,j,k)+subframes(i,j+1,k))>0 41 | subframes(i,j,k)=subframes(i,j,k)+1; 42 | end 43 | end 44 | if i==num_pixel &&j==1 45 | if (subframes(i-1,j,k)+subframes(i,j+1,k))>0 46 | subframes(i,j,k)=subframes(i,j,k)+1; 47 | end 48 | end 49 | if i==1 && j==num_pixel 50 | if (subframes(i+1,j,k)+subframes(i,j-1,k))>0 51 | subframes(i,j,k)=subframes(i,j,k)+1; 52 | end 53 | end 54 | if i==num_pixel && j==num_pixel 55 | if (subframes(i-1,j,k)+subframes(i,j-1,k))>0 56 | subframes(i,j,k)=subframes(i,j,k)+1; 57 | end 58 | end 59 | if i==1 && j>1 && j0 61 | subframes(i,j,k)=subframes(i,j,k)+1; 62 | end 63 | end 64 | if i==num_pixel && j>1 && j0 66 | subframes(i,j,k)=subframes(i,j,k)+1; 67 | end 68 | end 69 | if j==1 && i>1 && i0 71 | subframes(i,j,k)=subframes(i,j,k)+1; 72 | end 73 | end 74 | if j==num_pixel && i>1 && i0 76 | subframes(i,j,k)=subframes(i,j,k)+1; 77 | end 78 | end 79 | if i>1 && i1 && j0 81 | subframes(i,j,k)=subframes(i,j,k)+1; 82 | end 83 | end 84 | end 85 | end 86 | end 87 | end 88 | %% Generate a frame from subframes 89 | frame = zeros(num_pixel,1*num_pixel); 90 | for i = 1:num_pixel 91 | for j = 1:num_pixel*1 92 | for k = 1:num_sub 93 | frame(i,j) = frame(i,j) + subframes(i,j,k); 94 | end 95 | end 96 | end 97 | 98 | %% Dark count rate 99 | load('count_dark.mat'); 100 | randomdcr = zeros(32,32); %% used for 60000 frames at 0.02us integration time 101 | for i = 1:32 102 | for j = 1:32 103 | randomdcr(i,j) = random('Poisson',count_dark(i,j)); 104 | end 105 | end 106 | 107 | %% Add noise 108 | frame2 = frame + randomdcr/60000*num_sub; 109 | 110 | %% Normalize 111 | truthp = mat2gray(d); %% the truth ranges in (0-1) 112 | frameout = mat2gray(frame2); 113 | 114 | %% Display the final output 115 | figure; imshow(frameout); 116 | p = psnr(truthp, frameout); 117 | title([num2str(num_sub) ' frames, psnr = ' num2str(p) ]) 118 | disp([num2str(num_sub) ' frames, psnr = ' num2str(p) ]) 119 | %% Save the output frame 120 | imwrite(frameout,['.\output\test\' num2str(scale) 'mW_' num2str(num_sub) 'frames_' num2str(image_index) '.tiff']) 121 | end 122 | --------------------------------------------------------------------------------