├── LICENSE ├── MSGNet-release ├── misc │ ├── MSG_Net.png │ └── illus.png ├── models │ ├── MSGNet_x16.caffemodel │ ├── MSGNet_x16_deploy.prototxt │ ├── MSGNet_x2.caffemodel │ ├── MSGNet_x2_deploy.prototxt │ ├── MSGNet_x4.caffemodel │ ├── MSGNet_x4_deploy.prototxt │ ├── MSGNet_x8.caffemodel │ └── MSGNet_x8_deploy.prototxt ├── testing sets │ ├── A │ │ ├── Depth │ │ │ ├── art.png │ │ │ ├── books.png │ │ │ └── moebius.png │ │ └── RGB │ │ │ ├── art.png │ │ │ ├── books.png │ │ │ └── moebius.png │ ├── B │ │ ├── Depth │ │ │ ├── art.bmp │ │ │ ├── books.bmp │ │ │ ├── dolls.bmp │ │ │ ├── laundry.bmp │ │ │ ├── moebius.bmp │ │ │ └── reindeer.bmp │ │ └── RGB │ │ │ ├── art.bmp │ │ │ ├── books.bmp │ │ │ ├── dolls.bmp │ │ │ ├── laundry.bmp │ │ │ ├── moebius.bmp │ │ │ └── reindeer.bmp │ └── C │ │ ├── Depth │ │ ├── cones_2.mat │ │ ├── teddy_2.mat │ │ ├── tsukuba_2.mat │ │ └── venus_2.mat │ │ └── RGB │ │ ├── cones.bmp │ │ ├── teddy.bmp │ │ ├── tsukuba.bmp │ │ └── venus.bmp └── util │ ├── MSGNet.m │ ├── compute_RMSE.m │ ├── gen_train_data_x2.m │ ├── modcrop.m │ ├── normalizeIm.m │ ├── normalize_cleanIm.m │ └── store2hdf5_v2.m └── README.md /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Tak-Wai Hui. All rights reserved. 2 | 3 | This software and the associated documentation files (the "Software"), and the research paper 4 | Depth Map Super-Resolution by Deep Multi-Scale Guidance (the "Paper") including but not limited 5 | to the figures and tables are provided for non-commercial use only and without any warranty. 6 | Any commercial use requires a prior arrangement with the author (Tak-Wai Hui). When using any 7 | parts of the Software or the Paper in your work, please cite the following paper: 8 | 9 | @InProceedings{hui16msgnet, 10 | author = {Tak-Wai Hui and Chen Change Loy and and Xiaoou Tang}, 11 | title = {Depth Map Super-Resolution by Deep Multi-Scale Guidance}, 12 | booktitle = {Proceedings of European Conference on Computer Vision (ECCV)}, 13 | pages = {353--369}, 14 | year = {2016}, 15 | url = {http://mmlab.ie.cuhk.edu.hk/projects/guidance_SR_depth.html} 16 | } 17 | 18 | The above copyright notice and this permission notice shall be included in all copies or 19 | substantial portions of the Software. 20 | -------------------------------------------------------------------------------- /MSGNet-release/misc/MSG_Net.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/misc/MSG_Net.png -------------------------------------------------------------------------------- /MSGNet-release/misc/illus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/misc/illus.png -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x16.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/models/MSGNet_x16.caffemodel -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x16_deploy.prototxt: -------------------------------------------------------------------------------- 1 | # MSG-Net (published in ECCV 2016) 2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved. 3 | # 4 | # This software is licensed under the terms of the MSG-Net licence 5 | # which allows for non-commercial use only unless a prior arrangement 6 | # has been made with the author (Tak-Wai Hui), the full terms of which 7 | # are made available in the LICENSE file. 8 | 9 | name: "MSGNet_x16" 10 | 11 | input: "data-Y" 12 | input_dim: 1 13 | input_dim: 1 14 | input_dim: 127 15 | input_dim: 127 16 | 17 | input: "data-D" 18 | input_dim: 1 19 | input_dim: 1 20 | input_dim: 8 21 | input_dim: 8 22 | 23 | layer { 24 | name: "conv1-Y" 25 | type: "Convolution" 26 | bottom: "data-Y" 27 | top: "conv1-Y" 28 | convolution_param { 29 | num_output: 49 30 | kernel_size: 7 31 | stride: 1 32 | pad: 3 33 | } 34 | } 35 | 36 | layer { 37 | name: "prelu1-Y" 38 | type: "PReLU" 39 | bottom: "conv1-Y" 40 | top: "conv1-Y" 41 | } 42 | 43 | layer { 44 | name: "conv2-Y" 45 | type: "Convolution" 46 | bottom: "conv1-Y" 47 | top: "conv2-Y" 48 | convolution_param { 49 | num_output: 32 50 | kernel_size: 5 51 | stride: 1 52 | pad: 2 53 | } 54 | } 55 | 56 | layer { 57 | name: "prelu2-Y" 58 | type: "PReLU" 59 | bottom: "conv2-Y" 60 | top: "conv2-Y" 61 | } 62 | 63 | layer { 64 | name: "conv3-Y" 65 | type: "Convolution" 66 | bottom: "conv2-Y" 67 | top: "conv3-Y" 68 | convolution_param { 69 | num_output: 32 70 | kernel_size: 5 71 | stride: 1 72 | pad: 2 73 | } 74 | } 75 | 76 | layer { 77 | name: "prelu3-Y" 78 | type: "PReLU" 79 | bottom: "conv3-Y" 80 | top: "conv3-Y" 81 | } 82 | 83 | layer { 84 | name: "pool3-Y" 85 | type: "Pooling" 86 | bottom: "conv3-Y" 87 | top: "pool3-Y" 88 | pooling_param { 89 | pool: MAX 90 | kernel_size: 3 91 | stride: 2 92 | pad: 0 93 | } 94 | } 95 | 96 | layer { 97 | name: "conv4-Y" 98 | type: "Convolution" 99 | bottom: "pool3-Y" 100 | top: "conv4-Y" 101 | convolution_param { 102 | num_output: 32 103 | kernel_size: 5 104 | stride: 1 105 | pad: 2 106 | } 107 | } 108 | 109 | layer { 110 | name: "prelu4-Y" 111 | type: "PReLU" 112 | bottom: "conv4-Y" 113 | top: "conv4-Y" 114 | } 115 | 116 | layer { 117 | name: "pool4-Y" 118 | type: "Pooling" 119 | bottom: "conv4-Y" 120 | top: "pool4-Y" 121 | pooling_param { 122 | pool: MAX 123 | kernel_size: 3 124 | stride: 2 125 | pad: 0 126 | } 127 | } 128 | 129 | layer { 130 | name: "conv5-Y" 131 | type: "Convolution" 132 | bottom: "pool4-Y" 133 | top: "conv5-Y" 134 | convolution_param { 135 | num_output: 32 136 | kernel_size: 5 137 | stride: 1 138 | pad: 2 139 | } 140 | } 141 | 142 | layer { 143 | name: "prelu5-Y" 144 | type: "PReLU" 145 | bottom: "conv5-Y" 146 | top: "conv5-Y" 147 | } 148 | 149 | layer { 150 | name: "pool5-Y" 151 | type: "Pooling" 152 | bottom: "conv5-Y" 153 | top: "pool5-Y" 154 | pooling_param { 155 | pool: MAX 156 | kernel_size: 3 157 | stride: 2 158 | pad: 0 159 | } 160 | } 161 | 162 | layer { 163 | name: "conv1-D" 164 | type: "Convolution" 165 | bottom: "data-D" 166 | top: "conv1-D" 167 | convolution_param { 168 | num_output: 64 169 | kernel_size: 5 170 | stride: 1 171 | pad: 2 172 | } 173 | } 174 | 175 | layer { 176 | name: "prelu1-D" 177 | type: "PReLU" 178 | bottom: "conv1-D" 179 | top: "conv1-D" 180 | } 181 | 182 | layer { 183 | name: "deconv2-D" 184 | type: "Deconvolution" 185 | bottom: "conv1-D" 186 | top: "deconv2-D" 187 | convolution_param { 188 | num_output: 32 189 | kernel_size: 5 190 | stride: 2 191 | pad: 2 192 | } 193 | } 194 | 195 | layer { 196 | name: "prelu2-D" 197 | type: "PReLU" 198 | bottom: "deconv2-D" 199 | top: "deconv2-D" 200 | } 201 | 202 | layer { 203 | name: "concat2-x16" 204 | bottom: "pool5-Y" 205 | bottom: "deconv2-D" 206 | top: "concat2-x16" 207 | type: "Concat" 208 | concat_param { 209 | axis: 1 210 | } 211 | } 212 | 213 | layer { 214 | name: "conv2a-x4" 215 | type: "Convolution" 216 | bottom: "concat2-x16" 217 | top: "conv2a-x4" 218 | convolution_param { 219 | num_output: 32 220 | kernel_size: 5 221 | stride: 1 222 | pad: 2 223 | } 224 | } 225 | 226 | layer { 227 | name: "prelu2a-x4" 228 | type: "PReLU" 229 | bottom: "conv2a-x4" 230 | top: "conv2a-x4" 231 | } 232 | 233 | layer { 234 | name: "conv2b-x4" 235 | type: "Convolution" 236 | bottom: "conv2a-x4" 237 | top: "conv2b-x4" 238 | convolution_param { 239 | num_output: 32 240 | kernel_size: 5 241 | stride: 1 242 | pad: 2 243 | } 244 | } 245 | 246 | layer { 247 | name: "prelu2b-x4" 248 | type: "PReLU" 249 | bottom: "conv2b-x4" 250 | top: "conv2b-x4" 251 | } 252 | 253 | layer { 254 | name: "deconv3-x4" 255 | type: "Deconvolution" 256 | bottom: "conv2b-x4" 257 | top: "deconv3-x4" 258 | convolution_param { 259 | num_output: 32 260 | kernel_size: 5 261 | stride: 2 262 | pad: 1 263 | } 264 | } 265 | 266 | layer { 267 | name: "prelu3-x4" 268 | type: "PReLU" 269 | bottom: "deconv3-x4" 270 | top: "deconv3-x4" 271 | } 272 | 273 | layer { 274 | name: "concat3-x16" 275 | bottom: "pool4-Y" 276 | bottom: "deconv3-x4" 277 | top: "concat3-x16" 278 | type: "Concat" 279 | concat_param { 280 | axis: 1 281 | } 282 | } 283 | 284 | layer { 285 | name: "conv3a-x8" 286 | type: "Convolution" 287 | bottom: "concat3-x16" 288 | top: "conv3a-x8" 289 | convolution_param { 290 | num_output: 32 291 | kernel_size: 5 292 | stride: 1 293 | pad: 2 294 | } 295 | } 296 | 297 | layer { 298 | name: "prelu3a-x8" 299 | type: "PReLU" 300 | bottom: "conv3a-x8" 301 | top: "conv3a-x8" 302 | prelu_param { 303 | filler { 304 | type: "constant" 305 | value: 0.25 306 | } 307 | } 308 | } 309 | 310 | layer { 311 | name: "conv3b-x8" 312 | type: "Convolution" 313 | bottom: "conv3a-x8" 314 | top: "conv3b-x8" 315 | convolution_param { 316 | num_output: 32 317 | kernel_size: 5 318 | stride: 1 319 | pad: 2 320 | } 321 | } 322 | 323 | layer { 324 | name: "prelu3b-x8" 325 | type: "PReLU" 326 | bottom: "conv3b-x8" 327 | top: "conv3b-x8" 328 | } 329 | 330 | layer { 331 | name: "deconv4-x8" 332 | type: "Deconvolution" 333 | bottom: "conv3b-x8" 334 | top: "deconv4-x8" 335 | convolution_param { 336 | num_output: 32 337 | kernel_size: 5 338 | stride: 2 339 | pad: 1 340 | } 341 | } 342 | 343 | layer { 344 | name: "prelu4-x8" 345 | type: "PReLU" 346 | bottom: "deconv4-x8" 347 | top: "deconv4-x8" 348 | } 349 | 350 | layer { 351 | name: "concat4-x16" 352 | bottom: "pool3-Y" 353 | bottom: "deconv4-x8" 354 | top: "concat4-x16" 355 | type: "Concat" 356 | concat_param { 357 | axis: 1 358 | } 359 | } 360 | 361 | layer { 362 | name: "conv4a-x16" 363 | type: "Convolution" 364 | bottom: "concat4-x16" 365 | top: "conv4a-x16" 366 | convolution_param { 367 | num_output: 32 368 | kernel_size: 5 369 | stride: 1 370 | pad: 2 371 | } 372 | } 373 | 374 | layer { 375 | name: "prelu4a-x16" 376 | type: "PReLU" 377 | bottom: "conv4a-x16" 378 | top: "conv4a-x16" 379 | } 380 | 381 | layer { 382 | name: "conv4b-x16" 383 | type: "Convolution" 384 | bottom: "conv4a-x16" 385 | top: "conv4b-x16" 386 | convolution_param { 387 | num_output: 32 388 | kernel_size: 5 389 | stride: 1 390 | pad: 2 391 | } 392 | } 393 | 394 | layer { 395 | name: "prelu4b-x16" 396 | type: "PReLU" 397 | bottom: "conv4b-x16" 398 | top: "conv4b-x16" 399 | } 400 | 401 | layer { 402 | name: "deconv5-x16" 403 | type: "Deconvolution" 404 | bottom: "conv4b-x16" 405 | top: "deconv5-x16" 406 | convolution_param { 407 | num_output: 32 408 | kernel_size: 5 409 | stride: 2 410 | pad: 1 411 | } 412 | } 413 | 414 | layer { 415 | name: "prelu5-x16" 416 | type: "PReLU" 417 | bottom: "deconv5-x16" 418 | top: "deconv5-x16" 419 | } 420 | 421 | layer { 422 | name: "concat5-x16" 423 | bottom: "conv2-Y" 424 | bottom: "deconv5-x16" 425 | top: "concat5-x16" 426 | type: "Concat" 427 | concat_param { 428 | axis: 1 429 | } 430 | } 431 | 432 | layer { 433 | name: "conv5a" 434 | type: "Convolution" 435 | bottom: "concat5-x16" 436 | top: "conv5a" 437 | convolution_param { 438 | num_output: 32 439 | kernel_size: 5 440 | stride: 1 441 | pad: 2 442 | } 443 | } 444 | 445 | layer { 446 | name: "prelu5a" 447 | type: "PReLU" 448 | bottom: "conv5a" 449 | top: "conv5a" 450 | } 451 | 452 | layer { 453 | name: "conv5b" 454 | type: "Convolution" 455 | bottom: "conv5a" 456 | top: "conv5b" 457 | convolution_param { 458 | num_output: 32 459 | kernel_size: 5 460 | stride: 1 461 | pad: 2 462 | } 463 | } 464 | 465 | layer { 466 | name: "prelu5b" 467 | type: "PReLU" 468 | bottom: "conv5b" 469 | top: "conv5b" 470 | } 471 | 472 | layer { 473 | name: "conv5c" 474 | type: "Convolution" 475 | bottom: "conv5b" 476 | top: "conv5c" 477 | convolution_param { 478 | num_output: 32 479 | kernel_size: 5 480 | stride: 1 481 | pad: 2 482 | } 483 | } 484 | 485 | layer { 486 | name: "prelu5c" 487 | type: "PReLU" 488 | bottom: "conv5c" 489 | top: "conv5c" 490 | } 491 | 492 | layer { 493 | name: "conv6" 494 | type: "Convolution" 495 | bottom: "conv5c" 496 | top: "conv6" 497 | convolution_param { 498 | num_output: 1 499 | kernel_size: 5 500 | stride: 1 501 | pad: 2 502 | } 503 | } 504 | -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x2.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/models/MSGNet_x2.caffemodel -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x2_deploy.prototxt: -------------------------------------------------------------------------------- 1 | # MSG-Net (published in ECCV 2016) 2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved. 3 | # 4 | # This software is licensed under the terms of the MSG-Net licence 5 | # which allows for non-commercial use only unless a prior arrangement 6 | # has been made with the author (Tak-Wai Hui), the full terms of which 7 | # are made available in the LICENSE file. 8 | 9 | name: "MSGNet_x2" 10 | 11 | input: "data-Y" 12 | input_dim: 1 13 | input_dim: 1 14 | input_dim: 39 15 | input_dim: 39 16 | 17 | input: "data-D" 18 | input_dim: 1 19 | input_dim: 1 20 | input_dim: 20 21 | input_dim: 20 22 | 23 | layer { 24 | name: "conv1-Y" 25 | type: "Convolution" 26 | bottom: "data-Y" 27 | top: "conv1-Y" 28 | convolution_param { 29 | num_output: 49 30 | kernel_size: 7 31 | stride: 1 32 | pad: 3 33 | } 34 | } 35 | 36 | layer { 37 | name: "prelu1-Y" 38 | type: "PReLU" 39 | bottom: "conv1-Y" 40 | top: "conv1-Y" 41 | } 42 | 43 | layer { 44 | name: "conv2-Y" 45 | type: "Convolution" 46 | bottom: "conv1-Y" 47 | top: "conv2-Y" 48 | convolution_param { 49 | num_output: 32 50 | kernel_size: 5 51 | stride: 1 52 | pad: 2 53 | } 54 | } 55 | 56 | layer { 57 | name: "prelu2-Y" 58 | type: "PReLU" 59 | bottom: "conv2-Y" 60 | top: "conv2-Y" 61 | } 62 | 63 | layer { 64 | name: "conv1-D" 65 | type: "Convolution" 66 | bottom: "data-D" 67 | top: "conv1-D" 68 | convolution_param { 69 | num_output: 64 70 | kernel_size: 5 71 | stride: 1 72 | pad: 2 73 | } 74 | } 75 | 76 | layer { 77 | name: "prelu1-D" 78 | type: "PReLU" 79 | bottom: "conv1-D" 80 | top: "conv1-D" 81 | } 82 | 83 | layer { 84 | name: "deconv2-D" 85 | type: "Deconvolution" 86 | bottom: "conv1-D" 87 | top: "deconv2-D" 88 | convolution_param { 89 | num_output: 32 90 | kernel_size: 5 91 | stride: 2 92 | pad: 2 93 | } 94 | } 95 | 96 | layer { 97 | name: "prelu2-D" 98 | type: "PReLU" 99 | bottom: "deconv2-D" 100 | top: "deconv2-D" 101 | } 102 | 103 | layer { 104 | name: "concat2" 105 | bottom: "conv2-Y" 106 | bottom: "deconv2-D" 107 | top: "concat2" 108 | type: "Concat" 109 | concat_param { 110 | axis: 1 111 | } 112 | } 113 | 114 | layer { 115 | name: "conv2a" 116 | type: "Convolution" 117 | bottom: "concat2" 118 | top: "conv2a" 119 | convolution_param { 120 | num_output: 32 121 | kernel_size: 5 122 | stride: 1 123 | pad: 2 124 | } 125 | } 126 | 127 | layer { 128 | name: "prelu2a" 129 | type: "PReLU" 130 | bottom: "conv2a" 131 | top: "conv2a" 132 | } 133 | 134 | layer { 135 | name: "conv2b" 136 | type: "Convolution" 137 | bottom: "conv2a" 138 | top: "conv2b" 139 | convolution_param { 140 | num_output: 32 141 | kernel_size: 5 142 | stride: 1 143 | pad: 2 144 | } 145 | } 146 | 147 | layer { 148 | name: "prelu2b" 149 | type: "PReLU" 150 | bottom: "conv2b" 151 | top: "conv2b" 152 | } 153 | 154 | layer { 155 | name: "conv2c" 156 | type: "Convolution" 157 | bottom: "conv2b" 158 | top: "conv2c" 159 | convolution_param { 160 | num_output: 32 161 | kernel_size: 5 162 | stride: 1 163 | pad: 2 164 | } 165 | } 166 | 167 | layer { 168 | name: "prelu2c" 169 | type: "PReLU" 170 | bottom: "conv2c" 171 | top: "conv2c" 172 | } 173 | 174 | layer { 175 | name: "conv3" 176 | type: "Convolution" 177 | bottom: "conv2c" 178 | top: "conv3" 179 | convolution_param { 180 | num_output: 1 181 | kernel_size: 5 182 | stride: 1 183 | pad: 2 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x4.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/models/MSGNet_x4.caffemodel -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x4_deploy.prototxt: -------------------------------------------------------------------------------- 1 | # MSG-Net (published in ECCV 2016) 2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved. 3 | # 4 | # This software is licensed under the terms of the MSG-Net licence 5 | # which allows for non-commercial use only unless a prior arrangement 6 | # has been made with the author (Tak-Wai Hui), the full terms of which 7 | # are made available in the LICENSE file. 8 | 9 | name: "MSGNet_x4" 10 | 11 | input: "data-Y" 12 | input_dim: 1 13 | input_dim: 1 14 | input_dim: 63 15 | input_dim: 63 16 | 17 | input: "data-D" 18 | input_dim: 1 19 | input_dim: 1 20 | input_dim: 16 21 | input_dim: 16 22 | 23 | layer { 24 | name: "conv1-Y" 25 | type: "Convolution" 26 | bottom: "data-Y" 27 | top: "conv1-Y" 28 | convolution_param { 29 | num_output: 49 30 | kernel_size: 7 31 | stride: 1 32 | pad: 3 33 | } 34 | } 35 | 36 | layer { 37 | name: "prelu1-Y" 38 | type: "PReLU" 39 | bottom: "conv1-Y" 40 | top: "conv1-Y" 41 | } 42 | 43 | layer { 44 | name: "conv2-Y" 45 | type: "Convolution" 46 | bottom: "conv1-Y" 47 | top: "conv2-Y" 48 | convolution_param { 49 | num_output: 32 50 | kernel_size: 5 51 | stride: 1 52 | pad: 2 53 | } 54 | } 55 | 56 | layer { 57 | name: "prelu2-Y" 58 | type: "PReLU" 59 | bottom: "conv2-Y" 60 | top: "conv2-Y" 61 | } 62 | 63 | layer { 64 | name: "conv3-Y" 65 | type: "Convolution" 66 | bottom: "conv2-Y" 67 | top: "conv3-Y" 68 | convolution_param { 69 | num_output: 32 70 | kernel_size: 5 71 | stride: 1 72 | pad: 2 73 | } 74 | } 75 | 76 | layer { 77 | name: "prelu3-Y" 78 | type: "PReLU" 79 | bottom: "conv3-Y" 80 | top: "conv3-Y" 81 | } 82 | 83 | layer { 84 | name: "pool3-Y" 85 | type: "Pooling" 86 | bottom: "conv3-Y" 87 | top: "pool3-Y" 88 | pooling_param { 89 | pool: MAX 90 | kernel_size: 3 91 | stride: 2 92 | pad: 0 93 | } 94 | } 95 | 96 | layer { 97 | name: "conv1-D" 98 | type: "Convolution" 99 | bottom: "data-D" 100 | top: "conv1-D" 101 | convolution_param { 102 | num_output: 64 103 | kernel_size: 5 104 | stride: 1 105 | pad: 2 106 | } 107 | } 108 | 109 | layer { 110 | name: "prelu1-D" 111 | type: "PReLU" 112 | bottom: "conv1-D" 113 | top: "conv1-D" 114 | } 115 | 116 | layer { 117 | name: "deconv2-D" 118 | type: "Deconvolution" 119 | bottom: "conv1-D" 120 | top: "deconv2-D" 121 | convolution_param { 122 | num_output: 32 123 | kernel_size: 5 124 | stride: 2 125 | pad: 2 126 | } 127 | } 128 | 129 | layer { 130 | name: "prelu2-D" 131 | type: "PReLU" 132 | bottom: "deconv2-D" 133 | top: "deconv2-D" 134 | } 135 | 136 | layer { 137 | name: "concat2-x4" 138 | bottom: "pool3-Y" 139 | bottom: "deconv2-D" 140 | top: "concat2-x4" 141 | type: "Concat" 142 | concat_param { 143 | axis: 1 144 | } 145 | } 146 | 147 | layer { 148 | name: "conv2a-x4" 149 | type: "Convolution" 150 | bottom: "concat2-x4" 151 | top: "conv2a-x4" 152 | convolution_param { 153 | num_output: 32 154 | kernel_size: 5 155 | stride: 1 156 | pad: 2 157 | } 158 | } 159 | 160 | layer { 161 | name: "prelu2a-x4" 162 | type: "PReLU" 163 | bottom: "conv2a-x4" 164 | top: "conv2a-x4" 165 | } 166 | 167 | layer { 168 | name: "conv2b-x4" 169 | type: "Convolution" 170 | bottom: "conv2a-x4" 171 | top: "conv2b-x4" 172 | convolution_param { 173 | num_output: 32 174 | kernel_size: 5 175 | stride: 1 176 | pad: 2 177 | } 178 | } 179 | 180 | layer { 181 | name: "prelu2b-x4" 182 | type: "PReLU" 183 | bottom: "conv2b-x4" 184 | top: "conv2b-x4" 185 | } 186 | 187 | layer { 188 | name: "deconv3-x4" 189 | type: "Deconvolution" 190 | bottom: "conv2b-x4" 191 | top: "deconv3-x4" 192 | convolution_param { 193 | num_output: 32 194 | kernel_size: 5 195 | stride: 2 196 | pad: 1 197 | } 198 | } 199 | 200 | layer { 201 | name: "prelu3-x4" 202 | type: "PReLU" 203 | bottom: "deconv3-x4" 204 | top: "deconv3-x4" 205 | } 206 | 207 | layer { 208 | name: "concat3-x4" 209 | bottom: "conv2-Y" 210 | bottom: "deconv3-x4" 211 | top: "concat3-x4" 212 | type: "Concat" 213 | concat_param { 214 | axis: 1 215 | } 216 | } 217 | 218 | layer { 219 | name: "conv3a" 220 | type: "Convolution" 221 | bottom: "concat3-x4" 222 | top: "conv3a" 223 | convolution_param { 224 | num_output: 32 225 | kernel_size: 5 226 | stride: 1 227 | pad: 2 228 | } 229 | } 230 | 231 | layer { 232 | name: "prelu3a" 233 | type: "PReLU" 234 | bottom: "conv3a" 235 | top: "conv3a" 236 | } 237 | 238 | layer { 239 | name: "conv3b" 240 | type: "Convolution" 241 | bottom: "conv3a" 242 | top: "conv3b" 243 | convolution_param { 244 | num_output: 32 245 | kernel_size: 5 246 | stride: 1 247 | pad: 2 248 | } 249 | } 250 | 251 | layer { 252 | name: "prelu3b" 253 | type: "PReLU" 254 | bottom: "conv3b" 255 | top: "conv3b" 256 | } 257 | 258 | layer { 259 | name: "conv3c" 260 | type: "Convolution" 261 | bottom: "conv3b" 262 | top: "conv3c" 263 | convolution_param { 264 | num_output: 32 265 | kernel_size: 5 266 | stride: 1 267 | pad: 2 268 | } 269 | } 270 | 271 | layer { 272 | name: "prelu3c" 273 | type: "PReLU" 274 | bottom: "conv3c" 275 | top: "conv3c" 276 | } 277 | 278 | layer { 279 | name: "conv4" 280 | type: "Convolution" 281 | bottom: "conv3c" 282 | top: "conv4" 283 | convolution_param { 284 | num_output: 1 285 | kernel_size: 5 286 | stride: 1 287 | pad: 2 288 | } 289 | } 290 | 291 | -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x8.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/models/MSGNet_x8.caffemodel -------------------------------------------------------------------------------- /MSGNet-release/models/MSGNet_x8_deploy.prototxt: -------------------------------------------------------------------------------- 1 | # MSG-Net (published in ECCV 2016) 2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved. 3 | # 4 | # This software is licensed under the terms of the MSG-Net licence 5 | # which allows for non-commercial use only unless a prior arrangement 6 | # has been made with the author (Tak-Wai Hui), the full terms of which 7 | # are made available in the LICENSE file. 8 | 9 | name: "MSGNet_x8" 10 | 11 | input: "data-Y" 12 | input_dim: 1 13 | input_dim: 1 14 | input_dim: 95 15 | input_dim: 95 16 | 17 | input: "data-D" 18 | input_dim: 1 19 | input_dim: 1 20 | input_dim: 12 21 | input_dim: 12 22 | 23 | layer { 24 | name: "conv1-Y" 25 | type: "Convolution" 26 | bottom: "data-Y" 27 | top: "conv1-Y" 28 | convolution_param { 29 | num_output: 49 30 | kernel_size: 7 31 | stride: 1 32 | pad: 3 33 | } 34 | } 35 | 36 | layer { 37 | name: "prelu1-Y" 38 | type: "PReLU" 39 | bottom: "conv1-Y" 40 | top: "conv1-Y" 41 | } 42 | 43 | layer { 44 | name: "conv2-Y" 45 | type: "Convolution" 46 | bottom: "conv1-Y" 47 | top: "conv2-Y" 48 | convolution_param { 49 | num_output: 32 50 | kernel_size: 5 51 | stride: 1 52 | pad: 2 53 | } 54 | } 55 | 56 | layer { 57 | name: "prelu2-Y" 58 | type: "PReLU" 59 | bottom: "conv2-Y" 60 | top: "conv2-Y" 61 | } 62 | 63 | layer { 64 | name: "conv3-Y" 65 | type: "Convolution" 66 | bottom: "conv2-Y" 67 | top: "conv3-Y" 68 | convolution_param { 69 | num_output: 32 70 | kernel_size: 5 71 | stride: 1 72 | pad: 2 73 | } 74 | } 75 | 76 | layer { 77 | name: "prelu3-Y" 78 | type: "PReLU" 79 | bottom: "conv3-Y" 80 | top: "conv3-Y" 81 | } 82 | 83 | layer { 84 | name: "pool3-Y" 85 | type: "Pooling" 86 | bottom: "conv3-Y" 87 | top: "pool3-Y" 88 | pooling_param { 89 | pool: MAX 90 | kernel_size: 3 91 | stride: 2 92 | pad: 0 93 | } 94 | } 95 | 96 | layer { 97 | name: "conv4-Y" 98 | type: "Convolution" 99 | bottom: "pool3-Y" 100 | top: "conv4-Y" 101 | convolution_param { 102 | num_output: 32 103 | kernel_size: 5 104 | stride: 1 105 | pad: 2 106 | } 107 | } 108 | 109 | layer { 110 | name: "prelu4-Y" 111 | type: "PReLU" 112 | bottom: "conv4-Y" 113 | top: "conv4-Y" 114 | } 115 | 116 | layer { 117 | name: "pool4-Y" 118 | type: "Pooling" 119 | bottom: "conv4-Y" 120 | top: "pool4-Y" 121 | pooling_param { 122 | pool: MAX 123 | kernel_size: 3 124 | stride: 2 125 | pad: 0 126 | } 127 | } 128 | 129 | layer { 130 | name: "conv1-D" 131 | type: "Convolution" 132 | bottom: "data-D" 133 | top: "conv1-D" 134 | convolution_param { 135 | num_output: 64 136 | kernel_size: 5 137 | stride: 1 138 | pad: 2 139 | } 140 | } 141 | 142 | layer { 143 | name: "prelu1-D" 144 | type: "PReLU" 145 | bottom: "conv1-D" 146 | top: "conv1-D" 147 | } 148 | 149 | layer { 150 | name: "deconv2-D" 151 | type: "Deconvolution" 152 | bottom: "conv1-D" 153 | top: "deconv2-D" 154 | convolution_param { 155 | num_output: 32 156 | kernel_size: 5 157 | stride: 2 158 | pad: 2 159 | } 160 | } 161 | 162 | layer { 163 | name: "prelu2-D" 164 | type: "PReLU" 165 | bottom: "deconv2-D" 166 | top: "deconv2-D" 167 | } 168 | 169 | layer { 170 | name: "concat2-x8" 171 | bottom: "pool4-Y" 172 | bottom: "deconv2-D" 173 | top: "concat2-x8" 174 | type: "Concat" 175 | concat_param { 176 | axis: 1 177 | } 178 | } 179 | 180 | layer { 181 | name: "conv2a-x4" 182 | type: "Convolution" 183 | bottom: "concat2-x8" 184 | top: "conv2a-x4" 185 | convolution_param { 186 | num_output: 32 187 | kernel_size: 5 188 | stride: 1 189 | pad: 2 190 | } 191 | } 192 | 193 | layer { 194 | name: "prelu2a-x4" 195 | type: "PReLU" 196 | bottom: "conv2a-x4" 197 | top: "conv2a-x4" 198 | } 199 | 200 | layer { 201 | name: "conv2b-x4" 202 | type: "Convolution" 203 | bottom: "conv2a-x4" 204 | top: "conv2b-x4" 205 | convolution_param { 206 | num_output: 32 207 | kernel_size: 5 208 | stride: 1 209 | pad: 2 210 | } 211 | } 212 | 213 | layer { 214 | name: "prelu2b-x4" 215 | type: "PReLU" 216 | bottom: "conv2b-x4" 217 | top: "conv2b-x4" 218 | } 219 | 220 | layer { 221 | name: "deconv3-x4" 222 | type: "Deconvolution" 223 | bottom: "conv2b-x4" 224 | top: "deconv3-x4" 225 | convolution_param { 226 | num_output: 32 227 | kernel_size: 5 228 | stride: 2 229 | pad: 1 230 | } 231 | } 232 | 233 | layer { 234 | name: "prelu3-x4" 235 | type: "PReLU" 236 | bottom: "deconv3-x4" 237 | top: "deconv3-x4" 238 | } 239 | 240 | layer { 241 | name: "concat3-x8" 242 | bottom: "pool3-Y" 243 | bottom: "deconv3-x4" 244 | top: "concat3-x8" 245 | type: "Concat" 246 | concat_param { 247 | axis: 1 248 | } 249 | } 250 | 251 | layer { 252 | name: "conv3a-x8" 253 | type: "Convolution" 254 | bottom: "concat3-x8" 255 | top: "conv3a-x8" 256 | convolution_param { 257 | num_output: 32 258 | kernel_size: 5 259 | stride: 1 260 | pad: 2 261 | } 262 | } 263 | 264 | layer { 265 | name: "prelu3a-x8" 266 | type: "PReLU" 267 | bottom: "conv3a-x8" 268 | top: "conv3a-x8" 269 | } 270 | 271 | layer { 272 | name: "conv3b-x8" 273 | type: "Convolution" 274 | bottom: "conv3a-x8" 275 | top: "conv3b-x8" 276 | convolution_param { 277 | num_output: 32 278 | kernel_size: 5 279 | stride: 1 280 | pad: 2 281 | } 282 | } 283 | 284 | layer { 285 | name: "prelu3b-x8" 286 | type: "PReLU" 287 | bottom: "conv3b-x8" 288 | top: "conv3b-x8" 289 | } 290 | 291 | layer { 292 | name: "deconv4-x8" 293 | type: "Deconvolution" 294 | bottom: "conv3b-x8" 295 | top: "deconv4-x8" 296 | convolution_param { 297 | num_output: 32 298 | kernel_size: 5 299 | stride: 2 300 | pad: 1 301 | } 302 | } 303 | 304 | layer { 305 | name: "prelu4-x8" 306 | type: "PReLU" 307 | bottom: "deconv4-x8" 308 | top: "deconv4-x8" 309 | } 310 | 311 | layer { 312 | name: "concat4-x8" 313 | bottom: "conv2-Y" 314 | bottom: "deconv4-x8" 315 | top: "concat4-x8" 316 | type: "Concat" 317 | concat_param { 318 | axis: 1 319 | } 320 | } 321 | 322 | layer { 323 | name: "conv4a" 324 | type: "Convolution" 325 | bottom: "concat4-x8" 326 | top: "conv4a" 327 | convolution_param { 328 | num_output: 32 329 | kernel_size: 5 330 | stride: 1 331 | pad: 2 332 | } 333 | } 334 | 335 | layer { 336 | name: "prelu4a" 337 | type: "PReLU" 338 | bottom: "conv4a" 339 | top: "conv4a" 340 | } 341 | 342 | layer { 343 | name: "conv4b" 344 | type: "Convolution" 345 | bottom: "conv4a" 346 | top: "conv4b" 347 | convolution_param { 348 | num_output: 32 349 | kernel_size: 5 350 | stride: 1 351 | pad: 2 352 | } 353 | } 354 | 355 | layer { 356 | name: "prelu4b" 357 | type: "PReLU" 358 | bottom: "conv4b" 359 | top: "conv4b" 360 | } 361 | 362 | layer { 363 | name: "conv4c" 364 | type: "Convolution" 365 | bottom: "conv4b" 366 | top: "conv4c" 367 | convolution_param { 368 | num_output: 32 369 | kernel_size: 5 370 | stride: 1 371 | pad: 2 372 | } 373 | } 374 | 375 | layer { 376 | name: "prelu4c" 377 | type: "PReLU" 378 | bottom: "conv4c" 379 | top: "conv4c" 380 | } 381 | 382 | layer { 383 | name: "conv5" 384 | type: "Convolution" 385 | bottom: "conv4c" 386 | top: "conv5" 387 | convolution_param { 388 | num_output: 1 389 | kernel_size: 5 390 | stride: 1 391 | pad: 2 392 | } 393 | } 394 | 395 | -------------------------------------------------------------------------------- /MSGNet-release/testing sets/A/Depth/art.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/A/Depth/art.png -------------------------------------------------------------------------------- /MSGNet-release/testing sets/A/Depth/books.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/A/Depth/books.png -------------------------------------------------------------------------------- /MSGNet-release/testing sets/A/Depth/moebius.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/A/Depth/moebius.png -------------------------------------------------------------------------------- /MSGNet-release/testing sets/A/RGB/art.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/A/RGB/art.png -------------------------------------------------------------------------------- /MSGNet-release/testing sets/A/RGB/books.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/A/RGB/books.png -------------------------------------------------------------------------------- /MSGNet-release/testing sets/A/RGB/moebius.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/A/RGB/moebius.png -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/Depth/art.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/Depth/art.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/Depth/books.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/Depth/books.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/Depth/dolls.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/Depth/dolls.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/Depth/laundry.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/Depth/laundry.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/Depth/moebius.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/Depth/moebius.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/Depth/reindeer.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/Depth/reindeer.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/RGB/art.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/RGB/art.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/RGB/books.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/RGB/books.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/RGB/dolls.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/RGB/dolls.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/RGB/laundry.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/RGB/laundry.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/RGB/moebius.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/RGB/moebius.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/B/RGB/reindeer.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/B/RGB/reindeer.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/Depth/cones_2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/Depth/cones_2.mat -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/Depth/teddy_2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/Depth/teddy_2.mat -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/Depth/tsukuba_2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/Depth/tsukuba_2.mat -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/Depth/venus_2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/Depth/venus_2.mat -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/RGB/cones.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/RGB/cones.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/RGB/teddy.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/RGB/teddy.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/RGB/tsukuba.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/RGB/tsukuba.bmp -------------------------------------------------------------------------------- /MSGNet-release/testing sets/C/RGB/venus.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twhui/MSG-Net/852da0a093e530370ccb069507847220644df331/MSGNet-release/testing sets/C/RGB/venus.bmp -------------------------------------------------------------------------------- /MSGNet-release/util/MSGNet.m: -------------------------------------------------------------------------------- 1 | % MSG-Net (published in ECCV 2016) 2 | % Copyright (c) 2016 Tak-Wai Hui. All rights reserved. 3 | % 4 | % This software is licensed under the terms of the MSG-Net licence 5 | % which allows for non-commercial use only unless a prior arrangement 6 | % has been made with the author (Tak-Wai Hui), the full terms of which 7 | % are made available in the LICENSE file. 8 | 9 | clear all; 10 | close all; 11 | clc; 12 | addpath('./matlab'); 13 | root = './examples/MSGNet-release/'; 14 | 15 | %% CNN models 16 | cnn = './models/MSGNet_x2'; up_scale = 2; 17 | % cnn = './models/MSGNet_x4'; up_scale = 4; 18 | % cnn = './models/MSGNet_x8'; up_scale = 8; 19 | % cnn = './models/MSGNet_x16'; up_scale = 16; 20 | 21 | %% Testing image 22 | split = true; evaSet = 'A'; image_name = {'art', 'books', 'moebius'}; 23 | image_id = 1; 24 | 25 | % split = true; evaSet = 'B'; image_name = {'art', 'books', 'dolls', 'laundry', 'moebius', 'reindeer'}; 26 | % image_id = 3; 27 | 28 | % split = false; evaSet = 'C'; image_name = {'tsukuba', 'venus', 'teddy', 'cones'}; 29 | % image_id = 4; 30 | 31 | %% Parameters settings 32 | gpu_id = 7; 33 | h = ones(3,3)/9; 34 | hole_defect = false; 35 | margin = 8*up_scale; 36 | 37 | %% Read ground truth depth map and RGB image 38 | if strcmp(evaSet, 'A') 39 | imGtD0 = imread([root './testing sets/A/Depth/' image_name{image_id} '.png']); 40 | imRGB0 = imread([root './testing sets/A/RGB/' image_name{image_id} '.png']); 41 | [M, N] = size(imGtD0); 42 | [Mmb, Nmb, ~] = size(imRGB0); 43 | dd = [Mmb - M; Nmb - N]; 44 | imRGB0 = imRGB0(dd(1)/2+1:end-dd(1)/2, dd(2)/2+1:end-dd(2)/2, :); 45 | 46 | elseif strcmp(evaSet, 'B') 47 | imGtD0 = imread([root './testing sets/B/Depth/' image_name{image_id} '.bmp']); 48 | imRGB0 = imread([root './testing sets/B/RGB/' image_name{image_id} '.bmp']); 49 | 50 | if strcmp(image_name{image_id}, 'dolls') 51 | hole_defect = true; 52 | end 53 | 54 | elseif strcmp(evaSet, 'C') 55 | load([root './testing sets/C/Depth/' image_name{image_id} '_2']); 56 | imGtD0 = img_double; 57 | imRGB0 = imread([root './testing sets/C/RGB/' image_name{image_id} '.bmp']); 58 | [M, N] = size(imGtD0); 59 | [Mmb, Nmb, ~] = size(imRGB0); 60 | dd = [Mmb - M; Nmb - N]; 61 | imRGB0 = imRGB0(1:end-dd(1), 1:end-dd(2), :); 62 | end 63 | 64 | %% Main 65 | im_RGB0 = modcrop(imRGB0, up_scale); 66 | im_GtD0 = modcrop(imGtD0, up_scale); 67 | size_Dh = size(im_GtD0); 68 | size_Dl = size_Dh/up_scale; 69 | 70 | % Due to the limit of MAX_INT in Caffe, upsampling of depth map is 71 | % performed patch by patch. Final result is obtained by combining the two 72 | % patches together. 73 | if split 74 | num_split = 2; 75 | 76 | if margin > 0 77 | size_Dl(1) = size_Dl(1)/2; 78 | size_Dh(1) = size_Dh(1)/2; 79 | else 80 | error('Padded margin size must be greater than zero !\n'); 81 | end 82 | 83 | cropSz_Dh = (size_Dl(1) - floor(size_Dl(1)))*up_scale; 84 | if cropSz_Dh > 0 85 | fprintf('Increased margin by %d pixels \n\n', cropSz_Dh); 86 | margin = margin + cropSz_Dh; 87 | else 88 | fprintf('\n'); 89 | end 90 | 91 | ims_RGB{1} = im_RGB0(1:size_Dh(1)+margin,:,:); 92 | ims_GtD{1} = im_GtD0(1:size_Dh(1)+margin,:); 93 | 94 | ims_RGB{2} = im_RGB0(size_Dh(1)+1-margin:end,:,:); 95 | ims_GtD{2} = im_GtD0(size_Dh(1)+1-margin:end,:); 96 | 97 | else 98 | num_split = 1; 99 | ims_RGB{1} = im_RGB0; 100 | ims_GtD{1} = im_GtD0; 101 | end 102 | 103 | for k_sub = 1 : num_split 104 | 105 | if num_split > 1 106 | fprintf('Splitting %d of %d\n', k_sub, num_split); 107 | end 108 | 109 | % Y-channel 110 | if size(ims_RGB{k_sub}, 3) > 1 111 | im_I = rgb2ycbcr(ims_RGB{k_sub}); 112 | im_Y = double(im_I(:,:,1)); 113 | else 114 | im_Y = double(im_I); 115 | end 116 | im_Y = normalize_cleanIm(im_Y); 117 | im_Y_LF = imfilter(im_Y, h, 'symmetric'); 118 | im_Y = normalize_cleanIm(im_Y - im_Y_LF); 119 | 120 | im_Y = im_Y(1:end-1, 1:end-1); 121 | 122 | % D-channel 123 | im_GtD = double(ims_GtD{k_sub}); 124 | im_Dl = imresize(im_GtD, 1/up_scale, 'bicubic'); 125 | [im_Dl, min_D, max_D] = normalize_cleanIm(im_Dl); 126 | 127 | im_Dl_LF = imfilter(im_Dl, h, 'symmetric'); 128 | in_D = im_Dl - im_Dl_LF; 129 | 130 | % Run CNN 131 | caffe.reset_all(); 132 | caffe.set_mode_gpu(); 133 | caffe.set_device(gpu_id); 134 | 135 | net_model = [root cnn '_deploy.prototxt']; 136 | net_weights = [root cnn '.caffemodel']; 137 | 138 | net = caffe.Net(net_model, net_weights, 'test'); 139 | 140 | blobs_input{1} = permute(im_Y, [2 1 3]); 141 | blobs_input{2} = permute(in_D, [2 1 3]); 142 | 143 | net.blobs('data-Y').reshape([size(blobs_input{1},1) size(blobs_input{1},2) 1 1]); 144 | net.blobs('data-D').reshape([size(blobs_input{2},1) size(blobs_input{2},2) 1 1]); 145 | net.reshape(); 146 | 147 | fprintf('Run CNN ... \n'); 148 | blobs_output = net.forward(blobs_input); 149 | fprintf('Finished. \n'); 150 | 151 | im_Dh = permute(blobs_output{1}, [2 1 3]); 152 | im_D_LF = imresize(im_Dl_LF, up_scale, 'bicubic'); 153 | im_Dh = im_Dh + im_D_LF(1:end-1, 1:end-1); 154 | im_Dh = im_Dh*(max_D - min_D) + min_D; 155 | 156 | if split 157 | im_Dh_sub(:,:,k_sub) = im_Dh; 158 | end 159 | 160 | end 161 | 162 | %% Combine splitted images to a single one if necessary 163 | if split 164 | im_Dh = [im_Dh_sub(1:end-(margin-1),:,1); im_Dh_sub(1+margin:end,:,2)]; 165 | end 166 | 167 | %% Prepare ground truth 168 | im_GtD = double(im_GtD0); 169 | 170 | if hole_defect 171 | im_occ = im_GtD > 9; 172 | else 173 | im_occ = true(size(im_GtD)); 174 | end 175 | 176 | im_GtD = im_GtD(1:end-1, 1:end-1); 177 | im_occ = im_occ(1:end-1, 1:end-1); 178 | 179 | %% Convert to uint8 or uint16 180 | if max(max(im_GtD)) > 2^8-1 181 | im_Dh = uint16(im_Dh); 182 | im_GtD = uint16(im_GtD); 183 | else 184 | im_Dh = uint8(im_Dh); 185 | im_GtD = uint8(im_GtD); 186 | end 187 | 188 | %% Compute RMSE 189 | rmse_cnn = compute_RMSE(im_GtD, im_Dh, im_occ); 190 | 191 | fprintf('------------------------------------------\n'); 192 | fprintf('Model: %s\n', cnn); 193 | fprintf('Testing set %s: %s\n', evaSet, image_name{image_id}); 194 | fprintf('MSG-Net %dx upsampling, RMSE = %.3f\n', up_scale, rmse_cnn); 195 | 196 | -------------------------------------------------------------------------------- /MSGNet-release/util/compute_RMSE.m: -------------------------------------------------------------------------------- 1 | function rmse = compute_RMSE(im1, im2, occ) 2 | if size(im1, 3) == 3, 3 | im1 = rgb2ycbcr(im1); 4 | im1 = im1(:, :, 1); 5 | end 6 | 7 | if size(im2, 3) == 3, 8 | im2 = rgb2ycbcr(im2); 9 | im2 = im2(:, :, 1); 10 | end 11 | 12 | imdff = double(im1) - double(im2); 13 | imdff = imdff(occ == 1); 14 | 15 | rmse = sqrt(mean(imdff.^2)); 16 | -------------------------------------------------------------------------------- /MSGNet-release/util/gen_train_data_x2.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------------------- 2 | % Depth Map Super-Resolution by Deep Multi-Scale Guidance, ECCV16 3 | % written by T.-W. HUI 4 | %--------------------------------------------------------------- 5 | clear all; 6 | close all; 7 | clc; 8 | 9 | %% Settings 10 | dataDir = '/path/to/training_data/'; 11 | saveDir = '/path/to/h5/'; 12 | 13 | chunksz = 128; 14 | 15 | scale = 2; 16 | size_dataY = 40; 17 | size_dataD = 20; 18 | size_label = 40; 19 | border_dataY = 1; 20 | border_label = 1; 21 | stride = 22; 22 | data_aug = true; 23 | index_val = [1 20 28 58 64 66 69 73 75 79]; 24 | 25 | %% Initialization 26 | dataY = zeros(size_dataY, size_dataY, 1, 1); 27 | dataD = zeros(size_dataD, size_dataD, 1, 1); 28 | label = zeros(size_label, size_label, 1, 1); 29 | 30 | size_subIm = size_dataD*scale; 31 | 32 | pad_dataY = (size_subIm - size_dataY)/2; 33 | pad_label = (size_subIm - size_label)/2; 34 | assert(pad_dataY >= 0, 'pad_dataY cannot be an negative number!'); 35 | assert(pad_label >= 0, 'pad_label cannot be an negative number!'); 36 | 37 | count = 0; 38 | 39 | %% Generate training data 40 | load([dataDir '/RGB']); 41 | load([dataDir '/Df']); 42 | 43 | index_train = setdiff(1:numel(Df), index_val); 44 | fprintf('%d images is used for training.\n', numel(index_train)); 45 | 46 | RGB = RGB(index_train); 47 | Df = Df(index_train); 48 | 49 | h = ones(3,3)/9; 50 | 51 | for k = 1 : size(Df,1) 52 | 53 | fprintf('Processing %d of %d, id: %d, ', k, size(Df,1), index_train(k)); 54 | 55 | im_Dh = modcrop(Df{k}, scale); 56 | im_M = im_Dh > 0; 57 | im_Dh = normalizeIm(im_Dh, [0 1]); 58 | im_Dl = imresize(im_Dh, 1/scale, 'bicubic'); 59 | im_Dl_LF = imfilter(im_Dl, h, 'symmetric'); 60 | im_label = im_Dh - imresize(im_Dl_LF, scale, 'bicubic'); 61 | [H, W] = size(im_label); 62 | 63 | I = RGB{k}; 64 | I = rgb2ycbcr(I); 65 | Y = double(I(:,:,1)); 66 | im_Y = modcrop(Y, scale); 67 | im_Y = normalize_cleanIm(im_Y); 68 | im_Y_LF = imfilter(im_Y, h, 'symmetric'); 69 | im_inputY = normalize_cleanIm(im_Y - im_Y_LF); 70 | 71 | im_grad_Dh = imgradient(im_Dh, 'sobel'); 72 | 73 | stride_ = stride; 74 | 75 | if index_train(k) >= 75 76 | stride_ = floor(2.0*stride_); 77 | elseif index_train(k) >= 65 78 | stride_ = floor(1.5*stride_); 79 | end 80 | 81 | for x = 1 : stride_ : H-size_subIm+1 82 | for y = 1 :stride_ : W-size_subIm+1 83 | 84 | subIm_M = im_M(x : x+size_subIm-1, y : y+size_subIm-1); 85 | 86 | if sum(sum(subIm_M == 0)) == 0 87 | 88 | subim_dataY = im_inputY(x+pad_dataY : x+pad_dataY+size_dataY-1, y+pad_dataY : y+pad_dataY+size_dataY-1); 89 | 90 | subim_dataD = imresize(im_Dh(x : x+size_subIm-1, y : y+size_subIm-1), 1/scale, 'bicubic'); 91 | subim_dataD = subim_dataD - imfilter(subim_dataD, h, 'symmetric'); 92 | 93 | subim_label = im_label(x+pad_label : x+pad_label+size_label-1, y+pad_label : y+pad_label+size_label-1); 94 | 95 | count = count + 1; 96 | dataY(:, :, 1, count) = subim_dataY; 97 | dataD(:, :, 1, count) = subim_dataD; 98 | label(:, :, 1, count) = subim_label; 99 | 100 | end 101 | 102 | end 103 | end 104 | 105 | end 106 | 107 | fprintf('---------------------------------------\n') 108 | 109 | fprintf('Total: %d patches.\n', count); 110 | num_patches = count; 111 | 112 | % Data augmntation 113 | if data_aug 114 | num_patches = num_patches*2; 115 | dataY_tmp = dataY; clear dataY; 116 | dataD_tmp = dataD; clear dataD; 117 | label_tmp = label; clear label; 118 | dataY = cat(4, dataY_tmp, rot90(dataY_tmp,1)); 119 | dataD = cat(4, dataD_tmp, rot90(dataD_tmp,1)); 120 | label = cat(4, label_tmp, rot90(label_tmp,1)); 121 | end 122 | 123 | fprintf('Number of patches used: %d\n', num_patches); 124 | 125 | % Remove left-most column and bottom row of dataY and label 126 | dataY = dataY(1:end-border_dataY,1:end-border_dataY,:,:); 127 | label = label(1:end-border_label,1:end-border_label,:,:); 128 | 129 | % Re-arrange order 130 | order = randperm(num_patches); 131 | dataY = dataY(:,:,:,order); 132 | dataD = dataD(:,:,:,order); 133 | label = label(:,:,:,order); 134 | 135 | %% writing to HDF5 136 | created_flag = false; 137 | totalct = 0; 138 | 139 | for batchno = 1 : floor(num_patches/chunksz) 140 | last_read = (batchno - 1)*chunksz; 141 | batchdata = dataY(:, :, :, last_read+1 : last_read+chunksz); 142 | 143 | startloc = struct('dat', [1,1,1,totalct+1], 'lab', [1,1,1,totalct+1]); 144 | curr_dat_sz = store2hdf5_v2(savepathY, '-Y', batchdata, [], [], ~created_flag, startloc, chunksz); 145 | created_flag = true; 146 | totalct = curr_dat_sz(end); 147 | end 148 | 149 | h5disp(savepathY); 150 | 151 | 152 | created_flag = false; 153 | totalct = 0; 154 | 155 | for batchno = 1 : floor(num_patches/chunksz) 156 | last_read = (batchno - 1)*chunksz; 157 | batchdata = dataD(:, :, :, last_read+1 : last_read+chunksz); 158 | batchlabs = label(:, :, :, last_read+1 : last_read+chunksz); 159 | 160 | startloc = struct('dat', [1,1,1,totalct+1], 'lab', [1,1,1,totalct+1]); 161 | curr_dat_sz = store2hdf5_v2(savepathD, '-D', batchdata, [], batchlabs, ~created_flag, startloc, chunksz); 162 | created_flag = true; 163 | totalct = curr_dat_sz(end); 164 | end 165 | 166 | h5disp(savepathD); 167 | -------------------------------------------------------------------------------- /MSGNet-release/util/modcrop.m: -------------------------------------------------------------------------------- 1 | function imgs = modcrop(imgs, modulo) 2 | if size(imgs,3)==1 3 | sz = size(imgs); 4 | sz = sz - mod(sz, modulo); 5 | imgs = imgs(1:sz(1), 1:sz(2)); 6 | else 7 | tmpsz = size(imgs); 8 | sz = tmpsz(1:2); 9 | sz = sz - mod(sz, modulo); 10 | imgs = imgs(1:sz(1), 1:sz(2),:); 11 | end 12 | 13 | -------------------------------------------------------------------------------- /MSGNet-release/util/normalizeIm.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------------------- 2 | % Depth Map Super-Resolution by Deep Multi-Scale Guidance, ECCV16 3 | % written by T.-W. HUI 4 | %---------------------------------------------------------------- 5 | function Z = normalizeIm(Z, range) 6 | 7 | if nargin < 2 8 | range = [0.01 1]; 9 | end 10 | 11 | validMap = Z > 0; 12 | Z = Z - min(Z(validMap)); 13 | Z = Z/max(Z(validMap)); 14 | Z(~validMap) = 0; 15 | Z(validMap) = Z(validMap) + range(1); 16 | Z = range(2)*Z/max(max(Z)); 17 | 18 | end -------------------------------------------------------------------------------- /MSGNet-release/util/normalize_cleanIm.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------------------- 2 | % Depth Map Super-Resolution by Deep Multi-Scale Guidance, ECCV16 3 | % written by T.-W. HUI 4 | %---------------------------------------------------------------- 5 | function [Z, min_Z, max_Z] = normalize_cleanIm(Z, min_Z, max_Z) 6 | 7 | if nargin == 1 8 | min_Z = min(min(Z)); 9 | max_Z = max(max(Z)); 10 | end 11 | 12 | Z = (Z - min_Z)/(max_Z - min_Z); 13 | 14 | end -------------------------------------------------------------------------------- /MSGNet-release/util/store2hdf5_v2.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------------------- 2 | % Depth Map Super-Resolution by Deep Multi-Scale Guidance, ECCV16 3 | % written by T.-W. HUI 4 | %---------------------------------------------------------------- 5 | function [curr_dat_sz, curr_lab_sz] = store2hdf5_v2(filename, dataname, data, labelname, labels, create, startloc, chunksz) 6 | 7 | % *data* is W*H*C*N matrix of images should be normalized (e.g. to lie between 0 and 1) beforehand 8 | % *label* is D*N matrix of labels (D labels per sample) 9 | % *create* [0/1] specifies whether to create file newly or to append to previously created file, useful to store information in batches when a dataset is too big to be held in memory (default: 1) 10 | % *startloc* (point at which to start writing data). By default, 11 | % if create=1 (create mode), startloc.data = [1 1 1 1], and startloc.lab = [1 1]; 12 | % if create=0 (append mode), startloc.data = [1 1 1 K+1], and startloc.lab = [1 K+1]; where K is the current number of samples stored in the HDF 13 | % chunksz (used only in create mode), specifies number of samples to be stored per chunk (see HDF5 documentation on chunking) for creating HDF5 files with unbounded maximum size - TLDR; higher chunk sizes allow faster read-write operations 14 | 15 | % verify that format is right 16 | dat_dims = size(data); 17 | if ~isempty(labels) 18 | lab_dims = size(labels); 19 | num_samples = dat_dims(end); 20 | assert(lab_dims(end) == num_samples, 'Number of samples should be matched between data and labels'); 21 | end 22 | 23 | if ~exist('create','var') 24 | create = true; 25 | end 26 | 27 | if create 28 | %fprintf('Creating dataset with %d samples\n', num_samples); 29 | if ~exist('chunksz', 'var') 30 | chunksz = 1000; 31 | end 32 | if exist(filename, 'file') 33 | fprintf('Warning: replacing existing file %s \n', filename); 34 | delete(filename); 35 | end 36 | if ~isempty(dataname) 37 | h5create(filename, ['/data' dataname], [dat_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat_dims(1:end-1) chunksz]); % width, height, channels, number 38 | else 39 | h5create(filename, '/data', [dat_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat_dims(1:end-1) chunksz]); % width, height, channels, number 40 | end 41 | if ~isempty(labels) 42 | if ~isempty(labelname) 43 | h5create(filename, ['/label' labelname], [lab_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [lab_dims(1:end-1) chunksz]); % width, height, channels, number 44 | else 45 | h5create(filename, '/label', [lab_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [lab_dims(1:end-1) chunksz]); % width, height, channels, number 46 | end 47 | end 48 | if ~exist('startloc','var') 49 | startloc.dat = [ones(1,length(dat_dims)-1), 1]; 50 | if ~isempty(labelname) 51 | startloc.lab = [ones(1,length(lab_dims)-1), 1]; 52 | end 53 | end 54 | 55 | else % append mode 56 | if ~exist('startloc','var') 57 | info = h5info(filename); 58 | 59 | prev_dat_sz = info.Datasets(1).Dataspace.Size; 60 | assert(prev_dat_sz(1:end-1) == dat_dims(1:end-1), 'Data dimensions must match existing dimensions in dataset'); 61 | startloc.dat = [ones(1,length(dat_dims)-1), prev_dat_sz(end)+1]; 62 | 63 | if ~isempty(labels) 64 | prev_lab_sz = info.Datasets(2).Dataspace.Size; 65 | assert(prev_lab_sz(1:end-1) == lab_dims(1:end-1), 'Label dimensions must match existing dimensions in dataset'); 66 | startloc.lab = [ones(1,length(lab_dims)-1), prev_lab_sz(end)+1]; 67 | end 68 | end 69 | end 70 | 71 | if ~isempty(data) 72 | if ~isempty(dataname) 73 | h5write(filename, ['/data' dataname], single(data), startloc.dat, size(data)); 74 | else 75 | h5write(filename, '/data', single(data), startloc.dat, size(data)); 76 | end 77 | if ~isempty(labels) 78 | if ~isempty(labelname) 79 | h5write(filename, ['/label' labelname], single(labels), startloc.lab, size(labels)); 80 | else 81 | h5write(filename, '/label', single(labels), startloc.lab, size(labels)); 82 | end 83 | end 84 | end 85 | 86 | if nargout 87 | info = h5info(filename); 88 | curr_dat_sz = info.Datasets(1).Dataspace.Size; 89 | if ~isempty(labels) 90 | curr_lab_sz = info.Datasets(2).Dataspace.Size; 91 | end 92 | end 93 | 94 | end 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MSG-Net (Multi-scale guidance network) 2 |

3 |

4 |

Multi-scale guidance network

5 | 6 |

7 |

8 | 9 | This repository (https://github.com/twhui/MSG-Net) is the offical release of MSG-Net for our paper Depth Map Super-Resolution by Deep Multi-Scale Guidance in ECCV16. It comes with four trained networks (x2, x4, x8, and x16), one hole-filled RGBD training set, and three hole-filled RGBD testing sets (A, B, and C). 10 | 11 | To the best of our knowledge, MSG-Net is the FIRST convolution neural networkwhich attempts to upsample depth images under multi-scale guidance from the corresponding high-resolution RGB images. 12 | 13 | Another repository for MS-Net (without multi-scale guidance) is also available. 14 | 15 | For more details, please visit my project page. 16 | 17 | # License and Citation 18 | This software and associated documentation files (the "Software"), and the research paper (Depth Map Super-Resolution by Deep Multi-Scale Guidance) including but not limited to the figures, and tables (the "Paper") are provided for academic research purposes only and without any warranty. Any commercial use requires my consent. When using any parts of the Software or the Paper in your work, please cite the following paper 19 | 20 | 
@InProceedings{hui16msgnet,    
21 |  author = {Tak-Wai Hui and Chen Change Loy and and Xiaoou Tang},    
22 |  title  = {Depth Map Super-Resolution by Deep Multi-Scale Guidance},    
23 |  booktitle  = {Proceedings of European Conference on Computer Vision (ECCV)},    
24 |  pages = {353--369},  
25 |  year = {2016},    
26 |  url = {http://mmlab.ie.cuhk.edu.hk/projects/guidance_SR_depth.html}
27 | }
28 | 29 | # Dependency 30 | We train our models using caffe and evaluate the results on Matlab. 31 | 32 | # Installation and Running 33 | You need to install caffe and remeber to complie matcaffe. You can put the folder MSGNet-release in caffe/examples. Finally, you need to get into the the directory of examples/MSGNet-release/util, and run MSGNet.m. 34 | 35 | # Training data 36 | Our RGBD training set consists of 58 RGBD images from MPI Sintel depth dataset, and 34 RGBD images from Middlebury dataset. 82 images are used for training and 10 images (frames 1, 20, 28, 58, 64, 66, 69, 73, 75 and 79) are used for validation. 37 | 38 | # Training sub-images/patches (hdf5 format) 39 | An example for generating training patches of the scale factor 2 can be found here util/gen_train_data_x2.m. 40 | 41 | # Testing data 42 | Testig set is available at the folder MSGNet-release/testing sets. 43 | --------------------------------------------------------------------------------