├── LICENSE
├── MSNet-release
├── models
│ ├── MSNet_x16.caffemodel
│ ├── MSNet_x16_deploy.prototxt
│ ├── MSNet_x2.caffemodel
│ ├── MSNet_x2_deploy.prototxt
│ ├── MSNet_x4.caffemodel
│ ├── MSNet_x4_deploy.prototxt
│ ├── MSNet_x8.caffemodel
│ └── MSNet_x8_deploy.prototxt
├── testing sets
│ ├── A
│ │ ├── Depth
│ │ │ ├── art.png
│ │ │ ├── books.png
│ │ │ └── moebius.png
│ │ └── RGB
│ │ │ ├── art.png
│ │ │ ├── books.png
│ │ │ └── moebius.png
│ ├── B
│ │ ├── Depth
│ │ │ ├── art.bmp
│ │ │ ├── books.bmp
│ │ │ ├── dolls.bmp
│ │ │ ├── laundry.bmp
│ │ │ ├── moebius.bmp
│ │ │ └── reindeer.bmp
│ │ └── RGB
│ │ │ ├── art.bmp
│ │ │ ├── books.bmp
│ │ │ ├── dolls.bmp
│ │ │ ├── laundry.bmp
│ │ │ ├── moebius.bmp
│ │ │ └── reindeer.bmp
│ └── C
│ │ ├── Depth
│ │ ├── cones_2.mat
│ │ ├── teddy_2.mat
│ │ ├── tsukuba_2.mat
│ │ └── venus_2.mat
│ │ └── RGB
│ │ ├── cones.bmp
│ │ ├── teddy.bmp
│ │ ├── tsukuba.bmp
│ │ └── venus.bmp
└── util
│ ├── MSNet.m
│ ├── compute_RMSE.m
│ ├── modcrop.m
│ └── normalize_cleanIm.m
└── README.md
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2016 Tak-Wai Hui. All rights reserved.
2 |
3 | This software and the associated documentation files (the "Software"), and the research paper
4 | Depth Map Super-Resolution by Deep Multi-Scale Guidance (the "Paper") including but not limited
5 | to the figures and tables are provided for non-commercial use only and without any warranty.
6 | Any commercial use requires a prior arrangement with the author (Tak-Wai Hui). When using any
7 | parts of the Software or the Paper in your work, please cite the following paper:
8 |
9 | @InProceedings{hui16msgnet,
10 | author = {Tak-Wai Hui and Chen Change Loy and and Xiaoou Tang},
11 | title = {Depth Map Super-Resolution by Deep Multi-Scale Guidance},
12 | booktitle = {Proceedings of European Conference on Computer Vision (ECCV)},
13 | pages = {353--369},
14 | year = {2016},
15 | url = {http://mmlab.ie.cuhk.edu.hk/projects/guidance_SR_depth.html}
16 | }
17 |
18 | The above copyright notice and this permission notice shall be included in all copies or
19 | substantial portions of the Software.
20 |
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x16.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/models/MSNet_x16.caffemodel
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x16_deploy.prototxt:
--------------------------------------------------------------------------------
1 | # MS-Net (published in ECCV 2016)
2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved.
3 | #
4 | # This software is licensed under the terms of the MS-Net licence
5 | # which allows for non-commercial use only unless a prior arrangement
6 | # has been made with the author (Tak-Wai Hui), the full terms of which
7 | # are made available in the LICENSE file.
8 |
9 | name: "MSNet_x16"
10 |
11 | input: "data-D"
12 | input_dim: 1
13 | input_dim: 1
14 | input_dim: 8
15 | input_dim: 8
16 |
17 | layer {
18 | name: "conv1-D"
19 | type: "Convolution"
20 | bottom: "data-D"
21 | top: "conv1-D"
22 | convolution_param {
23 | num_output: 64
24 | kernel_size: 5
25 | stride: 1
26 | pad: 2
27 | }
28 | }
29 |
30 | layer {
31 | name: "prelu1-D"
32 | type: "PReLU"
33 | bottom: "conv1-D"
34 | top: "conv1-D"
35 | }
36 |
37 | layer {
38 | name: "deconv2-D"
39 | type: "Deconvolution"
40 | bottom: "conv1-D"
41 | top: "deconv2-D"
42 | convolution_param {
43 | num_output: 32
44 | kernel_size: 5
45 | stride: 2
46 | pad: 2
47 | }
48 | }
49 |
50 | layer {
51 | name: "prelu2-D"
52 | type: "PReLU"
53 | bottom: "deconv2-D"
54 | top: "deconv2-D"
55 | }
56 |
57 | layer {
58 | name: "deconv3-D"
59 | type: "Deconvolution"
60 | bottom: "deconv2-D"
61 | top: "deconv3-D"
62 | convolution_param {
63 | num_output: 32
64 | kernel_size: 5
65 | stride: 2
66 | pad: 1
67 | }
68 | }
69 |
70 | layer {
71 | name: "prelu3-D"
72 | type: "PReLU"
73 | bottom: "deconv3-D"
74 | top: "deconv3-D"
75 | }
76 |
77 | layer {
78 | name: "deconv4-D"
79 | type: "Deconvolution"
80 | bottom: "deconv3-D"
81 | top: "deconv4-D"
82 | convolution_param {
83 | num_output: 32
84 | kernel_size: 5
85 | stride: 2
86 | pad: 1
87 | }
88 | }
89 |
90 | layer {
91 | name: "prelu4-D"
92 | type: "PReLU"
93 | bottom: "deconv4-D"
94 | top: "deconv4-D"
95 | }
96 |
97 | layer {
98 | name: "deconv5-D"
99 | type: "Deconvolution"
100 | bottom: "deconv4-D"
101 | top: "deconv5-D"
102 | convolution_param {
103 | num_output: 32
104 | kernel_size: 5
105 | stride: 2
106 | pad: 1
107 | }
108 | }
109 |
110 | layer {
111 | name: "prelu5-D"
112 | type: "PReLU"
113 | bottom: "deconv5-D"
114 | top: "deconv5-D"
115 | }
116 |
117 | layer {
118 | name: "conv6-D"
119 | type: "Convolution"
120 | bottom: "deconv5-D"
121 | top: "conv6-D"
122 | convolution_param {
123 | num_output: 1
124 | kernel_size: 5
125 | stride: 1
126 | pad: 2
127 | }
128 | }
129 |
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x2.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/models/MSNet_x2.caffemodel
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x2_deploy.prototxt:
--------------------------------------------------------------------------------
1 | # MS-Net (published in ECCV 2016)
2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved.
3 | #
4 | # This software is licensed under the terms of the MS-Net licence
5 | # which allows for non-commercial use only unless a prior arrangement
6 | # has been made with the author (Tak-Wai Hui), the full terms of which
7 | # are made available in the LICENSE file.
8 |
9 | name: "MSNet_x2"
10 |
11 | input: "data-D"
12 | input_dim: 1
13 | input_dim: 1
14 | input_dim: 20
15 | input_dim: 20
16 |
17 | layer {
18 | name: "conv1-D"
19 | type: "Convolution"
20 | bottom: "data-D"
21 | top: "conv1-D"
22 | convolution_param {
23 | num_output: 64
24 | kernel_size: 5
25 | stride: 1
26 | pad: 2
27 | }
28 | }
29 |
30 | layer {
31 | name: "prelu1-D"
32 | type: "PReLU"
33 | bottom: "conv1-D"
34 | top: "conv1-D"
35 | }
36 |
37 | layer {
38 | name: "deconv2-D"
39 | type: "Deconvolution"
40 | bottom: "conv1-D"
41 | top: "deconv2-D"
42 | convolution_param {
43 | num_output: 32
44 | kernel_size: 5
45 | stride: 2
46 | pad: 2
47 | }
48 | }
49 |
50 | layer {
51 | name: "prelu2-D"
52 | type: "PReLU"
53 | bottom: "deconv2-D"
54 | top: "deconv2-D"
55 | }
56 |
57 | layer {
58 | name: "conv3-D"
59 | type: "Convolution"
60 | bottom: "deconv2-D"
61 | top: "conv3-D"
62 | convolution_param {
63 | num_output: 1
64 | kernel_size: 5
65 | stride: 1
66 | pad: 2
67 | }
68 | }
69 |
70 |
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x4.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/models/MSNet_x4.caffemodel
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x4_deploy.prototxt:
--------------------------------------------------------------------------------
1 | # MS-Net (published in ECCV 2016)
2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved.
3 | #
4 | # This software is licensed under the terms of the MS-Net licence
5 | # which allows for non-commercial use only unless a prior arrangement
6 | # has been made with the author (Tak-Wai Hui), the full terms of which
7 | # are made available in the LICENSE file.
8 |
9 | name: "MSNet_x4"
10 |
11 | input: "data-D"
12 | input_dim: 1
13 | input_dim: 1
14 | input_dim: 270
15 | input_dim: 330
16 |
17 | layer {
18 | name: "conv1-D"
19 | type: "Convolution"
20 | bottom: "data-D"
21 | top: "conv1-D"
22 | convolution_param {
23 | num_output: 64
24 | kernel_size: 5
25 | stride: 1
26 | pad: 2
27 | }
28 | }
29 |
30 | layer {
31 | name: "prelu1-D"
32 | type: "PReLU"
33 | bottom: "conv1-D"
34 | top: "conv1-D"
35 | }
36 |
37 | layer {
38 | name: "deconv2-D"
39 | type: "Deconvolution"
40 | bottom: "conv1-D"
41 | top: "deconv2-D"
42 | convolution_param {
43 | num_output: 32
44 | kernel_size: 5
45 | stride: 2
46 | pad: 2
47 | }
48 | }
49 |
50 | layer {
51 | name: "prelu2-D"
52 | type: "PReLU"
53 | bottom: "deconv2-D"
54 | top: "deconv2-D"
55 | }
56 |
57 | layer {
58 | name: "deconv3-D"
59 | type: "Deconvolution"
60 | bottom: "deconv2-D"
61 | top: "deconv3-D"
62 | convolution_param {
63 | num_output: 32
64 | kernel_size: 5
65 | stride: 2
66 | pad: 1
67 | }
68 | }
69 |
70 | layer {
71 | name: "prelu3-D"
72 | type: "PReLU"
73 | bottom: "deconv3-D"
74 | top: "deconv3-D"
75 | }
76 |
77 | layer {
78 | name: "conv4-D"
79 | type: "Convolution"
80 | bottom: "deconv3-D"
81 | top: "conv4-D"
82 | convolution_param {
83 | num_output: 1
84 | kernel_size: 5
85 | stride: 1
86 | pad: 2
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x8.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/models/MSNet_x8.caffemodel
--------------------------------------------------------------------------------
/MSNet-release/models/MSNet_x8_deploy.prototxt:
--------------------------------------------------------------------------------
1 | # MS-Net (published in ECCV 2016)
2 | # Copyright (c) 2016 Tak-Wai Hui. All rights reserved.
3 | #
4 | # This software is licensed under the terms of the MS-Net licence
5 | # which allows for non-commercial use only unless a prior arrangement
6 | # has been made with the author (Tak-Wai Hui), the full terms of which
7 | # are made available in the LICENSE file.
8 |
9 | name: "MSNet_x8"
10 |
11 | input: "data-D"
12 | input_dim: 1
13 | input_dim: 1
14 | input_dim: 12
15 | input_dim: 12
16 |
17 | layer {
18 | name: "conv1-D"
19 | type: "Convolution"
20 | bottom: "data-D"
21 | top: "conv1-D"
22 | convolution_param {
23 | num_output: 64
24 | kernel_size: 5
25 | stride: 1
26 | pad: 2
27 | }
28 | }
29 |
30 | layer {
31 | name: "prelu1-D"
32 | type: "PReLU"
33 | bottom: "conv1-D"
34 | top: "conv1-D"
35 | }
36 |
37 | layer {
38 | name: "deconv2-D"
39 | type: "Deconvolution"
40 | bottom: "conv1-D"
41 | top: "deconv2-D"
42 | convolution_param {
43 | num_output: 32
44 | kernel_size: 5
45 | stride: 2
46 | pad: 2
47 | }
48 | }
49 |
50 | layer {
51 | name: "prelu2-D"
52 | type: "PReLU"
53 | bottom: "deconv2-D"
54 | top: "deconv2-D"
55 | }
56 |
57 | layer {
58 | name: "deconv3-D"
59 | type: "Deconvolution"
60 | bottom: "deconv2-D"
61 | top: "deconv3-D"
62 | convolution_param {
63 | num_output: 32
64 | kernel_size: 5
65 | stride: 2
66 | pad: 1
67 | }
68 | }
69 |
70 | layer {
71 | name: "prelu3-D"
72 | type: "PReLU"
73 | bottom: "deconv3-D"
74 | top: "deconv3-D"
75 | }
76 |
77 | layer {
78 | name: "deconv4-D"
79 | type: "Deconvolution"
80 | bottom: "deconv3-D"
81 | top: "deconv4-D"
82 | convolution_param {
83 | num_output: 32
84 | kernel_size: 5
85 | stride: 2
86 | pad: 1
87 | }
88 | }
89 |
90 | layer {
91 | name: "prelu4-D"
92 | type: "PReLU"
93 | bottom: "deconv4-D"
94 | top: "deconv4-D"
95 | }
96 |
97 | layer {
98 | name: "conv5-D"
99 | type: "Convolution"
100 | bottom: "deconv4-D"
101 | top: "conv5-D"
102 | convolution_param {
103 | num_output: 1
104 | kernel_size: 5
105 | stride: 1
106 | pad: 2
107 | }
108 | }
109 |
--------------------------------------------------------------------------------
/MSNet-release/testing sets/A/Depth/art.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/A/Depth/art.png
--------------------------------------------------------------------------------
/MSNet-release/testing sets/A/Depth/books.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/A/Depth/books.png
--------------------------------------------------------------------------------
/MSNet-release/testing sets/A/Depth/moebius.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/A/Depth/moebius.png
--------------------------------------------------------------------------------
/MSNet-release/testing sets/A/RGB/art.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/A/RGB/art.png
--------------------------------------------------------------------------------
/MSNet-release/testing sets/A/RGB/books.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/A/RGB/books.png
--------------------------------------------------------------------------------
/MSNet-release/testing sets/A/RGB/moebius.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/A/RGB/moebius.png
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/Depth/art.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/Depth/art.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/Depth/books.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/Depth/books.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/Depth/dolls.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/Depth/dolls.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/Depth/laundry.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/Depth/laundry.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/Depth/moebius.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/Depth/moebius.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/Depth/reindeer.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/Depth/reindeer.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/RGB/art.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/RGB/art.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/RGB/books.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/RGB/books.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/RGB/dolls.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/RGB/dolls.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/RGB/laundry.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/RGB/laundry.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/RGB/moebius.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/RGB/moebius.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/B/RGB/reindeer.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/B/RGB/reindeer.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/Depth/cones_2.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/Depth/cones_2.mat
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/Depth/teddy_2.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/Depth/teddy_2.mat
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/Depth/tsukuba_2.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/Depth/tsukuba_2.mat
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/Depth/venus_2.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/Depth/venus_2.mat
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/RGB/cones.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/RGB/cones.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/RGB/teddy.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/RGB/teddy.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/RGB/tsukuba.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/RGB/tsukuba.bmp
--------------------------------------------------------------------------------
/MSNet-release/testing sets/C/RGB/venus.bmp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/twhui/MS-Net/2dd270812a61c31675fb431d7a914ad2ef3fe2bb/MSNet-release/testing sets/C/RGB/venus.bmp
--------------------------------------------------------------------------------
/MSNet-release/util/MSNet.m:
--------------------------------------------------------------------------------
1 | % MS-Net (published in ECCV 2016)
2 | % Copyright (c) 2016 Tak-Wai Hui. All rights reserved.
3 | %
4 | % This software is licensed under the terms of the MS-Net licence
5 | % which allows for non-commercial use only unless a prior arrangement
6 | % has been made with the author (Tak-Wai Hui), the full terms of which
7 | % are made available in the LICENSE file.
8 |
9 | clear all;
10 | close all;
11 | clc;
12 | addpath('./matlab');
13 | root = './examples/MSNet-release/';
14 |
15 | %% CNN models
16 | cnn = 'models/MSNet_x2'; up_scale = 2;
17 | % cnn = 'models/MSNet_x4'; up_scale = 4;
18 | % cnn = 'models/MSNet_x8'; up_scale = 8;
19 | % cnn = 'models/MSNet_x16'; up_scale = 16;
20 |
21 | %% Testing image
22 | evaSet = 'A'; image_name = {'art', 'books', 'moebius'};
23 | image_id = 1;
24 |
25 | % evaSet = 'B'; image_name = {'art', 'books', 'dolls', 'laundry', 'moebius', 'reindeer'};
26 | % image_id = 3;
27 |
28 | % evaSet = 'C'; image_name = {'tsukuba', 'venus', 'teddy', 'cones'};
29 | % image_id = 4;
30 |
31 | %% Parameters settings
32 | gpu_id = 7;
33 | h = ones(3,3)/9;
34 | hole_defect = false;
35 | margin = 8*up_scale;
36 |
37 | %% Read ground truth depth map and RGB image
38 | if strcmp(evaSet, 'A')
39 | imGtD0 = imread([root 'testing sets/A/Depth/' image_name{image_id} '.png']);
40 |
41 | elseif strcmp(evaSet, 'B')
42 | imGtD0 = imread([root 'testing sets/B/Depth/' image_name{image_id} '.bmp']);
43 |
44 | if strcmp(image_name{image_id}, 'dolls')
45 | hole_defect = true;
46 | end
47 |
48 | elseif strcmp(evaSet, 'C')
49 | load([root 'testing sets/C/Depth/' image_name{image_id} '_2']);
50 | imGtD0 = img_double;
51 | end
52 |
53 | %% Main
54 | im_GtD0 = modcrop(imGtD0, up_scale);
55 | size_Dh = size(im_GtD0);
56 | size_Dl = size_Dh/up_scale;
57 |
58 | % D-channel
59 | im_GtD = double(im_GtD0);
60 | im_Dl = imresize(im_GtD, 1/up_scale, 'bicubic');
61 | [im_Dl, min_D, max_D] = normalize_cleanIm(im_Dl);
62 |
63 | im_Dl_LF = imfilter(im_Dl, h, 'symmetric');
64 | in_D = im_Dl - im_Dl_LF;
65 |
66 | % Run CNN
67 | caffe.reset_all();
68 | caffe.set_mode_gpu();
69 | caffe.set_device(gpu_id);
70 |
71 | net_model = [root cnn '_deploy.prototxt'];
72 | net_weights = [root cnn '.caffemodel'];
73 |
74 | net = caffe.Net(net_model, net_weights, 'test');
75 |
76 | blobs_input = {permute(in_D, [2 1 3])};
77 |
78 | net.blobs('data-D').reshape([size(blobs_input{1},1) size(blobs_input{1},2) 1 1]);
79 | net.reshape();
80 |
81 | fprintf('Run CNN ... \n');
82 | blobs_output = net.forward(blobs_input);
83 | fprintf('Finished. \n');
84 |
85 | im_Dh = permute(blobs_output{1}, [2 1 3]);
86 | im_D_LF = imresize(im_Dl_LF, up_scale, 'bicubic');
87 | im_Dh = im_Dh + im_D_LF(1:end-1, 1:end-1);
88 | im_Dh = im_Dh*(max_D - min_D) + min_D;
89 |
90 | %% Prepare ground truth
91 | im_GtD = double(im_GtD0);
92 |
93 | if hole_defect
94 | im_occ = im_GtD > 9;
95 | else
96 | im_occ = true(size(im_GtD));
97 | end
98 |
99 | im_GtD = im_GtD(1:end-1, 1:end-1);
100 | im_occ = im_occ(1:end-1, 1:end-1);
101 |
102 | %% Convert to uint8 or uint16
103 | if max(max(im_GtD)) > 2^8-1
104 | im_Dh = uint16(im_Dh);
105 | im_GtD = uint16(im_GtD);
106 | else
107 | im_Dh = uint8(im_Dh);
108 | im_GtD = uint8(im_GtD);
109 | end
110 |
111 | %% Compute RMSE
112 | rmse_cnn = compute_RMSE(im_GtD, im_Dh, im_occ);
113 |
114 | fprintf('------------------------------------------\n');
115 | fprintf('Model: %s\n', cnn);
116 | fprintf('Testing set %s: %s\n', evaSet, image_name{image_id});
117 | fprintf('MS-Net %dx upsampling, RMSE = %.3f\n', up_scale, rmse_cnn);
118 |
119 |
--------------------------------------------------------------------------------
/MSNet-release/util/compute_RMSE.m:
--------------------------------------------------------------------------------
1 | function rmse = compute_RMSE(im1, im2, occ)
2 | if size(im1, 3) == 3,
3 | im1 = rgb2ycbcr(im1);
4 | im1 = im1(:, :, 1);
5 | end
6 |
7 | if size(im2, 3) == 3,
8 | im2 = rgb2ycbcr(im2);
9 | im2 = im2(:, :, 1);
10 | end
11 |
12 | imdff = double(im1) - double(im2);
13 | imdff = imdff(occ == 1);
14 |
15 | rmse = sqrt(mean(imdff.^2));
16 |
--------------------------------------------------------------------------------
/MSNet-release/util/modcrop.m:
--------------------------------------------------------------------------------
1 | function imgs = modcrop(imgs, modulo)
2 | if size(imgs,3)==1
3 | sz = size(imgs);
4 | sz = sz - mod(sz, modulo);
5 | imgs = imgs(1:sz(1), 1:sz(2));
6 | else
7 | tmpsz = size(imgs);
8 | sz = tmpsz(1:2);
9 | sz = sz - mod(sz, modulo);
10 | imgs = imgs(1:sz(1), 1:sz(2),:);
11 | end
12 |
13 |
--------------------------------------------------------------------------------
/MSNet-release/util/normalize_cleanIm.m:
--------------------------------------------------------------------------------
1 | function [Z, min_Z, max_Z] = normalize_cleanIm(Z, min_Z, max_Z)
2 |
3 | if nargin == 1
4 | min_Z = min(min(Z));
5 | max_Z = max(max(Z));
6 | end
7 |
8 | Z = (Z - min_Z)/(max_Z - min_Z);
9 |
10 | end
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # MS-Net (Multi-scale network)
2 | This repository (https://github.com/twhui/MS-Net) is the offical release of MS-Net for our paper Depth Map Super-Resolution by Deep Multi-Scale Guidance in ECCV16. It comes with four trained networks (x2, x4, x8, and x16) one hole-filled RGBD training set, and three hole-filled RGBD testing sets (A, B, and C).
3 |
4 | To the best of our knowledge, MS-Net is the FIRST convolution neural network which attempts to upsample depth images.
5 |
6 | Another repository for MSG-Net (multi-scale guidance network) is also available.
7 |
8 | For more details, please visit my project page.
9 |
10 | # License and Citation
11 | This software and associated documentation files (the "Software"), and the research paper (Depth Map Super-Resolution by Deep Multi-Scale Guidance) including but not limited to the figures, and tables (the "Paper") are provided for academic research purposes only and without any warranty. Any commercial use requires my consent. When using any parts of the Software or the Paper in your work, please cite the following paper
12 |
13 | @InProceedings{hui16msgnet,
14 | author = {Tak-Wai Hui and Chen Change Loy and and Xiaoou Tang},
15 | title = {Depth Map Super-Resolution by Deep Multi-Scale Guidance},
16 | booktitle = {Proceedings of European Conference on Computer Vision (ECCV)},
17 | pages = {353--369},
18 | year = {2016},
19 | url = {http://mmlab.ie.cuhk.edu.hk/projects/guidance_SR_depth.html}
20 | }
21 |
22 | # Dependency
23 | We train our models using caffe and evaluate the results on Matlab.
24 |
25 | # Installation and Running
26 | You need to install caffe and remeber to complie matcaffe. You can put the folder MSNet-release in caffe/examples. Finally, you need to get into the the directory of examples/MSNet-release/util, and run MSNet.m.
27 |
28 | # Training data
29 | Please refer to MSG-Net.
30 |
31 | # Testing data
32 | Testig set is available at the folder MSNet-release/testing sets.
33 |
--------------------------------------------------------------------------------