├── ColorSimilarity_term_gen.m ├── DepthMapFusion.asv ├── DepthMapFusion.m ├── Edge_term_gen.m ├── GDP_term_gen.m ├── NLS_term_gen.m ├── README.md ├── Segmentation_term_gen.m └── data ├── art-color.png ├── art-depth.png ├── art-edge.mat ├── art-seg.mat ├── books-color.png ├── books-depth.png ├── books-edge.mat ├── books-seg.mat ├── moebius-color.png ├── moebius-depth.png ├── moebius-edge.mat └── moebius-seg.mat /ColorSimilarity_term_gen.m: -------------------------------------------------------------------------------- 1 | %% ColorSimilarity_term_gen.m 2 | % This function generates the Color similarity term 3 | % this work was originally proposed by J. Park on iccv2011 4 | % "High Quality Depth Map Upsampling for 3D-ToF Cameras" 5 | % 6 | % Author: rui wang & sgzthu 7 | % Last Modified: 2019.03.19 8 | % 9 | function W_pq = ColorSimilarity_term_gen(im,para) 10 | %% 11 | [M N ~] = size(im); 12 | Num = M*N; 13 | im = double(im); 14 | rgb2yuv_coef = [0.299 -0.14713 0.615;... 15 | 0.587 -0.28886 -0.51499;... 16 | 0.114 0.436 -0.10001]; 17 | yuv = zeros(size(im)); 18 | yuv(:,:,1) = rgb2yuv_coef(1,1).*im(:,:,1)+rgb2yuv_coef(2,1).*im(:,:,2)+rgb2yuv_coef(3,1).*im(:,:,3); 19 | yuv(:,:,2) = rgb2yuv_coef(1,2).*im(:,:,1)+rgb2yuv_coef(2,2).*im(:,:,2)+rgb2yuv_coef(3,2).*im(:,:,3); 20 | yuv(:,:,3) = rgb2yuv_coef(1,3).*im(:,:,1)+rgb2yuv_coef(2,3).*im(:,:,2)+rgb2yuv_coef(3,3).*im(:,:,3); 21 | 22 | %% calculate the matrix W_pq 23 | % first order neighbor 24 | neighbor = [-1 1 0 0 0;... 25 | 0 0 0 -1 1]; 26 | sigma = para.sigma_Color; 27 | index_Matrix = reshape([1:Num],M,N); 28 | W_pq = sparse(Num,Num); 29 | for xy = neighbor 30 | x = xy(1); y = xy(2); 31 | % calculate the w_pq for all [p, q=p+(x,y)] pairs 32 | left = max(1+x,1); 33 | right = min(M+x,M); 34 | up = max(1+y,1); 35 | down = min(N+y,N); 36 | CD = zeros(size(yuv)); 37 | CD(left-x:right-x,up-y:down-y,:) = yuv(left-x:right-x,up-y:down-y,:)-yuv(left:right,up:down,:); 38 | CD = sum(CD.^2,3); 39 | W_q = exp(-CD.^2./sigma^2); 40 | % calculate the indes of [p,q] pairs 41 | q_index = index_Matrix(left:right,up:down); 42 | p_index = index_Matrix(left-x:right-x,up-y:down-y); 43 | % assign the valid k_pq value to k_pq matrix 44 | W_q = W_q(left-x:right-x,up-y:down-y); 45 | W_pq_temp = sparse(p_index(:),q_index(:),W_q(:),Num,Num); 46 | W_pq = W_pq+W_pq_temp; 47 | end 48 | 49 | end 50 | -------------------------------------------------------------------------------- /DepthMapFusion.asv: -------------------------------------------------------------------------------- 1 | %% DepthMapFusion.m 2 | % This script reproduces the depth map upsampling method based on image 3 | % fusion techniques, this work was originally proposed by J. Park on iccv2011 4 | % "High Quality Depth Map Upsampling for 3D-ToF Cameras" 5 | % 6 | % Author: rui wang & sgzthu 7 | % Last Modified: 2019.03.18 8 | % 9 | 10 | %% 11 | clear all; 12 | close all; 13 | clc; 14 | 15 | %% 16 | %% read ground truth image 17 | im = imread('data\art-color.png'); 18 | dm = imread('data\art-depth.png'); 19 | 20 | %% set parameters 21 | up_scale = 16; 22 | para.NLS_window_size = 2; 23 | para.lambda_NLS = 1; 24 | para.NLS_th = 0.1; 25 | para.sigma_NLS = 1; 26 | para.sigma_Color = 1; 27 | para.seg_penalty = 0.7; 28 | para.lambda_confWeight = 0.01; 29 | para.sigma_gdp = 1; 30 | 31 | %% iccv 2011 32 | % complete the depth map with nearest neibor upsampling 33 | [M N ~] = size(im); 34 | dm_comp = imresize(dm, [M,N], 'nearest'); 35 | figure 36 | imshow(dm) 37 | title('original depth map'); 38 | figure 39 | imshow(dm_comp) 40 | title('original depth map with nearest neibor upsampling'); 41 | Num = M*N; % Number of pixels to be optimize 42 | outliers = double(dm_comp(:) == 0); 43 | G = sparse(1:Num,1:Num,1-outliers,Num,Num); % assign the diagnal value to 1 if not outliers 44 | 45 | %% NLS term 46 | tic 47 | K_pq = NLS_term_gen(im,para); 48 | L_NLS = diag(sum(K_pq,1))+diag(sum(K_pq,2))-2*K_pq; 49 | disp('Non_local struct term calculated!') 50 | clear K_pq 51 | toc 52 | 53 | %% confidence weighting 54 | % color similarity term 55 | tic 56 | Wc = ColorSimilarity_term_gen(im,para); 57 | disp('Color Similarity term calculated!') 58 | toc 59 | % segmentation term 60 | tic 61 | load('data\art-seg.mat'); 62 | Ws = Segmentation_term_gen(seg_level,para); 63 | disp('segmentation term calculated!') 64 | toc 65 | % edge silience term 66 | tic 67 | load('data\art-edge.mat'); 68 | We = Edge_term_gen(edge_map,para); 69 | disp('edge term calculated!') 70 | toc 71 | % guided depth map term 72 | tic 73 | guided_depth_map = imresize(dm, [M,N], 'bicubic'); 74 | Wg = GDP_term_gen(guided_depth_map,para); 75 | disp('guided depth map term calculated!') 76 | toc 77 | 78 | % combine all weights 79 | W_pq = Wc;%.*Ws.*Wg;%.*Wg; 80 | clear Wc Ws We Wg 81 | % normalize each row (sum of coefficients for each p) 82 | norm = 1./sum(W_pq.'); 83 | [indx,indy,value] = find(W_pq); % ÕÒ³öËùÓзÇÁãÖµ×ø±ê 84 | norm = sparse(indx,indy,norm(indx),Num,Num); 85 | W_pq = W_pq .* norm; 86 | 87 | L_s = diag(sum(W_pq,1))+diag(sum(W_pq,2))-2*W_pq; 88 | clear W_pq 89 | 90 | %% solving 91 | para.lambda_NLS = 0; 92 | para.lambda_confWeight = 0; 93 | b = (1-outliers).*double(dm_comp(:)); 94 | A = para.lambda_NLS*L_NLS+para.lambda_confWeight*L_s+G; 95 | A(1,1) 96 | nnz(A) 97 | dm_sup_res = A\b; 98 | toc 99 | dm_sup_res = uint8(reshape(dm_sup_res,M,N)); 100 | figure 101 | imshow(dm_sup_res) 102 | -------------------------------------------------------------------------------- /DepthMapFusion.m: -------------------------------------------------------------------------------- 1 | %% DepthMapFusion.m 2 | % This script reproduces the depth map upsampling method based on image 3 | % fusion techniques, this work was originally proposed by J. Park on iccv2011 4 | % "High Quality Depth Map Upsampling for 3D-ToF Cameras" 5 | % 6 | % Author: rui wang & sgzthu 7 | % Last Modified: 2019.03.18 8 | % 9 | 10 | %% 11 | clear all; 12 | close all; 13 | clc; 14 | 15 | %% 16 | %% read ground truth image 17 | im = imread('data\art-color.png'); 18 | dm = imread('data\art-depth.png'); 19 | 20 | %% set parameters 21 | up_scale = 16; 22 | para.NLS_window_size = 2; 23 | para.lambda_NLS = 1; 24 | para.NLS_th = 0.1; 25 | para.sigma_NLS = 1; 26 | para.sigma_Color = 1; 27 | para.seg_penalty = 0.7; 28 | para.lambda_confWeight = 0.01; 29 | para.sigma_gdp = 1; 30 | 31 | %% iccv 2011 32 | % complete the depth map with nearest neibor upsampling 33 | [M N ~] = size(im); 34 | dm_comp = imresize(dm, [M,N], 'bicubic'); 35 | figure 36 | imshow(dm) 37 | title('original depth map'); 38 | figure 39 | imshow(dm_comp) 40 | title('original depth map with nearest neibor upsampling'); 41 | Num = M*N; % Number of pixels to be optimize 42 | outliers = double(dm_comp(:) == 0); 43 | G = sparse(1:Num,1:Num,1-outliers,Num,Num); % assign the diagnal value to 1 if not outliers 44 | 45 | %% NLS term 46 | tic 47 | K_pq = NLS_term_gen(im,para); 48 | L_NLS = diag(sum(K_pq,1))+diag(sum(K_pq,2))-2*K_pq; 49 | disp('Non_local struct term calculated!') 50 | clear K_pq 51 | toc 52 | 53 | %% confidence weighting 54 | % color similarity term 55 | tic 56 | Wc = ColorSimilarity_term_gen(im,para); 57 | disp('Color Similarity term calculated!') 58 | toc 59 | % segmentation term 60 | tic 61 | load('data\art-seg.mat'); 62 | Ws = Segmentation_term_gen(seg_level,para); 63 | disp('segmentation term calculated!') 64 | toc 65 | % edge silience term 66 | tic 67 | load('data\art-edge.mat'); 68 | We = Edge_term_gen(edge_map,para); 69 | disp('edge term calculated!') 70 | toc 71 | % guided depth map term 72 | tic 73 | guided_depth_map = imresize(dm, [M,N], 'bicubic'); 74 | Wg = GDP_term_gen(guided_depth_map,para); 75 | disp('guided depth map term calculated!') 76 | toc 77 | 78 | % combine all weights 79 | W_pq = Wc.*Ws.*We.*Wg; 80 | clear Wc Ws We Wg 81 | % normalize each row (sum of coefficients for each p) 82 | norm = 1./sum(W_pq.'); 83 | [indx,indy,value] = find(W_pq); % ÕÒ³öËùÓзÇÁãÖµ×ø±ê 84 | norm = sparse(indx,indy,norm(indx),Num,Num); 85 | W_pq = W_pq .* norm; 86 | 87 | L_s = diag(sum(W_pq,1))+diag(sum(W_pq,2))-2*W_pq; 88 | clear W_pq 89 | 90 | %% solving 91 | b = (1-outliers).*double(dm_comp(:)); 92 | A = para.lambda_NLS*L_NLS+para.lambda_confWeight*L_s+G+0e-3*speye(Num); 93 | A(1,1) 94 | nnz(A) 95 | dm_sup_res = A\b; 96 | toc 97 | dm_sup_res = uint8(reshape(dm_sup_res,M,N)); 98 | figure 99 | imshow(dm_sup_res) 100 | -------------------------------------------------------------------------------- /Edge_term_gen.m: -------------------------------------------------------------------------------- 1 | %% Edge_term_gen.m 2 | % This function generates the edge term 3 | % this work was originally proposed by J. Park on iccv2011 4 | % "High Quality Depth Map Upsampling for 3D-ToF Cameras" 5 | % 6 | % Author: rui wang & sgzthu 7 | % Last Modified: 2019.03.19 8 | % 9 | function W_pq = Edge_term_gen(edge,para) 10 | %% 11 | [M N ~] = size(edge); 12 | Num = M*N; 13 | 14 | %% calculate the matrix W_pq 15 | % first order neighbor 16 | index_Matrix = reshape([1:Num],M,N); 17 | W_pq = sparse(Num,Num); 18 | for x = [-1,0,1] 19 | y=0; 20 | % calculate the w_pq for all [p, q=p+(x,y)] pairs 21 | left = max(1+x,1); 22 | right = min(M+x,M); 23 | up = max(1+y,1); 24 | down = min(N+y,N); 25 | W_q = zeros(M,N); 26 | W_q(left-x:right-x,up-y:down-y) = 1./(sqrt(edge(left-x:right-x,up-y:down-y).^2+edge(left:right,up:down).^2)+1); 27 | % calculate the indes of [p,q] pairs 28 | q_index = index_Matrix(left:right,up:down); 29 | p_index = index_Matrix(left-x:right-x,up-y:down-y); 30 | % assign the valid k_pq value to k_pq matrix 31 | W_q = W_q(left-x:right-x,up-y:down-y); 32 | W_pq_temp = sparse(p_index(:),q_index(:),W_q(:),Num,Num); 33 | W_pq = W_pq+W_pq_temp; 34 | end 35 | 36 | end 37 | -------------------------------------------------------------------------------- /GDP_term_gen.m: -------------------------------------------------------------------------------- 1 | %% GDP_term_gen.m 2 | % This function generates the guided depth map term 3 | % this work was originally proposed by J. Park on iccv2011 4 | % "High Quality Depth Map Upsampling for 3D-ToF Cameras" 5 | % 6 | % Author: rui wang & sgzthu 7 | % Last Modified: 2019.03.19 8 | % 9 | function W_pq = GDP_term_gen(gdp,para) 10 | %% 11 | [M N ~] = size(gdp); 12 | Num = M*N; 13 | gdp = double(gdp); 14 | %% calculate the matrix W_pq 15 | % first order neighbor 16 | neighbor = [-1 1 0 0 0;... 17 | 0 0 0 -1 1]; 18 | index_Matrix = reshape([1:Num],M,N); 19 | W_pq = sparse(Num,Num); 20 | sigma = para.sigma_gdp; 21 | for xy = neighbor 22 | x = xy(1); y = xy(2); 23 | % calculate the w_pq for all [p, q=p+(x,y)] pairs 24 | left = max(1+x,1); 25 | right = min(M+x,M); 26 | up = max(1+y,1); 27 | down = min(N+y,N); 28 | W_q = zeros(M,N); 29 | W_q(left-x:right-x,up-y:down-y) = exp(-(gdp(left-x:right-x,up-y:down-y)-gdp(left:right,up:down))./2./sigma^2); 30 | % calculate the indes of [p,q] pairs 31 | q_index = index_Matrix(left:right,up:down); 32 | p_index = index_Matrix(left-x:right-x,up-y:down-y); 33 | % assign the valid k_pq value to k_pq matrix 34 | W_q = W_q(left-x:right-x,up-y:down-y); 35 | W_pq_temp = sparse(p_index(:),q_index(:),W_q(:),Num,Num); 36 | W_pq = W_pq+W_pq_temp; 37 | end 38 | 39 | end 40 | -------------------------------------------------------------------------------- /NLS_term_gen.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/NLS_term_gen.m -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Depth_Map_Upsampling_TOF 2 | This is a matlab implementation of J.Park's work "High Quality Depth Map Upsampling for 3D-TOF Cameras" . 3 | We test our implementation with a TOF-Simulated.mat file, and our results is close to the original authors. 4 | Here, I want to show my deeply thanks to my friend Gong-Zhe Su(graduate from Tsinghua University). 5 | -------------------------------------------------------------------------------- /Segmentation_term_gen.m: -------------------------------------------------------------------------------- 1 | %% Segmentation_term_gen.m 2 | % This function generates the segmentation term 3 | % this work was originally proposed by J. Park on iccv2011 4 | % "High Quality Depth Map Upsampling for 3D-ToF Cameras" 5 | % 6 | % Author: sgzthu @ Deptrum 7 | % Last Modified: 2019.03.19 8 | % 9 | function W_pq = Segmentation_term_gen(seg,para) 10 | %% 11 | [M N ~] = size(seg); 12 | Num = M*N; 13 | 14 | %% calculate the matrix W_pq 15 | % first order neighbor 16 | neighbor = [-1 1 0 0 0;... 17 | 0 0 0 -1 1]; 18 | index_Matrix = reshape([1:Num],M,N); 19 | W_pq = sparse(Num,Num); 20 | for xy = neighbor 21 | x = xy(1); y = xy(2); 22 | % calculate the w_pq for all [p, q=p+(x,y)] pairs 23 | left = max(1+x,1); 24 | right = min(M+x,M); 25 | up = max(1+y,1); 26 | down = min(N+y,N); 27 | W_q = zeros(M,N); 28 | W_q(left-x:right-x,up-y:down-y) = double(seg(left-x:right-x,up-y:down-y)==seg(left:right,up:down)); 29 | W_q(left-x:right-x,up-y:down-y) = W_q(left-x:right-x,up-y:down-y)*(1-para.seg_penalty)+para.seg_penalty; 30 | % calculate the indes of [p,q] pairs 31 | q_index = index_Matrix(left:right,up:down); 32 | p_index = index_Matrix(left-x:right-x,up-y:down-y); 33 | % assign the valid k_pq value to k_pq matrix 34 | W_q = W_q(left-x:right-x,up-y:down-y); 35 | W_pq_temp = sparse(p_index(:),q_index(:),W_q(:),Num,Num); 36 | W_pq = W_pq+W_pq_temp; 37 | end 38 | 39 | end -------------------------------------------------------------------------------- /data/art-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/art-color.png -------------------------------------------------------------------------------- /data/art-depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/art-depth.png -------------------------------------------------------------------------------- /data/art-edge.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/art-edge.mat -------------------------------------------------------------------------------- /data/art-seg.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/art-seg.mat -------------------------------------------------------------------------------- /data/books-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/books-color.png -------------------------------------------------------------------------------- /data/books-depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/books-depth.png -------------------------------------------------------------------------------- /data/books-edge.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/books-edge.mat -------------------------------------------------------------------------------- /data/books-seg.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/books-seg.mat -------------------------------------------------------------------------------- /data/moebius-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/moebius-color.png -------------------------------------------------------------------------------- /data/moebius-depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/moebius-depth.png -------------------------------------------------------------------------------- /data/moebius-edge.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/moebius-edge.mat -------------------------------------------------------------------------------- /data/moebius-seg.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GitWR/Depth_Map_Upsampling_TOF/de6ba81d8fb8d7de8b3da584093b70cb6b0e8b6f/data/moebius-seg.mat --------------------------------------------------------------------------------