├── Training4.png ├── Training4Trimap.png ├── common ├── affinityMatrixToLaplacian.m ├── localLinearEmbedding.m ├── solveForAlphas.m ├── localRGBnormalDistributions.m ├── detectHighlyTransparent.m ├── findNonlocalNeighbors.m └── getMattingParams.m ├── abmtSetup.m ├── demo.m ├── LICENSE ├── closedFormMatting.m ├── KNNMatting.m ├── sharedMattingMatteRefinement.m ├── affinity ├── colorMixtureAffinities.m ├── knownToUnknownColorMixture.m ├── colorSimilarityAffinities.m └── mattingAffinity.m ├── informationFlowMatteRefinement.m ├── trimming ├── patchBasedTrimming.m └── trimmingFromKnownUnknownEdges.m ├── readme.md └── informationFlowMatting.m /Training4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaksoy/AffinityBasedMattingToolbox/HEAD/Training4.png -------------------------------------------------------------------------------- /Training4Trimap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaksoy/AffinityBasedMattingToolbox/HEAD/Training4Trimap.png -------------------------------------------------------------------------------- /common/affinityMatrixToLaplacian.m: -------------------------------------------------------------------------------- 1 | 2 | function Lap = affinityMatrixToLaplacian(aff) 3 | N = size(aff, 1); 4 | Lap = spdiags(sum(aff, 2), 0 , N, N) - aff; 5 | end -------------------------------------------------------------------------------- /abmtSetup.m: -------------------------------------------------------------------------------- 1 | % Yagiz Aksoy 2 | % 2017 3 | 4 | tbloc = fileparts(mfilename('fullpath')); 5 | addpath(fullfile(tbloc, 'affinity')); 6 | addpath(fullfile(tbloc, 'trimming')); 7 | addpath(fullfile(tbloc, 'common')); 8 | clear tbloc; -------------------------------------------------------------------------------- /common/localLinearEmbedding.m: -------------------------------------------------------------------------------- 1 | 2 | % Local Linear Embedding 3 | % This function implements the weight computation defined in 4 | % Sam T. Roweis, Lawrence K. Saul, "Nonlinear Dimensionality 5 | % Reduction by Local Linear Embedding", Science, 2000. 6 | % 'w' is the weights for representing the row-vector 'pt' in terms 7 | % of the dimensions x neighborCount matrix 'neighbors'. 8 | % 'conditionerMult' is the multiplier of the identity matrix added 9 | % to the neighborhood correlation matrix before inversion. 10 | 11 | function w = localLinearEmbedding(pt, neighbors, conditionerMult) 12 | % each column of neighbors represent a neighbor, each row a dimension 13 | % pt is a row vector 14 | corr = neighbors' * neighbors + conditionerMult * eye(size(neighbors, 2)); 15 | ptDotN = neighbors' * pt; 16 | alpha = 1 - sum(corr \ ptDotN); 17 | beta = sum(corr \ ones(size(corr, 1), 1)); % sum of elements of inv(corr) 18 | lagrangeMult = alpha / beta; 19 | w = corr \ (ptDotN + lagrangeMult); 20 | end -------------------------------------------------------------------------------- /demo.m: -------------------------------------------------------------------------------- 1 | 2 | image = imread('Training4.png'); 3 | trimap = imread('Training4Trimap.png'); 4 | 5 | % Closed-form matting 6 | a_cf = closedFormMatting(image, trimap); 7 | % Some alternatives: 8 | % a_ifm = informationFlowMatting(image, trimap); 9 | % a_knn = KNNMatting(image, trimap); 10 | 11 | % Get the parameter struct and edit for customization if desired 12 | params = getMattingParams('IFM'); 13 | params.useKnownToUnknown = 0; 14 | % params.iu_xyw = 0.1; 15 | % params.loc_mult = 3; 16 | a_ifm = informationFlowMatting(image, trimap, params); 17 | 18 | % Trim the trimap 19 | trimmed = patchBasedTrimming(image, trimap); 20 | % An alternative: 21 | % trimmed = trimmingFromUnknownToKnownEdges(image, trimap); 22 | 23 | % Run K-to-U information flow to get a rough alpha and confidences 24 | [alphaHat, conf] = knownToUnknownColorMixture(image, trimmed); 25 | 26 | % Refine alphaHat shared matting 27 | a_sm_ref = sharedMattingMatteRefinement(image, trimmed, alphaHat, conf); 28 | % Alternative: 29 | % a_ifm_ref = informationFlowMatteRefinement(image, trimmed, alphaHat, conf); -------------------------------------------------------------------------------- /common/solveForAlphas.m: -------------------------------------------------------------------------------- 1 | 2 | % Constructs and solves the linear system 3 | 4 | function alphas = solveForAlphas(Lap, trimap, lambda, usePCG, alphaHat, conf, aHatMult) 5 | if ~exist('usePCG', 'var') || isempty(usePCG) 6 | usePCG = true; 7 | end 8 | [h, w, ~] = size(trimap); 9 | N = h * w; 10 | known = trimap > 0.8 | trimap < 0.2; 11 | A = lambda * spdiags(double(known(:)), 0, N, N); 12 | if exist('alphaHat', 'var') 13 | if ~exist('conf', 'var') || isempty(conf) 14 | conf = ones(size(alphaHat)); 15 | end 16 | if ~exist('aHatMult', 'var') || isempty(aHatMult) 17 | aHatMult = 0.1; 18 | end 19 | conf(known(:)) = 0; 20 | A = A + aHatMult * spdiags(conf(:), 0, N, N); 21 | b = A * alphaHat(:); 22 | else 23 | b = A * double(trimap(:) > 0.8); 24 | end 25 | A = A + Lap; 26 | if usePCG 27 | [alphas, ~] = pcg(A, b, [], 2000); 28 | else 29 | alphas = A \ b; 30 | end 31 | alphas(alphas < 0) = 0; 32 | alphas(alphas > 1) = 1; 33 | end -------------------------------------------------------------------------------- /common/localRGBnormalDistributions.m: -------------------------------------------------------------------------------- 1 | 2 | % RGB normal distributions fit to colors around each pixel 3 | 4 | function [meanImage, covarMat] = localRGBnormalDistributions(image, windowRadius, epsilon) 5 | 6 | if ~exist('windowRadius', 'var') || isempty(windowRadius) 7 | windowRadius = 1; 8 | end 9 | if ~exist('epsilon', 'var') || isempty(epsilon) 10 | epsilon = 1e-8; 11 | end 12 | 13 | [h, w, ~] = size(image); 14 | N = h * w; 15 | windowSize = 2 * windowRadius + 1; 16 | 17 | meanImage = imboxfilt(image, windowSize); 18 | covarMat = zeros(3, 3, N); 19 | 20 | for r = 1 : 3 21 | for c = r : 3 22 | temp = imboxfilt(image(:, :, r).*image(:, :, c), windowSize) - meanImage(:,:,r) .* meanImage(:,:,c); 23 | covarMat(r, c, :) = temp(:); 24 | end 25 | end 26 | 27 | for i = 1 : 3 28 | covarMat(i, i, :) = covarMat(i, i, :) + epsilon; 29 | end 30 | 31 | for r = 2 : 3 32 | for c = 1 : r - 1 33 | covarMat(r, c, :) = covarMat(c, r, :); 34 | end 35 | end 36 | 37 | end -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017, Yagiz Aksoy. All rights reserved. 2 | 3 | This software is for academic use only. A redistribution of this 4 | software, with or without modifications, has to be for academic 5 | use only, while giving the appropriate credit to the original 6 | authors of the software. The methods implemented as a part of 7 | this software may be covered under patents or patent applications. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED 10 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 11 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR 12 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 13 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 14 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 15 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 16 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 17 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /common/detectHighlyTransparent.m: -------------------------------------------------------------------------------- 1 | 2 | % Detect if the expected matte is a highly-transparent one 3 | % This function implements the energy selection method described in 4 | % Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective 5 | % Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017. 6 | % This is a very simple histogram-based classifier. 7 | 8 | function ht = detectHighlyTransparent(image, trimap) 9 | 10 | image = reshape(im2double(image), [size(image, 1) * size(image, 2), 3]); 11 | trimap = im2double(trimap(:,:,1)); 12 | 13 | fg = trimap > 0.8; 14 | bg = trimap < 0.2; 15 | unk = ~(fg | bg); 16 | fg = fg & imdilate(unk, ones(20)); 17 | bg = bg & imdilate(unk, ones(20)); 18 | 19 | fgi = image(fg, :); 20 | bgi = image(bg, :); 21 | uni = image(unk, :); 22 | 23 | fgh = [imhist(fgi(:, 1), 10); imhist(fgi(:, 3), 10); imhist(fgi(:, 3), 10);] / sum(fg(:)); 24 | bgh = [imhist(bgi(:, 1), 10); imhist(bgi(:, 3), 10); imhist(bgi(:, 3), 10);] / sum(bg(:)); 25 | unh = [imhist(uni(:, 1), 10); imhist(uni(:, 3), 10); imhist(uni(:, 3), 10);] / sum(unk(:)); 26 | 27 | weights = ([fgh bgh]' * [fgh bgh]) \ ([fgh bgh]' * unh); 28 | recError = [fgh bgh] * weights - unh; 29 | recError = sqrt(sum(recError(:) .* recError(:))) / size(recError(:), 1); 30 | 31 | ht = recError > 0.0099; 32 | 33 | end -------------------------------------------------------------------------------- /closedFormMatting.m: -------------------------------------------------------------------------------- 1 | 2 | % Closed-Form Matting 3 | % This function implements the image matting approach described in 4 | % Anat Levin, Dani Lischinski, Yair Weiss, "A Closed Form Solution to 5 | % Natural Image Matting", IEEE TPAMI, 2008. 6 | % Optional input parameter 'params' can be customized by editing the 7 | % default values in the struct returned by 'getMattingParams('CF'). 8 | % - loc_*** define the parameters for the matting Laplacian. 9 | 10 | function alpha = closedFormMatting(image, trimap, params, suppressMessages) 11 | abmtSetup 12 | tic; 13 | if ~exist('params', 'var') || isempty(params) 14 | params = getMattingParams('CF'); 15 | end 16 | if ~exist('suppressMessages', 'var') || isempty(suppressMessages) 17 | suppressMessages = false; 18 | end 19 | if(~suppressMessages) display('Closed-Form Matting started...'); end 20 | 21 | image = im2double(image); 22 | trimap = im2double(trimap(:,:,1)); 23 | 24 | % Compute matting Laplacian 25 | unk = trimap < 0.8 & trimap > 0.2; 26 | dilUnk = imdilate(unk, ones(2 * params.loc_win + 1)); 27 | if(~suppressMessages) display(' Computing matting Laplacian...'); end 28 | Lap = affinityMatrixToLaplacian(mattingAffinity(image, dilUnk, params.loc_win, params.loc_eps)); 29 | 30 | if(~suppressMessages) display(' Solving for alphas...'); end 31 | alpha = solveForAlphas(Lap, trimap, params.lambda, params.usePCGtoSolve); 32 | 33 | alpha = reshape(alpha, [size(image, 1), size(image, 2)]); 34 | 35 | dur = toc; 36 | if(~suppressMessages) display(['Done. It took ' num2str(dur) ' seconds.']); end 37 | end 38 | -------------------------------------------------------------------------------- /KNNMatting.m: -------------------------------------------------------------------------------- 1 | 2 | % KNN Matting 3 | % This function implements the image matting approach described in 4 | % Qifeng Chen, Dingzeyu Li, Chi-Keung Tang, "KNN Matting", IEEE 5 | % TPAMI, 2013. 6 | % Optional input parameter 'params' can be customized by editing the 7 | % values in the struct returned by 'getMattingParams('CF'). 8 | % - knn_K defines the number of nonlocal neighbors 9 | % - knn_xyw defines the weight of the spatial coordinates in KNN search 10 | % - knn_hsv defines the color space (RGB or HSV) for KNN search 11 | 12 | function alpha = KNNMatting(image, trimap, params, suppressMessages) 13 | abmtSetup 14 | tic; 15 | if ~exist('params', 'var') || isempty(params) 16 | params = getMattingParams('KNN'); 17 | end 18 | if ~exist('suppressMessages', 'var') || isempty(suppressMessages) 19 | suppressMessages = false; 20 | end 21 | if(~suppressMessages) display('KNN Matting started...'); end 22 | 23 | image = im2double(image); 24 | trimap = im2double(trimap(:,:,1)); 25 | 26 | % Compute KNN affinities 27 | unk = trimap < 0.8 & trimap > 0.2; 28 | dilUnk = imdilate(unk, ones(3, 3)); 29 | if(~suppressMessages) display(' Computing KNN affinities...'); end 30 | Lap = affinityMatrixToLaplacian(colorSimilarityAffinities(image, params.knn_K, [], [], params.knn_xyw, params.knn_hsv)); 31 | 32 | if(~suppressMessages) display(' Solving for alphas...'); end 33 | alpha = solveForAlphas(Lap, trimap, params.lambda, params.usePCGtoSolve); 34 | 35 | alpha = reshape(alpha, [size(image, 1), size(image, 2)]); 36 | 37 | dur = toc; 38 | if(~suppressMessages) display(['Done. It took ' num2str(dur) ' seconds.']); end 39 | end 40 | -------------------------------------------------------------------------------- /sharedMattingMatteRefinement.m: -------------------------------------------------------------------------------- 1 | 2 | % Shared Matting Matte Refinement 3 | % This function implements the matte refinement approach described in 4 | % Eduardo S. L. Gastal, Manuel M. Oliveira, "Shared Sampling for 5 | % Real-Time Alpha Matting", Computer Graphics Forum, 2010. 6 | % 'alphaHat' and 'confidences' parameters are typically obtained by a 7 | % sampling-based natural matting algorithm. 'confidences' is filled by 8 | % ones if not provided. Optional input parameter 'params' can be 9 | % customized by editing the default values in the struct returned 10 | % by 'getMattingParams('SharedMatting'). 11 | % - loc_*** define the parameters for the matting Laplacian. 12 | % - refinement_mult determines how much trust is given to the initial 13 | % alpha estimation 14 | 15 | function alpha = sharedMattingMatteRefinement(image, trimap, alphaHat, confidences, params, suppressMessages) 16 | abmtSetup 17 | tic; 18 | if ~exist('confidences', 'var') || isempty(confidences) 19 | confidences = ones(size(alphaHat(:,:,1))); 20 | end 21 | if ~exist('params', 'var') || isempty(params) 22 | params = getMattingParams('SharedMatting'); 23 | end 24 | if ~exist('suppressMessages', 'var') || isempty(suppressMessages) 25 | suppressMessages = false; 26 | end 27 | if(~suppressMessages) display('Matte refinement via Shared Matting...'); end 28 | 29 | image = im2double(image); 30 | trimap = im2double(trimap(:,:,1)); 31 | alphaHat = im2double(alphaHat(:,:,1)); 32 | 33 | % Compute matting Laplacian 34 | unk = trimap < 0.8 & trimap > 0.2; 35 | dilUnk = imdilate(unk, ones(3, 3)); 36 | if(~suppressMessages) display(' Computing matting Laplacian...'); end 37 | Lap = affinityMatrixToLaplacian(mattingAffinity(image, dilUnk, params.loc_win, params.loc_eps)); 38 | 39 | if(~suppressMessages) display(' Solving for alphas...'); end 40 | alpha = solveForAlphas(Lap, trimap, params.lambda, params.usePCGtoSolve, alphaHat, confidences, params.refinement_mult); 41 | 42 | alpha = reshape(alpha, [size(image, 1), size(image, 2)]); 43 | 44 | dur = toc; 45 | if(~suppressMessages) display(['Done. It took ' num2str(dur) ' seconds.']); end 46 | end 47 | -------------------------------------------------------------------------------- /affinity/colorMixtureAffinities.m: -------------------------------------------------------------------------------- 1 | 2 | % Color Mixture Non-local Pixel Affinities 3 | % This function implements the color-mixture information flow in 4 | % Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective 5 | % Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017 6 | % when the input parameter 'useXYinLLEcomp' is false (default), and 7 | % the affinity definition used in 8 | % Xiaowu Chen, Dongqing Zou, Qinping Zhao, Ping Tan, "Manifold 9 | % preserving edit propagation", ACM TOG, 2012 10 | % when 'useXYinLLEcomp' is true. 11 | % All parameters other than image are optional. The output is a sparse 12 | % matrix which has non-zero element for the non-local neighbors of 13 | % the pixels given by binary map inMap. 14 | % - K defines the number of neighbors from which LLE weights are 15 | % computed. 16 | % - outMap is a binary map that defines where the nearest neighbor 17 | % search is done. 18 | % - xyWeight determines how much importance is given to the spatial 19 | % coordinates in the nearest neighbor selection. 20 | 21 | function Wcm = colorMixtureAffinities(image, K, inMap, outMap, xyWeight, useXYinLLEcomp) 22 | 23 | [h, w, ~] = size(image); 24 | N = h * w; 25 | 26 | if ~exist('K', 'var') || isempty(K) 27 | K = 20; 28 | end 29 | if ~exist('inMap', 'var') || isempty(inMap) 30 | inMap = true(h, w); 31 | end 32 | if ~exist('outMap', 'var') || isempty(outMap) 33 | outMap = true(h, w); 34 | end 35 | if ~exist('xyWeight', 'var') || isempty(xyWeight) 36 | xyWeight = 1; 37 | end 38 | if ~exist('useXYinLLEcomp', 'var') || isempty(useXYinLLEcomp) 39 | useXYinLLEcomp = false; 40 | end 41 | 42 | [inInd, neighInd, features] = findNonlocalNeighbors(image, K, xyWeight, inMap, outMap); 43 | 44 | if ~useXYinLLEcomp 45 | features = features(:, 1 : end - 2); 46 | end 47 | flows = zeros(size(inInd, 1), size(neighInd, 2)); 48 | 49 | for i = 1 : size(inInd, 1) 50 | flows(i, :) = localLinearEmbedding(features(inInd(i), :)', features(neighInd(i, :), :)', 1e-10); 51 | end 52 | flows = flows ./ repmat(sum(flows, 2), [1, K]); 53 | 54 | inInd = repmat(inInd, [1, K]); 55 | Wcm = sparse(inInd(:), neighInd(:), flows, N, N); 56 | end -------------------------------------------------------------------------------- /common/findNonlocalNeighbors.m: -------------------------------------------------------------------------------- 1 | 2 | % Find neighbors using the pixel colors and spatial ccordinates 3 | % - K is the number of neighbors to be found 4 | % - Parameters other than K and image are optional. 5 | % - xyWeight sets the relative importance of spatial coordinates 6 | % - inMap and outMap are binary maps determining the query and 7 | % search regions 8 | % - Self matches are detected and removed if eraseSelfMatches is true 9 | % - inInd and neighInd give pixel indices of query pixels and their neighbors. 10 | % - features is noOfPixels X dimensions matrix used in neighbor search. 11 | 12 | function [inInd, neighInd, features] = findNonlocalNeighbors(image, K, xyWeight, inMap, outMap, eraseSelfMatches) 13 | 14 | [h, w, c] = size(image); 15 | 16 | if ~exist('xyWeight', 'var') || isempty(xyWeight) 17 | xyWeight = 1; 18 | end 19 | if ~exist('inMap', 'var') || isempty(inMap) 20 | inMap = true(h, w); 21 | end 22 | if ~exist('outMap', 'var') || isempty(outMap) 23 | outMap = true(h, w); 24 | end 25 | if ~exist('eraseSelfMatches', 'var') || isempty(eraseSelfMatches) 26 | eraseSelfMatches = true; 27 | end 28 | 29 | features = reshape(image, [h*w, c]); 30 | if xyWeight > 0 31 | [x, y] = meshgrid(1 : w, 1 : h); 32 | x = xyWeight * double(x) / w; 33 | y = xyWeight * double(y) / h; 34 | features = [features x(:) y(:)]; 35 | end 36 | 37 | inMap = inMap(:); 38 | outMap = outMap(:); 39 | indices = (1 : h * w)'; 40 | inInd = indices(inMap); 41 | outInd = indices(outMap); 42 | 43 | if eraseSelfMatches 44 | % Find K + 1 matches to count for self-matches 45 | neighbors = knnsearch(features(outMap, :), features(inMap, :), 'K', K + 1); 46 | % Get rid of self-matches 47 | validNeighMap = true(size(neighbors)); 48 | validNeighMap(inMap(inInd) & outMap(inInd), 1) = 0; 49 | validNeighMap(:, end) = ~validNeighMap(:, 1); 50 | validNeighbors = zeros(size(neighbors, 1), size(neighbors, 2) - 1); 51 | for i = 1 : size(validNeighbors, 1) 52 | validNeighbors(i, :) = neighbors(i, validNeighMap(i, :)); 53 | end 54 | neighInd = outInd(validNeighbors); 55 | else 56 | neighbors = knnsearch(features(outMap, :), features(inMap, :), 'K', K); 57 | neighInd = outInd(neighbors); 58 | end 59 | end -------------------------------------------------------------------------------- /affinity/knownToUnknownColorMixture.m: -------------------------------------------------------------------------------- 1 | 2 | % Known-to-Unknown Information Flow 3 | % This function implements the known-to-unknown information flow in 4 | % Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective 5 | % Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017. 6 | % All parameters other than image and the trimap are optional. The outputs 7 | % are the weight of FG pixels inside the unknown region, and the confidence 8 | % on these estimated values. 9 | % - K defines the number of neighbors found in FG and BG from which 10 | % LLE weights are computed. 11 | % - xyWeight determines how much importance is given to the spatial 12 | % coordinates in the nearest neighbor selection. 13 | 14 | function [alphaEst, conf] = knownToUnknownColorMixture(image, trimap, K, xyWeight) 15 | 16 | if ~exist('K', 'var') || isempty(K) 17 | K = 7; 18 | end 19 | if ~exist('xyWeight', 'var') || isempty(xyWeight) 20 | xyWeight = 10; 21 | end 22 | 23 | image= im2double(image); 24 | trimap = im2double(trimap(:,:,1)); 25 | bg = trimap < 0.2; 26 | fg = trimap > 0.8; 27 | unk = ~(bg | fg); 28 | 29 | % Find neighbors of unknown pixels in FG and BG 30 | [inInd, bgInd, features] = findNonlocalNeighbors(image, K, xyWeight, unk, bg); 31 | [~, fgInd] = findNonlocalNeighbors(image, K, xyWeight, unk, fg); 32 | neighInd = [fgInd, bgInd]; 33 | 34 | % Compute LLE weights and estimate FG and BG colors that got into the mixture 35 | features = features(:, 1 : end - 2); 36 | flows = zeros(size(inInd, 1), size(neighInd, 2)); 37 | fgCols = zeros(size(inInd, 1), 3); 38 | bgCols = zeros(size(inInd, 1), 3); 39 | for i = 1 : size(inInd, 1) 40 | flows(i, :) = localLinearEmbedding(features(inInd(i), :)', features(neighInd(i, :), :)', 1e-10); 41 | fgCols(i, :) = sum(features(neighInd(i, 1 : K), :) .* repmat(flows(i, 1 : K)', [1 3]), 1); 42 | bgCols(i, :) = sum(features(neighInd(i, K + 1 : end), :) .* repmat(flows(i, K + 1 : end)', [1 3]), 1); 43 | end 44 | 45 | % Estimated alpha is the sum of weights of FG neighbors 46 | alphaEst = trimap; 47 | alphaEst(unk) = sum(flows(:, 1 : K), 2); 48 | 49 | % Compute the confidence based on FG - BG color difference 50 | unConf = fgCols - bgCols; 51 | unConf = sum(unConf .* unConf, 2) / 3; 52 | conf = double(fg | bg); 53 | conf(unk) = unConf; 54 | end -------------------------------------------------------------------------------- /affinity/colorSimilarityAffinities.m: -------------------------------------------------------------------------------- 1 | 2 | % Color Similarity Non-local Pixel Affinities 3 | % This function implements the affinity based on color differences 4 | % first used for image matting in the paper 5 | % Qifeng Chen, Dingzeyu Li, Chi-Keung Tang, "KNN Matting", IEEE 6 | % TPAMI, 2013. 7 | % All parameters other than image are optional. The output is a sparse 8 | % matrix which has non-zero element for the non-local neighbors of 9 | % the pixels given by binary map inMap. 10 | % - K defines the number of neighbors from which LLE weights are 11 | % computed. 12 | % - outMap is a binary map that defines where the nearest neighbor 13 | % search is done. 14 | % - xyWeight determines how much importance is given to the spatial 15 | % coordinates in the nearest neighbor selection. 16 | % - When useHSV is false (default), the search is done i [r g b x y] space, 17 | % otherwise the feature space is [cos(h) sin(h), s, v, x, y]. 18 | 19 | function Wcs = colorSimilarityAffinities(image, K, inMap, outMap, xyWeight, useHSV) 20 | 21 | [h, w, ~] = size(image); 22 | N = h * w; 23 | 24 | if ~exist('K', 'var') || isempty(K) 25 | K = 5; 26 | end 27 | if ~exist('inMap', 'var') || isempty(inMap) 28 | inMap = true(h, w); 29 | end 30 | if ~exist('outMap', 'var') || isempty(outMap) 31 | outMap = true(h, w); 32 | end 33 | if ~exist('xyWeight', 'var') || isempty(xyWeight) 34 | xyWeight = 0.05; 35 | end 36 | if ~exist('useHSV', 'var') || isempty(useHSV) 37 | useHSV = false; 38 | end 39 | 40 | if useHSV 41 | image = rgb2hsv(image); 42 | image = cat(3, cos(image(:, :, 1)) * 2 * pi, sin(image(:, :, 1)) * 2 * pi, image(:,:,2:3)); 43 | end 44 | 45 | [~, neighInd, ~] = findNonlocalNeighbors(image, K, xyWeight, inMap, outMap); 46 | 47 | % This behaviour below, decreasing the xy-weight and finding a new set of neighbors, is taken 48 | % from the public implementation of KNN matting by Chen et al. 49 | [inInd, neighInd2, features] = findNonlocalNeighbors(image, ceil(K / 5), xyWeight / 100, inMap, outMap); 50 | neighInd = [neighInd, neighInd2]; 51 | features(:, end-1 : end) = features(:, end-1 : end) / 100; 52 | 53 | inInd = repmat(inInd, [1, size(neighInd, 2)]); 54 | flows = max(1 - sum(abs(features(inInd(:), :) - features(neighInd(:), :)), 2) / size(features, 2), 0); 55 | 56 | Wcs = sparse(inInd(:), neighInd(:), flows, N, N); 57 | Wcs = (Wcs + Wcs') / 2; % If p is a neighbor of q, make q a neighbor of p 58 | end -------------------------------------------------------------------------------- /common/getMattingParams.m: -------------------------------------------------------------------------------- 1 | 2 | % The returned struct can be customized before given as 3 | % input to each of the methods available in this toolbox. 4 | % algName should be: 5 | % - 'IFM' for information flow matting 6 | % - 'CF' for closed-form matting 7 | % - 'KNN' for KNN matting 8 | % - 'IFMRefinement' for information flow matte refinement 9 | % - 'SharedMatting' for shared matting matte refinement 10 | 11 | function params = getMattingParams(algName) 12 | if strcmpi(algName, 'IFM') || strcmpi(algName, 'InformationFlowMatting') || strcmpi(algName, 'IFMrefinement') 13 | params.lambda = 100; 14 | params.usePCGtoSolve = true; 15 | 16 | % Switch to use known-to-unknown information flow 17 | % -1: automatic selection, 0: do not use, 1: use 18 | params.useKnownToUnknown = -1; 19 | 20 | % Switch to apply edge-based trimming after matte estimation 21 | % The value reported in the paper is true, although we leave the 22 | % default as false here. 23 | params.mattePostTrim = false; 24 | 25 | % Color mixture information flow parameters 26 | params.cm_K = 20; 27 | params.cm_xyw = 1; 28 | params.cm_mult = 1; 29 | 30 | % Known-to-unknown information flow parameters 31 | params.ku_K = 7; 32 | params.ku_xyw = 10; 33 | params.ku_mult = 0.05; 34 | 35 | % Intra-unknown information flow parameters 36 | params.iu_K = 5; 37 | params.iu_xyw = 0.05; 38 | params.iu_mult = 0.01; 39 | 40 | % Local information flow parameters 41 | params.loc_win = 1; 42 | params.loc_eps = 1e-6; 43 | params.loc_mult = 1; 44 | 45 | % Parameter for Information Flow Matting matte refinement 46 | params.refinement_mult = 0.1; 47 | end 48 | if strcmpi(algName, 'ClosedForm') || strcmpi(algName, 'CF') || strcmpi(algName, 'ClosedFormMatting')... 49 | || strcmpi(algName, 'SharedMatting') || strcmpi(algName, 'SharedMattingRefinement') 50 | params.lambda = 100; 51 | params.usePCGtoSolve = false; 52 | 53 | % Matting Laplacian parameters 54 | params.loc_win = 1; 55 | params.loc_eps = 1e-7; 56 | 57 | % Parameter for Shared Matting matte refinement 58 | params.refinement_mult = 0.1; 59 | end 60 | if strcmpi(algName, 'KNN') || strcmpi(algName, 'KNNMatting') 61 | params.lambda = 1000; 62 | params.usePCGtoSolve = true; 63 | 64 | % Parameters for the neighbor selection 65 | params.knn_K = 20; 66 | params.knn_xyw = 1; 67 | params.knn_hsv = true; 68 | end 69 | end -------------------------------------------------------------------------------- /affinity/mattingAffinity.m: -------------------------------------------------------------------------------- 1 | 2 | % Matting Affinity 3 | % This function implements the image matting approach described in 4 | % Anat Levin, Dani Lischinski, Yair Weiss, "A Closed Form Solution to 5 | % Natural Image Matting", IEEE TPAMI, 2008. 6 | % All parameters other than image are optional. The output is a sparse 7 | % matrix which has non-zero element for the non-local neighbors of 8 | % the pixels given by binary map inMap. 9 | % - windowRadius defines the size of the window where the local normal 10 | % distributions are estimated. 11 | % - epsilon defines the regularization coefficient used before inverting 12 | % covariance matrices. It should be larger for noisy images. 13 | 14 | function W = mattingAffinity(image, inMap, windowRadius, epsilon) 15 | 16 | if ~exist('windowRadius', 'var') || isempty(windowRadius) 17 | windowRadius = 1; 18 | end 19 | if ~exist('epsilon', 'var') || isempty(epsilon) 20 | epsilon = 1e-7; 21 | end 22 | 23 | windowSize = 2 * windowRadius + 1; 24 | neighSize = windowSize^2; 25 | [h, w, c] = size(image); 26 | N = h * w; 27 | epsilon = epsilon / neighSize; 28 | 29 | % No need to compute affinities in known regions if a trimap is defined 30 | if nargin < 2 || isempty(inMap) 31 | inMap = true(size(image, 1), size(image, 2)); 32 | end 33 | 34 | [meanImage, covarMat] = localRGBnormalDistributions(image, windowRadius, epsilon); 35 | 36 | % Determine pixels and their local neighbors 37 | indices = reshape((1 : h * w), [h w]); 38 | neighInd = im2col(indices, [windowSize windowSize], 'sliding')'; 39 | inMap = inMap(windowRadius + 1 : end - windowRadius, windowRadius + 1 : end - windowRadius); 40 | neighInd = neighInd(inMap, :); 41 | inInd = neighInd(:, (neighSize + 1) / 2); 42 | pixCnt = size(inInd, 1); 43 | 44 | % Prepare in & out data 45 | image = reshape(image, [N, c]); 46 | meanImage = reshape(meanImage, [N, c]); 47 | flowRows = zeros(neighSize, neighSize, pixCnt); 48 | flowCols = zeros(neighSize, neighSize, pixCnt); 49 | flows = zeros(neighSize, neighSize, pixCnt); 50 | 51 | % Compute matting affinity 52 | for i = 1 : size(inInd, 1) 53 | neighs = neighInd(i, :); 54 | shiftedWinColors = image(neighs, :) - repmat(meanImage(inInd(i), :), [size(neighs, 2), 1]); 55 | flows(:, :, i) = shiftedWinColors * (covarMat(:, :, inInd(i)) \ shiftedWinColors'); 56 | neighs = repmat(neighs, [size(neighs, 2), 1]); 57 | flowRows(:, :, i) = neighs; 58 | flowCols(:, :, i) = neighs'; 59 | end 60 | flows = (flows + 1) / neighSize; 61 | W = sparse(flowRows(:), flowCols(:), flows(:), N, N); 62 | 63 | % Make sure it's symmetric 64 | W = W + W'; 65 | 66 | % Normalize 67 | sumW = full(sum(W, 2)); 68 | sumW(sumW < 0.05) = 1; 69 | W = spdiags(1 ./ sumW(:), 0, N, N) * W; 70 | end -------------------------------------------------------------------------------- /informationFlowMatteRefinement.m: -------------------------------------------------------------------------------- 1 | 2 | % Information-Flow Matte Refinement 3 | % This function implements the matte refinement approach described in 4 | % Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective 5 | % Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017. 6 | % 'alphaHat' and 'confidences' parameters are typically obtained by a 7 | % sampling-based natural matting algorithm. 'confidences' is filled by 8 | % ones if not provided. Optional input parameter 'params' can be 9 | % customized by editing the default values in the struct returned 10 | % by 'getMattingParams('IFM'). 11 | % - **_K parameters represent the number of nonlocal neighbors found 12 | % for color mixture, and intra-U flows, while **_xyw define the effect of 13 | % spatial proximity. 14 | % - **_mult define the weight of each information flow. 15 | % - loc_*** define the parameters for the matting Laplacian. 16 | % - refinement_mult determines how much trust is given to the initial 17 | % alpha estimation 18 | 19 | function alpha = informationFlowMatteRefinement(image, trimap, alphaHat, confidences, params, suppressMessages) 20 | abmtSetup 21 | tic; 22 | if ~exist('confidences', 'var') || isempty(confidences) 23 | confidences = ones(size(alphaHat(:,:,1))); 24 | end 25 | if ~exist('params', 'var') || isempty(params) 26 | params = getMattingParams('IFM'); 27 | end 28 | if ~exist('suppressMessages', 'var') || isempty(suppressMessages) 29 | suppressMessages = false; 30 | end 31 | if(~suppressMessages) display('Matte refinement via Information-Flow Matting...'); end 32 | 33 | image = im2double(image); 34 | trimap = im2double(trimap(:,:,1)); 35 | alphaHat = im2double(alphaHat(:,:,1)); 36 | 37 | % Compute L_IFM 38 | unk = trimap < 0.8 & trimap > 0.2; 39 | dilUnk = imdilate(unk, ones(3, 3)); 40 | if(~suppressMessages) display(' Computing color mixture flow...'); end 41 | Lap = affinityMatrixToLaplacian(colorMixtureAffinities(image, params.cm_K, dilUnk, [], params.cm_xyw)); 42 | Lap = params.cm_mult * (Lap' * Lap); 43 | if(~suppressMessages) display(' Computing matting Laplacian...'); end 44 | Lap = Lap + params.loc_mult * affinityMatrixToLaplacian(mattingAffinity(image, dilUnk, params.loc_win, params.loc_eps)); 45 | if(~suppressMessages) display(' Computing intra-U flow...'); end 46 | Lap = Lap + params.iu_mult * affinityMatrixToLaplacian(colorSimilarityAffinities(image, params.iu_K, unk, unk, params.iu_xyw)); 47 | 48 | if(~suppressMessages) display(' Solving for alphas...'); end 49 | alpha = solveForAlphas(Lap, trimap, params.lambda, params.usePCGtoSolve, alphaHat, confidences, params.refinement_mult); 50 | 51 | alpha = reshape(alpha, [size(image, 1), size(image, 2)]); 52 | 53 | dur = toc; 54 | if(~suppressMessages) display(['Done. It took ' num2str(dur) ' seconds.']); end 55 | end 56 | -------------------------------------------------------------------------------- /trimming/patchBasedTrimming.m: -------------------------------------------------------------------------------- 1 | 2 | % Patch-Based Trimming 3 | % This function implements the trimap trimming approach described in 4 | % Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective 5 | % Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017. 6 | % The input parameters other than image and trimap are optional. 7 | % - minDist and maxDist define a good match, and a match in BG that 8 | % rejects a good match in FG, and vice versa. 9 | % - windowRadius defines the size of the window where the local normal 10 | % distributions are estimated 11 | % - K defines the number of nearest neighbors found using the mean 12 | % vectors before the Bhattacharyya distance comparison. 13 | 14 | function trimap = patchBasedTrimming(image, trimap, minDist, maxDist, windowRadius, K) 15 | 16 | if ~exist('minDist', 'var') || isempty(minDist) 17 | minDist = 0.25; 18 | end 19 | if ~exist('maxDist', 'var') || isempty(maxDist) 20 | maxDist = 0.90; 21 | end 22 | if ~exist('windowRadius', 'var') || isempty(windowRadius) 23 | windowRadius = 1; 24 | end 25 | if ~exist('K', 'var') || isempty(K) 26 | K = 10; 27 | end 28 | 29 | image = im2double(image); 30 | trimap = im2double(trimap(:,:,1)); 31 | [h, w, ~] = size(image); 32 | 33 | epsilon = 1e-8; 34 | 35 | fg = trimap > 0.8; 36 | bg = trimap < 0.2; 37 | unk = ~(fg | bg); 38 | 39 | [meanImage, covarMat] = localRGBnormalDistributions(image, windowRadius, epsilon); 40 | 41 | [unkInd, fgNeigh] = findNonlocalNeighbors(meanImage, K, -1, unk, fg); 42 | [~, bgNeigh] = findNonlocalNeighbors(meanImage, K, -1, unk, bg); 43 | 44 | meanImage = reshape(meanImage, [h * w, size(meanImage, 3)]); 45 | 46 | fgBhatt = zeros(K, 1); 47 | bgBhatt = zeros(K, 1); 48 | for i = 1 : size(unkInd, 1) 49 | pixMean = meanImage(unkInd(i), :)'; 50 | pixCovar = covarMat(:, :, unkInd(i)); 51 | pixDet = det(pixCovar); 52 | for n = 1 : K 53 | nMean = meanImage(fgNeigh(i, n), :)' - pixMean; 54 | nCovar = covarMat(:, :, fgNeigh(i, n)); 55 | nDet = det(nCovar); 56 | nCovar = (pixCovar + nCovar) / 2; 57 | fgBhatt(n) = 0.125 * nMean' * (nCovar \ nMean) + 0.5 * log(det(nCovar) / sqrt(pixDet * nDet)); % Bhattacharyya distance 58 | end 59 | for n = 1 : K 60 | nMean = meanImage(bgNeigh(i, n), :)' - pixMean; 61 | nCovar = covarMat(:, :, bgNeigh(i, n)); 62 | nDet = det(nCovar); 63 | nCovar = (pixCovar + nCovar) / 2; 64 | bgBhatt(n) = 0.125 * nMean' * (nCovar \ nMean) + 0.5 * log(det(nCovar) / sqrt(pixDet * nDet)); % Bhattacharyya distance 65 | end 66 | minFGdist = min(fgBhatt); 67 | minBGdist = min(bgBhatt); 68 | if minFGdist < minDist 69 | if minBGdist > maxDist 70 | trimap(unkInd(i)) = 1; 71 | end 72 | elseif minBGdist < minDist 73 | if minFGdist > maxDist 74 | trimap(unkInd(i)) = 0; 75 | end 76 | end 77 | end 78 | end -------------------------------------------------------------------------------- /trimming/trimmingFromKnownUnknownEdges.m: -------------------------------------------------------------------------------- 1 | 2 | % Trimming from Edges of the Unknown Region 3 | % This function implements the trimap trimming approach described in 4 | % Ehsan Shahrian, Deepu Rajan, Brian Price, Scott Cohen, "Improving 5 | % Image Matting using Comprehensive Sampling Sets", CVPR 2013 6 | % The implementation uses the public source code provided by the 7 | % authors as a guideline and has an iterative structure not explained 8 | % in the paper. The input parameters other than image and trimap 9 | % are optional. 10 | % - Maximum Manhattan distance of trimming is determined by iterCnt 11 | % such that maxManhDist = sum(1:iterCnt). 12 | % - paramU determines the (maximum) color threshold in the iterations. 13 | % - paramD determines how much this threshold is lowered as the 14 | % iterations progress. 15 | 16 | function trimap = trimmingFromKnownUnknownEdges(image, trimap, paramU, paramD, iterCnt) 17 | 18 | if ~exist('iterCnt', 'var') || isempty(iterCnt) 19 | iterCnt = 9; 20 | end 21 | if ~exist('paramD', 'var') || isempty(paramD) 22 | paramD = 1 / 256; 23 | end 24 | if ~exist('paramU', 'var') || isempty(paramU) 25 | paramU = 9 / 256; 26 | end 27 | 28 | image = im2double(image); 29 | trimap = im2double(trimap); 30 | bg = (trimap < 0.2); 31 | fg = (trimap > 0.8); 32 | paramD = paramU - paramD; 33 | 34 | for i = 1 : iterCnt 35 | iterColorThresh = paramU - i * paramD / iterCnt; % color threshold = paramU - iterNo * (paramU - paramD) / maxIter 36 | trimap = LabelExpansion(image, trimap, i, iterColorThresh); % distance threshold 1 to iterCnt 37 | end 38 | end 39 | 40 | function [extendedTrimap] = LabelExpansion(image, trimap, maxDist, colorThresh) 41 | [h, w, ~] = size(image); 42 | 43 | fg = trimap > 0.8; 44 | bg = trimap < 0.2; 45 | knownReg = (bg | fg); 46 | extendedTrimap = trimap; 47 | 48 | searchReg= ((imdilate(fg, ones(2 * maxDist + 1)) & ~fg) | (imdilate(bg, ones(2 * maxDist + 1)) & ~bg)); 49 | [cols, rows] = meshgrid(1 : w, 1 : h); 50 | cols = cols(searchReg(:)); 51 | rows = rows(searchReg(:)); 52 | 53 | winCenter = (2 * maxDist) / 2 + 1; 54 | distPlane = repmat((1 : 2 * maxDist + 1)', [1, 2 * maxDist + 1])'; 55 | distPlane = sqrt((distPlane - winCenter) .^ 2 + (distPlane' - winCenter) .^ 2); 56 | 57 | for pixNo = 1 : size(cols, 1) 58 | r = rows(pixNo); 59 | c = cols(pixNo); 60 | minR = max(r - maxDist, 1); % pixel limits 61 | minC = max(c - maxDist, 1); 62 | maxR = min(r + maxDist , h); 63 | maxC = min(c + maxDist, w); 64 | winMinR = winCenter - (r - minR); % pixel limits in window 65 | winMinC = winCenter - (c - minC); 66 | winMaxR = winCenter + (maxR - r); 67 | winMaxC = winCenter + (maxC - c); 68 | 69 | pixColor = image(r, c, :); 70 | imgWin = image(minR : maxR, minC : maxC, :); % colors 71 | trimapWin = trimap(minR : maxR, minC : maxC); 72 | 73 | winColorDiff = imgWin(:, :, 1) - pixColor(1); 74 | winColorDiff(:, :, 2) = imgWin(:, :, 2) - pixColor(2); 75 | winColorDiff(:, :, 3) = imgWin(: ,:, 3) - pixColor(3); 76 | winColorDiff = sqrt(sum(winColorDiff .* winColorDiff, 3)); 77 | 78 | candidates= (winColorDiff < colorThresh) & knownReg(minR : maxR, minC : maxC); % known pixels under thresh 79 | if sum(candidates(:)) > 0 80 | distWin = distPlane(winMinR : winMaxR, winMinC : winMaxC); % distance plane 81 | distWin = distWin(candidates); % distances of known 82 | [~, minDistInd] = min(distWin); % location of minimum 83 | trimapWin = trimapWin(candidates); 84 | extendedTrimap(r, c) = trimapWin(minDistInd); 85 | end 86 | end 87 | 88 | end 89 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Affinity-Based Matting Toolbox 2 | ================================================================= 3 | 4 | About 5 | ------------ 6 | 7 | This toolbox includes a collection of common affinity-based image matting algorithms as well as matte refinement algorithms used by sampling-based image matting methods. 8 | It features the only public (re-)implementation of information-flow matting [AAP17], a faster matting Laplacian [LLW08] computation and a faster trimap trimming [SRPC13]. 9 | The parameters for each algorithm are easily customizable. 10 | 11 | The included matting algorithms are: 12 | 13 | - Information-flow matting [AAP17] 14 | - KNN matting [CLT13] 15 | - Closed-form matting [LLW08] 16 | 17 | The included matte refinement algorithms are: 18 | 19 | - Information-flow matte refinement [AAP17] 20 | - Shared matting matte refinement [GO10] 21 | 22 | The included trimap trimming methods are: 23 | 24 | - Patch-based trimming [AAP17] 25 | - Trimming from known-unknown edges [SRPC13] 26 | 27 | The toolbox is designed to be ease of use for an extended set of applications. 28 | Sparse affinity matrices defined and used in [AAP17, CLT13, CZZT12, LLW08] can be obtained by calling the corresponding functions inside 'affinity' directory. 29 | The functions in this directory allow defining regions for neighborhood search. 30 | 31 | An example image-trimap pair from the alpha matting benchmark [RRW09] is provided. 32 | Basic features are demonstrated in the demo file. 33 | Each function features an explanation and definitions of related parameters. 34 | 35 | The information-flow matting function in this toolbox is not the original implementation used in the paper. 36 | These are reimplementations of the original methods and may not give the exact same results as reported in the corresponding papers. 37 | 38 | Planned extensions 39 | ------------ 40 | I plan to add the layer color estimation methods, as well as LNSP matting and multiple-layer matte estimation methods in the near future. 41 | I may tweak the information-flow matting implementation to achieve its original performance on the benchmark. 42 | Feel free to contribute or propose a method to be added to the toolbox by [contacting me](http://people.inf.ethz.ch/aksoyy/contact/). 43 | 44 | License and citing the toolbox 45 | ------------ 46 | 47 | This toolbox is provided for academic use only. 48 | If you use this toolbox for an academic publication, please cite corresponding publications referenced in the description of each function, as well as this toolbox itself: 49 | 50 | @MISC{abmt, 51 | author={Ya\u{g}\{i}z Aksoy}, 52 | title={Affinity-based matting toolbox}, 53 | year={2017}, 54 | howpublished = {\url{https://github.com/yaksoy/AffinityBasedMattingToolbox}}, 55 | } 56 | 57 | References 58 | ------------ 59 | [AAP17] Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017. [[link](http://people.inf.ethz.ch/aksoyy/ifm/)] 60 | 61 | [CLT13] Qifeng Chen, Dingzeyu Li, Chi-Keung Tang, "KNN Matting", IEEE TPAMI, 2013. [[link](http://dingzeyu.li/projects/knn/)] 62 | 63 | [CZZT12] Xiaowu Chen, Dongqing Zou, Qinping Zhao, Ping Tan, "Manifold preserving edit propagation", ACM TOG, 2012 [[paper](http://www.cs.sfu.ca/~pingtan/Papers/sigasia12.pdf)] 64 | 65 | [GO10] Eduardo S. L. Gastal, Manuel M. Oliveira, "Shared Sampling for Real-Time Alpha Matting", Computer Graphics Forum, 2010. [[link](http://www.inf.ufrgs.br/~eslgastal/SharedMatting/)] 66 | 67 | [LLW08] Anat Levin, Dani Lischinski, Yair Weiss, "A Closed Form Solution to Natural Image Matting", IEEE TPAMI, 2008. [[paper](http://people.csail.mit.edu/alevin/papers/Matting-Levin-Lischinski-Weiss-PAMI.pdf)] 68 | 69 | [RRW09] Christoph Rhemann, Carsten Rother, Jue Wang, Margrit Gelautz, Pushmeet Kohli, Pamela Rott, "A Perceptually Motivated Online Benchmark for Image Matting", CVPR 2009. [[link](http://alphamatting.com)] 70 | 71 | [SRPC13] Ehsan Shahrian, Deepu Rajan, Brian Price, Scott Cohen, "Improving Image Matting using Comprehensive Sampling Sets", CVPR 2013 [[paper](http://www.cv-foundation.org/openaccess/content_cvpr_2013/papers/Shahrian_Improving_Image_Matting_2013_CVPR_paper.pdf)] -------------------------------------------------------------------------------- /informationFlowMatting.m: -------------------------------------------------------------------------------- 1 | 2 | % Information-Flow Matting 3 | % This function implements the image matting approach described in 4 | % Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "Designing Effective 5 | % Inter-Pixel Information Flow for Natural Image Matting", CVPR, 2017. 6 | % Optional input parameter 'params' can be customized by editing the 7 | % default values in the struct returned by 'getMattingParams('IFM'). 8 | % - The parameter useKnownToUnknown makes the decision to use 9 | % E_1 or E_2 as defined in the paper. A negative number means 10 | % automatic selection. 11 | % - **_K parameters represent the number of nonlocal neighbors found 12 | % for color mixture, K-to-U and intra-U flows, while **_xyw define the 13 | % effect of spatial proximity. 14 | % - **_mult define the weight of each information flow. 15 | % - loc_*** define the parameters for the matting Laplacian. 16 | % - mattePostTrim determines if edge-based trimming should be applied 17 | % as a post-processing to the estimated alpha. Here the defult is 18 | % false, but in the original paper it is reported to be 'true'. 19 | 20 | function alpha = informationFlowMatting(image, trimap, params, suppressMessages) 21 | abmtSetup 22 | tic; 23 | if ~exist('params', 'var') || isempty(params) 24 | params = getMattingParams('IFM'); 25 | end 26 | if ~exist('suppressMessages', 'var') || isempty(suppressMessages) 27 | suppressMessages = false; 28 | end 29 | if(~suppressMessages) display('Information-Flow Matting started...'); end 30 | 31 | image = im2double(image); 32 | trimap = im2double(trimap(:,:,1)); 33 | 34 | % Decide to use the K-to-U flow 35 | if params.useKnownToUnknown < 0 36 | useKU = ~detectHighlyTransparent(image, trimap); 37 | else 38 | useKU = params.useKnownToUnknown > 0; 39 | end 40 | if(~suppressMessages) 41 | if useKU 42 | display(' Known-to-unknown information flow will be used.'); 43 | else 44 | display(' Known-to-unknown information flow will NOT be used.'); 45 | end 46 | end 47 | 48 | if params.mattePostTrim || useKU 49 | % Trimap trimming for refining kToU flow or final matte 50 | if(~suppressMessages) display(' Trimming trimap from edges...'); end 51 | edgeTrimmed = trimmingFromKnownUnknownEdges(image, trimap); 52 | end 53 | 54 | % Compute L_IFM 55 | unk = trimap < 0.8 & trimap > 0.2; 56 | dilUnk = imdilate(unk, ones(2 * params.loc_win + 1)); 57 | if(~suppressMessages) display(' Computing color mixture flow...'); end 58 | Lap = affinityMatrixToLaplacian(colorMixtureAffinities(image, params.cm_K, unk, [], params.cm_xyw)); 59 | Lap = params.cm_mult * (Lap' * Lap); 60 | if(~suppressMessages) display(' Computing matting Laplacian...'); end 61 | Lap = Lap + params.loc_mult * affinityMatrixToLaplacian(mattingAffinity(image, dilUnk, params.loc_win, params.loc_eps)); 62 | if(~suppressMessages) display(' Computing intra-U flow...'); end 63 | Lap = Lap + params.iu_mult * affinityMatrixToLaplacian(colorSimilarityAffinities(image, params.iu_K, unk, unk, params.iu_xyw)); 64 | 65 | if useKU 66 | % Compute kToU flow 67 | if(~suppressMessages) display(' Trimming trimap using patch similarity...'); end 68 | patchTrimmed = patchBasedTrimming(image, trimap, [], [], [], 5); % We set K = 5 here for better computation time 69 | if(~suppressMessages) display(' Computing K-to-U flow...'); end 70 | [kToU, kToUconf] = knownToUnknownColorMixture(image, patchTrimmed, params.ku_K, params.ku_xyw); 71 | kToU(edgeTrimmed < 0.2) = 0; 72 | kToU(edgeTrimmed > 0.8) = 1; 73 | kToUconf(edgeTrimmed < 0.2) = 1; 74 | kToUconf(edgeTrimmed > 0.8) = 1; 75 | if(~suppressMessages) display(' Solving for alphas...'); end 76 | alpha = solveForAlphas(Lap, trimap, params.lambda, params.usePCGtoSolve, kToU, kToUconf, params.ku_mult); 77 | else 78 | if(~suppressMessages) display(' Solving for alphas...'); end 79 | alpha = solveForAlphas(Lap, trimap, params.lambda, params.usePCGtoSolve); 80 | end 81 | 82 | alpha = reshape(alpha, [size(image, 1), size(image, 2)]); 83 | 84 | if params.mattePostTrim 85 | alpha(edgeTrimmed < 0.2) = 0; 86 | alpha(edgeTrimmed > 0.8) = 1; 87 | end 88 | 89 | dur = toc; 90 | if(~suppressMessages) display(['Done. It took ' num2str(dur) ' seconds.']); end 91 | end 92 | --------------------------------------------------------------------------------