├── LICENSE ├── SemanticSoftSegmentation.m ├── SpectralMatting.m ├── Superpixels.m ├── affinityMatrixToLaplacian.m ├── demo.m ├── docia.png ├── groupSegments.m ├── imageGradient.m ├── mattingAffinity.m ├── preprocessFeatures.m ├── readme.md ├── replicability_instructions.htm ├── softSegmentsFromEigs.m ├── sparsifySegments.m ├── visualizeEigenvectorRedGreen.m └── visualizeSoftSegments.m /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018, Yagiz Aksoy. All rights reserved. 2 | 3 | This software is for academic use only. A redistribution of this 4 | software, with or without modifications, has to be for academic 5 | use only, while giving the appropriate credit to the original 6 | authors of the software. The methods implemented as a part of 7 | this software may be covered under patents or patent applications. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED 10 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 11 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR 12 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 13 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 14 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 15 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 16 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 17 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /SemanticSoftSegmentation.m: -------------------------------------------------------------------------------- 1 | 2 | % Semantic Soft Segmentation 3 | % This function implements the soft segmentation approach described in 4 | % Yagiz Aksoy, Tae-Hyun Oh, Sylvain Paris, Marc Pollefeys, Wojciech Matusik 5 | % "Semantic Soft Segmentation", ACM TOG (Proc. SIGGRAPH) 2018 6 | 7 | function [softSegments, initSoftSegments, Laplacian, affinities, features, superpixels, eigenvectors, eigenvalues] = SemanticSoftSegmentation(image, features) 8 | 9 | disp('Semantic Soft Segmentation') 10 | % Prepare the inputs and superpixels 11 | image = im2double(image); 12 | if size(features, 3) > 3 % If the features are raw, hyperdimensional, preprocess them 13 | features = preprocessFeatures(features, image); 14 | else 15 | features = im2double(features); 16 | end 17 | superpixels = Superpixels(image); 18 | [h, w, ~] = size(image); 19 | 20 | disp(' Computing affinities') 21 | % Compute the affinities and the Laplacian 22 | affinities{1} = mattingAffinity(image); 23 | affinities{2} = superpixels.neighborAffinities(features); % semantic affinity 24 | affinities{3} = superpixels.nearbyAffinities(image); % non-local color affinity 25 | Laplacian = affinityMatrixToLaplacian(affinities{1} + 0.01 * affinities{2} + 0.01 * affinities{3}); % Equation 6 26 | 27 | disp(' Computing eigenvectors') 28 | % Compute the eigendecomposition 29 | eigCnt = 100; % We use 100 eigenvectors in the optimization 30 | [eigenvectors, eigenvalues] = eigs(Laplacian, eigCnt, 'SM'); 31 | 32 | disp(' Initial optimization') 33 | % Compute initial soft segments 34 | initialSegmCnt = 40; 35 | sparsityParam = 0.8; 36 | iterCnt = 40; 37 | % feeding features to the function below triggers semantic intialization 38 | initSoftSegments = softSegmentsFromEigs(eigenvectors, eigenvalues, Laplacian, ... 39 | h, w, features, initialSegmCnt, iterCnt, sparsityParam, [], []); 40 | 41 | % Group segments w.r.t. their mean semantic feature vectors 42 | groupedSegments = groupSegments(initSoftSegments, features); 43 | 44 | disp(' Final optimization') 45 | % Do the final sparsification 46 | softSegments = sparsifySegments(groupedSegments, Laplacian, imageGradient(image, false, 6)); 47 | 48 | disp(' Done.') 49 | end -------------------------------------------------------------------------------- /SpectralMatting.m: -------------------------------------------------------------------------------- 1 | 2 | % Spectral Matting 3 | % This function implements the soft segmentation approach described in 4 | % Anat Levin, Dani Lischinski, Yair Weiss, "Spectral Matting", IEEE TPAMI, 2008. 5 | % The parameters here are set to their default values as reported by Levin et al. 6 | 7 | function [softSegments, Laplacian, eigenvectors, eigenvalues] = SpectralMatting(image) 8 | 9 | disp('Spectral Matting') 10 | image = im2double(image); 11 | [h, w, ~] = size(image); 12 | 13 | disp(' Computing affinities') 14 | % Compute the matting Laplacian 15 | Laplacian = affinityMatrixToLaplacian(mattingAffinity(image)); 16 | 17 | disp(' Computing eigenvectors') 18 | % Compute the eigendecomposition 19 | eigCnt = 50; 20 | [eigenvectors, eigenvalues] = eigs(Laplacian, eigCnt, 'SM'); 21 | 22 | disp(' Optimization') 23 | % Compute the soft segments = matting components 24 | initialSegmCnt = 20; 25 | sparsityParam = 0.8; 26 | iterCnt = 20; 27 | softSegments = softSegmentsFromEigs(eigenvectors, eigenvalues, Laplacian, ... 28 | h, w, [], initialSegmCnt, iterCnt, sparsityParam, [], []); 29 | 30 | disp(' Done.') 31 | end -------------------------------------------------------------------------------- /Superpixels.m: -------------------------------------------------------------------------------- 1 | 2 | % This class is for computing superpixel-based affinities described in the paper. 3 | % This class requires the image graphs methods by Steve Eddins. Find it here: 4 | % http://www.mathworks.com/matlabcentral/fileexchange/53614-image-graphs 5 | 6 | classdef Superpixels 7 | properties 8 | labels 9 | spcount 10 | neigh 11 | centroids 12 | end 13 | methods 14 | function obj = Superpixels(im, spcnt) 15 | if ~exist('spcnt', 'var') || isempty(spcnt) 16 | spcnt = 2500; 17 | end 18 | [L, N] = superpixels(im, spcnt, 'Compactness', 1e-20); 19 | obj.labels = L; 20 | obj.spcount = N; 21 | % Find neighboring superpixels 22 | g = adjacentRegionsGraph(L); 23 | obj.neigh = g.Edges.Labels; 24 | % Find centroids 25 | s = regionprops(L, 'centroid'); 26 | cent = cat(1, s.Centroid); 27 | obj.centroids = round(cent(:, 2:-1:1)); 28 | [h, w, ~] = size(im); 29 | obj.centroids(:, 3) = sub2ind([h, w], obj.centroids(:, 1), obj.centroids(:, 2)); 30 | end 31 | 32 | function regmeans = computeRegionMeans(obj, image) 33 | [h, w, c] = size(image); 34 | image = reshape(image, [h*w, c]); 35 | regmeans = zeros(obj.spcount, c); 36 | idx = label2idx(obj.labels); 37 | for i = 1 : length(idx) 38 | regmeans(i, :) = mean(image(idx{i}, :), 1); 39 | end 40 | end 41 | 42 | % This is for the semantic affinity, generates affinities in [-1, 1] 43 | function W = neighborAffinities(obj, features, erfSteepness, erfCenter) 44 | if ~exist('erfSteepness', 'var') || isempty(erfSteepness) 45 | erfSteepness = 20; 46 | end 47 | if ~exist('erfCenter', 'var') || isempty(erfCenter) 48 | erfCenter = 0.85; 49 | end 50 | [h, w, ~] = size(features); 51 | N = h * w; 52 | spMeans = obj.computeRegionMeans(features); 53 | affs = zeros(size(obj.neigh, 1), 1); 54 | inds1 = affs; 55 | inds2 = affs; 56 | for i = 1 : size(obj.neigh, 1) 57 | ind1 = obj.neigh(i, 1); 58 | ind2 = obj.neigh(i, 2); 59 | affs(i) = sigmoidAff(spMeans(ind1, :), spMeans(ind2, :), erfSteepness, erfCenter); 60 | inds1(i) = obj.centroids(ind1, 3); 61 | inds2(i) = obj.centroids(ind2, 3); 62 | end 63 | W = sparse(inds1, inds2, affs, N, N); 64 | W = W' + W; 65 | end 66 | 67 | % This is for the nonlocal color affinity, generates affinities in [0, 1] 68 | function W = nearbyAffinities(obj, image, erfSteepness, erfCenter, proxThresh) 69 | if ~exist('erfSteepness', 'var') || isempty(erfSteepness) 70 | erfSteepness = 50; 71 | end 72 | if ~exist('erfCenter', 'var') || isempty(erfCenter) 73 | erfCenter = 0.95; 74 | end 75 | if ~exist('proxThresh', 'var') || isempty(proxThresh) 76 | proxThresh = 0.2; 77 | end 78 | [h, w, ~] = size(image); 79 | N = h * w; 80 | spMeans = obj.computeRegionMeans(image); 81 | combinationCnt = obj.spcount; 82 | combinationCnt = combinationCnt * (combinationCnt - 1) / 2; 83 | affs = zeros(combinationCnt, 1); 84 | inds1 = affs; 85 | inds2 = affs; 86 | cnt = 1; 87 | cents = obj.centroids(:, 1:2); 88 | cents(:,1) = cents(:,1) / h; 89 | cents(:,2) = cents(:,2) / w; 90 | for i = 1 : obj.spcount 91 | for j = i + 1 : obj.spcount 92 | centdist = cents(i, 1:2) - cents(j, 1:2); 93 | centdist = sqrt(centdist * centdist'); 94 | if centdist > proxThresh 95 | affs(cnt) = 0; 96 | else 97 | affs(cnt) = sigmoidAffPos(spMeans(i, :), spMeans(j, :), erfSteepness, erfCenter); 98 | end 99 | inds1(cnt) = obj.centroids(i, 3); 100 | inds2(cnt) = obj.centroids(j, 3); 101 | cnt = cnt + 1; 102 | end 103 | end 104 | W = sparse(inds1, inds2, affs, N, N); 105 | W = W' + W; 106 | end 107 | 108 | function vis = visualizeRegionMeans(obj, im) 109 | vis = label2rgb(obj.labels, obj.computeRegionMeans(im)); 110 | end 111 | 112 | end 113 | end 114 | 115 | function aff = sigmoidAff(feat1, feat2, steepness, center) 116 | aff = abs(feat1 - feat2); 117 | aff = 1 - sqrt(aff * aff'); 118 | aff = (erf(steepness * (aff - center))); 119 | end 120 | 121 | function aff = sigmoidAffPos(feat1, feat2, steepness, center) 122 | aff = abs(feat1 - feat2); 123 | aff = 1 - sqrt(aff * aff'); 124 | aff = (erf(steepness * (aff - center)) + 1) / 2; 125 | end -------------------------------------------------------------------------------- /affinityMatrixToLaplacian.m: -------------------------------------------------------------------------------- 1 | 2 | function Lap = affinityMatrixToLaplacian(aff, normalize) 3 | if ~exist('normalize', 'var') || isempty(normalize) 4 | normalize = false ; 5 | end 6 | N = size(aff, 1); 7 | if normalize 8 | aa = sum(aff, 2); 9 | D = spdiags(aa, 0 , N, N); 10 | aa = sqrt(1./aa); 11 | D12 = spdiags(aa, 0 , N, N); 12 | Lap = D12 * (D - aff) * D12; 13 | else 14 | Lap = spdiags(sum(aff, 2), 0 , N, N) - aff; 15 | end 16 | end -------------------------------------------------------------------------------- /demo.m: -------------------------------------------------------------------------------- 1 | 2 | % Add the ImageGraph to path (in a folder named 'ImageGraphs'). Find it here: 3 | % http://www.mathworks.com/matlabcentral/fileexchange/53614-image-graphs 4 | addpath(fullfile(fileparts(mfilename('fullpath')), 'ImageGraphs')); 5 | 6 | %% Read the image and features from the sample file 7 | image = im2double(imread('docia.png')); 8 | features = image(:, size(image, 2) / 2 + 1 : end, :); 9 | image = image(:, 1 : size(image, 2) / 2, :); 10 | 11 | % The eigendecomposition uses a lot of memory and may render the computer 12 | % unresponsive, so better to test it first with a small image. 13 | image = imresize(image, 0.5); 14 | features = imresize(features, 0.5); 15 | 16 | %% Semantic soft segmentation 17 | % This function outputs many intermediate variables, if needed. 18 | % The results may vary a bit from run to run, as there are 2 stages that use 19 | % k-means for intialization & grouping. 20 | sss = SemanticSoftSegmentation(image, features); 21 | 22 | % To use the features generated using our network implementation, 23 | % just feed them as the 'features' variable to the function. It will do 24 | % the prepocessing described in the paper and give the processed 25 | % features as an output. 26 | % If you are dealing with many images, storing the features after 27 | % preprocessing is recommended as raw hyperdimensional features 28 | % take a lot of space. Check the 'preprocessFeatures.m' file. 29 | 30 | % Visualize 31 | figure; imshow([image features visualizeSoftSegments(sss)]); 32 | title('Semantic soft segments'); 33 | 34 | % There's also an implementation of Spectral Matting included 35 | sm = SpectralMatting(image); 36 | % You can group the soft segments from Spectral Matting using 37 | % semantic features, the way we presented our comparisons in the paper. 38 | sm_gr = groupSegments(sm, features); 39 | figure; imshow([image visualizeSoftSegments(sm) visualizeSoftSegments(sm_gr)]); 40 | title('Matting components'); 41 | -------------------------------------------------------------------------------- /docia.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaksoy/SemanticSoftSegmentation/3d0a57c40d58be61599e744f28a9cb8be6960509/docia.png -------------------------------------------------------------------------------- /groupSegments.m: -------------------------------------------------------------------------------- 1 | 2 | % A simple grouping of soft segments w.r.t. their semantic features 3 | % as described in Section 3.4. 4 | 5 | function groupedSegments = groupSegments(segments, features, segmCnt) 6 | if ~exist('segmCnt', 'var') || isempty(segmCnt) 7 | segmCnt = 5; 8 | end 9 | [h, w, cnt] = size(segments); 10 | compFeatures = zeros(cnt, size(features, 3)); 11 | for i = 1 : cnt 12 | cc = segments(:,:,i) .* features; 13 | cc = sum(sum(cc, 1), 2) / sum(sum(segments(:,:,i), 1), 2); 14 | compFeatures(i, :) = cc; 15 | end 16 | ids = kmeans(compFeatures, segmCnt); 17 | groupedSegments = zeros(h, w, segmCnt); 18 | for i = 1 : segmCnt 19 | groupedSegments(:,:,i) = sum(segments(:,:,ids==i), 3); 20 | end 21 | end -------------------------------------------------------------------------------- /imageGradient.m: -------------------------------------------------------------------------------- 1 | 2 | % Yagiz Aksoy, 2016 3 | % Implements H. Farid, E.P. Simoncelli, Differentiation of Discrete Multidimensional Signals, TIP 2004 4 | % outColor switch makes the output 3-channel (color) or 1-channel, default true 5 | % There are 3 different filtSize options (1, 4 or 6), which determines the number of taps in the filters 6 | % and hence which neighborhood is used to approximate the derivatives 7 | 8 | function [gradientMagnitude, gradientOrientation, xGradient, yGradient] = imageGradient(image, outColor, filtSize) 9 | % Set up variables legally 10 | if ~exist('outColor', 'var') || isempty(outColor) 11 | outColor = true; 12 | end 13 | if ~exist('filtSize', 'var') || isempty(filtSize) 14 | filtSize = 1; 15 | end 16 | if filtSize <= 3 17 | filtSize = 1; 18 | elseif filtSize <= 5 19 | filtSize = 4; 20 | else 21 | filtSize = 6; 22 | end 23 | image = im2double(image); 24 | 25 | % Set up one-dimensional filters 26 | switch filtSize 27 | case 1 28 | dk = [0.425287, -0.0000, -0.425287]; 29 | kk = [0.229879, 0.540242, 0.229879]; 30 | case 4 31 | dk = [0.0032, 0.0350, 0.1190, 0.1458, -0.0000, -0.1458, -0.1190, -0.0350, -0.0032]; 32 | kk = [0.0009, 0.0151, 0.0890, 0.2349, 0.3201, 0.2349, 0.0890, 0.0151, 0.0009]; 33 | case 6 34 | dk = [0.0001, 0.0019, 0.0142, 0.0509, 0.0963, 0.0878, 0.0000, -0.0878, -0.0963, -0.0509, -0.0142, -0.0019, -0.0001]; 35 | kk = [0.0000, 0.0007, 0.0071, 0.0374, 0.1126, 0.2119, 0.2605, 0.2119, 0.1126, 0.0374, 0.0071, 0.0007, 0.0000]; 36 | end 37 | 38 | % Repeat-pad image 39 | leftPad = image(:, 1, :); 40 | rightPad = image(:, end, :); 41 | image = [repmat(leftPad, [1 13 1]), image, repmat(rightPad, [1 13 1])]; 42 | upPad = image(1, :, :); 43 | downPad = image(end, :, :); 44 | image = [repmat(upPad, [13 1 1]); image; repmat(downPad, [13 1 1])]; 45 | 46 | % Compute gradients 47 | yGradient = zeros(size(image)); 48 | xGradient = zeros(size(image)); 49 | for i = 1 : size(image, 3) 50 | yGradient(:,:,i) = conv2(dk, kk, image(:,:,i), 'same'); 51 | xGradient(:,:,i) = conv2(kk, dk, image(:,:,i), 'same'); 52 | end 53 | 54 | % Remove padding 55 | yGradient = yGradient(14 : end - 13, 14 : end - 13, :); 56 | xGradient = xGradient(14 : end - 13, 14 : end - 13, :); 57 | 58 | % Compute pixel-wise L2 norm if no color option is selected 59 | if ~outColor 60 | yGradient = sqrt(sum(yGradient .* yGradient, 3)); 61 | xGradient = sqrt(sum(xGradient .* xGradient, 3)); 62 | end 63 | 64 | % Compute polar-coordinate representation 65 | gradientMagnitude = sqrt(yGradient .* yGradient + xGradient .* xGradient); 66 | gradientOrientation = atan2(xGradient, yGradient); 67 | end -------------------------------------------------------------------------------- /mattingAffinity.m: -------------------------------------------------------------------------------- 1 | 2 | % Matting Affinity 3 | % This function implements the image matting approach described in 4 | % Anat Levin, Dani Lischinski, Yair Weiss, "A Closed Form Solution to 5 | % Natural Image Matting", IEEE TPAMI, 2008. 6 | % All parameters other than image are optional. The output is a sparse 7 | % matrix which has non-zero element for the non-local neighbors of 8 | % the pixels given by binary map inMap. 9 | % - windowRadius defines the size of the window where the local normal 10 | % distributions are estimated. 11 | % - epsilon defines the regularization coefficient used before inverting 12 | % covariance matrices. It should be larger for noisy images. 13 | 14 | function W = mattingAffinity(image, inMap, windowRadius, epsilon) 15 | 16 | if ~exist('windowRadius', 'var') || isempty(windowRadius) 17 | windowRadius = 1; 18 | end 19 | if ~exist('epsilon', 'var') || isempty(epsilon) 20 | epsilon = 1e-7; 21 | end 22 | image = im2double(image); 23 | windowSize = 2 * windowRadius + 1; 24 | neighSize = windowSize^2; 25 | [h, w, c] = size(image); 26 | N = h * w; 27 | epsilon = epsilon / neighSize; 28 | 29 | % No need to compute affinities in known regions if a trimap is defined 30 | if nargin < 2 || isempty(inMap) 31 | inMap = true(size(image, 1), size(image, 2)); 32 | end 33 | 34 | [meanImage, covarMat] = localRGBnormalDistributions(image, windowRadius, epsilon); 35 | 36 | % Determine pixels and their local neighbors 37 | indices = reshape((1 : h * w), [h w]); 38 | neighInd = im2col(indices, [windowSize windowSize], 'sliding')'; 39 | inMap = inMap(windowRadius + 1 : end - windowRadius, windowRadius + 1 : end - windowRadius); 40 | neighInd = neighInd(inMap, :); 41 | inInd = neighInd(:, (neighSize + 1) / 2); 42 | pixCnt = size(inInd, 1); 43 | 44 | % Prepare in & out data 45 | image = reshape(image, [N, c]); 46 | meanImage = reshape(meanImage, [N, c]); 47 | flowRows = zeros(neighSize, neighSize, pixCnt); 48 | flowCols = zeros(neighSize, neighSize, pixCnt); 49 | flows = zeros(neighSize, neighSize, pixCnt); 50 | 51 | % Compute matting affinity 52 | for i = 1 : size(inInd, 1) 53 | neighs = neighInd(i, :); 54 | shiftedWinColors = image(neighs, :) - repmat(meanImage(inInd(i), :), [size(neighs, 2), 1]); 55 | flows(:, :, i) = shiftedWinColors * (covarMat(:, :, inInd(i)) \ shiftedWinColors'); 56 | neighs = repmat(neighs, [size(neighs, 2), 1]); 57 | flowRows(:, :, i) = neighs; 58 | flowCols(:, :, i) = neighs'; 59 | end 60 | flows = (flows + 1) / neighSize; 61 | 62 | W = sparse(flowRows(:), flowCols(:), flows(:), N, N); 63 | % Make sure it's symmetric 64 | W = (W + W') / 2; 65 | end 66 | 67 | 68 | function [meanImage, covarMat] = localRGBnormalDistributions(image, windowRadius, epsilon) 69 | 70 | if ~exist('windowRadius', 'var') || isempty(windowRadius) 71 | windowRadius = 1; 72 | end 73 | if ~exist('epsilon', 'var') || isempty(epsilon) 74 | epsilon = 1e-8; 75 | end 76 | 77 | [h, w, ~] = size(image); 78 | N = h * w; 79 | windowSize = 2 * windowRadius + 1; 80 | 81 | meanImage = imboxfilt(image, windowSize); 82 | covarMat = zeros(3, 3, N); 83 | 84 | for r = 1 : 3 85 | for c = r : 3 86 | temp = imboxfilt(image(:, :, r).*image(:, :, c), windowSize) - meanImage(:,:,r) .* meanImage(:,:,c); 87 | covarMat(r, c, :) = temp(:); 88 | end 89 | end 90 | 91 | for i = 1 : 3 92 | covarMat(i, i, :) = covarMat(i, i, :) + epsilon; 93 | end 94 | 95 | for r = 2 : 3 96 | for c = 1 : r - 1 97 | covarMat(r, c, :) = covarMat(c, r, :); 98 | end 99 | end 100 | 101 | end -------------------------------------------------------------------------------- /preprocessFeatures.m: -------------------------------------------------------------------------------- 1 | 2 | % Pre-processing hyper-dimensional semantic feature vectors 3 | % as described in Section 3.5. 4 | 5 | function simp = preprocessFeatures(features, image) 6 | % Filter out super high numbers due to some instability in the network 7 | features(features < -5) = -5; 8 | features(features > 5) = 5; 9 | 10 | % Filter each channel of features with image as the guide 11 | if exist('image', 'var') && ~isempty(image) 12 | fd = size(features, 3); 13 | maxfd = fd - rem(fd, 3); 14 | for i = 1 : 3 : maxfd 15 | features(:, :, i : i+2) = imguidedfilter(features(:, :, i : i+2), image, 'NeighborhoodSize', 10); 16 | end 17 | for i = maxfd + 1 : fd 18 | features(:, :, i) = imguidedfilter(features(:, :, i), image, 'NeighborhoodSize', 10); 19 | end 20 | end 21 | 22 | % Run PCA and normalize to [0, 1] 23 | simp = featuresPCA(features, 3); 24 | for i = 1 : 3 25 | simp(:,:,i) = simp(:,:,i) - min(min(simp(:,:,i))); 26 | simp(:,:,i) = simp(:,:,i) / max(max(simp(:,:,i))); 27 | end 28 | end 29 | 30 | function pcafeat = featuresPCA(features, dim) 31 | features = double(features); 32 | [h, w, d] = size(features); 33 | features = reshape(features, [h*w, d]); 34 | featmean = mean(features, 1); 35 | features = features - ones(h*w, 1) * featmean; 36 | covar = features' * features; 37 | [eigvecs, ~] = eigs(covar, dim, 'LA'); 38 | pcafeat = features * eigvecs; 39 | pcafeat = reshape(pcafeat, [h, w, dim]); 40 | end 41 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Semantic Soft Segmentation 2 | ================================================================= 3 | 4 | This repository includes the spectral segmentation approach presented in 5 | 6 | Yagiz Aksoy, Tae-Hyun Oh, Sylvain Paris, Marc Pollefeys and Wojciech Matusik, "Semantic Soft Segmentation", ACM Transactions on Graphics (Proc. SIGGRAPH), 2018 7 | 8 | The network for semantic feature generation can be found [[here](https://github.com/iyah4888/SIGGRAPH18SSS)]. 9 | 10 | Please refer to the [[project page](http://people.inf.ethz.ch/aksoyy/sss/)] for more information and example data. 11 | 12 | The Superpixels class requires [[ImageGraphs](http://www.mathworks.com/matlabcentral/fileexchange/53614-image-graphs)]. 13 | 14 | License and citation 15 | ------------ 16 | 17 | This toolbox is provided for academic use only. 18 | If you use this code, please cite our paper: 19 | 20 | @ARTICLE{sss, 21 | author={Ya\u{g}{\i}z Aksoy and Tae-Hyun Oh and Sylvain Paris and Marc Pollefeys and Wojciech Matusik}, 22 | title={Semantic Soft Segmentation}, 23 | journal={ACM Transactions on Graphics (Proc. SIGGRAPH)}, 24 | year={2018}, 25 | pages = {72:1-72:13}, 26 | volume = {37}, 27 | number = {4} 28 | } 29 | 30 | Credit 31 | ------------ 32 | Parts of this implementation are taken from 33 | 34 | Anat Levin, Alex Rav-Acha, Dani Lischinski, "Spectral Matting", IEEE TPAMI, 2008 35 | 36 | The original source code for Spectral Matting can be found [[here](http://www.vision.huji.ac.il/SpectralMatting/)]. -------------------------------------------------------------------------------- /replicability_instructions.htm: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 33 | 34 | 36 | 649 | 1117 | 1147 | 1148 | 1149 | 1150 | 1151 |
1152 | 1153 |

This code was evaluated for replicability by an independent 1161 | team of experts (see https://replicability.graphics/ ).

1163 | 1164 |

The remainder of this file contains the evaluation report, along with suggestions to successfully 1171 | compile and run the code.

1172 | 1173 |

Information

1175 | 1176 | 1256 | 1257 |

Source code 1261 | information

1262 | 1263 | 1298 | 1299 |

Comments

1302 | 1303 | 
The code is split in two parts, in two different projects. The second project is trivial to run in matlab without hassle, takes an input image that consists in standard RGB colors for the first half, and the second half should contain features. 
The first project of the code, which is supposed to generate features is much more difficult to run. It depends on TensorFlow 1.4 (while the readme indicates TensorFlow >= 1.4, it in fact only works with TensorFlow = 1.4). This old TensorFlow does not support Python 3.7, so I had to remove my 3.7 to install a 3.6 (which cannot be installed via Anaconda -- Anaconda spent an entire night trying to downgrade 3.7 to 3.6 but this didn't work). Finally, when the code runs, it outputs a matlab file which contains 128 features. This file should be processed with preprocessFeatures, along with the original image, in the second project.
The process is not documented, but can be understood from the context. In general, while it was relatively painful to run, results seem to be reproducible. It would however have been much less painful if I had a Python 3.6 already installed.
The lower replicability score is explained by the fact no code is provided for training and only the pre-trained model is given.
1362 | 1363 |

 

1364 | 1365 |
1366 | 1367 | 1368 | 1369 | 1370 | -------------------------------------------------------------------------------- /softSegmentsFromEigs.m: -------------------------------------------------------------------------------- 1 | 2 | % This function implements the constrained sparsification descibed in Section 3.4. 3 | % This methodology is intriduced by Levin et al. in "Spectral Matting", and 4 | % this function is an adapted version of their original source code. Find it here: 5 | % http://www.vision.huji.ac.il/SpectralMatting/ 6 | 7 | function [softSegments, initialSegments] = softSegmentsFromEigs(eigVecs, eigVals, Laplacian, h, w, features, compCnt, maxIter, sparsityParam, imageGrad, initialSegments) 8 | if (~exist('maxIter','var') || isempty(maxIter)) 9 | maxIter = 20; 10 | end 11 | if (~exist('sparsityParam','var') || isempty(sparsityParam)) 12 | sparsityParam = 0.9; 13 | end 14 | if (~exist('compCnt','var') || isempty(compCnt)) 15 | compCnt = 10; 16 | end 17 | eigValCnt = size(eigVecs, 2); 18 | if (~exist('eigVals','var') || isempty(eigVals)) 19 | eigVals = 1e-5*eye(eigValCnt); 20 | end 21 | 22 | % Initialize with k-means if an initialization (initialSegments) isn't provided 23 | if ~exist('initialSegments', 'var') || isempty(initialSegments) 24 | % Use the features for the k-means, if provided 25 | if ~(~exist('features', 'var') || isempty(features)) 26 | disp(' Computing k-means initialization using semantic features...'); 27 | %features = featuresPCA(features, 10); 28 | features = reshape(features, [size(features, 1) * size(features, 2), size(features, 3)]); 29 | [initialSegments, ~, ~, ~] = fastKmeans(features, compCnt); 30 | else 31 | % This is the default initialization from Spectral Matting 32 | disp(' Computing k-means initialization...'); 33 | eigVals = abs(eigVals); % Sometimes there are -epsilon numbers that mess up the thing 34 | initEigsCnt = 20; 35 | initEigsWeights = diag(1 ./ diag(eigVals(2 : initEigsCnt + 1, 2 : initEigsCnt + 1) .^ 0.5 )); 36 | initEigVecs = eigVecs(:, 2 : initEigsCnt + 1) * initEigsWeights; 37 | [initialSegments, ~, ~, ~] = fastKmeans(initEigVecs, compCnt); 38 | end 39 | end 40 | softSegments = zeros(length(initialSegments), compCnt); 41 | for i = 1 : compCnt 42 | softSegments(:, i) = double(initialSegments == i); 43 | end 44 | 45 | if ~exist('imageGrad', 'var') || isempty(imageGrad) 46 | spMat = sparsityParam; 47 | else 48 | imageGrad(imageGrad > 0.2) = 0.2; 49 | imageGrad = imageGrad + 0.8; 50 | spMat = repmat(imageGrad(:), [1, compCnt]); 51 | end 52 | 53 | % Second derivative of first and second terms in the sparsity penalty 54 | disp(' Starting optimization...'); 55 | thr_e = 1e-10; 56 | w1 = 0.3; 57 | w0 = 0.3; 58 | e1 = w1 .^ sparsityParam * max(abs(softSegments-1), thr_e) .^ (spMat - 2); 59 | e0 = w0 .^ sparsityParam * max(abs(softSegments), thr_e) .^ (spMat - 2); 60 | 61 | scld = 1; 62 | eig_vectors = eigVecs(:, 1 : eigValCnt); 63 | eig_values = eigVals(1 : eigValCnt, 1 : eigValCnt); 64 | 65 | % First iter no for removing zero components 66 | removeIter = ceil(maxIter / 4); 67 | removeIterCycle = ceil(maxIter / 4); 68 | 69 | % Compute matting component with sparsity prior 70 | for iter = 1 : maxIter 71 | if rem(iter, 10) == 0 72 | disp([' Iteration ' int2str(iter) ' of ' int2str(maxIter)]); 73 | end 74 | % Construct the matrices in Eq(9) in Spectral Matting 75 | tA = zeros((compCnt - 1) * eigValCnt); 76 | tb = zeros((compCnt - 1) * eigValCnt, 1); 77 | for k = 1 : compCnt - 1 78 | weighted_eigs = repmat(e1(:, k) + e0(:, k), 1, eigValCnt) .* eig_vectors; 79 | tA((k-1) * eigValCnt + 1 : k * eigValCnt, (k-1) * eigValCnt + 1 : k * eigValCnt) = eig_vectors' * weighted_eigs + scld * eig_values; 80 | tb((k-1) * eigValCnt + 1 : k * eigValCnt) = eig_vectors' * e1(:,k); 81 | end 82 | k = compCnt; 83 | weighted_eigs = repmat(e1(:, k) + e0(:, k), 1, eigValCnt) .* eig_vectors; 84 | ttA = eig_vectors' * weighted_eigs + scld * eig_values; 85 | ttb = eig_vectors' * e0(:, k) + scld * sum(eig_vectors' * Laplacian, 2); 86 | 87 | tA = tA + repmat(ttA, [compCnt - 1, compCnt - 1]); 88 | tb = tb + repmat(ttb, [compCnt - 1, 1]); 89 | 90 | % Solve for weights 91 | y = reshape(tA \ tb, eigValCnt, compCnt - 1); 92 | 93 | % Compute the matting comps from weights 94 | softSegments = eigVecs(:, 1 : eigValCnt) * y; 95 | softSegments(:, compCnt) = 1 - sum(softSegments(:, 1 : compCnt - 1), 2); % Sets the last one as 1-sum(others), guaranteeing \sum(all) = 1 96 | 97 | % Remove matting components which are close to zero every once in a while 98 | if iter > removeIter 99 | nzii = find(max(abs(softSegments)) > 0.1); 100 | compCnt = length(nzii); 101 | softSegments = softSegments(:, nzii); 102 | removeIter = removeIter + removeIterCycle; 103 | end 104 | 105 | % Recompute the derivatives of sparsity penalties 106 | if length(spMat(:)) == 1 107 | e1 = w1 .^ sparsityParam * max(abs(softSegments-1), thr_e) .^ (spMat - 2); 108 | e0 = w0 .^ sparsityParam * max(abs(softSegments), thr_e) .^ (spMat - 2); 109 | else 110 | e1 = w1 .^ sparsityParam * max(abs(softSegments-1), thr_e) .^ (spMat(:, 1 : size(softSegments, 2)) - 2); 111 | e0 = w0 .^ sparsityParam * max(abs(softSegments), thr_e) .^ (spMat(:, 1 : size(softSegments, 2)) - 2); 112 | end 113 | end 114 | softSegments = reshape(softSegments, [h, w, size(softSegments, 2)]); 115 | end 116 | 117 | function [idx, C, sumd, D]=fastKmeans(X,K) 118 | % X: points in the N-by-P data matrix 119 | % idx - an N-by-1 vector containing the cluster indices of each point 120 | % 121 | % c - the K cluster centroid locations in a K-by-P matrix. 122 | % sumd - the within-cluster sums of point-to-centroid distances in a 1-by-K vector. 123 | % distances from each point to every centroid in a N-by-K. 124 | 125 | startK = 5; 126 | startK = min(K,startK); 127 | maxIters = 100; % Defualt of matlab is 100 128 | 129 | X = sign(real(X)) .* abs(X); 130 | 131 | [idx, C, sumd, D]=kmeans(X,startK,'EmptyAction','singleton','Start','cluster', ... 132 | 'Maxiter', maxIters, 'Replicates', 7); 133 | 134 | valid_vec = zeros(1,startK); 135 | scr_vec = zeros(1,startK)-1; 136 | 137 | for compCnt = startK+1:K 138 | % create a new cluster by splitting each cluster to two... 139 | max_scr=-1; 140 | clear min_C; 141 | for cl = 1:compCnt-1 142 | cl_mask = idx == cl; 143 | cl_idxs = find(cl_mask); 144 | clX = X(cl_idxs,:); 145 | if (size(clX,1)> 2*size(clX,2)) 146 | if (valid_vec(cl) == 0) 147 | [tmp_idx, tmp_C, ~, tmp_D]=kmeans(clX,2,'EmptyAction','singleton','Start','cluster', 'Maxiter', maxIters); 148 | % chk how much the partition helps ... 149 | scr=sum(min(D(cl_idxs,:),[],2))-sum(min(tmp_D,[],2)); 150 | scr_vec(cl) = scr; 151 | else % we already saved it... 152 | scr = scr_vec(cl); 153 | end 154 | else 155 | scr=-2; 156 | scr_vec(cl) = scr; 157 | end 158 | if (scr > max_scr) 159 | if (valid_vec(cl)==1) % not for the scr. Just for the idxs. 160 | [tmp_idx, tmp_C, ~, ~]=kmeans(clX,2,'EmptyAction','singleton','Start','cluster', 'Maxiter', maxIters); 161 | end 162 | 163 | max_scr = scr; 164 | bestC = [C;tmp_C(2,:)]; 165 | bestC(cl,:) = tmp_C(1,:); 166 | best_cl = cl; 167 | 168 | best_idx = idx; 169 | best_idx(cl_idxs) = (tmp_idx == 1)*best_cl + (tmp_idx == 2)*compCnt; 170 | end 171 | valid_vec(cl) = 1; 172 | end 173 | C = bestC; 174 | idx = best_idx; 175 | 176 | valid_vec = [valid_vec, 0]; % the two new clusers are new, so their 177 | valid_vec(best_cl) = 0; % score have not been computed yet. 178 | scr_vec = [scr_vec, -1]; 179 | scr_vec(best_cl) = -1; 180 | 181 | if (compCnt < 13) 182 | [idx, C, sumd, D]=kmeans(X, compCnt, 'EmptyAction', 'singleton', 'Start', C, 'Maxiter', maxIters); 183 | valid_vec = zeros(1,compCnt); 184 | end 185 | end 186 | 187 | end -------------------------------------------------------------------------------- /sparsifySegments.m: -------------------------------------------------------------------------------- 1 | 2 | % This function implements the relaxed sparsification descibed in Section 3.4 3 | 4 | function softSegments = sparsifySegments(softSegments, Laplacian, imageGrad) 5 | 6 | sigmaS = 1; % sparsity 7 | sigmaF = 1; % fidelity 8 | delta = 100; % constraint 9 | [h, w, compCnt] = size(softSegments); 10 | N = h * w * compCnt; 11 | 12 | if ~exist('imageGrad', 'var') || isempty(imageGrad) 13 | % If image gradient is not provided, set the param to the default 0.9 14 | spPow = 0.90; 15 | else 16 | % Compute the per-pixel sparsity parameter from the gradient 17 | imageGrad(imageGrad > 0.1) = 0.1; 18 | imageGrad = imageGrad + 0.9; 19 | spPow = repmat(imageGrad(:), [compCnt, 1]); 20 | end 21 | 22 | % Iter count for pcg and main optimization 23 | itersBetweenUpdate = 100; 24 | highLevelIters = 20; 25 | 26 | % Get rid of very low/high alpha values and normalize 27 | softSegments(softSegments < 0.1) = 0; 28 | softSegments(softSegments > 0.9) = 1; 29 | softSegments = softSegments ./ repmat(sum(softSegments, 3), [1 1 size(softSegments, 3)]); 30 | 31 | % Construct the linear system 32 | lap = Laplacian; 33 | for i = 2 : compCnt 34 | Laplacian = blkdiag(Laplacian, lap); 35 | end 36 | 37 | % The alpha constraint 38 | C = repmat(speye(h*w), [1 compCnt]); 39 | C = C' * C; 40 | Laplacian = Laplacian + delta * C; 41 | 42 | % The sparsification optimization 43 | softSegments = softSegments(:); 44 | compInit = softSegments; % needed for fidelity energy 45 | for iter = 1 : highLevelIters 46 | if rem(iter, 5) == 0 47 | disp([' Iteration ' int2str(iter) ' of ' int2str(highLevelIters)]); 48 | end 49 | [u, v] = getUandV(softSegments, spPow); % The sparsity energy 50 | A = Laplacian + sigmaS * (spdiags(u, 0, N, N) + spdiags(v, 0, N, N)) + sigmaF * speye(N); 51 | b = sigmaS * v + sigmaF * compInit + delta; 52 | [softSegments, ~] = pcg(A, b, [], itersBetweenUpdate, [], [], softSegments); 53 | end 54 | 55 | % One final iter for good times (everything but sparsity) 56 | A = Laplacian + sigmaF * speye(N); 57 | b = sigmaF * softSegments + delta; 58 | softSegments = pcg(A, b, [], 10 * itersBetweenUpdate, [], [], softSegments); 59 | 60 | % Ta-dah 61 | softSegments = reshape(softSegments, [h w compCnt]); 62 | end 63 | 64 | function [u, v] = getUandV(comp, spPow) 65 | % Sparsity terms in the energy 66 | eps = 1e-2; 67 | u = max(abs(comp(:)), eps) .^ (spPow - 2); 68 | v = max(abs(1 - comp(:)), eps) .^ (spPow - 2); 69 | end -------------------------------------------------------------------------------- /visualizeEigenvectorRedGreen.m: -------------------------------------------------------------------------------- 1 | 2 | function vis = visualizeEigenvectorRedGreen(eigenvector) 3 | % negative values are shown in red, and positive in green 4 | vis = zeros(size(eigenvector, 1), size(eigenvector, 2), 3); 5 | vis(:,:,1) = -100 * eigenvector; 6 | vis(:,:,2) = 100 * eigenvector; 7 | vis(vis < 0) = 0; 8 | vis(vis > 1) = 1; 9 | end -------------------------------------------------------------------------------- /visualizeSoftSegments.m: -------------------------------------------------------------------------------- 1 | 2 | % Assigns a distinct solid color to eact soft segment and composites 3 | % them using the corresponding alpha calues 4 | 5 | function [vis, softSegments] = visualizeSoftSegments(softSegments, doOrdering) 6 | if ~exist('doOrdering', 'var') || isempty(doOrdering) 7 | doOrdering = false; 8 | end 9 | if doOrdering 10 | % Order layers w.r.t. sum(alpha(:)) -- makes visualizations more consistent across images 11 | order = determineAlphaOrder(softSegments); 12 | softSegments = softSegments(:, :, order); 13 | end 14 | % A hard-coded set of 'distinct' colors 15 | colors = createPalette(); 16 | % One solid color per segment, mixed w.r.t. alpha values 17 | vis = repmat(softSegments(:,:,1), [1 1 3]) .* repmat(colors(1,1,:), [size(softSegments, 1), size(softSegments, 2), 1]); 18 | for i = 2 : size(softSegments, 3) 19 | vis = vis + repmat(softSegments(:,:,i), [1 1 3]) .* repmat(colors(i,1,:), [size(softSegments, 1), size(softSegments, 2), 1]); 20 | end 21 | end 22 | 23 | function colors = createPalette() 24 | % https://graphicdesign.stackexchange.com/questions/3682/where-can-i-find-a-large-palette-set-of-contrasting-colors-for-coloring-many-d 25 | % From P. Green-Armytage (2010): A Colour Alphabet and the Limits of Colour Coding. 26 | % Colour: Design & Creativity (5) (2010): 10, 1-23 27 | % http://eleanormaclure.files.wordpress.com/2011/03/colour-coding.pdf 28 | colors = reshape(... 29 | [0,117,220; 255,255,128; 43,206,72; 153,0,0; 128,128,128; 240,163,255; 153,63,0; 76,0,92;... 30 | 0,92,49; 255,204,153; 148,255,181;... 31 | 143,124,0; 157,204,0; 194,0,136; 0,51,128; 255,164,5;... 32 | 255,168,187; 66,102,0; 255,0,16; 94,241,242; 0,153,143;... 33 | 224,255,102; 116,10,255; 255,255,0; 255,80,5; 25,25,25]... 34 | , [26, 1, 3]) / 255; 35 | colors = [colors; colors]; 36 | end 37 | 38 | function order = determineAlphaOrder(alphas) 39 | alphas = sum(alphas, 1); 40 | alphas = sum(alphas, 2); 41 | alphas = reshape(alphas, [size(alphas, 3) 1]); 42 | [~, order] = sort(alphas, 'descend'); 43 | end --------------------------------------------------------------------------------