├── LICENSE ├── PyKernelCut ├── 0_5_5303.bmp ├── 124084.jpg ├── 130066.bmp ├── 314016.jpg ├── README.md ├── kernelcut.py ├── requirements.txt ├── segmentation.py └── segutil.py ├── README.md ├── kernelcut ├── Image.h ├── PPBCBase.h ├── PPBCncut.h ├── SparseMatrix.h ├── basicutil.h ├── ezi │ ├── Basics2D.h │ ├── Image2D.h │ ├── Image2D.template │ ├── Table2D.h │ ├── Table2D.template │ └── myassert.h ├── knn.h ├── main.cpp ├── makefile ├── ncutknnbinary.h └── ncutknnmulti.h ├── libs ├── Bk_matlab │ ├── BK_AddVars.m │ ├── BK_BuildLib.m │ ├── BK_Create.m │ ├── BK_Delete.m │ ├── BK_GetLabeling.m │ ├── BK_ListHandles.m │ ├── BK_LoadLib.m │ ├── BK_Minimize.m │ ├── BK_SetNeighbors.m │ ├── BK_SetPairwise.m │ ├── BK_SetUnary.m │ ├── BK_UnitTest.m │ ├── bin │ │ └── bk_matlab.mexa64 │ ├── bk_matlab.cpp │ ├── block.h │ ├── energy.h │ ├── graph.cpp │ ├── graph.h │ └── maxflow.cpp ├── EasyBMP │ ├── BSD_(revised)_license.txt │ ├── EasyBMP.cpp │ ├── EasyBMP.h │ ├── EasyBMP_BMP.h │ ├── EasyBMP_ChangeLog.txt │ ├── EasyBMP_DataStructures.h │ ├── EasyBMP_UserManual.pdf │ ├── EasyBMP_VariousBMPutilities.h │ └── sample │ │ ├── EasyBMPbackground.bmp │ │ ├── EasyBMPsample.cpp │ │ ├── EasyBMPtext.bmp │ │ └── makefile ├── flow-code-matlab │ ├── README.txt │ ├── colorTest.m │ ├── computeColor.m │ ├── flowToColor.m │ ├── readFlowFile.m │ └── writeFlowFile.m ├── makefile └── maxflow │ ├── block.h │ ├── graph.cpp │ ├── graph.h │ ├── instances.inc │ └── maxflow.cpp ├── matlab ├── AverageAssociationEnergy.m ├── KernelBound.m ├── NC_clustering.png ├── NC_energy.png ├── NC_init.png ├── NormalizedCutEnergy.m ├── SpectralBound.m ├── SpectralEmbedding.m ├── genringkernelandinit.m └── syntheticclustering.m └── motionsegmentation ├── ducks01 ├── computeknn.m ├── computeopticalflow.m ├── getsubpixelimages.m ├── images │ ├── ducks01_0300.bmp │ ├── ducks01_0301.bmp │ ├── ducks01_0302.bmp │ ├── ducks01_0303.bmp │ ├── ducks01_0304.bmp │ ├── ducks01_0305.bmp │ ├── ducks01_0306.bmp │ ├── ducks01_0307.bmp │ ├── ducks01_0308.bmp │ ├── ducks01_0309.bmp │ ├── ducks01_0310.bmp │ ├── ducks01_0311.bmp │ ├── ducks01_0312.bmp │ ├── ducks01_0313.bmp │ ├── ducks01_0314.bmp │ ├── ducks01_0315.bmp │ ├── ducks01_0316.bmp │ ├── ducks01_0317.bmp │ ├── ducks01_0318.bmp │ ├── ducks01_0319.bmp │ ├── ducks01_0320.bmp │ ├── ducks01_0321.bmp │ ├── ducks01_0322.bmp │ ├── ducks01_0323.bmp │ ├── ducks01_0324.bmp │ ├── ducks01_0325.bmp │ ├── ducks01_0326.bmp │ ├── ducks01_0327.bmp │ ├── ducks01_0328.bmp │ ├── ducks01_0329.bmp │ ├── ducks01_0330.bmp │ ├── ducks01_0331.bmp │ ├── ducks01_0332.bmp │ ├── ducks01_0333.bmp │ ├── ducks01_0334.bmp │ ├── ducks01_0335.bmp │ ├── ducks01_0336.bmp │ ├── ducks01_0337.bmp │ ├── ducks01_0338.bmp │ ├── ducks01_0339.bmp │ ├── ducks01_0340.bmp │ └── ducks01_0341.bmp ├── output │ └── ducks01_0300_ncutknnmulti_s0.5.bmp ├── seedsmulti │ └── ducks01_0300.bmp └── showldof.m ├── horses01 ├── computeknn.m ├── computeopticalflow.m ├── getsubpixelimages.m ├── images │ ├── horses01_0233.bmp │ ├── horses01_0234.bmp │ ├── horses01_0235.bmp │ ├── horses01_0236.bmp │ ├── horses01_0237.bmp │ ├── horses01_0238.bmp │ ├── horses01_0239.bmp │ ├── horses01_0240.bmp │ ├── horses01_0241.bmp │ ├── horses01_0242.bmp │ ├── horses01_0243.bmp │ ├── horses01_0244.bmp │ ├── horses01_0245.bmp │ ├── horses01_0246.bmp │ ├── horses01_0247.bmp │ ├── horses01_0248.bmp │ ├── horses01_0249.bmp │ ├── horses01_0250.bmp │ ├── horses01_0251.bmp │ ├── horses01_0252.bmp │ ├── horses01_0253.bmp │ ├── horses01_0254.bmp │ ├── horses01_0255.bmp │ ├── horses01_0256.bmp │ ├── horses01_0257.bmp │ ├── horses01_0258.bmp │ ├── horses01_0259.bmp │ ├── horses01_0260.bmp │ ├── horses01_0261.bmp │ ├── horses01_0262.bmp │ ├── horses01_0263.bmp │ ├── horses01_0264.bmp │ ├── horses01_0265.bmp │ ├── horses01_0266.bmp │ ├── horses01_0267.bmp │ ├── horses01_0268.bmp │ ├── horses01_0269.bmp │ ├── horses01_0270.bmp │ ├── horses01_0271.bmp │ ├── horses01_0272.bmp │ └── horses01_0273.bmp ├── output │ └── horses01_0241_ncutknnbinary_s0.5.bmp ├── seeds │ └── horses01_0233.bmp └── showldof.m ├── motion_ducks01.sh ├── motion_horses01.sh ├── mycallback.m └── visualizeknnbyclick.m /LICENSE: -------------------------------------------------------------------------------- 1 | IMPORTANT: 2 | To use this software, you MUST cite the following in any resulting publication: 3 | 4 | Normalized Cut Meets MRF. 5 | Meng Tang, Dmitrii Marin, Ismail Ben Ayed, Yuri Boykov 6 | In European Conference on Computer Vision (ECCV), Amsterdam, the Netherlands, October, 2016 7 | 8 | COPYRIGHT, LICENSE & DISCLAIMER 9 | 10 | Copyright 2016-2017 Meng Tang 11 | Dmitrii Marin 12 | 13 | This software and its modifications can be used and distributed for 14 | research purposes only. Publications resulting from use of this code 15 | must cite publications according to the rules given above. Only 16 | Meng Tang and Dmitrii Marin can redistribute this code, unless expressed 17 | permission is given otherwise. Commercial use of this code, any of 18 | its parts, or its modifications is not permited. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /PyKernelCut/0_5_5303.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meng-tang/KernelCut/71cdae17285b9a74ae09b2a5f552debaf92dcd23/PyKernelCut/0_5_5303.bmp -------------------------------------------------------------------------------- /PyKernelCut/124084.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meng-tang/KernelCut/71cdae17285b9a74ae09b2a5f552debaf92dcd23/PyKernelCut/124084.jpg -------------------------------------------------------------------------------- /PyKernelCut/130066.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meng-tang/KernelCut/71cdae17285b9a74ae09b2a5f552debaf92dcd23/PyKernelCut/130066.bmp -------------------------------------------------------------------------------- /PyKernelCut/314016.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meng-tang/KernelCut/71cdae17285b9a74ae09b2a5f552debaf92dcd23/PyKernelCut/314016.jpg -------------------------------------------------------------------------------- /PyKernelCut/README.md: -------------------------------------------------------------------------------- 1 | ## KernelCut in Python ## 2 | Python implementation of KernelCut for binary segmentation is provided. 3 | Example Usage: 4 | ```{r, engine='bash'} 5 | $ python segmentation.py -h 6 | Usage: segmentation.py [options] 7 | 8 | Options: 9 | -h, --help show this help message and exit 10 | -i IMAGENAME, --image=IMAGENAME 11 | name of the image, can also be an URL 12 | -d IMAGEDIR, --dir=IMAGEDIR 13 | directory that contains image, defalt is .py file path 14 | -b BOX, --box=BOX bounding box (top, left, height, width) 15 | --hard enforce hard constraints, default off 16 | -k KNN_K, --knn=KNN_K 17 | K nearest neighbors, defualt 400 18 | -s WEIGHT_SMOOTHNESS, --smoothness=WEIGHT_SMOOTHNESS 19 | weight of smoothness term, defualt 0.0001 20 | --maxitr=MAX_ITERATION 21 | maximum number of iteration, defualt 80 22 | --xyscale=XYSCALE scale of x,y coordinate for KNN graph, defualt 0 23 | 24 | $ python segmentation.py -i 124084.jpg -b 10 20 300 300 -k 100 25 | $ python segmentation.py -i 0_5_5303.bmp -b 50 100 200 200 -s 0 26 | $ python segmentation.py -i 0_5_5303.bmp -b 50 100 200 200 27 | $ python segmentation.py -i 314016.jpg -b 10 80 300 300 --hard 28 | 29 | ``` 30 | Note that To use PyKernelCut several dependencies (skimage, scipy, [PyMaxflow](https://github.com/pmneila/PyMaxflow)) have to be installed. See [requirements.txt](requirements.txt) 31 | -------------------------------------------------------------------------------- /PyKernelCut/kernelcut.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # KERNELCUT - software for image segmentation / graph clustering # 3 | # http://vision.csd.uwo.ca/code # 4 | ################################################################################ 5 | import numpy as np 6 | import time 7 | from scipy import sparse 8 | from skimage.color import rgb2lab 9 | from skimage.util import random_noise 10 | from sklearn.neighbors import * 11 | from segutil import * 12 | from matplotlib import pyplot as plt 13 | import maxflow 14 | 15 | def NormalizedCutEnergy(A, clustering): 16 | if isinstance(A, np.ndarray): 17 | d = np.sum(A, axis=1) 18 | #print "degree vector" 19 | #print d 20 | #print d.shape 21 | #print type(d) 22 | elif isinstance(A, sparse.csr.csr_matrix): 23 | d = A.sum(axis=1) 24 | #print "degree vector" 25 | #print d 26 | #print d.shape 27 | #print type(d) 28 | maxclusterid = np.max(clustering) 29 | #print "max cluster id is: ", maxclusterid 30 | nassoc_e = 0; 31 | num_cluster = 0; 32 | for k in range(maxclusterid+1): 33 | #print k 34 | # binary indicators (N-by-1) for cluster k 35 | S_k = np.array(clustering == k,dtype=np.float) 36 | #print S_k 37 | if 0 == np.sum(clustering==k): 38 | continue # skip empty cluster 39 | num_cluster = num_cluster + 1 40 | if isinstance(A, np.ndarray): 41 | nassoc_e = nassoc_e + np.dot( np.dot(np.transpose(S_k), A) , S_k) / np.dot(np.transpose(d), S_k) 42 | elif isinstance(A, sparse.csr.csr_matrix): 43 | nassoc_e = nassoc_e + np.dot(np.transpose(S_k), A.dot(S_k)) / np.dot(np.transpose(d), S_k) 44 | nassoc_e = nassoc_e[0,0] 45 | #print "number of clusters: ", num_cluster 46 | ncut_e = num_cluster - nassoc_e 47 | return ncut_e 48 | 49 | def KernelBound(A, K, current_clustering): 50 | N = current_clustering.size 51 | unaries = np.zeros((N, K), dtype=np.float) 52 | d = A.sum(axis=1) 53 | for i in range(K): 54 | #print i 55 | S_i = np.array(current_clustering == i, dtype=np.float) 56 | volume_s_i = np.dot(np.transpose(d), S_i) 57 | volume_s_i = volume_s_i[0,0] 58 | #print volume_s_i 59 | temp = np.dot(np.transpose(S_i), A.dot(S_i)) / volume_s_i / volume_s_i 60 | temp = temp * d 61 | #print temp.shape 62 | temp2 = temp + np.reshape( - 2 * A.dot(S_i) / volume_s_i, (N,1)) 63 | #print type(temp2) 64 | unaries[:,i] = temp2.flatten() 65 | return unaries 66 | 67 | def ConstructKNNGraph(image, KNN_K=400, xyscale=0, samplerate=5): 68 | print "computing k nearest neighbors..." 69 | (height, width, channel) = image.shape 70 | # construct KNN graph in LAB color space. Find KNN_K neighbors and sample the neighbors 71 | #add gaussian noise to image (Images typically come with quantization artifcat, 72 | #which makes KNN graph not connected if noise not added) 73 | noisyimage = random_noise(image,mode='gaussian',var=0.0002); 74 | labimage = rgb2lab(noisyimage) 75 | # normalize so that each channel has unit variance 76 | labimage[:,:,0] /= np.std(labimage[:,:,0]) 77 | labimage[:,:,1] /= np.std(labimage[:,:,1]) 78 | labimage[:,:,2] /= np.std(labimage[:,:,2]) 79 | # compute K nearest neighbor 80 | X = np.reshape(labimage,(height * width, channel)) 81 | if xyscale > 1e-100: 82 | print "KNN over color + XY feature" 83 | rgbxy = np.zeros((height*width, 5), dtype=np.float) 84 | rgbxy[:,0:3] = X 85 | rgbxy[:,3] = np.repeat(range(height),width) * xyscale 86 | rgbxy[:,4] = np.tile(range(width),height).flatten() * xyscale 87 | X = np.copy(rgbxy) 88 | start = time.clock() 89 | tree = KDTree(X, leaf_size=50) 90 | neighbors = tree.query(X, k=KNN_K,return_distance=False) 91 | print time.clock() - start, "seconds process time" 92 | 93 | # sample the neighbors 94 | neighbors = neighbors[:,0::5] 95 | neighbors = perturbknn(neighbors, height, width) 96 | row = np.repeat(range(height*width), KNN_K/samplerate) 97 | col = neighbors.flatten() 98 | data = np.ones(height*width*KNN_K/samplerate, dtype=np.float) 99 | affinity_matrix = sparse.csr_matrix((data, (row, col)), shape=(height*width,height*width),dtype=np.float) 100 | affinity_matrix = (affinity_matrix + affinity_matrix.transpose(copy=True)) /2 101 | return affinity_matrix 102 | 103 | def KernelCut(image, opt): 104 | [height, width, channel] = image.shape 105 | top = opt.box[0] 106 | left = opt.box[1] 107 | boxheight = opt.box[2] 108 | boxwidth = opt.box[3] 109 | connecttype = 8 110 | gridvariance = computeGridVariance(np.asarray(image), connecttype) 111 | #print "Grid variance is: ", gridvariance 112 | 113 | # intial segmentation 114 | initsegmentation = np.zeros([height, width],dtype=int) 115 | initsegmentation[(top-1):(top+boxheight-2),(left-1):(left+boxwidth-2)] = 1 116 | # hard constraints 117 | hardconstraints = np.zeros([height, width], dtype=int) 118 | hardconstraints[(top-1):(top+boxheight-2),(left-1):(left+boxwidth-2)] = -1 119 | print "hardconstraintsflag", opt.hardconstraintsflag 120 | plt.figure(2) 121 | plt.imshow(maskoncolorimage(image, initsegmentation)) 122 | plt.show(block=False) 123 | 124 | affinity_matrix = ConstructKNNGraph(image, opt.KNN_K, opt.xyscale) 125 | 126 | energy = NormalizedCutEnergy(affinity_matrix, initsegmentation.flatten()) 127 | print "normalized cut energy is ", energy 128 | 129 | segmentation = np.copy(initsegmentation) 130 | itr_num = 1 131 | while True: 132 | itr_num = itr_num +1 133 | start = time.time() 134 | g = maxflow.Graph[float](height*width, height*width*4) 135 | nodeids = g.add_grid_nodes((height,width)) 136 | addsmoothness(g, opt.weight_smoothness, image, connecttype, gridvariance) 137 | unaries = KernelBound(affinity_matrix, 2, segmentation.flatten()) 138 | capsource = np.reshape(unaries[:,0],(height,width)) 139 | capsink = np.reshape(unaries[:,1],(height,width)) 140 | # hard constraints? 141 | if opt.hardconstraintsflag: 142 | capsink[hardconstraints==0] = 1e+100 143 | capsource[hardconstraints==1] = 1e+100 144 | 145 | g.add_grid_tedges(nodeids, capsink, capsource) 146 | g.maxflow() 147 | newsegmentation = np.asarray(g.get_grid_segments(nodeids), dtype=np.int) 148 | print "# of pixels with new label", np.sum(newsegmentation.flatten()!=segmentation.flatten()) 149 | if np.sum(newsegmentation.flatten()!=segmentation.flatten()) >10: 150 | segmentation = np.copy(newsegmentation) 151 | energy = NormalizedCutEnergy(affinity_matrix, segmentation.flatten()) 152 | end = time.time() 153 | print "normalized cut energy is ", energy 154 | else: 155 | print "converged" 156 | break 157 | if itr_num>opt.MAX_ITERATION: 158 | print "max number of iteration" 159 | break; 160 | #print end - start 161 | return segmentation 162 | 163 | -------------------------------------------------------------------------------- /PyKernelCut/requirements.txt: -------------------------------------------------------------------------------- 1 | # This file lists the python dependencies, 2 | # it is meant to be used with pip (and/or possibly virtualenv, pbundler, etc) 3 | # See http://pip.readthedocs.org/en/latest/user_guide.html#requirements-files 4 | # You will probably want to run 5 | # something similar to `pip install -r requirements.txt` 6 | # or `pip install --user -r requirements.txt` 7 | 8 | scikit-learn 9 | scikit-image 10 | PyMaxflow 11 | 12 | 13 | -------------------------------------------------------------------------------- /PyKernelCut/segmentation.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # KERNELCUT - software for image segmentation / graph clustering # 3 | # http://vision.csd.uwo.ca/code # 4 | # https://github.com/meng-tang/KernelCut # 5 | # Meng Tang (mtang73@uwo.ca) (contact author) # 6 | # Dmitrii Marin (dmitrii.a.marin@gmail.com) # 7 | # Ismail Ben Ayed (ismail.benayed@etsmtl.ca) # 8 | # Yuri Boykov (yuri@csd.uwo.ca) # 9 | ################################################################################ 10 | #!/usr/bin/python 11 | 12 | import sys, getopt 13 | import os, time 14 | import skimage.io as io 15 | import numpy as np 16 | from optparse import OptionParser 17 | import timeit 18 | 19 | from segutil import * 20 | from kernelcut import * 21 | from scipy import sparse 22 | from matplotlib import pyplot as plt 23 | 24 | 25 | def parsearguments(): 26 | parser = OptionParser() 27 | parser.add_option("-i", "--image", dest="imagename", 28 | help="name of the image, can also be an URL") 29 | parser.add_option("-d", "--dir", dest="imagedir", 30 | help="directory that contains image, defalt is .py file path", type="string", default=os.path.dirname(os.path.realpath(__file__))) 31 | parser.add_option("-b", "--box", dest="box", 32 | help="bounding box (top, left, height, width)",type="int", nargs=4) 33 | parser.add_option("--hard", dest="hardconstraintsflag", 34 | help="enforce hard constraints, default off", action="store_true", default=False) 35 | parser.add_option("-k", "--knn", dest="KNN_K", 36 | help="K nearest neighbors, defualt 400",type="int", default=400) 37 | parser.add_option("-s", "--smoothness", dest="weight_smoothness", 38 | help="weight of smoothness term, default 0.0001", type="float", default=0.0001) 39 | parser.add_option("--maxitr", dest="MAX_ITERATION", 40 | help="maximum number of iteration, default 80", type="int", default=80) 41 | parser.add_option("--xyscale", dest="xyscale", 42 | help="scale of x,y coordinate for KNN graph, default 0", type="float", default=0) 43 | 44 | (opt, args) = parser.parse_args() 45 | print 'image directory is ', opt.imagedir 46 | print 'image name is ', opt.imagename 47 | return opt 48 | 49 | def main(): 50 | opt = parsearguments() 51 | if "http" in opt.imagename: 52 | image = io.imread(opt.imagename) 53 | else: 54 | image = io.imread(os.path.join(opt.imagedir, opt.imagename)) 55 | (height, width, channel) = image.shape 56 | plt.figure(1) 57 | plt.imshow(image) 58 | plt.show(block=False) 59 | 60 | segmentation = KernelCut(image, opt) 61 | plt.figure(3) 62 | plt.imshow(maskoncolorimage(image, segmentation)) 63 | plt.show() 64 | return 65 | 66 | if __name__ == "__main__": 67 | main() 68 | -------------------------------------------------------------------------------- /PyKernelCut/segutil.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # KERNELCUT - software for image segmentation / graph clustering # 3 | # http://vision.csd.uwo.ca/code # 4 | ################################################################################ 5 | #!/usr/bin/python 6 | import numpy as np 7 | from scipy import sparse 8 | 9 | 10 | def addsmoothness(g, weight, image, connecttype, gridvariance): 11 | if weight < 1e-100: 12 | return 13 | pixelshifts = np.asarray([[1,0],[0,1],[1,1],[1,-1],]) 14 | if not (connecttype in [4,8]): 15 | print "Grid connect type should be 4 or 8" 16 | exit(-2) 17 | [h, w, c] = image.shape 18 | for i in range(connecttype/2): 19 | shift_x = pixelshifts[i,0] 20 | shift_y = pixelshifts[i,1] 21 | imageshifted = image[max(0,(0+shift_y)):min(h,(h+shift_y)), 22 | max(0,(0+shift_x)):min(w,(w+shift_x)),:] 23 | nodeids = np.reshape(range(h*w), (h,w)) 24 | if shift_x>0: 25 | imagecrop = image[:, 0:-shift_x, :] 26 | else: 27 | imagecrop = image[:, -shift_x:, :] 28 | if shift_y>0: 29 | imagecrop = imagecrop[0:-shift_y, :, :] 30 | else: 31 | imagecrop = imagecrop[-shift_y:, :, :] 32 | if shift_x>0: 33 | nodeidscrop = nodeids[:, 0:-shift_x] 34 | else: 35 | nodeidscrop = nodeids[:, -shift_x:] 36 | if shift_y>0: 37 | nodeidscrop = nodeidscrop[0:-shift_y, :] 38 | else: 39 | nodeidscrop = nodeidscrop[-shift_y:, :] 40 | diff = imageshifted.astype(float) - imagecrop.astype(float) 41 | diff = diff*diff 42 | edgeweight = np.exp( - np.sum(diff,axis=2) / 2 / gridvariance) / np.sqrt(shift_x*shift_x + shift_y*shift_y) * weight 43 | structure = np.zeros((3,3)) 44 | structure[shift_y+1, shift_x+1] = 1 45 | g.add_grid_edges(nodeidscrop, weights=edgeweight, structure=structure, symmetric=True) 46 | 47 | def maskoncolorimage(colorimage, mask, bkgcolor=[255,255,255]): 48 | maskedimage = np.copy(colorimage) 49 | for i in range(3): 50 | onechannel = maskedimage[:,:,i] 51 | onechannel[mask==0] = bkgcolor[i] 52 | maskedimage[:,:,i] = onechannel 53 | return maskedimage 54 | 55 | def computeGridVariance(image, connecttype): 56 | pixelshifts = np.asarray([[1,0],[0,1],[1,1],[1,-1],]) 57 | if not (connecttype in [4,8]): 58 | print "Grid connect type should be 4 or 8" 59 | exit(-2) 60 | [h, w, c] = image.shape 61 | counter = int(0) 62 | totalvariance = 0 63 | for i in range(connecttype/2): 64 | shift_x = pixelshifts[i,0] 65 | shift_y = pixelshifts[i,1] 66 | imageshifted = image[max(0,(0+shift_y)):min(h,(h+shift_y)), 67 | max(0,(0+shift_x)):min(w,(w+shift_x)),:] 68 | if shift_x>0: 69 | imagecrop = image[:, 0:-shift_x, :] 70 | else: 71 | imagecrop = image[:, -shift_x:, :] 72 | if shift_y>0: 73 | imagecrop = imagecrop[0:-shift_y, :, :] 74 | else: 75 | imagecrop = imagecrop[-shift_y:, :, :] 76 | diff = imageshifted.astype(float) - imagecrop.astype(float) 77 | totalvariance += np.sum(diff*diff) 78 | counter += diff.size / 3 79 | return totalvariance / counter 80 | 81 | # perturb the knn in 'image grid', this is a trick to reach out to large 82 | # neighborhood in the color space without knn search of large K 83 | def perturbknn(neighbors, height, width): 84 | knnidx_c = np.remainder(neighbors,width); 85 | knnidx_r = (neighbors-knnidx_c)/width; 86 | knnidx_r = knnidx_r + np.random.randint(-2, 3, size=neighbors.shape) 87 | knnidx_c = knnidx_c + np.random.randint(-2, 3, size=neighbors.shape) 88 | knnidx_r[knnidx_r<0] = 0 89 | knnidx_r[knnidx_r>=height] = height-1 90 | knnidx_c[knnidx_c<0] = 0 91 | knnidx_c[knnidx_c>=width] = width-1 92 | return knnidx_r * width + knnidx_c 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is the code for the paper: 2 | 3 | "Normalized Cut Meets MRF" 4 | Meng Tang, Dmitrii Marin, Ismail Ben Ayed, Yuri Boykov 5 | In European Conference on Computer Vision (ECCV), Amsterdam, the Netherlands, October, 2016 6 | 7 | The CORE of our algorithm is linearization or unary bound for Normalized Cut (NC).
8 | Simple implementation of such linearization is given in a FEW lines in "matlab/KernelBound.m" and "matlab/SpectralBound.m".
9 | 10 | ```matlab 11 | [ unaries ] = KernelBound( A, K, current_clustering) 12 | % KERNELBOUND simply takes affinity, cluster number and current clustering and gives unary terms. 13 | [ unaries ] = SpectralBound( A, K, dim, current_clustering) 14 | % SPECTRALBOUND requires an extra argument dim, the dimensionality of spectral embedding 15 | % When dim is the number of data points, SpectralBound is algorithmically EQUIVALENT to KernelCut. 16 | ``` 17 | Example of optimizing NC or AA (avearge association) ONLY is in **"matlab/syntheticclustering.m"**. Below is sample result with KernelCut for NC:
18 | 19 | 20 | 21 | 22 | ## Motion Segmentation using KernelCut ## 23 | Multilabel example: Input image frames: directory "motionsegmentation/ducks01/images" 24 | Initial Strokes for the first frame: directory "motionsegmentation/ducks01/seedsmulti" 25 | Binary example: Input image frames: directory "motionsegmentation/horses01/images" 26 | Initial Strokes for the first frame: directory "motionsegmentation/horses01/seeds" 27 | 28 | 29 | 30 |
31 | 32 | 33 | 34 | Build dependency libraries (maxflow and easybmp) 35 | ```{r, engine='bash'} 36 | $ cd libs 37 | $ make all 38 | ``` 39 | Build main program 40 | ```{r, engine='bash'} 41 | $ cd ../kernelcut 42 | $ make main 43 | $ cd ../ 44 | ``` 45 | Download executable for optical flow 46 | ```{r, engine='bash'} 47 | $ wget http://lmb.informatik.uni-freiburg.de/resources/binaries/pami2010Linux64.zip 48 | $ unzip pami2010Linux64.zip -d libs/LDOF 49 | ``` 50 | Compute optical flow, flow and its visualization saved into opticalflow directory 51 | ```{r, engine='bash'} 52 | $ chmod +x libs/LDOF/ldof 53 | $ matlab -nojvm -nosplash -nodisplay -r "cd motionsegmentation/ducks01; computeopticalflow; exit()" 54 | ``` 55 | Compute KNN graph for joint LAB + XY + M space 56 | ```{r, engine='bash'} 57 | $ matlab -nojvm -nosplash -nodisplay -r "cd motionsegmentation/ducks01; getsubpixelimages; exit();" 58 | $ matlab -nojvm -nosplash -nodisplay -r "cd motionsegmentation/ducks01; computeknn; exit()" 59 | ``` 60 | (Visualization of KNN graph is by clicking on image pixel, simply run motionsegmentation/visualizeknnbyclick.m) 61 | 62 | Go to motionsegmentation/motion.sh, change codepath, and run script 63 | ```{r, engine='bash'} 64 | $ chmod +x ./motionsegmentation/motion_ducks01.sh 65 | $ ./motionsegmentation/motion_ducks01.sh 66 | ``` 67 | Output segmentations are in the directory "motionsegmentation/ducks01/output". 68 | 69 | (note that if initialized from seeds, the colors has to be of the following: {white,red,blue,green,black,navy}) 70 | 71 | Binary example "horses01" is similar to multilabel example "ducks01". 72 | 73 | ## KernelCut in Python ## 74 | Python implementation of KernelCut for binary segmentation is provided. See directory 'PyKernelCut'. 75 | Example Usage: 76 | ```{r, engine='bash'} 77 | $ cd PyKernelCut 78 | $ python segmentation.py -h 79 | $ python segmentation.py -i 124084.jpg -b 10 20 300 300 -k 100 80 | $ python segmentation.py -i 0_5_5303.bmp -b 50 100 200 200 -s 0 81 | $ python segmentation.py -i 0_5_5303.bmp -b 50 100 200 200 82 | $ python segmentation.py -i 314016.jpg -b 10 80 300 300 --hard 83 | ``` 84 | Note that To use PyKernelCut several dependencies (skimage, scipy, [PyMaxflow](https://github.com/pmneila/PyMaxflow)) have to be installed. 85 | 86 | ## License & Copyright 87 | See [LICENSE](LICENSE). 88 | -------------------------------------------------------------------------------- /kernelcut/Image.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "ezi/Image2D.h" 3 | #include "ezi/Table2D.h" 4 | #include "SparseMatrix.h" 5 | #include 6 | 7 | 8 | // get average sigma_square for smoothness term 9 | double getsmoothnessterm(const Table2D &img,vector & pointpairs,int connecttype); 10 | void rgb2indeximg(Table2D & indeximg,const Table2D & img,double channelstep); 11 | int getcompactlabel(Table2D & colorlabel,double channelstep,vector & compacthist); 12 | // edge-constrast sensitive smoothness penalty 13 | inline double fn(const double dI, double lambda,double sigma_square); 14 | 15 | class Image{ 16 | public: 17 | Image(); 18 | Image(Table2D img_, double channelstep, const char * imgname_ = "", int connecttype_ = 16); 19 | Image(const char * imgpath, const char * imgname_, double channelstep, int connecttype_); 20 | void computesmoothnesscost(); 21 | void print(); 22 | 23 | void addboxsmooth(Table2D box); 24 | Table2D img; 25 | const char * imgname; 26 | int img_w; 27 | int img_h; 28 | int img_size; 29 | double sigma_square; 30 | vector pointpairs; 31 | vector smoothnesscosts; 32 | int colorbinnum; 33 | Table2D colorlabel; 34 | vector compacthist; // color bin histogram (whole image) 35 | int connecttype; // can be 4 or 8 or 16 36 | 37 | vector > pair_arcs; 38 | }; 39 | 40 | Image::Image() 41 | { 42 | 43 | } 44 | 45 | void Image::computesmoothnesscost() 46 | { 47 | // number of neighboring pairs of pixels 48 | int numNeighbor = pointpairs.size(); 49 | smoothnesscosts = vector(numNeighbor); 50 | for(int i=0;i >(); 57 | for(int i=0;i(p1.x+p1.y*img_w,p2.x+p2.y*img_w,smoothnesscosts[i])); 62 | pair_arcs.push_back(Trituple(p2.x+p2.y*img_w,p1.x+p1.y*img_w,smoothnesscosts[i])); 63 | } 64 | 65 | } 66 | 67 | void Image::addboxsmooth(Table2D box) 68 | { 69 | int numNeighbor = pointpairs.size(); 70 | for(int i=0;i img_, double channelstep, const char * imgname_, int connecttype_) 81 | { 82 | Assert((connecttype_==4)||(connecttype_==8)||(connecttype_==16),"wrong connect type!"); 83 | imgname = imgname_; 84 | img = Table2D(img_); 85 | img_w = img.getWidth(); 86 | img_h = img.getHeight(); 87 | img_size = img_w * img_h; 88 | 89 | connecttype = connecttype_; 90 | sigma_square = getsmoothnessterm(img,pointpairs,connecttype); 91 | //sigma_square = sigma_square*2; 92 | 93 | colorlabel= Table2D(img_w,img_h); 94 | rgb2indeximg(colorlabel,img,channelstep); 95 | colorbinnum = getcompactlabel(colorlabel,channelstep,compacthist); 96 | 97 | computesmoothnesscost(); 98 | 99 | } 100 | Image::Image(const char * imgpath, const char * imgname_, double channelstep, int connecttype_) 101 | { 102 | Table2D img_ = loadImage(imgpath); 103 | new (this)Image(img_, channelstep, imgname_, connecttype_); 104 | } 105 | void Image::print() 106 | { 107 | cout<<"sigma_square: "< 7 | #include 8 | 9 | class BreakPoint{ 10 | public: 11 | bool operator<(const BreakPoint& b) const {return para < b.para;} 12 | double para; 13 | Table2D