├── README ├── bike_a.png ├── bike_b.png ├── main.py └── patchmatch.c /README: -------------------------------------------------------------------------------- 1 | # CLPatchMatch # 2 | 3 | This is a Python and C OpenCL implementation of patch match image processing algorithm that runs in real-time. 4 | 5 | Patch match is a quick approximation algorithm for finding similar areas of one image in another one, developed by @ConnellyBarnes (http://gfx.cs.princeton.edu/pubs/Barnes_2009_PAR/index.php). 6 | 7 | The sequential patch match is used in Photoshop CS5 (http://www.adobe.com/technology/projects/patchmatch.html), but requires a few seconds even for small patches and minimal iterations. This OpenCL implementation is capable of performing more than a dozen iterations in less than a second, and 500 iterations in a few seconds. 8 | 9 | There is a demo mode included, so just download and try running it using Python. 10 | -------------------------------------------------------------------------------- /bike_a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abiusx/CLPatchMatch/b4c21fd32f739a187724538af84fef7f79430681/bike_a.png -------------------------------------------------------------------------------- /bike_b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abiusx/CLPatchMatch/b4c21fd32f739a187724538af84fef7f79430681/bike_b.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | CLPatchMatch by AbiusX 3 | This code runs the patch match algorithm on two images, using OpenCL and runs in realtime. 4 | 5 | Performance: 6 | 50 iterations of this take less than a second on the development machine, where as the sequential 7 | code requires 5+ seconds for each iteration. 8 | ''' 9 | import pyopencl as cl 10 | import numpy 11 | import pylab 12 | import matplotlib; 13 | import skimage 14 | import skimage.io 15 | import skimage.transform 16 | import datetime 17 | 18 | from operator import itemgetter 19 | import sys 20 | import math 21 | import random 22 | import os 23 | 24 | 25 | files=["bike_a.png","bike_b.png"]; 26 | def getTime(): 27 | return datetime.datetime.now(); 28 | 29 | class CLPatchMatch: 30 | ''' 31 | CLSeamCarving class, 32 | performs the seam carving algorithm on an image to reduce its size without scaling 33 | its main features, using OpenCL in realtime 34 | ''' 35 | def __init__(self): 36 | ''' 37 | Inits the opencl environment and parameters 38 | ''' 39 | #profiling operation times 40 | self.times={k:datetime.timedelta() for k in ("execute", 41 | "init","randomfill")}; 42 | #show opencl compiler errors 43 | os.environ["PYOPENCL_COMPILER_OUTPUT"]="1"; 44 | 45 | t=getTime(); 46 | 47 | self.ctx = cl.create_some_context(False) 48 | self.queue = cl.CommandQueue(self.ctx) 49 | 50 | self.times["init"]+=getTime()-t; 51 | 52 | def loadProgram(self, filename): 53 | t=getTime(); 54 | 55 | f = open(filename, 'r') 56 | fstr = "".join(f.readlines()) 57 | 58 | #create the program 59 | self.program = cl.Program(self.ctx, fstr).build() 60 | self.times["init"]+=getTime()-t; 61 | 62 | def loadImages(self,files): 63 | ''' 64 | load both images into self.img[0,1] 65 | ''' 66 | t=getTime(); 67 | self.img = [skimage.img_as_float(skimage.io.imread(files[i])) for i in (0,1)] 68 | self.loadProgram("patchmatch.c") 69 | self.times["init"]+=getTime()-t; 70 | 71 | def randomfill(self): 72 | t=getTime(); 73 | mf= cl.mem_flags 74 | self.inputBuf=[cl.Buffer(self.ctx,mf.READ_ONLY | mf.COPY_HOST_PTR,hostbuf=self.img[i]) for i in [0,1]]; 75 | self.outputBuf=cl.Buffer(self.ctx,mf.WRITE_ONLY | mf.COPY_HOST_PTR ,hostbuf=self.nff) 76 | 77 | 78 | self.program.randomfill(self.queue, self.effectiveSize, None, 79 | numpy.int32(self.patchSize[0]), #patchHeight 80 | numpy.int32(self.patchSize[1]), #patchWidth 81 | numpy.int32(self.size[0]), #height 82 | numpy.int32(self.size[1]), #width 83 | self.inputBuf[0], 84 | self.inputBuf[1], 85 | self.outputBuf) 86 | c = numpy.empty_like(self.nff) 87 | cl.enqueue_read_buffer(self.queue, self.outputBuf, c).wait() 88 | self.nff=numpy.copy(c); 89 | 90 | self.times["randomfill"]+=getTime()-t; 91 | 92 | def execute(self): 93 | ''' 94 | execute an iteration of patchMatch 95 | ''' 96 | t=getTime(); 97 | mf= cl.mem_flags 98 | self.inputBuf=[cl.Buffer(self.ctx,mf.READ_ONLY | mf.COPY_HOST_PTR,hostbuf=self.img[i]) for i in [0,1]]; 99 | self.outputBuf=cl.Buffer(self.ctx,mf.READ_WRITE | mf.COPY_HOST_PTR ,hostbuf=self.nff) 100 | 101 | 102 | self.program.propagate(self.queue, self.effectiveSize, None, 103 | numpy.int32(self.patchSize[0]), #patchHeight 104 | numpy.int32(self.patchSize[1]), #patchWidth 105 | numpy.int32(self.size[0]), #height 106 | numpy.int32(self.size[1]), #width 107 | numpy.int32(self.iteration), 108 | self.inputBuf[0], 109 | self.inputBuf[1], 110 | self.outputBuf) 111 | c = numpy.empty_like(self.nff) 112 | cl.enqueue_read_buffer(self.queue, self.outputBuf, c).wait() 113 | self.nff=numpy.copy(c); 114 | 115 | self.times["execute"]+=getTime()-t; 116 | 117 | def _drawRect(self,img,y,x,height,width,color=(1,0,0)): 118 | ''' 119 | used for demo, showing which rectangles match 120 | ''' 121 | for i in range(0,width+1): 122 | img[y][x+i]=color; 123 | img[y+height][x+i]=color; 124 | for i in range(0,height+1): 125 | img[y+i][x]=color 126 | img[y+i][x+width]=color; 127 | 128 | def show(self,nffs=True): 129 | ''' 130 | shows times and images 131 | ''' 132 | samples=5; 133 | for i in range(0,samples): 134 | color=(random.random(),random.random(),random.random()) 135 | randomPoint=[(int)(random.random()*i) for i in self.effectiveSize]; 136 | self._drawRect(self.img[0],randomPoint[0],randomPoint[1],self.patchSize[0],self.patchSize[1],color); 137 | self._drawRect(self.img[1],self.nff[randomPoint[0]][randomPoint[1]][0],self.nff[randomPoint[0]][randomPoint[1]][1],self.patchSize[0],self.patchSize[1],color); 138 | 139 | for i in self.times.keys(): 140 | print i,":", (self.times[i].seconds*1000+self.times[i].microseconds/1000)/1000.0,"seconds" 141 | 142 | if nffs: 143 | for i in range(0,3): 144 | pylab.imshow(self.nff[:,:,i]) 145 | pylab.show(); 146 | f=pylab.figure() 147 | f.add_subplot(1,2,0); 148 | pylab.imshow(self.img[0],cmap=matplotlib.cm.Greys_r); 149 | f.add_subplot(1,2,1); 150 | pylab.imshow(self.img[1],cmap=matplotlib.cm.Greys_r); 151 | pylab.title("Patch Match (by AbiusX)") 152 | pylab.show(); 153 | 154 | def D(self,y1,x1,y2,x2): 155 | ''' 156 | compute difference of two patches 157 | ''' 158 | return ((self.img[0][y1:y1+self.patchSize[0],x1:x1+self.patchSize[1]]- 159 | self.img[1][y2:y2+self.patchSize[0],x2:x2+self.patchSize[1]])**2).sum(); 160 | 161 | def match(self,files,patchSize=(7,7),iterations=20,Demo=False): 162 | ''' 163 | run the patchMatch algorithm on the images, returning nff array 164 | ''' 165 | self.loadImages(files); 166 | 167 | 168 | self.size=self.img[0].shape; 169 | self.patchSize=patchSize; 170 | self.effectiveSize=[self.size[i]-patchSize[i] for i in (0,1)]; 171 | self.nff=numpy.ndarray((self.effectiveSize[0],self.effectiveSize[1],3)); 172 | 173 | self.randomfill(); 174 | 175 | for i in range(0,iterations): 176 | self.iteration=i+1; 177 | if (Demo): 178 | print "iteration",self.iteration 179 | print "mean block difference:", self.nff[:,:,2].mean(); 180 | self.execute(); 181 | 182 | if (Demo): self.show(); 183 | return self.nff; 184 | 185 | 186 | if __name__ == "__main__": 187 | patchmatch = CLPatchMatch() 188 | print "Please wait a few seconds..."; 189 | patchmatch.match(files,Demo=True); 190 | print "Done." 191 | -------------------------------------------------------------------------------- /patchmatch.c: -------------------------------------------------------------------------------- 1 | unsigned int getIndex(const int y,const int x, const int z, const int width); 2 | double dis( 3 | __global double *img1,const int y1, const int x1, 4 | __global double *img2,const int y2, const int x2, 5 | const int patchWidth, const int patchHeight, 6 | const int width 7 | ); 8 | int random (int start, int end,unsigned int seed); 9 | unsigned int getIndex(const int y, const int x, const int z, const int width) 10 | { 11 | return ((y*width+x)*3+z); 12 | 13 | } 14 | 15 | double dis( 16 | __global double *img1,const int y1, const int x1, 17 | __global double *img2,const int y2, const int x2, 18 | const int patchWidth, const int patchHeight, 19 | const int width 20 | ) 21 | { 22 | double diff=0; 23 | for (int j=0;j=effectiveHeight || x+1>=effectiveWidth) return; 75 | 76 | double currentD,topD,leftD; 77 | //compute intensitive part 78 | dir=direction; 79 | if (nff(y-dir,x,0)+dir>=effectiveHeight || nff(y-dir,x,0)+dir<0) 80 | topD=MAXINT; 81 | else 82 | topD=D(y,x , nff(y-dir,x,0)+dir , nff(y-dir,x,1)); 83 | 84 | if (nff(y,x-dir,1)+dir>=effectiveWidth || nff(y,x-dir,1)+dir<0) 85 | leftD=MAXINT; 86 | else 87 | leftD=D(y,x , nff(y,x-dir,0) , nff(y,x-dir,1)+dir); 88 | 89 | 90 | dir=direction; 91 | currentD=nff(y,x,2); 92 | 93 | if (topD1 && w>1) 111 | { 112 | int x1,y1,x2,y2; 113 | y1=y-h/2; 114 | x1=x-w/2; 115 | y2=y+h/2; 116 | x2=x+w/2; 117 | if (x1<0) x1=0; 118 | if (y1<0) y1=0; 119 | if (x2>=effectiveWidth) 120 | x2=effectiveWidth-1; 121 | if (y2>=effectiveHeight) 122 | y2=effectiveHeight-2; 123 | 124 | int targetX=seed=random(x1,x2,seed); 125 | int targetY=seed=random(y1,y2,seed); 126 | 127 | double newD=D(y,x, targetY,targetX); 128 | if (newD