├── CMakeLists.txt ├── GaussianPyramid.cpp ├── GaussianPyramid.h ├── Image.h ├── ImageIO.h ├── ImageProcessing.h ├── OpticalFlow.h ├── OpticalFlowCode.cpp ├── README.md ├── img1.jpg ├── img2.jpg ├── init.lua ├── legend.png ├── liuflow.cpp └── project.h /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR) 2 | CMAKE_POLICY(VERSION 2.6) 3 | FIND_PACKAGE(Torch REQUIRED) 4 | 5 | SET(src liuflow.cpp 6 | GaussianPyramid.cpp 7 | GaussianPyramid.h 8 | Image.h 9 | ImageIO.h 10 | ImageProcessing.h 11 | OpticalFlowCode.cpp 12 | OpticalFlow.h 13 | project.h) 14 | 15 | SET(luasrc init.lua 16 | img1.jpg 17 | img2.jpg 18 | legend.png) 19 | 20 | SET(QT_MIN_VERSION "4.3.0") 21 | FIND_PACKAGE(Qt4) 22 | 23 | IF (QT4_FOUND) 24 | INCLUDE_DIRECTORIES (${QT_INCLUDES}) 25 | INCLUDE (${QT_USE_FILE}) 26 | LINK_DIRECTORIES (${QT_LIBRARY_DIR}) 27 | ADD_TORCH_PACKAGE(liuflow "${src}" "${luasrc}" "Vision") 28 | TARGET_LINK_LIBRARIES(liuflow luaT TH ${QT_LIBRARIES}) 29 | ENDIF (QT4_FOUND) 30 | -------------------------------------------------------------------------------- /GaussianPyramid.cpp: -------------------------------------------------------------------------------- 1 | #include "GaussianPyramid.h" 2 | #include "math.h" 3 | 4 | GaussianPyramid::GaussianPyramid(void) 5 | { 6 | ImPyramid=NULL; 7 | } 8 | 9 | GaussianPyramid::~GaussianPyramid(void) 10 | { 11 | if(ImPyramid!=NULL) 12 | delete []ImPyramid; 13 | } 14 | 15 | //--------------------------------------------------------------------------------------- 16 | // function to construct the pyramid 17 | // this is the slow way 18 | //--------------------------------------------------------------------------------------- 19 | /*void GaussianPyramid::ConstructPyramid(const DImage &image, double ratio, int minWidth) 20 | { 21 | // the ratio cannot be arbitrary numbers 22 | if(ratio>0.98 || ratio<0.4) 23 | ratio=0.75; 24 | // first decide how many levels 25 | nLevels=log((double)minWidth/image.width())/log(ratio); 26 | if(ImPyramid!=NULL) 27 | delete []ImPyramid; 28 | ImPyramid=new DImage[nLevels]; 29 | ImPyramid[0].copyData(image); 30 | double baseSigma=(1/ratio-1); 31 | for(int i=1;i0.98 || ratio<0.4) 48 | ratio=0.75; 49 | // first decide how many levels 50 | nLevels=log((double)minWidth/image.width())/log(ratio); 51 | if(ImPyramid!=NULL) 52 | delete []ImPyramid; 53 | ImPyramid=new DImage[nLevels]; 54 | ImPyramid[0].copyData(image); 55 | double baseSigma=(1/ratio-1); 56 | int n=log(0.25)/log(ratio); 57 | double nSigma=baseSigma*n; 58 | for(int i=1;i 9 | 10 | #ifndef _MATLAB 11 | #include 12 | #include "ImageIO.h" 13 | #else 14 | #include "mex.h" 15 | #endif 16 | 17 | using namespace std; 18 | 19 | // template class for image 20 | template 21 | class Image 22 | { 23 | protected: 24 | T* pData; 25 | int imWidth,imHeight,nChannels; 26 | int nPixels,nElements; 27 | bool IsDerivativeImage; 28 | public: 29 | Image(void); 30 | Image(int width,int height,int nchannels=1); 31 | Image(const T& value,int _width,int _height,int _nchannels=1); 32 | #ifndef _MATLAB 33 | Image(const QImage& image); 34 | #endif 35 | Image(const Image& other); 36 | ~Image(void); 37 | virtual Image& operator=(const Image& other); 38 | 39 | virtual inline void computeDimension(){nPixels=imWidth*imHeight;nElements=nPixels*nChannels;}; 40 | 41 | virtual void allocate(int width,int height,int nchannels=1); 42 | 43 | template 44 | void allocate(const Image& other); 45 | 46 | virtual void clear(); 47 | virtual void reset(); 48 | virtual void copyData(const Image& other); 49 | void setValue(const T& value); 50 | void setValue(const T& value,int _width,int _height,int _nchannels=1); 51 | 52 | template 53 | void copy(const Image& other); 54 | 55 | void im2double(); 56 | 57 | // function to access the member variables 58 | inline T*& data(){return pData;}; 59 | inline const T*& data() const{return (const T*&)pData;}; 60 | inline int width() const {return imWidth;}; 61 | inline int height() const {return imHeight;}; 62 | inline int nchannels() const {return nChannels;}; 63 | inline int npixels() const {return nPixels;}; 64 | inline int nelements() const {return nElements;}; 65 | inline bool isDerivativeImage() const {return IsDerivativeImage;}; 66 | bool IsFloat () const; 67 | 68 | template 69 | bool matchDimension (const Image& image) const; 70 | 71 | inline void setDerivative(bool isDerivativeImage=true){IsDerivativeImage=isDerivativeImage;}; 72 | 73 | // function to move this image to another one 74 | template 75 | void moveto(Image& image,int x,int y,int width=0,int height=0); 76 | 77 | // function of basic image operations 78 | virtual bool imresize(double ratio); 79 | template 80 | void imresize(Image& result,double ratio); 81 | void imresize(int dstWidth,int dstHeight); 82 | 83 | #ifndef _MATLAB 84 | virtual bool imread(const QString& filename); 85 | virtual void imread(const QImage& image); 86 | 87 | virtual bool imwrite(const QString& filename,int quality=100) const; 88 | virtual bool imwrite(const QString& filename,ImageIO::ImageType imagetype,int quality=100) const; 89 | #else 90 | virtual bool imread(const char* filename) const {return true;}; 91 | virtual bool imwrite(const char* filename) const {return true;}; 92 | #endif 93 | 94 | template 95 | Image dx(bool IsAdvancedFilter=false) const; 96 | 97 | template 98 | void dx(Image& image,bool IsAdvancedFilter=false) const; 99 | 100 | template 101 | Image dy(bool IsAdvancedFilter=false) const; 102 | 103 | template 104 | void dy(Image& image,bool IsAdvancedFilter=false) const; 105 | 106 | template 107 | void GaussianSmoothing(Image& image,double sigma,int fsize) const; 108 | 109 | template 110 | void smoothing(Image& image,double factor=4); 111 | 112 | template 113 | Image smoothing(double factor=4); 114 | 115 | void smoothing(double factor=4); 116 | 117 | // funciton for filtering 118 | template 119 | void imfilter(Image& image,double* filter,int fsize) const; 120 | 121 | template 122 | Image imfilter(double* filter,int fsize); 123 | 124 | template 125 | void imfilter_h(Image& image,double* filter,int fsize) const; 126 | 127 | template 128 | void imfilter_v(Image& image,double* filter,int fsize) const; 129 | 130 | template 131 | void imfilter_hv(Image& image,double* hfilter,int hfsize,double* vfilter,int vfsize) const; 132 | 133 | // function to desaturating 134 | template 135 | void desaturate(Image& image) const; 136 | 137 | template 138 | void collapse(Image& image) const; 139 | 140 | // function to concatenate images 141 | template 142 | void concatenate(Image& destImage,const Image& addImage) const; 143 | 144 | template 145 | Image concatenate(const Image& addImage) const; 146 | 147 | // function to separate the channels of the image 148 | template 149 | void separate(unsigned firstNChannels,Image& image1,Image& image2) const; 150 | 151 | // function to sample patch 152 | template 153 | void getPatch(Image& patch,double x,double y,int fsize) const; 154 | 155 | // function to crop the image 156 | template 157 | void crop(Image& patch,int Lef,int Top,int Width,int Height) const; 158 | 159 | // basic numerics of images 160 | template 161 | void Multiply(const Image& image1,const Image& image2); 162 | 163 | template 164 | void Multiply(const Image& image1,const Image& image2,const Image& image3); 165 | 166 | template 167 | void Multiplywith(const Image& image1); 168 | 169 | void Multiplywith(double value); 170 | 171 | template 172 | void Add(const Image& image1,const Image& image2); 173 | 174 | template 175 | void Add(const Image& image1,const Image& image2,double ratio); 176 | 177 | void Add(const T value); 178 | 179 | template 180 | void Add(const Image& image1,const double value); 181 | 182 | template 183 | void Subtract(const Image& image1,const Image& image2); 184 | 185 | // function to normalize an image 186 | void normalize(Image& image); 187 | 188 | // function to compute the statistics of the image 189 | double norm2() const; 190 | 191 | template 192 | double innerproduct(Image& image) const; 193 | 194 | #ifdef _MATLAB 195 | template 196 | void LoadMatlabImage(const mxArray* image); 197 | 198 | template 199 | void ConvertFromMatlab(const T1* pMatlabPlane,int _width,int _height,int _nchannels); 200 | 201 | void OutputToMatlab(mxArray*& matrix); 202 | 203 | template 204 | void ConvertToMatlab(T1* pMatlabPlane); 205 | #endif 206 | }; 207 | 208 | 209 | typedef Image BiImage; 210 | typedef Image IntImage; 211 | typedef Image FImage; 212 | typedef Image DImage; 213 | 214 | //------------------------------------------------------------------------------------------ 215 | // constructor 216 | //------------------------------------------------------------------------------------------ 217 | template 218 | Image::Image() 219 | { 220 | pData=NULL; 221 | imWidth=imHeight=nChannels=nPixels=nElements=0; 222 | IsDerivativeImage=false; 223 | } 224 | 225 | //------------------------------------------------------------------------------------------ 226 | // constructor with specified dimensions 227 | //------------------------------------------------------------------------------------------ 228 | template 229 | Image::Image(int width,int height,int nchannels) 230 | { 231 | imWidth=width; 232 | imHeight=height; 233 | nChannels=nchannels; 234 | computeDimension(); 235 | pData=NULL; 236 | pData=new T[nElements]; 237 | if(nElements>0) 238 | memset(pData,0,sizeof(T)*nElements); 239 | IsDerivativeImage=false; 240 | } 241 | 242 | template 243 | Image::Image(const T& value,int _width,int _height,int _nchannels) 244 | { 245 | pData=NULL; 246 | allocate(_width,_height,_nchannels); 247 | setValue(value); 248 | } 249 | 250 | #ifndef _MATLAB 251 | template 252 | Image::Image(const QImage& image) 253 | { 254 | pData=NULL; 255 | imread(image); 256 | } 257 | #endif 258 | 259 | template 260 | void Image::allocate(int width,int height,int nchannels) 261 | { 262 | clear(); 263 | imWidth=width; 264 | imHeight=height; 265 | nChannels=nchannels; 266 | computeDimension(); 267 | pData=NULL; 268 | pData=new T[nElements]; 269 | if(nElements>0) 270 | memset(pData,0,sizeof(T)*nElements); 271 | } 272 | 273 | template 274 | template 275 | void Image::allocate(const Image &other) 276 | { 277 | allocate(other.width(),other.height(),other.nchannels()); 278 | } 279 | 280 | //------------------------------------------------------------------------------------------ 281 | // copy constructor 282 | //------------------------------------------------------------------------------------------ 283 | template 284 | Image::Image(const Image& other) 285 | { 286 | pData=NULL; 287 | copyData(other); 288 | } 289 | 290 | //------------------------------------------------------------------------------------------ 291 | // destructor 292 | //------------------------------------------------------------------------------------------ 293 | template 294 | Image::~Image() 295 | { 296 | if(pData!=NULL) 297 | delete []pData; 298 | } 299 | 300 | //------------------------------------------------------------------------------------------ 301 | // clear the image 302 | //------------------------------------------------------------------------------------------ 303 | template 304 | void Image::clear() 305 | { 306 | if(pData!=NULL) 307 | delete []pData; 308 | pData=NULL; 309 | imWidth=imHeight=nChannels=nPixels=nElements=0; 310 | } 311 | 312 | //------------------------------------------------------------------------------------------ 313 | // reset the image (reset the buffer to zero) 314 | //------------------------------------------------------------------------------------------ 315 | template 316 | void Image::reset() 317 | { 318 | if(pData!=NULL) 319 | memset(pData,0,sizeof(T)*nElements); 320 | } 321 | 322 | template 323 | void Image::setValue(const T &value) 324 | { 325 | for(int i=0;i 330 | void Image::setValue(const T& value,int _width,int _height,int _nchannels) 331 | { 332 | if(imWidth!=_width || imHeight!=_height || nChannels!=_nchannels) 333 | allocate(_width,_height,_nchannels); 334 | setValue(value); 335 | } 336 | 337 | //------------------------------------------------------------------------------------------ 338 | // copy from other image 339 | //------------------------------------------------------------------------------------------ 340 | template 341 | void Image::copyData(const Image& other) 342 | { 343 | imWidth=other.imWidth; 344 | imHeight=other.imHeight; 345 | nChannels=other.nChannels; 346 | nPixels=other.nPixels; 347 | IsDerivativeImage=other.IsDerivativeImage; 348 | 349 | if(nElements!=other.nElements) 350 | { 351 | nElements=other.nElements; 352 | if(pData!=NULL) 353 | delete []pData; 354 | pData=NULL; 355 | pData=new T[nElements]; 356 | } 357 | if(nElements>0) 358 | memcpy(pData,other.pData,sizeof(T)*nElements); 359 | } 360 | 361 | template 362 | template 363 | void Image::copy(const Image& other) 364 | { 365 | clear(); 366 | 367 | imWidth=other.width(); 368 | imHeight=other.height(); 369 | nChannels=other.nchannels(); 370 | computeDimension(); 371 | 372 | IsDerivativeImage=other.isDerivativeImage(); 373 | 374 | pData=NULL; 375 | pData=new T[nElements]; 376 | const T1*& srcData=other.data(); 377 | for(int i=0;i 382 | void Image::im2double() 383 | { 384 | if(IsFloat()) 385 | for(int i=0;i 393 | Image& Image::operator=(const Image& other) 394 | { 395 | copyData(other); 396 | return *this; 397 | } 398 | 399 | template 400 | bool Image::IsFloat() const 401 | { 402 | if(typeid(T)==typeid(float) || typeid(T)==typeid(double) || typeid(T)==typeid(long double)) 403 | return true; 404 | else 405 | return false; 406 | } 407 | 408 | template 409 | template 410 | bool Image::matchDimension(const Image& image) const 411 | { 412 | if(imWidth==image.width() && imHeight==image.height() && nChannels==image.nchannels()) 413 | return true; 414 | else 415 | return false; 416 | } 417 | 418 | //------------------------------------------------------------------------------------------ 419 | // function to move this image to a dest image at (x,y) with specified width and height 420 | //------------------------------------------------------------------------------------------ 421 | template 422 | template 423 | void Image::moveto(Image& image,int x0,int y0,int width,int height) 424 | { 425 | if(width==0) 426 | width=imWidth; 427 | if(height==0) 428 | height=imHeight; 429 | int NChannels=__min(nChannels,image.nchannels()); 430 | 431 | int x,y; 432 | for(int i=0;i=image.height()) 436 | break; 437 | for(int j=0;j=image.width()) 441 | break; 442 | for(int k=0;k 453 | bool Image::imresize(double ratio) 454 | { 455 | if(pData==NULL) 456 | return false; 457 | 458 | T* pDstData; 459 | int DstWidth,DstHeight; 460 | DstWidth=(double)imWidth*ratio; 461 | DstHeight=(double)imHeight*ratio; 462 | pDstData=new T[DstWidth*DstHeight*nChannels]; 463 | 464 | ImageProcessing::ResizeImage(pData,pDstData,imWidth,imHeight,nChannels,ratio); 465 | 466 | delete []pData; 467 | pData=pDstData; 468 | imWidth=DstWidth; 469 | imHeight=DstHeight; 470 | computeDimension(); 471 | return true; 472 | } 473 | 474 | template 475 | template 476 | void Image::imresize(Image& result,double ratio) 477 | { 478 | int DstWidth,DstHeight; 479 | DstWidth=(double)imWidth*ratio; 480 | DstHeight=(double)imHeight*ratio; 481 | if(result.width()!=DstWidth || result.height()!=DstHeight || result.nchannels()!=nChannels) 482 | result.allocate(DstWidth,DstHeight,nChannels); 483 | ImageProcessing::ResizeImage(pData,result.data(),imWidth,imHeight,nChannels,ratio); 484 | } 485 | 486 | template 487 | void Image::imresize(int dstWidth,int dstHeight) 488 | { 489 | DImage foo(dstWidth,dstHeight,nChannels); 490 | ImageProcessing::ResizeImage(pData,foo.data(),imWidth,imHeight,nChannels,dstWidth,dstHeight); 491 | copyData(foo); 492 | } 493 | 494 | //------------------------------------------------------------------------------------------ 495 | // function to load the image 496 | //------------------------------------------------------------------------------------------ 497 | #ifndef _MATLAB 498 | template 499 | bool Image::imread(const QString &filename) 500 | { 501 | clear(); 502 | if(ImageIO::loadImage(filename,pData,imWidth,imHeight,nChannels)) 503 | { 504 | computeDimension(); 505 | return true; 506 | } 507 | return false; 508 | } 509 | 510 | template 511 | void Image::imread(const QImage& image) 512 | { 513 | clear(); 514 | ImageIO::loadImage(image,pData,imWidth,imHeight,nChannels); 515 | computeDimension(); 516 | } 517 | 518 | //------------------------------------------------------------------------------------------ 519 | // function to write the image 520 | //------------------------------------------------------------------------------------------ 521 | template 522 | bool Image::imwrite(const QString& filename,int quality) const 523 | { 524 | ImageIO::ImageType type; 525 | if(IsDerivativeImage) 526 | type=ImageIO::derivative; 527 | else 528 | type=ImageIO::standard; 529 | 530 | return ImageIO::writeImage(filename,(const T*&)pData,imWidth,imHeight,nChannels,type,quality); 531 | } 532 | 533 | template 534 | bool Image::imwrite(const QString &filename, ImageIO::ImageType imagetype, int quality) const 535 | { 536 | return ImageIO::writeImage(filename,(const T*&)pData,imWidth,imHeight,nChannels,imagetype,quality); 537 | } 538 | #endif 539 | 540 | //------------------------------------------------------------------------------------------ 541 | // function to get x-derivative of the image 542 | //------------------------------------------------------------------------------------------ 543 | template 544 | template 545 | void Image::dx(Image& result,bool IsAdvancedFilter) const 546 | { 547 | if(matchDimension(result)==false) 548 | result.allocate(imWidth,imHeight,nChannels); 549 | result.reset(); 550 | result.setDerivative(); 551 | T1*& data=result.data(); 552 | int i,j,k,offset; 553 | if(IsAdvancedFilter==false) 554 | for(i=0;i 571 | template 572 | Image Image::dx(bool IsAdvancedFilter) const 573 | { 574 | Image result; 575 | dx(result,IsAdvancedFilter); 576 | return result; 577 | } 578 | 579 | //------------------------------------------------------------------------------------------ 580 | // function to get y-derivative of the image 581 | //------------------------------------------------------------------------------------------ 582 | template 583 | template 584 | void Image::dy(Image& result,bool IsAdvancedFilter) const 585 | { 586 | if(matchDimension(result)==false) 587 | result.allocate(imWidth,imHeight,nChannels); 588 | result.setDerivative(); 589 | T1*& data=result.data(); 590 | int i,j,k,offset; 591 | if(IsAdvancedFilter==false) 592 | for(i=0;i 609 | template 610 | Image Image::dy(bool IsAdvancedFilter) const 611 | { 612 | Image result; 613 | dy(result,IsAdvancedFilter); 614 | return result; 615 | } 616 | 617 | //------------------------------------------------------------------------------------------ 618 | // function to do Gaussian smoothing 619 | //------------------------------------------------------------------------------------------ 620 | template 621 | template 622 | void Image::GaussianSmoothing(Image& image,double sigma,int fsize) const 623 | { 624 | Image foo; 625 | // constructing the 1D gaussian filter 626 | double* gFilter; 627 | gFilter=new double[fsize*2+1]; 628 | double sum=0; 629 | sigma=sigma*sigma*2; 630 | for(int i=-fsize;i<=fsize;i++) 631 | { 632 | gFilter[i+fsize]=exp(-(double)(i*i)/sigma); 633 | sum+=gFilter[i+fsize]; 634 | } 635 | for(int i=0;i<2*fsize+1;i++) 636 | gFilter[i]/=sum; 637 | 638 | // apply filtering 639 | imfilter_hv(image,gFilter,fsize,gFilter,fsize); 640 | 641 | delete gFilter; 642 | } 643 | 644 | //------------------------------------------------------------------------------------------ 645 | // function to smooth the image using a simple 3x3 filter 646 | // the filter is [1 factor 1]/(factor+2), applied horizontally and vertically 647 | //------------------------------------------------------------------------------------------ 648 | template 649 | template 650 | void Image::smoothing(Image& image,double factor) 651 | { 652 | // build 653 | double filter2D[9]={1,0,1,0, 0, 0,1, 0,1}; 654 | filter2D[1]=filter2D[3]=filter2D[5]=filter2D[7]=factor; 655 | filter2D[4]=factor*factor; 656 | for(int i=0;i<9;i++) 657 | filter2D[i]/=(factor+2)*(factor+2); 658 | 659 | if(matchDimension(image)==false) 660 | image.allocate(imWidth,imHeight,nChannels); 661 | imfilter(image,filter2D,1); 662 | } 663 | 664 | template 665 | template 666 | Image Image::smoothing(double factor) 667 | { 668 | Image result; 669 | smoothing(result,factor); 670 | return result; 671 | } 672 | 673 | template 674 | void Image::smoothing(double factor) 675 | { 676 | Image result(imWidth,imHeight,nChannels); 677 | smoothing(result,factor); 678 | copyData(result); 679 | } 680 | 681 | //------------------------------------------------------------------------------------------ 682 | // function of image filtering 683 | //------------------------------------------------------------------------------------------ 684 | template 685 | template 686 | void Image::imfilter(Image& image,double* filter,int fsize) const 687 | { 688 | if(matchDimension(image)==false) 689 | image.allocate(imWidth,imHeight,nChannels); 690 | ImageProcessing::filtering(pData,image.data(),imWidth,imHeight,nChannels,filter,fsize); 691 | } 692 | 693 | template 694 | template 695 | Image Image::imfilter(double *filter, int fsize) 696 | { 697 | Image result; 698 | imfilter(result,filter,fsize); 699 | return result; 700 | } 701 | 702 | template 703 | template 704 | void Image::imfilter_h(Image& image,double* filter,int fsize) const 705 | { 706 | if(matchDimension(image)==false) 707 | image.allocate(imWidth,imHeight,nChannels); 708 | ImageProcessing::hfiltering(pData,image.data(),imWidth,imHeight,nChannels,filter,fsize); 709 | } 710 | 711 | template 712 | template 713 | void Image::imfilter_v(Image& image,double* filter,int fsize) const 714 | { 715 | if(matchDimension(image)==false) 716 | image.allocate(imWidth,imHeight,nChannels); 717 | ImageProcessing::vfiltering(pData,image.data(),imWidth,imHeight,nChannels,filter,fsize); 718 | } 719 | 720 | 721 | template 722 | template 723 | void Image::imfilter_hv(Image &image, double *hfilter, int hfsize, double *vfilter, int vfsize) const 724 | { 725 | if(matchDimension(image)==false) 726 | image.allocate(imWidth,imHeight,nChannels); 727 | T1* pTempBuffer; 728 | pTempBuffer=new T1[nElements]; 729 | ImageProcessing::hfiltering(pData,pTempBuffer,imWidth,imHeight,nChannels,hfilter,hfsize); 730 | ImageProcessing::vfiltering(pTempBuffer,image.data(),imWidth,imHeight,nChannels,vfilter,vfsize); 731 | delete pTempBuffer; 732 | } 733 | 734 | //------------------------------------------------------------------------------------------ 735 | // function for desaturation 736 | //------------------------------------------------------------------------------------------ 737 | template 738 | template 739 | void Image::desaturate(Image &image) const 740 | { 741 | if(nChannels!=3) 742 | { 743 | collapse(image); 744 | return; 745 | } 746 | if(!(image.width()==imWidth && image.height()==imHeight && image.nChannels==1)) 747 | image.allocate(imWidth,imHeight,1); 748 | T1* data=image.data(); 749 | int offset; 750 | for(int i=0;i 758 | template 759 | void Image::collapse(Image &image) const 760 | { 761 | if(!(image.width()==imWidth && image.height()==imHeight && image.nChannels==1)) 762 | image.allocate(imWidth,imHeight,1); 763 | T1* data=image.data(); 764 | int offset; 765 | double temp; 766 | for(int i=0;i 780 | template 781 | void Image::concatenate(Image &destImage, const Image &addImage) const 782 | { 783 | if(addImage.width()!=imWidth || addImage.height()!=imHeight) 784 | { 785 | destImage.copy(*this); 786 | return; 787 | } 788 | int extNChannels=nChannels+addImage.nchannels(); 789 | if(destImage.width()!=imWidth || destImage.height()!=imHeight || destImage.nchannels()!=extNChannels) 790 | destImage.allocate(imWidth,imHeight,extNChannels); 791 | int offset; 792 | T1*& pDestData=destImage.data(); 793 | const T2*& pAddData=addImage.data(); 794 | for(int i=0;i 806 | template 807 | Image Image::concatenate(const Image &addImage) const 808 | { 809 | Image destImage; 810 | concatenate(destImage,addImage); 811 | return destImage; 812 | } 813 | 814 | //------------------------------------------------------------------------------------------ 815 | // function to separate the image into two 816 | //------------------------------------------------------------------------------------------ 817 | template 818 | template 819 | void Image::separate(unsigned int firstNChannels, Image &image1, Image &image2) const 820 | { 821 | image1.IsDerivativeImage=IsDerivativeImage; 822 | image2.IsDerivativeImage=IsDerivativeImage; 823 | 824 | if(firstNChannels>=nChannels) 825 | { 826 | image1=*this; 827 | image2.allocate(imWidth,imHeight,0); 828 | return; 829 | } 830 | if(firstNChannels==0) 831 | { 832 | image1.allocate(imWidth,imHeight,0); 833 | image2=*this; 834 | return; 835 | } 836 | int secondNChannels=nChannels-firstNChannels; 837 | if(image1.width()!=imWidth || image1.height()!=imHeight || image1.nchannels()!=firstNChannels) 838 | image1.allocate(imWidth,imHeight,firstNChannels); 839 | if(image2.width()!=imWidth || image2.height()!=imHeight || image2.nchannels()!=secondNChannels) 840 | image2.allocate(imWidth,imHeight,secondNChannels); 841 | 842 | for(int i=0;i 857 | template 858 | void Image::getPatch(Image& patch,double x,double y,int wsize) const 859 | { 860 | int wlength=wsize*2+1; 861 | if(patch.width()!=wlength || patch.height()!=wlength || patch.nchannels()!=nChannels) 862 | patch.allocate(wlength,wlength,nChannels); 863 | else 864 | patch.reset(); 865 | ImageProcessing::getPatch(pData,patch.data(),imWidth,imHeight,nChannels,x,y,wsize); 866 | } 867 | 868 | //------------------------------------------------------------------------------------------ 869 | // function to crop an image 870 | //------------------------------------------------------------------------------------------ 871 | template 872 | template 873 | void Image::crop(Image& patch,int Left,int Top,int Width,int Height) const 874 | { 875 | if(patch.width()!=Width || patch.height()!=Height || patch.nchannels()!=nChannels) 876 | patch.allocate(Width,Height,nChannels); 877 | // make sure that the cropping is valid 878 | if(Left<0 || Top<0 || Left>=imWidth || Top>=imHeight) 879 | { 880 | cout<<"The cropping coordinate is outside the image boundary!"<imWidth || Height+Top>imHeight) 884 | { 885 | cout<<"The patch to crop is invalid!"< 895 | template 896 | void Image::Multiply(const Image& image1,const Image& image2,const Image& image3) 897 | { 898 | if(image1.matchDimension(image2)==false || image2.matchDimension(image3)==false) 899 | { 900 | cout<<"Error in image dimensions--function Image::Multiply()!"< 915 | template 916 | void Image::Multiply(const Image& image1,const Image& image2) 917 | { 918 | if(image1.matchDimension(image2)==false) 919 | { 920 | cout<<"Error in image dimensions--function Image::Multiply()!"< 934 | template 935 | void Image::Multiplywith(const Image &image1) 936 | { 937 | if(matchDimension(image1)==false) 938 | { 939 | cout<<"Error in image dimensions--function Image::Multiplywith()!"< 948 | void Image::Multiplywith(double value) 949 | { 950 | for(int i=0;i 958 | template 959 | void Image::Add(const Image& image1,const Image& image2) 960 | { 961 | if(image1.matchDimension(image2)==false) 962 | { 963 | cout<<"Error in image dimensions--function Image::Add()!"< 976 | template 977 | void Image::Add(const Image& image1,const Image& image2,double ratio) 978 | { 979 | if(image1.matchDimension(image2)==false) 980 | { 981 | cout<<"Error in image dimensions--function Image::Add()!"< 994 | template 995 | void Image::Add(const Image& image1,const double ratio) 996 | { 997 | if(matchDimension(image1)==false) 998 | { 999 | cout<<"Error in image dimensions--function Image::Add()!"< 1008 | void Image::Add(const T value) 1009 | { 1010 | for(int i=0;i 1018 | template 1019 | void Image::Subtract(const Image &image1, const Image &image2) 1020 | { 1021 | if(image1.matchDimension(image2)==false) 1022 | { 1023 | cout<<"Error in image dimensions--function Image::Add()!"< 1039 | void Image::normalize(Image& image) 1040 | { 1041 | if(image.width()!=imWidth || image.height()!=imHeight || image.nchannels()!=nChannels) 1042 | image.allocate(imWidth,imHeight,nChannels); 1043 | T Max,Min; 1044 | Max=Min=pData[0]; 1045 | for(int i=0;i 1061 | double Image::norm2() const 1062 | { 1063 | double result=0; 1064 | for(int i=0;i 1070 | template 1071 | double Image::innerproduct(Image &image) const 1072 | { 1073 | double result=0; 1074 | const T1* pData1=image.data(); 1075 | for(int i=0;i 1083 | template 1084 | void Image::LoadMatlabImage(const mxArray *image) 1085 | { 1086 | int nDim = mxGetNumberOfDimensions(image); 1087 | const int* imDim = mxGetDimensions(image); 1088 | if(nDim==2) 1089 | allocate(imDim[1],imDim[0]); 1090 | else if(nDim==3) 1091 | allocate(imDim[1],imDim[0],imDim[2]); 1092 | else 1093 | mexErrMsgTxt("The image doesn't have the appropriate dimension!"); 1094 | T1* pMatlabPlane=(T1*)mxGetData(image); 1095 | bool IsMatlabFloat; 1096 | if(typeid(T1)==typeid(float) || typeid(T1)==typeid(double) || typeid(T1)==typeid(long double)) 1097 | IsMatlabFloat=true; 1098 | else 1099 | IsMatlabFloat=false; 1100 | bool isfloat=IsFloat(); 1101 | if(isfloat==IsMatlabFloat) 1102 | { 1103 | ConvertFromMatlab(pMatlabPlane,imWidth,imHeight,nChannels); 1104 | return; 1105 | } 1106 | int offset=0; 1107 | if(isfloat==true) 1108 | for(int i=0;i 1120 | template 1121 | void Image::ConvertFromMatlab(const T1 *pMatlabPlane, int _width, int _height, int _nchannels) 1122 | { 1123 | if(imWidth!=_width || imHeight!=_height || nChannels!=_nchannels) 1124 | allocate(_width,_height,_nchannels); 1125 | int offset=0; 1126 | for(int i=0;i 1134 | template 1135 | void Image::ConvertToMatlab(T1 *pMatlabPlane) 1136 | { 1137 | int offset=0; 1138 | for(int i=0;i 1145 | void Image::OutputToMatlab(mxArray *&matrix) 1146 | { 1147 | int dims[3]; 1148 | dims[0]=imHeight; 1149 | dims[1]=imWidth; 1150 | dims[2]=nChannels; 1151 | if(nChannels==1) 1152 | matrix=mxCreateNumericArray(2, dims,mxDOUBLE_CLASS, mxREAL); 1153 | else 1154 | matrix=mxCreateNumericArray(3, dims,mxDOUBLE_CLASS, mxREAL); 1155 | ConvertToMatlab((double*)mxGetData(matrix)); 1156 | } 1157 | 1158 | #endif 1159 | 1160 | 1161 | #endif 1162 | -------------------------------------------------------------------------------- /ImageIO.h: -------------------------------------------------------------------------------- 1 | #ifndef _ImageIO_h 2 | #define _ImageIO_h 3 | 4 | #include 5 | #include 6 | #include 7 | #include "math.h" 8 | //----------------------------------------------------------------------------------------- 9 | // this class is a wrapper to use QImage to load image into image planes 10 | //----------------------------------------------------------------------------------------- 11 | 12 | class ImageIO 13 | { 14 | public: 15 | enum ImageType{standard, derivative, normalized}; 16 | ImageIO(void); 17 | ~ImageIO(void); 18 | public: 19 | template 20 | static void loadImage(const QImage& image,T*& pImagePlane,int& width,int& height,int& nchannels); 21 | template 22 | static bool loadImage(const QString& filename,T*& pImagePlane,int& width,int& height,int& nchannels); 23 | 24 | template 25 | static unsigned char convertPixel(const T& value,bool IsFloat,ImageType type,T& _Max,T& _Min); 26 | 27 | template 28 | static bool writeImage(const QString& filename, const T*& pImagePlane,int width,int height,int nchannels,ImageType type=standard,int quality=-1); 29 | }; 30 | 31 | template 32 | void ImageIO::loadImage(const QImage& image, T*& pImagePlane,int& width,int& height,int& nchannels) 33 | { 34 | // get the image information 35 | width=image.width(); 36 | height=image.height(); 37 | nchannels=3; 38 | pImagePlane=new T[width*height*nchannels]; 39 | 40 | // check whether the type is float point 41 | bool IsFloat=false; 42 | if(typeid(T)==typeid(double) || typeid(T)==typeid(float) || typeid(T)==typeid(long double)) 43 | IsFloat=true; 44 | 45 | const unsigned char* plinebuffer; 46 | for(int i=0;i 68 | bool ImageIO::loadImage(const QString&filename, T*& pImagePlane,int& width,int& height,int& nchannels) 69 | { 70 | QImage image; 71 | if(image.load(filename)==false) 72 | return false; 73 | if(image.format()!=QImage::Format_RGB32) 74 | { 75 | QImage temp=image.convertToFormat(QImage::Format_RGB32); 76 | image=temp; 77 | } 78 | loadImage(image,pImagePlane,width,height,nchannels); 79 | return true; 80 | } 81 | 82 | template 83 | bool ImageIO::writeImage(const QString& filename, const T*& pImagePlane,int width,int height,int nchannels,ImageType type,int quality) 84 | { 85 | int nPixels=width*height,nElements; 86 | nElements=nPixels*nchannels; 87 | unsigned char* pTempBuffer; 88 | pTempBuffer=new unsigned char[nPixels*4]; 89 | memset(pTempBuffer,0,nPixels*4); 90 | 91 | // check whether the type is float point 92 | bool IsFloat=false; 93 | if(typeid(T)==typeid(double) || typeid(T)==typeid(float) || typeid(T)==typeid(long double)) 94 | IsFloat=true; 95 | 96 | T _Max=0,_Min=0; 97 | switch(type){ 98 | case standard: 99 | break; 100 | case derivative: 101 | _Max=0; 102 | for(int i=0;i=3) 123 | { 124 | pTempBuffer[i*4]=convertPixel(pImagePlane[i*nchannels],IsFloat,type,_Max,_Min); 125 | pTempBuffer[i*4+1]=convertPixel(pImagePlane[i*nchannels+1],IsFloat,type,_Max,_Min); 126 | pTempBuffer[i*4+2]=convertPixel(pImagePlane[i*nchannels+2],IsFloat,type,_Max,_Min); 127 | } 128 | else 129 | for (int j=0;j<3;j++) 130 | pTempBuffer[i*4+j]=convertPixel(pImagePlane[i*nchannels],IsFloat,type,_Max,_Min); 131 | pTempBuffer[i*4+3]=255; 132 | } 133 | QImage *pQImage=new QImage(pTempBuffer,width,height,QImage::Format_RGB32); 134 | bool result= pQImage->save(filename,0,quality); 135 | delete pQImage; 136 | delete pTempBuffer; 137 | return result; 138 | } 139 | 140 | template 141 | unsigned char ImageIO::convertPixel(const T& value,bool IsFloat,ImageType type,T& _Max,T& _Min) 142 | { 143 | switch(type){ 144 | case standard: 145 | if(IsFloat) 146 | return __max(__min(value*255,255),0); 147 | else 148 | return __max(__min(value,255),0); 149 | break; 150 | case derivative: 151 | return (double)((double)value/_Max+1)/2*255; 152 | break; 153 | case normalized: 154 | return (double)(value-_Min)/(_Max-_Min)*255; 155 | break; 156 | } 157 | return 0; 158 | } 159 | 160 | #endif -------------------------------------------------------------------------------- /ImageProcessing.h: -------------------------------------------------------------------------------- 1 | #ifndef _ImageProcessing_h 2 | #define _ImageProcessing_h 3 | 4 | #include "math.h" 5 | #include "stdio.h" 6 | #include "stdlib.h" 7 | #include 8 | //---------------------------------------------------------------------------------- 9 | // class to handle basic image processing functions 10 | // this is a collection of template functions. These template functions are 11 | // used in other image classes such as BiImage, IntImage and FImage 12 | //---------------------------------------------------------------------------------- 13 | 14 | class ImageProcessing 15 | { 16 | public: 17 | ImageProcessing(void); 18 | ~ImageProcessing(void); 19 | public: 20 | // basic functions 21 | template 22 | static inline T EnforceRange(const T& x,const int& MaxValue) {return __min(__max(x,0),MaxValue-1);}; 23 | 24 | //--------------------------------------------------------------------------------- 25 | // function to interpolate the image plane 26 | //--------------------------------------------------------------------------------- 27 | template 28 | static inline void BilinearInterpolate(const T1* pImage,int width,int height,int nChannels,double x,double y,T2* result); 29 | 30 | template 31 | static void ResizeImage(const T1* pSrcImage,T2* pDstImage,int SrcWidth,int SrcHeight,int nChannels,double Ratio); 32 | 33 | template 34 | static void ResizeImage(const T1* pSrcImage,T2* pDstImage,int SrcWidth,int SrcHeight,int nChannels,int DstWidth,int DstHeight); 35 | 36 | //--------------------------------------------------------------------------------- 37 | // functions for 1D filtering 38 | //--------------------------------------------------------------------------------- 39 | template 40 | static void hfiltering(const T1* pSrcImage,T2* pDstImage,int width,int height,int nChannels,double* pfilter1D,int fsize); 41 | 42 | template 43 | static void vfiltering(const T1* pSrcImage,T2* pDstImage,int width,int height,int nChannels,double* pfilter1D,int fsize); 44 | 45 | //--------------------------------------------------------------------------------- 46 | // functions for 2D filtering 47 | //--------------------------------------------------------------------------------- 48 | template 49 | static void filtering(const T1* pSrcImage,T2* pDstImage,int width,int height,int nChannels,double* pfilter2D,int fsize); 50 | 51 | //--------------------------------------------------------------------------------- 52 | // functions for sample a patch from the image 53 | //--------------------------------------------------------------------------------- 54 | template 55 | static void getPatch(const T1* pSrcImgae,T2* pPatch,int width,int height,int nChannels,double x,double y,int wsize); 56 | 57 | //--------------------------------------------------------------------------------- 58 | // function to warp image 59 | //--------------------------------------------------------------------------------- 60 | template 61 | static void warpImage(T1* pWarpIm2,const T1* pIm1,const T1* pIm2,const T2* pVx,const T2* pVy,int width,int height,int nChannels); 62 | 63 | //--------------------------------------------------------------------------------- 64 | // function to crop an image 65 | //--------------------------------------------------------------------------------- 66 | template 67 | static void cropImage(const T1* pSrcImage,int SrcWidth,int SrcHeight,int nChannels,T2* pDstImage,int Left,int Top,int DstWidth,int DstHeight); 68 | //--------------------------------------------------------------------------------- 69 | 70 | //--------------------------------------------------------------------------------- 71 | // function to generate a 2D Gaussian 72 | //--------------------------------------------------------------------------------- 73 | template 74 | static void generate2DGaussian(T*& pImage,int wsize,double sigma=-1); 75 | }; 76 | 77 | //-------------------------------------------------------------------------------------------------- 78 | // function to interplate multi-channel image plane for (x,y) 79 | // -------------------------------------------------------------------------------------------------- 80 | template 81 | inline void ImageProcessing::BilinearInterpolate(const T1* pImage,int width,int height,int nChannels,double x,double y,T2* result) 82 | { 83 | int xx,yy,m,n,u,v,l,offset; 84 | xx=x; 85 | yy=y; 86 | double dx,dy,s; 87 | dx=__max(__min(x-xx,1),0); 88 | dy=__max(__min(y-yy,1),0); 89 | 90 | memset(result,0,sizeof(T2)*nChannels); 91 | 92 | for(m=0;m<=1;m++) 93 | for(n=0;n<=1;n++) 94 | { 95 | u=EnforceRange(xx+m,width); 96 | v=EnforceRange(yy+n,height); 97 | offset=(v*width+u)*nChannels; 98 | s=fabs(1-m-dx)*fabs(1-n-dy); 99 | for(l=0;l 109 | void ImageProcessing::ResizeImage(const T1* pSrcImage,T2* pDstImage,int SrcWidth,int SrcHeight,int nChannels,double Ratio) 110 | { 111 | int DstWidth,DstHeight; 112 | DstWidth=(double)SrcWidth*Ratio; 113 | DstHeight=(double)SrcHeight*Ratio; 114 | memset(pDstImage,sizeof(T2)*DstWidth*DstHeight*nChannels,0); 115 | 116 | double x,y; 117 | 118 | for(int i=0;i 130 | void ImageProcessing::ResizeImage(const T1 *pSrcImage, T2 *pDstImage, int SrcWidth, int SrcHeight, int nChannels, int DstWidth, int DstHeight) 131 | { 132 | double xRatio=(double)DstWidth/SrcWidth; 133 | double yRatio=(double)DstHeight/SrcHeight; 134 | memset(pDstImage,sizeof(T2)*DstWidth*DstHeight*nChannels,0); 135 | 136 | double x,y; 137 | 138 | for(int i=0;i 153 | void ImageProcessing::hfiltering(const T1* pSrcImage,T2* pDstImage,int width,int height,int nChannels,double* pfilter1D,int fsize) 154 | { 155 | memset(pDstImage,0,sizeof(T2)*width*height*nChannels); 156 | T2* pBuffer; 157 | double w; 158 | int i,j,l,k,offset,jj; 159 | for(i=0;i 178 | void ImageProcessing::vfiltering(const T1* pSrcImage,T2* pDstImage,int width,int height,int nChannels,double* pfilter1D,int fsize) 179 | { 180 | memset(pDstImage,0,sizeof(T2)*width*height*nChannels); 181 | T2* pBuffer; 182 | double w; 183 | int i,j,l,k,offset,ii; 184 | for(i=0;i 202 | void ImageProcessing::filtering(const T1* pSrcImage,T2* pDstImage,int width,int height,int nChannels,double* pfilter2D,int fsize) 203 | { 204 | double w; 205 | int i,j,u,v,k,ii,jj,wsize,offset; 206 | wsize=fsize*2+1; 207 | double* pBuffer=new double[nChannels]; 208 | for(i=0;i 234 | void ImageProcessing::getPatch(const T1* pSrcImage,T2* pPatch,int width,int height,int nChannels,double x0,double y0,int wsize) 235 | { 236 | // suppose pPatch has been allocated and cleared before calling the function 237 | int wlength=wsize*2+1; 238 | double x,y; 239 | for(int i=-wsize;i<=wsize;i++) 240 | for(int j=-wsize;j<=wsize;j++) 241 | { 242 | y=y0+i; 243 | x=x0+j; 244 | if(x<0 || x>width-1 || y<0 || y>height-1) 245 | continue; 246 | BilinearInterpolate(pSrcImage,width,height,nChannels,x,y,pPatch+((i+wsize)*wlength+j+wsize)*nChannels); 247 | } 248 | } 249 | 250 | //------------------------------------------------------------------------------------------------------------ 251 | // function to warp an image with respect to flow field 252 | // pWarpIm2 has to be allocated before hands 253 | //------------------------------------------------------------------------------------------------------------ 254 | template 255 | void ImageProcessing::warpImage(T1 *pWarpIm2, const T1 *pIm1, const T1 *pIm2, const T2 *pVx, const T2 *pVy, int width, int height, int nChannels) 256 | { 257 | for(int i=0;iwidth-1 || y<0 || y>height-1) 266 | { 267 | for(int k=0;k 282 | void ImageProcessing::cropImage(const T1 *pSrcImage, int SrcWidth, int SrcHeight, int nChannels, T2 *pDstImage, int Left, int Top, int DstWidth, int DstHeight) 283 | { 284 | if(typeid(T1)==typeid(T2)) 285 | { 286 | for(int i=0;i 306 | void ImageProcessing::generate2DGaussian(T*& pImage, int wsize, double sigma) 307 | { 308 | if(sigma==-1) 309 | sigma=wsize; 310 | double alpha=1/(2*sigma*sigma); 311 | int winlength=wsize*2+1; 312 | if(pImage==NULL) 313 | pImage=new T[winlength*winlength]; 314 | for(int i=-wsize;i<=wsize;i++) 315 | for(int j=-wsize;j<=wsize;j++) 316 | pImage[(i+wsize)*winlength+j+wsize]=exp(-(double)(i*i+j*j)*alpha); 317 | } 318 | #endif -------------------------------------------------------------------------------- /OpticalFlow.h: -------------------------------------------------------------------------------- 1 | #ifndef _OpticalFlow_h 2 | #define _OpticalFlow_h 3 | 4 | #include "Image.h" 5 | 6 | class OpticalFlow 7 | { 8 | private: 9 | static bool IsDisplay; 10 | public: 11 | OpticalFlow(void); 12 | ~OpticalFlow(void); 13 | public: 14 | static void getDxs(DImage& imdx,DImage& imdy,DImage& imdt,const DImage& im1,const DImage& im2); 15 | static void SanityCheck(const DImage& imdx,const DImage& imdy,const DImage& imdt,double du,double dv); 16 | static void warpFL(DImage& warpIm2,const DImage& Im1,const DImage& Im2,const DImage& vx,const DImage& vy); 17 | static void genConstFlow(DImage& flow,double value,int width,int height); 18 | static void genInImageMask(DImage& mask,const DImage& vx,const DImage& vy); 19 | static void SmoothFlowPDE(const DImage& Im1,const DImage& Im2, DImage& warpIm2,DImage& vx,DImage& vy, 20 | double alpha,int nOuterFPIterations,int nInnerFPIterations,int nCGIterations); 21 | static void Laplacian(DImage& output,const DImage& input,const DImage& weight); 22 | static void testLaplacian(int dim=3); 23 | 24 | // function of coarse to fine optical flow 25 | static void Coarse2FineFlow(DImage& vx,DImage& vy,DImage &warpI2,const DImage& Im1,const DImage& Im2,double alpha,double ratio,int minWidth, 26 | int nOuterFPIterations,int nInnerFPIterations,int nCGIterations); 27 | // function to convert image to features 28 | static void im2feature(DImage& imfeature,const DImage& im); 29 | }; 30 | 31 | #endif -------------------------------------------------------------------------------- /OpticalFlowCode.cpp: -------------------------------------------------------------------------------- 1 | #include "OpticalFlow.h" 2 | #include "ImageProcessing.h" 3 | #include "GaussianPyramid.h" 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | 9 | bool OpticalFlow::IsDisplay=false; 10 | 11 | OpticalFlow::OpticalFlow(void) 12 | { 13 | } 14 | 15 | OpticalFlow::~OpticalFlow(void) 16 | { 17 | } 18 | 19 | //-------------------------------------------------------------------------------------------------------- 20 | // function to compute dx, dy and dt for motion estimation 21 | //-------------------------------------------------------------------------------------------------------- 22 | void OpticalFlow::getDxs(DImage &imdx, DImage &imdy, DImage &imdt, const DImage &im1, const DImage &im2) 23 | { 24 | // Im1 and Im2 are the smoothed version of im1 and im2 25 | DImage Im1,Im2; 26 | double gfilter[5]={0.05,0.2,0.5,0.2,0.05}; 27 | im1.imfilter_hv(Im1,gfilter,2,gfilter,2); 28 | im2.imfilter_hv(Im2,gfilter,2,gfilter,2); 29 | 30 | //Im1.copyData(im1); 31 | //Im2.copyData(im2); 32 | 33 | Im2.dx(imdx,true); 34 | Im2.dy(imdy,true); 35 | imdt.Subtract(Im2,Im1); 36 | imdx.setDerivative(); 37 | imdy.setDerivative(); 38 | imdt.setDerivative(); 39 | } 40 | 41 | //-------------------------------------------------------------------------------------------------------- 42 | // function to do sanity check: imdx*du+imdy*dy+imdt=0 43 | //-------------------------------------------------------------------------------------------------------- 44 | void OpticalFlow::SanityCheck(const DImage &imdx, const DImage &imdy, const DImage &imdt, double du, double dv) 45 | { 46 | if(imdx.matchDimension(imdy)==false || imdx.matchDimension(imdt)==false) 47 | { 48 | cout<<"The dimensions of the derivatives don't match!"<imWidth-1 || y<0 || y>imHeight-1) 102 | continue; 103 | pMask[offset]=1; 104 | } 105 | } 106 | 107 | //-------------------------------------------------------------------------------------------------------- 108 | // function to compute optical flow field using two fixed point iterations 109 | // Input arguments: 110 | // Im1, Im2: frame 1 and frame 2 111 | // warpIm2: the warped frame 2 according to the current flow field u and v 112 | // u,v: the current flow field, NOTICE that they are also output arguments 113 | // 114 | //-------------------------------------------------------------------------------------------------------- 115 | void OpticalFlow::SmoothFlowPDE(const DImage &Im1, const DImage &Im2, DImage &warpIm2, DImage &u, DImage &v, 116 | double alpha, int nOuterFPIterations, int nInnerFPIterations, int nCGIterations) 117 | { 118 | DImage mask,imdx,imdy,imdt; 119 | int imWidth,imHeight,nChannels,nPixels; 120 | imWidth=Im1.width(); 121 | imHeight=Im1.height(); 122 | nChannels=Im1.nchannels(); 123 | nPixels=imWidth*imHeight; 124 | 125 | DImage du(imWidth,imHeight),dv(imWidth,imHeight); 126 | DImage uu(imWidth,imHeight),vv(imWidth,imHeight); 127 | DImage ux(imWidth,imHeight),uy(imWidth,imHeight); 128 | DImage vx(imWidth,imHeight),vy(imWidth,imHeight); 129 | DImage Phi_1st(imWidth,imHeight); 130 | DImage Psi_1st(imWidth,imHeight,nChannels); 131 | 132 | DImage imdxy,imdx2,imdy2,imdtdx,imdtdy; 133 | DImage ImDxy,ImDx2,ImDy2,ImDtDx,ImDtDy; 134 | DImage A11,A12,A22,b1,b2; 135 | DImage foo1,foo2; 136 | 137 | // variables for conjugate gradient 138 | DImage r1,r2,p1,p2,q1,q2; 139 | double* rou; 140 | rou=new double[nCGIterations]; 141 | 142 | double varepsilon_phi=pow(0.001,2); 143 | double varepsilon_psi=pow(0.001,2); 144 | 145 | //-------------------------------------------------------------------------- 146 | // the outer fixed point iteration 147 | //-------------------------------------------------------------------------- 148 | for(int count=0;count1) 239 | { 240 | ImDxy.collapse(imdxy); 241 | ImDx2.collapse(imdx2); 242 | ImDy2.collapse(imdy2); 243 | ImDtDx.collapse(imdtdx); 244 | ImDtDy.collapse(imdtdy); 245 | } 246 | else 247 | { 248 | imdxy.copyData(ImDxy); 249 | imdx2.copyData(ImDx2); 250 | imdy2.copyData(ImDy2); 251 | imdtdx.copyData(ImDtDx); 252 | imdtdy.copyData(ImDtDy); 253 | } 254 | 255 | // filtering 256 | imdx2.smoothing(A11,3); 257 | imdxy.smoothing(A12,3); 258 | imdy2.smoothing(A22,3); 259 | 260 | // add epsilon to A11 and A22 261 | A11.Add(alpha*0.1); 262 | A22.Add(alpha*0.1); 263 | 264 | // form b 265 | imdtdx.smoothing(b1,3); 266 | imdtdy.smoothing(b2,3); 267 | // laplacian filtering of the current flow field 268 | Laplacian(foo1,u,Phi_1st); 269 | Laplacian(foo2,v,Phi_1st); 270 | double *b1Data,*b2Data; 271 | const double *foo1Data,*foo2Data; 272 | b1Data=b1.data(); 273 | b2Data=b2.data(); 274 | foo1Data=foo1.data(); 275 | foo2Data=foo2.data(); 276 | 277 | for(int i=0;i0) 385 | outputData[offset]+=fooData[offset-1]; 386 | } 387 | foo.reset(); 388 | // vertical filtering 389 | for(int i=0;i0) 402 | outputData[offset]+=fooData[offset-width]; 403 | } 404 | } 405 | 406 | void OpticalFlow::testLaplacian(int dim) 407 | { 408 | // generate the random weight 409 | DImage weight(dim,dim); 410 | for(int i=0;i=0) 431 | printf(" "); 432 | printf(" %1.0f ",sysMatrix.data()[i*dim*dim+j]); 433 | } 434 | printf("\n"); 435 | } 436 | } 437 | 438 | //-------------------------------------------------------------------------------------- 439 | // function to perfomr coarse to fine optical flow estimation 440 | //-------------------------------------------------------------------------------------- 441 | void OpticalFlow::Coarse2FineFlow(DImage &vx, DImage &vy, DImage &warpI2,const DImage &Im1, const DImage &Im2, double alpha, double ratio, int minWidth, 442 | int nOuterFPIterations, int nInnerFPIterations, int nCGIterations) 443 | { 444 | // first build the pyramid of the two images 445 | GaussianPyramid GPyramid1; 446 | GaussianPyramid GPyramid2; 447 | if(IsDisplay) 448 | cout<<"Constructing pyramid..."; 449 | GPyramid1.ConstructPyramid(Im1,ratio,minWidth); 450 | GPyramid2.ConstructPyramid(Im2,ratio,minWidth); 451 | if(IsDisplay) 452 | cout<<"done!"<=0;k--) 458 | { 459 | if(IsDisplay) 460 | cout<<"Pyramid level "<img2, then warp(img2,vx,vy) will compute\n' 117 | ..'a reconstruction of img1', 118 | {arg='image', type='torch.Tensor', help='input image (NxHxW tensor)', req=true}, 119 | {arg='flow_x', type='torch.Tensor', help='x component of flow field', req=true}, 120 | {arg='flow_y', type='torch.Tensor', help='y component of flow field', req=true} 121 | ) 122 | if inp:nDimension() ~= 3 then 123 | xerror('image should be a NxHxW tensor',nil,args.usage) 124 | end 125 | return libliuflow.warp(inp, vx, vy) 126 | end 127 | 128 | ------------------------------------------------------------ 129 | -- Computes the optical flow on some example images 130 | -- 131 | -- @see opticalFlow.infer 132 | ------------------------------------------------------------ 133 | liuflow.testme = function() 134 | require 'image' 135 | 136 | local img1 = image.load(paths.concat(paths.install_lua_path, 'liuflow/img1.jpg')):float() 137 | local img2 = image.load(paths.concat(paths.install_lua_path, 'liuflow/img2.jpg')):float() 138 | local img1s = image.scale(img1,img1:size(3)/2,img1:size(2)/2) 139 | local img2s = image.scale(img2,img1:size(3)/2,img1:size(2)/2) 140 | 141 | print('computing optical on ' .. img1s:size(3) .. 'x' .. img1s:size(2) .. ' image') 142 | 143 | local resn,resa,warp,resx,resy = liuflow.infer{ pair={img1s,img2s}, 144 | alpha=0.005, 145 | ratio=0.6, 146 | minWidth=50, 147 | nOuterFPIterations=6, 148 | nInnerFPIterations=1, 149 | nCGIterations=40 } 150 | 151 | local resn_q = resn:clone():div(resn:max()):mul(6):floor():div(8) 152 | local resa_q = resa:clone():div(360/16):floor():mul(360/16) 153 | 154 | image.display{image={img1s, liuflow.field2rgb(resn,resa), 155 | img1s, (img2s-img1s):abs(), 156 | img2s, liuflow.field2rgb(resn_q,resa_q), 157 | warp, (warp-img1s):abs()}, 158 | zoom=1, 159 | min=0, max=1, 160 | nrow=4, 161 | legends={'input 1', 'flow field', 'input 1', 162 | 'input 1 - input 2', 163 | 'input 2', 'quantized flow field', 164 | 'warped(input 2)', 165 | 'input 1 - warped(input 2)'}, 166 | legend="optical flow, method = C.Liu"} 167 | 168 | return resn, resa, warp 169 | end 170 | 171 | ------------------------------------------------------------ 172 | -- computes norm (size) of flow field from flow_x and flow_y, 173 | -- 174 | -- @usage opticalFlow.computeNorm() -- prints online help 175 | -- 176 | -- @param flow_x flow field (x), (WxH) [required] [type = torch.Tensor] 177 | -- @param flow_y flow field (y), (WxH) [required] [type = torch.Tensor] 178 | ------------------------------------------------------------ 179 | liuflow.computeNorm = function(...) 180 | -- check args 181 | local args, flow_x, flow_y = dok.unpack( 182 | {...}, 183 | 'opticalFlow.computeNorm', 184 | 'computes norm (size) of flow field from flow_x and flow_y,\n', 185 | {arg='flow_x', type='torch.Tensor', help='flow field (x), (WxH)', req=true}, 186 | {arg='flow_y', type='torch.Tensor', help='flow field (y), (WxH)', req=true} 187 | ) 188 | local flow_norm = flow_y:clone():cmul(flow_y) 189 | local x_squared = flow_x:clone():cmul(flow_x) 190 | flow_norm:add(x_squared):sqrt() 191 | return flow_norm 192 | end 193 | 194 | ------------------------------------------------------------ 195 | -- computes angle (direction) of flow field from flow_x and flow_y, 196 | -- 197 | -- @usage opticalFlow.computeAngle() -- prints online help 198 | -- 199 | -- @param flow_x flow field (x), (WxH) [required] [type = torch.Tensor] 200 | -- @param flow_y flow field (y), (WxH) [required] [type = torch.Tensor] 201 | ------------------------------------------------------------ 202 | liuflow.computeAngle = function(...) 203 | -- check args 204 | local args, flow_x, flow_y = dok.unpack( 205 | {...}, 206 | 'opticalFlow.computeAngle', 207 | 'computes angle (direction) of flow field from flow_x and flow_y,\n', 208 | {arg='flow_x', type='torch.Tensor', help='flow field (x), (WxH)', req=true}, 209 | {arg='flow_y', type='torch.Tensor', help='flow field (y), (WxH)', req=true} 210 | ) 211 | local flow_angle = flow_y:clone():cdiv(flow_x):abs():atan():mul(180/math.pi) 212 | flow_angle:map2(flow_x, flow_y, function(h,x,y) 213 | if x == 0 and y >= 0 then 214 | return 90 215 | elseif x == 0 and y <= 0 then 216 | return 270 217 | elseif x >= 0 and y >= 0 then 218 | -- all good 219 | elseif x >= 0 and y < 0 then 220 | return 360 - h 221 | elseif x < 0 and y >= 0 then 222 | return 180 - h 223 | elseif x < 0 and y < 0 then 224 | return 180 + h 225 | end 226 | end) 227 | return flow_angle 228 | end 229 | 230 | ------------------------------------------------------------ 231 | -- merges Norm and Angle flow fields into a single RGB image, 232 | -- where saturation=intensity, and hue=direction 233 | -- 234 | -- @usage opticalFlow.field2rgb() -- prints online help 235 | -- 236 | -- @param norm flow field (norm), (WxH) [required] [type = torch.Tensor] 237 | -- @param angle flow field (angle), (WxH) [required] [type = torch.Tensor] 238 | -- @param max if not provided, norm:max() is used [type = number] 239 | -- @param legend prints a legend on the image [type = boolean] 240 | ------------------------------------------------------------ 241 | liuflow.field2rgb = function (...) 242 | -- check args 243 | local args, norm, angle, max, legend = dok.unpack( 244 | {...}, 245 | 'opticalFlow.field2rgb', 246 | 'merges Norm and Angle flow fields into a single RGB image,\n' 247 | .. 'where saturation=intensity, and hue=direction', 248 | {arg='norm', type='torch.Tensor', help='flow field (norm), (WxH)', req=true}, 249 | {arg='angle', type='torch.Tensor', help='flow field (angle), (WxH)', req=true}, 250 | {arg='max', type='number', help='if not provided, norm:max() is used'}, 251 | {arg='legend', type='boolean', help='prints a legend on the image', default=false} 252 | ) 253 | 254 | -- max 255 | local saturate = false 256 | if max then saturate = true end 257 | max = math.max(max or norm:max(), 1e-2) 258 | 259 | -- redim? 260 | if norm:nDimension() == 3 then 261 | norm = norm[1] 262 | end 263 | if angle:nDimension() == 3 then 264 | angle = angle[1] 265 | end 266 | 267 | -- merge them into an HSL image 268 | local hsl = torch.Tensor(3, norm:size(1), norm:size(2)) 269 | -- hue = angle: 270 | hsl:select(1,1):copy(angle):div(360) 271 | -- saturation = normalized intensity: 272 | hsl:select(1,2):copy(norm):div(max) 273 | if saturate then hsl:select(1,2):tanh() end 274 | -- light varies inversely from saturation (null flow = white): 275 | hsl:select(1,3):copy(hsl:select(1,2)):mul(-0.5):add(1) 276 | 277 | -- convert HSL to RGB 278 | local rgb = image.hsl2rgb(hsl) 279 | 280 | -- legend 281 | if legend then 282 | _legend_ = _legend_ 283 | or image.load(paths.concat(paths.install_lua_path, 'liuflow/legend.png')) 284 | local legend = image.scale(_legend_, hsl:size(2)/8, hsl:size(2)/8) 285 | rgb:narrow(3,1,legend:size(2)):narrow(2,hsl:size(2)-legend:size(2)+1,legend:size(2)):copy(legend) 286 | end 287 | 288 | -- done 289 | return rgb 290 | end 291 | 292 | ------------------------------------------------------------ 293 | -- Simplifies display of flow field in HSV colorspace when the 294 | -- available field is in x,y displacement 295 | -- 296 | -- @usage opticalFlow.xy2rgb() -- prints online help 297 | -- 298 | -- @param x flow field (x), (WxH) [required] [type = torch.Tensor] 299 | -- @param y flow field (y), (WxH) [required] [type = torch.Tensor] 300 | ------------------------------------------------------------ 301 | liuflow.xy2rgb = function (...) 302 | -- check args 303 | local args, x, y, max = dok.unpack( 304 | {...}, 305 | 'opticalFlow.xy2rgb', 306 | 'merges x and y flow fields into a single RGB image,\n' 307 | .. 'where saturation=intensity, and hue=direction', 308 | {arg='x', type='torch.Tensor', help='flow field (norm), (WxH)', req=true}, 309 | {arg='y', type='torch.Tensor', help='flow field (angle), (WxH)', req=true}, 310 | {arg='max', type='number', help='if not provided, norm:max() is used'} 311 | ) 312 | 313 | local norm = liuflow.computeNorm(x,y) 314 | local angle = liuflow.computeAngle(x,y) 315 | return liuflow.field2rgb(norm,angle,max) 316 | end 317 | -------------------------------------------------------------------------------- /legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clementfarabet/lua---liuflow/0c5d1064c848085eb7a65b39e98f07c0b918908f/legend.png -------------------------------------------------------------------------------- /liuflow.cpp: -------------------------------------------------------------------------------- 1 | 2 | // To load this lib in LUA: 3 | // require 'libliuflow' 4 | 5 | #include 6 | #include 7 | 8 | #include "project.h" 9 | #include "Image.h" 10 | #include "OpticalFlow.h" 11 | #include 12 | 13 | using namespace std; 14 | 15 | // conversion functions 16 | static DImage *tensor_to_image(THFloatTensor *tensor) { 17 | // create output 18 | int w = tensor->size[2]; 19 | int h = tensor->size[1]; 20 | int c = tensor->size[0]; 21 | DImage *img = new DImage(w,h,c); 22 | 23 | // copy data 24 | int i1,i0,i2; 25 | double *dest = img->data(); 26 | int offset = 0; 27 | for (i1=0; i1size[1]; i1++) { 28 | for (i2=0; i2size[2]; i2++) { 29 | for (i0=0; i0size[0]; i0++) { 30 | dest[offset++] = THFloatTensor_get3d(tensor, i0, i1, i2); 31 | } 32 | } 33 | } 34 | 35 | // return result 36 | return img; 37 | } 38 | 39 | static THFloatTensor *image_to_tensor(DImage *img) { 40 | // create output 41 | THFloatTensor *tensor = THFloatTensor_newWithSize3d(img->nchannels(), img->height(), img->width()); 42 | 43 | // copy data 44 | int i1,i0,i2; 45 | double *src = img->data(); 46 | int offset = 0; 47 | for (i1=0; i1size[1]; i1++) { 48 | for (i2=0; i2size[2]; i2++) { 49 | for (i0=0; i0size[0]; i0++) { 50 | THFloatTensor_set3d(tensor, i0, i1, i2, src[offset++]); 51 | } 52 | } 53 | } 54 | 55 | // return result 56 | return tensor; 57 | } 58 | 59 | int optflow_lua(lua_State *L) { 60 | // defaults 61 | double alpha=0.01; 62 | double ratio=0.75; 63 | int minWidth=30; 64 | int nOuterFPIterations=15; 65 | int nInnerFPIterations=1; 66 | int nCGIterations=40; 67 | 68 | // get args 69 | THFloatTensor *ten1 = (THFloatTensor *)luaT_checkudata(L, 1, luaT_checktypename2id(L, "torch.FloatTensor")); 70 | THFloatTensor *ten2 = (THFloatTensor *)luaT_checkudata(L, 2, luaT_checktypename2id(L, "torch.FloatTensor")); 71 | if (lua_isnumber(L, 3)) alpha = lua_tonumber(L, 3); 72 | if (lua_isnumber(L, 4)) ratio = lua_tonumber(L, 4); 73 | if (lua_isnumber(L, 5)) minWidth = lua_tonumber(L, 5); 74 | if (lua_isnumber(L, 6)) nOuterFPIterations = lua_tonumber(L, 6); 75 | if (lua_isnumber(L, 7)) nInnerFPIterations = lua_tonumber(L, 7); 76 | if (lua_isnumber(L, 8)) nCGIterations = lua_tonumber(L, 8); 77 | 78 | // copy tensors to images 79 | DImage *img1 = tensor_to_image(ten1); 80 | DImage *img2 = tensor_to_image(ten2); 81 | 82 | // declare outputs, and process 83 | DImage vx,vy,warpI2; 84 | OpticalFlow::Coarse2FineFlow(vx,vy,warpI2, // outputs 85 | *img1,*img2, // inputs 86 | alpha,ratio,minWidth, // params 87 | nOuterFPIterations,nInnerFPIterations,nCGIterations); 88 | 89 | // return result 90 | THFloatTensor *ten_vx = image_to_tensor(&vx); 91 | THFloatTensor *ten_vy = image_to_tensor(&vy); 92 | THFloatTensor *ten_warp = image_to_tensor(&warpI2); 93 | luaT_pushudata(L, ten_vx, luaT_checktypename2id(L, "torch.FloatTensor")); 94 | luaT_pushudata(L, ten_vy, luaT_checktypename2id(L, "torch.FloatTensor")); 95 | luaT_pushudata(L, ten_warp, luaT_checktypename2id(L, "torch.FloatTensor")); 96 | 97 | // cleanup 98 | delete(img1); 99 | delete(img2); 100 | 101 | return 3; 102 | } 103 | 104 | int warp_lua(lua_State *L) { 105 | // get args 106 | THFloatTensor *ten_inp = (THFloatTensor *)luaT_checkudata(L, 1, luaT_checktypename2id(L, "torch.Tensor")); 107 | THFloatTensor *ten_vx = (THFloatTensor *)luaT_checkudata(L, 2, luaT_checktypename2id(L, "torch.Tensor")); 108 | THFloatTensor *ten_vy = (THFloatTensor *)luaT_checkudata(L, 3, luaT_checktypename2id(L, "torch.Tensor")); 109 | 110 | // copy tensors to images 111 | DImage *input = tensor_to_image(ten_inp); 112 | DImage *vx = tensor_to_image(ten_vx); 113 | DImage *vy = tensor_to_image(ten_vy); 114 | 115 | // declare outputs, and process 116 | DImage warpedInput; 117 | OpticalFlow::warpFL(warpedInput, // warped input 118 | *input,*input, // input 119 | *vx, *vy // flow 120 | ); 121 | 122 | // return result 123 | THFloatTensor *ten_warp = image_to_tensor(&warpedInput); 124 | luaT_pushudata(L, ten_warp, luaT_checktypename2id(L, "torch.Tensor")); 125 | 126 | // cleanup 127 | delete(input); 128 | delete(vx); 129 | delete(vy); 130 | 131 | return 1; 132 | } 133 | 134 | // Register functions in LUA 135 | static const struct luaL_reg liuflow [] = { 136 | {"infer", optflow_lua}, 137 | {"warp", warp_lua}, 138 | {NULL, NULL} /* sentinel */ 139 | }; 140 | 141 | extern "C" { 142 | int luaopen_libliuflow (lua_State *L) { 143 | luaL_openlib(L, "libliuflow", liuflow, 0); 144 | return 1; 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /project.h: -------------------------------------------------------------------------------- 1 | 2 | // some global macros 3 | #define __min(x,y) ((x)<(y)?(x):(y)) 4 | #define __max(x,y) ((x)>(y)?(x):(y)) 5 | --------------------------------------------------------------------------------