├── .gitignore ├── ReadMe.md ├── cmd.example ├── color_flow ├── .gitignore ├── Makefile ├── README.txt ├── color_flow ├── color_flow.cpp ├── colorcode.cpp ├── colorcode.h ├── colortest ├── colortest.cpp ├── flowIO.cpp ├── flowIO.h └── imageLib │ ├── Convert.cpp │ ├── Convert.h │ ├── Convolve.cpp │ ├── Convolve.h │ ├── Copyright.h │ ├── Error.h │ ├── Image.cpp │ ├── Image.h │ ├── ImageIO.cpp │ ├── ImageIO.h │ ├── ImageIOpng.cpp │ ├── Makefile │ ├── README.txt │ ├── RefCntMem.cpp │ ├── RefCntMem.h │ └── imageLib.h ├── deep_flow2 ├── .gitignore ├── COPYING ├── Makefile ├── Makefile.ddk ├── Makefile.zhujin.s ├── README ├── deepflow2 ├── deepflow2-static ├── deepflow2.c ├── deepflow2.i ├── deepflow2.m ├── deepflow2.py ├── deepflow2_matlab.cpp ├── deepflow2_wrap.c ├── image.c ├── image.h ├── io.c ├── io.h ├── opticalflow.c ├── opticalflow.h ├── opticalflow_aux.c ├── opticalflow_aux.h ├── solver.c └── solver.h ├── deep_matching ├── .gitignore ├── Makefile ├── Makefile.bak ├── README.txt ├── caffe ├── caffe.zip ├── deep_matching_gpu.py ├── extra_layers.cpp ├── extra_layers.cu ├── extra_layers.hpp ├── flow_utils.py ├── gpudm.py ├── gpudm.swig ├── gpudm_wrap.cxx ├── helper.py ├── my_im2col.cpp ├── my_im2col.cu ├── my_im2col.hpp ├── net.py ├── numpy_image.h └── numpy_image.swg └── optical_flow ├── bbc_pose.sh ├── flic_movies.sh ├── flow_pipeline.py ├── one_pic.sh ├── origin_pipeline.py └── origin_pipeline.sh /.gitignore: -------------------------------------------------------------------------------- 1 | ## General 2 | 3 | # Compiled Object files 4 | *.slo 5 | *.lo 6 | *.o 7 | *.cuo 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | *.dylib 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | 18 | # Compiled protocol buffers 19 | *.pb.h 20 | *.pb.cc 21 | *_pb2.py 22 | 23 | # Compiled python 24 | *.pyc 25 | 26 | # Compiled MATLAB 27 | *.mex* 28 | 29 | # IPython notebook checkpoints 30 | .ipynb_checkpoints 31 | 32 | # Editor temporaries 33 | *.swp 34 | *~ 35 | 36 | # Sublime Text settings 37 | *.sublime-workspace 38 | *.sublime-project 39 | 40 | # Eclipse Project settings 41 | *.*project 42 | .settings 43 | 44 | # QtCreator files 45 | *.user 46 | 47 | # PyCharm files 48 | .idea 49 | 50 | # OSX dir files 51 | .DS_Store 52 | 53 | ## Caffe 54 | 55 | # User's build configuration 56 | Makefile.config 57 | 58 | # Data and models are either 59 | # 1. reference, and not casually committed 60 | # 2. custom, and live on their own unless they're deliberated contributed 61 | data/* 62 | models/* 63 | *.caffemodel 64 | *.caffemodel.h5 65 | *.solverstate 66 | *.solverstate.h5 67 | *.binaryproto 68 | *leveldb 69 | *lmdb 70 | 71 | # build, distribute, and bins (+ python proto bindings) 72 | build 73 | .build_debug/* 74 | .build_release/* 75 | distribute/* 76 | *.testbin 77 | *.bin 78 | python/caffe/proto/ 79 | cmake_build 80 | .cmake_build 81 | 82 | # Generated documentation 83 | docs/_site 84 | docs/gathered 85 | _site 86 | doxygen 87 | docs/dev 88 | 89 | # LevelDB files 90 | *.sst 91 | *.ldb 92 | LOCK 93 | LOG* 94 | CURRENT 95 | MANIFEST-* 96 | 97 | # images 98 | images/ -------------------------------------------------------------------------------- /ReadMe.md: -------------------------------------------------------------------------------- 1 | ### Declaration #### 2 | This code is used for computing optical flow between two images. 3 | 4 | This code is mentioned only for scientific or personal use. 5 | Please contact `DeepFlow`, `DeepMatching` and `ColorFlow` for commercial use. 6 | 7 | DeepFlow: 8 | http://lear.inrialpes.fr/src/deepflow/ 9 | 10 | DeepMatching: 11 | http://lear.inrialpes.fr/src/deepmatching/ 12 | 13 | ColorFlow: 14 | http://vision.middlebury.edu/flow/data/ 15 | 16 | ### Pipeline ### 17 | ``` 18 | 0: prepare the images pairs 19 | 1: compute matches using `DeepMatching` 20 | 2: compute flo file using `DeepFlow` 21 | 3: compute optical flow using `ColorFlow` 22 | ``` 23 | 24 | 25 | ### Installation ### 26 | ``` 27 | 1 download the code, put them into some directory 28 | 29 | 2 compiling `deep_matching`, `deep_flow2`, and `color_flow` 30 | please refer to `README` of each of them for more details. 31 | ``` 32 | **The program was only tested under a 64-bit Linux distribution (Ubuntu 14.04).** 33 | 34 | ### Example ### 35 | cd optical_flow && sh flow_pipeline.sh 36 | 37 | or see `cmd.example` for more examples 38 | 39 | 40 | -------------------------------------------------------------------------------- /cmd.example: -------------------------------------------------------------------------------- 1 | # demo 0 2 | cd /path/to/web_gpudm_1 3 | python deep_matching_gpu.py demo1.jpg demo2.jpg -GPU -v --downscale 1 --ngh_rad 256 --use_sparse |cat 4 | python deep_matching_gpu.py demo1.jpg demo2.jpg -GPU -v --downscale 1 --ngh_rad 256 --use_sparse -out demo1.txt 5 | 6 | # demo 1 7 | cd /path/to/DeepFlow_release2.0 8 | 9 | no match: 10 | ./deepflow2 demo1.jpg demo2.jpg demo.flo 11 | 12 | with match: 13 | python ../deep_matching/deep_matching_gpu.py demo1.jpg demo2.jpg -GPU -v --downscale 1 --ngh_rad 256 --use_sparse |cat | ./deepflow2 demo1.jpg demo2.jpg demo.flo 14 | python ../deep_matching/deep_matching_gpu.py demo1.jpg demo2.jpg -GPU -v --downscale 1 --ngh_rad 256 --use_sparse |cat | \ 15 | ./deepflow2 demo1.jpg demo2.jpg demo.flo -match 16 | -------------------------------------------------------------------------------- /color_flow/.gitignore: -------------------------------------------------------------------------------- 1 | ## General 2 | 3 | # Compiled Object files 4 | *.slo 5 | *.lo 6 | *.o 7 | *.cuo 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | *.dylib 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | 18 | # Compiled protocol buffers 19 | *.pb.h 20 | *.pb.cc 21 | *_pb2.py 22 | 23 | # Compiled python 24 | *.pyc 25 | 26 | # Compiled MATLAB 27 | *.mex* 28 | 29 | # IPython notebook checkpoints 30 | .ipynb_checkpoints 31 | 32 | # Editor temporaries 33 | *.swp 34 | *~ 35 | 36 | # Sublime Text settings 37 | *.sublime-workspace 38 | *.sublime-project 39 | 40 | # Eclipse Project settings 41 | *.*project 42 | .settings 43 | 44 | # QtCreator files 45 | *.user 46 | 47 | # PyCharm files 48 | .idea 49 | 50 | # OSX dir files 51 | .DS_Store 52 | 53 | ## Caffe 54 | 55 | # User's build configuration 56 | Makefile.config 57 | 58 | # Data and models are either 59 | # 1. reference, and not casually committed 60 | # 2. custom, and live on their own unless they're deliberated contributed 61 | data/* 62 | models/* 63 | *.caffemodel 64 | *.caffemodel.h5 65 | *.solverstate 66 | *.solverstate.h5 67 | *.binaryproto 68 | *leveldb 69 | *lmdb 70 | 71 | # build, distribute, and bins (+ python proto bindings) 72 | build 73 | .build_debug/* 74 | .build_release/* 75 | distribute/* 76 | *.testbin 77 | *.bin 78 | python/caffe/proto/ 79 | cmake_build 80 | .cmake_build 81 | 82 | # Generated documentation 83 | docs/_site 84 | docs/gathered 85 | _site 86 | doxygen 87 | docs/dev 88 | 89 | # LevelDB files 90 | *.sst 91 | *.ldb 92 | LOCK 93 | LOG* 94 | CURRENT 95 | MANIFEST-* 96 | 97 | # images 98 | *.jpg 99 | *.png 100 | *.flo 101 | -------------------------------------------------------------------------------- /color_flow/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for flow evaluation code 2 | 3 | SRC = flowIO.cpp colorcode.cpp colortest.cpp color_flow.cpp 4 | BIN = colortest color_flow 5 | 6 | IMGLIB = imageLib 7 | 8 | CC = g++ 9 | WARN = -W -Wall 10 | OPT ?= -O3 11 | CPPFLAGS = $(OPT) $(WARN) -I$(IMGLIB) 12 | LDLIBS = -L$(IMGLIB) -lImg -lpng -lz 13 | EXE = $(SRC:.cpp=.exe) 14 | 15 | all: $(BIN) 16 | 17 | colortest: colortest.cpp colorcode.cpp 18 | color_flow: color_flow.cpp flowIO.cpp colorcode.cpp 19 | 20 | clean: 21 | rm -f core *.stackdump 22 | 23 | allclean: clean 24 | rm -f $(BIN) $(EXE) 25 | -------------------------------------------------------------------------------- /color_flow/README.txt: -------------------------------------------------------------------------------- 1 | Some utilities for reading, writing, and color-coding .flo images 2 | 3 | Daniel Scharstein, 7/2/07 4 | updated 2/9/08 to fix bug in color_flow.cpp 5 | updated 6/9/09 to make robust to NaN or constant 0 flow (thanks Jan Bouecke) 6 | 7 | See flowIO.cpp for sample code for reading and writing .flo files. 8 | Here's an excerpt from this file describing the flow file format: 9 | 10 | // ".flo" file format used for optical flow evaluation 11 | // 12 | // Stores 2-band float image for horizontal (u) and vertical (v) flow components. 13 | // Floats are stored in little-endian order. 14 | // A flow value is considered "unknown" if either |u| or |v| is greater than 1e9. 15 | // 16 | // bytes contents 17 | // 18 | // 0-3 tag: "PIEH" in ASCII, which in little endian happens to be the float 202021.25 19 | // (just a sanity check that floats are represented correctly) 20 | // 4-7 width as an integer 21 | // 8-11 height as an integer 22 | // 12-end data (width*height*2*4 bytes total) 23 | // the float values for u and v, interleaved, in row order, i.e., 24 | // u[row0,col0], v[row0,col0], u[row0,col1], v[row0,col1], ... 25 | // 26 | 27 | 28 | Once you have a .flo file, you can create a color coding of it using 29 | color_flow 30 | 31 | Use colortest to visualize the encoding 32 | 33 | 34 | To compile 35 | 36 | cd imageLib 37 | make 38 | cd .. 39 | make 40 | ./colortest 10 colors.png 41 | 42 | /// optical flow 43 | ./color_flow mude.flo mude.png 44 | 45 | -------------------------------------------------------------------------------- /color_flow/color_flow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/color_flow -------------------------------------------------------------------------------- /color_flow/color_flow.cpp: -------------------------------------------------------------------------------- 1 | // color_flow.cpp 2 | // color-code motion field 3 | // normalizes based on specified value, or on maximum motion present otherwise 4 | 5 | // DS 2/9/08 fixed bug in MotionToColor concerning reallocation of colim (thanks Yunpeng!) 6 | 7 | static const char *usage = "\n usage: %s [-quiet] in.flo out.png [maxmotion]\n"; 8 | 9 | #include 10 | #include 11 | #include "imageLib.h" 12 | #include "flowIO.h" 13 | #include "colorcode.h" 14 | 15 | int verbose = 1; 16 | 17 | void MotionToColor(CFloatImage motim, CByteImage &colim, float maxmotion) 18 | { 19 | CShape sh = motim.Shape(); 20 | int width = sh.width, height = sh.height; 21 | colim.ReAllocate(CShape(width, height, 3)); 22 | int x, y; 23 | // determine motion range: 24 | float maxx = -999, maxy = -999; 25 | float minx = 999, miny = 999; 26 | float maxrad = -1; 27 | for (y = 0; y < height; y++) { 28 | for (x = 0; x < width; x++) { 29 | float fx = motim.Pixel(x, y, 0); 30 | float fy = motim.Pixel(x, y, 1); 31 | if (unknown_flow(fx, fy)) 32 | continue; 33 | maxx = __max(maxx, fx); 34 | maxy = __max(maxy, fy); 35 | minx = __min(minx, fx); 36 | miny = __min(miny, fy); 37 | float rad = sqrt(fx * fx + fy * fy); 38 | maxrad = __max(maxrad, rad); 39 | } 40 | } 41 | printf("max motion: %.4f motion range: u = %.3f .. %.3f; v = %.3f .. %.3f\n", 42 | maxrad, minx, maxx, miny, maxy); 43 | 44 | 45 | if (maxmotion > 0) // i.e., specified on commandline 46 | maxrad = maxmotion; 47 | 48 | if (maxrad == 0) // if flow == 0 everywhere 49 | maxrad = 1; 50 | 51 | if (verbose) 52 | fprintf(stderr, "normalizing by %g\n", maxrad); 53 | 54 | for (y = 0; y < height; y++) { 55 | for (x = 0; x < width; x++) { 56 | float fx = motim.Pixel(x, y, 0); 57 | float fy = motim.Pixel(x, y, 1); 58 | uchar *pix = &colim.Pixel(x, y, 0); 59 | if (unknown_flow(fx, fy)) { 60 | pix[0] = pix[1] = pix[2] = 0; 61 | } else { 62 | computeColor(fx/maxrad, fy/maxrad, pix); 63 | } 64 | } 65 | } 66 | } 67 | 68 | int main(int argc, char *argv[]) 69 | { 70 | try { 71 | int argn = 1; 72 | if (argc > 1 && argv[1][0]=='-' && argv[1][1]=='q') { 73 | verbose = 0; 74 | argn++; 75 | } 76 | if (argn >= argc-3 && argn <= argc-2) { 77 | char *flowname = argv[argn++]; 78 | char *outname = argv[argn++]; 79 | float maxmotion = argn < argc ? atof(argv[argn++]) : -1; 80 | CFloatImage im, fband; 81 | ReadFlowFile(im, flowname); 82 | CByteImage band, outim; 83 | CShape sh = im.Shape(); 84 | sh.nBands = 3; 85 | outim.ReAllocate(sh); 86 | outim.ClearPixels(); 87 | MotionToColor(im, outim, maxmotion); 88 | WriteImageVerb(outim, outname, verbose); 89 | } else 90 | throw CError(usage, argv[0]); 91 | } 92 | catch (CError &err) { 93 | fprintf(stderr, err.message); 94 | fprintf(stderr, "\n"); 95 | return -1; 96 | } 97 | 98 | return 0; 99 | } 100 | -------------------------------------------------------------------------------- /color_flow/colorcode.cpp: -------------------------------------------------------------------------------- 1 | // colorcode.cpp 2 | // 3 | // Color encoding of flow vectors 4 | // adapted from the color circle idea described at 5 | // http://members.shaw.ca/quadibloc/other/colint.htm 6 | // 7 | // Daniel Scharstein, 4/2007 8 | // added tick marks and out-of-range coding 6/05/07 9 | 10 | #include 11 | #include 12 | typedef unsigned char uchar; 13 | 14 | int ncols = 0; 15 | #define MAXCOLS 60 16 | int colorwheel[MAXCOLS][3]; 17 | 18 | 19 | void setcols(int r, int g, int b, int k) 20 | { 21 | colorwheel[k][0] = r; 22 | colorwheel[k][1] = g; 23 | colorwheel[k][2] = b; 24 | } 25 | 26 | void makecolorwheel() 27 | { 28 | // relative lengths of color transitions: 29 | // these are chosen based on perceptual similarity 30 | // (e.g. one can distinguish more shades between red and yellow 31 | // than between yellow and green) 32 | int RY = 15; 33 | int YG = 6; 34 | int GC = 4; 35 | int CB = 11; 36 | int BM = 13; 37 | int MR = 6; 38 | ncols = RY + YG + GC + CB + BM + MR; 39 | //printf("ncols = %d\n", ncols); 40 | if (ncols > MAXCOLS) 41 | exit(1); 42 | int i; 43 | int k = 0; 44 | for (i = 0; i < RY; i++) setcols(255, 255*i/RY, 0, k++); 45 | for (i = 0; i < YG; i++) setcols(255-255*i/YG, 255, 0, k++); 46 | for (i = 0; i < GC; i++) setcols(0, 255, 255*i/GC, k++); 47 | for (i = 0; i < CB; i++) setcols(0, 255-255*i/CB, 255, k++); 48 | for (i = 0; i < BM; i++) setcols(255*i/BM, 0, 255, k++); 49 | for (i = 0; i < MR; i++) setcols(255, 0, 255-255*i/MR, k++); 50 | } 51 | 52 | void computeColor(float fx, float fy, uchar *pix) 53 | { 54 | if (ncols == 0) 55 | makecolorwheel(); 56 | 57 | float rad = sqrt(fx * fx + fy * fy); 58 | float a = atan2(-fy, -fx) / M_PI; 59 | float fk = (a + 1.0) / 2.0 * (ncols-1); 60 | int k0 = (int)fk; 61 | int k1 = (k0 + 1) % ncols; 62 | float f = fk - k0; 63 | //f = 0; // uncomment to see original color wheel 64 | for (int b = 0; b < 3; b++) { 65 | float col0 = colorwheel[k0][b] / 255.0; 66 | float col1 = colorwheel[k1][b] / 255.0; 67 | float col = (1 - f) * col0 + f * col1; 68 | if (rad <= 1) 69 | col = 1 - rad * (1 - col); // increase saturation with radius 70 | else 71 | col *= .75; // out of range 72 | pix[2 - b] = (int)(255.0 * col); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /color_flow/colorcode.h: -------------------------------------------------------------------------------- 1 | void computeColor(float fx, float fy, uchar *pix); 2 | -------------------------------------------------------------------------------- /color_flow/colortest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/colortest -------------------------------------------------------------------------------- /color_flow/colortest.cpp: -------------------------------------------------------------------------------- 1 | // colortest.cpp 2 | 3 | // create a test image showing the color encoding 4 | 5 | static char usage[] = "usage: %s range outimage [size]\n"; 6 | 7 | # include 8 | # include 9 | #include "imageLib.h" 10 | #include "colorcode.h" 11 | 12 | int main(int argc, char **argv) 13 | { 14 | int verbose = 1; 15 | if (argc < 3) { 16 | fprintf(stderr, usage, argv[0]); 17 | exit(1); 18 | } 19 | int optind = 1; 20 | float truerange = atof(argv[optind++]); 21 | char *outname = argv[optind++]; 22 | int size = optind < argc ? atoi(argv[optind++]) : 151; 23 | 24 | float range = 1.04 * truerange; // make picture a bit bigger to show out-of-range coding 25 | try { 26 | CShape sh(size, size, 3); 27 | CByteImage out(sh); 28 | 29 | int s2 = size/2; 30 | for (int y = 0; y < size; y++) { 31 | for (int x = 0; x < size; x++) { 32 | float fx = (float)x / (float)s2 * range - range; 33 | float fy = (float)y / (float)s2 * range - range; 34 | if (x == s2 || y == s2) // make black coordinate axes 35 | continue; 36 | uchar *pix = &out.Pixel(x, y, 0); 37 | //fx = rintf(fx); 38 | //fy = rintf(fy); 39 | computeColor(fx/truerange, fy/truerange, pix); 40 | } 41 | } 42 | int ir = (int)truerange; 43 | int ticksize = size < 120 ? 1 : 2; 44 | for (int k = -ir; k <= ir; k++) { 45 | int ik = (int)(k / range * s2) + s2; 46 | for (int t = -ticksize; t <= ticksize; t++) { 47 | uchar *pix; 48 | pix = &out.Pixel(ik, s2 + t, 0); pix[0] = pix[1] = pix[2] = 0; 49 | pix = &out.Pixel(s2 + t, ik, 0); pix[0] = pix[1] = pix[2] = 0; 50 | } 51 | } 52 | 53 | WriteImageVerb(out, outname, verbose); 54 | } 55 | catch (CError &err) { 56 | fprintf(stderr, err.message); 57 | fprintf(stderr, "\n"); 58 | exit(1); 59 | } 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /color_flow/flowIO.cpp: -------------------------------------------------------------------------------- 1 | // flow_io.cpp 2 | // 3 | // read and write our simple .flo flow file format 4 | 5 | // ".flo" file format used for optical flow evaluation 6 | // 7 | // Stores 2-band float image for horizontal (u) and vertical (v) flow components. 8 | // Floats are stored in little-endian order. 9 | // A flow value is considered "unknown" if either |u| or |v| is greater than 1e9. 10 | // 11 | // bytes contents 12 | // 13 | // 0-3 tag: "PIEH" in ASCII, which in little endian happens to be the float 202021.25 14 | // (just a sanity check that floats are represented correctly) 15 | // 4-7 width as an integer 16 | // 8-11 height as an integer 17 | // 12-end data (width*height*2*4 bytes total) 18 | // the float values for u and v, interleaved, in row order, i.e., 19 | // u[row0,col0], v[row0,col0], u[row0,col1], v[row0,col1], ... 20 | // 21 | 22 | 23 | // first four bytes, should be the same in little endian 24 | #define TAG_FLOAT 202021.25 // check for this when READING the file 25 | #define TAG_STRING "PIEH" // use this when WRITING the file 26 | 27 | 28 | #include 29 | #include 30 | #include 31 | #include "imageLib.h" 32 | #include "flowIO.h" 33 | 34 | // return whether flow vector is unknown 35 | bool unknown_flow(float u, float v) { 36 | return (fabs(u) > UNKNOWN_FLOW_THRESH) 37 | || (fabs(v) > UNKNOWN_FLOW_THRESH) 38 | || isnan(u) || isnan(v); 39 | } 40 | 41 | bool unknown_flow(float *f) { 42 | return unknown_flow(f[0], f[1]); 43 | } 44 | 45 | // read a flow file into 2-band image 46 | void ReadFlowFile(CFloatImage& img, const char* filename) 47 | { 48 | if (filename == NULL) 49 | throw CError("ReadFlowFile: empty filename"); 50 | 51 | const char *dot = strrchr(filename, '.'); 52 | if (strcmp(dot, ".flo") != 0) 53 | throw CError("ReadFlowFile (%s): extension .flo expected", filename); 54 | 55 | FILE *stream = fopen(filename, "rb"); 56 | if (stream == 0) 57 | throw CError("ReadFlowFile: could not open %s", filename); 58 | 59 | int width, height; 60 | float tag; 61 | 62 | if ((int)fread(&tag, sizeof(float), 1, stream) != 1 || 63 | (int)fread(&width, sizeof(int), 1, stream) != 1 || 64 | (int)fread(&height, sizeof(int), 1, stream) != 1) 65 | throw CError("ReadFlowFile: problem reading file %s", filename); 66 | 67 | if (tag != TAG_FLOAT) // simple test for correct endian-ness 68 | throw CError("ReadFlowFile(%s): wrong tag (possibly due to big-endian machine?)", filename); 69 | 70 | // another sanity check to see that integers were read correctly (99999 should do the trick...) 71 | if (width < 1 || width > 99999) 72 | throw CError("ReadFlowFile(%s): illegal width %d", filename, width); 73 | 74 | if (height < 1 || height > 99999) 75 | throw CError("ReadFlowFile(%s): illegal height %d", filename, height); 76 | 77 | int nBands = 2; 78 | CShape sh(width, height, nBands); 79 | img.ReAllocate(sh); 80 | 81 | //printf("reading %d x %d x 2 = %d floats\n", width, height, width*height*2); 82 | int n = nBands * width; 83 | for (int y = 0; y < height; y++) { 84 | float* ptr = &img.Pixel(0, y, 0); 85 | if ((int)fread(ptr, sizeof(float), n, stream) != n) 86 | throw CError("ReadFlowFile(%s): file is too short", filename); 87 | } 88 | 89 | if (fgetc(stream) != EOF) 90 | throw CError("ReadFlowFile(%s): file is too long", filename); 91 | 92 | fclose(stream); 93 | } 94 | 95 | // write a 2-band image into flow file 96 | void WriteFlowFile(CFloatImage img, const char* filename) 97 | { 98 | if (filename == NULL) 99 | throw CError("WriteFlowFile: empty filename"); 100 | 101 | const char *dot = strrchr(filename, '.'); 102 | if (dot == NULL) 103 | throw CError("WriteFlowFile: extension required in filename '%s'", filename); 104 | 105 | if (strcmp(dot, ".flo") != 0) 106 | throw CError("WriteFlowFile: filename '%s' should have extension '.flo'", filename); 107 | 108 | CShape sh = img.Shape(); 109 | int width = sh.width, height = sh.height, nBands = sh.nBands; 110 | 111 | if (nBands != 2) 112 | throw CError("WriteFlowFile(%s): image must have 2 bands", filename); 113 | 114 | FILE *stream = fopen(filename, "wb"); 115 | if (stream == 0) 116 | throw CError("WriteFlowFile: could not open %s", filename); 117 | 118 | // write the header 119 | fprintf(stream, TAG_STRING); 120 | if ((int)fwrite(&width, sizeof(int), 1, stream) != 1 || 121 | (int)fwrite(&height, sizeof(int), 1, stream) != 1) 122 | throw CError("WriteFlowFile(%s): problem writing header", filename); 123 | 124 | // write the rows 125 | int n = nBands * width; 126 | for (int y = 0; y < height; y++) { 127 | float* ptr = &img.Pixel(0, y, 0); 128 | if ((int)fwrite(ptr, sizeof(float), n, stream) != n) 129 | throw CError("WriteFlowFile(%s): problem writing data", filename); 130 | } 131 | 132 | fclose(stream); 133 | } 134 | 135 | 136 | /* 137 | int main() { 138 | 139 | try { 140 | CShape sh(5, 1, 2); 141 | CFloatImage img(sh); 142 | img.ClearPixels(); 143 | img.Pixel(0, 0, 0) = -5.0f; 144 | char *filename = "test.flo"; 145 | 146 | WriteFlowFile(img, filename); 147 | ReadFlowFile(img, filename); 148 | } 149 | catch (CError &err) { 150 | fprintf(stderr, err.message); 151 | fprintf(stderr, "\n"); 152 | exit(1); 153 | } 154 | 155 | return 0; 156 | } 157 | */ 158 | -------------------------------------------------------------------------------- /color_flow/flowIO.h: -------------------------------------------------------------------------------- 1 | // flowIO.h 2 | 3 | // the "official" threshold - if the absolute value of either 4 | // flow component is greater, it's considered unknown 5 | #define UNKNOWN_FLOW_THRESH 1e9 6 | 7 | // value to use to represent unknown flow 8 | #define UNKNOWN_FLOW 1e10 9 | 10 | // return whether flow vector is unknown 11 | bool unknown_flow(float u, float v); 12 | bool unknown_flow(float *f); 13 | 14 | // read a flow file into 2-band image 15 | void ReadFlowFile(CFloatImage& img, const char* filename); 16 | 17 | // write a 2-band image into flow file 18 | void WriteFlowFile(CFloatImage img, const char* filename); 19 | 20 | 21 | -------------------------------------------------------------------------------- /color_flow/imageLib/Convert.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Convert.cpp -------------------------------------------------------------------------------- /color_flow/imageLib/Convert.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Convert.h -------------------------------------------------------------------------------- /color_flow/imageLib/Convolve.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Convolve.cpp -------------------------------------------------------------------------------- /color_flow/imageLib/Convolve.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Convolve.h -------------------------------------------------------------------------------- /color_flow/imageLib/Copyright.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | StereoMatcher 1.0 4 | 5 | This Microsoft Research Source Code License Agreement ("MSR-SCLA") is 6 | a legal agreement between you and Microsoft Corporation 7 | (``Microsoft'' or ``we'') for the above pre-release Software. 8 | 9 | By installing, copying, compiling or otherwise using the Software found at 10 | http://research.microsoft.com/downloads/, 11 | you agree to be bound by the terms of this MSR-SCLA. If you do not agree, 12 | do not install, copy, compile or use the Software. The Software is protected 13 | by copyright and other intellectual property laws and is licensed, not sold. 14 | 15 | You may do anything you want with the Software for non-commercial research or 16 | teaching purposes, free of charge. In return, we simply ask that you agree to 17 | the following: 18 | 19 | * To leave in place all copyright notices and licensing information that you 20 | might find in the Software. 21 | 22 | * That you will not use the Software in a live operating environment where it 23 | may be relied upon to perform in the same manner as a commercially released 24 | product, or with data that has not been sufficiently backed up. 25 | 26 | * To make freely available to others the source code of any modifications or 27 | additions you make to the Software source code, and any related documentation, 28 | solely and exclusively under the same terms as this License. 29 | 30 | * That Microsoft is granted back, without limitations, the rights to reproduce, 31 | install, use, modify, distribute and transfer your source modifications or 32 | additions in source and/or object code formats. 33 | 34 | * That any feedback about the Software provided by you to us is voluntarily 35 | given, and Microsoft shall be free to use the feedback as it sees fit without 36 | obligation or restriction of any kind, even if the feedback is designated by 37 | you as confidential. 38 | 39 | * NO WARRANTIES WHATSOEVER: That the Software comes ``AS IS'', with all 40 | faults and with no warranties, conditions or representations. None. 41 | Not even the implied warranty of merchantability, warranty of fitness for 42 | a particular purpose, or any warranty against interference with your 43 | enjoyment of the Software or against infringement. The entire risk as to 44 | satisfactory quality, performance, accuracy, and effort is with you. 45 | There is no warranty that this Software will fulfill any of your particular 46 | purposes or needs. 47 | 48 | * That we have no duty of reasonable care or lack of negligence and that we 49 | are not obligated to (and will not) provide technical support or updates for 50 | the Software. 51 | 52 | * That we will not be liable for any damages, including those known as direct, 53 | indirect, special, consequential, or incidental damages related to the Software 54 | or this MSR-SCLA, nor any damages for negligence, to the maximum extent the 55 | law permits. 56 | 57 | * That if you sue or threaten to sue anyone over patents that you think may 58 | apply to the Software or if you breach this MSR-SCLA in any way, your license 59 | to the Software ends automatically. 60 | 61 | * That this MSR-SCLA shall be construed and controlled by the laws of the 62 | State of Washington, USA, without regard to conflicts of law. 63 | 64 | 65 | Copyright (c) Microsoft Corporation. All rights reserved 66 | 67 | */ 68 | -------------------------------------------------------------------------------- /color_flow/imageLib/Error.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Error.h -------------------------------------------------------------------------------- /color_flow/imageLib/Image.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Image.cpp -------------------------------------------------------------------------------- /color_flow/imageLib/Image.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/Image.h -------------------------------------------------------------------------------- /color_flow/imageLib/ImageIO.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/ImageIO.cpp -------------------------------------------------------------------------------- /color_flow/imageLib/ImageIO.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/ImageIO.h -------------------------------------------------------------------------------- /color_flow/imageLib/ImageIOpng.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/ImageIOpng.cpp -------------------------------------------------------------------------------- /color_flow/imageLib/Makefile: -------------------------------------------------------------------------------- 1 | SRC = Convert.cpp Convolve.cpp Image.cpp ImageIO.cpp ImageIOpng.cpp RefCntMem.cpp 2 | 3 | CC = g++ 4 | WARN = -W -Wall 5 | OPT ?= -O3 6 | CPPFLAGS = $(OPT) $(WARN) 7 | 8 | OBJ = $(SRC:.cpp=.o) 9 | 10 | all: libImg.a 11 | 12 | libImg.a: $(OBJ) 13 | rm -f libImg.a 14 | ar ruc libImg.a $(OBJ) 15 | ranlib libImg.a 16 | 17 | clean: 18 | rm -f $(OBJ) core *.stackdump *.bak 19 | 20 | allclean: clean 21 | rm -f libImg.a 22 | 23 | depend: 24 | @makedepend -Y -- $(CPPFLAGS) -- $(SRC) 2>> /dev/null 25 | 26 | # DO NOT DELETE THIS LINE -- make depend depends on it. 27 | 28 | Convert.o: Image.h RefCntMem.h Error.h Convert.h 29 | Convolve.o: Image.h RefCntMem.h Error.h Convert.h Convolve.h 30 | Image.o: Image.h RefCntMem.h Error.h 31 | ImageIO.o: Image.h RefCntMem.h Error.h ImageIO.h 32 | ImageIOpng.o: Image.h RefCntMem.h Error.h 33 | RefCntMem.o: RefCntMem.h 34 | -------------------------------------------------------------------------------- /color_flow/imageLib/README.txt: -------------------------------------------------------------------------------- 1 | imageLib is a small C++ library for 2D multi-band images. See Image.h 2 | for more detail. Reading and writing of images is supported in png 3 | and pgm/ppm formats, as well as a subset of Targa. See ImageIO.h for 4 | more detail. 5 | 6 | This code is derived from a subset of the StereoMatcher code by Rick 7 | Szeliski and Daniel Scharstein, which is available at 8 | http://research.microsoft.com/downloads/. 9 | 10 | This code is distributed under the same Source Code License Agreement 11 | as the original version. Please see the file Copyright.h 12 | 13 | Last change: 8/30/2012 - to get to compile under gcc 4 without warnings, and some minor fixes 14 | -------------------------------------------------------------------------------- /color_flow/imageLib/RefCntMem.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/RefCntMem.cpp -------------------------------------------------------------------------------- /color_flow/imageLib/RefCntMem.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/color_flow/imageLib/RefCntMem.h -------------------------------------------------------------------------------- /color_flow/imageLib/imageLib.h: -------------------------------------------------------------------------------- 1 | // imageLib.h 2 | 3 | // common includes 4 | 5 | #include "Error.h" 6 | #include "Image.h" 7 | #include "ImageIO.h" 8 | #include "Convert.h" 9 | -------------------------------------------------------------------------------- /deep_flow2/.gitignore: -------------------------------------------------------------------------------- 1 | ## General 2 | 3 | # Compiled Object files 4 | *.slo 5 | *.lo 6 | *.o 7 | *.cuo 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | *.dylib 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | 18 | # Compiled protocol buffers 19 | *.pb.h 20 | *.pb.cc 21 | *_pb2.py 22 | 23 | # Compiled python 24 | *.pyc 25 | 26 | # Compiled MATLAB 27 | *.mex* 28 | 29 | # IPython notebook checkpoints 30 | .ipynb_checkpoints 31 | 32 | # Editor temporaries 33 | *.swp 34 | *~ 35 | 36 | # Sublime Text settings 37 | *.sublime-workspace 38 | *.sublime-project 39 | 40 | # Eclipse Project settings 41 | *.*project 42 | .settings 43 | 44 | # QtCreator files 45 | *.user 46 | 47 | # PyCharm files 48 | .idea 49 | 50 | # OSX dir files 51 | .DS_Store 52 | 53 | ## Caffe 54 | 55 | # User's build configuration 56 | Makefile.config 57 | 58 | # Data and models are either 59 | # 1. reference, and not casually committed 60 | # 2. custom, and live on their own unless they're deliberated contributed 61 | data/* 62 | models/* 63 | *.caffemodel 64 | *.caffemodel.h5 65 | *.solverstate 66 | *.solverstate.h5 67 | *.binaryproto 68 | *leveldb 69 | *lmdb 70 | 71 | # build, distribute, and bins (+ python proto bindings) 72 | build 73 | .build_debug/* 74 | .build_release/* 75 | distribute/* 76 | *.testbin 77 | *.bin 78 | python/caffe/proto/ 79 | cmake_build 80 | .cmake_build 81 | 82 | # Generated documentation 83 | docs/_site 84 | docs/gathered 85 | _site 86 | doxygen 87 | docs/dev 88 | 89 | # LevelDB files 90 | *.sst 91 | *.ldb 92 | LOCK 93 | LOG* 94 | CURRENT 95 | MANIFEST-* 96 | 97 | # images 98 | *.jpg 99 | *.png 100 | *.flo 101 | -------------------------------------------------------------------------------- /deep_flow2/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | 3 | CFLAGS=-Wall -g -O3 -msse4 -fPIC 4 | LDFLAGS=-g -Wall -O3 -msse4 -fPIC 5 | LIBFLAGS=-lm -ljpeg -lpng -L/usr/lib/x86_64-linux-gnu 6 | LIBAFLAGS=-static /usr/lib/x86_64-linux-gnu/libjpeg.a /usr/lib/x86_64-linux-gnu/libpng.a /usr/lib/x86_64-linux-gnu/libz.a /usr/lib/x86_64-linux-gnu/libm.a 7 | CPYTHONFLAGS=-I/usr/include/python2.7 8 | 9 | SOURCES := $(shell find . -name '*.c' ! -name 'deepflow2_wrap.c') 10 | OBJ := $(SOURCES:%.c=%.o) 11 | HEADERS := $(shell find . -name '*.h') 12 | 13 | all: deepflow2 14 | 15 | deepflow2: $(OBJ) 16 | $(CC) $(LDFLAGS) $^ $(LIBFLAGS) -o $@ 17 | 18 | deepflow2-static: $(OBJ) 19 | $(CC) -o $@ $^ $(LIBAFLAGS) 20 | 21 | %.o: %.c 22 | $(CC) -o $@ $(CFLAGS) -c $+ 23 | 24 | python: all 25 | swig -python $(CPYTHONFLAGS) deepflow2.i 26 | gcc $(CFLAGS) -c deepflow2_wrap.c $(CPYTHONFLAGS) 27 | gcc -shared $(LDFLAGS) deepflow2_wrap.o $(OBJ) -o _deepflow2.so $(LIBFLAGS) 28 | 29 | 30 | clean: 31 | rm -f *.o deepflow2 _deepflow2.so deepflow2.pyc deepflow2.mex??? 32 | -------------------------------------------------------------------------------- /deep_flow2/Makefile.ddk: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | 3 | CFLAGS=-Wall -g -O3 -msse4 -fPIC 4 | LDFLAGS=-g -Wall -O3 -msse4 -fPIC 5 | LIBFLAGS=-lm -ljpeg -lpng 6 | LIBAFLAGS=-static /usr/lib/x86_64-linux-gnu/libjpeg.a /usr/lib/x86_64-linux-gnu/libpng.a /usr/lib/x86_64-linux-gnu/libz.a /usr/lib/x86_64-linux-gnu/libm.a 7 | CPYTHONFLAGS=-I/usr/include/python2.7 8 | 9 | SOURCES := $(shell find . -name '*.c' ! -name 'deepflow2_wrap.c') 10 | OBJ := $(SOURCES:%.c=%.o) 11 | HEADERS := $(shell find . -name '*.h') 12 | 13 | all: deepflow2 14 | 15 | deepflow2: $(OBJ) 16 | $(CC) $(LDFLAGS) $(LIBFLAGS) -o $@ $^ 17 | 18 | deepflow2-static: $(OBJ) 19 | $(CC) -o $@ $^ $(LIBAFLAGS) 20 | 21 | %.o: %.c 22 | $(CC) -o $@ $(CFLAGS) -c $+ 23 | 24 | python: all 25 | swig -python $(CPYTHONFLAGS) deepflow2.i 26 | gcc $(CFLAGS) -c deepflow2_wrap.c $(CPYTHONFLAGS) 27 | gcc -shared $(LDFLAGS) deepflow2_wrap.o $(OBJ) -o _deepflow2.so $(LIBFLAGS) 28 | 29 | 30 | clean: 31 | rm -f *.o deepflow2 _deepflow2.so deepflow2.pyc deepflow2.mex??? 32 | -------------------------------------------------------------------------------- /deep_flow2/Makefile.zhujin.s: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | 3 | CFLAGS=-Wall -g -O3 -msse4 -fPIC 4 | LDFLAGS=-g -Wall -O3 -msse4 -fPIC 5 | LIBFLAGS=-lm -ljpeg -lpng -L/usr/lib/x86_64-linux-gnu 6 | LIBAFLAGS=-static /usr/lib/x86_64-linux-gnu/libjpeg.a /usr/lib/x86_64-linux-gnu/libpng.a /usr/lib/x86_64-linux-gnu/libz.a /usr/lib/x86_64-linux-gnu/libm.a 7 | CPYTHONFLAGS=-I/usr/include/python2.7 8 | 9 | SOURCES := $(shell find . -name '*.c' ! -name 'deepflow2_wrap.c') 10 | OBJ := $(SOURCES:%.c=%.o) 11 | HEADERS := $(shell find . -name '*.h') 12 | 13 | all: deepflow2 14 | 15 | deepflow2: $(OBJ) 16 | $(CC) $(LDFLAGS) $^ $(LIBFLAGS) -o $@ 17 | 18 | deepflow2-static: $(OBJ) 19 | $(CC) -o $@ $^ $(LIBAFLAGS) 20 | 21 | %.o: %.c 22 | $(CC) -o $@ $(CFLAGS) -c $+ 23 | 24 | python: all 25 | swig -python $(CPYTHONFLAGS) deepflow2.i 26 | gcc $(CFLAGS) -c deepflow2_wrap.c $(CPYTHONFLAGS) 27 | gcc -shared $(LDFLAGS) deepflow2_wrap.o $(OBJ) -o _deepflow2.so $(LIBFLAGS) 28 | 29 | 30 | clean: 31 | rm -f *.o deepflow2 _deepflow2.so deepflow2.pyc deepflow2.mex??? 32 | -------------------------------------------------------------------------------- /deep_flow2/README: -------------------------------------------------------------------------------- 1 | DeepFlow, v2.0 2 | 3 | Our code is mentioned only for scientific or personal use. 4 | Please contact us for commercial use. 5 | 6 | DeepFlow2 is an improvement of DeepFlow that takes benefit of SSE instructions to speed up the computation and that incorporates a non-local weight for the smoothness term. 7 | 8 | ### Compiling ### 9 | 10 | Simply type 'make' and the code must compile. 11 | The program was only tested under a 64-bit Linux distribution. 12 | A static version is also provided in case of the compilation failure. 13 | We do not give any support for other OS or compilation issues. 14 | 15 | Dependencies: (my own on ubuntu 14.04) 16 | LIBAFLAGS=-static /usr/lib/x86_64-linux-gnu/libjpeg.a /usr/lib/x86_64-linux-gnu/libpng.a /usr/lib/x86_64-linux-gnu/libz.a /usr/lib/x86_64-linux-gnu/libm.a 17 | 18 | ### Using DeepFlow2 ### 19 | 20 | The general command to use DeepFlow2 is: 21 | ./deepflow2 [options] 22 | 23 | e.g: 24 | ./deepflow2 sintel1.png sintel2.png sintel.flo 25 | ./deepflow2 mude1.jpg mude2.jpg mude.flo 26 | 27 | For a list of the available options, you can type: 28 | ./deepflow2 -h 29 | >>> 30 | -h, --help print this message 31 | -a, -alpha (12.0) weight of smoothness terms 32 | -b, -beta (300.0) weight of descriptor matching 33 | -g, -gamma (3.0) weight of gradient constancy assumption 34 | -d, -delta (2.0) weight of color constancy assumption 35 | -s, -sigma (0.8) standard deviation of Gaussian presmoothing kernel 36 | -e, -eta (0.95) ratio factor for coarse-to-fine scheme 37 | -minsize (25) size of the coarsest level 38 | -inner (5) number of inner fixed point iterations 39 | -iter (25) number of iterations for the solver 40 | -soromega (1.6) omega parameter of the sor method 41 | -bk (0.0) use decreasing beta i.e. beta(k) = beta*( k / kmax )^betak, if 0, a last iteration is done with beta=0 42 | 43 | -match '-match filename' reads matches from a file and '-match' from stdin. Each line is the match with the first four numbers being x1 y1 x2 y2. 44 | 45 | -sintel set the parameters to the one used in the arXiv paper for MPI-Sintel dataset 46 | -middlebury set the parameters to the one used in the arXiv paper for middlebury dataset 47 | -kitti set the parameters to the one used in the arXiv paper for KITTI dataset 48 | 49 | The option -match allows to give input matches, either from an input file or from stdin. 50 | In particular, results from the paper use the results from DeepMatching. 51 | 52 | ### Example including DeepMatching ### 53 | 54 | To compute DeepFlow2 including the DeepMatching, you need to download the Deep Matching code on the dedicated webpage: http://lear.inrialpes.fr/src/deepmatching/. 55 | The matches output by DeepMatching can directly be piped to DeepFlow2 like in the following example: 56 | 57 | /deepmatching sintel1.png sintel2.png | ./deepflow2 sintel1.png sintel2.png sintel.flo -match -sintel 58 | (for small size images, you can add the option "-downscale 0" when extracting deepmatching for a better accuracy) 59 | 60 | It creates a .flo file in a standard format. 61 | For instance, code for reading and displaying such format is available with the Middlebury dataset. 62 | 63 | ### Warning: without matching ### 64 | 65 | If you don't use any input match (i.e. you don't specify any -match argument), the method will be limited to a standard variational method without matching integration and thus will lose accuracy, specially for large displacements. This is the case in the opencv implementation. 66 | 67 | ### Bugs and extensions ### 68 | 69 | If you find bugs, etc., please feel free to contact me. 70 | Contact details are available on my webpage. 71 | http://lear.inrialpes.fr/people/pweinzae 72 | 73 | ### History ### 74 | 75 | December 2013 v1.0.0 76 | March 2014 v1.0.1: memory leak fix + support for png images 77 | March 2014 FastDeepFlow_v1.0.1: faster (~2x) version using SSE instructions. Results are slightly different due to floating point approximations. 78 | September 2015 v2.0: incorporation of a local smoothness weight and of the computation of matches' weights 79 | 80 | ### Matlab wrapper ### 81 | 82 | To compile the wrapper, first compile the code using make, then open matlab and compile the mex-file: 83 | >> mex deepflow2_matlab.cpp image.o io.o opticalflow.o opticalflow_aux.o solver.o -ljpeg -lpng -lm -output deepflow2 84 | The wrapper has been tested on Matlab14b under Linux-64 bit. We do not provide any support for compilation issue. 85 | 86 | Usage with DeepMatching (assuming that DeepMatching has been added to the path with the matlab wrapper compiled; note that matlab must be opened with atlas preloaded for DeepMatching computation: LD_PRELOAD=/usr/lib64/atlas/libtatlas.so.3 matlab): 87 | >> im1 = single(imread('sintel1.png')); 88 | >> im2 = single(imread('sintel2.png')); 89 | >> matches = deepmatching(im1, im2); 90 | >> flow = deepflow2(im1, im2, matches, '-sintel'); 91 | Calling deepflow2() will show the list of available options. 92 | 93 | ### Python wrapper ### 94 | 95 | The python wrapper requires numpy (and swig if you modify the code and recompile it). To compile it, 96 | type 'make python'. The wrapper has been tested under Linux-64 bit only. We do not provide any support for compilation issue. 97 | make sure you have installed swig -> "sudo apt-get install swig" 98 | 99 | Usage with DeepMatching (assuming that DeepMatching has been added to the pythonpath with the python wrapper compiled): 100 | >> from deepmatching import deepmatching 101 | >> from deepflow2 import deepflow2 102 | >> import numpy 103 | >> from PIL import Image 104 | >> im1 = numpy.array(Image.open('sintel1.png')) 105 | >> im2 = numpy.array(Image.open('sintel2.png')) 106 | >> matches = deepmatching(im1, im2) 107 | >> flow = deepflow2(im1, im2, matches, '-sintel') 108 | Callind deepflow2() will show the list of available options. 109 | 110 | ### LICENCE CONDITIONS ### 111 | 112 | Copyright (C) 2013 Philippe Weinzaepfel 113 | 114 | This program is free software: you can redistribute it and/or modify 115 | it under the terms of the GNU General Public License as published by 116 | the Free Software Foundation, either version 3 of the License, or 117 | (at your option) any later version. 118 | 119 | This program is distributed in the hope that it will be useful, 120 | but WITHOUT ANY WARRANTY; without even the implied warranty of 121 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 122 | GNU General Public License for more details. 123 | 124 | You should have received a copy of the GNU General Public License 125 | along with this program. If not, see . 126 | -------------------------------------------------------------------------------- /deep_flow2/deepflow2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/deep_flow2/deepflow2 -------------------------------------------------------------------------------- /deep_flow2/deepflow2-static: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/deep_flow2/deepflow2-static -------------------------------------------------------------------------------- /deep_flow2/deepflow2.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #include 18 | #include 19 | #include 20 | 21 | #include "image.h" 22 | #include "opticalflow.h" 23 | #include "io.h" 24 | 25 | int main(int argc, char ** argv){ 26 | 27 | // load images 28 | if(argc < 4){ 29 | fprintf(stderr,"Wrong command, require at least 3 arguments.\n\n"); 30 | usage(EXE_OPTIONS); 31 | exit(1); 32 | } 33 | color_image_t *im1 = color_image_load(argv[1]), *im2 = color_image_load(argv[2]); 34 | if(im1->width != im2->width || im1->height != im2->height){ 35 | fprintf(stderr,"Image dimensions does not match\n"); 36 | exit(1); 37 | } 38 | 39 | // set params to default 40 | optical_flow_params_t* params = (optical_flow_params_t*) malloc(sizeof(optical_flow_params_t)); 41 | if(!params){ 42 | fprintf(stderr,"error deepflow2(): not enough memory\n"); 43 | exit(1); 44 | } 45 | optical_flow_params_default(params); 46 | 47 | // parse options 48 | image_t **matches = parse_options(params, argc-4, &argv[4], EXE_OPTIONS, im1->width, im1->height); 49 | image_t *match_x = matches[0], *match_y = matches[1], *match_z = matches[2]; 50 | free(matches); 51 | 52 | image_t *wx = image_new(im1->width,im1->height), *wy = image_new(im1->width,im1->height); 53 | optical_flow(wx, wy, im1, im2, params, match_x, match_y, match_z); 54 | writeFlowFile(argv[3], wx, wy); 55 | image_delete(wx); 56 | image_delete(wy); 57 | image_delete(match_x); image_delete(match_y); image_delete(match_z); 58 | color_image_delete(im1); color_image_delete(im2); 59 | free(params); 60 | 61 | return 0; 62 | } 63 | -------------------------------------------------------------------------------- /deep_flow2/deepflow2.i: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | /* 19 | Python wrapper 20 | */ 21 | 22 | %module(docstring="Module to compute DeepFlow") deepflow2 23 | 24 | %{ 25 | #define SWIG_FILE_WITH_INIT 26 | 27 | #include 28 | 29 | 30 | #define CHECK_NUMPY_ARRAY(a, expected_npy) \ 31 | if(!a) { \ 32 | fprintf(stderr,"error in %s(): NULL input\n",__PRETTY_FUNCTION__); \ 33 | return NULL; \ 34 | } \ 35 | if(!PyArray_Check(a)) { \ 36 | fprintf(stderr,"error in %s(): input not numpy array\n",__PRETTY_FUNCTION__); \ 37 | return NULL; \ 38 | } \ 39 | if(!PyArray_ISCONTIGUOUS(a)) { \ 40 | fprintf(stderr,"error in %s(): array is not C-contiguous\n",__PRETTY_FUNCTION__); \ 41 | return NULL; \ 42 | } \ 43 | if(PyArray_TYPE(a)!=expected_npy) { \ 44 | fprintf(stderr,"error in %s(): input has bad type (type id %d != " #expected_npy " %d)\n",__PRETTY_FUNCTION__, \ 45 | PyArray_TYPE(a),expected_npy); \ 46 | return NULL; \ 47 | } 48 | 49 | %} 50 | 51 | %init %{ 52 | import_array(); 53 | %} 54 | 55 | 56 | %{ 57 | #include "image.h" 58 | %} 59 | 60 | %typemap(in) 61 | (image_t* img) 62 | (image_t image) { 63 | 64 | PyObject* a = $input; 65 | if(a==Py_None) { 66 | $1 = NULL; 67 | } else { 68 | CHECK_NUMPY_ARRAY(a, NPY_FLOAT) 69 | image.data = (float*) PyArray_DATA(a); 70 | a = PyObject_GetAttrString($input,"shape"); 71 | assert(PyTuple_Size(a)==2); 72 | image.height = PyInt_AsLong(PyTuple_GetItem(a,0)); 73 | image.width = PyInt_AsLong(PyTuple_GetItem(a,1)); 74 | assert( image.width%4==0); 75 | image.stride = image.width; 76 | $1=ℑ 77 | } 78 | } 79 | %apply (image_t* img) {(image_t* )}; 80 | 81 | %typemap(in) 82 | (color_image_t* cimg) 83 | (color_image_t cimage) { 84 | 85 | PyObject* a = $input; 86 | if(a==Py_None) { 87 | $1 = NULL; 88 | } else { 89 | CHECK_NUMPY_ARRAY(a, NPY_FLOAT) 90 | cimage.c1 = (float*) PyArray_DATA(a); 91 | a = PyObject_GetAttrString($input,"shape"); 92 | assert(PyTuple_Size(a)==3); 93 | assert( PyInt_AsLong(PyTuple_GetItem(a,0)) == 3); 94 | cimage.height = PyInt_AsLong(PyTuple_GetItem(a,1)); 95 | cimage.width = PyInt_AsLong(PyTuple_GetItem(a,2)); 96 | assert( cimage.width%4==0); 97 | cimage.stride = cimage.width; 98 | cimage.c2 = cimage.c1 + cimage.stride*cimage.height; 99 | cimage.c3 = cimage.c2 + cimage.stride*cimage.height; 100 | $1=&cimage; 101 | } 102 | } 103 | %apply (color_image_t* cimg) {(color_image_t* )}; 104 | 105 | void deepflow2_numpy( int w, image_t* wx, image_t* wy, color_image_t* im1, color_image_t* im2, image_t* match, char *options); 106 | 107 | void usage_python(); 108 | 109 | %{ 110 | #include "io.h" 111 | #include "image.h" 112 | #include "opticalflow.h" 113 | #include 114 | 115 | void deepflow2_numpy( int w, image_t* wx, image_t* wy, color_image_t* im1, color_image_t* im2, image_t* match, char *options){ 116 | // correct the width on the inputs 117 | wx->width = w; 118 | wy->width = w; 119 | im1->width = w; 120 | im2->width = w; 121 | int h = im1->height; 122 | 123 | // read the matches 124 | image_t *match_x = NULL, *match_y = NULL, *match_z = NULL; 125 | if( match != NULL){ 126 | match_x = image_new(w, h); 127 | match_y = image_new(w, h); 128 | match_z = image_new(w, h); 129 | image_erase(match_x); image_erase(match_y); image_erase(match_z); 130 | int i; 131 | for( i = 0 ; iheight ; i++){ 132 | float x1 = match->data[i*match->stride+0], y1 = match->data[i*match->stride+1], x2 = match->data[i*match->stride+2], y2 = match->data[i*match->stride+3]; 133 | if( x1<0 || y1<0 || x2<0 || y2<0 || x1>=w || y1>=h || x2>=w || y2>=h){ 134 | fprintf(stderr, "Warning: match out of bound: %f %f -> %f %f\n", x1, y1, x2, y2); 135 | x1 = MINMAX(x1, w); 136 | x2 = MINMAX(x2, w); 137 | y1 = MINMAX(y1, h); 138 | y2 = MINMAX(y2, h); 139 | } 140 | int pos = (int) (y1*match_x->stride+x1); 141 | match_x->data[ pos ] = x2-x1; 142 | match_y->data[ pos ] = y2-y1; 143 | match_z->data[ pos ] = 1.0f; 144 | } 145 | } 146 | 147 | // set params to default 148 | optical_flow_params_t* params = (optical_flow_params_t*) malloc(sizeof(optical_flow_params_t)); 149 | if(!params){ 150 | fprintf(stderr,"error deepflow2(): not enough memory\n"); 151 | exit(1); 152 | } 153 | optical_flow_params_default(params); 154 | 155 | // read options 156 | if( options!=NULL ){ 157 | int argc=0; 158 | char* argv[256]; 159 | argv[argc]=strtok(options," "); 160 | while(argv[argc]!=NULL) 161 | { 162 | argv[++argc]=strtok(NULL," "); 163 | } 164 | parse_options(params, argc, argv, PYTHON_OPTIONS, w, h); 165 | } 166 | 167 | // launch flow 168 | image_t *flowx = image_new(w, h), *flowy = image_new(w,h); // optical_flow changes the pointer to the data 169 | optical_flow(flowx, flowy, im1, im2, params, match_x, match_y, match_z); 170 | memcpy( wx->data, flowx->data, sizeof(float)*flowx->stride*h); 171 | memcpy( wy->data, flowy->data, sizeof(float)*flowx->stride*h); 172 | 173 | free(params); 174 | image_delete(flowx); 175 | image_delete(flowy); 176 | if(match!=NULL){ 177 | image_delete(match_x); 178 | image_delete(match_y); 179 | image_delete(match_z); 180 | } 181 | } 182 | 183 | void usage_python(){ 184 | usage(PYTHON_OPTIONS); 185 | } 186 | 187 | %} 188 | 189 | 190 | %pythoncode %{ 191 | from numpy import float32, concatenate, empty, rollaxis, ascontiguousarray, pad 192 | def deepflow2( im1=None, im2=None, match=None, options=""): 193 | """ 194 | flow = deepflow2.deepflow2(image1, image2, match=None, options='') 195 | Compute the flow between two images, eventually using given matches. 196 | Images must be HxWx3 numpy arrays (convert to float32). 197 | Match is an optional numpy array argument (None by default, ie no input match), where each row starts by x1 y1 x2 y2. 198 | Options is an optional string argument ('' by default), to set the options. Type deepflow2() to see the list of available options. 199 | The function returns the optical flow as a HxWx2 numpy array.""" 200 | #convert images 201 | if None in (im1,im2): 202 | usage_python() 203 | return 204 | assert im1.shape == im2.shape, "images must have the same shape" 205 | if im1.dtype != float32: 206 | im1 = im1.astype(float32) 207 | if im2.dtype != float32: 208 | im2 = im2.astype(float32) 209 | h, w, nchannels = im1.shape 210 | assert nchannels==3, "images must have 3 channels" 211 | stride = 4*((w+3)//4) 212 | im1 = pad( rollaxis(im1,2), ((0,0),(0,0),(0, stride-w)), 'constant') 213 | im2 = pad( rollaxis(im2,2), ((0,0),(0,0),(0, stride-w)), 'constant') 214 | # allocate flow 215 | flowx = empty((h,stride), dtype=float32) 216 | flowy = empty((h,stride), dtype=float32) 217 | # compute flow 218 | if match is not None: 219 | assert match.shape[1]>=4 220 | match = ascontiguousarray(match[:,:4], dtype=float32) 221 | deepflow2_numpy( w, flowx, flowy, im1, im2, match, options) 222 | return concatenate ( (flowx[:,:w,None], flowy[:,:w,None]), axis=2) 223 | %} 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | -------------------------------------------------------------------------------- /deep_flow2/deepflow2.m: -------------------------------------------------------------------------------- 1 | % "flow = deepflow2(image1, image2, match, options) 2 | % 3 | % "Compute the flow between two images, eventually using given matches. 4 | % 5 | % "Images are HxWx3 single matrices. 6 | % "Match is an optional argument ([] by default), where each row starts by x1 y1 x2 y2. 7 | % "Options is an optional string argument ('' by default), to set the options. Type deepflow2() to see the list of available options. 8 | % 9 | % "The function returns the optical flow as a HxWx2 single matrix. 10 | -------------------------------------------------------------------------------- /deep_flow2/deepflow2.py: -------------------------------------------------------------------------------- 1 | # This file was automatically generated by SWIG (http://www.swig.org). 2 | # Version 2.0.11 3 | # 4 | # Do not make changes to this file unless you know what you are doing--modify 5 | # the SWIG interface file instead. 6 | 7 | 8 | 9 | 10 | """ 11 | Module to compute DeepFlow 12 | """ 13 | 14 | 15 | from sys import version_info 16 | if version_info >= (2,6,0): 17 | def swig_import_helper(): 18 | from os.path import dirname 19 | import imp 20 | fp = None 21 | try: 22 | fp, pathname, description = imp.find_module('_deepflow2', [dirname(__file__)]) 23 | except ImportError: 24 | import _deepflow2 25 | return _deepflow2 26 | if fp is not None: 27 | try: 28 | _mod = imp.load_module('_deepflow2', fp, pathname, description) 29 | finally: 30 | fp.close() 31 | return _mod 32 | _deepflow2 = swig_import_helper() 33 | del swig_import_helper 34 | else: 35 | import _deepflow2 36 | del version_info 37 | try: 38 | _swig_property = property 39 | except NameError: 40 | pass # Python < 2.2 doesn't have 'property'. 41 | def _swig_setattr_nondynamic(self,class_type,name,value,static=1): 42 | if (name == "thisown"): return self.this.own(value) 43 | if (name == "this"): 44 | if type(value).__name__ == 'SwigPyObject': 45 | self.__dict__[name] = value 46 | return 47 | method = class_type.__swig_setmethods__.get(name,None) 48 | if method: return method(self,value) 49 | if (not static): 50 | self.__dict__[name] = value 51 | else: 52 | raise AttributeError("You cannot add attributes to %s" % self) 53 | 54 | def _swig_setattr(self,class_type,name,value): 55 | return _swig_setattr_nondynamic(self,class_type,name,value,0) 56 | 57 | def _swig_getattr(self,class_type,name): 58 | if (name == "thisown"): return self.this.own() 59 | method = class_type.__swig_getmethods__.get(name,None) 60 | if method: return method(self) 61 | raise AttributeError(name) 62 | 63 | def _swig_repr(self): 64 | try: strthis = "proxy of " + self.this.__repr__() 65 | except: strthis = "" 66 | return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) 67 | 68 | try: 69 | _object = object 70 | _newclass = 1 71 | except AttributeError: 72 | class _object : pass 73 | _newclass = 0 74 | 75 | 76 | 77 | def deepflow2_numpy(*args): 78 | return _deepflow2.deepflow2_numpy(*args) 79 | deepflow2_numpy = _deepflow2.deepflow2_numpy 80 | 81 | def usage_python(): 82 | return _deepflow2.usage_python() 83 | usage_python = _deepflow2.usage_python 84 | from numpy import float32, concatenate, empty, rollaxis, ascontiguousarray, pad 85 | def deepflow2( im1=None, im2=None, match=None, options=""): 86 | """ 87 | flow = deepflow2.deepflow2(image1, image2, match=None, options='') 88 | Compute the flow between two images, eventually using given matches. 89 | Images must be HxWx3 numpy arrays (convert to float32). 90 | Match is an optional numpy array argument (None by default, ie no input match), where each row starts by x1 y1 x2 y2. 91 | Options is an optional string argument ('' by default), to set the options. Type deepflow2() to see the list of available options. 92 | The function returns the optical flow as a HxWx2 numpy array.""" 93 | #convert images 94 | if None in (im1,im2): 95 | usage_python() 96 | return 97 | assert im1.shape == im2.shape, "images must have the same shape" 98 | if im1.dtype != float32: 99 | im1 = im1.astype(float32) 100 | if im2.dtype != float32: 101 | im2 = im2.astype(float32) 102 | h, w, nchannels = im1.shape 103 | assert nchannels==3, "images must have 3 channels" 104 | stride = 4*((w+3)//4) 105 | im1 = pad( rollaxis(im1,2), ((0,0),(0,0),(0, stride-w)), 'constant') 106 | im2 = pad( rollaxis(im2,2), ((0,0),(0,0),(0, stride-w)), 'constant') 107 | # allocate flow 108 | flowx = empty((h,stride), dtype=float32) 109 | flowy = empty((h,stride), dtype=float32) 110 | # compute flow 111 | if match is not None: 112 | assert match.shape[1]>=4 113 | match = ascontiguousarray(match[:,:4], dtype=float32) 114 | deepflow2_numpy( w, flowx, flowy, im1, im2, match, options) 115 | return concatenate ( (flowx[:,:w,None], flowy[:,:w,None]), axis=2) 116 | 117 | # This file is compatible with both classic and new-style classes. 118 | 119 | 120 | -------------------------------------------------------------------------------- /deep_flow2/deepflow2_matlab.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | /* 19 | MATLAB Wrapper 20 | */ 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | #define printf mexPrintf 29 | 30 | extern "C" { 31 | #include "image.h" 32 | #include "opticalflow.h" 33 | #include "io.h" 34 | } 35 | 36 | color_image_t *input3darray_to_color_image(const mxArray *p){ 37 | const int *dims = mxGetDimensions(p); 38 | const int h = dims[0], w = dims[1]; 39 | assert( dims[2]==3 ); 40 | float *in = (float*) mxGetData(p); 41 | color_image_t *out = color_image_new(w, h); 42 | const int s = out->stride; 43 | for(int c=0 ; c<3 ; c++){ 44 | float *inptr = in + c*w*h; 45 | float *outptr = out->c1 + c*s*h; 46 | for( int j=0 ; jwidth, h = match_x->height, s = match_x->stride; 58 | float *data = (float*) mxGetData(p); 59 | image_erase(match_x); image_erase(match_y); image_erase(match_z); 60 | for( int i=0 ; i=w || y1>=h || x2>=w || y2>=h){ 63 | fprintf(stderr, "Warning: match out of bound: %f %f -> %f %f\n", x1, y1, x2, y2); 64 | x1 = MINMAX(x1,w); 65 | x2 = MINMAX(x2,w); 66 | y1 = MINMAX(y1,h); 67 | y2 = MINMAX(y2,h); 68 | } 69 | int pos = (int) (y1*s+x1); 70 | match_x->data[ pos ] = x2-x1; 71 | match_y->data[ pos ] = y2-y1; 72 | match_z->data[ pos ] = 1.0f; 73 | } 74 | } 75 | 76 | 77 | 78 | void flow_to_output3darray(image_t *wx, image_t *wy, mxArray *p){ 79 | const int h = wx->height, w = wx->width, s = wx->stride; 80 | float *data = (float*) mxGetData(p); 81 | for( int j=0 ; jdata[j*s+i]; 84 | data[(i+w)*h+j] = wy->data[j*s+i]; 85 | } 86 | } 87 | } 88 | 89 | void mexFunction( int nl, mxArray *pl[], int nr, const mxArray *pr[] ) { 90 | 91 | if( nr==0 ){ 92 | usage(MATLAB_OPTIONS); 93 | return; 94 | } 95 | if ( nl != 1){ 96 | usage(MATLAB_OPTIONS); 97 | mexErrMsgTxt("error: returns one output"); 98 | return; 99 | } 100 | if( nr < 2 || nr > 4){ 101 | usage(MATLAB_OPTIONS); 102 | mexErrMsgTxt("error: takes two to four inputs"); 103 | return; 104 | } 105 | 106 | // The code is originally written for C-order arrays. 107 | // We thus transpose all arrays in this mex-function which is not efficient... 108 | 109 | const int *pDims; 110 | if( mxGetNumberOfDimensions(pr[0]) != 3 ) mexErrMsgTxt("input images must have 3 dimensions"); 111 | if( !mxIsClass(pr[0], "single") ) mexErrMsgTxt("input images must be single"); 112 | pDims = mxGetDimensions(pr[0]); 113 | if( pDims[2]!=3 ) mexErrMsgTxt("input images must have 3 channels"); 114 | const int h = pDims[0], w = pDims[1]; 115 | color_image_t *im1 = input3darray_to_color_image( pr[0] ); 116 | 117 | if( mxGetNumberOfDimensions(pr[1]) != 3 ) mexErrMsgTxt("input images must have 3 dimensions"); 118 | if( !mxIsClass(pr[1], "single") ) mexErrMsgTxt("input images must be single"); 119 | pDims = mxGetDimensions(pr[1]); 120 | if( pDims[0]!=h || pDims[1]!=w || pDims[2]!=3) mexErrMsgTxt( "input images must have the same size" ); 121 | color_image_t *im2 = input3darray_to_color_image( pr[1] ); 122 | 123 | image_t *match_x = NULL, *match_y = NULL, *match_z = NULL; 124 | if( nr>2 && !mxIsEmpty(pr[2]) ){ 125 | if( mxGetNumberOfDimensions(pr[2]) != 2 ) mexErrMsgTxt("input matches must be a 2d-matrix"); 126 | if( !mxIsClass(pr[2], "single")) mexErrMsgTxt("input matches must be single"); 127 | pDims = mxGetDimensions(pr[1]); 128 | if( pDims[1]<4) mexErrMsgTxt( "input matches must have at least 4 columns: x1 y1 x2 y2" ); 129 | match_x = image_new(w, h); match_y = image_new(w, h); match_z = image_new(w, h); 130 | input2darray_to_matches( match_x, match_y, match_z, pr[2]); 131 | } 132 | 133 | // set params to default 134 | optical_flow_params_t* params = (optical_flow_params_t*) malloc(sizeof(optical_flow_params_t)); 135 | if(!params){ 136 | fprintf(stderr,"error deepflow2(): not enough memory\n"); 137 | exit(1); 138 | } 139 | optical_flow_params_default(params); 140 | 141 | // read options 142 | if( nr > 3 ){ 143 | char *options = mxArrayToString(pr[3]); 144 | if( !options ) mexErrMsgTxt("Fourth parameter must be a string"); 145 | int argc=0; 146 | char* argv[256]; 147 | argv[argc]=strtok(options," "); 148 | while(argv[argc]!=NULL) 149 | { 150 | argv[++argc]=strtok(NULL," "); 151 | } 152 | parse_options(params, argc, argv, MATLAB_OPTIONS, w, h); 153 | } 154 | 155 | 156 | image_t *wx = image_new(im1->width,im1->height), *wy = image_new(im1->width,im1->height); 157 | optical_flow(wx, wy, im1, im2, params, match_x, match_y, match_z); 158 | 159 | int dims[3] = {h,w,2}; 160 | pl[0] = mxCreateNumericArray(3, dims, mxSINGLE_CLASS, mxREAL); 161 | flow_to_output3darray(wx, wy, pl[0]); 162 | 163 | image_delete(wx); 164 | image_delete(wy); 165 | image_delete(match_x); image_delete(match_y); image_delete(match_z); 166 | color_image_delete(im1); color_image_delete(im2); 167 | free(params); 168 | 169 | } 170 | -------------------------------------------------------------------------------- /deep_flow2/image.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #ifndef __IMAGE_H_ 18 | #define __IMAGE_H_ 19 | 20 | #include 21 | 22 | #define MIN(a, b) ((a) < (b) ? (a) : (b)) 23 | #define MAX(a, b) ((a) > (b) ? (a) : (b)) 24 | #define MINMAX(a,b) MIN( MAX(a,0) , b-1 ) 25 | 26 | /********** STRUCTURES *********/ 27 | 28 | /* structure for 1-channel image */ 29 | typedef struct image_s 30 | { 31 | int width; /* Width of the image */ 32 | int height; /* Height of the image */ 33 | int stride; /* Width of the memory (width + paddind such that it is a multiple of 4) */ 34 | float *data; /* Image data, aligned */ 35 | } image_t; 36 | 37 | /* structure for 3-channels image stored with one layer per color, it assumes that c2 = c1+width*height and c3 = c2+width*height. */ 38 | typedef struct color_image_s 39 | { 40 | int width; /* Width of the image */ 41 | int height; /* Height of the image */ 42 | int stride; /* Width of the memory (width + paddind such that it is a multiple of 4) */ 43 | float *c1; /* Color 1, aligned */ 44 | float *c2; /* Color 2, consecutive to c1*/ 45 | float *c3; /* Color 3, consecutive to c2 */ 46 | } color_image_t; 47 | 48 | /* structure for color image pyramid */ 49 | typedef struct color_image_pyramid_s 50 | { 51 | float scale_factor; /* difference of scale between two levels */ 52 | int min_size; /* minimum size for width or height at the coarsest level */ 53 | int size; /* number of levels in the pyramid */ 54 | color_image_t **images; /* list of images with images[0] the original one, images[size-1] the finest one */ 55 | } color_image_pyramid_t; 56 | 57 | /* structure for convolutions */ 58 | typedef struct convolution_s 59 | { 60 | int order; /* Order of the convolution */ 61 | float *coeffs; /* Coefficients */ 62 | float *coeffs_accu; /* Accumulated coefficients */ 63 | } convolution_t; 64 | 65 | /********** Create/Delete **********/ 66 | 67 | /* allocate a new image of size width x height */ 68 | image_t *image_new(const int width, const int height); 69 | 70 | /* allocate a new image and copy the content from src */ 71 | image_t *image_cpy(const image_t *src); 72 | 73 | /* set all pixels values to zeros */ 74 | void image_erase(image_t *image); 75 | 76 | /* free memory of an image */ 77 | void image_delete(image_t *image); 78 | 79 | /* multiply an image by a scalar */ 80 | void image_mul_scalar(image_t *image, const float scalar); 81 | 82 | /* allocate a new color image of size width x height */ 83 | color_image_t *color_image_new(const int width, const int height); 84 | 85 | /* allocate a new color image and copy the content from src */ 86 | color_image_t *color_image_cpy(const color_image_t *src); 87 | 88 | /* set all pixels values to zeros */ 89 | void color_image_erase(color_image_t *image); 90 | 91 | /* free memory of a color image */ 92 | void color_image_delete(color_image_t *image); 93 | 94 | /* reallocate the memory of an image to fit the new width height */ 95 | void resize_if_needed_newsize(image_t *im, const int w, const int h); 96 | 97 | /************ Resizing *********/ 98 | 99 | /* resize an image with bilinear interpolation */ 100 | image_t *image_resize_bilinear(const image_t *src, const float scale); 101 | 102 | /* resize an image with bilinear interpolation to fit the new weidht, height ; reallocation is done if necessary */ 103 | void image_resize_bilinear_newsize(image_t *dst, const image_t *src, const int new_width, const int new_height); 104 | 105 | /* resize a color image with bilinear interpolation */ 106 | color_image_t *color_image_resize_bilinear(const color_image_t *src, const float scale); 107 | 108 | /************ Convolution ******/ 109 | 110 | /* return half coefficient of a gaussian filter */ 111 | float *gaussian_filter(const float sigma, int *fSize); 112 | 113 | /* create a convolution structure with a given order, half_coeffs, symmetric or anti-symmetric according to even parameter */ 114 | convolution_t *convolution_new(int order, const float *half_coeffs, const int even); 115 | 116 | /* perform an horizontal convolution of an image */ 117 | void convolve_horiz(image_t *dest, const image_t *src, const convolution_t *conv); 118 | 119 | /* perform a vertical convolution of an image */ 120 | void convolve_vert(image_t *dest, const image_t *src, const convolution_t *conv); 121 | 122 | /* free memory of a convolution structure */ 123 | void convolution_delete(convolution_t *conv); 124 | 125 | /* perform horizontal and/or vertical convolution to a color image */ 126 | void color_image_convolve_hv(color_image_t *dst, const color_image_t *src, const convolution_t *horiz_conv, const convolution_t *vert_conv); 127 | 128 | /************ Pyramid **********/ 129 | 130 | /* create a pyramid of color images using a given scale factor, stopping when one dimension reach min_size and with applying a gaussian smoothing of standard deviation spyr (no smoothing if 0) */ 131 | color_image_pyramid_t *color_image_pyramid_create(const color_image_t *src, const float scale_factor, const int min_size, const float spyr); 132 | 133 | /* delete the structure of a pyramid of color images */ 134 | void color_image_pyramid_delete(color_image_pyramid_t *pyr); 135 | 136 | #endif 137 | -------------------------------------------------------------------------------- /deep_flow2/io.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #ifndef __IO_H__ 18 | #define __IO_H__ 19 | 20 | #include 21 | 22 | #include "image.h" 23 | #include "opticalflow.h" 24 | 25 | /* read a flow file and returns a pointer with two images containing the flow along x and y axis */ 26 | image_t** readFlowFile(const char* filename); 27 | 28 | /* write a flow to a file */ 29 | void writeFlowFile(const char* filename, const image_t *flowx, const image_t *flowy); 30 | 31 | /* load a color image from a file in jpg, png or ppm*/ 32 | color_image_t *color_image_load(const char *fname); 33 | 34 | 35 | #define EXE_OPTIONS 0 36 | #define MATLAB_OPTIONS 1 37 | #define PYTHON_OPTIONS 2 38 | /* print usage */ 39 | void usage(const int language); 40 | 41 | /* parse options and eventually return the matches given as argument */ 42 | image_t** parse_options(optical_flow_params_t* params, int argc, char **argv, const int language, const int width, const int height); 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /deep_flow2/opticalflow.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "opticalflow.h" 23 | #include "opticalflow_aux.h" 24 | #include "solver.h" 25 | #include "image.h" 26 | 27 | #include 28 | typedef __v4sf v4sf; 29 | 30 | convolution_t *deriv, *deriv_flow; 31 | float half_alpha, half_delta_over3, half_beta, half_gamma_over3; 32 | 33 | /* perform flow computation at one level of the pyramid */ 34 | void compute_one_level(image_t *wx, image_t *wy, color_image_t *im1, color_image_t *im2, image_t *desc_flow_x, image_t *desc_flow_y, image_t *desc_weight, const optical_flow_params_t *params){ 35 | const int width = wx->width, height = wx->height, stride=wx->stride; 36 | int i_inner_iteration; 37 | 38 | image_t *du = image_new(width,height), *dv = image_new(width,height), // the flow increment 39 | *mask = image_new(width,height), // mask containing 0 if a point goes outside image boundary, 1 otherwise 40 | *smooth_horiz = image_new(width,height), *smooth_vert = image_new(width,height), // horiz: (i,j) contains the diffusivity coeff from (i,j) to (i+1,j) 41 | *uu = image_new(width,height), *vv = image_new(width,height), // flow plus flow increment 42 | *a11 = image_new(width,height), *a12 = image_new(width,height), *a22 = image_new(width,height), // system matrix A of Ax=b for each pixel 43 | *b1 = image_new(width,height), *b2 = image_new(width,height), // system matrix b of Ax=b for each pixel 44 | *smooth_weight = compute_smoothness_weight(im1, 5.0f, deriv); // local weights for smoothness 45 | 46 | color_image_t *w_im2 = color_image_new(width,height), // warped second image 47 | *Ix = color_image_new(width,height), *Iy = color_image_new(width,height), *Iz = color_image_new(width,height), // first order derivatives 48 | *Ixx = color_image_new(width,height), *Ixy = color_image_new(width,height), *Iyy = color_image_new(width,height), *Ixz = color_image_new(width,height), *Iyz = color_image_new(width,height); // second order derivatives 49 | 50 | // warp second image 51 | image_warp(w_im2, mask, im2, wx, wy); 52 | // compute derivatives 53 | get_derivatives(im1, w_im2, deriv, Ix, Iy, Iz, Ixx, Ixy, Iyy, Ixz, Iyz); 54 | // erase du and dv 55 | image_erase(du); 56 | image_erase(dv); 57 | // initialize uu and vv 58 | memcpy(uu->data,wx->data,wx->stride*wx->height*sizeof(float)); 59 | memcpy(vv->data,wy->data,wy->stride*wy->height*sizeof(float)); 60 | // inner fixed point iterations 61 | for(i_inner_iteration = 0 ; i_inner_iteration < params->n_inner_iteration ; i_inner_iteration++){ 62 | // compute robust function and system 63 | compute_smoothness(smooth_horiz, smooth_vert, uu, vv, smooth_weight, deriv_flow, half_alpha ); 64 | compute_data_and_match(a11, a12, a22, b1, b2, mask, wx, wy, du, dv, uu, vv, Ix, Iy, Iz, Ixx, Ixy, Iyy, Ixz, Iyz, desc_weight, desc_flow_x, desc_flow_y, half_delta_over3, half_beta, half_gamma_over3); 65 | sub_laplacian(b1, wx, smooth_horiz, smooth_vert); 66 | sub_laplacian(b2, wy, smooth_horiz, smooth_vert); 67 | // solve system 68 | sor_coupled(du, dv, a11, a12, a22, b1, b2, smooth_horiz, smooth_vert, params->n_solver_iteration, params->sor_omega); 69 | // update flow plus flow increment 70 | int i; 71 | v4sf *uup = (v4sf*) uu->data, *vvp = (v4sf*) vv->data, *wxp = (v4sf*) wx->data, *wyp = (v4sf*) wy->data, *dup = (v4sf*) du->data, *dvp = (v4sf*) dv->data; 72 | for( i=0 ; idata,uu->data,uu->stride*uu->height*sizeof(float)); 80 | memcpy(wy->data,vv->data,vv->stride*vv->height*sizeof(float)); 81 | // free memory 82 | image_delete(du); image_delete(dv); 83 | image_delete(mask); 84 | image_delete(smooth_horiz); image_delete(smooth_vert); 85 | image_delete(uu); image_delete(vv); 86 | image_delete(a11); image_delete(a12); image_delete(a22); 87 | image_delete(b1); image_delete(b2); 88 | image_delete(smooth_weight); 89 | color_image_delete(w_im2); 90 | color_image_delete(Ix); color_image_delete(Iy); color_image_delete(Iz); 91 | color_image_delete(Ixx); color_image_delete(Ixy); color_image_delete(Iyy); color_image_delete(Ixz); color_image_delete(Iyz); 92 | } 93 | 94 | /* set flow parameters to default */ 95 | void optical_flow_params_default(optical_flow_params_t *params){ 96 | if(!params){ 97 | fprintf(stderr,"Error optical_flow_params_default: argument is null\n"); 98 | exit(1); 99 | } 100 | params->alpha = 1.0f; 101 | params->beta = 20.0f; 102 | params->gamma = 0.7f; 103 | params->delta = 0.1f; 104 | params->sigma = 0.6f; 105 | params->bk = 0.5f; 106 | params->eta = 0.95f; 107 | params->min_size = 25; 108 | params->n_inner_iteration = 5; 109 | params->n_solver_iteration = 25; 110 | params->sor_omega = 1.60f; 111 | } 112 | 113 | /* set flow parameters to sintel one */ 114 | void optical_flow_params_sintel(optical_flow_params_t *params){ 115 | if(!params){ 116 | fprintf(stderr,"Error optical_flow_params_sintel: argument is null\n"); 117 | exit(1); 118 | } 119 | params->alpha = 1.0f; 120 | params->beta = 32.0f; 121 | params->gamma = 0.56f; 122 | params->delta = 0.0f; 123 | params->sigma = 0.45f; 124 | params->bk = 0.45f; 125 | params->eta = 0.95f; 126 | params->min_size = 25; 127 | params->n_inner_iteration = 5; 128 | params->n_solver_iteration = 25; 129 | params->sor_omega = 1.60f; 130 | } 131 | 132 | /* set flow parameters to middlebury one */ 133 | void optical_flow_params_middlebury(optical_flow_params_t *params){ 134 | if(!params){ 135 | fprintf(stderr,"Error optical_flow_params_middlebury: argument is null\n"); 136 | exit(1); 137 | } 138 | params->alpha = 1.0f; 139 | params->beta = 11.5f; 140 | params->gamma = 1.09f; 141 | params->delta = 0.04f; 142 | params->sigma = 0.65f; 143 | params->bk = 0.45f; 144 | params->eta = 0.95f; 145 | params->min_size = 25; 146 | params->n_inner_iteration = 5; 147 | params->n_solver_iteration = 25; 148 | params->sor_omega = 1.60f; 149 | } 150 | 151 | /* set flow parameters to kitti one */ 152 | void optical_flow_params_kitti(optical_flow_params_t *params){ 153 | if(!params){ 154 | fprintf(stderr,"Error optical_flow_params_kitti: argument is null\n"); 155 | exit(1); 156 | } 157 | params->alpha = 1.0f; 158 | params->beta = 0.6f; 159 | params->gamma = 1.09f; 160 | params->delta = 0.40f; 161 | params->sigma = 0.60f; 162 | params->bk = 0.0f; 163 | params->eta = 0.95f; 164 | params->min_size = 25; 165 | params->n_inner_iteration = 5; 166 | params->n_solver_iteration = 25; 167 | params->sor_omega = 1.60f; 168 | } 169 | 170 | /* Compute the optical flow between im1 and im2 and store it as two 1-channel images in wx for flow along x-axis and wy for flow along y-axis. match_x, match_y and match_z contains eventually the input matches (NULL for no match) at any scale. */ 171 | void optical_flow(image_t *wx, image_t *wy, const color_image_t *im1, const color_image_t *im2, optical_flow_params_t *params, const image_t *match_x, const image_t *match_y, image_t *match_z){ 172 | 173 | // Check parameters 174 | if(!params){ 175 | params = (optical_flow_params_t*) malloc(sizeof(optical_flow_params_t)); 176 | if(!params){ 177 | fprintf(stderr,"error color_image_convolve_hv(): not enough memory\n"); 178 | exit(1); 179 | } 180 | optical_flow_params_default(params); 181 | } 182 | 183 | // initialize global variables 184 | half_alpha = 0.5f*params->alpha; 185 | half_gamma_over3 = params->gamma*0.5f/3.0f; 186 | half_delta_over3 = params->delta*0.5f/3.0f; 187 | half_beta = params->beta*0.5f; 188 | float deriv_filter[3] = {0.0f, -8.0f/12.0f, 1.0f/12.0f}; 189 | deriv = convolution_new(2, deriv_filter, 0); 190 | float deriv_filter_flow[2] = {0.0f, -0.5f}; 191 | deriv_flow = convolution_new(1, deriv_filter_flow, 0); 192 | 193 | // presmooth images 194 | int width = im1->width, height = im1->height, filter_size; 195 | color_image_t *smooth_im1 = color_image_new(width, height), *smooth_im2 = color_image_new(width, height); 196 | float *presmooth_filter = gaussian_filter(params->sigma, &filter_size); 197 | convolution_t *presmoothing = convolution_new(filter_size, presmooth_filter, 1); 198 | color_image_convolve_hv(smooth_im1, im1, presmoothing, presmoothing); 199 | color_image_convolve_hv(smooth_im2, im2, presmoothing, presmoothing); 200 | convolution_delete(presmoothing); 201 | free(presmooth_filter); 202 | 203 | // check descriptors 204 | image_t *desc_flow_x, *desc_flow_y, *desc_weight, *desc_flow_x_original=NULL, *desc_flow_y_original=NULL, *desc_weight_original=NULL; 205 | desc_flow_x = image_new(0,0); 206 | desc_flow_y = image_new(0,0); 207 | desc_weight = image_new(0,0); 208 | if(params->beta){ 209 | if(match_x == NULL){ 210 | printf("No descriptor input, weight of descriptor term set to 0\n"); 211 | params->beta = 0.0f; 212 | half_beta = 0.0f; 213 | }else{ 214 | compute_desc_weight(im1, im2, match_x, match_y, match_z, deriv_flow, deriv); 215 | desc_flow_x_original = image_cpy(match_x); 216 | desc_flow_y_original = image_cpy(match_y); 217 | desc_weight_original = image_cpy(match_z); 218 | } 219 | } 220 | 221 | // building pyramid 222 | color_image_pyramid_t *pyr1 = color_image_pyramid_create(smooth_im1, 1.0f/params->eta, params->min_size, 0.0f), 223 | *pyr2 = color_image_pyramid_create(smooth_im2, 1.0f/params->eta, params->min_size, 0.0f); 224 | 225 | // loop over levels 226 | int k; 227 | for(k=pyr1->size-1; k>=0 ; k--){ 228 | if(params->bk>0.0f) half_beta = 0.5f*params->beta * pow(((float)k)/((float)pyr1->size-1),params->bk); 229 | if(k == pyr1->size-1){ 230 | // first level 231 | // allocate wx and wy 232 | resize_if_needed_newsize(wx, pyr1->images[k]->width, pyr1->images[k]->height); 233 | resize_if_needed_newsize(wy, pyr1->images[k]->width, pyr1->images[k]->height); 234 | image_erase(wx); image_erase(wy); 235 | }else{ 236 | // resize flow to the new pyramid level size and multiply it by 1/eta 237 | image_t *tmp = image_new(pyr1->images[k]->width, pyr1->images[k]->height); 238 | image_resize_bilinear_newsize(tmp, wx, pyr1->images[k]->width, pyr1->images[k]->height); 239 | resize_if_needed_newsize(wx, pyr1->images[k]->width, pyr1->images[k]->height); 240 | memcpy(wx->data, tmp->data, tmp->stride*tmp->height*sizeof(float)); 241 | image_mul_scalar(wx, 1.0f/params->eta); 242 | image_resize_bilinear_newsize(tmp, wy, pyr1->images[k]->width, pyr1->images[k]->height); 243 | resize_if_needed_newsize(wy, pyr1->images[k]->width, pyr1->images[k]->height); 244 | memcpy(wy->data, tmp->data, tmp->stride*tmp->height*sizeof(float)); 245 | image_mul_scalar(wy, 1.0f/params->eta); 246 | image_delete(tmp); 247 | } 248 | 249 | // resize descriptors 250 | if(params->beta){ 251 | resize_if_needed_newsize(desc_flow_x, pyr1->images[k]->width, pyr1->images[k]->height); 252 | resize_if_needed_newsize(desc_flow_y, pyr1->images[k]->width, pyr1->images[k]->height); 253 | resize_if_needed_newsize(desc_weight, pyr1->images[k]->width, pyr1->images[k]->height); 254 | descflow_resize(desc_flow_x,desc_flow_y,desc_weight,desc_flow_x_original,desc_flow_y_original,desc_weight_original); 255 | } 256 | 257 | compute_one_level(wx, wy, pyr1->images[k], pyr2->images[k], desc_flow_x, desc_flow_y, desc_weight, params); 258 | 259 | } 260 | 261 | color_image_pyramid_delete(pyr1); color_image_pyramid_delete(pyr2); 262 | 263 | // do a last iteration without descriptor if bk==0 264 | if(params->beta>0.0f && params->bk==0.0f){ 265 | half_beta = 0.0f; 266 | compute_one_level(wx, wy, smooth_im1, smooth_im2, desc_flow_x, desc_flow_y, desc_weight, params); 267 | half_beta = 0.5f*params->beta; 268 | } 269 | 270 | // free memory 271 | color_image_delete(smooth_im1); 272 | color_image_delete(smooth_im2); 273 | image_delete(desc_flow_x); image_delete(desc_flow_y); image_delete(desc_weight); 274 | convolution_delete(deriv); 275 | convolution_delete(deriv_flow); 276 | if(params->beta){image_delete(desc_flow_x_original); image_delete(desc_flow_y_original); image_delete(desc_weight_original);} 277 | } 278 | -------------------------------------------------------------------------------- /deep_flow2/opticalflow.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #ifndef __OPTICALFLOW_H__ 18 | #define __OPTICALFLOW_H__ 19 | 20 | #include 21 | #include 22 | 23 | #include "image.h" 24 | 25 | typedef struct optical_flow_params_s { 26 | float alpha; // smoothness weight 27 | float beta; // matching weight 28 | float gamma; // gradient constancy assumption weight 29 | float delta; // color constancy assumption weight 30 | float sigma; // presmoothing of the images 31 | float bk; // downweight parameter of the matching weight 32 | float eta; // downscale factor 33 | int min_size; // minimum size of the first level 34 | int n_inner_iteration; // number of inner fixed point iterations 35 | int n_solver_iteration; // number of solver iterations 36 | float sor_omega; // omega parameter of sor method 37 | } optical_flow_params_t; 38 | 39 | /* set flow parameters to default */ 40 | void optical_flow_params_default(optical_flow_params_t *params); 41 | 42 | /* set flow parameters to sintel one */ 43 | void optical_flow_params_sintel(optical_flow_params_t *params); 44 | 45 | /* set flow parameters to middlebury one */ 46 | void optical_flow_params_middlebury(optical_flow_params_t *params); 47 | 48 | /* set flow parameters to kitti one */ 49 | void optical_flow_params_kitti(optical_flow_params_t *params); 50 | 51 | /* Compute the optical flow between im1 and im2 and store it as two 1-channel images in wx for flow along x-axis and wy for flow along y-axis. match_x, match_y and match_z contains eventually the input matches (NULL for no match) at any scale. */ 52 | void optical_flow(image_t *wx, image_t *wy, const color_image_t *im1, const color_image_t *im2, optical_flow_params_t *params, const image_t *match_x, const image_t *match_y, image_t *match_z); 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /deep_flow2/opticalflow_aux.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #ifndef __OPTICALFLOW_AUX_ 18 | #define __OPTICALFLOW_AUX_ 19 | 20 | #include 21 | 22 | #include "image.h" 23 | 24 | /* warp a color image according to a flow. src is the input image, wx and wy, the input flow. dst is the warped image and mask contains 0 or 1 if the pixels goes outside/inside image boundaries */ 25 | void image_warp(color_image_t *dst, image_t *mask, const color_image_t *src, const image_t *wx, const image_t *wy); 26 | 27 | /* compute image first and second order spatio-temporal derivatives of a color image */ 28 | void get_derivatives(const color_image_t *im1, const color_image_t *im2, const convolution_t *deriv, color_image_t *dx, color_image_t *dy, color_image_t *dt, color_image_t *dxx, color_image_t *dxy, color_image_t *dyy, color_image_t *dxt, color_image_t *dyt); 29 | 30 | /* compute smoothness weight using image gradient's norm as exp(-coef*grad(luminance))*/ 31 | image_t* compute_smoothness_weight(color_image_t *im, float coef, const convolution_t *deriv); 32 | 33 | /* compute the smoothness term */ 34 | void compute_smoothness(image_t *dst_horiz, image_t *dst_vert, const image_t *uu, const image_t *vv, const image_t *smooth_weight, const convolution_t *deriv_flow, const float quarter_alpha); 35 | 36 | /* sub the laplacian (smoothness term) to the right-hand term */ 37 | void sub_laplacian(image_t *dst, const image_t *src, const image_t *weight_horiz, const image_t *weight_vert); 38 | 39 | /* compute the dataterm and the matching term 40 | a11 a12 a22 represents the 2x2 diagonal matrix, b1 and b2 the right hand side 41 | other (color) images are input */ 42 | void compute_data_and_match(image_t *a11, image_t *a12, image_t *a22, image_t *b1, image_t *b2, image_t *mask, image_t *wx, image_t *wy, image_t *du, image_t *dv, image_t *uu, image_t *vv, color_image_t *Ix, color_image_t *Iy, color_image_t *Iz, color_image_t *Ixx, color_image_t *Ixy, color_image_t *Iyy, color_image_t *Ixz, color_image_t *Iyz, image_t *desc_weight, image_t *desc_flow_x, image_t *desc_flow_y, const float half_delta_over3, const float half_beta, const float half_gamma_over3); 43 | 44 | /* compute score for matches based on autocorrelation matrix and similarity in color/gradient */ 45 | void compute_desc_weight(const color_image_t* im1, const color_image_t* im2, const image_t* desc_x, const image_t *desc_y, image_t *desc_w, const convolution_t* deriv_autocor, const convolution_t *deriv_flow); 46 | 47 | /* resize the descriptors to the new size using a weighted mean */ 48 | void descflow_resize(image_t *dst_flow_x, image_t *dst_flow_y, image_t *dst_weight, const image_t *src_flow_x, const image_t *src_flow_y, const image_t *src_weight); 49 | 50 | /* resize the descriptors to the new size using a nearest neighbor method while keeping the descriptor with the higher weight at the end */ 51 | void descflow_resize_nn(image_t *dst_flow_x, image_t *dst_flow_y, image_t *dst_weight, const image_t *src_flow_x, const image_t *src_flow_y, const image_t *src_weight); 52 | 53 | 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /deep_flow2/solver.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2013 Philippe Weinzaepfel 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | #include 18 | #include 19 | 20 | #include "image.h" 21 | 22 | // Perform n iterations of the sor_coupled algorithm for a system of the form as described in opticalflow.c 23 | void sor_coupled(image_t *du, image_t *dv, image_t *a11, image_t *a12, image_t *a22, const image_t *b1, const image_t *b2, const image_t *dpsis_horiz, const image_t *dpsis_vert, const int iterations, const float omega); 24 | -------------------------------------------------------------------------------- /deep_matching/.gitignore: -------------------------------------------------------------------------------- 1 | ## General 2 | 3 | # Compiled Object files 4 | *.slo 5 | *.lo 6 | *.o 7 | *.cuo 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | *.dylib 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | 18 | # Compiled protocol buffers 19 | *.pb.h 20 | *.pb.cc 21 | *_pb2.py 22 | 23 | # Compiled python 24 | *.pyc 25 | 26 | # Compiled MATLAB 27 | *.mex* 28 | 29 | # IPython notebook checkpoints 30 | .ipynb_checkpoints 31 | 32 | # Editor temporaries 33 | *.swp 34 | *~ 35 | 36 | # Sublime Text settings 37 | *.sublime-workspace 38 | *.sublime-project 39 | 40 | # Eclipse Project settings 41 | *.*project 42 | .settings 43 | 44 | # QtCreator files 45 | *.user 46 | 47 | # PyCharm files 48 | .idea 49 | 50 | # OSX dir files 51 | .DS_Store 52 | 53 | ## Caffe 54 | 55 | # User's build configuration 56 | Makefile.config 57 | 58 | # Data and models are either 59 | # 1. reference, and not casually committed 60 | # 2. custom, and live on their own unless they're deliberated contributed 61 | data/* 62 | models/* 63 | *.caffemodel 64 | *.caffemodel.h5 65 | *.solverstate 66 | *.solverstate.h5 67 | *.binaryproto 68 | *leveldb 69 | *lmdb 70 | 71 | # build, distribute, and bins (+ python proto bindings) 72 | build 73 | .build_debug/* 74 | .build_release/* 75 | distribute/* 76 | *.testbin 77 | *.bin 78 | python/caffe/proto/ 79 | cmake_build 80 | .cmake_build 81 | 82 | # Generated documentation 83 | docs/_site 84 | docs/gathered 85 | _site 86 | doxygen 87 | docs/dev 88 | 89 | # LevelDB files 90 | *.sst 91 | *.ldb 92 | LOCK 93 | LOG* 94 | CURRENT 95 | MANIFEST-* 96 | 97 | # images 98 | *.jpg 99 | *.png 100 | *.flo 101 | -------------------------------------------------------------------------------- /deep_matching/Makefile: -------------------------------------------------------------------------------- 1 | # Path to gpudm and libcaffe.so 2 | CAFFEDIR=caffe 3 | CAFFELIB=$(CAFFEDIR)/build/lib 4 | CAFFE_BUIL_SRC=$(CAFFEDIR)/.build_release/src 5 | 6 | # Path to google tools (protobuf, glogs) # GOOGLETOOLS=/usr 7 | GOOGLETOOLS=/home/ddk/softwares/caffe/zhouxia.env 8 | 9 | CUDA=/usr/local/cuda 10 | 11 | MKL=/home/ddk/softwares/caffe/mkl.install/intel/mkl 12 | 13 | INCLUDES=-I/usr/include/python2.7 \ 14 | -I$(CAFFEDIR)/include \ 15 | -I$(CUDA)/include \ 16 | -I$(GOOGLETOOLS)/include \ 17 | -I$(MKL)/include \ 18 | -I$(CAFFEDIR)/include \ 19 | -I$(CAFFE_BUIL_SRC) 20 | 21 | #include gpudm/Makefile.config 22 | CUDA_ARCH := \ 23 | -gencode arch=compute_35,code=sm_35 \ 24 | -gencode arch=compute_50,code=sm_50 25 | 26 | HEADERS := $(shell find . -name '*.hpp') 27 | EXTRA_LAYERS := $(shell find . -name '*.hpp') 28 | 29 | OPTFLAGS=-g -O2 30 | 31 | all: _gpudm.so 32 | 33 | _gpudm.so: gpudm_wrap.o $(EXTRA_LAYERS:.hpp=.o) $(EXTRA_LAYERS:.hpp=.cuo) 34 | g++ $(OPTFLAGS) -fPIC $(INCLUDES) -L$(CAFFELIB) $^ -shared -o $@ -lcaffe -L$(CUDA)/lib64 -lcusparse 35 | CAFFEDIR=$(CAFFEDIR); GOOGLETOOLS=$(GOOGLETOOLS); CUDA=$(CUDA); MKL=$(MKL); LD_LIBRARY_PATH="$(CAFFELIB):$(CUDA)/lib64:$(MKL)/lib/intel64:/usr/lib64/openmpi/lib/:$(GOOGLETOOLS)/lib:$(LD_LIBRARY_PATH)" python -c "import gpudm" 36 | 37 | %.cuo: %.cu %.hpp 38 | $(CUDA)/bin/nvcc $(CUDA_ARCH) -Xcompiler -fPIC $(INCLUDES) $(OPTFLAGS) -c $< -o $@ 39 | 40 | gpudm_wrap.cxx: gpudm.swig $(HEADERS) 41 | swig -python -c++ $(INCLUDES) gpudm.swig 42 | 43 | gpudm_wrap.o: gpudm_wrap.cxx 44 | g++ $(OPTFLAGS) -c gpudm_wrap.cxx -fPIC $(INCLUDES) -o gpudm_wrap.o 45 | 46 | %.o: %.cpp %.hpp 47 | g++ $(OPTFLAGS) -c $< -fPIC $(INCLUDES) -L$(CAFFEDIR) -o $@ 48 | 49 | clean: 50 | rm -f *.pyc *~ _gpudm.so gpudm_wrap.o $(EXTRA_LAYERS:.hpp=.o) $(EXTRA_LAYERS:.hpp=.cuo) 51 | 52 | cleanswig: clean 53 | rm -f gpudm.py gpudm_wrap.cxx gpudm_wrap.o 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /deep_matching/Makefile.bak: -------------------------------------------------------------------------------- 1 | # Path to gpudm and libcaffe.so 2 | CAFFEDIR=caffe 3 | CAFFELIB=$(CAFFEDIR)/build/lib 4 | CAFFE_BUIL_SRC=$(CAFFEDIR)/.build_release/src 5 | 6 | # Path to google tools (protobuf, glogs) 7 | GOOGLETOOLS=google_tools 8 | 9 | # Path to CUDA 10 | CUDA=/scratch/hydrus/cuda-6.5 11 | 12 | # Path to MKL (or BLAS) 13 | MKL=/home/clear/lear/intel/mkl 14 | 15 | INCLUDES=-I/usr/include/python2.7 \ 16 | -I$(CAFFEDIR)/include \ 17 | -I$(CUDA)/include \ 18 | -I$(GOOGLETOOLS)/include/ \ 19 | -I$(MKL)/include \ 20 | -I$(CAFFEDIR)/include \ 21 | -I$(CAFFE_BUIL_SRC) 22 | 23 | #include gpudm/Makefile.config 24 | CUDA_ARCH := \ 25 | -gencode arch=compute_35,code=sm_35 \ 26 | -gencode arch=compute_50,code=sm_50 27 | 28 | HEADERS := $(shell find . -name '*.hpp') 29 | EXTRA_LAYERS := $(shell find . -name '*.hpp') 30 | 31 | OPTFLAGS=-g -O2 32 | 33 | all: _gpudm.so 34 | 35 | _gpudm.so: gpudm_wrap.o $(EXTRA_LAYERS:.hpp=.o) $(EXTRA_LAYERS:.hpp=.cuo) 36 | g++ $(OPTFLAGS) -fPIC $(INCLUDES) -L$(CAFFELIB) $^ -shared -o $@ -lcaffe -L$(CUDA)/lib64 -lcusparse 37 | CAFFEDIR=$(CAFFEDIR); GOOGLETOOLS=$(GOOGLETOOLS); CUDA=$(CUDA); MKL=$(MKL); LD_LIBRARY_PATH="$(CAFFELIB):$(CUDA)/lib64:$(MKL)/lib/intel64:/usr/lib64/openmpi/lib/:$(GOOGLETOOLS)/lib:$(LD_LIBRARY_PATH)" python -c "import gpudm" 38 | 39 | %.cuo: %.cu %.hpp 40 | $(CUDA)/bin/nvcc $(CUDA_ARCH) -Xcompiler -fPIC $(INCLUDES) $(OPTFLAGS) -c $< -o $@ 41 | 42 | gpudm_wrap.cxx: gpudm.swig $(HEADERS) 43 | swig -python -c++ $(INCLUDES) gpudm.swig 44 | 45 | gpudm_wrap.o: gpudm_wrap.cxx 46 | g++ $(OPTFLAGS) -c gpudm_wrap.cxx -fPIC $(INCLUDES) -o gpudm_wrap.o 47 | 48 | %.o: %.cpp %.hpp 49 | g++ $(OPTFLAGS) -c $< -fPIC $(INCLUDES) -L$(CAFFEDIR) -o $@ 50 | 51 | clean: 52 | rm -f *.pyc *~ _gpudm.so gpudm_wrap.o $(EXTRA_LAYERS:.hpp=.o) $(EXTRA_LAYERS:.hpp=.cuo) 53 | 54 | cleanswig: clean 55 | rm -f gpudm.py gpudm_wrap.cxx gpudm_wrap.o 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /deep_matching/README.txt: -------------------------------------------------------------------------------- 1 | Implementation of the Deep Matching algorithm on GPU. 2 | See paper "DeepMatching: Hierarchical Deformable Dense Matching", 3 | at http://lear.inrialpes.fr/src/deepmatching/ by Jerome Revaud, 4 | Philippe Weinzaepfel, Zaid Harchaoui and Cordelia Schmid. 5 | Main code by Jerome Revaud, INRIA. The code is only for scientific 6 | or personnal use. Please contact me/INRIA for commercial use. 7 | Email: jerome.revaud@inria.fr 8 | 9 | Copyright (C) 2015 Jerome Revaud 10 | 11 | Version 1.0 12 | 13 | License: 14 | 15 | This program is free software: you can redistribute it and/or modify 16 | it under the terms of the GNU General Public License as published by 17 | the Free Software Foundation, either version 3 of the License, or 18 | (at your option) any later version. 19 | 20 | This program is distributed in the hope that it will be useful, 21 | but WITHOUT ANY WARRANTY; without even the implied warranty of 22 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 23 | GNU General Public License for more details. 24 | 25 | You should have received a copy of the GNU General Public License 26 | along with this program. If not, see 27 | 28 | 29 | Installation: 30 | 31 | 1) Install 'caffe' 32 | a) extract `caffe.zip` to get `caffe` directory 33 | b) mv `caffe` to some where, e.g. `mv caffe/ ../` 34 | c) cd `/path/to/caffe` and check the `Makefile` and `Makefile.config` to be sure than env settings are right. 35 | d) make -j8 36 | i use `Makefile` and `Makefile.config` of `faster-rcnn` instead of origin because of my env 37 | ( 38 | and now 39 | the `Makefile` and `Makefile.config` of caffe.zip are from `faster-rcnn` (i have replaced) 40 | ) 41 | 42 | 2) Add a softlink named 'caffe' in the current directory that points 43 | toward the caffe directory (just to make things simpler): 44 | ln -s /path/to/caffe caffe, e.g. `ln -s ../caffe caffe` 45 | 46 | 3) Edit Makefile 47 | Set the different paths for caffe and other dependencies 48 | set `GOOGLETOOLS`, `CUDA`, `MKL` path, 49 | 50 | 4) Compile 51 | make all 52 | 53 | (IMPORTANT NOTE: 54 | if you compile the SWIG file, it is normal that you see lots of errors. 55 | However, as long as it generates 'gpudm_warp.cxx', you can just relaunch 56 | 'make' and the rest of the compilation should go smoothly. ) 57 | 58 | 5) Test 59 | Set your environment variables: 60 | LD_LIBRARY_PATH should point to your libcaffe directory, to mkl, etc. 61 | (basically to all the paths you edited in the Makefile) 62 | >>> 63 | vim ~/.bashrc 64 | add "export LD_LIBRARY_PATH=/home/ddk/dongdk/deep-matching/deep_matching/caffe/.build_release/lib:/usr/local/cuda-7.0/lib64/:$LD_LIBRARY_PATH" and save 65 | source ~/.bashrc 66 | 67 | 68 | Try executing the following command: 69 | >>> python deep_matching_gpu.py liberty1.png liberty2.png -v -viz corres 70 | or 71 | >>> python deep_matching_gpu.py liberty1.png liberty2.png -GPU 0 -v -viz corres 72 | >>> python deep_matching_gpu.py liberty1.png liberty2.png -GPU 0 -v -viz flow 73 | >>> python deep_matching_gpu.py climb1.png climb2.png -GPU 0 -v -viz following 74 | 75 | if missing `libcaffe.so`, `libcusparse.so.7.0`, ..., all you do is to set the paths in environment variables 76 | e.g. 77 | `libcaffe.so` is located in `caffe/.build_release/lib` 78 | `libcusparse.so.7.0` is located in `/usr/local/cuda-7.0/lib64` (use `locate libcusparse.so.7.0` command to find where it is) 79 | and then set their paths in `~/.bashrc` file and `source ~/.bashrc` 80 | e.g. 81 | `export LD_LIBRARY_PATH=/home/ddk/dongdk/deep-matching/web_gpudm_1.0/caffe/.build_release/lib:/usr/local/cuda-7.0/lib64/:$LD_LIBRARY_PATH` 82 | 83 | You should see a rainbow visualization of correspondences. 84 | When you continue (type 'c'+enter), you should get the following output: 85 | 36 36 36 26 3.77558 13 86 | 36 44 36 34 3.65536 24 87 | [...] 88 | 28 4 22 6 3.59854 94 89 | 28 20 28 10 3.75238 90 90 | 28 28 28 18 3.77126 83 91 | 92 | 93 | Example usages and explanations: 94 | 95 | To get detailed information on parameters: 96 | python deep_matching_gpu.py -h 97 | python deep_matching_gpu.py --help 98 | >>> 99 | usage: deep_matching_gpu.py [-h] [-GPU [{-1,0,1,2,3,4}]] [-ds D] [-sp] 100 | [-ngh RAD] [-pow G] [--crop W H] [-out OUTPUT] 101 | [-v] 102 | [-viz {net,mem,pxl_desc,patch_corr,rmap,corres,flow}] 103 | img1 img2 104 | 105 | positional arguments: 106 | img1 Path to the first image 107 | img2 Path to the second image 108 | 109 | optional arguments: 110 | -h, --help show this help message and exit 111 | -GPU [{-1,0,1,2,3,4}] 112 | GPU device number (default=0), or -1 for CPU (default) 113 | -ds D, --downscale D Prior downscale of input images by 2^D 114 | -sp, --use_sparse Use CUSPARSE for ligther convolutions (GPU only) 115 | -ngh RAD, --ngh_rad RAD 116 | Restrict matching to local neighborhood of RAD pixels 117 | -pow G, --powerlaw G Non-linear power-law rectification (default = 1.4) 118 | --crop W H [Pre-processing] crop the images to a given shape 119 | -out OUTPUT, --output OUTPUT 120 | Output the matching to a text file 121 | -v, --verbose Increase verbosity 122 | -viz {net,mem,pxl_desc,patch_corr,rmap,corres,flow} 123 | Vizualisation options 124 | 125 | 126 | Typical command to match Sintel images: 127 | # Requires 4.8 Go of memory on your GPU 128 | >>> python deep_matching_gpu.py .../SINTEL/training/final/temple_3/frame_0041.png \ 129 | .../SINTEL/training/final/temple_3/frame_0042.png \ 130 | -GPU -v --downscale 1 --ngh_rad 256 --use_sparse -viz mem -viz flow 131 | 132 | >>> python deep_matching_gpu.py climb1.png climb2.png -GPU -v --downscale 1 --ngh_rad 256 --use_sparse -viz mem -viz flow 133 | 134 | >>> python deep_matching_gpu.py mude1.jpg mude2.jpg -GPU -v --downscale 1 --ngh_rad 256 --use_sparse -viz mem -viz flow 135 | 136 | >>> python deep_matching_gpu.py mude1.jpg mude2.jpg -GPU -v --downscale 1 --ngh_rad 256 --use_sparse -out mude1.txt 137 | 138 | 139 | # Note: if you don't use CUsparse, (no --use_sparse), it requires 5.3 Go. 140 | 141 | Nice vizualizations options: 142 | # e.g. use "-viz rmap" to examine response_maps 143 | # (Then click on the top image to select a patch) 144 | python deep_matching_gpu.py liberty1.png liberty2.png -v --downscale 0 -viz net -viz mem -viz rmap 145 | 146 | 147 | For details about the options, please refer to the help, the papers or the code. 148 | 149 | 150 | Important tip: 151 | If the program stops with an error in memset/memcpy, then it means that your GPU 152 | doesn't have enough memory. In this case, you should consider: 153 | * decreasing the neighborhood radius with "--ngh_rad" 154 | * incrementing the "--downscale" parameter (i.e. to downscale images before matching). 155 | * You can also investigate the memory usage with '-viz mem', but keep in mind that 156 | it under-estimates the actual memory usage. 157 | * if your image pair is very large, please use `ds`, or you will see 158 | `F0316 10:10:52.342209 24363 blob.cpp:33] Check failed: shape[i] <= 2147483647 / count_ (513 vs. 242) blob size exceeds INT_MAX*** Check failure stack trace: ***` 159 | and, for now it's not fixed. 160 | 161 | 162 | 163 | Versions of the different modules: 164 | * python 2.7 165 | * fedora 21 166 | * gcc/g++ 4.9.2 167 | * swig 3.0.7 168 | * cuda 6.5 169 | * caffe version is from November 12th (included in zip) 170 | * protobuf 2.5.0 171 | * glog 0.3.3 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | -------------------------------------------------------------------------------- /deep_matching/caffe: -------------------------------------------------------------------------------- 1 | ../caffe -------------------------------------------------------------------------------- /deep_matching/caffe.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimenglan-sysu-512/deep-flow/fd127e9a2ccd2e5b73dcef39a3438ee77b2476a4/deep_matching/caffe.zip -------------------------------------------------------------------------------- /deep_matching/extra_layers.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2015 Jerome Revaud 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see 16 | */ 17 | #ifndef __EXTA_LAYERS_H__ 18 | #define __EXTA_LAYERS_H__ 19 | 20 | #include "caffe/layer.hpp" 21 | #include "caffe/vision_layers.hpp" 22 | #include "caffe/proto/caffe.pb.h" 23 | #include 24 | 25 | #include "cusparse.h" 26 | #define CUSPARSE_CHECK(condition) CHECK_EQ((condition), CUSPARSE_STATUS_SUCCESS) 27 | 28 | #define recast reinterpret_cast 29 | 30 | namespace caffe { 31 | 32 | //////////////////////////////////////////////////////////// 33 | // CONVOLUTION WITH CSR-SPARSE WEIGHTS // 34 | ////////////////////////////////////////////////////////// 35 | 36 | class CusparseHandle { 37 | public: 38 | static cusparseHandle_t cusparse_handle() { 39 | if( instance_.cusparse_handle_ == NULL ) { // not yet initalized 40 | CUSPARSE_CHECK(cusparseCreate(&instance_.cusparse_handle_)); 41 | } 42 | return instance_.cusparse_handle_; 43 | } 44 | 45 | private: 46 | static CusparseHandle instance_; 47 | cusparseHandle_t cusparse_handle_; 48 | 49 | CusparseHandle() : 50 | cusparse_handle_(NULL) {} 51 | ~CusparseHandle() { 52 | if (cusparse_handle_) 53 | CUSPARSE_CHECK(cusparseDestroy(cusparse_handle_)); 54 | } 55 | }; 56 | 57 | /* Convolution layer but with sparse filters. 58 | This layer computes: Out = Filters x im2col( In ) 59 | 60 | In and Out are dense matrices (NCHW fully-packed). 61 | Filters are given as a CSR matrix instead as dense matrix. 62 | */ 63 | template 64 | class CSR_SparseConvolutionLayer : public Layer { 65 | public: 66 | explicit CSR_SparseConvolutionLayer(const LayerParameter& param ) 67 | : Layer(param) {} 68 | 69 | virtual void LayerSetUp(const vector*>& bottom, const vector*>& top); 70 | virtual void Reshape(const vector*>& bottom, const vector*>& top); 71 | 72 | // Blob is cast to Blob 73 | void SetSparsityPattern( const int nnz, 74 | const Blob* row_ptr, const Blob* col_ind, const Blob* data = NULL ); 75 | 76 | protected: 77 | virtual void Forward_cpu(const vector*>& bottom, const vector*>& top) 78 | {NOT_IMPLEMENTED;} 79 | virtual void Forward_gpu(const vector*>& bottom, const vector*>& top); 80 | virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 81 | {NOT_IMPLEMENTED;} 82 | virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 83 | {NOT_IMPLEMENTED;} 84 | 85 | int KSIZE_, STRIDE_, PAD_, NUM_OUTPUT_; 86 | int NUM_, CHANNELS_, HEIGHT_, WIDTH_; 87 | bool biasterm_; 88 | int M_, N_, K_; // matrix multiplication sizes 89 | Blob col_buffer_; 90 | 91 | int nnz_; // number of non-zero elements 92 | shared_ptr weight_rowptr_, weight_colind_; 93 | 94 | void load_csr_gpu( const int** weight_rowptr, const int** weight_colind, 95 | const Dtype** weight_data, Dtype** weight_diff ); 96 | 97 | bool do_im2col_; 98 | shared_ptr bias_multiplier_, transpose_res_; 99 | }; 100 | 101 | 102 | //////////////////////////////////////////////////////////// 103 | // BORDER RECTIFICATION // 104 | ////////////////////////////////////////////////////////// 105 | 106 | template 107 | class BorderRectifyLayer : public Layer { 108 | public: 109 | explicit BorderRectifyLayer(const LayerParameter& param, int ksize) 110 | : Layer(param), KSIZE_(ksize) {} 111 | 112 | virtual void LayerSetUp(const vector*>& bottom, const vector*>& top); 113 | virtual void Reshape(const vector*>& bottom, const vector*>& top); 114 | 115 | protected: 116 | virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); 117 | virtual void Forward_gpu(const vector*>& bottom, const vector*>& top); 118 | virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 119 | {NOT_IMPLEMENTED;} 120 | virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 121 | {NOT_IMPLEMENTED;} 122 | 123 | const int KSIZE_; 124 | }; 125 | 126 | 127 | //////////////////////////////////////////////////////////// 128 | // RECTIFIED SIGMOID // 129 | ////////////////////////////////////////////////////////// 130 | 131 | template 132 | class RectifiedSigmoidLayer : public NeuronLayer { 133 | public: 134 | explicit RectifiedSigmoidLayer(const LayerParameter& param) 135 | : NeuronLayer(param) {} 136 | 137 | protected: 138 | virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); 139 | virtual void Forward_gpu(const vector*>& bottom, const vector*>& top); 140 | virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 141 | {NOT_IMPLEMENTED;} 142 | virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 143 | {NOT_IMPLEMENTED;} 144 | }; 145 | 146 | 147 | //////////////////////////////////////////////////////////// 148 | // PIXEL NORMALIZATION // 149 | ////////////////////////////////////////////////////////// 150 | 151 | template 152 | class PixelNormLayer : public NeuronLayer { 153 | public: 154 | explicit PixelNormLayer(const LayerParameter& param, Dtype norm) 155 | : NeuronLayer(param), alpha_(norm) {} 156 | 157 | protected: 158 | virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); 159 | virtual void Forward_gpu(const vector*>& bottom, const vector*>& top); 160 | virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 161 | {NOT_IMPLEMENTED;} 162 | virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 163 | {NOT_IMPLEMENTED;} 164 | 165 | const Dtype alpha_; 166 | }; 167 | 168 | 169 | //////////////////////////////////////////////////////////// 170 | // PATCH CONVOLUTION // 171 | ////////////////////////////////////////////////////////// 172 | 173 | template 174 | class PatchConvolutionLayer : public Layer { 175 | public: 176 | explicit PatchConvolutionLayer(const LayerParameter& param, int ksize, int pad, int ngh_rad, char normalize_borders) 177 | : Layer(param), KSIZE_(ksize), PAD_(pad), STRIDE_(1), ngh_rad_(ngh_rad), normalize_borders_(normalize_borders) {} 178 | 179 | virtual void LayerSetUp(const vector*>& bottom, const vector*>& top); 180 | virtual void Reshape(const vector*>& bottom, const vector*>& top); 181 | 182 | protected: 183 | virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); 184 | virtual void Forward_gpu(const vector*>& bottom, const vector*>& top); 185 | virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 186 | {NOT_IMPLEMENTED;} 187 | virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 188 | {NOT_IMPLEMENTED;} 189 | 190 | const char normalize_borders_; // if 'd': general case (dynamic) border normalization 191 | // if 's': static border normalization => faster 192 | // (DeepMatching-like with normalized pixel descriptors) 193 | int num_, channels_, height_, width_; 194 | int channels_out_; 195 | const int KSIZE_, STRIDE_; 196 | int PAD_; // if PAD_ is negative, then it is applied to the first image (subsampling center patches only) 197 | int K_; 198 | const int ngh_rad_; // 0 if no offsets, else 2*nghrad+1 199 | int virtual_pad_; // PAD_ + virtual_pad_ = theoretical_pad 200 | int CROP_; // crop of the first image in case of negative padding 201 | 202 | Blob patches_; // first image transformed into consecutive patches 203 | Blob col_buffer_; // temporary matrix for convolve-ready second image (4x4 bigger) 204 | Blob ones_col_; 205 | Blob masks_; 206 | Blob dense_buf_; // buffer to transfer results from dense GEMM to sparse nghrad 207 | }; 208 | 209 | 210 | //////////////////////////////////////////////////////////// 211 | // DeepMatching ArgMax layer // 212 | ////////////////////////////////////////////////////////// 213 | 214 | template 215 | class DeepMatchingArgMaxLayer : public Layer { 216 | public: 217 | explicit DeepMatchingArgMaxLayer(const LayerParameter& param, 218 | const int img_height, const int img_width, const int step, const int ngh_rad) 219 | : Layer(param), img_height_(img_height), img_width_(img_width), step_(step), 220 | ngh_rad_(ngh_rad) {} 221 | 222 | virtual void LayerSetUp(const vector*>& bottom, const vector*>& top); 223 | virtual void Reshape(const vector*>& bottom, const vector*>& top); 224 | 225 | protected: 226 | virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); 227 | virtual void Forward_gpu(const vector*>& bottom, const vector*>& top); 228 | virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 229 | {NOT_IMPLEMENTED;} 230 | virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) 231 | {NOT_IMPLEMENTED;} 232 | 233 | const int img_height_, img_width_, step_, ngh_rad_; 234 | }; 235 | 236 | 237 | 238 | }; 239 | 240 | 241 | 242 | #endif 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | -------------------------------------------------------------------------------- /deep_matching/flow_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Contains utilities function for flows: read/write, convert to color, displaying the color coding, 3 | The functions are similar to the Middelbury function (which were written in matlab by Deqing Sun. 4 | Author: Philippe Weinzaepfel 5 | Version: 1.0 6 | Date: 19 November 2012 7 | """ 8 | 9 | import numpy as np 10 | """import matplotlib.pylab as plt""" 11 | import struct 12 | from PIL import Image 13 | 14 | # for read/write 15 | TAG_FLOAT = 202021.25 # tag to check the sanity of the file 16 | TAG_STRING = 'PIEH' # string containing the tag 17 | MIN_WIDTH = 1 18 | MAX_WIDTH = 99999 19 | MIN_HEIGHT = 1 20 | MAX_HEIGHT = 99999 21 | 22 | # for colors 23 | RY = 15 24 | YG = 6 25 | GC = 4 26 | CB = 11 27 | BM = 13 28 | MR = 6 29 | 30 | # for flow 31 | UNKNOWN_THRESH = 1e9 32 | 33 | def flowToColor(flow, maxflow=None, maxmaxflow=None, saturate=False): 34 | """ 35 | flow_utils.flowToColor(flow): return a color code flow field, normalized based on the maximum l2-norm of the flow 36 | flow_utils.flowToColor(flow,maxflow): return a color code flow field, normalized by maxflow 37 | 38 | ---- PARAMETERS ---- 39 | flow: flow to display of shape (height x width x 2) 40 | maxflow (default:None): if given, normalize the flow by its value, otherwise by the flow norm 41 | maxmaxflow (default:None): if given, normalize the flow by the max of its value and the flow norm 42 | 43 | ---- OUTPUT ---- 44 | an np.array of shape (height x width x 3) of type uint8 containing a color code of the flow 45 | """ 46 | h,w,n = flow.shape 47 | # check size of flow 48 | if not n == 2: 49 | raise Exception("flow_utils.flowToColor(flow): flow must have 2 bands") 50 | # compute max flow if needed 51 | if maxflow is None: 52 | maxflow = flowMaxNorm(flow) 53 | if maxmaxflow is not None: 54 | maxflow = min(maxmaxflow, maxflow) 55 | # fix unknown flow 56 | unknown_idx = np.max(np.abs(flow),2)>UNKNOWN_THRESH 57 | flow[unknown_idx] = 0.0 58 | # normalize flow 59 | eps = np.spacing(1) # minimum positive float value to avoid division by 0 60 | # compute the flow 61 | img = _computeColor(flow/(maxflow+eps), saturate=saturate) 62 | # put black pixels in unknown location 63 | img[ np.tile( unknown_idx[:,:,np.newaxis],[1,1,3]) ] = 0.0 64 | return img 65 | 66 | def flowMaxNorm(flow): 67 | """ 68 | flow_utils.flowMaxNorm(flow): return the maximum of the l2-norm of the given flow 69 | 70 | ---- PARAMETERS ---- 71 | flow: the flow 72 | 73 | ---- OUTPUT ---- 74 | a float containing the maximum of the l2-norm of the flow 75 | """ 76 | return np.max( np.sqrt( np.sum( np.square( flow ) , 2) ) ) 77 | 78 | 79 | def _computeColor(flow, saturate=True): 80 | """ 81 | flow_utils._computeColor(flow): compute color codes for the flow field flow 82 | 83 | ---- PARAMETERS ---- 84 | flow: np.array of dimension (height x width x 2) containing the flow to display 85 | 86 | ---- OUTPUTS ---- 87 | an np.array of dimension (height x width x 3) containing the color conversion of the flow 88 | """ 89 | # set nan to 0 90 | nanidx = np.isnan(flow[:,:,0]) 91 | flow[nanidx] = 0.0 92 | 93 | # colorwheel 94 | ncols = RY + YG + GC + CB + BM + MR 95 | nchans = 3 96 | colorwheel = np.zeros((ncols,nchans),'uint8') 97 | col = 0; 98 | #RY 99 | colorwheel[:RY,0] = 255 100 | colorwheel[:RY,1] = [(255*i) // RY for i in range(RY)] 101 | col += RY 102 | # YG 103 | colorwheel[col:col+YG,0] = [255 - (255*i) // YG for i in range(YG)] 104 | colorwheel[col:col+YG,1] = 255 105 | col += YG 106 | # GC 107 | colorwheel[col:col+GC,1] = 255 108 | colorwheel[col:col+GC,2] = [(255*i) // GC for i in range(GC)] 109 | col += GC 110 | # CB 111 | colorwheel[col:col+CB,1] = [255 - (255*i) // CB for i in range(CB)] 112 | colorwheel[col:col+CB,2] = 255 113 | col += CB 114 | # BM 115 | colorwheel[col:col+BM,0] = [(255*i) // BM for i in range(BM)] 116 | colorwheel[col:col+BM,2] = 255 117 | col += BM 118 | # MR 119 | colorwheel[col:col+MR,0] = 255 120 | colorwheel[col:col+MR,2] = [255 - (255*i) // MR for i in range(MR)] 121 | 122 | # compute utility variables 123 | rad = np.sqrt( np.sum( np.square(flow) , 2) ) # magnitude 124 | a = np.arctan2( -flow[:,:,1] , -flow[:,:,0]) / np.pi # angle 125 | fk = (a+1)/2 * (ncols-1) # map [-1,1] to [0,ncols-1] 126 | k0 = np.floor(fk).astype('int') 127 | k1 = k0+1 128 | k1[k1==ncols] = 0 129 | f = fk-k0 130 | 131 | if not saturate: 132 | rad = np.minimum(rad,1) 133 | 134 | # compute the image 135 | img = np.zeros( (flow.shape[0],flow.shape[1],nchans), 'uint8' ) 136 | for i in range(nchans): 137 | tmp = colorwheel[:,i].astype('float') 138 | col0 = tmp[k0]/255 139 | col1 = tmp[k1]/255 140 | col = (1-f)*col0 + f*col1 141 | idx = (rad <= 1) 142 | col[idx] = 1-rad[idx]*(1-col[idx]) # increase saturation with radius 143 | col[~idx] *= 0.75 # out of range 144 | img[:,:,i] = (255*col*(1-nanidx.astype('float'))).astype('uint8') 145 | 146 | return img 147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /deep_matching/gpudm.swig: -------------------------------------------------------------------------------- 1 | // -*- c++ -*- 2 | /* 3 | Copyright (C) 2015 Jerome Revaud 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see 17 | */ 18 | 19 | %module gpudm; 20 | 21 | %{ 22 | 23 | #include "caffe/caffe.hpp" 24 | #include "extra_layers.hpp" 25 | 26 | using namespace caffe; 27 | using namespace std; 28 | 29 | %} 30 | 31 | %include "std_string.i" 32 | %include "std_pair.i" 33 | %include "std_set.i" 34 | %include "std_map.i" 35 | %include "std_vector.i" 36 | %include "boost_shared_ptr.i" 37 | 38 | 39 | %shared_ptr(caffe::Blob< float >); 40 | %shared_ptr(caffe::Layer< float >); 41 | %shared_ptr(caffe::NeuronLayer); 42 | %shared_ptr(caffe::ReLULayer< float >); 43 | %shared_ptr(caffe::PowerLayer< float >); 44 | %shared_ptr(caffe::RectifiedSigmoidLayer< float >); 45 | %shared_ptr(caffe::ReshapeLayer< float >); 46 | %shared_ptr(caffe::PixelNormLayer< float >); 47 | %shared_ptr(caffe::BaseConvolutionLayer< float >); 48 | %shared_ptr(caffe::ConvolutionLayer< float >); 49 | %shared_ptr(caffe::PatchConvolutionLayer); 50 | %shared_ptr(caffe::CSR_SparseConvolutionLayer); 51 | %shared_ptr(caffe::BorderRectifyLayer< float >); 52 | %shared_ptr(caffe::PoolingLayer); 53 | %shared_ptr(caffe::InnerProductLayer); 54 | %shared_ptr(caffe::DeepMatchingArgMaxLayer); 55 | 56 | 57 | %include "caffe/caffe.hpp" 58 | %include "caffe/common.hpp" 59 | %include "caffe/blob.hpp" 60 | %include "caffe/filler.hpp" 61 | %include "caffe/layer.hpp" 62 | 63 | using namespace std; 64 | using namespace boost; 65 | %template(BlobVector) vector< shared_ptr< caffe::Blob< float > > >; 66 | %template(BlobPtrVector) vector< caffe::Blob< float >* >; 67 | %template(LayerVector) vector< shared_ptr< caffe::Layer< float > > >; 68 | %template(FloatVector) vector< float >; 69 | 70 | %include "caffe/net.hpp" 71 | %include "caffe/common_layers.hpp" 72 | %include "caffe/neuron_layers.hpp" 73 | %include "caffe/vision_layers.hpp" 74 | %include "caffe/util/math_functions.hpp" 75 | %include "extra_layers.hpp" 76 | 77 | #define GOOGLE_PROTOBUF_VERSION 2005000 78 | #define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 2005000 79 | 80 | namespace google { 81 | namespace protobuf { 82 | class Message { 83 | private: 84 | Message(); 85 | int x; 86 | }; 87 | typedef unsigned int uint32; 88 | typedef int int32; 89 | typedef long long int64; 90 | } 91 | } 92 | 93 | %include "caffe/proto/caffe.pb.h" 94 | 95 | 96 | %template(NetFloat) caffe::Net; 97 | %template(BlobFloat) caffe::Blob; 98 | %template(LayerFloat) caffe::Layer; 99 | %template(NeuronLayerFloat) caffe::NeuronLayer; 100 | %template(ReLULayerFloat) caffe::ReLULayer; 101 | %template(PowerLayerFloat) caffe::PowerLayer; 102 | %template(ReshapeLayerFloat) caffe::ReshapeLayer< float >; 103 | %template(RectifiedSigmoidLayerFloat) caffe::RectifiedSigmoidLayer; 104 | %template(PixelNormLayerFloat) caffe::PixelNormLayer< float >; 105 | %template(BaseConvolutionLayerFloat) caffe::BaseConvolutionLayer< float >; 106 | %template(ConvolutionLayerFloat) caffe::ConvolutionLayer< float >; 107 | %template(PatchConvolutionLayerFloat) caffe::PatchConvolutionLayer; 108 | %template(CSR_SparseConvolutionLayerFloat) caffe::CSR_SparseConvolutionLayer; 109 | %template(BorderRectifyLayerFloat) caffe::BorderRectifyLayer< float >; 110 | %template(PoolingLayerFloat) caffe::PoolingLayer; 111 | %template(DeepMatchingArgMaxLayerFloat) caffe::DeepMatchingArgMaxLayer; 112 | 113 | // function releases GIL while doing slow action 114 | %exception { 115 | Py_BEGIN_ALLOW_THREADS 116 | $action 117 | Py_END_ALLOW_THREADS 118 | } 119 | 120 | %exception; 121 | 122 | 123 | %{ 124 | #define SWIG_FILE_WITH_INIT 125 | #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION 126 | #include 127 | 128 | PyObject * floats_to_numpy_ref(float *src, long size) { 129 | npy_intp shp = size; 130 | return PyArray_SimpleNewFromData(1, &shp, NPY_FLOAT32, src); 131 | } 132 | 133 | %} 134 | 135 | PyObject * floats_to_numpy_ref(float *src, long size); 136 | 137 | %init %{ 138 | /* needed, else crash at runtime */ 139 | import_array(); 140 | %} 141 | 142 | 143 | %pythoncode %{ 144 | 145 | import numpy 146 | 147 | def BlobFloat_get_shape(self): 148 | return (self.num(), self.channels(), self.height(), self.width()) 149 | 150 | def BlobFloat_to_numpy_ref(self): 151 | b = floats_to_numpy_ref(self.cpu_data(), self.count()) 152 | return b.reshape(self.get_shape()) 153 | 154 | def BlobFloat_diff_to_numpy_ref(self): 155 | b = floats_to_numpy_ref(self.cpu_diff(), self.count()) 156 | return b.reshape(self.get_shape()) 157 | 158 | def BlobFloat_mutable_to_numpy_ref(self): 159 | b = floats_to_numpy_ref(self.mutable_cpu_data(), self.count()) 160 | return b.reshape(self.get_shape()) 161 | 162 | def BlobFloat_mutable_diff_to_numpy_ref(self): 163 | b = floats_to_numpy_ref(self.mutable_cpu_diff(), self.count()) 164 | return b.reshape(self.get_shape()) 165 | 166 | BlobFloat.get_shape = BlobFloat_get_shape 167 | BlobFloat.to_numpy_ref = BlobFloat_to_numpy_ref 168 | BlobFloat.diff_to_numpy_ref = BlobFloat_diff_to_numpy_ref 169 | BlobFloat.mutable_to_numpy_ref = BlobFloat_mutable_to_numpy_ref 170 | BlobFloat.mutable_diff_to_numpy_ref = BlobFloat_mutable_diff_to_numpy_ref 171 | 172 | %} 173 | 174 | 175 | /*-------- DeepMatching functions ------------*/ 176 | %{ 177 | 178 | #include "numpy_image.h" 179 | #include 180 | using std::min; 181 | using std::max; 182 | static inline float pow2(float x) {return x*x;} 183 | #define nullptr 0 184 | 185 | static inline int retrieve_children( const int x, const int y, const int_cube* child_grid ) { 186 | const int size0_div2 = child_grid->pixels[0]; 187 | const int step0 = child_grid->tx==1 && child_grid->ty==1 ? 1 : 188 | max( child_grid->pixels[2]-child_grid->pixels[0], 189 | child_grid->pixels[1+2*child_grid->tx]-child_grid->pixels[1] ); 190 | int i = (x-size0_div2)/step0; 191 | int j = (y-size0_div2)/step0; 192 | assert( x==(i*step0+size0_div2) || !"error: child_grid does not match current grid" ); 193 | assert( y==(j*step0+size0_div2) || !"error: child_grid does not match current grid" ); 194 | if( i<0 || i>=child_grid->tx ) return -1; 195 | if( j<0 || j>=child_grid->ty ) return -1; 196 | return i+j*child_grid->tx; 197 | } 198 | 199 | static inline void prepare_gaps( const int parent_psize, const int nc, int gaps[3] ) { 200 | const int hs = parent_psize/2; 201 | if(nc==2) { // 4 children per parent patch 202 | gaps[0] = hs/2 - hs; 203 | gaps[1] = hs/2; 204 | } else if(nc==3) { // 9 children per parent patch 205 | gaps[0] = hs/2 - hs; 206 | gaps[1] = 0; 207 | gaps[2] = hs/2; 208 | } else assert(0); 209 | } 210 | 211 | /* Prepare a grid of cell positions in the first image for a given scale. Big cells inherit the cell at the previous scale. 212 | size = size of cells at current scale 213 | offset, step = grid generator: (offset + i*step, offset + j*step) 214 | child_grid = grid of the previous layer (or None if first layer) 215 | child_norms = image containing the norms of the patch at the previous level 216 | grid = result center positions of cells in current scale 217 | children = index of cells in previous scale used to construct big cells 218 | norms = norms of the cells of this level 219 | */ 220 | void _prepare_big_cells( int size, int offset, int step, 221 | int_cube* child_grid, float_image* child_norms, 222 | int_cube* grid, int_cube* children, float_image* norms ) { 223 | assert(grid->tz==2); 224 | const int ntx = grid->tx; // should be == 1+(tx-size)/step so that patches do not pass the border 225 | const int nty = grid->ty; // should be == 1+(ty-size)/step so that patches do not pass the border 226 | 227 | /* grid[i,j] = ( offset + i*step, offset + j*step ) 228 | 229 | connection between two scales: 230 | x cell position in lower scale == x position of children in upper scale 231 | child_offset + child_i*child_step = offset + i*step + U*size/4 232 | with U = (2*u/(nc-1)-1) \in {-1,0,1} 233 | */ 234 | 235 | int i,j,u,v; 236 | int* r = grid->pixels; 237 | 238 | if( !child_grid ) { 239 | // this is the first scale: 240 | // we just return a grid of step size*(1-overlap/2) in [0, tx[ x [0, ty[ 241 | 242 | for(j=0; jtz==2); 249 | assert( (child_norms!=NULL) == (norms!=NULL) || !"both must be null or non-null at the same time" ); 250 | if(norms) ASSERT_SAME_SIZE( child_grid, child_norms ); 251 | assert( children ); 252 | const int nc = sqrt(children->tz); // number of children per row or col 253 | assert( children->tz==pow2(nc) ); 254 | ASSERT_SAME_SIZE( grid, children ); 255 | if(norms) ASSERT_SAME_SIZE( grid, norms ); 256 | // this is at least second scale 257 | // we return a grid of step size*(1-overlap/2) in [0, tx[ x [0, ty[ 258 | 259 | int gaps[3]; 260 | prepare_gaps( size, nc, gaps ); // usually, returns [-q,q] for nc==2 with q=size/4 261 | 262 | int* c = children->pixels; 263 | float *n = norms ? norms->pixels : nullptr; 264 | if(n) memset(n,0,ntx*nty*sizeof(float)); 265 | for(j=0; j=0) *n += child_norms->pixels[*c]; 279 | } 280 | if(n) n++; 281 | } 282 | } 283 | } 284 | 285 | #define NEWA(type,n) (type*)malloc(sizeof(type)*(n)) 286 | 287 | static float** get_list_corres( const float_cube* map, int* nb ) { 288 | const int tz = map->tz; 289 | float* m = map->pixels; 290 | const long npix = map->tx*map->ty; 291 | float** res = NEWA(float*,npix); 292 | 293 | int i,n=0; 294 | for(i=0; itz==tz && map1->tz==tz ); 312 | 313 | // build the list of triplets 314 | int n0,n1; 315 | float** const corres0 = get_list_corres(map0,&n0); 316 | float** const corres1 = get_list_corres(map1,&n1); 317 | 318 | // arg-sort the lists 319 | qsort( corres0, n0, sizeof(float*), cmp_corres ); 320 | qsort( corres1, n1, sizeof(float*), cmp_corres ); 321 | 322 | // remove all correspondences from map0/map1 that is not shared 323 | float** c0 = corres0; 324 | float** c1 = corres1; 325 | float** const c0max = corres0 + n0; 326 | float** const c1max = corres1 + n1; 327 | float* res = corres->pixels; 328 | float* r = res; 329 | while(c00) { // corres0 > corres1 335 | c1++; 336 | } else { // corres0 == corres1 337 | if( r==res || memcmp( r-tz, *c0, tz*sizeof(float) ) ) { // if not already copied 338 | memcpy( r, *c0, tz*sizeof(float) ); 339 | r += tz; 340 | } 341 | c0++; 342 | c1++; 343 | } 344 | } 345 | 346 | free(corres0); 347 | free(corres1); 348 | return (r-res)/tz; 349 | } 350 | 351 | %} 352 | 353 | %include 354 | 355 | void _prepare_big_cells( int size, int offset, int step, 356 | int_cube* child_grid, float_image* child_norms, 357 | int_cube* grid, int_cube* children, float_image* norms ); 358 | 359 | int _intersect_corres( const float_cube* map0, const float_cube* map1, float_image* corres ); 360 | 361 | 362 | %pythoncode %{ 363 | 364 | def prepare_big_cells( imshape, cell_size, overlap, child_overlap, child_grid, child_norms, dense_step=0, offset=None ): 365 | if offset is None: offset = cell_size/2 if not dense_step else 0 366 | step = cell_size/(overlap+1) if not dense_step else dense_step 367 | grid_size = lambda imsize: 1+max(0,imsize-2*offset)/step 368 | gtx = grid_size(imshape[1]) 369 | gty = grid_size(imshape[0]) 370 | grid = numpy.empty((gty,gtx,2),numpy.int32) 371 | norms = numpy.zeros((gty,gtx),numpy.float32) if child_norms is not None else None 372 | 373 | assert overlap in {0,1} 374 | nc = (2+child_overlap)**2 # number of children per cell 375 | children = numpy.empty((gty,gtx,nc),numpy.int32) if child_grid!=None else None 376 | 377 | _prepare_big_cells( cell_size, offset, step, child_grid, child_norms, grid, children, norms ) 378 | 379 | if norms is None: 380 | return step, grid, children 381 | else: 382 | return step, grid, children, norms 383 | 384 | def intersect_corres( c0, c1 ): 385 | n = min(c0.size, c1.size) / 6 386 | res = numpy.empty((n,6), numpy.float32) 387 | n = _intersect_corres( c0, c1, res ) 388 | return res[:n] 389 | 390 | 391 | %} 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | -------------------------------------------------------------------------------- /deep_matching/helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2015 Jerome Revaud 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see 16 | """ 17 | import os, sys, pdb, cPickle 18 | from collections import namedtuple 19 | from PIL import Image 20 | from numpy import * 21 | try: 22 | from matplotlib.pyplot import * 23 | ion() 24 | except: 25 | pass 26 | 27 | ######################################################## 28 | # visualization 29 | 30 | def noticks(): 31 | xticks([]) 32 | yticks([]) 33 | 34 | def plot_rect(l,t,r,b,ls='-',rescale=True,**kwargs): 35 | (L,R), (B,T) = xlim(), ylim() 36 | plot([l,r,r,l,l],[t,t,b,b,t],ls,scalex=0,scaley=0,**kwargs) 37 | if rescale: 38 | xlim((min(L,l),max(R,r))) 39 | ylim((max(B,b),min(T,t))) 40 | 41 | def plot_square(cx,cy,rad,ls='-',**kwargs): 42 | plot_rect(cx-rad,cy-rad,cx+rad,cy+rad,ls,**kwargs) 43 | 44 | 45 | def get_assign(assign, n_maps): 46 | return arange(n_maps) if assign==None else assign 47 | 48 | 49 | def show_conv( levels, rot45=None, nshow=0, img0=None, img1=None, **kwargs ): 50 | level = levels[-1] 51 | grid = level.grid.reshape(-1,2) 52 | if rot45: grid = apply_rot45(rot45,grid) 53 | rad = level.psize/2 54 | assign = get_assign(level.assign,len(level.res_map)) 55 | 56 | ax1 = subplot(311) 57 | ax1.numplot = 1 58 | imshow(img0, interpolation='nearest') 59 | ax2 = subplot(312) 60 | ax2.numplot = 2 61 | imshow(img1, interpolation='nearest') 62 | ax3 = subplot(313) 63 | ax3.numplot = 3 64 | fig = get_current_fig_manager().canvas.figure 65 | 66 | def redraw(): 67 | # we redraw only the concerned axes 68 | renderer = fig.canvas.get_renderer() 69 | ax1.draw(renderer) 70 | ax2.draw(renderer) 71 | ax3.draw(renderer) 72 | fig.canvas.blit(ax1.bbox) 73 | fig.canvas.blit(ax2.bbox) 74 | fig.canvas.blit(ax3.bbox) 75 | 76 | global cur # ugly but wo cares 77 | cur = None 78 | def motion_notify_callback(event): 79 | global cur 80 | if not event.inaxes: return 81 | x,y = event.xdata, event.ydata 82 | if x and y: # we are somewhere on a plot 83 | if cur is not None and event.inaxes.numplot in (2,3): 84 | ax2.lines = ax2.lines[:1] 85 | ax3.lines = [] 86 | offx,offy = (0,0) if level.offsets is None else level.offsets.reshape(-1,2)[cur] 87 | if event.inaxes.numplot==2: 88 | f = level.f 89 | x,y = int(0.5 + x/f), int(0.5 + y/f) 90 | else: 91 | f = 1 92 | x,y = int(0.5 + (offx+x)/f), int(0.5 + (offy+y)/f) 93 | fig.add_subplot(312) 94 | xl,yl=xlim(),ylim() 95 | ax2.plot(x*level.f,y*level.f,'+',c=(0,1,0),ms=10,scalex=0,scaley=0) 96 | plot_square(x*level.f-0.5,y*level.f-0.5,rad,color='b') 97 | xlim(xl);ylim(yl) 98 | ax3.plot(x-offx,y-offy,'+k',ms=20,scalex=0,scaley=0) 99 | redraw() 100 | 101 | def mouse_click_callback(event): 102 | global cur 103 | if not event.inaxes: return 104 | x,y = event.xdata, event.ydata 105 | if x and y: # we are somewhere on a plot 106 | if event.inaxes.numplot==1: 107 | cur = sum((grid - [x,y])**2,1).argmin() # find nearest point 108 | x,y = grid[cur] 109 | ax1.lines = [] 110 | ax2.lines = [] 111 | ax3.lines = [] 112 | fig.add_subplot(311) 113 | xl,yl=xlim(),ylim() 114 | plot(x,y,'+',color=(0,1,0),ms=10,mew=1) 115 | plot_square(x-0.5,y-0.5,rad,color='b') 116 | xlim(xl);ylim(yl) 117 | ax3.images = [] 118 | ax3.imshow(level.res_map[assign[cur]], vmin=0, vmax=1.1, interpolation='nearest') 119 | if level.offsets is not None: 120 | ox, oy = level.offsets.reshape(-1,2)[cur] 121 | sx, sy = level.res_map.shape[1:] 122 | subplot(312) 123 | plot_rect(level.f*ox,level.f*oy,level.f*(ox+sx),level.f*(oy+sy),'-',c=(0,1,0),rescale=False) 124 | redraw() 125 | elif cur is not None and event.inaxes.numplot>1: 126 | offx,offy = (0,0) if level.offsets is None else level.offsets.reshape(-1,2)[cur] 127 | offx,offy = (0,0) if level.offsets is None else level.offsets.reshape(-1,2)[cur] 128 | if event.inaxes.numplot==2: 129 | f = level.f 130 | x,y = int(0.5 + x/f), int(0.5 + y/f) 131 | score = level.res_map[cur,y-offy,x-offx] 132 | else: 133 | x,y = int(0.5 + x), int(0.5 + y) 134 | score = level.res_map[cur,y,x] 135 | print 'res_map[%d, %d, %d] = %g' % (cur, y, x, score) 136 | 137 | class FakeEvent: 138 | def __init__(self, x,y,ax): 139 | self.xdata=x; self.ydata=y; self.inaxes=ax 140 | mouse_click_callback(FakeEvent(1,1,ax1)) 141 | 142 | subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0.02, hspace=0.02) 143 | cid_move = fig.canvas.mpl_connect('motion_notify_event',motion_notify_callback) 144 | cid_clic = fig.canvas.mpl_connect('button_press_event',mouse_click_callback) 145 | print "Click on the top image to select a patch..." 146 | pdb.set_trace() 147 | fig.canvas.mpl_disconnect(cid_move) 148 | fig.canvas.mpl_disconnect(cid_clic) 149 | 150 | 151 | def get_imatches(matches, shape, psize=8): 152 | half = psize/2 153 | imatches = -ones(shape, dtype=int32) 154 | nums = arange(len(matches)) 155 | for j in range(-half,half+(psize%2)): 156 | for i in range(-half,half+(psize%2)): 157 | imatches[ clip(matches[:,1]+i,0,shape[0]-1), 158 | clip(matches[:,0]+j,0,shape[1]-1)] = nums 159 | return imatches 160 | 161 | 162 | def show_flow( lm, maxima, corr, img0=None, img1=None, mode='flow', full_corres=None, psize=None, **viz ): 163 | assert img0!=None 164 | assert img1!=None 165 | if type(corr)==tuple: corr = corr[0] 166 | assert corr.size, 'error: empty correspondences' 167 | if corr.ndim==3: 168 | corr = corr[corr[:,:,4]>0] 169 | set_max = set(corr[:,5]) 170 | colors = {m:i for i,m in enumerate(set_max)} 171 | colors = {m:cm.jet(i/float(len(colors))) for m,i in colors.items()} 172 | for key in viz: 173 | if key.startswith('mode_') and viz[key] is True: 174 | mode = key[5:] 175 | 176 | def motion_notify_callback(event): 177 | if not event.inaxes: return 178 | x,y = event.xdata, event.ydata 179 | if x and y: # we are somewhere on a plot 180 | ax1.lines = [] 181 | ax2.lines = [] 182 | if event.inaxes.numplot==0: 183 | if mode=='score_path': 184 | ax3.lines = [] 185 | col = fc0[int(y/step),int(x/step)] 186 | x0, y0, x1, y1 = col[:4] 187 | ax3.plot( col[6:], '+-', color='k' ) 188 | elif mode in ('comatches','argmax'): 189 | n = sum((corr[:,0:2] - [x,y])**2,1).argmin() # find nearest point 190 | x0,y0,x1,y1,_,m = corr[n,0:6] 191 | # print leading correspondences 192 | ax1.plot(x0,y0,'o',ms=10,mew=2,color='blue',scalex=False,scaley=False) 193 | ax2.plot(x1,y1,'o',ms=10,mew=2,color='red',scalex=False,scaley=False) 194 | # find co-matches 195 | corres0 = retrieve_one_maxima_corres( lm, maxima[m], **viz['params'] ) 196 | corres0 = set(map(tuple,corres0[:,0:4])) 197 | if mode == 'comatches': 198 | real0 = set(map(tuple,corr[:,0:4])) 199 | intersect = corres0 & real0 200 | else: 201 | intersect = corres0 # no filtering 202 | x0,y0,x1,y1 = zip(*list(intersect)) 203 | 204 | else: 205 | n = sum((corr[:,0:2] - [x,y])**2,1).argmin() # find nearest point 206 | x0,y0,x1,y1,score,m = corr[n,0:6] 207 | print "\rmatch #%d (%d,%d) --> (%d,%d) (len=%.1f), score=%.3f from maxima %d" % (n, 208 | x0,y0,x1,y1,sqrt((x0-x1)**2+(y0-y1)**2),score,m),;sys.stdout.flush() 209 | 210 | ax1.plot(x0,y0,'+',ms=10,mew=2,color='blue',scalex=False,scaley=False) 211 | ax2.plot(x1,y1,'+',ms=10,mew=2,color='red',scalex=False,scaley=False) 212 | 213 | elif event.inaxes.numplot==1: 214 | if mode=='score_path': 215 | ax3.lines = [] 216 | col = fc1[int(y/step),int(x/step)] 217 | x0, y0, x1, y1 = col[:4] 218 | ax3.plot( col[6:], '+-', color='k' ) 219 | else: 220 | n = sum((corr[:,2:4] - [x,y])**2,1).argmin() # find nearest point 221 | x0,y0,x1,y1,score,m = corr[n,0:6] 222 | print "\rmatch #%d (%d,%d) --> (%d,%d) (len=%.1f), score=%.3f from maxima %d" % (n, 223 | x0,y0,x1,y1,sqrt((x0-x1)**2+(y0-y1)**2),score,m),;sys.stdout.flush() 224 | ax1.plot(x0,y0,'+',ms=10,mew=2,color='red',scalex=False,scaley=False) 225 | ax2.plot(x1,y1,'+',ms=10,mew=2,color='blue',scalex=False,scaley=False) 226 | # we redraw only the concerned axes 227 | renderer = fig.canvas.get_renderer() 228 | ax1.draw(renderer) 229 | ax2.draw(renderer) 230 | fig.canvas.blit(ax1.bbox) 231 | fig.canvas.blit(ax2.bbox) 232 | if mode=='score_path': 233 | ax3.set_ylim((0,1)) 234 | ax3.draw(renderer) 235 | fig.canvas.blit(ax3.bbox) 236 | 237 | wider_than_high = (img0.shape[0]+img1.shape[0]