├── .gitignore ├── 1 preprocessing ├── s4_whole_sequence_L0smooth_on_time_axis │ ├── rose-unrotate.HOW │ ├── rose-timewise.HOW │ ├── l0smooth1Dsparse │ │ ├── mask_h.pyxbld │ │ ├── mask_h.pyx │ │ ├── ptsv.py │ │ └── l0smooth1Dsparse.py │ ├── README.txt │ ├── l0smooth.HOW │ ├── timerotate.py │ └── timeunrotate.py ├── s1_whole_sequence_colorshift │ ├── README.txt │ ├── colorshift.json │ └── colorshift_filter.cpp ├── s5_get_albedo_sequence │ ├── README.txt │ └── extract_albedo_image_sequence.py ├── s2_extract_keyframes │ ├── README.txt │ └── detect_keyframe.cpp └── s3_subsequence_movestd_movemedian │ ├── README.txt │ ├── moving_median_with_mask_function.cpp │ ├── moving_standard_deviation_mask_occlusion.py │ └── new_rolling_median.h ├── 2 layer extraction ├── PD_3by3_method_layer_extraction │ ├── test_case │ │ ├── after.png │ │ ├── alpha.png │ │ ├── before.png │ │ ├── layer.png │ │ ├── paint.png │ │ └── ground_truth-189_214_110.png │ ├── README.txt │ └── PD_spatial_coherency_3by3_layer_extraction.cpp └── KM_PD_layer_extraction │ ├── README.txt │ ├── rose_layer_extraction.json │ └── layer_extraction.cpp └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.pyo 4 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/rose-unrotate.HOW: -------------------------------------------------------------------------------- 1 | ## On Windows, don't put quotes. 2 | python timeunrotate.py --glob 'rose-timewise-320x480/*.png' rose-unrotate/rose 3 | -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/test_case/after.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CraGL/timelapse/HEAD/2 layer extraction/PD_3by3_method_layer_extraction/test_case/after.png -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/test_case/alpha.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CraGL/timelapse/HEAD/2 layer extraction/PD_3by3_method_layer_extraction/test_case/alpha.png -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/test_case/before.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CraGL/timelapse/HEAD/2 layer extraction/PD_3by3_method_layer_extraction/test_case/before.png -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/test_case/layer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CraGL/timelapse/HEAD/2 layer extraction/PD_3by3_method_layer_extraction/test_case/layer.png -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/test_case/paint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CraGL/timelapse/HEAD/2 layer extraction/PD_3by3_method_layer_extraction/test_case/paint.png -------------------------------------------------------------------------------- /1 preprocessing/s1_whole_sequence_colorshift/README.txt: -------------------------------------------------------------------------------- 1 | Compile `colorshift_filter.cpp`. Run `colorshift_filter path/to/parameters.json`. In the example `colorshift.json` file, you may need to modify some file paths. 2 | -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/test_case/ground_truth-189_214_110.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CraGL/timelapse/HEAD/2 layer extraction/PD_3by3_method_layer_extraction/test_case/ground_truth-189_214_110.png -------------------------------------------------------------------------------- /1 preprocessing/s5_get_albedo_sequence/README.txt: -------------------------------------------------------------------------------- 1 | Modify the paths inside `extract_albedo_image_sequence.py` and run it. 2 | 3 | This code uses the l0-smoothed image sequence (result of step 4). 4 | The output is a sequence of albedo images. 5 | -------------------------------------------------------------------------------- /1 preprocessing/s2_extract_keyframes/README.txt: -------------------------------------------------------------------------------- 1 | Modify the paths inside `detect_keyframe.cpp`, compile, and run it. 2 | 3 | This code takes as input the color-shifted image sequence output from `s1_whole_sequence_colorshift` (step 1). 4 | It outputs a keyframe sequence, keyframe difference masks, and a new sequence (after performing subsequence color shifts). 5 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/rose-timewise.HOW: -------------------------------------------------------------------------------- 1 | ## The quotes around the thing with the * are important, because we want 2 | ## python to interpret the *, not the shell. 3 | 4 | ## On Windows, don't put quotes. 5 | python timerotate.py --glob 3200 --max-GB 2 'rose_subsequence_movestd_movingmedian_recover/*.png' rose-timewise/rose-timewise 6 | -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/README.txt: -------------------------------------------------------------------------------- 1 | Compile and run `PD_spatial_coherency_3by3_layer_extraction.cpp`. You will need to include and link OpenCV and Eigen. You can change the paths in `main()`. 2 | 3 | The code uses a before/after image pair as input. The output is the RGBA paint stroke (`layer.png` which is a combination of `alpha.png` and `paint.png`). 4 | -------------------------------------------------------------------------------- /2 layer extraction/KM_PD_layer_extraction/README.txt: -------------------------------------------------------------------------------- 1 | Compile `layer_extraction.cpp`. You will need to include and link: opencv, json, zlib. 2 | Run `layer_extraction path/to/parameters.json`. In the example `rose_layer_extraction.json`, you may need to modify some paths. 3 | 4 | This code uses a clean albedo video or image sequence (resulting from the preprocessing steps). It outputs KM layers or PD layers or both. 5 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/l0smooth1Dsparse/mask_h.pyxbld: -------------------------------------------------------------------------------- 1 | ## From: http://stackoverflow.com/questions/26833947/how-can-i-set-cython-compiler-flags-when-using-pyximport 2 | def make_ext(modname, pyxfilename): 3 | from distutils.extension import Extension 4 | return Extension(name=modname, 5 | sources=[pyxfilename], 6 | extra_compile_args=['-O3']) 7 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/README.txt: -------------------------------------------------------------------------------- 1 | This step depends on NumPy and LAPACK and PIL (or Pillow). 2 | 3 | First, run `timerotate.py` to convert an image sequence (the result of step 3) to a set of timewise images (each row is a pixel across time). See `rose-timewise.HOW`. 4 | 5 | Then, run `l0smooth1Dsparse.py`, to get l0-smoothed timewise images. See `l0smooth.HOW`. 6 | 7 | Finally, run `timeunrotate.py` to convert the l0-smoothed timewise images back to an image sequence. See `rose-unrotate.HOW`. 8 | -------------------------------------------------------------------------------- /1 preprocessing/s3_subsequence_movestd_movemedian/README.txt: -------------------------------------------------------------------------------- 1 | This step is a Python script which calls through to a compiled binary. The Python script depends on [Bottleneck](https://pypi.python.org/pypi/Bottleneck) (`pip install bottleneck`). 2 | Compile `moving_median_with_mask_function.cpp` to get `moving_median_with_mask_function.exe`. 3 | Run `moving_standard_deviation_mask_occlusion.py`. You may need to change some paths and the keyframe indices inside. 4 | 5 | This code uses the keyframe mask sequence and colorshift image sequence resulting from the previous step (2). It output a new image sequence with occlusions removed. 6 | -------------------------------------------------------------------------------- /1 preprocessing/s1_whole_sequence_colorshift/colorshift.json: -------------------------------------------------------------------------------- 1 | { 2 | "operations": 3 | [ 4 | { 5 | "filter": "VideoReader", 6 | "params": { 7 | "filename": "rose.mp4", 8 | "frameStop" : 5283, 9 | "frameStart" : 0 10 | } 11 | }, 12 | 13 | { 14 | "filter": "ColorShift", 15 | "params": { 16 | "Percent": 0.5, 17 | "Threshold": 0.0 18 | } 19 | }, 20 | 21 | { 22 | "filter": "VideoSaver", 23 | "params": { 24 | "filename": "rose_colorshift_" 25 | } 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/l0smooth.HOW: -------------------------------------------------------------------------------- 1 | python l0smooth1Dsparse.py --lambda 1e-3 --kappa 1.1 --constraint-weight hard rose-timewise/rose-timewise-320x480-0000.png 1 rose-timewise/l0_1e-3_1.1_hard_rose-timewise-320x480-0000.png 2 | 3 | python l0smooth1Dsparse.py --lambda 1e-3 --kappa 1.1 --constraint-weight hard rose-timewise/rose-timewise-320x480-0001.png 1 rose-timewise/l0_1e-3_1.1_hard_rose-timewise-320x480-0001.png 4 | 5 | python l0smooth1Dsparse.py --lambda 1e-3 --kappa 1.1 --constraint-weight hard rose-timewise/rose-timewise-320x480-0002.png 1 rose-timewise/l0_1e-3_1.1_hard_rose-timewise-320x480-0002.png 6 | 7 | # ... 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Decomposing Time-Lapse Paintings into Layers 2 | 3 | This code implements the pipeline described in the paper "Decomposing Time-Lapse Paintings into Layers" by Jianchao Tan, Marek Dvorožňák, Daniel Sýkora, Yotam Gingold from SIGGRAPH 2015. The pipeline is divided into two stages. 4 | 5 | ### 1 Preprocessing: 6 | - Input: raw time-lapse video 7 | - Output: albedo video 8 | 9 | The substeps are: 10 | 11 | 1. Color shift the whole sequence 12 | 2. Extract keyframes and color shift each sub-sequence 13 | 3. For each sub-sequence, perform moving std. deviation and moving median 14 | 4. Whole sequence L0 smoothing 15 | 5. Perform albedo conversion 16 | 17 | ### 2 Layer extraction 18 | - Input: albedo video 19 | - Output: KM layers and PD layers 20 | 21 | The programs are: 22 | 23 | - PD layer extraction and KM layer extraction 24 | - PD using the spatial coherency solution: The 3-by-3 layer extraction described in the paper 25 | 26 | ## Dependencies 27 | 28 | - OpenCV 2.4 29 | - Eigen 3 30 | - JsonCpp 0.5 31 | - zlib 32 | - [Bottleneck](https://pypi.python.org/pypi/Bottleneck): `pip install bottleneck` 33 | - PIL or Pillow (Python Image Library): `pip install Pillow` 34 | - NumPy 35 | - LAPACK 36 | -------------------------------------------------------------------------------- /2 layer extraction/KM_PD_layer_extraction/rose_layer_extraction.json: -------------------------------------------------------------------------------- 1 | { 2 | "operations": 3 | [ 4 | { 5 | "type": "KM_stroke_extraction", 6 | "params": { 7 | "source_name": "rose_albedo_%04d.png", 8 | "firstframe_path": "rose_albedo_0000.png", 9 | "threshold": 0.0, 10 | "frame_num": 5334, 11 | "final_replay_output_path": "rose_KM_replay_final.png", 12 | "KM_R_video_name": "rose_KM_R.avi", 13 | "KM_T_video_name": "rose_KM_T.avi", 14 | "KM_stroke_video_name": "rose_KM_stroke.avi", 15 | "KM_stroke_image_base_path": "rose_KM_stroke_", 16 | "width": 480, 17 | "height": 320, 18 | "FPS": 25.0, 19 | "R_gzfile_base_name": "rose_KM_R_", 20 | "T_gzfile_base_name": "rose_KM_T_" 21 | } 22 | }, 23 | 24 | { 25 | "type": "PD_stroke_extraction", 26 | "params": { 27 | "source_name": "rose_albedo_%04d.png", 28 | "firstframe_path": "rose_albedo_0000.png", 29 | "threshold": 0.0, 30 | "frame_num": 5334, 31 | "PD_paint_base_name": "rose_PD_paint_", 32 | "PD_alpha_base_name": "rose_PD_alpha_", 33 | "PD_stroke_base_name": "rose_PD_stroke_", 34 | "final_replay_output_path": "rose_PD_replay_final.png", 35 | "PD_stroke_video_name": "rose_PD_stroke.avi", 36 | "width": 480, 37 | "height": 320, 38 | "FPS": 25.0 39 | } 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /1 preprocessing/s5_get_albedo_sequence/extract_albedo_image_sequence.py: -------------------------------------------------------------------------------- 1 | sequence_name="rose-unrotate-%04d.png" 2 | output_path="rose_albedo_" 3 | num_frames=5334 4 | Reflectance=0.66 #### choose the value that smaller than 1./all_time_maxval. 5 | 6 | 7 | import numpy as np 8 | import cv2 9 | 10 | capture=cv2.VideoCapture(sequence_name) 11 | 12 | ret,frame=capture.read() 13 | _first=frame 14 | 15 | first_modified=np.zeros(frame.shape,dtype=np.uint8) 16 | 17 | first=cv2.medianBlur(_first,55) 18 | 19 | first_modified[:,:,0]=first.max(axis=2) 20 | first_modified[:,:,1]=first.max(axis=2) 21 | first_modified[:,:,2]=first.max(axis=2) 22 | 23 | all_time_maxval = 0. 24 | 25 | for i in range(1,num_frames): 26 | ## Load each frame 27 | new_frame =frame*1.0/first_modified 28 | ## Compute the maximum value in any channel after applying per-pixel-scale. 29 | maxval = new_frame.max() 30 | ## Find the all-time-maximum value. 31 | if maxval > all_time_maxval: 32 | all_time_maxval = maxval 33 | ## Print it if it changes: 34 | print all_time_maxval 35 | ret,frame = capture.read() 36 | 37 | print 'Final all time max:', all_time_maxval 38 | print 1./all_time_maxval ###### for rose, this value is 0.665217391304 39 | 40 | 41 | capture3=cv2.VideoCapture(sequence_name) 42 | for i in range(0,num_frames): 43 | 44 | ret3,frame3=capture3.read() 45 | 46 | modified=frame3*255.0*Reflectance/first_modified 47 | 48 | test1=modified[:,:,0] 49 | test2=modified[:,:,1] 50 | test3=modified[:,:,2] 51 | 52 | if test1[test1>255.0].shape[0]>0: 53 | print test1[test1>255.0] 54 | 55 | if test2[test2>255.0].shape[0]>0: 56 | print test2[test2>255.0] 57 | 58 | if test3[test3>255.0].shape[0]>0: 59 | print test3[test3>255.0] 60 | 61 | division3=(modified).clip(0,255) 62 | 63 | cv2.imwrite(output_path+'{0:04}'.format(i)+".png",division3) 64 | 65 | print 'finished' 66 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/l0smooth1Dsparse/mask_h.pyx: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport numpy as np 3 | ctypedef np.float64_t DTYPE_t 4 | 5 | cimport cython 6 | 7 | @cython.boundscheck(False) 8 | @cython.wraparound(False) 9 | def mask_h_threechannel( np.ndarray[DTYPE_t, ndim=2] h not None, DTYPE_t threshold, np.ndarray[DTYPE_t, ndim=1] maskscratch not None ): 10 | assert h.shape[1] % 3 == 0 11 | 12 | cdef int row, col 13 | cdef int row_max = h.shape[0] 14 | cdef int col_max = h.shape[1] 15 | cdef DTYPE_t r 16 | 17 | ## Assume h is a contiguous column-major (Fortran) array 18 | cdef DTYPE_t* rawh = &h[0,0] 19 | cdef DTYPE_t *rawh0, *rawh1, *rawh2 20 | ## Assume maskscratch is a contiguous 1D array 21 | cdef DTYPE_t* rawmask0 = &maskscratch[0] 22 | cdef DTYPE_t* rawmask 23 | 24 | for col in range( 0, col_max, 3 ): 25 | 26 | rawmask = rawmask0 27 | for row in range( row_max ): 28 | # r = h[ row, col ] 29 | # rawmask[row] = r*r 30 | r = rawh[0] 31 | rawmask[0] = r*r 32 | rawh += 1 33 | rawmask += 1 34 | 35 | rawmask = rawmask0 36 | for row in range( row_max ): 37 | # r = h[ row, (col+1) ] 38 | # rawmask[row] += r*r 39 | r = rawh[0] 40 | rawmask[0] += r*r 41 | rawh += 1 42 | rawmask += 1 43 | 44 | rawmask = rawmask0 45 | for row in range( row_max ): 46 | # r = h[ row, (col+2) ] 47 | # rawmask[row] += r*r 48 | r = rawh[0] 49 | rawmask[0] += r*r 50 | rawh += 1 51 | rawmask += 1 52 | 53 | rawmask = rawmask0 54 | rawh2 = rawh - row_max 55 | rawh1 = rawh2 - row_max 56 | rawh0 = rawh1 - row_max 57 | for row in range( row_max ): 58 | if rawmask[0] < threshold: 59 | rawh0[0] = 0 60 | rawh1[0] = 0 61 | rawh2[0] = 0 62 | rawh0 += 1 63 | rawh1 += 1 64 | rawh2 += 1 65 | rawmask += 1 66 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/l0smooth1Dsparse/ptsv.py: -------------------------------------------------------------------------------- 1 | import ctypes.util 2 | import numpy 3 | import os, sys 4 | 5 | def platform_shared_library_suffix(): 6 | import sys 7 | result = 'so' 8 | if 'win' in sys.platform.lower(): result = 'dll' 9 | ## No else if, because we want darwin to override win (which is a substring of darwin) 10 | if 'darwin' in sys.platform.lower(): result = 'dylib' 11 | return result 12 | 13 | ''' 14 | lib = None 15 | libdirs = [ os.path.join( 'usr', 'lib' ) ] 16 | for libdir in libdirs: 17 | libpath = os.path.join( libdir, 'liblapack.' + platform_shared_library_suffix() ) 18 | if not os.path.exists( libpath ): continue 19 | 20 | lib = ctypes.cdll.LoadLibrary( libpath ) 21 | if lib is None: 22 | raise ImportError( 'Unable to find lapack' ) 23 | ''' 24 | 25 | lib = ctypes.util.find_library( 'lapack' ) 26 | if lib is None: 27 | ## Search for it right here: 28 | lib = os.path.join( os.path.dirname( os.path.abspath( __file__ ) ), 'liblapack.' + platform_shared_library_suffix() ) 29 | if not os.path.isfile( lib ): lib = None 30 | if lib is None: 31 | raise ImportError( 'Unable to find lapack' ) 32 | lib = ctypes.cdll.LoadLibrary( lib ) 33 | 34 | _dptsv = lib.dptsv_ 35 | _dptsv.restype = None 36 | _dptsv.argtypes = \ 37 | [ 38 | # N 39 | ctypes.POINTER( ctypes.c_int ), 40 | # NRHS 41 | ctypes.POINTER( ctypes.c_int ), 42 | # D 43 | ctypes.POINTER( ctypes.c_double ), 44 | # E 45 | ctypes.POINTER( ctypes.c_double ), 46 | # B 47 | ctypes.POINTER( ctypes.c_double ), 48 | # LDB 49 | ctypes.POINTER( ctypes.c_int ), 50 | # INFO 51 | ctypes.POINTER( ctypes.c_int ) 52 | ] 53 | def ptsv( D, E, B ): 54 | assert len(D) == B.shape[0] 55 | assert len(E)+1 >= len(D) 56 | 57 | D = numpy.require( D, dtype = ctypes.c_double, requirements = [ 'F_CONTIGUOUS', 'WRITEABLE' ] ) 58 | E = numpy.require( E, dtype = ctypes.c_double, requirements = [ 'F_CONTIGUOUS', 'WRITEABLE' ] ) 59 | B = numpy.require( B, dtype = ctypes.c_double, requirements = [ 'F_CONTIGUOUS', 'WRITEABLE' ] ) 60 | 61 | N = ctypes.c_int( len(D) ) 62 | NRHS = ctypes.c_int( B.shape[1] ) 63 | LDB = ctypes.c_int( max( 1, B.shape[0] ) ) 64 | INFO = ctypes.c_int( 0 ) 65 | 66 | _dptsv( 67 | ctypes.byref( N ), 68 | ctypes.byref( NRHS ), 69 | 70 | D.ctypes.data_as( ctypes.POINTER( ctypes.c_double ) ), 71 | E.ctypes.data_as( ctypes.POINTER( ctypes.c_double ) ), 72 | B.ctypes.data_as( ctypes.POINTER( ctypes.c_double ) ), 73 | 74 | ctypes.byref( LDB ), 75 | ctypes.byref( INFO ) 76 | ) 77 | 78 | if INFO.value < 0: 79 | raise RuntimeError( 'dptsv: The %d-th argument had an illegal value' % (-INFO.value) ) 80 | elif INFO.value > 0: 81 | raise RuntimeError( 'dptsv: The leading minor of order %d is not positive definite; the solution has not been computed.' % INFO.value ) 82 | 83 | return B 84 | -------------------------------------------------------------------------------- /1 preprocessing/s3_subsequence_movestd_movemedian/moving_median_with_mask_function.cpp: -------------------------------------------------------------------------------- 1 | #include "stdafx.h" 2 | #include "new_rolling_median.h" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | using namespace cv; 25 | 26 | int main(int argc, const char* argv[]) // argv[1] maskname. argv[2] imgname. argv[3] buffersize. argv[4] outputname. 27 | { 28 | Mat mask=imread(argv[1],1);//F:\\mask_rose_270.png 29 | 30 | Mat img=imread(argv[2],1);//F:\\img_rose_270,png 31 | 32 | mm_handle **mm; 33 | value_t **Buffer; 34 | int kBufferSize; 35 | 36 | Mat output=img.clone(); 37 | 38 | kBufferSize=atoi(argv[3]); // 9 for rose and egg 39 | 40 | mm=new mm_handle *[img.rows*3]; // for each row, and three channels 41 | 42 | for(int i=0;i(i); 54 | for(int j=0;j(i); 74 | Vec3b *Ni=mask.ptr(i); 75 | Vec3b *Qi=output.ptr(i); 76 | for(int k=0;k<3;k++) 77 | { 78 | for(int index=0;index=img.cols) 90 | { 91 | temp[kBufferSize+count]=Mi[img.cols-1][k]; 92 | count++; 93 | } 94 | 95 | else if(Ni[j+q][k]!=255) 96 | { 97 | 98 | temp[kBufferSize+count]=Mi[j+q][k]; 99 | count++; 100 | } 101 | 102 | } 103 | 104 | for(int q=1;q<=kBufferSize;q++) 105 | { 106 | if((j-q)<0) 107 | { 108 | temp[kBufferSize-q]=Mi[0][k]; 109 | 110 | } 111 | else if(Ni[j-q][k]!=255) 112 | { 113 | temp[kBufferSize-q]=Qi[j-q][k]; 114 | 115 | } 116 | 117 | } 118 | 119 | for(int q=count;q> sys.stderr, 'Usage:', sys.argv[0], '[--glob] [--max-GB 2] spatial_pixels_per_timewise_image path/to/image1 [path/to/image2 ... path/to/imageN] path/to/output_basename' 11 | sys.exit(-1) 12 | 13 | argv = list( sys.argv[1:] ) 14 | 15 | glob_filenames = False 16 | try: 17 | i = argv.index( '--glob' ) 18 | glob_filenames = True 19 | del argv[ i ] 20 | except ValueError: pass 21 | 22 | ## The maximum memory buffer in GB 23 | max_GB = 2 24 | try: 25 | i = argv.index( '--max-GB' ) 26 | max_GB = float( argv[ i + 1 ] ) 27 | del argv[ i : i + 2 ] 28 | except IndexError: usage() 29 | except ValueError: pass 30 | 31 | if len( argv ) < 3: 32 | usage() 33 | 34 | spatial_pixels_per_output = int( argv.pop(0) ) 35 | filenames = argv[:-1] 36 | outname_basename = argv[-1] 37 | 38 | if glob_filenames: 39 | from glob import glob 40 | from itertools import chain 41 | filenames = list( chain( *[ glob( fname ) for fname in filenames ] ) ) 42 | 43 | ## Check this later, so that the image data will be printed first. 44 | #assert max_pixels_per_image >= len( filenames ) 45 | 46 | ## TODO: I can't assert this yet. 47 | # if os.path.exists( outname ): 48 | # print >> sys.stderr, 'ERROR: Output file exists, will not clobber:', outname 49 | # usage() 50 | 51 | image_shape = asarray( Image.open( filenames[0] ).convert( 'RGB' ), dtype = uint8 ).shape 52 | # spatial_pixels_per_output = min( max_pixels_per_image // len( filenames ), prod( image_shape[:2] ) ) 53 | print 'Image dimensions:', image_shape[0], 'by', image_shape[1], 'and', len( filenames ), 'frames.' 54 | print 'Each time-wise image will have', spatial_pixels_per_output, 'of the pixels (row-major)', 55 | num_images = (prod( image_shape[:2] )+spatial_pixels_per_output-1)//spatial_pixels_per_output 56 | print 'for a total of', num_images, 'images.' 57 | 58 | if spatial_pixels_per_output < 1: 59 | print >> sys.stderr, 'ERROR: spatial_pixels_per_timewise_image must be a positive integer.' 60 | usage() 61 | 62 | ## 2 GB 63 | max_ram_usage_bytes = int( max_GB*1024*1024*1024 ) 64 | max_spatial_pixels_in_memory = max_ram_usage_bytes // ( len(filenames) * image_shape[2] ) 65 | spatial_pixels_in_memory = spatial_pixels_per_output * ( max_spatial_pixels_in_memory // spatial_pixels_per_output ) 66 | if spatial_pixels_in_memory < spatial_pixels_per_output: 67 | print >> sys.stderr, 'ERROR: Maximum RAM usage would be exceeded to save even one image', 68 | print >> sys.stderr, '(which would take', ( spatial_pixels_per_output * len(filenames) * image_shape[2] ), 'bytes).' 69 | usage() 70 | 71 | stack = empty( ( spatial_pixels_in_memory, len(filenames), image_shape[2] ), dtype = uint8 ) 72 | count = 0 73 | for mem_off in xrange( 0, prod( image_shape[:2] ), spatial_pixels_in_memory ): 74 | mem_end = min( mem_off + spatial_pixels_in_memory, prod( image_shape[:2] ) ) 75 | 76 | for frame_count, fname in enumerate( filenames ): 77 | print 'Loading "%s"' % ( fname, ) 78 | img = Image.open( fname ).convert( 'RGB' ) 79 | arr = asarray( img, dtype = uint8 ) 80 | arr = arr.reshape( -1, arr.shape[2] )[ mem_off : mem_end ] 81 | stack[ :mem_end-mem_off, frame_count, : ] = arr 82 | 83 | for save_off in xrange( 0, mem_end-mem_off, spatial_pixels_per_output ): 84 | save_end = min( save_off + spatial_pixels_per_output, mem_end-mem_off ) 85 | 86 | outname = outname_basename + '-%dx%d-%04d.png' % ( image_shape[0], image_shape[1], count ) 87 | Image.fromarray( stack[ save_off:save_end ] ).save( outname ) 88 | print 'Saved', outname 89 | 90 | count += 1 91 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/timeunrotate.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python2.7 2 | 3 | from numpy import * 4 | ## Only Pillow (PIL) works for Jianchao on Windows 5 | from PIL import Image 6 | 7 | import os, sys 8 | 9 | def usage(): 10 | print >> sys.stderr, 'Usage:', sys.argv[0], '[--glob] [--max-GB 2] path/to/image1 [path/to/image2 ... path/to/imageN] path/to/output_basename' 11 | sys.exit(-1) 12 | 13 | argv = list( sys.argv[1:] ) 14 | 15 | glob_filenames = False 16 | try: 17 | i = argv.index( '--glob' ) 18 | glob_filenames = True 19 | del argv[ i ] 20 | except ValueError: pass 21 | 22 | ## The maximum memory buffer in GB 23 | max_GB = 2 24 | try: 25 | i = argv.index( '--max-GB' ) 26 | max_GB = float( argv[ i + 1 ] ) 27 | del argv[ i : i + 2 ] 28 | except IndexError: usage() 29 | except ValueError: pass 30 | 31 | if len( argv ) < 2: 32 | usage() 33 | 34 | filenames = argv[:-1] 35 | outname_basename = argv[-1] 36 | 37 | if glob_filenames: 38 | from glob import glob 39 | from itertools import chain 40 | filenames = list( chain( *[ glob( fname ) for fname in filenames ] ) ) 41 | 42 | ## Check this later, so that the image data will be printed first. 43 | #assert max_pixels_per_image >= len( filenames ) 44 | 45 | ## TODO: I can't assert this yet. 46 | # if os.path.exists( outname ): 47 | # print >> sys.stderr, 'ERROR: Output file exists, will not clobber:', outname 48 | # usage() 49 | 50 | output_shape = set([ tuple(list(map( int, filename.split('-')[-2].lower().split('x') ))) for filename in filenames ]) 51 | if len( output_shape ) != 1: 52 | print >> sys.stderr, "Multiple output image shapes (read from the second-to-last _NxM_ part of the filename) detected." 53 | usage() 54 | output_shape = output_shape.pop() 55 | 56 | image_shape = asarray( Image.open( filenames[0] ).convert( 'RGB' ), dtype = uint8 ).shape 57 | output_shape = ( output_shape[0], output_shape[1], image_shape[2] ) 58 | # spatial_pixels_per_output = min( max_pixels_per_image // len( filenames ), prod( image_shape[:2] ) ) 59 | print 'Time-wise image dimensions:', image_shape[0], 'by', image_shape[1], 'and', len( filenames ), 'frames.' 60 | print 'Each output image will be', output_shape[0], 'by', output_shape[1], 'by', output_shape[2] 61 | #max_output_images = image_shape[0] * len( filenames ) // ( output_shape[0]*output_shape[1] ) 62 | num_output_images = image_shape[1] 63 | 64 | ## 2 GB 65 | max_ram_usage_bytes = int( max_GB*1024*1024*1024 ) 66 | ## Testing: 67 | # max_ram_usage_bytes = 2*prod( output_shape ) + 10 68 | # num_output_images = 5 69 | max_images_in_memory = max_ram_usage_bytes // prod( output_shape ) 70 | images_in_memory = min( max_images_in_memory, num_output_images ) 71 | print 'Loading', images_in_memory, 'out of', num_output_images, 'into memory at once.' 72 | if max_images_in_memory < 1: 73 | print >> sys.stderr, 'ERROR: Maximum RAM usage would be exceeded to save even one image', 74 | print >> sys.stderr, '(which would take', prod( output_shape ), 'bytes).' 75 | usage() 76 | 77 | stack = empty( ( output_shape[0]*output_shape[1], images_in_memory, output_shape[2] ), dtype = uint8 ) 78 | count = 0 79 | for mem_off in xrange( 0, num_output_images, images_in_memory ): 80 | mem_end = min( mem_off + images_in_memory, num_output_images ) 81 | 82 | for frame_count, fname in enumerate( filenames ): 83 | print 'Loading "%s"' % ( fname, ) 84 | img = Image.open( fname ).convert( 'RGB' ) 85 | arr = asarray( img, dtype = uint8 ) 86 | #arr = arr.reshape( -1, arr.shape[2] )[ mem_off : mem_end ] 87 | stack[ frame_count*image_shape[0] : frame_count*image_shape[0] + arr.shape[0], :mem_end-mem_off, : ] = arr[:,mem_off:mem_end,:] 88 | 89 | for save_off in xrange( 0, mem_end-mem_off ): 90 | outname = outname_basename + '-%04d.png' % ( count, ) 91 | Image.fromarray( stack[ :, save_off, : ].reshape( output_shape ) ).save( outname ) 92 | print 'Saved', outname 93 | 94 | count += 1 95 | -------------------------------------------------------------------------------- /1 preprocessing/s3_subsequence_movestd_movemedian/moving_standard_deviation_mask_occlusion.py: -------------------------------------------------------------------------------- 1 | INPUT_PATH_PREFIX = "rose_" 2 | OUTPUT_PATH_PREFIX = "rose_" 3 | #### These indices are the keyframe indices of the input sequence. 4 | KEYFRAME_INDICES=np.array([0,255,291,298,361,591,692,779,835,1024,1130, 5 | 1142,1509,1753,1790,2037,2097,2101,2456,3116, 6 | 3492,3576,3754,3981,4029,4051,4199,4219,4267, 7 | 4574,4627,4685,4813,4894,4917,4968,4974]) 8 | 9 | import cv2 10 | import numpy as np 11 | import bottleneck 12 | import subprocess 13 | 14 | #function that modifies those functions in bottleneck module 15 | def bottleneck_centered( func, data, window, axis = -1 ): 16 | assert window % 2 == 1 17 | result = func( data, window = window, axis = axis ) 18 | shift = window//2 19 | result = np.roll( result, -shift, axis = axis ) 20 | ## I don't know how to write a general axis selection, so let's use swapaxis 21 | if -1 != axis: result = result.swapaxes( axis, -1 ) 22 | result[ ...,:shift ] = result[ ..., shift ][...,np.newaxis] 23 | result[ ..., -shift-1: ] = result[ ..., -shift-1 ][...,np.newaxis] 24 | if -1 != axis: result = result.swapaxes( axis, -1 ) 25 | return result 26 | 27 | def lerp_image(first,last,index,length,mask): #mask !=0 means common part. 28 | interpolation=np.zeros(first.shape,dtype=np.uint8) 29 | if index>length: 30 | index=length 31 | interpolation=np.uint8(first*(1.0-index*1.0/length)+last*(index*1.0/length)).clip(0,255) 32 | return interpolation 33 | 34 | print KEYFRAME_INDICES.shape 35 | 36 | 37 | capture=cv2.VideoCapture( INPUT_PATH_PREFIX + "subsequence_colorshift_%04d.png") 38 | capture1=cv2.VideoCapture( INPUT_PATH_PREFIX + "keyframe_mask_%04d.png") 39 | capture2=cv2.VideoCapture( INPUT_PATH_PREFIX + "subsequence_last_%04d.png") 40 | 41 | ret,firstframe=capture.read() 42 | 43 | first=firstframe 44 | firstframe2=firstframe 45 | 46 | first_keyframe=firstframe 47 | ret2,last_keyframe=capture2.read() 48 | 49 | kbuffersize=9 50 | count=0 51 | std_threshold=1.5 52 | window = 7 53 | window_mov_mean=3 54 | 55 | kernel = np.ones((5,5),np.uint8) 56 | kernel2 = np.ones((3,3),np.uint8) 57 | 58 | recover_image_output_path=OUTPUT_PATH_PREFIX + "subsequence_movestd_movingmedian_recover" 59 | 60 | for index in range(1,KEYFRAME_INDICES.shape[0]): 61 | 62 | retval1,mask=capture1.read() 63 | 64 | mask=cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel2) 65 | 66 | cv2.imwrite(OUTPUT_PATH_PREFIX + "keyframe_mask_closed_"+'{0:04}'.format(index)+".png",mask) 67 | 68 | nonzero_num=cv2.countNonZero(mask) 69 | print nonzero_num 70 | print mask.shape 71 | 72 | if nonzero_num=1: 82 | retval,frame=capture.read() 83 | frame_lab=cv2.cvtColor(frame,cv2.COLOR_BGR2LAB) 84 | img[:,i,:]=(frame_lab[mask==0]).reshape((zero_num,3)) 85 | temp=frame_lab 86 | 87 | for i in range(KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1]+1,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1]+kbuffersize+1): 88 | img[:,i,:]=(temp[mask==0]).reshape((zero_num,3)) 89 | 90 | firstframe=cv2.cvtColor(temp,cv2.COLOR_LAB2BGR) 91 | 92 | img_RGB= cv2.cvtColor(img,cv2.COLOR_LAB2BGR) 93 | 94 | rose_subsequence_img_name=OUTPUT_PATH_PREFIX + "subsequence_img_"+'{0:04}'.format(index)+".png" 95 | cv2.imwrite(rose_subsequence_img_name,img_RGB) 96 | 97 | stdevs=bottleneck_centered(bottleneck.move_std,img,window=window,axis=1) 98 | 99 | stdevs_per_pixel = stdevs.sum( axis = 2 )/(stdevs.shape[2]) 100 | 101 | highlighted_bad_pixels = img_RGB.copy() 102 | badmask=np.zeros((zero_num,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1]+kbuffersize+1),dtype=np.uint8) 103 | 104 | badmask[ stdevs_per_pixel > std_threshold] = 255 105 | 106 | rose_subsequence_mask_name=OUTPUT_PATH_PREFIX + "subsequence_mask_"+'{0:04}'.format(index)+".png" 107 | cv2.imwrite(rose_subsequence_mask_name,badmask) 108 | 109 | highlighted_bad_pixels[ stdevs_per_pixel > std_threshold ] = (255,0,0) 110 | rose_subsequence_highlighted_name=OUTPUT_PATH_PREFIX + "subsequence_highlighted_"+'{0:04}'.format(index)+".png" 111 | cv2.imwrite(rose_subsequence_highlighted_name,highlighted_bad_pixels) 112 | 113 | 114 | rose_subsequence_mm_outputname=OUTPUT_PATH_PREFIX + "subsequence_movingmedian_output_"+'{0:04}'.format(index)+".png" 115 | 116 | 117 | #recover by moving masked_median using C code 118 | subprocess.call(['moving_median_with_mask_function.exe', rose_subsequence_mask_name, rose_subsequence_img_name,str(kbuffersize),rose_subsequence_mm_outputname]) 119 | 120 | frame_mask=mask 121 | recover_base=cv2.imread(rose_subsequence_mm_outputname) 122 | 123 | 124 | #using bilateral filtering 125 | temp=np.zeros((recover_base.shape[0]*3,recover_base.shape[1],recover_base.shape[2]),dtype=np.uint8) 126 | for i in range(0,recover_base.shape[0]): 127 | temp[3*i+0,:,:]=recover_base[i,:,:] 128 | temp[3*i+1,:,:]=recover_base[i,:,:] 129 | temp[3*i+2,:,:]=recover_base[i,:,:] 130 | 131 | temp2=cv2.adaptiveBilateralFilter(temp,(3,15),200.0) 132 | 133 | for i in range(0,recover_base.shape[0]): 134 | recover_base[i,:,:]=temp2[3*i+1,:,:] 135 | 136 | _frame=np.zeros(firstframe.shape,dtype=np.uint8) 137 | 138 | for i in range (0,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1]+kbuffersize+1): 139 | 140 | _frame=lerp_image(first_keyframe,last_keyframe,i,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1],frame_mask) 141 | _frame[frame_mask==0]=(recover_base[:,i,:]).reshape((zero_num,3)) 142 | cv2.imwrite(recover_image_output_path+'{0:04}'.format(count)+".png",_frame) 143 | count=count+1 144 | 145 | firstframe2=firstframe 146 | 147 | print count 148 | 149 | else: 150 | 151 | _frame=np.zeros(firstframe.shape,dtype=np.uint8) 152 | frame_mask=mask 153 | for i in range(0,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1]+1): 154 | if i==0: 155 | frame=firstframe 156 | if i>=1: 157 | retval,frame=capture.read() 158 | firstframe=frame 159 | 160 | for i in range(0,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1]+1): 161 | 162 | _frame=lerp_image(first_keyframe,last_keyframe,i,KEYFRAME_INDICES[index]-KEYFRAME_INDICES[index-1],frame_mask) 163 | cv2.imwrite(recover_image_output_path+'{0:04}'.format(count)+".png",_frame) 164 | count=count+1 165 | 166 | firstframe2=firstframe 167 | print count 168 | 169 | first_keyframe=last_keyframe 170 | ret2,last_keyframe=capture2.read() 171 | 172 | 173 | -------------------------------------------------------------------------------- /1 preprocessing/s4_whole_sequence_L0smooth_on_time_axis/l0smooth1Dsparse/l0smooth1Dsparse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from numpy import * 4 | from ptsv import ptsv 5 | 6 | kUseCython = True 7 | if kUseCython: 8 | import pyximport 9 | pyximport.install( setup_args={ 'script_args': ['--compiler=mingw32'], 'include_dirs': get_include() } ) 10 | from mask_h import mask_h_threechannel 11 | 12 | class Struct( object ): pass 13 | 14 | def compute_h( S, num_channels, Lambda, beta, prepared ): 15 | assert num_channels >= 1 16 | assert len( S.shape ) == 2 17 | assert S.shape[1] % num_channels == 0 18 | 19 | h = prepared.h 20 | 21 | ## Simple: 22 | # h[:] = S[1:] - S[:-1] 23 | h[:] = S[1:] 24 | h[:] -= S[:-1] 25 | 26 | if kUseCython and num_channels == 3: 27 | mask_h_threechannel( h, Lambda/beta, prepared.maskscratch ) 28 | return h 29 | 30 | h2 = prepared.h2 31 | h2sum = prepared.h2sum 32 | mask = prepared.mask 33 | 34 | ## Simple: 35 | # h[ (h**2).sum(-1) < Lambda/beta ] = 0. 36 | ## Treat each channel separately. 37 | # h2[:] = h**2 38 | square( h, out = h2 ) 39 | 40 | h2sum[:] = 0 41 | for c in xrange( num_channels ): 42 | h2sum += h2[:,c::num_channels] 43 | del h2 44 | 45 | ## Set masked values of h to zero. 46 | # mask = mask < Lambda/beta 47 | # mask = repeat( mask, num_channels, axis = -1 ) 48 | # h[ mask ] = 0. 49 | 50 | ## This is a little faster: 51 | # masked = empty( h.shape, dtype = bool ) 52 | # masked[:,::num_channels] = mask < Lambda/beta 53 | # del mask 54 | # for c in xrange( 1, num_channels ): 55 | # masked[:,c::num_channels] = masked[...,::num_channels] 56 | # h[ masked ] = 0. 57 | 58 | ## This is faster still: 59 | mask[:] = h2sum < Lambda/beta 60 | for c in xrange( num_channels ): 61 | h[:,c::num_channels][ mask ] = 0. 62 | 63 | return h 64 | 65 | def prepare_solve_for_S( S, num_channels ): 66 | assert num_channels >= 1 67 | assert len( S.shape ) == 2 68 | assert S.shape[1] % num_channels == 0 69 | 70 | N = len(S) 71 | 72 | result = Struct() 73 | 74 | ## compute_h() 75 | h = empty( ( S.shape[0]-1, S.shape[1] ), order = 'F' ) 76 | result.h = h 77 | if num_channels == 3 and kUseCython: 78 | result.maskscratch = empty( h.shape[0], order = 'F' ) 79 | else: 80 | h2 = empty( h.shape, order = 'F' ) 81 | h2sum = empty( ( h.shape[0], h.shape[1]//num_channels ), order = 'F' ) 82 | mask = empty( h2sum.shape, dtype = bool, order = 'F' ) 83 | 84 | result.h2 = h2 85 | result.h2sum = h2sum 86 | result.mask = mask 87 | 88 | ## solve_for_S() 89 | result.system_diag = empty( (N,1), order = 'F' ) 90 | result.system_offdiag = empty( (N-1,1), order = 'F' ) 91 | 92 | return result 93 | 94 | def solve_for_S( S, I, h, beta, prepared, constrained = None, w_lsq = None ): 95 | ### WARNING: This function replaces the memory inside S with the return value. 96 | 97 | assert S.shape == I.shape 98 | 99 | N = len(S) 100 | 101 | # system = beta * gradTgrad + scipy.sparse.eye(N) 102 | system_diag = prepared.system_diag 103 | system_offdiag = prepared.system_offdiag 104 | system_offdiag[:] = -beta 105 | system_diag[:] = 2.*beta + 1. 106 | system_diag[ 0 ] = beta + 1. 107 | system_diag[ -1 ] = beta + 1. 108 | 109 | ## Simpler: 110 | # rhs = I + beta * ( grad.T * h ) 111 | ## Faster: 112 | # rhs = (beta*grad.T) * h 113 | # rhs += I 114 | ## Faster still: 115 | rhs = S 116 | rhs[0, :] = 0. 117 | rhs[1:, :] = h 118 | rhs[ :-1, :] -= h 119 | rhs *= beta 120 | rhs += I 121 | 122 | #import pdb 123 | #pdb.set_trace() 124 | ## scipy.sparse.spdiags( system, ( -1, 0, 1 ), N, N ) 125 | 126 | if constrained is not None and w_lsq != 0.: 127 | if w_lsq is None: w_lsq = 100. 128 | 129 | constrained = list( constrained ) 130 | 131 | if w_lsq == 'hard': 132 | ## Zero the constrained rows and columns (update the RHS for the column), 133 | ## and set the diagonal to 1. 134 | 135 | ## Zero the constrained rows/columns. 136 | for c in constrained: 137 | if c > 0: rhs[ c-1,: ] -= system_offdiag[ c-1 ] * I[ c, : ] 138 | if c+1 < N: rhs[ c+1,: ] -= system_offdiag[ c ] * I[ c, : ] 139 | 140 | rhs[ c,: ] = I[ c,: ] 141 | 142 | system_diag[ c ] = 1. 143 | if c < N-1: system_offdiag[ c ] = 0. 144 | if c > 0: system_offdiag[ c-1 ] = 0. 145 | 146 | else: 147 | ## There is no += :-( 148 | system_diag[ constrained ] += w_lsq 149 | rhs[ constrained, : ] += w_lsq*I[ constrained, : ] 150 | 151 | ## lapack's ptsv directly: 152 | result = ptsv( system_diag, system_offdiag, rhs ) 153 | assert may_share_memory( rhs, result ) 154 | ## rhs is Snext 155 | return rhs 156 | 157 | ## Default parameters following the matlab code 158 | kDefaultLambda = 2e-2 159 | kDefaultKappa = 1.5 160 | def l0smooth1D( I, axis = 0, channels = False, Lambda = None, kappa = None, harmonic = False, constrained = None, w_lsq = None ): 161 | ''' 162 | The following documentation on parameters is taken from the matlab implentation 163 | provided with the paper "Image Smoothing via L0 Gradient Minimization" by 164 | Li Xu, Cewu Lu, Yi Xu, Jiaya Jia (SIGGRAPH Asia 2011): 165 | 166 | L0Smooth - Image Smoothing via L0 Gradient Minimization 167 | S = L0Smooth(Im, Lambda, kappa) performs L0 graidient smoothing of input 168 | image Im, with smoothness weight Lambda and rate kappa. 169 | 170 | Paras: 171 | @Im : Input UINT8 image, both grayscale and color images are acceptable. 172 | @Lambda: Smoothing parameter controlling the degree of smooth. (See [1]) 173 | Typically it is within the range [1e-3, 1e-1], 2e-2 by default. 174 | @kappa : Parameter that controls the rate. (See [1]) 175 | Small kappa results in more iteratioins and with sharper edges. 176 | We select kappa in (1, 2]. 177 | kappa = 2 is suggested for natural images. 178 | 179 | Example 180 | ========== 181 | Im = imread('pflower.jpg'); 182 | S = L0Smooth(Im); # Default Parameters (Lambda = 2e-2, kappa = 2) 183 | figure, imshow(Im), figure, imshow(S); 184 | 185 | I added additional parameters: 186 | axis : an integer specifying which axis of I is the 1D axis along which to smooth. 187 | channels: a boolean specifying whether-or-not the last axis of I is taken to be 188 | the channels of a multichannel image. If channels is True, axis cannot 189 | refer to the last axis. 190 | harmonic: a boolean specifying whether the l0 sparsity should be on the 191 | color gradient or on the color laplacian. A sparse color gradient 192 | is an image with constant color regions. A sparse color laplacian 193 | is an image with linear color gradients. 194 | constrained: a list of indices along axis which will be constrained to remain 195 | the same. 196 | w_lsq : a floating point value (default: 100) specifying how strongly the 197 | constrain on the constrained indices should be enforced. 198 | ''' 199 | 200 | ## Default parameters following the matlab code 201 | if Lambda is None: Lambda = kDefaultLambda 202 | if kappa is None: kappa = kDefaultKappa 203 | beta0 = 2*Lambda 204 | betamax = 1e5 205 | 206 | I = asfarray(I).copy( order = 'F' ) 207 | 208 | assert not channels or ( axis != -1 and axis < len( I.shape )-1 ) 209 | num_channels = 1 210 | if channels: 211 | num_channels = I.shape[-1] 212 | 213 | if axis != 0: 214 | I = swapaxes( I, 0, axis ) 215 | I_shape = I.shape 216 | I = I.reshape( I.shape[0], -1 ) 217 | 218 | S = I.copy( order = 'F' ) 219 | beta = beta0 220 | i = 0 221 | prepared = prepare_solve_for_S( S, num_channels ) 222 | while beta < betamax: 223 | if i % 10 == 0: 224 | print beta, betamax 225 | 226 | h = compute_h( S, num_channels, Lambda, beta, prepared ) 227 | S = solve_for_S( S, I, h, beta, prepared, constrained = constrained, w_lsq = w_lsq ) 228 | 229 | i += 1 230 | beta *= kappa 231 | 232 | S = S.reshape( I_shape ) 233 | if axis != 0: 234 | S = swapaxes( S, 0, axis ) 235 | return S 236 | 237 | def smooth_and_save( inpath, axis = 0, outpath = None, Lambda = None, kappa = None, w_lsq = None, constrained = None ): 238 | 239 | if outpath is None: 240 | import os 241 | inpath_base = os.path.splitext( inpath )[0] 242 | outpath = inpath_base + "-l0-axis_%d-Lambda_%3g-kappa_%g.png" % ( axis, kDefaultLambda if Lambda is None else Lambda, kDefaultKappa if kappa is None else kappa ) 243 | 244 | import skimage.io 245 | print 'Loading:', inpath 246 | I = skimage.img_as_float( skimage.io.imread( inpath ) ) 247 | assert len( I.shape ) == 3 248 | 249 | if w_lsq is not None and constrained is None: 250 | constrained = [ 0, I.shape[ axis ]-1 ] 251 | 252 | print 'Lambda:', Lambda 253 | print 'kappa:', kappa 254 | print 'Constraint weight:', w_lsq 255 | print 'Constraints:', constrained 256 | 257 | S = l0smooth1D( I, axis = axis, channels = True, Lambda = Lambda, kappa = kappa, constrained = constrained, w_lsq = w_lsq ) 258 | S = S.clip(0,1) 259 | assert len( S.shape ) == 3 260 | 261 | skimage.io.imsave( outpath, S ) 262 | print 'Saved:', outpath 263 | 264 | def main(): 265 | import sys 266 | argv = list( sys.argv[1:] ) 267 | 268 | def usage(): 269 | print >> sys.stderr, "Usage:", sys.argv[0], "[--lambda 2e-2] [--kappa 1.5] [--constraint-weight 0.] [--constraints [frame index, frame index, ...] (default: [0,num_frames-1])] path/to/input axis path/to/output" 270 | ## We liked Lambda = 5e-3, Kappa = 1.1 271 | sys.exit(-1) 272 | 273 | if len( argv ) == 0: usage() 274 | 275 | Lambda = None 276 | try: 277 | i = argv.index( '--lambda' ) 278 | Lambda = float( argv[ i + 1 ] ) 279 | del argv[ i : i + 2 ] 280 | except IndexError: usage() 281 | except ValueError: pass 282 | 283 | kappa = None 284 | try: 285 | i = argv.index( '--kappa' ) 286 | kappa = float( argv[ i + 1 ] ) 287 | del argv[ i : i + 2 ] 288 | except IndexError: usage() 289 | except ValueError: pass 290 | 291 | w_lsq = None 292 | try: 293 | i = argv.index( '--constraint-weight' ) 294 | w_lsq = argv[ i + 1 ] 295 | if w_lsq != 'hard': w_lsq = float( w_lsq ) 296 | del argv[ i : i + 2 ] 297 | except IndexError: usage() 298 | except ValueError: pass 299 | 300 | constrained = None 301 | try: 302 | i = argv.index( '--constraints' ) 303 | import json 304 | constrained = list( json.loads( argv[ i + 1 ] ) ) 305 | del argv[ i : i + 2 ] 306 | except IndexError: usage() 307 | except ValueError: pass 308 | 309 | inpath = argv.pop(0) 310 | axis = int( argv.pop(0) ) 311 | outpath = argv.pop(0) 312 | 313 | import os 314 | if os.path.exists( outpath ): 315 | print >> sys.stderr, 'ERROR: Refusing to clobber output path:', outpath 316 | usage() 317 | if not os.path.isfile( inpath ): 318 | print >> sys.stderr, 'ERROR: Input path is not a file:', inpath 319 | usage() 320 | 321 | if len( argv ) != 0: usage() 322 | 323 | smooth_and_save( inpath, axis = axis, outpath = outpath, Lambda = Lambda, kappa = kappa, w_lsq = w_lsq, constrained = constrained ) 324 | 325 | if __name__ == '__main__': main() 326 | -------------------------------------------------------------------------------- /1 preprocessing/s3_subsequence_movestd_movemedian/new_rolling_median.h: -------------------------------------------------------------------------------- 1 | #ifndef __new_rolling_median_h__ 2 | #define __new_rolling_median_h__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | //#include "C:\Python27\Lib\site-packages\numpy\core\include\numpy\arrayobject.h" 10 | 11 | //#define HIDE_SWAP 1 12 | 13 | /* 14 | Copyright (c) 2011 J. David Lee. All rights reserved. 15 | 16 | Redistribution and use in source and binary forms, with or without 17 | modification, are permitted provided that the following conditions are 18 | met: 19 | 20 | 1. Redistributions of source code must retain the above copyright 21 | notice, this list of conditions and the following disclaimer. 22 | 23 | 2. Redistributions in binary form must reproduce the above 24 | copyright notice, this list of conditions and the following 25 | disclaimer in the documentation and/or other materials provided 26 | with the distribution. 27 | 28 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 | HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 33 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 34 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 35 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 36 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 37 | TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 38 | USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 39 | DAMAGE. 40 | */ 41 | 42 | typedef int _size_t; 43 | typedef uint8_t value_t; 44 | 45 | 46 | /* 47 | * The number of children has a maximum of 8 due to the manual loop- 48 | * unrolling used in the code below. 49 | */ 50 | #define NUM_CHILDREN 8 51 | 52 | // Minimum of two numbers. 53 | #define min(a, b) (((a) < (b)) ? (a) : (b)) 54 | 55 | // Find indices of parent, first, and last child. 56 | #define P_IDX(i) ((i) - 1) / NUM_CHILDREN 57 | #define FC_IDX(i) NUM_CHILDREN * (i) + 1 58 | 59 | 60 | struct _mm_node { 61 | int small; // 1 if the node is in the small heap. 62 | _size_t idx; // The node's index in the heap array. 63 | value_t val; // The node's value. 64 | struct _mm_node *next; // The next node in order of insertion. 65 | }; 66 | 67 | typedef struct _mm_node mm_node; 68 | 69 | 70 | struct _mm_handle { 71 | int odd; // 1 if the window size is odd, 0 otherwise. 72 | _size_t n_s; // The number of elements in the min heap. 73 | _size_t n_l; // The number of elements in the max heap. 74 | mm_node **s_heap; // The min heap. 75 | mm_node **l_heap; // The max heap. 76 | mm_node **nodes; // All the nodes. s_heap and l_heap point into 77 | // this array. 78 | mm_node *node_data; // Pointer to memory location where nodes live. 79 | mm_node *first; // The node added first to the list of nodes. 80 | mm_node *last; // The last (most recent) node added. 81 | 82 | // Most nodes are leaf nodes, therefore it makes sense to have a 83 | // quick way to check if a node is a leaf to avoid processing. 84 | _size_t s_first_leaf; // First leaf index in the small heap. 85 | _size_t l_first_leaf; // First leaf index in the large heap. 86 | }; 87 | 88 | typedef struct _mm_handle mm_handle; 89 | 90 | 91 | /* 92 | * Return the index of the smallest child of the node. The pointer 93 | * child will also be set. 94 | */ 95 | __inline _size_t get_smallest_child(mm_node **heap, 96 | _size_t size, 97 | _size_t idx, 98 | mm_node *node, 99 | mm_node **child) { 100 | _size_t i0 = FC_IDX(idx); 101 | _size_t i1 = i0 + NUM_CHILDREN; 102 | i1 = min(i1, size); 103 | 104 | switch(i1 - i0) { 105 | case 8: if(heap[i0 + 7]->val < heap[idx]->val) { idx = i0 + 7; } 106 | case 7: if(heap[i0 + 6]->val < heap[idx]->val) { idx = i0 + 6; } 107 | case 6: if(heap[i0 + 5]->val < heap[idx]->val) { idx = i0 + 5; } 108 | case 5: if(heap[i0 + 4]->val < heap[idx]->val) { idx = i0 + 4; } 109 | case 4: if(heap[i0 + 3]->val < heap[idx]->val) { idx = i0 + 3; } 110 | case 3: if(heap[i0 + 2]->val < heap[idx]->val) { idx = i0 + 2; } 111 | case 2: if(heap[i0 + 1]->val < heap[idx]->val) { idx = i0 + 1; } 112 | case 1: if(heap[i0 ]->val < heap[idx]->val) { idx = i0; } 113 | } 114 | 115 | *child = heap[idx]; 116 | return idx; 117 | } 118 | 119 | 120 | /* 121 | * Return the index of the largest child of the node. The pointer 122 | * child will also be set. 123 | */ 124 | __inline _size_t get_largest_child(mm_node **heap, 125 | _size_t size, 126 | _size_t idx, 127 | mm_node *node, 128 | mm_node **child) { 129 | _size_t i0 = FC_IDX(idx); 130 | _size_t i1 = i0 + NUM_CHILDREN; 131 | i1 = min(i1, size); 132 | 133 | switch(i1 - i0) { 134 | case 8: if(heap[i0 + 7]->val > heap[idx]->val) { idx = i0 + 7; } 135 | case 7: if(heap[i0 + 6]->val > heap[idx]->val) { idx = i0 + 6; } 136 | case 6: if(heap[i0 + 5]->val > heap[idx]->val) { idx = i0 + 5; } 137 | case 5: if(heap[i0 + 4]->val > heap[idx]->val) { idx = i0 + 4; } 138 | case 4: if(heap[i0 + 3]->val > heap[idx]->val) { idx = i0 + 3; } 139 | case 3: if(heap[i0 + 2]->val > heap[idx]->val) { idx = i0 + 2; } 140 | case 2: if(heap[i0 + 1]->val > heap[idx]->val) { idx = i0 + 1; } 141 | case 1: if(heap[i0 ]->val > heap[idx]->val) { idx = i0; } 142 | } 143 | 144 | *child = heap[idx]; 145 | return idx; 146 | } 147 | 148 | 149 | /* 150 | * Swap nodes. 151 | */ 152 | #define SWAP_NODES(heap, idx1, node1, idx2, node2) \ 153 | heap[idx1] = node2; \ 154 | heap[idx2] = node1; \ 155 | node1->idx = idx2; \ 156 | node2->idx = idx1; \ 157 | idx1 = idx2 158 | 159 | 160 | /* 161 | * Move the given node up through the heap to the appropriate position. 162 | */ 163 | __inline void move_up_small(mm_node **heap, 164 | _size_t size, 165 | _size_t idx, 166 | mm_node *node, 167 | _size_t p_idx, 168 | mm_node *parent) { 169 | do { 170 | SWAP_NODES(heap, idx, node, p_idx, parent); 171 | if(idx == 0) { 172 | break; 173 | } 174 | p_idx = P_IDX(idx); 175 | parent = heap[p_idx]; 176 | } while (node->val > parent->val); 177 | } 178 | 179 | 180 | /* 181 | * Move the given node down through the heap to the appropriate position. 182 | */ 183 | __inline void move_down_small(mm_node **heap, 184 | _size_t size, 185 | _size_t idx, 186 | mm_node *node) { 187 | mm_node *child; 188 | value_t val = node->val; 189 | _size_t c_idx = get_largest_child(heap, size, idx, node, &child); 190 | 191 | while(val < child->val) { 192 | SWAP_NODES(heap, idx, node, c_idx, child); 193 | c_idx = get_largest_child(heap, size, idx, node, &child); 194 | } 195 | } 196 | 197 | 198 | /* 199 | * Move the given node down through the heap to the appropriate 200 | * position. 201 | */ 202 | __inline void move_down_large(mm_node **heap, 203 | _size_t size, 204 | _size_t idx, 205 | mm_node *node, 206 | _size_t p_idx, 207 | mm_node *parent) { 208 | do { 209 | SWAP_NODES(heap, idx, node, p_idx, parent); 210 | if(idx == 0) { 211 | break; 212 | } 213 | p_idx = P_IDX(idx); 214 | parent = heap[p_idx]; 215 | } while (node->val < parent->val); 216 | } 217 | 218 | 219 | 220 | /* 221 | * Move the given node up through the heap to the appropriate position. 222 | */ 223 | __inline void move_up_large(mm_node **heap, 224 | _size_t size, 225 | _size_t idx, 226 | mm_node *node) { 227 | mm_node *child; 228 | value_t val = node->val; 229 | _size_t c_idx = get_smallest_child(heap, size, idx, node, &child); 230 | 231 | while(val > child->val) { 232 | SWAP_NODES(heap, idx, node, c_idx, child); 233 | c_idx = get_smallest_child(heap, size, idx, node, &child); 234 | } 235 | } 236 | 237 | 238 | /* 239 | * Swap the heap heads. 240 | */ 241 | __inline void swap_heap_heads(mm_node **s_heap, 242 | _size_t n_s, 243 | mm_node **l_heap, 244 | _size_t n_l, 245 | mm_node *s_node, 246 | mm_node *l_node) { 247 | s_node->small = 0; 248 | l_node->small = 1; 249 | s_heap[0] = l_node; 250 | l_heap[0] = s_node; 251 | move_down_small(s_heap, n_s, 0, l_node); 252 | move_up_large(l_heap, n_l, 0, s_node); 253 | } 254 | 255 | 256 | /* 257 | * ---------------- 258 | * Public functions 259 | * ---------------- 260 | */ 261 | 262 | 263 | /* 264 | * Construct the double heap with the given total number of values. 265 | * 266 | * Arguments: 267 | * size -- The total number of values in the double heap. 268 | * 269 | * Return: The mm_handle structure, uninitialized. 270 | */ 271 | mm_handle *mm_new(const _size_t size) { 272 | mm_handle *mm = ( mm_handle *)malloc(sizeof(mm_handle)); 273 | mm->odd = size % 2; 274 | mm->n_l = 0; 275 | mm->n_s = 0; 276 | 277 | mm->nodes = (mm_node **) malloc(size * sizeof(mm_node*)); 278 | mm->node_data =( mm_node *)malloc(size * sizeof(mm_node)); 279 | 280 | mm->s_heap = mm->nodes; 281 | mm->l_heap = &mm->nodes[size/2 + size % 2]; 282 | 283 | return mm; 284 | } 285 | 286 | 287 | /* 288 | * Update the running median with a new value. 289 | */ 290 | void mm_update(mm_handle *mm, value_t val) { 291 | 292 | // Local variables. 293 | mm_node **l_heap = mm->l_heap; 294 | mm_node **s_heap = mm->s_heap; 295 | _size_t n_s = mm->n_s; 296 | _size_t n_l = mm->n_l; 297 | 298 | // Nodes and indices. 299 | mm_node *node = mm->first; 300 | _size_t idx = node->idx; 301 | 302 | mm_node *node2; 303 | _size_t idx2; 304 | 305 | // Replace value of first inserted node, and update first, last. 306 | node->val = val; 307 | mm->first = mm->first->next; 308 | mm->last->next = node; 309 | mm->last = node; 310 | 311 | // In small heap. 312 | if(node->small) { 313 | 314 | // Internal or leaf node. 315 | if(idx > 0) { 316 | idx2 = P_IDX(idx); 317 | node2 = s_heap[idx2]; 318 | 319 | // Move up. 320 | if(val > node2->val) { 321 | move_up_small(s_heap, n_s, idx, node, idx2, node2); 322 | 323 | // Maybe swap between heaps. 324 | node2 = l_heap[0]; 325 | if(val > node2->val) { 326 | swap_heap_heads(s_heap, n_s, l_heap, n_l, node, node2); 327 | } 328 | } 329 | 330 | // Move down. 331 | else if(idx < mm->s_first_leaf) { 332 | move_down_small(s_heap, n_s, idx, node); 333 | } 334 | } 335 | 336 | // Head node. 337 | else { 338 | node2 = l_heap[0]; 339 | if(val > node2->val) { 340 | swap_heap_heads(s_heap, n_s, l_heap, n_l, node, node2); 341 | } else { 342 | move_down_small(s_heap, n_s, idx, node); 343 | } 344 | } 345 | } 346 | 347 | // In large heap. 348 | else { 349 | 350 | // Internal or leaf node. 351 | if(idx > 0) { 352 | idx2 = P_IDX(idx); 353 | node2 = l_heap[idx2]; 354 | 355 | // Move down. 356 | if(val < node2->val) { 357 | move_down_large(l_heap, n_l, idx, node, idx2, node2); 358 | 359 | // Maybe swap between heaps. 360 | node2 = s_heap[0]; 361 | if(val < node2->val) { 362 | swap_heap_heads(s_heap, n_s, l_heap, n_l, node2, node); 363 | } 364 | } 365 | 366 | // Move up. 367 | else if(idx < mm->l_first_leaf) { 368 | move_up_large(l_heap, n_l, idx, node); 369 | } 370 | } 371 | 372 | // Head node. 373 | else { 374 | node2 = s_heap[0]; 375 | if(val < node2->val) { 376 | swap_heap_heads(s_heap, n_s, l_heap, n_l, node2, node); 377 | } else { 378 | move_up_large(l_heap, n_l, idx, node); 379 | } 380 | } 381 | } 382 | } 383 | 384 | 385 | /* 386 | * Insert initial values into the double heap structure. 387 | * 388 | * Arguments: 389 | * mm -- The double heap structure. 390 | * idx -- The index of the value running from 0 to size - 1. 391 | * val -- The value to insert. 392 | */ 393 | void mm_insert_init(mm_handle *mm, value_t val) { 394 | // Some local variables. 395 | mm_node *node; 396 | _size_t n_s = mm->n_s; 397 | _size_t n_l = mm->n_l; 398 | 399 | 400 | // The first node. 401 | if(n_s == 0) { 402 | node = mm->s_heap[0] = &mm->node_data[0]; 403 | node->small = 1; 404 | node->idx = 0; 405 | node->val = val; 406 | node->next = mm->l_heap[0]; 407 | 408 | mm->n_s = 1; 409 | mm->first = mm->last = node; 410 | mm->s_first_leaf = 0; 411 | } 412 | 413 | // Nodes after the first. 414 | else { 415 | 416 | // Add to the large heap. 417 | if(n_s > n_l) { 418 | node = mm->l_heap[n_l] = &mm->node_data[n_s + n_l]; 419 | node->small = 0; 420 | node->idx = n_l; 421 | node->next = mm->first; 422 | 423 | mm->first = node; 424 | ++mm->n_l; 425 | mm->l_first_leaf = ceil((mm->n_l - 1) / (double)NUM_CHILDREN); 426 | mm_update(mm, val); 427 | } 428 | 429 | // Add to the small heap. 430 | else { 431 | node = mm->s_heap[n_s] = &mm->node_data[n_s + n_l]; 432 | node->small = 1; 433 | node->idx = n_s; 434 | node->next = mm->first; 435 | 436 | mm->first = node; 437 | ++mm->n_s; 438 | mm->s_first_leaf = ceil((mm->n_s - 1) / (double)NUM_CHILDREN); 439 | mm_update(mm, val); 440 | } 441 | } 442 | } 443 | 444 | 445 | /* 446 | * Return the current median value. 447 | */ 448 | __inline value_t mm_get_median(mm_handle *mm) { 449 | if(mm->odd) { 450 | return mm->s_heap[0]->val; 451 | } else { 452 | return (mm->s_heap[0]->val + mm->l_heap[0]->val) / 2.0; 453 | } 454 | } 455 | 456 | 457 | /* 458 | * Print the two heaps to the screen. 459 | */ 460 | void mm_dump(mm_handle *mm) { 461 | _size_t i; 462 | printf("\n\nFirst: %f\n", (double)mm->first->val); 463 | printf("Last: %f\n", (double)mm->last->val); 464 | 465 | printf("\n\nSmall heap:\n"); 466 | for(i = 0; i < mm->n_s; ++i) { 467 | printf("%i %f\n", (int)mm->s_heap[i]->idx, mm->s_heap[i]->val); 468 | } 469 | 470 | printf("\n\nLarge heap:\n"); 471 | for(i = 0; i < mm->n_l; ++i) { 472 | printf("%i %f\n", (int)mm->l_heap[i]->idx, mm->l_heap[i]->val); 473 | } 474 | } 475 | 476 | 477 | /* 478 | * Free memory allocated in mm_new. 479 | */ 480 | __inline void mm_free(mm_handle *mm) { 481 | free(mm->node_data); 482 | free(mm->nodes); 483 | free(mm); 484 | } 485 | 486 | 487 | 488 | 489 | #endif /* __rolling_median_h__ */ 490 | -------------------------------------------------------------------------------- /2 layer extraction/PD_3by3_method_layer_extraction/PD_spatial_coherency_3by3_layer_extraction.cpp: -------------------------------------------------------------------------------- 1 | #include "stdafx.h" 2 | #include 3 | #include 4 | #include 5 | 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | 25 | using namespace std; 26 | using namespace cv; 27 | using namespace Eigen; 28 | 29 | typedef double timemap_t; 30 | typedef Eigen::SparseMatrix SparseMatrixT; 31 | 32 | 33 | unsigned char d2b( double value ) 34 | { 35 | if( value < 0. ) value = 0.; 36 | long int result = ( value * 255. + .5 ); 37 | if( result < 0 ) result = 0; 38 | if( result > 255 ) result = 255; 39 | return (unsigned char) result; 40 | } 41 | 42 | cv::Mat toLab( const cv::Mat& bgr ) 43 | { 44 | using namespace cv; 45 | 46 | Mat bgr32, lab; 47 | bgr.convertTo(bgr32,CV_32FC3,1.0/255.0); 48 | return bgr32; 49 | //cvtColor(bgr32,lab,CV_BGR2Lab); 50 | //return lab; 51 | } 52 | 53 | cv::Mat diff( cv::Mat mat1, cv::Mat mat2 ) 54 | { 55 | using namespace cv; 56 | 57 | Mat diff; 58 | Mat mat1_lab=toLab(mat1); 59 | Mat mat2_lab=toLab(mat2); 60 | 61 | subtract( mat1_lab, mat2_lab, diff ); 62 | 63 | std::vector lab; 64 | split(diff,lab); 65 | 66 | Mat L2=lab[0].mul(lab[0]); 67 | Mat a2=lab[1].mul(lab[1]); 68 | Mat b2=lab[2].mul(lab[2]); 69 | 70 | Mat DIFF = Mat::zeros(diff.rows, diff.cols, CV_32FC1); 71 | 72 | 73 | for(int i=0; i(i)[j]=sqrt( L2.ptr(i)[j]+ a2.ptr(i)[j]+ b2.ptr(i)[j])/1.8;///376.0;///1.8;///376.0; 76 | 77 | 78 | Mat imgout; 79 | DIFF.convertTo(imgout,CV_8UC1,255.0); 80 | //imshow("imgout",imgout); 81 | //waitKey(0); 82 | return imgout; 83 | 84 | } 85 | 86 | void Compute_three_by_three_Paint_Matrix_on_just_one_component(cv::Mat img1, cv::Mat img2, std::vector< std::pair< int, int > > &differing_pixels, SparseMatrixT& M, Eigen::VectorXd& N) 87 | { 88 | using namespace cv; 89 | 90 | 91 | Eigen::SparseMatrix A(3,3); 92 | Eigen::VectorXd B(3); 93 | 94 | M.resize(3,3); 95 | N.resize(3); 96 | 97 | double A00=0.0; 98 | double A01=0.0; 99 | double A02=0.0; 100 | double A10=0.0; 101 | double A11=0.0; 102 | double A12=0.0; 103 | double A20=0.0; 104 | double A21=0.0; 105 | double A22=0.0; 106 | double B00=0.0; 107 | double B10=0.0; 108 | double B20=0.0; 109 | 110 | 111 | int n=differing_pixels.size(); 112 | for(int index=0; index(i); 119 | Vec3d *Ai=img2.ptr(i); 120 | 121 | 122 | double ui[3]={0.0}; 123 | double vm=std::sqrt( (Ai[j][0]-Bi[j][0])*(Ai[j][0]-Bi[j][0])+ (Ai[j][1]-Bi[j][1])*(Ai[j][1]-Bi[j][1]) + (Ai[j][2]-Bi[j][2])*(Ai[j][2]-Bi[j][2]) ); 124 | for(int k=0;k<3;k++) 125 | ui[k]=(Ai[j][k]-Bi[j][k])/vm; 126 | 127 | //double weight=1.0; 128 | double weight=vm*vm; 129 | 130 | double ci[3]={0.0}; 131 | double b_u=Bi[j][0]*ui[0]+ Bi[j][1]*ui[1] + Bi[j][2]*ui[2]; 132 | for(int k=0;k<3;k++) 133 | ci[k]=b_u*ui[k]-Bi[j][k]; 134 | 135 | double common=ui[0]*ui[0]+ui[1]*ui[1]+ui[2]*ui[2]-2.0; 136 | 137 | double tempA00=(1-ui[0]*ui[0])*(1-ui[0]*ui[0]) + ui[0]*ui[0]*(ui[1]*ui[1]+ui[2]*ui[2]); 138 | A00+=(weight*tempA00); 139 | double tempA01= ui[0]*ui[1]*common; 140 | A01+=(weight*tempA01); 141 | double tempA02= ui[0]*ui[2]*common; 142 | A02+=(weight*tempA02); 143 | double tempA10= ui[0]*ui[1]*common; 144 | A10+=(weight*tempA10); 145 | double tempA11=(1-ui[1]*ui[1])*(1-ui[1]*ui[1]) + ui[1]*ui[1]*(ui[0]*ui[0]+ui[2]*ui[2]); 146 | A11+=(weight*tempA11); 147 | double tempA12= ui[1]*ui[2]*common; 148 | A12+=(weight*tempA12); 149 | double tempA20= ui[0]*ui[2]*common; 150 | A20+=(weight*tempA20); 151 | double tempA21= ui[1]*ui[2]*common; 152 | A21+=(weight*tempA21); 153 | double tempA22= (1-ui[2]*ui[2])*(1-ui[2]*ui[2]) + ui[2]*ui[2]*(ui[1]*ui[1]+ui[0]*ui[0]); 154 | A22+=(weight*tempA22); 155 | double tempB00= ci[0]*(ui[0]*ui[0]-1) + ci[1]*ui[0]*ui[1] + ci[2]*ui[0]*ui[2]; 156 | B00+=(weight*tempB00); 157 | double tempB10= ci[0]*ui[0]*ui[1] + ci[1]*(ui[1]*ui[1]-1) + ci[2]*ui[1]*ui[2]; 158 | B10+=(weight*tempB10); 159 | double tempB20= ci[0]*ui[0]*ui[2] + ci[1]*ui[2]*ui[1] + ci[2]*(ui[2]*ui[2]-1); 160 | B20+=(weight*tempB20); 161 | 162 | 163 | } 164 | 165 | A.insert(0,0)=A00; 166 | A.insert(0,1)=A01; 167 | A.insert(0,2)=A02; 168 | A.insert(1,0)=A10; 169 | A.insert(1,1)=A11; 170 | A.insert(1,2)=A12; 171 | A.insert(2,0)=A20; 172 | A.insert(2,1)=A21; 173 | A.insert(2,2)=A22; 174 | B(0)=B00; 175 | B(1)=B10; 176 | B(2)=B20; 177 | 178 | //std::ofstream file("D:\\A_Matrix.txt"); 179 | //if (file.is_open()) 180 | //{ 181 | // file << "Here is the RowMajor matrix A:\n" << A << '\n'; 182 | // /* file << "m" << '\n' << Eigen::colm(_M) << '\n';*/ 183 | //} 184 | //file.close(); 185 | 186 | //std::ofstream file2("D:\\B_Vector.txt"); 187 | //if (file2.is_open()) 188 | //{ 189 | // file2 << "Here is the matrix B:\n" << B << '\n'; 190 | // /* file << "m" << '\n' << Eigen::colm(_M) << '\n';*/ 191 | //} 192 | 193 | //file2.close(); 194 | 195 | M=A.transpose()*A; 196 | N=A.transpose()*B; 197 | 198 | } 199 | 200 | 201 | void Repair_Paint_Alpha(cv::Mat img1, cv::Mat img2, std::vector< std::pair< int, int > > &differing_pixels, std::vector< std::pair< int, int > > &sub_differing_pixels, cv::Mat &T_frame, timemap_t currentT, Eigen::VectorXd X,cv::Mat &Paint,cv::Mat &Alpha) 202 | { 203 | std::vector< std::pair< int, int > > rest_differing_pixels; 204 | int n=differing_pixels.size(); 205 | for(int index=0; index(i); 211 | Vec3d *Ai=img2.ptr(i); 212 | Vec3b *Pi=Paint.ptr(i); 213 | Vec3b *Alphai=Alpha.ptr(i); 214 | 215 | 216 | #define Reprojection 0 217 | #if Reprojection==0 218 | 219 | double u_pb[3]={0.0}; 220 | double vm_pb=std::sqrt( (X[0]-Bi[j][0])*(X[0]-Bi[j][0])+ (X[1]-Bi[j][1])*(X[1]-Bi[j][1]) + (X[2]-Bi[j][2])*(X[2]-Bi[j][2]) ); 221 | for(int k=0;k<3;k++) 222 | u_pb[k]=(X[k]-Bi[j][k])/vm_pb; 223 | 224 | double _a[3]={0.0}; 225 | double t_pb=0.0; 226 | 227 | for(int k=0;k<3;k++) 228 | { 229 | t_pb+=(Ai[j][k]-Bi[j][k])*u_pb[k]; 230 | } 231 | 232 | for(int k=0;k<3;k++) 233 | { 234 | _a[k]=Bi[j][k]+t_pb*u_pb[k]; 235 | } 236 | 237 | 238 | 239 | double a_new[3]={0.0}; 240 | 241 | double r=1.0; 242 | 243 | double e=1.0/512; 244 | 245 | 246 | for(int k=0;k<3;k++) 247 | { 248 | if((_a[k]-Ai[j][k])>0) 249 | if(r>=e/(_a[k]-Ai[j][k])) 250 | r=e/(_a[k]-Ai[j][k]); 251 | if((_a[k]-Ai[j][k])<0) 252 | if(r>=-e/(_a[k]-Ai[j][k])) 253 | r=-e/(_a[k]-Ai[j][k]); 254 | } 255 | 256 | assert((r>0)&&(r<=1)); 257 | 258 | 259 | 260 | for(int k=0;k<3;k++) 261 | a_new[k]=Ai[j][k]+r*(_a[k]-Ai[j][k]); 262 | 263 | 264 | #elif Reprojection==1 265 | double a_new[3]={0.0}; 266 | for(int k=0;k<3;k++) 267 | a_new[k]=Ai[j][k]; 268 | #endif 269 | 270 | //here, we get new a: a_new[], then we will use a_new[] to replace old Ai[j][k] to do projection. 271 | 272 | 273 | 274 | double ui[3]={0.0}; 275 | double vm=std::sqrt( (a_new[0]-Bi[j][0])*(a_new[0]-Bi[j][0])+ (a_new[1]-Bi[j][1])*(a_new[1]-Bi[j][1]) + (a_new[2]-Bi[j][2])*(a_new[2]-Bi[j][2]) ); 276 | for(int k=0;k<3;k++) 277 | ui[k]=(a_new[k]-Bi[j][k])/vm; 278 | 279 | 280 | 281 | 282 | double t=0.0; 283 | for(int k=0;k<3;k++) 284 | { 285 | t+=(X[k]-Bi[j][k])*ui[k]; 286 | } 287 | 288 | if(t<0.0) 289 | t=abs(t); 290 | 291 | 292 | //{ 293 | // for(int k=0;k<3;k++) 294 | // { 295 | // X[k]=2*Bi[j][k]-X[k]; 296 | // } 297 | //} 298 | 299 | double a_b=0.0; 300 | for(int k=0;k<3;k++) 301 | { 302 | a_b+=(a_new[k]-Bi[j][k])*ui[k]; 303 | } 304 | 305 | 306 | 307 | 308 | double l_b=1000; 309 | 310 | for(int k=0;k<3;k++) 311 | { 312 | if((1-Bi[j][k])/ui[k]>0) 313 | if(l_b>=(1-Bi[j][k])/ui[k]) 314 | l_b=(1-Bi[j][k])/ui[k]; 315 | if((0-Bi[j][k])/ui[k]>0) 316 | if(l_b>=(0-Bi[j][k])/ui[k]) 317 | l_b=(0-Bi[j][k])/ui[k]; 318 | } 319 | 320 | if(l_b>sqrt(3.0)) 321 | { 322 | cout.precision(15); 323 | cout<<"error0: "<l_b) 330 | { 331 | cout<<"error1"<(l_b+sqrt(3.0)/512.0)) 349 | { 350 | cout<<"error3"<l_b)) 356 | t=l_b; 357 | 358 | 359 | 360 | 361 | double alpha=a_b/t; 362 | 363 | if(alpha>1.0) 364 | { 365 | alpha=1.0; 366 | cout<<"error4: "<255? 255:p; 384 | Alphai[j][k]=(alpha)*255; 385 | } 386 | 387 | T_frame.ptr(i)[j]=T_frame.ptr(i)[j]*(1-alpha) + alpha*currentT; 388 | 389 | 390 | } 391 | 392 | differing_pixels.~vector(); 393 | cout< > &differing_pixels,cv::Mat &Paint,cv::Mat &Alpha, int Myflag) 404 | { 405 | using namespace cv; 406 | int width=img1.cols; 407 | int height=img1.rows; 408 | 409 | Eigen::SparseMatrix M; 410 | Eigen::VectorXd N; 411 | 412 | 413 | Compute_three_by_three_Paint_Matrix_on_just_one_component(img1,img2,differing_pixels,M,N); 414 | 415 | Eigen::VectorXd X(3); 416 | 417 | Eigen::SimplicialLDLT> ldlt(M); 418 | X=ldlt.solve(N); 419 | 420 | cout<<"X: "<<(int)d2b(X[0])<<" "<<(int)d2b(X[1])<<" "<<(int)d2b(X[2])< > sub_differing_pixels; 431 | 432 | Repair_Paint_Alpha(img1,img2,differing_pixels,sub_differing_pixels,T_frame,currentT,X,Paint,Alpha); 433 | 434 | 435 | if(sub_differing_pixels.size()>0) 436 | { 437 | Compute_three_by_three_Paint_Matrix_on_just_one_component(img1,img2,sub_differing_pixels,M,N); 438 | Eigen::VectorXd Y(3); 439 | 440 | Eigen::SimplicialLDLT> ldlt(M); 441 | Y=ldlt.solve(N); 442 | 443 | cout<<"Y: "<<(int)d2b(Y[0])<<" "<<(int)d2b(Y[1])<<" "<<(int)d2b(Y[2])< > sub_sub_differing_pixels; 447 | 448 | Repair_Paint_Alpha(img1,img2,sub_differing_pixels,sub_sub_differing_pixels,T_frame,currentT,Y,Paint,Alpha); 449 | 450 | if(sub_sub_differing_pixels.size()>0) // means still can not get proper result, then use magic! 451 | { 452 | cout<<"Magic"<(i); 460 | Vec3d * Ai=img2.ptr(i); 461 | Vec3b * Pi=Paint.ptr(i); 462 | Vec3b * Ui=Alpha.ptr(i); 463 | 464 | double alpha=0.0; 465 | vector alpha_list; 466 | for(int k=0;k<3;k++) 467 | { 468 | if(Bi[j][k]!=0) 469 | { 470 | double value=(Bi[j][k]-Ai[j][k])/Bi[j][k]; 471 | if((value<=1.0)&&(value>=0.0)) 472 | alpha_list.push_back(value); 473 | } 474 | if((1.0-Bi[j][k])!=0) 475 | { 476 | double value=(Ai[j][k]-Bi[j][k])/(1.0-Bi[j][k]); 477 | if((value<=1.0)&&(value>=0.0)) 478 | alpha_list.push_back(value); 479 | } 480 | 481 | } 482 | 483 | //cout<(i)[j]=T_frame.ptr(i)[j]*(1-alpha) + alpha*currentT; 491 | 492 | for(int k=0;k<3;k++) 493 | { 494 | Pi[j][k]=((Ai[j][k]-Bi[j][k])/alpha+Bi[j][k])*255; 495 | Ui[j][k]=alpha*255; 496 | } 497 | 498 | } 499 | 500 | } 501 | 502 | } 503 | 504 | } 505 | 506 | 507 | if(ldlt.info()==Eigen::NumericalIssue) 508 | { 509 | std::cout<<"fail"<(i); 517 | Vec3d * Ai=img2.ptr(i); 518 | Vec3b * Pi=Paint.ptr(i); 519 | Vec3b * Ui=Alpha.ptr(i); 520 | 521 | double alpha=0.0; 522 | vector alpha_list; 523 | for(int k=0;k<3;k++) 524 | { 525 | if(Bi[j][k]!=0) 526 | { 527 | double value=(Bi[j][k]-Ai[j][k])/Bi[j][k]; 528 | if((value<=1.0)&&(value>=0.0)) 529 | alpha_list.push_back(value); 530 | } 531 | if((1.0-Bi[j][k])!=0) 532 | { 533 | double value=(Ai[j][k]-Bi[j][k])/(1.0-Bi[j][k]); 534 | if((value<=1.0)&&(value>=0.0)) 535 | alpha_list.push_back(value); 536 | } 537 | 538 | } 539 | 540 | //cout<(i)[j]=T_frame.ptr(i)[j]*(1-alpha) + alpha*currentT; 548 | 549 | for(int k=0;k<3;k++) 550 | { 551 | Pi[j][k]=((Ai[j][k]-Bi[j][k])/alpha+Bi[j][k])*255; 552 | Ui[j][k]=alpha*255; 553 | } 554 | 555 | 556 | } 557 | 558 | 559 | } 560 | 561 | 562 | 563 | } 564 | 565 | 566 | double difference_Vec3b(Vec3b A, Vec3b B) 567 | { 568 | return(sqrt(double((A[0]-B[0])*(A[0]-B[0])+(A[1]-B[1])*(A[1]-B[1])+(A[2]-B[2])*(A[2]-B[2])))); 569 | } 570 | 571 | //use rgb space to compute diff pixels. input is CV_8UC3 572 | void get_diff_pixels(Mat before,Mat after,vector> &diff_pixels) 573 | { 574 | for(int i=0;i(i); 577 | Vec3b *Pi=after.ptr(i); 578 | for(int j=0;j0.0) 582 | { 583 | diff_pixels.push_back(std::make_pair(i,j)); 584 | } 585 | 586 | } 587 | } 588 | } 589 | 590 | 591 | 592 | int main() 593 | { 594 | 595 | Mat before=imread("test_case/before.png",1); 596 | Mat after= imread("test_case/after.png",1); 597 | 598 | Mat T_frame=Mat::zeros(before.cols,before.rows,CV_64FC1); 599 | 600 | //compute with just one component 601 | std::vector> differing_pixels; 602 | Mat Paint=Mat::zeros(before.size(),CV_8UC3); 603 | Mat Alpha=Mat::zeros(before.size(),CV_8UC3); 604 | 605 | Mat before_double; 606 | Mat after_double; 607 | 608 | before.convertTo(before_double,CV_64FC3,1.0/255.0); 609 | after.convertTo(after_double,CV_64FC3,1.0/255.0); 610 | 611 | get_diff_pixels(before,after,differing_pixels); 612 | 613 | Mat Diff_BinImage; 614 | PD_spatial_coherency_closet_paint_3by3_layer_extraction(before_double,after_double,T_frame,0.0,Diff_BinImage,differing_pixels,Paint,Alpha,0); 615 | 616 | imwrite("test_case/paint.png",Paint); 617 | imwrite("test_case/alpha.png",Alpha); 618 | 619 | } 620 | -------------------------------------------------------------------------------- /1 preprocessing/s1_whole_sequence_colorshift/colorshift_filter.cpp: -------------------------------------------------------------------------------- 1 | //#include "stdafx.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | //json library 21 | #include "autolink.h" 22 | #include "config.h" 23 | #include "features.h" 24 | #include "forwards.h" 25 | #include "json.h" 26 | #include "reader.h" 27 | #include "value.h" 28 | #include "writer.h" 29 | 30 | 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | 48 | 49 | using namespace std; 50 | using namespace cv; 51 | 52 | namespace 53 | { 54 | const bool SHOW_IMAGE = false; 55 | } 56 | 57 | /// Superclass 58 | class VideoFilter 59 | { 60 | public: 61 | VideoFilter() : m_output(0) {} 62 | virtual ~VideoFilter() {} 63 | 64 | void SetOutput( VideoFilter* out ) { m_output = out; } 65 | 66 | Mat NextFrame( const Mat& frame ) 67 | { 68 | Mat result=Mat::zeros(frame.rows,frame.cols,CV_8UC3); 69 | 70 | bool success = this->doNextFrame( frame, result ); 71 | if( !success ) return result; 72 | 73 | 74 | 75 | assert( !result.empty() ); 76 | 77 | if( m_output ) 78 | { 79 | return m_output->NextFrame( result ); 80 | } 81 | else 82 | { 83 | return result; 84 | } 85 | } 86 | 87 | protected: 88 | // subclasses fill this in. 89 | // Return false to skip this frame, true otherwise. 90 | virtual bool doNextFrame( const Mat& inputFrame, Mat& outputFrame ) = 0; 91 | 92 | private: 93 | VideoFilter* m_output; 94 | }; 95 | 96 | 97 | class VideoReader : public VideoFilter 98 | { 99 | public: 100 | VideoReader( const Json::Value& params ) 101 | { 102 | m_video = VideoCapture( params["filename"].asString() ); 103 | count=0; 104 | FrameStop=params["frameStop"].asInt(); 105 | FrameStart=params["frameStart"].asInt(); 106 | outputFrame=Mat::zeros((int)m_video.get(CV_CAP_PROP_FRAME_HEIGHT),(int)m_video.get(CV_CAP_PROP_FRAME_WIDTH),CV_8UC3); 107 | } 108 | 109 | bool IsFinished() const 110 | { 111 | return count>FrameStop? true: false; 112 | } 113 | 114 | int getcount() 115 | { 116 | return count; 117 | } 118 | 119 | protected: 120 | bool doNextFrame( const Mat&, Mat& outputFrame ) override 121 | { 122 | if(countpercent% 177 | Mat temp; 178 | threshold(current_diff,temp,Threshold,1.0,THRESH_BINARY); 179 | if(countNonZero(temp)>percent*height*width) 180 | ColorShiftRecover(inputFrame,Threshold,outputFrame,fixedFrame); 181 | else 182 | outputFrame=inputFrame.clone(); 183 | 184 | return true; 185 | } 186 | 187 | Mat toLab( const cv::Mat& bgr ) 188 | { 189 | using namespace cv; 190 | 191 | Mat bgr32, lab; 192 | bgr.convertTo(bgr32,CV_32FC3,1.0/255.0); 193 | return bgr32; 194 | //cvtColor(bgr32,lab,CV_BGR2Lab); 195 | //return lab; 196 | } 197 | 198 | 199 | Mat diff( cv::Mat mat1, cv::Mat mat2 ) 200 | { 201 | using namespace cv; 202 | 203 | Mat diff; 204 | Mat mat1_lab=toLab(mat1); 205 | Mat mat2_lab=toLab(mat2); 206 | 207 | subtract( mat1_lab, mat2_lab, diff ); 208 | 209 | std::vector lab; 210 | split(diff,lab); 211 | 212 | Mat L2=lab[0].mul(lab[0]); 213 | Mat a2=lab[1].mul(lab[1]); 214 | Mat b2=lab[2].mul(lab[2]); 215 | 216 | Mat DIFF = Mat::zeros(diff.rows, diff.cols, CV_32FC1); 217 | 218 | 219 | for(int i=0; i(i)[j]=sqrt( L2.ptr(i)[j]+ a2.ptr(i)[j]+ b2.ptr(i)[j])/1.8;//376.0;//1.8;//376.0;/*/1.8;*////376.0;///1.8;///376.0; 222 | 223 | 224 | Mat imgout; 225 | DIFF.convertTo(imgout,CV_8UC1,255.0); 226 | //imshow("imgout",imgout); 227 | //waitKey(0); 228 | return imgout; 229 | 230 | } 231 | 232 | 233 | 234 | void ColorShiftRecover(const Mat& inputFrame, double Threshold, Mat& outputFrame, Mat& fixedFrame) 235 | { 236 | int height=inputFrame.rows; 237 | int width=inputFrame.cols; 238 | std::vector RGB_old; 239 | split(fixedFrame,RGB_old); //fixedframe is last old frame 240 | Mat R_old=RGB_old[0]; 241 | Mat G_old=RGB_old[1]; 242 | Mat B_old=RGB_old[2]; 243 | 244 | std::vector RGB_new; 245 | split(inputFrame,RGB_new); //inputframe is new frame. 246 | Mat R_new=RGB_new[0]; 247 | Mat G_new=RGB_new[1]; 248 | Mat B_new=RGB_new[2]; 249 | 250 | Mat R_recover,G_recover,B_recover,RGB_recover; 251 | 252 | 253 | std::vector INDEX; 254 | 255 | Mat mask; 256 | 257 | 258 | #define MASK_CHOICE 0 // 0 for sorting diff color value and then choose a range. 1 for sorting old and new imgs' color. and then compute diff. then sort diff and then choose a range. 259 | #if MASK_CHOICE==0 260 | 261 | mask=Mat::zeros(height,width,CV_8UC1); 262 | getmask(fixedFrame,inputFrame,mask); // get the mask of the selected pixels that are in N/8 to N/4 range of color differences . 263 | 264 | #elif MASK_CHOICE==1 265 | 266 | 267 | std::vector Diff_R,Diff_G,Diff_B; 268 | 269 | getsortedDiff(R_old,R_new,Diff_R); 270 | getsortedDiff(G_old,G_new,Diff_G); 271 | getsortedDiff(B_old,B_new,Diff_B); 272 | 273 | std::vector> DIFF; 274 | for(int i = 0;i &left, const std::pair &right) { 282 | return left.first < right.first; 283 | } 284 | }; 285 | 286 | sort(DIFF.begin(),DIFF.end(),sort_pred()); 287 | 288 | int N=DIFF.size(); 289 | 290 | for(int i=N/10; i Recover; 301 | Recover.push_back(R_recover); 302 | Recover.push_back(G_recover); 303 | Recover.push_back(B_recover); 304 | merge(Recover,RGB_recover); 305 | 306 | outputFrame=RGB_recover.clone(); 307 | fixedFrame=outputFrame.clone(); 308 | // since 10/16/2014, we do colorshift based on fixedframe for a small 309 | // sequence of frame, and do not update the fixedframe. that means, we compare each 310 | // frame with first frame to do colorshift recovering. 311 | 312 | } 313 | 314 | 315 | struct Triple 316 | { 317 | 318 | double diff; 319 | int row; 320 | int col; 321 | 322 | Triple(double k, int i, int j) : diff(k), row(i), col(j) {} 323 | 324 | bool operator < (const Triple& str) const 325 | { 326 | return (diff < str.diff); 327 | } 328 | }; 329 | 330 | 331 | void getmask(Mat _before,Mat _after, Mat &mask) 332 | { 333 | 334 | Mat before,after; 335 | _before.convertTo(before,CV_64FC3,1.0/255.0); 336 | _after.convertTo(after,CV_64FC3,1.0/255.0); 337 | 338 | int height=before.rows; 339 | int width=before.cols; 340 | 341 | Mat diff=Mat::zeros(height,width,CV_64FC1); 342 | 343 | for(int i=0;i(i)[j]+=pow(abs(before.ptr(i)[j][k]-after.ptr(i)[j][k]),2.0); 347 | 348 | 349 | std::vector diff_list; 350 | 351 | 352 | 353 | for(int i=0;i(i)[j],i,j)); 357 | } 358 | 359 | 360 | std::sort(diff_list.begin(),diff_list.end()); 361 | 362 | //for(int index=0;index<1000;index++) 363 | //cout<<(diff_list[index].diff)<(diff_list[index].row)[diff_list[index].col]=1; 369 | 370 | } 371 | 372 | 373 | void getsortedDiff(Mat _OldImg, Mat _NewImg,vector &diff) 374 | { 375 | 376 | Mat OldImg,NewImg; 377 | _OldImg.convertTo(OldImg,CV_64FC1,1.0/255.0); 378 | _NewImg.convertTo(NewImg,CV_64FC1,1.0/255.0); 379 | 380 | int height=NewImg.rows; 381 | int width=NewImg.cols; 382 | 383 | std::vector OLD,NEW; 384 | 385 | for(int i = 0;i (i); 388 | double *Ni= NewImg.ptr(i); 389 | 390 | for(int j =0;j &INDEX) 406 | { 407 | Mat OldImg,NewImg; 408 | _OldImg.convertTo(OldImg,CV_64FC1,1.0/255.0); 409 | _NewImg.convertTo(NewImg,CV_64FC1,1.0/255.0); 410 | 411 | Eigen::SparseMatrix M(2,2); 412 | Eigen::VectorXd N(2); 413 | Eigen::VectorXd X(2); 414 | 415 | int height=OldImg.rows; 416 | int width= OldImg.cols; 417 | 418 | double count=0; 419 | double new_sum=0; 420 | double new_sum_square=0; 421 | double old_sum=0; 422 | double old_new_sum=0; 423 | 424 | 425 | #define MASK 1 // 0 for not using mask, 1 for using mask. 2 for using vector INDEX 426 | 427 | #if MASK==0 428 | for(int i = 0;i (i); 431 | double *Ni= NewImg.ptr(i); 432 | 433 | for(int j =0;j =0) 436 | { 437 | count+=1.0; 438 | new_sum+=Ni[j]; 439 | old_sum+=Mi[j]; 440 | new_sum_square+=Ni[j]*Ni[j]; 441 | old_new_sum+=Ni[j]*Mi[j]; 442 | } 443 | } 444 | } 445 | 446 | #elif MASK==1 447 | for(int i = 0;i (i); 450 | double *Ni= NewImg.ptr(i); 451 | unsigned char *Maski=mask.ptr(i); 452 | 453 | for(int j =0;j OLD,NEW; 473 | 474 | for(int i = 0;i (i); 477 | double *Ni= NewImg.ptr(i); 478 | 479 | for(int j =0;j > chol(M); //solve MX=N 510 | X=chol.solve(N); 511 | 512 | Mat temp_recover=OldImg.clone(); 513 | 514 | for(int i = 0;i (i); 517 | double *Ni= NewImg.ptr(i); 518 | double *Qi = temp_recover.ptr(i); 519 | 520 | for(int j =0;j =0) 523 | { 524 | double value=(X[0]+X[1]*Ni[j]); // ||a+b*new-old|| so, recover=a+b*new 525 | 526 | if(value>1.0) 527 | Qi[j]=1.0; 528 | 529 | else if(value<0.0) 530 | Qi[j]=0.0; 531 | else 532 | Qi[j]=value; 533 | } 534 | } 535 | } 536 | 537 | temp_recover.convertTo(recover,CV_8UC1,255.0); 538 | 539 | } 540 | 541 | 542 | private: 543 | 544 | double percent; 545 | Mat fixedFrame; 546 | double Threshold; 547 | }; 548 | 549 | 550 | class VideoWriter : public VideoFilter 551 | { 552 | public: 553 | VideoWriter( const Json::Value& params ) 554 | { 555 | m_video = cv::VideoWriter( params["filename"].asString(), CV_FOURCC('D','I','V','X'), params["FPS"].asDouble(), Size(params["width"].asInt(),params["height"].asInt()), true ); 556 | } 557 | 558 | protected: 559 | virtual bool doNextFrame( const Mat& inputFrame, Mat& outputFrame ) 560 | { 561 | outputFrame = inputFrame.clone(); 562 | m_video.write( inputFrame ); 563 | return true; 564 | } 565 | 566 | private: 567 | cv::VideoWriter m_video; 568 | }; 569 | 570 | 571 | class VideoSaver : public VideoFilter 572 | { 573 | public: 574 | VideoSaver( const Json::Value& params ) 575 | { 576 | m_path = params["filename"].asString(); 577 | number=0; 578 | } 579 | 580 | protected: 581 | virtual bool doNextFrame( const Mat& inputFrame, Mat& outputFrame ) 582 | { 583 | string fullpathname=m_path+ GetNextNumber(number)+".png"; 584 | outputFrame = inputFrame.clone(); 585 | cv::imwrite( fullpathname, inputFrame ); 586 | number++; 587 | return true; 588 | } 589 | 590 | std::string GetNextNumber( int lastNum ) 591 | { 592 | std::stringstream ss; 593 | ss << std::setfill('0') << std::setw(4) << lastNum; 594 | return ss.str(); 595 | } 596 | 597 | private: 598 | std::string m_path; 599 | int number; 600 | }; 601 | 602 | VideoFilter* CreateVideoFilterByName( const std::string& filterName, const Json::Value& params ) 603 | { 604 | if( filterName == std::string("VideoReader") ) return new VideoReader(params); 605 | else if( filterName == std::string("VideoWriter") ) return new ::VideoWriter(params); 606 | else if( filterName == std::string("VideoSaver") ) return new VideoSaver(params); 607 | else if( filterName == std::string("ColorShift") ) return new ColorShift(params); 608 | } 609 | 610 | 611 | 612 | 613 | std::string get_file_contents(const char *filename) 614 | { 615 | std::ifstream in(filename, std::ios::in | std::ios::binary); 616 | if (in) 617 | { 618 | std::string contents; 619 | in.seekg(0, std::ios::end); 620 | contents.resize(in.tellg()); 621 | in.seekg(0, std::ios::beg); 622 | in.read(&contents[0], contents.size()); 623 | in.close(); 624 | return(contents); 625 | } 626 | throw(errno); 627 | } 628 | 629 | 630 | int main( int argc, const char* argv[] ) 631 | { 632 | 633 | if( argc != 2 ) 634 | { 635 | std::cerr << "Usage: " << argv[0] << "path/to/params.json\n"; 636 | return -1; 637 | } 638 | const char *json_name = argv[1]; 639 | std::string str=get_file_contents(json_name); 640 | cout< filters; 667 | for( int i = 0; i < operations.size(); ++i ) 668 | { 669 | VideoFilter* nextFilter = CreateVideoFilterByName( operations[i]["filter"].asString(), operations[i]["params"] ); 670 | filters.push_back( nextFilter ); 671 | if( i > 0 ) filters.at(i-1)->SetOutput( nextFilter ); 672 | } 673 | 674 | // The first filter must be a VideoReader 675 | assert( filters.size() > 0 ); 676 | VideoReader* videoreader = dynamic_cast< VideoReader* >( filters.front() ); 677 | Mat ignore, result; 678 | int num=0; 679 | while( !videoreader->IsFinished() ) 680 | { 681 | cout<NextFrame( ignore ); 684 | num++; 685 | 686 | } 687 | 688 | // Cleanup 689 | for( int i = 0; i < filters.size(); ++i ) 690 | { 691 | delete filters.at(i); 692 | } 693 | filters.clear(); 694 | 695 | } 696 | -------------------------------------------------------------------------------- /1 preprocessing/s2_extract_keyframes/detect_keyframe.cpp: -------------------------------------------------------------------------------- 1 | #include "stdafx.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | using namespace cv; 29 | using namespace std; 30 | 31 | std::string GetNextNumber( int lastNum ) 32 | { 33 | std::stringstream ss; 34 | ss << std::setfill('0') << std::setw(4) << lastNum; 35 | return ss.str(); 36 | } 37 | 38 | struct Triple 39 | { 40 | 41 | double diff; 42 | int row; 43 | int col; 44 | 45 | Triple(double k, int i, int j) : diff(k), row(i), col(j) {} 46 | 47 | bool operator < (const Triple& str) const 48 | { 49 | return (diff < str.diff); 50 | } 51 | }; 52 | 53 | void getmask(Mat _before,Mat _after, Mat &mask) 54 | { 55 | 56 | Mat before,after; 57 | _before.convertTo(before,CV_64FC3,1.0/255.0); 58 | _after.convertTo(after,CV_64FC3,1.0/255.0); 59 | 60 | int height=before.rows; 61 | int width=before.cols; 62 | 63 | Mat diff=Mat::zeros(height,width,CV_64FC1); 64 | 65 | for(int i=0;i(i)[j]+=pow(abs(before.ptr(i)[j][k]-after.ptr(i)[j][k]),2.0); 69 | 70 | 71 | std::vector diff_list; 72 | 73 | 74 | 75 | for(int i=0;i(i)[j],i,j)); 79 | } 80 | 81 | 82 | std::sort(diff_list.begin(),diff_list.end()); 83 | 84 | //for(int index=0;index<1000;index++) 85 | //cout<<(diff_list[index].diff)<(diff_list[index].row)[diff_list[index].col]=1; 91 | 92 | } 93 | 94 | void SolveLSM(Mat _OldImg, Mat _NewImg, Mat &mask, Mat &recover, vector &INDEX) 95 | { 96 | Mat OldImg,NewImg; 97 | _OldImg.convertTo(OldImg,CV_64FC1,1.0/255.0); 98 | _NewImg.convertTo(NewImg,CV_64FC1,1.0/255.0); 99 | 100 | Eigen::SparseMatrix M(2,2); 101 | Eigen::VectorXd N(2); 102 | Eigen::VectorXd X(2); 103 | 104 | int height=OldImg.rows; 105 | int width= OldImg.cols; 106 | 107 | double count=0; 108 | double new_sum=0; 109 | double new_sum_square=0; 110 | double old_sum=0; 111 | double old_new_sum=0; 112 | 113 | 114 | #define MASK 1 // 0 for not using mask, 1 for using mask. 2 for using vector INDEX 115 | 116 | #if MASK==0 117 | for(int i = 0;i (i); 120 | double *Ni= NewImg.ptr(i); 121 | 122 | for(int j =0;j =0) 125 | { 126 | count+=1.0; 127 | new_sum+=Ni[j]; 128 | old_sum+=Mi[j]; 129 | new_sum_square+=Ni[j]*Ni[j]; 130 | old_new_sum+=Ni[j]*Mi[j]; 131 | } 132 | } 133 | } 134 | 135 | #elif MASK==1 136 | for(int i = 0;i (i); 139 | double *Ni= NewImg.ptr(i); 140 | unsigned char *Maski=mask.ptr(i); 141 | 142 | for(int j =0;j OLD,NEW; 158 | 159 | for(int i = 0;i (i); 162 | double *Ni= NewImg.ptr(i); 163 | 164 | for(int j =0;j > chol(M); //solve MX=N 195 | X=chol.solve(N); 196 | 197 | Mat temp_recover=OldImg.clone(); 198 | 199 | for(int i = 0;i (i); 202 | double *Ni= NewImg.ptr(i); 203 | double *Qi = temp_recover.ptr(i); 204 | 205 | for(int j =0;j =0) 208 | { 209 | double value=(X[0]+X[1]*Ni[j]); // ||a+b*new-old|| so, recover=a+b*new 210 | 211 | if(value>1.0) 212 | Qi[j]=1.0; 213 | 214 | else if(value<0.0) 215 | Qi[j]=0.0; 216 | else 217 | Qi[j]=value; 218 | } 219 | } 220 | } 221 | 222 | temp_recover.convertTo(recover,CV_8UC1,255.0); 223 | 224 | } 225 | 226 | void ColorShiftRecover(const Mat& inputFrame, double Threshold, Mat& outputFrame, Mat& fixedFrame) 227 | { 228 | int height=inputFrame.rows; 229 | int width=inputFrame.cols; 230 | std::vector RGB_old; 231 | split(fixedFrame,RGB_old); //fixedframe is last old frame 232 | Mat R_old=RGB_old[0]; 233 | Mat G_old=RGB_old[1]; 234 | Mat B_old=RGB_old[2]; 235 | 236 | std::vector RGB_new; 237 | split(inputFrame,RGB_new); //inputframe is new frame. 238 | Mat R_new=RGB_new[0]; 239 | Mat G_new=RGB_new[1]; 240 | Mat B_new=RGB_new[2]; 241 | 242 | Mat R_recover,G_recover,B_recover,RGB_recover; 243 | 244 | 245 | std::vector INDEX; 246 | 247 | Mat mask; 248 | 249 | 250 | #define MASK_CHOICE 0 // 0 for sorting diff color value and then choose a range. 1 for sorting old and new imgs' color. and then compute diff. then sort diff and then choose a range. 251 | #if MASK_CHOICE==0 252 | 253 | mask=Mat::zeros(height,width,CV_8UC1); 254 | getmask(fixedFrame,inputFrame,mask); // get the mask of the selected pixels that are in N/4 to N/2 range of color differences . 255 | 256 | #elif MASK_CHOICE==1 257 | 258 | 259 | std::vector Diff_R,Diff_G,Diff_B; 260 | 261 | getsortedDiff(R_old,R_new,Diff_R); 262 | getsortedDiff(G_old,G_new,Diff_G); 263 | getsortedDiff(B_old,B_new,Diff_B); 264 | 265 | std::vector> DIFF; 266 | for(int i = 0;i &left, const std::pair &right) { 274 | return left.first < right.first; 275 | } 276 | }; 277 | 278 | sort(DIFF.begin(),DIFF.end(),sort_pred()); 279 | 280 | int N=DIFF.size(); 281 | 282 | for(int i=N/10; i Recover; 293 | Recover.push_back(R_recover); 294 | Recover.push_back(G_recover); 295 | Recover.push_back(B_recover); 296 | merge(Recover,RGB_recover); 297 | 298 | outputFrame=RGB_recover.clone(); 299 | //fixedFrame=outputFrame.clone();// ?? inputframe.clone or outputframe.clone??????? 300 | // since 10/16/2014, we do colorshift based on fixedframe for a small 301 | // sequence of frame, and do not update the fixedframe. that means, we compare each 302 | // frame with first frame to do colorshift recovering. 303 | 304 | } 305 | 306 | 307 | void getsortedDiff(Mat _OldImg, Mat _NewImg,vector &diff) 308 | { 309 | 310 | Mat OldImg,NewImg; 311 | _OldImg.convertTo(OldImg,CV_64FC1,1.0/255.0); 312 | _NewImg.convertTo(NewImg,CV_64FC1,1.0/255.0); 313 | 314 | int height=NewImg.rows; 315 | int width=NewImg.cols; 316 | 317 | std::vector OLD,NEW; 318 | 319 | for(int i = 0;i (i); 322 | double *Ni= NewImg.ptr(i); 323 | 324 | for(int j =0;j lab; 361 | split(diff,lab); 362 | 363 | Mat L2=lab[0].mul(lab[0]); 364 | Mat a2=lab[1].mul(lab[1]); 365 | Mat b2=lab[2].mul(lab[2]); 366 | 367 | Mat DIFF = Mat::zeros(diff.rows, diff.cols, CV_32FC1); 368 | 369 | 370 | for(int i=0; i(i)[j]=sqrt( L2.ptr(i)[j]+ a2.ptr(i)[j]+ b2.ptr(i)[j])/1.8;//376.0;/*/1.8;*////376.0;///1.8;///376.0; 373 | 374 | 375 | Mat imgout; 376 | DIFF.convertTo(imgout,CV_8UC1,255.0); 377 | return imgout; 378 | } 379 | 380 | 381 | void subsequence_colorshift(Mat fixedFrame, Mat inputFrame, Mat &outputFrame) 382 | { 383 | double Threshold=0.0; 384 | double percent=0.5; 385 | Mat current_diff = Mat::zeros(inputFrame.rows,inputFrame.cols, CV_8UC1); 386 | 387 | current_diff = diff(fixedFrame,inputFrame); 388 | 389 | int height=inputFrame.rows; 390 | int width=inputFrame.cols; 391 | 392 | //count the non zero element, and do not change tframe corresponding to the frame whose number >percent% 393 | Mat temp; 394 | threshold(current_diff,temp,Threshold,1.0,THRESH_BINARY); 395 | if(countNonZero(temp)>percent*height*width) 396 | ColorShiftRecover(inputFrame,Threshold,outputFrame,fixedFrame); 397 | else 398 | outputFrame=inputFrame.clone(); 399 | } 400 | 401 | 402 | void extract_difference_mask_between_keyframe(std::string keyframe_sequence_name, std::string first_keyframe_name, double keyframe_diff_threshold, int keyframe_num, std::string keyframe_mask_output_path) 403 | { 404 | VideoCapture capture=VideoCapture(keyframe_sequence_name); 405 | Mat first=imread(first_keyframe_name,1); 406 | Mat _frame; 407 | Mat frame; 408 | Mat first_lab; 409 | Mat first_gaussian; 410 | GaussianBlur(first,first_gaussian,Size(3,3),1.0,1.0); 411 | cvtColor(first_gaussian,first_lab,CV_BGR2Lab); 412 | 413 | for(int i=0;i(m); 427 | Vec3b *Tm=frame_lab.ptr(m); 428 | 429 | unsigned char *Qm=diff.ptr(m); 430 | for(int n=0;n(m); 479 | Vec3b *Nm=temp.ptr(m); 480 | for(int n=0;nthreshold) 484 | Mm[n]=255; 485 | } 486 | } 487 | img_lab=frame_lab.clone(); 488 | cout< averaged_index; 546 | 547 | int last_position; 548 | for(int i=N-1;i>=0;i--) 549 | { 550 | if(selected_sign[i]==1) 551 | { 552 | last_position=i; 553 | break; 554 | } 555 | } 556 | 557 | 558 | for(int i=0;i 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | 29 | 30 | #include 31 | 32 | 33 | #include "autolink.h" 34 | #include "config.h" 35 | #include "features.h" 36 | #include "forwards.h" 37 | #include "json.h" 38 | #include "reader.h" 39 | #include "value.h" 40 | #include "writer.h" 41 | 42 | 43 | 44 | using namespace std; 45 | using namespace cv; 46 | 47 | std::string GetNextNumber( int lastNum ) 48 | { 49 | std::stringstream ss; 50 | ss << std::setfill('0') << std::setw(4) << lastNum; 51 | return ss.str(); 52 | } 53 | 54 | double Round(double x) 55 | { 56 | double y; 57 | //if (x >= floor(x)+0.5) 58 | // y = ceil(x); 59 | // else 60 | // y = floor(x); 61 | y=floor(x+0.5); 62 | 63 | return y; 64 | } 65 | 66 | void Uchar2Double(cv::Mat &img, cv::Mat &dst) 67 | { 68 | int width=img.cols; 69 | int height=img.rows; 70 | for(int i = 0;i < height;i++) 71 | { 72 | Vec3b * Mi = img.ptr(i); 73 | Vec3d * Ni = dst.ptr(i); 74 | for(int j =0;j < width;j++) 75 | { 76 | for(int k=0;k<3;k++) 77 | { 78 | Ni[j][k]=(double)(Mi[j][k]/255.0); 79 | } 80 | } 81 | } 82 | 83 | } 84 | 85 | double difference_Vec3b(Vec3b A, Vec3b B) 86 | { 87 | return(sqrt(double((A[0]-B[0])*(A[0]-B[0])+(A[1]-B[1])*(A[1]-B[1])+(A[2]-B[2])*(A[2]-B[2])))); 88 | } 89 | 90 | 91 | void PD_alpha_interpolation(Mat firstframe,Mat eachframe,Mat &T_frame,double threshold, int currentT,Mat &Paint,Mat &Alpha) 92 | { 93 | int width=firstframe.cols; 94 | int height=firstframe.rows; 95 | Mat _firstframe=Mat::zeros(height,width,CV_64FC3); 96 | Mat _eachframe=Mat::zeros(height,width,CV_64FC3); 97 | Mat firstframe_lab; 98 | Mat eachframe_lab; 99 | cvtColor(firstframe,firstframe_lab,CV_BGR2Lab); 100 | cvtColor(eachframe,eachframe_lab,CV_BGR2Lab); 101 | 102 | Uchar2Double(firstframe,_firstframe); 103 | Uchar2Double(eachframe,_eachframe); 104 | 105 | for(int i=0;i(i); 108 | Vec3d * Ai=_eachframe.ptr(i); 109 | 110 | Vec3b *_Bi=firstframe_lab.ptr(i); 111 | Vec3b *_Ai=eachframe_lab.ptr(i); 112 | 113 | Vec3b * Pi=Paint.ptr(i); 114 | Vec3b * Ui=Alpha.ptr(i); 115 | 116 | double alpha=0.0; 117 | 118 | for(int j=0;jthreshold) 122 | { 123 | vector alpha_list; 124 | for(int k=0;k<3;k++) 125 | { 126 | if(Bi[j][k]!=0) 127 | { 128 | double value=(Bi[j][k]-Ai[j][k])/Bi[j][k]; 129 | if((value<=1.0)&&(value>=0.0)) 130 | alpha_list.push_back(value); 131 | } 132 | if((1.0-Bi[j][k])!=0) 133 | { 134 | double value=(Ai[j][k]-Bi[j][k])/(1.0-Bi[j][k]); 135 | if((value<=1.0)&&(value>=0.0)) 136 | alpha_list.push_back(value); 137 | } 138 | //if(Bi[j][k]!=0) 139 | // if(alpha<=(Bi[j][k]-Ai[j][k])/Bi[j][k]) 140 | // alpha=(Bi[j][k]-Ai[j][k])/Bi[j][k]; 141 | //if((1.0-Bi[j][k])!=0) 142 | // if(alpha<=(Ai[j][k]-Bi[j][k])/(1.0-Bi[j][k])) 143 | // alpha=(Ai[j][k]-Bi[j][k])/(1.0-Bi[j][k]); 144 | } 145 | 146 | //cout<(i)[j]=T_frame.ptr(i)[j]*(1-alpha) + alpha*currentT; 167 | } 168 | 169 | } 170 | } 171 | 172 | } 173 | 174 | 175 | Mat toLab( const cv::Mat& bgr ) 176 | { 177 | using namespace cv; 178 | 179 | Mat bgr32, lab; 180 | bgr.convertTo(bgr32,CV_32FC3,1.0/255.0); 181 | return bgr32; 182 | //cvtColor(bgr32,lab,CV_BGR2Lab); 183 | //return lab; 184 | } 185 | 186 | Mat diff( cv::Mat mat1, cv::Mat mat2 ) 187 | { 188 | using namespace cv; 189 | 190 | Mat diff; 191 | Mat mat1_lab=toLab(mat1); 192 | Mat mat2_lab=toLab(mat2); 193 | 194 | subtract( mat1_lab, mat2_lab, diff ); 195 | 196 | std::vector lab; 197 | split(diff,lab); 198 | 199 | Mat L2=lab[0].mul(lab[0]); 200 | Mat a2=lab[1].mul(lab[1]); 201 | Mat b2=lab[2].mul(lab[2]); 202 | 203 | Mat DIFF = Mat::zeros(diff.rows, diff.cols, CV_32FC1); 204 | 205 | 206 | for(int i=0; i(i)[j]=sqrt( L2.ptr(i)[j]+ a2.ptr(i)[j]+ b2.ptr(i)[j])/1.8;//376.0;/*/1.8;*////376.0;///1.8;///376.0; 209 | 210 | 211 | Mat imgout; 212 | DIFF.convertTo(imgout,CV_8UC1,255.0); 213 | return imgout; 214 | } 215 | 216 | 217 | double KM_equation(double R, double T, double b) 218 | { 219 | //cout<<"b"<(i); 240 | Vec3d * Ai=_after.ptr(i); 241 | 242 | Vec3b * BBi=before.ptr(i); 243 | Vec3b * AAi=after.ptr(i); 244 | 245 | Vec3b *_Bi=before_lab.ptr(i); 246 | Vec3b *_Ai=after_lab.ptr(i); 247 | 248 | Vec3d * Pi=R.ptr(i); // R and T are double now! 249 | Vec3d * Ui=T.ptr(i); 250 | 251 | for(int j=0;j(1.0+1/512.0)) 295 | { 296 | cout.precision(15); 297 | cout<<"error1: "<1.0)&&(R<(1.0+1.0/512.0))) 301 | R=1.0; 302 | 303 | if((R+T)>(1.0+1.0/512)) 304 | { 305 | cout.precision(15); 306 | cout<<"error2: "<1.0)&&((R+T)<(1.0+1.0/512))) 311 | T=1.0-R; 312 | 313 | } 314 | } 315 | 316 | } 317 | 318 | Pi[j][k]=R; 319 | Ui[j][k]=T; 320 | 321 | } 322 | 323 | } 324 | } 325 | 326 | } 327 | 328 | void writeMatToFile(const Mat &I, string path) 329 | { 330 | //load the matrix size 331 | int matWidth = I.size().width, matHeight = I.size().height; 332 | 333 | //read type from Mat 334 | int type = I.type(); 335 | gzFile file = gzopen(path.c_str(),"wb"); 336 | 337 | //write type and size of the matrix first 338 | gzwrite(file,(const char*) &type, sizeof(type)); 339 | gzwrite(file,(const char*) &matWidth, sizeof(matWidth)); 340 | gzwrite(file,(const char*) &matHeight, sizeof(matHeight)); 341 | 342 | //write data depending on the image's type 343 | switch (type) 344 | { 345 | default: 346 | cout << "Error: wrong Mat type: must be CV_32F, CV_64F, CV_32FC3 or CV_64FC3" << endl; 347 | break; 348 | 349 | case CV_64FC3: 350 | // cout << "Writing CV_64FC3 image" << endl; 351 | gzwrite(file,(const char*) &I.at(0)[0],sizeof(double)*3*matWidth*matHeight); 352 | break; 353 | 354 | } 355 | 356 | gzclose(file); 357 | } 358 | 359 | void readFileToMat(Mat &I, string path) 360 | { 361 | 362 | 363 | //declare image parameters 364 | int matWidth, matHeight, type; 365 | 366 | Vec3d vdvalue; 367 | 368 | gzFile file =gzopen(path.c_str(),"rb"); 369 | 370 | 371 | 372 | //read type and size of the matrix first 373 | gzread(file,(char*) &type, sizeof(type)); 374 | gzread(file,(char*) &matWidth, sizeof(matWidth)); 375 | gzread(file,(char*) &matHeight, sizeof(matHeight)); 376 | 377 | //change Mat type 378 | I = Mat::zeros(matHeight, matWidth, type); 379 | 380 | //write data depending on the image's type 381 | switch (type) 382 | { 383 | default: 384 | cout << "Error: wrong Mat type: must be CV_32F, CV_64F, CV_32FC3 or CV_64FC3" << endl; 385 | break; 386 | 387 | case CV_64FC3: 388 | //cout << "Reading CV_64FC3 image" << endl; 389 | for (int i=0; i < matWidth*matHeight; ++i) 390 | { 391 | gzread(file,(char*) &vdvalue, sizeof(vdvalue)); 392 | I.at(i) = vdvalue; 393 | } 394 | break; 395 | 396 | } 397 | 398 | gzclose(file); 399 | 400 | } 401 | 402 | 403 | void process_KM_stroke_extraction(const Json::Value ¶ms) 404 | { 405 | string source_name=params["source_name"].asString(); 406 | VideoCapture capture=VideoCapture(source_name); 407 | double threshold=params["threshold"].asDouble(); 408 | int frame_num=params["frame_num"].asInt(); 409 | string final_replay_output_path=params["final_replay_output_path"].asString(); 410 | string KM_R_video_name=params["KM_R_video_name"].asString(); 411 | string KM_T_video_name=params["KM_T_video_name"].asString(); 412 | string KM_stroke_video_name=params["KM_stroke_video_name"].asString(); 413 | string firstframe_path=params["firstframe_path"].asString(); 414 | string KM_stroke_image_base_path=params["KM_stroke_image_base_path"].asString(); 415 | 416 | 417 | int width=params["width"].asInt(); 418 | int height=params["height"].asInt(); 419 | double FPS=params["FPS"].asDouble(); 420 | string R_gzfile_base_name=params["R_gzfile_base_name"].asString(); 421 | string T_gzfile_base_name=params["T_gzfile_base_name"].asString(); 422 | 423 | Mat frame; 424 | capture.read(frame); 425 | Mat first=frame.clone(); 426 | Mat timemap=Mat::zeros(frame.rows,frame.cols,CV_64FC1); 427 | 428 | 429 | VideoWriter writer_R=VideoWriter(KM_R_video_name,CV_FOURCC('D','I','V','X'),FPS,Size(width,height),1); 430 | VideoWriter writer_T=VideoWriter(KM_T_video_name,CV_FOURCC('D','I','V','X'),FPS,Size(width,height),1); 431 | 432 | for(int i=1;i(m); //R double 481 | Vec3d *Tm=frame3.ptr(m); //T double 482 | Vec3d *Mm=replay_double.ptr(m);// double! 483 | 484 | Vec3b *Qm=replay.ptr(m); 485 | Vec3b *Sm=stroke.ptr(m); 486 | 487 | for(int n=0;n1.0) 494 | { 495 | cout.precision(15); 496 | cout<<"value: "<1.0? 1.0:value; 499 | Qm[n][k]=Round(Mm[n][k]*255); 500 | 501 | //stroke. 502 | double value2=Fm[n][k]+Tm[n][k]*Tm[n][k]*1.0/(1.0-Fm[n][k]*1.0); 503 | // if(value2>1.0) 504 | //{ 505 | // cout.precision(15); 506 | // cout<<"value2: "<1.0? 1.0:value2; 510 | Sm[n][k]=Round(value2*255.0); 511 | } 512 | } 513 | } 514 | 515 | writer_stroke<(m); 587 | Vec3b *Tm=frame3.ptr(m); 588 | Vec3b *Qm=replay.ptr(m); 589 | Vec3d *_Qm=replay_double.ptr(m); 590 | Vec3b *Pm=stroke.ptr(m); 591 | for(int n=0;n