├── media └── guitar.mp4 ├── test_filtering.py ├── .gitignore ├── README.md ├── pyr2arr.py ├── phasebasedMoMag.py └── temporal_filters.py /media/guitar.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mobizt/pbMoMa/master/media/guitar.mp4 -------------------------------------------------------------------------------- /test_filtering.py: -------------------------------------------------------------------------------- 1 | import temporal_filters 2 | from temporal_filters import * 3 | from matplotlib.pylab import * 4 | import numpy as np 5 | 6 | fps = 60 7 | secs = 3 8 | n = fps * secs 9 | ts = np.linspace(0, secs, n) 10 | noise = np.random.rand(n) * .2 11 | fq1 = sin(ts * 2 * pi * 1) 12 | fq2 = sin(ts * 2 * pi * 7) 13 | fq3 = sin(ts * 2 * pi * 12) 14 | data = fq1*1 + fq2*1 + fq3 + noise 15 | 16 | win = IdealFilterWindowed(60, 4, 9, fps, outfun=lambda x: x[0]) 17 | #win = ButterBandpassFilter(1, 4, 9, fps) 18 | #win = ButterFilter(5, 9, fps) 19 | win.update(data) 20 | out = win.collect() 21 | 22 | if 1: 23 | # create plot 24 | clf() 25 | plot(data, 'k:') 26 | plot(fq2, 'k-') 27 | plot(out, 'r', linewidth=2) 28 | #out2 = scipy.signal.lfilter(win.b, win.a, data) 29 | #plot(out2, '--') 30 | 31 | show() -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pbMoMa: Phase Based video MOtion MAgnification 2 | 3 | A Python source code implementation of motion magnification based on the paper: [Phase Based Video Motion Processing](http://people.csail.mit.edu/mrub/papers/phasevid-siggraph13.pdf) by Neal Wadhwa, Michael Rubinstein, Frédo Durand, William T. Freeman, ACM Transactions on Graphics, Volume 32, Number 4 (Proc. SIGGRAPH), 2013. [project](http://people.csail.mit.edu/nwadhwa/phase-video/). 4 | 5 | #### Note: this follow up code can also handle large motion https://acceleration-magnification.github.io/ 6 | 7 | 8 | ### Requirements: 9 | 10 | - python 2.7 11 | - numpy 12 | - [perceptual](https://github.com/andreydung/Steerable-filter) (Complex steerable pyramid, install with: sudo pip install perceptual) 13 | 14 | ### Organization 15 | 16 | phasebasedMoMag.py # Main file 17 | pyramid2arr.py # Help class to convert a pyramid to a 1d array 18 | media/guitar.mp4 # Example video 19 | 20 | ### Example video 21 | 22 | ./media/guitar.mp4 23 | 24 | When you run the code 'python phasebasedMoMag.py' it expects an example video in the 'media' folder. Here we use the [http://people.csail.mit.edu/mrub/evm/video/guitar.mp4](guitar.mp4) video from the motion magnification website. 25 | 26 | 27 | ### About 28 | 29 | The pbMoMA implementation is based only on the paper. It was developed independent of the source code that can be requested from the paper authors (this pyton code was written without having access to that code). Therefore, the results from the pbMoMA code may differ from the results by the paper authors. Differences include: using a sliding window, only an Ideal filter, no sub-octave pyramid, and no color. 30 | 31 | The code was implemented during the [Lorentz Center](http://www.lorentzcenter.nl/) workshop [ICT with Industry: motion microscope](http://www.lorentzcenter.nl/lc/web/2015/775/info.php3?wsid=775&venue=Oort). Participants: Joao Bastos, Elsbeth van Dam, Coert van Gemeren, Jan van Gemert, Amogh Gudi, Julian Kooij, Malte Lorbach, Claudio Martella, Ronald Poppe. 32 | 33 | -------------------------------------------------------------------------------- /pyr2arr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Dec 21 18:19:12 2015 4 | 5 | @author: jkooij 6 | """ 7 | 8 | import numpy as np 9 | 10 | class Pyramid2arr: 11 | '''Class for converting a pyramid to/from a 1d array''' 12 | 13 | def __init__(self, steer, coeff=None): 14 | """ 15 | Initialize class with sizes from pyramid coeff 16 | """ 17 | self.levels = range(1, steer.height-1) 18 | self.bands = range(steer.nbands) 19 | 20 | self._indices = None 21 | if coeff is not None: 22 | self.init_coeff(coeff) 23 | 24 | def init_coeff(self, coeff): 25 | shapes = [coeff[0].shape] 26 | for lvl in self.levels: 27 | for b in self.bands: 28 | shapes.append( coeff[lvl][b].shape ) 29 | shapes.append(coeff[-1].shape) 30 | 31 | # compute the total sizes 32 | sizes = [np.prod(shape) for shape in shapes] 33 | 34 | # precompute indices of each band 35 | offsets = np.cumsum([0] + sizes) 36 | self._indices = zip(offsets[:-1], offsets[1:], shapes) 37 | 38 | def p2a(self, coeff): 39 | """ 40 | Convert pyramid as a 1d Array 41 | """ 42 | 43 | if self._indices is None: 44 | self.init_coeff(coeff) 45 | 46 | bandArray = np.hstack([ np.ravel( coeff[lvl][b] ) for lvl in self.levels for b in self.bands ]) 47 | bandArray = np.hstack((np.ravel(coeff[0]), bandArray, np.ravel(coeff[-1]))) 48 | 49 | return bandArray 50 | 51 | 52 | def a2p(self, bandArray): 53 | """ 54 | Convert 1d array back to Pyramid 55 | """ 56 | 57 | assert self._indices is not None, 'Initialize Pyramid2arr first with init_coeff() or p2a()' 58 | 59 | # create iterator that convert array to images 60 | it = (np.reshape(bandArray[istart:iend], size) for (istart,iend,size) in self._indices) 61 | 62 | coeffs = [it.next()] 63 | for lvl in self.levels: 64 | coeffs.append([it.next() for band in self.bands]) 65 | coeffs.append(it.next()) 66 | 67 | return coeffs 68 | 69 | -------------------------------------------------------------------------------- /phasebasedMoMag.py: -------------------------------------------------------------------------------- 1 | from perceptual.filterbank import * 2 | 3 | import cv2 4 | 5 | # determine what OpenCV version we are using 6 | try: 7 | import cv2.cv as cv 8 | USE_CV2 = True 9 | except ImportError: 10 | # OpenCV 3.x does not have cv2.cv submodule 11 | USE_CV2 = False 12 | 13 | import sys 14 | import numpy as np 15 | 16 | from pyr2arr import Pyramid2arr 17 | from temporal_filters import IdealFilterWindowed, ButterBandpassFilter 18 | 19 | 20 | def phaseBasedMagnify(vidFname, vidFnameOut, maxFrames, windowSize, factor, fpsForBandPass, lowFreq, highFreq): 21 | 22 | # initialize the steerable complex pyramid 23 | steer = Steerable(5) 24 | pyArr = Pyramid2arr(steer) 25 | 26 | print "Reading:", vidFname, 27 | 28 | # get vid properties 29 | vidReader = cv2.VideoCapture(vidFname) 30 | if USE_CV2: 31 | # OpenCV 2.x interface 32 | vidFrames = int(vidReader.get(cv.CV_CAP_PROP_FRAME_COUNT)) 33 | width = int(vidReader.get(cv.CV_CAP_PROP_FRAME_WIDTH)) 34 | height = int(vidReader.get(cv.CV_CAP_PROP_FRAME_HEIGHT)) 35 | fps = int(vidReader.get(cv.CV_CAP_PROP_FPS)) 36 | func_fourcc = cv.CV_FOURCC 37 | else: 38 | # OpenCV 3.x interface 39 | vidFrames = int(vidReader.get(cv2.CAP_PROP_FRAME_COUNT)) 40 | width = int(vidReader.get(cv2.CAP_PROP_FRAME_WIDTH)) 41 | height = int(vidReader.get(cv2.CAP_PROP_FRAME_HEIGHT)) 42 | fps = int(vidReader.get(cv2.CAP_PROP_FPS)) 43 | func_fourcc = cv2.VideoWriter_fourcc 44 | 45 | if np.isnan(fps): 46 | fps = 30 47 | 48 | print ' %d frames' % vidFrames, 49 | print ' (%d x %d)' % (width, height), 50 | print ' FPS:%d' % fps 51 | 52 | # video Writer 53 | fourcc = func_fourcc('M', 'J', 'P', 'G') 54 | vidWriter = cv2.VideoWriter(vidFnameOut, fourcc, int(fps), (width,height), 1) 55 | print 'Writing:', vidFnameOut 56 | 57 | # how many frames 58 | nrFrames = min(vidFrames, maxFrames) 59 | 60 | # read video 61 | #print steer.height, steer.nbands 62 | 63 | # setup temporal filter 64 | filter = IdealFilterWindowed(windowSize, lowFreq, highFreq, fps=fpsForBandPass, outfun=lambda x: x[0]) 65 | #filter = ButterBandpassFilter(1, lowFreq, highFreq, fps=fpsForBandPass) 66 | 67 | print 'FrameNr:', 68 | for frameNr in range( nrFrames + windowSize ): 69 | print frameNr, 70 | sys.stdout.flush() 71 | 72 | if frameNr < nrFrames: 73 | # read frame 74 | _, im = vidReader.read() 75 | 76 | if im is None: 77 | # if unexpected, quit 78 | break 79 | 80 | # convert to gray image 81 | if len(im.shape) > 2: 82 | grayIm = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY) 83 | else: 84 | # already a grayscale image? 85 | grayIm = im 86 | 87 | # get coeffs for pyramid 88 | coeff = steer.buildSCFpyr(grayIm) 89 | 90 | # add image pyramid to video array 91 | # NOTE: on first frame, this will init rotating array to store the pyramid coeffs 92 | arr = pyArr.p2a(coeff) 93 | 94 | 95 | phases = np.angle(arr) 96 | 97 | # add to temporal filter 98 | filter.update([phases]) 99 | 100 | # try to get filtered output to continue 101 | try: 102 | filteredPhases = filter.next() 103 | except StopIteration: 104 | continue 105 | 106 | print '*', 107 | 108 | # motion magnification 109 | magnifiedPhases = (phases - filteredPhases) + filteredPhases*factor 110 | 111 | # create new array 112 | newArr = np.abs(arr) * np.exp(magnifiedPhases * 1j) 113 | 114 | # create pyramid coeffs 115 | newCoeff = pyArr.a2p(newArr) 116 | 117 | # reconstruct pyramid 118 | out = steer.reconSCFpyr(newCoeff) 119 | 120 | # clip values out of range 121 | out[out>255] = 255 122 | out[out<0] = 0 123 | 124 | # make a RGB image 125 | rgbIm = np.empty( (out.shape[0], out.shape[1], 3 ) ) 126 | rgbIm[:,:,0] = out 127 | rgbIm[:,:,1] = out 128 | rgbIm[:,:,2] = out 129 | 130 | #write to disk 131 | res = cv2.convertScaleAbs(rgbIm) 132 | vidWriter.write(res) 133 | 134 | # free the video reader/writer 135 | vidReader.release() 136 | vidWriter.release() 137 | 138 | 139 | ################# main script 140 | 141 | #vidFname = 'media/baby.mp4'; 142 | #vidFname = 'media/WIN_20151208_17_11_27_Pro.mp4.normalized.avi' 143 | #vidFname = 'media/embryos01_30s.mp4' 144 | vidFname = 'media/guitar.mp4' 145 | 146 | # maximum nr of frames to process 147 | maxFrames = 60000 148 | # the size of the sliding window 149 | windowSize = 30 150 | # the magnifaction factor 151 | factor = 20 152 | # the fps used for the bandpass 153 | fpsForBandPass = 600 # use -1 for input video fps 154 | # low ideal filter 155 | lowFreq = 72 156 | # high ideal filter 157 | highFreq = 92 158 | # output video filename 159 | vidFnameOut = vidFname + '-Mag%dIdeal-lo%d-hi%d.avi' % (factor, lowFreq, highFreq) 160 | 161 | phaseBasedMagnify(vidFname, vidFnameOut, maxFrames, windowSize, factor, fpsForBandPass, lowFreq, highFreq) 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /temporal_filters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Dec 21 17:52:29 2015 4 | 5 | @author: jkooij 6 | """ 7 | 8 | import numpy as np 9 | import scipy.signal 10 | 11 | import scipy.fftpack as fftpack 12 | #import pyfftw.interfaces.scipy_fftpack as fftpack 13 | 14 | 15 | class SlidingWindow (object): 16 | 17 | def __init__(self, size, step=1): 18 | self.size = size 19 | self.step = step 20 | self.memory = None 21 | 22 | assert(self.step > 0) 23 | 24 | def process(self, data_itr): 25 | """ Generator for windows after giving it more data. 26 | 27 | Example: 28 | 29 | winsize = 2 30 | win = SlidingWindow(winsize) 31 | batches = (np.random.randint(0,9, 3) for _ in range(3)) 32 | for w in win.process(batches): 33 | print '<<<', w 34 | """ 35 | for data in data_itr: 36 | self.update(data) 37 | while True: 38 | try: 39 | out = self.next() 40 | yield out 41 | except StopIteration: 42 | break 43 | 44 | def update(self, data): 45 | if self.memory is None: 46 | self.memory = np.asarray(data) 47 | else: 48 | self.memory = np.concatenate((self.memory, data), axis=0) 49 | 50 | def next(self): 51 | if self.memory is not None and self.memory.shape[0] >= self.size: 52 | # get window 53 | out = self.memory[:self.size] 54 | 55 | # slide 56 | self.memory = self.memory[self.step:] 57 | 58 | return out 59 | else: 60 | raise StopIteration() 61 | 62 | def collect(self): 63 | # collect remainder of sliding windows 64 | out = [] 65 | while True: 66 | try: 67 | out.append(self.next()) 68 | except StopIteration: 69 | break 70 | return np.array(out) 71 | 72 | class IdealFilter (object): 73 | """ Implements ideal_bandpassing as in EVM_MAtlab. """ 74 | 75 | def __init__(self, wl=.5, wh=.75, fps=1, NFFT=None): 76 | """Ideal bandpass filter using FFT """ 77 | 78 | self.fps = fps 79 | self.wl = wl 80 | self.wh = wh 81 | self.NFFT = NFFT 82 | 83 | if self.NFFT is not None: 84 | self.__set_mask() 85 | 86 | def __set_mask(self): 87 | self.frequencies = fftpack.fftfreq(self.NFFT, d=1.0/self.fps) 88 | 89 | # determine what indices in Fourier transform should be set to 0 90 | self.mask = (np.abs(self.frequencies) < self.wl) | (np.abs(self.frequencies) > self.wh) 91 | 92 | 93 | def __call__(self, data, axis=0): 94 | if self.NFFT is None: 95 | self.NFFT = data.shape[0] 96 | self.__set_mask() 97 | 98 | fft = fftpack.fft(data, axis=axis) 99 | fft[self.mask] = 0 100 | return np.real( fftpack.ifft(fft, axis=axis) ) 101 | 102 | class IdealFilterWindowed (SlidingWindow): 103 | 104 | def __init__(self, winsize, wl=.5, wh=.75, fps=1, step=1, outfun=None): 105 | SlidingWindow.__init__(self, winsize, step) 106 | self.filter = IdealFilter(wl, wh, fps=fps, NFFT=winsize) 107 | self.outfun = outfun 108 | 109 | def next(self): 110 | out = SlidingWindow.next(self) 111 | out = self.filter(out) 112 | if self.outfun is not None: 113 | # apply output function, e.g. to return first (most recent) item 114 | out = self.outfun(out) 115 | return out 116 | 117 | 118 | class IIRFilter (SlidingWindow): 119 | """ 120 | Implements the IIR filter 121 | a[0]*y[n] = b[0]*x[n] + b[1]*x[n-1] + ... + b[nb]*x[n-nb] 122 | - a[1]*y[n-1] - ... - a[na]*y[n-na] 123 | See scipy.signal.lfilter 124 | """ 125 | 126 | def __init__(self, b, a): 127 | 128 | self.b = b 129 | self.a = a 130 | self.nb = len(b) 131 | self.na = len(a) 132 | 133 | # put parameters in right order for calculation 134 | # (i.e. parameter of most recent time step last) 135 | self.b_ = b[::-1] 136 | self.a_ = a[-1:0:-1] # exclude a[0], it's used to scale output 137 | 138 | # setup sliding windows for input x and output y 139 | self.windowy = SlidingWindow(self.na-1) 140 | SlidingWindow.__init__(self, self.nb) 141 | 142 | def update(self, data): 143 | if self.memory is None: 144 | # prepend zeros 145 | data = np.asarray(data) 146 | zsize = (self.nb-1,) + data.shape[1:] 147 | data = np.concatenate((np.zeros(zsize), data), axis=0) 148 | 149 | # initialize output memory with zerostoo 150 | zsize = (self.na-1,) + data.shape[1:] 151 | self.windowy.update(np.zeros(zsize)) 152 | 153 | SlidingWindow.update(self, data) 154 | 155 | def next(self): 156 | winx = SlidingWindow.next(self) 157 | winy = self.windowy.next() 158 | y = np.dot(self.b_, winx) - np.dot(self.a_, winy) 159 | 160 | self.windowy.update([y]) 161 | 162 | return y / self.a[0] 163 | 164 | 165 | class ButterFilter (IIRFilter): 166 | def __init__(self, n, freq, fps=1, btype='low'): 167 | freq = float(freq) / fps 168 | (b,a) = scipy.signal.butter(n, freq, btype) 169 | IIRFilter.__init__(self, b, a) 170 | 171 | 172 | class ButterBandpassFilter (ButterFilter): 173 | 174 | def __init__(self, n, freq_low=.25, freq_high=.5, fps=1): 175 | ButterFilter.__init__(self, n, freq_high, fps=fps, btype='low') 176 | 177 | # additional low-pass 178 | self.lowpass = ButterFilter(n, freq_low, fps=fps, btype='low') 179 | 180 | def update(self, data): 181 | ButterFilter.update(self, data) 182 | self.lowpass.update(data) 183 | 184 | def next(self): 185 | out = ButterFilter.next(self) 186 | out_low = self.lowpass.next() 187 | return (out - out_low) 188 | --------------------------------------------------------------------------------