├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── image_files ├── forest │ ├── forest.bmp │ ├── forest_mask.png │ └── forest_pruned.bmp └── inpainting │ ├── image │ ├── image_00000.png │ ├── image_00001.png │ └── image_00002.png │ ├── mask │ ├── mask_00000.png │ ├── mask_00001.png │ └── mask_00002.png │ ├── masked_image │ ├── masked_image_00000.png │ ├── masked_image_00001.png │ └── masked_image_00002.png │ ├── metrics.log │ ├── output │ ├── output_00000.png │ ├── output_00001.png │ └── output_00002.png │ ├── prepare.py │ └── py_wrapper.py ├── include ├── defineall.h ├── inpaint.h ├── maskedimage.h ├── nearestneighborfield.h ├── qualitymesures.h └── structdef.h ├── man └── man1 │ └── patchmatch.1 ├── run.sh └── src ├── inpaint.cpp ├── main.cpp ├── maskedimage.cpp ├── nearestneighborfield.cpp └── qualitymesures.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | 3 | *.orig 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8.3) 2 | PROJECT(deploy_tvm) 3 | 4 | SET(CMAKE_BUILD_TYPE Release) 5 | set(CMAKE_C_COMPILER "/usr/bin/gcc") 6 | SET(CMAKE_C_FLAGS ${CMAKE_C_FLAGS} "-std=c99") 7 | SET(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} "-std=c++0x") 8 | 9 | FIND_PACKAGE(OpenCV 3 REQUIRED) 10 | 11 | 12 | # include and libs path 13 | SET(HOME /home/pzq) 14 | SET(OpenCV_INCLUDE_DIRS ${OpenCV_INCLUDE_DIRS} /home/pzq/local/include) 15 | SET(OpenCV_LIBRARY_DIRS ${OpenCV_LIBRARY_DIRS} /home/pzq/local/lib) 16 | 17 | SET(INCLUDE_PATH ./include /usr/include /usr/local/include 18 | ${OpenCV_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS}/opencv) 19 | SET(LINK_PATH ./lib /usr/local/lib 20 | ${OpenCV_LIBRARY_DIRS}) 21 | 22 | INCLUDE_DIRECTORIES(${INCLUDE_PATH}) 23 | LINK_DIRECTORIES(${LINK_PATH}) 24 | 25 | 26 | # get executable file 27 | ADD_EXECUTABLE(patchmatch ./src/main.cpp ./src/inpaint.cpp ./src/maskedimage.cpp ./src/nearestneighborfield.cpp ./src/qualitymesures.cpp) 28 | TARGET_LINK_LIBRARIES(patchmatch opencv_core opencv_imgproc opencv_highgui) 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Public Domain Mark 1.0 2 | No Copyright 3 | 4 | This work has been identified as being free of known restrictions 5 | under copyright law, including all related and neighboring rights. 6 | 7 | You can copy, modify, distribute and perform the work, even for 8 | commercial purposes, all without asking permission. See Other 9 | Information below. 10 | 11 | Other Information 12 | 13 | The work may not be free of known copyright restrictions in all 14 | jurisdictions. 15 | 16 | Persons may have other rights in or related to the work, such as 17 | patent or trademark rights, and others may have rights in how the 18 | work is used, such as publicity or privacy rights. 19 | 20 | In some jurisdictions moral rights of the author may persist beyond 21 | the term of copyright. These rights may include the right to be 22 | identified as the author and the right to object to derogatory 23 | treatments. 24 | 25 | Unless expressly stated otherwise, the person who identified the work 26 | makes no warranties about the work, and disclaims liability for all 27 | uses of the work, to the fullest extent permitted by applicable law. 28 | 29 | When using or citing the work, you should not imply endorsement by 30 | the author or the person who identified the work. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PatchMatch for inpainting 2 | 3 | ![](https://img.shields.io/badge/build-passing-green) ![](https://img.shields.io/badge/opencv-3.x-green) ![](https://img.shields.io/badge/cmake-%3E2.8-orange) ![](https://img.shields.io/badge/Lisence-MIT-yellow) 4 | 5 | ## Introduction 6 | 7 | This repository borrows most of the code from [younesse-cv](https://github.com/younesse-cv/PatchMatch). However that repository is C style, which cannot be compiled using `opencv 3.x`. What I did is wrapping the code using `opencv 3.x` API in C++ style. Thanks [zvezdochiot](https://github.com/zvezdochiot) for suggestion. 8 | 9 | 10 | 11 | ## Dependencies 12 | 13 | - cmake > 2.8 14 | - opencv 3.x 15 | - g++-4.7 16 | 17 | 18 | 19 | ## How to use 20 | 21 | - Download this repository 22 | 23 | ```bash 24 | git clone https://github.com/ZQPei/patchmatch_inpainting.git 25 | ``` 26 | 27 | - Compile 28 | 29 | ```bash 30 | mkdir build 31 | cd build 32 | cmake .. 33 | make 34 | cd .. 35 | ``` 36 | 37 | - Run 38 | As what does in `run.sh`. 39 | 40 | ```bash 41 | for i in 0 1 2; do 42 | ./build/patchmatch image_files/inpainting/image/image_0000$i.png \ 43 | image_files/inpainting/mask/mask_0000$i.png \ 44 | image_files/inpainting/output/output_0000$i.png \ 45 | image_files/inpainting/metrics.log \ 46 | $i; 47 | done 48 | ``` 49 | 50 | Or simple: 51 | ```sh 52 | ./build/patchmatch image_files/forest/forest.bmp \ 53 | image_files/forest/forest_mask.png \ 54 | image_files/forest/forest_inpaint.png 55 | ``` 56 | 57 | 58 | ## Demo 59 | 60 | | mask | masked image | inpainting image using PatchMatch | origin image | 61 | | ----------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------ | 62 | | ![](./image_files/inpainting/mask/mask_00001.png) | ![](./image_files/inpainting/masked_image/masked_image_00001.png) | ![](./image_files/inpainting/output/output_00001.png) | ![](./image_files/inpainting/image/image_00001.png) | 63 | 64 | 65 | 66 | 67 | 68 | 69 | ## References 70 | 71 | https://github.com/younesse-cv/PatchMatch 72 | 73 | -------------------------------------------------------------------------------- /image_files/forest/forest.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/forest/forest.bmp -------------------------------------------------------------------------------- /image_files/forest/forest_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/forest/forest_mask.png -------------------------------------------------------------------------------- /image_files/forest/forest_pruned.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/forest/forest_pruned.bmp -------------------------------------------------------------------------------- /image_files/inpainting/image/image_00000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/image/image_00000.png -------------------------------------------------------------------------------- /image_files/inpainting/image/image_00001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/image/image_00001.png -------------------------------------------------------------------------------- /image_files/inpainting/image/image_00002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/image/image_00002.png -------------------------------------------------------------------------------- /image_files/inpainting/mask/mask_00000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/mask/mask_00000.png -------------------------------------------------------------------------------- /image_files/inpainting/mask/mask_00001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/mask/mask_00001.png -------------------------------------------------------------------------------- /image_files/inpainting/mask/mask_00002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/mask/mask_00002.png -------------------------------------------------------------------------------- /image_files/inpainting/masked_image/masked_image_00000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/masked_image/masked_image_00000.png -------------------------------------------------------------------------------- /image_files/inpainting/masked_image/masked_image_00001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/masked_image/masked_image_00001.png -------------------------------------------------------------------------------- /image_files/inpainting/masked_image/masked_image_00002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/masked_image/masked_image_00002.png -------------------------------------------------------------------------------- /image_files/inpainting/metrics.log: -------------------------------------------------------------------------------- 1 | 0 27.168239 0.964703 0.000000 2 | 1 30.166819 0.966201 0.000000 3 | 2 28.294660 0.967397 0.000000 4 | 0 27.177405 0.965692 0.000000 5 | 1 29.551508 0.963841 0.000000 6 | 2 28.025298 0.968335 0.000000 7 | -------------------------------------------------------------------------------- /image_files/inpainting/output/output_00000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/output/output_00000.png -------------------------------------------------------------------------------- /image_files/inpainting/output/output_00001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/output/output_00001.png -------------------------------------------------------------------------------- /image_files/inpainting/output/output_00002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZQPei/patchmatch_inpainting/21a4d499a5632679203268d356ac29a38c0a4d27/image_files/inpainting/output/output_00002.png -------------------------------------------------------------------------------- /image_files/inpainting/prepare.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import PIL.Image as Image 4 | import random 5 | 6 | 7 | os.makedirs("image", exist_ok=True) 8 | os.makedirs("mask", exist_ok=True) 9 | os.makedirs("masked_image", exist_ok=True) 10 | os.makedirs("output", exist_ok=True) 11 | 12 | 13 | image_filst = [os.path.join("./image", fn) for fn in os.listdir("./image")] 14 | image_filst.sort() 15 | 16 | mask_filst = [os.path.join("./mask", fn) for fn in os.listdir("./mask")] 17 | mask_filst.sort() 18 | 19 | for idx in range(3): 20 | image_path = image_filst[idx] 21 | mask_path = mask_filst[idx] 22 | 23 | image = Image.open(image_path) 24 | image = image.resize((256,256)) 25 | image = image.convert("RGB") 26 | mask = Image.open(mask_path) 27 | mask = mask.resize((256,256)) 28 | mask = mask.convert("RGB") 29 | masked_image = Image.fromarray(np.maximum(np.array(image), np.array(mask))) 30 | 31 | print("\rProgress %05d/%05d"%(idx+1, len(image_filst)), end="") 32 | image.save("./image/image_%05d.png"%idx) 33 | mask.save("./mask/mask_%05d.png"%idx) 34 | masked_image.save("./masked_image/masked_image_%05d.png"%idx) 35 | 36 | -------------------------------------------------------------------------------- /image_files/inpainting/py_wrapper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from glob import glob 4 | 5 | def load_flist(flist, substr='', ext=['.jpg', '.png'], recursive=False): 6 | if isinstance(flist, list): 7 | return flist 8 | 9 | if isinstance(flist, np.ndarray): 10 | return flist 11 | 12 | # flist: image file path, image directory path, text file flist path 13 | if isinstance(flist, str): 14 | if os.path.isdir(flist): 15 | if not recursive: 16 | _flist = flist 17 | flist = [] 18 | for _ext in ext: 19 | flist += [os.path.abspath(fn) for fn in glob(_flist + '/*%s' %(_ext)) if substr in os.path.basename(fn)] 20 | else: 21 | walk = os.walk(flist) 22 | flist = [] 23 | for parentname, _, filelist in walk: 24 | flist += [os.path.abspath(os.path.join(parentname, fn)) for fn in filelist if os.path.splitext(fn)[-1].lower() in ext and substr in fn] 25 | 26 | flist.sort() 27 | return flist 28 | 29 | if os.path.isfile(flist): 30 | try: 31 | flist = np.genfromtxt(flist, dtype=np.str, encoding='utf-8') 32 | if flist.ndim > 1 : 33 | flist = flist[:, 0] 34 | return flist.tolist() 35 | except: 36 | return [flist] 37 | 38 | return [] 39 | 40 | 41 | if __name__ == "__main__": 42 | size = 256 43 | 44 | image_root = "images" 45 | mask_root = "masks" 46 | output_root = "output" 47 | time_log = "test_time.log" 48 | 49 | # flist = np.genfromtxt("./image_files/inpainting/places2_testlist_large", dtype=np.str, encoding="utf-8") 50 | # image_flist = [os.path.join(image_root, os.path.basename(fn)) for fn in flist[:,0]] 51 | # mask_flist = [os.path.join(mask_root, "mask_"+os.path.basename(fn)) for fn in flist[:,1]] 52 | 53 | image_flist = load_flist(image_root) 54 | mask_flist = load_flist(mask_root) 55 | 56 | # import ipdb; ipdb.set_trace() 57 | 58 | for i in range(len(image_flist)): 59 | print("\rprogress: %d/%d"%(i, len(image_flist)), end="") 60 | image_path = image_flist[i] 61 | mask_path = mask_flist[i] 62 | basename = os.path.basename(image_path)[:-4] 63 | output_path = os.path.join(output_root, "%s.png"%basename) 64 | log_path = "./metrics.log" 65 | cmd = "../../build/patchmatch %s %s %s %s %05d >> %s"%(image_path, mask_path, output_path, log_path, i, time_log) 66 | #print(cmd) 67 | 68 | # import ipdb; ipdb.set_trace() 69 | os.system(cmd) -------------------------------------------------------------------------------- /include/defineall.h: -------------------------------------------------------------------------------- 1 | #ifndef __GLOBAL_INCLUDE_PATCHMATCH_H__ 2 | #define __GLOBAL_INCLUDE_PATCHMATCH_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "structdef.h" 13 | #include "inpaint.h" 14 | #include "maskedimage.h" 15 | #include "nearestneighborfield.h" 16 | #include "qualitymesures.h" 17 | 18 | double max1(double a, double b); 19 | double min1(double a, double b); 20 | 21 | //#define VERBOSE 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /include/inpaint.h: -------------------------------------------------------------------------------- 1 | #ifndef __INPAINT_H_ 2 | #define __INPAINT_H_ 3 | #include "defineall.h" 4 | 5 | Inpaint_P initInpaint(); 6 | 7 | void addEltInpaintingPyramid(Inpaint_P imp, MaskedImage_P elt); 8 | MaskedImage_P ExpectationMaximization(Inpaint_P imp, int level); 9 | void ExpectationStep(NNF_P nnf, int sourceToTarget, double*** vote, MaskedImage_P source, int upscale); 10 | void weightedCopy(MaskedImage_P src, int xs, int ys, double*** vote, int xd, int yd, double w); 11 | void MaximizationStep(MaskedImage_P target, double*** vote); 12 | IplImage* inpaint(Inpaint_P imp, IplImage* input, int ** mask, int radius); 13 | void freeInpaintingPyramid(Inpaint_P imp); 14 | 15 | // Variables globales 16 | extern double* G_globalSimilarity; 17 | extern int G_initSim; 18 | #endif 19 | -------------------------------------------------------------------------------- /include/maskedimage.h: -------------------------------------------------------------------------------- 1 | #ifndef __MASKED_IMAGE_H_ 2 | #define __MASKED_IMAGE_H_ 3 | #include "defineall.h" 4 | 5 | void initSimilarity(); 6 | MaskedImage_P initMaskedImage(IplImage* image, int** mask); 7 | MaskedImage_P initNewMaskedImage(int width, int height); 8 | void freeMaskedImage(MaskedImage_P mIm); 9 | int isMasked(MaskedImage_P mIm, int x, int y); 10 | void setMask(MaskedImage_P mIm, int x, int y, int value); 11 | int constainsMasked(MaskedImage_P mIm, int x, int y, int S); 12 | int distanceMaskedImage(MaskedImage_P source,int xs,int ys, MaskedImage_P target,int xt,int yt, int S); 13 | MaskedImage_P downsample(MaskedImage_P source); 14 | MaskedImage_P upscale(MaskedImage_P source, int newW,int newH); 15 | MaskedImage_P copyMaskedImage(MaskedImage_P mIm); 16 | 17 | int getSampleMaskedImage(MaskedImage_P mIm, int x, int y, int band);//////////// 18 | void setSampleMaskedImage(MaskedImage_P mIm, int x, int y, int band, int value);///////////// 19 | 20 | 21 | // Variables globales 22 | extern double* G_globalSimilarity; 23 | extern int G_initSim; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /include/nearestneighborfield.h: -------------------------------------------------------------------------------- 1 | #ifndef __NNF_H_ 2 | #define __NNF_H_ 3 | #include "defineall.h" 4 | 5 | NNF_P initNNF(MaskedImage_P input, MaskedImage_P output, int patchsize); 6 | void allocNNFField(NNF_P nnf); 7 | void freeNNFField(NNF_P nnf); 8 | void freeNNF(NNF_P nnf); 9 | void randomize(NNF_P nnf); 10 | void initializeNNFFromOtherNNF(NNF_P nnf, NNF_P otherNnf); 11 | void initializeNNF(NNF_P nnf); 12 | void minimizeNNF(NNF_P nnf, int pass); 13 | void minimizeLinkNNF(NNF_P nnf, int x, int y, int dir); 14 | int distanceNNF(NNF_P nnf, int x, int y, int xp,int yp); 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /include/qualitymesures.h: -------------------------------------------------------------------------------- 1 | #ifndef __quality_mesure_H_ 2 | #define __quality_mesure_H_ 3 | #include "defineall.h" 4 | 5 | double PSNR(IplImage *original,IplImage *distorted); 6 | double SSIM(IplImage *original,IplImage *distorted); 7 | #endif 8 | -------------------------------------------------------------------------------- /include/structdef.h: -------------------------------------------------------------------------------- 1 | #ifndef __STRUCT_PATCHMATCH_H_ 2 | #define __STRUCT_PATCHMATCH_H_ 3 | 4 | // the maximum value returned by MaskedImage.distance() 5 | #define DSCALE 65535 6 | #define INCREASE_PYRAMID_SIZE_RATE 2 7 | 8 | 9 | typedef struct 10 | { 11 | // image data 12 | int ** mask; 13 | IplImage* image; 14 | 15 | // array for converting distance to similarity 16 | double * similarity; 17 | 18 | int isNew; 19 | } MaskedImage_T; 20 | 21 | typedef MaskedImage_T* MaskedImage_P; 22 | 23 | typedef struct 24 | { 25 | // image 26 | MaskedImage_P input; 27 | MaskedImage_P output; 28 | 29 | // patch size 30 | int S; 31 | 32 | // Nearest-Neighbor Field 1 pixel = { x_target, y_target, distance_scaled } 33 | int *** field; 34 | int fieldH; 35 | int fieldW; 36 | } NNF_T; 37 | typedef NNF_T* NNF_P; 38 | 39 | typedef struct 40 | { 41 | //initial image 42 | MaskedImage_P initial; 43 | 44 | // Nearest-Neighbor Fields 45 | NNF_P nnf_SourceToTarget; 46 | NNF_P nnf_TargetToSource; 47 | 48 | // patch radius 49 | int radius; 50 | 51 | // Pyramid of downsampled initial images 52 | MaskedImage_P* pyramid; 53 | int nbEltPyramid; 54 | int nbEltMaxPyramid; 55 | } Inpaint_T; 56 | 57 | typedef Inpaint_T* Inpaint_P; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /man/man1/patchmatch.1: -------------------------------------------------------------------------------- 1 | .TH "PatchMatch" 1 "05 Apr 2020" 0.2 " User Manual" 2 | 3 | .SH NAME 4 | patchmatch 5 | 6 | .SH SYNOPSIS 7 | \fBpatchmatch\fR \fI[image_path] [mask_path] [output_path] [log_path] [misc]\fR 8 | \fBpatchmatch\fR \fI[image] [mask] [output]\fR 9 | 10 | .SH DESCRIPTION 11 | PatchMatch for inpainting using OpenCV 3.x 12 | 13 | .SH EXAMPLE 14 | .TP 15 | log mode: 16 | \fIfor i in $(seq 0 2); do \fBpatchmatch\fI image_files/inpainting/image/image_0000$i.png image_files/inpainting/mask/mask_0000$i.png image_files/inpainting/output/output_0000$i.png image_files/inpainting/metrics.log $i; done\fR 17 | .TP 18 | simple: 19 | \fBpatchmatch\fI image_files/forest/forest.bmp image_files/forest/forest_mask.png image_files/forest/forest_inpaint.png\fR 20 | 21 | .SH COPYRIGHT 22 | Public Domain Mark 1.0 23 | No Copyright 24 | 25 | .SH SEE ALSO 26 | convert(1), 27 | inpaint(1) 28 | 29 | .SH CONTACTS 30 | Homepage: https://github.com/ZQPei/patchmatch_inpainting 31 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | # build 2 | 3 | if [ ! -d "./build" ];then 4 | mkdir build 5 | fi 6 | 7 | cd build 8 | cmake .. 9 | make -j8 10 | cd .. 11 | 12 | 13 | # run and test on some local cases 14 | for i in $(seq 0 2); do 15 | ./build/patchmatch image_files/inpainting/image/image_0000$i.png \ 16 | image_files/inpainting/mask/mask_0000$i.png \ 17 | image_files/inpainting/output/output_0000$i.png \ 18 | image_files/inpainting/metrics.log \ 19 | $i; 20 | done -------------------------------------------------------------------------------- /src/inpaint.cpp: -------------------------------------------------------------------------------- 1 | #include "inpaint.h" 2 | 3 | 4 | /** This algorithme uses a version proposed by Xavier Philippeau 5 | */ 6 | 7 | Inpaint_P initInpaint() 8 | { 9 | Inpaint_P inp = (Inpaint_P)malloc(sizeof(Inpaint_T)); 10 | //initial image 11 | inp->initial = NULL; 12 | 13 | // Nearest-Neighbor Fields 14 | inp->nnf_SourceToTarget = NULL; 15 | inp->nnf_TargetToSource = NULL; 16 | 17 | // Pyramid of downsampled initial images 18 | inp->pyramid = NULL; 19 | inp->nbEltPyramid = 0; 20 | inp->nbEltMaxPyramid = 0; 21 | 22 | return inp; 23 | } 24 | 25 | void addEltInpaintingPyramid(Inpaint_P imp, MaskedImage_P elt) 26 | { 27 | int inc = INCREASE_PYRAMID_SIZE_RATE; 28 | if (inc<2) 29 | inc = 2; 30 | 31 | if (imp->pyramid == NULL || imp->nbEltMaxPyramid == 0) 32 | { 33 | imp->nbEltMaxPyramid = inc; 34 | imp->pyramid = (MaskedImage_P*)malloc(sizeof(MaskedImage_P)*imp->nbEltMaxPyramid); 35 | } 36 | else if (imp->nbEltPyramid == imp->nbEltMaxPyramid) 37 | { 38 | imp->nbEltMaxPyramid = imp->nbEltMaxPyramid*inc; 39 | imp->pyramid = (MaskedImage_P*)realloc(imp->pyramid, sizeof(MaskedImage_P)*imp->nbEltMaxPyramid); 40 | } 41 | 42 | imp->pyramid[imp->nbEltPyramid] = elt; 43 | imp->nbEltPyramid++; 44 | } 45 | 46 | // EM-Like algorithm (see "PatchMatch" - page 6) 47 | // Returns a double sized target image 48 | MaskedImage_P ExpectationMaximization(Inpaint_P imp, int level) 49 | { 50 | int emloop, x, y, H, W, i, j; 51 | double*** vote; 52 | 53 | int iterEM = 1+2*level; 54 | int iterNNF = int(min1(7,1+level)); 55 | 56 | int upscaled; 57 | MaskedImage_P newsource; 58 | MaskedImage_P source = imp->nnf_SourceToTarget->input; 59 | MaskedImage_P target = imp->nnf_SourceToTarget->output; 60 | MaskedImage_P newtarget = NULL; 61 | 62 | CvSize size; 63 | 64 | #ifdef VERBOSE 65 | printf("EM loop (em=%d,nnf=%d) : ", iterEM, iterNNF); 66 | #endif 67 | 68 | // EM Loop 69 | for (emloop=1; emloop<=iterEM; emloop++) 70 | { 71 | #ifdef VERBOSE 72 | printf(" %d", 1+iterEM-emloop); 73 | #endif 74 | // set the new target as current target 75 | if (newtarget!=NULL) 76 | { 77 | imp->nnf_SourceToTarget->output = newtarget; 78 | imp->nnf_TargetToSource->input = newtarget; 79 | target = newtarget; 80 | newtarget = NULL; 81 | } 82 | // -- add constraint to the NNF 83 | H = source->image->height; 84 | W = source->image->width; 85 | // we force the link between unmasked patch in source/target 86 | for ( x=0 ; xradius)) 89 | { 90 | imp->nnf_SourceToTarget->field[x][y][0] = x; 91 | imp->nnf_SourceToTarget->field[x][y][1] = y; 92 | imp->nnf_SourceToTarget->field[x][y][2] = 0; 93 | } 94 | 95 | H = target->image->height; 96 | W = target->image->width; 97 | for ( x=0 ; xradius)) 100 | { 101 | imp->nnf_TargetToSource->field[x][y][0] = x; 102 | imp->nnf_TargetToSource->field[x][y][1] = y; 103 | imp->nnf_TargetToSource->field[x][y][2] = 0; 104 | } 105 | // -- minimize the NNF 106 | minimizeNNF(imp->nnf_SourceToTarget, iterNNF); 107 | minimizeNNF(imp->nnf_TargetToSource, iterNNF); 108 | 109 | // -- Now we rebuild the target using best patches from source 110 | 111 | upscaled = 0; 112 | 113 | // Instead of upsizing the final target, we build the last target from the next level source image 114 | // So the final target is less blurry (see "Space-Time Video Completion" - page 5) 115 | if (level>=1 && (emloop==iterEM)) 116 | { 117 | newsource = imp->pyramid[level-1]; 118 | newtarget = upscale(target, newsource->image->width,newsource->image->height); 119 | upscaled = 1; 120 | } 121 | else 122 | { 123 | newsource = imp->pyramid[level]; 124 | newtarget = copyMaskedImage(target); 125 | upscaled = 0; 126 | } 127 | 128 | // --- EXPECTATION STEP --- 129 | 130 | // votes for best patch from NNF Source->Target (completeness) and Target->Source (coherence) 131 | 132 | vote = (double ***)malloc(newtarget->image->height*sizeof(double **)); 133 | for ( i=0 ; iimage->height ; ++i ) 134 | { 135 | vote[i] = (double **)malloc(newtarget->image->width*sizeof(double *)); 136 | for ( j=0 ; jimage->width ; ++j) 137 | { 138 | vote[i][j] = (double *)calloc(4, sizeof(double)); 139 | } 140 | } 141 | 142 | ExpectationStep(imp->nnf_SourceToTarget, 1, vote, newsource, upscaled); 143 | ExpectationStep(imp->nnf_TargetToSource, 0, vote, newsource, upscaled); 144 | 145 | // --- MAXIMIZATION STEP --- 146 | 147 | // compile votes and update pixel values 148 | MaximizationStep(newtarget, vote); 149 | 150 | size = cvSize(imp->initial->image->width,imp->initial->image->height); 151 | IplImage* result=cvCreateImage(size,IPL_DEPTH_8U,3); 152 | 153 | cvResize(newtarget->image,result,CV_INTER_LINEAR); 154 | 155 | for (i=0; iimage->height; i++) 156 | { 157 | for (j=0; jimage->width; j++) 158 | free(vote[i][j]); 159 | 160 | free(vote[i]); 161 | } 162 | free(vote); 163 | } 164 | #ifdef VERBOSE 165 | printf("\n"); 166 | #endif 167 | 168 | return newtarget; 169 | } 170 | 171 | 172 | // Expectation Step : vote for best estimations of each pixel 173 | void ExpectationStep(NNF_P nnf, int sourceToTarget, double*** vote, MaskedImage_P source, int upscale) 174 | { 175 | int y, x, H, W, Ho, Wo, xp, yp, dp, dy, dx; 176 | int xs,ys,xt,yt; 177 | int*** field = nnf->field; 178 | int R = nnf->S; /////////////int R = nnf->PatchSize; 179 | double w; 180 | 181 | H = nnf->input->image->height; 182 | W = nnf->input->image->width; 183 | Ho = nnf->output->image->height; 184 | Wo = nnf->output->image->width; 185 | for ( x=0 ; x=H) continue; 222 | if (ys<0 || ys>=W) continue; 223 | if (xt<0 || xt>=Ho) continue; 224 | if (yt<0 || yt>=Wo) continue; 225 | 226 | // add vote for the value 227 | if (upscale) 228 | { 229 | weightedCopy(source, 2*xs, 2*ys, vote, 2*xt, 2*yt, w); 230 | weightedCopy(source, 2*xs+1, 2*ys, vote, 2*xt+1, 2*yt, w); 231 | weightedCopy(source, 2*xs, 2*ys+1, vote, 2*xt, 2*yt+1, w); 232 | weightedCopy(source, 2*xs+1, 2*ys+1, vote, 2*xt+1, 2*yt+1, w); 233 | } 234 | else 235 | { 236 | weightedCopy(source, xs, ys, vote, xt, yt, w); 237 | } 238 | } 239 | } 240 | } 241 | } 242 | } 243 | 244 | void weightedCopy(MaskedImage_P src, int xs, int ys, double*** vote, int xd,int yd, double w) 245 | { 246 | if (isMasked(src, xs, ys)) 247 | return; 248 | 249 | vote[xd][yd][0] += w*getSampleMaskedImage(src, xs, ys, 0); 250 | vote[xd][yd][1] += w*getSampleMaskedImage(src, xs, ys, 1); 251 | vote[xd][yd][2] += w*getSampleMaskedImage(src, xs, ys, 2); 252 | vote[xd][yd][3] += w; 253 | } 254 | 255 | 256 | // Maximization Step : Maximum likelihood of target pixel 257 | void MaximizationStep(MaskedImage_P target, double*** vote) 258 | { 259 | int y, x, H, W, r, g, b; 260 | H = target->image->height; 261 | W = target->image->width; 262 | for( x=0 ; x0) 267 | { 268 | r = (int) (vote[x][y][0]/vote[x][y][3]); 269 | g = (int) (vote[x][y][1]/vote[x][y][3]); 270 | b = (int) (vote[x][y][2]/vote[x][y][3]); 271 | 272 | setSampleMaskedImage(target, x, y, 0, r ); 273 | setSampleMaskedImage(target, x, y, 1, g ); 274 | setSampleMaskedImage(target, x, y, 2, b ); 275 | setMask(target, x, y, 0); 276 | } 277 | 278 | 279 | } 280 | } 281 | } 282 | IplImage* inpaint(Inpaint_P imp, IplImage* input, int ** mask, int radius) 283 | { 284 | int level, y, x; 285 | NNF_P new_nnf, new_nnf_rev; 286 | // initial image 287 | imp->initial = initMaskedImage(input, mask); 288 | IplImage* tmp = NULL; 289 | CvSize size; 290 | size = cvSize(imp->initial->image->width,imp->initial->image->height); 291 | 292 | // patch radius 293 | imp->radius = radius; 294 | 295 | // working copies 296 | MaskedImage_P source = imp->initial; 297 | MaskedImage_P target = NULL; 298 | #ifdef VERBOSE 299 | printf("Build pyramid of images...\n"); 300 | #endif 301 | 302 | // build pyramid of downscaled images 303 | addEltInpaintingPyramid(imp, source); 304 | while (source->image->width>radius && source->image->height>radius) 305 | { 306 | source = downsample(source); 307 | addEltInpaintingPyramid(imp, source); 308 | } 309 | int maxlevel=imp->nbEltPyramid; 310 | 311 | // for each level of the pyramid 312 | #ifdef VERBOSE 313 | cv::namedWindow("Progression", CV_WINDOW_AUTOSIZE); 314 | #endif 315 | for (level=maxlevel-1 ; level>0 ; level--) 316 | { 317 | #ifdef VERBOSE 318 | printf( "\n*** Processing - Zoom 1:%d ***" , 1<pyramid[level]; 323 | 324 | #ifdef VERBOSE 325 | printf("initialize NNF..."); 326 | #endif 327 | if (level==maxlevel-1) 328 | { 329 | // at first, we use the same image for target and source 330 | // and use random data as initial guess 331 | target = copyMaskedImage(source); 332 | 333 | // we consider that the target contains no masked pixels in the firt time 334 | 335 | for( x = 0 ; x < target -> image -> height ; x++ ) 336 | for( y = 0 ; y < target -> image -> width ; y++ ) 337 | setMask(target, x, y, 0); 338 | 339 | imp->nnf_SourceToTarget = initNNF(source, target, radius); 340 | randomize(imp->nnf_SourceToTarget); 341 | 342 | imp->nnf_TargetToSource = initNNF(target, source, radius); 343 | randomize(imp->nnf_TargetToSource); 344 | 345 | } 346 | else 347 | { 348 | // then, we use the rebuilt (upscaled) target 349 | // and re-use the previous NNF as initial guess 350 | new_nnf = initNNF(source, target, radius); 351 | initializeNNFFromOtherNNF(new_nnf, imp->nnf_SourceToTarget); 352 | imp->nnf_SourceToTarget = new_nnf; 353 | 354 | new_nnf_rev = initNNF(target, source, radius); 355 | initializeNNFFromOtherNNF(new_nnf_rev, imp->nnf_TargetToSource); 356 | imp->nnf_TargetToSource = new_nnf_rev; 357 | } 358 | 359 | target = ExpectationMaximization(imp, level); 360 | if (tmp != NULL ) 361 | cvReleaseImage(&tmp); 362 | 363 | tmp=cvCreateImage(size, IPL_DEPTH_8U, 3); 364 | cvResize(target->image, tmp, CV_INTER_LINEAR); 365 | 366 | #ifdef VERBOSE 367 | cv::imshow("Progression", cv::Mat(tmp)); 368 | cv::waitKey(1); 369 | #endif 370 | } 371 | 372 | cvReleaseImage(&tmp); 373 | 374 | return target->image; 375 | } 376 | 377 | void freeInpaintingPyramid(Inpaint_P inp) 378 | { 379 | int i; 380 | if (inp->pyramid != NULL) 381 | { 382 | for ( i=0 ; inbEltPyramid ; ++i) 383 | freeMaskedImage(inp->pyramid[i]); 384 | 385 | free(inp->pyramid); 386 | inp->pyramid = NULL; 387 | } 388 | } 389 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | #include "defineall.h" 2 | 3 | // Disable to measure the PSNR or the SSIM measure 4 | 5 | #ifdef VERBOSE 6 | #define DISPLAY 1 7 | #else 8 | #define DISPLAY 0 9 | #endif 10 | 11 | #define MESURES 1 12 | 13 | double* G_globalSimilarity = NULL; 14 | int G_initSim = 0; 15 | 16 | double max1(double a, double b) 17 | { 18 | return (a + b + fabs(a - b)) / 2; 19 | } 20 | 21 | double min1(double a, double b) 22 | { 23 | return (a + b - fabs(a - b)) / 2; 24 | } 25 | 26 | int process(const char *image_path, const char *mask_path, const char *output_path, double *psnr_total, double *ssim_total, double *time_total) 27 | { 28 | clock_t tic, toc; 29 | float cpu_time; /* Total CPU time in minutes */ 30 | 31 | // set random seed 32 | srand((unsigned)time(0)); 33 | 34 | //Put your file name here 35 | std::string input_image_path = image_path; 36 | std::string input_mask_path = mask_path; 37 | std::string output_image_path = output_path; 38 | 39 | cv::Mat ori_image = cv::imread(input_image_path, cv::IMREAD_COLOR); 40 | cv::Mat mask_image = cv::imread(input_mask_path, cv::IMREAD_GRAYSCALE); 41 | if ((ori_image.empty()) || (mask_image.empty())) return 1; 42 | IplImage ori_ipl_img_data = IplImage(ori_image); 43 | IplImage *ori_ipl_img = &ori_ipl_img_data; 44 | 45 | assert(ori_image.rows == mask_image.rows && ori_image.cols == mask_image.cols); 46 | 47 | int height = ori_image.rows, width = ori_image.cols; 48 | 49 | //display images 50 | if (DISPLAY) 51 | { 52 | cv::namedWindow("Original image", cv::WINDOW_AUTOSIZE); 53 | cv::imshow("Original image", ori_image); 54 | 55 | cv::namedWindow("MASK", cv::WINDOW_AUTOSIZE); 56 | cv::imshow("MASK", mask_image); 57 | } 58 | 59 | // generate mask array from mask image 60 | int channels = mask_image.channels(); 61 | int step = mask_image.cols / sizeof(uchar); 62 | int ** mask = (int **)calloc(int(height), sizeof(int*)); 63 | for (int i = 0; i < height; i++) 64 | mask[i] = (int *)calloc(int(width), sizeof(int)); 65 | 66 | printf("----------------------------------------------------------------------\n"); 67 | printf("\n"); 68 | printf("Computing, please wait, this operation may take several minutes...\n"); 69 | 70 | uchar* data = (uchar *)mask_image.data; 71 | //Timer: tic, toc 72 | tic = clock(); 73 | for (int i = 0; i < height; ++i) 74 | for (int j = 0; j < width; ++j) 75 | if (data[i*step + j*channels] == 255) 76 | mask[i][j] = 1; 77 | 78 | Inpaint_P inp = initInpaint(); 79 | IplImage * output_ipl_img = inpaint(inp, ori_ipl_img, (int**)mask, 2); 80 | if (output_ipl_img == NULL) return 1; 81 | cv::Mat output_image = cv::cvarrToMat(output_ipl_img); 82 | if (!cv::imwrite(output_image_path, output_image)) 83 | printf("/!\\/!\\/!\\/!\\/!\\/!\\Could not save the resultant image. Check the path of saving.../!\\/!\\/!\\/!\\/!\\/!\\\n"); 84 | 85 | /*---------------------------------------------- Quality Mesure ----------------------------------------------*/ 86 | // It will be useful so far to measure the quality of the recontructed areas against the original ones 87 | 88 | #if MESURES 89 | double ssim; 90 | double psnr; 91 | 92 | psnr = PSNR(ori_ipl_img, output_ipl_img); 93 | ssim = SSIM(ori_ipl_img, output_ipl_img); 94 | 95 | *psnr_total += psnr; 96 | *ssim_total += ssim; 97 | 98 | printf("Quality mesures: \n"); 99 | 100 | printf("--> PSNR=%f dB\n", psnr); 101 | printf("--> SSIM= %f \n", ssim); 102 | #endif 103 | /*------------------------------------------------------------------------------------------------------------*/ 104 | printf("DONE\n"); 105 | printf("\n"); 106 | printf("The result is saved in: %s\n", output_image_path.c_str()); 107 | toc = clock(); 108 | cpu_time = float((toc - tic) / CLOCKS_PER_SEC); 109 | 110 | *time_total += cpu_time; 111 | 112 | int seconds; 113 | int time = (int)(cpu_time / 60); 114 | seconds = int(cpu_time - time * 60); 115 | 116 | printf("The CPU time is %i minutes and %i seconds ", time, seconds); 117 | 118 | if (DISPLAY) 119 | { 120 | cv::waitKey(0); 121 | cv::destroyAllWindows(); 122 | } 123 | 124 | //free memory 125 | cvReleaseImage(& output_ipl_img); 126 | for (int i = 0; i < height; ++i) 127 | free(mask[i]); 128 | free(mask); 129 | 130 | return 0; 131 | } 132 | 133 | int main(int argc, char** argv) 134 | { 135 | 136 | if ((argc != 4) && (argc != 6)) 137 | { 138 | printf("Usage:\n"); 139 | printf("log mode: %s [image_path] [mask_path] [output_path] [log_path] [misc]\n", argv[0]); 140 | printf("simple : %s [image] [mask] [output]\n", argv[0]); 141 | exit(1); 142 | } 143 | 144 | double psnr_total = 0.0; 145 | double ssim_total = 0.0; 146 | double time_total = 0.0; 147 | 148 | process(argv[1], argv[2], argv[3], &psnr_total, &ssim_total, &time_total); 149 | 150 | if (argc == 4) 151 | { 152 | printf("average psnr: %lf\taverage ssim: %lf\taverage time: %lf\n", psnr_total, ssim_total, time_total); 153 | } 154 | else 155 | { 156 | FILE * fp = fopen(argv[4], "a"); 157 | fprintf(fp, "%s\t%lf\t%lf\t%lf\n", argv[5], psnr_total, ssim_total, time_total); 158 | fclose(fp); 159 | } 160 | 161 | return 0; 162 | } 163 | 164 | -------------------------------------------------------------------------------- /src/maskedimage.cpp: -------------------------------------------------------------------------------- 1 | #include "maskedimage.h" 2 | 3 | void initSimilarity() 4 | { 5 | int i, j, k, length; 6 | double base[11] = {1.0, 0.99, 0.96, 0.83, 0.38, 0.11, 0.02, 0.005, 0.0006, 0.0001, 0 }; 7 | double t, vj, vk; 8 | length = (DSCALE+1); 9 | if (!G_initSim) 10 | { 11 | G_globalSimilarity = (double *) calloc(length, sizeof(double)); 12 | for ( i=0 ; imask = mask; 48 | mIm->image = image; 49 | 50 | initSimilarity(); 51 | mIm->similarity = G_globalSimilarity; 52 | 53 | mIm->isNew=0; 54 | 55 | return mIm; 56 | } 57 | 58 | MaskedImage_P initMaskedImageFromImage(IplImage* image, int** mask) 59 | { 60 | MaskedImage_P mIm = (MaskedImage_P)malloc(sizeof(MaskedImage_T)); 61 | 62 | // image data 63 | mIm->mask = mask; 64 | mIm->image = image; 65 | 66 | initSimilarity(); 67 | mIm->similarity = G_globalSimilarity; 68 | 69 | mIm->isNew=0; 70 | 71 | return mIm; 72 | } 73 | 74 | MaskedImage_P initNewMaskedImage(int width, int height) 75 | { 76 | int j; 77 | MaskedImage_P mIm = (MaskedImage_P)malloc(sizeof(MaskedImage_T)); 78 | 79 | // image data 80 | mIm->mask = (int**)malloc(sizeof(int*)*height); 81 | for (j=0; jmask[j] = (int*)calloc(width, sizeof(int)); 83 | 84 | mIm->image = cvCreateImage(cvSize(width,height),IPL_DEPTH_8U,3); 85 | 86 | initSimilarity(); 87 | mIm->similarity = G_globalSimilarity; 88 | 89 | mIm->isNew=1; 90 | 91 | return mIm; 92 | } 93 | 94 | void freeMaskedImage(MaskedImage_P mIm) 95 | { 96 | int j; 97 | if (mIm!=NULL) 98 | { 99 | if (mIm->isNew) 100 | { 101 | if (mIm->mask!=NULL) 102 | { 103 | for (j=0; jimage->height; j++) 104 | free(mIm->mask[j]); 105 | 106 | free(mIm->mask); 107 | mIm->mask=NULL; 108 | } 109 | if (mIm->image!=NULL) 110 | { 111 | cvReleaseImage(&mIm->image); 112 | mIm->image=NULL; 113 | } 114 | } 115 | free(mIm); 116 | mIm = NULL; 117 | } 118 | } 119 | 120 | 121 | int getSampleMaskedImage(MaskedImage_P mIm, int x, int y, int band) 122 | { 123 | uchar* data; 124 | int channels=mIm->image->nChannels; 125 | int step = mIm->image->widthStep/sizeof(uchar); 126 | data = (uchar *)mIm->image->imageData; 127 | return data[x*step+y*channels+band]; 128 | } 129 | 130 | void setSampleMaskedImage(MaskedImage_P mIm, int x, int y, int band, int value) 131 | { 132 | uchar* data; 133 | int channels=mIm->image->nChannels; 134 | int step = mIm->image->widthStep/sizeof(uchar); 135 | data = (uchar *)mIm->image->imageData; 136 | data[x*step+y*channels+band] = value; 137 | 138 | } 139 | 140 | int isMasked(MaskedImage_P mIm, int x, int y) 141 | { 142 | if (mIm==NULL || mIm->mask==NULL) 143 | return 0; 144 | return mIm->mask[x][y]; 145 | } 146 | 147 | void setMask(MaskedImage_P mIm, int x, int y, int value) 148 | { 149 | if (mIm==NULL || mIm->mask==NULL) 150 | return; 151 | mIm->mask[x][y]=0=mIm->image->height) 166 | continue; 167 | if (ys<0 || ys>=mIm->image->width) 168 | continue; 169 | if (mIm->mask[xs][ys]) 170 | return 1; 171 | } 172 | } 173 | return 0; 174 | } 175 | 176 | // distance between two patches in two images 177 | int distanceMaskedImage(MaskedImage_P source,int xs,int ys, MaskedImage_P target,int xt,int yt, int S) 178 | { 179 | long double distance=0; 180 | long double wsum=0, ssdmax = 9*255*255; 181 | int dy, dx, band; 182 | int xks, yks; 183 | int xkt, ykt; 184 | long double ssd; 185 | long res; 186 | int s_value, t_value, s_gx, t_gx, s_gy, t_gy; 187 | 188 | // for each pixel in the source patch 189 | for ( dy=-S ; dy<=S ; ++dy ) 190 | { 191 | for ( dx=-S ; dx<=S ; ++dx ) 192 | { 193 | 194 | xks = xs+dx; 195 | yks = ys+dy; 196 | xkt=xt+dx; 197 | ykt=yt+dy; 198 | wsum++; 199 | 200 | if ( xks<1 || xks>=source->image->height-1 ) 201 | { 202 | distance++; 203 | continue; 204 | } 205 | if ( yks<1 || yks>=source->image->width-1 ) 206 | { 207 | distance++; 208 | continue; 209 | } 210 | 211 | // cannot use masked pixels as a valid source of information 212 | if (isMasked(source, xks, yks)) 213 | { 214 | distance++; 215 | continue; 216 | } 217 | 218 | // corresponding pixel in the target patch 219 | if (xkt<1 || xkt>=target->image->height-1) 220 | { 221 | distance++; 222 | continue; 223 | } 224 | if (ykt<1 || ykt>=target->image->width-1) 225 | { 226 | distance++; 227 | continue; 228 | } 229 | 230 | // cannot use masked pixels as a valid source of information 231 | if (isMasked(target, xkt, ykt)) 232 | { 233 | distance++; 234 | continue; 235 | } 236 | 237 | ssd=0; 238 | for (band=0; band<3; ++band) 239 | { 240 | // pixel values 241 | s_value = getSampleMaskedImage(source, xks, yks, band); 242 | t_value = getSampleMaskedImage(source, xkt, ykt, band); 243 | 244 | // pixel horizontal gradients (Gx) 245 | s_gx = 128+(getSampleMaskedImage(source, xks+1, yks, band) - getSampleMaskedImage(source, xks-1, yks, band))/2; 246 | t_gx = 128+(getSampleMaskedImage(target, xkt+1, ykt, band) - getSampleMaskedImage(target, xkt-1, ykt, band))/2; 247 | 248 | // pixel vertical gradients (Gy) 249 | s_gy = 128+(getSampleMaskedImage(source, xks, yks+1, band) - getSampleMaskedImage(source, xks, yks-1, band))/2; 250 | t_gy = 128+(getSampleMaskedImage(target, xkt, ykt+1, band) - getSampleMaskedImage(target, xkt, ykt-1, band))/2; 251 | 252 | ssd += pow((long double)s_value-t_value , 2); // distance between values in [0,255^2] 253 | ssd += pow((long double)s_gx-t_gx , 2); // distance between Gx in [0,255^2] 254 | ssd += pow((long double)s_gy-t_gy , 2); // distance between Gy in [0,255^2] 255 | } 256 | 257 | // add pixel distance to global patch distance 258 | distance += ssd/ssdmax; 259 | } 260 | } 261 | 262 | res = (int)(DSCALE*distance/wsum); 263 | if (res < 0 || res > DSCALE) return DSCALE; 264 | return res; 265 | } 266 | 267 | // return a copy of the image 268 | MaskedImage_P copyMaskedImage(MaskedImage_P mIm) 269 | { 270 | int j, x; 271 | int W = mIm->image->width; 272 | int H = mIm->image->height; 273 | int ** newmask; 274 | MaskedImage_P copy; 275 | newmask = (int**)malloc(sizeof(int*)*H); 276 | for (j=0; jmask[j][x]; 281 | } 282 | 283 | IplImage* newimage = cvCloneImage(mIm->image); 284 | 285 | copy = initMaskedImage(newimage,newmask); 286 | copy->isNew=1; 287 | 288 | return copy; 289 | } 290 | 291 | // return a downsampled image (factor 1/2) 292 | MaskedImage_P downsample2(MaskedImage_P source) 293 | { 294 | int kernel[6] = {1,2,4,4,2,1}; 295 | int H, W; 296 | int x, y; 297 | int xs, ys; 298 | int dx, dy; 299 | int xk, yk, ky, k; 300 | int r=0,g=0,b=0,m=0,ksum=0; 301 | H=source->image->height; 302 | W=source->image->width; 303 | int newW=W/2, newH=H/2; 304 | 305 | MaskedImage_P newimage = initNewMaskedImage(newW, newH); 306 | xs=0; 307 | for(x=0; x=W) continue; 322 | ky = kernel[2+dy]; 323 | for(dx=-2; dx<=3; dx++) 324 | { 325 | xk = xs+dx; 326 | if (xk<0 || xk>=H) continue; 327 | 328 | if (source->mask[xk][yk]) continue; 329 | 330 | k = kernel[2+dx]*ky; 331 | r+= k*getSampleMaskedImage(source, xk, yk, 0); 332 | g+= k*getSampleMaskedImage(source, xk, yk, 1); 333 | b+= k*getSampleMaskedImage(source, xk, yk, 2); 334 | ksum+=k; 335 | m++; 336 | } 337 | } 338 | if (ksum>0) 339 | { 340 | r/=ksum; 341 | g/=ksum; 342 | b/=ksum; 343 | } 344 | 345 | if (m!=0) 346 | { 347 | setSampleMaskedImage(newimage, x, y, 0, r); 348 | setSampleMaskedImage(newimage, x, y, 1, g); 349 | setSampleMaskedImage(newimage, x, y, 2, b); 350 | setMask(newimage, x, y, 0); 351 | } 352 | else 353 | { 354 | setMask(newimage, x, y, 1); 355 | setSampleMaskedImage(newimage, x, y, 0, 0); 356 | setSampleMaskedImage(newimage, x, y, 1, 0); 357 | setSampleMaskedImage(newimage, x, y, 2, 0); 358 | } 359 | ys+=2; 360 | } 361 | xs+=2; 362 | } 363 | 364 | return newimage; 365 | } 366 | 367 | // return a downsampled image (factor 1/2) 368 | MaskedImage_P downsample(MaskedImage_P source) 369 | { 370 | int kernel[6] = {1,5,10,10,5,1}; 371 | int H, W; 372 | int x, y; 373 | int dx, dy; 374 | int xk, yk, ky, k; 375 | int r=0,g=0,b=0,m=0,ksum=0; 376 | H=source->image->height; 377 | W=source->image->width; 378 | int newW=W/2, newH=H/2; 379 | 380 | MaskedImage_P newimage = initNewMaskedImage(newW, newH); 381 | for (x=0; x=W) 395 | continue; 396 | ky = kernel[2+dy]; 397 | for (dx=-2; dx<=3; ++dx) 398 | { 399 | xk = x+dx; 400 | if (xk<0 || xk>=H) 401 | continue; 402 | 403 | if (source->mask[xk][yk]) 404 | continue; 405 | 406 | k = kernel[2+dx]*ky; 407 | r+= k*getSampleMaskedImage(source, xk, yk, 0); 408 | g+= k*getSampleMaskedImage(source, xk, yk, 1); 409 | b+= k*getSampleMaskedImage(source, xk, yk, 2); 410 | ksum+=k; 411 | m++; 412 | } 413 | } 414 | if (ksum>0) 415 | { 416 | r/=ksum; 417 | g/=ksum; 418 | b/=ksum; 419 | } 420 | 421 | if (m!=0) 422 | { 423 | setSampleMaskedImage(newimage, x/2, y/2, 0, r); 424 | setSampleMaskedImage(newimage, x/2, y/2, 1, g); 425 | setSampleMaskedImage(newimage, x/2, y/2, 2, b); 426 | setMask(newimage, x/2, y/2, 0); 427 | } 428 | else 429 | { 430 | setMask(newimage, x/2, y/2, 1); 431 | } 432 | } 433 | } 434 | 435 | return newimage; 436 | } 437 | 438 | // return an upscaled image 439 | MaskedImage_P upscale(MaskedImage_P source, int newW,int newH) 440 | { 441 | int x, y; 442 | int xs, ys; 443 | int H, W; 444 | H=source->image->height; 445 | W=source->image->width; 446 | MaskedImage_P newimage = initNewMaskedImage(newW, newH); 447 | 448 | for (x=0; xmask[xs][ys]) 459 | { 460 | setSampleMaskedImage(newimage, x, y, 0, getSampleMaskedImage(source, xs, ys, 0)); 461 | setSampleMaskedImage(newimage, x, y, 1, getSampleMaskedImage(source, xs, ys, 1)); 462 | setSampleMaskedImage(newimage, x, y, 2, getSampleMaskedImage(source, xs, ys, 2)); 463 | setMask(newimage, x, y, 0); 464 | } 465 | else 466 | { 467 | setSampleMaskedImage(newimage, x, y, 0, 0); 468 | setSampleMaskedImage(newimage, x, y, 1, 0); 469 | setSampleMaskedImage(newimage, x, y, 2, 0); 470 | setMask(newimage, x, y, 1); 471 | } 472 | } 473 | } 474 | 475 | return newimage; 476 | } 477 | -------------------------------------------------------------------------------- /src/nearestneighborfield.cpp: -------------------------------------------------------------------------------- 1 | #include "nearestneighborfield.h" 2 | 3 | /** 4 | * Nearest-Neighbor Field (see PatchMatch algorithm) 5 | * This algorithme uses a version proposed by Xavier Philippeau 6 | * 7 | */ 8 | 9 | NNF_P initNNF(MaskedImage_P input, MaskedImage_P output, int patchsize) 10 | { 11 | NNF_P nnf = (NNF_P)malloc(sizeof(NNF_T)); 12 | nnf->input = input; 13 | nnf->output= output; 14 | nnf->S = patchsize; 15 | nnf->field=NULL; 16 | 17 | return nnf; 18 | } 19 | 20 | void allocNNFField(NNF_P nnf) 21 | { 22 | int i, j; 23 | if (nnf!=NULL) 24 | { 25 | nnf->fieldH=nnf->input->image->height; 26 | nnf->fieldW=nnf->input->image->width; 27 | nnf->field = (int ***) malloc(sizeof(int**)*nnf->fieldH); 28 | 29 | for ( i=0 ; i < nnf->fieldH ; i++ ) 30 | { 31 | nnf->field[i] = (int **) malloc(sizeof(int*)*nnf->fieldW); 32 | for (j=0 ; jfieldW ; j++ ) 33 | { 34 | nnf->field[i][j] = (int *) calloc(3,sizeof(int)); 35 | } 36 | } 37 | } 38 | } 39 | 40 | void freeNNFField(NNF_P nnf) 41 | { 42 | int i, j; 43 | if ( nnf->field != NULL ) 44 | { 45 | for ( i=0 ; i < nnf->fieldH ; ++i ) 46 | { 47 | for ( j=0 ; j < nnf->fieldW ; ++j ) 48 | { 49 | free( nnf->field[i][j] ); 50 | } 51 | free(nnf->field[i]); 52 | } 53 | free(nnf->field); 54 | nnf->field=NULL; 55 | } 56 | } 57 | 58 | void freeNNF(NNF_P nnf) 59 | { 60 | if (nnf!=NULL) 61 | { 62 | freeNNFField(nnf); 63 | free(nnf); 64 | nnf=NULL; 65 | } 66 | } 67 | 68 | // initialize field with random values 69 | void randomize(NNF_P nnf) 70 | { 71 | int i, j; 72 | // field 73 | allocNNFField(nnf); 74 | for (i=0; iinput->image->height; ++i) 75 | { 76 | for (j=0; jinput->image->width; ++j) 77 | { 78 | nnf->field[i][j][0] = rand() % nnf->output->image->height +1; 79 | nnf->field[i][j][1] = rand() % nnf->output->image->width +1; 80 | nnf->field[i][j][2] = DSCALE; 81 | } 82 | } 83 | initializeNNF(nnf); 84 | } 85 | 86 | // initialize field from an existing (possibily smaller) NNF 87 | void initializeNNFFromOtherNNF(NNF_P nnf, NNF_P otherNnf) 88 | { 89 | int fx, fy, x, y, xlow, ylow; 90 | // field 91 | allocNNFField(nnf); 92 | fy = nnf->fieldW/otherNnf->fieldW; 93 | fx = nnf->fieldH/otherNnf->fieldH; 94 | for (x=0; xfieldH; ++x) 95 | { 96 | for (y=0; yfieldW; ++y) 97 | { 98 | xlow = int(min1(x/fx, otherNnf->input->image->height-1)); 99 | ylow = int(min1(y/fy, otherNnf->input->image->width-1)); 100 | nnf->field[x][y][0] = otherNnf->field[xlow][ylow][0]*fx; 101 | nnf->field[x][y][1] = otherNnf->field[xlow][ylow][1]*fy; 102 | nnf->field[x][y][2] = DSCALE; 103 | } 104 | } 105 | initializeNNF(nnf); 106 | } 107 | 108 | // compute initial value of the distance term 109 | void initializeNNF(NNF_P nnf) 110 | { 111 | int y, x; 112 | int iter=0, maxretry=20; 113 | for (x=0; xfieldH; ++x) 114 | { 115 | for (y=0; yfieldW; ++y) 116 | { 117 | 118 | nnf->field[x][y][2] = distanceNNF(nnf, x,y, nnf->field[x][y][0],nnf->field[x][y][1]); 119 | // if the distance is INFINITY (all pixels masked ?), try to find a better link 120 | iter=0; 121 | while ( nnf->field[x][y][2] == DSCALE && iterfield[x][y][0] = rand() % nnf->output->image->height +1; 124 | nnf->field[x][y][1] = rand() % nnf->output->image->width +1; 125 | nnf->field[x][y][2] = distanceNNF(nnf, x,y, nnf->field[x][y][0],nnf->field[x][y][1]); 126 | iter++; 127 | } 128 | } 129 | } 130 | } 131 | 132 | // multi-pass NN-field minimization (see "PatchMatch" - page 4) 133 | void minimizeNNF(NNF_P nnf, int pass) 134 | { 135 | int i, y, x; 136 | int min_x=0, min_y=0, max_y=nnf->input->image->width-1, max_x=nnf->input->image->height-1; 137 | // multi-pass minimization 138 | for (i=0; ifield[x][y][2]>0) 144 | minimizeLinkNNF(nnf, x,y,+1); 145 | 146 | // reverse scanline order 147 | for (x=max_x; x>=min_x; x--) 148 | for (y=max_y; y>=min_y; y--) 149 | if (nnf->field[x][y][2]>0) 150 | minimizeLinkNNF(nnf, x,y,-1); 151 | } 152 | } 153 | 154 | // minimize a single link (see "PatchMatch" - page 4) 155 | void minimizeLinkNNF(NNF_P nnf, int x, int y, int dir) 156 | { 157 | int xp,yp,dp,wi, xpi, ypi; 158 | //Propagation Up/Down 159 | if (x-dir>0 && x-dirinput->image->height) 160 | { 161 | xp = nnf->field[x-dir][y][0]+dir; 162 | yp = nnf->field[x-dir][y][1]; 163 | dp = distanceNNF(nnf,x,y, xp,yp); 164 | if (dpfield[x][y][2]) 165 | { 166 | nnf->field[x][y][0] = xp; 167 | nnf->field[x][y][1] = yp; 168 | nnf->field[x][y][2] = dp; 169 | } 170 | } 171 | //Propagation Left/Right 172 | if (y-dir>0 && y-dirinput->image->width) 173 | { 174 | xp = nnf->field[x][y-dir][0]; 175 | yp = nnf->field[x][y-dir][1]+dir; 176 | dp = distanceNNF(nnf,x,y, xp,yp); 177 | if (dpfield[x][y][2]) 178 | { 179 | nnf->field[x][y][0] = xp; 180 | nnf->field[x][y][1] = yp; 181 | nnf->field[x][y][2] = dp; 182 | } 183 | } 184 | //Random search 185 | wi=nnf->output->image->width; 186 | xpi=nnf->field[x][y][0]; 187 | ypi=nnf->field[x][y][1]; 188 | int r=0; 189 | while (wi>0) 190 | { 191 | r=(rand() % (2*wi)) -wi; 192 | xp = xpi + r; 193 | r=(rand() % (2*wi)) -wi; 194 | yp = ypi + r; 195 | xp = int(max1(0, min1(nnf->output->image->height-1, xp ))); 196 | yp = int(max1(0, min1(nnf->output->image->width-1, yp ))); 197 | 198 | dp = distanceNNF(nnf,x,y, xp,yp); 199 | if (dpfield[x][y][2]) 200 | { 201 | nnf->field[x][y][0] = xp; 202 | nnf->field[x][y][1] = yp; 203 | nnf->field[x][y][2] = dp; 204 | } 205 | wi/=2; 206 | } 207 | } 208 | 209 | // compute distance between two patch 210 | int distanceNNF(NNF_P nnf, int x,int y, int xp,int yp) 211 | { 212 | return distanceMaskedImage(nnf->input,x,y, nnf->output,xp,yp, nnf->S); 213 | } 214 | 215 | -------------------------------------------------------------------------------- /src/qualitymesures.cpp: -------------------------------------------------------------------------------- 1 | #include "defineall.h" 2 | 3 | double PSNR(IplImage *Original , IplImage *inpainted) 4 | { 5 | //First let's convert the images to Y component only 6 | double sumR = 0., sumG = 0., sumB = 0., sumRGB; 7 | double psnr, EQM = 0.; 8 | double h= Original->height; 9 | double w= Original->width; 10 | 11 | for (int i=0 ; iwidth, y=original->height; 50 | int nChan=original->nChannels, d=IPL_DEPTH_32F; 51 | CvSize size = cvSize(x, y); 52 | 53 | img1 = cvCreateImage( size, d, nChan); 54 | img2 = cvCreateImage( size, d, nChan); 55 | 56 | cvConvert(original, img1); 57 | cvConvert(distorted, img2); 58 | 59 | 60 | img1_sq = cvCreateImage( size, d, nChan); 61 | img2_sq = cvCreateImage( size, d, nChan); 62 | img1_img2 = cvCreateImage( size, d, nChan); 63 | 64 | cvPow( img1, img1_sq, 2 ); 65 | cvPow( img2, img2_sq, 2 ); 66 | cvMul( img1, img2, img1_img2, 1 ); 67 | 68 | mu1 = cvCreateImage( size, d, nChan); 69 | mu2 = cvCreateImage( size, d, nChan); 70 | 71 | mu1_sq = cvCreateImage( size, d, nChan); 72 | mu2_sq = cvCreateImage( size, d, nChan); 73 | mu1_mu2 = cvCreateImage( size, d, nChan); 74 | 75 | 76 | sigma1_sq = cvCreateImage( size, d, nChan); 77 | sigma2_sq = cvCreateImage( size, d, nChan); 78 | sigma12 = cvCreateImage( size, d, nChan); 79 | 80 | temp1 = cvCreateImage( size, d, nChan); 81 | temp2 = cvCreateImage( size, d, nChan); 82 | temp3 = cvCreateImage( size, d, nChan); 83 | 84 | ssim_map = cvCreateImage( size, d, nChan); 85 | 86 | /*************************** END INITS **********************************/ 87 | ////////////////////////////////////////////////////////////////////////// 88 | // PRELIMINARY COMPUTING 89 | cvSmooth( img1, mu1, CV_GAUSSIAN, 11, 11, 1.10 ); 90 | cvSmooth( img2, mu2, CV_GAUSSIAN, 11, 11, 1.10 ); 91 | 92 | cvPow( mu1, mu1_sq, 2 ); 93 | cvPow( mu2, mu2_sq, 2 ); 94 | cvMul( mu1, mu2, mu1_mu2, 1 ); 95 | 96 | cvSmooth( img1_sq, sigma1_sq, CV_GAUSSIAN, 11, 11, 1.10 ); 97 | cvAddWeighted( sigma1_sq, 1, mu1_sq, -1, 0, sigma1_sq ); 98 | 99 | cvSmooth( img2_sq, sigma2_sq, CV_GAUSSIAN, 11, 11, 1.10 ); 100 | cvAddWeighted( sigma2_sq, 1, mu2_sq, -1, 0, sigma2_sq ); 101 | 102 | cvSmooth( img1_img2, sigma12, CV_GAUSSIAN, 11, 11, 1.10 ); 103 | cvAddWeighted( sigma12, 1, mu1_mu2, -1, 0, sigma12 ); 104 | 105 | ////////////////////////////////////////////////////////////////////////// 106 | // FORMULA 107 | // (2*mu1_mu2 + C1) 108 | cvScale( mu1_mu2, temp1, 2 ); 109 | cvAddS( temp1, cvScalarAll(C1), temp1 ); 110 | 111 | // (2*sigma12 + C2) 112 | cvScale( sigma12, temp2, 2 ); 113 | cvAddS( temp2, cvScalarAll(C2), temp2 ); 114 | 115 | // ((2*mu1_mu2 + C1).*(2*sigma12 + C2)) 116 | cvMul( temp1, temp2, temp3, 1 ); 117 | 118 | // (mu1_sq + mu2_sq + C1) 119 | cvAdd( mu1_sq, mu2_sq, temp1 ); 120 | cvAddS( temp1, cvScalarAll(C1), temp1 ); 121 | 122 | // (sigma1_sq + sigma2_sq + C2) 123 | cvAdd( sigma1_sq, sigma2_sq, temp2 ); 124 | cvAddS( temp2, cvScalarAll(C2), temp2 ); 125 | 126 | // ((mu1_sq + mu2_sq + C1).*(sigma1_sq + sigma2_sq + C2)) 127 | cvMul( temp1, temp2, temp1, 1 ); 128 | 129 | // ((2*mu1_mu2 + C1).*(2*sigma12 + C2))./((mu1_sq + mu2_sq + C1).*(sigma1_sq + sigma2_sq + C2)) 130 | cvDiv( temp3, temp1, ssim_map, 1 ); 131 | CvScalar index_scalar = cvAvg( ssim_map ); 132 | // through observation, there is approximately 133 | // error max with the original matlab program 134 | SSIM=(index_scalar.val[2]+index_scalar.val[1]+index_scalar.val[0])/3; 135 | 136 | return SSIM; 137 | } 138 | --------------------------------------------------------------------------------