├── README.md ├── background_smooth.PNG ├── compilation ├── EdgeComputation.cu ├── EdgeComputation.h ├── EdgeComputation.lua ├── EdgeDetector.cu ├── EdgeDetector.h ├── EdgeDetector.lua ├── SmoothAndEdgeTerm.cu ├── SmoothAndEdgeTerm.h ├── SmoothAndEdgeTerm.lua └── adam_state.lua ├── data ├── VOC2012_test.txt └── VOC2012_train.txt ├── detail_enhancement.PNG ├── foreground_enhance.PNG ├── foreground_smooth.PNG ├── images ├── content_smoothing │ ├── blur-previous-5-predict-detail.png │ ├── blur-previous-5.png │ ├── blur10-500-10-input-predict-detail.png │ ├── blur10-500-10-input-predict.png │ ├── blur10-500-10-input.png │ ├── blur10-500-11-input-predict-detail.png │ ├── blur10-500-11-input.png │ ├── blur10-500-5-input-predict.png │ ├── blur10-500-5-input.png │ ├── blur10-500-7-input-predict.png │ ├── blur10-500-7-input.png │ ├── blur11-500-1-input-predict.png │ ├── blur11-500-1-input.png │ ├── blur11-500-3-input-predict-detail.png │ ├── blur11-500-3-input.png │ ├── blur11-500-4-input-predict.png │ ├── blur11-500-4-input.png │ ├── blur11-500-8-input-predict-detail.png │ ├── blur11-500-8-input.png │ ├── demo11-500-13-input-detail.png │ ├── demo11-500-13-input-predict-detail.png │ ├── demo11-500-22-input-predict-detail.png │ ├── demo11-500-22-input-predict.png │ ├── demo11-500-22-input.png │ ├── demo12-500-25-input-predict-detail.png │ ├── demo12-500-25-input-predict.png │ ├── demo12-500-25-input.png │ ├── demo12-500-9-input-predict-detail.png │ ├── demo12-500-9-input.png │ ├── sig_v3_input_19-predict-detail.png │ ├── sig_v3_input_19-predict.png │ └── sig_v3_input_19.png ├── detail_enhancement │ ├── detail-previous-2-predict-detail.png │ ├── detail-previous-2-predict.png │ ├── detail-previous-2.png │ ├── detail-previous-6-predict-detail.png │ ├── detail-previous-6-predict.png │ ├── detail-previous-6.png │ ├── detail_2_4-predict-detail.png │ ├── detail_2_4-predict.png │ ├── detail_2_4.png │ ├── detail_2_5-predict-detail.png │ ├── detail_2_5-predict.png │ ├── detail_2_5.png │ ├── detail_2_6-predict-detail.png │ ├── detail_2_6-predict.png │ ├── detail_2_6.png │ ├── flower10-500-11-input-predict-detail.png │ ├── flower10-500-11-input-predict.png │ ├── flower10-500-11-input.png │ ├── flower10-500-14-input-predict-detail.png │ ├── flower10-500-14-input-predict.png │ ├── flower10-500-14-input.png │ ├── flower10-500-17-input-predict-detail.png │ ├── flower10-500-17-input-predict.png │ ├── flower10-500-17-input.png │ ├── flower10-500-3-input-predict-detail.png │ ├── flower10-500-3-input-predict.png │ ├── flower10-500-3-input.png │ ├── flower10-500-5-input-predict-detail.png │ ├── flower10-500-5-input-predict.png │ ├── flower10-500-5-input.png │ ├── flower10-500-6-input-predict-detail.png │ ├── flower10-500-6-input-predict.png │ ├── flower10-500-6-input.png │ ├── sig_v2_input_76-predict-detail.png │ ├── sig_v2_input_76-predict.png │ └── sig_v2_input_76.png ├── edge_preserving_smoothing │ ├── demo-13-predict.png │ ├── demo-13.png │ ├── demo-new5-16-predict.png │ ├── demo-new5-16.png │ ├── demo9-600-5-input-predict.png │ ├── demo9-600-5-input.png │ ├── sig_input_47-predict.png │ ├── sig_input_47.png │ ├── sig_v2_input_42-predict.png │ ├── sig_v2_input_42.png │ ├── sig_v2_input_72-predict.png │ ├── sig_v2_input_72.png │ ├── sig_v3_input_45-predict.png │ ├── sig_v3_input_45.png │ ├── smooth-63-input-predict.png │ ├── smooth-63-input.png │ ├── smooth-previous-1-predict.png │ ├── smooth-previous-1.png │ ├── smooth_2_10-predict.png │ ├── smooth_2_10.png │ ├── smooth_2_15-predict.png │ ├── smooth_2_15.png │ ├── smooth_2_2-predict.png │ ├── smooth_2_2.png │ ├── smooth_2_4-predict.png │ └── smooth_2_4.png ├── stylization │ ├── demo-new-21-predict-draw.png │ ├── demo-new-21.png │ ├── demo8-2-input-predict-draw.png │ ├── demo8-2-input.png │ ├── sig_v2_input_31-predict-draw.png │ ├── sig_v2_input_31.png │ ├── sig_v2_input_46-predict-draw.png │ ├── sig_v2_input_46.png │ ├── smooth_2_11-predict-draw-woTone.png │ ├── smooth_2_11-predict-draw.png │ ├── smooth_2_11.png │ ├── smooth_2_18-predict-draw-woTone.png │ ├── smooth_2_18-predict-draw.png │ ├── smooth_2_18.png │ ├── smooth_2_19-predict-draw-woTone.png │ ├── smooth_2_19-predict-draw.png │ ├── smooth_2_19.png │ ├── style-1-predict-draw-woTone.png │ ├── style-1-predict-draw.png │ ├── style-1.png │ ├── style-22-predict-draw-woTone.png │ ├── style-22-predict-draw.png │ ├── style-22.png │ ├── style-40-predict-draw-woTone.png │ ├── style-40-predict-draw.png │ └── style-40.png └── texture_removal │ ├── RTV-51-predict.png │ ├── RTV-51.png │ ├── texture10-500-10-input-predict.png │ ├── texture10-500-10-input.png │ ├── texture10-500-3-input-predict21.png │ ├── texture10-500-3-input.png │ ├── texture_2_1-predict.png │ └── texture_2_1.png ├── netfiles ├── model_smooth_background-smooth_30.net ├── model_smooth_detail-enhance_30.net ├── model_smooth_foreground-enhance_30.net ├── model_smooth_stylization_30.net └── model_smooth_texture-removal_30.net ├── pencils ├── pencil0.png ├── pencil1.png ├── pencil2.png ├── pencil3.png └── pencil4.png ├── post_detail_enhance.m ├── post_stylization.m ├── run_edgedetector.lua ├── smoothing.PNG ├── stylization.PNG ├── test_detail_enhance ├── deer-input.png └── deer-smooth.png ├── test_smooth.lua ├── test_stylization └── tree-smooth.png ├── train_background-smooth.lua ├── train_detail-enhance.lua ├── train_foreground-enhance.lua ├── train_stylization.lua ├── train_texture-removal.lua └── util ├── GenPencil.m ├── GenStroke.m ├── GenToneMap.m ├── PencilDrawing.m ├── sigmoid.m ├── tonemapLAB_simple.m └── zeroone.m /README.md: -------------------------------------------------------------------------------- 1 | Image Smoothing via Unsupervised Learning 2 | ======= 3 | 4 | This is the implementation of SIGGRAPH Asia 2018 paper *"Image Smoothing via Unsupervised Learning"* by [Qingnan Fan](), *et al.* 5 | 6 | Our work is the first unsupervised learning approach for the image smoothing problem. Our proposed objective function can be slightly modifed to achieve different image processing effects, for example, 7 | 8 | Edge-preserving smoothing 9 | ![image](smoothing.PNG) 10 | Image abstraction 11 | ![image](stylization.PNG) 12 | Detail enhancement 13 | ![image](detail_enhancement.PNG) 14 | Background smooth 15 | ![image](background_smooth.PNG) 16 | Foreground enhancement 17 | ![image](foreground_enhance.PNG) 18 | 19 | ## Getting Started 20 | 21 | This paper is implemented with Torch framework. 22 | 23 | Compilation 24 | ---- 25 | 26 | Our codes are implemented in Torch framework. Some self-defined layers are included in the '**compilation**' folder. Before testing or training the models, you need to install the latest torch framework and compile the ***.lua** under the nn module, the ***.cu** and ***.h** under the cunn module, and **adam_state.lua** under optim module. 27 | 28 | To be specific, the lua files have to be put in **./torch/extra/nn/** module directory, and editing **init.lua** file to include the corresponding file. Finally run the command under nn directory, **luarocks make ./rocks/nn-scm-1.rockspec**. Then nn module will be independently compiled. 29 | 30 | Similarly for the cuda files, they need to be put into **./torch/extra/cunn/lib/THCUNN/** module directory, then editing the **./torch/extra/cunn/lib/THCUNN/generic/THCUNN.h** to include the corresponding files, and finally run the command **luarocks make ./rocks/cunn-scm-1.rockspec** under **./torch/extra/cunn** folder to compile them. 31 | 32 | Accordingly, the adam lua file has to be put in **./torch/pkg/optim**, edit the **init.lua** file and then run **luarocks make optim-1.0.5-0.rockspec**. 33 | 34 | Data Generation 35 | ---- 36 | 37 | Run **run_edgedetector.lua** to compute the edges for edge-preserving smoothing or texture removal. To learn the content-aware image smoothing effects, we need saliency map for each training sample, which could be generated by existing saliency prediction methods. 38 | 39 | Training 40 | ---- 41 | 42 | The trained models are all in the **netfiles** folder. 43 | 44 | Run **train_*.lua** to learn specific smoothing effects. 45 | 46 | Evaluation 47 | ---- 48 | To test our trained model, run **test_smooth.lua**. 49 | 50 | For the application of stylization, run **post_stylization.m** after predicting the smooth images. Similarly, for detail enhancement, run **post_detail_enhance.m**. 51 | 52 | Cite 53 | ---- 54 | 55 | You can use our codes for research purpose only. And please cite our paper when you use our codes. 56 | ``` 57 | @Article{fan2018smoothing, 58 | Title = {Image Smoothing via Unsupervised Learning}, 59 | Author = {Qingnan Fan and Jiaolong Yang and David Wipf and Baoquan Chen and Xin Tong}, 60 | Journal = {ACM Transactions on Graphics (Proceedings of SIGGRAPH ASIA 2018)}, 61 | Year = {2018}, 62 | Volume = {37} 63 | Number = {6}, 64 | } 65 | ``` 66 | Contact 67 | ------- 68 | 69 | If you find any bugs or have any ideas of optimizing these codes, please contact me via fqnchina [at] gmail [dot] com 70 | -------------------------------------------------------------------------------- /background_smooth.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/background_smooth.PNG -------------------------------------------------------------------------------- /compilation/EdgeComputation.cu: -------------------------------------------------------------------------------- 1 | #include "THCUNN.h" 2 | #include "common.h" 3 | #include "THCHalf.h" 4 | #include "THCHalfAutoNumerics.cuh" 5 | 6 | #include "EdgeComputation.h" 7 | 8 | void THNN_CudaEdgeComputation_updateOutput(THCState *state, THCudaTensor *input, THCudaTensor *output) { 9 | 10 | long batchSize = input->size[0]; 11 | long plane = input->size[1]; 12 | long height = input->size[2]; 13 | long width = input->size[3]; 14 | 15 | THCudaTensor *input_n = THCudaTensor_new(state); 16 | THCudaTensor *output_n = THCudaTensor_new(state); 17 | 18 | // For each elt in batch, do: 19 | for (int elt = 0; elt < batchSize; elt ++) { 20 | // Matrix mulitply per output: 21 | THCudaTensor_select(state, input_n, input, 0, elt); 22 | THCudaTensor_select(state, output_n, output, 0, elt); 23 | 24 | EdgeComputation(THCState_getCurrentStream(state), 25 | THCudaTensor_data(state, input_n), 26 | THCudaTensor_data(state, output_n), 27 | height, width); 28 | } 29 | 30 | THCudaTensor_free(state, input_n); 31 | THCudaTensor_free(state, output_n); 32 | } 33 | 34 | void THNN_CudaEdgeComputation_updateGradInput(THCState *state, THCudaTensor *input, THCudaTensor *gradOutput, THCudaTensor *gradInput) { 35 | 36 | long batchSize = input->size[0]; 37 | long plane = input->size[1]; 38 | long height = input->size[2]; 39 | long width = input->size[3]; 40 | 41 | THCudaTensor *input_n = THCudaTensor_new(state); 42 | THCudaTensor *gradOutput_n = THCudaTensor_new(state); 43 | THCudaTensor *gradInput_n = THCudaTensor_new(state); 44 | 45 | // For each elt in batch, do: 46 | for (int elt = 0; elt < batchSize; elt ++) { 47 | // Matrix mulitply per output: 48 | THCudaTensor_select(state, input_n, input, 0, elt); 49 | THCudaTensor_select(state, gradOutput_n, gradOutput, 0, elt); 50 | THCudaTensor_select(state, gradInput_n, gradInput, 0, elt); 51 | 52 | EdgeComputation_backward(THCState_getCurrentStream(state), 53 | THCudaTensor_data(state, input_n), 54 | THCudaTensor_data(state, gradOutput_n), 55 | THCudaTensor_data(state, gradInput_n), 56 | height, width); 57 | } 58 | 59 | THCudaTensor_free(state, input_n); 60 | THCudaTensor_free(state, gradOutput_n); 61 | THCudaTensor_free(state, gradInput_n); 62 | } -------------------------------------------------------------------------------- /compilation/EdgeComputation.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #define MAX(x, y) (((x) > (y)) ? (x) : (y)) 4 | #define MIN(x, y) (((x) < (y)) ? (x) : (y)) 5 | 6 | template 7 | __global__ void EdgeComputation_kernel(const int num_kernels, Dtype* input, Dtype* output, int height, int width) { 8 | 9 | CUDA_KERNEL_LOOP(index, num_kernels) 10 | { 11 | int point_offset = index; 12 | int x = index % width; 13 | int y = index / width; 14 | 15 | int window_size = 1; 16 | for (int m = -window_size; m <= window_size; m++) { 17 | for (int n = -window_size; n <= window_size; n++) { 18 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 19 | continue; 20 | int image_offset = (y + m) * width + x + n; 21 | *(output + point_offset) += fabs(*(input + point_offset)-*(input + image_offset)); 22 | } 23 | } 24 | 25 | if (y-2 >= 0) 26 | *(output + point_offset) += fabs(*(input + point_offset)-*(input + (y - 2) * width + x)); 27 | if (y+2 < height) 28 | *(output + point_offset) += fabs(*(input + point_offset)-*(input + (y + 2) * width + x)); 29 | if (x-2 >= 0) 30 | *(output + point_offset) += fabs(*(input + point_offset)-*(input + y * width + x - 2)); 31 | if (x+2 < width) 32 | *(output + point_offset) += fabs(*(input + point_offset)-*(input + y * width + x + 2)); 33 | 34 | *(output + point_offset) = *(output + point_offset)/6; 35 | } 36 | } 37 | 38 | template 39 | void EdgeComputation(cudaStream_t stream, Dtype* input, Dtype* output, int height, int width) 40 | { 41 | int dimSize = 1024; 42 | int num_kernels = height * width; 43 | int grid = (num_kernels + dimSize - 1) / dimSize; 44 | EdgeComputation_kernel<<>>(num_kernels, input, output, height, width); 45 | } 46 | 47 | template 48 | __global__ void EdgeComputation_backward_kernel(const int num_kernels, Dtype* input, Dtype* gradOutput, Dtype* gradInput, int height, int width) { 49 | 50 | CUDA_KERNEL_LOOP(index, num_kernels) 51 | { 52 | int point_offset = index; 53 | int x = index % width; 54 | int y = index / width; 55 | 56 | int window_size = 1; 57 | for (int m = -window_size; m <= window_size; m++) { 58 | for (int n = -window_size; n <= window_size; n++) { 59 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 60 | continue; 61 | int image_offset = (y + m) * width + x + n; 62 | 63 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + image_offset) ? 1 : -1) * *(gradOutput + point_offset); 64 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + image_offset) ? 1 : -1) * *(gradOutput + image_offset); 65 | } 66 | } 67 | 68 | if (y-2 >= 0) 69 | { 70 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + (y - 2) * width + x) ? 1 : -1) * *(gradOutput + point_offset); 71 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + (y - 2) * width + x) ? 1 : -1) * *(gradOutput + (y - 2) * width + x); 72 | } 73 | if (y+2 < height) 74 | { 75 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + (y + 2) * width + x) ? 1 : -1) * *(gradOutput + point_offset); 76 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + (y + 2) * width + x) ? 1 : -1) * *(gradOutput + (y + 2) * width + x); 77 | } 78 | if (x-2 >= 0) 79 | { 80 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + y * width + x - 2) ? 1 : -1) * *(gradOutput + point_offset); 81 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + y * width + x - 2) ? 1 : -1) * *(gradOutput + y * width + x - 2); 82 | } 83 | if (x+2 < width) 84 | { 85 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + y * width + x + 2) ? 1 : -1) * *(gradOutput + point_offset); 86 | *(gradInput + point_offset) += (*(input + point_offset) > *(input + y * width + x + 2) ? 1 : -1) * *(gradOutput + y * width + x + 2); 87 | } 88 | 89 | *(gradInput + point_offset) = *(gradInput + point_offset)/6; 90 | 91 | } 92 | } 93 | 94 | template 95 | void EdgeComputation_backward(cudaStream_t stream, Dtype* input, Dtype* gradOutput, Dtype* gradInput, int height, int width) 96 | { 97 | int dimSize = 1024; 98 | int num_kernels = height * width; 99 | int grid = (num_kernels + dimSize - 1) / dimSize; 100 | EdgeComputation_backward_kernel<<>>(num_kernels, input, gradOutput, gradInput, height, width); 101 | } -------------------------------------------------------------------------------- /compilation/EdgeComputation.lua: -------------------------------------------------------------------------------- 1 | local THNN = require 'nn.THNN' 2 | local EdgeComputation, parent = torch.class('nn.EdgeComputation', 'nn.Module') 3 | 4 | function EdgeComputation:__init(scale) 5 | parent.__init(self) 6 | self.scale = scale or 1 7 | end 8 | 9 | function EdgeComputation:updateOutput(input) 10 | self.scale = self.scale or 1 11 | input = input / self.scale 12 | 13 | local bs,dim,height,width = input:size(1),input:size(2),input:size(3),input:size(4) 14 | input = torch.sum(input,2) 15 | self.output = torch.CudaTensor():resizeAs(input):fill(0) 16 | 17 | input.THNN.EdgeComputation_updateOutput( 18 | input:cdata(), 19 | self.output:cdata() 20 | ) 21 | return self.output 22 | end 23 | 24 | function EdgeComputation:updateGradInput(input, gradOutput) 25 | local bs,dim,height,width = input:size(1),input:size(2),input:size(3),input:size(4) 26 | self.gradInput = torch.CudaTensor():resizeAs(gradOutput):zero() 27 | 28 | input.THNN.EdgeComputation_updateGradInput( 29 | input:cdata(), 30 | gradOutput:cdata(), 31 | self.gradInput:cdata() 32 | ) 33 | self.gradInput = torch.expand(self.gradInput,bs,dim,height,width) / self.scale 34 | 35 | return self.gradInput 36 | end -------------------------------------------------------------------------------- /compilation/EdgeDetector.cu: -------------------------------------------------------------------------------- 1 | #include "THCUNN.h" 2 | #include "common.h" 3 | #include "THCHalf.h" 4 | #include "THCHalfAutoNumerics.cuh" 5 | 6 | #include "EdgeDetector.h" 7 | 8 | void THNN_CudaEdgeDetector_updateOutput(THCState *state, THCudaTensor *input_image, THCudaTensor *input_edge, THCudaTensor *label_preserve, THCudaTensor *label_eliminate, int isSmoothing) { 9 | 10 | long batchSize = input_image->size[0]; 11 | long plane = input_image->size[1]; 12 | long height = input_image->size[2]; 13 | long width = input_image->size[3]; 14 | 15 | THCudaTensor *input_image_n = THCudaTensor_new(state); 16 | THCudaTensor *input_edge_n = THCudaTensor_new(state); 17 | THCudaTensor *label_preserve_n = THCudaTensor_new(state); 18 | THCudaTensor *label_eliminate_n = THCudaTensor_new(state); 19 | 20 | for (int elt = 0; elt < batchSize; elt ++) { 21 | THCudaTensor_select(state, input_image_n, input_image, 0, elt); 22 | THCudaTensor_select(state, input_edge_n, input_edge, 0, elt); 23 | THCudaTensor_select(state, label_preserve_n, label_preserve, 0, elt); 24 | THCudaTensor_select(state, label_eliminate_n, label_eliminate, 0, elt); 25 | 26 | EdgeDetector(THCState_getCurrentStream(state), 27 | THCudaTensor_data(state, input_image_n), 28 | THCudaTensor_data(state, input_edge_n), 29 | THCudaTensor_data(state, label_preserve_n), 30 | THCudaTensor_data(state, label_eliminate_n), 31 | height, width, isSmoothing); 32 | } 33 | 34 | THCudaTensor_free(state, input_image_n); 35 | THCudaTensor_free(state, input_edge_n); 36 | THCudaTensor_free(state, label_preserve_n); 37 | THCudaTensor_free(state, label_eliminate_n); 38 | } -------------------------------------------------------------------------------- /compilation/EdgeDetector.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #define MAX(x, y) (((x) > (y)) ? (x) : (y)) 4 | #define MIN(x, y) (((x) < (y)) ? (x) : (y)) 5 | 6 | #define PI 3.1415926535897932384626433832 7 | 8 | template 9 | __global__ void EdgeSelection_kernel(const int num_kernels, Dtype* input_edge, Dtype* output, int height, int width) { 10 | 11 | CUDA_KERNEL_LOOP(index, num_kernels) 12 | { 13 | int point_offset = index; 14 | int x = index % width; 15 | int y = index / width; 16 | 17 | Dtype* input_edge_center = input_edge + point_offset; 18 | if (*input_edge_center > 25 && *input_edge_center < 60) 19 | { 20 | int sharpCount = 0; 21 | int smoothCount = 0; 22 | 23 | int window_size = 10; 24 | for (int m = -window_size; m <= window_size; m++) { 25 | for (int n = -window_size; n <= window_size; n++) { 26 | if (m == 0 && n == 0) 27 | continue; 28 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 29 | continue; 30 | 31 | int image_offset = (y + m) * width + x + n; 32 | Dtype* input_edge_offset = input_edge + image_offset; 33 | if (*input_edge_center - *input_edge_offset > 20) 34 | smoothCount++; 35 | } 36 | } 37 | if (smoothCount > 200) 38 | *(output + point_offset) = 1; 39 | } 40 | 41 | if (*input_edge_center >= 60) 42 | { 43 | int sharpCount = 0; 44 | int smoothCount = 0; 45 | 46 | int window_size = 10; 47 | for (int m = -window_size; m <= window_size; m++) { 48 | for (int n = -window_size; n <= window_size; n++) { 49 | if (m == 0 && n == 0) 50 | continue; 51 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 52 | continue; 53 | 54 | int image_offset = (y + m) * width + x + n; 55 | Dtype* input_edge_offset = input_edge + image_offset; 56 | if (*input_edge_center - *input_edge_offset > 30) 57 | smoothCount++; 58 | } 59 | } 60 | if (smoothCount > 200) 61 | *(output + point_offset) = 1; 62 | } 63 | 64 | } 65 | } 66 | 67 | template 68 | __global__ void EdgeFiltering_kernel(const int num_kernels, Dtype* input_edge, Dtype* output, int height, int width) { 69 | 70 | CUDA_KERNEL_LOOP(index, num_kernels) 71 | { 72 | int point_offset = index; 73 | int x = index % width; 74 | int y = index / width; 75 | 76 | Dtype* output_center = output + point_offset; 77 | if(*output_center == 1) 78 | { 79 | int thres = 20; 80 | int count = 0; 81 | int countAll = 0; 82 | 83 | // horizontal 84 | int window_size = 25; 85 | int temp_x = x; 86 | for (int m = 1; m <= window_size; m++) 87 | { 88 | if (y+m < 0 || y+m >= height) 89 | continue; 90 | int precount = count; 91 | for (int n = -1; n <= 1; n++) 92 | { 93 | if (temp_x+n < 0 || temp_x+n >= width) 94 | continue; 95 | Dtype* output_offset = output + (y + m) * width + temp_x + n; 96 | if (*output_offset == 1) 97 | { 98 | temp_x = temp_x + n; 99 | count++; 100 | break; 101 | } 102 | } 103 | if(precount == count) 104 | break; 105 | } 106 | 107 | temp_x = x; 108 | for (int m = -1; m >= -window_size; m--) 109 | { 110 | if (y+m < 0 || y+m >= height) 111 | continue; 112 | int precount = count; 113 | for (int n = -1; n <= 1; n++) 114 | { 115 | if(temp_x+n < 0 || temp_x+n >= width) 116 | continue; 117 | Dtype* output_offset = output + (y + m) * width + temp_x + n; 118 | if (*output_offset == 1) 119 | { 120 | temp_x = temp_x + n; 121 | count++; 122 | break; 123 | } 124 | } 125 | if(precount == count) 126 | break; 127 | } 128 | if (count < thres) 129 | countAll++; 130 | 131 | 132 | //vertical 133 | count = 0; 134 | int temp_y = y; 135 | for (int n = 1; n <= window_size; n++) 136 | { 137 | if (x+n < 0 || x+n >= width) 138 | continue; 139 | int precount = count; 140 | for (int m = -1; m <= 1; m++) 141 | { 142 | if(temp_y+m < 0 || temp_y+m >= height) 143 | continue; 144 | Dtype* output_offset = output + (temp_y + m) * width + x + n; 145 | if (*output_offset == 1) 146 | { 147 | temp_y = temp_y + m; 148 | count++; 149 | break; 150 | } 151 | } 152 | if(precount == count) 153 | break; 154 | } 155 | 156 | temp_y = y; 157 | for (int n = -1; n >= -window_size; n--) 158 | { 159 | if (x+n < 0 || x+n >= width) 160 | continue; 161 | int precount = count; 162 | for (int m = -1; m <= 1; m++) 163 | { 164 | if(temp_y+m < 0 || temp_y+m >= height) 165 | continue; 166 | Dtype* output_offset = output + (temp_y + m) * width + x + n; 167 | if (*output_offset == 1) 168 | { 169 | temp_y = temp_y + m; 170 | count++; 171 | break; 172 | } 173 | } 174 | if(precount == count) 175 | break; 176 | } 177 | if (count < thres) 178 | countAll++; 179 | 180 | 181 | 182 | //diagonal 183 | count = 0; 184 | temp_x = x, temp_y = y; 185 | for (int p = 1; p <= window_size; p++) 186 | { 187 | int m = 0, n = 1; 188 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 189 | continue; 190 | Dtype* output_offset = output + (temp_y + m) * width + temp_x + n; 191 | if (*output_offset == 1) 192 | { 193 | temp_y = temp_y + m; 194 | temp_x = temp_x + n; 195 | count++; 196 | continue; 197 | } 198 | 199 | m = 1, n = 0; 200 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 201 | continue; 202 | output_offset = output + (temp_y + m) * width + temp_x + n; 203 | if (*output_offset == 1) 204 | { 205 | temp_y = temp_y + m; 206 | temp_x = temp_x + n; 207 | count++; 208 | continue; 209 | } 210 | 211 | m = 1, n = 1; 212 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 213 | continue; 214 | output_offset = output + (temp_y + m) * width + temp_x + n; 215 | if (*output_offset == 1) 216 | { 217 | temp_y = temp_y + m; 218 | temp_x = temp_x + n; 219 | count++; 220 | continue; 221 | } 222 | break; 223 | } 224 | 225 | temp_x = x, temp_y = y; 226 | for (int p = 1; p <= window_size; p++) 227 | { 228 | int m = 0, n = -1; 229 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 230 | continue; 231 | Dtype* output_offset = output + (temp_y + m) * width + temp_x + n; 232 | if (*output_offset == 1) 233 | { 234 | temp_y = temp_y + m; 235 | temp_x = temp_x + n; 236 | count++; 237 | continue; 238 | } 239 | 240 | m = -1, n = 0; 241 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 242 | continue; 243 | output_offset = output + (temp_y + m) * width + temp_x + n; 244 | if (*output_offset == 1) 245 | { 246 | temp_y = temp_y + m; 247 | temp_x = temp_x + n; 248 | count++; 249 | continue; 250 | } 251 | 252 | m = -1, n = -1; 253 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 254 | continue; 255 | output_offset = output + (temp_y + m) * width + temp_x + n; 256 | if (*output_offset == 1) 257 | { 258 | temp_y = temp_y + m; 259 | temp_x = temp_x + n; 260 | count++; 261 | continue; 262 | } 263 | break; 264 | } 265 | if (count < thres) 266 | countAll++; 267 | 268 | 269 | 270 | //diagonal -1 271 | count = 0; 272 | temp_x = x, temp_y = y; 273 | for (int p = 1; p <= window_size; p++) 274 | { 275 | int m = 0, n = 1; 276 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 277 | continue; 278 | Dtype* output_offset = output + (temp_y + m) * width + temp_x + n; 279 | if (*output_offset == 1) 280 | { 281 | temp_y = temp_y + m; 282 | temp_x = temp_x + n; 283 | count++; 284 | continue; 285 | } 286 | 287 | m = -1, n = 0; 288 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 289 | continue; 290 | output_offset = output + (temp_y + m) * width + temp_x + n; 291 | if (*output_offset == 1) 292 | { 293 | temp_y = temp_y + m; 294 | temp_x = temp_x + n; 295 | count++; 296 | continue; 297 | } 298 | 299 | m = -1, n = 1; 300 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 301 | continue; 302 | output_offset = output + (temp_y + m) * width + temp_x + n; 303 | if (*output_offset == 1) 304 | { 305 | temp_y = temp_y + m; 306 | temp_x = temp_x + n; 307 | count++; 308 | continue; 309 | } 310 | break; 311 | } 312 | 313 | temp_x = x, temp_y = y; 314 | for (int p = 1; p <= window_size; p++) 315 | { 316 | int m = 0, n = -1; 317 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 318 | continue; 319 | Dtype* output_offset = output + (temp_y + m) * width + temp_x + n; 320 | if (*output_offset == 1) 321 | { 322 | temp_y = temp_y + m; 323 | temp_x = temp_x + n; 324 | count++; 325 | continue; 326 | } 327 | 328 | m = 1, n = 0; 329 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 330 | continue; 331 | output_offset = output + (temp_y + m) * width + temp_x + n; 332 | if (*output_offset == 1) 333 | { 334 | temp_y = temp_y + m; 335 | temp_x = temp_x + n; 336 | count++; 337 | continue; 338 | } 339 | 340 | m = 1, n = -1; 341 | if(temp_y+m < 0 || temp_y+m >= height || temp_x+n < 0 || temp_x+n >= width) 342 | continue; 343 | output_offset = output + (temp_y + m) * width + temp_x + n; 344 | if (*output_offset == 1) 345 | { 346 | temp_y = temp_y + m; 347 | temp_x = temp_x + n; 348 | count++; 349 | continue; 350 | } 351 | break; 352 | } 353 | if (count <= thres) 354 | countAll++; 355 | 356 | if (countAll == 4) 357 | *output_center = 0; 358 | } 359 | } 360 | } 361 | 362 | template 363 | __global__ void EdgeTexture_kernel(const int num_kernels, Dtype* input_image, Dtype* input_edge, Dtype* output, int height, int width) { 364 | 365 | CUDA_KERNEL_LOOP(index, num_kernels) 366 | { 367 | int point_offset = index; 368 | int x = index % width; 369 | int y = index / width; 370 | int image_length = height*width; 371 | 372 | Dtype* input_image_center = input_image + point_offset; 373 | Dtype* input_edge_center = input_edge + point_offset; 374 | if (*input_edge_center > 25) 375 | { 376 | 377 | if (y-1 >= 0 && y+1 < height && x-1 >= 0 && x+1 < width) 378 | { 379 | double y_off = 2 * (*(input_image + (y + 1) * width + x) - *(input_image + (y - 1) * width + x)) + (*(input_image + (y + 1) * width + x + 1) - *(input_image + (y - 1) * width + x + 1)) + (*(input_image + (y + 1) * width + x - 1) - *(input_image + (y - 1) * width + x - 1)); 380 | double x_off = 2 * (*(input_image + y * width + x + 1) - *(input_image + y * width + x - 1)) + (*(input_image + (y+1) * width + x + 1) - *(input_image + (y+1) * width + x - 1)) + (*(input_image + (y-1) * width + x + 1) - *(input_image + (y-1) * width + x - 1)); 381 | 382 | double angle = 0; 383 | if (x_off == 0){ 384 | if (y_off > 0) 385 | angle = PI / 2; 386 | else if (y_off <= 0) 387 | angle = PI*1.5; 388 | } 389 | else if (y_off == 0){ 390 | if (x_off > 0) 391 | angle = 0; 392 | else if (x_off <= 0) 393 | angle = PI; 394 | } 395 | else 396 | angle = atan2(y_off, x_off); 397 | 398 | if (angle < 0) 399 | angle += PI; 400 | 401 | int point_dis = 2; 402 | int a_x = x + point_dis*cos(angle); 403 | int a_y = y + point_dis*sin(angle); 404 | int b_x = x + point_dis*cos(angle + PI); 405 | int b_y = y + point_dis*sin(angle + PI); 406 | 407 | double averageA = 0; 408 | int countA = 0; 409 | int window_size = 1; 410 | for (int m = -window_size; m <= window_size; m++) { 411 | for (int n = -window_size; n <= window_size; n++) { 412 | if (m == 0 && n == 0) 413 | continue; 414 | if (a_y+m < 0 || a_y+m >= height || a_x+n < 0 || a_x+n >= width) 415 | continue; 416 | 417 | int image_offset = (a_y + m) * width + a_x + n; 418 | Dtype* input_image_offset = input_image + image_offset; 419 | // if (fabs(*input_image_center - *input_image_offset) > 20) 420 | { 421 | averageA = averageA + *input_image_offset; 422 | countA++; 423 | } 424 | } 425 | } 426 | averageA = averageA / countA; 427 | 428 | double averageB = 0; 429 | int countB = 0; 430 | for (int m = -window_size; m <= window_size; m++) { 431 | for (int n = -window_size; n <= window_size; n++) { 432 | if (m == 0 && n == 0) 433 | continue; 434 | if (b_y+m < 0 || b_y+m >= height || b_x+n < 0 || b_x+n >= width) 435 | continue; 436 | 437 | int image_offset = (b_y + m) * width + b_x + n; 438 | Dtype* input_image_offset = input_image + image_offset; 439 | // if (fabs(*input_image_center - *input_image_offset) > 20) 440 | { 441 | averageB = averageB + *input_image_offset; 442 | countB++; 443 | } 444 | } 445 | } 446 | averageB = averageB / countB; 447 | 448 | if (fabs(averageA-averageB) < 50) 449 | *(output + point_offset) = 1; 450 | } 451 | } 452 | } 453 | } 454 | 455 | template 456 | void EdgeDetector(cudaStream_t stream, Dtype* input_image, Dtype* input_edge, Dtype* output_preserve, Dtype* output_eliminate, int height, int width, int isSmoothing) 457 | { 458 | int dimSize = 1024; 459 | int num_kernels = height * width; 460 | int grid = (num_kernels + dimSize - 1) / dimSize; 461 | 462 | if (isSmoothing == 1){ 463 | // structure extraction for edge-preserving smoothing 464 | EdgeSelection_kernel<<>>(num_kernels, input_edge, output_preserve, height, width); 465 | EdgeFiltering_kernel<<>>(num_kernels, input_edge, output_preserve, height, width); 466 | }else{ 467 | // structure extraction for texture removal 468 | EdgeTexture_kernel<<>>(num_kernels, input_image, input_edge, output_eliminate, height, width); 469 | } 470 | } -------------------------------------------------------------------------------- /compilation/EdgeDetector.lua: -------------------------------------------------------------------------------- 1 | local THNN = require 'nn.THNN' 2 | local EdgeDetector, parent = torch.class('nn.EdgeDetector', 'nn.Module') 3 | 4 | function EdgeDetector:__init(isSmoothing) 5 | parent.__init(self) 6 | self.isSmoothing = isSmoothing 7 | end 8 | 9 | function EdgeDetector:updateOutput(input) 10 | local input_image = input[{{},{1,3},{},{}}] 11 | local input_edge = input[{{},{4},{},{}}] 12 | local bs,dim,height,width = input_edge:size(1),input_edge:size(2),input_edge:size(3),input_edge:size(4) 13 | 14 | input_image = torch.sum(input_image,2)/3 15 | self.output_preserve = torch.CudaTensor():resizeAs(input_edge):fill(0) 16 | self.output_eliminate = torch.CudaTensor():resizeAs(input_edge):fill(0) 17 | self.output = torch.CudaTensor():resize(bs,dim*2,height,width):fill(0) 18 | 19 | input.THNN.EdgeDetector_updateOutput( 20 | input_image:cdata(), 21 | input_edge:cdata(), 22 | self.output_preserve:cdata(), 23 | self.output_eliminate:cdata(), 24 | self.isSmoothing 25 | ) 26 | 27 | self.output[{{},{1},{},{}}] = self.output_preserve 28 | self.output[{{},{2},{},{}}] = self.output_eliminate 29 | 30 | return self.output 31 | end -------------------------------------------------------------------------------- /compilation/SmoothAndEdgeTerm.cu: -------------------------------------------------------------------------------- 1 | #include "THCUNN.h" 2 | #include "common.h" 3 | #include "im2col.h" 4 | #include "SmoothAndEdgeTerm.h" 5 | 6 | void THNN_CudaSmoothAndEdgeTerm_updateOutput(THCState *state, THCudaTensor *input_cnn, THCudaTensor *input_edge, THCudaTensor *target_yuv, THCudaTensor *target_edge, THCudaTensor *target_edge_mask, THCudaTensor *smooth_mask_pre, THCudaTensor *smooth_mask, THCudaTensor *weight, THCudaTensor *output, float sigma_color, float sigma_space, int window_size, float lp, int isDetailEnhancement, int isStylization, int w_L2) { 7 | 8 | long batchSize = input_cnn->size[0]; 9 | long plane = input_cnn->size[1]; 10 | long height = input_cnn->size[2]; 11 | long width = input_cnn->size[3]; 12 | 13 | THCudaTensor_resize4d(state, weight, batchSize, (window_size*2+1) * (window_size*2+1), 14 | height, width); 15 | THCudaTensor_fill(state, weight, 0); 16 | 17 | THCudaTensor *input_cnn_n = THCudaTensor_new(state); 18 | THCudaTensor *input_edge_n = THCudaTensor_new(state); 19 | THCudaTensor *target_yuv_n = THCudaTensor_new(state); 20 | THCudaTensor *target_edge_n = THCudaTensor_new(state); 21 | THCudaTensor *target_edge_mask_n = THCudaTensor_new(state); 22 | THCudaTensor *smooth_mask_pre_n = THCudaTensor_new(state); 23 | THCudaTensor *smooth_mask_n = THCudaTensor_new(state); 24 | THCudaTensor *weight_n = THCudaTensor_new(state); 25 | THCudaTensor *output_n = THCudaTensor_new(state); 26 | 27 | for (int elt = 0; elt < batchSize; elt++) { 28 | // Matrix mulitply per output: 29 | THCudaTensor_select(state, input_cnn_n, input_cnn, 0, elt); 30 | THCudaTensor_select(state, input_edge_n, input_edge, 0, elt); 31 | THCudaTensor_select(state, target_yuv_n, target_yuv, 0, elt); 32 | THCudaTensor_select(state, target_edge_n, target_edge, 0, elt); 33 | THCudaTensor_select(state, target_edge_mask_n, target_edge_mask, 0, elt); 34 | THCudaTensor_select(state, smooth_mask_pre_n, smooth_mask_pre, 0, elt); 35 | THCudaTensor_select(state, smooth_mask_n, smooth_mask, 0, elt); 36 | THCudaTensor_select(state, weight_n, weight, 0, elt); 37 | THCudaTensor_select(state, output_n, output, 0, elt); 38 | 39 | SmoothAndEdgeTerm_Loss_forward(THCState_getCurrentStream(state), 40 | THCudaTensor_data(state, input_cnn_n), 41 | THCudaTensor_data(state, input_edge_n), 42 | THCudaTensor_data(state, target_yuv_n), 43 | THCudaTensor_data(state, target_edge_n), 44 | THCudaTensor_data(state, target_edge_mask_n), 45 | THCudaTensor_data(state, smooth_mask_pre_n), 46 | THCudaTensor_data(state, smooth_mask_n), 47 | THCudaTensor_data(state, weight_n), 48 | THCudaTensor_data(state, output_n), 49 | sigma_color, sigma_space, window_size, lp, height, width, isDetailEnhancement, isStylization, w_L2); 50 | } 51 | 52 | // Free 53 | THCudaTensor_free(state, input_cnn_n); 54 | THCudaTensor_free(state, input_edge_n); 55 | THCudaTensor_free(state, target_yuv_n); 56 | THCudaTensor_free(state, target_edge_n); 57 | THCudaTensor_free(state, target_edge_mask_n); 58 | THCudaTensor_free(state, smooth_mask_pre_n); 59 | THCudaTensor_free(state, smooth_mask_n); 60 | THCudaTensor_free(state, weight_n); 61 | THCudaTensor_free(state, output_n); 62 | } 63 | 64 | void THNN_CudaSmoothAndEdgeTerm_updateGradInput(THCState *state, THCudaTensor *input_cnn, THCudaTensor *smooth_mask, THCudaTensor *target_edge_mask, THCudaTensor *weight, THCudaTensor *gradInput, float sigma_color, int window_size, float lp, int w_L2) { 65 | 66 | long batchSize = input_cnn->size[0]; 67 | long plane = input_cnn->size[1]; 68 | long height = input_cnn->size[2]; 69 | long width = input_cnn->size[3]; 70 | 71 | THCudaTensor *input_cnn_n = THCudaTensor_new(state); 72 | THCudaTensor *smooth_mask_n = THCudaTensor_new(state); 73 | THCudaTensor *target_edge_mask_n = THCudaTensor_new(state); 74 | THCudaTensor *weight_n = THCudaTensor_new(state); 75 | THCudaTensor *gradInput_n = THCudaTensor_new(state); 76 | 77 | for (int elt = 0; elt < batchSize; elt++) { 78 | THCudaTensor_select(state, input_cnn_n, input_cnn, 0, elt); 79 | THCudaTensor_select(state, smooth_mask_n, smooth_mask, 0, elt); 80 | THCudaTensor_select(state, target_edge_mask_n, target_edge_mask, 0, elt); 81 | THCudaTensor_select(state, weight_n, weight, 0, elt); 82 | THCudaTensor_select(state, gradInput_n, gradInput, 0, elt); 83 | 84 | SmoothAndEdgeTerm_Loss_backward(THCState_getCurrentStream(state), 85 | THCudaTensor_data(state, input_cnn_n), 86 | THCudaTensor_data(state, smooth_mask_n), 87 | THCudaTensor_data(state, target_edge_mask_n), 88 | THCudaTensor_data(state, weight_n), 89 | THCudaTensor_data(state, gradInput_n), 90 | sigma_color, window_size, lp, height, width, w_L2); 91 | } 92 | 93 | // Free 94 | THCudaTensor_free(state, input_cnn_n); 95 | THCudaTensor_free(state, smooth_mask_n); 96 | THCudaTensor_free(state, target_edge_mask_n); 97 | THCudaTensor_free(state, weight_n); 98 | THCudaTensor_free(state, gradInput_n); 99 | } 100 | -------------------------------------------------------------------------------- /compilation/SmoothAndEdgeTerm.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #define MAX(x, y) (((x) > (y)) ? (x) : (y)) 4 | #define MIN(x, y) (((x) < (y)) ? (x) : (y)) 5 | 6 | template 7 | __global__ void SmoothAndEdgeTerm_Loss_forward_main_kernel(const int num_kernels, Dtype* input_cnn, Dtype* target_yuv, Dtype* smooth_mask, Dtype* target_edge_mask, Dtype* weight, Dtype* output, float sigma_color, float sigma_space, int window_size, float lp, int height, int width, int w_L2) { 8 | 9 | CUDA_KERNEL_LOOP(index, num_kernels) 10 | { 11 | int point_offset = index; 12 | int x = index % width; 13 | int y = index / width; 14 | int window_length = window_size * 2 + 1; 15 | int image_length = height * width; 16 | 17 | for (int m = -window_size; m <= window_size; m++) { 18 | for (int n = -window_size; n <= window_size; n++) { 19 | 20 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 21 | continue; 22 | int image_offset = (y + m) * width + x + n; 23 | 24 | int window_offset = point_offset + ((m+window_size) * window_length + n+window_size) * image_length; 25 | Dtype* target_yuv_center = target_yuv + point_offset; 26 | Dtype* target_yuv_offset = target_yuv + image_offset; 27 | Dtype* input_cnn_center = input_cnn + point_offset; 28 | Dtype* input_cnn_offset = input_cnn + image_offset; 29 | 30 | if (*(smooth_mask + point_offset) == 0) 31 | { 32 | *(weight + window_offset) = exp((powf(fabs(*target_yuv_offset - *target_yuv_center),2) + powf(fabs(*(target_yuv_offset + image_length) - *(target_yuv_center + image_length)),2) + powf(fabs(*(target_yuv_offset + image_length * 2) - *(target_yuv_center + image_length * 2)),2)) * sigma_color); 33 | if (*(weight + window_offset) < 0.3) 34 | *(weight + window_offset) = 0; 35 | }else{ 36 | *(weight + window_offset) = exp((m*m+n*n) * sigma_space); 37 | } 38 | 39 | for (int p = 0; p < 3; p++) 40 | { 41 | if (*(target_edge_mask + point_offset) == 0) 42 | { 43 | if (*(smooth_mask + point_offset) == 0) 44 | *(output + point_offset + image_length*p) += *(weight + window_offset) * powf(fabs(*(input_cnn_center + image_length*p) - *(input_cnn_offset + image_length*p)),lp); 45 | else 46 | *(output + point_offset + image_length*p) += *(weight + window_offset) * powf(fabs(*(input_cnn_center + image_length*p) - *(input_cnn_offset + image_length*p)),2) * w_L2; 47 | } 48 | } 49 | } 50 | } 51 | } 52 | } 53 | 54 | template 55 | __global__ void SmoothAndEdgeTerm_Loss_forward_pre1_kernel(const int num_kernels, Dtype* input_edge, Dtype* target_edge, Dtype* smooth_mask_pre, int height, int width, int isDetailEnhancement) { 56 | 57 | CUDA_KERNEL_LOOP(index, num_kernels) 58 | { 59 | int point_offset = index; 60 | int x = index % width; 61 | int y = index / width; 62 | 63 | Dtype* input_center = input_edge + point_offset; 64 | Dtype* target_center = target_edge + point_offset; 65 | 66 | if (isDetailEnhancement == 1){ 67 | if (*input_center - *target_center > 0) 68 | *(smooth_mask_pre + point_offset) = 1; 69 | } 70 | else{ 71 | if (*target_center < 20 && *input_center - *target_center > 10) 72 | *(smooth_mask_pre + point_offset) = 1; 73 | } 74 | if (*target_center == 0 && *input_center - *target_center > 0) 75 | *(smooth_mask_pre + point_offset) = 1; 76 | } 77 | } 78 | 79 | template 80 | __global__ void SmoothAndEdgeTerm_Loss_forward_pre2_kernel(const int num_kernels, Dtype* smooth_mask_pre, Dtype* smooth_mask, int height, int width, int isStylization) { 81 | 82 | CUDA_KERNEL_LOOP(index, num_kernels) 83 | { 84 | int point_offset = index; 85 | int x = index % width; 86 | int y = index / width; 87 | 88 | if (isStylization == 1){ 89 | int window_size = 3; 90 | for (int m = -window_size; m <= window_size; m++) { 91 | for (int n = -window_size; n <= window_size; n++) { 92 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 93 | continue; 94 | int image_offset = (y + m) * width + x + n; 95 | 96 | if (*(smooth_mask_pre + image_offset) == 1) 97 | { 98 | *(smooth_mask + point_offset) = 1; 99 | break; 100 | } 101 | } 102 | } 103 | } 104 | 105 | if (*(smooth_mask_pre + point_offset) == 1) 106 | *(smooth_mask + point_offset) = 1; 107 | } 108 | } 109 | 110 | 111 | template 112 | void SmoothAndEdgeTerm_Loss_forward(cudaStream_t stream, Dtype* input_cnn, Dtype* input_edge, Dtype* target_yuv, Dtype* target_edge, Dtype* target_edge_mask, Dtype* smooth_mask_pre, Dtype* smooth_mask, Dtype* weight, Dtype* output, float sigma_color, float sigma_space, int window_size, float lp, int height, int width, int isDetailEnhancement, int isStylization, int w_L2) 113 | { 114 | int dimSize = 1024; 115 | int num_kernels = height * width; 116 | int grid = (num_kernels + dimSize - 1) / dimSize; 117 | 118 | SmoothAndEdgeTerm_Loss_forward_pre1_kernel<<>>(num_kernels, input_edge, target_edge, smooth_mask_pre, height, width, isDetailEnhancement); 119 | SmoothAndEdgeTerm_Loss_forward_pre2_kernel<<>>(num_kernels, smooth_mask_pre, smooth_mask, height, width, isStylization); 120 | 121 | SmoothAndEdgeTerm_Loss_forward_main_kernel<<>>(num_kernels, input_cnn, target_yuv, smooth_mask, target_edge_mask, weight, output, sigma_color, sigma_space, window_size, lp, height, width, w_L2); 122 | } 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | template 131 | __global__ void SmoothAndEdgeTerm_Loss_backward_kernel(const int num_kernels, Dtype* input_cnn, Dtype* smooth_mask, Dtype* target_edge_mask, Dtype* weight, Dtype* gradInput, float sigma_color, int window_size, float lp, int height, int width, int w_L2) { 132 | 133 | CUDA_KERNEL_LOOP(index, num_kernels) 134 | { 135 | int point_offset = index; 136 | int x = index % width; 137 | int y = index / width; 138 | int window_length = window_size * 2 + 1; 139 | int image_length = height * width; 140 | 141 | for (int m = -window_size; m <= window_size; m++) { 142 | for (int n = -window_size; n <= window_size; n++) { 143 | if (y+m < 0 || y+m >= height || x+n < 0 || x+n >= width) 144 | continue; 145 | int image_offset = (y + m) * width + x + n; 146 | 147 | int window_offset = image_offset + ((-m + window_size) * window_length - n + window_size) * image_length; 148 | Dtype* input_cnn_center = input_cnn + point_offset; 149 | Dtype* input_cnn_offset = input_cnn + image_offset; 150 | 151 | for (int p = 0; p < 3; p++) 152 | { 153 | if (*(target_edge_mask + point_offset) == 0) 154 | { 155 | if (*(smooth_mask + point_offset) == 0) 156 | *(gradInput + point_offset + image_length*p) += *(weight + window_offset) * lp * MIN(powf(fabs(*(input_cnn_center + image_length*p) - *(input_cnn_offset + image_length*p)),lp-1),1) * (*(input_cnn_center + image_length*p) > *(input_cnn_offset + image_length*p) ? 1 : -1); 157 | else 158 | *(gradInput + point_offset + image_length*p) += *(weight + window_offset) * 2 * MIN(powf(fabs(*(input_cnn_center + image_length*p) - *(input_cnn_offset + image_length*p)),2 - 1),1) * (*(input_cnn_center + image_length*p) > *(input_cnn_offset + image_length*p) ? 1 : -1) * w_L2; 159 | } 160 | if (*(target_edge_mask + image_offset) == 0) 161 | { 162 | if (*(smooth_mask + image_offset) == 0) 163 | *(gradInput + point_offset + image_length*p) += *(weight + window_offset) * lp * MIN(powf(fabs(*(input_cnn_center + image_length*p) - *(input_cnn_offset + image_length*p)),lp-1),1) * (*(input_cnn_center + image_length*p) > *(input_cnn_offset + image_length*p) ? 1 : -1); 164 | else 165 | *(gradInput + point_offset + image_length*p) += *(weight + window_offset) * 2 * MIN(powf(fabs(*(input_cnn_center + image_length*p) - *(input_cnn_offset + image_length*p)),2 - 1),1) * (*(input_cnn_center + image_length*p) > *(input_cnn_offset + image_length*p) ? 1 : -1) * w_L2; 166 | } 167 | } 168 | } 169 | } 170 | 171 | } 172 | } 173 | 174 | template 175 | void SmoothAndEdgeTerm_Loss_backward(cudaStream_t stream, Dtype* input_cnn, Dtype* smooth_mask, Dtype* target_edge_mask, Dtype* weight, Dtype* gradInput, float sigma_color,int window_size, float lp, int height, int width, int w_L2) 176 | { 177 | int dimSize = 1024; 178 | int num_kernels = height * width; 179 | int grid = (num_kernels + dimSize - 1) / dimSize; 180 | SmoothAndEdgeTerm_Loss_backward_kernel<<>>(num_kernels, input_cnn, smooth_mask, target_edge_mask, weight, gradInput, sigma_color, window_size, lp, height, width, w_L2); 181 | } -------------------------------------------------------------------------------- /compilation/SmoothAndEdgeTerm.lua: -------------------------------------------------------------------------------- 1 | local THNN = require 'nn.THNN' 2 | local SmoothAndEdgeTerm, parent = torch.class('nn.SmoothAndEdgeTerm', 'nn.Criterion') 3 | 4 | function SmoothAndEdgeTerm:__init(sigma_color,sigma_space,window_size,lp,w_smooth,w_edge,w_L2,isDetailEnhancement,isStylization) 5 | parent.__init(self) 6 | 7 | self.sigma_color = - 1 / (sigma_color*sigma_color*2) 8 | self.sigma_space = - 1 / (sigma_space*sigma_space*2) 9 | self.window_size = window_size 10 | self.lp = lp 11 | self.w_smooth = w_smooth 12 | self.w_edge = w_edge 13 | self.w_L2 = w_L2 14 | self.isDetailEnhancement = isDetailEnhancement 15 | self.isStylization = isStylization 16 | end 17 | 18 | function SmoothAndEdgeTerm:updateOutput(input, target) 19 | local input_cnn = input[{{},{1,3},{},{}}] 20 | local input_edge = input[{{},{4},{},{}}] 21 | local target_yuv = target[{{},{1,3},{},{}}] 22 | local target_edge = target[{{},{4},{},{}}] 23 | local target_edge_mask = target[{{},{5},{},{}}] 24 | 25 | self.output_smooth = torch.CudaTensor():resizeAs(input_cnn):zero() 26 | self.smooth_mask_pre = torch.CudaTensor():resizeAs(input_edge):zero() 27 | self.smooth_mask = torch.CudaTensor():resizeAs(input_edge):zero() 28 | self.w = torch.CudaTensor():zero() 29 | 30 | input.THNN.SmoothAndEdgeTerm_updateOutput( 31 | input_cnn:cdata(), 32 | input_edge:cdata(), 33 | target_yuv:cdata(), 34 | target_edge:cdata(), 35 | target_edge_mask:cdata(), 36 | self.smooth_mask_pre:cdata(), 37 | self.smooth_mask:cdata(), 38 | self.w:cdata(), 39 | self.output_smooth:cdata(), 40 | self.sigma_color, 41 | self.sigma_space, 42 | self.window_size, 43 | self.lp, 44 | self.isDetailEnhancement, 45 | self.isStylization, 46 | self.w_L2 47 | ) 48 | 49 | local loss_edge = 0 50 | if torch.sum(target_edge_mask) ~= 0 then 51 | local sub = torch.csub(input_edge,target_edge) 52 | self.gt = sub[target_edge_mask] 53 | loss_edge = torch.mean(torch.pow(self.gt,2)) 54 | end 55 | local loss_smooth = torch.mean(self.output_smooth) 56 | 57 | self.output = loss_edge * self.w_edge + loss_smooth * self.w_smooth 58 | 59 | return self.output 60 | end 61 | 62 | function SmoothAndEdgeTerm:updateGradInput(input, target) 63 | local input_cnn = input[{{},{1,3},{},{}}] 64 | local input_edge = input[{{},{4},{},{}}] 65 | local target_yuv = target[{{},{1,3},{},{}}] 66 | local target_edge = target[{{},{4},{},{}}] 67 | local target_edge_mask = target[{{},{5},{},{}}] 68 | local gradInput_smooth = torch.CudaTensor():resizeAs(input_cnn):zero() 69 | local gradInput_edge = torch.CudaTensor():resizeAs(input_edge):zero() 70 | 71 | input.THNN.SmoothAndEdgeTerm_updateGradInput( 72 | input_cnn:cdata(), 73 | self.smooth_mask:cdata(), 74 | target_edge_mask:cdata(), 75 | self.w:cdata(), 76 | gradInput_smooth:cdata(), 77 | self.sigma_color, 78 | self.window_size, 79 | self.lp, 80 | self.w_L2 81 | ) 82 | 83 | if torch.sum(target_edge_mask) ~= 0 then 84 | gradInput_edge[target_edge_mask] = 2*self.gt/self.gt:size(1) 85 | end 86 | 87 | self.gradInput = torch.CudaTensor():resizeAs(input):zero() 88 | self.gradInput[{{},{1,3},{},{}}] = self.w_smooth * gradInput_smooth / (3 * input_cnn:size(3) * input_cnn:size(4)) 89 | self.gradInput[{{},{4},{},{}}] = self.w_edge * gradInput_edge 90 | 91 | return self.gradInput 92 | end 93 | 94 | function SmoothAndEdgeTerm:clearState() 95 | nn.utils.clear(self, 'w', 'smooth_mask_pre', 'smooth_mask', 'gt', 'output_smooth') 96 | return parent.clearState(self) 97 | end -------------------------------------------------------------------------------- /compilation/adam_state.lua: -------------------------------------------------------------------------------- 1 | --[[ An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf 2 | 3 | ARGS: 4 | 5 | - 'opfunc' : a function that takes a single input (X), the point 6 | of a evaluation, and returns f(X) and df/dX 7 | - 'x' : the initial point 8 | - 'config` : a table with configuration parameters for the optimizer 9 | - 'config.learningRate' : learning rate 10 | - 'config.beta1' : first moment coefficient 11 | - 'config.beta2' : second moment coefficient 12 | - 'config.epsilon' : for numerical stability 13 | - 'state' : a table describing the state of the optimizer; after each 14 | call the state is modified 15 | 16 | RETURN: 17 | - `x` : the new x vector 18 | - `f(x)` : the function, evaluated before the update 19 | 20 | ]] 21 | 22 | function optim.adam_state(opfunc, x, config, state) 23 | -- (0) get/update state 24 | local config = config or {} 25 | local state = state or config 26 | local lr = config.learningRate or 0.001 27 | 28 | local beta1 = config.beta1 or 0.9 29 | local beta2 = config.beta2 or 0.999 30 | local epsilon = config.epsilon or 1e-8 31 | 32 | -- (1) evaluate f(x) and df/dx 33 | local fx, dfdx = opfunc(x) 34 | 35 | -- Initialization 36 | state.t = state.t or 0 37 | -- Exponential moving average of gradient values 38 | state.m = state.m or x.new(dfdx:size()):zero() 39 | -- Exponential moving average of squared gradient values 40 | state.v = state.v or x.new(dfdx:size()):zero() 41 | -- A tmp tensor to hold the sqrt(v) + epsilon 42 | state.denom = state.denom or x.new(dfdx:size()):zero() 43 | 44 | state.t = state.t + 1 45 | 46 | -- Decay the first and second moment running average coefficient 47 | state.m:mul(beta1):add(1-beta1, dfdx) 48 | state.v:mul(beta2):addcmul(1-beta2, dfdx, dfdx) 49 | 50 | state.denom:copy(state.v):sqrt():add(epsilon) 51 | 52 | local biasCorrection1 = 1 - beta1^state.t 53 | local biasCorrection2 = 1 - beta2^state.t 54 | local stepSize = lr * math.sqrt(biasCorrection2)/biasCorrection1 55 | -- (2) update x 56 | x:addcdiv(-stepSize, state.m, state.denom) 57 | 58 | -- return x*, f(x) before optimization 59 | return x, {fx}, state 60 | end 61 | -------------------------------------------------------------------------------- /detail_enhancement.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/detail_enhancement.PNG -------------------------------------------------------------------------------- /foreground_enhance.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/foreground_enhance.PNG -------------------------------------------------------------------------------- /foreground_smooth.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/foreground_smooth.PNG -------------------------------------------------------------------------------- /images/content_smoothing/blur-previous-5-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur-previous-5-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/blur-previous-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur-previous-5.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-10-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-10-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-10-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-10-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-10-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-10-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-11-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-11-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-11-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-11-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-5-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-5-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-5-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-5-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-7-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-7-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/blur10-500-7-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur10-500-7-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-1-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-1-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-1-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-1-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-3-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-3-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-3-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-3-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-4-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-4-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-4-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-4-input.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-8-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-8-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/blur11-500-8-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/blur11-500-8-input.png -------------------------------------------------------------------------------- /images/content_smoothing/demo11-500-13-input-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo11-500-13-input-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/demo11-500-13-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo11-500-13-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/demo11-500-22-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo11-500-22-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/demo11-500-22-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo11-500-22-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/demo11-500-22-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo11-500-22-input.png -------------------------------------------------------------------------------- /images/content_smoothing/demo12-500-25-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo12-500-25-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/demo12-500-25-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo12-500-25-input-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/demo12-500-25-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo12-500-25-input.png -------------------------------------------------------------------------------- /images/content_smoothing/demo12-500-9-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo12-500-9-input-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/demo12-500-9-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/demo12-500-9-input.png -------------------------------------------------------------------------------- /images/content_smoothing/sig_v3_input_19-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/sig_v3_input_19-predict-detail.png -------------------------------------------------------------------------------- /images/content_smoothing/sig_v3_input_19-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/sig_v3_input_19-predict.png -------------------------------------------------------------------------------- /images/content_smoothing/sig_v3_input_19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/content_smoothing/sig_v3_input_19.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail-previous-2-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail-previous-2-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail-previous-2-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail-previous-2-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail-previous-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail-previous-2.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail-previous-6-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail-previous-6-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail-previous-6-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail-previous-6-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail-previous-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail-previous-6.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_4-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_4-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_4-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_4-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_4.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_5-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_5-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_5-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_5-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_5.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_6-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_6-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_6-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_6-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/detail_2_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/detail_2_6.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-11-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-11-input-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-11-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-11-input-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-11-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-11-input.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-14-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-14-input-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-14-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-14-input-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-14-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-14-input.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-17-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-17-input-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-17-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-17-input-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-17-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-17-input.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-3-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-3-input-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-3-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-3-input-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-3-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-3-input.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-5-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-5-input-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-5-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-5-input-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-5-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-5-input.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-6-input-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-6-input-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-6-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-6-input-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/flower10-500-6-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/flower10-500-6-input.png -------------------------------------------------------------------------------- /images/detail_enhancement/sig_v2_input_76-predict-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/sig_v2_input_76-predict-detail.png -------------------------------------------------------------------------------- /images/detail_enhancement/sig_v2_input_76-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/sig_v2_input_76-predict.png -------------------------------------------------------------------------------- /images/detail_enhancement/sig_v2_input_76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/detail_enhancement/sig_v2_input_76.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/demo-13-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/demo-13-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/demo-13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/demo-13.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/demo-new5-16-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/demo-new5-16-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/demo-new5-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/demo-new5-16.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/demo9-600-5-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/demo9-600-5-input-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/demo9-600-5-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/demo9-600-5-input.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_input_47-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_input_47-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_input_47.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_input_47.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_v2_input_42-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_v2_input_42-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_v2_input_42.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_v2_input_42.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_v2_input_72-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_v2_input_72-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_v2_input_72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_v2_input_72.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_v3_input_45-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_v3_input_45-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/sig_v3_input_45.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/sig_v3_input_45.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth-63-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth-63-input-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth-63-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth-63-input.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth-previous-1-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth-previous-1-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth-previous-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth-previous-1.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_10-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_10-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_10.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_15-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_15-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_15.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_2-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_2-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_2.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_4-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_4-predict.png -------------------------------------------------------------------------------- /images/edge_preserving_smoothing/smooth_2_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/edge_preserving_smoothing/smooth_2_4.png -------------------------------------------------------------------------------- /images/stylization/demo-new-21-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/demo-new-21-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/demo-new-21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/demo-new-21.png -------------------------------------------------------------------------------- /images/stylization/demo8-2-input-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/demo8-2-input-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/demo8-2-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/demo8-2-input.png -------------------------------------------------------------------------------- /images/stylization/sig_v2_input_31-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/sig_v2_input_31-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/sig_v2_input_31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/sig_v2_input_31.png -------------------------------------------------------------------------------- /images/stylization/sig_v2_input_46-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/sig_v2_input_46-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/sig_v2_input_46.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/sig_v2_input_46.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_11-predict-draw-woTone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_11-predict-draw-woTone.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_11-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_11-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_11.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_18-predict-draw-woTone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_18-predict-draw-woTone.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_18-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_18-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_18.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_19-predict-draw-woTone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_19-predict-draw-woTone.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_19-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_19-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/smooth_2_19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/smooth_2_19.png -------------------------------------------------------------------------------- /images/stylization/style-1-predict-draw-woTone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-1-predict-draw-woTone.png -------------------------------------------------------------------------------- /images/stylization/style-1-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-1-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/style-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-1.png -------------------------------------------------------------------------------- /images/stylization/style-22-predict-draw-woTone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-22-predict-draw-woTone.png -------------------------------------------------------------------------------- /images/stylization/style-22-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-22-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/style-22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-22.png -------------------------------------------------------------------------------- /images/stylization/style-40-predict-draw-woTone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-40-predict-draw-woTone.png -------------------------------------------------------------------------------- /images/stylization/style-40-predict-draw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-40-predict-draw.png -------------------------------------------------------------------------------- /images/stylization/style-40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/stylization/style-40.png -------------------------------------------------------------------------------- /images/texture_removal/RTV-51-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/RTV-51-predict.png -------------------------------------------------------------------------------- /images/texture_removal/RTV-51.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/RTV-51.png -------------------------------------------------------------------------------- /images/texture_removal/texture10-500-10-input-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/texture10-500-10-input-predict.png -------------------------------------------------------------------------------- /images/texture_removal/texture10-500-10-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/texture10-500-10-input.png -------------------------------------------------------------------------------- /images/texture_removal/texture10-500-3-input-predict21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/texture10-500-3-input-predict21.png -------------------------------------------------------------------------------- /images/texture_removal/texture10-500-3-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/texture10-500-3-input.png -------------------------------------------------------------------------------- /images/texture_removal/texture_2_1-predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/texture_2_1-predict.png -------------------------------------------------------------------------------- /images/texture_removal/texture_2_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/images/texture_removal/texture_2_1.png -------------------------------------------------------------------------------- /netfiles/model_smooth_background-smooth_30.net: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/netfiles/model_smooth_background-smooth_30.net -------------------------------------------------------------------------------- /netfiles/model_smooth_detail-enhance_30.net: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/netfiles/model_smooth_detail-enhance_30.net -------------------------------------------------------------------------------- /netfiles/model_smooth_foreground-enhance_30.net: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/netfiles/model_smooth_foreground-enhance_30.net -------------------------------------------------------------------------------- /netfiles/model_smooth_stylization_30.net: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/netfiles/model_smooth_stylization_30.net -------------------------------------------------------------------------------- /netfiles/model_smooth_texture-removal_30.net: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/netfiles/model_smooth_texture-removal_30.net -------------------------------------------------------------------------------- /pencils/pencil0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/pencils/pencil0.png -------------------------------------------------------------------------------- /pencils/pencil1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/pencils/pencil1.png -------------------------------------------------------------------------------- /pencils/pencil2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/pencils/pencil2.png -------------------------------------------------------------------------------- /pencils/pencil3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/pencils/pencil3.png -------------------------------------------------------------------------------- /pencils/pencil4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/pencils/pencil4.png -------------------------------------------------------------------------------- /post_detail_enhance.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | addpath('./util'); 3 | 4 | inputDir = '.\test_detail_enhance\'; 5 | outputDir = '.\test_detail_enhance\'; 6 | images = dir([inputDir '*-input.png']); 7 | for m = 1:length(images) 8 | inputname = [inputDir images(m).name]; 9 | outputname = strrep(inputname,'input','detail-enhance'); 10 | input = double(imread(inputname))/255; 11 | [rows, columns, numberOfColorChannels] = size(input); 12 | if numberOfColorChannels == 1 13 | temp = input; 14 | input = zeros(rows,columns,3); 15 | input(:,:,1) = temp; 16 | input(:,:,2) = temp; 17 | input(:,:,3) = temp; 18 | end 19 | cform = makecform('srgb2lab'); 20 | input_lab = applycform(input, cform); 21 | input_l = input_lab(:,:,1); 22 | 23 | smoothname = strrep(inputname,'input.png','smooth.png'); 24 | smooth = double(imread(smoothname))/255; 25 | smooth_lab = applycform(smooth, cform); 26 | smooth_l = smooth_lab(:,:,1); 27 | 28 | val0 = 15; 29 | val2 = 1; 30 | exposure = 1.0; 31 | saturation = 1.0; 32 | gamma = 1.0; 33 | 34 | output = tonemapLAB_simple(input_lab,smooth_l,input_l,val0,val2,exposure,gamma,saturation); 35 | 36 | imwrite(output, outputname); 37 | end -------------------------------------------------------------------------------- /post_stylization.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | addpath('./util'); 3 | 4 | inputDir = './test_stylization/'; 5 | outputDir = './test_stylization/'; 6 | images = dir([inputDir '*-smooth.png']); 7 | for m = 1:length(images) 8 | inputname = [inputDir images(m).name]; 9 | strokename = [outputDir images(m).name]; 10 | strokename = strrep(strokename,'smooth.png','stroke.png'); 11 | stylename = [outputDir images(m).name]; 12 | stylename = strrep(stylename,'smooth.png','draw.png'); 13 | 14 | input = im2double(imread(inputname)); 15 | output_style = PencilDrawing(input, 8, 1, 8, 2, 1, 0); 16 | imwrite(output_style,stylename); 17 | 18 | input_ycbcr = rgb2ycbcr(input); 19 | input_y = input_ycbcr(:,:,1); 20 | output_stroke = GenStroke(input_y, 8, 1, 8).^2; 21 | imwrite(output_stroke,strokename); 22 | end -------------------------------------------------------------------------------- /run_edgedetector.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | 13 | -- structure extraction for edge-preserving smoothing 14 | smoothing = '1' 15 | -- structure extraction for texture removal 16 | -- smoothing = '0' 17 | 18 | 19 | imgPath = '/mnt/data/VOC2012_input/' 20 | 21 | if smoothing == '1' then 22 | savePath = '/mnt/data/VOC2012_input_edge_default/' 23 | else 24 | savePath = '/mnt/data/VOC2012_input_edge_texture/' 25 | end 26 | 27 | h0 = nn.Identity()() 28 | h0_edge = h0 - nn.EdgeComputation() 29 | h1 = {h0,h0_edge} - nn.JoinTable(2) 30 | h2 = h1 - nn.EdgeDetector(smoothing) 31 | model_edgeDetector = nn.gModule({h0},{h2}) 32 | model_edgeDetector = model_edgeDetector:cuda() 33 | 34 | files = {} 35 | for file in paths.files(imgPath) do 36 | if string.find(file,'.png') then 37 | table.insert(files, paths.concat(imgPath,file)) 38 | end 39 | end 40 | 41 | for _,inputFile in ipairs(files) do 42 | 43 | local inputImg = image.load(inputFile) 44 | local savColor = string.gsub(inputFile,imgPath,savePath) 45 | 46 | local height = inputImg:size(2) 47 | local width = inputImg:size(3) 48 | 49 | local input = torch.CudaTensor(1, 3, height, width) 50 | input[1] = inputImg:cuda() 51 | input = input * 255 52 | 53 | edge_label = model_edgeDetector:forward(input) 54 | edge_label_preserve = edge_label[{{},{1},{},{}}] 55 | edge_label_eliminate = edge_label[{{},{2},{},{}}] 56 | 57 | local sav = string.gsub(savColor,'%.png','-edge.png') 58 | if smoothing == '1' then 59 | image.save(sav,edge_label_preserve[1]) 60 | else 61 | image.save(sav,edge_label_eliminate[1]) 62 | end 63 | end -------------------------------------------------------------------------------- /smoothing.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/smoothing.PNG -------------------------------------------------------------------------------- /stylization.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/stylization.PNG -------------------------------------------------------------------------------- /test_detail_enhance/deer-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/test_detail_enhance/deer-input.png -------------------------------------------------------------------------------- /test_detail_enhance/deer-smooth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/test_detail_enhance/deer-smooth.png -------------------------------------------------------------------------------- /test_smooth.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | local matio = require 'matio' 13 | matio.compression = matio.ffi.COMPRESSION_wDiminish 14 | 15 | imgPath = './images' 16 | savePath = './test_stylization' 17 | 18 | files = {} 19 | for file in paths.files(imgPath) do 20 | if string.find(file,'input.png') then 21 | table.insert(files, paths.concat(imgPath,file)) 22 | end 23 | end 24 | 25 | -- change the model file to any others to try different smoothing effects. 26 | modelfile = './netfiles/model_smooth_stylization_30.net' 27 | model = torch.load(modelfile) 28 | model = model:cuda() 29 | model:training() 30 | 31 | for _,inputFile in ipairs(files) do 32 | print(inputFile) 33 | 34 | local inputImg = image.load(inputFile) 35 | local savColor = string.gsub(inputFile,imgPath,savePath) 36 | local height = inputImg:size(2) 37 | local width = inputImg:size(3) 38 | 39 | local input = torch.CudaTensor(1, 3, height, width) 40 | input[1] = inputImg:cuda() 41 | local input_origin = input:clone() 42 | input = input * 255 43 | 44 | local inputs = {input - 115,input} 45 | predictions = model:forward(inputs) 46 | predictions_final = predictions[1] 47 | 48 | for m = 1,3 do 49 | local numerator = torch.dot(predictions_final[1][m], input[1][m]) 50 | local denominator = torch.dot(predictions_final[1][m], predictions_final[1][m]) 51 | local alpha = numerator/denominator 52 | predictions_final[1][m] = predictions_final[1][m] * alpha 53 | end 54 | 55 | predictions_final = predictions_final/255 56 | local sav = string.gsub(savColor,'.png','-predict_30.png') 57 | image.save(sav,predictions_final[1]) 58 | end -------------------------------------------------------------------------------- /test_stylization/tree-smooth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fqnchina/ImageSmoothing/11a8718310f0d64ee2a7140ec87300eb33f719e0/test_stylization/tree-smooth.png -------------------------------------------------------------------------------- /train_background-smooth.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | 13 | local function subnet1() 14 | 15 | sub = nn.Sequential() 16 | 17 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 18 | sub:add(cudnn.SpatialBatchNormalization(64)) 19 | sub:add(cudnn.ReLU(true)) 20 | 21 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 22 | sub:add(cudnn.SpatialBatchNormalization(64)) 23 | 24 | cont = nn.ConcatTable() 25 | cont:add(sub) 26 | cont:add(nn.Identity()) 27 | 28 | return cont 29 | end 30 | 31 | local function subnet2() 32 | 33 | sub = nn.Sequential() 34 | 35 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 36 | sub:add(cudnn.SpatialBatchNormalization(64)) 37 | sub:add(cudnn.ReLU(true)) 38 | 39 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 40 | sub:add(cudnn.SpatialBatchNormalization(64)) 41 | 42 | cont = nn.ConcatTable() 43 | cont:add(sub) 44 | cont:add(nn.Identity()) 45 | 46 | return cont 47 | end 48 | 49 | local function subnet4() 50 | 51 | sub = nn.Sequential() 52 | 53 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 54 | sub:add(cudnn.SpatialBatchNormalization(64)) 55 | sub:add(cudnn.ReLU(true)) 56 | 57 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 58 | sub:add(cudnn.SpatialBatchNormalization(64)) 59 | 60 | cont = nn.ConcatTable() 61 | cont:add(sub) 62 | cont:add(nn.Identity()) 63 | 64 | return cont 65 | end 66 | 67 | local function subnet8() 68 | 69 | sub = nn.Sequential() 70 | 71 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 72 | sub:add(cudnn.SpatialBatchNormalization(64)) 73 | sub:add(cudnn.ReLU(true)) 74 | 75 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 76 | sub:add(cudnn.SpatialBatchNormalization(64)) 77 | 78 | cont = nn.ConcatTable() 79 | cont:add(sub) 80 | cont:add(nn.Identity()) 81 | 82 | return cont 83 | end 84 | 85 | local function subnet16() 86 | 87 | sub = nn.Sequential() 88 | 89 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 90 | sub:add(cudnn.SpatialBatchNormalization(64)) 91 | sub:add(cudnn.ReLU(true)) 92 | 93 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 94 | sub:add(cudnn.SpatialBatchNormalization(64)) 95 | 96 | cont = nn.ConcatTable() 97 | cont:add(sub) 98 | cont:add(nn.Identity()) 99 | 100 | return cont 101 | end 102 | 103 | local function subnet32() 104 | 105 | sub = nn.Sequential() 106 | 107 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 108 | sub:add(cudnn.SpatialBatchNormalization(64)) 109 | sub:add(cudnn.ReLU(true)) 110 | 111 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 112 | sub:add(cudnn.SpatialBatchNormalization(64)) 113 | 114 | cont = nn.ConcatTable() 115 | cont:add(sub) 116 | cont:add(nn.Identity()) 117 | 118 | return cont 119 | end 120 | 121 | h0 = nn.Identity()() 122 | h0_origin = nn.Identity()() 123 | h1 = h0 - cudnn.SpatialConvolution(3, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 124 | h2 = h1 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 125 | h3 = h2 - cudnn.SpatialConvolution(64, 64, 3, 3, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 126 | 127 | sub1 = h3 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 128 | sub2 = sub1 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 129 | sub3 = sub2 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 130 | sub4 = sub3 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 131 | sub5 = sub4 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 132 | sub6 = sub5 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 133 | sub7 = sub6 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 134 | sub8 = sub7 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 135 | sub9 = sub8 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 136 | sub10 = sub9 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 137 | 138 | h4 = sub10 - cudnn.SpatialFullConvolution(64, 64, 4, 4, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 139 | h5 = h4 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 140 | h6 = h5 - cudnn.SpatialConvolution(64, 3, 1, 1) 141 | h7 = {h6,h0_origin} - nn.CAddTable() 142 | h7_edge = h7 - nn.EdgeComputation() 143 | h7_grad = {h7,h7_edge} - nn.JoinTable(2) 144 | 145 | model = nn.gModule({h0,h0_origin},{h7,h7_grad}) 146 | model = model:cuda() 147 | 148 | criterion = nn.ParallelCriterion():add(nn.MSECriterion(),1):add(nn.SmoothAndEdgeTerm(0.1,7,2,0.8,1,0.1,5,0,0),1) 149 | criterion = criterion:cuda() 150 | 151 | model_computeEdge = nn.EdgeComputation() 152 | 153 | for i,module in ipairs(model:listModules()) do 154 | local m = module 155 | if m.__typename == 'cudnn.SpatialConvolution' or m.__typename == 'cudnn.SpatialFullConvolution' then 156 | local stdv = math.sqrt(12/(m.nInputPlane*m.kH*m.kW + m.nOutputPlane*m.kH*m.kW)) 157 | m.weight:uniform(-stdv, stdv) 158 | m.bias:zero() 159 | end 160 | if m.__typename == 'cudnn.SpatialBatchNormalization' then 161 | m.weight:fill(1) 162 | m.bias:zero() 163 | end 164 | end 165 | 166 | 167 | 168 | postfix = 'smooth_background-smooth' 169 | max_iters = 30 170 | batch_size = 1 171 | 172 | model:training() 173 | collectgarbage() 174 | 175 | parameters, gradParameters = model:getParameters() 176 | 177 | sgd_params = { 178 | learningRate = 1e-2, 179 | learningRateDecay = 1e-8, 180 | weightDecay = 0.0005, 181 | momentum = 0.9, 182 | dampening = 0, 183 | nesterov = true 184 | } 185 | 186 | adam_params = { 187 | learningRate = 0.01, 188 | weightDecay = 0.0005, 189 | beta1 = 0.9, 190 | beta2 = 0.999 191 | } 192 | 193 | rmsprop_params = { 194 | learningRate = 1e-2, 195 | weightDecay = 0.0005, 196 | alpha = 0.9 197 | } 198 | 199 | savePath = './smoothing/' 200 | 201 | local file = './smoothing_codes/train_background-smooth.lua' 202 | local f = io.open(file, "rb") 203 | local line = f:read("*all") 204 | f:close() 205 | print('*******************train file*******************') 206 | print(line) 207 | print('*******************train file*******************') 208 | 209 | local file = './data/VOC2012_train.txt' 210 | local trainSet = {} 211 | local f = io.open(file, "rb") 212 | while true do 213 | local line = f:read() 214 | if line == nil then break end 215 | table.insert(trainSet, line) 216 | end 217 | f:close() 218 | local trainsetSize = #trainSet 219 | 220 | local file = './data/VOC2012_test.txt' 221 | local testSet = {} 222 | local f = io.open(file, "rb") 223 | while true do 224 | local line = f:read() 225 | if line == nil then break end 226 | table.insert(testSet, line) 227 | end 228 | f:close() 229 | local testsetSize = #testSet 230 | 231 | local iter = 0 232 | local epoch_judge = false 233 | step = function(batch_size) 234 | local testCount = 1 235 | local current_loss = 0 236 | local current_testloss = 0 237 | local count = 0 238 | local testcount = 0 239 | batch_size = batch_size or 4 240 | local order = torch.randperm(trainsetSize) 241 | 242 | for t = 1,trainsetSize,batch_size do 243 | iter = iter + 1 244 | local size = math.min(t + batch_size, trainsetSize + 1) - t 245 | 246 | local feval = function(x_new) 247 | -- reset data 248 | if parameters ~= x_new then parameters:copy(x_new) end 249 | gradParameters:zero() 250 | 251 | local loss = 0 252 | for i = 1,size do 253 | local inputFile = trainSet[order[t+i-1]] 254 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 255 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 256 | local inputSaliency = string.gsub(inputFile,'VOC2012_input','VOC2012_input_saliency') 257 | 258 | local tempInput = image.load(inputFile) 259 | local height = tempInput:size(2) 260 | local width = tempInput:size(3) 261 | local input = torch.CudaTensor(1, 3, height, width) 262 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 263 | 264 | input[1] = tempInput 265 | local input_origin = input:clone() 266 | input = input * 255 267 | label = input:clone() 268 | local inputs = {input - 115,input} 269 | 270 | local input_edge = model_computeEdge:forward(input) 271 | local saliency = image.load(inputSaliency) 272 | local edgeLabel = image.load(inputEdgeFile) 273 | saliency = torch.gt(saliency,0.5) 274 | saliency = saliency:cuda() 275 | edgeLabel = edgeLabel:cuda() 276 | 277 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 278 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 279 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 280 | label_all[{{},{4},{},{}}] = torch.cmul(saliency,input_edge) 281 | label_all[{{},{5},{},{}}] = torch.cmul(saliency,edgeLabel) 282 | local labels = {label,label_all} 283 | 284 | local pred = model:forward(inputs) 285 | local tempLoss = criterion:forward(pred, labels) 286 | loss = loss + tempLoss 287 | local grad = criterion:backward(pred, labels) 288 | 289 | model:backward(inputs, grad) 290 | end 291 | gradParameters:div(size) 292 | loss = loss/size 293 | 294 | return loss, gradParameters 295 | end 296 | 297 | if epoch_judge then 298 | adam_params.learningRate = adam_params.learningRate*0.1 299 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params, adam_params) 300 | epoch_judge = false 301 | else 302 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params) 303 | end 304 | 305 | count = count + 1 306 | current_loss = current_loss + fs[1] 307 | print(string.format('Iter: %d Current loss: %4f', iter, fs[1])) 308 | 309 | if iter % 20 == 0 then 310 | local loss = 0 311 | for i = 1,size do 312 | local inputFile = testSet[testCount] 313 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 314 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 315 | local inputSaliency = string.gsub(inputFile,'VOC2012_input','VOC2012_input_saliency') 316 | 317 | local tempInput = image.load(inputFile) 318 | local height = tempInput:size(2) 319 | local width = tempInput:size(3) 320 | local input = torch.CudaTensor(1, 3, height, width) 321 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 322 | 323 | input[1] = tempInput 324 | local input_origin = input:clone() 325 | input = input * 255 326 | label = input:clone() 327 | local inputs = {input - 115,input} 328 | 329 | local input_edge = model_computeEdge:forward(input) 330 | local saliency = image.load(inputSaliency) 331 | local edgeLabel = image.load(inputEdgeFile) 332 | saliency = torch.gt(saliency,0.5) 333 | saliency = saliency:cuda() 334 | edgeLabel = edgeLabel:cuda() 335 | 336 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 337 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 338 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 339 | label_all[{{},{4},{},{}}] = torch.cmul(saliency,input_edge) 340 | label_all[{{},{5},{},{}}] = torch.cmul(saliency,edgeLabel) 341 | local labels = {label,label_all} 342 | 343 | local pred = model:forward(inputs) 344 | local tempLoss = criterion:forward(pred, labels) 345 | loss = loss + tempLoss 346 | testCount = testCount + 1 347 | end 348 | loss = loss/size 349 | testcount = testcount + 1 350 | current_testloss = current_testloss + loss 351 | 352 | print(string.format('TestIter: %d Current loss: %4f', iter, loss)) 353 | end 354 | end 355 | 356 | return current_loss / count, current_testloss / testcount 357 | end 358 | 359 | netfiles = './smoothing/netfiles/' 360 | timer = torch.Timer() 361 | do 362 | for i = 1,max_iters do 363 | localTimer = torch.Timer() 364 | local loss,testloss = step(batch_size,i) 365 | print(string.format('Epoch: %d Current loss: %4f', i, loss)) 366 | print(string.format('Epoch: %d Current test loss: %4f', i, testloss)) 367 | 368 | local filename = string.format('%smodel_%s_%d.net',netfiles,postfix,i) 369 | model:clearState() 370 | torch.save(filename, model) 371 | local filename = string.format('%sstate_%s_%d.t7',netfiles,postfix,i) 372 | torch.save(filename, adam_state_save) 373 | print('Time elapsed (epoch): ' .. localTimer:time().real/(3600) .. ' hours') 374 | end 375 | end 376 | print('Time elapsed: ' .. timer:time().real/(3600*24) .. ' days') 377 | -------------------------------------------------------------------------------- /train_detail-enhance.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | 13 | local function subnet1() 14 | 15 | sub = nn.Sequential() 16 | 17 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 18 | sub:add(cudnn.SpatialBatchNormalization(64)) 19 | sub:add(cudnn.ReLU(true)) 20 | 21 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 22 | sub:add(cudnn.SpatialBatchNormalization(64)) 23 | 24 | cont = nn.ConcatTable() 25 | cont:add(sub) 26 | cont:add(nn.Identity()) 27 | 28 | return cont 29 | end 30 | 31 | local function subnet2() 32 | 33 | sub = nn.Sequential() 34 | 35 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 36 | sub:add(cudnn.SpatialBatchNormalization(64)) 37 | sub:add(cudnn.ReLU(true)) 38 | 39 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 40 | sub:add(cudnn.SpatialBatchNormalization(64)) 41 | 42 | cont = nn.ConcatTable() 43 | cont:add(sub) 44 | cont:add(nn.Identity()) 45 | 46 | return cont 47 | end 48 | 49 | local function subnet4() 50 | 51 | sub = nn.Sequential() 52 | 53 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 54 | sub:add(cudnn.SpatialBatchNormalization(64)) 55 | sub:add(cudnn.ReLU(true)) 56 | 57 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 58 | sub:add(cudnn.SpatialBatchNormalization(64)) 59 | 60 | cont = nn.ConcatTable() 61 | cont:add(sub) 62 | cont:add(nn.Identity()) 63 | 64 | return cont 65 | end 66 | 67 | local function subnet8() 68 | 69 | sub = nn.Sequential() 70 | 71 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 72 | sub:add(cudnn.SpatialBatchNormalization(64)) 73 | sub:add(cudnn.ReLU(true)) 74 | 75 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 76 | sub:add(cudnn.SpatialBatchNormalization(64)) 77 | 78 | cont = nn.ConcatTable() 79 | cont:add(sub) 80 | cont:add(nn.Identity()) 81 | 82 | return cont 83 | end 84 | 85 | local function subnet16() 86 | 87 | sub = nn.Sequential() 88 | 89 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 90 | sub:add(cudnn.SpatialBatchNormalization(64)) 91 | sub:add(cudnn.ReLU(true)) 92 | 93 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 94 | sub:add(cudnn.SpatialBatchNormalization(64)) 95 | 96 | cont = nn.ConcatTable() 97 | cont:add(sub) 98 | cont:add(nn.Identity()) 99 | 100 | return cont 101 | end 102 | 103 | local function subnet32() 104 | 105 | sub = nn.Sequential() 106 | 107 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 108 | sub:add(cudnn.SpatialBatchNormalization(64)) 109 | sub:add(cudnn.ReLU(true)) 110 | 111 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 112 | sub:add(cudnn.SpatialBatchNormalization(64)) 113 | 114 | cont = nn.ConcatTable() 115 | cont:add(sub) 116 | cont:add(nn.Identity()) 117 | 118 | return cont 119 | end 120 | 121 | h0 = nn.Identity()() 122 | h0_origin = nn.Identity()() 123 | h1 = h0 - cudnn.SpatialConvolution(3, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 124 | h2 = h1 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 125 | h3 = h2 - cudnn.SpatialConvolution(64, 64, 3, 3, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 126 | 127 | sub1 = h3 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 128 | sub2 = sub1 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 129 | sub3 = sub2 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 130 | sub4 = sub3 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 131 | sub5 = sub4 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 132 | sub6 = sub5 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 133 | sub7 = sub6 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 134 | sub8 = sub7 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 135 | sub9 = sub8 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 136 | sub10 = sub9 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 137 | 138 | h4 = sub10 - cudnn.SpatialFullConvolution(64, 64, 4, 4, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 139 | h5 = h4 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 140 | h6 = h5 - cudnn.SpatialConvolution(64, 3, 1, 1) 141 | h7 = {h6,h0_origin} - nn.CAddTable() 142 | h7_edge = h7 - nn.EdgeComputation() 143 | h7_grad = {h7,h7_edge} - nn.JoinTable(2) 144 | 145 | model = nn.gModule({h0,h0_origin},{h7,h7_grad}) 146 | model = model:cuda() 147 | 148 | criterion = nn.ParallelCriterion():add(nn.MSECriterion(),1):add(nn.SmoothAndEdgeTerm(0.1,7,10,0.8,1,0.1,15,1,0),1) 149 | criterion = criterion:cuda() 150 | 151 | model_computeEdge = nn.EdgeComputation() 152 | 153 | for i,module in ipairs(model:listModules()) do 154 | local m = module 155 | if m.__typename == 'cudnn.SpatialConvolution' or m.__typename == 'cudnn.SpatialFullConvolution' then 156 | local stdv = math.sqrt(12/(m.nInputPlane*m.kH*m.kW + m.nOutputPlane*m.kH*m.kW)) 157 | m.weight:uniform(-stdv, stdv) 158 | m.bias:zero() 159 | end 160 | if m.__typename == 'cudnn.SpatialBatchNormalization' then 161 | m.weight:fill(1) 162 | m.bias:zero() 163 | end 164 | end 165 | 166 | 167 | 168 | postfix = 'smooth_detail-enhance' 169 | max_iters = 30 170 | batch_size = 1 171 | 172 | model:training() 173 | collectgarbage() 174 | 175 | parameters, gradParameters = model:getParameters() 176 | 177 | sgd_params = { 178 | learningRate = 1e-2, 179 | learningRateDecay = 1e-8, 180 | weightDecay = 0.0005, 181 | momentum = 0.9, 182 | dampening = 0, 183 | nesterov = true 184 | } 185 | 186 | adam_params = { 187 | learningRate = 0.01, 188 | weightDecay = 0.0005, 189 | beta1 = 0.9, 190 | beta2 = 0.999 191 | } 192 | 193 | rmsprop_params = { 194 | learningRate = 1e-2, 195 | weightDecay = 0.0005, 196 | alpha = 0.9 197 | } 198 | 199 | savePath = './smoothing/' 200 | 201 | local file = './smoothing_codes/train_detail-enhance.lua' 202 | local f = io.open(file, "rb") 203 | local line = f:read("*all") 204 | f:close() 205 | print('*******************train file*******************') 206 | print(line) 207 | print('*******************train file*******************') 208 | 209 | local file = './data/VOC2012_train.txt' 210 | local trainSet = {} 211 | local f = io.open(file, "rb") 212 | while true do 213 | local line = f:read() 214 | if line == nil then break end 215 | table.insert(trainSet, line) 216 | end 217 | f:close() 218 | local trainsetSize = #trainSet 219 | 220 | local file = './data/VOC2012_test.txt' 221 | local testSet = {} 222 | local f = io.open(file, "rb") 223 | while true do 224 | local line = f:read() 225 | if line == nil then break end 226 | table.insert(testSet, line) 227 | end 228 | f:close() 229 | local testsetSize = #testSet 230 | 231 | local iter = 0 232 | local epoch_judge = false 233 | step = function(batch_size) 234 | local testCount = 1 235 | local current_loss = 0 236 | local current_testloss = 0 237 | local count = 0 238 | local testcount = 0 239 | batch_size = batch_size or 4 240 | local order = torch.randperm(trainsetSize) 241 | 242 | for t = 1,trainsetSize,batch_size do 243 | iter = iter + 1 244 | local size = math.min(t + batch_size, trainsetSize + 1) - t 245 | 246 | local feval = function(x_new) 247 | -- reset data 248 | if parameters ~= x_new then parameters:copy(x_new) end 249 | gradParameters:zero() 250 | 251 | local loss = 0 252 | for i = 1,size do 253 | local inputFile = trainSet[order[t+i-1]] 254 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 255 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 256 | 257 | local tempInput = image.load(inputFile) 258 | local height = tempInput:size(2) 259 | local width = tempInput:size(3) 260 | local input = torch.CudaTensor(1, 3, height, width) 261 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 262 | 263 | input[1] = tempInput 264 | local input_origin = input:clone() 265 | input = input * 255 266 | label = input:clone() 267 | local inputs = {input - 115,input} 268 | 269 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 270 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 271 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 272 | label_all[{{},{4},{},{}}] = model_computeEdge:forward(input) 273 | label_all[{{},{5},{},{}}] = image.load(inputEdgeFile) 274 | local labels = {label,label_all} 275 | 276 | local pred = model:forward(inputs) 277 | local tempLoss = criterion:forward(pred, labels) 278 | loss = loss + tempLoss 279 | local grad = criterion:backward(pred, labels) 280 | 281 | model:backward(inputs, grad) 282 | end 283 | gradParameters:div(size) 284 | loss = loss/size 285 | 286 | return loss, gradParameters 287 | end 288 | 289 | if epoch_judge then 290 | adam_params.learningRate = adam_params.learningRate*0.1 291 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params, adam_params) 292 | epoch_judge = false 293 | else 294 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params) 295 | end 296 | 297 | count = count + 1 298 | current_loss = current_loss + fs[1] 299 | print(string.format('Iter: %d Current loss: %4f', iter, fs[1])) 300 | 301 | if iter % 20 == 0 then 302 | local loss = 0 303 | for i = 1,size do 304 | local inputFile = testSet[testCount] 305 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 306 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 307 | 308 | local tempInput = image.load(inputFile) 309 | local height = tempInput:size(2) 310 | local width = tempInput:size(3) 311 | local input = torch.CudaTensor(1, 3, height, width) 312 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 313 | 314 | input[1] = tempInput 315 | local input_origin = input:clone() 316 | input = input * 255 317 | label = input:clone() 318 | local inputs = {input - 115,input} 319 | 320 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 321 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 322 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 323 | label_all[{{},{4},{},{}}] = model_computeEdge:forward(input) 324 | label_all[{{},{5},{},{}}] = image.load(inputEdgeFile) 325 | local labels = {label,label_all} 326 | 327 | local pred = model:forward(inputs) 328 | local tempLoss = criterion:forward(pred, labels) 329 | loss = loss + tempLoss 330 | testCount = testCount + 1 331 | end 332 | loss = loss/size 333 | testcount = testcount + 1 334 | current_testloss = current_testloss + loss 335 | 336 | print(string.format('TestIter: %d Current loss: %4f', iter, loss)) 337 | end 338 | end 339 | 340 | return current_loss / count, current_testloss / testcount 341 | end 342 | 343 | netfiles = './smoothing/netfiles/' 344 | timer = torch.Timer() 345 | do 346 | for i = 1,max_iters do 347 | localTimer = torch.Timer() 348 | local loss,testloss = step(batch_size,i) 349 | print(string.format('Epoch: %d Current loss: %4f', i, loss)) 350 | print(string.format('Epoch: %d Current test loss: %4f', i, testloss)) 351 | 352 | local filename = string.format('%smodel_%s_%d.net',netfiles,postfix,i) 353 | model:clearState() 354 | torch.save(filename, model) 355 | local filename = string.format('%sstate_%s_%d.t7',netfiles,postfix,i) 356 | torch.save(filename, adam_state_save) 357 | print('Time elapsed (epoch): ' .. localTimer:time().real/(3600) .. ' hours') 358 | end 359 | end 360 | print('Time elapsed: ' .. timer:time().real/(3600*24) .. ' days') 361 | -------------------------------------------------------------------------------- /train_foreground-enhance.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | 13 | local function subnet1() 14 | 15 | sub = nn.Sequential() 16 | 17 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 18 | sub:add(cudnn.SpatialBatchNormalization(64)) 19 | sub:add(cudnn.ReLU(true)) 20 | 21 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 22 | sub:add(cudnn.SpatialBatchNormalization(64)) 23 | 24 | cont = nn.ConcatTable() 25 | cont:add(sub) 26 | cont:add(nn.Identity()) 27 | 28 | return cont 29 | end 30 | 31 | local function subnet2() 32 | 33 | sub = nn.Sequential() 34 | 35 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 36 | sub:add(cudnn.SpatialBatchNormalization(64)) 37 | sub:add(cudnn.ReLU(true)) 38 | 39 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 40 | sub:add(cudnn.SpatialBatchNormalization(64)) 41 | 42 | cont = nn.ConcatTable() 43 | cont:add(sub) 44 | cont:add(nn.Identity()) 45 | 46 | return cont 47 | end 48 | 49 | local function subnet4() 50 | 51 | sub = nn.Sequential() 52 | 53 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 54 | sub:add(cudnn.SpatialBatchNormalization(64)) 55 | sub:add(cudnn.ReLU(true)) 56 | 57 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 58 | sub:add(cudnn.SpatialBatchNormalization(64)) 59 | 60 | cont = nn.ConcatTable() 61 | cont:add(sub) 62 | cont:add(nn.Identity()) 63 | 64 | return cont 65 | end 66 | 67 | local function subnet8() 68 | 69 | sub = nn.Sequential() 70 | 71 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 72 | sub:add(cudnn.SpatialBatchNormalization(64)) 73 | sub:add(cudnn.ReLU(true)) 74 | 75 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 76 | sub:add(cudnn.SpatialBatchNormalization(64)) 77 | 78 | cont = nn.ConcatTable() 79 | cont:add(sub) 80 | cont:add(nn.Identity()) 81 | 82 | return cont 83 | end 84 | 85 | local function subnet16() 86 | 87 | sub = nn.Sequential() 88 | 89 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 90 | sub:add(cudnn.SpatialBatchNormalization(64)) 91 | sub:add(cudnn.ReLU(true)) 92 | 93 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 94 | sub:add(cudnn.SpatialBatchNormalization(64)) 95 | 96 | cont = nn.ConcatTable() 97 | cont:add(sub) 98 | cont:add(nn.Identity()) 99 | 100 | return cont 101 | end 102 | 103 | local function subnet32() 104 | 105 | sub = nn.Sequential() 106 | 107 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 108 | sub:add(cudnn.SpatialBatchNormalization(64)) 109 | sub:add(cudnn.ReLU(true)) 110 | 111 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 112 | sub:add(cudnn.SpatialBatchNormalization(64)) 113 | 114 | cont = nn.ConcatTable() 115 | cont:add(sub) 116 | cont:add(nn.Identity()) 117 | 118 | return cont 119 | end 120 | 121 | h0 = nn.Identity()() 122 | h0_origin = nn.Identity()() 123 | h1 = h0 - cudnn.SpatialConvolution(3, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 124 | h2 = h1 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 125 | h3 = h2 - cudnn.SpatialConvolution(64, 64, 3, 3, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 126 | 127 | sub1 = h3 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 128 | sub2 = sub1 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 129 | sub3 = sub2 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 130 | sub4 = sub3 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 131 | sub5 = sub4 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 132 | sub6 = sub5 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 133 | sub7 = sub6 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 134 | sub8 = sub7 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 135 | sub9 = sub8 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 136 | sub10 = sub9 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 137 | 138 | h4 = sub10 - cudnn.SpatialFullConvolution(64, 64, 4, 4, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 139 | h5 = h4 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 140 | h6 = h5 - cudnn.SpatialConvolution(64, 3, 1, 1) 141 | h7 = {h6,h0_origin} - nn.CAddTable() 142 | h7_edge = h7 - nn.EdgeComputation() 143 | h7_grad = {h7,h7_edge} - nn.JoinTable(2) 144 | 145 | model = nn.gModule({h0,h0_origin},{h7,h7_grad}) 146 | model = model:cuda() 147 | 148 | criterion = nn.ParallelCriterion():add(nn.MSECriterion(),1):add(nn.SmoothAndEdgeTerm(0.1,7,1,0.8,1,0.1,5,0,0),1) 149 | criterion = criterion:cuda() 150 | 151 | model_computeEdge = nn.EdgeComputation() 152 | 153 | for i,module in ipairs(model:listModules()) do 154 | local m = module 155 | if m.__typename == 'cudnn.SpatialConvolution' or m.__typename == 'cudnn.SpatialFullConvolution' then 156 | local stdv = math.sqrt(12/(m.nInputPlane*m.kH*m.kW + m.nOutputPlane*m.kH*m.kW)) 157 | m.weight:uniform(-stdv, stdv) 158 | m.bias:zero() 159 | end 160 | if m.__typename == 'cudnn.SpatialBatchNormalization' then 161 | m.weight:fill(1) 162 | m.bias:zero() 163 | end 164 | end 165 | 166 | 167 | 168 | postfix = 'smooth_foreground-enhance' 169 | max_iters = 30 170 | batch_size = 1 171 | 172 | model:training() 173 | collectgarbage() 174 | 175 | parameters, gradParameters = model:getParameters() 176 | 177 | sgd_params = { 178 | learningRate = 1e-2, 179 | learningRateDecay = 1e-8, 180 | weightDecay = 0.0005, 181 | momentum = 0.9, 182 | dampening = 0, 183 | nesterov = true 184 | } 185 | 186 | adam_params = { 187 | learningRate = 0.01, 188 | weightDecay = 0.0005, 189 | beta1 = 0.9, 190 | beta2 = 0.999 191 | } 192 | 193 | rmsprop_params = { 194 | learningRate = 1e-2, 195 | weightDecay = 0.0005, 196 | alpha = 0.9 197 | } 198 | 199 | savePath = './smoothing/' 200 | 201 | local file = './smoothing_codes/train_foreground-enhance.lua' 202 | local f = io.open(file, "rb") 203 | local line = f:read("*all") 204 | f:close() 205 | print('*******************train file*******************') 206 | print(line) 207 | print('*******************train file*******************') 208 | 209 | local file = './data/VOC2012_train.txt' 210 | local trainSet = {} 211 | local f = io.open(file, "rb") 212 | while true do 213 | local line = f:read() 214 | if line == nil then break end 215 | table.insert(trainSet, line) 216 | end 217 | f:close() 218 | local trainsetSize = #trainSet 219 | 220 | local file = './data/VOC2012_test.txt' 221 | local testSet = {} 222 | local f = io.open(file, "rb") 223 | while true do 224 | local line = f:read() 225 | if line == nil then break end 226 | table.insert(testSet, line) 227 | end 228 | f:close() 229 | local testsetSize = #testSet 230 | 231 | local iter = 0 232 | local epoch_judge = false 233 | step = function(batch_size) 234 | local testCount = 1 235 | local current_loss = 0 236 | local current_testloss = 0 237 | local count = 0 238 | local testcount = 0 239 | batch_size = batch_size or 4 240 | local order = torch.randperm(trainsetSize) 241 | 242 | for t = 1,trainsetSize,batch_size do 243 | iter = iter + 1 244 | local size = math.min(t + batch_size, trainsetSize + 1) - t 245 | 246 | local feval = function(x_new) 247 | -- reset data 248 | if parameters ~= x_new then parameters:copy(x_new) end 249 | gradParameters:zero() 250 | 251 | local loss = 0 252 | for i = 1,size do 253 | local inputFile = trainSet[order[t+i-1]] 254 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 255 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 256 | local inputSaliency = string.gsub(inputFile,'VOC2012_input','VOC2012_input_saliency') 257 | 258 | local tempInput = image.load(inputFile) 259 | local height = tempInput:size(2) 260 | local width = tempInput:size(3) 261 | local input = torch.CudaTensor(1, 3, height, width) 262 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 263 | 264 | input[1] = tempInput 265 | local input_origin = input:clone() 266 | input = input * 255 267 | label = input:clone() 268 | local inputs = {input - 115,input} 269 | 270 | local input_edge = model_computeEdge:forward(input) 271 | local saliency = image.load(inputSaliency) 272 | local edgeLabel = image.load(inputEdgeFile) 273 | saliency = torch.lt(saliency,0.5) 274 | saliency = saliency:cuda() 275 | edgeLabel = edgeLabel:cuda() 276 | 277 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 278 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 279 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 280 | label_all[{{},{4},{},{}}] = torch.cmul(saliency,input_edge) 281 | label_all[{{},{5},{},{}}] = torch.cmul(saliency,edgeLabel) 282 | local labels = {label,label_all} 283 | 284 | local pred = model:forward(inputs) 285 | local tempLoss = criterion:forward(pred, labels) 286 | loss = loss + tempLoss 287 | local grad = criterion:backward(pred, labels) 288 | 289 | model:backward(inputs, grad) 290 | end 291 | gradParameters:div(size) 292 | loss = loss/size 293 | 294 | return loss, gradParameters 295 | end 296 | 297 | if epoch_judge then 298 | adam_params.learningRate = adam_params.learningRate*0.1 299 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params, adam_params) 300 | epoch_judge = false 301 | else 302 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params) 303 | end 304 | 305 | count = count + 1 306 | current_loss = current_loss + fs[1] 307 | print(string.format('Iter: %d Current loss: %4f', iter, fs[1])) 308 | 309 | if iter % 20 == 0 then 310 | local loss = 0 311 | for i = 1,size do 312 | local inputFile = testSet[testCount] 313 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 314 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 315 | local inputSaliency = string.gsub(inputFile,'VOC2012_input','VOC2012_input_saliency') 316 | 317 | local tempInput = image.load(inputFile) 318 | local height = tempInput:size(2) 319 | local width = tempInput:size(3) 320 | local input = torch.CudaTensor(1, 3, height, width) 321 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 322 | 323 | input[1] = tempInput 324 | local input_origin = input:clone() 325 | input = input * 255 326 | label = input:clone() 327 | local inputs = {input - 115,input} 328 | 329 | local input_edge = model_computeEdge:forward(input) 330 | local saliency = image.load(inputSaliency) 331 | local edgeLabel = image.load(inputEdgeFile) 332 | saliency = torch.lt(saliency,0.5) 333 | saliency = saliency:cuda() 334 | edgeLabel = edgeLabel:cuda() 335 | 336 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 337 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 338 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 339 | label_all[{{},{4},{},{}}] = torch.cmul(saliency,input_edge) 340 | label_all[{{},{5},{},{}}] = torch.cmul(saliency,edgeLabel) 341 | local labels = {label,label_all} 342 | 343 | local pred = model:forward(inputs) 344 | local tempLoss = criterion:forward(pred, labels) 345 | loss = loss + tempLoss 346 | testCount = testCount + 1 347 | end 348 | loss = loss/size 349 | testcount = testcount + 1 350 | current_testloss = current_testloss + loss 351 | 352 | print(string.format('TestIter: %d Current loss: %4f', iter, loss)) 353 | end 354 | end 355 | 356 | return current_loss / count, current_testloss / testcount 357 | end 358 | 359 | netfiles = './smoothing/netfiles/' 360 | timer = torch.Timer() 361 | do 362 | for i = 1,max_iters do 363 | localTimer = torch.Timer() 364 | local loss,testloss = step(batch_size,i) 365 | print(string.format('Epoch: %d Current loss: %4f', i, loss)) 366 | print(string.format('Epoch: %d Current test loss: %4f', i, testloss)) 367 | 368 | local filename = string.format('%smodel_%s_%d.net',netfiles,postfix,i) 369 | model:clearState() 370 | torch.save(filename, model) 371 | local filename = string.format('%sstate_%s_%d.t7',netfiles,postfix,i) 372 | torch.save(filename, adam_state_save) 373 | print('Time elapsed (epoch): ' .. localTimer:time().real/(3600) .. ' hours') 374 | end 375 | end 376 | print('Time elapsed: ' .. timer:time().real/(3600*24) .. ' days') 377 | -------------------------------------------------------------------------------- /train_stylization.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | 13 | local function subnet1() 14 | 15 | sub = nn.Sequential() 16 | 17 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 18 | sub:add(cudnn.SpatialBatchNormalization(64)) 19 | sub:add(cudnn.ReLU(true)) 20 | 21 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 22 | sub:add(cudnn.SpatialBatchNormalization(64)) 23 | 24 | cont = nn.ConcatTable() 25 | cont:add(sub) 26 | cont:add(nn.Identity()) 27 | 28 | return cont 29 | end 30 | 31 | local function subnet2() 32 | 33 | sub = nn.Sequential() 34 | 35 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 36 | sub:add(cudnn.SpatialBatchNormalization(64)) 37 | sub:add(cudnn.ReLU(true)) 38 | 39 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 40 | sub:add(cudnn.SpatialBatchNormalization(64)) 41 | 42 | cont = nn.ConcatTable() 43 | cont:add(sub) 44 | cont:add(nn.Identity()) 45 | 46 | return cont 47 | end 48 | 49 | local function subnet4() 50 | 51 | sub = nn.Sequential() 52 | 53 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 54 | sub:add(cudnn.SpatialBatchNormalization(64)) 55 | sub:add(cudnn.ReLU(true)) 56 | 57 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 58 | sub:add(cudnn.SpatialBatchNormalization(64)) 59 | 60 | cont = nn.ConcatTable() 61 | cont:add(sub) 62 | cont:add(nn.Identity()) 63 | 64 | return cont 65 | end 66 | 67 | local function subnet8() 68 | 69 | sub = nn.Sequential() 70 | 71 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 72 | sub:add(cudnn.SpatialBatchNormalization(64)) 73 | sub:add(cudnn.ReLU(true)) 74 | 75 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 76 | sub:add(cudnn.SpatialBatchNormalization(64)) 77 | 78 | cont = nn.ConcatTable() 79 | cont:add(sub) 80 | cont:add(nn.Identity()) 81 | 82 | return cont 83 | end 84 | 85 | local function subnet16() 86 | 87 | sub = nn.Sequential() 88 | 89 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 90 | sub:add(cudnn.SpatialBatchNormalization(64)) 91 | sub:add(cudnn.ReLU(true)) 92 | 93 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 94 | sub:add(cudnn.SpatialBatchNormalization(64)) 95 | 96 | cont = nn.ConcatTable() 97 | cont:add(sub) 98 | cont:add(nn.Identity()) 99 | 100 | return cont 101 | end 102 | 103 | local function subnet32() 104 | 105 | sub = nn.Sequential() 106 | 107 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 108 | sub:add(cudnn.SpatialBatchNormalization(64)) 109 | sub:add(cudnn.ReLU(true)) 110 | 111 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 112 | sub:add(cudnn.SpatialBatchNormalization(64)) 113 | 114 | cont = nn.ConcatTable() 115 | cont:add(sub) 116 | cont:add(nn.Identity()) 117 | 118 | return cont 119 | end 120 | 121 | h0 = nn.Identity()() 122 | h0_origin = nn.Identity()() 123 | h1 = h0 - cudnn.SpatialConvolution(3, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 124 | h2 = h1 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 125 | h3 = h2 - cudnn.SpatialConvolution(64, 64, 3, 3, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 126 | 127 | sub1 = h3 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 128 | sub2 = sub1 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 129 | sub3 = sub2 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 130 | sub4 = sub3 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 131 | sub5 = sub4 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 132 | sub6 = sub5 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 133 | sub7 = sub6 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 134 | sub8 = sub7 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 135 | sub9 = sub8 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 136 | sub10 = sub9 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 137 | 138 | h4 = sub10 - cudnn.SpatialFullConvolution(64, 64, 4, 4, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 139 | h5 = h4 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 140 | h6 = h5 - cudnn.SpatialConvolution(64, 3, 1, 1) 141 | h7 = {h6,h0_origin} - nn.CAddTable() 142 | h7_edge = h7 - nn.EdgeComputation() 143 | h7_grad = {h7,h7_edge} - nn.JoinTable(2) 144 | 145 | model = nn.gModule({h0,h0_origin},{h7,h7_grad}) 146 | model = model:cuda() 147 | 148 | criterion = nn.ParallelCriterion():add(nn.MSECriterion(),1):add(nn.SmoothAndEdgeTerm(0.1,7,10,0.8,1,0.1,5,0,1),1) 149 | criterion = criterion:cuda() 150 | 151 | model_computeEdge = nn.EdgeComputation() 152 | 153 | for i,module in ipairs(model:listModules()) do 154 | local m = module 155 | if m.__typename == 'cudnn.SpatialConvolution' or m.__typename == 'cudnn.SpatialFullConvolution' then 156 | local stdv = math.sqrt(12/(m.nInputPlane*m.kH*m.kW + m.nOutputPlane*m.kH*m.kW)) 157 | m.weight:uniform(-stdv, stdv) 158 | m.bias:zero() 159 | end 160 | if m.__typename == 'cudnn.SpatialBatchNormalization' then 161 | m.weight:fill(1) 162 | m.bias:zero() 163 | end 164 | end 165 | 166 | 167 | 168 | postfix = 'smooth_stylization' 169 | max_iters = 30 170 | batch_size = 1 171 | 172 | model:training() 173 | collectgarbage() 174 | 175 | parameters, gradParameters = model:getParameters() 176 | 177 | sgd_params = { 178 | learningRate = 1e-2, 179 | learningRateDecay = 1e-8, 180 | weightDecay = 0.0005, 181 | momentum = 0.9, 182 | dampening = 0, 183 | nesterov = true 184 | } 185 | 186 | adam_params = { 187 | learningRate = 0.01, 188 | weightDecay = 0.0005, 189 | beta1 = 0.9, 190 | beta2 = 0.999 191 | } 192 | 193 | rmsprop_params = { 194 | learningRate = 1e-2, 195 | weightDecay = 0.0005, 196 | alpha = 0.9 197 | } 198 | 199 | savePath = './smoothing/' 200 | 201 | local file = './smoothing_codes/train_stylization.lua' 202 | local f = io.open(file, "rb") 203 | local line = f:read("*all") 204 | f:close() 205 | print('*******************train file*******************') 206 | print(line) 207 | print('*******************train file*******************') 208 | 209 | local file = './data/VOC2012_train.txt' 210 | local trainSet = {} 211 | local f = io.open(file, "rb") 212 | while true do 213 | local line = f:read() 214 | if line == nil then break end 215 | table.insert(trainSet, line) 216 | end 217 | f:close() 218 | local trainsetSize = #trainSet 219 | 220 | local file = './data/VOC2012_test.txt' 221 | local testSet = {} 222 | local f = io.open(file, "rb") 223 | while true do 224 | local line = f:read() 225 | if line == nil then break end 226 | table.insert(testSet, line) 227 | end 228 | f:close() 229 | local testsetSize = #testSet 230 | 231 | local iter = 0 232 | local epoch_judge = false 233 | step = function(batch_size) 234 | local testCount = 1 235 | local current_loss = 0 236 | local current_testloss = 0 237 | local count = 0 238 | local testcount = 0 239 | batch_size = batch_size or 4 240 | local order = torch.randperm(trainsetSize) 241 | 242 | for t = 1,trainsetSize,batch_size do 243 | iter = iter + 1 244 | local size = math.min(t + batch_size, trainsetSize + 1) - t 245 | 246 | local feval = function(x_new) 247 | -- reset data 248 | if parameters ~= x_new then parameters:copy(x_new) end 249 | gradParameters:zero() 250 | 251 | local loss = 0 252 | for i = 1,size do 253 | local inputFile = trainSet[order[t+i-1]] 254 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 255 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 256 | 257 | local tempInput = image.load(inputFile) 258 | local height = tempInput:size(2) 259 | local width = tempInput:size(3) 260 | local input = torch.CudaTensor(1, 3, height, width) 261 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 262 | 263 | input[1] = tempInput 264 | local input_origin = input:clone() 265 | input = input * 255 266 | label = input:clone() 267 | local inputs = {input - 115,input} 268 | 269 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 270 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 271 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 272 | label_all[{{},{4},{},{}}] = model_computeEdge:forward(input) 273 | label_all[{{},{5},{},{}}] = image.load(inputEdgeFile) 274 | local labels = {label,label_all} 275 | 276 | local pred = model:forward(inputs) 277 | local tempLoss = criterion:forward(pred, labels) 278 | loss = loss + tempLoss 279 | local grad = criterion:backward(pred, labels) 280 | 281 | model:backward(inputs, grad) 282 | end 283 | gradParameters:div(size) 284 | loss = loss/size 285 | 286 | return loss, gradParameters 287 | end 288 | 289 | if epoch_judge then 290 | adam_params.learningRate = adam_params.learningRate*0.1 291 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params, adam_params) 292 | epoch_judge = false 293 | else 294 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params) 295 | end 296 | 297 | count = count + 1 298 | current_loss = current_loss + fs[1] 299 | print(string.format('Iter: %d Current loss: %4f', iter, fs[1])) 300 | 301 | if iter % 20 == 0 then 302 | local loss = 0 303 | for i = 1,size do 304 | local inputFile = testSet[testCount] 305 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_default') 306 | inputEdgeFile = string.gsub(inputEdgeFile,'%.png','-edge.png') 307 | 308 | local tempInput = image.load(inputFile) 309 | local height = tempInput:size(2) 310 | local width = tempInput:size(3) 311 | local input = torch.CudaTensor(1, 3, height, width) 312 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 313 | 314 | input[1] = tempInput 315 | local input_origin = input:clone() 316 | input = input * 255 317 | label = input:clone() 318 | local inputs = {input - 115,input} 319 | 320 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 321 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 322 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 323 | label_all[{{},{4},{},{}}] = model_computeEdge:forward(input) 324 | label_all[{{},{5},{},{}}] = image.load(inputEdgeFile) 325 | local labels = {label,label_all} 326 | 327 | local pred = model:forward(inputs) 328 | local tempLoss = criterion:forward(pred, labels) 329 | loss = loss + tempLoss 330 | testCount = testCount + 1 331 | end 332 | loss = loss/size 333 | testcount = testcount + 1 334 | current_testloss = current_testloss + loss 335 | 336 | print(string.format('TestIter: %d Current loss: %4f', iter, loss)) 337 | end 338 | end 339 | 340 | return current_loss / count, current_testloss / testcount 341 | end 342 | 343 | netfiles = './smoothing/netfiles/' 344 | timer = torch.Timer() 345 | do 346 | for i = 1,max_iters do 347 | localTimer = torch.Timer() 348 | local loss,testloss = step(batch_size,i) 349 | print(string.format('Epoch: %d Current loss: %4f', i, loss)) 350 | print(string.format('Epoch: %d Current test loss: %4f', i, testloss)) 351 | 352 | local filename = string.format('%smodel_%s_%d.net',netfiles,postfix,i) 353 | model:clearState() 354 | torch.save(filename, model) 355 | local filename = string.format('%sstate_%s_%d.t7',netfiles,postfix,i) 356 | torch.save(filename, adam_state_save) 357 | print('Time elapsed (epoch): ' .. localTimer:time().real/(3600) .. ' hours') 358 | end 359 | end 360 | print('Time elapsed: ' .. timer:time().real/(3600*24) .. ' days') 361 | -------------------------------------------------------------------------------- /train_texture-removal.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'optim' 3 | require 'torch' 4 | require 'cutorch' 5 | require 'cunn' 6 | require 'image' 7 | require 'sys' 8 | require 'cudnn' 9 | require 'nngraph' 10 | cudnn.fastest = true 11 | cudnn.benchmark = true 12 | 13 | local function subnet1() 14 | 15 | sub = nn.Sequential() 16 | 17 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 18 | sub:add(cudnn.SpatialBatchNormalization(64)) 19 | sub:add(cudnn.ReLU(true)) 20 | 21 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 1, 1, 1, 1)) 22 | sub:add(cudnn.SpatialBatchNormalization(64)) 23 | 24 | cont = nn.ConcatTable() 25 | cont:add(sub) 26 | cont:add(nn.Identity()) 27 | 28 | return cont 29 | end 30 | 31 | local function subnet2() 32 | 33 | sub = nn.Sequential() 34 | 35 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 36 | sub:add(cudnn.SpatialBatchNormalization(64)) 37 | sub:add(cudnn.ReLU(true)) 38 | 39 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 2, 2, 2, 2)) 40 | sub:add(cudnn.SpatialBatchNormalization(64)) 41 | 42 | cont = nn.ConcatTable() 43 | cont:add(sub) 44 | cont:add(nn.Identity()) 45 | 46 | return cont 47 | end 48 | 49 | local function subnet4() 50 | 51 | sub = nn.Sequential() 52 | 53 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 54 | sub:add(cudnn.SpatialBatchNormalization(64)) 55 | sub:add(cudnn.ReLU(true)) 56 | 57 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 4, 4, 4, 4)) 58 | sub:add(cudnn.SpatialBatchNormalization(64)) 59 | 60 | cont = nn.ConcatTable() 61 | cont:add(sub) 62 | cont:add(nn.Identity()) 63 | 64 | return cont 65 | end 66 | 67 | local function subnet8() 68 | 69 | sub = nn.Sequential() 70 | 71 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 72 | sub:add(cudnn.SpatialBatchNormalization(64)) 73 | sub:add(cudnn.ReLU(true)) 74 | 75 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 8, 8, 8, 8)) 76 | sub:add(cudnn.SpatialBatchNormalization(64)) 77 | 78 | cont = nn.ConcatTable() 79 | cont:add(sub) 80 | cont:add(nn.Identity()) 81 | 82 | return cont 83 | end 84 | 85 | local function subnet16() 86 | 87 | sub = nn.Sequential() 88 | 89 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 90 | sub:add(cudnn.SpatialBatchNormalization(64)) 91 | sub:add(cudnn.ReLU(true)) 92 | 93 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 16, 16, 16, 16)) 94 | sub:add(cudnn.SpatialBatchNormalization(64)) 95 | 96 | cont = nn.ConcatTable() 97 | cont:add(sub) 98 | cont:add(nn.Identity()) 99 | 100 | return cont 101 | end 102 | 103 | local function subnet32() 104 | 105 | sub = nn.Sequential() 106 | 107 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 108 | sub:add(cudnn.SpatialBatchNormalization(64)) 109 | sub:add(cudnn.ReLU(true)) 110 | 111 | sub:add(cudnn.SpatialDilatedConvolution(64, 64, 3, 3, 1, 1, 32, 32, 32, 32)) 112 | sub:add(cudnn.SpatialBatchNormalization(64)) 113 | 114 | cont = nn.ConcatTable() 115 | cont:add(sub) 116 | cont:add(nn.Identity()) 117 | 118 | return cont 119 | end 120 | 121 | h0 = nn.Identity()() 122 | h0_origin = nn.Identity()() 123 | h1 = h0 - cudnn.SpatialConvolution(3, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 124 | h2 = h1 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 125 | h3 = h2 - cudnn.SpatialConvolution(64, 64, 3, 3, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 126 | 127 | sub1 = h3 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 128 | sub2 = sub1 - subnet2() - nn.CAddTable() - cudnn.ReLU(true) 129 | sub3 = sub2 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 130 | sub4 = sub3 - subnet4() - nn.CAddTable() - cudnn.ReLU(true) 131 | sub5 = sub4 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 132 | sub6 = sub5 - subnet8() - nn.CAddTable() - cudnn.ReLU(true) 133 | sub7 = sub6 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 134 | sub8 = sub7 - subnet16() - nn.CAddTable() - cudnn.ReLU(true) 135 | sub9 = sub8 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 136 | sub10 = sub9 - subnet1() - nn.CAddTable() - cudnn.ReLU(true) 137 | 138 | h4 = sub10 - cudnn.SpatialFullConvolution(64, 64, 4, 4, 2, 2, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 139 | h5 = h4 - cudnn.SpatialConvolution(64, 64, 3, 3, 1, 1, 1, 1) - cudnn.SpatialBatchNormalization(64) - cudnn.ReLU(true) 140 | h6 = h5 - cudnn.SpatialConvolution(64, 3, 1, 1) 141 | h7 = {h6,h0_origin} - nn.CAddTable() 142 | h7_edge = h7 - nn.EdgeComputation() 143 | h7_grad = {h7,h7_edge} - nn.JoinTable(2) 144 | 145 | model = nn.gModule({h0,h0_origin},{h7,h7_grad}) 146 | model = model:cuda() 147 | 148 | criterion = nn.ParallelCriterion():add(nn.MSECriterion(),1):add(nn.SmoothAndEdgeTerm(0.1,7,2,0.8,1,0.1,20,0,0),1) 149 | criterion = criterion:cuda() 150 | 151 | model_computeEdge = nn.EdgeComputation() 152 | 153 | for i,module in ipairs(model:listModules()) do 154 | local m = module 155 | if m.__typename == 'cudnn.SpatialConvolution' or m.__typename == 'cudnn.SpatialFullConvolution' then 156 | local stdv = math.sqrt(12/(m.nInputPlane*m.kH*m.kW + m.nOutputPlane*m.kH*m.kW)) 157 | m.weight:uniform(-stdv, stdv) 158 | m.bias:zero() 159 | end 160 | if m.__typename == 'cudnn.SpatialBatchNormalization' then 161 | m.weight:fill(1) 162 | m.bias:zero() 163 | end 164 | end 165 | 166 | 167 | 168 | postfix = 'smooth_texture-removal' 169 | max_iters = 30 170 | batch_size = 1 171 | 172 | model:training() 173 | collectgarbage() 174 | 175 | parameters, gradParameters = model:getParameters() 176 | 177 | sgd_params = { 178 | learningRate = 1e-2, 179 | learningRateDecay = 1e-8, 180 | weightDecay = 0.0005, 181 | momentum = 0.9, 182 | dampening = 0, 183 | nesterov = true 184 | } 185 | 186 | adam_params = { 187 | learningRate = 0.01, 188 | weightDecay = 0.0005, 189 | beta1 = 0.9, 190 | beta2 = 0.999 191 | } 192 | 193 | rmsprop_params = { 194 | learningRate = 1e-2, 195 | weightDecay = 0.0005, 196 | alpha = 0.9 197 | } 198 | 199 | savePath = './smoothing/' 200 | 201 | local file = './smoothing_codes/train_texture-removal.lua' 202 | local f = io.open(file, "rb") 203 | local line = f:read("*all") 204 | f:close() 205 | print('*******************train file*******************') 206 | print(line) 207 | print('*******************train file*******************') 208 | 209 | local file = './data/VOC2012_train.txt' 210 | local trainSet = {} 211 | local f = io.open(file, "rb") 212 | while true do 213 | local line = f:read() 214 | if line == nil then break end 215 | table.insert(trainSet, line) 216 | end 217 | f:close() 218 | local trainsetSize = #trainSet 219 | 220 | local file = './data/VOC2012_test.txt' 221 | local testSet = {} 222 | local f = io.open(file, "rb") 223 | while true do 224 | local line = f:read() 225 | if line == nil then break end 226 | table.insert(testSet, line) 227 | end 228 | f:close() 229 | local testsetSize = #testSet 230 | 231 | local iter = 0 232 | local epoch_judge = false 233 | step = function(batch_size) 234 | local testCount = 1 235 | local current_loss = 0 236 | local current_testloss = 0 237 | local count = 0 238 | local testcount = 0 239 | batch_size = batch_size or 4 240 | local order = torch.randperm(trainsetSize) 241 | 242 | for t = 1,trainsetSize,batch_size do 243 | iter = iter + 1 244 | local size = math.min(t + batch_size, trainsetSize + 1) - t 245 | 246 | local feval = function(x_new) 247 | -- reset data 248 | if parameters ~= x_new then parameters:copy(x_new) end 249 | gradParameters:zero() 250 | 251 | local loss = 0 252 | for i = 1,size do 253 | local inputFile = trainSet[order[t+i-1]] 254 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_texture') 255 | inputEdgeFileLabelSaliency = string.gsub(inputEdgeFile,'%.png','-edge.png') 256 | 257 | local tempInput = image.load(inputFile) 258 | local height = tempInput:size(2) 259 | local width = tempInput:size(3) 260 | local input = torch.CudaTensor(1, 3, height, width) 261 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 262 | 263 | input[1] = tempInput 264 | local input_origin = input:clone() 265 | input = input * 255 266 | label = input:clone() 267 | local inputs = {input - 115,input} 268 | 269 | local input_edge = model_computeEdge:forward(input) 270 | local saliency = image.load(inputEdgeFileLabelSaliency) 271 | saliency = torch.lt(saliency,0.5) 272 | saliency = saliency:cuda() 273 | 274 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 275 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 276 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 277 | label_all[{{},{4},{},{}}] = torch.cmul(saliency,input_edge) 278 | local labels = {label,label_all} 279 | 280 | local pred = model:forward(inputs) 281 | local tempLoss = criterion:forward(pred, labels) 282 | loss = loss + tempLoss 283 | local grad = criterion:backward(pred, labels) 284 | 285 | model:backward(inputs, grad) 286 | end 287 | gradParameters:div(size) 288 | loss = loss/size 289 | 290 | return loss, gradParameters 291 | end 292 | 293 | if epoch_judge then 294 | adam_params.learningRate = adam_params.learningRate*0.1 295 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params, adam_params) 296 | epoch_judge = false 297 | else 298 | _, fs, adam_state_save = optim.adam_state(feval, parameters, adam_params) 299 | end 300 | 301 | count = count + 1 302 | current_loss = current_loss + fs[1] 303 | print(string.format('Iter: %d Current loss: %4f', iter, fs[1])) 304 | 305 | if iter % 20 == 0 then 306 | local loss = 0 307 | for i = 1,size do 308 | local inputFile = testSet[testCount] 309 | local inputEdgeFile = string.gsub(inputFile,'VOC2012_input','VOC2012_input_edge_texture') 310 | inputEdgeFileLabelSaliency = string.gsub(inputEdgeFile,'%.png','-edge.png') 311 | 312 | local tempInput = image.load(inputFile) 313 | local height = tempInput:size(2) 314 | local width = tempInput:size(3) 315 | local input = torch.CudaTensor(1, 3, height, width) 316 | local label_all = torch.CudaTensor(1, 5, height, width):fill(0) 317 | 318 | input[1] = tempInput 319 | local input_origin = input:clone() 320 | input = input * 255 321 | label = input:clone() 322 | local inputs = {input - 115,input} 323 | 324 | local input_edge = model_computeEdge:forward(input) 325 | local saliency = image.load(inputEdgeFileLabelSaliency) 326 | saliency = torch.lt(saliency,0.5) 327 | saliency = saliency:cuda() 328 | 329 | label_all[{{},{1},{},{}}] = image.rgb2y(tempInput) 330 | label_all[{{},{2},{},{}}] = 0.492 * torch.csub(input_origin[{{},{3},{},{}}],label_all[{{},{1},{},{}}]) 331 | label_all[{{},{3},{},{}}] = 0.877 * torch.csub(input_origin[{{},{1},{},{}}],label_all[{{},{1},{},{}}]) 332 | label_all[{{},{4},{},{}}] = torch.cmul(saliency,input_edge) 333 | local labels = {label,label_all} 334 | 335 | local pred = model:forward(inputs) 336 | local tempLoss = criterion:forward(pred, labels) 337 | loss = loss + tempLoss 338 | testCount = testCount + 1 339 | end 340 | loss = loss/size 341 | testcount = testcount + 1 342 | current_testloss = current_testloss + loss 343 | 344 | print(string.format('TestIter: %d Current loss: %4f', iter, loss)) 345 | end 346 | end 347 | 348 | return current_loss / count, current_testloss / testcount 349 | end 350 | 351 | netfiles = './smoothing/netfiles/' 352 | timer = torch.Timer() 353 | do 354 | for i = 1,max_iters do 355 | localTimer = torch.Timer() 356 | local loss,testloss = step(batch_size,i) 357 | print(string.format('Epoch: %d Current loss: %4f', i, loss)) 358 | print(string.format('Epoch: %d Current test loss: %4f', i, testloss)) 359 | 360 | local filename = string.format('%smodel_%s_%d.net',netfiles,postfix,i) 361 | model:clearState() 362 | torch.save(filename, model) 363 | local filename = string.format('%sstate_%s_%d.t7',netfiles,postfix,i) 364 | torch.save(filename, adam_state_save) 365 | print('Time elapsed (epoch): ' .. localTimer:time().real/(3600) .. ' hours') 366 | end 367 | end 368 | print('Time elapsed: ' .. timer:time().real/(3600*24) .. ' days') 369 | -------------------------------------------------------------------------------- /util/GenPencil.m: -------------------------------------------------------------------------------- 1 | function T = GenPencil(im, P, J) 2 | % ============================================== 3 | % Compute the pencil map 'T' 4 | % 5 | % Paras: 6 | % @im : input image ranging value from 0 to 1. 7 | % @P : the pencil texture. 8 | % @J : the tone map. 9 | % 10 | 11 | %% Parameters 12 | theta = 0.2; 13 | 14 | [H, W, ~] = size(im); 15 | 16 | %% Initialization 17 | P = imresize(P, [H, W]); 18 | P = reshape(P, H*W, 1); 19 | logP = log(P); 20 | logP = spdiags(logP, 0, H*W, H*W); 21 | 22 | J = imresize(J, [H, W]); 23 | J = reshape(J, H*W, 1); 24 | logJ = log(J); 25 | 26 | e = ones(H*W, 1); 27 | Dx = spdiags([-e, e], [0, H], H*W, H*W); 28 | Dy = spdiags([-e, e], [0, 1], H*W, H*W); 29 | 30 | %% Compute matrix A and b 31 | A = theta * (Dx * Dx' + Dy * Dy') + (logP)' * logP; 32 | b = (logP)' * logJ; 33 | 34 | %% Conjugate gradient 35 | beta = pcg(A, b, 1e-6, 60); 36 | 37 | %% Compute the result 38 | beta = reshape(beta, H, W); 39 | 40 | P = reshape(P, H, W); 41 | 42 | T = P .^ beta; 43 | end -------------------------------------------------------------------------------- /util/GenStroke.m: -------------------------------------------------------------------------------- 1 | function S = GenStroke(im, ks, width, dirNum) 2 | % ============================================== 3 | % Compute the stroke structure 'S' 4 | % 5 | % Paras: 6 | % @im : input image ranging value from 0 to 1. 7 | % @ks : kernel size. 8 | % @width : width of the strocke 9 | % @dirNum : number of directions. 10 | % 11 | 12 | %% Initialization 13 | [H, W, ~] = size(im); 14 | 15 | %% Smoothing 16 | % im = medfilt2(im, [3 3]); 17 | 18 | %% Image gradient 19 | imX = [abs(im(:,1:(end-1)) - im(:,2:end)),zeros(H,1)]; 20 | imY = [abs(im(1:(end-1),:) - im(2:end,:));zeros(1,W)]; 21 | imEdge = imX + imY; 22 | 23 | %% Convolution kernel with horizontal direction 24 | kerRef = zeros(ks*2+1); 25 | kerRef(ks+1,:) = 1; 26 | 27 | %% Classification 28 | response = zeros(H,W,dirNum); 29 | for n = 1 : dirNum 30 | ker = imrotate(kerRef, (n-1)*180/dirNum, 'bilinear', 'crop'); 31 | response(:,:,n) = conv2(imEdge, ker, 'same'); 32 | end 33 | 34 | [~, index] = max(response,[], 3); 35 | 36 | %% Create the stroke 37 | C = zeros(H, W, dirNum); 38 | for n = 1 : dirNum 39 | C(:,:,n) = imEdge .* (index == n); 40 | end 41 | 42 | kerRef = zeros(ks*2+1); 43 | kerRef(ks+1,:) = 1; 44 | for n = 1 : width 45 | if (ks+1-n) > 0 46 | kerRef(ks+1-n,:) = 1; 47 | end 48 | if (ks+1+n) < (ks*2+1) 49 | kerRef(ks+1+n,:) = 1; 50 | end 51 | end 52 | 53 | Spn = zeros(H, W, dirNum); 54 | for n = 1 : dirNum 55 | ker = imrotate(kerRef, (n-1)*180/dirNum, 'bilinear', 'crop'); 56 | Spn(:,:,n) = conv2(C(:,:,n), ker, 'same'); 57 | end 58 | 59 | Sp = sum(Spn, 3); 60 | Sp = (Sp - min(Sp(:))) / (max(Sp(:)) - min(Sp(:))); 61 | S = 1 - Sp; 62 | end -------------------------------------------------------------------------------- /util/GenToneMap.m: -------------------------------------------------------------------------------- 1 | function J = GenToneMap(im) 2 | % ============================================== 3 | % Compute the tone map 'T' 4 | % 5 | % Paras: 6 | % @im : input image ranging value from 0 to 1. 7 | % 8 | 9 | %% Parameters 10 | Ub = 225; 11 | Ua = 105; 12 | 13 | Mud = 90; 14 | 15 | DeltaB = 9; 16 | DeltaD = 11; 17 | 18 | % groups from dark to light 19 | % 1st group 20 | % Omega1 = 42; 21 | % Omega2 = 29; 22 | % Omega3 = 29; 23 | % 2nd group 24 | % Omega1 = 52; 25 | % Omega2 = 37; 26 | % Omega3 = 11; 27 | % 3rd group 28 | % Omega1 = 76; 29 | % Omega2 = 22; 30 | % Omega3 = 2; 31 | 32 | Omega1 = 76; 33 | Omega2 = 22; 34 | Omega3 = 2; 35 | 36 | %% Compute the target histgram 37 | histgramTarget = zeros(256, 1); 38 | total = 0; 39 | for ii = 0 : 255 40 | if ii < Ua || ii > Ub 41 | p = 0; 42 | else 43 | p = 1 / (Ub - Ua); 44 | end 45 | 46 | histgramTarget(ii+1, 1) = (... 47 | Omega1 * 1/DeltaB * exp(-(255-ii)/DeltaB) + ... 48 | Omega2 * p + ... 49 | Omega3 * 1/sqrt(2 * pi * DeltaD) * exp(-(ii-Mud)^2/(2*DeltaD^2))) * 0.01; 50 | 51 | total = total + histgramTarget(ii+1, 1); 52 | end 53 | histgramTarget(:, 1) = histgramTarget(:, 1)/total; 54 | 55 | % %% Smoothing 56 | % im = medfilt2(im, [5 5]); 57 | 58 | %% Histgram matching 59 | J = histeq(im, histgramTarget); 60 | 61 | %% Smoothing 62 | G = fspecial('average', 10); 63 | J = imfilter(J, G,'same'); 64 | end -------------------------------------------------------------------------------- /util/PencilDrawing.m: -------------------------------------------------------------------------------- 1 | function I = PencilDrawing(im, ks, width, dirNum, gammaS, gammaI, toneMap) 2 | % ============================================== 3 | % Generate the pencil drawing I based on the method described in 4 | % "Combining Sketch and Tone for Pencil Drawing Production" Cewu Lu, Li Xu, Jiaya Jia 5 | % International Symposium on Non-Photorealistic Animation and Rendering (NPAR 2012), June, 2012 6 | % 7 | % Paras: 8 | % @im : the input image. 9 | % @ks : the length of convolution line. 10 | % @width : the width of the stroke. 11 | % @dirNum : the number of directions. 12 | % @gammaS : the darkness of the stroke. 13 | % @gammaI : the darkness of the resulted image. 14 | % @toneMap : 1 for tonemapped output, and 0 for normal output 15 | 16 | %% Read the image 17 | im = im2double(im); 18 | [H, W, sc] = size(im); 19 | 20 | %% Convert from rgb to yuv when nessesary 21 | if (sc == 3) 22 | yuvIm = rgb2ycbcr(im); 23 | lumIm = yuvIm(:,:,1); 24 | else 25 | lumIm = im; 26 | end 27 | 28 | %% Generate the stroke map 29 | S = GenStroke(lumIm, ks, width, dirNum) .^ gammaS; % darken the result by gamma 30 | 31 | %% Generate the tone map 32 | J = GenToneMap(lumIm) .^ gammaI; % darken the result by gamma 33 | 34 | %% Read the pencil texture 35 | P = im2double(imread('pencils/pencil2.png')); 36 | P = rgb2gray(P); 37 | 38 | %% Generate the pencil map 39 | T = GenPencil(lumIm, P, J); 40 | 41 | %% Compute the result 42 | if toneMap == 1 43 | lumIm = S .* T; 44 | else 45 | lumIm = S .* lumIm; 46 | end 47 | 48 | if (sc == 3) 49 | yuvIm(:,:,1) = lumIm; 50 | I = ycbcr2rgb(yuvIm); 51 | else 52 | I = lumIm; 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /util/sigmoid.m: -------------------------------------------------------------------------------- 1 | function y = sigmoid(x, a) 2 | % DESCR: 3 | % Applies a sigmoid function on the data x in [0-1] range. Then rescales 4 | % the result so 0.5 will be mapped to itself. 5 | 6 | % Apply Sigmoid 7 | y = 1./(1+exp(-a*x)) - 0.5; 8 | 9 | % Re-scale 10 | y05 = 1./(1+exp(-a*0.5)) - 0.5; 11 | y = y*(0.5/y05); 12 | 13 | end 14 | -------------------------------------------------------------------------------- /util/tonemapLAB_simple.m: -------------------------------------------------------------------------------- 1 | function res = tonemapLAB_simple(lab,L0,L,val0,val2,exposure,gamma,saturation) 2 | if val0==0 3 | diff0 = L-L0; 4 | else 5 | if val0>0 6 | diff0 = sigmoid((L-L0)/100,val0)*100; 7 | else 8 | diff0 = (1+val0)*(L-L0); 9 | end 10 | end 11 | 12 | if val2==0 13 | base = exposure*L0; 14 | else 15 | if val2>0 16 | base = (sigmoid((exposure*L0-56)/100,val2)*100)+56; 17 | else 18 | base = (1+val2)*(exposure*L0-56) + 56; 19 | end 20 | end 21 | 22 | if gamma == 1 23 | res = base + diff0; 24 | else 25 | maxBase = max(base(:)); 26 | res = (zeroone(base).^gamma)*maxBase + diff0; 27 | end 28 | 29 | if saturation == 0 30 | lab(:,:,1) = res; 31 | else 32 | lab(:,:,1) = res; 33 | lab(:,:,2) = lab(:,:,2) * saturation; 34 | lab(:,:,3) = lab(:,:,3) * saturation; 35 | end 36 | 37 | cform = makecform('lab2srgb'); 38 | res = applycform(lab, cform); -------------------------------------------------------------------------------- /util/zeroone.m: -------------------------------------------------------------------------------- 1 | function out = zeroone(in) 2 | 3 | maxIn = max(in(:)); 4 | minIn = min(in(:)); 5 | 6 | out = (in-minIn)/(maxIn-minIn); --------------------------------------------------------------------------------