├── _ext
    ├── __init__.py
    └── cunnex
    │   └── __init__.py
├── .gitignore
├── floyd_requirements.txt
├── AdaConv.pptx
├── .floydexpt
├── images
    ├── first.png
    ├── second.png
    └── README.md
├── .idea
    ├── markdown-navigator
    │   └── profiles_settings.xml
    ├── misc.xml
    ├── modules.xml
    ├── pytorch-sepconv-master.iml
    ├── markdown-navigator.xml
    └── workspace.xml
├── src
    ├── SeparableConvolution_cuda.h
    ├── SeparableConvolution_kernel.h
    ├── SeparableConvolution_cuda.c
    └── SeparableConvolution_kernel.cu
├── .floydignore
├── install.bash
├── floyd.yml
├── install.py
├── SeparableConvolution.py
├── README.md
└── run.py


/_ext/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /.project
2 | 
3 | .pytorch
4 | .o


--------------------------------------------------------------------------------
/floyd_requirements.txt:
--------------------------------------------------------------------------------
1 | moviepy
2 | pillow
3 | cffi


--------------------------------------------------------------------------------
/AdaConv.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/youyuge34/sepconv_video/HEAD/AdaConv.pptx


--------------------------------------------------------------------------------
/.floydexpt:
--------------------------------------------------------------------------------
1 | {"name": "adaconv", "family_id": "prj_cdHJokKGnyjctrfr", "namespace": "youyuge34"}


--------------------------------------------------------------------------------
/images/first.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/youyuge34/sepconv_video/HEAD/images/first.png


--------------------------------------------------------------------------------
/images/second.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/youyuge34/sepconv_video/HEAD/images/second.png


--------------------------------------------------------------------------------
/images/README.md:
--------------------------------------------------------------------------------
1 | The used example originates from the Middlebury benchmark for Optical Flow: http://vision.middlebury.edu/flow


--------------------------------------------------------------------------------
/.idea/markdown-navigator/profiles_settings.xml:
--------------------------------------------------------------------------------
1 | <component name="MarkdownNavigator.ProfileManager">
2 |   <settings default="" pdf-export="" />
3 | </component>


--------------------------------------------------------------------------------
/src/SeparableConvolution_cuda.h:
--------------------------------------------------------------------------------
1 | int SeparableConvolution_cuda_forward(
2 | 	THCudaTensor* input,
3 | 	THCudaTensor* vertical,
4 | 	THCudaTensor* horizontal,
5 | 	THCudaTensor* output
6 | );


--------------------------------------------------------------------------------
/.floydignore:
--------------------------------------------------------------------------------
 1 | 
 2 | # Directories and files to ignore when uploading code to floyd
 3 | 
 4 | .git
 5 | .eggs
 6 | eggs
 7 | lib
 8 | lib64
 9 | parts
10 | sdist
11 | var
12 | *.pyc
13 | *.swp
14 | .DS_Store
15 | 


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.13 (C:\Python27\python.exe)" project-jdk-type="Python SDK" />
4 | </project>


--------------------------------------------------------------------------------
/src/SeparableConvolution_kernel.h:
--------------------------------------------------------------------------------
 1 | #ifdef __cplusplus
 2 | 	extern "C" {
 3 | #endif
 4 | 
 5 | void SeparableConvolution_kernel_forward(
 6 | 	THCState* state,
 7 | 	THCudaTensor* input,
 8 | 	THCudaTensor* vertical,
 9 | 	THCudaTensor* horizontal,
10 | 	THCudaTensor* output
11 | );
12 | 
13 | #ifdef __cplusplus
14 | 	}
15 | #endif


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/pytorch-sepconv-master.iml" filepath="$PROJECT_DIR$/.idea/pytorch-sepconv-master.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/pytorch-sepconv-master.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
10 |   </component>
11 | </module>


--------------------------------------------------------------------------------
/_ext/cunnex/__init__.py:
--------------------------------------------------------------------------------
 1 | 
 2 | from torch.utils.ffi import _wrap_function
 3 | from ._cunnex import lib as _lib, ffi as _ffi
 4 | 
 5 | __all__ = []
 6 | def _import_symbols(locals):
 7 |     for symbol in dir(_lib):
 8 |         fn = getattr(_lib, symbol)
 9 |         if callable(fn):
10 |             locals[symbol] = _wrap_function(fn, _ffi)
11 |         else:
12 |             locals[symbol] = fn
13 |         __all__.append(symbol)
14 | 
15 | _import_symbols(locals())
16 | 


--------------------------------------------------------------------------------
/src/SeparableConvolution_cuda.c:
--------------------------------------------------------------------------------
 1 | #include <THC.h>
 2 | #include <THCGeneral.h>
 3 | 
 4 | #include "SeparableConvolution_kernel.h"
 5 | 
 6 | extern THCState* state;
 7 | 
 8 | int SeparableConvolution_cuda_forward(
 9 | 	THCudaTensor* input,
10 | 	THCudaTensor* vertical,
11 | 	THCudaTensor* horizontal,
12 | 	THCudaTensor* output
13 | ) {
14 | 	SeparableConvolution_kernel_forward(
15 | 		state,
16 | 		input,
17 | 		vertical,
18 | 		horizontal,
19 | 		output
20 | 	);
21 | 
22 | 	return 1;
23 | }


--------------------------------------------------------------------------------
/install.bash:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | TORCH=$(python -c "import os; import torch; print(os.path.dirname(torch.__file__))")
 4 | 
 5 | nvcc -c -o src/SeparableConvolution_kernel.o src/SeparableConvolution_kernel.cu --gpu-architecture=compute_52 --gpu-code=compute_52 --compiler-options -fPIC -I ${TORCH}/lib/include/TH -I ${TORCH}/lib/include/THC
 6 | 
 7 | python install.py
 8 | 
 9 | wget --timestamping http://content.sniklaus.com/sepconv/network-l1.pytorch
10 | wget --timestamping http://content.sniklaus.com/sepconv/network-lf.pytorch


--------------------------------------------------------------------------------
/floyd.yml:
--------------------------------------------------------------------------------
 1 | # see: https://docs.floydhub.com/floyd_config
 2 | # All supported configs:
 3 | #
 4 | #machine: cpu
 5 | #env: tensorflow-1.8
 6 | #input:
 7 | #  - destination: input
 8 | #    source: foo/datasets/yelp-food/1
 9 | #  - foo/datasets/yelp-food-test/1:test
10 | #description: this is a test
11 | #max_runtime: 3600
12 | #command: python train.py
13 | 
14 | # You can also define multiple tasks to use with --task argument:
15 | #
16 | #task:
17 | #  evaluate:
18 | #    machine: gpu
19 | #    command: python evaluate.py
20 | #
21 | #  serve:
22 | #    machine: cpu
23 | #    mode: serve
24 | 


--------------------------------------------------------------------------------
/install.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import torch
 3 | import torch.utils.ffi
 4 | 
 5 | strBasepath = os.path.split(os.path.abspath(__file__))[0] + '/'
 6 | strHeaders = []
 7 | strSources = []
 8 | strDefines = []
 9 | strObjects = []
10 | 
11 | if torch.cuda.is_available() == True:
12 | 	strHeaders += ['src/SeparableConvolution_cuda.h']
13 | 	strSources += ['src/SeparableConvolution_cuda.c']
14 | 	strDefines += [('WITH_CUDA', None)]
15 | 	strObjects += ['src/SeparableConvolution_kernel.o']
16 | # end
17 | 
18 | objectExtension = torch.utils.ffi.create_extension(
19 | 	name='_ext.cunnex',
20 | 	headers=strHeaders,
21 | 	sources=strSources,
22 | 	verbose=False,
23 | 	with_cuda=any(strDefine[0] == 'WITH_CUDA' for strDefine in strDefines),
24 | 	package=False,
25 | 	relative_to=strBasepath,
26 | 	include_dirs=[os.path.expandvars('$CUDA_HOME') + '/include'],
27 | 	define_macros=strDefines,
28 | 	extra_objects=[os.path.join(strBasepath, strObject) for strObject in strObjects]
29 | )
30 | 
31 | if __name__ == '__main__':
32 | 	objectExtension.build()
33 | # end


--------------------------------------------------------------------------------
/SeparableConvolution.py:
--------------------------------------------------------------------------------
 1 | import torch
 2 | 
 3 | import _ext.cunnex
 4 | 
 5 | class SeparableConvolution(torch.autograd.Function):
 6 | 	def __init__(self):
 7 | 		super(SeparableConvolution, self).__init__()
 8 | 	# end
 9 | 
10 | 	def forward(self, input, vertical, horizontal):
11 | 		intBatches = input.size(0)
12 | 		intInputDepth = input.size(1)
13 | 		intInputHeight = input.size(2)
14 | 		intInputWidth = input.size(3)
15 | 		intFilterSize = min(vertical.size(1), horizontal.size(1))
16 | 		intOutputHeight = min(vertical.size(2), horizontal.size(2))
17 | 		intOutputWidth = min(vertical.size(3), horizontal.size(3))
18 | 
19 | 		assert(intInputHeight - 51 == intOutputHeight - 1)
20 | 		assert(intInputWidth - 51 == intOutputWidth - 1)
21 | 		assert(intFilterSize == 51)
22 | 
23 | 		assert(input.is_contiguous() == True)
24 | 		assert(vertical.is_contiguous() == True)
25 | 		assert(horizontal.is_contiguous() == True)
26 | 
27 | 		output = input.new().resize_(intBatches, intInputDepth, intOutputHeight, intOutputWidth).zero_()
28 | 
29 | 		if input.is_cuda == True:
30 | 			_ext.cunnex.SeparableConvolution_cuda_forward(
31 | 				input,
32 | 				vertical,
33 | 				horizontal,
34 | 				output
35 | 			)
36 | 
37 | 		elif input.is_cuda == False:
38 | 			raise NotImplementedError() # CPU VERSION NOT IMPLEMENTED
39 | 
40 | 		# end
41 | 
42 | 		return output
43 | 	# end
44 | 
45 | 	def backward(self, gradOutput):
46 | 		raise NotImplementedError() # BACKPROPAGATION NOT IMPLEMENTED
47 | 	# end
48 | # end
49 | 


--------------------------------------------------------------------------------
/src/SeparableConvolution_kernel.cu:
--------------------------------------------------------------------------------
 1 | #include <THC.h>
 2 | #include <THCGeneral.h>
 3 | 
 4 | #define VEC_0(ARRAY) ((ARRAY).x)
 5 | #define VEC_1(ARRAY) ((ARRAY).y)
 6 | #define VEC_2(ARRAY) ((ARRAY).z)
 7 | #define VEC_3(ARRAY) ((ARRAY).w)
 8 | 
 9 | #define IDX_1(ARRAY, X)          ((ARRAY)[((X) * (ARRAY##_stride.x))])
10 | #define IDX_2(ARRAY, X, Y)       ((ARRAY)[((X) * (ARRAY##_stride.x)) + ((Y) * (ARRAY##_stride.y))])
11 | #define IDX_3(ARRAY, X, Y, Z)    ((ARRAY)[((X) * (ARRAY##_stride.x)) + ((Y) * (ARRAY##_stride.y)) + ((Z) * (ARRAY##_stride.z))])
12 | #define IDX_4(ARRAY, X, Y, Z, W) ((ARRAY)[((X) * (ARRAY##_stride.x)) + ((Y) * (ARRAY##_stride.y)) + ((Z) * (ARRAY##_stride.z)) + ((W) * (ARRAY##_stride.w))])
13 | 
14 | #ifdef __cplusplus
15 | 	extern "C" {
16 | #endif
17 | 
18 | __global__ void kernel_SeparableConvolution_updateOutput(
19 | 	const int n,
20 | 	const float* input, const long4 input_size, const long4 input_stride,
21 | 	const float* vertical, const long4 vertical_size, const long4 vertical_stride,
22 | 	const float* horizontal, const long4 horizontal_size, const long4 horizontal_stride,
23 | 	float* output, const long4 output_size, const long4 output_stride
24 | ) {
25 | 	int intIndex = blockIdx.x * blockDim.x + threadIdx.x;
26 | 
27 | 	if (intIndex >= n) {
28 | 		return;
29 | 	}
30 | 
31 | 	float dblOutput = 0.0;
32 | 
33 | 	int intBatch = ( intIndex / VEC_3(output_size) / VEC_2(output_size) / VEC_1(output_size) ) % VEC_0(output_size);
34 | 	int intDepth = ( intIndex / VEC_3(output_size) / VEC_2(output_size)                      ) % VEC_1(output_size);
35 | 	int intY     = ( intIndex / VEC_3(output_size)                                           ) % VEC_2(output_size);
36 | 	int intX     = ( intIndex                                                                ) % VEC_3(output_size);
37 | 
38 | 	for (int intFilterY = 0; intFilterY < 51; intFilterY += 1) {
39 | 		for (int intFilterX = 0; intFilterX < 51; intFilterX += 1) {
40 | 			dblOutput += IDX_4(input, intBatch, intDepth, intY + intFilterY, intX + intFilterX) * IDX_4(vertical, intBatch, intFilterY, intY, intX) * IDX_4(horizontal, intBatch, intFilterX, intY, intX);
41 | 		}
42 | 	}
43 | 
44 | 	output[intIndex] = dblOutput;
45 | }
46 | 
47 | void SeparableConvolution_kernel_forward(
48 | 	THCState* state,
49 | 	THCudaTensor* input,
50 | 	THCudaTensor* vertical,
51 | 	THCudaTensor* horizontal,
52 | 	THCudaTensor* output
53 | ) {
54 | 	int n = 0;
55 | 
56 | 	n = THCudaTensor_nElement(state, output);
57 | 	kernel_SeparableConvolution_updateOutput<<< (n + 512 - 1) / 512, 512, 0, THCState_getCurrentStream(state) >>>(
58 | 		n,
59 | 		THCudaTensor_data(state, input), make_long4(input->size[0], input->size[1], input->size[2], input->size[3]), make_long4(input->stride[0], input->stride[1], input->stride[2], input->stride[3]),
60 | 		THCudaTensor_data(state, vertical), make_long4(vertical->size[0], vertical->size[1], vertical->size[2], vertical->size[3]), make_long4(vertical->stride[0], vertical->stride[1], vertical->stride[2], vertical->stride[3]),
61 | 		THCudaTensor_data(state, horizontal), make_long4(horizontal->size[0], horizontal->size[1], horizontal->size[2], horizontal->size[3]), make_long4(horizontal->stride[0], horizontal->stride[1], horizontal->stride[2], horizontal->stride[3]),
62 | 		THCudaTensor_data(state, output), make_long4(output->size[0], output->size[1], output->size[2], output->size[3]), make_long4(output->stride[0], output->stride[1], output->stride[2], output->stride[3])
63 | 	);
64 | 
65 | 	THCudaCheck(cudaGetLastError());
66 | }
67 | 
68 | #ifdef __cplusplus
69 | 	}
70 | #endif


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Introduction
 2 | This project is updated from [dagf2101/pytorch-sepconv](https://github.com/dagf2101/pytorch-sepconv)    
 3 | 
 4 | updates:
 5 | 
 6 | 1. some code robustness     
 7 | 2. now the output video has audio     
 8 | 3. increase a progress bar when processing     
 9 | 
10 | **Chinese introduction and practice lesson video:** [【中文】 基于SepConv深度学习的视频补帧 插帧 论文+代码+实战教程](https://www.bilibili.com/video/av29090385/)
11 | 
12 | 
13 | ## Warning!!!
14 |  The code is old and has bugs,that it needs a high calculation capability GPU（`K80` is not enough）to run successfully. Also it is not friendly to Windows users.     
15 | 
16 | 
17 |  So u should refer to the origin author [sniklaus/pytorch-sepconv](https://github.com/sniklaus/pytorch-sepconv) to get newest code which used cupy to compile automatically. But it can only handle photos.
18 | 
19 | > Below is the origin 'readme'    
20 | 
21 | ------------------------------
22 | 
23 | # pytorch-sepconv
24 | This is a reference implementation of Video Frame Interpolation via Adaptive Separable Convolution [1] using PyTorch. Given two frames, it will make use of <a href="http://graphics.cs.pdx.edu/project/adaconv">adaptive convolution</a> [2] in a separable manner to interpolate the intermediate frame. Should you be making use of our work, please cite our paper [1].
25 | 
26 | <a href="https://arxiv.org/abs/1708.01692" rel="Paper"><img src="http://content.sniklaus.com/SepConv/Paper.jpg" alt="Paper" width="100%"></a>
27 | 
28 | For the Torch version of this work, please see: https://github.com/sniklaus/torch-sepconv
29 | 
30 | ## setup
31 | To build the implementation and download the pre-trained networks, run `bash install.bash` and make sure that you configured the `CUDA_HOME` environment variable. After successfully completing this step, run `python run.py` to test it. Should you receive an error message regarding an invalid device function during execution, configure the utilized CUDA architecture within `install.bash` to something your graphics card supports.
32 | 
33 | ## usage
34 | To run it on your own pair of frames, use the following command. You can either select the `l1` or the `lf` model, please see our paper for more details.
35 | 
36 | Image:
37 | ```
38 | python run.py --model lf --first ./images/first.png --second ./images/second.png --out ./result.png
39 | ```
40 | Video:
41 | ```
42 | python run.py --model lf --video ./video.mp4 --video-out ./result.mp4
43 | ```
44 | 
45 | ## video
46 | <a href="http://web.cecs.pdx.edu/~fliu/project/sepconv/demo.mp4" rel="Video"><img src="http://web.cecs.pdx.edu/~fliu/project/sepconv/screen.jpg" alt="Video" width="100%"></a>
47 | 
48 | ## license
49 | The provided implementation is strictly for academic purposes only. Should you be interested in using our technology for any commercial use, please feel free to contact us.
50 | 
51 | ## references
52 | ```
53 | [1]  @inproceedings{Niklaus_ICCV_2017,
54 |          author = {Simon Niklaus and Long Mai and Feng Liu},
55 |          title = {Video Frame Interpolation via Adaptive Separable Convolution},
56 |          booktitle = {IEEE International Conference on Computer Vision},
57 |          year = {2017}
58 |      }
59 | ```
60 | 
61 | ```
62 | [2]  @inproceedings{Niklaus_CVPR_2017,
63 |          author = {Simon Niklaus and Long Mai and Feng Liu},
64 |          title = {Video Frame Interpolation via Adaptive Convolution},
65 |          booktitle = {IEEE Conference on Computer Vision and Pattern Recognition},
66 |          year = {2017}
67 |      }
68 | ```
69 | 
70 | ## acknowledgment
71 | This work was supported by NSF IIS-1321119. The video above uses materials under a Creative Common license or with the owner's permission, as detailed at the end.
72 | 


--------------------------------------------------------------------------------
/.idea/markdown-navigator.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project version="4">
 3 |   <component name="MarkdownProjectSettings" wasCopied="true">
 4 |     <PreviewSettings splitEditorLayout="SPLIT" splitEditorPreview="PREVIEW" useGrayscaleRendering="false" zoomFactor="1.0" maxImageWidth="0" showGitHubPageIfSynced="false" allowBrowsingInPreview="false" synchronizePreviewPosition="true" highlightPreviewType="NONE" highlightFadeOut="5" highlightOnTyping="true" synchronizeSourcePosition="true" verticallyAlignSourceAndPreviewSyncPosition="true" showSearchHighlightsInPreview="false" showSelectionInPreview="true" openRemoteLinks="true" replaceUnicodeEmoji="false" lastLayoutSetsDefault="false">
 5 |       <PanelProvider>
 6 |         <provider providerId="com.vladsch.idea.multimarkdown.editor.swing.html.panel" providerName="Default - Swing" />
 7 |       </PanelProvider>
 8 |     </PreviewSettings>
 9 |     <ParserSettings gitHubSyntaxChange="false" emojiShortcuts="1" emojiImages="0">
10 |       <PegdownExtensions>
11 |         <option name="ABBREVIATIONS" value="false" />
12 |         <option name="ANCHORLINKS" value="true" />
13 |         <option name="ASIDE" value="false" />
14 |         <option name="ATXHEADERSPACE" value="true" />
15 |         <option name="AUTOLINKS" value="true" />
16 |         <option name="DEFINITIONS" value="false" />
17 |         <option name="DEFINITION_BREAK_DOUBLE_BLANK_LINE" value="false" />
18 |         <option name="FENCED_CODE_BLOCKS" value="true" />
19 |         <option name="FOOTNOTES" value="false" />
20 |         <option name="HARDWRAPS" value="false" />
21 |         <option name="HTML_DEEP_PARSER" value="false" />
22 |         <option name="INSERTED" value="false" />
23 |         <option name="QUOTES" value="false" />
24 |         <option name="RELAXEDHRULES" value="true" />
25 |         <option name="SMARTS" value="false" />
26 |         <option name="STRIKETHROUGH" value="true" />
27 |         <option name="SUBSCRIPT" value="false" />
28 |         <option name="SUPERSCRIPT" value="false" />
29 |         <option name="SUPPRESS_HTML_BLOCKS" value="false" />
30 |         <option name="SUPPRESS_INLINE_HTML" value="false" />
31 |         <option name="TABLES" value="true" />
32 |         <option name="TASKLISTITEMS" value="true" />
33 |         <option name="TOC" value="false" />
34 |         <option name="WIKILINKS" value="true" />
35 |       </PegdownExtensions>
36 |       <ParserOptions>
37 |         <option name="ADMONITION_EXT" value="false" />
38 |         <option name="ATTRIBUTES_EXT" value="false" />
39 |         <option name="COMMONMARK_LISTS" value="true" />
40 |         <option name="DUMMY" value="false" />
41 |         <option name="EMOJI_SHORTCUTS" value="true" />
42 |         <option name="ENUMERATED_REFERENCES_EXT" value="false" />
43 |         <option name="FLEXMARK_FRONT_MATTER" value="false" />
44 |         <option name="GFM_LOOSE_BLANK_LINE_AFTER_ITEM_PARA" value="false" />
45 |         <option name="GFM_TABLE_RENDERING" value="true" />
46 |         <option name="GITBOOK_URL_ENCODING" value="false" />
47 |         <option name="GITHUB_LISTS" value="false" />
48 |         <option name="GITHUB_WIKI_LINKS" value="true" />
49 |         <option name="HEADER_ID_NO_DUPED_DASHES" value="false" />
50 |         <option name="JEKYLL_FRONT_MATTER" value="false" />
51 |         <option name="NO_TEXT_ATTRIBUTES" value="false" />
52 |         <option name="PARSE_HTML_ANCHOR_ID" value="false" />
53 |         <option name="SIM_TOC_BLANK_LINE_SPACER" value="true" />
54 |       </ParserOptions>
55 |     </ParserSettings>
56 |     <HtmlSettings headerTopEnabled="false" headerBottomEnabled="false" bodyTopEnabled="false" bodyBottomEnabled="false" embedUrlContent="false" addPageHeader="true" embedImages="false" embedHttpImages="false" imageUriSerials="false">
57 |       <GeneratorProvider>
58 |         <provider providerId="com.vladsch.idea.multimarkdown.editor.swing.html.generator" providerName="Default Swing HTML Generator" />
59 |       </GeneratorProvider>
60 |       <headerTop />
61 |       <headerBottom />
62 |       <bodyTop />
63 |       <bodyBottom />
64 |     </HtmlSettings>
65 |     <CssSettings previewScheme="UI_SCHEME" cssUri="" isCssUriEnabled="false" isCssUriSerial="false" isCssTextEnabled="false" isDynamicPageWidth="true">
66 |       <StylesheetProvider>
67 |         <provider providerId="com.vladsch.idea.multimarkdown.editor.swing.html.css" providerName="Default Swing Stylesheet" />
68 |       </StylesheetProvider>
69 |       <ScriptProviders />
70 |       <cssText />
71 |       <cssUriHistory />
72 |     </CssSettings>
73 |     <HtmlExportSettings updateOnSave="false" parentDir="" targetDir="" cssDir="" scriptDir="" plainHtml="false" imageDir="" copyLinkedImages="false" imageUniquifyType="0" targetExt="" useTargetExt="false" noCssNoScripts="false" linkToExportedHtml="true" exportOnSettingsChange="true" regenerateOnProjectOpen="false" linkFormatType="HTTP_ABSOLUTE" />
74 |     <LinkMapSettings>
75 |       <textMaps />
76 |     </LinkMapSettings>
77 |   </component>
78 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="ChangeListManager">
  4 |     <list default="true" id="05ffb335-bff1-4815-80f2-3ebb6b0d49b7" name="Default" comment="" />
  5 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  6 |     <option name="TRACKING_ENABLED" value="true" />
  7 |     <option name="SHOW_DIALOG" value="false" />
  8 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
  9 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
 10 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 11 |   </component>
 12 |   <component name="FileEditorManager">
 13 |     <leaf>
 14 |       <file leaf-file-name="run.py" pinned="false" current-in-tab="true">
 15 |         <entry file="file://$PROJECT_DIR$/run.py">
 16 |           <provider selected="true" editor-type-id="text-editor">
 17 |             <state relative-caret-position="452">
 18 |               <caret line="254" column="17" lean-forward="false" selection-start-line="254" selection-start-column="17" selection-end-line="254" selection-end-column="17" />
 19 |               <folding>
 20 |                 <element signature="e#26#36#0" expanded="true" />
 21 |               </folding>
 22 |             </state>
 23 |           </provider>
 24 |         </entry>
 25 |       </file>
 26 |     </leaf>
 27 |   </component>
 28 |   <component name="IdeDocumentHistory">
 29 |     <option name="CHANGED_PATHS">
 30 |       <list>
 31 |         <option value="$PROJECT_DIR$/run.py" />
 32 |       </list>
 33 |     </option>
 34 |   </component>
 35 |   <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
 36 |   <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
 37 |   <component name="JsGulpfileManager">
 38 |     <detection-done>true</detection-done>
 39 |     <sorting>DEFINITION_ORDER</sorting>
 40 |   </component>
 41 |   <component name="ProjectFrameBounds" extendedState="7">
 42 |     <option name="x" value="15" />
 43 |     <option name="y" value="36" />
 44 |     <option name="width" value="1890" />
 45 |     <option name="height" value="930" />
 46 |   </component>
 47 |   <component name="ProjectView">
 48 |     <navigator currentView="ProjectPane" proportions="" version="1">
 49 |       <flattenPackages />
 50 |       <showMembers />
 51 |       <showModules />
 52 |       <showLibraryContents />
 53 |       <hideEmptyPackages />
 54 |       <abbreviatePackageNames />
 55 |       <autoscrollToSource />
 56 |       <autoscrollFromSource />
 57 |       <sortByType />
 58 |       <manualOrder />
 59 |       <foldersAlwaysOnTop value="true" />
 60 |     </navigator>
 61 |     <panes>
 62 |       <pane id="Scratches" />
 63 |       <pane id="ProjectPane">
 64 |         <subPane>
 65 |           <expand>
 66 |             <path>
 67 |               <item name="pytorch-sepconv-master" type="b2602c69:ProjectViewProjectNode" />
 68 |               <item name="pytorch-sepconv-master" type="462c0819:PsiDirectoryNode" />
 69 |             </path>
 70 |           </expand>
 71 |           <select />
 72 |         </subPane>
 73 |       </pane>
 74 |       <pane id="Scope" />
 75 |     </panes>
 76 |   </component>
 77 |   <component name="PropertiesComponent">
 78 |     <property name="last_opened_file_path" value="$PROJECT_DIR$" />
 79 |     <property name="WebServerToolWindowFactoryState" value="false" />
 80 |   </component>
 81 |   <component name="RunDashboard">
 82 |     <option name="ruleStates">
 83 |       <list>
 84 |         <RuleState>
 85 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
 86 |         </RuleState>
 87 |         <RuleState>
 88 |           <option name="name" value="StatusDashboardGroupingRule" />
 89 |         </RuleState>
 90 |       </list>
 91 |     </option>
 92 |   </component>
 93 |   <component name="ShelveChangesManager" show_recycled="false">
 94 |     <option name="remove_strategy" value="false" />
 95 |   </component>
 96 |   <component name="TaskManager">
 97 |     <task active="true" id="Default" summary="Default task">
 98 |       <changelist id="05ffb335-bff1-4815-80f2-3ebb6b0d49b7" name="Default" comment="" />
 99 |       <created>1533642385436</created>
100 |       <option name="number" value="Default" />
101 |       <option name="presentableId" value="Default" />
102 |       <updated>1533642385436</updated>
103 |     </task>
104 |     <servers />
105 |   </component>
106 |   <component name="ToolWindowManager">
107 |     <frame x="-9" y="-9" width="1938" height="1048" extended-state="7" />
108 |     <editor active="true" />
109 |     <layout>
110 |       <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
111 |       <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
112 |       <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
113 |       <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
114 |       <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
115 |       <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
116 |       <window_info id="JSON Viewer" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
117 |       <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24356224" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
118 |       <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
119 |       <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
120 |       <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
121 |       <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
122 |       <window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
123 |       <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
124 |       <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
125 |       <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
126 |       <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
127 |       <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
128 |       <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
129 |       <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
130 |     </layout>
131 |   </component>
132 |   <component name="TypeScriptGeneratedFilesManager">
133 |     <option name="version" value="1" />
134 |   </component>
135 |   <component name="VcsContentAnnotationSettings">
136 |     <option name="myLimit" value="2678400000" />
137 |   </component>
138 |   <component name="XDebuggerManager">
139 |     <breakpoint-manager />
140 |     <watches-manager />
141 |   </component>
142 |   <component name="editorHistoryManager">
143 |     <entry file="file://$PROJECT_DIR$/run.py">
144 |       <provider selected="true" editor-type-id="text-editor">
145 |         <state relative-caret-position="452">
146 |           <caret line="254" column="17" lean-forward="false" selection-start-line="254" selection-start-column="17" selection-end-line="254" selection-end-column="17" />
147 |           <folding>
148 |             <element signature="e#26#36#0" expanded="true" />
149 |           </folding>
150 |         </state>
151 |       </provider>
152 |     </entry>
153 |   </component>
154 | </project>


--------------------------------------------------------------------------------
/run.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python2.7
  2 | 
  3 | import sys
  4 | import os
  5 | import getopt
  6 | import math
  7 | import numpy
  8 | import torch
  9 | import torch.utils.serialization
 10 | import PIL
 11 | import PIL.Image
 12 | from moviepy.editor import *
 13 | 
 14 | from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader
 15 | from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter
 16 | 
 17 | from SeparableConvolution import SeparableConvolution # the custom SeparableConvolution layer
 18 | 
 19 | torch.cuda.device(1) # change this if you have a multiple graphics cards and you want to utilize them
 20 | 
 21 | torch.backends.cudnn.enabled = True # make sure to use cudnn for computational performance
 22 | 
 23 | ##########################################################
 24 | 
 25 | arguments_strModel = 'lf'
 26 | arguments_strFirst = './images/first.png'
 27 | arguments_strSecond = './images/second.png'
 28 | arguments_strOut = './result.png'
 29 | arguments_strVideo = ''
 30 | arguments_strVideoOut = ''
 31 | 
 32 | for strOption, strArgument in getopt.getopt(sys.argv[1:], '', [ strParameter[2:] + '=' for strParameter in sys.argv[1::2] ])[0]:
 33 | 	if strOption == '--model':
 34 | 		arguments_strModel = strArgument # which model to use, l1 or lf, please see our paper for more details
 35 | 
 36 | 	elif strOption == '--first':
 37 | 		arguments_strFirst = strArgument # path to the first frame
 38 | 
 39 | 	elif strOption == '--second':
 40 | 		arguments_strSecond = strArgument # path to the second frame
 41 | 
 42 | 	elif strOption == '--out':
 43 | 		arguments_strOut = strArgument # path to where the output should be stored
 44 | 
 45 | 	elif strOption == '--video':
 46 | 		arguments_strVideo = strArgument # path to the video
 47 | 
 48 | 	elif strOption == '--video-out':
 49 | 		arguments_strVideoOut = strArgument # path to the video
 50 | 	# end
 51 | # end
 52 | 
 53 | ##########################################################
 54 | 
 55 | class Network(torch.nn.Module):
 56 | 	def __init__(self):
 57 | 		super(Network, self).__init__()
 58 | 
 59 | 		def Basic(intInput, intOutput):
 60 | 			return torch.nn.Sequential(
 61 | 				torch.nn.Conv2d(in_channels=intInput, out_channels=intOutput, kernel_size=3, stride=1, padding=1),
 62 | 				torch.nn.ReLU(inplace=False),
 63 | 				torch.nn.Conv2d(in_channels=intOutput, out_channels=intOutput, kernel_size=3, stride=1, padding=1),
 64 | 				torch.nn.ReLU(inplace=False),
 65 | 				torch.nn.Conv2d(in_channels=intOutput, out_channels=intOutput, kernel_size=3, stride=1, padding=1),
 66 | 				torch.nn.ReLU(inplace=False)
 67 | 			)
 68 | 		# end
 69 | 
 70 | 		def Subnet():
 71 | 			return torch.nn.Sequential(
 72 | 				torch.nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1),
 73 | 				torch.nn.ReLU(inplace=False),
 74 | 				torch.nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1),
 75 | 				torch.nn.ReLU(inplace=False),
 76 | 				torch.nn.Conv2d(in_channels=64, out_channels=51, kernel_size=3, stride=1, padding=1),
 77 | 				torch.nn.ReLU(inplace=False),
 78 | 				torch.nn.Upsample(scale_factor=2, mode='bilinear'),
 79 | 				torch.nn.Conv2d(in_channels=51, out_channels=51, kernel_size=3, stride=1, padding=1)
 80 | 			)
 81 | 		# end
 82 | 
 83 | 		self.moduleConv1 = Basic(6, 32)
 84 | 		self.modulePool1 = torch.nn.AvgPool2d(kernel_size=2, stride=2)
 85 | 
 86 | 		self.moduleConv2 = Basic(32, 64)
 87 | 		self.modulePool2 = torch.nn.AvgPool2d(kernel_size=2, stride=2)
 88 | 
 89 | 		self.moduleConv3 = Basic(64, 128)
 90 | 		self.modulePool3 = torch.nn.AvgPool2d(kernel_size=2, stride=2)
 91 | 
 92 | 		self.moduleConv4 = Basic(128, 256)
 93 | 		self.modulePool4 = torch.nn.AvgPool2d(kernel_size=2, stride=2)
 94 | 
 95 | 		self.moduleConv5 = Basic(256, 512)
 96 | 		self.modulePool5 = torch.nn.AvgPool2d(kernel_size=2, stride=2)
 97 | 
 98 | 		self.moduleDeconv5 = Basic(512, 512)
 99 | 		self.moduleUpsample5 = torch.nn.Sequential(
100 | 			torch.nn.Upsample(scale_factor=2, mode='bilinear'),
101 | 			torch.nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1),
102 | 			torch.nn.ReLU(inplace=False)
103 | 		)
104 | 
105 | 		self.moduleDeconv4 = Basic(512, 256)
106 | 		self.moduleUpsample4 = torch.nn.Sequential(
107 | 			torch.nn.Upsample(scale_factor=2, mode='bilinear'),
108 | 			torch.nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1),
109 | 			torch.nn.ReLU(inplace=False)
110 | 		)
111 | 
112 | 		self.moduleDeconv3 = Basic(256, 128)
113 | 		self.moduleUpsample3 = torch.nn.Sequential(
114 | 			torch.nn.Upsample(scale_factor=2, mode='bilinear'),
115 | 			torch.nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1),
116 | 			torch.nn.ReLU(inplace=False)
117 | 		)
118 | 
119 | 		self.moduleDeconv2 = Basic(128, 64)
120 | 		self.moduleUpsample2 = torch.nn.Sequential(
121 | 			torch.nn.Upsample(scale_factor=2, mode='bilinear'),
122 | 			torch.nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1),
123 | 			torch.nn.ReLU(inplace=False)
124 | 		)
125 | 
126 | 		self.moduleVertical1 = Subnet()
127 | 		self.moduleVertical2 = Subnet()
128 | 		self.moduleHorizontal1 = Subnet()
129 | 		self.moduleHorizontal2 = Subnet()
130 | 
131 | 		self.modulePad = torch.nn.ReplicationPad2d([ int(math.floor(51 / 2.0)), int(math.floor(51 / 2.0)), int(math.floor(51 / 2.0)), int(math.floor(51 / 2.0)) ])
132 | 
133 | 		self.load_state_dict(torch.load('./network-' + arguments_strModel + '.pytorch'))
134 | 	# end
135 | 
136 | 	def forward(self, variableInput1, variableInput2):
137 | 		variableJoin = torch.cat([variableInput1, variableInput2], 1)
138 | 
139 | 		variableConv1 = self.moduleConv1(variableJoin)
140 | 		variablePool1 = self.modulePool1(variableConv1)
141 | 
142 | 		variableConv2 = self.moduleConv2(variablePool1)
143 | 		variablePool2 = self.modulePool2(variableConv2)
144 | 
145 | 		variableConv3 = self.moduleConv3(variablePool2)
146 | 		variablePool3 = self.modulePool3(variableConv3)
147 | 
148 | 		variableConv4 = self.moduleConv4(variablePool3)
149 | 		variablePool4 = self.modulePool4(variableConv4)
150 | 
151 | 		variableConv5 = self.moduleConv5(variablePool4)
152 | 		variablePool5 = self.modulePool5(variableConv5)
153 | 
154 | 		variableDeconv5 = self.moduleDeconv5(variablePool5)
155 | 		variableUpsample5 = self.moduleUpsample5(variableDeconv5)
156 | 
157 | 		variableDeconv4 = self.moduleDeconv4(variableUpsample5 + variableConv5)
158 | 		variableUpsample4 = self.moduleUpsample4(variableDeconv4)
159 | 
160 | 		variableDeconv3 = self.moduleDeconv3(variableUpsample4 + variableConv4)
161 | 		variableUpsample3 = self.moduleUpsample3(variableDeconv3)
162 | 
163 | 		variableDeconv2 = self.moduleDeconv2(variableUpsample3 + variableConv3)
164 | 		variableUpsample2 = self.moduleUpsample2(variableDeconv2)
165 | 
166 | 		variableCombine = variableUpsample2 + variableConv2
167 | 
168 | 		variableDot1 = SeparableConvolution()(self.modulePad(variableInput1), self.moduleVertical1(variableCombine), self.moduleHorizontal1(variableCombine))
169 | 		variableDot2 = SeparableConvolution()(self.modulePad(variableInput2), self.moduleVertical2(variableCombine), self.moduleHorizontal2(variableCombine))
170 | 
171 | 		return variableDot1 + variableDot2
172 | 	# end
173 | # end
174 | 
175 | moduleNetwork = Network().cuda()
176 | 
177 | ##########################################################
178 | 
179 | def process(tensorInputFirst, tensorInputSecond, tensorOutput):
180 | 	assert(tensorInputFirst.size(1) == tensorInputSecond.size(1))
181 | 	assert(tensorInputFirst.size(2) == tensorInputSecond.size(2))
182 | 
183 | 	intWidth = tensorInputFirst.size(2)
184 | 	intHeight = tensorInputFirst.size(1)
185 | 
186 | 	assert(intWidth <= 1920) # while our approach works with larger images, we do not recommend it unless you are aware of the implications
187 | 	assert(intHeight <= 1080) # while our approach works with larger images, we do not recommend it unless you are aware of the implications
188 | 
189 | 	intPaddingLeft = int(math.floor(51 / 2.0))
190 | 	intPaddingTop = int(math.floor(51 / 2.0))
191 | 	intPaddingRight = int(math.floor(51 / 2.0))
192 | 	intPaddingBottom = int(math.floor(51 / 2.0))
193 | 	modulePaddingInput = torch.nn.Module()
194 | 	modulePaddingOutput = torch.nn.Module()
195 | 
196 | 	if True:
197 | 		intPaddingWidth = intPaddingLeft + intWidth + intPaddingRight
198 | 		intPaddingHeight = intPaddingTop + intHeight + intPaddingBottom
199 | 
200 | 		if intPaddingWidth != ((intPaddingWidth >> 7) << 7):
201 | 			intPaddingWidth = (((intPaddingWidth >> 7) + 1) << 7) # more than necessary
202 | 		# end
203 | 		
204 | 		if intPaddingHeight != ((intPaddingHeight >> 7) << 7):
205 | 			intPaddingHeight = (((intPaddingHeight >> 7) + 1) << 7) # more than necessary
206 | 		# end
207 | 
208 | 		intPaddingWidth = intPaddingWidth - (intPaddingLeft + intWidth + intPaddingRight)
209 | 		intPaddingHeight = intPaddingHeight - (intPaddingTop + intHeight + intPaddingBottom)
210 | 
211 | 		modulePaddingInput = torch.nn.ReplicationPad2d([intPaddingLeft, intPaddingRight + intPaddingWidth, intPaddingTop, intPaddingBottom + intPaddingHeight])
212 | 		modulePaddingOutput = torch.nn.ReplicationPad2d([0 - intPaddingLeft, 0 - intPaddingRight - intPaddingWidth, 0 - intPaddingTop, 0 - intPaddingBottom - intPaddingHeight])
213 | 	# end
214 | 
215 | 	if True:
216 | 		tensorInputFirst = tensorInputFirst.cuda()
217 | 		tensorInputSecond = tensorInputSecond.cuda()
218 | 
219 | 		modulePaddingInput = modulePaddingInput.cuda()
220 | 		modulePaddingOutput = modulePaddingOutput.cuda()
221 | 	# end
222 | 
223 | 	if True:
224 | 		variablePaddingFirst = modulePaddingInput(torch.autograd.Variable(data=tensorInputFirst.view(1, 3, intHeight, intWidth), volatile=True))
225 | 		variablePaddingSecond = modulePaddingInput(torch.autograd.Variable(data=tensorInputSecond.view(1, 3, intHeight, intWidth), volatile=True))
226 | 		variablePaddingOutput = modulePaddingOutput(moduleNetwork(variablePaddingFirst, variablePaddingSecond))
227 | 
228 | 		tensorOutput.resize_(3, intHeight, intWidth).copy_(variablePaddingOutput.data[0])
229 | 	# end
230 | 
231 | 	if True:
232 | 		tensorInputFirst.cpu()
233 | 		tensorInputSecond.cpu()
234 | 		tensorOutput.cpu()
235 | 	# end
236 | #end
237 | 
238 | tensorOutput = torch.FloatTensor()
239 | 
240 | if arguments_strVideo and arguments_strVideoOut:
241 | 
242 | 	if not os.path.exists(arguments_strVideo):
243 | 		print('{} is not exits! Plz check~'.format(arguments_strVideo))
244 | 		break
245 | 
246 | 	# Process video
247 | 	reader = FFMPEG_VideoReader(arguments_strVideo, False)
248 | 	temp = './temp.mp4'
249 | 	writer = FFMPEG_VideoWriter(temp, reader.size, reader.fps*2)
250 | 	reader.initialize()
251 | 	print('##### processing {} fps={} #########'.format(arguments_strVideo, reader.fps))
252 | 	nextFrame = reader.read_frame()
253 | 	num = reader.nframes  # number of frames
254 | 	for x in range(0, num):
255 | 		# progress bar
256 | 		if x % (reader.fps * 10) == 0:
257 | 			sys.stdout.write(' ' * 10 + '\r')
258 | 			sys.stdout.flush()
259 | 			sys.stdout.write('{:.1f}%'.format(x * 100 / num) + '\r')
260 | 			sys.stdout.flush()
261 | 
262 | 		firstFrame = nextFrame
263 | 		nextFrame = reader.read_frame()
264 | 		tensorInputFirst = torch.FloatTensor(numpy.rollaxis(firstFrame[:,:,::-1], 2, 0) / 255.0)
265 | 		tensorInputSecond = torch.FloatTensor(numpy.rollaxis(nextFrame[:,:,::-1], 2, 0) / 255.0)
266 | 		process(tensorInputFirst, tensorInputSecond, tensorOutput)
267 | 		writer.write_frame(firstFrame)
268 | 		writer.write_frame((numpy.rollaxis(tensorOutput.clamp(0.0, 1.0).numpy(), 0, 3)[:,:,::-1] * 255.0).astype(numpy.uint8))
269 | 	#end
270 | 	writer.write_frame(nextFrame)
271 | 	writer.close()
272 | 	reader.close()
273 | 
274 | 	# mix the video and audio
275 | 	audioclip = AudioFileClip(arguments_strVideo)
276 | 	videoclip = VideoFileClip(temp)
277 | 
278 | 	videoclip2 = videoclip.set_audio(audioclip)
279 | 	videoclip2.write_videofile(arguments_strVideoOut, progress_bar=False)
280 | 
281 | 	if os.path.exists(temp):
282 | 		os.remove(temp)
283 | 
284 | else:
285 | 	# Process image
286 | 	tensorInputFirst = torch.FloatTensor(numpy.rollaxis(numpy.asarray(PIL.Image.open(arguments_strFirst))[:,:,::-1], 2, 0).astype(numpy.float32) / 255.0)
287 | 	tensorInputSecond = torch.FloatTensor(numpy.rollaxis(numpy.asarray(PIL.Image.open(arguments_strSecond))[:,:,::-1], 2, 0).astype(numpy.float32) / 255.0)	
288 | 	process(tensorInputFirst, tensorInputSecond, tensorOutput)
289 | 	PIL.Image.fromarray((numpy.rollaxis(tensorOutput.clamp(0.0, 1.0).numpy(), 0, 3)[:,:,::-1] * 255.0).astype(numpy.uint8)).save(arguments_strOut)
290 | #end
291 | 


--------------------------------------------------------------------------------