├── .gitignore
├── README.md
├── arxiv_videos
    ├── .DS_Store
    ├── all_stack.mp4
    ├── fb_pairs
    │   ├── cityscapes_guidance.gif
    │   ├── cityscapes_vid2vid.gif
    │   ├── mannequin_guidance.gif
    │   ├── mannequin_vid2vid.gif
    │   ├── scannet_guidance.gif
    │   └── scannet_vid2vid.gif
    ├── inputs_outputs.mp4
    ├── mann_scan
    │   ├── 01_n.mp4
    │   ├── 02_n.mp4
    │   ├── 03_n.mp4
    │   └── 04_n.mp4
    ├── stereo
    │   ├── stereo_stuttgart_00_346_flow.mp4
    │   ├── stereo_stuttgart_00_346_guidance.mp4
    │   ├── stereo_stuttgart_02_6132_flow.mp4
    │   └── stereo_stuttgart_02_6132_guidance.mp4
    ├── stuttgart_01_06_flow.mp4
    ├── stuttgart_01_06_guidance.mp4
    ├── stuttgart_01_06_inpainting.mp4
    ├── stuttgart_01_06_single.mp4
    ├── stuttgart_01_06_vid2vid.mp4
    ├── stuttgart_02_06_flow.mp4
    ├── stuttgart_02_06_guidance.mp4
    ├── stuttgart_02_06_inpainting.mp4
    ├── stuttgart_02_06_single.mp4
    └── stuttgart_02_06_vid2vid.mp4
├── files
    └── wc-vid2vid.pdf
├── images
    ├── favicon.ico
    ├── guidance.png
    └── overview.png
├── index.html
└── videos
    ├── fb_guidance.mp4
    ├── fb_vid2vid.mp4
    ├── mannequin.mp4
    ├── sample_output.gif
    ├── side_by_side.gif
    ├── side_by_side.mp4
    ├── stereo_1.mp4
    ├── stereo_2.mp4
    ├── stuttgart_00_02.mp4
    ├── stuttgart_01_04.mp4
    ├── stuttgart_01_04_all.mp4
    ├── stuttgart_01_06.mp4
    ├── what_is_guidance.mp4
    └── what_is_vid2vid.mp4


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ### This repository is only for creating the project page. Our code will be released in [Imaginaire](https://github.com/nvlabs/imaginaire)
2 | 


--------------------------------------------------------------------------------
/arxiv_videos/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/.DS_Store


--------------------------------------------------------------------------------
/arxiv_videos/all_stack.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/all_stack.mp4


--------------------------------------------------------------------------------
/arxiv_videos/fb_pairs/cityscapes_guidance.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/fb_pairs/cityscapes_guidance.gif


--------------------------------------------------------------------------------
/arxiv_videos/fb_pairs/cityscapes_vid2vid.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/fb_pairs/cityscapes_vid2vid.gif


--------------------------------------------------------------------------------
/arxiv_videos/fb_pairs/mannequin_guidance.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/fb_pairs/mannequin_guidance.gif


--------------------------------------------------------------------------------
/arxiv_videos/fb_pairs/mannequin_vid2vid.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/fb_pairs/mannequin_vid2vid.gif


--------------------------------------------------------------------------------
/arxiv_videos/fb_pairs/scannet_guidance.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/fb_pairs/scannet_guidance.gif


--------------------------------------------------------------------------------
/arxiv_videos/fb_pairs/scannet_vid2vid.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/fb_pairs/scannet_vid2vid.gif


--------------------------------------------------------------------------------
/arxiv_videos/inputs_outputs.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/inputs_outputs.mp4


--------------------------------------------------------------------------------
/arxiv_videos/mann_scan/01_n.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/mann_scan/01_n.mp4


--------------------------------------------------------------------------------
/arxiv_videos/mann_scan/02_n.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/mann_scan/02_n.mp4


--------------------------------------------------------------------------------
/arxiv_videos/mann_scan/03_n.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/mann_scan/03_n.mp4


--------------------------------------------------------------------------------
/arxiv_videos/mann_scan/04_n.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/mann_scan/04_n.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stereo/stereo_stuttgart_00_346_flow.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stereo/stereo_stuttgart_00_346_flow.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stereo/stereo_stuttgart_00_346_guidance.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stereo/stereo_stuttgart_00_346_guidance.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stereo/stereo_stuttgart_02_6132_flow.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stereo/stereo_stuttgart_02_6132_flow.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stereo/stereo_stuttgart_02_6132_guidance.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stereo/stereo_stuttgart_02_6132_guidance.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_01_06_flow.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_01_06_flow.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_01_06_guidance.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_01_06_guidance.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_01_06_inpainting.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_01_06_inpainting.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_01_06_single.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_01_06_single.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_01_06_vid2vid.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_01_06_vid2vid.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_02_06_flow.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_02_06_flow.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_02_06_guidance.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_02_06_guidance.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_02_06_inpainting.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_02_06_inpainting.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_02_06_single.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_02_06_single.mp4


--------------------------------------------------------------------------------
/arxiv_videos/stuttgart_02_06_vid2vid.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/arxiv_videos/stuttgart_02_06_vid2vid.mp4


--------------------------------------------------------------------------------
/files/wc-vid2vid.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/files/wc-vid2vid.pdf


--------------------------------------------------------------------------------
/images/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/images/favicon.ico


--------------------------------------------------------------------------------
/images/guidance.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/images/guidance.png


--------------------------------------------------------------------------------
/images/overview.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/images/overview.png


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
  1 | 
  2 | <!doctype html>
  3 | <html lang="en">
  4 |   <head>
  5 |     <!-- Global site tag (gtag.js) - Google Analytics -->
  6 |     <script async src="https://www.googletagmanager.com/gtag/js?id=UA-140332927-4"></script>
  7 |     <script>
  8 |         window.dataLayer = window.dataLayer || [];
  9 |         function gtag(){dataLayer.push(arguments);}
 10 |         gtag('js', new Date());
 11 | 
 12 |         gtag('config', 'UA-140332927-4');
 13 |     </script>
 14 | 	<!-- Global site tag (gtag.js) - Google Analytics -->
 15 | 	<script async src="https://www.googletagmanager.com/gtag/js?id=UA-174254593-3"></script>
 16 | 	<script>
 17 | 	  window.dataLayer = window.dataLayer || [];
 18 | 	  function gtag(){dataLayer.push(arguments);}
 19 | 	  gtag('js', new Date());
 20 | 
 21 | 	  gtag('config', 'UA-174254593-3');
 22 | 	</script>
 23 | 
 24 |     <meta charset="utf-8">
 25 |     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 26 |     <meta name="description" content="">
 27 |     <meta name="author" content="">
 28 |     <link rel="icon" href="images/favicon.ico">
 29 | 
 30 |     <title>World-Consistent Video-to-Video Synthesis</title>
 31 | 
 32 |     <!-- Bootstrap core CSS -->
 33 |     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
 34 | 
 35 | 
 36 |     <!-- Custom styles for this template -->
 37 |     <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Sans+Pro">
 38 |     <link rel="dns-prefetch" href="//cdn.jsdelivr.net" />
 39 |     <link href="https://cdn.jsdelivr.net/npm/prismjs/themes/prism.min.css" rel="stylesheet" />
 40 |     <link href="https://cdn.jsdelivr.net/gh/jablonczay/code-box-copy/code-box-copy/css/code-box-copy.min.css" rel="stylesheet" />
 41 | 
 42 |     <style>
 43 |         body {
 44 |             font-family: 'Source Sans Pro', sans-serif;
 45 |             padding-bottom: 50px;
 46 |         }
 47 |         hr {
 48 |             background: #80808083;
 49 |         }
 50 |         .embed-responsive-2by1 {
 51 |             padding-bottom: 50%;
 52 |         }
 53 |         .embed-responsive-4by1 {
 54 |             padding-bottom: 25%;
 55 |         }
 56 |         .embed-responsive-6by1 {
 57 |             padding-bottom: 16.67%;
 58 |         }
 59 |         .embed-responsive-8by1 {
 60 |             padding-bottom: 12.50%;
 61 |         }
 62 |         .embed-responsive-teaser {
 63 |             padding-bottom: 29%;
 64 |         }
 65 |         .float-button {
 66 |             position: fixed;
 67 |             right: 1%;
 68 |             z-index: 9999;
 69 |         }
 70 |     </style>
 71 |   </head>
 72 | 
 73 |   <body>
 74 |     <header>
 75 |     <main role="main">
 76 | 
 77 |       <section class="jumbotron text-center" style="padding: 2%; background-color: #e6e9ec;">
 78 |         <div class="container">
 79 |           <h1 class="jumbotron-heading">World-Consistent Video-to-Video Synthesis</h1>
 80 |         </div>
 81 | 
 82 |         <div class="container" style="max-width: 768px;">
 83 |             <div class="row">
 84 |                 <div class="col-md">
 85 |                     <h5 id="authors" class="text-center" style="margin: 0px;"><a class="text-center" href="http://arunmallya.github.io/" target="_blank">Arun Mallya*</a></h5>
 86 |                 </div>
 87 |                 <div class="col-md">
 88 |                     <h5 id="authors" class="text-center" style="margin: 0px;"><a class="text-center" href="https://tcwang0509.github.io/" target="_blank">Ting-Chun Wang*</a></h5>
 89 |                 </div>
 90 |                 <div class="col-md">
 91 |                     <h5 id="authors" class="text-center" style="margin: 0px;"><a class="text-center" href="https://karansapra.github.io/" target="_blank">Karan Sapra</a></h5>
 92 |                 </div>
 93 |                 <div class="col-md">
 94 |                     <h5 id="authors" class="text-center" style="margin: 0px;"><a class="text-center" href="http://mingyuliu.net/" target="_blank">Ming-Yu Liu</a></h5>
 95 |                 </div>
 96 |             </div>
 97 | 
 98 |             <div class="row">
 99 |             	<div class="col-md">
100 |                     <h5 style="color: #76b900; margin-bottom: 0;"><a class="text-center" href="https://www.nvidia.com/en-us/research/" target="_blank" style="color: #76b900;"><b>NVIDIA</b></a></h5>
101 |                     <h5 style="margin-bottom: 0;">Published at the <a class="text-center" href="https://eccv2020.eu/" target="_blank" style="color: black;">European Conference on Computer Vision, 2020</a></h5>
102 |             	</div>
103 |             </div>
104 |         </div>
105 | 
106 |         <div class="buttons">
107 |             <a href="https://arxiv.org/abs/2007.08509" target="_blank" class="btn btn-primary my-2">Paper (arxiv)</a>
108 |             <a href="https://nvlabs.github.io/wc-vid2vid/files/wc-vid2vid.pdf" target="_blank" class="btn btn-primary my-2">Paper (embedded videos)</a>
109 |             <a href="https://github.com/NVlabs/imaginaire" target="_blank" class="btn btn-secondary my-2">Code (GitHub)</a>
110 |           </p>
111 |         </div>
112 | 
113 |         <div class="container" style="max-width: 900px;">
114 |             <div class="row">
115 |                 <div class="col-md">
116 |                     <p>We present a GAN-based approach to generate 2D world renderings that are consistent over time and viewpoints, which was not possible with prior approaches.
117 |                     Our method colors the 3D point cloud of the world as the camera moves through the world, coloring new regions in a manner consistent with the already colored world.
118 |                     It learns to render images based on the 2D projections of the point cloud to the camera in a semantically consistent manner while robustly dealing with incorrect and incomplete point clouds.
119 |                     Our proposed approach further shortens the gap between classical graphics rendering and neural rendering.</p>
120 |                 </div>
121 |             </div>
122 | 
123 |             <div class="row">
124 |                 <div class="col text-center"><b>Colorization of the world's 3D point cloud</b></div>
125 |                 <div class="col text-center"><b>Simultaneously rendered 2D output</b></div>
126 |             </div>
127 |             <div class="row">
128 |                 <div class="embed-responsive embed-responsive-teaser" style="margin: 0;">
129 |                     <video controls loop class="embed-responsive-item">
130 |                         <source src="videos/side_by_side.mp4" type="video/mp4">
131 |                     </video>
132 |                 </div></br>
133 |             </div>
134 |         </div>
135 | 
136 |       </section>
137 |     </main>
138 | 
139 |     <!-- TL;DR floating button -->
140 |     <div class="float-button">
141 |         <p class="float-right">
142 |             <a href="#Summary">TL;DR</a>
143 |         </p>
144 |     </div>
145 | 
146 |     <!-- Summary video -->
147 |     <div class="container" style="max-width: 768px;">
148 |         <h5 class="text-center">Summary Video</h5>
149 |         <div class="embed-responsive embed-responsive-16by9" style="margin: 0;">
150 |             <iframe class="embed-responsive-item" src="https://www.youtube.com/embed/rlCh6-2NfSg" allowfullscreen></iframe>
151 |         </div>
152 |         </br>
153 | 
154 |         <h5 class="text-center">Presentation Video</h5>
155 |         <div class="embed-responsive embed-responsive-16by9" style="margin: 0;">
156 |             <iframe class="embed-responsive-item" src="https://www.youtube.com/embed/b2P39sS2kKo" allowfullscreen></iframe>
157 |         </div>
158 |         <!-- <p class="text-center"><a href="https://www.youtube.com/watch?v=rlCh6-2NfSg" target="_blank">Previous video</a></p> -->
159 |     </div>
160 | 
161 |     <!-- Overview -->
162 |     <hr style="max-width: 768px;">
163 |     <div class="container" style="max-width: 768px;">
164 |         <div class="row">
165 |             <div class="col-md-12">
166 |                 <h2>Overview</h2>
167 | 
168 |                 <!-- What is the task of vid2vid? -->
169 |                 <h5 class="text-center">What is the task of vid2vid?</h5>
170 |                 Video-to-video synthesis (vid2vid) is a powerful tool for converting high-level semantic inputs to photorealistic videos. An example of this task is shown in the video below. Given per-frame labels such as the semantic segmentation and depth map, our goal is to generate the video shown on the right side. You can imagine the inputs being generated by a game engine, with the goal of producing a hyper-realistic gaming experience. Alternatively, you might want to convert a video captured by a user to one that looks like a stylized game.
171 |             </div>
172 |         </div>
173 |         </br>
174 |         <div class="row">
175 |             <div class="col text-center"><b>Semantic Segmentation</b></div>
176 |             <div class="col text-center"><b>Depth</b></div>
177 |             <div class="col text-center"><b>Rendered Video</b></div>
178 |         </div>
179 |         <div class="row">
180 |             <div class="embed-responsive embed-responsive-6by1" style="margin: 0;">
181 |                 <video controls loop class="embed-responsive-item">
182 |                     <source src="videos/what_is_vid2vid.mp4" type="video/mp4">
183 |                 </video>
184 |             </div>
185 |         </div>
186 | 
187 | 
188 |         <!-- Issues with prior work. -->
189 |         <div class="row">
190 |             <div class="col-md-12">
191 |                 <hr style="max-width: 768px;">
192 |                 <h5 class="text-center">Issues with prior work</h5>
193 |                 The most relevant prior work in this area is <a class="text-center" href="https://tcwang0509.github.io/vid2vid/" target="_blank">Video-to-Video Synthesis</a>, published at NeurIPS 2018. One of the major shortcomings of this work is that it fails to ensure long-term consistency in the output video. This is because it generates each frame only based on the past few generated frames and lacks knowledge of the structure of the 3D world being generated. Some of the issues are demonstrated in the video below, where we drive forward and then backward to the starting point.
194 | 
195 |                 <!-- However, while existing vid2vid methods can maintain short-term temporal consistency, they fail to ensure long-term consistency in the outputs. This is because they generate each frame only based on the past few frames. They lack knowledge of the 3D world being generated. In this work, we propose a framework for utilizing all past generated frames when synthesizing each frame. This is achieved by condensing the 3D world generated so far into a physically-grounded estimate of the current frame, which we call the <span style="background-color: #ffff00d5"><em><strong>guidance image</strong></em></span>. A novel Multi-SPADE module is also proposed to take advantage of the information stored in the guidance images. Extensive experimental results on several challenging datasets verify the effectiveness of our method in achieving world consistency – the output video is consistent within the entire generated 3D world. -->
196 |             </div>
197 |         </div>
198 |         </br>
199 |         <div class="row">
200 |             <div class="embed-responsive embed-responsive-4by1" style="margin: 0;">
201 |                 <video controls class="embed-responsive-item">
202 |                     <source src="videos/fb_vid2vid.mp4" type="video/mp4">
203 |                 </video>
204 |             </div>
205 |         </div>
206 |         </br>
207 |         <div class="row">
208 |             <div class="col-md-12">
209 |                 We can immediately notice a few issues:
210 |                 <ul>
211 |                     <li>As highlighted at the end of the video, compared to the first frame, a lot of features are different in the last frame, even though we return back to the same viewpoint. In other words, the video is not world-consistent  over long timeframes.</li>
212 |                     <li>We observe flickering in the outputs, indicating a lack of fine short-term consistency and realism in features such as the trees, the road markings, etc.</li>
213 |                 </ul>
214 |             </div>
215 |         </div>
216 | 
217 |         <!-- Introduction to guidance images. -->
218 |         <div class="row">
219 |             <div class="col-md-12">
220 |                 <hr style="max-width: 768px;">
221 |                 <h5 class="text-center">Achieving consistency across views and time</h5>
222 |                 <p>We believe that in order to produce realistic outputs that are consistent over time and viewpoint change, the method must be aware of the 3D structure of the world.
223 |                 To achieve this, we introduce the concept of <span style="background-color: #ffff00d5"><em><strong>guidance images</strong></em></span>, which are physically-grounded estimates of what the next output frame should look like, based on how the world has been generated so far. As alluded to in their name, the role of these <span style="background-color: #ffff00d5"><em><strong>guidance images</strong></em></span> is to guide the generative model to produce colors and textures that respect previous outputs.</p>
224 | 
225 |                 <p>While prior works use optical flow to warp prior outputs, our guidance image differs from this in two aspects.
226 |                 First, instead of using optical flow, the guidance image should be generated by using the motion field, or scene flow, which describes the true motion of each 3D point in the world. Second, the guidance image should aggregate information from all past viewpoints (and thus frames), instead of only the direct previous frames as in vid2vid. This makes sure that the generated frame is consistent with the entire history.</p>
227 | 
228 |                 <p>The figure below shows one method to generate guidance images by using point clouds and camera locations obtained by performing <a class="text-center" href="https://en.wikipedia.org/wiki/Structure_from_Motion#:~:text=Structure%20from%20Motion%20(SfM)%20is,computer%20vision%20and%20visual%20perception." target="_blank">Structure from Motion (SfM)</a> on an input video. In case of a game rendering engine, the ground truth scene flow can be obtained and used to generate guidance images.</p>
229 | 
230 | 
231 |                 <img src="images/guidance.png" style="width: 100%;margin-bottom: 8px;">
232 |                 <div class="text-center">
233 |                     <p style="font-size: 0.9rem; margin-bottom: 0;"><strong>Generating 3D-aware guidance images</strong></p>
234 |                     <p style="color: #808080; font-size: 0.85rem;">
235 |                         A camera(s) with known parameters and positions travels over time \( t = 0,\cdots,N \). At \( t = 0 \), the scene is textureless and an output image is generated for this viewpoint. The output image is then back-projected to the scene and a guidance image for a subsequent camera position is generated by projecting the partially textured point cloud. Using this <span style="background-color: #ffff00d5"><em><strong>guidance image</strong></em></span>, the generative method can produce an output that is consistent across views and smooth over time.
236 |                         The guidance image can be noisy, misaligned, and have holes, and the generation method should be robust to such inputs.
237 |                     </p>
238 |                 </div>
239 |             </div>
240 |         </div>
241 |         <div class="row">
242 |             <div class="col-md-12">
243 |                 <p>As input to our model, we provide the guidance images along with the input labels such as semantic segmentation and depth maps. The inputs and generated outputs are visualized in the below video. As new 3D points become visible to the camera, colors are assigned to them by our image generator and the point cloud is updated. Note that the guidance images have holes and incorrect projections due to noisy point cloud information. Our method is robust to such noise and produces meaningful outputs.</p>
244 |             </div>
245 |         </div>
246 |         <div class="row">
247 |             <div class="col text-center"><b>Semantic Segmentation</b></div>
248 |             <div class="col text-center"><b>Depth</b></div>
249 |             <div class="col text-center"><span style="background-color: #ffff00d5"><strong>Guidance Image</strong></span></div>
250 |             <div class="col text-center"><b>Rendered Video</b></div>
251 |         </div>
252 |         <div class="row">
253 |             <div class="embed-responsive embed-responsive-8by1" style="margin: 0;">
254 |                 <video controls loop class="embed-responsive-item">
255 |                     <source src="videos/what_is_guidance.mp4" type="video/mp4">
256 |                 </video>
257 |             </div>
258 |         </div></br>
259 | 
260 |         <div class="row">
261 |             <div class="col-md-12">
262 |                 <p>Here, we show how the application of our method solves the issues with temporal consistency observed above with prior work. As the viewer returns back to the starting position, the produced output is very similar to that of the first image, respecting previously produced textures. The output images and transitions over time also look more realistic.</p>
263 |             </div>
264 |         </div>
265 |         <div class="row">
266 |             <div class="embed-responsive embed-responsive-4by1" style="margin: 0;">
267 |                 <video controls class="embed-responsive-item">
268 |                     <source src="videos/fb_guidance.mp4" type="video/mp4">
269 |                 </video>
270 |             </div>
271 |         </div>
272 |         <br>
273 | 
274 |         <!-- The network architecture. -->
275 |         <div class="row">
276 |             <div class="col-md-12">
277 |                 <hr style="max-width: 768px;">
278 |                 <h5 class="text-center">The complete network architecture</h5>
279 | 
280 |                 <p>Here, we visualize our generator architecture and all its components. The main component in our generator is the novel Multi-SPADE block, which is composed of multiple <a class="text-center" href="https://nvlabs.github.io/SPADE/" target="_blank">SPADE</a> layers. Each SPADE layer takes in a spatial conditioning map such as the semantic segmentation, or the optical flow-warped previous output, or the guidance image, and applies appropriate transformations on the intermediate feature maps so that the finally generated output respects the required constraints and looks realistic in the spatial and temporal domain.
281 |                 </p>
282 | 
283 |                 <div class="text-center">
284 |                     <img src="images/overview.png" style="width: 100%;margin-bottom: 8px;">
285 |                     <p style="font-size: 0.9rem; margin-bottom: 0;"><strong>The overall architecture based on the Multi-SPADE module</strong></p>
286 |                     <p style="color: #808080; font-size: 0.85rem;">
287 |                         Each Multi-SPADE module takes input label features, warped previous frame features, and guidance images to modulate the features in each layer of our generator. The <span style="background-color: #ff88009c">labels decide the semantic content of the output frames</span>, the <span style="background-color: #00ff37a2">optical-flow warped previous frames ensure short-term consistency</span>, and the <span style="background-color: #ff00d469">guidance images ensure long-term consistency</span>.
288 |                     </p>
289 |                 </div>
290 |             </div>
291 |         </div>
292 |     </div>
293 | 
294 |     <!-- Outputs. -->
295 |     <hr style="max-width: 768px;">
296 |     <div class="container" style="max-width: 768px;">
297 |         <div class="row">
298 |             <div class="col-md-12">
299 |                 <h2>Sample Video Generation Results</h2>
300 |             </div>
301 |         </div>
302 | 
303 |         <div style="width: 75%; margin: 0 auto;">
304 |             <div class="embed-responsive embed-responsive-2by1" style="width: 100%; margin: 0 auto;">
305 |                 <iframe class="embed-responsive-item" style="padding: 0;" src="videos/stuttgart_00_02.mp4" allowfullscreen></iframe>
306 |             </div></br>
307 |             <div class="embed-responsive embed-responsive-2by1" style="width: 100%; margin: 0 auto;">
308 |                 <iframe class="embed-responsive-item" style="padding: 0;" src="videos/stuttgart_01_06.mp4" allowfullscreen></iframe>
309 |             </div></br>
310 |             <div class="embed-responsive embed-responsive-2by1" style="width: 100%; margin: 0 auto;">
311 |                 <iframe class="embed-responsive-item" style="padding: 0;" src="videos/mannequin.mp4" allowfullscreen></iframe>
312 |             </div></br>
313 |         </div>
314 | 
315 |         <hr style="max-width: 768px;">
316 |         <p class="text-center" style="margin-bottom: 0;">
317 |             <h5 class="text-center">Consistent Multiview Generation</h5>
318 |             We can also simultaneously generate videos that are consistent across multiple viewpoints or users.
319 |             In this setting, the first user generates the first frame and appropriate colors are assigned to the world's 3D point cloud. The 3D point cloud is then splatted/projected to the viewpoint of the second user to be used as guidance and our network generates the final semantically consistent output image after resolving holes and errors. Colors are then assigned to the newly visible 3D points and the point cloud is updated.
320 |             Below are examples of stereo pair generations, with the point cloud being alternately updated by the left and right views.
321 |         </p>
322 |         <div class="embed-responsive embed-responsive-4by1" style="width: 100%; margin: 0 auto;">
323 |             <iframe class="embed-responsive-item" style="padding: 0;" src="videos/stereo_1.mp4" allowfullscreen></iframe>
324 |         </div></br>
325 |         <div class="embed-responsive embed-responsive-4by1" style="width: 100%; margin: 0 auto;">
326 |             <iframe class="embed-responsive-item" style="padding: 0;" src="videos/stereo_2.mp4" allowfullscreen></iframe>
327 |         </div>
328 |     </div>
329 | 
330 |     <!-- Summary. -->
331 |     <hr style="max-width: 768px;">
332 |     <div class="container" style="max-width: 768px;" id="Summary">
333 |         <div class="row">
334 |             <div class="col-md-12">
335 |                 <h2>Summary</h2>
336 |                 <ul>
337 |                     <li>Video-to-video synthesis is a powerful tool for converting semantic inputs to photorealistic videos.</li>
338 |                     <li>Existing vid2vid methods are unable to maintain long-term consistency (such as during loop closure) due to lack of the 3D structure of the world.</li>
339 |                     <li>We provide information about the 3D structure of the world using guidance images - projecting point clouds of the world colored so far (which can be noisy and incomplete) to the of the current camera view.</li>
340 |                     <li>We introduce a new architecture based on the Multi-SPADE module, which uses semantic labels, optical-flow warping, and guidance images as conditioning inputs.</li>
341 |                     <li>Our models improve upon the realism, short-term, and long-term consistency of generated videos.</li>
342 |                 </ul>
343 |             </div>
344 |         </div>
345 |     </div>
346 | 
347 |     <!-- Citation. -->
348 |     <hr style="max-width: 768px;">
349 |     <div class="container" style="max-width: 768px;">
350 |         <div class="row">
351 |             <div class="col-md-12">
352 |                 <h2>Citation</h2>
353 |             </div>
354 |         </div>
355 |     <div class="code-box-copy"  style="font-size: 0.9rem !important; max-width: 768px; margin: 0 auto;">
356 |         <button class="code-box-copy__btn" data-clipboard-target="#citation" title="Copy" style="opacity: 1;"></button>
357 |         <pre><code class="language-bib" style="font-size: 0.9rem;" id="citation">@inproceedings{mallya2020world,
358 |     title={World-Consistent Video-to-Video Synthesis},
359 |     author={Arun Mallya and Ting-Chun Wang and Karan Sapra and Ming-Yu Liu},
360 |     booktitle={Proceedings of the European Conference on Computer Vision},
361 |     year={2020}
362 | }</code></pre>
363 |     </div>
364 |     </div>
365 | 
366 | 
367 |     <footer class="text-muted">
368 |       <div class="container">
369 |         <p class="float-right">
370 |           <a href="#">Back to top</a>
371 |         </p>
372 |       </div>
373 |     </footer>
374 | 
375 |     <!-- Bootstrap core JavaScript
376 |     ================================================== -->
377 |     <!-- Placed at the end of the document so the pages load faster -->
378 |     <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
379 | 	<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
380 |     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
381 |     <script type="text/javascript" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js?config=TeX-MML-AM_CHTML"></script>
382 |     <script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
383 |     <script src="https://cdn.jsdelivr.net/npm/prismjs/prism.min.js"></script>
384 |     <script src="https://cdn.jsdelivr.net/combine/gh/jablonczay/code-box-copy/clipboard/clipboard.min.js,gh/jablonczay/code-box-copy/code-box-copy/js/code-box-copy.min.js"></script>
385 |     <script src="https://saswatpadhi.github.io/prismjs-bibtex/prism-bibtex.min.js"></script>
386 |     <script>
387 |         (function($) {
388 |             $('.code-box-copy').codeBoxCopy();
389 |         })(jQuery);
390 |     </script>
391 |   </body>
392 | </html>
393 | 


--------------------------------------------------------------------------------
/videos/fb_guidance.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/fb_guidance.mp4


--------------------------------------------------------------------------------
/videos/fb_vid2vid.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/fb_vid2vid.mp4


--------------------------------------------------------------------------------
/videos/mannequin.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/mannequin.mp4


--------------------------------------------------------------------------------
/videos/sample_output.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/sample_output.gif


--------------------------------------------------------------------------------
/videos/side_by_side.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/side_by_side.gif


--------------------------------------------------------------------------------
/videos/side_by_side.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/side_by_side.mp4


--------------------------------------------------------------------------------
/videos/stereo_1.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/stereo_1.mp4


--------------------------------------------------------------------------------
/videos/stereo_2.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/stereo_2.mp4


--------------------------------------------------------------------------------
/videos/stuttgart_00_02.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/stuttgart_00_02.mp4


--------------------------------------------------------------------------------
/videos/stuttgart_01_04.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/stuttgart_01_04.mp4


--------------------------------------------------------------------------------
/videos/stuttgart_01_04_all.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/stuttgart_01_04_all.mp4


--------------------------------------------------------------------------------
/videos/stuttgart_01_06.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/stuttgart_01_06.mp4


--------------------------------------------------------------------------------
/videos/what_is_guidance.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/what_is_guidance.mp4


--------------------------------------------------------------------------------
/videos/what_is_vid2vid.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NVlabs/wc-vid2vid/b795e72b9693168fbacef36dd3491c9dd4c5e203/videos/what_is_vid2vid.mp4


--------------------------------------------------------------------------------