├── LICENSE ├── README.md ├── dataset ├── __init__.py ├── real_benchmark_dataset.py ├── syn_sintel_dataset.py ├── syn_test_dataset.py ├── syn_vimeo_dataset.py └── tog13_online_align_dataset.py ├── docs ├── README.md ├── _site │ ├── README.md │ ├── index.html │ └── static │ │ ├── css │ │ ├── bulma-carousel.min.css │ │ ├── bulma-slider.min.css │ │ ├── bulma.css.map.txt │ │ ├── bulma.min.css │ │ ├── fontawesome.all.min.css │ │ └── index.css │ │ ├── images │ │ └── icon.png │ │ ├── js │ │ ├── bulma-carousel.js │ │ ├── bulma-carousel.min.js │ │ ├── bulma-slider.js │ │ ├── bulma-slider.min.js │ │ ├── fontawesome.all.min.js │ │ └── index.js │ │ ├── pdfs │ │ └── sample.pdf │ │ └── videos │ │ └── teaser.mp4 ├── index.html └── static │ ├── css │ ├── bulma-carousel.min.css │ ├── bulma-slider.min.css │ ├── bulma.css.map.txt │ ├── bulma.min.css │ ├── fontawesome.all.min.css │ └── index.css │ ├── images │ ├── comp_attention.png │ ├── comp_flow.png │ ├── icon.png │ ├── teaser.png │ └── teaser_v2.png │ ├── js │ ├── bulma-carousel.js │ ├── bulma-carousel.min.js │ ├── bulma-slider.js │ ├── bulma-slider.min.js │ ├── fontawesome.all.min.js │ └── index.js │ ├── pdfs │ └── sample.pdf │ └── videos │ ├── teaser.mp4 │ └── teaser_v2.mp4 ├── models ├── flow_2E.py ├── flow_3E.py ├── fusion_2E.py ├── fusion_3E.py ├── loss.py ├── model_2E.py ├── model_3E.py └── network_utils.py ├── pretrained_models ├── 2E │ └── checkpoint.pth └── 3E │ └── checkpoint.pth ├── requirements.txt ├── test_2E.py ├── test_3E.py ├── test_tog13_2E.py ├── test_tog13_3E.py ├── train_2E.py ├── train_3E.py └── utils ├── __init__.py ├── flow_viz.py ├── frame_utils.py └── utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 OpenImagingLab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HDRFlow: Real-Time HDR Video Reconstruction with Large Motions 2 | ### [Project Page](https://openimaginglab.github.io/HDRFlow/) | [Paper](https://openaccess.thecvf.com/content/CVPR2024/papers/Xu_HDRFlow_Real-Time_HDR_Video_Reconstruction_with_Large_Motions_CVPR_2024_paper.pdf)
3 | 4 | Gangwei Xu, Yujin Wang, Jinwei Gu, Tianfan Xue, Xin Yang
5 | CVPR 2024

6 | 7 | ![teaser](docs/static/images/teaser.png) 8 | We propose a robust and efficient flow estimator tailored for real-time HDR video reconstruction, named HDRFlow. HDRFlow predicts HDR-oriented optical flow and exhibits robustness to large motions. We compare our HDR-oriented flow with RAFT's flow. RAFT's flow is sub-optimal for HDR fusion, and alignment may fail in occluded regions, leading to significant ghosting artifacts in the HDR output.
9 | Compared to previous SOTA methods, our HDRFlow enables real-time reconstruction of HDR video from video sequences captured with alternating exposures. 10 | 11 | ## Installation 12 | 13 | ### Set up the python environment 14 | 15 | ``` 16 | conda create -n hdrflow python=3.10 17 | conda activate hdrflow 18 | conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 cudatoolkit=11.3 -c pytorch 19 | pip install -r requirements.txt 20 | ``` 21 | 22 | ### Set up datasets 23 | 24 | #### 0. Set up training datasets 25 | We utilize Vimeo-90K and Sintel datasets as our training datasets. The Vimeo-90K dataset can be downloaded at [DeepHDRVideo-Dataset](https://github.com/guanyingc/DeepHDRVideo-Dataset). The Sintel dataset can be downloaded at [BaiduYun](https://pan.baidu.com/s/1GBRyIWZmlGbTGptYX1j-zQ?pwd=gnqj). The training datasets are organized as follows: 26 | ``` 27 | ├── HDRFlow/data 28 | ├── vimeo_septuplet 29 | ├── sequences 30 | ├── Sintel 31 | ├── training 32 | ├── clean 33 | ├── final 34 | ├── flow 35 | ├── reverse_flow 36 | ├── flow_2 37 | ├── reverse_flow_2 38 | ``` 39 | 40 | #### 1. Set up test datasets 41 | We evaluate our method on HDR_Synthetic_Test_Dataset (Cinematic Video dataset), DeepHDRVideo, and TOG13_Dynamic_Dataset (HDRVideo dataset). These datasets can be downloaded at [DeepHDRVideo-Dataset](https://github.com/guanyingc/DeepHDRVideo-Dataset). The HDR_Synthetic_Test_Dataset contains two synthetic videos (POKER FULLSHOT and CAROUSEL FIREWORKS), DeepHDRVideo consists of both real-world dynamic scenes and static scenes that have been augmented with random global motion. The TOG13_Dynamic_Dataset does not have ground truth, so we use it for qualitative evaluation. The test datasets are organized as follows: 42 | 43 | ``` 44 | ├── HDRFlow/data 45 | ├── HDR_Synthetic_Test_Dataset 46 | ├── dynamic_RGB_data_2exp_release 47 | ├── static_RGB_data_2exp_rand_motion_release 48 | ├── dynamic_RGB_data_3exp_release 49 | ├── static_RGB_data_3exp_rand_motion_release 50 | ├── TOG13_Dynamic_Dataset 51 | ``` 52 | 53 | ## Evaluation and Training 54 | ### Demo 55 | You can demo a pre-trained model on ThrowingTowel-2Exp-3Stop from TOG13_Dynamic_Dataset. The TOG13_Dynamic_Dataset can be downloaded at [BaiduYun](https://pan.baidu.com/s/1GBRyIWZmlGbTGptYX1j-zQ?pwd=gnqj). 56 | ``` 57 | python test_tog13_2E.py 58 | ``` 59 | 60 | ### Evaluation 61 | 2 Exposures 62 | ``` 63 | python test_2E.py --dataset DeepHDRVideo --dataset_dir data/dynamic_RGB_data_2exp_release 64 | python test_2E.py --dataset DeepHDRVideo --dataset_dir data/static_RGB_data_2exp_rand_motion_release 65 | python test_2E.py --dataset CinematicVideo --dataset_dir data/HDR_Synthetic_Test_Dataset 66 | python test_tog13_2E.py 67 | ``` 68 | 69 | 3 Exposures 70 | ``` 71 | python test_3E.py --dataset DeepHDRVideo --dataset_dir data/dynamic_RGB_data_3exp_release 72 | python test_3E.py --dataset DeepHDRVideo --dataset_dir data/static_RGB_data_3exp_rand_motion_release 73 | python test_3E.py --dataset CinematicVideo --dataset_dir data/HDR_Synthetic_Test_Dataset 74 | python test_tog13_3E.py 75 | ``` 76 | ### Training 77 | 2 Exposures 78 | ``` 79 | python train_2E.py 80 | ``` 81 | 82 | 3 Exposures 83 | ``` 84 | python train_3E.py 85 | ``` 86 | 87 | ## Citation 88 | 89 | If you find this code useful for your research, please use the following BibTeX entry. 90 | 91 | ``` 92 | @inproceedings{xu2024hdrflow, 93 | title={HDRFlow: Real-Time HDR Video Reconstruction with Large Motions}, 94 | author={Xu, Gangwei and Wang, Yujin and Gu, Jinwei and Xue, Tianfan and Yang, Xin}, 95 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 96 | pages={24851--24860}, 97 | year={2024} 98 | } 99 | 100 | ``` 101 | 102 | ## Acknowledgement 103 | This project is based on [DeepHDRVideo](https://github.com/guanyingc/DeepHDRVideo), we thank the original authors for their excellent work. 104 | 105 | -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | from torch.utils.data import DataLoader, ConcatDataset 4 | from .syn_vimeo_dataset import Syn_Vimeo_Dataset 5 | from .syn_sintel_dataset import Syn_Sintel_Dataset 6 | 7 | def fetch_dataloader_2E(args): 8 | train_vimeo = Syn_Vimeo_Dataset(root_dir=args.dataset_vimeo_dir, nframes=3, nexps=2, is_training=True) 9 | train_sintel_clean = Syn_Sintel_Dataset(root_dir=args.dataset_sintel_dir, dtype='clean', nframes=3, nexps=2, is_training=True) 10 | train_sintel_final = Syn_Sintel_Dataset(root_dir=args.dataset_sintel_dir, dtype='final', nframes=3, nexps=2, is_training=True) 11 | train_dataset = ConcatDataset([train_vimeo, train_sintel_clean, train_sintel_final]) 12 | train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) 13 | val_vimeo = Syn_Vimeo_Dataset(root_dir=args.dataset_vimeo_dir, nframes=3, nexps=2, is_training=False) 14 | val_loader = DataLoader(val_vimeo, batch_size=args.val_batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True) 15 | return train_loader, val_loader 16 | 17 | def fetch_dataloader_3E(args): 18 | train_vimeo = Syn_Vimeo_Dataset(root_dir=args.dataset_vimeo_dir, nframes=5, nexps=3, is_training=True) 19 | train_sintel_clean = Syn_Sintel_Dataset(root_dir=args.dataset_sintel_dir, dtype='clean', nframes=5, nexps=3, is_training=True) 20 | train_sintel_final = Syn_Sintel_Dataset(root_dir=args.dataset_sintel_dir, dtype='final', nframes=5, nexps=3, is_training=True) 21 | train_dataset = ConcatDataset([train_vimeo, train_sintel_clean, train_sintel_final]) 22 | train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) 23 | val_vimeo = Syn_Vimeo_Dataset(root_dir=args.dataset_vimeo_dir, nframes=5, nexps=3, is_training=False) 24 | val_loader = DataLoader(val_vimeo, batch_size=args.val_batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True) 25 | return train_loader, val_loader 26 | 27 | 28 | -------------------------------------------------------------------------------- /dataset/real_benchmark_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dataloader for processing vimeo videos into training data 3 | """ 4 | import os 5 | import numpy as np 6 | from imageio import imread 7 | import sys 8 | sys.path.append('..') 9 | import torch 10 | from torch.utils.data import Dataset 11 | from utils.utils import * 12 | np.random.seed(0) 13 | 14 | class Real_Benchmark_Dataset(Dataset): 15 | def __init__(self, root_dir, nframes, nexps): 16 | 17 | self.root_dir = root_dir 18 | self.nframes = nframes 19 | self.nexps = nexps 20 | self.scene_list = read_list(os.path.join(self.root_dir, 'scene_all.txt')) 21 | 22 | self.expos_list = [] 23 | self.img_list = [] 24 | self.hdrs_list = [] 25 | 26 | for i in range(len(self.scene_list)): 27 | img_dir = os.path.join(self.root_dir, self.scene_list[i]) 28 | img_list, hdr_list = self._load_img_hdr_list(img_dir) 29 | e_list = self._load_exposure_list(os.path.join(img_dir, 'Exposures.txt'), img_num=len(img_list)) 30 | 31 | for i in range(0, len(img_list)-(self.nframes+2)+1): 32 | sub_img_list = img_list[i:i+self.nframes+2] 33 | sub_hdr_list = hdr_list[i:i+self.nframes+2] 34 | sub_e_list = e_list[i:i+self.nframes+2] 35 | 36 | self.expos_list.append(sub_e_list) 37 | self.img_list.append(sub_img_list) 38 | self.hdrs_list.append(sub_hdr_list) 39 | 40 | print('[%s] totaling %d ldrs' % (self.__class__.__name__, len(self.img_list))) 41 | 42 | def _load_img_hdr_list(self, img_dir): 43 | if os.path.exists(os.path.join(img_dir, 'img_hdr_list.txt')): 44 | img_hdr_list = np.genfromtxt(os.path.join(img_dir, 'img_hdr_list.txt'), dtype='str') 45 | img_list = img_hdr_list[:, 0] 46 | hdr_list = img_hdr_list[:, 1] 47 | else: 48 | img_list = np.genfromtxt(os.path.join(img_dir, 'img_list.txt'), dtype='str') 49 | hdr_list = ['None'] * len(img_list) 50 | img_list =[os.path.join(img_dir, img_path) for img_path in img_list] 51 | hdr_list =[os.path.join(img_dir, hdr_path) for hdr_path in hdr_list] 52 | return img_list, hdr_list 53 | 54 | def _load_exposure_list(self, expos_path, img_num): 55 | expos = np.genfromtxt(expos_path, dtype='float') 56 | expos = np.power(2, expos - expos.min()).astype(np.float32) 57 | expo_list = np.tile(expos, int(img_num / len(expos) + 1))[:img_num] 58 | return expo_list 59 | 60 | def __getitem__(self, index): 61 | 62 | ldrs = [] 63 | expos = [] 64 | 65 | if self.nexps == 2: 66 | img_paths, hdr_path = self.img_list[index], self.hdrs_list[index][2] 67 | exposures_all = np.array(self.expos_list[index]).astype(np.float32) 68 | for i in range(1, 4): 69 | if img_paths[i][-4:] == '.tif': 70 | img = read_16bit_tif(img_paths[i]) 71 | else: 72 | img = imread(img_paths[i]) / 255.0 73 | ldrs.append(img) 74 | expos.append(exposures_all[i]) 75 | 76 | elif self.nexps == 3: 77 | img_paths, hdr_path = self.img_list[index], self.hdrs_list[index][3] 78 | exposures_all = np.array(self.expos_list[index]).astype(np.float32) 79 | for i in range(1, 6): 80 | if img_paths[i][-4:] == '.tif': 81 | img = read_16bit_tif(img_paths[i]) 82 | else: 83 | img = imread(img_paths[i]) / 255.0 84 | ldrs.append(img) 85 | expos.append(exposures_all[i]) 86 | 87 | else: 88 | raise Exception("Unknow exposures") 89 | 90 | if os.path.exists(hdr_path): 91 | hdr = read_hdr(hdr_path) 92 | if hdr.max() > 1: 93 | hdr = hdr / hdr.max() 94 | hdr = hdr.astype(np.float32).transpose(2, 0, 1) 95 | hdr_tensor = torch.from_numpy(hdr) 96 | ldrs_tensor = [] 97 | expos_tensor = [] 98 | for i in range(len(ldrs)): 99 | ldr = ldrs[i].astype(np.float32).transpose(2, 0, 1) 100 | ldr_tensor = torch.from_numpy(ldr) 101 | ldrs_tensor.append(ldr_tensor) 102 | expos_tensor.append(torch.tensor(expos[i])) 103 | 104 | sample = { 105 | 'hdr_path': hdr_path.split('/')[-2]+'_'+hdr_path.split('/')[-1], 106 | 'hdr': hdr_tensor, 107 | 'ldrs': ldrs_tensor, 108 | 'expos': expos_tensor 109 | } 110 | return sample 111 | 112 | def __len__(self): 113 | return len(self.img_list) -------------------------------------------------------------------------------- /dataset/syn_sintel_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | # from imageio import imread 4 | import sys 5 | sys.path.append('..') 6 | import torch 7 | from torch.utils.data import Dataset 8 | from utils.utils import * 9 | from utils import frame_utils 10 | from imageio import imread 11 | 12 | np.random.seed(0) 13 | 14 | class Syn_Sintel_Dataset(Dataset): 15 | 16 | def __init__(self, root_dir, dtype, nframes, nexps, is_training=True): 17 | self.root_dir = os.path.join(root_dir, dtype) 18 | self.dtype = dtype 19 | self.nframes = nframes 20 | self.nexps = nexps 21 | 22 | if is_training: 23 | list_name = 'trainlist.txt' 24 | self.repeat = 20 25 | else: 26 | list_name = 'testlist.txt' 27 | self.repeat = 1 28 | 29 | self.scene_list = read_list(os.path.join(self.root_dir, list_name)) 30 | image_list = [] 31 | for scene in self.scene_list: 32 | files = os.listdir(os.path.join(self.root_dir, scene)) 33 | files.sort() 34 | for i in range(len(files)- (self.nframes-1)): 35 | image = files[i:i+self.nframes] 36 | if all(idx.endswith('.png') for idx in image): 37 | image_list.append([os.path.join(self.root_dir, scene, ip) for ip in image]) 38 | 39 | self.image_list = image_list 40 | 41 | def __getitem__(self, index): 42 | 43 | img_paths = self.image_list[index//self.repeat] 44 | if self.nexps == 2: 45 | exposures = self._get_2exposures(index) 46 | elif self.nexps == 3: 47 | exposures = self._get_3exposures(index) 48 | else: 49 | raise Exception("Unknow exposures") 50 | 51 | """ sample parameters for the camera curves""" 52 | n, sigma = self.sample_camera_curve() # print(n, sigma) 53 | 54 | hdrs = [] 55 | for img_path in img_paths: 56 | img = (imread(img_path).astype(np.float32) / 255.0).clip(0, 1) 57 | # img = read_16bit_tif(img_path) 58 | """ convert the LDR images to linear HDR image""" 59 | linear_img = self.apply_inv_sigmoid_curve(img, n, sigma) 60 | linear_img = self.discretize_to_uint16(linear_img) 61 | hdrs.append(linear_img) 62 | 63 | # p_hdr, c_hdr, n_hdr = hdrs[0], hdrs[1], hdrs[2] 64 | ### flow 65 | if self.nexps == 2: 66 | prev_flow_path = img_paths[1].replace(self.dtype, 'reverse_flow').replace('png', 'flo') 67 | nxt_flow_path = img_paths[1].replace(self.dtype, 'flow').replace('png', 'flo') 68 | prev_flow = frame_utils.read_gen(prev_flow_path) 69 | nxt_flow = frame_utils.read_gen(nxt_flow_path) 70 | flow_gts = [prev_flow, nxt_flow] 71 | 72 | elif self.nexps == 3: 73 | prev2_flow_path = img_paths[2].replace(self.dtype, 'reverse_flow_2').replace('png', 'flo') 74 | nxt1_flow_path = img_paths[2].replace(self.dtype, 'flow').replace('png', 'flo') 75 | prev1_flow_path = img_paths[2].replace(self.dtype, 'reverse_flow').replace('png', 'flo') 76 | nxt2_flow_path = img_paths[2].replace(self.dtype, 'flow_2').replace('png', 'flo') 77 | 78 | prev2_flow = frame_utils.read_gen(prev2_flow_path) 79 | nxt1_flow = frame_utils.read_gen(nxt1_flow_path) 80 | prev1_flow = frame_utils.read_gen(prev1_flow_path) 81 | nxt2_flow = frame_utils.read_gen(nxt2_flow_path) 82 | 83 | flow_gts = [prev2_flow, nxt1_flow, prev1_flow, nxt2_flow] 84 | 85 | else: 86 | raise Exception("Unknow exposures") 87 | 88 | # hdrs = prob_center_crop(hdrs) 89 | h, w, c = hdrs[0].shape 90 | # crop_h, crop_w = self.args.crop_h, self.args.crop_w 91 | 92 | crop_h, crop_w = 256, 256 93 | 94 | _hdrs = [] 95 | _flow_gts = [] 96 | 97 | ### random flip 98 | if np.random.rand() < 0.3: # h-flip 99 | for hdr in hdrs: 100 | _hdrs.append(hdr[:, ::-1]) 101 | for flow in flow_gts: 102 | _flow_gts.append(flow[:, ::-1] * [-1.0, 1.0]) 103 | 104 | hdrs = _hdrs 105 | flow_gts = _flow_gts 106 | 107 | 108 | _hdrs = [] 109 | _flow_gts = [] 110 | 111 | if np.random.rand() < 0.1: # v-flip 112 | 113 | for hdr in hdrs: 114 | _hdrs.append(hdr[::-1, :]) 115 | for flow in flow_gts: 116 | _flow_gts.append(flow[::-1, :] * [1.0, -1.0]) 117 | 118 | hdrs = _hdrs 119 | flow_gts = _flow_gts 120 | 121 | ### random crop 122 | y0 = np.random.randint(0, hdrs[0].shape[0] - crop_h) 123 | x0 = np.random.randint(0, hdrs[0].shape[1] - crop_w) 124 | 125 | _hdrs = [] 126 | _flow_gts = [] 127 | 128 | for hdr in hdrs: 129 | _hdrs.append(hdr[y0:y0+crop_h, x0:x0+crop_w]) 130 | for flow in flow_gts: 131 | _flow_gts.append(flow[y0:y0+crop_h, x0:x0+crop_w]) 132 | 133 | hdrs = _hdrs 134 | flow_gts = _flow_gts 135 | 136 | color_permute = np.random.permutation(3) 137 | for i in range(len(hdrs)): 138 | hdrs[i] = hdrs[i][:,:,color_permute] 139 | 140 | hdrs, ldrs = self.re_expose_ldrs(hdrs, exposures) 141 | ldrs_tensor = [] 142 | hdrs_tensor = [] 143 | expos_tensor = [] 144 | for i in range(len(ldrs)): 145 | ldr = ldrs[i].astype(np.float32).transpose(2, 0, 1) 146 | ldr_tensor = torch.from_numpy(ldr) 147 | ldrs_tensor.append(ldr_tensor) 148 | 149 | hdr = hdrs[i].astype(np.float32).transpose(2, 0, 1) 150 | hdr_tensor = torch.from_numpy(hdr) 151 | hdrs_tensor.append(hdr_tensor) 152 | 153 | expos_tensor.append(torch.tensor(exposures[i])) 154 | 155 | flow_gts_tensor = [] 156 | for i in range(len(flow_gts)): 157 | flow = flow_gts[i].astype(np.float32).transpose(2, 0, 1) 158 | flow_tensor = torch.from_numpy(flow) 159 | flow_gts_tensor.append(flow_tensor) 160 | 161 | flow_mask = torch.tensor(1.) 162 | 163 | sample = { 164 | 'hdrs': hdrs_tensor, 165 | 'ldrs': ldrs_tensor, 166 | 'expos': expos_tensor, 167 | 'flow_gts': flow_gts_tensor, 168 | 'flow_mask': flow_mask 169 | } 170 | return sample 171 | 172 | def sample_camera_curve(self): 173 | n = np.clip(np.random.normal(0.65, 0.1), 0.4, 0.9) 174 | sigma = np.clip(np.random.normal(0.6, 0.1), 0.4, 0.8) 175 | return n, sigma 176 | 177 | def apply_sigmoid_curve(self, x, n, sigma): 178 | y = (1 + sigma) * np.power(x, n) / (np.power(x, n) + sigma) 179 | return y 180 | 181 | def apply_inv_sigmoid_curve(self, y, n, sigma): 182 | x = np.power((sigma * y) / (1 + sigma - y), 1/n) 183 | return x 184 | 185 | def apply_inv_s_curve(self, y): 186 | x = 0.5 - np.sin(np.arcsin(1 - 2*y)/3.0) 187 | return x 188 | 189 | def discretize_to_uint16(self, img): 190 | max_int = 2**16-1 191 | img_uint16 = np.uint16(img * max_int).astype(np.float32) / max_int 192 | return img_uint16 193 | 194 | def _get_2exposures(self, index): 195 | cur_high = True if np.random.uniform() > 0.5 else False 196 | exposures = np.ones(self.nframes, dtype=np.float32) 197 | high_expo = np.random.choice([4., 8.]) 198 | 199 | if cur_high: 200 | for i in range(0, self.nframes, 2): 201 | exposures[i] = high_expo 202 | else: 203 | for i in range(1, self.nframes, 2): 204 | exposures[i] = high_expo 205 | return exposures 206 | 207 | def _get_3exposures(self, index): 208 | if index % self.nexps == 0: 209 | exp1 = 1 210 | elif index % self.nexps == 1: 211 | exp1 = 4 212 | else: 213 | exp1 = 16 214 | expos = [exp1] 215 | for i in range(1, self.nframes): 216 | if expos[-1] == 1: 217 | expos.append(4) 218 | elif expos[-1] == 4: 219 | expos.append(16) 220 | elif expos[-1] == 16: 221 | expos.append(1) 222 | else: 223 | raise Exception('Unknown expos %d' % expos[-1]) 224 | exposures = np.array(expos).astype(np.float32) 225 | return exposures 226 | 227 | def re_expose_ldrs(self, hdrs, exposures): 228 | mid = len(hdrs) // 2 229 | 230 | new_hdrs = [] 231 | if self.nexps == 3: 232 | if exposures[mid] == 1: 233 | factor = np.random.uniform(0.1, 0.8) 234 | anchor = hdrs[mid].max() 235 | new_anchor = anchor * factor 236 | else: # exposures[mid] == 4 or 8 237 | percent = np.random.uniform(98, 100) 238 | anchor = np.percentile(hdrs[mid], percent) 239 | new_anchor = np.random.uniform(anchor, 1) 240 | else: 241 | if exposures[mid] == 1: # low exposure reference 242 | factor = np.random.uniform(0.1, 0.8) 243 | anchor = hdrs[mid].max() 244 | new_anchor = anchor * factor 245 | else: # high exposure reference 246 | percent = np.random.uniform(98, 100) 247 | anchor = np.percentile(hdrs[mid], percent) 248 | new_anchor = np.random.uniform(anchor, 1) 249 | 250 | for idx, hdr in enumerate(hdrs): 251 | new_hdr = (hdr / (anchor + 1e-8) * new_anchor).clip(0, 1) 252 | new_hdrs.append(new_hdr) 253 | 254 | ldrs = [] 255 | for i in range(len(new_hdrs)): 256 | ldr = hdr_to_ldr(new_hdrs[i], exposures[i]) 257 | ldrs.append(ldr) 258 | return new_hdrs, ldrs 259 | 260 | def __len__(self): 261 | return len(self.image_list) * self.repeat -------------------------------------------------------------------------------- /dataset/syn_test_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from imageio import imread 4 | import sys 5 | sys.path.append('..') 6 | import torch 7 | from torch.utils.data import Dataset 8 | from utils.utils import * 9 | np.random.seed(0) 10 | 11 | class Syn_Test_Dataset(Dataset): 12 | def __init__(self, root_dir, nframes, nexps): 13 | 14 | self.root_dir = root_dir 15 | self.nframes = nframes 16 | self.nexps = nexps 17 | self.scene_list = read_list(os.path.join(self.root_dir, 'scenes_2expo.txt')) 18 | 19 | self.expos_list = [] 20 | self.img_list = [] 21 | self.hdrs_list = [] 22 | 23 | for i in range(len(self.scene_list)): 24 | img_dir = os.path.join(self.root_dir, 'Images', self.scene_list[i]) 25 | img_list, hdr_list = self._load_img_hdr_list(img_dir) 26 | e_list = self._load_exposure_list(os.path.join(img_dir, 'Exposures.txt'), img_num=len(img_list)) 27 | img_list, hdr_list, e_list = self._lists_to_paired_lists([img_list, hdr_list, e_list]) 28 | 29 | self.expos_list += e_list 30 | self.img_list += img_list 31 | self.hdrs_list += hdr_list 32 | 33 | print('[%s] totaling %d ldrs' % (self.__class__.__name__, len(self.img_list))) 34 | 35 | def _load_img_hdr_list(self, img_dir): 36 | scene_list = np.genfromtxt(os.path.join(img_dir, 'img_list.txt'), dtype='str') 37 | img_list = ['%s.tif' % name for name in scene_list] 38 | hdr_list = ['%s.hdr' % name for name in scene_list] 39 | img_list =[os.path.join(img_dir, img_path) for img_path in img_list] 40 | hdr_list =[os.path.join(img_dir, hdr_path) for hdr_path in hdr_list] 41 | return img_list, hdr_list 42 | 43 | def _load_exposure_list(self, expos_path, img_num): 44 | expos = np.genfromtxt(expos_path, dtype='float') 45 | expos = np.power(2, expos - expos.min()).astype(np.float32) 46 | expo_list = np.tile(expos, int(img_num / len(expos) + 1))[:img_num] 47 | return expo_list 48 | 49 | def _lists_to_paired_lists(self, lists): 50 | paired_lists = [] 51 | 52 | for l in lists: 53 | if (self.nexps == 2 and self.nframes == 3) or (self.nexps == 3 and self.nframes == 5): 54 | l = l[1:-1] 55 | paired_list = [] 56 | paired_list.append(l[: len(l) - self.nframes + 1]) 57 | for j in range(1, self.nframes): 58 | start_idx, end_idx = j, len(l) - self.nframes + 1 + j 59 | paired_list.append(l[start_idx: end_idx]) 60 | paired_lists.append(np.stack(paired_list, 1).tolist()) # Nxframes 61 | return paired_lists 62 | 63 | def __getitem__(self, index): 64 | 65 | ldrs = [] 66 | expos = [] 67 | if self.nexps == 2: 68 | img_paths, hdr_path = self.img_list[index], self.hdrs_list[index][1] 69 | exposures_all = np.array(self.expos_list[index]).astype(np.float32) 70 | for i in range(0, 3): 71 | img = read_16bit_tif(img_paths[i]) 72 | ldrs.append(img) 73 | expos.append(exposures_all[i]) 74 | 75 | elif self.nexps == 3: 76 | img_paths, hdr_path = self.img_list[index], self.hdrs_list[index][2] 77 | exposures_all = np.array(self.expos_list[index]).astype(np.float32) 78 | for i in range(0, 5): 79 | img = read_16bit_tif(img_paths[i]) 80 | ldrs.append(img) 81 | expos.append(exposures_all[i]) 82 | 83 | else: 84 | raise Exception("Unknow exposures") 85 | 86 | if os.path.exists(hdr_path): 87 | hdr = read_hdr(hdr_path) 88 | hdr = hdr.astype(np.float32).transpose(2, 0, 1) 89 | hdr_tensor = torch.from_numpy(hdr) 90 | ldrs_tensor = [] 91 | expos_tensor = [] 92 | for i in range(len(ldrs)): 93 | ldr = ldrs[i].astype(np.float32).transpose(2, 0, 1) 94 | ldr_tensor = torch.from_numpy(ldr) 95 | ldrs_tensor.append(ldr_tensor) 96 | expos_tensor.append(torch.tensor(expos[i])) 97 | 98 | sample = { 99 | 'hdr_path': hdr_path.split('/')[-1], 100 | 'hdr': hdr_tensor, 101 | 'ldrs': ldrs_tensor, 102 | 'expos': expos_tensor 103 | } 104 | return sample 105 | 106 | def __len__(self): 107 | return len(self.img_list) 108 | 109 | -------------------------------------------------------------------------------- /dataset/syn_vimeo_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dataloader for processing vimeo videos into training data 3 | """ 4 | import os 5 | import numpy as np 6 | from imageio import imread 7 | 8 | import sys 9 | sys.path.append('..') 10 | import torch 11 | from torch.utils.data import Dataset 12 | from utils.utils import * 13 | 14 | np.random.seed(0) 15 | 16 | class Syn_Vimeo_Dataset(Dataset): 17 | 18 | def __init__(self, root_dir, nframes, nexps, is_training=True): 19 | 20 | self.root_dir = root_dir 21 | self.nframes = nframes 22 | self.nexps = nexps 23 | 24 | if is_training: 25 | list_name = 'sep_trainlist.txt' 26 | self.repeat = 1 27 | else: 28 | list_name = 'sep_testlist.txt' 29 | self.repeat = 1 30 | 31 | self.patch_list = read_list(os.path.join(self.root_dir, list_name)) 32 | 33 | if not is_training: 34 | self.patch_list = self.patch_list[:100] # only use 100 val patches 35 | 36 | def __getitem__(self, index): 37 | 38 | img_dir = os.path.join(self.root_dir, 'sequences', self.patch_list[index // self.repeat]) 39 | img_idxs = sorted(np.random.permutation(7)[:self.nframes] + 1) 40 | if np.random.random() > 0.5: # inverse time order 41 | img_idxs = img_idxs[::-1] 42 | img_paths = [os.path.join(img_dir, 'im%d.png' % idx) for idx in img_idxs] 43 | 44 | if self.nexps == 2: 45 | exposures = self._get_2exposures(index) 46 | elif self.nexps == 3: 47 | exposures = self._get_3exposures(index) 48 | else: 49 | raise Exception("Unknow exposures") 50 | 51 | """ sample parameters for the camera curves""" 52 | n, sigma = self.sample_camera_curve() # print(n, sigma) 53 | 54 | hdrs = [] 55 | for img_path in img_paths: 56 | img = (imread(img_path).astype(np.float32) / 255.0).clip(0, 1) 57 | 58 | """ convert the LDR images to linear HDR image""" 59 | linear_img = self.apply_inv_sigmoid_curve(img, n, sigma) 60 | linear_img = self.discretize_to_uint16(linear_img) 61 | hdrs.append(linear_img) 62 | 63 | h, w, c = hdrs[0].shape 64 | crop_h, crop_w = 256, 256 65 | 66 | hdrs = random_flip_lrud(hdrs) 67 | hdrs = random_crop(hdrs, [crop_h, crop_w]) 68 | color_permute = np.random.permutation(3) 69 | for i in range(len(hdrs)): 70 | hdrs[i] = hdrs[i][:,:,color_permute] 71 | 72 | hdrs, ldrs = self.re_expose_ldrs(hdrs, exposures) 73 | ldrs_tensor = [] 74 | hdrs_tensor = [] 75 | expos_tensor = [] 76 | for i in range(len(ldrs)): 77 | ldr = ldrs[i].astype(np.float32).transpose(2, 0, 1) 78 | ldr_tensor = torch.from_numpy(ldr) 79 | ldrs_tensor.append(ldr_tensor) 80 | 81 | hdr = hdrs[i].astype(np.float32).transpose(2, 0, 1) 82 | hdr_tensor = torch.from_numpy(hdr) 83 | hdrs_tensor.append(hdr_tensor) 84 | 85 | expos_tensor.append(torch.tensor(exposures[i])) 86 | 87 | flow_gts = [] 88 | if self.nexps == 2: 89 | 90 | prev_flow = torch.zeros(2, crop_h, crop_w) 91 | nxt_flow = torch.zeros(2, crop_h, crop_w) 92 | flow_gts.append(prev_flow) 93 | flow_gts.append(nxt_flow) 94 | 95 | elif self.nexps == 3: 96 | prev2_flow = torch.zeros(2, crop_h, crop_w) 97 | nxt1_flow = torch.zeros(2, crop_h, crop_w) 98 | prev1_flow = torch.zeros(2, crop_h, crop_w) 99 | nxt2_flow = torch.zeros(2, crop_h, crop_w) 100 | 101 | flow_gts.append(prev2_flow) 102 | flow_gts.append(nxt1_flow) 103 | flow_gts.append(prev1_flow) 104 | flow_gts.append(nxt2_flow) 105 | 106 | else: 107 | raise Exception("Unknow exposures") 108 | 109 | flow_mask = torch.tensor(0.) 110 | 111 | sample = { 112 | 'hdrs': hdrs_tensor, 113 | 'ldrs': ldrs_tensor, 114 | 'expos': expos_tensor, 115 | 'flow_gts': flow_gts, 116 | 'flow_mask': flow_mask 117 | } 118 | return sample 119 | 120 | def sample_camera_curve(self): 121 | n = np.clip(np.random.normal(0.65, 0.1), 0.4, 0.9) 122 | sigma = np.clip(np.random.normal(0.6, 0.1), 0.4, 0.8) 123 | return n, sigma 124 | 125 | def apply_sigmoid_curve(self, x, n, sigma): 126 | y = (1 + sigma) * np.power(x, n) / (np.power(x, n) + sigma) 127 | return y 128 | 129 | def apply_inv_sigmoid_curve(self, y, n, sigma): 130 | x = np.power((sigma * y) / (1 + sigma - y), 1/n) 131 | return x 132 | 133 | def apply_inv_s_curve(self, y): 134 | x = 0.5 - np.sin(np.arcsin(1 - 2*y)/3.0) 135 | return x 136 | 137 | def discretize_to_uint16(self, img): 138 | max_int = 2**16-1 139 | img_uint16 = np.uint16(img * max_int).astype(np.float32) / max_int 140 | return img_uint16 141 | 142 | def _get_2exposures(self, index): 143 | cur_high = True if np.random.uniform() > 0.5 else False 144 | exposures = np.ones(self.nframes, dtype=np.float32) 145 | high_expo = np.random.choice([4., 8.]) 146 | 147 | if cur_high: 148 | for i in range(0, self.nframes, 2): 149 | exposures[i] = high_expo 150 | else: 151 | for i in range(1, self.nframes, 2): 152 | exposures[i] = high_expo 153 | return exposures 154 | 155 | def _get_3exposures(self, index): 156 | if index % self.nexps == 0: 157 | exp1 = 1 158 | elif index % self.nexps == 1: 159 | exp1 = 4 160 | else: 161 | exp1 = 16 162 | expos = [exp1] 163 | for i in range(1, self.nframes): 164 | if expos[-1] == 1: 165 | expos.append(4) 166 | elif expos[-1] == 4: 167 | expos.append(16) 168 | elif expos[-1] == 16: 169 | expos.append(1) 170 | else: 171 | raise Exception('Unknown expos %d' % expos[-1]) 172 | exposures = np.array(expos).astype(np.float32) 173 | return exposures 174 | 175 | def re_expose_ldrs(self, hdrs, exposures): 176 | mid = len(hdrs) // 2 177 | new_hdrs = [] 178 | if self.nexps == 3: 179 | if exposures[mid] == 1: 180 | factor = np.random.uniform(0.1, 0.8) 181 | anchor = hdrs[mid].max() 182 | new_anchor = anchor * factor 183 | else: # exposures[mid] == 4 or 8 184 | percent = np.random.uniform(98, 100) 185 | anchor = np.percentile(hdrs[mid], percent) 186 | new_anchor = np.random.uniform(anchor, 1) 187 | else: 188 | if exposures[mid] == 1: # low exposure reference 189 | factor = np.random.uniform(0.1, 0.8) 190 | anchor = hdrs[mid].max() 191 | new_anchor = anchor * factor 192 | else: # high exposure reference 193 | percent = np.random.uniform(98, 100) 194 | anchor = np.percentile(hdrs[mid], percent) 195 | new_anchor = np.random.uniform(anchor, 1) 196 | for idx, hdr in enumerate(hdrs): 197 | new_hdr = (hdr / (anchor + 1e-8) * new_anchor).clip(0, 1) 198 | new_hdrs.append(new_hdr) 199 | 200 | ldrs = [] 201 | for i in range(len(new_hdrs)): 202 | ldr = hdr_to_ldr(new_hdrs[i], exposures[i]) 203 | ldrs.append(ldr) 204 | return new_hdrs, ldrs 205 | 206 | def __len__(self): 207 | return len(self.patch_list) * self.repeat -------------------------------------------------------------------------------- /dataset/tog13_online_align_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from imageio import imread 4 | import sys 5 | sys.path.append('..') 6 | import torch 7 | from torch.utils.data import Dataset 8 | import scipy.io as sio 9 | from utils.utils import * 10 | np.random.seed(0) 11 | 12 | class TOG13_online_align_Dataset(Dataset): 13 | def __init__(self, root_dir, nframes, nexps, align=True): 14 | 15 | self.root_dir = root_dir 16 | self.nframes = nframes 17 | self.nexps = nexps 18 | crf_path = root_dir.split('/') 19 | crf_path = crf_path[:-1] 20 | crf_path = '/'.join(crf_path) 21 | self.crf = sio.loadmat(os.path.join(crf_path, 'BaslerCRF.mat'))['BaslerCRF'] 22 | self.align = align 23 | img_list, hdr_list = self._load_img_hdr_list(self.root_dir) 24 | e_list = self._load_exposure_list(os.path.join(self.root_dir, 'Exposures.txt'), img_num=len(img_list)) 25 | self.imgs_list, self.hdrs_list, self.expos_list = self._lists_to_paired_lists([img_list, hdr_list, e_list]) 26 | print('[%s] totaling %d ldrs' % (self.__class__.__name__, len(self.imgs_list))) 27 | 28 | def _load_img_hdr_list(self, img_dir): 29 | if os.path.exists(os.path.join(img_dir, 'img_hdr_list.txt')): 30 | img_hdr_list = np.genfromtxt(os.path.join(img_dir, 'img_hdr_list.txt'), dtype='str') 31 | img_list = img_hdr_list[:, 0] 32 | hdr_list = img_hdr_list[:, 1] 33 | else: 34 | img_list = np.genfromtxt(os.path.join(img_dir, 'img_list.txt'), dtype='str') 35 | hdr_list = ['None'] * len(img_list) 36 | img_list =[os.path.join(img_dir, img_path) for img_path in img_list] 37 | hdr_list =[os.path.join(img_dir, hdr_path) for hdr_path in hdr_list] 38 | return img_list, hdr_list 39 | 40 | def _load_exposure_list(self, expos_path, img_num): 41 | expos = np.genfromtxt(expos_path, dtype='float') 42 | expos = np.power(2, expos - expos.min()).astype(np.float32) 43 | expo_list = np.tile(expos, int(img_num / len(expos) + 1))[:img_num] 44 | return expo_list 45 | 46 | def _lists_to_paired_lists(self, lists): 47 | paired_lists = [] 48 | 49 | for l in lists: 50 | if (self.nexps == 2 and self.nframes == 3) or (self.nexps == 3 and self.nframes == 5): 51 | l = l[1:-1] 52 | paired_list = [] 53 | paired_list.append(l[: len(l) - self.nframes + 1]) 54 | for j in range(1, self.nframes): 55 | start_idx, end_idx = j, len(l) - self.nframes + 1 + j 56 | paired_list.append(l[start_idx: end_idx]) 57 | paired_lists.append(np.stack(paired_list, 1).tolist()) # Nxframes 58 | return paired_lists 59 | 60 | def load_affine_matrices(self, img_path, h, w): 61 | dir_name, img_name = os.path.dirname(img_path), os.path.basename(img_path) 62 | cv2_match = np.genfromtxt(os.path.join(dir_name, 'Affine_Trans_Matrices', img_name[:-4]+'_match.txt'), dtype=np.float32) 63 | # For two exposure: cv2_match [2, 6], row 1->2: cur->prev, cur->next 64 | # For three exposure: cv2_match [4, 6], row1->4: cur->prev2, cur->prev, cur->next, cur->next2 65 | 66 | n_matches =cv2_match.shape[0] 67 | if self.nexps == 2: 68 | assert (n_matches == 2) 69 | elif self.nexps == 3: 70 | assert (n_matches == 4) 71 | 72 | cv2_match = cv2_match.reshape(n_matches, 2, 3) 73 | theta = np.zeros((n_matches, 2, 3)).astype(np.float32) # Theta for affine transformation in pytorch 74 | for mi in range(n_matches): 75 | theta[mi] = cvt_MToTheta(cv2_match[mi], w, h) 76 | return theta 77 | 78 | def __getitem__(self, index): 79 | ldrs = [] 80 | expos = [] 81 | matches = [] 82 | img_paths = self.imgs_list[index] 83 | exposures_all = np.array(self.expos_list[index]).astype(np.float32) 84 | for i in range(0, self.nframes): 85 | img = apply_gamma(read_16bit_tif(img_paths[i], crf=self.crf), gamma=2.2) 86 | ldrs.append(img) 87 | expos.append(exposures_all[i]) 88 | if self.align: 89 | match = self.load_affine_matrices(img_paths[i], img.shape[0], img.shape[1]) 90 | matches.append(match) 91 | ldrs_tensor = [] 92 | expos_tensor = [] 93 | matches_tensor = [] 94 | for i in range(len(ldrs)): 95 | ldr = ldrs[i].astype(np.float32).transpose(2, 0, 1) 96 | ldr_tensor = torch.from_numpy(ldr) 97 | ldrs_tensor.append(ldr_tensor) 98 | expos_tensor.append(torch.tensor(expos[i])) 99 | matches_tensor.append(torch.tensor(matches[i])) 100 | 101 | sample = { 102 | 'ldrs': ldrs_tensor, 103 | 'expos': expos_tensor, 104 | 'matches': matches_tensor 105 | } 106 | return sample 107 | 108 | def __len__(self): 109 | return len(self.imgs_list) 110 | 111 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Academic Project Page Template 2 | This is an academic paper project page template. 3 | 4 | 5 | Example project pages built using this template are: 6 | - https://www.vision.huji.ac.il/deepsim/ 7 | - https://www.vision.huji.ac.il/3d_ads/ 8 | - https://www.vision.huji.ac.il/ssrl_ad/ 9 | - https://www.vision.huji.ac.il/conffusion/ 10 | 11 | 12 | ## Start using the template 13 | To start using the template click on `Use this Template`. 14 | 15 | The template uses html for controlling the content and css for controlling the style. 16 | To edit the websites contents edit the `index.html` file. It contains different HTML "building blocks", use whichever ones you need and comment out the rest. 17 | 18 | **IMPORTANT!** Make sure to replace the `favicon.ico` under `static/images/` with one of your own, otherwise your favicon is going to be a dreambooth image of me. 19 | 20 | ## Components 21 | - Teaser video 22 | - Images Carousel 23 | - Youtube embedding 24 | - Video Carousel 25 | - PDF Poster 26 | - Bibtex citation 27 | 28 | ## Tips: 29 | - The `index.html` file contains comments instructing you what to replace, you should follow these comments. 30 | - The `meta` tags in the `index.html` file are used to provide metadata about your paper 31 | (e.g. helping search engine index the website, showing a preview image when sharing the website, etc.) 32 | - The resolution of images and videos can usually be around 1920-2048, there rarely a need for better resolution that take longer to load. 33 | - All the images and videos you use should be compressed to allow for fast loading of the website (and thus better indexing by search engines). For images, you can use [TinyPNG](https://tinypng.com), for videos you can need to find the tradeoff between size and quality. 34 | - When using large video files (larger than 10MB), it's better to use youtube for hosting the video as serving the video from the website can take time. 35 | - Using a tracker can help you analyze the traffic and see where users came from. [statcounter](https://statcounter.com) is a free, easy to use tracker that takes under 5 minutes to set up. 36 | - This project page can also be made into a github pages website. 37 | - Replace the favicon to one of your choosing (the default one is of the Hebrew University). 38 | - Suggestions, improvements and comments are welcome, simply open an issue or contact me. You can find my contact information at [https://pages.cs.huji.ac.il/eliahu-horwitz/](https://pages.cs.huji.ac.il/eliahu-horwitz/) 39 | 40 | ## Acknowledgments 41 | Parts of this project page were adopted from the [Nerfies](https://nerfies.github.io/) page. 42 | 43 | ## Website License 44 | Creative Commons License
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License. 45 | -------------------------------------------------------------------------------- /docs/_site/README.md: -------------------------------------------------------------------------------- 1 | # Academic Project Page Template 2 | This is an academic paper project page template. 3 | 4 | 5 | Example project pages built using this template are: 6 | - https://www.vision.huji.ac.il/deepsim/ 7 | - https://www.vision.huji.ac.il/3d_ads/ 8 | - https://www.vision.huji.ac.il/ssrl_ad/ 9 | - https://www.vision.huji.ac.il/conffusion/ 10 | 11 | 12 | ## Start using the template 13 | To start using the template click on `Use this Template`. 14 | 15 | The template uses html for controlling the content and css for controlling the style. 16 | To edit the websites contents edit the `index.html` file. It contains different HTML "building blocks", use whichever ones you need and comment out the rest. 17 | 18 | **IMPORTANT!** Make sure to replace the `favicon.ico` under `static/images/` with one of your own, otherwise your favicon is going to be a dreambooth image of me. 19 | 20 | ## Components 21 | - Teaser video 22 | - Images Carousel 23 | - Youtube embedding 24 | - Video Carousel 25 | - PDF Poster 26 | - Bibtex citation 27 | 28 | ## Tips: 29 | - The `index.html` file contains comments instructing you what to replace, you should follow these comments. 30 | - The `meta` tags in the `index.html` file are used to provide metadata about your paper 31 | (e.g. helping search engine index the website, showing a preview image when sharing the website, etc.) 32 | - The resolution of images and videos can usually be around 1920-2048, there rarely a need for better resolution that take longer to load. 33 | - All the images and videos you use should be compressed to allow for fast loading of the website (and thus better indexing by search engines). For images, you can use [TinyPNG](https://tinypng.com), for videos you can need to find the tradeoff between size and quality. 34 | - When using large video files (larger than 10MB), it's better to use youtube for hosting the video as serving the video from the website can take time. 35 | - Using a tracker can help you analyze the traffic and see where users came from. [statcounter](https://statcounter.com) is a free, easy to use tracker that takes under 5 minutes to set up. 36 | - This project page can also be made into a github pages website. 37 | - Replace the favicon to one of your choosing (the default one is of the Hebrew University). 38 | - Suggestions, improvements and comments are welcome, simply open an issue or contact me. You can find my contact information at [https://pages.cs.huji.ac.il/eliahu-horwitz/](https://pages.cs.huji.ac.il/eliahu-horwitz/) 39 | 40 | ## Acknowledgments 41 | Parts of this project page were adopted from the [Nerfies](https://nerfies.github.io/) page. 42 | 43 | ## Website License 44 | Creative Commons License
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License. 45 | -------------------------------------------------------------------------------- /docs/_site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Event-Based Motion Magnification 28 | 29 | 31 | 32 | 33 | 34 | 35 | 36 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 |
51 |
52 |
53 |
54 |
55 |

Event-Based Motion Magnification

56 |
57 | 58 | 59 | Yutian Chen1,2, 60 | 61 | Shi Guo1,†, 62 | 63 | Fangzheng Yu1,2, 64 | 65 | Feng Zhang1, 66 | 67 | Jinwei Gu3, 68 | 69 | Tianfan Xue3 70 | 71 |
72 | 73 |
74 | 1Shanghai AI Laboratory, 2Zhejiang University, 75 | 3The Chinese University of Hong Kong 76 |
†Indicates Corresponding Author 77 |
78 | 79 |
80 |
81 | 82 | 83 | 85 | 86 | 87 | 88 | Paper 89 | 90 | 91 | 92 | 93 | 94 | 96 | 97 | 98 | 99 | Supplementary 100 | 101 | 102 | 103 | 104 | 105 | 107 | 108 | 109 | 110 | Code 111 | 112 | 113 | 114 | 115 | 116 | 118 | 119 | 120 | 121 | arXiv 122 | 123 | 124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 | 132 |
133 |
134 |
135 | 139 |

140 | Out system enables a broad and cost-effective amplification of high-frequency motions. 141 |

142 |
143 |
144 |
145 | 146 |
147 |
148 | 149 |
150 |
151 |

Abstract

152 |
153 |

154 | Detecting and magnifying imperceptible high-frequency motions in real-world scenarios has substantial implications for industrial and medical applications. These motions are characterized by small amplitudes and high frequencies. Traditional motion magnification methods rely on costly high-speed cameras or active light sources, which limit the scope of their applications. In this work, we propose a dual-camera system consisting of an event camera and a conventional RGB camera for video motion magnification, containing temporally-dense information from the event stream and spatially-dense data from the RGB images. This innovative combination enables a broad and cost-effective amplification of high-frequency motions. By revisiting the physical camera model, we observe that estimating motion direction and magnitude necessitates the integration of event streams with additional image features. On this basis, we propose a novel deep network for event-based video motion magnification that addresses two primary challenges: firstly, the high frequency of motion induces a large number of interpolated frames (up to 80), which our network mitigates with a Second-order Recurrent Propagation module for better handling of long-term frame interpolations; and secondly, magnifying subtle motions is sensitive to noise, which we address by utilizing a temporal filter to amplify motion at specific frequencies and reduce noise impact. We demonstrate the effectiveness and accuracy of our dual-camera system and network through extensive experiments in magnifying small-amplitude, high-frequency motions, offering a cost-effective and flexible solution for motion detection and magnification. 155 |

156 |

157 |

158 |
159 |
160 |
161 | 162 | 163 | 164 |
165 |
166 |

Video

167 |
168 | 173 |
174 |
175 |
176 | 177 |
178 |
179 | 180 | 181 | 182 | 183 | 188 | 189 | 198 | 199 | 200 | 201 | 202 | 203 |
204 |
205 |

BibTeX

206 | 
BibTex Code Here
207 |
208 |
209 | 210 | 211 | 212 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /docs/_site/static/css/bulma-carousel.min.css: -------------------------------------------------------------------------------- 1 | @-webkit-keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.slider{position:relative;width:100%}.slider-container{display:flex;flex-wrap:nowrap;flex-direction:row;overflow:hidden;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0);min-height:100%}.slider-container.is-vertical{flex-direction:column}.slider-container .slider-item{flex:none}.slider-container .slider-item .image.is-covered img{-o-object-fit:cover;object-fit:cover;-o-object-position:center center;object-position:center center;height:100%;width:100%}.slider-container .slider-item .video-container{height:0;padding-bottom:0;padding-top:56.25%;margin:0;position:relative}.slider-container .slider-item .video-container.is-1by1,.slider-container .slider-item .video-container.is-square{padding-top:100%}.slider-container .slider-item .video-container.is-4by3{padding-top:75%}.slider-container .slider-item .video-container.is-21by9{padding-top:42.857143%}.slider-container .slider-item .video-container embed,.slider-container .slider-item .video-container iframe,.slider-container .slider-item .video-container object{position:absolute;top:0;left:0;width:100%!important;height:100%!important}.slider-navigation-next,.slider-navigation-previous{display:flex;justify-content:center;align-items:center;position:absolute;width:42px;height:42px;background:#fff center center no-repeat;background-size:20px 20px;border:1px solid #fff;border-radius:25091983px;box-shadow:0 2px 5px #3232321a;top:50%;margin-top:-20px;left:0;cursor:pointer;transition:opacity .3s,-webkit-transform .3s;transition:transform .3s,opacity .3s;transition:transform .3s,opacity .3s,-webkit-transform .3s}.slider-navigation-next:hover,.slider-navigation-previous:hover{-webkit-transform:scale(1.2);transform:scale(1.2)}.slider-navigation-next.is-hidden,.slider-navigation-previous.is-hidden{display:none;opacity:0}.slider-navigation-next svg,.slider-navigation-previous svg{width:25%}.slider-navigation-next{left:auto;right:0;background:#fff center center no-repeat;background-size:20px 20px}.slider-pagination{display:none;justify-content:center;align-items:center;position:absolute;bottom:0;left:0;right:0;padding:.5rem 1rem;text-align:center}.slider-pagination .slider-page{background:#fff;width:10px;height:10px;border-radius:25091983px;display:inline-block;margin:0 3px;box-shadow:0 2px 5px #3232321a;transition:-webkit-transform .3s;transition:transform .3s;transition:transform .3s,-webkit-transform .3s;cursor:pointer}.slider-pagination .slider-page.is-active,.slider-pagination .slider-page:hover{-webkit-transform:scale(1.4);transform:scale(1.4)}@media screen and (min-width:800px){.slider-pagination{display:flex}}.hero.has-carousel{position:relative}.hero.has-carousel+.hero-body,.hero.has-carousel+.hero-footer,.hero.has-carousel+.hero-head{z-index:10;overflow:hidden}.hero.has-carousel .hero-carousel{position:absolute;top:0;left:0;bottom:0;right:0;height:auto;border:none;margin:auto;padding:0;z-index:0}.hero.has-carousel .hero-carousel .slider{width:100%;max-width:100%;overflow:hidden;height:100%!important;max-height:100%;z-index:0}.hero.has-carousel .hero-carousel .slider .has-background{max-height:100%}.hero.has-carousel .hero-carousel .slider .has-background .is-background{-o-object-fit:cover;object-fit:cover;-o-object-position:center center;object-position:center center;height:100%;width:100%}.hero.has-carousel .hero-body{margin:0 3rem;z-index:10} -------------------------------------------------------------------------------- /docs/_site/static/css/bulma-slider.min.css: -------------------------------------------------------------------------------- 1 | @-webkit-keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}input[type=range].slider{-webkit-appearance:none;-moz-appearance:none;appearance:none;margin:1rem 0;background:0 0;touch-action:none}input[type=range].slider.is-fullwidth{display:block;width:100%}input[type=range].slider:focus{outline:0}input[type=range].slider:not([orient=vertical])::-webkit-slider-runnable-track{width:100%}input[type=range].slider:not([orient=vertical])::-moz-range-track{width:100%}input[type=range].slider:not([orient=vertical])::-ms-track{width:100%}input[type=range].slider:not([orient=vertical]).has-output+output,input[type=range].slider:not([orient=vertical]).has-output-tooltip+output{width:3rem;background:#4a4a4a;border-radius:4px;padding:.4rem .8rem;font-size:.75rem;line-height:.75rem;text-align:center;text-overflow:ellipsis;white-space:nowrap;color:#fff;overflow:hidden;pointer-events:none;z-index:200}input[type=range].slider:not([orient=vertical]).has-output-tooltip:disabled+output,input[type=range].slider:not([orient=vertical]).has-output:disabled+output{opacity:.5}input[type=range].slider:not([orient=vertical]).has-output{display:inline-block;vertical-align:middle;width:calc(100% - (4.2rem))}input[type=range].slider:not([orient=vertical]).has-output+output{display:inline-block;margin-left:.75rem;vertical-align:middle}input[type=range].slider:not([orient=vertical]).has-output-tooltip{display:block}input[type=range].slider:not([orient=vertical]).has-output-tooltip+output{position:absolute;left:0;top:-.1rem}input[type=range].slider[orient=vertical]{-webkit-appearance:slider-vertical;-moz-appearance:slider-vertical;appearance:slider-vertical;-webkit-writing-mode:bt-lr;-ms-writing-mode:bt-lr;writing-mode:bt-lr}input[type=range].slider[orient=vertical]::-webkit-slider-runnable-track{height:100%}input[type=range].slider[orient=vertical]::-moz-range-track{height:100%}input[type=range].slider[orient=vertical]::-ms-track{height:100%}input[type=range].slider::-webkit-slider-runnable-track{cursor:pointer;animate:.2s;box-shadow:0 0 0 #7a7a7a;background:#dbdbdb;border-radius:4px;border:0 solid #7a7a7a}input[type=range].slider::-moz-range-track{cursor:pointer;animate:.2s;box-shadow:0 0 0 #7a7a7a;background:#dbdbdb;border-radius:4px;border:0 solid #7a7a7a}input[type=range].slider::-ms-track{cursor:pointer;animate:.2s;box-shadow:0 0 0 #7a7a7a;background:#dbdbdb;border-radius:4px;border:0 solid #7a7a7a}input[type=range].slider::-ms-fill-lower{background:#dbdbdb;border-radius:4px}input[type=range].slider::-ms-fill-upper{background:#dbdbdb;border-radius:4px}input[type=range].slider::-webkit-slider-thumb{box-shadow:none;border:1px solid #b5b5b5;border-radius:4px;background:#fff;cursor:pointer}input[type=range].slider::-moz-range-thumb{box-shadow:none;border:1px solid #b5b5b5;border-radius:4px;background:#fff;cursor:pointer}input[type=range].slider::-ms-thumb{box-shadow:none;border:1px solid #b5b5b5;border-radius:4px;background:#fff;cursor:pointer}input[type=range].slider::-webkit-slider-thumb{-webkit-appearance:none;appearance:none}input[type=range].slider.is-circle::-webkit-slider-thumb{border-radius:290486px}input[type=range].slider.is-circle::-moz-range-thumb{border-radius:290486px}input[type=range].slider.is-circle::-ms-thumb{border-radius:290486px}input[type=range].slider:active::-webkit-slider-thumb{-webkit-transform:scale(1.25);transform:scale(1.25)}input[type=range].slider:active::-moz-range-thumb{transform:scale(1.25)}input[type=range].slider:active::-ms-thumb{transform:scale(1.25)}input[type=range].slider:disabled{opacity:.5;cursor:not-allowed}input[type=range].slider:disabled::-webkit-slider-thumb{cursor:not-allowed;-webkit-transform:scale(1);transform:scale(1)}input[type=range].slider:disabled::-moz-range-thumb{cursor:not-allowed;transform:scale(1)}input[type=range].slider:disabled::-ms-thumb{cursor:not-allowed;transform:scale(1)}input[type=range].slider:not([orient=vertical]){min-height:calc((1rem + 2px) * 1.25)}input[type=range].slider:not([orient=vertical])::-webkit-slider-runnable-track{height:.5rem}input[type=range].slider:not([orient=vertical])::-moz-range-track{height:.5rem}input[type=range].slider:not([orient=vertical])::-ms-track{height:.5rem}input[type=range].slider[orient=vertical]::-webkit-slider-runnable-track{width:.5rem}input[type=range].slider[orient=vertical]::-moz-range-track{width:.5rem}input[type=range].slider[orient=vertical]::-ms-track{width:.5rem}input[type=range].slider::-webkit-slider-thumb{height:1rem;width:1rem}input[type=range].slider::-moz-range-thumb{height:1rem;width:1rem}input[type=range].slider::-ms-thumb{height:1rem;width:1rem}input[type=range].slider::-ms-thumb{margin-top:0}input[type=range].slider::-webkit-slider-thumb{margin-top:-.25rem}input[type=range].slider[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.25rem}input[type=range].slider.is-small:not([orient=vertical]){min-height:calc((.75rem + 2px) * 1.25)}input[type=range].slider.is-small:not([orient=vertical])::-webkit-slider-runnable-track{height:.375rem}input[type=range].slider.is-small:not([orient=vertical])::-moz-range-track{height:.375rem}input[type=range].slider.is-small:not([orient=vertical])::-ms-track{height:.375rem}input[type=range].slider.is-small[orient=vertical]::-webkit-slider-runnable-track{width:.375rem}input[type=range].slider.is-small[orient=vertical]::-moz-range-track{width:.375rem}input[type=range].slider.is-small[orient=vertical]::-ms-track{width:.375rem}input[type=range].slider.is-small::-webkit-slider-thumb{height:.75rem;width:.75rem}input[type=range].slider.is-small::-moz-range-thumb{height:.75rem;width:.75rem}input[type=range].slider.is-small::-ms-thumb{height:.75rem;width:.75rem}input[type=range].slider.is-small::-ms-thumb{margin-top:0}input[type=range].slider.is-small::-webkit-slider-thumb{margin-top:-.1875rem}input[type=range].slider.is-small[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.1875rem}input[type=range].slider.is-medium:not([orient=vertical]){min-height:calc((1.25rem + 2px) * 1.25)}input[type=range].slider.is-medium:not([orient=vertical])::-webkit-slider-runnable-track{height:.625rem}input[type=range].slider.is-medium:not([orient=vertical])::-moz-range-track{height:.625rem}input[type=range].slider.is-medium:not([orient=vertical])::-ms-track{height:.625rem}input[type=range].slider.is-medium[orient=vertical]::-webkit-slider-runnable-track{width:.625rem}input[type=range].slider.is-medium[orient=vertical]::-moz-range-track{width:.625rem}input[type=range].slider.is-medium[orient=vertical]::-ms-track{width:.625rem}input[type=range].slider.is-medium::-webkit-slider-thumb{height:1.25rem;width:1.25rem}input[type=range].slider.is-medium::-moz-range-thumb{height:1.25rem;width:1.25rem}input[type=range].slider.is-medium::-ms-thumb{height:1.25rem;width:1.25rem}input[type=range].slider.is-medium::-ms-thumb{margin-top:0}input[type=range].slider.is-medium::-webkit-slider-thumb{margin-top:-.3125rem}input[type=range].slider.is-medium[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.3125rem}input[type=range].slider.is-large:not([orient=vertical]){min-height:calc((1.5rem + 2px) * 1.25)}input[type=range].slider.is-large:not([orient=vertical])::-webkit-slider-runnable-track{height:.75rem}input[type=range].slider.is-large:not([orient=vertical])::-moz-range-track{height:.75rem}input[type=range].slider.is-large:not([orient=vertical])::-ms-track{height:.75rem}input[type=range].slider.is-large[orient=vertical]::-webkit-slider-runnable-track{width:.75rem}input[type=range].slider.is-large[orient=vertical]::-moz-range-track{width:.75rem}input[type=range].slider.is-large[orient=vertical]::-ms-track{width:.75rem}input[type=range].slider.is-large::-webkit-slider-thumb{height:1.5rem;width:1.5rem}input[type=range].slider.is-large::-moz-range-thumb{height:1.5rem;width:1.5rem}input[type=range].slider.is-large::-ms-thumb{height:1.5rem;width:1.5rem}input[type=range].slider.is-large::-ms-thumb{margin-top:0}input[type=range].slider.is-large::-webkit-slider-thumb{margin-top:-.375rem}input[type=range].slider.is-large[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.375rem}input[type=range].slider.is-white::-moz-range-track{background:#fff!important}input[type=range].slider.is-white::-webkit-slider-runnable-track{background:#fff!important}input[type=range].slider.is-white::-ms-track{background:#fff!important}input[type=range].slider.is-white::-ms-fill-lower{background:#fff}input[type=range].slider.is-white::-ms-fill-upper{background:#fff}input[type=range].slider.is-white .has-output-tooltip+output,input[type=range].slider.is-white.has-output+output{background-color:#fff;color:#0a0a0a}input[type=range].slider.is-black::-moz-range-track{background:#0a0a0a!important}input[type=range].slider.is-black::-webkit-slider-runnable-track{background:#0a0a0a!important}input[type=range].slider.is-black::-ms-track{background:#0a0a0a!important}input[type=range].slider.is-black::-ms-fill-lower{background:#0a0a0a}input[type=range].slider.is-black::-ms-fill-upper{background:#0a0a0a}input[type=range].slider.is-black .has-output-tooltip+output,input[type=range].slider.is-black.has-output+output{background-color:#0a0a0a;color:#fff}input[type=range].slider.is-light::-moz-range-track{background:#f5f5f5!important}input[type=range].slider.is-light::-webkit-slider-runnable-track{background:#f5f5f5!important}input[type=range].slider.is-light::-ms-track{background:#f5f5f5!important}input[type=range].slider.is-light::-ms-fill-lower{background:#f5f5f5}input[type=range].slider.is-light::-ms-fill-upper{background:#f5f5f5}input[type=range].slider.is-light .has-output-tooltip+output,input[type=range].slider.is-light.has-output+output{background-color:#f5f5f5;color:#363636}input[type=range].slider.is-dark::-moz-range-track{background:#363636!important}input[type=range].slider.is-dark::-webkit-slider-runnable-track{background:#363636!important}input[type=range].slider.is-dark::-ms-track{background:#363636!important}input[type=range].slider.is-dark::-ms-fill-lower{background:#363636}input[type=range].slider.is-dark::-ms-fill-upper{background:#363636}input[type=range].slider.is-dark .has-output-tooltip+output,input[type=range].slider.is-dark.has-output+output{background-color:#363636;color:#f5f5f5}input[type=range].slider.is-primary::-moz-range-track{background:#00d1b2!important}input[type=range].slider.is-primary::-webkit-slider-runnable-track{background:#00d1b2!important}input[type=range].slider.is-primary::-ms-track{background:#00d1b2!important}input[type=range].slider.is-primary::-ms-fill-lower{background:#00d1b2}input[type=range].slider.is-primary::-ms-fill-upper{background:#00d1b2}input[type=range].slider.is-primary .has-output-tooltip+output,input[type=range].slider.is-primary.has-output+output{background-color:#00d1b2;color:#fff}input[type=range].slider.is-link::-moz-range-track{background:#3273dc!important}input[type=range].slider.is-link::-webkit-slider-runnable-track{background:#3273dc!important}input[type=range].slider.is-link::-ms-track{background:#3273dc!important}input[type=range].slider.is-link::-ms-fill-lower{background:#3273dc}input[type=range].slider.is-link::-ms-fill-upper{background:#3273dc}input[type=range].slider.is-link .has-output-tooltip+output,input[type=range].slider.is-link.has-output+output{background-color:#3273dc;color:#fff}input[type=range].slider.is-info::-moz-range-track{background:#209cee!important}input[type=range].slider.is-info::-webkit-slider-runnable-track{background:#209cee!important}input[type=range].slider.is-info::-ms-track{background:#209cee!important}input[type=range].slider.is-info::-ms-fill-lower{background:#209cee}input[type=range].slider.is-info::-ms-fill-upper{background:#209cee}input[type=range].slider.is-info .has-output-tooltip+output,input[type=range].slider.is-info.has-output+output{background-color:#209cee;color:#fff}input[type=range].slider.is-success::-moz-range-track{background:#23d160!important}input[type=range].slider.is-success::-webkit-slider-runnable-track{background:#23d160!important}input[type=range].slider.is-success::-ms-track{background:#23d160!important}input[type=range].slider.is-success::-ms-fill-lower{background:#23d160}input[type=range].slider.is-success::-ms-fill-upper{background:#23d160}input[type=range].slider.is-success .has-output-tooltip+output,input[type=range].slider.is-success.has-output+output{background-color:#23d160;color:#fff}input[type=range].slider.is-warning::-moz-range-track{background:#ffdd57!important}input[type=range].slider.is-warning::-webkit-slider-runnable-track{background:#ffdd57!important}input[type=range].slider.is-warning::-ms-track{background:#ffdd57!important}input[type=range].slider.is-warning::-ms-fill-lower{background:#ffdd57}input[type=range].slider.is-warning::-ms-fill-upper{background:#ffdd57}input[type=range].slider.is-warning .has-output-tooltip+output,input[type=range].slider.is-warning.has-output+output{background-color:#ffdd57;color:rgba(0,0,0,.7)}input[type=range].slider.is-danger::-moz-range-track{background:#ff3860!important}input[type=range].slider.is-danger::-webkit-slider-runnable-track{background:#ff3860!important}input[type=range].slider.is-danger::-ms-track{background:#ff3860!important}input[type=range].slider.is-danger::-ms-fill-lower{background:#ff3860}input[type=range].slider.is-danger::-ms-fill-upper{background:#ff3860}input[type=range].slider.is-danger .has-output-tooltip+output,input[type=range].slider.is-danger.has-output+output{background-color:#ff3860;color:#fff} -------------------------------------------------------------------------------- /docs/_site/static/css/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: 'Noto Sans', sans-serif; 3 | } 4 | 5 | 6 | .footer .icon-link { 7 | font-size: 25px; 8 | color: #000; 9 | } 10 | 11 | .link-block a { 12 | margin-top: 5px; 13 | margin-bottom: 5px; 14 | } 15 | 16 | .dnerf { 17 | font-variant: small-caps; 18 | } 19 | 20 | 21 | .teaser .hero-body { 22 | padding-top: 0; 23 | padding-bottom: 3rem; 24 | } 25 | 26 | .teaser { 27 | font-family: 'Google Sans', sans-serif; 28 | } 29 | 30 | 31 | .publication-title { 32 | } 33 | 34 | .publication-banner { 35 | max-height: parent; 36 | 37 | } 38 | 39 | .publication-banner video { 40 | position: relative; 41 | left: auto; 42 | top: auto; 43 | transform: none; 44 | object-fit: fit; 45 | } 46 | 47 | .publication-header .hero-body { 48 | } 49 | 50 | .publication-title { 51 | font-family: 'Google Sans', sans-serif; 52 | } 53 | 54 | .publication-authors { 55 | font-family: 'Google Sans', sans-serif; 56 | } 57 | 58 | .publication-venue { 59 | color: #555; 60 | width: fit-content; 61 | font-weight: bold; 62 | } 63 | 64 | .publication-awards { 65 | color: #ff3860; 66 | width: fit-content; 67 | font-weight: bolder; 68 | } 69 | 70 | .publication-authors { 71 | } 72 | 73 | .publication-authors a { 74 | color: hsl(204, 86%, 53%) !important; 75 | } 76 | 77 | .publication-authors a:hover { 78 | text-decoration: underline; 79 | } 80 | 81 | .author-block { 82 | display: inline-block; 83 | } 84 | 85 | .publication-banner img { 86 | } 87 | 88 | .publication-authors { 89 | /*color: #4286f4;*/ 90 | } 91 | 92 | .publication-video { 93 | position: relative; 94 | width: 100%; 95 | height: 0; 96 | padding-bottom: 56.25%; 97 | 98 | overflow: hidden; 99 | border-radius: 10px !important; 100 | } 101 | 102 | .publication-video iframe { 103 | position: absolute; 104 | top: 0; 105 | left: 0; 106 | width: 100%; 107 | height: 100%; 108 | } 109 | 110 | .publication-body img { 111 | } 112 | 113 | .results-carousel { 114 | overflow: hidden; 115 | } 116 | 117 | .results-carousel .item { 118 | margin: 5px; 119 | overflow: hidden; 120 | padding: 20px; 121 | font-size: 0; 122 | } 123 | 124 | .results-carousel video { 125 | margin: 0; 126 | } 127 | 128 | .slider-pagination .slider-page { 129 | background: #000000; 130 | } 131 | 132 | .eql-cntrb { 133 | font-size: smaller; 134 | } 135 | 136 | 137 | 138 | -------------------------------------------------------------------------------- /docs/_site/static/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/_site/static/images/icon.png -------------------------------------------------------------------------------- /docs/_site/static/js/bulma-slider.min.js: -------------------------------------------------------------------------------- 1 | !function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.bulmaSlider=e():t.bulmaSlider=e()}("undefined"!=typeof self?self:this,function(){return function(n){var r={};function i(t){if(r[t])return r[t].exports;var e=r[t]={i:t,l:!1,exports:{}};return n[t].call(e.exports,e,e.exports,i),e.l=!0,e.exports}return i.m=n,i.c=r,i.d=function(t,e,n){i.o(t,e)||Object.defineProperty(t,e,{configurable:!1,enumerable:!0,get:n})},i.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return i.d(e,"a",e),e},i.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},i.p="",i(i.s=0)}([function(t,e,n){"use strict";Object.defineProperty(e,"__esModule",{value:!0}),n.d(e,"isString",function(){return l});var r=n(1),i=Object.assign||function(t){for(var e=1;e=l.length&&(s=!0)):s=!0),s&&(t.once&&(u[e]=null),t.callback(r))});-1!==u.indexOf(null);)u.splice(u.indexOf(null),1)}}]),e}();e.a=i}]).default}); -------------------------------------------------------------------------------- /docs/_site/static/js/index.js: -------------------------------------------------------------------------------- 1 | window.HELP_IMPROVE_VIDEOJS = false; 2 | 3 | 4 | $(document).ready(function() { 5 | // Check for click events on the navbar burger icon 6 | 7 | var options = { 8 | slidesToScroll: 1, 9 | slidesToShow: 1, 10 | loop: true, 11 | infinite: true, 12 | autoplay: true, 13 | autoplaySpeed: 5000, 14 | } 15 | 16 | // Initialize all div with carousel class 17 | var carousels = bulmaCarousel.attach('.carousel', options); 18 | 19 | bulmaSlider.attach(); 20 | 21 | }) 22 | -------------------------------------------------------------------------------- /docs/_site/static/pdfs/sample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/_site/static/pdfs/sample.pdf -------------------------------------------------------------------------------- /docs/_site/static/videos/teaser.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/_site/static/videos/teaser.mp4 -------------------------------------------------------------------------------- /docs/static/css/bulma-carousel.min.css: -------------------------------------------------------------------------------- 1 | @-webkit-keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.slider{position:relative;width:100%}.slider-container{display:flex;flex-wrap:nowrap;flex-direction:row;overflow:hidden;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0);min-height:100%}.slider-container.is-vertical{flex-direction:column}.slider-container .slider-item{flex:none}.slider-container .slider-item .image.is-covered img{-o-object-fit:cover;object-fit:cover;-o-object-position:center center;object-position:center center;height:100%;width:100%}.slider-container .slider-item .video-container{height:0;padding-bottom:0;padding-top:56.25%;margin:0;position:relative}.slider-container .slider-item .video-container.is-1by1,.slider-container .slider-item .video-container.is-square{padding-top:100%}.slider-container .slider-item .video-container.is-4by3{padding-top:75%}.slider-container .slider-item .video-container.is-21by9{padding-top:42.857143%}.slider-container .slider-item .video-container embed,.slider-container .slider-item .video-container iframe,.slider-container .slider-item .video-container object{position:absolute;top:0;left:0;width:100%!important;height:100%!important}.slider-navigation-next,.slider-navigation-previous{display:flex;justify-content:center;align-items:center;position:absolute;width:42px;height:42px;background:#fff center center no-repeat;background-size:20px 20px;border:1px solid #fff;border-radius:25091983px;box-shadow:0 2px 5px #3232321a;top:50%;margin-top:-20px;left:0;cursor:pointer;transition:opacity .3s,-webkit-transform .3s;transition:transform .3s,opacity .3s;transition:transform .3s,opacity .3s,-webkit-transform .3s}.slider-navigation-next:hover,.slider-navigation-previous:hover{-webkit-transform:scale(1.2);transform:scale(1.2)}.slider-navigation-next.is-hidden,.slider-navigation-previous.is-hidden{display:none;opacity:0}.slider-navigation-next svg,.slider-navigation-previous svg{width:25%}.slider-navigation-next{left:auto;right:0;background:#fff center center no-repeat;background-size:20px 20px}.slider-pagination{display:none;justify-content:center;align-items:center;position:absolute;bottom:0;left:0;right:0;padding:.5rem 1rem;text-align:center}.slider-pagination .slider-page{background:#fff;width:10px;height:10px;border-radius:25091983px;display:inline-block;margin:0 3px;box-shadow:0 2px 5px #3232321a;transition:-webkit-transform .3s;transition:transform .3s;transition:transform .3s,-webkit-transform .3s;cursor:pointer}.slider-pagination .slider-page.is-active,.slider-pagination .slider-page:hover{-webkit-transform:scale(1.4);transform:scale(1.4)}@media screen and (min-width:800px){.slider-pagination{display:flex}}.hero.has-carousel{position:relative}.hero.has-carousel+.hero-body,.hero.has-carousel+.hero-footer,.hero.has-carousel+.hero-head{z-index:10;overflow:hidden}.hero.has-carousel .hero-carousel{position:absolute;top:0;left:0;bottom:0;right:0;height:auto;border:none;margin:auto;padding:0;z-index:0}.hero.has-carousel .hero-carousel .slider{width:100%;max-width:100%;overflow:hidden;height:100%!important;max-height:100%;z-index:0}.hero.has-carousel .hero-carousel .slider .has-background{max-height:100%}.hero.has-carousel .hero-carousel .slider .has-background .is-background{-o-object-fit:cover;object-fit:cover;-o-object-position:center center;object-position:center center;height:100%;width:100%}.hero.has-carousel .hero-body{margin:0 3rem;z-index:10} -------------------------------------------------------------------------------- /docs/static/css/bulma-slider.min.css: -------------------------------------------------------------------------------- 1 | @-webkit-keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes spinAround{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}input[type=range].slider{-webkit-appearance:none;-moz-appearance:none;appearance:none;margin:1rem 0;background:0 0;touch-action:none}input[type=range].slider.is-fullwidth{display:block;width:100%}input[type=range].slider:focus{outline:0}input[type=range].slider:not([orient=vertical])::-webkit-slider-runnable-track{width:100%}input[type=range].slider:not([orient=vertical])::-moz-range-track{width:100%}input[type=range].slider:not([orient=vertical])::-ms-track{width:100%}input[type=range].slider:not([orient=vertical]).has-output+output,input[type=range].slider:not([orient=vertical]).has-output-tooltip+output{width:3rem;background:#4a4a4a;border-radius:4px;padding:.4rem .8rem;font-size:.75rem;line-height:.75rem;text-align:center;text-overflow:ellipsis;white-space:nowrap;color:#fff;overflow:hidden;pointer-events:none;z-index:200}input[type=range].slider:not([orient=vertical]).has-output-tooltip:disabled+output,input[type=range].slider:not([orient=vertical]).has-output:disabled+output{opacity:.5}input[type=range].slider:not([orient=vertical]).has-output{display:inline-block;vertical-align:middle;width:calc(100% - (4.2rem))}input[type=range].slider:not([orient=vertical]).has-output+output{display:inline-block;margin-left:.75rem;vertical-align:middle}input[type=range].slider:not([orient=vertical]).has-output-tooltip{display:block}input[type=range].slider:not([orient=vertical]).has-output-tooltip+output{position:absolute;left:0;top:-.1rem}input[type=range].slider[orient=vertical]{-webkit-appearance:slider-vertical;-moz-appearance:slider-vertical;appearance:slider-vertical;-webkit-writing-mode:bt-lr;-ms-writing-mode:bt-lr;writing-mode:bt-lr}input[type=range].slider[orient=vertical]::-webkit-slider-runnable-track{height:100%}input[type=range].slider[orient=vertical]::-moz-range-track{height:100%}input[type=range].slider[orient=vertical]::-ms-track{height:100%}input[type=range].slider::-webkit-slider-runnable-track{cursor:pointer;animate:.2s;box-shadow:0 0 0 #7a7a7a;background:#dbdbdb;border-radius:4px;border:0 solid #7a7a7a}input[type=range].slider::-moz-range-track{cursor:pointer;animate:.2s;box-shadow:0 0 0 #7a7a7a;background:#dbdbdb;border-radius:4px;border:0 solid #7a7a7a}input[type=range].slider::-ms-track{cursor:pointer;animate:.2s;box-shadow:0 0 0 #7a7a7a;background:#dbdbdb;border-radius:4px;border:0 solid #7a7a7a}input[type=range].slider::-ms-fill-lower{background:#dbdbdb;border-radius:4px}input[type=range].slider::-ms-fill-upper{background:#dbdbdb;border-radius:4px}input[type=range].slider::-webkit-slider-thumb{box-shadow:none;border:1px solid #b5b5b5;border-radius:4px;background:#fff;cursor:pointer}input[type=range].slider::-moz-range-thumb{box-shadow:none;border:1px solid #b5b5b5;border-radius:4px;background:#fff;cursor:pointer}input[type=range].slider::-ms-thumb{box-shadow:none;border:1px solid #b5b5b5;border-radius:4px;background:#fff;cursor:pointer}input[type=range].slider::-webkit-slider-thumb{-webkit-appearance:none;appearance:none}input[type=range].slider.is-circle::-webkit-slider-thumb{border-radius:290486px}input[type=range].slider.is-circle::-moz-range-thumb{border-radius:290486px}input[type=range].slider.is-circle::-ms-thumb{border-radius:290486px}input[type=range].slider:active::-webkit-slider-thumb{-webkit-transform:scale(1.25);transform:scale(1.25)}input[type=range].slider:active::-moz-range-thumb{transform:scale(1.25)}input[type=range].slider:active::-ms-thumb{transform:scale(1.25)}input[type=range].slider:disabled{opacity:.5;cursor:not-allowed}input[type=range].slider:disabled::-webkit-slider-thumb{cursor:not-allowed;-webkit-transform:scale(1);transform:scale(1)}input[type=range].slider:disabled::-moz-range-thumb{cursor:not-allowed;transform:scale(1)}input[type=range].slider:disabled::-ms-thumb{cursor:not-allowed;transform:scale(1)}input[type=range].slider:not([orient=vertical]){min-height:calc((1rem + 2px) * 1.25)}input[type=range].slider:not([orient=vertical])::-webkit-slider-runnable-track{height:.5rem}input[type=range].slider:not([orient=vertical])::-moz-range-track{height:.5rem}input[type=range].slider:not([orient=vertical])::-ms-track{height:.5rem}input[type=range].slider[orient=vertical]::-webkit-slider-runnable-track{width:.5rem}input[type=range].slider[orient=vertical]::-moz-range-track{width:.5rem}input[type=range].slider[orient=vertical]::-ms-track{width:.5rem}input[type=range].slider::-webkit-slider-thumb{height:1rem;width:1rem}input[type=range].slider::-moz-range-thumb{height:1rem;width:1rem}input[type=range].slider::-ms-thumb{height:1rem;width:1rem}input[type=range].slider::-ms-thumb{margin-top:0}input[type=range].slider::-webkit-slider-thumb{margin-top:-.25rem}input[type=range].slider[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.25rem}input[type=range].slider.is-small:not([orient=vertical]){min-height:calc((.75rem + 2px) * 1.25)}input[type=range].slider.is-small:not([orient=vertical])::-webkit-slider-runnable-track{height:.375rem}input[type=range].slider.is-small:not([orient=vertical])::-moz-range-track{height:.375rem}input[type=range].slider.is-small:not([orient=vertical])::-ms-track{height:.375rem}input[type=range].slider.is-small[orient=vertical]::-webkit-slider-runnable-track{width:.375rem}input[type=range].slider.is-small[orient=vertical]::-moz-range-track{width:.375rem}input[type=range].slider.is-small[orient=vertical]::-ms-track{width:.375rem}input[type=range].slider.is-small::-webkit-slider-thumb{height:.75rem;width:.75rem}input[type=range].slider.is-small::-moz-range-thumb{height:.75rem;width:.75rem}input[type=range].slider.is-small::-ms-thumb{height:.75rem;width:.75rem}input[type=range].slider.is-small::-ms-thumb{margin-top:0}input[type=range].slider.is-small::-webkit-slider-thumb{margin-top:-.1875rem}input[type=range].slider.is-small[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.1875rem}input[type=range].slider.is-medium:not([orient=vertical]){min-height:calc((1.25rem + 2px) * 1.25)}input[type=range].slider.is-medium:not([orient=vertical])::-webkit-slider-runnable-track{height:.625rem}input[type=range].slider.is-medium:not([orient=vertical])::-moz-range-track{height:.625rem}input[type=range].slider.is-medium:not([orient=vertical])::-ms-track{height:.625rem}input[type=range].slider.is-medium[orient=vertical]::-webkit-slider-runnable-track{width:.625rem}input[type=range].slider.is-medium[orient=vertical]::-moz-range-track{width:.625rem}input[type=range].slider.is-medium[orient=vertical]::-ms-track{width:.625rem}input[type=range].slider.is-medium::-webkit-slider-thumb{height:1.25rem;width:1.25rem}input[type=range].slider.is-medium::-moz-range-thumb{height:1.25rem;width:1.25rem}input[type=range].slider.is-medium::-ms-thumb{height:1.25rem;width:1.25rem}input[type=range].slider.is-medium::-ms-thumb{margin-top:0}input[type=range].slider.is-medium::-webkit-slider-thumb{margin-top:-.3125rem}input[type=range].slider.is-medium[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.3125rem}input[type=range].slider.is-large:not([orient=vertical]){min-height:calc((1.5rem + 2px) * 1.25)}input[type=range].slider.is-large:not([orient=vertical])::-webkit-slider-runnable-track{height:.75rem}input[type=range].slider.is-large:not([orient=vertical])::-moz-range-track{height:.75rem}input[type=range].slider.is-large:not([orient=vertical])::-ms-track{height:.75rem}input[type=range].slider.is-large[orient=vertical]::-webkit-slider-runnable-track{width:.75rem}input[type=range].slider.is-large[orient=vertical]::-moz-range-track{width:.75rem}input[type=range].slider.is-large[orient=vertical]::-ms-track{width:.75rem}input[type=range].slider.is-large::-webkit-slider-thumb{height:1.5rem;width:1.5rem}input[type=range].slider.is-large::-moz-range-thumb{height:1.5rem;width:1.5rem}input[type=range].slider.is-large::-ms-thumb{height:1.5rem;width:1.5rem}input[type=range].slider.is-large::-ms-thumb{margin-top:0}input[type=range].slider.is-large::-webkit-slider-thumb{margin-top:-.375rem}input[type=range].slider.is-large[orient=vertical]::-webkit-slider-thumb{margin-top:auto;margin-left:-.375rem}input[type=range].slider.is-white::-moz-range-track{background:#fff!important}input[type=range].slider.is-white::-webkit-slider-runnable-track{background:#fff!important}input[type=range].slider.is-white::-ms-track{background:#fff!important}input[type=range].slider.is-white::-ms-fill-lower{background:#fff}input[type=range].slider.is-white::-ms-fill-upper{background:#fff}input[type=range].slider.is-white .has-output-tooltip+output,input[type=range].slider.is-white.has-output+output{background-color:#fff;color:#0a0a0a}input[type=range].slider.is-black::-moz-range-track{background:#0a0a0a!important}input[type=range].slider.is-black::-webkit-slider-runnable-track{background:#0a0a0a!important}input[type=range].slider.is-black::-ms-track{background:#0a0a0a!important}input[type=range].slider.is-black::-ms-fill-lower{background:#0a0a0a}input[type=range].slider.is-black::-ms-fill-upper{background:#0a0a0a}input[type=range].slider.is-black .has-output-tooltip+output,input[type=range].slider.is-black.has-output+output{background-color:#0a0a0a;color:#fff}input[type=range].slider.is-light::-moz-range-track{background:#f5f5f5!important}input[type=range].slider.is-light::-webkit-slider-runnable-track{background:#f5f5f5!important}input[type=range].slider.is-light::-ms-track{background:#f5f5f5!important}input[type=range].slider.is-light::-ms-fill-lower{background:#f5f5f5}input[type=range].slider.is-light::-ms-fill-upper{background:#f5f5f5}input[type=range].slider.is-light .has-output-tooltip+output,input[type=range].slider.is-light.has-output+output{background-color:#f5f5f5;color:#363636}input[type=range].slider.is-dark::-moz-range-track{background:#363636!important}input[type=range].slider.is-dark::-webkit-slider-runnable-track{background:#363636!important}input[type=range].slider.is-dark::-ms-track{background:#363636!important}input[type=range].slider.is-dark::-ms-fill-lower{background:#363636}input[type=range].slider.is-dark::-ms-fill-upper{background:#363636}input[type=range].slider.is-dark .has-output-tooltip+output,input[type=range].slider.is-dark.has-output+output{background-color:#363636;color:#f5f5f5}input[type=range].slider.is-primary::-moz-range-track{background:#00d1b2!important}input[type=range].slider.is-primary::-webkit-slider-runnable-track{background:#00d1b2!important}input[type=range].slider.is-primary::-ms-track{background:#00d1b2!important}input[type=range].slider.is-primary::-ms-fill-lower{background:#00d1b2}input[type=range].slider.is-primary::-ms-fill-upper{background:#00d1b2}input[type=range].slider.is-primary .has-output-tooltip+output,input[type=range].slider.is-primary.has-output+output{background-color:#00d1b2;color:#fff}input[type=range].slider.is-link::-moz-range-track{background:#3273dc!important}input[type=range].slider.is-link::-webkit-slider-runnable-track{background:#3273dc!important}input[type=range].slider.is-link::-ms-track{background:#3273dc!important}input[type=range].slider.is-link::-ms-fill-lower{background:#3273dc}input[type=range].slider.is-link::-ms-fill-upper{background:#3273dc}input[type=range].slider.is-link .has-output-tooltip+output,input[type=range].slider.is-link.has-output+output{background-color:#3273dc;color:#fff}input[type=range].slider.is-info::-moz-range-track{background:#209cee!important}input[type=range].slider.is-info::-webkit-slider-runnable-track{background:#209cee!important}input[type=range].slider.is-info::-ms-track{background:#209cee!important}input[type=range].slider.is-info::-ms-fill-lower{background:#209cee}input[type=range].slider.is-info::-ms-fill-upper{background:#209cee}input[type=range].slider.is-info .has-output-tooltip+output,input[type=range].slider.is-info.has-output+output{background-color:#209cee;color:#fff}input[type=range].slider.is-success::-moz-range-track{background:#23d160!important}input[type=range].slider.is-success::-webkit-slider-runnable-track{background:#23d160!important}input[type=range].slider.is-success::-ms-track{background:#23d160!important}input[type=range].slider.is-success::-ms-fill-lower{background:#23d160}input[type=range].slider.is-success::-ms-fill-upper{background:#23d160}input[type=range].slider.is-success .has-output-tooltip+output,input[type=range].slider.is-success.has-output+output{background-color:#23d160;color:#fff}input[type=range].slider.is-warning::-moz-range-track{background:#ffdd57!important}input[type=range].slider.is-warning::-webkit-slider-runnable-track{background:#ffdd57!important}input[type=range].slider.is-warning::-ms-track{background:#ffdd57!important}input[type=range].slider.is-warning::-ms-fill-lower{background:#ffdd57}input[type=range].slider.is-warning::-ms-fill-upper{background:#ffdd57}input[type=range].slider.is-warning .has-output-tooltip+output,input[type=range].slider.is-warning.has-output+output{background-color:#ffdd57;color:rgba(0,0,0,.7)}input[type=range].slider.is-danger::-moz-range-track{background:#ff3860!important}input[type=range].slider.is-danger::-webkit-slider-runnable-track{background:#ff3860!important}input[type=range].slider.is-danger::-ms-track{background:#ff3860!important}input[type=range].slider.is-danger::-ms-fill-lower{background:#ff3860}input[type=range].slider.is-danger::-ms-fill-upper{background:#ff3860}input[type=range].slider.is-danger .has-output-tooltip+output,input[type=range].slider.is-danger.has-output+output{background-color:#ff3860;color:#fff} -------------------------------------------------------------------------------- /docs/static/css/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: 'Noto Sans', sans-serif; 3 | } 4 | 5 | 6 | .footer .icon-link { 7 | font-size: 25px; 8 | color: #000; 9 | } 10 | 11 | .link-block a { 12 | margin-top: 5px; 13 | margin-bottom: 5px; 14 | } 15 | 16 | .dnerf { 17 | font-variant: small-caps; 18 | } 19 | 20 | 21 | .teaser .hero-body { 22 | padding-top: 0; 23 | padding-bottom: 3rem; 24 | } 25 | 26 | .teaser { 27 | font-family: 'Google Sans', sans-serif; 28 | } 29 | 30 | 31 | .publication-title { 32 | } 33 | 34 | .publication-banner { 35 | max-height: parent; 36 | 37 | } 38 | 39 | .publication-banner video { 40 | position: relative; 41 | left: auto; 42 | top: auto; 43 | transform: none; 44 | object-fit: fit; 45 | } 46 | 47 | .publication-header .hero-body { 48 | } 49 | 50 | .publication-title { 51 | font-family: 'Google Sans', sans-serif; 52 | } 53 | 54 | .publication-authors { 55 | font-family: 'Google Sans', sans-serif; 56 | } 57 | 58 | .publication-venue { 59 | color: #555; 60 | width: fit-content; 61 | font-weight: bold; 62 | } 63 | 64 | .publication-awards { 65 | color: #ff3860; 66 | width: fit-content; 67 | font-weight: bolder; 68 | } 69 | 70 | .publication-authors { 71 | } 72 | 73 | .publication-authors a { 74 | color: hsl(204, 86%, 53%) !important; 75 | } 76 | 77 | .publication-authors a:hover { 78 | text-decoration: underline; 79 | } 80 | 81 | .author-block { 82 | display: inline-block; 83 | } 84 | 85 | .publication-banner img { 86 | } 87 | 88 | .publication-authors { 89 | /*color: #4286f4;*/ 90 | } 91 | 92 | .publication-video { 93 | position: relative; 94 | width: 100%; 95 | height: 0; 96 | padding-bottom: 56.25%; 97 | 98 | overflow: hidden; 99 | border-radius: 10px !important; 100 | } 101 | 102 | .publication-video iframe { 103 | position: absolute; 104 | top: 0; 105 | left: 0; 106 | width: 100%; 107 | height: 100%; 108 | } 109 | 110 | .publication-body img { 111 | } 112 | 113 | .results-carousel { 114 | overflow: hidden; 115 | } 116 | 117 | .results-carousel .item { 118 | margin: 5px; 119 | overflow: hidden; 120 | padding: 20px; 121 | font-size: 0; 122 | } 123 | 124 | .results-carousel video { 125 | margin: 0; 126 | } 127 | 128 | .slider-pagination .slider-page { 129 | background: #000000; 130 | } 131 | 132 | .eql-cntrb { 133 | font-size: smaller; 134 | } 135 | 136 | 137 | 138 | -------------------------------------------------------------------------------- /docs/static/images/comp_attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/images/comp_attention.png -------------------------------------------------------------------------------- /docs/static/images/comp_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/images/comp_flow.png -------------------------------------------------------------------------------- /docs/static/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/images/icon.png -------------------------------------------------------------------------------- /docs/static/images/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/images/teaser.png -------------------------------------------------------------------------------- /docs/static/images/teaser_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/images/teaser_v2.png -------------------------------------------------------------------------------- /docs/static/js/bulma-slider.min.js: -------------------------------------------------------------------------------- 1 | !function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.bulmaSlider=e():t.bulmaSlider=e()}("undefined"!=typeof self?self:this,function(){return function(n){var r={};function i(t){if(r[t])return r[t].exports;var e=r[t]={i:t,l:!1,exports:{}};return n[t].call(e.exports,e,e.exports,i),e.l=!0,e.exports}return i.m=n,i.c=r,i.d=function(t,e,n){i.o(t,e)||Object.defineProperty(t,e,{configurable:!1,enumerable:!0,get:n})},i.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return i.d(e,"a",e),e},i.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},i.p="",i(i.s=0)}([function(t,e,n){"use strict";Object.defineProperty(e,"__esModule",{value:!0}),n.d(e,"isString",function(){return l});var r=n(1),i=Object.assign||function(t){for(var e=1;e=l.length&&(s=!0)):s=!0),s&&(t.once&&(u[e]=null),t.callback(r))});-1!==u.indexOf(null);)u.splice(u.indexOf(null),1)}}]),e}();e.a=i}]).default}); -------------------------------------------------------------------------------- /docs/static/js/index.js: -------------------------------------------------------------------------------- 1 | window.HELP_IMPROVE_VIDEOJS = false; 2 | 3 | 4 | $(document).ready(function() { 5 | // Check for click events on the navbar burger icon 6 | 7 | var options = { 8 | slidesToScroll: 1, 9 | slidesToShow: 1, 10 | loop: true, 11 | infinite: true, 12 | autoplay: true, 13 | autoplaySpeed: 5000, 14 | } 15 | 16 | // Initialize all div with carousel class 17 | var carousels = bulmaCarousel.attach('.carousel', options); 18 | 19 | bulmaSlider.attach(); 20 | 21 | }) 22 | -------------------------------------------------------------------------------- /docs/static/pdfs/sample.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/pdfs/sample.pdf -------------------------------------------------------------------------------- /docs/static/videos/teaser.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/videos/teaser.mp4 -------------------------------------------------------------------------------- /docs/static/videos/teaser_v2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/docs/static/videos/teaser_v2.mp4 -------------------------------------------------------------------------------- /models/flow_2E.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .network_utils import * 4 | import torch.nn.functional as F 5 | 6 | class BottleneckBlock(nn.Module): 7 | def __init__(self, in_planes, planes, norm_fn='group', stride=1): 8 | super(BottleneckBlock, self).__init__() 9 | 10 | self.conv1 = nn.Conv2d(in_planes, planes//4, kernel_size=1, padding=0) 11 | self.conv2 = nn.Conv2d(planes//4, planes//4, kernel_size=3, padding=1, stride=stride) 12 | self.conv3 = nn.Conv2d(planes//4, planes, kernel_size=1, padding=0) 13 | self.relu = nn.ReLU(inplace=True) 14 | 15 | num_groups = planes // 8 16 | 17 | if norm_fn == 'group': 18 | self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=planes//4) 19 | self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=planes//4) 20 | self.norm3 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 21 | if not stride == 1: 22 | self.norm4 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 23 | 24 | elif norm_fn == 'batch': 25 | self.norm1 = nn.BatchNorm2d(planes//4) 26 | self.norm2 = nn.BatchNorm2d(planes//4) 27 | self.norm3 = nn.BatchNorm2d(planes) 28 | if not stride == 1: 29 | self.norm4 = nn.BatchNorm2d(planes) 30 | 31 | elif norm_fn == 'instance': 32 | self.norm1 = nn.InstanceNorm2d(planes//4) 33 | self.norm2 = nn.InstanceNorm2d(planes//4) 34 | self.norm3 = nn.InstanceNorm2d(planes) 35 | if not stride == 1: 36 | self.norm4 = nn.InstanceNorm2d(planes) 37 | 38 | elif norm_fn == 'none': 39 | self.norm1 = nn.Sequential() 40 | self.norm2 = nn.Sequential() 41 | self.norm3 = nn.Sequential() 42 | if not stride == 1: 43 | self.norm4 = nn.Sequential() 44 | 45 | if stride == 1: 46 | self.downsample = None 47 | 48 | else: 49 | self.downsample = nn.Sequential( 50 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride), self.norm4) 51 | 52 | 53 | def forward(self, x): 54 | y = x 55 | y = self.relu(self.norm1(self.conv1(y))) 56 | y = self.relu(self.norm2(self.conv2(y))) 57 | y = self.relu(self.norm3(self.conv3(y))) 58 | 59 | if self.downsample is not None: 60 | x = self.downsample(x) 61 | 62 | return self.relu(x+y) 63 | 64 | 65 | class Flow_Net(nn.Module): 66 | """ for 2-exposure """ 67 | def __init__(self): 68 | super(Flow_Net, self).__init__() 69 | 70 | self.norm_fn = 'batch' 71 | self.in_planes = 32 72 | self.conv1 = nn.Sequential(nn.Conv2d(9, 32, kernel_size=7, stride=2, padding=3), 73 | nn.BatchNorm2d(32), 74 | nn.ReLU(inplace=True)) 75 | 76 | self.imconv4 = nn.Sequential(nn.Conv2d(9, 64, kernel_size=3, stride=1, padding=1), 77 | nn.BatchNorm2d(64), 78 | nn.ReLU(inplace=True)) 79 | 80 | self.combine4 = nn.Sequential(nn.Conv2d(64*2, 64, kernel_size=1, stride=1, padding=0), 81 | nn.BatchNorm2d(64), 82 | nn.ReLU(inplace=True), 83 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), 84 | nn.BatchNorm2d(64), 85 | nn.ReLU(inplace=True)) 86 | 87 | self.imconv8 = nn.Sequential(nn.Conv2d(9, 128, kernel_size=3, stride=1, padding=1), 88 | nn.BatchNorm2d(128), 89 | nn.ReLU(inplace=True)) 90 | 91 | self.combine8 = nn.Sequential(nn.Conv2d(128*2, 128, kernel_size=1, stride=1, padding=0), 92 | nn.BatchNorm2d(128), 93 | nn.ReLU(inplace=True), 94 | nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), 95 | nn.BatchNorm2d(128), 96 | nn.ReLU(inplace=True)) 97 | 98 | self.imconv16 = nn.Sequential(nn.Conv2d(9, 256, kernel_size=3, stride=1, padding=1), 99 | nn.BatchNorm2d(256), 100 | nn.ReLU(inplace=True)) 101 | 102 | self.combine16 = nn.Sequential(nn.Conv2d(256*2, 256, kernel_size=1, stride=1, padding=0), 103 | nn.BatchNorm2d(256), 104 | nn.ReLU(inplace=True), 105 | nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1), 106 | nn.BatchNorm2d(256), 107 | nn.ReLU(inplace=True)) 108 | 109 | self.layer1 = self._make_layer(32, stride=1) # 1/2 110 | self.layer2 = self._make_layer(64, stride=2) # 1/4 111 | self.layer3 = self._make_layer(128, stride=2) # 1/8 112 | self.layer4 = self._make_layer(256, stride=2) # 1/16 113 | 114 | self.upconv4 = nn.Sequential(nn.ConvTranspose2d(256, 128, kernel_size=4, stride=2, padding=1), 115 | nn.BatchNorm2d(128), 116 | nn.ReLU(inplace=True)) 117 | 118 | self.iconv3 = nn.Sequential(nn.Conv2d(128*2, 128, kernel_size=1, stride=1, padding=0), 119 | nn.BatchNorm2d(128), 120 | nn.ReLU(inplace=True), 121 | nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), 122 | nn.BatchNorm2d(128), 123 | nn.ReLU(inplace=True)) 124 | 125 | self.upconv3 = nn.Sequential(nn.ConvTranspose2d(128, 64, kernel_size=4, stride=2, padding=1), 126 | nn.BatchNorm2d(64), 127 | nn.ReLU(inplace=True)) 128 | 129 | self.iconv2 = nn.Sequential(nn.Conv2d(64*2, 64, kernel_size=1, stride=1, padding=0), 130 | nn.BatchNorm2d(64), 131 | nn.ReLU(inplace=True), 132 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), 133 | nn.BatchNorm2d(64), 134 | nn.ReLU(inplace=True)) 135 | 136 | self.conv7x7 = nn.Conv2d(256, 256, 7, stride=1, padding=7//2, groups=256) 137 | self.conv9x9 = nn.Conv2d(256, 256, 9, stride=1, padding=9//2, groups=256) 138 | self.conv11x11 = nn.Conv2d(256, 256, 11, stride=1, padding=11//2, groups=256) 139 | 140 | self.merge_features = nn.Conv2d(256*3, 256, 1, stride=1, padding=0) 141 | 142 | self.flow_head = nn.Sequential(nn.Conv2d(64, 64, kernel_size=5, stride=1, padding=2), 143 | nn.ReLU(inplace=True), 144 | nn.Conv2d(64, 64, kernel_size=5, stride=1, padding=2), 145 | nn.ReLU(inplace=True), 146 | nn.Conv2d(64, 4, kernel_size=5, stride=1, padding=2)) 147 | 148 | def _make_layer(self, dim, stride=1): 149 | layer1 = BottleneckBlock(self.in_planes, dim, self.norm_fn, stride=stride) 150 | layer2 = BottleneckBlock(dim, dim, self.norm_fn, stride=1) 151 | layers = (layer1, layer2) 152 | 153 | self.in_planes = dim 154 | return nn.Sequential(*layers) 155 | 156 | 157 | def forward(self, ldrs, expos): 158 | prev, cur, nxt = ldrs 159 | p_exp, c_exp, n_exp = expos 160 | cur_adj_exp = adj_expo_ldr_to_ldr(cur, c_exp, p_exp) 161 | 162 | p4 = F.avg_pool2d(prev, 4) 163 | c4 = F.avg_pool2d(cur_adj_exp, 4) 164 | n4 = F.avg_pool2d(nxt, 4) 165 | 166 | p8 = F.avg_pool2d(prev, 8) 167 | c8 = F.avg_pool2d(cur_adj_exp, 8) 168 | n8 = F.avg_pool2d(nxt, 8) 169 | 170 | p16 = F.avg_pool2d(prev, 16) 171 | c16 = F.avg_pool2d(cur_adj_exp, 16) 172 | n16 = F.avg_pool2d(nxt, 16) 173 | 174 | x = torch.cat((prev, cur_adj_exp, nxt), dim=1) 175 | 176 | x2 = self.conv1(x) 177 | x2 = self.layer1(x2) 178 | x4 = self.layer2(x2) 179 | x4_ = self.imconv4(torch.cat((p4, c4, n4), dim=1)) 180 | x4 = self.combine4(torch.cat((x4, x4_), dim=1)) 181 | 182 | x8 = self.layer3(x4) 183 | x8_ = self.imconv8(torch.cat((p8, c8, n8), dim=1)) 184 | x8 = self.combine8(torch.cat((x8, x8_), dim=1)) 185 | 186 | x16 = self.layer4(x8) 187 | x16_ = self.imconv16(torch.cat((p16, c16, n16), dim=1)) 188 | x16 = self.combine16(torch.cat((x16, x16_), dim=1)) 189 | 190 | x16_7x7 = self.conv7x7(x16) 191 | x16_9x9 = self.conv9x9(x16) 192 | x16_11x11 = self.conv11x11(x16) 193 | lk_features = torch.cat((x16_7x7, x16_9x9, x16_11x11), dim=1) 194 | lk_features = self.merge_features(lk_features) 195 | x16 = F.relu(x16+lk_features) 196 | 197 | x16_up = self.upconv4(x16) 198 | x8 = self.iconv3(torch.cat((x8, x16_up), dim=1)) 199 | 200 | x8_up = self.upconv3(x8) 201 | x4 = self.iconv2(torch.cat((x4, x8_up), dim=1)) 202 | flow4 = self.flow_head(x4) 203 | 204 | p_flow4 = flow4[:,0:2].clamp(-100, 100) 205 | n_flow4 = flow4[:,2:].clamp(-100, 100) 206 | 207 | p_flow = F.interpolate(p_flow4, scale_factor=4, mode='bilinear', align_corners=True) * 4.0 208 | n_flow = F.interpolate(n_flow4, scale_factor=4, mode='bilinear', align_corners=True) * 4.0 209 | 210 | return [p_flow, n_flow] -------------------------------------------------------------------------------- /models/flow_3E.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .network_utils import * 4 | import torch.nn.functional as F 5 | 6 | class BottleneckBlock(nn.Module): 7 | def __init__(self, in_planes, planes, norm_fn='group', stride=1): 8 | super(BottleneckBlock, self).__init__() 9 | 10 | self.conv1 = nn.Conv2d(in_planes, planes//4, kernel_size=1, padding=0) 11 | self.conv2 = nn.Conv2d(planes//4, planes//4, kernel_size=3, padding=1, stride=stride) 12 | self.conv3 = nn.Conv2d(planes//4, planes, kernel_size=1, padding=0) 13 | self.relu = nn.ReLU(inplace=True) 14 | 15 | num_groups = planes // 8 16 | 17 | if norm_fn == 'group': 18 | self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=planes//4) 19 | self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=planes//4) 20 | self.norm3 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 21 | if not stride == 1: 22 | self.norm4 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 23 | 24 | elif norm_fn == 'batch': 25 | self.norm1 = nn.BatchNorm2d(planes//4) 26 | self.norm2 = nn.BatchNorm2d(planes//4) 27 | self.norm3 = nn.BatchNorm2d(planes) 28 | if not stride == 1: 29 | self.norm4 = nn.BatchNorm2d(planes) 30 | 31 | elif norm_fn == 'instance': 32 | self.norm1 = nn.InstanceNorm2d(planes//4) 33 | self.norm2 = nn.InstanceNorm2d(planes//4) 34 | self.norm3 = nn.InstanceNorm2d(planes) 35 | if not stride == 1: 36 | self.norm4 = nn.InstanceNorm2d(planes) 37 | 38 | elif norm_fn == 'none': 39 | self.norm1 = nn.Sequential() 40 | self.norm2 = nn.Sequential() 41 | self.norm3 = nn.Sequential() 42 | if not stride == 1: 43 | self.norm4 = nn.Sequential() 44 | 45 | if stride == 1: 46 | self.downsample = None 47 | 48 | else: 49 | self.downsample = nn.Sequential( 50 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride), self.norm4) 51 | 52 | 53 | def forward(self, x): 54 | y = x 55 | y = self.relu(self.norm1(self.conv1(y))) 56 | y = self.relu(self.norm2(self.conv2(y))) 57 | y = self.relu(self.norm3(self.conv3(y))) 58 | 59 | if self.downsample is not None: 60 | x = self.downsample(x) 61 | 62 | return self.relu(x+y) 63 | 64 | 65 | class Flow_Net(nn.Module): 66 | """ for 2-exposure """ 67 | def __init__(self): 68 | super(Flow_Net, self).__init__() 69 | 70 | self.norm_fn = 'batch' 71 | self.in_planes = 32 72 | self.conv1 = nn.Sequential(nn.Conv2d(18, 32, kernel_size=7, stride=2, padding=3), 73 | nn.BatchNorm2d(32), 74 | nn.ReLU(inplace=True)) 75 | 76 | self.imconv4 = nn.Sequential(nn.Conv2d(18, 64, kernel_size=3, stride=1, padding=1), 77 | nn.BatchNorm2d(64), 78 | nn.ReLU(inplace=True)) 79 | 80 | self.combine4 = nn.Sequential(nn.Conv2d(64*2, 64, kernel_size=1, stride=1, padding=0), 81 | nn.BatchNorm2d(64), 82 | nn.ReLU(inplace=True), 83 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), 84 | nn.BatchNorm2d(64), 85 | nn.ReLU(inplace=True)) 86 | 87 | self.imconv8 = nn.Sequential(nn.Conv2d(18, 128, kernel_size=3, stride=1, padding=1), 88 | nn.BatchNorm2d(128), 89 | nn.ReLU(inplace=True)) 90 | 91 | self.combine8 = nn.Sequential(nn.Conv2d(128*2, 128, kernel_size=1, stride=1, padding=0), 92 | nn.BatchNorm2d(128), 93 | nn.ReLU(inplace=True), 94 | nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), 95 | nn.BatchNorm2d(128), 96 | nn.ReLU(inplace=True)) 97 | 98 | self.imconv16 = nn.Sequential(nn.Conv2d(18, 256, kernel_size=3, stride=1, padding=1), 99 | nn.BatchNorm2d(256), 100 | nn.ReLU(inplace=True)) 101 | 102 | self.combine16 = nn.Sequential(nn.Conv2d(256*2, 256, kernel_size=1, stride=1, padding=0), 103 | nn.BatchNorm2d(256), 104 | nn.ReLU(inplace=True), 105 | nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1), 106 | nn.BatchNorm2d(256), 107 | nn.ReLU(inplace=True)) 108 | 109 | self.layer1 = self._make_layer(32, stride=1) # 1/2 110 | self.layer2 = self._make_layer(64, stride=2) # 1/4 111 | self.layer3 = self._make_layer(128, stride=2) # 1/8 112 | self.layer4 = self._make_layer(256, stride=2) # 1/16 113 | 114 | self.upconv4 = nn.Sequential(nn.ConvTranspose2d(256, 128, kernel_size=4, stride=2, padding=1), 115 | nn.BatchNorm2d(128), 116 | nn.ReLU(inplace=True)) 117 | 118 | self.iconv3 = nn.Sequential(nn.Conv2d(128*2, 128, kernel_size=1, stride=1, padding=0), 119 | nn.BatchNorm2d(128), 120 | nn.ReLU(inplace=True), 121 | nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), 122 | nn.BatchNorm2d(128), 123 | nn.ReLU(inplace=True)) 124 | 125 | self.upconv3 = nn.Sequential(nn.ConvTranspose2d(128, 64, kernel_size=4, stride=2, padding=1), 126 | nn.BatchNorm2d(64), 127 | nn.ReLU(inplace=True)) 128 | 129 | self.iconv2 = nn.Sequential(nn.Conv2d(64*2, 64, kernel_size=1, stride=1, padding=0), 130 | nn.BatchNorm2d(64), 131 | nn.ReLU(inplace=True), 132 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), 133 | nn.BatchNorm2d(64), 134 | nn.ReLU(inplace=True)) 135 | 136 | self.conv7x7 = nn.Conv2d(256, 256, 7, stride=1, padding=7//2, groups=256) 137 | self.conv9x9 = nn.Conv2d(256, 256, 9, stride=1, padding=9//2, groups=256) 138 | self.conv11x11 = nn.Conv2d(256, 256, 11, stride=1, padding=11//2, groups=256) 139 | 140 | self.merge_features = nn.Conv2d(256*3, 256, 1, stride=1, padding=0) 141 | 142 | self.flow_head = nn.Sequential(nn.Conv2d(64, 128, kernel_size=5, stride=1, padding=2), 143 | nn.ReLU(inplace=True), 144 | nn.Conv2d(128, 128, kernel_size=5, stride=1, padding=2), 145 | nn.ReLU(inplace=True), 146 | nn.Conv2d(128, 8, kernel_size=5, stride=1, padding=2)) 147 | 148 | def _make_layer(self, dim, stride=1): 149 | layer1 = BottleneckBlock(self.in_planes, dim, self.norm_fn, stride=stride) 150 | layer2 = BottleneckBlock(dim, dim, self.norm_fn, stride=1) 151 | layers = (layer1, layer2) 152 | 153 | self.in_planes = dim 154 | return nn.Sequential(*layers) 155 | 156 | 157 | def forward(self, ldrs, expos): 158 | 159 | p2, p1, c, n1, n2 = ldrs 160 | p2_exp, p1_exp, c_exp, n1_exp, n2_exp = expos 161 | c_adj_p2 = adj_expo_ldr_to_ldr(c, c_exp, p2_exp) 162 | c_adj_p1 = adj_expo_ldr_to_ldr(c, c_exp, p1_exp) 163 | 164 | p2_4 = F.avg_pool2d(p2, 4) 165 | c_adj_p2_4 = F.avg_pool2d(c_adj_p2, 4) 166 | n1_4 = F.avg_pool2d(n1, 4) 167 | 168 | p1_4 = F.avg_pool2d(p1, 4) 169 | c_adj_p1_4 = F.avg_pool2d(c_adj_p1, 4) 170 | n2_4 = F.avg_pool2d(n2, 4) 171 | 172 | p2_8 = F.avg_pool2d(p2, 8) 173 | c_adj_p2_8 = F.avg_pool2d(c_adj_p2, 8) 174 | n1_8 = F.avg_pool2d(n1, 8) 175 | 176 | p1_8 = F.avg_pool2d(p1, 8) 177 | c_adj_p1_8 = F.avg_pool2d(c_adj_p1, 8) 178 | n2_8 = F.avg_pool2d(n2, 8) 179 | 180 | p2_16 = F.avg_pool2d(p2, 16) 181 | c_adj_p2_16 = F.avg_pool2d(c_adj_p2, 16) 182 | n1_16 = F.avg_pool2d(n1, 16) 183 | 184 | p1_16 = F.avg_pool2d(p1, 16) 185 | c_adj_p1_16 = F.avg_pool2d(c_adj_p1, 16) 186 | n2_16 = F.avg_pool2d(n2, 16) 187 | 188 | x = torch.cat((p2, c_adj_p2, n1, p1, c_adj_p1, n2), dim=1) 189 | 190 | x2 = self.conv1(x) 191 | x2 = self.layer1(x2) 192 | x4 = self.layer2(x2) 193 | x4_ = self.imconv4(torch.cat((p2_4, c_adj_p2_4, n1_4, p1_4, c_adj_p1_4, n2_4), dim=1)) 194 | x4 = self.combine4(torch.cat((x4, x4_), dim=1)) 195 | 196 | x8 = self.layer3(x4) 197 | x8_ = self.imconv8(torch.cat((p2_8, c_adj_p2_8, n1_8, p1_8, c_adj_p1_8, n2_8), dim=1)) 198 | x8 = self.combine8(torch.cat((x8, x8_), dim=1)) 199 | 200 | x16 = self.layer4(x8) 201 | x16_ = self.imconv16(torch.cat((p2_16, c_adj_p2_16, n1_16, p1_16, c_adj_p1_16, n2_16), dim=1)) 202 | x16 = self.combine16(torch.cat((x16, x16_), dim=1)) 203 | # x16 = self.conv2(x16) 204 | 205 | x16_7x7 = self.conv7x7(x16) 206 | x16_9x9 = self.conv9x9(x16) 207 | x16_11x11 = self.conv11x11(x16) 208 | lk_features = torch.cat((x16_7x7, x16_9x9, x16_11x11), dim=1) 209 | lk_features = self.merge_features(lk_features) 210 | x16 = F.relu(x16+lk_features) 211 | 212 | x16_up = self.upconv4(x16) 213 | x8 = self.iconv3(torch.cat((x8, x16_up), dim=1)) 214 | 215 | x8_up = self.upconv3(x8) 216 | x4 = self.iconv2(torch.cat((x4, x8_up), dim=1)) 217 | flow4 = self.flow_head(x4) 218 | 219 | p2_flow4 = flow4[:,:2].clamp(-64, 64) 220 | n1_flow4 = flow4[:,2:4].clamp(-64, 64) 221 | 222 | p1_flow4 = flow4[:,4:6].clamp(-64, 64) 223 | n2_flow4 = flow4[:,6:].clamp(-64, 64) 224 | 225 | p2_flow = F.interpolate(p2_flow4, scale_factor=4, mode='bilinear', align_corners=True) * 4.0 226 | n1_flow = F.interpolate(n1_flow4, scale_factor=4, mode='bilinear', align_corners=True) * 4.0 227 | 228 | p1_flow = F.interpolate(p1_flow4, scale_factor=4, mode='bilinear', align_corners=True) * 4.0 229 | n2_flow = F.interpolate(n2_flow4, scale_factor=4, mode='bilinear', align_corners=True) * 4.0 230 | 231 | return [p2_flow, n1_flow, p1_flow, n2_flow] -------------------------------------------------------------------------------- /models/fusion_2E.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .network_utils import * 4 | 5 | class Downsample(nn.Module): 6 | def __init__(self, c_in, c_out=512, use_bn=True, afunc='LReLU'): 7 | super(Downsample, self).__init__() 8 | 9 | self.layers = torch.nn.Sequential( 10 | conv_layer(c_in, c_out//4, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn), 11 | conv_layer(c_out//4, c_out//4, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 12 | conv_layer(c_out//4, c_out//2, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn), 13 | conv_layer(c_out//2, c_out//2, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 14 | conv_layer(c_out//2, c_out, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn), 15 | conv_layer(c_out, c_out, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 16 | ) 17 | 18 | def forward(self, inputs): 19 | out = self.layers(inputs) 20 | return out 21 | 22 | class Upsample(nn.Module): 23 | def __init__(self, c_in, c_out, use_bn=True, afunc='LReLU'): 24 | super(Upsample, self).__init__() 25 | last_c = max(128, c_in // 8) 26 | self.layers = torch.nn.Sequential( 27 | deconv_layer(c_in, c_in//2, use_bn=use_bn, afunc=afunc), 28 | conv_layer(c_in//2, c_in//2, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 29 | deconv_layer(c_in//2, c_in//2, use_bn=use_bn, afunc=afunc), 30 | conv_layer(c_in//2, c_in//2, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 31 | deconv_layer(c_in//2, last_c, use_bn=use_bn, afunc=afunc), 32 | output_conv(last_c, c_out, k=3, stride=1, pad=1), 33 | ) 34 | 35 | def forward(self, inputs): 36 | out = self.layers(inputs) 37 | out = torch.sigmoid(out) 38 | return out 39 | 40 | class Fusion_Net(nn.Module): 41 | def __init__(self, c_in=30, c_out=15, c_mid=256): 42 | super(Fusion_Net, self).__init__() 43 | self.conv1 = torch.nn.Sequential( 44 | nn.Conv2d(c_in, c_mid//4, kernel_size=3, stride=2, padding=1), 45 | nn.BatchNorm2d(c_mid//4), 46 | nn.LeakyReLU(0.1, inplace=True)) 47 | self.conv2 = torch.nn.Sequential( 48 | nn.Conv2d(c_mid//4, c_mid//4, kernel_size=3, stride=1, padding=1), 49 | nn.BatchNorm2d(c_mid//4), 50 | nn.LeakyReLU(0.1, inplace=True)) 51 | self.conv3 = torch.nn.Sequential( 52 | nn.Conv2d(c_mid//4, c_mid//2, kernel_size=3, stride=2, padding=1), 53 | nn.BatchNorm2d(c_mid//2), 54 | nn.LeakyReLU(0.1, inplace=True)) 55 | self.conv4 = torch.nn.Sequential( 56 | nn.Conv2d(c_mid//2, c_mid//2, kernel_size=3, stride=1, padding=1), 57 | nn.BatchNorm2d(c_mid//2), 58 | nn.LeakyReLU(0.1, inplace=True)) 59 | 60 | self.conv5 = torch.nn.Sequential( 61 | nn.Conv2d(c_mid//2, c_mid, kernel_size=3, stride=2, padding=1), 62 | nn.BatchNorm2d(c_mid), 63 | nn.LeakyReLU(0.1, inplace=True)) 64 | self.conv6 = torch.nn.Sequential( 65 | nn.Conv2d(c_mid, c_mid, kernel_size=3, stride=1, padding=1), 66 | nn.BatchNorm2d(c_mid), 67 | nn.LeakyReLU(0.1, inplace=True)) 68 | 69 | self.conv7 = torch.nn.Sequential( 70 | nn.ConvTranspose2d(c_mid, c_mid//2, kernel_size=4, stride=2, padding=1), 71 | nn.BatchNorm2d(c_mid//2), 72 | nn.LeakyReLU(0.1, inplace=True)) 73 | self.conv8 = torch.nn.Sequential( 74 | nn.Conv2d(c_mid, c_mid//2, kernel_size=3, stride=1, padding=1), 75 | nn.BatchNorm2d(c_mid//2), 76 | nn.LeakyReLU(0.1, inplace=True)) 77 | self.conv9 = torch.nn.Sequential( 78 | nn.ConvTranspose2d(c_mid//2, c_mid//2, kernel_size=4, stride=2, padding=1), 79 | nn.BatchNorm2d(c_mid//2), 80 | nn.LeakyReLU(0.1, inplace=True)) 81 | self.conv10 = torch.nn.Sequential( 82 | nn.Conv2d(c_mid//2+c_mid//4, c_mid//2, kernel_size=3, stride=1, padding=1), 83 | nn.BatchNorm2d(c_mid//2), 84 | nn.LeakyReLU(0.1, inplace=True)) 85 | 86 | self.conv11 = torch.nn.Sequential( 87 | nn.ConvTranspose2d(c_mid//2, 64, kernel_size=4, stride=2, padding=1), 88 | nn.BatchNorm2d(64), 89 | nn.LeakyReLU(0.1, inplace=True)) 90 | 91 | self.conv12 = torch.nn.Sequential( 92 | nn.Conv2d(64, c_out, kernel_size=3, stride=1, padding=1)) 93 | 94 | 95 | self.merge_HDR = MergeHDRModule() 96 | 97 | def forward(self, inputs, hdrs): 98 | pred = OrderedDict() 99 | n, c, h, w = inputs.shape 100 | ### 1/2 101 | conv1 = self.conv1(inputs) 102 | conv2 = self.conv2(conv1) 103 | 104 | ### 1/4 105 | conv3 = self.conv3(conv2) 106 | conv4 = self.conv4(conv3) 107 | 108 | ### 1/8 109 | conv5 = self.conv5(conv4) 110 | conv6 = self.conv6(conv5) 111 | ### 1/4 112 | conv7 = self.conv7(conv6) 113 | conv8 = self.conv8(torch.cat((conv7, conv4), dim=1)) 114 | ### 1/2 115 | conv9 = self.conv9(conv8) 116 | conv10 = self.conv10(torch.cat((conv9, conv2), dim=1)) 117 | ### output 118 | conv11 = self.conv11(conv10) 119 | conv12 = self.conv12(conv11) 120 | weights = torch.sigmoid(conv12) 121 | 122 | ws = torch.split(weights, 1, 1) 123 | hdr, weights = self.merge_HDR(ws, hdrs) 124 | return hdr -------------------------------------------------------------------------------- /models/fusion_3E.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .network_utils import * 4 | class Downsample(nn.Module): 5 | def __init__(self, c_in, c_out=512, use_bn=True, afunc='LReLU'): 6 | super(Downsample, self).__init__() 7 | 8 | self.layers = torch.nn.Sequential( 9 | conv_layer(c_in, c_out//4, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn), 10 | conv_layer(c_out//4, c_out//4, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 11 | conv_layer(c_out//4, c_out//2, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn), 12 | conv_layer(c_out//2, c_out//2, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 13 | conv_layer(c_out//2, c_out, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn), 14 | conv_layer(c_out, c_out, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 15 | ) 16 | 17 | def forward(self, inputs): 18 | out = self.layers(inputs) 19 | return out 20 | 21 | class Upsample(nn.Module): 22 | def __init__(self, c_in, c_out, use_bn=True, afunc='LReLU'): 23 | super(Upsample, self).__init__() 24 | last_c = max(128, c_in // 8) 25 | self.layers = torch.nn.Sequential( 26 | deconv_layer(c_in, c_in//2, use_bn=use_bn, afunc=afunc), 27 | conv_layer(c_in//2, c_in//2, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 28 | deconv_layer(c_in//2, c_in//2, use_bn=use_bn, afunc=afunc), 29 | conv_layer(c_in//2, c_in//2, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn), 30 | deconv_layer(c_in//2, last_c, use_bn=use_bn, afunc=afunc), 31 | output_conv(last_c, c_out, k=3, stride=1, pad=1), 32 | ) 33 | 34 | def forward(self, inputs): 35 | out = self.layers(inputs) 36 | out = torch.sigmoid(out) 37 | return out 38 | 39 | class Fusion_Net(nn.Module): 40 | def __init__(self, c_in=30, c_out=15, c_mid=256): 41 | super(Fusion_Net, self).__init__() 42 | 43 | self.conv1 = torch.nn.Sequential( 44 | nn.Conv2d(c_in, c_mid//4, kernel_size=3, stride=2, padding=1), 45 | nn.BatchNorm2d(c_mid//4), 46 | nn.LeakyReLU(0.1, inplace=True)) 47 | self.conv2 = torch.nn.Sequential( 48 | nn.Conv2d(c_mid//4, c_mid//4, kernel_size=3, stride=1, padding=1), 49 | nn.BatchNorm2d(c_mid//4), 50 | nn.LeakyReLU(0.1, inplace=True)) 51 | self.conv3 = torch.nn.Sequential( 52 | nn.Conv2d(c_mid//4, c_mid//2, kernel_size=3, stride=2, padding=1), 53 | nn.BatchNorm2d(c_mid//2), 54 | nn.LeakyReLU(0.1, inplace=True)) 55 | self.conv4 = torch.nn.Sequential( 56 | nn.Conv2d(c_mid//2, c_mid//2, kernel_size=3, stride=1, padding=1), 57 | nn.BatchNorm2d(c_mid//2), 58 | nn.LeakyReLU(0.1, inplace=True)) 59 | 60 | self.conv5 = torch.nn.Sequential( 61 | nn.Conv2d(c_mid//2, c_mid, kernel_size=3, stride=2, padding=1), 62 | nn.BatchNorm2d(c_mid), 63 | nn.LeakyReLU(0.1, inplace=True)) 64 | self.conv6 = torch.nn.Sequential( 65 | nn.Conv2d(c_mid, c_mid, kernel_size=3, stride=1, padding=1), 66 | nn.BatchNorm2d(c_mid), 67 | nn.LeakyReLU(0.1, inplace=True)) 68 | 69 | self.conv7 = torch.nn.Sequential( 70 | nn.ConvTranspose2d(c_mid, c_mid//2, kernel_size=4, stride=2, padding=1), 71 | nn.BatchNorm2d(c_mid//2), 72 | nn.LeakyReLU(0.1, inplace=True)) 73 | self.conv8 = torch.nn.Sequential( 74 | nn.Conv2d(c_mid, c_mid//2, kernel_size=3, stride=1, padding=1), 75 | nn.BatchNorm2d(c_mid//2), 76 | nn.LeakyReLU(0.1, inplace=True)) 77 | self.conv9 = torch.nn.Sequential( 78 | nn.ConvTranspose2d(c_mid//2, c_mid//4, kernel_size=4, stride=2, padding=1), 79 | nn.BatchNorm2d(c_mid//4), 80 | nn.LeakyReLU(0.1, inplace=True)) 81 | self.conv10 = torch.nn.Sequential( 82 | nn.Conv2d(c_mid//2, c_mid//4, kernel_size=3, stride=1, padding=1), 83 | nn.BatchNorm2d(c_mid//4), 84 | nn.LeakyReLU(0.1, inplace=True)) 85 | 86 | self.conv11 = torch.nn.Sequential( 87 | nn.ConvTranspose2d(c_mid//4, 64, kernel_size=4, stride=2, padding=1), 88 | nn.BatchNorm2d(64), 89 | nn.LeakyReLU(0.1, inplace=True)) 90 | 91 | self.conv12 = torch.nn.Sequential( 92 | nn.Conv2d(64, c_out, kernel_size=3, stride=1, padding=1)) 93 | 94 | 95 | self.merge_HDR = MergeHDRModule() 96 | 97 | def forward(self, inputs, hdrs): 98 | pred = OrderedDict() 99 | n, c, h, w = inputs.shape 100 | ### 1/2 101 | conv1 = self.conv1(inputs) 102 | conv2 = self.conv2(conv1) 103 | 104 | ### 1/4 105 | conv3 = self.conv3(conv2) 106 | conv4 = self.conv4(conv3) 107 | 108 | ### 1/8 109 | conv5 = self.conv5(conv4) 110 | conv6 = self.conv6(conv5) 111 | ### 1/4 112 | conv7 = self.conv7(conv6) 113 | conv8 = self.conv8(torch.cat((conv7, conv4), dim=1)) 114 | ### 1/2 115 | conv9 = self.conv9(conv8) 116 | conv10 = self.conv10(torch.cat((conv9, conv2), dim=1)) 117 | ### output 118 | conv11 = self.conv11(conv10) 119 | conv12 = self.conv12(conv11) 120 | weights = torch.sigmoid(conv12) 121 | 122 | ws = torch.split(weights, 1, 1) 123 | hdr, weights = self.merge_HDR(ws, hdrs) 124 | return hdr -------------------------------------------------------------------------------- /models/loss.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torchvision 5 | import numpy as np 6 | import torch.nn.functional as F 7 | from .network_utils import backward_warp 8 | 9 | def tonemap(hdr_img, mu=5000): 10 | return (torch.log(1 + mu * hdr_img)) / math.log(1 + mu) 11 | 12 | class HDRFlow_Loss_2E(nn.Module): 13 | def __init__(self, mu=5000): 14 | super(HDRFlow_Loss_2E, self).__init__() 15 | self.mu = mu 16 | 17 | def forward(self, pred, hdrs, flow_preds, cur_ldr, flow_mask, flow_gts): 18 | gt = hdrs[1] 19 | mu_pred = tonemap(pred, self.mu) 20 | mu_gt = tonemap(gt, self.mu) 21 | recon_loss = nn.L1Loss()(mu_pred, mu_gt) 22 | loss = recon_loss 23 | 24 | Y = 0.299 * cur_ldr[:, 0] + 0.587 * cur_ldr[:, 1] + 0.114 * cur_ldr[:, 2] 25 | Y = Y[:, None] 26 | mask = (Y>0.8) | (Y<0.2) 27 | mask = mask.repeat(1, 3, 1, 1) 28 | p_flow, n_flow = flow_preds 29 | if mask.sum() > 0: 30 | p_warp_hdr = backward_warp(hdrs[0], p_flow) 31 | n_warp_hdr = backward_warp(hdrs[2], n_flow) 32 | 33 | mu_p_warp_hdr = tonemap(p_warp_hdr, self.mu) 34 | mu_n_warp_hdr = tonemap(n_warp_hdr, self.mu) 35 | 36 | p_align_loss = nn.L1Loss()(mu_p_warp_hdr[mask], mu_gt[mask]) 37 | n_align_loss = nn.L1Loss()(mu_n_warp_hdr[mask], mu_gt[mask]) 38 | 39 | hdr_align_loss = p_align_loss + n_align_loss 40 | loss += 0.5 * hdr_align_loss 41 | 42 | b, c, h, w = p_flow.shape 43 | flow_mask = flow_mask[:, None, None, None].repeat(1, 2, h, w) 44 | flow_mask = flow_mask > 0.5 45 | if flow_mask.sum() > 0: 46 | p_flow_loss = nn.L1Loss()(p_flow[flow_mask], flow_gts[0][flow_mask]) 47 | n_flow_loss = nn.L1Loss()(n_flow[flow_mask], flow_gts[1][flow_mask]) 48 | 49 | flow_loss = p_flow_loss + n_flow_loss 50 | loss += 0.001 * flow_loss 51 | 52 | return loss 53 | 54 | class HDRFlow_Loss_3E(nn.Module): 55 | def __init__(self, mu=5000): 56 | super(HDRFlow_Loss_3E, self).__init__() 57 | self.mu = mu 58 | 59 | def forward(self, pred, hdrs, flow_preds, cur_ldr, flow_mask, flow_gts): 60 | gt = hdrs[2] 61 | mu_pred = tonemap(pred, self.mu) 62 | mu_gt = tonemap(gt, self.mu) 63 | recon_loss = nn.L1Loss()(mu_pred, mu_gt) 64 | loss = recon_loss 65 | 66 | Y = 0.299 * cur_ldr[:, 0] + 0.587 * cur_ldr[:, 1] + 0.114 * cur_ldr[:, 2] 67 | Y = Y[:, None] 68 | mask = (Y>0.8) | (Y<0.2) 69 | mask = mask.repeat(1, 3, 1, 1) 70 | p2_flow, n1_flow, p1_flow, n2_flow = flow_preds 71 | if mask.sum() > 0: 72 | p2_warp_hdr = backward_warp(hdrs[0], p2_flow) 73 | n1_warp_hdr = backward_warp(hdrs[3], n1_flow) 74 | p1_warp_hdr = backward_warp(hdrs[1], p1_flow) 75 | n2_warp_hdr = backward_warp(hdrs[4], n2_flow) 76 | 77 | mu_p2_warp_hdr = tonemap(p2_warp_hdr, self.mu) 78 | mu_n1_warp_hdr = tonemap(n1_warp_hdr, self.mu) 79 | mu_p1_warp_hdr = tonemap(p1_warp_hdr, self.mu) 80 | mu_n2_warp_hdr = tonemap(n2_warp_hdr, self.mu) 81 | 82 | p2_align_loss = nn.L1Loss()(mu_p2_warp_hdr[mask], mu_gt[mask]) 83 | n1_align_loss = nn.L1Loss()(mu_n1_warp_hdr[mask], mu_gt[mask]) 84 | p1_align_loss = nn.L1Loss()(mu_p1_warp_hdr[mask], mu_gt[mask]) 85 | n2_align_loss = nn.L1Loss()(mu_n2_warp_hdr[mask], mu_gt[mask]) 86 | 87 | hdr_align_loss = p2_align_loss + n1_align_loss + p1_align_loss + n2_align_loss 88 | loss += 0.2 * hdr_align_loss 89 | 90 | b, c, h, w = p2_flow.shape 91 | flow_mask = flow_mask[:, None, None, None].repeat(1, 2, h, w) 92 | flow_mask = flow_mask > 0.5 93 | if flow_mask.sum() > 0: 94 | p2_flow_loss = nn.L1Loss()(p2_flow[flow_mask], flow_gts[0][flow_mask]) 95 | n1_flow_loss = nn.L1Loss()(n1_flow[flow_mask], flow_gts[1][flow_mask]) 96 | p1_flow_loss = nn.L1Loss()(p1_flow[flow_mask], flow_gts[2][flow_mask]) 97 | n2_flow_loss = nn.L1Loss()(n2_flow[flow_mask], flow_gts[3][flow_mask]) 98 | 99 | flow_loss = p2_flow_loss + n1_flow_loss + p1_flow_loss + n2_flow_loss 100 | loss += 0.0005 * flow_loss 101 | 102 | return loss 103 | -------------------------------------------------------------------------------- /models/model_2E.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .network_utils import * 4 | from .flow_2E import Flow_Net 5 | from .fusion_2E import Fusion_Net 6 | 7 | def cur_tone_perturb(cur, test_mode, d=0.7): 8 | if not test_mode: 9 | b, c, h, w = cur.shape 10 | gamma_aug = torch.exp(torch.rand(b, 3, 1, 1) * 2 * d - d) 11 | gamma_aug = gamma_aug.to(cur.device) 12 | cur_aug = torch.pow(cur, 1.0 / gamma_aug) 13 | else: 14 | cur_aug = cur 15 | return cur_aug 16 | 17 | def prepare_fusion_inputs(ldrs, pt_cur, expos, flows): 18 | prev, cur, nxt = ldrs 19 | p_exp, c_exp, n_exp = expos 20 | p_flow, n_flow = flows 21 | p_warp = backward_warp(prev, p_flow) 22 | n_warp = backward_warp(nxt, n_flow) 23 | p_warp_hdr = ldr_to_hdr(p_warp, p_exp) 24 | n_warp_hdr = ldr_to_hdr(n_warp, n_exp) 25 | c_hdr = ldr_to_hdr(cur, c_exp) 26 | p_hdr = ldr_to_hdr(prev, p_exp) 27 | n_hdr = ldr_to_hdr(nxt, n_exp) 28 | pt_c_hdr = ldr_to_hdr(pt_cur, c_exp) 29 | ldrs = [pt_cur, p_warp, n_warp, prev, nxt] 30 | hdrs = [pt_c_hdr, p_warp_hdr, n_warp_hdr, p_hdr, n_hdr] 31 | fusion_in = hdrs 32 | fusion_in += ldrs 33 | fusion_in = torch.cat(fusion_in, 1) 34 | fusion_hdrs = [c_hdr, p_warp_hdr, n_warp_hdr, p_hdr, n_hdr] 35 | return fusion_in, fusion_hdrs 36 | 37 | class HDRFlow(nn.Module): 38 | def __init__(self): 39 | super(HDRFlow, self).__init__() 40 | self.flow_net = Flow_Net() 41 | self.fusion_net = Fusion_Net(c_in=30, c_out=5, c_mid=128) 42 | def forward(self, ldrs, expos, test_mode=False): 43 | prev, cur, nxt = ldrs 44 | pt_cur = cur_tone_perturb(cur, test_mode) 45 | # flow net 46 | flow_preds = self.flow_net([prev, pt_cur, nxt], expos) 47 | fusion_in, fusion_hdrs = prepare_fusion_inputs(ldrs, pt_cur, expos, flow_preds) 48 | # fusion net 49 | pred_hdr = self.fusion_net(fusion_in, fusion_hdrs) 50 | return pred_hdr, flow_preds 51 | -------------------------------------------------------------------------------- /models/model_3E.py: -------------------------------------------------------------------------------- 1 | import math 2 | import time 3 | import torch 4 | import torch.nn as nn 5 | from .network_utils import * 6 | from .flow_3E import Flow_Net 7 | from .fusion_3E import Fusion_Net 8 | 9 | def cur_tone_perturb(cur, test_mode, d=0.7): 10 | if not test_mode: 11 | b, c, h, w = cur.shape 12 | gamma_aug = torch.exp(torch.rand(b, 3, 1, 1) * 2 * d - d) 13 | gamma_aug = gamma_aug.to(cur.device) 14 | cur_aug = torch.pow(cur, 1.0 / gamma_aug) 15 | else: 16 | cur_aug = cur 17 | return cur_aug 18 | 19 | def prepare_fusion_inputs(ldrs, pt_c, expos, flow_preds): 20 | p2, p1, c, n1, n2 = ldrs 21 | p2_exp, p1_exp, c_exp, n1_exp, n2_exp = expos 22 | p2_flow, n1_flow, p1_flow, n2_flow = flow_preds 23 | 24 | p2_warp = backward_warp(p2, p2_flow) 25 | n1_warp = backward_warp(n1, n1_flow) 26 | p1_warp = backward_warp(p1, p1_flow) 27 | n2_warp = backward_warp(n2, n2_flow) 28 | 29 | p2_warp_hdr = ldr_to_hdr(p2_warp, p2_exp) 30 | n1_warp_hdr = ldr_to_hdr(n1_warp, n1_exp) 31 | p1_warp_hdr = ldr_to_hdr(p1_warp, p1_exp) 32 | n2_warp_hdr = ldr_to_hdr(n2_warp, n2_exp) 33 | c_hdr = ldr_to_hdr(c, c_exp) 34 | p2_hdr = ldr_to_hdr(p2, p2_exp) 35 | p1_hdr = ldr_to_hdr(p1, p1_exp) 36 | n1_hdr = ldr_to_hdr(n1, n1_exp) 37 | n2_hdr = ldr_to_hdr(n2, n2_exp) 38 | pt_c_hdr = ldr_to_hdr(pt_c, c_exp) 39 | 40 | fusion_in = [pt_c, pt_c_hdr, p2_warp, p2_warp_hdr, n1_warp, n1_warp_hdr, p1_warp, p1_warp_hdr, n2_warp, n2_warp_hdr, p2, p2_hdr, n1, n1_hdr, p1, p1_hdr, n2, n2_hdr] 41 | fusion_hdrs = [c_hdr, p2_warp_hdr, n1_warp_hdr, p1_warp_hdr, n2_warp_hdr, p2_hdr, n1_hdr, p1_hdr, n2_hdr] 42 | fusion_in = torch.cat(fusion_in, dim=1) 43 | return fusion_in, fusion_hdrs 44 | 45 | class HDRFlow(nn.Module): 46 | def __init__(self): 47 | super(HDRFlow, self).__init__() 48 | self.flow_net = Flow_Net() 49 | self.fusion_net = Fusion_Net(c_in=54, c_out=9, c_mid=256) 50 | 51 | def forward(self, ldrs, expos, test_mode=False): 52 | prev2, prev1, cur, nxt1, nxt2 = ldrs 53 | pt_cur = cur_tone_perturb(cur, test_mode) 54 | # flow net 55 | flow_preds = self.flow_net([prev2, prev1, pt_cur, nxt1, nxt2], expos) 56 | fusion_in, fusion_hdrs = prepare_fusion_inputs(ldrs, pt_cur, expos, flow_preds) 57 | # fusion net 58 | pred_hdr = self.fusion_net(fusion_in, fusion_hdrs) 59 | return pred_hdr, flow_preds 60 | -------------------------------------------------------------------------------- /models/network_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Util functions for network construction 3 | """ 4 | import os 5 | import importlib 6 | import torch 7 | import numpy as np 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | from collections import OrderedDict 11 | 12 | def import_network(net_name, arch_dir='models.archs', backup_module='networks'): 13 | print('%s/%s' % (arch_dir.replace('.', '/'), net_name)) 14 | if os.path.exists('%s/%s.py' % (arch_dir.replace('.', '/'), net_name)): 15 | network_file = importlib.import_module('%s.%s' % (arch_dir, net_name)) 16 | else: 17 | network_file = importlib.import_module('%s.%s' % (arch_dir, backup_module)) 18 | network = getattr(network_file, net_name) 19 | return network 20 | 21 | ## Common Network Blocks 22 | def activation(afunc='LReLU', inplace=True): 23 | if afunc == 'LReLU': 24 | return nn.LeakyReLU(0.1, inplace=inplace) 25 | elif afunc == 'LReLU02': 26 | return nn.LeakyReLU(0.2, inplace=inplace) 27 | elif afunc == 'ReLU': 28 | return nn.ReLU(inplace=inplace) 29 | elif afunc == 'Sigmoid': 30 | return nn.Sigmoid() 31 | elif afunc == 'Tanh': 32 | return nn.Tanh() 33 | else: 34 | raise Exception('Unknown activation function') 35 | 36 | def conv_layer(cin, cout, k=3, stride=1, pad=-1, dilation=1, afunc='LReLU', use_bn=False, bias=True, inplace=True): 37 | if type(pad) != tuple: 38 | pad = pad if pad >= 0 else (k - 1) // 2 39 | block = [nn.Conv2d(cin, cout, kernel_size=k, stride=stride, padding=pad, bias=bias, dilation=dilation)] 40 | if use_bn: 41 | # print('=> convolutional layer with bachnorm') 42 | block += [nn.BatchNorm2d(cout)] 43 | if afunc != '': 44 | block += [activation(afunc, inplace=inplace)] 45 | return nn.Sequential(*block) 46 | 47 | def deconv_layer(cin, cout, afunc='LReLU', use_bn=False, bias=False): 48 | block = [nn.ConvTranspose2d(cin, cout, kernel_size=4, stride=2, padding=1, bias=bias)] 49 | if use_bn: 50 | # print('=> deconvolutional layer with bachnorm') 51 | block += [nn.BatchNorm2d(cout)] 52 | block += [activation(afunc)] 53 | return nn.Sequential(*block) 54 | 55 | def upconv_layer(cin, cout, afunc='LReLU', mode='bilinear', use_bn=False, bias=True): 56 | if mode == 'bilinear': 57 | block = [nn.Upsample(scale_factor=2, mode=mode, align_corners=True)] 58 | elif mode == 'nearest': 59 | block = [nn.Upsample(scale_factor=2, mode=mode)] 60 | else: 61 | raise Exception('Unknonw mode: %s' % mode) 62 | block += [nn.Conv2d(cin, cout, kernel_size=3, stride=1, padding=1, bias=bias)] 63 | if use_bn: 64 | # print('=> deconvolutional layer with bachnorm') 65 | block += [nn.BatchNorm2d(cout)] 66 | block += [activation(afunc)] 67 | return nn.Sequential(*block) 68 | 69 | def up_nearest_layer(cin, cout, afunc='LReLU', use_bn=False, bias=True): 70 | block = [nn.Upsample(scale_factor=2, mode='nearest'), 71 | nn.Conv2d(cin, cout, kernel_size=3, stride=1, padding=1, bias=bias)] 72 | if use_bn: 73 | # print('=> deconvolutional layer with bachnorm') 74 | block += [nn.BatchNorm2d(cout)] 75 | block += [activation(afunc)] 76 | return nn.Sequential(*block) 77 | 78 | def output_conv(cin, cout, k=1, stride=1, pad=0, bias=True): 79 | pad = (k - 1) // 2 80 | return nn.Sequential( 81 | nn.Conv2d(cin, cout, kernel_size=k, stride=stride, padding=pad, bias=bias) 82 | ) 83 | 84 | def fc_layer(cin, cout): 85 | return nn.Sequential( 86 | nn.Linear(cin, cout), 87 | nn.LeakyReLU(0.1, inplace=True)) 88 | 89 | def down_block(cin, cout, k=3, layers=2, down='conv', afunc='LReLU', use_bn=True): 90 | block = [] 91 | if down == 'conv': 92 | block += [conv_layer(cin, cout, k=3, stride=2, pad=1, afunc=afunc, use_bn=use_bn)] 93 | elif down == 'max': 94 | block += [conv_layer(cin, cout, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn)] 95 | block += [torch.nn.MaxPool2d(2)] 96 | elif down == 'avg': 97 | block += [conv_layer(cin, cout, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn)] 98 | block += [torch.nn.AvgPool2d(2)] 99 | else: 100 | raise Exception('Unknown downsample mode %s' % (down)) 101 | for i in range(1, layers): 102 | block += [conv_layer(cout, cout, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn)] 103 | return nn.Sequential(*block) 104 | 105 | def up_block(cin, cout, layers=2, up='bilinear', afunc='LReLU', use_bn=True): 106 | block = [] 107 | if up == 'deconv': 108 | block += [deconv_layer(cin, cout, afunc=afunc, use_bn=use_bn)] 109 | elif up == 'bilinear': 110 | block += [upconv_layer(cin, cout, afunc=afunc, mode='bilinear', use_bn=use_bn)] 111 | elif up == 'nearest': 112 | block += [upconv_layer(cin, cout, afunc=afunc, mode='nearest', use_bn=use_bn)] 113 | else: 114 | raise Exception('Unknown upsample mode: %s' % up) 115 | for i in range(1, layers): 116 | block += [conv_layer(cout, cout, k=3, stride=1, pad=1, afunc=afunc, use_bn=use_bn)] 117 | return nn.Sequential(*block) 118 | 119 | # For ResNet 120 | def make_layer(block, cin, channel, layer_num, use_bn=True): # TODO: clean 121 | layers = [] 122 | for i in range(0, layer_num): 123 | layers.append(block(cin, channel, stride=1, downsample=None, use_bn=use_bn)) 124 | return nn.Sequential(*layers) 125 | 126 | def conv3x3(in_planes, out_planes, stride=1): 127 | "3x3 convolution with padding" 128 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) 129 | 130 | class BasicBlock(nn.Module): 131 | expansion = 1 132 | def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None, use_bn=True): 133 | super(BasicBlock, self).__init__() 134 | self.use_bn = use_bn 135 | self.conv1 = conv3x3(inplanes, planes, stride) 136 | if use_bn: 137 | self.bn1 = nn.BatchNorm2d(planes) 138 | self.relu = nn.ReLU(inplace=True) 139 | self.conv2 = conv3x3(planes, planes) 140 | if use_bn: 141 | self.bn2 = nn.BatchNorm2d(planes) 142 | self.downsample = downsample 143 | self.stride = stride 144 | 145 | def forward(self, x): 146 | residual = x 147 | 148 | out = self.conv1(x) 149 | if self.use_bn: 150 | out = self.bn1(out) 151 | out = self.relu(out) 152 | 153 | out = self.conv2(out) 154 | if self.use_bn: 155 | out = self.bn2(out) 156 | if self.downsample is not None: 157 | residual = self.downsample(x) 158 | out += residual 159 | out = self.relu(out) 160 | return out 161 | 162 | def merge_feats(ws, feats): 163 | assert(len(ws) == len(feats)) 164 | w_sum = torch.stack(ws, 1).sum(1) 165 | ws = [w / (w_sum + 1e-8) for w in ws] 166 | 167 | feat = ws[0] * feats[0] 168 | for i in range(1, len(ws)): 169 | feat += ws[i] * feats[i] 170 | return feat, ws 171 | 172 | ## HDR output processing 173 | def merge_hdr(ws, hdrs): 174 | assert(len(ws) == len(hdrs)) 175 | w_sum = torch.stack(ws, 1).sum(1) 176 | ws = [w / (w_sum + 1e-8) for w in ws] 177 | 178 | hdr = ws[0] * hdrs[0] 179 | for i in range(1, len(ws)): 180 | hdr += ws[i] * hdrs[i] 181 | return hdr, ws 182 | 183 | class MergeHDRModule(nn.Module): 184 | def __init__(self): 185 | super(MergeHDRModule, self).__init__() 186 | 187 | def forward(self, ws, hdrs): 188 | assert(len(ws) == len(hdrs)) 189 | w_sum = torch.stack(ws, 1).sum(1) 190 | ws = [w / (w_sum + 1e-8) for w in ws] 191 | 192 | hdr = ws[0] * hdrs[0] 193 | for i in range(1, len(ws)): 194 | hdr += ws[i] * hdrs[i] 195 | return hdr, ws 196 | 197 | def coords_grid(b, h, w, device): 198 | coords = torch.meshgrid(torch.arange(h, device=device), torch.arange(w, device=device)) 199 | coords = torch.stack(coords[::-1], dim=0).float() 200 | return coords[None].repeat(b, 1, 1, 1) 201 | 202 | def backward_warp(img, flow, pad='zeros'): 203 | b, c, h, w = img.shape 204 | grid = coords_grid(b, h, w, device=img.device) 205 | grid = grid + flow 206 | xgrid, ygrid = grid.split([1,1], dim=1) 207 | xgrid = 2*xgrid/(w-1) - 1 208 | ygrid = 2*ygrid/(h-1) - 1 209 | grid = torch.cat([xgrid, ygrid], dim=1) 210 | warped_img = F.grid_sample(input=img, grid=grid.permute(0,2,3,1), mode='bilinear', padding_mode='zeros') 211 | return warped_img 212 | 213 | def affine_warp(img, theta): # warp img1 to img2 214 | n, c, h, w = img.shape 215 | affine_grid = F.affine_grid(theta, img.shape) 216 | invalid_mask = ((affine_grid.narrow(3, 0, 1).abs() > 1) + (affine_grid.narrow(3, 1, 1).abs() > 1)) >= 1 217 | invalid_mask = invalid_mask.view(n, 1, h, w).float() 218 | img1_to_img2 = F.grid_sample(img, affine_grid) 219 | img1_to_img2 = img * invalid_mask + img1_to_img2 * (1 - invalid_mask) 220 | 221 | return img1_to_img2 222 | 223 | def ldr_to_hdr(ldr, expo, gamma=2.2): 224 | ldr = ldr.clamp(0, 1) 225 | ldr = torch.pow(ldr, gamma) 226 | expo = expo.view(-1, 1, 1, 1) 227 | hdr = ldr / expo 228 | return hdr 229 | 230 | def adj_expo_ldr_to_ldr(ldr, c_exp, p_exp, gamma=2.2): 231 | gain = torch.pow(p_exp / c_exp, 1.0 / gamma) 232 | gain = gain.view(-1, 1, 1, 1) 233 | ldr = (ldr * gain).clamp(0, 1) 234 | return ldr 235 | 236 | def resize_flow(flow, h, w): 237 | b, c, c_h, c_w = flow.shape 238 | flow = F.interpolate(input=flow, size=(h, w), mode='bilinear', align_corners=False) 239 | flow[:, 0] *= float(w) / float(c_w) # rescale flow magnitude after rescaling spatial size 240 | flow[:, 1] *= float(h) / float(c_h) 241 | return flow 242 | 243 | def resize_img_to_factor_of_k(img, k=64, mode='bilinear'): 244 | b, c, h, w = img.shape 245 | new_h = int(np.ceil(h / float(k)) * k) 246 | new_w = int(np.ceil(w / float(k)) * k) 247 | img = F.interpolate(input=img, size=(new_h, new_w), mode=mode, align_corners=False) 248 | return img 249 | 250 | def pad_img_to_factor_of_k(img, k=64, mode='replicate'): 251 | if img.ndimension() == 4: 252 | b, c, h, w = img.shape 253 | pad_h, pad_w = k - h % k, k - w % k 254 | img = F.pad(img, (0, pad_w, 0, pad_h), mode='replicate') 255 | elif img.ndimension() == 5: 256 | h, w = img.shape[-2], img.shape[-1] 257 | pad_h, pad_w = k - h % k, k - w % k 258 | img = F.pad(img, (0, pad_w, 0, pad_h, 0, 0), mode='replicate') 259 | return img 260 | 261 | -------------------------------------------------------------------------------- /pretrained_models/2E/checkpoint.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/pretrained_models/2E/checkpoint.pth -------------------------------------------------------------------------------- /pretrained_models/3E/checkpoint.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/pretrained_models/3E/checkpoint.pth -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | imageio 3 | opencv-python 4 | scikit-image -------------------------------------------------------------------------------- /test_2E.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 3 | import os.path as osp 4 | import sys 5 | import time 6 | import argparse 7 | from tqdm import tqdm 8 | import glob 9 | import logging 10 | import torch 11 | import torch.nn as nn 12 | from torch.utils.data import DataLoader 13 | from dataset.real_benchmark_dataset import Real_Benchmark_Dataset 14 | from dataset.syn_test_dataset import Syn_Test_Dataset 15 | from models.model_2E import HDRFlow 16 | from utils.utils import * 17 | from skimage.metrics import structural_similarity as ssim 18 | from skimage.metrics import peak_signal_noise_ratio as psnr 19 | from utils import flow_viz 20 | 21 | parser = argparse.ArgumentParser(description="Test Setting") 22 | parser.add_argument("--dataset", type=str, default='DeepHDRVideo', choices=['DeepHDRVideo', 'CinematicVideo'], 23 | help='dataset directory') 24 | parser.add_argument("--dataset_dir", type=str, default='data/dynamic_RGB_data_2exp_release', 25 | help='dataset directory') 26 | # parser.add_argument("--dataset_dir", type=str, default='data/static_RGB_data_2exp_rand_motion_release', 27 | # help='dataset directory') 28 | # parser.add_argument("--dataset_dir", type=str, default='data/HDR_Synthetic_Test_Dataset', 29 | # help='dataset directory') 30 | parser.add_argument('--pretrained_model', type=str, default='./pretrained_models/2E/checkpoint.pth') 31 | parser.add_argument('--save_results', action='store_true', default=True) 32 | parser.add_argument('--save_dir', type=str, default="./output_results/2E") 33 | 34 | def save_flo(flow_preds, i, args): 35 | p_flo, n_flo = flow_preds 36 | p_flo = torch.squeeze(p_flo).permute(1,2,0).cpu().numpy() 37 | p_flo = flow_viz.flow_to_image(p_flo) 38 | n_flo = torch.squeeze(n_flo).permute(1,2,0).cpu().numpy() 39 | n_flo = flow_viz.flow_to_image(n_flo) 40 | 41 | concat_flo = np.concatenate([p_flo, n_flo], axis=1) 42 | dataset_name = args.dataset_dir.split('/')[-1] 43 | save_dir = os.path.join(args.save_dir, dataset_name, 'flow_preds') 44 | os.makedirs(save_dir, exist_ok=True) 45 | flo_path = os.path.join(save_dir, f'{i}_flow.png') 46 | cv2.imwrite(flo_path, concat_flo[:, :, [2,1,0]].astype('uint8')) 47 | 48 | def main(): 49 | # Settings 50 | args = parser.parse_args() 51 | # pretrained_model 52 | print(">>>>>>>>> Start Testing >>>>>>>>>") 53 | print("Load weights from: ", args.pretrained_model) 54 | device = torch.device('cuda') 55 | model = HDRFlow() 56 | model.to(device) 57 | model = nn.DataParallel(model) 58 | model.load_state_dict(torch.load(args.pretrained_model)['state_dict']) 59 | model.eval() 60 | 61 | # test_loader = fetch_benchmarkloader(args) 62 | if args.dataset == 'DeepHDRVideo': 63 | test_dataset = Real_Benchmark_Dataset(root_dir=args.dataset_dir, nframes=3, nexps=2) 64 | elif args.dataset == 'CinematicVideo': 65 | test_dataset = Syn_Test_Dataset(root_dir=args.dataset_dir, nframes=3, nexps=2) 66 | else: 67 | print('Unknown dataset') 68 | 69 | test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) 70 | 71 | 72 | psnrL_list = AverageMeter() 73 | ssimL_list = AverageMeter() 74 | psnrT_list = AverageMeter() 75 | ssimT_list = AverageMeter() 76 | 77 | low_psnrL_list = AverageMeter() 78 | low_psnrT_list = AverageMeter() 79 | high_psnrL_list = AverageMeter() 80 | high_psnrT_list = AverageMeter() 81 | 82 | with torch.no_grad(): 83 | for idx, img_data in enumerate(test_loader): 84 | ldrs = [x.to(device) for x in img_data['ldrs']] 85 | expos = [x.to(device) for x in img_data['expos']] 86 | # hdrs = [x.to(device) for x in img_data['hdrs']] 87 | gt_hdr = img_data['hdr'] 88 | padder = InputPadder(ldrs[0].shape, divis_by=16) 89 | pad_ldrs = padder.pad(ldrs) 90 | pred_hdr, flow_preds = model(pad_ldrs, expos, test_mode=True) 91 | pred_hdr = padder.unpad(pred_hdr) 92 | pred_hdr = torch.squeeze(pred_hdr.detach().cpu()).numpy().astype(np.float32).transpose(1,2,0) 93 | 94 | save_flo(flow_preds, idx+1, args) 95 | cur_ldr = torch.squeeze(ldrs[1].cpu()).numpy().astype(np.float32).transpose(1,2,0) 96 | Y = 0.299 * cur_ldr[:, :, 0] + 0.587 * cur_ldr[:, :, 1] + 0.114 * cur_ldr[:, :, 2] 97 | Y = Y[:, :, None] 98 | if expos[1] <= 1.: 99 | mask = Y < 0.2 100 | else: 101 | mask = Y > 0.8 102 | cur_linear_ldr = ldr_to_hdr(ldrs[1], expos[1]) 103 | cur_linear_ldr = torch.squeeze(cur_linear_ldr.cpu()).numpy().astype(np.float32).transpose(1,2,0) 104 | pred_hdr = (~mask) * cur_linear_ldr + (mask) * pred_hdr 105 | gt_hdr = torch.squeeze(gt_hdr).numpy().astype(np.float32).transpose(1,2,0) 106 | gt_hdr_tm = tonemap(gt_hdr) 107 | pred_hdr_tm = tonemap(pred_hdr) 108 | 109 | psnrL = psnr(gt_hdr, pred_hdr) 110 | ssimL = ssim(gt_hdr, pred_hdr, multichannel=True, channel_axis=2, data_range=gt_hdr.max()-gt_hdr.min()) 111 | 112 | psnrT = psnr(gt_hdr_tm, pred_hdr_tm) 113 | ssimT = ssim(gt_hdr_tm, pred_hdr_tm, multichannel=True, channel_axis=2, data_range=gt_hdr_tm.max()-gt_hdr_tm.min()) 114 | 115 | psnrL_list.update(psnrL) 116 | ssimL_list.update(ssimL) 117 | psnrT_list.update(psnrT) 118 | ssimT_list.update(ssimT) 119 | 120 | if expos[1] <= 1.: 121 | print("Iter: [{}/{}] Low Exposure".format(idx+1, len(test_loader))) 122 | low_psnrL_list.update(psnrL) 123 | low_psnrT_list.update(psnrT) 124 | else: 125 | print("Iter: [{}/{}] High Exposure".format(idx+1, len(test_loader))) 126 | high_psnrL_list.update(psnrL) 127 | high_psnrT_list.update(psnrT) 128 | 129 | print("PSNR_mu: {:.4f} PSNR_l: {:.4f}".format(psnrT, psnrL)) 130 | print("SSIM_mu: {:.4f} SSIM_l: {:.4f}".format(ssimT, ssimL)) 131 | 132 | # save results 133 | if args.save_results: 134 | dataset_name = args.dataset_dir.split('/')[-1] 135 | hdr_output_dir = os.path.join(args.save_dir, dataset_name, 'hdr_output') 136 | if not osp.exists(hdr_output_dir): 137 | os.makedirs(hdr_output_dir) 138 | cv2.imwrite(os.path.join(hdr_output_dir, '{}_pred.png'.format(idx+1)), (pred_hdr_tm*255.)[:,:,[2,1,0]].astype('uint8')) 139 | # save_hdr(os.path.join(args.save_dir, img_data['hdr_path'][0]), pred_hdr) 140 | 141 | print("Low Average PSNRT: {:.4f} PSNRL: {:.4f}".format(low_psnrT_list.avg, low_psnrL_list.avg)) 142 | print("High Average PSNRT: {:.4f} PSNRL: {:.4f}".format(high_psnrT_list.avg, high_psnrL_list.avg)) 143 | print("Average PSNRT: {:.4f} PSNRL: {:.4f}".format(psnrT_list.avg, psnrL_list.avg)) 144 | print("Average SSIMT: {:.4f} SSIML: {:.4f}".format(ssimT_list.avg, ssimL_list.avg)) 145 | print(">>>>>>>>> Finish Testing >>>>>>>>>") 146 | 147 | 148 | if __name__ == '__main__': 149 | main() 150 | -------------------------------------------------------------------------------- /test_3E.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 5 | import time 6 | import argparse 7 | from tqdm import tqdm 8 | import glob 9 | import logging 10 | import torch 11 | import torch.nn as nn 12 | from torch.utils.data import DataLoader 13 | from dataset.real_benchmark_dataset import Real_Benchmark_Dataset 14 | from dataset.syn_test_dataset import Syn_Test_Dataset 15 | from models.model_3E import HDRFlow 16 | from utils.utils import * 17 | from skimage.metrics import structural_similarity as ssim 18 | from skimage.metrics import peak_signal_noise_ratio as psnr 19 | from utils import flow_viz 20 | 21 | parser = argparse.ArgumentParser(description="Test Setting") 22 | parser.add_argument("--dataset", type=str, default='DeepHDRVideo', choices=['DeepHDRVideo', 'CinematicVideo'], 23 | help='dataset directory') 24 | parser.add_argument("--dataset_dir", type=str, default='./data/dynamic_RGB_data_3exp_release', 25 | help='dataset directory') 26 | # parser.add_argument("--dataset_dir", type=str, default='./data/static_RGB_data_3exp_rand_motion_release', 27 | # help='dataset directory') 28 | # parser.add_argument("--dataset_dir", type=str, default='./data/HDR_Synthetic_Test_Dataset', 29 | # help='dataset directory') 30 | parser.add_argument('--pretrained_model', type=str, default='./pretrained_models/3E/checkpoint.pth') 31 | parser.add_argument('--save_results', action='store_true', default=True) 32 | parser.add_argument('--save_dir', type=str, default="./output_results/3E") 33 | 34 | def save_flo(flow_preds, i, args): 35 | p2_flo, n1_flo, p1_flo, n2_flo = flow_preds 36 | p1_flo = torch.squeeze(p1_flo).permute(1,2,0).cpu().numpy() 37 | p1_flo = flow_viz.flow_to_image(p1_flo) 38 | n1_flo = torch.squeeze(n1_flo).permute(1,2,0).cpu().numpy() 39 | n1_flo = flow_viz.flow_to_image(n1_flo) 40 | p2_flo = torch.squeeze(p2_flo).permute(1,2,0).cpu().numpy() 41 | p2_flo = flow_viz.flow_to_image(p2_flo) 42 | n2_flo = torch.squeeze(n2_flo).permute(1,2,0).cpu().numpy() 43 | n2_flo = flow_viz.flow_to_image(n2_flo) 44 | concat_flo = np.concatenate([p2_flo, n1_flo, p1_flo, n2_flo], axis=1) 45 | dataset_name = args.dataset_dir.split('/')[-1] 46 | save_dir = os.path.join(args.save_dir, dataset_name, 'flow_preds') 47 | os.makedirs(save_dir, exist_ok=True) 48 | flo_path = os.path.join(save_dir, f'{i}_flow.png') 49 | cv2.imwrite(flo_path, concat_flo[:, :, [2,1,0]].astype('uint8')) 50 | 51 | def main(): 52 | # Settings 53 | args = parser.parse_args() 54 | # pretrained_model 55 | print(">>>>>>>>> Start Testing >>>>>>>>>") 56 | print("Load weights from: ", args.pretrained_model) 57 | device = torch.device('cuda') 58 | model = HDRFlow() 59 | model.to(device) 60 | model = nn.DataParallel(model) 61 | model.load_state_dict(torch.load(args.pretrained_model)['state_dict']) 62 | model.eval() 63 | 64 | # test_loader = fetch_benchmarkloader(args) 65 | if args.dataset == 'DeepHDRVideo': 66 | test_dataset = Real_Benchmark_Dataset(root_dir=args.dataset_dir, nframes=5, nexps=3) 67 | elif args.dataset == 'CinematicVideo': 68 | test_dataset = Syn_Test_Dataset(root_dir=args.dataset_dir, nframes=5, nexps=3) 69 | # elif args.dataset == 'HDRVideo_TOG13': 70 | # test_dataset = TOG13_online_align_Dataset(root_dir=args.dataset_dir, nframes=3, nexps=2) 71 | else: 72 | print('Unknown dataset') 73 | test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) 74 | 75 | psnr_l = AverageMeter() 76 | ssim_l = AverageMeter() 77 | psnr_mu = AverageMeter() 78 | ssim_mu = AverageMeter() 79 | 80 | low_psnr_l = AverageMeter() 81 | low_psnr_mu = AverageMeter() 82 | mid_psnr_l = AverageMeter() 83 | mid_psnr_mu = AverageMeter() 84 | high_psnr_l = AverageMeter() 85 | high_psnr_mu = AverageMeter() 86 | 87 | with torch.no_grad(): 88 | for idx, img_data in enumerate(test_loader): 89 | ldrs = [x.to(device) for x in img_data['ldrs']] 90 | expos = [x.to(device) for x in img_data['expos']] 91 | # hdrs = [x.to(device) for x in img_data['hdrs']] 92 | gt_hdr = img_data['hdr'] 93 | 94 | padder = InputPadder(ldrs[0].shape, divis_by=16) 95 | pad_ldrs = padder.pad(ldrs) 96 | pred_hdr, flow_preds = model(pad_ldrs, expos, test_mode=True) 97 | pred_hdr = padder.unpad(pred_hdr) 98 | pred_hdr = torch.squeeze(pred_hdr.detach().cpu()).numpy().astype(np.float32).transpose(1,2,0) 99 | save_flo(flow_preds, idx+1, args) 100 | cur_ldr = torch.squeeze(ldrs[2].cpu()).numpy().astype(np.float32).transpose(1,2,0) 101 | Y = 0.299 * cur_ldr[:, :, 0] + 0.587 * cur_ldr[:, :, 1] + 0.114 * cur_ldr[:, :, 2] 102 | Y = Y[:, :, None] 103 | 104 | if expos[2] == 1.: 105 | mask = Y < 0.2 106 | elif expos[2] == 4.: 107 | mask = (Y < 0.2) | (Y > 0.8) 108 | else: 109 | mask = Y > 0.8 110 | cur_linear_ldr = ldr_to_hdr(ldrs[2], expos[2]) 111 | cur_linear_ldr = torch.squeeze(cur_linear_ldr.cpu()).numpy().astype(np.float32).transpose(1,2,0) 112 | pred_hdr = (~mask) * cur_linear_ldr + (mask) * pred_hdr 113 | gt_hdr = torch.squeeze(gt_hdr).numpy().astype(np.float32).transpose(1,2,0) 114 | pred_hdr = pred_hdr.copy() 115 | psnrL = psnr(gt_hdr, pred_hdr) 116 | 117 | gt_hdr_tm = tonemap(gt_hdr) 118 | pred_hdr_tm = tonemap(pred_hdr) 119 | psnrT = psnr(gt_hdr_tm, pred_hdr_tm) 120 | 121 | ssimL = ssim(gt_hdr, pred_hdr, multichannel=True, channel_axis=2, data_range=gt_hdr.max()-gt_hdr.min()) 122 | ssimT = ssim(gt_hdr_tm, pred_hdr_tm, multichannel=True, channel_axis=2, data_range=gt_hdr_tm.max()-gt_hdr_tm.min()) 123 | 124 | psnr_l.update(psnrL) 125 | ssim_l.update(ssimL) 126 | psnr_mu.update(psnrT) 127 | ssim_mu.update(ssimT) 128 | 129 | if expos[2] == 1.: 130 | print("Iter: [{}/{}] Low Exposure".format(idx+1, len(test_loader))) 131 | low_psnr_l.update(psnrL) 132 | low_psnr_mu.update(psnrT) 133 | 134 | elif expos[2] == 4.: 135 | print("Iter: [{}/{}] Mid Exposure".format(idx+1, len(test_loader))) 136 | mid_psnr_l.update(psnrL) 137 | mid_psnr_mu.update(psnrT) 138 | 139 | else: 140 | print("Iter: [{}/{}] High Exposure".format(idx+1, len(test_loader))) 141 | high_psnr_l.update(psnrL) 142 | high_psnr_mu.update(psnrT) 143 | 144 | print("PSNR_mu: {:.4f} PSNR_l: {:.4f}".format(psnrT, psnrL)) 145 | print("SSIM_mu: {:.4f} SSIM_l: {:.4f}".format(ssimT, ssimL)) 146 | 147 | # save results 148 | if args.save_results: 149 | dataset_name = args.dataset_dir.split('/')[-1] 150 | hdr_output_dir = os.path.join(args.save_dir, dataset_name, 'hdr_output') 151 | if not osp.exists(hdr_output_dir): 152 | os.makedirs(hdr_output_dir) 153 | cv2.imwrite(os.path.join(hdr_output_dir, '{}_pred.png'.format(idx+1)), (pred_hdr_tm*255.)[:,:,[2,1,0]].astype('uint8')) 154 | # save_hdr(os.path.join(args.save_dir, img_data['hdr_path'][0]), pred_hdr) 155 | print("Low Average PSNR_mu: {:.4f} PSNR_l: {:.4f}".format(low_psnr_mu.avg, low_psnr_l.avg)) 156 | print("Mid Average PSNR_mu: {:.4f} PSNR_l: {:.4f}".format(mid_psnr_mu.avg, mid_psnr_l.avg)) 157 | print("High Average PSNR_mu: {:.4f} PSNR_l: {:.4f}".format(high_psnr_mu.avg, high_psnr_l.avg)) 158 | print("Average PSNR_mu: {:.4f} PSNR_l: {:.4f}".format(psnr_mu.avg, psnr_l.avg)) 159 | print("Average SSIM_mu: {:.4f} SSIM_l: {:.4f}".format(ssim_mu.avg, ssim_l.avg)) 160 | print(">>>>>>>>> Finish Testing >>>>>>>>>") 161 | 162 | 163 | if __name__ == '__main__': 164 | main() 165 | 166 | 167 | 168 | 169 | -------------------------------------------------------------------------------- /test_tog13_2E.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 5 | import time 6 | import argparse 7 | from tqdm import tqdm 8 | import glob 9 | import logging 10 | import torch 11 | import torch.nn as nn 12 | from torch.utils.data import DataLoader 13 | from dataset.tog13_online_align_dataset import TOG13_online_align_Dataset 14 | from models.model_2E import HDRFlow 15 | from utils.utils import * 16 | from utils import flow_viz 17 | 18 | parser = argparse.ArgumentParser(description="Test Setting") 19 | parser.add_argument("--dataset_dir", type=str, default='data/TOG13_Dynamic_Dataset/ThrowingTowel-2Exp-3Stop', 20 | help='dataset directory') 21 | parser.add_argument('--pretrained_model', type=str, default='./pretrained_models/2E/checkpoint.pth') 22 | parser.add_argument('--save_results', action='store_true', default=True) 23 | parser.add_argument('--save_dir', type=str, default="output_results/2E/") 24 | 25 | def save_flo(flow_preds, i, args): 26 | p_flo, n_flo = flow_preds 27 | p_flo = torch.squeeze(p_flo).permute(1,2,0).cpu().numpy() 28 | p_flo = flow_viz.flow_to_image(p_flo) 29 | n_flo = torch.squeeze(n_flo).permute(1,2,0).cpu().numpy() 30 | n_flo = flow_viz.flow_to_image(n_flo) 31 | 32 | concat_flo = np.concatenate([p_flo, n_flo], axis=1) 33 | dataset_name = args.dataset_dir.split('/')[-1] 34 | save_dir = os.path.join(args.save_dir, dataset_name, 'flow_preds') 35 | os.makedirs(save_dir, exist_ok=True) 36 | flo_path = os.path.join(save_dir, f'{i}_flow.png') 37 | cv2.imwrite(flo_path, concat_flo[:, :, [2,1,0]].astype('uint8')) 38 | 39 | def main(): 40 | # Settings 41 | args = parser.parse_args() 42 | # pretrained_model 43 | print(">>>>>>>>> Start Testing >>>>>>>>>") 44 | print("Load weights from: ", args.pretrained_model) 45 | device = torch.device('cuda') 46 | model = HDRFlow() 47 | model.to(device) 48 | model = nn.DataParallel(model) 49 | model.load_state_dict(torch.load(args.pretrained_model)['state_dict']) 50 | model.eval() 51 | 52 | test_dataset = TOG13_online_align_Dataset(root_dir=args.dataset_dir, nframes=3, nexps=2) 53 | test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) 54 | with torch.no_grad(): 55 | for idx, img_data in enumerate(test_loader): 56 | ldrs = [x.to(device) for x in img_data['ldrs']] 57 | expos = [x.to(device) for x in img_data['expos']] 58 | matches = [x.to(device) for x in img_data['matches']] 59 | align_ldrs = global_align_nbr_ldrs(ldrs, matches) 60 | padder = InputPadder(align_ldrs[0].shape, divis_by=16) 61 | pad_ldrs = padder.pad(align_ldrs) 62 | pred_hdr, flow_preds = model(pad_ldrs, expos, test_mode=True) 63 | pred_hdr = padder.unpad(pred_hdr) 64 | pred_hdr = torch.squeeze(pred_hdr.detach().cpu()).numpy().astype(np.float32).transpose(1,2,0) 65 | save_flo(flow_preds, idx+1, args) 66 | 67 | cur_ldr = torch.squeeze(ldrs[1].cpu()).numpy().astype(np.float32).transpose(1,2,0) 68 | Y = 0.299 * cur_ldr[:, :, 0] + 0.587 * cur_ldr[:, :, 1] + 0.114 * cur_ldr[:, :, 2] 69 | Y = Y[:, :, None] 70 | 71 | if expos[1] <= 1.: 72 | mask = Y < 0.2 73 | else: 74 | mask = Y > 0.8 75 | 76 | cur_linear_ldr = ldr_to_hdr(ldrs[1], expos[1]) 77 | cur_linear_ldr = torch.squeeze(cur_linear_ldr.cpu()).numpy().astype(np.float32).transpose(1,2,0) 78 | pred_hdr = (~mask) * cur_linear_ldr + (mask) * pred_hdr 79 | pred_hdr_tm = tonemap(pred_hdr) 80 | 81 | # save results 82 | if args.save_results: 83 | dataset_name = args.dataset_dir.split('/')[-1] 84 | hdr_output_dir = os.path.join(args.save_dir, dataset_name, 'hdr_output') 85 | if not osp.exists(hdr_output_dir): 86 | os.makedirs(hdr_output_dir) 87 | save_hdr(os.path.join(hdr_output_dir, '{}_pred.hdr'.format(idx+1)), pred_hdr) 88 | 89 | if __name__ == '__main__': 90 | main() 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /test_tog13_3E.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 5 | import time 6 | import argparse 7 | from tqdm import tqdm 8 | import glob 9 | import logging 10 | import torch 11 | import torch.nn as nn 12 | from torch.utils.data import DataLoader 13 | from dataset.tog13_online_align_dataset import TOG13_online_align_Dataset 14 | from models.model_3E import HDRFlow 15 | from utils.utils import * 16 | from utils import flow_viz 17 | 18 | parser = argparse.ArgumentParser(description="Test Setting") 19 | parser.add_argument("--dataset_dir", type=str, default='data/TOG13_Dynamic_Dataset/Dog-3Exp-2Stop', 20 | help='dataset directory') 21 | parser.add_argument('--pretrained_model', type=str, default='./pretrained_models/3E/checkpoint.pth') 22 | parser.add_argument('--save_results', action='store_true', default=True) 23 | parser.add_argument('--save_dir', type=str, default="output_results/3E/") 24 | 25 | def save_flo(flow_preds, i, args): 26 | p2_flo, n1_flo, p1_flo, n2_flo = flow_preds 27 | p1_flo = torch.squeeze(p1_flo).permute(1,2,0).cpu().numpy() 28 | p1_flo = flow_viz.flow_to_image(p1_flo) 29 | n1_flo = torch.squeeze(n1_flo).permute(1,2,0).cpu().numpy() 30 | n1_flo = flow_viz.flow_to_image(n1_flo) 31 | p2_flo = torch.squeeze(p2_flo).permute(1,2,0).cpu().numpy() 32 | p2_flo = flow_viz.flow_to_image(p2_flo) 33 | n2_flo = torch.squeeze(n2_flo).permute(1,2,0).cpu().numpy() 34 | n2_flo = flow_viz.flow_to_image(n2_flo) 35 | concat_flo = np.concatenate([p2_flo, n1_flo, p1_flo, n2_flo], axis=1) 36 | dataset_name = args.dataset_dir.split('/')[-1] 37 | save_dir = os.path.join(args.save_dir, dataset_name, 'flow_preds') 38 | os.makedirs(save_dir, exist_ok=True) 39 | flo_path = os.path.join(save_dir, f'{i}_flow.png') 40 | cv2.imwrite(flo_path, concat_flo[:, :, [2,1,0]].astype('uint8')) 41 | 42 | def main(): 43 | # Settings 44 | args = parser.parse_args() 45 | # pretrained_model 46 | print(">>>>>>>>> Start Testing >>>>>>>>>") 47 | print("Load weights from: ", args.pretrained_model) 48 | device = torch.device('cuda') 49 | model = HDRFlow() 50 | model.to(device) 51 | model = nn.DataParallel(model) 52 | model.load_state_dict(torch.load(args.pretrained_model)['state_dict']) 53 | model.eval() 54 | test_dataset = TOG13_online_align_Dataset(root_dir=args.dataset_dir, nframes=5, nexps=3) 55 | test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) 56 | with torch.no_grad(): 57 | for idx, img_data in enumerate(test_loader): 58 | ldrs = [x.to(device) for x in img_data['ldrs']] 59 | expos = [x.to(device) for x in img_data['expos']] 60 | matches = [x.to(device) for x in img_data['matches']] 61 | # hdrs = [x.to(device) for x in img_data['hdrs']] 62 | align_ldrs = global_align_nbr_ldrs(ldrs, matches) 63 | padder = InputPadder(align_ldrs[0].shape, divis_by=16) 64 | pad_ldrs = padder.pad(align_ldrs) 65 | pred_hdr, flow_preds = model(pad_ldrs, expos, test_mode=True) 66 | pred_hdr = padder.unpad(pred_hdr) 67 | pred_hdr = torch.squeeze(pred_hdr.detach().cpu()).numpy().astype(np.float32).transpose(1,2,0) 68 | save_flo(flow_preds, idx+1, args) 69 | cur_ldr = torch.squeeze(ldrs[2].cpu()).numpy().astype(np.float32).transpose(1,2,0) 70 | Y = 0.299 * cur_ldr[:, :, 0] + 0.587 * cur_ldr[:, :, 1] + 0.114 * cur_ldr[:, :, 2] 71 | Y = Y[:, :, None] 72 | if expos[2] == 1.: 73 | mask = Y < 0.2 74 | elif expos[2] == 4.: 75 | mask = (Y<0.2) | (Y>0.8) 76 | else: 77 | mask = Y > 0.8 78 | 79 | cur_linear_ldr = ldr_to_hdr(ldrs[2], expos[2]) 80 | cur_linear_ldr = torch.squeeze(cur_linear_ldr.cpu()).numpy().astype(np.float32).transpose(1,2,0) 81 | pred_hdr = (~mask) * cur_linear_ldr + (mask) * pred_hdr 82 | # save results 83 | if args.save_results: 84 | dataset_name = args.dataset_dir.split('/')[-1] 85 | hdr_output_dir = os.path.join(args.save_dir, dataset_name, 'hdr_output') 86 | if not osp.exists(hdr_output_dir): 87 | os.makedirs(hdr_output_dir) 88 | save_hdr(os.path.join(hdr_output_dir, '{}_pred.hdr'.format(idx+1)), pred_hdr) 89 | 90 | if __name__ == '__main__': 91 | main() 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /train_2E.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 3 | import time 4 | import argparse 5 | from tqdm import tqdm 6 | import torch 7 | import torch.nn as nn 8 | from torch.utils.data import DataLoader 9 | from dataset import fetch_dataloader_2E 10 | from models.loss import HDRFlow_Loss_2E 11 | from models.model_2E import HDRFlow 12 | from utils.utils import * 13 | 14 | def get_args(): 15 | parser = argparse.ArgumentParser(description='HDRFlow', 16 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 17 | parser.add_argument("--dataset_vimeo_dir", type=str, default='data/vimeo_septuplet', 18 | help='dataset directory'), 19 | parser.add_argument("--dataset_sintel_dir", type=str, default='data/Sintel/training/', 20 | help='dataset directory'), 21 | parser.add_argument('--logdir', type=str, default='./checkpoints_2E', 22 | help='target log directory') 23 | parser.add_argument('--num_workers', type=int, default=8, metavar='N', 24 | help='number of workers to fetch data (default: 8)') 25 | parser.add_argument('--resume', type=str, default=None, 26 | help='load model from a .pth file') 27 | parser.add_argument('--seed', type=int, default=443, metavar='S', 28 | help='random seed (default: 443)') 29 | parser.add_argument('--init_weights', action='store_true', default=False, 30 | help='init model weights') 31 | parser.add_argument('--lr', type=float, default=0.0001, metavar='LR', 32 | help='learning rate (default: 0.0002)') 33 | parser.add_argument('--lr_decay_epochs', type=str, 34 | default="20,30:2", help='the epochs to decay lr: the downscale rate') 35 | parser.add_argument('--start_epoch', type=int, default=0, metavar='N', 36 | help='start epoch of training (default: 1)') 37 | parser.add_argument('--epochs', type=int, default=40, metavar='N', 38 | help='number of epochs to train (default: 100)') 39 | parser.add_argument('--batch_size', type=int, default=16, metavar='N', 40 | help='training batch size (default: 16)') 41 | parser.add_argument('--val_batch_size', type=int, default=8, metavar='N', 42 | help='testing batch size (default: 1)') 43 | parser.add_argument('--log_interval', type=int, default=100, metavar='N', 44 | help='how many batches to wait before logging training status') 45 | return parser.parse_args() 46 | 47 | 48 | def train(args, model, device, train_loader, optimizer, epoch, hdrflow_loss): 49 | model.train() 50 | batch_time = AverageMeter() 51 | data_time = AverageMeter() 52 | end = time.time() 53 | for batch_idx, batch_data in enumerate(train_loader): 54 | data_time.update(time.time() - end) 55 | ldrs = [x.to(device) for x in batch_data['ldrs']] 56 | expos = [x.to(device) for x in batch_data['expos']] 57 | hdrs = [x.to(device) for x in batch_data['hdrs']] 58 | flow_gts = [x.to(device) for x in batch_data['flow_gts']] 59 | flow_mask = batch_data['flow_mask'].to(device) 60 | pred_hdr, flow_preds = model(ldrs, expos) 61 | cur_ldr = ldrs[1] 62 | loss = hdrflow_loss(pred_hdr, hdrs, flow_preds, cur_ldr, flow_mask, flow_gts) 63 | optimizer.zero_grad() 64 | loss.backward() 65 | optimizer.step() 66 | batch_time.update(time.time() - end) 67 | end = time.time() 68 | if batch_idx % args.log_interval == 0: 69 | print('Train Epoch: {} [{}/{} ({:.0f} %)]\tLoss: {:.6f}\t' 70 | 'Time: {batch_time.val:.3f} ({batch_time.avg:3f})\t' 71 | 'Data: {data_time.val:.3f} ({data_time.avg:3f})'.format( 72 | epoch, 73 | batch_idx, 74 | len(train_loader), 75 | 100. * batch_idx / len(train_loader), 76 | loss.item(), 77 | batch_time=batch_time, 78 | data_time=data_time 79 | )) 80 | 81 | def validation(args, model, device, val_loader, optimizer, epoch): 82 | model.eval() 83 | n_val = len(val_loader) 84 | val_psnr = AverageMeter() 85 | val_mu_psnr = AverageMeter() 86 | with torch.no_grad(): 87 | for batch_idx, batch_data in enumerate(val_loader): 88 | ldrs = [x.to(device) for x in batch_data['ldrs']] 89 | expos = [x.to(device) for x in batch_data['expos']] 90 | hdrs = [x.to(device) for x in batch_data['hdrs']] 91 | gt_hdr = hdrs[1] 92 | pred_hdr, _ = model(ldrs, expos) 93 | psnr = batch_psnr(pred_hdr, gt_hdr, 1.0) 94 | mu_psnr = batch_psnr_mu(pred_hdr, gt_hdr, 1.0) 95 | val_psnr.update(psnr.item()) 96 | val_mu_psnr.update(mu_psnr.item()) 97 | 98 | print('Validation set: Number: {}'.format(n_val)) 99 | print('Validation set: Average PSNR-l: {:.4f}, PSNR-mu: {:.4f}'.format(val_psnr.avg, val_mu_psnr.avg)) 100 | save_dict = { 101 | 'epoch': epoch + 1, 102 | 'state_dict': model.state_dict(), 103 | 'optimizer': optimizer.state_dict()} 104 | torch.save(save_dict, os.path.join(args.logdir, 'checkpoint_%s.pth' % (epoch+1))) 105 | with open(os.path.join(args.logdir, 'checkpoint.json'), 'a') as f: 106 | f.write('epoch:' + str(epoch) + '\n') 107 | f.write('Validation set: Average PSNR-l: {:.4f}, PSNR-mu: {:.4f}\n'.format(val_psnr.avg, val_mu_psnr.avg)) 108 | 109 | 110 | def main(): 111 | args = get_args() 112 | if args.seed is not None: 113 | set_random_seed(args.seed) 114 | if not os.path.exists(args.logdir): 115 | os.makedirs(args.logdir) 116 | device = torch.device('cuda') 117 | # model 118 | model = HDRFlow() 119 | if args.init_weights: 120 | init_parameters(model) 121 | hdrflow_loss = HDRFlow_Loss_2E().to(device) 122 | # optimizer 123 | optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-08) 124 | model.to(device) 125 | model = nn.DataParallel(model) 126 | 127 | if args.resume: 128 | if os.path.isfile(args.resume): 129 | print("===> Loading checkpoint from: {}".format(args.resume)) 130 | checkpoint = torch.load(args.resume) 131 | args.start_epoch = checkpoint['epoch'] 132 | model.load_state_dict(checkpoint['state_dict']) 133 | optimizer.load_state_dict(checkpoint['optimizer']) 134 | print("===> Loaded checkpoint: epoch {}".format(checkpoint['epoch'])) 135 | else: 136 | print("===> No checkpoint is founded at {}.".format(args.resume)) 137 | 138 | train_loader, val_loader = fetch_dataloader_2E(args) 139 | 140 | for epoch in range(args.start_epoch, args.epochs): 141 | adjust_learning_rate(args, optimizer, epoch) 142 | train(args, model, device, train_loader, optimizer, epoch, hdrflow_loss) 143 | validation(args, model, device, val_loader, optimizer, epoch) 144 | 145 | if __name__ == '__main__': 146 | main() 147 | -------------------------------------------------------------------------------- /train_3E.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 3 | import time 4 | import argparse 5 | from tqdm import tqdm 6 | import torch 7 | import torch.nn as nn 8 | from torch.utils.data import DataLoader 9 | from dataset import fetch_dataloader 10 | from models.loss import HDRFlow_Loss_3E 11 | from models.model_3E import HDRFlow 12 | from utils.utils import * 13 | 14 | def get_args(): 15 | parser = argparse.ArgumentParser(description='HDRFlow', 16 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 17 | parser.add_argument("--dataset_vimeo_dir", type=str, default='data/vimeo_septuplet', 18 | help='dataset directory'), 19 | parser.add_argument("--dataset_sintel_dir", type=str, default='data/Sintel/training/', 20 | help='dataset directory'), 21 | parser.add_argument('--logdir', type=str, default='./checkpoints_3E', 22 | help='target log directory') 23 | parser.add_argument('--num_workers', type=int, default=8, metavar='N', 24 | help='number of workers to fetch data (default: 8)') 25 | # Training 26 | parser.add_argument('--resume', type=str, default=None, 27 | help='load model from a .pth file') 28 | parser.add_argument('--seed', type=int, default=443, metavar='S', 29 | help='random seed (default: 443)') 30 | parser.add_argument('--init_weights', action='store_true', default=False, 31 | help='init model weights') 32 | parser.add_argument('--lr', type=float, default=0.0001, metavar='LR', 33 | help='learning rate (default: 0.0002)') 34 | parser.add_argument('--lr_decay_epochs', type=str, 35 | default="20,30:2", help='the epochs to decay lr: the downscale rate') 36 | parser.add_argument('--start_epoch', type=int, default=1, metavar='N', 37 | help='start epoch of training (default: 1)') 38 | parser.add_argument('--epochs', type=int, default=40, metavar='N', 39 | help='number of epochs to train (default: 100)') 40 | parser.add_argument('--batch_size', type=int, default=16, metavar='N', 41 | help='training batch size (default: 16)') 42 | parser.add_argument('--val_batch_size', type=int, default=8, metavar='N', 43 | help='testing batch size (default: 1)') 44 | parser.add_argument('--log_interval', type=int, default=100, metavar='N', 45 | help='how many batches to wait before logging training status') 46 | return parser.parse_args() 47 | 48 | 49 | def train(args, model, device, train_loader, optimizer, epoch, hdrflow_loss): 50 | model.train() 51 | batch_time = AverageMeter() 52 | data_time = AverageMeter() 53 | end = time.time() 54 | for batch_idx, batch_data in enumerate(train_loader): 55 | data_time.update(time.time() - end) 56 | ldrs = [x.to(device) for x in batch_data['ldrs']] 57 | expos = [x.to(device) for x in batch_data['expos']] 58 | hdrs = [x.to(device) for x in batch_data['hdrs']] 59 | flow_gts = [x.to(device) for x in batch_data['flow_gts']] 60 | flow_mask = batch_data['flow_mask'].to(device) 61 | pred_hdr, flow_preds = model(ldrs, expos) 62 | cur_ldr = ldrs[2] 63 | loss = hdrflow_loss(pred, hdrs, flow_preds, cur_ldr, flow_mask, flow_gts) 64 | optimizer.zero_grad() 65 | loss.backward() 66 | optimizer.step() 67 | batch_time.update(time.time() - end) 68 | end = time.time() 69 | if batch_idx % args.log_interval == 0: 70 | print('Train Epoch: {} [{}/{} ({:.0f} %)]\tLoss: {:.6f}\t' 71 | 'Time: {batch_time.val:.3f} ({batch_time.avg:3f})\t' 72 | 'Data: {data_time.val:.3f} ({data_time.avg:3f})'.format( 73 | epoch, 74 | batch_idx, 75 | len(train_loader), 76 | 100. * batch_idx / len(train_loader), 77 | loss.item(), 78 | batch_time=batch_time, 79 | data_time=data_time 80 | )) 81 | 82 | def validation(args, model, device, val_loader, optimizer, epoch): 83 | model.eval() 84 | n_val = len(val_loader) 85 | val_psnr = AverageMeter() 86 | val_mu_psnr = AverageMeter() 87 | with torch.no_grad(): 88 | for batch_idx, batch_data in enumerate(val_loader): 89 | ldrs = [x.to(device) for x in batch_data['ldrs']] 90 | expos = [x.to(device) for x in batch_data['expos']] 91 | hdrs = [x.to(device) for x in batch_data['hdrs']] 92 | gt_hdr = hdrs[2] 93 | pred_hdr, _ = model(ldrs, expos) 94 | psnr = batch_psnr(pred_hdr, gt_hdr, 1.0) 95 | mu_psnr = batch_psnr_mu(pred_hdr, gt_hdr, 1.0) 96 | val_psnr.update(psnr.item()) 97 | val_mu_psnr.update(mu_psnr.item()) 98 | 99 | print('Validation set: Number: {}'.format(n_val)) 100 | print('Validation set: Average PSNR-l: {:.4f}, PSNR-mu: {:.4f}'.format(val_psnr.avg, val_mu_psnr.avg)) 101 | 102 | save_dict = { 103 | 'epoch': epoch + 1, 104 | 'state_dict': model.state_dict(), 105 | 'optimizer': optimizer.state_dict()} 106 | torch.save(save_dict, os.path.join(args.logdir, 'checkpoint_%s.pth' % (epoch+1))) 107 | 108 | with open(os.path.join(args.logdir, 'checkpoint.json'), 'a') as f: 109 | f.write('epoch:' + str(epoch) + '\n') 110 | f.write('Validation set: Average PSNR-l: {:.4f}, PSNR-mu: {:.4f}\n'.format(val_psnr.avg, val_mu_psnr.avg)) 111 | 112 | def main(): 113 | args = get_args() 114 | if args.seed is not None: 115 | set_random_seed(args.seed) 116 | if not os.path.exists(args.logdir): 117 | os.makedirs(args.logdir) 118 | device = torch.device('cuda') 119 | # model 120 | model = HDRFlow() 121 | if args.init_weights: 122 | init_parameters(model) 123 | hdrflow_loss = HDRFlow_Loss_3E().to(device) 124 | # optimizer 125 | optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-08) 126 | model.to(device) 127 | model = nn.DataParallel(model) 128 | 129 | if args.resume: 130 | if os.path.isfile(args.resume): 131 | print("===> Loading checkpoint from: {}".format(args.resume)) 132 | checkpoint = torch.load(args.resume) 133 | args.start_epoch = checkpoint['epoch'] 134 | model.load_state_dict(checkpoint['state_dict']) 135 | optimizer.load_state_dict(checkpoint['optimizer']) 136 | print("===> Loaded checkpoint: epoch {}".format(checkpoint['epoch'])) 137 | else: 138 | print("===> No checkpoint is founded at {}.".format(args.resume)) 139 | 140 | train_loader, val_loader = fetch_dataloader(args) 141 | 142 | for epoch in range(args.epochs): 143 | adjust_learning_rate(args, optimizer, epoch) 144 | train(args, model, device, train_loader, optimizer, epoch, hdrflow_loss) 145 | validation(args, model, device, val_loader, optimizer, epoch) 146 | 147 | if __name__ == '__main__': 148 | main() 149 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenImagingLab/HDRFlow/c891759c5eba11869a038679da444e78c54f5a22/utils/__init__.py -------------------------------------------------------------------------------- /utils/flow_viz.py: -------------------------------------------------------------------------------- 1 | # Flow visualization code used from https://github.com/tomrunia/OpticalFlow_Visualization 2 | 3 | 4 | # MIT License 5 | # 6 | # Copyright (c) 2018 Tom Runia 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to conditions. 14 | # 15 | # Author: Tom Runia 16 | # Date Created: 2018-08-03 17 | 18 | import numpy as np 19 | 20 | def make_colorwheel(): 21 | """ 22 | Generates a color wheel for optical flow visualization as presented in: 23 | Baker et al. "A Database and Evaluation Methodology for Optical Flow" (ICCV, 2007) 24 | URL: http://vision.middlebury.edu/flow/flowEval-iccv07.pdf 25 | 26 | Code follows the original C++ source code of Daniel Scharstein. 27 | Code follows the the Matlab source code of Deqing Sun. 28 | 29 | Returns: 30 | np.ndarray: Color wheel 31 | """ 32 | 33 | RY = 15 34 | YG = 6 35 | GC = 4 36 | CB = 11 37 | BM = 13 38 | MR = 6 39 | 40 | ncols = RY + YG + GC + CB + BM + MR 41 | colorwheel = np.zeros((ncols, 3)) 42 | col = 0 43 | 44 | # RY 45 | colorwheel[0:RY, 0] = 255 46 | colorwheel[0:RY, 1] = np.floor(255*np.arange(0,RY)/RY) 47 | col = col+RY 48 | # YG 49 | colorwheel[col:col+YG, 0] = 255 - np.floor(255*np.arange(0,YG)/YG) 50 | colorwheel[col:col+YG, 1] = 255 51 | col = col+YG 52 | # GC 53 | colorwheel[col:col+GC, 1] = 255 54 | colorwheel[col:col+GC, 2] = np.floor(255*np.arange(0,GC)/GC) 55 | col = col+GC 56 | # CB 57 | colorwheel[col:col+CB, 1] = 255 - np.floor(255*np.arange(CB)/CB) 58 | colorwheel[col:col+CB, 2] = 255 59 | col = col+CB 60 | # BM 61 | colorwheel[col:col+BM, 2] = 255 62 | colorwheel[col:col+BM, 0] = np.floor(255*np.arange(0,BM)/BM) 63 | col = col+BM 64 | # MR 65 | colorwheel[col:col+MR, 2] = 255 - np.floor(255*np.arange(MR)/MR) 66 | colorwheel[col:col+MR, 0] = 255 67 | return colorwheel 68 | 69 | 70 | def flow_uv_to_colors(u, v, convert_to_bgr=False): 71 | """ 72 | Applies the flow color wheel to (possibly clipped) flow components u and v. 73 | 74 | According to the C++ source code of Daniel Scharstein 75 | According to the Matlab source code of Deqing Sun 76 | 77 | Args: 78 | u (np.ndarray): Input horizontal flow of shape [H,W] 79 | v (np.ndarray): Input vertical flow of shape [H,W] 80 | convert_to_bgr (bool, optional): Convert output image to BGR. Defaults to False. 81 | 82 | Returns: 83 | np.ndarray: Flow visualization image of shape [H,W,3] 84 | """ 85 | flow_image = np.zeros((u.shape[0], u.shape[1], 3), np.uint8) 86 | colorwheel = make_colorwheel() # shape [55x3] 87 | ncols = colorwheel.shape[0] 88 | rad = np.sqrt(np.square(u) + np.square(v)) 89 | a = np.arctan2(-v, -u)/np.pi 90 | fk = (a+1) / 2*(ncols-1) 91 | k0 = np.floor(fk).astype(np.int32) 92 | k1 = k0 + 1 93 | k1[k1 == ncols] = 0 94 | f = fk - k0 95 | for i in range(colorwheel.shape[1]): 96 | tmp = colorwheel[:,i] 97 | col0 = tmp[k0] / 255.0 98 | col1 = tmp[k1] / 255.0 99 | col = (1-f)*col0 + f*col1 100 | idx = (rad <= 1) 101 | col[idx] = 1 - rad[idx] * (1-col[idx]) 102 | col[~idx] = col[~idx] * 0.75 # out of range 103 | # Note the 2-i => BGR instead of RGB 104 | ch_idx = 2-i if convert_to_bgr else i 105 | flow_image[:,:,ch_idx] = np.floor(255 * col) 106 | return flow_image 107 | 108 | 109 | def flow_to_image(flow_uv, clip_flow=None, convert_to_bgr=False): 110 | """ 111 | Expects a two dimensional flow image of shape. 112 | 113 | Args: 114 | flow_uv (np.ndarray): Flow UV image of shape [H,W,2] 115 | clip_flow (float, optional): Clip maximum of flow values. Defaults to None. 116 | convert_to_bgr (bool, optional): Convert output image to BGR. Defaults to False. 117 | 118 | Returns: 119 | np.ndarray: Flow visualization image of shape [H,W,3] 120 | """ 121 | assert flow_uv.ndim == 3, 'input flow must have three dimensions' 122 | assert flow_uv.shape[2] == 2, 'input flow must have shape [H,W,2]' 123 | if clip_flow is not None: 124 | flow_uv = np.clip(flow_uv, 0, clip_flow) 125 | u = flow_uv[:,:,0] 126 | v = flow_uv[:,:,1] 127 | rad = np.sqrt(np.square(u) + np.square(v)) 128 | rad_max = np.max(rad) 129 | epsilon = 1e-5 130 | u = u / (rad_max + epsilon) 131 | v = v / (rad_max + epsilon) 132 | return flow_uv_to_colors(u, v, convert_to_bgr) -------------------------------------------------------------------------------- /utils/frame_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | from os.path import * 4 | import re 5 | 6 | import cv2 7 | cv2.setNumThreads(0) 8 | cv2.ocl.setUseOpenCL(False) 9 | 10 | TAG_CHAR = np.array([202021.25], np.float32) 11 | 12 | def readFlow(fn): 13 | """ Read .flo file in Middlebury format""" 14 | # Code adapted from: 15 | # http://stackoverflow.com/questions/28013200/reading-middlebury-flow-files-with-python-bytes-array-numpy 16 | 17 | # WARNING: this will work on little-endian architectures (eg Intel x86) only! 18 | # print 'fn = %s'%(fn) 19 | with open(fn, 'rb') as f: 20 | magic = np.fromfile(f, np.float32, count=1) 21 | if 202021.25 != magic: 22 | print('Magic number incorrect. Invalid .flo file') 23 | return None 24 | else: 25 | w = np.fromfile(f, np.int32, count=1) 26 | h = np.fromfile(f, np.int32, count=1) 27 | # print 'Reading %d x %d flo file\n' % (w, h) 28 | data = np.fromfile(f, np.float32, count=2*int(w)*int(h)) 29 | # Reshape data into 3D array (columns, rows, bands) 30 | # The reshape here is for visualization, the original code is (w,h,2) 31 | return np.resize(data, (int(h), int(w), 2)) 32 | 33 | def readPFM(file): 34 | file = open(file, 'rb') 35 | 36 | color = None 37 | width = None 38 | height = None 39 | scale = None 40 | endian = None 41 | 42 | header = file.readline().rstrip() 43 | if header == b'PF': 44 | color = True 45 | elif header == b'Pf': 46 | color = False 47 | else: 48 | raise Exception('Not a PFM file.') 49 | 50 | dim_match = re.match(rb'^(\d+)\s(\d+)\s$', file.readline()) 51 | if dim_match: 52 | width, height = map(int, dim_match.groups()) 53 | else: 54 | raise Exception('Malformed PFM header.') 55 | 56 | scale = float(file.readline().rstrip()) 57 | if scale < 0: # little-endian 58 | endian = '<' 59 | scale = -scale 60 | else: 61 | endian = '>' # big-endian 62 | 63 | data = np.fromfile(file, endian + 'f') 64 | shape = (height, width, 3) if color else (height, width) 65 | 66 | data = np.reshape(data, shape) 67 | data = np.flipud(data) 68 | return data 69 | 70 | def writeFlow(filename,uv,v=None): 71 | """ Write optical flow to file. 72 | 73 | If v is None, uv is assumed to contain both u and v channels, 74 | stacked in depth. 75 | Original code by Deqing Sun, adapted from Daniel Scharstein. 76 | """ 77 | nBands = 2 78 | 79 | if v is None: 80 | assert(uv.ndim == 3) 81 | assert(uv.shape[2] == 2) 82 | u = uv[:,:,0] 83 | v = uv[:,:,1] 84 | else: 85 | u = uv 86 | 87 | assert(u.shape == v.shape) 88 | height,width = u.shape 89 | f = open(filename,'wb') 90 | # write the header 91 | f.write(TAG_CHAR) 92 | np.array(width).astype(np.int32).tofile(f) 93 | np.array(height).astype(np.int32).tofile(f) 94 | # arrange into matrix form 95 | tmp = np.zeros((height, width*nBands)) 96 | tmp[:,np.arange(width)*2] = u 97 | tmp[:,np.arange(width)*2 + 1] = v 98 | tmp.astype(np.float32).tofile(f) 99 | f.close() 100 | 101 | 102 | def readFlowKITTI(filename): 103 | flow = cv2.imread(filename, cv2.IMREAD_ANYDEPTH|cv2.IMREAD_COLOR) 104 | flow = flow[:,:,::-1].astype(np.float32) 105 | flow, valid = flow[:, :, :2], flow[:, :, 2] 106 | flow = (flow - 2**15) / 64.0 107 | return flow, valid 108 | 109 | def readDispKITTI(filename): 110 | disp = cv2.imread(filename, cv2.IMREAD_ANYDEPTH) / 256.0 111 | valid = disp > 0.0 112 | flow = np.stack([-disp, np.zeros_like(disp)], -1) 113 | return flow, valid 114 | 115 | 116 | def writeFlowKITTI(filename, uv): 117 | uv = 64.0 * uv + 2**15 118 | valid = np.ones([uv.shape[0], uv.shape[1], 1]) 119 | uv = np.concatenate([uv, valid], axis=-1).astype(np.uint16) 120 | cv2.imwrite(filename, uv[..., ::-1]) 121 | 122 | 123 | def read_gen(file_name, pil=False): 124 | ext = splitext(file_name)[-1] 125 | if ext == '.png' or ext == '.jpeg' or ext == '.ppm' or ext == '.jpg': 126 | return Image.open(file_name) 127 | elif ext == '.bin' or ext == '.raw': 128 | return np.load(file_name) 129 | elif ext == '.flo': 130 | return readFlow(file_name).astype(np.float32) 131 | elif ext == '.pfm': 132 | flow = readPFM(file_name).astype(np.float32) 133 | if len(flow.shape) == 2: 134 | return flow 135 | else: 136 | return flow[:, :, :-1] 137 | return [] -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os, glob 3 | import cv2 4 | import math 5 | import imageio 6 | from math import log10 7 | import random 8 | import torch 9 | import torch.nn as nn 10 | import torch.nn.init as init 11 | from skimage.metrics import peak_signal_noise_ratio 12 | from skimage.transform import resize, rotate 13 | import torch.nn.functional as F 14 | 15 | def list_all_files_sorted(folder_name, extension=""): 16 | return sorted(glob.glob(os.path.join(folder_name, "*" + extension))) 17 | 18 | def read_expo_times(file_name): 19 | return np.power(2, np.loadtxt(file_name)) 20 | 21 | def read_images(file_names): 22 | imgs = [] 23 | for img_str in file_names: 24 | img = cv2.imread(img_str, -1) 25 | # equivalent to im2single from Matlab 26 | img = img / 2 ** 16 27 | img = np.float32(img) 28 | img.clip(0, 1) 29 | imgs.append(img) 30 | return np.array(imgs) 31 | 32 | def read_label(file_path, file_name): 33 | label = imageio.imread(os.path.join(file_path, file_name), 'hdr') 34 | label = label[:, :, [2, 1, 0]] ##cv2 35 | return label 36 | 37 | def ldr_to_hdr(img, expo, gamma=2.2): 38 | return (img ** gamma) / (expo + 1e-8) 39 | 40 | # def hdr_to_ldr(img, expo, gamma=2.2, stdv1=1e-3, stdv2=1e-3): 41 | def hdr_to_ldr(img, expo, gamma=2.2, stdv1=1e-3, stdv2=1e-3): 42 | # add noise to low expo 43 | if expo == 1. or expo == 4.: 44 | stdv = np.random.rand(*img.shape) * (stdv2 - stdv1) + stdv1 45 | noise = np.random.normal(0, stdv) 46 | img = (img + noise).clip(0, 1) 47 | img = np.power(img * expo, 1.0 / gamma) 48 | img = img.clip(0, 1) 49 | return img 50 | 51 | def tonemap(x): 52 | return (np.log(1 + 5000 * x)) / np.log(1 + 5000) 53 | 54 | def range_compressor_cuda(hdr_img, mu=5000): 55 | return (torch.log(1 + mu * hdr_img)) / math.log(1 + mu) 56 | 57 | def range_compressor_tensor(x, device): 58 | a = torch.tensor(1.0, device=device, requires_grad=False) 59 | mu = torch.tensor(5000.0, device=device, requires_grad=False) 60 | return (torch.log(a + mu * x)) / torch.log(a + mu) 61 | 62 | # def psnr(x, target): 63 | # sqrdErr = np.mean((x - target) ** 2) 64 | # return 10 * log10(1/sqrdErr) 65 | 66 | def batch_psnr(img, imclean, data_range): 67 | Img = img.data.cpu().numpy().astype(np.float32) 68 | Iclean = imclean.data.cpu().numpy().astype(np.float32) 69 | psnr = 0 70 | for i in range(Img.shape[0]): 71 | psnr += peak_signal_noise_ratio(Iclean[i,:,:,:], Img[i,:,:,:], data_range=data_range) 72 | return (psnr/Img.shape[0]) 73 | 74 | def batch_psnr_mu(img, imclean, data_range): 75 | img = range_compressor_cuda(img) 76 | imclean = range_compressor_cuda(imclean) 77 | Img = img.data.cpu().numpy().astype(np.float32) 78 | Iclean = imclean.data.cpu().numpy().astype(np.float32) 79 | psnr = 0 80 | for i in range(Img.shape[0]): 81 | psnr += peak_signal_noise_ratio(Iclean[i,:,:,:], Img[i,:,:,:], data_range=data_range) 82 | return (psnr/Img.shape[0]) 83 | 84 | # def adjust_learning_rate(args, optimizer, epoch): 85 | # lr = args.lr * (0.5 ** (epoch // args.lr_decay_interval)) 86 | # for param_group in optimizer.param_groups: 87 | # param_group['lr'] = lr 88 | 89 | def adjust_learning_rate(args, optimizer, epoch): 90 | splits = args.lr_decay_epochs.split(':') 91 | assert len(splits) == 2 92 | 93 | # parse the epochs to downscale the learning rate (before :) 94 | downscale_epochs = [int(eid_str) for eid_str in splits[0].split(',')] 95 | downscale_rate = float(splits[1]) 96 | print("downscale epochs: {}, downscale rate: {}".format(downscale_epochs, downscale_rate)) 97 | 98 | lr = args.lr 99 | for eid in downscale_epochs: 100 | if epoch >= eid: 101 | lr /= downscale_rate 102 | else: 103 | break 104 | print("setting learning rate to {}".format(lr)) 105 | for param_group in optimizer.param_groups: 106 | param_group['lr'] = lr 107 | 108 | def init_parameters(net): 109 | """Init layer parameters""" 110 | for m in net.modules(): 111 | if isinstance(m, nn.Conv2d): 112 | init.kaiming_normal_(m.weight, mode='fan_out') 113 | if m.bias is not None: 114 | init.constant_(m.bias, 0) 115 | elif isinstance(m, nn.BatchNorm2d): 116 | init.constant_(m.weight, 1) 117 | init.constant_(m.bias, 0) 118 | elif isinstance(m, nn.Linear): 119 | init.xavier_normal_(m.weight) 120 | init.constant_(m.bias, 0) 121 | 122 | def set_random_seed(seed): 123 | """Set random seed for reproduce""" 124 | random.seed(seed) 125 | np.random.seed(seed) 126 | torch.manual_seed(seed) 127 | torch.cuda.manual_seed_all(seed) 128 | # torch.backends.cudnn.deterministic = True 129 | # torch.backends.cudnn.benchmark = False 130 | 131 | class AverageMeter(object): 132 | """Computes and stores the average and current value""" 133 | def __init__(self): 134 | self.reset() 135 | 136 | def reset(self): 137 | self.val = 0 138 | self.avg = 0 139 | self.sum = 0 140 | self.count = 0 141 | 142 | def update(self, val, n=1): 143 | self.val = val 144 | self.sum += val * n 145 | self.count += n 146 | self.avg = self.sum / self.count 147 | 148 | # def ssim(img1, img2): 149 | # C1 = (0.01 * 255)**2 150 | # C2 = (0.03 * 255)**2 151 | 152 | # img1 = img1.astype(np.float64) 153 | # img2 = img2.astype(np.float64) 154 | # kernel = cv2.getGaussianKernel(11, 1.5) 155 | # window = np.outer(kernel, kernel.transpose()) 156 | 157 | # mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5] # valid 158 | # mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5] 159 | # mu1_sq = mu1**2 160 | # mu2_sq = mu2**2 161 | # mu1_mu2 = mu1 * mu2 162 | # sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq 163 | # sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq 164 | # sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2 165 | 166 | # ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * 167 | # (sigma1_sq + sigma2_sq + C2)) 168 | # return ssim_map.mean() 169 | 170 | def calculate_ssim(img1, img2): 171 | """ 172 | calculate SSIM 173 | 174 | :param img1: [0, 255] 175 | :param img2: [0, 255] 176 | :return: 177 | """ 178 | if not img1.shape == img2.shape: 179 | raise ValueError('Input images must have the same dimensions.') 180 | if img1.ndim == 2: 181 | return ssim(img1, img2) 182 | elif img1.ndim == 3: 183 | if img1.shape[2] == 3: 184 | ssims = [] 185 | for i in range(3): 186 | ssims.append(ssim(img1, img2)) 187 | return np.array(ssims).mean() 188 | elif img1.shape[2] == 1: 189 | return ssim(np.squeeze(img1), np.squeeze(img2)) 190 | else: 191 | raise ValueError('Wrong input image dimensions.') 192 | 193 | def radiance_writer(out_path, image): 194 | 195 | with open(out_path, "wb") as f: 196 | f.write(b"#?RADIANCE\n# Made with Python & Numpy\nFORMAT=32-bit_rle_rgbe\n\n") 197 | f.write(b"-Y %d +X %d\n" %(image.shape[0], image.shape[1])) 198 | 199 | brightest = np.maximum(np.maximum(image[...,0], image[...,1]), image[...,2]) 200 | mantissa = np.zeros_like(brightest) 201 | exponent = np.zeros_like(brightest) 202 | np.frexp(brightest, mantissa, exponent) 203 | scaled_mantissa = mantissa * 255.0 / brightest 204 | rgbe = np.zeros((image.shape[0], image.shape[1], 4), dtype=np.uint8) 205 | rgbe[...,0:3] = np.around(image[...,0:3] * scaled_mantissa[...,None]) 206 | rgbe[...,3] = np.around(exponent + 128) 207 | 208 | rgbe.flatten().tofile(f) 209 | 210 | def save_hdr(path, image): 211 | return radiance_writer(path, image) 212 | 213 | def read_16bit_tif(img_name, crf=None): 214 | img = cv2.imread(img_name, -1) #/ 65535.0 # normalize to [0, 1] 215 | img = img[:, :, [2, 1, 0]] # BGR to RGB 216 | if crf is not None: 217 | img = reverse_crf(img, crf) 218 | img = img / crf.max() 219 | else: 220 | img = img / 65535.0 221 | return img 222 | 223 | def prob_center_crop(imgs): 224 | if np.random.uniform() > 0.9: 225 | return imgs 226 | new_imgs = [] 227 | for img in imgs: 228 | h, w, c = img.shape 229 | t, l = h // 4, w // 6 230 | new_imgs.append(img[t:, l:w-l]) 231 | return new_imgs 232 | 233 | def random_crop(inputs, size, margin=0): 234 | is_list = type(inputs) == list 235 | if not is_list: inputs = [inputs] 236 | 237 | outputs = [] 238 | h, w, _ = inputs[0].shape 239 | c_h, c_w = size 240 | if h != c_h or w != c_w: 241 | t = random.randint(0+margin, h - c_h-margin) 242 | l = random.randint(0+margin, w - c_w-margin) 243 | for img in inputs: 244 | outputs.append(img[t: t+c_h, l: l+c_w]) 245 | else: 246 | outputs = inputs 247 | if not is_list: outputs = outputs[0] 248 | return outputs 249 | 250 | def random_flip_lrud(inputs): 251 | if np.random.random() > 0.5: 252 | return inputs 253 | is_list = type(inputs) == list 254 | if not is_list: inputs = [inputs] 255 | 256 | outputs = [] 257 | vertical_flip = True if np.random.random() > 0.5 else False # vertical flip 258 | for img in inputs: 259 | flip_img = np.fliplr(img) 260 | if vertical_flip: 261 | flip_img = np.flipud(flip_img) 262 | outputs.append(flip_img.copy()) 263 | if not is_list: outputs = outputs[0] 264 | return outputs 265 | 266 | def random_rotate(inputs, angle=90.0): 267 | is_list = type(inputs) == list 268 | if not is_list: inputs = [inputs] 269 | 270 | outputs = [] 271 | ang = np.random.uniform(0, angle) 272 | for img in inputs: 273 | outputs.append(rotate(img, angle=ang, mode='constant')) 274 | if not is_list: outputs = outputs[0] 275 | return outputs 276 | 277 | def read_list(list_path,ignore_head=False, sort=False): 278 | lists = [] 279 | with open(list_path) as f: 280 | lists = f.read().splitlines() 281 | if ignore_head: 282 | lists = lists[1:] 283 | if sort: 284 | lists.sort(key=natural_keys) 285 | return lists 286 | 287 | def read_hdr(filename, use_cv2=True): 288 | ext = os.path.splitext(filename)[1] 289 | if use_cv2: 290 | hdr = cv2.imread(filename, -1)[:,:,::-1].clip(0) 291 | elif ext == '.exr': 292 | hdr = read_exr(filename) 293 | elif ext == '.hdr': 294 | hdr = cv2.imread(filename, -1) 295 | elif ext == '.npy': 296 | hdr = np.load(filenmae) 297 | else: 298 | raise_not_defined() 299 | return hdr 300 | 301 | class InputPadder: 302 | """ Pads images such that dimensions are divisible by 8 """ 303 | def __init__(self, dims, mode='sintel', divis_by=8): 304 | self.ht, self.wd = dims[-2:] 305 | pad_ht = (((self.ht // divis_by) + 1) * divis_by - self.ht) % divis_by 306 | pad_wd = (((self.wd // divis_by) + 1) * divis_by - self.wd) % divis_by 307 | self._pad = [pad_wd//2, pad_wd - pad_wd//2, pad_ht//2, pad_ht - pad_ht//2] 308 | 309 | def pad(self, inputs): 310 | return [F.pad(x, self._pad, mode='replicate') for x in inputs] 311 | 312 | def unpad(self,x): 313 | ht, wd = x.shape[-2:] 314 | c = [self._pad[2], ht-self._pad[3], self._pad[0], wd-self._pad[1]] 315 | return x[..., c[0]:c[1], c[2]:c[3]] 316 | 317 | def coords_grid(b, h, w, device): 318 | coords = torch.meshgrid(torch.arange(h, device=device), torch.arange(w, device=device)) 319 | coords = torch.stack(coords[::-1], dim=0).float() 320 | return coords[None].repeat(b, 1, 1, 1) 321 | 322 | def backward_warp(img, flow, pad='zeros'): 323 | b, c, h, w = img.shape 324 | grid = coords_grid(b, h, w, device=img.device) 325 | grid = grid + flow 326 | xgrid, ygrid = grid.split([1,1], dim=1) 327 | xgrid = 2*xgrid/(w-1) - 1 328 | ygrid = 2*ygrid/(h-1) - 1 329 | grid = torch.cat([xgrid, ygrid], dim=1) 330 | 331 | warped_img = F.grid_sample(input=img, grid=grid.permute(0,2,3,1), mode='bilinear', padding_mode='zeros') 332 | return warped_img 333 | 334 | def reverse_crf(img, crf): 335 | img = img.astype(int) 336 | out = img.astype(float) 337 | for i in range(img.shape[2]): 338 | out[:,:,i] = crf[:,i][img[:,:,i]] # crf shape [65536, 3] 339 | return out 340 | 341 | # For online global alignment 342 | def cvt_MToTheta(M, w, h): 343 | M_aug = np.concatenate([M, np.zeros((1, 3))], axis=0) 344 | M_aug[-1, -1] = 1.0 345 | N = get_N(w, h) 346 | N_inv = get_N_inv(w, h) 347 | theta = N @ M_aug @ N_inv 348 | theta = np.linalg.inv(theta) 349 | return theta[:2, :] 350 | 351 | def get_N(W, H): 352 | """N that maps from unnormalized to normalized coordinates""" 353 | N = np.zeros((3, 3), dtype=np.float64) 354 | N[0, 0] = 2.0 / W 355 | N[0, 1] = 0 356 | N[1, 1] = 2.0 / H 357 | N[1, 0] = 0 358 | N[0, -1] = -1.0 359 | N[1, -1] = -1.0 360 | N[-1, -1] = 1.0 361 | return N 362 | 363 | def get_N_inv(W, H): 364 | """N that maps from normalized to unnormalized coordinates""" 365 | # TODO: do this analytically maybe? 366 | N = get_N(W, H) 367 | return np.linalg.inv(N) 368 | 369 | def apply_gamma(image, gamma=2.2): 370 | image = image.clip(1e-8, 1) 371 | image = np.power(image, 1.0 / gamma) 372 | return image 373 | 374 | def affine_warp(img, theta): # warp img1 to img2 375 | n, c, h, w = img.shape 376 | affine_grid = F.affine_grid(theta, img.shape) 377 | invalid_mask = ((affine_grid.narrow(3, 0, 1).abs() > 1) + (affine_grid.narrow(3, 1, 1).abs() > 1)) >= 1 378 | invalid_mask = invalid_mask.view(n, 1, h, w).float() 379 | img1_to_img2 = F.grid_sample(img, affine_grid) 380 | img1_to_img2 = img * invalid_mask + img1_to_img2 * (1 - invalid_mask) 381 | return img1_to_img2 382 | 383 | def global_align_nbr_ldrs(ldrs, matches): 384 | if len(ldrs) == 3: 385 | match_p = matches[0][:,1].view(-1, 2, 3) 386 | match_n = matches[2][:,0].view(-1, 2, 3) 387 | p_to_c = affine_warp(ldrs[0], match_p) 388 | n_to_c = affine_warp(ldrs[2], match_n) 389 | return [p_to_c, ldrs[1], n_to_c] 390 | 391 | elif len(ldrs) == 5: 392 | match_p2 = matches[0][:,3].view(-1, 2, 3) 393 | match_p1 = matches[1][:,2].view(-1, 2, 3) 394 | match_n1 = matches[3][:,1].view(-1, 2, 3) 395 | match_n2 = matches[4][:,0].view(-1, 2, 3) 396 | p2_to_c = affine_warp(ldrs[0], match_p2) 397 | p1_to_c = affine_warp(ldrs[1], match_p1) 398 | n1_to_c = affine_warp(ldrs[3], match_n1) 399 | n2_to_c = affine_warp(ldrs[4], match_n2) 400 | return [p2_to_c, p1_to_c, ldrs[2], n1_to_c, n2_to_c] 401 | else: 402 | return 0 403 | --------------------------------------------------------------------------------