├── Img
    ├── dataset_example.png
    └── sensor_system.png
├── MS2dataset
    ├── readme.txt
    ├── test_day_list.txt
    ├── test_night_list.txt
    ├── test_rainy_list.txt
    ├── train_list.txt
    └── val_list.txt
├── README.md
├── dataloader
    └── MS2_dataset.py
├── demo.py
└── utils
    └── utils.py


/Img/dataset_example.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/UkcheolShin/MS2-MultiSpectralStereoDataset/0bb0fc1a544d8ad58c7fda5bc025cb0cbb400284/Img/dataset_example.png


--------------------------------------------------------------------------------
/Img/sensor_system.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/UkcheolShin/MS2-MultiSpectralStereoDataset/0bb0fc1a544d8ad58c7fda5bc025cb0cbb400284/Img/sensor_system.png


--------------------------------------------------------------------------------
/MS2dataset/readme.txt:
--------------------------------------------------------------------------------
 1 | * File Structure of "MS2 dataset"
 2 | |--sync_data
 3 |     |-- <Seq name>
 4 |         |-- rgb, nir, thr
 5 |             |-- img_left
 6 |             |-- img_right
 7 |             |-- depth_gt
 8 |             |-- depth_refl_gt
 9 |             |-- odom
10 |         |-- lidar
11 |             |-- left
12 |             |-- right
13 |         |-- gps_imu
14 |             |-- data
15 |         |-- calib.npy
16 | |--proj_depth
17 |     |-- <Seq name>
18 |         |-- rgb, nir, thr
19 |             |-- depth
20 |             |-- intensity
21 |             |-- depth_multi
22 |             |-- intensity_multi
23 |             |-- depth_filtered 
24 | |--odom
25 |     |-- <Seq name>
26 |         |-- rgb, nir, thr, odom
27 | |-- train_list.txt
28 | |-- val_list.txt
29 | |-- test_day_list.txt
30 | |-- test_night_list.txt
31 | |-- test_rainy_list.txt
32 | 


--------------------------------------------------------------------------------
/MS2dataset/test_day_list.txt:
--------------------------------------------------------------------------------
1 | _2021-08-06-11-23-45
2 | _2021-08-13-15-46-56
3 | _2021-08-13-16-31-10
4 | 


--------------------------------------------------------------------------------
/MS2dataset/test_night_list.txt:
--------------------------------------------------------------------------------
1 | _2021-08-13-21-18-04
2 | _2021-08-13-22-03-03
3 | _2021-08-13-22-16-02
4 | 


--------------------------------------------------------------------------------
/MS2dataset/test_rainy_list.txt:
--------------------------------------------------------------------------------
1 | _2021-08-06-16-19-00
2 | _2021-08-06-16-45-28
3 | _2021-08-06-16-59-13
4 | 


--------------------------------------------------------------------------------
/MS2dataset/train_list.txt:
--------------------------------------------------------------------------------
1 | _2021-08-06-10-59-33
2 | _2021-08-06-11-37-46
3 | _2021-08-06-17-21-04
4 | _2021-08-06-17-44-55
5 | _2021-08-13-16-50-57
6 | _2021-08-13-17-06-04
7 | _2021-08-13-21-36-10
8 | _2021-08-13-22-36-41
9 | 


--------------------------------------------------------------------------------
/MS2dataset/val_list.txt:
--------------------------------------------------------------------------------
1 | _2021-08-13-16-08-46
2 | _2021-08-13-16-14-48
3 | _2021-08-13-21-58-13
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Multi-Spectral Stereo ($MS^2$) Outdoor Driving Dataset
 2 | 
 3 | This is the official github page of the $MS^2$ dataset described in the following paper.
 4 | 
 5 | This page provides a dataloader and simple python code for $MS^2$ dataset.
 6 | 
 7 | If you want to download the dataset and see the details, please visit the [dataset page](https://sites.google.com/view/multi-spectral-stereo-dataset).
 8 | 
 9 |  >Deep Depth Estimation from Thermal Image
10 |  >
11 |  >[Ukcheol Shin](https://ukcheolshin.github.io/), Jinsun Park, In So Kweon
12 |  >
13 |  >IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 2023
14 |  >
15 |  >[[Paper](https://openaccess.thecvf.com/content/CVPR2023/papers/Shin_Deep_Depth_Estimation_From_Thermal_Image_CVPR_2023_paper.pdf)] [[Dataset page](https://sites.google.com/view/multi-spectral-stereo-dataset)]
16 | 
17 | ## Updates
18 | - 2023.03.30: Open Github page.
19 | - 2023.05.30: Release $MS^2$ dataset, dataloader, and demo code.
20 | 
21 | ## $MS^2$ Dataset Specification
22 | MS2 dataset provides:
23 | - (Synchronized) Stereo RGB images / Stereo NIR images / Stereo thermal images
24 | - (Synchronized) Stereo LiDAR scans / GPS/IMU navigation data
25 | - Projected depth map (in RGB, NIR, thermal image planes)
26 | - Odometry data (in RGB, NIR, thermal cameras, and LiDAR coordinates)
27 | 
28 | ## Usage
29 | 1. Download the datasets and place them in 'MS2dataset' folder in the following structure:
30 | 
31 | ```shell
32 | MS2dataset
33 | ├── sync_data
34 | │   ├── <Sequence Name1>
35 | │   ├── <Sequence Name2>
36 | │   ├── ...
37 | │   └── <Sequence NameN>
38 | ├── proj_depth
39 | │   ├── <Sequence Name1>
40 | │   ├── <Sequence Name2>
41 | │   ├── ...
42 | │   └── <Sequence NameN>
43 | └── odom
44 |     ├── <Sequence Name1>
45 |     ├── <Sequence Name2>
46 |     ├── ...
47 |     └── <Sequence NameN>
48 | ```
49 | 
50 | 2. We provide a simple python code (demo.py) along with a dataloader to take a look at the provided dataset.
51 | To run the code, you need any version of Pytorch library.
52 | ```bash
53 | python demo.py --seq_name <Sequence Name> --modality rgb --data_format MonoDepth
54 | python demo.py --seq_name <Sequence Name> --modality nir --data_format StereoMatch
55 | python demo.py --seq_name <Sequence Name> --modality thr --data_format MultiViewImg
56 | ```
57 | 


--------------------------------------------------------------------------------
/dataloader/MS2_dataset.py:
--------------------------------------------------------------------------------
  1 | # Written by Ukcheol Shin (shinwc159[at]gmail.com)
  2 | import torch
  3 | import torch.utils.data as data
  4 | import numpy as np
  5 | import os.path as osp
  6 | 
  7 | import random
  8 | from path import Path
  9 | from utils.utils import load_as_float_img, load_as_float_depth
 10 | 
 11 | class DataLoader_MS2(data.Dataset):
 12 |     """A data loader where the files are arranged in this way:
 13 |         * Structure of "KAIST MS2 dataset"
 14 |         |--sync_data
 15 |             |-- <Seq name>
 16 |                 |-- rgb, nir, thr
 17 |                     |-- img_left
 18 |                     |-- img_right
 19 |                 |-- lidar
 20 |                     |-- left
 21 |                     |-- right
 22 |                 |-- gps_imu
 23 |                     |-- data
 24 |                 |-- calib.npy
 25 |         |--proj_depth
 26 |             |-- <Seq name>
 27 |                 |-- rgb, nir, thr
 28 |                     |-- depth
 29 |                     |-- intensity
 30 |                     |-- depth_multi
 31 |                     |-- intensity_multi
 32 |                     |-- depth_filtered 
 33 |         |--odom
 34 |             |-- <Seq name>
 35 |                 |-- rgb, nir, thr, odom
 36 |         |-- train_list.txt
 37 |         |-- val_list.txt
 38 |         |-- test_list.txt
 39 |     """
 40 |     def __init__(self, root, seed=None, data_split='train', modality='thr', \
 41 |                  data_format='MonoDepth', sampling_step=3, set_length=3, set_interval=1, opt=None):
 42 |         super(DataLoader_MS2, self).__init__()
 43 | 
 44 |         np.random.seed(seed)
 45 |         random.seed(seed)
 46 | 
 47 |         # read (train/val/test) data list
 48 |         self.root = Path(root)
 49 |         if data_split == 'train':
 50 |             data_list_file = self.root/'train_list.txt' 
 51 |         elif data_split == 'val':
 52 |             data_list_file = self.root/'val_list.txt' 
 53 |         elif data_split == 'test':
 54 |             data_list_file = self.root/'test_list.txt' 
 55 |         elif data_split == 'test_day':
 56 |             data_list_file = self.root/'test_day_list.txt' 
 57 |         elif data_split == 'test_night':
 58 |             data_list_file = self.root/'test_night_list.txt' 
 59 |         elif data_split == 'test_rain':
 60 |             data_list_file = self.root/'test_rainy_list.txt' 
 61 |         else: # when data_split is a specific sequence name
 62 |             data_list_file = data_split 
 63 | 
 64 |         # check if data_list_file has the .txt extension and create a list of folders
 65 |         if 'txt' in data_list_file:                
 66 |             self.seq_list = [seq_name[:-1] for seq_name in open(data_list_file)]
 67 |         else:
 68 |             self.seq_list = [data_list_file]
 69 | 
 70 |         self.modality = modality
 71 |         self.extrinsics = self.set_extrinsics()
 72 | 
 73 |         # determine which data getter function to use 
 74 |         if data_format == 'MonoDepth': # Monocular depth estimation, dict: {'img', 'depth'}
 75 |             self.data_getter = self.get_data_MonoDepth
 76 |             self.crawl_folders_depth(sampling_step, set_length, set_interval)
 77 |         elif data_format == 'StereoMatch':
 78 |             self.data_getter = self.get_data_StereoMatching
 79 |             self.crawl_folders_depth(sampling_step, set_length, set_interval)
 80 |         elif data_format == 'MultiViewImg':
 81 |             self.data_getter = self.get_data_MultiViewImg
 82 |             self.crawl_folders_depth(sampling_step, set_length, set_interval)
 83 |         elif data_format == 'Odometry':
 84 |             self.data_getter = self.get_data_Odometry
 85 |             self.crawl_folders_pose(sampling_step, set_length, set_interval)
 86 |         else:
 87 |             raise NotImplementedError(f"not supported type {data_format} in KAIST MS2 dataset.")
 88 | 
 89 |     def __getitem__(self, index):
 90 |         if isinstance(self.modality, list):
 91 |             results = {}
 92 |             if "rgb" in self.modality: results["rgb"] = self.data_getter(index, 'rgb')
 93 |             if "nir" in self.modality: results["nir"] = self.data_getter(index, 'nir')
 94 |             if "thr" in self.modality: results["thr"] = self.data_getter(index, 'thr')
 95 |             results["extrinsics"] = self.get_extrinsic()
 96 |             return results
 97 |         else:
 98 |             return self.data_getter(index, self.modality)
 99 | 
100 |     def __len__(self):
101 |         if isinstance(self.modality, list):
102 |             return len(self.samples['rgb'])
103 |         else:
104 |             return len(self.samples[self.modality])
105 | 
106 |     def set_extrinsics(self) :
107 |         # extrinsics matries are all same across the sequences, thus use the same values
108 |         calib_path = osp.join(self.root, "sync_data", self.seq_list[0], "calib.npy")
109 |         calib = np.load(calib_path, allow_pickle=True).item()
110 | 
111 |         ext_NIR2THR = np.concatenate([calib['R_nir2thr'], calib['T_nir2thr']*0.001], axis=1) # mm -> m scale conversion.
112 |         ext_NIR2RGB = np.concatenate([calib['R_nir2rgb'], calib['T_nir2rgb']*0.001], axis=1)
113 | 
114 |         ext_THR2NIR = np.linalg.inv(np.concatenate([ext_NIR2THR, [[0,0,0,1]]],axis=0))
115 |         ext_THR2RGB = np.matmul(np.concatenate([ext_NIR2RGB, [[0,0,0, 1]]],axis=0), ext_THR2NIR)
116 | 
117 |         ext_RGB2NIR = np.linalg.inv(np.concatenate([ext_NIR2RGB, [[0,0,0,1]]],axis=0))
118 |         ext_RGB2THR = np.linalg.inv(ext_THR2RGB)
119 | 
120 |         extrinsics = {}
121 |         extrinsics["NIR2THR"] = torch.as_tensor(ext_NIR2THR)
122 |         extrinsics["NIR2RGB"] = torch.as_tensor(ext_NIR2RGB)
123 | 
124 |         extrinsics["THR2NIR"] = torch.as_tensor(ext_THR2NIR[0:3,:])
125 |         extrinsics["THR2RGB"] = torch.as_tensor(ext_THR2RGB[0:3,:])
126 | 
127 |         extrinsics["RGB2NIR"] = torch.as_tensor(ext_RGB2NIR[0:3,:])
128 |         extrinsics["RGB2THR"] = torch.as_tensor(ext_RGB2THR[0:3,:])
129 |         return extrinsics
130 | 
131 |     def crawl_folders_depth(self, sampling_step, set_length, set_interval):
132 |         # Args:
133 |         # - sampling_step: sampling data from the crawled sample sets every N step. 
134 |         # - set_length: control length of each sample set 
135 |         #               if length = 1, each sample set only has time T sample, (i.e., single-view)
136 |         #               if length > 1, each sample set has {T-N, ..., T, ..., T+N} (i.e., multi-view)
137 |         # - set_interval: control interval between samples within sample set 
138 |         #               if set_interval = N, each sample set has {T-N*M, ..., T-N, T, T+N, .., T+N*M}
139 |         # if you want to do monocular vision task, set set_length = 1, set_interval = 1
140 |         # if you want to do multi-view vision task, set set_length > 1 (odd number), set_interval >= 1
141 |         # e.g., sampling_step = 2, set_length = 3, set_interval = 1
142 |         # final sample sets = {{T-1, T, T+1}, {(T+2)-1, (T+2), (T+2)+1}, ...}
143 | 
144 |         # define shifts to select reference frames
145 |         demi_length = (set_length-1)//2 
146 |         shifts = [set_interval*i for i in range(-demi_length, demi_length + 1)]
147 |         for i in range(1, 2*demi_length):
148 |             shifts.pop(1)
149 | 
150 |         # crawl the request modality list
151 |         sensor_list = []
152 |         if isinstance(self.modality, list):
153 |             for modal in self.modality: 
154 |                 sensor_list.append(modal)
155 |         else:
156 |             sensor_list.append(self.modality)
157 | 
158 |         # iterate over sensor modalities
159 |         sample_sets = {}
160 |         for sensor in sensor_list:               
161 |             # create an empty list to store sample set for current sensor modality
162 |             sample_set = []
163 |             for seq in self.seq_list: # iterate over each sequence
164 |                 calib_path = osp.join(self.root, "sync_data", seq, "calib.npy")
165 |                 calib = np.load(calib_path, allow_pickle=True).item()
166 | 
167 |                 # read intrinsics and extrinsics (L/R) for current sensor modality
168 |                 if sensor == "rgb":
169 |                     intrinsics     = calib['K_rgbL'].astype(np.float32)
170 |                     extrinsics_L2R = np.concatenate([calib['R_rgbR'],calib['T_rgbR']*0.001], axis=1).astype(np.float32)
171 |                     baseline       = abs(calib['T_rgbR'][0].astype(np.float32))*0.001 # convert to the meter scale
172 |                 elif sensor == "nir":
173 |                     intrinsics     = calib['K_nirL'].astype(np.float32)
174 |                     extrinsics_L2R = np.concatenate([calib['R_nirR'],calib['T_nirR']*0.001], axis=1).astype(np.float32)
175 |                     baseline       = abs(calib['T_nirR'][0].astype(np.float32))*0.001 
176 |                 elif sensor == "thr":
177 |                     intrinsics     = calib['K_thrL'].astype(np.float32)
178 |                     extrinsics_L2R = np.concatenate([calib['R_thrR'],calib['T_thrR']*0.001], axis=1).astype(np.float32)
179 |                     baseline       = abs(calib['T_thrR'][0].astype(np.float32))*0.001 
180 | 
181 |                 img_list_left      = sorted((self.root/"sync_data"/seq/sensor/"img_left").files('*.png')) 
182 |                 img_list_right     = sorted((self.root/"sync_data"/seq/sensor/"img_right").files('*.png')) 
183 | 
184 |                 # crawl the data path
185 |                 init_offset = demi_length*set_interval
186 |                 for i in range(init_offset, len(img_list_left)-init_offset):
187 |                     # file name is all same across the folders (left, right, depth, depth_filterd, intensity, odom)
188 |                     tgt_name  = img_list_left[i].name[:-4]
189 |                     depth_in  = self.root/"proj_depth"/seq/sensor/"depth"/(tgt_name + '.png')
190 |                     inten_in  = self.root/"proj_depth"/seq/sensor/"intensity"/(tgt_name + '.png')
191 |                     depth_gt  = self.root/"proj_depth"/seq/sensor/"depth_filtered"/(tgt_name + '.png')
192 | 
193 |                     # create a dictionary containing information about the target and reference images, depth maps, etc.
194 |                     sample = {'intrinsics': intrinsics, 
195 |                               'tgt_img_left': img_list_left[i], 'tgt_img_right': img_list_right[i], 
196 |                               'ref_imgs_left': [], 'ref_imgs_right': [],
197 |                               'tgt_depth_in' : depth_in, 'tgt_depth_gt' : depth_gt,
198 |                               'tgt_inten_in' : inten_in, 'ref_intens_in' : [],
199 |                               'ref_depths_in' : [], 'ref_depths_gt' : [],
200 |                               'baseline' : baseline, 'extrinsics_L2R' : extrinsics_L2R,
201 |                              }
202 | 
203 |                     # Loop over the neighboring images to create the reference image list for the central image
204 |                     for j in shifts:
205 |                         ref_name       = img_list_left[i+j].name[:-4]
206 |                         depth_in_  = self.root/"proj_depth"/seq/sensor/"depth"/(ref_name + '.png')
207 |                         inten_in_  = self.root/"proj_depth"/seq/sensor/"intensity"/(ref_name + '.png')
208 |                         depth_gt_  = self.root/"proj_depth"/seq/sensor/"depth_filtered"/(ref_name + '.png')
209 | 
210 |                         # append reference image, depth map, etc. to the sample dictionary
211 |                         sample['ref_imgs_left'].append(img_list_left[i+j])
212 |                         sample['ref_imgs_right'].append(img_list_right[i+j])
213 |                         sample['ref_depths_in'].append(depth_in_)
214 |                         sample['ref_intens_in'].append(inten_in_)
215 |                         sample['ref_depths_gt'].append(depth_gt_)
216 | 
217 |                     sample_set.append(sample)
218 | 
219 |             # Subsampling the list of images according to the sampling step
220 |             sample_set  = sample_set[0:-1:sampling_step]
221 |             sample_sets[sensor] = sample_set
222 |         self.samples = sample_sets
223 | 
224 |     def crawl_folders_pose(self, sampling_step, set_length, set_interval):
225 |         # define shifts to select reference frames
226 |         demi_length = (set_length-1)//2
227 |         shift_range = np.array([set_interval*i for i in range(-demi_length, demi_length + 1)]).reshape(1, -1)
228 | 
229 |         # crawl the request modality list
230 |         sensor_list = []
231 |         if isinstance(self.modality, list):
232 |             for modal in self.modality: 
233 |                 sensor_list.append(modal)
234 |         else:
235 |             sensor_list.append(self.modality)
236 | 
237 |         # iterate over each sensor modalitiy
238 |         sample_sets = {}
239 |         for sensor in sensor_list:               
240 |             # create an empty list to store samples for this sensor modality
241 |             sample_set = []
242 |             for seq in self.seq_list: # iterate over each folder
243 |                 calib_path = osp.join(self.root, "sync_data", seq, "calib.npy")
244 |                 calib = np.load(calib_path, allow_pickle=True).item()
245 | 
246 |                 img_list_left      = sorted((self.root/"sync_data"/seq/sensor/"img_left").files('*.png')) 
247 |                 img_list_right     = sorted((self.root/"sync_data"/seq/sensor/"img_right").files('*.png')) 
248 | 
249 |                 # construct N-snippet sequences (note: N=set_length)
250 |                 init_offset = demi_length*set_interval
251 |                 tgt_indices = np.arange(init_offset, len(img_list_left) - init_offset).reshape(-1, 1)
252 |                 snippet_indices = shift_range + tgt_indices
253 |     
254 |                 for indices in snippet_indices :
255 |                     sample = {'imgs' : [], 'poses' : []}
256 |                     for i in indices :
257 |                         tgt_name = img_list_left[i].name[:-4]
258 |                         pose  = self.root/"odom"/seq/sensor/(tgt_name + '.txt')
259 |                         sample['imgs'].append(img_list_left[i])
260 |                         sample['poses'].append(pose)
261 |                     sample_set.append(sample)
262 | 
263 |             # Subsampling the list of images according to the sampling step
264 |             sample_set  = sample_set[0:-1:sampling_step]
265 |             sample_sets[sensor] = sample_set
266 | 
267 |         self.samples = sample_sets
268 | 
269 |     def depth2disp(self, depth, focal, baseline):
270 |         min_depth = 1e-3
271 |         mask = (depth < min_depth) 
272 | 
273 |         disp = baseline * focal / (depth +1e-10)
274 |         disp[mask] = 0.
275 |         return disp
276 | 
277 |     def get_extrinsic(self):
278 |         return self.extrinsics
279 | 
280 |     # For monocular depth estimation
281 |     def get_data_MonoDepth(self, index, modality):
282 |         sample = self.samples[modality][index]
283 | 
284 |         tgt_img = load_as_float_img(sample['tgt_img_left'])
285 |         tgt_depth_gt = load_as_float_depth(sample['tgt_depth_gt'])/256.0
286 | 
287 |         result = {}
288 |         result["tgt_image"]    = tgt_img
289 |         result["tgt_depth_gt"] = tgt_depth_gt
290 |         return result
291 | 
292 |     # For stereo matching 
293 |     def get_data_StereoMatching(self, index, modality):
294 |         sample = self.samples[modality][index]
295 | 
296 |         tgt_img_left  = load_as_float_img(sample['tgt_img_left'])
297 |         tgt_img_right = load_as_float_img(sample['tgt_img_right'])
298 |         tgt_depth_gt  = load_as_float_depth(sample['tgt_depth_gt'])/256.0
299 | 
300 |         result = {}
301 |         result["tgt_left"]       = tgt_img_left
302 |         result["tgt_right"]      = tgt_img_right
303 |         result["tgt_disp_gt"]    = torch.as_tensor(self.depth2disp(tgt_depth_gt,\
304 |                                      focal=torch.as_tensor(np.copy(sample['intrinsics'][0,0])),\
305 |                                      baseline=torch.as_tensor(sample['baseline'])))
306 |         return result
307 | 
308 |     # For self-supervised monocular depth estimation
309 |     def get_data_MultiViewImg(self, index, modality):
310 |         sample = self.samples[modality][index]
311 | 
312 |         tgt_img  = load_as_float_img(sample['tgt_img_left'])
313 |         ref_imgs = [load_as_float_img(ref_img) for ref_img in sample['ref_imgs_left']]
314 | 
315 |         result = {}
316 |         result["tgt_image"]     = tgt_img
317 |         result["ref_images"]    = ref_imgs
318 |         result["intrinsics"]    = np.copy(sample['intrinsics'])
319 |         return result
320 | 
321 |     # For multi-view pose estimation
322 |     def get_data_Odometry(self, index, modality):
323 |         sample = self.samples[modality][index]
324 | 
325 |         tgt_imgs = [load_as_float_img(img) for img in sample['imgs']]
326 |         pose    = np.stack([np.genfromtxt(pose).astype(np.float64).reshape(4,4)[:3,:] for pose in sample['poses']]) # 3x4
327 | 
328 |         # uncomment if you need pose w.r.t img 0 within the list "tgt_imgs"
329 |         # first_pose = poses[0]
330 |         # poses[:,:,-1] -= first_pose[:,-1]
331 |         # compensated_poses = np.linalg.inv(first_pose[:,:3]) @ poses
332 | 
333 |         result = {}
334 |         result["image"]    = tgt_imgs
335 |         result["pose"]     = torch.as_tensor(pose) # Abloste pose (w.r.t first image of the sequence)
336 |         return result


--------------------------------------------------------------------------------
/demo.py:
--------------------------------------------------------------------------------
  1 | # Written by Ukcheol Shin (shinwc159[at]gmail.com)
  2 | import torch
  3 | from tqdm import tqdm
  4 | from argparse import ArgumentParser
  5 | from torch.utils.data import DataLoader
  6 | 
  7 | import matplotlib.pyplot as plt
  8 | from dataloader.MS2_dataset import DataLoader_MS2
  9 | from utils.utils import visualize_disp_as_numpy, visualize_depth_as_numpy, Raw2Celsius
 10 | 
 11 | def parse_args():
 12 |     parser = ArgumentParser()
 13 |     parser.add_argument('--dataset_dir' , type=str, default='./MS2dataset')
 14 |     parser.add_argument('--modality', type=str, default='rgb', help='sensor modality: [rgb, nir ,thr]')
 15 |     parser.add_argument('--seq_name', type=str, default='_2021-08-06-10-59-33', help='sequence name')
 16 |     parser.add_argument('--data_format', type=str, default='MonoDepth', help='[MonoDepth, StereoMatch, MultiViewImg]')
 17 |     return parser.parse_args()
 18 | 
 19 | def main():
 20 |     args = parse_args()
 21 | 
 22 |     dataset_dir = args.dataset_dir
 23 |     seq_name    = args.seq_name
 24 |     data_format = args.data_format
 25 |     modality    = args.modality
 26 | 
 27 |     if data_format == 'MonoDepth':
 28 |       sampling_step = 50
 29 |       set_length = 1
 30 |       set_interval = 1
 31 |     elif data_format == 'StereoMatch':
 32 |       sampling_step = 50
 33 |       set_length = 1
 34 |       set_interval = 1
 35 |     elif data_format == 'MultiViewImg':
 36 |       sampling_step = 50
 37 |       set_length = 3
 38 |       set_interval = 5
 39 | 
 40 |     dataset       =   DataLoader_MS2(
 41 |                       dataset_dir,
 42 |                       data_split = seq_name,                                  
 43 |                       data_format = data_format,
 44 |                       modality=modality,
 45 |                       sampling_step=sampling_step,
 46 |                       set_length=set_length,
 47 |                       set_interval=set_interval
 48 |                   )
 49 | 
 50 |     demo_loader   = DataLoader(dataset, 
 51 |                               batch_size=1,
 52 |                               shuffle=False, 
 53 |                               num_workers=1, 
 54 |                               drop_last=False)
 55 | 
 56 |     print('{} samples found for evaluation'.format(len(demo_loader)))
 57 | 
 58 |     for i, batch in enumerate(tqdm(demo_loader)):
 59 |       if data_format == 'MonoDepth':
 60 |           if modality == 'thr':
 61 |             img = Raw2Celsius(batch["tgt_image"])
 62 |           else:
 63 |             img = batch["tgt_image"].type(torch.uint8)
 64 | 
 65 |           depth_gt = batch["tgt_depth_gt"]
 66 | 
 67 |           plt.subplot(2,1,1)
 68 |           plt.imshow(img[0])
 69 |           plt.subplot(2,1,2)
 70 |           plt.imshow(visualize_depth_as_numpy(depth_gt[0]))
 71 |           plt.pause(0.5)
 72 | 
 73 |       elif data_format == 'StereoMatch':
 74 |           if modality == 'thr':
 75 |             imgL = Raw2Celsius(batch["tgt_left"])
 76 |             imgR = Raw2Celsius(batch["tgt_right"])
 77 |           else:
 78 |             imgL = batch["tgt_left"].type(torch.uint8)
 79 |             imgR = batch["tgt_right"].type(torch.uint8)
 80 | 
 81 |           disp_gt = batch["tgt_disp_gt"]
 82 | 
 83 |           plt.subplot(3,1,1)
 84 |           plt.imshow(imgL[0])
 85 |           plt.subplot(3,1,2)
 86 |           plt.imshow(imgR[0])
 87 |           plt.subplot(3,1,3)
 88 |           plt.imshow(visualize_disp_as_numpy(disp_gt[0]))
 89 |           plt.pause(0.5)
 90 | 
 91 |       elif data_format == 'MultiViewImg':
 92 |           if modality == 'thr':
 93 |             tgt_img  = Raw2Celsius(batch["tgt_image"])
 94 |             ref_imgs = [Raw2Celsius(img) for img in batch["ref_images"]]
 95 |           else:
 96 |             tgt_img  = batch["tgt_image"].type(torch.uint8)
 97 |             ref_imgs = [img.type(torch.uint8) for img in batch["ref_images"]]
 98 | 
 99 |           plt.subplot(3,1,1) 
100 |           plt.imshow(ref_imgs[0][0]) # T-N
101 |           plt.subplot(3,1,2)
102 |           plt.imshow(tgt_img[0]) # T
103 |           plt.subplot(3,1,3)
104 |           plt.imshow(ref_imgs[1][0]) # T+N
105 |           plt.pause(0.5)
106 | 
107 | if __name__ == '__main__':
108 |     main()
109 | 


--------------------------------------------------------------------------------
/utils/utils.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | from imageio import imread
 3 | from scipy.io import loadmat
 4 | import matplotlib as mpl
 5 | import matplotlib.cm as cm
 6 | 
 7 | def load_as_float_img(path):
 8 |     img =  imread(path).astype(np.float32)
 9 |     if len(img.shape) == 2: # for NIR and thermal images
10 |         img = np.expand_dims(img, axis=2)
11 |     return img
12 | 
13 | def load_as_float_depth(path):
14 |     if 'png' in path:
15 |         depth =  np.array(imread(path).astype(np.float32))
16 |     elif 'npy' in path:
17 |         depth =  np.load(path).astype(np.float32)
18 |     elif 'mat' in path:
19 |         depth =  loadmat(path).astype(np.float32)
20 |     return depth
21 | 
22 | # Tempearture to raw thermal radiation value
23 | # For the parameters (R,B,F,O), we use default values of FLIR A65C camera
24 | def Celsius2Raw(celcius_degree):
25 |     R = 380747
26 |     B = 1428
27 |     F = 1
28 |     O = -88.539
29 |     raw_value = R / (np.exp(B / (celcius_degree + 273.15)) - F) + O;
30 |     return raw_value
31 | 
32 | # Raw thermal radiation value to tempearture 
33 | def Raw2Celsius(Raw):
34 |     R = 380747
35 |     B = 1428
36 |     F = 1
37 |     O = -88.539
38 |     Celsius = B / np.log(R / (Raw - O) + F) - 273.15;
39 |     return Celsius
40 | 
41 | def visualize_disp_as_numpy(disp, cmap='jet'):
42 |     """
43 |     Args:
44 |         data (HxW): disp data
45 |         cmap: color map (inferno, plasma, jet, turbo, magma, rainbow)
46 |     Returns:
47 |         vis_data (HxWx3): disp visualization (RGB)
48 |     """
49 | 
50 |     disp = disp.cpu().numpy()
51 |     disp = np.nan_to_num(disp)  # change nan to 0
52 | 
53 |     vmin = np.percentile(disp[disp!=0], 0)
54 |     vmax = np.percentile(disp[disp!=0], 95)
55 | 
56 |     normalizer = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
57 |     mapper = cm.ScalarMappable(norm=normalizer, cmap=cmap)
58 |     vis_data = (mapper.to_rgba(disp)[:, :, :3] * 255).astype(np.uint8)
59 |     return vis_data
60 | 
61 | def visualize_depth_as_numpy(depth, cmap='jet', is_sparse=True):
62 |     """
63 |     Args:
64 |         data (HxW): depth data
65 |         cmap: color map (inferno, plasma, jet, turbo, magma, rainbow)
66 |     Returns:
67 |         vis_data (HxWx3): depth visualization (RGB)
68 |     """
69 | 
70 |     x = depth.cpu().numpy()
71 |     x = np.nan_to_num(x)  # change nan to 0
72 | 
73 |     inv_depth = 1 / (x + 1e-6)
74 | 
75 |     if is_sparse:
76 |         vmax = 1/np.percentile(x[x!=0], 5)
77 |     else:
78 |         vmax = np.percentile(inv_depth, 95)
79 | 
80 |     normalizer = mpl.colors.Normalize(vmin=inv_depth.min(), vmax=vmax)
81 |     mapper = cm.ScalarMappable(norm=normalizer, cmap=cmap)
82 |     vis_data = (mapper.to_rgba(inv_depth)[:, :, :3] * 255).astype(np.uint8)
83 |     if is_sparse:
84 |         vis_data[inv_depth>vmax] = 0
85 |     return vis_data


--------------------------------------------------------------------------------