├── .gitignore ├── LICENSE ├── README.md └── scripts ├── README.md ├── __init__.py ├── check_disparity_submission.py ├── check_optical_flow_submission.py ├── dataloading_example.py ├── dataset ├── provider.py ├── representations.py ├── sequence.py └── visualization.py ├── events_to_video.py ├── utils ├── __init__.py └── eventslicer.py └── visualization ├── __init__.py └── eventreader.py /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | pip-wheel-metadata/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | db.sqlite3 64 | db.sqlite3-journal 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 97 | __pypackages__/ 98 | 99 | # Celery stuff 100 | celerybeat-schedule 101 | celerybeat.pid 102 | 103 | # SageMath parsed files 104 | *.sage.py 105 | 106 | # Environments 107 | .env 108 | .venv 109 | env/ 110 | venv/ 111 | ENV/ 112 | env.bak/ 113 | venv.bak/ 114 | 115 | # Spyder project settings 116 | .spyderproject 117 | .spyproject 118 | 119 | # Rope project settings 120 | .ropeproject 121 | 122 | # mkdocs documentation 123 | /site 124 | 125 | # mypy 126 | .mypy_cache/ 127 | .dmypy.json 128 | dmypy.json 129 | 130 | # Pyre type checker 131 | .pyre/ 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Robotics and Perception Group 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # News 2 | 3 | - **Nov. 26, 2022** - Lidar and IMU data is now available on the [download page](https://dsec.ifi.uzh.ch/dsec-datasets/download/). 4 | 5 | # DSEC 6 | 7 |

8 | 9 | 10 |

11 | 12 | **DSEC**: A Stereo Event Camera Dataset for Driving Scenarios 13 | 14 | This is code accompanying the dataset and paper by [Mathias Gehrig](https://magehrig.github.io/), Willem Aarents, [Daniel Gehrig](https://danielgehrig18.github.io/) and [Davide Scaramuzza](http://rpg.ifi.uzh.ch/people_scaramuzza.html) 15 | 16 | Visit the [project webpage](https://dsec.ifi.uzh.ch/) to download the dataset. 17 | 18 | If you use this code in an academic context, please cite the following work: 19 | 20 | ```bibtex 21 | @InProceedings{Gehrig21ral, 22 | author = {Mathias Gehrig and Willem Aarents and Daniel Gehrig and Davide Scaramuzza}, 23 | title = {{DSEC}: A Stereo Event Camera Dataset for Driving Scenarios}, 24 | journal = {{IEEE} Robotics and Automation Letters}, 25 | year = {2021}, 26 | doi = {10.1109/LRA.2021.3068942} 27 | } 28 | ``` 29 | and 30 | ```bibtex 31 | @InProceedings{Gehrig3dv2021, 32 | author = {Mathias Gehrig and Mario Millh\"ausler and Daniel Gehrig and Davide Scaramuzza}, 33 | title = {E-RAFT: Dense Optical Flow from Event Cameras}, 34 | booktitle = {International Conference on 3D Vision (3DV)}, 35 | year = {2021} 36 | } 37 | ``` 38 | 39 | ## Install 40 | 41 | In this repository we provide code for loading data and verifying the submission for the benchmarks. For details regarding the dataset, visit the [DSEC webpage](https://dsec.ifi.uzh.ch/). 42 | 43 | 1. Clone 44 | 45 | ```bash 46 | git clone git@github.com:uzh-rpg/DSEC.git 47 | ``` 48 | 49 | 2. Install conda environment to run example code 50 | ```bash 51 | conda create -n dsec python=3.8 52 | conda activate dsec 53 | conda install -y -c anaconda numpy 54 | conda install -y -c numba numba 55 | conda install -y -c conda-forge h5py blosc-hdf5-plugin opencv scikit-video tqdm prettytable imageio 56 | # only for dataset loading: 57 | conda install -y -c pytorch pytorch torchvision cudatoolkit=10.2 58 | # only for visualilzation in the dataset loading: 59 | conda install -y -c conda-forge matplotlib 60 | ``` 61 | 62 | ## Disparity Evaluation 63 | 64 | We provide a [python script](scripts/check_disparity_submission.py) to ensure that the structure of the submission directory is correct. 65 | Usage example: 66 | 67 | ```Python 68 | python check_disparity_submission.py SUBMISSION_DIR EVAL_DISPARITY_TIMESTAMPS_DIR 69 | ``` 70 | 71 | where `EVAL_DISPARITY_TIMESTAMPS_DIR` is the path to the unzipped directory containing evaluation timestamps. It can [downloaded on the webpage](https://dsec.ifi.uzh.ch/dsec-datasets/download/) or directly [here](https://download.ifi.uzh.ch/rpg/DSEC/test_disparity_timestamps.zip). 72 | `SUBMISSION_DIR` is the path to the directory containing your submission. 73 | 74 | Follow the instructions on the [webpage](https://dsec.ifi.uzh.ch/disparity-submission-format/) for a detailed description of the submission format. 75 | 76 | ## Optical Flow Evaluation 77 | 78 | We provide a [python script](scripts/check_optical_flow_submission.py) to ensure that the structure of the submission directory is correct. 79 | Usage example: 80 | 81 | ```Python 82 | python check_optical_flow_submission.py SUBMISSION_DIR EVAL_FLOW_TIMESTAMPS_DIR 83 | ``` 84 | 85 | where `EVAL_FLOW_TIMESTAMPS_DIR` is the path to the unzipped directory containing evaluation timestamps. It can [downloaded on the webpage](https://dsec.ifi.uzh.ch/dsec-datasets/download/) or directly [here](https://download.ifi.uzh.ch/rpg/DSEC/test_forward_optical_flow_timestamps.zip). 86 | `SUBMISSION_DIR` is the path to the directory containing your submission. 87 | 88 | Follow the instructions on the [webpage](https://dsec.ifi.uzh.ch/optical-flow-submission-format/) for a detailed description of the submission format. 89 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | This directory contains example code 2 | - to convert an h5 event file into a video file for visualization, and 3 | - for loading data in pytorch. 4 | - to check the structure of the submission directory 5 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uzh-rpg/DSEC/af90ef112c175ddcb29d7bfefde517a877c18172/scripts/__init__.py -------------------------------------------------------------------------------- /scripts/check_disparity_submission.py: -------------------------------------------------------------------------------- 1 | import sys 2 | version = sys.version_info 3 | assert version[0] >= 3, 'Python 2 is not supported' 4 | assert version[1] >= 6, 'Requires Python 3.6 or higher' 5 | 6 | import argparse 7 | import os 8 | from pathlib import Path 9 | from typing import Dict 10 | 11 | import numpy as np 12 | 13 | has_cv2 = True 14 | try: 15 | import cv2 16 | except ImportError: 17 | has_cv2 = False 18 | 19 | try: 20 | from PIL import Image 21 | except ImportError: 22 | assert has_cv2, 'Either install opencv-python or Pillow' 23 | 24 | 25 | def is_string_swiss(input_str: str) -> bool: 26 | is_swiss = False 27 | is_swiss |= 'thun_' in input_str 28 | is_swiss |= 'interlaken_' in input_str 29 | is_swiss |= 'zurich_city_' in input_str 30 | return is_swiss 31 | 32 | 33 | def load_disparity(filepath: Path): 34 | assert filepath.is_file() 35 | assert filepath.suffix == '.png', filepath.suffix 36 | if has_cv2: 37 | disp = cv2.imread(str(filepath), cv2.IMREAD_ANYDEPTH).astype("float32") / 256.0 38 | else: 39 | disp = np.array(Image.open(str(filepath))).astype("float32") / 256.0 40 | return disp 41 | 42 | 43 | def files_per_sequence(disparity_timestamps_dir: Path) -> Dict[str, int]: 44 | out_dict = dict() 45 | for entry in disparity_timestamps_dir.iterdir(): 46 | assert entry.is_file() 47 | assert entry.suffix == '.csv', entry.suffix 48 | assert is_string_swiss(entry.stem), entry.stem 49 | data = np.loadtxt(entry, dtype=np.int64, delimiter=', ', comments='#') 50 | assert data.ndim == 2, data.ndim 51 | num_files = data.shape[0] 52 | out_dict[entry.stem] = num_files 53 | return out_dict 54 | 55 | 56 | if __name__ == '__main__': 57 | parser = argparse.ArgumentParser() 58 | parser.add_argument('submission_dir', help='Path to submission directory') 59 | parser.add_argument('disparity_timestamps_dir', help='Path to directory containing the disparity timestamps for evaluation.') 60 | 61 | args = parser.parse_args() 62 | 63 | ref_disp_ts_dir = Path(args.disparity_timestamps_dir) 64 | assert ref_disp_ts_dir.is_dir() 65 | 66 | submission_dir = Path(args.submission_dir) 67 | assert submission_dir.is_dir() 68 | 69 | name2num = files_per_sequence(ref_disp_ts_dir) 70 | 71 | expected_disparity_shape = (480, 640) 72 | 73 | expected_dir_names = set([*name2num]) 74 | actual_dir_names = set(os.listdir(submission_dir)) 75 | assert expected_dir_names == actual_dir_names, f'Expected directories in your submission: {expected_dir_names}.\nMissing directories: {expected_dir_names.difference(actual_dir_names)}' 76 | 77 | for seq in submission_dir.iterdir(): 78 | assert seq.is_dir() 79 | assert is_string_swiss(seq.name), seq.name 80 | num_files = 0 81 | for prediction in seq.iterdir(): 82 | disparity = load_disparity(prediction) 83 | assert disparity.shape == expected_disparity_shape, f'Expected shape:{expected_disparity_shape}, actual shape: {disparity.shape}' 84 | assert disparity.min() >= 0, disparity.min() 85 | num_files += 1 86 | assert seq.name in [*name2num], f'{seq.name} not in {[*name2num]}' 87 | assert num_files == name2num[seq.name], f'expected {name2num[seq.name]} files in {str(seq)} but only found {num_files} files' 88 | 89 | print('Your submission directory has the correct structure: Ready to submit!\n') 90 | print('Note, that we will sort the files according to their names in each directory and evaluate them sequentially. Follow the exact naming instructions if you are unsure.') 91 | -------------------------------------------------------------------------------- /scripts/check_optical_flow_submission.py: -------------------------------------------------------------------------------- 1 | import sys 2 | version = sys.version_info 3 | assert version[0] >= 3, 'Python 2 is not supported' 4 | assert version[1] >= 6, 'Requires Python 3.6 or higher' 5 | 6 | import argparse 7 | from enum import Enum, auto 8 | import os 9 | os.environ['IMAGEIO_USERDIR'] = '/var/tmp' 10 | from pathlib import Path 11 | from typing import Dict 12 | 13 | import imageio 14 | imageio.plugins.freeimage.download() 15 | import numpy as np 16 | 17 | 18 | class WriteFormat(Enum): 19 | OPENCV = auto() 20 | IMAGEIO = auto() 21 | 22 | 23 | def is_string_swiss(input_str: str) -> bool: 24 | is_swiss = False 25 | is_swiss |= 'thun_' in input_str 26 | is_swiss |= 'interlaken_' in input_str 27 | is_swiss |= 'zurich_city_' in input_str 28 | return is_swiss 29 | 30 | 31 | def flow_16bit_to_float(flow_16bit: np.ndarray, valid_in_3rd_channel: bool): 32 | assert flow_16bit.dtype == np.uint16 33 | assert flow_16bit.ndim == 3 34 | h, w, c = flow_16bit.shape 35 | assert c == 3 36 | 37 | if valid_in_3rd_channel: 38 | valid2D = flow_16bit[..., 2] == 1 39 | assert valid2D.shape == (h, w) 40 | assert np.all(flow_16bit[~valid2D, -1] == 0) 41 | else: 42 | valid2D = np.ones_like(flow_16bit[..., 2], dtype=np.bool) 43 | valid_map = np.where(valid2D) 44 | 45 | # to actually compute something useful: 46 | flow_16bit = flow_16bit.astype('float') 47 | 48 | flow_map = np.zeros((h, w, 2)) 49 | flow_map[valid_map[0], valid_map[1], 0] = (flow_16bit[valid_map[0], valid_map[1], 0] - 2**15) / 128 50 | flow_map[valid_map[0], valid_map[1], 1] = (flow_16bit[valid_map[0], valid_map[1], 1] - 2**15) / 128 51 | return flow_map, valid2D 52 | 53 | 54 | def load_flow(flowfile: Path, valid_in_3rd_channel: bool, write_format=WriteFormat): 55 | assert flowfile.exists() 56 | assert flowfile.suffix == '.png' 57 | 58 | # imageio reading assumes write format was rgb 59 | flow_16bit = imageio.imread(str(flowfile), format='PNG-FI') 60 | if write_format == WriteFormat.OPENCV: 61 | # opencv writes as bgr -> flip last axis to get rgb 62 | flow_16bit = np.flip(flow_16bit, axis=-1) 63 | else: 64 | assert write_format == WriteFormat.IMAGEIO 65 | 66 | channel3 = flow_16bit[..., -1] 67 | assert channel3.max() <= 1, f'Maximum value in last channel should be 1: {flowfile}' 68 | flow, valid2D = flow_16bit_to_float(flow_16bit, valid_in_3rd_channel) 69 | return flow, valid2D 70 | 71 | 72 | def list_of_dirs(dirpath: Path): 73 | return next(os.walk(dirpath))[1] 74 | 75 | 76 | def files_per_sequence(flow_timestamps_dir: Path) -> Dict[str, int]: 77 | out_dict = dict() 78 | for entry in flow_timestamps_dir.iterdir(): 79 | assert entry.is_file() 80 | assert entry.suffix == '.csv', entry.suffix 81 | assert is_string_swiss(entry.stem), entry.stem 82 | data = np.loadtxt(entry, dtype=np.int64, delimiter=', ', comments='#') 83 | assert data.ndim == 2, data.ndim 84 | num_files = data.shape[0] 85 | out_dict[entry.stem] = num_files 86 | return out_dict 87 | 88 | 89 | def check_submission(submission_dir: Path, flow_timestamps_dir: Path): 90 | assert flow_timestamps_dir.is_dir() 91 | assert submission_dir.is_dir() 92 | 93 | name2num = files_per_sequence(flow_timestamps_dir) 94 | 95 | expected_flow_shape = (480, 640, 2) 96 | expected_valid_shape = (480, 640) 97 | expected_dir_names = set([*name2num]) 98 | actual_dir_names = set(list_of_dirs(submission_dir)) 99 | assert expected_dir_names == actual_dir_names, f'Expected directories in your submission: {expected_dir_names}.\nMissing directories: {expected_dir_names.difference(actual_dir_names)}' 100 | 101 | for seq in submission_dir.iterdir(): 102 | if not seq.is_dir(): 103 | continue 104 | assert seq.is_dir() 105 | assert is_string_swiss(seq.name), seq.name 106 | num_files = 0 107 | for prediction in seq.iterdir(): 108 | flow, valid_map = load_flow(prediction, valid_in_3rd_channel=False, write_format=WriteFormat.IMAGEIO) 109 | assert flow.shape == expected_flow_shape, f'Expected shape: {expected_flow_shape}, actual shape: {flow.shape}' 110 | assert valid_map.shape == expected_valid_shape, f'Expected shape: {expected_valid_shape}, actual shape: {valid_map.shape}' 111 | num_files += 1 112 | assert seq.name in [*name2num], f'{seq.name} not in {[*name2num]}' 113 | assert num_files == name2num[seq.name], f'expected {name2num[seq.name]} files in {str(seq)} but only found {num_files} files' 114 | 115 | return True 116 | 117 | if __name__ == '__main__': 118 | parser = argparse.ArgumentParser() 119 | parser.add_argument('submission_dir', help='Path to submission directory') 120 | parser.add_argument('flow_timestamps_dir', help='Path to directory containing the flow timestamps for evaluation.') 121 | 122 | args = parser.parse_args() 123 | 124 | print('start checking submission') 125 | check_submission(Path(args.submission_dir), Path(args.flow_timestamps_dir)) 126 | print('Your submission directory has the correct structure: Ready to submit!\n') 127 | print('Note, that we will sort the files according to their names in each directory and evaluate them sequentially. Follow the exact naming instructions if you are unsure.') 128 | -------------------------------------------------------------------------------- /scripts/dataloading_example.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import cv2 4 | import numpy as np 5 | import torch 6 | from torch.utils.data import DataLoader 7 | from tqdm import tqdm 8 | 9 | from dataset.provider import DatasetProvider 10 | from dataset.visualization import disp_img_to_rgb_img, show_disp_overlay, show_image 11 | 12 | 13 | if __name__ == "__main__": 14 | import argparse 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('dsec_dir', help='Path to DSEC dataset directory') 17 | parser.add_argument('--visualize', action='store_true', help='Visualize data') 18 | parser.add_argument('--overlay', action='store_true', help='If visualizing, overlay disparity and voxel grid image') 19 | args = parser.parse_args() 20 | 21 | visualize = args.visualize 22 | dsec_dir = Path(args.dsec_dir) 23 | assert dsec_dir.is_dir() 24 | 25 | dataset_provider = DatasetProvider(dsec_dir) 26 | train_dataset = dataset_provider.get_train_dataset() 27 | 28 | batch_size = 1 29 | num_workers = 0 30 | train_loader = DataLoader( 31 | dataset=train_dataset, 32 | batch_size=batch_size, 33 | shuffle=True, 34 | num_workers=num_workers, 35 | drop_last=False) 36 | with torch.no_grad(): 37 | for data in tqdm(train_loader): 38 | if batch_size == 1 and visualize: 39 | disp = data['disparity_gt'].numpy().squeeze() 40 | disp_img = disp_img_to_rgb_img(disp) 41 | if args.overlay: 42 | left_voxel_grid = data['representation']['left'].squeeze() 43 | ev_img = torch.sum(left_voxel_grid, axis=0).numpy() 44 | ev_img = (ev_img/ev_img.max()*256).astype('uint8') 45 | show_disp_overlay(ev_img, disp_img, height=480, width=640) 46 | else: 47 | show_image(disp_img) 48 | -------------------------------------------------------------------------------- /scripts/dataset/provider.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import torch 4 | 5 | from dataset.sequence import Sequence 6 | 7 | class DatasetProvider: 8 | def __init__(self, dataset_path: Path, delta_t_ms: int=50, num_bins=15): 9 | train_path = dataset_path / 'train' 10 | assert dataset_path.is_dir(), str(dataset_path) 11 | assert train_path.is_dir(), str(train_path) 12 | 13 | train_sequences = list() 14 | for child in train_path.iterdir(): 15 | train_sequences.append(Sequence(child, 'train', delta_t_ms, num_bins)) 16 | 17 | self.train_dataset = torch.utils.data.ConcatDataset(train_sequences) 18 | 19 | def get_train_dataset(self): 20 | return self.train_dataset 21 | 22 | def get_val_dataset(self): 23 | # Implement this according to your needs. 24 | raise NotImplementedError 25 | 26 | def get_test_dataset(self): 27 | # Implement this according to your needs. 28 | raise NotImplementedError 29 | -------------------------------------------------------------------------------- /scripts/dataset/representations.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class EventRepresentation: 5 | def convert(self, x: torch.Tensor, y: torch.Tensor, pol: torch.Tensor, time: torch.Tensor): 6 | raise NotImplementedError 7 | 8 | 9 | class VoxelGrid(EventRepresentation): 10 | def __init__(self, channels: int, height: int, width: int, normalize: bool): 11 | self.voxel_grid = torch.zeros((channels, height, width), dtype=torch.float, requires_grad=False) 12 | self.nb_channels = channels 13 | self.normalize = normalize 14 | 15 | def convert(self, x: torch.Tensor, y: torch.Tensor, pol: torch.Tensor, time: torch.Tensor): 16 | assert x.shape == y.shape == pol.shape == time.shape 17 | assert x.ndim == 1 18 | 19 | C, H, W = self.voxel_grid.shape 20 | with torch.no_grad(): 21 | self.voxel_grid = self.voxel_grid.to(pol.device) 22 | voxel_grid = self.voxel_grid.clone() 23 | 24 | t_norm = time 25 | t_norm = (C - 1) * (t_norm-t_norm[0]) / (t_norm[-1]-t_norm[0]) 26 | 27 | x0 = x.int() 28 | y0 = y.int() 29 | t0 = t_norm.int() 30 | 31 | value = 2*pol-1 32 | 33 | for xlim in [x0,x0+1]: 34 | for ylim in [y0,y0+1]: 35 | for tlim in [t0,t0+1]: 36 | 37 | mask = (xlim < W) & (xlim >= 0) & (ylim < H) & (ylim >= 0) & (tlim >= 0) & (tlim < self.nb_channels) 38 | interp_weights = value * (1 - (xlim-x).abs()) * (1 - (ylim-y).abs()) * (1 - (tlim - t_norm).abs()) 39 | 40 | index = H * W * tlim.long() + \ 41 | W * ylim.long() + \ 42 | xlim.long() 43 | 44 | voxel_grid.put_(index[mask], interp_weights[mask], accumulate=True) 45 | 46 | if self.normalize: 47 | mask = torch.nonzero(voxel_grid, as_tuple=True) 48 | if mask[0].size()[0] > 0: 49 | mean = voxel_grid[mask].mean() 50 | std = voxel_grid[mask].std() 51 | if std > 0: 52 | voxel_grid[mask] = (voxel_grid[mask] - mean) / std 53 | else: 54 | voxel_grid[mask] = voxel_grid[mask] - mean 55 | 56 | return voxel_grid 57 | -------------------------------------------------------------------------------- /scripts/dataset/sequence.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import weakref 3 | 4 | import cv2 5 | import h5py 6 | import numpy as np 7 | import torch 8 | from torch.utils.data import Dataset 9 | 10 | from dataset.representations import VoxelGrid 11 | from utils.eventslicer import EventSlicer 12 | 13 | 14 | class Sequence(Dataset): 15 | # NOTE: This is just an EXAMPLE class for convenience. Adapt it to your case. 16 | # In this example, we use the voxel grid representation. 17 | # 18 | # This class assumes the following structure in a sequence directory: 19 | # 20 | # seq_name (e.g. zurich_city_11_a) 21 | # ├── disparity 22 | # │   ├── event 23 | # │   │   ├── 000000.png 24 | # │   │   └── ... 25 | # │   └── timestamps.txt 26 | # └── events 27 | #    ├── left 28 | #    │   ├── events.h5 29 | #    │   └── rectify_map.h5 30 | #    └── right 31 | #    ├── events.h5 32 | #    └── rectify_map.h5 33 | 34 | def __init__(self, seq_path: Path, mode: str='train', delta_t_ms: int=50, num_bins: int=15): 35 | assert num_bins >= 1 36 | assert delta_t_ms <= 100, 'adapt this code, if duration is higher than 100 ms' 37 | assert seq_path.is_dir() 38 | 39 | # NOTE: Adapt this code according to the present mode (e.g. train, val or test). 40 | self.mode = mode 41 | 42 | # Save output dimensions 43 | self.height = 480 44 | self.width = 640 45 | self.num_bins = num_bins 46 | 47 | # Set event representation 48 | self.voxel_grid = VoxelGrid(self.num_bins, self.height, self.width, normalize=True) 49 | 50 | self.locations = ['left', 'right'] 51 | 52 | # Save delta timestamp in ms 53 | self.delta_t_us = delta_t_ms * 1000 54 | 55 | # load disparity timestamps 56 | disp_dir = seq_path / 'disparity' 57 | assert disp_dir.is_dir() 58 | self.timestamps = np.loadtxt(disp_dir / 'timestamps.txt', dtype='int64') 59 | 60 | # load disparity paths 61 | ev_disp_dir = disp_dir / 'event' 62 | assert ev_disp_dir.is_dir() 63 | disp_gt_pathstrings = list() 64 | for entry in ev_disp_dir.iterdir(): 65 | assert str(entry.name).endswith('.png') 66 | disp_gt_pathstrings.append(str(entry)) 67 | disp_gt_pathstrings.sort() 68 | self.disp_gt_pathstrings = disp_gt_pathstrings 69 | 70 | assert len(self.disp_gt_pathstrings) == self.timestamps.size 71 | 72 | # Remove first disparity path and corresponding timestamp. 73 | # This is necessary because we do not have events before the first disparity map. 74 | assert int(Path(self.disp_gt_pathstrings[0]).stem) == 0 75 | self.disp_gt_pathstrings.pop(0) 76 | self.timestamps = self.timestamps[1:] 77 | 78 | self.h5f = dict() 79 | self.rectify_ev_maps = dict() 80 | self.event_slicers = dict() 81 | 82 | ev_dir = seq_path / 'events' 83 | for location in self.locations: 84 | ev_dir_location = ev_dir / location 85 | ev_data_file = ev_dir_location / 'events.h5' 86 | ev_rect_file = ev_dir_location / 'rectify_map.h5' 87 | 88 | h5f_location = h5py.File(str(ev_data_file), 'r') 89 | self.h5f[location] = h5f_location 90 | self.event_slicers[location] = EventSlicer(h5f_location) 91 | with h5py.File(str(ev_rect_file), 'r') as h5_rect: 92 | self.rectify_ev_maps[location] = h5_rect['rectify_map'][()] 93 | 94 | 95 | self._finalizer = weakref.finalize(self, self.close_callback, self.h5f) 96 | 97 | def events_to_voxel_grid(self, x, y, p, t, device: str='cpu'): 98 | t = (t - t[0]).astype('float32') 99 | t = (t/t[-1]) 100 | x = x.astype('float32') 101 | y = y.astype('float32') 102 | pol = p.astype('float32') 103 | return self.voxel_grid.convert( 104 | torch.from_numpy(x), 105 | torch.from_numpy(y), 106 | torch.from_numpy(pol), 107 | torch.from_numpy(t)) 108 | 109 | def getHeightAndWidth(self): 110 | return self.height, self.width 111 | 112 | @staticmethod 113 | def get_disparity_map(filepath: Path): 114 | assert filepath.is_file() 115 | disp_16bit = cv2.imread(str(filepath), cv2.IMREAD_ANYDEPTH) 116 | return disp_16bit.astype('float32')/256 117 | 118 | @staticmethod 119 | def close_callback(h5f_dict): 120 | for k, h5f in h5f_dict.items(): 121 | h5f.close() 122 | 123 | def __len__(self): 124 | return len(self.disp_gt_pathstrings) 125 | 126 | def rectify_events(self, x: np.ndarray, y: np.ndarray, location: str): 127 | assert location in self.locations 128 | # From distorted to undistorted 129 | rectify_map = self.rectify_ev_maps[location] 130 | assert rectify_map.shape == (self.height, self.width, 2), rectify_map.shape 131 | assert x.max() < self.width 132 | assert y.max() < self.height 133 | return rectify_map[y, x] 134 | 135 | def __getitem__(self, index): 136 | ts_end = self.timestamps[index] 137 | # ts_start should be fine (within the window as we removed the first disparity map) 138 | ts_start = ts_end - self.delta_t_us 139 | 140 | disp_gt_path = Path(self.disp_gt_pathstrings[index]) 141 | file_index = int(disp_gt_path.stem) 142 | output = { 143 | 'disparity_gt': self.get_disparity_map(disp_gt_path), 144 | 'file_index': file_index, 145 | } 146 | for location in self.locations: 147 | event_data = self.event_slicers[location].get_events(ts_start, ts_end) 148 | 149 | p = event_data['p'] 150 | t = event_data['t'] 151 | x = event_data['x'] 152 | y = event_data['y'] 153 | 154 | xy_rect = self.rectify_events(x, y, location) 155 | x_rect = xy_rect[:, 0] 156 | y_rect = xy_rect[:, 1] 157 | 158 | event_representation = self.events_to_voxel_grid(x_rect, y_rect, p, t) 159 | if 'representation' not in output: 160 | output['representation'] = dict() 161 | output['representation'][location] = event_representation 162 | 163 | return output 164 | -------------------------------------------------------------------------------- /scripts/dataset/visualization.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import matplotlib as mpl 3 | import matplotlib.cm as cm 4 | import numpy as np 5 | 6 | 7 | def disp_img_to_rgb_img(disp_array: np.ndarray): 8 | disp_pixels = np.argwhere(disp_array > 0) 9 | u_indices = disp_pixels[:, 1] 10 | v_indices = disp_pixels[:, 0] 11 | disp = disp_array[v_indices, u_indices] 12 | max_disp = 80 13 | 14 | norm = mpl.colors.Normalize(vmin=0, vmax=max_disp, clip=True) 15 | mapper = cm.ScalarMappable(norm=norm, cmap='inferno') 16 | 17 | disp_color = mapper.to_rgba(disp)[..., :3] 18 | output_image = np.zeros((disp_array.shape[0], disp_array.shape[1], 3)) 19 | output_image[v_indices, u_indices, :] = disp_color 20 | output_image = (255 * output_image).astype("uint8") 21 | output_image = cv2.cvtColor(output_image, cv2.COLOR_RGB2BGR) 22 | return output_image 23 | 24 | def show_image(image): 25 | cv2.namedWindow('viz', cv2.WND_PROP_FULLSCREEN) 26 | cv2.imshow('viz', image) 27 | cv2.waitKey(0) 28 | 29 | def get_disp_overlay(image_1c, disp_rgb_image, height, width): 30 | image = np.repeat(image_1c[..., np.newaxis], 3, axis=2) 31 | overlay = cv2.addWeighted(image, 0.1, disp_rgb_image, 0.9, 0) 32 | return overlay 33 | 34 | def show_disp_overlay(image_1c, disp_rgb_image, height, width): 35 | overlay = get_disp_overlay(image_1c, disp_rgb_image, height, width) 36 | show_image(overlay) 37 | -------------------------------------------------------------------------------- /scripts/events_to_video.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | 4 | import numpy as np 5 | import skvideo.io 6 | from tqdm import tqdm 7 | 8 | from visualization.eventreader import EventReader 9 | 10 | 11 | def render(x: np.ndarray, y: np.ndarray, pol: np.ndarray, H: int, W: int) -> np.ndarray: 12 | assert x.size == y.size == pol.size 13 | assert H > 0 14 | assert W > 0 15 | img = np.full((H,W,3), fill_value=255,dtype='uint8') 16 | mask = np.zeros((H,W),dtype='int32') 17 | pol = pol.astype('int') 18 | pol[pol==0]=-1 19 | mask1 = (x>=0)&(y>=0)&(W>x)&(H>y) 20 | mask[y[mask1],x[mask1]]=pol[mask1] 21 | img[mask==0]=[255,255,255] 22 | img[mask==-1]=[255,0,0] 23 | img[mask==1]=[0,0,255] 24 | return img 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser('Visualize Events') 29 | parser.add_argument('event_file', type=str, help='Path to events.h5 file') 30 | parser.add_argument('output_file', help='Path to write video file') 31 | parser.add_argument('--delta_time_ms', '-dt_ms', type=float, default=50.0, help='Time window (in milliseconds) to summarize events for visualization') 32 | args = parser.parse_args() 33 | 34 | event_filepath = Path(args.event_file) 35 | video_filepath = Path(args.output_file) 36 | dt = args.delta_time_ms 37 | 38 | height = 480 39 | width = 640 40 | 41 | assert video_filepath.parent.is_dir(), "Directory {} does not exist".format(str(video_filepath.parent)) 42 | 43 | writer = skvideo.io.FFmpegWriter(video_filepath) 44 | for events in tqdm(EventReader(event_filepath, dt)): 45 | p = events['p'] 46 | x = events['x'] 47 | y = events['y'] 48 | t = events['t'] 49 | img = render(x, y, p, height, width) 50 | writer.writeFrame(img) 51 | writer.close() 52 | -------------------------------------------------------------------------------- /scripts/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uzh-rpg/DSEC/af90ef112c175ddcb29d7bfefde517a877c18172/scripts/utils/__init__.py -------------------------------------------------------------------------------- /scripts/utils/eventslicer.py: -------------------------------------------------------------------------------- 1 | import math 2 | from typing import Dict, Tuple 3 | 4 | import h5py 5 | from numba import jit 6 | import numpy as np 7 | 8 | 9 | class EventSlicer: 10 | def __init__(self, h5f: h5py.File): 11 | self.h5f = h5f 12 | 13 | self.events = dict() 14 | for dset_str in ['p', 'x', 'y', 't']: 15 | self.events[dset_str] = self.h5f['events/{}'.format(dset_str)] 16 | 17 | # This is the mapping from milliseconds to event index: 18 | # It is defined such that 19 | # (1) t[ms_to_idx[ms]] >= ms*1000, for ms > 0 20 | # (2) t[ms_to_idx[ms] - 1] < ms*1000, for ms > 0 21 | # (3) ms_to_idx[0] == 0 22 | # , where 'ms' is the time in milliseconds and 't' the event timestamps in microseconds. 23 | # 24 | # As an example, given 't' and 'ms': 25 | # t: 0 500 2100 5000 5000 7100 7200 7200 8100 9000 26 | # ms: 0 1 2 3 4 5 6 7 8 9 27 | # 28 | # we get 29 | # 30 | # ms_to_idx: 31 | # 0 2 2 3 3 3 5 5 8 9 32 | self.ms_to_idx = np.asarray(self.h5f['ms_to_idx'], dtype='int64') 33 | 34 | if "t_offset" in list(h5f.keys()): 35 | self.t_offset = int(h5f['t_offset'][()]) 36 | else: 37 | self.t_offset = 0 38 | self.t_final = int(self.events['t'][-1]) + self.t_offset 39 | 40 | def get_start_time_us(self): 41 | return self.t_offset 42 | 43 | def get_final_time_us(self): 44 | return self.t_final 45 | 46 | def get_events(self, t_start_us: int, t_end_us: int) -> Dict[str, np.ndarray]: 47 | """Get events (p, x, y, t) within the specified time window 48 | Parameters 49 | ---------- 50 | t_start_us: start time in microseconds 51 | t_end_us: end time in microseconds 52 | Returns 53 | ------- 54 | events: dictionary of (p, x, y, t) or None if the time window cannot be retrieved 55 | """ 56 | assert t_start_us < t_end_us 57 | 58 | # We assume that the times are top-off-day, hence subtract offset: 59 | t_start_us -= self.t_offset 60 | t_end_us -= self.t_offset 61 | 62 | t_start_ms, t_end_ms = self.get_conservative_window_ms(t_start_us, t_end_us) 63 | t_start_ms_idx = self.ms2idx(t_start_ms) 64 | t_end_ms_idx = self.ms2idx(t_end_ms) 65 | 66 | if t_start_ms_idx is None or t_end_ms_idx is None: 67 | # Cannot guarantee window size anymore 68 | return None 69 | 70 | events = dict() 71 | time_array_conservative = np.asarray(self.events['t'][t_start_ms_idx:t_end_ms_idx]) 72 | idx_start_offset, idx_end_offset = self.get_time_indices_offsets(time_array_conservative, t_start_us, t_end_us) 73 | t_start_us_idx = t_start_ms_idx + idx_start_offset 74 | t_end_us_idx = t_start_ms_idx + idx_end_offset 75 | # Again add t_offset to get gps time 76 | events['t'] = time_array_conservative[idx_start_offset:idx_end_offset] + self.t_offset 77 | for dset_str in ['p', 'x', 'y']: 78 | events[dset_str] = np.asarray(self.events[dset_str][t_start_us_idx:t_end_us_idx]) 79 | assert events[dset_str].size == events['t'].size 80 | return events 81 | 82 | 83 | @staticmethod 84 | def get_conservative_window_ms(ts_start_us: int, ts_end_us) -> Tuple[int, int]: 85 | """Compute a conservative time window of time with millisecond resolution. 86 | We have a time to index mapping for each millisecond. Hence, we need 87 | to compute the lower and upper millisecond to retrieve events. 88 | Parameters 89 | ---------- 90 | ts_start_us: start time in microseconds 91 | ts_end_us: end time in microseconds 92 | Returns 93 | ------- 94 | window_start_ms: conservative start time in milliseconds 95 | window_end_ms: conservative end time in milliseconds 96 | """ 97 | assert ts_end_us > ts_start_us 98 | window_start_ms = math.floor(ts_start_us/1000) 99 | window_end_ms = math.ceil(ts_end_us/1000) 100 | return window_start_ms, window_end_ms 101 | 102 | @staticmethod 103 | @jit(nopython=True) 104 | def get_time_indices_offsets( 105 | time_array: np.ndarray, 106 | time_start_us: int, 107 | time_end_us: int) -> Tuple[int, int]: 108 | """Compute index offset of start and end timestamps in microseconds 109 | Parameters 110 | ---------- 111 | time_array: timestamps (in us) of the events 112 | time_start_us: start timestamp (in us) 113 | time_end_us: end timestamp (in us) 114 | Returns 115 | ------- 116 | idx_start: Index within this array corresponding to time_start_us 117 | idx_end: Index within this array corresponding to time_end_us 118 | such that (in non-edge cases) 119 | time_array[idx_start] >= time_start_us 120 | time_array[idx_end] >= time_end_us 121 | time_array[idx_start - 1] < time_start_us 122 | time_array[idx_end - 1] < time_end_us 123 | this means that 124 | time_start_us <= time_array[idx_start:idx_end] < time_end_us 125 | """ 126 | 127 | assert time_array.ndim == 1 128 | 129 | idx_start = -1 130 | if time_array[-1] < time_start_us: 131 | # This can happen in extreme corner cases. E.g. 132 | # time_array[0] = 1016 133 | # time_array[-1] = 1984 134 | # time_start_us = 1990 135 | # time_end_us = 2000 136 | 137 | # Return same index twice: array[x:x] is empty. 138 | return time_array.size, time_array.size 139 | else: 140 | for idx_from_start in range(0, time_array.size, 1): 141 | if time_array[idx_from_start] >= time_start_us: 142 | idx_start = idx_from_start 143 | break 144 | assert idx_start >= 0 145 | 146 | idx_end = time_array.size 147 | for idx_from_end in range(time_array.size - 1, -1, -1): 148 | if time_array[idx_from_end] >= time_end_us: 149 | idx_end = idx_from_end 150 | else: 151 | break 152 | 153 | assert time_array[idx_start] >= time_start_us 154 | if idx_end < time_array.size: 155 | assert time_array[idx_end] >= time_end_us 156 | if idx_start > 0: 157 | assert time_array[idx_start - 1] < time_start_us 158 | if idx_end > 0: 159 | assert time_array[idx_end - 1] < time_end_us 160 | return idx_start, idx_end 161 | 162 | def ms2idx(self, time_ms: int) -> int: 163 | assert time_ms >= 0 164 | if time_ms >= self.ms_to_idx.size: 165 | return None 166 | return self.ms_to_idx[time_ms] 167 | -------------------------------------------------------------------------------- /scripts/visualization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uzh-rpg/DSEC/af90ef112c175ddcb29d7bfefde517a877c18172/scripts/visualization/__init__.py -------------------------------------------------------------------------------- /scripts/visualization/eventreader.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import weakref 3 | 4 | import h5py 5 | 6 | from utils.eventslicer import EventSlicer 7 | 8 | 9 | class EventReaderAbstract: 10 | def __init__(self, filepath: Path): 11 | assert filepath.is_file() 12 | assert filepath.name.endswith('.h5') 13 | self.h5f = h5py.File(str(filepath), 'r') 14 | self._finalizer = weakref.finalize(self, self.close_callback, self.h5f) 15 | 16 | @staticmethod 17 | def close_callback(h5f: h5py.File): 18 | h5f.close() 19 | 20 | def __enter__(self): 21 | return self 22 | 23 | def __exit__(self, exc_type, exc_value, traceback): 24 | self._finalizer() 25 | 26 | def __iter__(self): 27 | return self 28 | 29 | def __next__(self): 30 | raise NotImplementedError 31 | 32 | 33 | class EventReader(EventReaderAbstract): 34 | def __init__(self, filepath: Path, dt_milliseconds: int): 35 | super().__init__(filepath) 36 | self.event_slicer = EventSlicer(self.h5f) 37 | 38 | self.dt_us = int(dt_milliseconds * 1000) 39 | self.t_start_us = self.event_slicer.get_start_time_us() 40 | self.t_end_us = self.event_slicer.get_final_time_us() 41 | 42 | self._length = (self.t_end_us - self.t_start_us)//self.dt_us 43 | 44 | def __len__(self): 45 | return self._length 46 | 47 | def __next__(self): 48 | t_end_us = self.t_start_us + self.dt_us 49 | if t_end_us > self.t_end_us: 50 | raise StopIteration 51 | events = self.event_slicer.get_events(self.t_start_us, t_end_us) 52 | if events is None: 53 | raise StopIteration 54 | 55 | self.t_start_us = t_end_us 56 | return events 57 | --------------------------------------------------------------------------------