├── src └── rohbau3d │ ├── __init__.py │ ├── core │ ├── __init__.py │ ├── rohbau3d_hub.py │ └── dataverse.py │ └── misc │ ├── __init__.py │ ├── _logging.py │ ├── config.py │ └── helper.py ├── requirements.txt ├── config ├── dataverse.yaml └── dataverse_file_index.json ├── pyproject.toml ├── LICENSE ├── scripts └── download.py ├── .gitignore └── README.md /src/rohbau3d/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/rohbau3d/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/rohbau3d/misc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Add your project dependencies here 2 | pooch 3 | tqdm 4 | zstandard 5 | yaml 6 | -------------------------------------------------------------------------------- /config/dataverse.yaml: -------------------------------------------------------------------------------- 1 | # CONFIGURATION 2 | # Rohbau3D 3 | 4 | # GENERAL 5 | config_dir: config 6 | log_dir: log 7 | log_level: INFO 8 | 9 | # DOWNLOAD 10 | download_hub: dataverse 11 | download_dir: data/download 12 | feature_index_file: dataverse_file_index.json 13 | feature_selection: [all] 14 | scene_selection: [all] 15 | 16 | # FILE EXTRACT 17 | extract_dir: data/extract 18 | clean_download_files: False 19 | 20 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "rohbau3d" 7 | version = "0.1.0" 8 | description = "Rohbau3D Dataset Access" 9 | authors = [ 10 | { name = "Lukas Rauch", email = "lukas.rauch@unibw.de" } 11 | ] 12 | license = {text = "MIT"} 13 | keywords = ["dataset", "point cloud", "download",] 14 | readme = "README.md" 15 | requires-python = ">=3.7" 16 | dependencies = [ 17 | "pooch", 18 | "tqdm", 19 | "zstandard", 20 | "pyyaml" 21 | ] 22 | 23 | 24 | [tool.setuptools.packages.find] 25 | where = ["src"] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Lukas Rauch 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts/download.py: -------------------------------------------------------------------------------- 1 | # rohbau3d script 2 | # author: lukas rauch 3 | # date: 2025-09-02 4 | 5 | from rohbau3d.misc.config import load_config 6 | from rohbau3d.misc.helper import session_summary 7 | from rohbau3d.core.rohbau3d_hub import Rohbau3DHub 8 | 9 | from rohbau3d.misc._logging import setup_logging 10 | import logging 11 | import argparse 12 | 13 | def _argparse(): 14 | parser = argparse.ArgumentParser(description="Rohbau3D Download Script") 15 | parser.add_argument("--config", type=str, required=True, help="Path to the config file") 16 | parser.add_argument("--download", action="store_true", help="Enable download") 17 | parser.add_argument("--extract", action="store_true", help="Enable extraction") 18 | args = parser.parse_args() 19 | 20 | return args 21 | 22 | 23 | def main(): 24 | 25 | args = _argparse() 26 | config = load_config(args.config) 27 | 28 | log_file = config.get("log_file", "logs") + "/rohbau3d.log" 29 | log_level = config.get("log_level", "DEBUG") 30 | 31 | setup_logging(log_file=log_file, level=log_level, to_console=True) 32 | log = logging.getLogger(__name__) 33 | log.info("/"*50) 34 | log.info(">>> Starting Rohbau3D download script ...") 35 | 36 | # LOGIT ----------------------------------------------------------- 37 | rohbau3d = Rohbau3DHub(config) 38 | stats = {} 39 | 40 | # RUN ------------------------------------------------------------ 41 | if not args.download and not args.extract: 42 | log.warning("No action specified. Use --download and/or --extract flags.") 43 | return 44 | else: 45 | if args.download: 46 | stats["download"] = rohbau3d.download() 47 | if args.extract: 48 | stats["extraction"] = rohbau3d.extract() 49 | 50 | # Exit 51 | if config.clean_download_files: 52 | rohbau3d.clean_download_files() 53 | 54 | session_summary(stats) 55 | 56 | log.info("Done ...") 57 | 58 | 59 | 60 | if __name__ == "__main__": 61 | main() -------------------------------------------------------------------------------- /src/rohbau3d/misc/_logging.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from pathlib import Path 3 | import logging 4 | import logging.config 5 | 6 | 7 | _DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d — %(message)s" 8 | 9 | class SkipConsoleFilter(logging.Filter): 10 | def filter(self, record: logging.LogRecord) -> bool: 11 | # If a record carries no_console=True, drop it from the console handler 12 | return not getattr(record, "no_console", False) 13 | 14 | def setup_logging( 15 | log_file: str | Path, 16 | level: str = "INFO", 17 | *, 18 | to_console: bool = False, 19 | rotate: bool = True, 20 | max_bytes: int = 10_000_000, 21 | backup_count: int = 5, 22 | ) -> None: 23 | log_path = Path(log_file) 24 | log_path.parent.mkdir(parents=True, exist_ok=True) 25 | 26 | # Choose file handler class 27 | file_handler_class = ( 28 | "logging.handlers.RotatingFileHandler" if rotate else "logging.FileHandler" 29 | ) 30 | 31 | config = { 32 | "version": 1, 33 | "disable_existing_loggers": False, 34 | "formatters": { 35 | "std": {"format": _DEFAULT_FORMAT}, 36 | }, 37 | "filters": { 38 | # IMPORTANT: use your package path here 39 | "skip_console": {"()": "rohbau3d.misc._logging.SkipConsoleFilter"}, 40 | }, 41 | "handlers": { 42 | "file": { 43 | "class": file_handler_class, 44 | "level": level, 45 | "formatter": "std", 46 | "filename": str(log_path), 47 | "encoding": "utf-8", 48 | **({"maxBytes": max_bytes, "backupCount": backup_count} if rotate else {}), 49 | }, 50 | }, 51 | "root": { 52 | "level": level, 53 | "handlers": ["file"], 54 | }, 55 | } 56 | 57 | if to_console: 58 | # Send console output to stderr to play nicely with tqdm 59 | config["handlers"]["console"] = { 60 | "class": "logging.StreamHandler", 61 | "level": level, 62 | "formatter": "std", 63 | "filters": ["skip_console"], 64 | "stream": "ext://sys.stderr", 65 | } 66 | config["root"]["handlers"].append("console") 67 | 68 | logging.config.dictConfig(config) 69 | -------------------------------------------------------------------------------- /src/rohbau3d/misc/config.py: -------------------------------------------------------------------------------- 1 | 2 | from pathlib import Path 3 | from typing import Any 4 | import yaml 5 | 6 | 7 | class Config(dict): 8 | """dict with attribute (dot) access. Recursively converts nested dicts/lists. 9 | Note: dot access only works for keys that are valid identifiers. 10 | """ 11 | 12 | def __init__(self, *args, **kwargs): 13 | super().__init__() 14 | self.update(*args, **kwargs) 15 | 16 | # --- attribute access --- 17 | def __getattr__(self, name: str) -> Any: 18 | try: 19 | return self[name] 20 | except KeyError as e: 21 | # also allow underscore→dash alias (handy for YAML keys with -) 22 | alt = name.replace("_", "-") 23 | if alt in self: 24 | return self[alt] 25 | raise AttributeError(name) from e 26 | 27 | def __setattr__(self, name: str, value: Any) -> None: 28 | # write to dict; supports cfg.foo = 1 29 | self[name] = value 30 | 31 | def __delattr__(self, name: str) -> None: 32 | try: 33 | del self[name] 34 | except KeyError as e: 35 | raise AttributeError(name) from e 36 | 37 | def __setitem__(self, key: str, value: Any) -> None: 38 | super().__setitem__(key, self._convert(value)) 39 | 40 | def update(self, *args, **kwargs) -> None: 41 | data = {} 42 | if args: 43 | data.update(args[0]) 44 | data.update(kwargs) 45 | for k, v in data.items(): 46 | super().__setitem__(k, self._convert(v)) 47 | 48 | @staticmethod 49 | def _convert(v: Any) -> Any: 50 | if isinstance(v, dict): 51 | return Config(v) 52 | if isinstance(v, list): 53 | return [Config._convert(i) for i in v] 54 | return v 55 | 56 | def to_dict(self) -> dict[str, Any]: 57 | """Back to plain dict (for saving).""" 58 | def un(v): 59 | if isinstance(v, Config): 60 | return {k: un(x) for k, x in v.items()} 61 | if isinstance(v, list): 62 | return [un(i) for i in v] 63 | return v 64 | return un(self) 65 | 66 | 67 | 68 | def load_config(path: str | Path) -> Config: 69 | """Read YAML → Config (dot-access).""" 70 | p = Path(path) 71 | data = yaml.safe_load(p.read_text(encoding="utf-8")) or {} 72 | if not isinstance(data, dict): 73 | raise TypeError(f"Top-level YAML must be a mapping, got {type(data)}") 74 | return Config(data) -------------------------------------------------------------------------------- /src/rohbau3d/misc/helper.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import os 4 | from tqdm import tqdm 5 | from pathlib import Path 6 | import re 7 | import tarfile 8 | import zstandard as zstd 9 | import json 10 | 11 | import logging 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | def read_json_dict(file_path): 17 | if not os.path.exists(file_path): 18 | raise FileNotFoundError(f"File not found: {file_path}") 19 | 20 | with open(file_path, 'r') as f: 21 | data = json.load(f) 22 | return data 23 | 24 | 25 | def session_summary(stats:dict): 26 | # Implement a function to summarize the current session 27 | log.info("/"*50) 28 | log.info("Session Summary:") 29 | log.info("="*50) 30 | for key, value in stats.items(): 31 | 32 | path = value.pop("path", None) 33 | time = value.pop("total_time", None) 34 | log.info(f"{key.upper()} Summary:") 35 | log.info(f" Total Time: {time:.2f} seconds" if time else " Total Time: N/A") 36 | log.info(f" Path: {path}" if path else " Path: N/A") 37 | 38 | for key, val in value.items(): 39 | log.info(f" > {key}: {val}") 40 | 41 | log.info("="*50) 42 | return None 43 | 44 | 45 | def extract_tar_zstd(zst_path, output_dir): 46 | """Extract a .tar.zst file directly into a folder without saving the intermediate .tar.""" 47 | name = Path(zst_path).name 48 | 49 | with open(zst_path, "rb") as compressed: 50 | dctx = zstd.ZstdDecompressor() 51 | with dctx.stream_reader(compressed) as reader: 52 | with tarfile.open(fileobj=reader, mode='r|') as tar: 53 | with tqdm(desc=f"📂 Extracting {name}", unit=" files") as pbar: 54 | for member in tar: 55 | tar.extract(member, path=output_dir) 56 | pbar.update(1) 57 | 58 | 59 | def extract_all_tar_zstd_parts(root_dir, output_root, feature): 60 | pattern = re.compile(r"site_(\d+)\.(\w+)\.part(\d+)\.tar\.zst$") 61 | 62 | stats = { 63 | "num_files_extracted": 0, 64 | "num_files_failed": 0, 65 | "corrupted_files": [], 66 | 67 | } 68 | 69 | feature_dir = Path(root_dir) 70 | for file_name in sorted(os.listdir(feature_dir)): 71 | match = pattern.match(file_name) 72 | if not match: 73 | continue 74 | 75 | site_id, feature_name, part_id = match.groups() 76 | if feature_name != feature: 77 | continue 78 | 79 | zst_path = feature_dir / file_name 80 | output_dir = Path(output_root) 81 | 82 | try: 83 | extract_tar_zstd(zst_path, output_dir) 84 | stats["num_files_extracted"] += 1 85 | log.info(f"Extracted {zst_path} to {output_dir}", extra={"no_console": True}) 86 | except Exception as e: 87 | log.error(f"Failed to extract {zst_path}: {e}") 88 | stats["num_files_failed"] += 1 89 | stats["corrupted_files"].append(str(zst_path)) 90 | return stats -------------------------------------------------------------------------------- /src/rohbau3d/core/rohbau3d_hub.py: -------------------------------------------------------------------------------- 1 | # rohbau3d script 2 | # author: lukas rauch 3 | # date: 2025-09-02 4 | 5 | from os.path import join, exists 6 | from pathlib import Path 7 | import shutil 8 | from time import time 9 | 10 | from rohbau3d.misc.helper import extract_all_tar_zstd_parts 11 | 12 | import logging 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | RHOBAU3D_HEADER = """ 17 | ____ __ __ _____ ____ __ __ __ 18 | / __ \____ / /_ / /_ ____ ___ _|__ // __ \ / / / /_ __/ /_ 19 | / /_/ / __ \/ __ \/ __ \/ __ `/ / / //_ >> Rohbau3D Hub <<< 23 | \n""" 24 | 25 | 26 | ROHBAU3D_FEATURES = [ 27 | "coord", "color", "intensity", "normal" 28 | ] 29 | 30 | 31 | class Rohbau3DHub: 32 | def __init__(self, cfg): 33 | print(RHOBAU3D_HEADER) 34 | 35 | self.cfg = cfg 36 | 37 | self.feature_selection = cfg.get("feature_selection") 38 | self.hub = self.get_hub(cfg["download_hub"]) 39 | 40 | self.download = self.hub.download 41 | 42 | 43 | def get_hub(self, hub: str): 44 | if hub.lower() == "default": 45 | from rohbau3d.core.dataverse import Dataverse 46 | hub = Dataverse(self.cfg) 47 | 48 | if hub.lower() == "dataverse": 49 | from rohbau3d.core.dataverse import Dataverse 50 | hub = Dataverse(self.cfg) 51 | 52 | return hub 53 | 54 | 55 | def extract(self): 56 | 57 | download_dir = self.cfg["download_dir"] 58 | extract_dir = self.cfg["extract_dir"] + "/rohbau3d" 59 | 60 | log.info(f"Extracting files to PATH: {extract_dir}") 61 | 62 | stats = { 63 | "path": extract_dir, 64 | "num_files_extracted": 0, 65 | "num_files_failed": 0, 66 | "total_time": 0 67 | } 68 | 69 | start_time = time() 70 | for feature in self.feature_selection: 71 | feature_dir = Path(download_dir, feature) 72 | if not feature_dir.exists(): 73 | log.warning(f"Feature directory {feature_dir} does not exist, skipping extraction.") 74 | continue 75 | 76 | temp_stats = extract_all_tar_zstd_parts(feature_dir, Path(extract_dir), feature=feature) 77 | 78 | stats["num_files_extracted"] += temp_stats["num_files_extracted"] 79 | stats["num_files_failed"] += temp_stats["num_files_failed"] 80 | stats["corrupted_files"] = stats.get("corrupted_files", []) + temp_stats.get("corrupted_files", []) 81 | 82 | stats["total_time"] = time() - start_time 83 | return stats 84 | 85 | 86 | def clean_download_files(self): 87 | # Implement file cleanup logic here 88 | download_dir = Path(self.cfg["download_dir"]) 89 | 90 | if download_dir.exists(): 91 | log.info(f"Cleaning downloaded features {ROHBAU3D_FEATURES} in PATH: {download_dir}") 92 | 93 | for feature in ROHBAU3D_FEATURES: 94 | path = join(download_dir, feature) 95 | if exists(path): 96 | log.info(f"Removing {path}") 97 | shutil.rmtree(path) 98 | 99 | return True 100 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | 7 | # logs 8 | *logs 9 | logs/ 10 | 11 | # data 12 | *data 13 | data/ 14 | 15 | # IDE 16 | *.vscode 17 | .vscode/ 18 | *.idea 19 | .idea/ 20 | 21 | # C extensions 22 | *.so 23 | 24 | # Distribution / packaging 25 | .Python 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | sdist/ 36 | var/ 37 | wheels/ 38 | share/python-wheels/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | MANIFEST 43 | 44 | # PyInstaller 45 | # Usually these files are written by a python script from a template 46 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 47 | *.manifest 48 | *.spec 49 | 50 | # Installer logs 51 | pip-log.txt 52 | pip-delete-this-directory.txt 53 | 54 | # Unit test / coverage reports 55 | htmlcov/ 56 | .tox/ 57 | .nox/ 58 | .coverage 59 | .coverage.* 60 | .cache 61 | nosetests.xml 62 | coverage.xml 63 | *.cover 64 | *.py,cover 65 | .hypothesis/ 66 | .pytest_cache/ 67 | cover/ 68 | 69 | # Translations 70 | *.mo 71 | *.pot 72 | 73 | # Django stuff: 74 | *.log 75 | local_settings.py 76 | db.sqlite3 77 | db.sqlite3-journal 78 | 79 | # Flask stuff: 80 | instance/ 81 | .webassets-cache 82 | 83 | # Scrapy stuff: 84 | .scrapy 85 | 86 | # Sphinx documentation 87 | docs/_build/ 88 | 89 | # PyBuilder 90 | .pybuilder/ 91 | target/ 92 | 93 | # Jupyter Notebook 94 | .ipynb_checkpoints 95 | 96 | # IPython 97 | profile_default/ 98 | ipython_config.py 99 | 100 | # pyenv 101 | # For a library or package, you might want to ignore these files since the code is 102 | # intended to run in multiple environments; otherwise, check them in: 103 | # .python-version 104 | 105 | # pipenv 106 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 107 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 108 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 109 | # install all needed dependencies. 110 | #Pipfile.lock 111 | 112 | # UV 113 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 114 | # This is especially recommended for binary packages to ensure reproducibility, and is more 115 | # commonly ignored for libraries. 116 | #uv.lock 117 | 118 | # poetry 119 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 120 | # This is especially recommended for binary packages to ensure reproducibility, and is more 121 | # commonly ignored for libraries. 122 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 123 | #poetry.lock 124 | 125 | # pdm 126 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 127 | #pdm.lock 128 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 129 | # in version control. 130 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 131 | .pdm.toml 132 | .pdm-python 133 | .pdm-build/ 134 | 135 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 136 | __pypackages__/ 137 | 138 | # Celery stuff 139 | celerybeat-schedule 140 | celerybeat.pid 141 | 142 | # SageMath parsed files 143 | *.sage.py 144 | 145 | # Environments 146 | .env 147 | .venv 148 | env/ 149 | venv/ 150 | ENV/ 151 | env.bak/ 152 | venv.bak/ 153 | 154 | # Spyder project settings 155 | .spyderproject 156 | .spyproject 157 | 158 | # Rope project settings 159 | .ropeproject 160 | 161 | # mkdocs documentation 162 | /site 163 | 164 | # mypy 165 | .mypy_cache/ 166 | .dmypy.json 167 | dmypy.json 168 | 169 | # Pyre type checker 170 | .pyre/ 171 | 172 | # pytype static type analyzer 173 | .pytype/ 174 | 175 | # Cython debug symbols 176 | cython_debug/ 177 | 178 | # PyCharm 179 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 180 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 181 | # and can be added to the global gitignore or merged into this file. For a more nuclear 182 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 183 | #.idea/ 184 | 185 | # Ruff stuff: 186 | .ruff_cache/ 187 | 188 | # PyPI configuration file 189 | .pypirc 190 | -------------------------------------------------------------------------------- /src/rohbau3d/core/dataverse.py: -------------------------------------------------------------------------------- 1 | # rohbau3d script 2 | # author: lukas rauch 3 | # date: 2025-09-02 4 | 5 | import os 6 | from os.path import join, exists 7 | from pooch import DOIDownloader 8 | from time import time 9 | 10 | from rohbau3d.misc.helper import read_json_dict 11 | 12 | import logging 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | # DATAVERSE REGISTRY --------------------------------------------- 17 | 18 | DATAVERSE_BASE_URL = "doi:10.60776/ZWJFI4" 19 | 20 | ROHBAU3D_FEATURES = [ 21 | "coord", "color", "intensity", "normal" 22 | ] 23 | 24 | ROHBAU3D_SCENES = [ 25 | "site_00", "site_01", "site_02", "site_03", 26 | "site_04", "site_05", "site_06", "site_07", 27 | "site_08", "site_09", "site_10", "site_11", 28 | "site_12", "site_13", 29 | ] 30 | # --------------------------------------------------------------- 31 | 32 | 33 | 34 | class Dataverse: 35 | def __init__(self, cfg): 36 | self.cfg = cfg 37 | 38 | self.base_url = DATAVERSE_BASE_URL 39 | self.output_dir = cfg["download_dir"] 40 | self.config_dir = cfg["config_dir"] 41 | 42 | self.feature_index_file = cfg.get("feature_index_file") 43 | self.feature_index = self._get_file_index(join(self.config_dir, self.feature_index_file)) 44 | 45 | self.feature_selection = self._get_feature_selection() 46 | self.scene_selection = self._get_scene_selection() 47 | self.data_selection = self._get_data_selection() 48 | 49 | self.stats = { 50 | "path": self.output_dir, 51 | "num_files_downloaded": 0, 52 | "num_files_skipped": 0, 53 | "skipped_files": [], 54 | "corrupted_files": [], 55 | "total_time": 0, 56 | } 57 | 58 | log.info(">>> Dataverse hub initialized <<<") 59 | 60 | 61 | @staticmethod 62 | def summarize_data_selection(data_selection): 63 | summary = {} 64 | for feature, sites in data_selection.items(): 65 | summary[feature] = { 66 | "num_sites": 0, 67 | "num_files": 0 68 | } 69 | for site_id, files in sites.items(): 70 | summary[feature]["num_sites"] += 1 71 | summary[feature]["num_files"] += len(files) 72 | 73 | for key, value in summary.items(): 74 | log.info(f"{key}: {value['num_sites']} sites, {value['num_files']} scenes") 75 | 76 | log.info("-------------------------------------------------------------") 77 | log.info(f"Total number of download files: {sum(value['num_files'] for value in summary.values())}") 78 | log.info("-------------------------------------------------------------") 79 | 80 | 81 | def download(self): 82 | log.info(f"Downloading files to {self.output_dir}") 83 | self.summarize_data_selection(self.data_selection) 84 | 85 | downloader = DOIDownloader(progressbar=True) 86 | 87 | start_time = time() 88 | for feature, sites in self.data_selection.items(): 89 | for site_id, files in sites.items(): 90 | for file_name in files: 91 | # Construct the full URL for the file 92 | url = f"{self.base_url}/{file_name}" 93 | 94 | output_file = join(self.output_dir, feature, file_name) 95 | 96 | if exists(output_file,): 97 | log.info(f"File {file_name} already exists, skipping download.") 98 | self.stats["num_files_skipped"] += 1 99 | self.stats["skipped_files"] = self.stats.get("skipped_files", []) + [file_name] 100 | continue 101 | 102 | os.makedirs(os.path.dirname(output_file), exist_ok=True) 103 | 104 | # Download the file 105 | try: 106 | log.info(f"🍕 Downloading {file_name} from {url} to {output_file}") 107 | downloader(url=url, output_file=output_file, pooch=None) 108 | self.stats["num_files_downloaded"] += 1 109 | except Exception as e: 110 | log.error(f"Failed to download {file_name}: {e}") 111 | self.stats["corrupted_files"].append(file_name) 112 | 113 | self.stats["total_time"] = time() - start_time 114 | return self.stats 115 | 116 | 117 | def _get_data_selection(self): 118 | database = self.feature_index 119 | 120 | # convert scene_selection to id strings only for json indexing 121 | scene_selection = [str(int(s.split("_")[-1])) for s in self.scene_selection] 122 | 123 | # compile a reduced dict 124 | data_selection = {} 125 | for feature in self.feature_selection: 126 | if feature not in database: 127 | log.warning(f"Feature '{feature}' not found in database.") 128 | continue 129 | data_selection[feature] = {} 130 | for site in scene_selection: 131 | if site not in database[feature]: 132 | log.warning(f"Site '{site}' not found for feature '{feature}'.") 133 | continue 134 | data_selection[feature][site] = database[feature][site] 135 | 136 | return data_selection 137 | 138 | 139 | @staticmethod 140 | def _get_file_index(path): 141 | try: 142 | database = read_json_dict(path) 143 | log.info(f"Dataverse File Index loaded from {path}:") 144 | return database 145 | except FileNotFoundError as e: 146 | log.error(e) 147 | return {} 148 | 149 | 150 | def _get_feature_selection(self): 151 | 152 | cfg_feature_selection = self.cfg.get("feature_selection", None) 153 | 154 | if cfg_feature_selection is None: 155 | log.warning("No feature selection provided, using all features.") 156 | return ROHBAU3D_FEATURES 157 | 158 | if cfg_feature_selection == "all" or next(iter(cfg_feature_selection)) == "all": 159 | return ROHBAU3D_FEATURES 160 | 161 | if type(cfg_feature_selection) is list: 162 | features = [] 163 | for feature in cfg_feature_selection: 164 | if feature in ROHBAU3D_FEATURES: 165 | features.append(feature) 166 | else: 167 | log.warning(f"Selected feature {feature} is not a valid feature. Skipping.") 168 | 169 | return features 170 | 171 | def _get_scene_selection(self): 172 | cfg_scene_selection = self.cfg.get("scene_selection", None) 173 | 174 | if cfg_scene_selection is None: 175 | log.warning("No scene selection provided, using all scenes.") 176 | return ROHBAU3D_SCENES 177 | 178 | if cfg_scene_selection == "all" or next(iter(cfg_scene_selection)) == "all": 179 | return ROHBAU3D_SCENES 180 | 181 | if type(cfg_scene_selection) is list: 182 | scenes = [] 183 | for scene in cfg_scene_selection: 184 | if scene in ROHBAU3D_SCENES: 185 | scenes.append(scene) 186 | else: 187 | log.warning(f"Selected scene {scene} is not a valid scene. Skipping.") 188 | 189 | return scenes 190 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rohbau3D 2 | A Shell Construction Site 3D Point Cloud Dataset 3 | 4 |

5 | Rohbau3D point cloud feature maps 6 |

7 |

Figure: Rohbau3D point cloud feature maps

8 | 9 | ## Abstract 10 | 11 | We introduce Rohbau3D, a novel dataset of 3D point clouds that realistically represent indoor construction environments. The dataset comprises 504 high-resolution LiDAR scans captured with a terrestrial laser scanner across 14 distinct construction sites, including residential buildings, a large-scale office complex, educational facilities, and an underground parking garage—all in various stages of shell construction or renovation. Each point cloud is enriched with scalar laser reflectance intensity, RGB color values, and reconstructed surface normal vectors. In addition to the 3D data, the dataset includes high-resolution 2D panoramic renderings of each scene and its associated point cloud features. Designed to reflect the complexity and variability of real-world construction sites, Rohbau3D supports research in geometric processing, scene understanding, and intelligent computing in structural and civil engineering. To our knowledge, it is the first dataset of its kind and scale to be publicly released. Rohbau3D is intended as a foundation for ongoing work, with plans to extend it through additional scenes and targeted annotations to support future research. 12 | 13 | ### Paper 14 | :page_facing_up: [Rohbau3D: A Shell Construction Site 3D Point Cloud Dataset](https://www.nature.com/articles/s41597-025-05827-7) 15 | 16 | 17 | ## Overview 18 | 19 | * [Data Records](#data-records) 20 | * [The Scope Of The Data](#the-scope-of-the-data) 21 | * [The Dataset Structure](#the-dataset-structure) 22 | * [Installation](#installation) 23 | * [Download and Extract the Data](#download-and-extract-the-data) 24 | * [Citation](#citation) 25 | * [Acknowledgement](#acknowledgement) 26 | 27 | 28 | ## Data Records 29 | 30 | The Rohbau3D data records can be summarized as a medium-scale repository of terrestrial laser scan point clouds covering static scenes from a wide variety of shell construction sides. The records include the spatial coordinates annotated with the sensor-specific (1) RGB color, (2) surface reflection intensity information, (3) the reconstruction of surface normal vectors, and (4) panoramic 2D image representations of all feature spaces 31 | 32 | 33 | ### The Scope Of The Data 34 | 35 | The repository contains in total a set of 504 scenes captured in one of 14 different building environments. 36 | 37 | 38 | File ID | Acquisition Site Overview 39 | -----------|-------------------------- 40 | site_000 | Multi-story apartment block with small to medium-sized rooms in brick wall construction. Some walls plastered, some exposed. Windows present; no doors. Floor mostly dry. 41 | site_001 | Multi-story apartment block with small to medium-sized rooms. Sloping ceilings, brick wall construction, walls partially plastered. Windows present; no doors. Floor mostly dry. 42 | site_002 | Reinforced concrete underground parking structure with low to high ceilings and column grid. Poor lighting. Water puddles on floor. 43 | site_003 | Multi-story school building with large rooms. Reinforced concrete skeleton construction. Good lighting. Water puddles on floor. 44 | site_004 | Large hall in reinforced concrete with round ceiling elements. Large floor opening. No facade installed. 45 | site_005 | Multi-story school building with rooms of varying sizes. Drywall partitions. Semi-transparent temporary facade covering. 46 | site_006 | Multi-story school building with medium to large rooms. Drywall partitions in some areas. Open facade surfaces. Technical equipment installed on ceilings. 47 | site_007 | Large hall with high ceiling. Reinforced concrete construction. 48 | site_008 | Multi-story office building with small to large rooms and freestanding drywall supports. Glazed facade installed. Technical equipment on ceilings installed. 49 | site_009\* | Multi-story brick building under renovation. Historic features. Small rooms and narrow staircases. Windows present; no doors. Poor lighting. 50 | site_010\* | Vaulted cellar of brick structure. Small rooms. Uneven floors. Poor lighting. 51 | site_011 | Two-story structure with basement. Mixed brick and precast concrete construction. Small to medium rooms. Water on floors. Poor lighting. 52 | site_012 | Multi-story apartment block with basement. Reinforced concrete prefabricated construction. Large window and door openings. Some scenes contain water on the floor and show poor lighting. 53 | site_013\* | Multi-story brick building under renovation. Small rooms connected by corridors. Walls partly plastered, partly exposed. Mostly clean floors. 54 | -------------------------------------- 55 | *Renovation sites are indicated with an asterisk (*).* 56 | 57 | 58 | ### The Dataset Structure 59 | ``` 60 | rohbau3d 61 | |-- metadata 62 | | |-- site_list.yaml 63 | | |-- site_metadata.yaml 64 | | '-- ... 65 | | 66 | |-- site_00 67 | | |-- scan_00000 68 | | | |-- coord.npy 69 | | | |-- color.npy 70 | | | |-- intensity.npy 71 | | | |-- normal.npy 72 | | | |-- panorama.png 73 | | | '-- ... 74 | | | 75 | | |-- scan_00001 76 | | |-- scan_00002 77 | | '-- ... 78 | | 79 | |-- site_001 80 | | |-- scan_01000 81 | | '-- ... 82 | | 83 | ... 84 | '-- site_013 85 | ``` 86 | 87 | 88 | ## Installation 89 | 90 | ### Requirements 91 | * git 92 | * pooch 93 | * tqdm 94 | * zstandard 95 | * yaml 96 | 97 | ### Clone Repository 98 | Clone the Rohbau3D Repository to a local space. 99 | 100 | ```bash 101 | git clone https://github.com/RauchLukas/Rohbau3D.git 102 | ``` 103 | 104 | ### Conda Environment 105 | Manually create a conda environment and install the package 106 | 107 | ```bash 108 | conda create -n rohbau3d python=3.11 -y 109 | conda activate rohbau3d 110 | 111 | cd Rohbau3D 112 | pip install . 113 | ``` 114 | 115 | ## Download and Extract the Data 116 | 117 | The Dataset can be directly downloaded in chunks from Dataverse | OpenData UniBw M: 118 | 119 | > Download Link: [https://open-data.unibw.de/dataset](https://open-data.unibw.de/dataset.xhtml?persistentId=doi:10.60776/ZWJFI4) 120 | 121 | Conveniently, this repository offers also the option of downloading the entire dataset or individual pieces using scripts. [**RECOMMENDED**] 122 | 123 | 124 | 125 | - **Short Version:** 126 | 127 | Inside the `Rohbau3D` folder, run the `scripts/download.py` script to download all dataset point cloud files. 128 | 129 | ```bash 130 | python scripts/download.py --config config/dataverse.yaml --download --extract 131 | ``` 132 | **Options:** 133 | - `--config` [required] : set the path to the configuration script. 134 | - `--download` [optional] : Flag to enable download. Default=False. 135 | - `--extract` [optional] : Flag to enable file extraction. Default=True. 136 | 137 | 138 | - **Manual Configuration:** 139 | 140 | Customize the configuration inside the `config/dataverse.yaml` file: 141 | 142 | ```yaml 143 | # CONFIGURATION 144 | # Rohbau3D 145 | 146 | # GENERAL 147 | config_dir: config 148 | log_dir: log 149 | log_level: INFO 150 | 151 | # DOWNLOAD 152 | download_hub: dataverse 153 | download_dir: data/download 154 | feature_index_file: dataverse_file_index.json 155 | feature_selection: [all] 156 | scene_selection: [all] 157 | 158 | # FILE EXTRACT 159 | extract_dir: data/extract 160 | clean_download_files: False 161 | ``` 162 | 163 | **Options:** 164 | 165 | - `config_dir` : Set the *path/to/the/configuration/files* location. 166 | - `log_dir`: Set the logging *path/to/the/logging* location. 167 | - `log_level` : Set the logging Level. 168 | - `download_hub`: Set the download server / hub. [Allowed options: `dataverse` and `default`] 169 | > *Note: At the moment, the data can only be downloaded from Dataverse [https://open-data.unibw.de/](https://open-data.unibw.de/)*. 170 | - `download_dir` : Set the *path/to/the/download* location. 171 | - `feature_index_file` : Name the content index file for the download hub. 172 | - `feature_selection` : Select the point cloud features to download as a `list`. Options include: 173 | - `coord` : the actual xyz coordinate of the points. 174 | - `color` : the RGB color annotation od the points. 175 | - `intensity` : the LiDAR reflection intensity annotation of the points. 176 | - `normal` : the reconstructed surface normal annotation of the points. 177 | - `all` : selects all available point cloud features. 178 | - `extract_dir` : Set the *path/to/the/file/extraction* location. 179 | - `clean_download_files` : Set the Flag `True`, `False` to delete the download directory at the end of the script. 180 | 181 | 182 | 183 | 184 | 185 | ## Citation 186 | 187 | If you find our work useful in your research, please cite our paper: 188 | 189 | ``` 190 | @article{rauch.Rohbau3D.2025, 191 | title = {Rohbau3D: A Shell Construction Site 3D Point Cloud Dataset}, 192 | shorttitle = {Rohbau3D}, 193 | author = {Rauch, Lukas and Braml, Thomas}, 194 | year = {2025}, 195 | journal = {Scientific Data}, 196 | volume = {12}, 197 | number = {1}, 198 | pages = {1478}, 199 | publisher = {Nature Publishing Group}, 200 | issn = {2052-4463}, 201 | doi = {10.1038/s41597-025-05827-7}, 202 | } 203 | ``` 204 | 205 | 206 | ## Acknowledgement 207 | The surface normal estimation in this repo is based on/inspired by great works, including but not limited to: 208 | [SHS-Net](https://github.com/LeoQLi/SHS-Net) 209 | -------------------------------------------------------------------------------- /config/dataverse_file_index.json: -------------------------------------------------------------------------------- 1 | { 2 | "color": { 3 | "0": [ 4 | "site_00.color.part000.tar.zst" 5 | ], 6 | "1": [ 7 | "site_01.color.part000.tar.zst" 8 | ], 9 | "2": [ 10 | "site_02.color.part000.tar.zst" 11 | ], 12 | "3": [ 13 | "site_03.color.part000.tar.zst", 14 | "site_03.color.part001.tar.zst", 15 | "site_03.color.part002.tar.zst" 16 | ], 17 | "4": [ 18 | "site_04.color.part000.tar.zst" 19 | ], 20 | "5": [ 21 | "site_05.color.part000.tar.zst", 22 | "site_05.color.part001.tar.zst", 23 | "site_05.color.part002.tar.zst" 24 | ], 25 | "6": [ 26 | "site_06.color.part000.tar.zst", 27 | "site_06.color.part001.tar.zst", 28 | "site_06.color.part002.tar.zst" 29 | ], 30 | "7": [ 31 | "site_07.color.part000.tar.zst" 32 | ], 33 | "8": [ 34 | "site_08.color.part000.tar.zst", 35 | "site_08.color.part001.tar.zst", 36 | "site_08.color.part002.tar.zst" 37 | ], 38 | "9": [ 39 | "site_09.color.part000.tar.zst", 40 | "site_09.color.part001.tar.zst" 41 | ], 42 | "10": [ 43 | "site_10.color.part000.tar.zst" 44 | ], 45 | "11": [ 46 | "site_11.color.part000.tar.zst", 47 | "site_11.color.part001.tar.zst" 48 | ], 49 | "12": [ 50 | "site_12.color.part000.tar.zst", 51 | "site_12.color.part001.tar.zst" 52 | ], 53 | "13": [ 54 | "site_13.color.part000.tar.zst", 55 | "site_13.color.part001.tar.zst" 56 | ] 57 | }, 58 | "coord": { 59 | "0": [ 60 | "site_00.coord.part000.tar.zst", 61 | "site_00.coord.part001.tar.zst", 62 | "site_00.coord.part002.tar.zst", 63 | "site_00.coord.part003.tar.zst" 64 | ], 65 | "1": [ 66 | "site_01.coord.part000.tar.zst", 67 | "site_01.coord.part001.tar.zst" 68 | ], 69 | "2": [ 70 | "site_02.coord.part000.tar.zst" 71 | ], 72 | "3": [ 73 | "site_03.coord.part000.tar.zst", 74 | "site_03.coord.part001.tar.zst", 75 | "site_03.coord.part002.tar.zst", 76 | "site_03.coord.part003.tar.zst", 77 | "site_03.coord.part004.tar.zst", 78 | "site_03.coord.part005.tar.zst", 79 | "site_03.coord.part006.tar.zst", 80 | "site_03.coord.part007.tar.zst", 81 | "site_03.coord.part008.tar.zst", 82 | "site_03.coord.part009.tar.zst" 83 | ], 84 | "4": [ 85 | "site_04.coord.part000.tar.zst", 86 | "site_04.coord.part001.tar.zst", 87 | "site_04.coord.part002.tar.zst" 88 | ], 89 | "5": [ 90 | "site_05.coord.part000.tar.zst", 91 | "site_05.coord.part001.tar.zst", 92 | "site_05.coord.part002.tar.zst", 93 | "site_05.coord.part003.tar.zst", 94 | "site_05.coord.part004.tar.zst", 95 | "site_05.coord.part005.tar.zst", 96 | "site_05.coord.part006.tar.zst", 97 | "site_05.coord.part007.tar.zst", 98 | "site_05.coord.part008.tar.zst", 99 | "site_05.coord.part009.tar.zst", 100 | "site_05.coord.part010.tar.zst" 101 | ], 102 | "6": [ 103 | "site_06.coord.part000.tar.zst", 104 | "site_06.coord.part001.tar.zst", 105 | "site_06.coord.part002.tar.zst", 106 | "site_06.coord.part003.tar.zst", 107 | "site_06.coord.part004.tar.zst", 108 | "site_06.coord.part005.tar.zst", 109 | "site_06.coord.part006.tar.zst", 110 | "site_06.coord.part007.tar.zst", 111 | "site_06.coord.part008.tar.zst", 112 | "site_06.coord.part009.tar.zst", 113 | "site_06.coord.part010.tar.zst" 114 | ], 115 | "7": [ 116 | "site_07.coord.part000.tar.zst" 117 | ], 118 | "8": [ 119 | "site_08.coord.part000.tar.zst", 120 | "site_08.coord.part001.tar.zst", 121 | "site_08.coord.part002.tar.zst", 122 | "site_08.coord.part003.tar.zst", 123 | "site_08.coord.part004.tar.zst", 124 | "site_08.coord.part005.tar.zst", 125 | "site_08.coord.part006.tar.zst", 126 | "site_08.coord.part007.tar.zst", 127 | "site_08.coord.part008.tar.zst", 128 | "site_08.coord.part009.tar.zst" 129 | ], 130 | "9": [ 131 | "site_09.coord.part000.tar.zst", 132 | "site_09.coord.part001.tar.zst", 133 | "site_09.coord.part002.tar.zst", 134 | "site_09.coord.part003.tar.zst", 135 | "site_09.coord.part004.tar.zst", 136 | "site_09.coord.part005.tar.zst" 137 | ], 138 | "10": [ 139 | "site_10.coord.part000.tar.zst" 140 | ], 141 | "11": [ 142 | "site_11.coord.part000.tar.zst", 143 | "site_11.coord.part001.tar.zst", 144 | "site_11.coord.part002.tar.zst", 145 | "site_11.coord.part003.tar.zst", 146 | "site_11.coord.part004.tar.zst", 147 | "site_11.coord.part005.tar.zst", 148 | "site_11.coord.part006.tar.zst" 149 | ], 150 | "12": [ 151 | "site_12.coord.part000.tar.zst", 152 | "site_12.coord.part001.tar.zst", 153 | "site_12.coord.part002.tar.zst", 154 | "site_12.coord.part003.tar.zst", 155 | "site_12.coord.part004.tar.zst", 156 | "site_12.coord.part005.tar.zst", 157 | "site_12.coord.part006.tar.zst" 158 | ], 159 | "13": [ 160 | "site_13.coord.part000.tar.zst", 161 | "site_13.coord.part001.tar.zst", 162 | "site_13.coord.part002.tar.zst", 163 | "site_13.coord.part003.tar.zst", 164 | "site_13.coord.part004.tar.zst" 165 | ] 166 | }, 167 | "intensity": { 168 | "0": [ 169 | "site_00.intensity.part000.tar.zst", 170 | "site_00.intensity.part001.tar.zst" 171 | ], 172 | "1": [ 173 | "site_01.intensity.part000.tar.zst" 174 | ], 175 | "2": [ 176 | "site_02.intensity.part000.tar.zst" 177 | ], 178 | "3": [ 179 | "site_03.intensity.part000.tar.zst", 180 | "site_03.intensity.part001.tar.zst", 181 | "site_03.intensity.part002.tar.zst", 182 | "site_03.intensity.part003.tar.zst" 183 | ], 184 | "4": [ 185 | "site_04.intensity.part000.tar.zst" 186 | ], 187 | "5": [ 188 | "site_05.intensity.part000.tar.zst", 189 | "site_05.intensity.part001.tar.zst", 190 | "site_05.intensity.part002.tar.zst", 191 | "site_05.intensity.part003.tar.zst" 192 | ], 193 | "6": [ 194 | "site_06.intensity.part000.tar.zst", 195 | "site_06.intensity.part001.tar.zst", 196 | "site_06.intensity.part002.tar.zst", 197 | "site_06.intensity.part003.tar.zst" 198 | ], 199 | "7": [ 200 | "site_07.intensity.part000.tar.zst" 201 | ], 202 | "8": [ 203 | "site_08.intensity.part000.tar.zst", 204 | "site_08.intensity.part001.tar.zst", 205 | "site_08.intensity.part002.tar.zst", 206 | "site_08.intensity.part003.tar.zst" 207 | ], 208 | "9": [ 209 | "site_09.intensity.part000.tar.zst", 210 | "site_09.intensity.part001.tar.zst" 211 | ], 212 | "10": [ 213 | "site_10.intensity.part000.tar.zst" 214 | ], 215 | "11": [ 216 | "site_11.intensity.part000.tar.zst", 217 | "site_11.intensity.part001.tar.zst", 218 | "site_11.intensity.part002.tar.zst" 219 | ], 220 | "12": [ 221 | "site_12.intensity.part000.tar.zst", 222 | "site_12.intensity.part001.tar.zst", 223 | "site_12.intensity.part002.tar.zst" 224 | ], 225 | "13": [ 226 | "site_13.intensity.part000.tar.zst", 227 | "site_13.intensity.part001.tar.zst" 228 | ] 229 | }, 230 | "normal": { 231 | "0": [ 232 | "site_00.normal.part000.tar.zst", 233 | "site_00.normal.part001.tar.zst", 234 | "site_00.normal.part002.tar.zst", 235 | "site_00.normal.part003.tar.zst" 236 | ], 237 | "1": [ 238 | "site_01.normal.part000.tar.zst", 239 | "site_01.normal.part001.tar.zst" 240 | ], 241 | "2": [ 242 | "site_02.normal.part000.tar.zst" 243 | ], 244 | "3": [ 245 | "site_03.normal.part000.tar.zst", 246 | "site_03.normal.part001.tar.zst", 247 | "site_03.normal.part002.tar.zst", 248 | "site_03.normal.part003.tar.zst", 249 | "site_03.normal.part004.tar.zst", 250 | "site_03.normal.part005.tar.zst", 251 | "site_03.normal.part006.tar.zst", 252 | "site_03.normal.part007.tar.zst", 253 | "site_03.normal.part008.tar.zst", 254 | "site_03.normal.part009.tar.zst" 255 | ], 256 | "4": [ 257 | "site_04.normal.part000.tar.zst", 258 | "site_04.normal.part001.tar.zst", 259 | "site_04.normal.part002.tar.zst" 260 | ], 261 | "5": [ 262 | "site_05.normal.part000.tar.zst", 263 | "site_05.normal.part001.tar.zst", 264 | "site_05.normal.part002.tar.zst", 265 | "site_05.normal.part003.tar.zst", 266 | "site_05.normal.part004.tar.zst", 267 | "site_05.normal.part005.tar.zst", 268 | "site_05.normal.part006.tar.zst", 269 | "site_05.normal.part007.tar.zst", 270 | "site_05.normal.part008.tar.zst", 271 | "site_05.normal.part009.tar.zst", 272 | "site_05.normal.part010.tar.zst" 273 | ], 274 | "6": [ 275 | "site_06.normal.part000.tar.zst", 276 | "site_06.normal.part001.tar.zst", 277 | "site_06.normal.part002.tar.zst", 278 | "site_06.normal.part003.tar.zst", 279 | "site_06.normal.part004.tar.zst", 280 | "site_06.normal.part005.tar.zst", 281 | "site_06.normal.part006.tar.zst", 282 | "site_06.normal.part007.tar.zst", 283 | "site_06.normal.part008.tar.zst", 284 | "site_06.normal.part009.tar.zst", 285 | "site_06.normal.part010.tar.zst" 286 | ], 287 | "7": [ 288 | "site_07.normal.part000.tar.zst" 289 | ], 290 | "8": [ 291 | "site_08.normal.part000.tar.zst", 292 | "site_08.normal.part001.tar.zst", 293 | "site_08.normal.part002.tar.zst", 294 | "site_08.normal.part003.tar.zst", 295 | "site_08.normal.part004.tar.zst", 296 | "site_08.normal.part005.tar.zst", 297 | "site_08.normal.part006.tar.zst", 298 | "site_08.normal.part007.tar.zst", 299 | "site_08.normal.part008.tar.zst", 300 | "site_08.normal.part009.tar.zst" 301 | ], 302 | "9": [ 303 | "site_09.normal.part000.tar.zst", 304 | "site_09.normal.part001.tar.zst", 305 | "site_09.normal.part002.tar.zst", 306 | "site_09.normal.part003.tar.zst", 307 | "site_09.normal.part004.tar.zst", 308 | "site_09.normal.part005.tar.zst" 309 | ], 310 | "10": [ 311 | "site_10.normal.part000.tar.zst" 312 | ], 313 | "11": [ 314 | "site_11.normal.part000.tar.zst", 315 | "site_11.normal.part001.tar.zst", 316 | "site_11.normal.part002.tar.zst", 317 | "site_11.normal.part003.tar.zst", 318 | "site_11.normal.part004.tar.zst", 319 | "site_11.normal.part005.tar.zst", 320 | "site_11.normal.part006.tar.zst" 321 | ], 322 | "12": [ 323 | "site_12.normal.part000.tar.zst", 324 | "site_12.normal.part001.tar.zst", 325 | "site_12.normal.part002.tar.zst", 326 | "site_12.normal.part003.tar.zst", 327 | "site_12.normal.part004.tar.zst", 328 | "site_12.normal.part005.tar.zst", 329 | "site_12.normal.part006.tar.zst" 330 | ], 331 | "13": [ 332 | "site_13.normal.part000.tar.zst", 333 | "site_13.normal.part001.tar.zst", 334 | "site_13.normal.part002.tar.zst", 335 | "site_13.normal.part003.tar.zst", 336 | "site_13.normal.part004.tar.zst" 337 | ] 338 | } 339 | } --------------------------------------------------------------------------------