├── .flake8 ├── .gitattributes ├── .github └── workflows │ ├── deploy-docs.yml │ ├── unit-tests-dev.yml │ └── unit-tests-main.yml ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── Makefile ├── _static │ └── custom.css ├── conf.py ├── index.rst ├── make.bat ├── refs.bib └── source │ ├── bibliography.rst │ ├── camera.rst │ ├── datasets.rst │ ├── datasplit.rst │ ├── examples.rst │ ├── geometry │ ├── common.rst │ ├── contraction.rst │ ├── index.rst │ ├── interpolation.rst │ ├── primitives.rst │ ├── projections.rst │ ├── quaternions.rst │ ├── rigid.rst │ └── trajectories.rst │ ├── installation.rst │ ├── io │ └── index.rst │ ├── loaders │ ├── dynamic.rst │ ├── index.rst │ └── static.rst │ ├── mvdataset.rst │ ├── tensorreel.rst │ ├── utils │ └── index.rst │ └── visualization │ └── index.rst ├── examples ├── .deprecated │ ├── chained_bbs_tranformations.py │ ├── convert_to_ingp_format.py │ ├── convert_to_opengl_format.py │ ├── mesh_rendering.py │ ├── ray_bb_intersect.py │ ├── save_video.py │ ├── scene_rendering.py │ └── spiral_trajectory.py ├── __init__.py ├── assets │ ├── .deprecated │ │ └── assetsnerf │ │ │ └── dmsr │ │ │ └── dinning │ │ │ └── icp │ │ │ ├── chair.npy │ │ │ └── vase.npy │ ├── error_maps │ │ └── plushy_test_0.png │ ├── meshes │ │ ├── blendernerf │ │ │ └── plushy.ply │ │ ├── dmsr │ │ │ └── dinning.ply │ │ ├── dtu │ │ │ └── dtu_scan83.ply │ │ └── shelly │ │ │ └── khady.ply │ └── point_clouds │ │ ├── llff │ │ └── fern.ply │ │ ├── mipnerf360 │ │ ├── bicycle.ply │ │ └── garden.ply │ │ └── nerf_synthetic │ │ └── lego.ply ├── bounding_primitives_vis.py ├── camera_data_vis.py ├── camera_depth_unprojection_vis.py ├── camera_points_projection_vis.py ├── camera_rays_sampling_vis.py ├── camera_trajectory_vis.py ├── dataset_config_to_yaml.py ├── dataset_splits_vis.py ├── pixels_sampling_vis.py ├── sample_cameras_on_hemisphere.py ├── training_loop_dataloader.py └── training_loop_tensorreel.py ├── imgs ├── MVD.png ├── art.webp ├── blender_test_cameras.png ├── blender_train_cameras.png ├── pose_and_intrinsics.png └── projection_with_principal_point_offset.png ├── mvdatasets ├── __init__.py ├── camera.py ├── configs │ ├── __init__.py │ ├── base_config.py │ ├── dataset_config.py │ ├── datasets_configs.py │ ├── example_config.py │ └── machine_config.py ├── datasplit.py ├── geometry │ ├── __init__.py │ ├── common.py │ ├── contraction.py │ ├── interpolation.py │ ├── primitives │ │ ├── __init__.py │ │ ├── bounding_box.py │ │ ├── bounding_sphere.py │ │ └── point_cloud.py │ ├── projections.py │ ├── quaternions.py │ ├── rigid.py │ └── trajectories.py ├── io │ ├── __init__.py │ └── yaml_utils.py ├── loaders │ ├── __init__.py │ ├── dynamic │ │ ├── d_nerf.py │ │ ├── flow3d.py │ │ ├── iphone.py │ │ ├── monst3r.py │ │ ├── nerfies.py │ │ ├── neu3d.py │ │ ├── panoptic_sports.py │ │ └── visor.py │ └── static │ │ ├── blender.py │ │ ├── colmap.py │ │ ├── dmsr.py │ │ └── dtu.py ├── mvdataset.py ├── tensorreel.py ├── utils │ ├── .deprecated │ │ ├── data_converter.py │ │ ├── loader.py │ │ └── pycolmap.py │ ├── __init__.py │ ├── camera_utils.py │ ├── images.py │ ├── loader_utils.py │ ├── memory.py │ ├── mesh.py │ ├── open3d_rendering.py │ ├── point_clouds.py │ ├── printing.py │ ├── profiler.py │ ├── raycasting.py │ ├── tensor_mesh.py │ ├── tensor_texture.py │ ├── texture.py │ ├── video_utils.py │ └── virtual_cameras.py └── visualization │ ├── __init__.py │ ├── colormaps.py │ ├── matplotlib.py │ └── video_gen.py ├── pyproject.toml ├── scripts └── download │ ├── dtu.sh │ ├── hypernerf.sh │ ├── iphone.md │ ├── iphone_som.sh │ ├── mipnerf360.sh │ ├── nerf_furry.sh │ ├── nerf_llff.sh │ ├── nerf_synthetic.sh │ ├── neu3d.sh │ ├── panoptic_sports.sh │ ├── ref_nerf.sh │ └── shelly.sh ├── setup.py └── tests ├── __init__.py ├── test_projection_utils.py ├── test_quaternion_utils.py └── test_rigid_geometry_utils.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | experimental merge=ours 2 | -------------------------------------------------------------------------------- /.github/workflows/deploy-docs.yml: -------------------------------------------------------------------------------- 1 | name: deploy-docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | deploy: 13 | runs-on: ubuntu-22.04 14 | 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v3 18 | 19 | - name: Set up Python 20 | uses: actions/setup-python@v4 21 | with: 22 | python-version: 3.10.15 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install sphinx sphinx-rtd-theme sphinxcontrib-mermaid sphinxcontrib-bibtex myst-parser 28 | 29 | - name: Build documentation 30 | run: sphinx-build -b html docs/ docs/_build/html/ 31 | 32 | - name: Deploy to GitHub Pages 33 | uses: peaceiris/actions-gh-pages@v3 34 | with: 35 | github_token: ${{ secrets.GITHUB_TOKEN }} 36 | publish_dir: docs/_build/html/ 37 | git_config_user_name: github-actions[bot] 38 | git_config_user_email: github-actions[bot]@users.noreply.github.com 39 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests-dev.yml: -------------------------------------------------------------------------------- 1 | name: unit-tests-dev 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | 8 | jobs: 9 | format: 10 | runs-on: ubuntu-22.04 11 | 12 | steps: 13 | - name: Checkout repository 14 | uses: actions/checkout@v3 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: 3.10.15 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install -e ".[tests]" 25 | 26 | - name: Run unit tests 27 | run: | 28 | pytest --maxfail=1 -------------------------------------------------------------------------------- /.github/workflows/unit-tests-main.yml: -------------------------------------------------------------------------------- 1 | name: unit-tests-main 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | format: 10 | runs-on: ubuntu-22.04 11 | 12 | steps: 13 | - name: Checkout repository 14 | uses: actions/checkout@v3 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: 3.10.15 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install -e ".[tests]" 25 | 26 | - name: Run unit tests 27 | run: | 28 | pytest --maxfail=1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | mvdatasets.egg-info 3 | *.pickle 4 | *.pyo 5 | *.pyc 6 | data 7 | outputs 8 | experimental 9 | docs/_build 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Attribution 4.0 International License (CC BY 4.0) 2 | 3 | Copyright (c) 2025 Stefano Esposito and Andreas Geiger 4 | 5 | You are free to: 6 | - Share — copy and redistribute the material in any medium, format, or platform 7 | - Adapt — remix, transform, and build upon the material for any purpose, even commercially 8 | 9 | Under the following terms: 10 | - Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. 11 | 12 | Full license text: https://creativecommons.org/licenses/by/4.0/ 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MVDatasets 2 | 3 | 4 | 5 | 6 | ### Standardized DataLoaders for 3D Computer Vision 7 | 8 | [Stefano Esposito](https://s-esposito.github.io/), [Andreas Geiger](https://www.cvlibs.net/) 9 |
10 | University of Tübingen, [Autonomous Vision Group](https://uni-tuebingen.de/fakultaeten/mathematisch-naturwissenschaftliche-fakultaet/fachbereiche/informatik/lehrstuehle/autonomous-vision/home/) 11 | 12 | [![unit-tests-main](https://github.com/autonomousvision/mvdatasets/actions/workflows/unit-tests-main.yml/badge.svg)](https://github.com/autonomousvision/mvdatasets/actions/workflows/unit-tests-main.yml) [![unit-tests-dev](https://github.com/autonomousvision/mvdatasets/actions/workflows/unit-tests-dev.yml/badge.svg)](https://github.com/autonomousvision/mvdatasets/actions/workflows/unit-tests-dev.yml) [![deploy-docs](https://github.com/autonomousvision/mvdatasets/actions/workflows/deploy-docs.yml/badge.svg)](https://github.com/autonomousvision/mvdatasets/actions/workflows/deploy-docs.yml) 13 | 14 | ``` 15 | ⚠️ This is a work in progress research codebase designed with a focus on modularity; future updates *will try* not to disrupt existing functionalities. 16 | ``` 17 | 18 | 19 | Our goal is to provide a plug and play library to quickly develop and test new research ideas. We offer various data loaders for commonly used multi-view datasets in 3D reconstruction and view-synthesis, that work out of the box without further data processing. 20 | 21 | ## Documentation 22 | 23 | Don't miss out on our [documentation](https://autonomousvision.github.io/mvdatasets/index.html). It's still a work in progress, but together with our [examples](examples), it is the best way to get started. 24 | 25 | ### Installation 26 | 27 | Install the module from source using `pip install .`. More details [here](https://autonomousvision.github.io/mvdatasets/source/installation.html). 28 | 29 | ### Datasets 30 | 31 | Checkout the state of currently supported datasets or work in progress [here](https://autonomousvision.github.io/mvdatasets/source/datasets.html). 32 | 33 | ### Camera 34 | 35 | Our standardized camera uses the OpenCV camera coordinate system and it is described [here](https://autonomousvision.github.io/mvdatasets/source/datasets.html). 36 | 37 |

38 | 39 | 40 |

41 | 42 | Images taken from Andreas Geiger's Computer Vision [lectures](https://uni-tuebingen.de/fakultaeten/mathematisch-naturwissenschaftliche-fakultaet/fachbereiche/informatik/lehrstuehle/autonomous-vision/lectures/computer-vision/) at the University of Tübingen. 43 | 44 | ## Examples 45 | 46 | ```bash 47 | # download data in ./data 48 | bash scripts/download/nerf_synthetic.sh 49 | # visualize dataset splits 50 | python examples/dataset_splits_vis.py --scene-name lego data:nerf-synthetic 51 | ``` 52 | 53 |

54 | 55 | 56 |

57 | 58 | ## Disclaimer 59 | 60 | Functions located in any `.deprecated` folder may no longer work as expected. While they might be supported again in the future, this is not guaranteed. 61 | 62 | ## License 63 | 64 | This project is licensed under the Creative Commons Attribution 4.0 International License (CC BY 4.0). See the [LICENSE](LICENSE) file for details. 65 | 66 | You are free to use, modify, and distribute this code as long as you provide proper attribution to the original author(s). 67 | 68 | ## Citation 69 | 70 | If you use this library for your research, please consider citing: 71 | 72 | ```bibtex 73 | @misc{Esposito2024MVDatasets, 74 | author = {Stefano Esposito and Andreas Geiger}, 75 | title = {MVDatasets: Standardized DataLoaders for 3D Computer Vision}, 76 | year = {2025}, 77 | url = {https://github.com/autonomousvision/mvdatasets}, 78 | note = {GitHub repository} 79 | } 80 | ``` -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Arial, sans-serif; 3 | } -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(0, os.path.abspath("../mvdatasets")) 5 | 6 | # Configuration file for the Sphinx documentation builder. 7 | # 8 | # For the full list of built-in configuration values, see the documentation: 9 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 10 | 11 | # -- Project information ----------------------------------------------------- 12 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 13 | 14 | project = "MVDatasets" 15 | copyright = "2025, Stefano Esposito, Andreas Geiger" 16 | author = "Stefano Esposito, Andreas Geiger" 17 | release = "1.0.0" 18 | 19 | # -- General configuration --------------------------------------------------- 20 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 21 | 22 | extensions = [ 23 | "sphinx.ext.autodoc", # Enables automatic docstring extraction 24 | "sphinx.ext.napoleon", # Supports Google-style and NumPy-style docstrings 25 | "sphinxcontrib.mermaid", # Enables Mermaid diagrams 26 | "sphinxcontrib.bibtex", # Enables BibTeX citations 27 | # "sphinxcontrib.osexample", # Enable tabs for multiple code examples 28 | "sphinx.ext.autosectionlabel", # Enables autosectionlabel 29 | ] 30 | 31 | 32 | templates_path = ["_templates"] 33 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 34 | 35 | 36 | # -- Options for HTML output ------------------------------------------------- 37 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 38 | 39 | html_theme = "sphinx_rtd_theme" 40 | html_theme_options = { 41 | "analytics_id": "G-1T0QG65M3V", 42 | "analytics_anonymize_ip": False, 43 | "logo_only": False, 44 | "prev_next_buttons_location": "bottom", 45 | "style_external_links": False, 46 | "vcs_pageview_mode": "", 47 | "style_nav_header_background": "black", 48 | "flyout_display": "hidden", 49 | "version_selector": False, 50 | "language_selector": False, 51 | # Toc options 52 | "collapse_navigation": True, 53 | "sticky_navigation": True, 54 | "navigation_depth": 4, 55 | "includehidden": True, 56 | "titles_only": False, 57 | } 58 | html_static_path = ["_static"] 59 | html_css_files = ["custom.css"] 60 | html_logo = "../imgs/MVD.png" 61 | 62 | # bibtex 63 | bibtex_bibfiles = ["refs.bib"] 64 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | MVDatasets 2 | ========== 3 | 4 | Welcome to the documentation of the MVDatasets: Standardized DataLoaders for 3D Computer Vision. 5 | Have a look around and feel free to reach out if you have any questions or suggestions. 6 | Examples are the best way to get started! 7 | 8 | ⚠️ This is a work in progress research codebase designed with a focus on modularity; future updates *will try* not to disrupt existing functionalities. 9 | 10 | -------- 11 | 12 | .. toctree:: 13 | :maxdepth: 1 14 | :caption: Get started 15 | 16 | source/installation 17 | source/examples 18 | source/datasets 19 | 20 | ------------ 21 | 22 | .. toctree:: 23 | :maxdepth: 1 24 | :caption: Main Modules 25 | 26 | source/camera 27 | source/mvdataset 28 | source/datasplit 29 | source/tensorreel 30 | 31 | --------------- 32 | 33 | .. toctree:: 34 | :maxdepth: 1 35 | :caption: Utility Modules 36 | 37 | source/io/index 38 | source/geometry/index 39 | source/loaders/index 40 | source/utils/index 41 | source/visualization/index 42 | 43 | --------------- 44 | 45 | License 46 | ------- 47 | 48 | This project is licensed under the Creative Commons Attribution 4.0 International License (CC BY 4.0). 49 | You are free to use, modify, and distribute this code as long as you provide proper attribution to the original author(s). 50 | 51 | Citation 52 | -------- 53 | 54 | If you use this library for your research, please consider citing: 55 | 56 | .. code-block:: bibtex 57 | 58 | @misc{Esposito2024MVDatasets, 59 | author = {Stefano Esposito and Andreas Geiger}, 60 | title = {MVDatasets: Standardized DataLoaders for 3D Computer Vision}, 61 | year = {2025}, 62 | url = {https://github.com/autonomousvision/mvdatasets}, 63 | note = {GitHub repository} 64 | } 65 | 66 | ------------ 67 | 68 | .. toctree:: 69 | :maxdepth: 1 70 | :caption: Credits 71 | 72 | source/bibliography -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/refs.bib: -------------------------------------------------------------------------------- 1 | @misc{Esposito2024MVDatasets, 2 | author = {Stefano Esposito and Andreas Geiger}, 3 | title = {MVDatasets: Standardized DataLoaders for 3D Computer Vision}, 4 | year = {2025}, 5 | url = {https://github.com/autonomousvision/mvdatasets}, 6 | note = {GitHub repository} 7 | } 8 | 9 | @inproceedings{zhou2019continuity, 10 | title={On the continuity of rotation representations in neural networks}, 11 | author={Zhou, Yi and Barnes, Connelly and Lu, Jingwan and Yang, Jimei and Li, Hao}, 12 | booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, 13 | pages={5745--5753}, 14 | year={2019} 15 | } 16 | 17 | @article{barron2022mipnerf360, 18 | title={Mip-NeRF 360: Unbounded Anti-Aliased Neural Radiance Fields}, 19 | author={Jonathan T. Barron and Ben Mildenhall and 20 | Dor Verbin and Pratul P. Srinivasan and Peter Hedman}, 21 | journal={CVPR}, 22 | year={2022} 23 | } 24 | 25 | @inproceedings{gao2022dynamic, 26 | title={Monocular Dynamic View Synthesis: A Reality Check}, 27 | author={Gao, Hang and Li, Ruilong and Tulsiani, Shubham and Russell, Bryan and Kanazawa, Angjoo}, 28 | booktitle={NeurIPS}, 29 | year={2022}, 30 | } -------------------------------------------------------------------------------- /docs/source/bibliography.rst: -------------------------------------------------------------------------------- 1 | Bibliography 2 | ============ 3 | 4 | .. bibliography:: 5 | :style: plain -------------------------------------------------------------------------------- /docs/source/camera.rst: -------------------------------------------------------------------------------- 1 | Camera 2 | ====== 3 | 4 | Coordinate system 5 | ----------------- 6 | 7 | Central class of the library; it provides a standardized format for camera parameters, its data and methods to manipulate them. 8 | All data in stored in CPU. 9 | 10 | We use the OpenCV camera coordinate system: 11 | 12 | - X axis: Points to the right of the camera's sensor. It extends horizontally from the left side to the right side of the image. Increasing values move towards the right side of the image. 13 | - Y axis: Points downward from the camera's sensor. It extends vertically from the top to the bottom of the image. Increasing values move towards the bottom of the image. 14 | - Z axis: Represents depth and points away from the camera lens. It extends from the camera's lens outward into the scene. Increasing values move away from the camera. 15 | 16 | The Camera Class 17 | ---------------- 18 | 19 | .. automodule:: mvdatasets.camera 20 | :members: 21 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/datasets.rst: -------------------------------------------------------------------------------- 1 | Datasets 2 | ======== 3 | 4 | Currently supported datasets in the library: 5 | 6 | .. raw:: html 7 | 8 |
9 | 10 | 11 | 24 | 36 | 37 |
12 | 13 | Static 14 | 22 | 23 | 25 | 26 | Dynamic 27 | 34 | 35 |
38 |
39 | 40 | 41 | Soon to be added (or not tested): 42 | 43 | .. raw:: html 44 | 45 |
46 | 47 | 48 | 57 | 71 | 72 |
49 | 50 | Static 51 | 55 | 56 | 58 | 59 | Dynamic 60 | 69 | 70 |
73 |
74 | 75 | A dataset labelled as "bounded" means that the dataset is limited to a specific volume, while "unbounded" means that the dataset is not limited to a specific volume, and can be extended indefinitely. 76 | A dataset labelled as "monocular" means that the dataset is captured with a single camera (e.g. casual captures), while "multi-view" means that the dataset is captured with multiple synchronized cameras simultaneosly. 77 | A dataset labelled as "semi-monocular" (e.g. D-NeRF) means that the dataset contain either teleporting cameras or quasi-static scenes :cite:t:`gao2022dynamic`. 78 | 79 | Download 80 | -------- 81 | 82 | Download each dataset by running scripts in the `download `_ directory (``scripts/download``). The scripts will download the data and save it to the `data` directory. 83 | 84 | Configuration 85 | ------------- 86 | 87 | Each dataset has a individual configuration (e.g. ``BlenderConfig``) that extends more general configuration ``DatasetConfig`` that all datasets share. 88 | Configuration can be overridden by command line arguments or loaded from file. -------------------------------------------------------------------------------- /docs/source/datasplit.rst: -------------------------------------------------------------------------------- 1 | DataSplit 2 | ------------------- 3 | 4 | Given a dataset split, it aggregated data for frames or pixels level indexing. 5 | All data in stored in CPU. 6 | 7 | .. code-block:: python 8 | 9 | # index frames 10 | data_split = DataSplit( 11 | cameras=mv_data.get_split("train"), 12 | nr_sequence_frames=nr_sequence_frames, 13 | modalities=mv_data.get_split_modalities("train"), 14 | index_pixels=False 15 | ) 16 | 17 | 18 | .. code-block:: python 19 | 20 | # or, index pixels 21 | data_split = DataSplit( 22 | cameras=mv_data.get_split("train"), 23 | nr_sequence_frames=nr_sequence_frames, 24 | modalities=mv_data.get_split_modalities("train"), 25 | index_pixels=True 26 | ) 27 | 28 | .. automodule:: mvdatasets.datasplit 29 | :members: 30 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | Dataset splits visualization 5 | ---------------------------- 6 | 7 | .. automodule:: examples.dataset_splits_vis 8 | :members: 9 | :undoc-members: 10 | 11 | Camera data visualization 12 | ------------------------- 13 | 14 | .. automodule:: examples.camera_data_vis 15 | :members: 16 | :undoc-members: 17 | 18 | Camera depth unprojection visualization 19 | --------------------------------------- 20 | 21 | For datasets with depth maps in training split. 22 | 23 | .. automodule:: examples.camera_depth_unprojection_vis 24 | :members: 25 | :undoc-members: 26 | 27 | Camera points projection visualization 28 | -------------------------------------- 29 | 30 | For datasets a 3D point cloud. 31 | 32 | .. automodule:: examples.camera_points_projection_vis 33 | :members: 34 | :undoc-members: 35 | 36 | Camera rays sampling visualization 37 | ---------------------------------- 38 | 39 | .. automodule:: examples.camera_rays_sampling_vis 40 | :members: 41 | :undoc-members: 42 | 43 | Camera trajectory visualization 44 | ------------------------------- 45 | 46 | For monocular sequences. 47 | 48 | .. automodule:: examples.camera_trajectory_vis 49 | :members: 50 | :undoc-members: 51 | 52 | 53 | Bounding primitives visualization 54 | --------------------------------- 55 | 56 | .. automodule:: examples.bounding_primitives_vis 57 | :members: 58 | :undoc-members: 59 | 60 | Pixels sampling visualization 61 | ----------------------------- 62 | 63 | Uniform or jittered sampling. 64 | 65 | .. automodule:: examples.pixels_sampling_vis 66 | :members: 67 | :undoc-members: 68 | 69 | Training loop DataLoader 70 | ------------------------ 71 | 72 | Example, speed benchmark and visualization example. 73 | 74 | .. automodule:: examples.training_loop_dataloader 75 | :members: 76 | :undoc-members: 77 | 78 | Training loop TensorReel 79 | ------------------------ 80 | 81 | Example, speed benchmark and visualization example. 82 | 83 | .. automodule:: examples.training_loop_tensorreel 84 | :members: 85 | :undoc-members: 86 | 87 | Sample cameras on hemisphere 88 | ---------------------------- 89 | 90 | Visualizations and speed benchmark. 91 | 92 | .. automodule:: examples.sample_cameras_on_hemisphere 93 | :members: 94 | :undoc-members: 95 | 96 | Dataset configuration to YAML 97 | ----------------------------- 98 | 99 | Load and save dataset configuration to YAML. 100 | 101 | .. automodule:: examples.dataset_config_to_yaml 102 | :members: 103 | :undoc-members: 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /docs/source/geometry/common.rst: -------------------------------------------------------------------------------- 1 | Common 2 | ====== 3 | 4 | .. automodule:: mvdatasets.geometry.common 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/geometry/contraction.rst: -------------------------------------------------------------------------------- 1 | Space contraction 2 | ================= 3 | 4 | .. automodule:: mvdatasets.geometry.contraction 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/geometry/index.rst: -------------------------------------------------------------------------------- 1 | Geometry 2 | ======== 3 | 4 | Primitives 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | 10 | primitives 11 | 12 | Functions 13 | --------- 14 | 15 | .. toctree:: 16 | :maxdepth: 1 17 | 18 | common 19 | contraction 20 | interpolation 21 | projections 22 | quaternions 23 | rigid 24 | trajectories 25 | 26 | -------------------------------------------------------------------------------- /docs/source/geometry/interpolation.rst: -------------------------------------------------------------------------------- 1 | Interpolation 2 | ============= 3 | 4 | .. automodule:: mvdatasets.geometry.interpolation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/geometry/primitives.rst: -------------------------------------------------------------------------------- 1 | Bounding Box 2 | ------------ 3 | 4 | .. automodule:: mvdatasets.geometry.primitives.bounding_box 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | Bounding Sphere 10 | --------------- 11 | 12 | .. automodule:: mvdatasets.geometry.primitives.bounding_sphere 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | 17 | Point Cloud 18 | ----------- 19 | 20 | .. automodule:: mvdatasets.geometry.primitives.point_cloud 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | -------------------------------------------------------------------------------- /docs/source/geometry/projections.rst: -------------------------------------------------------------------------------- 1 | Projections 2 | =========== 3 | 4 | .. automodule:: mvdatasets.geometry.projections 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/geometry/quaternions.rst: -------------------------------------------------------------------------------- 1 | Quaternions 2 | =========== 3 | 4 | .. automodule:: mvdatasets.geometry.quaternions 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/geometry/rigid.rst: -------------------------------------------------------------------------------- 1 | Rigid geometry 2 | ============== 3 | 4 | .. automodule:: mvdatasets.geometry.rigid 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/geometry/trajectories.rst: -------------------------------------------------------------------------------- 1 | Trajectories 2 | ============ 3 | 4 | .. automodule:: mvdatasets.geometry.trajectories 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | How to install the library depends on your use case. 5 | 6 | If you want to install the package for *regular usage* without needing further modifications, install the module from source using ``pip install .``. 7 | 8 | To perform an *editable installation* (also known as a development mode installation), install the module from source using ``pip install -e .``. 9 | Changes made to the code in the local directory will take effect immediately when you import the package in Python. 10 | 11 | To run tests, install the module with the ``tests`` extra (e.g. ``pip install ".[tests]"``). 12 | To compile the documentation, install the module with the ``docs`` extra (e.g. ``pip install ".[docs]"``). 13 | 14 | The library is tested with ``Python 3.10``; other versions (``>=3.8``) may work, but are not officially supported. -------------------------------------------------------------------------------- /docs/source/io/index.rst: -------------------------------------------------------------------------------- 1 | I/O 2 | === 3 | 4 | Toolset of I/O utilities. 5 | 6 | mvdatasets.io.yaml\_utils 7 | ------------------------------------------ 8 | 9 | .. automodule:: mvdatasets.io.yaml_utils 10 | :members: 11 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/loaders/dynamic.rst: -------------------------------------------------------------------------------- 1 | Dynamic scenes 2 | ============== 3 | 4 | Dynamic scenes loaders. -------------------------------------------------------------------------------- /docs/source/loaders/index.rst: -------------------------------------------------------------------------------- 1 | Datasets loaders 2 | ================ 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | static 8 | dynamic 9 | 10 | -------------------------------------------------------------------------------- /docs/source/loaders/static.rst: -------------------------------------------------------------------------------- 1 | Static scenes 2 | ============= 3 | 4 | Static scenes loaders. -------------------------------------------------------------------------------- /docs/source/mvdataset.rst: -------------------------------------------------------------------------------- 1 | MVDataset 2 | --------- 3 | 4 | .. automodule:: mvdatasets.mvdataset 5 | :members: 6 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/tensorreel.rst: -------------------------------------------------------------------------------- 1 | TensorReel 2 | ---------- 3 | 4 | .. automodule:: mvdatasets.tensorreel 5 | :members: 6 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/utils/index.rst: -------------------------------------------------------------------------------- 1 | Utility functions 2 | ================= 3 | 4 | Toolset of utility functions. 5 | 6 | mvdatasets.utils.camera\_utils module 7 | ------------------------------------- 8 | 9 | .. automodule:: mvdatasets.utils.camera_utils 10 | :members: 11 | :undoc-members: 12 | :show-inheritance: 13 | 14 | mvdatasets.utils.images module 15 | ------------------------------ 16 | 17 | .. automodule:: mvdatasets.utils.images 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | mvdatasets.utils.memory module 23 | ------------------------------ 24 | 25 | .. automodule:: mvdatasets.utils.memory 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | mvdatasets.utils.mesh module 31 | ---------------------------- 32 | 33 | .. automodule:: mvdatasets.utils.mesh 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | mvdatasets.utils.open3d\_rendering module 39 | ----------------------------------------- 40 | 41 | .. automodule:: mvdatasets.utils.open3d_rendering 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | mvdatasets.utils.point\_clouds module 47 | ------------------------------------- 48 | 49 | .. automodule:: mvdatasets.utils.point_clouds 50 | :members: 51 | :undoc-members: 52 | :show-inheritance: 53 | 54 | mvdatasets.utils.printing module 55 | -------------------------------- 56 | 57 | .. automodule:: mvdatasets.utils.printing 58 | :members: 59 | :undoc-members: 60 | :show-inheritance: 61 | 62 | mvdatasets.utils.profiler module 63 | -------------------------------- 64 | 65 | .. automodule:: mvdatasets.utils.profiler 66 | :members: 67 | :undoc-members: 68 | :show-inheritance: 69 | 70 | mvdatasets.utils.raycasting module 71 | ---------------------------------- 72 | 73 | .. automodule:: mvdatasets.utils.raycasting 74 | :members: 75 | :undoc-members: 76 | :show-inheritance: 77 | 78 | mvdatasets.utils.tensor\_mesh module 79 | ------------------------------------ 80 | 81 | .. automodule:: mvdatasets.utils.tensor_mesh 82 | :members: 83 | :undoc-members: 84 | :show-inheritance: 85 | 86 | mvdatasets.utils.tensor\_texture module 87 | --------------------------------------- 88 | 89 | .. automodule:: mvdatasets.utils.tensor_texture 90 | :members: 91 | :undoc-members: 92 | :show-inheritance: 93 | 94 | mvdatasets.utils.texture module 95 | ------------------------------- 96 | 97 | .. automodule:: mvdatasets.utils.texture 98 | :members: 99 | :undoc-members: 100 | :show-inheritance: 101 | 102 | mvdatasets.utils.video\_utils module 103 | ------------------------------------ 104 | 105 | .. automodule:: mvdatasets.utils.video_utils 106 | :members: 107 | :undoc-members: 108 | :show-inheritance: 109 | 110 | mvdatasets.utils.virtual\_cameras module 111 | ---------------------------------------- 112 | 113 | .. automodule:: mvdatasets.utils.virtual_cameras 114 | :members: 115 | :undoc-members: 116 | :show-inheritance: -------------------------------------------------------------------------------- /docs/source/visualization/index.rst: -------------------------------------------------------------------------------- 1 | Visualizations 2 | ============== 3 | 4 | Toolset of visualization utilities. 5 | 6 | mvdatasets.visualization.colormaps module 7 | ----------------------------------------- 8 | 9 | .. automodule:: mvdatasets.visualization.colormaps 10 | :members: 11 | :undoc-members: 12 | :show-inheritance: 13 | 14 | mvdatasets.visualization.matplotlib module 15 | ------------------------------------------ 16 | 17 | .. automodule:: mvdatasets.visualization.matplotlib 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | mvdatasets.visualization.video\_gen module 23 | ------------------------------------------ 24 | 25 | .. automodule:: mvdatasets.visualization.video_gen 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: -------------------------------------------------------------------------------- /examples/.deprecated/chained_bbs_tranformations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL 3 | import os 4 | import sys 5 | import time 6 | import torch 7 | from copy import deepcopy 8 | import matplotlib.pyplot as plt 9 | import open3d as o3d 10 | from tqdm import tqdm 11 | import imageio 12 | 13 | # load mvdatasets from parent directory 14 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 15 | 16 | from mvdatasets.visualization.matplotlib import plot_3d 17 | from mvdatasets.utils.profiler import Profiler 18 | from config import get_dataset_test_preset 19 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 20 | from mvdatasets.utils.point_clouds import load_point_cloud 21 | from mvdatasets.geometry.rigid import apply_transformation_3d 22 | 23 | 24 | def test(): 25 | 26 | # load data 27 | 28 | data_path = "examples/assets/assetsnerf/chair.npy" 29 | data = np.load(data_path, allow_pickle=True).item() 30 | print(data["chair"].keys()) 31 | 32 | # create bounding boxes 33 | bounding_boxes = [] 34 | point_clouds = [] 35 | 36 | bb_data = data["chair"].pop("bb") 37 | bb_pose = np.eye(4) 38 | bb_pose[:3, 3] = bb_data["center"] 39 | sizes = bb_data["dimensions"] 40 | father_bb = BoundingBox( 41 | pose=bb_pose, 42 | sizes=sizes, 43 | label="father_bb", 44 | color="orange", 45 | line_width=5.0, 46 | ) 47 | bounding_boxes.append(father_bb) 48 | 49 | # load point cloud from ply 50 | points_3d = load_point_cloud("examples/assets/assetsnerf/5.ply") 51 | point_clouds.append(points_3d) 52 | 53 | # test save 54 | father_bb.save_as_ply(".", father_bb.label) 55 | 56 | for bb_key, bb_data in data["chair"].items(): 57 | print("instance:", bb_key) 58 | bb = BoundingBox( 59 | pose=bb_data["transformation"], # father to child transform 60 | father_bb=father_bb, 61 | label=bb_key, 62 | color="blue", 63 | line_width=5.0, 64 | ) 65 | bounding_boxes.append(bb) 66 | # load instance point cloud (in world space) 67 | points_3d = load_point_cloud(f"examples/assets/assetsnerf/{bb_key}.ply") 68 | # align to father pc (in world space) 69 | points_3d = apply_transformation_3d(points_3d, bb_data["transformation"]) 70 | point_clouds.append(points_3d) 71 | 72 | # bb_pose = np.eye(4) 73 | # bb_pose[:3, 3] = np.array([1.0, 0.0, 0.0]) 74 | # bb_scale = np.array([0.7, 0.8, 0.9]) 75 | # bb_pose[:3, :3] = rot_y_3d(deg2rad(45)) @ rot_x_3d(deg2rad(45)) 76 | # bb_pose[:3, :3] *= bb_scale 77 | # bb = BoundingBox( 78 | # pose=bb_pose, 79 | # father_bb=father_bb, 80 | # ) 81 | # bounding_boxes.append(bb) 82 | 83 | # bb_pose = np.eye(4) 84 | # bb_pose[:3, 3] = np.array([-0.5, 0.5, 0.0]) 85 | # bb_scale = np.array([0.4, 0.3, 0.2]) 86 | # bb_pose[:3, :3] = rot_y_3d(deg2rad(45)) @ rot_x_3d(deg2rad(45)) 87 | # bb_pose[:3, :3] *= bb_scale 88 | # bb = BoundingBox( 89 | # pose=bb_pose, 90 | # father_bb=father_bb, 91 | # ) 92 | # bounding_boxes.append(bb) 93 | 94 | # visualize bounding boxes 95 | fig = plot_3d( 96 | bounding_boxes=bounding_boxes, 97 | points_3d=np.concatenate(point_clouds, axis=0), 98 | azimuth_deg=90, 99 | elevation_deg=60, 100 | scene_radius=1.0, 101 | up="z", 102 | figsize=(15, 15), 103 | draw_origin=True, 104 | draw_frame=True, 105 | title="", 106 | ) 107 | # plt.show() 108 | 109 | plt.savefig( 110 | os.path.join(output_path, f"bbs.png"), 111 | transparent=False, 112 | bbox_inches="tight", 113 | pad_inches=0, 114 | dpi=300, 115 | ) 116 | plt.close() 117 | 118 | 119 | if __name__ == "__main__": 120 | 121 | # Set a random seed for reproducibility 122 | torch.manual_seed(SEED) 123 | np.random.seed(SEED) 124 | 125 | # # Check if CUDA (GPU support) is available 126 | if torch.cuda.is_available(): 127 | device = "cuda" 128 | torch.cuda.manual_seed(SEED) # Set a random seed for GPU 129 | else: 130 | device = "cpu" 131 | torch.set_default_device(DEVICE) 132 | 133 | # Set default tensor type 134 | torch.set_default_dtype(torch.float32) 135 | 136 | test() 137 | -------------------------------------------------------------------------------- /examples/.deprecated/convert_to_ingp_format.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL 3 | import os 4 | import sys 5 | import time 6 | import torch 7 | from copy import deepcopy 8 | import matplotlib.pyplot as plt 9 | import open3d as o3d 10 | from tqdm import tqdm 11 | import imageio 12 | 13 | # load mvdatasets from parent directory 14 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 15 | 16 | from mvdatasets.visualization.matplotlib import plot_3d 17 | from mvdatasets.mvdataset import MVDataset 18 | from mvdatasets.utils.profiler import Profiler 19 | from mvdatasets.config import get_dataset_test_preset 20 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 21 | from mvdatasets.utils.data_converter import convert_to_ingp_format 22 | 23 | 24 | if __name__ == "__main__": 25 | 26 | # Set a random seed for reproducibility 27 | torch.manual_seed(SEED) 28 | np.random.seed(SEED) 29 | 30 | # # Check if CUDA (GPU support) is available 31 | if torch.cuda.is_available(): 32 | device = "cuda" 33 | torch.cuda.manual_seed(SEED) # Set a random seed for GPU 34 | else: 35 | device = "cpu" 36 | torch.set_default_device(DEVICE) 37 | 38 | # Set default tensor type 39 | torch.set_default_dtype(torch.float32) 40 | 41 | # Set profiler 42 | profiler = Profiler() # nb: might slow down the code 43 | 44 | # Get dataset test preset 45 | if len(sys.argv) > 1: 46 | dataset_name = sys.argv[1] 47 | else: 48 | dataset_name = "dtu" 49 | scene_name, pc_paths, config = get_dataset_test_preset(args.dataset_name) 50 | 51 | # dataset loading 52 | mv_data = MVDataset( 53 | args.dataset_name, 54 | scene_name, 55 | args.datasets_path, 56 | point_clouds_paths=pc_paths, 57 | splits=["train", "test"], 58 | verbose=True, 59 | ) 60 | 61 | # convert_to_ingp_format(mv_data, f"/home/stefano/Data/{dataset_name}_ingp") 62 | -------------------------------------------------------------------------------- /examples/.deprecated/convert_to_opengl_format.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL 3 | import os 4 | import sys 5 | import time 6 | import torch 7 | from copy import deepcopy 8 | import matplotlib.pyplot as plt 9 | import open3d as o3d 10 | from tqdm import tqdm 11 | import imageio 12 | import json 13 | 14 | # load mvdatasets from parent directory 15 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 16 | 17 | from mvdatasets.visualization.matplotlib import plot_3d 18 | from mvdatasets.mvdataset import MVDataset 19 | from mvdatasets.utils.profiler import Profiler 20 | from mvdatasets.config import get_dataset_test_preset 21 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 22 | from mvdatasets.utils.data_converter import convert_to_ingp_format 23 | 24 | 25 | if __name__ == "__main__": 26 | 27 | # Set a random seed for reproducibility 28 | torch.manual_seed(SEED) 29 | np.random.seed(SEED) 30 | 31 | # # Check if CUDA (GPU support) is available 32 | if torch.cuda.is_available(): 33 | device = "cuda" 34 | torch.cuda.manual_seed(SEED) # Set a random seed for GPU 35 | else: 36 | device = "cpu" 37 | torch.set_default_device(DEVICE) 38 | 39 | # Set default tensor type 40 | torch.set_default_dtype(torch.float32) 41 | 42 | # Set profiler 43 | profiler = Profiler() # nb: might slow down the code 44 | 45 | # Get dataset test preset 46 | if len(sys.argv) > 1: 47 | dataset_name = sys.argv[1] 48 | else: 49 | dataset_name = "dtu" 50 | scene_name, pc_paths, config = get_dataset_test_preset(args.dataset_name) 51 | 52 | # dataset loading 53 | mv_data = MVDataset( 54 | args.dataset_name, 55 | scene_name, 56 | args.datasets_path, 57 | point_clouds_paths=pc_paths, 58 | splits=["train", "test"], 59 | config=config, 60 | verbose=True, 61 | ) 62 | 63 | scene = {} 64 | scene["resolution"] = [mv_data.get_width(), mv_data.get_height()] 65 | scene["meshes"] = [] 66 | scene["cameras"] = {"test": {}, "train": {}} 67 | for camera in mv_data.get_split("test"): 68 | camera_idx = camera.camera_label 69 | projectionMatrix = camera.get_opengl_projection_matrix() 70 | matrixWorld = camera.get_opengl_matrix_world() 71 | scene["cameras"]["test"][camera_idx] = { 72 | "projectionMatrix": projectionMatrix.tolist(), 73 | "matrixWorld": matrixWorld.tolist(), # c2w 74 | } 75 | 76 | # Save the projections as a json file 77 | with open("out/scene.json", "w") as f: 78 | json.dump(scene, f, indent=4) 79 | -------------------------------------------------------------------------------- /examples/.deprecated/mesh_rendering.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import open3d as o3d 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | # load mvdatasets from parent directory 8 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 9 | 10 | from mvdatasets.mvdataset import MVDataset 11 | from mvdatasets.utils.profiler import Profiler 12 | from mvdatasets.utils.open3d_rendering import render_o3d_mesh 13 | from mvdatasets.utils.images import numpy_to_image 14 | from mvdatasets.config import get_dataset_test_preset 15 | 16 | if __name__ == "__main__": 17 | 18 | dataset_name = "dtu" 19 | scene_name, pc_paths, config = get_dataset_test_preset(args.dataset_name) 20 | mesh_file_path = "examples/assets/meshes/dtu/dtu_scan83.ply" 21 | 22 | # dataset loading 23 | mv_data = MVDataset( 24 | args.dataset_name, 25 | scene_name, 26 | args.datasets_path, 27 | splits=["train", "test"], 28 | config=config, 29 | verbose=True, 30 | ) 31 | 32 | # load mesh 33 | 34 | triangle_mesh = o3d.io.read_triangle_mesh(mesh_file_path) 35 | triangle_mesh.compute_vertex_normals() 36 | 37 | # render mesh 38 | camera = mv_data.get_split("test")[2] 39 | imgs = render_o3d_mesh(camera, triangle_mesh) 40 | plt.imshow(imgs["depth"]) 41 | plt.colorbar() 42 | plt.show() 43 | 44 | # plt.savefig(os.path.join(output_path, f"{dataset_name}_{scene_name}_mesh_depth.png"), transparent=True, dpi=300) 45 | # plt.close() 46 | -------------------------------------------------------------------------------- /examples/.deprecated/save_video.py: -------------------------------------------------------------------------------- 1 | import imageio 2 | 3 | out_dir = "./videos" 4 | mp4_name = "test.mp4" 5 | 6 | writer = imageio.get_writer( 7 | f"{out_dir}/{mp4_name}", mode="I", fps=30, codec="libx264", bitrate="16M" 8 | ) 9 | -------------------------------------------------------------------------------- /examples/.deprecated/scene_rendering.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import open3d as o3d 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | # load mvdatasets from parent directory 8 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 9 | 10 | from mvdatasets.mvdataset import MVDataset 11 | from mvdatasets.utils.profiler import Profiler 12 | from mvdatasets.utils.open3d_rendering import render_o3d_scene 13 | from mvdatasets.geometry.rigid import apply_transformation_3d 14 | from mvdatasets.config import get_dataset_test_preset 15 | 16 | if __name__ == "__main__": 17 | 18 | # Set profiler 19 | profiler = Profiler() # nb: might slow down the code 20 | 21 | # Get dataset test preset 22 | # if len(sys.argv) > 1: 23 | # dataset_name = sys.argv[1] 24 | # else: 25 | dataset_name = "dmsr" 26 | scene_name, pc_paths, config = get_dataset_test_preset(args.dataset_name) 27 | 28 | # dataset loading 29 | mv_data = MVDataset( 30 | args.dataset_name, 31 | scene_name, 32 | args.datasets_path, 33 | splits=["train", "test"], 34 | config=config, 35 | verbose=True, 36 | ) 37 | 38 | # scene path (folder containing only mesh files) 39 | scene_dir = "/home/stefano/Data/dmsr/dinning/meshes" 40 | 41 | # get all files in scene dir 42 | files = os.listdir(scene_dir) 43 | nr_objects = len(files) 44 | print(files) 45 | 46 | # setup scene 47 | o3d_scene = o3d.t.geometry.RaycastingScene() 48 | 49 | # load meshes and add to scene 50 | rotation = mv_data.global_transform[:3, :3] 51 | translation = mv_data.global_transform[:3, 3] 52 | for mesh_file in files: 53 | o3d_mesh = o3d.io.read_triangle_mesh(os.path.join(scene_dir, mesh_file)) 54 | o3d_mesh_vertices = np.asarray(o3d_mesh.vertices) 55 | o3d_mesh_vertices = apply_transformation_3d( 56 | o3d_mesh_vertices, mv_data.global_transform 57 | ) 58 | o3d_mesh.vertices = o3d.utility.Vector3dVector(o3d_mesh_vertices) 59 | o3d_mesh.compute_vertex_normals() 60 | o3d_scene.add_triangles(o3d.t.geometry.TriangleMesh.from_legacy(o3d_mesh)) 61 | 62 | # render mesh 63 | splits = ["test", "train"] 64 | for split in splits: 65 | save_path = os.path.join(args.datasets_path, dataset_name, scene_name, split) 66 | for camera in mv_data.get_split(split): 67 | # print(camera.camera_label) 68 | imgs = render_o3d_scene(camera, o3d_scene) 69 | geom_ids = imgs["geom_ids"] 70 | depth = imgs["depth"] 71 | print("min depth", np.min(depth)) 72 | print("max depth", np.max(depth)) 73 | # print(geom_ids.shape) 74 | # print(np.unique(geom_ids)) 75 | # plt.imshow(geom_ids, vmax=nr_objects) 76 | plt.imshow(depth, cmap="jet") 77 | plt.colorbar() 78 | plt.show() 79 | break 80 | # save_nr = format(camera.camera_label, "04d") 81 | # os.makedirs(os.path.join(save_path, "depth"), exist_ok=True) 82 | # os.makedirs(os.path.join(save_path, "instance_mask"), exist_ok=True) 83 | # np.save(os.path.join(save_path, "depth", f"d_{save_nr}"), depth) 84 | # np.save(os.path.join(save_path, "instance_mask", f"instance_mask_{save_nr}"), geom_ids) 85 | -------------------------------------------------------------------------------- /examples/.deprecated/spiral_trajectory.py: -------------------------------------------------------------------------------- 1 | # TODO: add tests for this functions 2 | 3 | import numpy as np 4 | 5 | # horizontal coordinate system to cartesian coordinate system 6 | # azimuth: 0 to 360 7 | # elevation: -90 to 90 8 | 9 | 10 | class SphereSpiral: 11 | 12 | def __init__(self, turns=5, points_per_turn=20): 13 | self.turns = turns 14 | self.points_per_turn = points_per_turn 15 | self.points = self.create_points_trajectory() 16 | 17 | def _spiral(self, azimuth_deg, elevation_deg): 18 | 19 | # azimuth_min = 0, azimuth_max = 360 20 | # elevation_min = -90, elevation_max = 90 21 | 22 | def deg2rad(deg): 23 | return deg * (np.pi / 180) 24 | 25 | azimuth_rad = deg2rad(azimuth_deg) 26 | elevation_rad = deg2rad(elevation_deg) 27 | 28 | x = np.cos(azimuth_rad) * np.cos(elevation_rad) 29 | y = np.sin(azimuth_rad) * np.cos(elevation_rad) 30 | z = np.sin(elevation_rad) 31 | 32 | x = np.array(x) 33 | y = np.array(y) 34 | z = np.array(z) 35 | 36 | return np.column_stack((x, y, z)) 37 | 38 | def create_points_trajectory(self): 39 | 40 | # points can be seen as unit vectors pointing from the origin to a point on the surface of the sphere 41 | 42 | p = self._spiral( 43 | azimuth_deg=np.tile( 44 | np.linspace(1, 360, self.points_per_turn), self.turns 45 | ), # angle around the z-axis 46 | elevation_deg=np.linspace( 47 | -90, 90, self.turns * self.points_per_turn 48 | ), # angle from the xy-plane 49 | ) 50 | 51 | return p # (turns * points_per_turn, 3) 52 | 53 | 54 | # spiral_points = SphereSpiral(turns=4, points_per_turn=10).points 55 | # plot_points_trajectory(spiral_points, show_plot=False, save_fig=True, save_dir=output_path) 56 | # view_dirs = -1 * spiral_points 57 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union, Tuple, Optional, Type 2 | from mvdatasets.utils.printing import print_error 3 | import traceback 4 | from rich import print 5 | from pathlib import Path 6 | 7 | 8 | def custom_exception_handler(exc_type, exc_value, exc_traceback): 9 | """ 10 | Custom exception handler to print detailed information for uncaught exceptions. 11 | """ 12 | # if issubclass(exc_type, KeyboardInterrupt): 13 | # # Allow program to exit quietly on Ctrl+C 14 | # sys.__excepthook__(exc_type, exc_value, exc_traceback) 15 | # return 16 | 17 | # destroy_context() # Clean up Dear PyGui 18 | 19 | # Format the exception message 20 | # message = f"{exc_type.__name__}: {exc_value}" 21 | # Pass detailed exception info to the print_error function 22 | # print_error(message, exc_type, exc_value, exc_traceback) 23 | 24 | # print(f"[bold red]ERROR:[/bold red] {message}") 25 | if exc_type and exc_traceback: 26 | print("\n[bold blue]Stack Trace:[/bold blue]") 27 | # Format the traceback into a readable format 28 | detailed_traceback = "".join( 29 | traceback.format_exception(exc_type, exc_value, exc_traceback) 30 | ) 31 | print(f"[dim]{detailed_traceback}[/dim]") 32 | 33 | 34 | def get_dataset_test_preset(dataset_name: str = "dtu") -> Tuple[str, List[str], dict]: 35 | 36 | # test dtu 37 | if dataset_name == "dtu": 38 | scene_name = "dtu_scan83" 39 | pc_paths = [Path(f"examples/assets/meshes/{dataset_name}/{scene_name}.ply")] 40 | 41 | # test blended-mvs 42 | elif dataset_name == "blended-mvs": 43 | scene_name = "bmvs_bear" 44 | pc_paths = [] 45 | 46 | # test nerf_synthetic 47 | elif dataset_name == "nerf_synthetic": 48 | scene_name = "lego" 49 | pc_paths = [ 50 | Path(f"examples/assets/point_clouds/{dataset_name}/{scene_name}.ply") 51 | ] 52 | 53 | # test shelly 54 | elif dataset_name == "shelly": 55 | scene_name = "khady" 56 | pc_paths = [Path(f"examples/assets/meshes/{dataset_name}/{scene_name}.ply")] 57 | 58 | # test nerf_furry 59 | elif dataset_name == "nerf_furry": 60 | scene_name = "plushy" 61 | pc_paths = [Path(f"examples/assets/meshes/{dataset_name}/{scene_name}.ply")] 62 | 63 | # test dmsr 64 | elif dataset_name == "dmsr": 65 | scene_name = "dinning" 66 | pc_paths = [Path(f"examples/assets/meshes/{dataset_name}/{scene_name}.ply")] 67 | 68 | # test refnerf 69 | elif dataset_name == "refnerf": 70 | scene_name = "car" 71 | pc_paths = [] 72 | 73 | # test ingp 74 | elif dataset_name == "ingp": 75 | scene_name = "fox" 76 | pc_paths = [] 77 | 78 | # test llff 79 | elif dataset_name == "llff": 80 | scene_name = "fern" 81 | pc_paths = ["examples/assets/point_clouds/llff/fern.ply"] 82 | 83 | # test mipnerf360 84 | elif dataset_name == "mipnerf360": 85 | scene_name = "garden" 86 | pc_paths = [] 87 | 88 | # test d-nerf 89 | elif dataset_name == "d-nerf": 90 | scene_name = "bouncingballs" 91 | pc_paths = [] 92 | 93 | # test visor 94 | elif dataset_name == "visor": 95 | scene_name = "P01_01" 96 | pc_paths = [] 97 | 98 | # test neu3d 99 | elif dataset_name == "neu3d": 100 | scene_name = "coffee_martini" 101 | pc_paths = [] 102 | 103 | # test panoptic-sports 104 | elif dataset_name == "panoptic-sports": 105 | scene_name = "basketball" 106 | pc_paths = [] 107 | 108 | # test iphone 109 | elif dataset_name == "iphone": 110 | scene_name = "paper-windmill" 111 | pc_paths = [] 112 | 113 | # test iphone_som 114 | elif dataset_name == "iphone_som": 115 | scene_name = "paper-windmill" 116 | pc_paths = [] 117 | 118 | # test monst3r 119 | elif dataset_name == "monst3r": 120 | scene_name = "car-turn" 121 | pc_paths = [] 122 | 123 | else: 124 | print_error(f"Dataset {dataset_name} does not have a test preset.") 125 | 126 | return {"scene_name": scene_name, "pc_paths": pc_paths} 127 | -------------------------------------------------------------------------------- /examples/assets/.deprecated/assetsnerf/dmsr/dinning/icp/chair.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/.deprecated/assetsnerf/dmsr/dinning/icp/chair.npy -------------------------------------------------------------------------------- /examples/assets/.deprecated/assetsnerf/dmsr/dinning/icp/vase.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/.deprecated/assetsnerf/dmsr/dinning/icp/vase.npy -------------------------------------------------------------------------------- /examples/assets/error_maps/plushy_test_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/error_maps/plushy_test_0.png -------------------------------------------------------------------------------- /examples/assets/meshes/blendernerf/plushy.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/meshes/blendernerf/plushy.ply -------------------------------------------------------------------------------- /examples/assets/meshes/dmsr/dinning.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/meshes/dmsr/dinning.ply -------------------------------------------------------------------------------- /examples/assets/meshes/dtu/dtu_scan83.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/meshes/dtu/dtu_scan83.ply -------------------------------------------------------------------------------- /examples/assets/meshes/shelly/khady.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/meshes/shelly/khady.ply -------------------------------------------------------------------------------- /examples/assets/point_clouds/llff/fern.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/point_clouds/llff/fern.ply -------------------------------------------------------------------------------- /examples/assets/point_clouds/mipnerf360/bicycle.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/point_clouds/mipnerf360/bicycle.ply -------------------------------------------------------------------------------- /examples/assets/point_clouds/mipnerf360/garden.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/point_clouds/mipnerf360/garden.ply -------------------------------------------------------------------------------- /examples/assets/point_clouds/nerf_synthetic/lego.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/examples/assets/point_clouds/nerf_synthetic/lego.ply -------------------------------------------------------------------------------- /examples/bounding_primitives_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import os 3 | import sys 4 | from typing import List 5 | from pathlib import Path 6 | import numpy as np 7 | from copy import deepcopy 8 | import matplotlib.pyplot as plt 9 | from mvdatasets.mvdataset import MVDataset 10 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 11 | from mvdatasets.geometry.primitives.bounding_sphere import BoundingSphere 12 | from mvdatasets.visualization.matplotlib import plot_image 13 | from mvdatasets.configs.example_config import ExampleConfig 14 | from examples import get_dataset_test_preset, custom_exception_handler 15 | from mvdatasets.utils.printing import print_warning 16 | 17 | 18 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 19 | 20 | device = cfg.machine.device 21 | datasets_path = cfg.datasets_path 22 | output_path = cfg.output_path 23 | scene_name = cfg.scene_name 24 | dataset_name = cfg.data.dataset_name 25 | 26 | # dataset loading 27 | mv_data = MVDataset( 28 | dataset_name, 29 | scene_name, 30 | datasets_path, 31 | config=cfg.data.asdict(), 32 | point_clouds_paths=pc_paths, 33 | verbose=True, 34 | ) 35 | 36 | # random camera index 37 | rand_idx = 0 # torch.randint(0, len(mv_data.get_split("test")), (1,)).item() 38 | camera = deepcopy(mv_data.get_split("test")[rand_idx]) 39 | # shoot rays from camera 40 | rays_o, rays_d, points_2d_screen = camera.get_rays(device=device) 41 | 42 | # bounding box 43 | bounding_volume = BoundingBox( 44 | pose=np.eye(4), 45 | local_scale=mv_data.get_foreground_radius() * 2, 46 | device=device, 47 | ) 48 | # bounding_volume intersection test 49 | is_hit, t_near, t_far, p_near, p_far = bounding_volume.intersect(rays_o, rays_d) 50 | hit_range = t_far - t_near 51 | hit_range = hit_range.cpu().numpy() 52 | # get the color map 53 | color_map = plt.colormaps.get_cmap("jet") 54 | # apply the colormap 55 | hit_range = color_map(hit_range)[:, :3] 56 | 57 | data = camera.get_data(keys=["rgbs"]) 58 | rgbs = data["rgbs"].cpu().numpy() 59 | img_np = (rgbs * 0.5) + (hit_range * 0.5) 60 | img_np = img_np.reshape(camera.width, camera.height, -1) 61 | 62 | plot_image( 63 | image=img_np, 64 | title="Bounding Box", 65 | show=cfg.with_viewer, 66 | save_path=os.path.join( 67 | output_path, f"{dataset_name}_{scene_name}_bounding_box.png" 68 | ), 69 | ) 70 | 71 | # bounding sphere 72 | bounding_volume = BoundingSphere( 73 | pose=np.eye(4), local_scale=mv_data.get_foreground_radius(), device=device 74 | ) 75 | # bounding_volume intersection test 76 | is_hit, t_near, t_far, p_near, p_far = bounding_volume.intersect(rays_o, rays_d) 77 | hit_range = t_far - t_near 78 | hit_range = hit_range.cpu().numpy() 79 | # get the color map 80 | color_map = plt.colormaps.get_cmap("jet") 81 | # apply the colormap 82 | hit_range = color_map(hit_range)[:, :3] 83 | 84 | data = camera.get_data(keys=["rgbs"]) 85 | rgbs = data["rgbs"].cpu().numpy() 86 | img_np = (rgbs * 0.5) + (hit_range * 0.5) 87 | img_np = img_np.reshape(camera.width, camera.height, -1) 88 | 89 | plot_image( 90 | image=img_np, 91 | title="Bounding Sphere", 92 | show=cfg.with_viewer, 93 | save_path=os.path.join( 94 | output_path, f"{dataset_name}_{scene_name}_bounding_sphere.png" 95 | ), 96 | ) 97 | 98 | 99 | if __name__ == "__main__": 100 | 101 | # custom exception handler 102 | sys.excepthook = custom_exception_handler 103 | 104 | # parse arguments 105 | args = tyro.cli(ExampleConfig) 106 | 107 | # get test preset 108 | test_preset = get_dataset_test_preset(args.data.dataset_name) 109 | # scene name 110 | if args.scene_name is None: 111 | args.scene_name = test_preset["scene_name"] 112 | print_warning( 113 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 114 | ) 115 | # additional point clouds paths (if any) 116 | pc_paths = test_preset["pc_paths"] 117 | 118 | # start the example program 119 | main(args, pc_paths) 120 | -------------------------------------------------------------------------------- /examples/camera_data_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import sys 3 | from typing import List 4 | from pathlib import Path 5 | from examples import get_dataset_test_preset 6 | from mvdatasets.mvdataset import MVDataset 7 | from mvdatasets.visualization.matplotlib import plot_cameras_2d 8 | from mvdatasets.configs.example_config import ExampleConfig 9 | from examples import get_dataset_test_preset, custom_exception_handler 10 | from mvdatasets.utils.printing import print_warning 11 | 12 | 13 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 14 | 15 | device = cfg.machine.device 16 | datasets_path = cfg.datasets_path 17 | output_path = cfg.output_path 18 | scene_name = cfg.scene_name 19 | dataset_name = cfg.data.dataset_name 20 | 21 | # dataset loading 22 | mv_data = MVDataset( 23 | dataset_name, 24 | scene_name, 25 | datasets_path, 26 | config=cfg.data.asdict(), 27 | point_clouds_paths=pc_paths, 28 | verbose=True, 29 | ) 30 | 31 | # # random camera index 32 | # rand_idx = np.random.randint(0, len(mv_data.get_split("test"))) 33 | # camera = deepcopy(mv_data.get_split("test")[rand_idx]) 34 | 35 | plot_cameras_2d( 36 | cameras=mv_data.get_split("train"), 37 | ) 38 | 39 | 40 | if __name__ == "__main__": 41 | 42 | # custom exception handler 43 | sys.excepthook = custom_exception_handler 44 | 45 | # parse arguments 46 | args = tyro.cli(ExampleConfig) 47 | 48 | # get test preset 49 | test_preset = get_dataset_test_preset(args.data.dataset_name) 50 | # scene name 51 | if args.scene_name is None: 52 | args.scene_name = test_preset["scene_name"] 53 | print_warning( 54 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 55 | ) 56 | # additional point clouds paths (if any) 57 | pc_paths = test_preset["pc_paths"] 58 | 59 | # start the example program 60 | main(args, pc_paths) 61 | -------------------------------------------------------------------------------- /examples/camera_depth_unprojection_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | from rich import print 3 | import os 4 | import sys 5 | from pathlib import Path 6 | from typing import List 7 | from copy import deepcopy 8 | import matplotlib.pyplot as plt 9 | from mvdatasets.geometry.primitives.point_cloud import PointCloud 10 | from mvdatasets.visualization.matplotlib import plot_camera_2d, plot_3d 11 | from mvdatasets.mvdataset import MVDataset 12 | from mvdatasets.utils.printing import print_error, print_warning 13 | from mvdatasets.visualization.video_gen import make_video_depth_unproject 14 | from mvdatasets.configs.example_config import ExampleConfig 15 | from examples import get_dataset_test_preset, custom_exception_handler 16 | 17 | 18 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 19 | 20 | device = cfg.machine.device 21 | datasets_path = cfg.datasets_path 22 | output_path = cfg.output_path 23 | scene_name = cfg.scene_name 24 | dataset_name = cfg.data.dataset_name 25 | 26 | # dataset loading 27 | mv_data = MVDataset( 28 | dataset_name, 29 | scene_name, 30 | datasets_path, 31 | config=cfg.data.asdict(), 32 | point_clouds_paths=pc_paths, 33 | verbose=True, 34 | ) 35 | 36 | # 37 | split_modalities = mv_data.get_split_modalities("train") 38 | print("split_modalities", split_modalities) 39 | # make sure mv_data has depth modality 40 | if "depths" not in split_modalities: 41 | raise ValueError("Dataset does not have depth modality") 42 | 43 | # check if camera trajectory is available 44 | print("nr_sequence_frames:", mv_data.get_nr_sequence_frames()) 45 | if mv_data.get_nr_sequence_frames() <= 1: 46 | raise ValueError( 47 | f"{dataset_name} is a static dataset and does not have camera trajectory" 48 | ) 49 | return 50 | 51 | # check if monocular sequence 52 | print("nr_per_camera_frames:", mv_data.get_nr_per_camera_frames()) 53 | if mv_data.get_nr_per_camera_frames() > 1: 54 | raise ValueError(f"{dataset_name} is not a monocular sequence") 55 | return 56 | 57 | from mvdatasets.utils.raycasting import get_points_2d_screen_from_pixels 58 | 59 | # iterate over training cameras 60 | pcs = [] 61 | for camera in mv_data.get_split("train"): 62 | # get rgb and depth images 63 | depth = camera.get_depth() # (H, W, 1) 64 | # invert H and W 65 | depth = depth.transpose(1, 0, 2) # (W, H, 1) 66 | # flatten depth image 67 | depth = depth.flatten() # (H*W,) 68 | # 69 | zero_depth_mask = depth < 1e-3 70 | # get rgb 71 | rgb = camera.get_rgb() # (H, W, 3) 72 | # invert H and W 73 | rgb = rgb.transpose(1, 0, 2) # (W, H, 3) 74 | points_rgb = rgb.reshape(-1, 3) # (H*W, 3) 75 | # get pixel coordinates 76 | pixels = camera.get_pixels() # (W, H, 2) 77 | # get pixels centers 78 | points_2d_screen = get_points_2d_screen_from_pixels(pixels) # (H*W, 2) 79 | # filtering 80 | points_2d_screen = points_2d_screen[~zero_depth_mask] 81 | depth = depth[~zero_depth_mask] 82 | points_rgb = points_rgb[~zero_depth_mask] 83 | # unproject depth to 3D 84 | points_3d = camera.unproject_points_2d_screen_to_3d_world( 85 | points_2d_screen=points_2d_screen, depth=depth 86 | ) 87 | # create point cloud 88 | pc = PointCloud( 89 | points_3d=points_3d, 90 | points_rgb=points_rgb, 91 | ) 92 | # append 93 | pcs.append(pc) 94 | 95 | # # create mask for filtering points 96 | # max_nr_points = 10000 97 | # if max_nr_points >= points_3d.shape[0]: 98 | # # no need to filter 99 | # pass 100 | # else: 101 | # idxs = np.random.choice(points_3d.shape[0], max_nr_points, replace=False) 102 | # for pc in pcs: 103 | # pc.mask(idxs) 104 | 105 | # make video 106 | make_video_depth_unproject( 107 | cameras=mv_data.get_split("train"), 108 | point_clouds=pcs, 109 | dataset_name=dataset_name, 110 | remove_tmp_files=True, 111 | scene_radius=mv_data.get_scene_radius(), 112 | azimuth_deg=280.0, 113 | elevation_deg=5.0, 114 | save_path=Path( 115 | os.path.join( 116 | output_path, f"{dataset_name}_{scene_name}_depth_unproject.mp4" 117 | ) 118 | ), 119 | fps=10, 120 | ) 121 | 122 | # # plot point clouds and camera 123 | # plot_3d( 124 | # cameras=mv_data.get_split("train"), 125 | # point_clouds=pcs, 126 | # azimuth_deg=20, 127 | # elevation_deg=30, 128 | # up="z", 129 | # scene_radius=mv_data.get_scene_radius(), 130 | # draw_bounding_cube=True, 131 | # draw_image_planes=True, 132 | # figsize=(15, 15), 133 | # title="point cloud unprojection", 134 | # show=cfg.with_viewer, 135 | # # save_path=os.path.join(output_path, f"{dataset_name}_{scene_name}_point_cloud_from_depths.png"), 136 | # ) 137 | 138 | 139 | if __name__ == "__main__": 140 | 141 | # custom exception handler 142 | sys.excepthook = custom_exception_handler 143 | 144 | # parse arguments 145 | args = tyro.cli(ExampleConfig) 146 | 147 | # get test preset 148 | test_preset = get_dataset_test_preset(args.data.dataset_name) 149 | # scene name 150 | if args.scene_name is None: 151 | args.scene_name = test_preset["scene_name"] 152 | print_warning( 153 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 154 | ) 155 | # additional point clouds paths (if any) 156 | pc_paths = test_preset["pc_paths"] 157 | 158 | # start the example program 159 | main(args, pc_paths) 160 | -------------------------------------------------------------------------------- /examples/camera_points_projection_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | from rich import print 3 | import os 4 | import sys 5 | import torch 6 | from pathlib import Path 7 | import numpy as np 8 | from typing import List 9 | from mvdatasets.geometry.primitives.point_cloud import PointCloud 10 | from mvdatasets.visualization.matplotlib import plot_camera_2d, plot_3d 11 | from mvdatasets.mvdataset import MVDataset 12 | from mvdatasets.utils.printing import print_error, print_warning 13 | from mvdatasets.configs.example_config import ExampleConfig 14 | from examples import get_dataset_test_preset, custom_exception_handler 15 | 16 | 17 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 18 | 19 | device = cfg.machine.device 20 | datasets_path = cfg.datasets_path 21 | output_path = cfg.output_path 22 | scene_name = cfg.scene_name 23 | dataset_name = cfg.data.dataset_name 24 | 25 | # dataset loading 26 | mv_data = MVDataset( 27 | dataset_name, 28 | scene_name, 29 | datasets_path, 30 | config=cfg.data.asdict(), 31 | point_clouds_paths=pc_paths, 32 | verbose=True, 33 | ) 34 | 35 | # random camera index 36 | rand_idx = 0 # torch.randint(0, len(mv_data.get_split("train")), (1,)).item() 37 | camera = mv_data.get_split("train")[rand_idx] 38 | 39 | # random frame index 40 | frame_idx = torch.randint(0, camera.temporal_dim, (1,)).item() 41 | 42 | if len(mv_data.point_clouds) > 0: 43 | point_cloud = mv_data.point_clouds[0] 44 | else: 45 | # dataset has not examples/assets point cloud 46 | raise ValueError("No point cloud found in the dataset") 47 | 48 | points_3d = point_cloud.points_3d 49 | 50 | points_2d_screen, points_mask = camera.project_points_3d_world_to_2d_screen( 51 | points_3d=points_3d, filter_points=True 52 | ) 53 | print("points_2d_screen", points_2d_screen.shape) 54 | 55 | points_3d = points_3d[points_mask] 56 | 57 | # 3d points distance from camera center 58 | camera_points_dists = camera.distance_to_points_3d_world(points_3d) 59 | print("camera_points_dist", camera_points_dists.shape) 60 | 61 | plot_camera_2d( 62 | camera, 63 | points_2d_screen, 64 | frame_idx=frame_idx, 65 | title="point cloud projection", 66 | points_norms=camera_points_dists, 67 | show=cfg.with_viewer, 68 | save_path=os.path.join( 69 | output_path, f"{dataset_name}_{scene_name}_point_cloud_projection.png" 70 | ), 71 | ) 72 | 73 | # # reproject to 3D 74 | # points_3d_r = camera.unproject_points_2d_screen_to_3d_world( 75 | # points_2d_screen=points_2d_screen, depth=camera_points_dists 76 | # ) 77 | 78 | # # filter out random number of points 79 | # num_points = 100 80 | # if len(points_3d) > num_points: 81 | # idx = np.random.choice(range(len(points_3d)), num_points, replace=False) 82 | # points_3d = points_3d[idx] 83 | # points_3d_r = points_3d_r[idx] 84 | 85 | # # create new point clouds for visualization 86 | # pcs = [] 87 | # pcs.append( 88 | # PointCloud( 89 | # points_3d=points_3d, 90 | # color="red", 91 | # label="point cloud", 92 | # marker="o", 93 | # size=150, 94 | # ) 95 | # ) 96 | # pcs.append( 97 | # PointCloud( 98 | # points_3d=points_3d_r, 99 | # color="blue", 100 | # label="reprojected points", 101 | # marker="x", 102 | # size=100, 103 | # ) 104 | # ) 105 | 106 | # # plot point clouds and camera 107 | # plot_3d( 108 | # cameras=[camera], 109 | # point_clouds=pcs, 110 | # azimuth_deg=20, 111 | # elevation_deg=30, 112 | # up="z", 113 | # scene_radius=mv_data.get_scene_radius(), 114 | # draw_bounding_cube=True, 115 | # draw_image_planes=True, 116 | # figsize=(15, 15), 117 | # title="point cloud unprojection", 118 | # show=cfg.with_viewer, 119 | # save_path=os.path.join( 120 | # output_path, f"{dataset_name}_{scene_name}_point_cloud_unprojection.png" 121 | # ), 122 | # ) 123 | 124 | # error = np.mean(np.abs(points_3d_r - points_3d)) 125 | # print("error", error.item()) 126 | 127 | 128 | if __name__ == "__main__": 129 | 130 | # custom exception handler 131 | sys.excepthook = custom_exception_handler 132 | 133 | # parse arguments 134 | args = tyro.cli(ExampleConfig) 135 | 136 | # get test preset 137 | test_preset = get_dataset_test_preset(args.data.dataset_name) 138 | # scene name 139 | if args.scene_name is None: 140 | args.scene_name = test_preset["scene_name"] 141 | print_warning( 142 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 143 | ) 144 | # additional point clouds paths (if any) 145 | pc_paths = test_preset["pc_paths"] 146 | 147 | # start the example program 148 | main(args, pc_paths) 149 | -------------------------------------------------------------------------------- /examples/camera_rays_sampling_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import os 3 | import sys 4 | import torch 5 | from pathlib import Path 6 | import numpy as np 7 | from copy import deepcopy 8 | from typing import List 9 | from mvdatasets.visualization.matplotlib import plot_3d, plot_rays_samples 10 | from mvdatasets.mvdataset import MVDataset 11 | from mvdatasets.geometry.primitives import BoundingBox, BoundingSphere 12 | from mvdatasets.utils.printing import print_error, print_warning 13 | from mvdatasets.configs.example_config import ExampleConfig 14 | from examples import get_dataset_test_preset, custom_exception_handler 15 | 16 | 17 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 18 | 19 | device = cfg.machine.device 20 | datasets_path = cfg.datasets_path 21 | output_path = cfg.output_path 22 | scene_name = cfg.scene_name 23 | dataset_name = cfg.data.dataset_name 24 | 25 | # dataset loading 26 | mv_data = MVDataset( 27 | dataset_name, 28 | scene_name, 29 | datasets_path, 30 | config=cfg.data.asdict(), 31 | point_clouds_paths=pc_paths, 32 | verbose=True, 33 | ) 34 | 35 | # list of bounding boxes to draw 36 | bb = None 37 | bs = None 38 | 39 | scene_type = mv_data.get_scene_type() 40 | if scene_type == "bounded": 41 | draw_bounding_cube = True 42 | draw_contraction_spheres = False 43 | # bounding box 44 | bb = BoundingBox( 45 | pose=np.eye(4), 46 | local_scale=mv_data.get_foreground_radius() * 2, 47 | device=device, 48 | ) 49 | elif scene_type == "unbounded": 50 | draw_bounding_cube = False 51 | draw_contraction_spheres = True 52 | if mv_data.get_scene_radius() > 1.0: 53 | print_warning( 54 | "scene radius is greater than 1.0, contraction spheres will not be displayed" 55 | ) 56 | draw_contraction_spheres = False 57 | 58 | else: 59 | raise ValueError("scene_type not supported") 60 | 61 | # random camera index 62 | rand_idx = 0 # torch.randint(0, len(mv_data.get_split("test")), (1,)).item() 63 | camera = deepcopy(mv_data.get_split("test")[rand_idx]) 64 | 65 | # resize camera 66 | camera.resize(subsample_factor=10) 67 | 68 | # Visualize cameras 69 | plot_3d( 70 | cameras=[camera], 71 | point_clouds=mv_data.point_clouds, 72 | # bounding_boxes=[bb] if bb is not None else None, 73 | nr_rays=256, 74 | azimuth_deg=20, 75 | elevation_deg=30, 76 | scene_radius=mv_data.get_scene_radius(), 77 | draw_bounding_cube=draw_bounding_cube, 78 | up="z", 79 | draw_image_planes=True, 80 | draw_cameras_frustums=False, 81 | draw_contraction_spheres=draw_contraction_spheres, 82 | figsize=(15, 15), 83 | title=f"test camera {rand_idx} rays", 84 | show=cfg.with_viewer, 85 | save_path=os.path.join( 86 | output_path, f"{dataset_name}_{scene_name}_camera_rays.png" 87 | ), 88 | ) 89 | 90 | # Visualize intersections with bounding box 91 | 92 | # shoot rays from camera 93 | rays_o, rays_d, points_2d_screen = camera.get_rays(device=device) 94 | 95 | # bounding primitive intersection test 96 | if scene_type == "bounded": 97 | is_hit, t_near, t_far, p_near, p_far = bb.intersect(rays_o, rays_d) 98 | elif scene_type == "unbounded": 99 | # bounding sphere 100 | bs = BoundingSphere( 101 | pose=np.eye(4), 102 | local_scale=0.5, 103 | device=device, 104 | ) 105 | is_hit, t_near, t_far, p_near, p_far = bs.intersect(rays_o, rays_d) 106 | if mv_data.get_scene_radius() > 1.0: 107 | print_warning( 108 | "scene radius is greater than 1.0, bounding box is not defined" 109 | ) 110 | exit(0) 111 | else: 112 | raise ValueError("scene_type not supported") 113 | 114 | plot_rays_samples( 115 | rays_o=rays_o.cpu().numpy(), 116 | rays_d=rays_d.cpu().numpy(), 117 | t_near=t_near.cpu().numpy(), 118 | t_far=t_far.cpu().numpy(), 119 | nr_rays=32, 120 | camera=camera, 121 | bounding_boxes=[bb] if bb is not None else None, 122 | azimuth_deg=20, 123 | elevation_deg=30, 124 | scene_radius=mv_data.get_scene_radius(), 125 | draw_bounding_cube=draw_bounding_cube, 126 | draw_contraction_spheres=draw_contraction_spheres, 127 | title="bounding box intersections", 128 | show=False, 129 | save_path=os.path.join( 130 | output_path, f"{dataset_name}_{scene_name}_bounding_box_intersections.png" 131 | ), 132 | ) 133 | 134 | 135 | if __name__ == "__main__": 136 | 137 | # custom exception handler 138 | sys.excepthook = custom_exception_handler 139 | 140 | # parse arguments 141 | args = tyro.cli(ExampleConfig) 142 | 143 | # get test preset 144 | test_preset = get_dataset_test_preset(args.data.dataset_name) 145 | # scene name 146 | if args.scene_name is None: 147 | args.scene_name = test_preset["scene_name"] 148 | print_warning( 149 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 150 | ) 151 | # additional point clouds paths (if any) 152 | pc_paths = test_preset["pc_paths"] 153 | 154 | # start the example program 155 | main(args, pc_paths) 156 | -------------------------------------------------------------------------------- /examples/camera_trajectory_vis.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import tyro 3 | import os 4 | from typing import List 5 | from pathlib import Path 6 | from mvdatasets.visualization.matplotlib import plot_camera_trajectory 7 | from mvdatasets.visualization.video_gen import make_video_camera_trajectory 8 | from mvdatasets.mvdataset import MVDataset 9 | from mvdatasets.utils.printing import print_warning, print_log, print_error 10 | from mvdatasets.configs.example_config import ExampleConfig 11 | from examples import get_dataset_test_preset, custom_exception_handler 12 | 13 | 14 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 15 | 16 | device = cfg.machine.device 17 | datasets_path = cfg.datasets_path 18 | output_path = cfg.output_path 19 | scene_name = cfg.scene_name 20 | dataset_name = cfg.data.dataset_name 21 | 22 | # dataset loading 23 | mv_data = MVDataset( 24 | dataset_name, 25 | scene_name, 26 | datasets_path, 27 | config=cfg.data.asdict(), 28 | point_clouds_paths=pc_paths, 29 | verbose=True, 30 | ) 31 | 32 | # check if camera trajectory is available 33 | print("nr_sequence_frames:", mv_data.get_nr_sequence_frames()) 34 | if mv_data.get_nr_sequence_frames() <= 1: 35 | raise ValueError( 36 | f"{dataset_name} is a static dataset and does not have camera trajectory" 37 | ) 38 | return 39 | 40 | # check if monocular sequence 41 | print("nr_per_camera_frames:", mv_data.get_nr_per_camera_frames()) 42 | if mv_data.get_nr_per_camera_frames() > 1: 43 | raise ValueError(f"{dataset_name} is not a monocular sequence") 44 | return 45 | 46 | # make video 47 | make_video_camera_trajectory( 48 | cameras=mv_data.get_split("train"), 49 | point_clouds=mv_data.point_clouds, 50 | dataset_name=dataset_name, 51 | nr_frames=-1, # -1 means all frames 52 | remove_tmp_files=True, 53 | scene_radius=mv_data.get_scene_radius(), 54 | save_path=Path( 55 | os.path.join(output_path, f"{dataset_name}_{scene_name}_trajectory.mp4") 56 | ), 57 | fps=mv_data.get_frame_rate(), 58 | ) 59 | 60 | # # create output folder 61 | # output_path = os.path.join(output_path, f"{dataset_name}_{scene_name}_trajectory") 62 | # os.makedirs(output_path, exist_ok=True) 63 | 64 | # # Visualize cameras 65 | # for _, last_frame_idx in enumerate(frames_idxs): 66 | 67 | # # get camera 68 | # camera = mv_data.get_split("train")[last_frame_idx] 69 | 70 | # # get timestamp 71 | # ts = camera.get_timestamps()[0] 72 | # # round to 3 decimal places 73 | # ts = round(ts, 3) 74 | 75 | # plot_camera_trajectory( 76 | # cameras=mv_data.get_split("train"), 77 | # last_frame_idx=last_frame_idx, 78 | # draw_every_n_cameras=1, 79 | # points_3d=[point_cloud], 80 | # points_3d_colors=["black"], 81 | # azimuth_deg=60, 82 | # elevation_deg=30, 83 | # max_nr_points=None, 84 | # up="z", 85 | # scene_radius=mv_data.get_scene_radius(), 86 | # draw_rgb_frame=True, 87 | # draw_all_cameras_frames=False, 88 | # draw_image_planes=True, 89 | # draw_cameras_frustums=True, 90 | # figsize=(15, 15), 91 | # title=f"{dataset_name} camera trajectory up to time {ts} [s]", 92 | # show=cfg.with_viewer, 93 | # save_path=os.path.join(output_path, f"{dataset_name}_{scene_name}_trajectory_{format(last_frame_idx, '09d')}.png"), 94 | # ) 95 | 96 | # # make video 97 | # video_path = os.path.join(output_path, f"{dataset_name}_{scene_name}_trajectory.mp4") 98 | # os.system(f'ffmpeg -y -r 10 -i {output_path}/{dataset_name}_trajectory_%09d.png -vf scale="trunc(iw/2)*2:trunc(ih/2)*2" -vcodec libx264 -crf 25 -pix_fmt yuv420p {video_path}') 99 | # print_log(f"Video saved at {video_path}") 100 | 101 | 102 | if __name__ == "__main__": 103 | 104 | # custom exception handler 105 | sys.excepthook = custom_exception_handler 106 | 107 | # parse arguments 108 | args = tyro.cli(ExampleConfig) 109 | 110 | # get test preset 111 | test_preset = get_dataset_test_preset(args.data.dataset_name) 112 | # scene name 113 | if args.scene_name is None: 114 | args.scene_name = test_preset["scene_name"] 115 | print_warning( 116 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 117 | ) 118 | # additional point clouds paths (if any) 119 | pc_paths = test_preset["pc_paths"] 120 | 121 | # start the example program 122 | main(args, pc_paths) 123 | -------------------------------------------------------------------------------- /examples/dataset_config_to_yaml.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import sys 3 | from typing import List 4 | from pathlib import Path 5 | from mvdatasets.mvdataset import MVDataset 6 | from mvdatasets.utils.printing import print_error, print_warning, print_success 7 | from mvdatasets.configs.example_config import ExampleConfig 8 | from mvdatasets.io import load_yaml, save_yaml 9 | from examples import get_dataset_test_preset, custom_exception_handler 10 | 11 | 12 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 13 | 14 | device = cfg.machine.device 15 | datasets_path = cfg.datasets_path 16 | output_path = cfg.output_path 17 | scene_name = cfg.scene_name 18 | dataset_name = cfg.data.dataset_name 19 | 20 | # dataset loading 21 | mv_data = MVDataset( 22 | dataset_name, 23 | scene_name, 24 | datasets_path, 25 | config=cfg.data.asdict(), 26 | point_clouds_paths=pc_paths, 27 | verbose=True, 28 | ) 29 | 30 | # Save to a YAML file 31 | save_yaml(cfg.data.asdict(), output_path / "data_config.yaml") 32 | print_success("Config saved to file") 33 | 34 | # Load from the YAML file 35 | cfg_dict = load_yaml(output_path / "data_config.yaml") 36 | print_success("Config loaded from file") 37 | print(cfg_dict) 38 | 39 | print("done") 40 | 41 | 42 | if __name__ == "__main__": 43 | 44 | # custom exception handler 45 | sys.excepthook = custom_exception_handler 46 | 47 | # parse arguments 48 | args = tyro.cli(ExampleConfig) 49 | 50 | # get test preset 51 | test_preset = get_dataset_test_preset(args.data.dataset_name) 52 | # scene name 53 | if args.scene_name is None: 54 | args.scene_name = test_preset["scene_name"] 55 | print_warning( 56 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 57 | ) 58 | # additional point clouds paths (if any) 59 | pc_paths = test_preset["pc_paths"] 60 | 61 | # start the example program 62 | main(args, pc_paths) 63 | -------------------------------------------------------------------------------- /examples/dataset_splits_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import numpy as np 3 | import os 4 | import sys 5 | from typing import List 6 | from pathlib import Path 7 | from mvdatasets.visualization.matplotlib import plot_3d 8 | from mvdatasets.mvdataset import MVDataset 9 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 10 | from mvdatasets.geometry.primitives.bounding_sphere import BoundingSphere 11 | from mvdatasets.utils.printing import print_error, print_warning 12 | from mvdatasets.configs.example_config import ExampleConfig 13 | from examples import get_dataset_test_preset, custom_exception_handler 14 | 15 | 16 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 17 | 18 | device = cfg.machine.device 19 | datasets_path = cfg.datasets_path 20 | output_path = cfg.output_path 21 | scene_name = cfg.scene_name 22 | dataset_name = cfg.data.dataset_name 23 | 24 | # dataset loading 25 | mv_data = MVDataset( 26 | dataset_name, 27 | scene_name, 28 | datasets_path, 29 | config=cfg.data.asdict(), 30 | point_clouds_paths=pc_paths, 31 | verbose=True, 32 | ) 33 | 34 | # # sdf init 35 | # bs = BoundingSphere( 36 | # pose=np.eye(4), 37 | # local_scale=mv_data.get_sphere_init_radius(), 38 | # device=device, 39 | # verbose=True, 40 | # ) 41 | 42 | # foreground bb 43 | bb = BoundingBox( 44 | pose=np.eye(4), 45 | local_scale=mv_data.get_foreground_radius() * 2, 46 | device=device, 47 | ) 48 | 49 | # scene type 50 | scene_type = mv_data.get_scene_type() 51 | if scene_type == "bounded": 52 | draw_bounding_cube = True 53 | draw_contraction_spheres = False 54 | 55 | if scene_type == "unbounded": 56 | draw_bounding_cube = False 57 | draw_contraction_spheres = True 58 | 59 | if mv_data.get_scene_radius() > 1.0: 60 | print_warning( 61 | "scene radius is greater than 1.0, contraction spheres will not be displayed" 62 | ) 63 | bb = None 64 | draw_contraction_spheres = False 65 | 66 | # Visualize cameras 67 | for split in mv_data.get_splits(): 68 | 69 | print("visualizing cameras for split: ", split) 70 | 71 | nr_cameras = len(mv_data.get_split(split)) 72 | if nr_cameras // 50 > 1: 73 | draw_every_n_cameras = nr_cameras // 50 74 | print_warning( 75 | f"{split} has too many cameras; displaying one every {draw_every_n_cameras}" 76 | ) 77 | else: 78 | draw_every_n_cameras = 1 79 | 80 | plot_3d( 81 | cameras=mv_data.get_split(split), 82 | draw_every_n_cameras=draw_every_n_cameras, 83 | point_clouds=mv_data.point_clouds, 84 | bounding_boxes=[bb] if bb is not None else [], 85 | # bounding_spheres=[bs], 86 | azimuth_deg=20, 87 | elevation_deg=30, 88 | up="z", 89 | scene_radius=mv_data.get_scene_radius(), 90 | draw_bounding_cube=draw_bounding_cube, 91 | draw_image_planes=True, 92 | draw_cameras_frustums=False, 93 | draw_contraction_spheres=draw_contraction_spheres, 94 | figsize=(15, 15), 95 | title=f"{split} cameras", 96 | show=cfg.with_viewer, 97 | save_path=os.path.join( 98 | output_path, f"{dataset_name}_{scene_name}_{split}_cameras.png" 99 | ), 100 | ) 101 | 102 | print("done") 103 | 104 | 105 | if __name__ == "__main__": 106 | 107 | # custom exception handler 108 | sys.excepthook = custom_exception_handler 109 | 110 | # parse arguments 111 | args = tyro.cli(ExampleConfig) 112 | 113 | # get test preset 114 | test_preset = get_dataset_test_preset(args.data.dataset_name) 115 | # scene name 116 | if args.scene_name is None: 117 | args.scene_name = test_preset["scene_name"] 118 | print_warning( 119 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 120 | ) 121 | # additional point clouds paths (if any) 122 | pc_paths = test_preset["pc_paths"] 123 | 124 | # start the example program 125 | main(args, pc_paths) 126 | -------------------------------------------------------------------------------- /examples/pixels_sampling_vis.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import sys 3 | import os 4 | from typing import List 5 | from pathlib import Path 6 | from copy import deepcopy 7 | from mvdatasets.visualization.matplotlib import plot_camera_2d 8 | from mvdatasets.mvdataset import MVDataset 9 | from mvdatasets.configs.example_config import ExampleConfig 10 | from mvdatasets.utils.printing import print_warning 11 | from examples import get_dataset_test_preset, custom_exception_handler 12 | 13 | 14 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 15 | 16 | device = cfg.machine.device 17 | datasets_path = cfg.datasets_path 18 | output_path = cfg.output_path 19 | scene_name = cfg.scene_name 20 | dataset_name = cfg.data.dataset_name 21 | 22 | # dataset loading 23 | mv_data = MVDataset( 24 | dataset_name, 25 | scene_name, 26 | datasets_path, 27 | config=cfg.data.asdict(), 28 | point_clouds_paths=pc_paths, 29 | verbose=True, 30 | ) 31 | 32 | # random camera index 33 | rand_idx = 2 # torch.randint(0, len(mv_data.get_split("test")), (1,)).item() 34 | camera = deepcopy(mv_data.get_split("train")[rand_idx]) 35 | 36 | # resize camera 37 | taget_dim = 100 38 | min_dim = min(camera.width, camera.height) 39 | print("min_dim", min_dim) 40 | subsample_factor = min_dim // taget_dim 41 | print("subsample_factor", subsample_factor) 42 | camera.resize(subsample_factor=subsample_factor) 43 | 44 | print(camera) 45 | 46 | # gen rays 47 | rays_o, rays_d, points_2d_screen = camera.get_rays(jitter_pixels=True) 48 | plot_camera_2d( 49 | camera, 50 | points_2d_screen, 51 | show_ticks=True, 52 | figsize=(15, 15), 53 | title="screen space sampling (jittered)", 54 | show=cfg.with_viewer, 55 | save_path=os.path.join( 56 | output_path, 57 | f"{dataset_name}_{scene_name}_screen_space_sampling_jittered.png", 58 | ), 59 | ) 60 | 61 | # gen rays 62 | rays_o, rays_d, points_2d_screen = camera.get_rays(jitter_pixels=False) 63 | plot_camera_2d( 64 | camera, 65 | points_2d_screen, 66 | show_ticks=True, 67 | figsize=(15, 15), 68 | title="screen space sampling", 69 | show=cfg.with_viewer, 70 | save_path=os.path.join( 71 | output_path, f"{dataset_name}_{scene_name}_screen_space_sampling.png" 72 | ), 73 | ) 74 | 75 | 76 | if __name__ == "__main__": 77 | 78 | # custom exception handler 79 | sys.excepthook = custom_exception_handler 80 | 81 | # parse arguments 82 | args = tyro.cli(ExampleConfig) 83 | 84 | # get test preset 85 | test_preset = get_dataset_test_preset(args.data.dataset_name) 86 | # scene name 87 | if args.scene_name is None: 88 | args.scene_name = test_preset["scene_name"] 89 | print_warning( 90 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 91 | ) 92 | # additional point clouds paths (if any) 93 | pc_paths = test_preset["pc_paths"] 94 | 95 | # start the example program 96 | main(args, pc_paths) 97 | -------------------------------------------------------------------------------- /examples/sample_cameras_on_hemisphere.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import numpy as np 3 | import os 4 | import sys 5 | from tqdm import tqdm 6 | from pathlib import Path 7 | from typing import List 8 | from mvdatasets.visualization.matplotlib import plot_3d 9 | from mvdatasets.visualization.matplotlib import plot_current_batch 10 | from mvdatasets.visualization.matplotlib import plot_rays_samples 11 | from mvdatasets.mvdataset import MVDataset 12 | from mvdatasets.utils.profiler import Profiler 13 | from mvdatasets.tensorreel import TensorReel 14 | from mvdatasets.utils.virtual_cameras import sample_cameras_on_hemisphere 15 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 16 | from mvdatasets.utils.printing import print_warning, print_error 17 | from mvdatasets.configs.example_config import ExampleConfig 18 | from examples import get_dataset_test_preset, custom_exception_handler 19 | 20 | 21 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 22 | 23 | device = cfg.machine.device 24 | datasets_path = cfg.datasets_path 25 | output_path = cfg.output_path 26 | scene_name = cfg.scene_name 27 | dataset_name = cfg.data.dataset_name 28 | 29 | # dataset loading 30 | mv_data = MVDataset( 31 | dataset_name, 32 | scene_name, 33 | datasets_path, 34 | config=cfg.data.asdict(), 35 | point_clouds_paths=pc_paths, 36 | verbose=True, 37 | ) 38 | 39 | bb = BoundingBox( 40 | pose=np.eye(4), 41 | local_scale=mv_data.get_foreground_radius() * 2, 42 | device=device, 43 | ) 44 | 45 | # only available for object centric datasets 46 | if not mv_data.cameras_on_hemisphere: 47 | raise ValueError(f"{dataset_name} does not have cameras on hemisphere") 48 | 49 | intrinsics = mv_data.get_split("train")[0].get_intrinsics() 50 | width = mv_data.get_split("train")[0].width 51 | height = mv_data.get_split("train")[0].height 52 | camera_center = mv_data.get_split("train")[0].get_center() 53 | camera_radius = np.linalg.norm(camera_center) 54 | 55 | sampled_cameras = sample_cameras_on_hemisphere( 56 | intrinsics=intrinsics, 57 | width=width, 58 | height=height, 59 | radius=camera_radius, 60 | up="z", 61 | nr_cameras=100, 62 | ) 63 | 64 | camera = sampled_cameras[0] 65 | 66 | # shoot rays from camera 67 | rays_o, rays_d, points_2d_screen = camera.get_rays(device=device) 68 | 69 | plot_rays_samples( 70 | rays_o=rays_o.cpu().numpy(), 71 | rays_d=rays_d.cpu().numpy(), 72 | # t_near=t_near.cpu().numpy(), 73 | # t_far=t_far.cpu().numpy(), 74 | nr_rays=32, 75 | camera=camera, 76 | # bounding_boxes=[bb] if bb is not None else None, 77 | azimuth_deg=20, 78 | elevation_deg=30, 79 | scene_radius=camera_radius, 80 | # draw_bounding_cube=draw_bounding_cube, 81 | # draw_contraction_spheres=draw_contraction_spheres, 82 | title="bounding box intersections", 83 | show=cfg.with_viewer, 84 | save_path=os.path.join(output_path, "virtual_camera_rays.png"), 85 | ) 86 | 87 | # Visualize cameras 88 | plot_3d( 89 | cameras=sampled_cameras, 90 | point_clouds=mv_data.point_clouds, 91 | bounding_boxes=[bb], 92 | azimuth_deg=20, 93 | elevation_deg=30, 94 | scene_radius=mv_data.get_scene_radius(), 95 | up="z", 96 | figsize=(15, 15), 97 | title="sampled cameras", 98 | show=cfg.with_viewer, 99 | save_path=os.path.join(output_path, "virtual_cameras.png"), 100 | ) 101 | 102 | # Create tensor reel 103 | tensorreel = TensorReel(sampled_cameras, modalities=[], device=device) # no data 104 | print(tensorreel) 105 | 106 | batch_size = 512 107 | 108 | benchmark = False 109 | 110 | if benchmark: 111 | # Set profiler 112 | profiler = Profiler() # nb: might slow down the code 113 | nr_iterations = 10000 114 | else: 115 | profiler = None 116 | nr_iterations = 10 117 | 118 | # use a subset of cameras and frames 119 | # cameras_idx = np.random.permutation(len(mv_data.get_split("train")))[:5] 120 | # frames_idx = np.random.permutation(mv_data.get_nr_per_camera_frames())[:2] 121 | # or use all 122 | cameras_idx = None 123 | frames_idx = None 124 | print("cameras_idx", cameras_idx) 125 | print("frames_idx", frames_idx) 126 | 127 | # ------------------------------------------------------------------------- 128 | 129 | pbar = tqdm(range(nr_iterations), desc="ray casting", ncols=100) 130 | azimuth_deg = 20 131 | azimuth_deg_delta = 1 132 | for i in pbar: 133 | 134 | # cameras_idx = np.random.permutation(len(mv_data.get_split("train")))[:2] 135 | 136 | if profiler is not None: 137 | profiler.start("get_next_rays_batch") 138 | 139 | # get rays and gt values 140 | batch = tensorreel.get_next_rays_batch( 141 | batch_size=batch_size, 142 | cameras_idx=cameras_idx, 143 | frames_idx=frames_idx, 144 | jitter_pixels=True, 145 | nr_rays_per_pixel=1, 146 | ) 147 | 148 | if profiler is not None: 149 | profiler.end("get_next_rays_batch") 150 | 151 | if not benchmark: 152 | 153 | # unpack batch 154 | batch_cameras_idx = batch["cameras_idx"] 155 | batch_rays_o = batch["rays_o"] 156 | batch_rays_d = batch["rays_d"] 157 | 158 | # print data shapes 159 | for k, v in batch.items(): 160 | # if v is a dict 161 | if isinstance(v, dict): 162 | for k1, v1 in v.items(): 163 | print(f"{k}, " f"{k1}", v1.shape, v1.device, v1.dtype) 164 | else: 165 | print(f"{k}", v.shape, v.device, v.dtype) 166 | 167 | plot_current_batch( 168 | cameras=sampled_cameras, 169 | cameras_idx=batch_cameras_idx.cpu().numpy(), 170 | rays_o=batch_rays_o.cpu().numpy(), 171 | rays_d=batch_rays_d.cpu().numpy(), 172 | rgbs=None, 173 | masks=None, 174 | bounding_boxes=[bb], 175 | azimuth_deg=azimuth_deg, 176 | elevation_deg=30, 177 | scene_radius=mv_data.get_scene_radius(), 178 | up="z", 179 | figsize=(15, 15), 180 | title=f"rays batch sampling {i}", 181 | show=cfg.with_viewer, 182 | save_path=os.path.join( 183 | output_path, 184 | f"virtual_cameras_batch_{i}.png", 185 | ), 186 | ) 187 | 188 | # update azimuth 189 | azimuth_deg += azimuth_deg_delta 190 | 191 | if profiler is not None: 192 | profiler.print_avg_times() 193 | 194 | 195 | if __name__ == "__main__": 196 | 197 | # custom exception handler 198 | sys.excepthook = custom_exception_handler 199 | 200 | # parse arguments 201 | args = tyro.cli(ExampleConfig) 202 | 203 | # get test preset 204 | test_preset = get_dataset_test_preset(args.data.dataset_name) 205 | # scene name 206 | if args.scene_name is None: 207 | args.scene_name = test_preset["scene_name"] 208 | print_warning( 209 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 210 | ) 211 | # additional point clouds paths (if any) 212 | pc_paths = test_preset["pc_paths"] 213 | 214 | # start the example program 215 | main(args, pc_paths) 216 | -------------------------------------------------------------------------------- /examples/training_loop_tensorreel.py: -------------------------------------------------------------------------------- 1 | import tyro 2 | import os 3 | import sys 4 | from pathlib import Path 5 | import numpy as np 6 | from tqdm import tqdm 7 | from typing import List 8 | from mvdatasets.visualization.matplotlib import plot_current_batch 9 | from mvdatasets.mvdataset import MVDataset 10 | from mvdatasets.tensorreel import TensorReel 11 | from mvdatasets.utils.profiler import Profiler 12 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 13 | from mvdatasets.configs.example_config import ExampleConfig 14 | from mvdatasets.utils.printing import print_warning 15 | from examples import get_dataset_test_preset, custom_exception_handler 16 | 17 | 18 | def main(cfg: ExampleConfig, pc_paths: List[Path]): 19 | 20 | device = cfg.machine.device 21 | datasets_path = cfg.datasets_path 22 | output_path = cfg.output_path 23 | scene_name = cfg.scene_name 24 | dataset_name = cfg.data.dataset_name 25 | 26 | # dataset loading 27 | mv_data = MVDataset( 28 | dataset_name, 29 | scene_name, 30 | datasets_path, 31 | config=cfg.data.asdict(), 32 | point_clouds_paths=pc_paths, 33 | verbose=True, 34 | ) 35 | 36 | # create bounding box 37 | bb = BoundingBox( 38 | pose=np.eye(4), 39 | local_scale=mv_data.get_foreground_radius() * 2, 40 | line_width=2.0, 41 | device=device, 42 | ) 43 | 44 | # TensorReel (~1300 it/s), camera's data in concatenated in big tensors on GPU 45 | 46 | tensorreel = TensorReel(mv_data.get_split("train"), device=device) 47 | print(tensorreel) 48 | 49 | batch_size = 512 50 | 51 | benchmark = False 52 | 53 | if benchmark: 54 | # Set profiler 55 | profiler = Profiler() # nb: might slow down the code 56 | nr_iterations = 10000 57 | else: 58 | profiler = None 59 | nr_iterations = 10 60 | 61 | # use a subset of cameras and frames 62 | # cameras_idx = np.random.permutation(len(mv_data.get_split("train")))[:5] 63 | # frames_idx = np.random.permutation(mv_data.get_nr_per_camera_frames())[:2] 64 | # or use all 65 | cameras_idx = None 66 | frames_idx = None 67 | print("cameras_idx", cameras_idx) 68 | print("frames_idx", frames_idx) 69 | 70 | # ------------------------------------------------------------------------- 71 | 72 | pbar = tqdm(range(nr_iterations), desc="ray casting", ncols=100) 73 | azimuth_deg = 20 74 | azimuth_deg_delta = 1 75 | for i in pbar: 76 | 77 | if profiler is not None: 78 | profiler.start("get_next_rays_batch") 79 | 80 | # get rays and gt values 81 | batch = tensorreel.get_next_rays_batch( 82 | batch_size=batch_size, 83 | cameras_idx=cameras_idx, 84 | frames_idx=frames_idx, 85 | jitter_pixels=True, 86 | nr_rays_per_pixel=1, 87 | ) 88 | 89 | if profiler is not None: 90 | profiler.end("get_next_rays_batch") 91 | 92 | if not benchmark: 93 | 94 | # unpack batch 95 | batch_cameras_idx = batch["cameras_idx"] 96 | batch_rays_o = batch["rays_o"] 97 | batch_rays_d = batch["rays_d"] 98 | batch_vals = batch["vals"] 99 | 100 | # print data shapes 101 | for k, v in batch.items(): 102 | # if v is a dict 103 | if isinstance(v, dict): 104 | for k1, v1 in v.items(): 105 | print(f"{k}, " f"{k1}", v1.shape, v1.device, v1.dtype) 106 | else: 107 | print(f"{k}", v.shape, v.device, v.dtype) 108 | 109 | # get gt values 110 | gt_rgb = batch_vals["rgbs"] 111 | if "masks" in batch_vals: 112 | gt_mask = batch_vals["masks"] 113 | else: 114 | gt_mask = None 115 | 116 | plot_current_batch( 117 | cameras=mv_data.get_split("train"), 118 | cameras_idx=batch_cameras_idx.cpu().numpy(), 119 | rays_o=batch_rays_o.cpu().numpy(), 120 | rays_d=batch_rays_d.cpu().numpy(), 121 | rgbs=gt_rgb.cpu().numpy(), 122 | masks=gt_mask.cpu().numpy() if gt_mask is not None else None, 123 | bounding_boxes=[bb], 124 | azimuth_deg=azimuth_deg, 125 | elevation_deg=30, 126 | scene_radius=mv_data.get_scene_radius(), 127 | up="z", 128 | figsize=(15, 15), 129 | title=f"rays batch sampling {i}", 130 | show=cfg.with_viewer, 131 | save_path=os.path.join( 132 | output_path, f"{dataset_name}_{scene_name}_batch_{i}.png" 133 | ), 134 | ) 135 | 136 | # update azimuth 137 | azimuth_deg += azimuth_deg_delta 138 | 139 | if profiler is not None: 140 | profiler.print_avg_times() 141 | 142 | 143 | if __name__ == "__main__": 144 | 145 | # custom exception handler 146 | sys.excepthook = custom_exception_handler 147 | 148 | # parse arguments 149 | args = tyro.cli(ExampleConfig) 150 | 151 | # get test preset 152 | test_preset = get_dataset_test_preset(args.data.dataset_name) 153 | # scene name 154 | if args.scene_name is None: 155 | args.scene_name = test_preset["scene_name"] 156 | print_warning( 157 | f"scene_name is None, using preset test scene {args.scene_name} for dataset" 158 | ) 159 | # additional point clouds paths (if any) 160 | pc_paths = test_preset["pc_paths"] 161 | 162 | # start the example program 163 | main(args, pc_paths) 164 | -------------------------------------------------------------------------------- /imgs/MVD.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/imgs/MVD.png -------------------------------------------------------------------------------- /imgs/art.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/imgs/art.webp -------------------------------------------------------------------------------- /imgs/blender_test_cameras.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/imgs/blender_test_cameras.png -------------------------------------------------------------------------------- /imgs/blender_train_cameras.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/imgs/blender_train_cameras.png -------------------------------------------------------------------------------- /imgs/pose_and_intrinsics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/imgs/pose_and_intrinsics.png -------------------------------------------------------------------------------- /imgs/projection_with_principal_point_offset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/imgs/projection_with_principal_point_offset.png -------------------------------------------------------------------------------- /mvdatasets/__init__.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | from mvdatasets.utils.printing import ( 3 | print_error, 4 | print_warning, 5 | print_info, 6 | print_log, 7 | print_success, 8 | ) 9 | from mvdatasets.camera import Camera 10 | from mvdatasets.mvdataset import MVDataset 11 | from mvdatasets.tensorreel import TensorReel 12 | from mvdatasets.utils.profiler import Profiler 13 | from mvdatasets.datasplit import DataSplit 14 | -------------------------------------------------------------------------------- /mvdatasets/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/mvdatasets/configs/__init__.py -------------------------------------------------------------------------------- /mvdatasets/configs/base_config.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | from pathlib import Path 3 | from typing import Tuple, Any, Type, Literal, List, Optional 4 | import numpy as np 5 | from dataclasses import dataclass, field, asdict 6 | 7 | 8 | @dataclass 9 | class PrintableConfig: 10 | """Printable Config defining str function""" 11 | 12 | def asdict(self): 13 | return asdict(self) 14 | 15 | def __str__(self): 16 | lines = [self.__class__.__name__ + ":"] 17 | for key, val in vars(self).items(): 18 | if isinstance(val, Tuple): 19 | flattened_val = "[" 20 | for item in val: 21 | flattened_val += str(item) + "\n" 22 | flattened_val = flattened_val.rstrip("\n") 23 | val = flattened_val + "]" 24 | lines += f"{key}: {str(val)}".split("\n") 25 | return "\n ".join(lines) 26 | -------------------------------------------------------------------------------- /mvdatasets/configs/dataset_config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Tuple, Any, Type, Literal, List, Optional 3 | import numpy as np 4 | import tyro 5 | from dataclasses import dataclass, field 6 | from mvdatasets.configs.base_config import PrintableConfig 7 | 8 | # TODO: use enums for scene types 9 | # from enum import Enum 10 | 11 | # 12 | SCENE_TYPES = ["bounded", "unbounded"] 13 | 14 | 15 | @dataclass 16 | class DatasetConfig(PrintableConfig): 17 | """General dataset configuration""" 18 | 19 | # Default generic configuration 20 | 21 | dataset_name: tyro.conf.Suppress[str] = "" 22 | """Name of the dataset (e.g., "dtu", "shelly", ...)""" 23 | scene_type: str = "unbounded" 24 | """Type of scene (bounded or unbounded)""" 25 | splits: List[str] = field(default_factory=lambda: ["train"]) 26 | """Dataset splits""" 27 | auto_center: bool = False 28 | """Shift the average of cameras centers to the origin""" 29 | rotate_deg: List[float] = field(default_factory=lambda: [0.0, 0.0, 0.0]) 30 | """Scene rotation angles in degrees""" 31 | max_cameras_distance: Optional[float] = None 32 | """Maximum distance of the furthest camera from the origin (for scaling), if None, no scaling""" 33 | foreground_scale_mult: float = 1.0 34 | """Foreground area scale factor (<= 1.0), (1.0 = no scaling), e.g. 0.5, 1.0, 2.0, ...""" 35 | subsample_factor: int = 1 36 | """Subsampling factor (>= 1), (1 = no subsampling), e.g. 2, 3, 4, ...""" 37 | init_sphere_radius_mult: float = 0.1 38 | """Initial sphere radius multiplier (<= 1.0), (for SDF initialization)""" 39 | pose_only: bool = False 40 | """Load only poses (no images)""" 41 | 42 | def __post__init__(self): 43 | # 44 | self.rotate_deg = np.array(self.rotate_deg, dtype=np.float32) 45 | 46 | # Check configuration values 47 | # dataset_name 48 | if type(self.dataset_name) is not str or len(self.dataset_name) == 0: 49 | raise ValueError("dataset_name must be a non-empty string") 50 | # scene_type 51 | if self.scene_type not in SCENE_TYPES: 52 | raise ValueError(f"scene_type must be one of {SCENE_TYPES}") 53 | # auto_center 54 | if type(self.auto_center) is not bool: 55 | raise ValueError("auto_center must be a boolean") 56 | # max_cameras_distance 57 | if self.max_cameras_distance is not None and self.max_cameras_distance <= 0: 58 | raise ValueError("max_cameras_distance must be > 0 or None") 59 | # subsample factor 60 | if type(self.subsample_factor) is not int or self.subsample_factor < 1: 61 | raise ValueError("subsample_factor must be an integer >= 1") 62 | # rotate_deg 63 | if type(self.rotate_deg) is not np.ndarray or len(self.rotate_deg) != 3: 64 | raise ValueError("rotate_deg must be a numpy array of 3 elements") 65 | # multipliers 66 | if self.foreground_scale_mult <= 0 or self.foreground_scale_mult > 1: 67 | raise ValueError("foreground_scale_mult must be in (0, 1]") 68 | if self.init_sphere_radius_mult <= 0 or self.init_sphere_radius_mult > 1: 69 | raise ValueError("init_sphere_radius_mult must be in (0, 1]") 70 | # pose_only 71 | if type(self.pose_only) is not bool: 72 | raise ValueError("pose_only must be a boolean") 73 | -------------------------------------------------------------------------------- /mvdatasets/configs/example_config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from dataclasses import dataclass, field 3 | from typing import Optional, Union 4 | from mvdatasets.configs.base_config import PrintableConfig 5 | from mvdatasets.configs.machine_config import MachineConfig 6 | from mvdatasets.configs.datasets_configs import AnnotatedDatasetsConfigUnion 7 | from mvdatasets.configs.datasets_configs import datasets_configs 8 | from mvdatasets.utils.printing import print_warning 9 | 10 | 11 | @dataclass 12 | class ExampleConfig(PrintableConfig): 13 | """Example configuration.""" 14 | 15 | # dataset configuration 16 | data: AnnotatedDatasetsConfigUnion 17 | """Dataset configuration""" 18 | 19 | scene_name: Optional[str] = None 20 | """Name of the scene (e.g., "dtu_scan83", "khady", ...)""" 21 | 22 | # paths 23 | datasets_path: Path = Path("data") 24 | """Relative or absolute path to the root datasets directory""" 25 | output_path: Path = Path("outputs") 26 | """Relative or absolute path to the output directory to save splots, videos, etc...""" 27 | 28 | with_viewer: bool = False 29 | """Show viewers to visualize the examples""" 30 | 31 | # nested configs 32 | machine: MachineConfig = field(default_factory=MachineConfig) 33 | """Machine configuration""" 34 | 35 | def __post_init__(self): 36 | # datasets_path 37 | if not self.datasets_path.exists(): 38 | raise ValueError(f"Dataset path {self.datasets_path} does not exist.") 39 | # create output directory if it does not exist 40 | self.output_path.mkdir(parents=True, exist_ok=True) 41 | -------------------------------------------------------------------------------- /mvdatasets/configs/machine_config.py: -------------------------------------------------------------------------------- 1 | from typing import Literal 2 | from dataclasses import dataclass 3 | from mvdatasets.configs.base_config import PrintableConfig 4 | 5 | 6 | @dataclass 7 | class MachineConfig(PrintableConfig): 8 | """Configuration of machine setup""" 9 | 10 | seed: int = 42 11 | """random seed initialization""" 12 | 13 | device: Literal["cpu", "cuda", "mps"] = "cuda" 14 | """device type to use for training""" 15 | 16 | def __post__init__(self): 17 | 18 | import torch 19 | import numpy as np 20 | import random 21 | 22 | # Set a random seed for reproducibility 23 | random.seed(self.seed) 24 | np.random.seed(self.seed) 25 | torch.manual_seed(self.seed) 26 | 27 | # Check if CUDA (GPU support) is available 28 | if "cuda" in self.device: 29 | if not torch.cuda.is_available(): 30 | raise ValueError("CUDA is not available, change device to 'cpu'") 31 | else: 32 | # Set a random seed for GPU 33 | torch.cuda.manual_seed(self.seed) 34 | -------------------------------------------------------------------------------- /mvdatasets/geometry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/mvdatasets/geometry/__init__.py -------------------------------------------------------------------------------- /mvdatasets/geometry/contraction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from typing import Union 4 | 5 | 6 | def contract_points(points, scale=2) -> Union[np.ndarray, torch.Tensor]: 7 | """ 8 | Warping function that smoothly maps all coordinates outside of a ball of radius 0.5 into a ball of radius 1. 9 | From :cite:t:`barron2022mipnerf360`. 10 | 11 | Args: 12 | points: (N, 3) numpy array or torch tensor 13 | scale: float 14 | Returns: 15 | (N, 3) numpy array or torch tensor of contracted points 16 | """ 17 | # compute points norm 18 | if isinstance(points, np.ndarray): 19 | points_norm = np.linalg.norm(points * scale, axis=1)[:, np.newaxis] # (N, 1) 20 | return np.where( 21 | points_norm < 1.0, points, (2 - 1.0 / points_norm) * points / points_norm 22 | ) 23 | elif isinstance(points, torch.Tensor): 24 | points_norm = torch.norm(points * scale, dim=1, keepdim=True) 25 | return torch.where( 26 | points_norm < 1.0, points, (2 - 1.0 / points_norm) * points / points_norm 27 | ) 28 | else: 29 | raise ValueError("points must be a numpy array or a torch tensor") 30 | 31 | 32 | def uncontract_points(points, scale=2) -> Union[np.ndarray, torch.Tensor]: 33 | """ 34 | Inverse of contract_points. 35 | 36 | Args: 37 | points: (N, 3) numpy array or torch tensor 38 | scale: float 39 | Returns: 40 | (N, 3) numpy array or torch tensor of uncontracted points 41 | """ 42 | # compute points norm 43 | if isinstance(points, np.ndarray): 44 | points_norm = np.linalg.norm(points * scale, axis=1)[:, np.newaxis] # (N, 1) 45 | return np.where( 46 | points_norm < 1.0, points, 1.0 / (2 - points_norm) * (points / points_norm) 47 | ) 48 | elif isinstance(points, torch.Tensor): 49 | points_norm = torch.norm(points * scale, dim=1, keepdim=True) 50 | return torch.where( 51 | points_norm < 1.0, points, 1.0 / (2 - points_norm) * (points / points_norm) 52 | ) 53 | else: 54 | raise ValueError("points must be a numpy array or a torch tensor") 55 | -------------------------------------------------------------------------------- /mvdatasets/geometry/interpolation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | def lerp(x0, x1, h): 6 | h = torch.clip(h, 0, 1) 7 | return (1 - h) * x0 + h * x1 8 | 9 | 10 | def slerp(q1, q2, t): 11 | """ 12 | Performs spherical linear interpolation (SLERP) between batches of quaternions q1 and q2. 13 | 14 | Args: 15 | q1: Tensor of shape (N, T, 4) representing the first batch of quaternions. 16 | q2: Tensor of shape (N, T, 4) representing the second batch of quaternions. 17 | t: A scalar or tensor of shape (N, T) or (N, 1) representing the interpolation factors (0 <= t <= 1). 18 | 19 | Returns: 20 | Interpolated quaternion tensor of shape (N, T, 4). 21 | """ 22 | 23 | # Convert to torch tensors if not already 24 | is_numpy = False 25 | if isinstance(q1, np.ndarray) and isinstance(q2, np.ndarray): 26 | q1 = torch.tensor(q1, dtype=torch.float32) 27 | q2 = torch.tensor(q2, dtype=torch.float32) 28 | t = torch.tensor(t, dtype=torch.float32) 29 | is_numpy = True 30 | 31 | # Ensure quaternions are normalized 32 | q1 = q1 / torch.linalg.norm(q1, dim=-1, keepdim=True) 33 | q2 = q2 / torch.linalg.norm(q2, dim=-1, keepdim=True) 34 | 35 | # Compute the dot product (cosine of the angle between the quaternions) 36 | dot_product = torch.sum(q1 * q2, dim=-1, keepdim=True) # Shape: (N, T, 1) 37 | 38 | # If the dot product is negative, negate q2 to take the shortest path 39 | mask = (dot_product < 0.0).squeeze(-1) 40 | q2[mask] = -q2[mask] 41 | 42 | dot_product = torch.abs(dot_product) 43 | 44 | # Clamp dot product to avoid numerical issues (should be in [-1, 1]) 45 | dot_product = torch.clamp(dot_product, -1.0, 1.0) 46 | 47 | # Compute the angle between the two quaternions 48 | theta_0 = torch.acos(dot_product) # Shape: (N, T, 1) 49 | 50 | # If the angle is very small, fall back to LERP (Linear interpolation) 51 | small_angle_mask = (theta_0 < 1e-6).squeeze(-1) 52 | if small_angle_mask.any(): 53 | q_lerp = (1.0 - t[..., None]) * q1 + t[..., None] * q2 54 | q_lerp = q_lerp / torch.linalg.norm(q_lerp, dim=-1, keepdim=True) 55 | q1[small_angle_mask] = q_lerp[small_angle_mask] 56 | return q1 57 | 58 | # Compute sin(theta_0) 59 | sin_theta_0 = torch.sin(theta_0) # Shape: (N, T, 1) 60 | 61 | # Compute the two interpolation terms 62 | s1 = torch.sin((1.0 - t)[..., None] * theta_0) / sin_theta_0 # Shape: (N, T, 1) 63 | s2 = torch.sin(t[..., None] * theta_0) / sin_theta_0 # Shape: (N, T, 1) 64 | 65 | # Compute the interpolated quaternion 66 | q_slerp = s1 * q1 + s2 * q2 # Shape: (N, T, 4) 67 | 68 | # Return the normalized interpolated quaternion 69 | q_slerp = q_slerp / torch.linalg.norm(q_slerp, dim=-1, keepdim=True) 70 | 71 | # Convert back to numpy if necessary 72 | if is_numpy: 73 | q_slerp = q_slerp.cpu().numpy() 74 | 75 | return q_slerp 76 | -------------------------------------------------------------------------------- /mvdatasets/geometry/primitives/__init__.py: -------------------------------------------------------------------------------- 1 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 2 | from mvdatasets.geometry.primitives.bounding_sphere import BoundingSphere 3 | from mvdatasets.geometry.primitives.point_cloud import PointCloud 4 | -------------------------------------------------------------------------------- /mvdatasets/geometry/primitives/point_cloud.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from typing import Optional 3 | from mvdatasets.geometry.rigid import apply_transformation_3d 4 | 5 | 6 | class PointCloud: 7 | 8 | def __init__( 9 | self, 10 | points_3d: np.ndarray, 11 | points_rgb: Optional[np.ndarray] = None, # (N, 3) or (3,) 12 | color: Optional[str] = None, 13 | label: Optional[str] = None, 14 | size: Optional[float] = None, 15 | marker: Optional[str] = None, 16 | ): 17 | self.points_3d = points_3d 18 | self.points_rgb = points_rgb 19 | 20 | if self.points_rgb is not None: 21 | # check if dimensions are correct 22 | if self.points_rgb.ndim == 2: 23 | # first dimension must be the same as points_3d 24 | if self.points_rgb.shape[0] != self.points_3d.shape[0]: 25 | raise ValueError( 26 | f"Points RGB must have the same number of points as points 3D, got {self.points_rgb.shape[0]} and {self.points_3d.shape[0]}" 27 | ) 28 | # second dimension must be 3 29 | if self.points_rgb.shape[1] != 3: 30 | raise ValueError( 31 | f"Points RGB must have shape (N, 3), got {self.points_rgb.shape}" 32 | ) 33 | elif self.points_rgb.ndim == 1: 34 | # first dimension must be 3 35 | if self.points_rgb.shape[0] != 3: 36 | raise ValueError( 37 | f"Points RGB must have shape (3,), got {self.points_rgb.shape}" 38 | ) 39 | else: 40 | raise ValueError( 41 | f"Points RGB must have shape (N, 3) or (3,), got {self.points_rgb.shape}" 42 | ) 43 | 44 | # plotting attributes 45 | self.color = color 46 | self.label = label 47 | self.size = size 48 | self.marker = marker 49 | 50 | def downsample(self, nr_points: int): 51 | if nr_points >= self.points_3d.shape[0]: 52 | # do nothing 53 | return 54 | 55 | idxs = np.random.choice(self.points_3d.shape[0], nr_points, replace=False) 56 | self.points_3d = self.points_3d[idxs] 57 | 58 | if self.points_rgb is not None: 59 | self.points_rgb = self.points_rgb[idxs] 60 | 61 | def mask(self, mask: np.ndarray): 62 | self.points_3d = self.points_3d[mask] 63 | 64 | if self.points_rgb is not None: 65 | self.points_rgb = self.points_rgb[mask] 66 | 67 | def shape(self): 68 | return self.points_3d.shape 69 | 70 | def __str__(self) -> str: 71 | return f"PointCloud with {self.points_3d.shape[0]} points" 72 | 73 | def transform(self, transformation: np.ndarray): 74 | self.points_3d = apply_transformation_3d(self.points_3d, transformation) 75 | -------------------------------------------------------------------------------- /mvdatasets/io/__init__.py: -------------------------------------------------------------------------------- 1 | from mvdatasets.io.yaml_utils import load_yaml, save_yaml 2 | -------------------------------------------------------------------------------- /mvdatasets/io/yaml_utils.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from pathlib import Path 3 | 4 | 5 | def save_yaml(data_dict: dict, file_path: Path): 6 | """ 7 | Save function. Convert Paths to strings. 8 | 9 | Args: 10 | data: Data to save. 11 | file_path: Path to save the data. 12 | """ 13 | # Convert Path objects to strings 14 | serializable_data = { 15 | key: str(value) if isinstance(value, Path) else value 16 | for key, value in data_dict.items() 17 | } 18 | # Save to YAML file 19 | with open(file_path, "w") as f: 20 | yaml.dump(serializable_data, f) 21 | 22 | 23 | def load_yaml(file_path: Path) -> dict: 24 | """ 25 | Load function. Convert strings whose key ends with '_path' back to Paths. 26 | 27 | Args: 28 | file_path: Path to load the data. 29 | Returns: 30 | Loaded deserialized_data data as a dictionary. 31 | """ 32 | with open(file_path, "r") as f: 33 | loaded_data = yaml.safe_load(f) 34 | # Convert strings back to Path objects 35 | data_dict = { 36 | key: Path(value) if key.endswith("_path") else value 37 | for key, value in loaded_data.items() 38 | } 39 | return data_dict 40 | -------------------------------------------------------------------------------- /mvdatasets/loaders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/mvdatasets/loaders/__init__.py -------------------------------------------------------------------------------- /mvdatasets/loaders/dynamic/nerfies.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | import numpy as np 3 | from pathlib import Path 4 | import os 5 | import json 6 | from PIL import Image 7 | from tqdm import tqdm 8 | from mvdatasets.utils.images import image_to_numpy 9 | from mvdatasets import Camera 10 | from mvdatasets.geometry.primitives.point_cloud import PointCloud 11 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 12 | from mvdatasets.utils.printing import print_error, print_warning, print_success 13 | from mvdatasets.utils.loader_utils import rescale 14 | from mvdatasets.geometry.common import rot_euler_3d_deg 15 | 16 | 17 | def load( 18 | dataset_path: Path, 19 | scene_name: str, 20 | config: dict, 21 | verbose: bool = False, 22 | ): 23 | """nerfies data format loader. 24 | 25 | Args: 26 | dataset_path (Path): Path to the dataset folder. 27 | scene_name (str): Name of the scene / sequence to load. 28 | splits (list): Splits to load (e.g., ["train", "val"]). 29 | config (DatasetConfig): Dataset configuration parameters. 30 | verbose (bool, optional): Whether to print debug information. Defaults to False. 31 | 32 | Returns: 33 | dict: Dictionary of splits with lists of Camera objects. 34 | np.ndarray: Global transform (4, 4) 35 | str: Scene type 36 | List[PointCloud]: List of PointClouds 37 | float: Minimum camera distance 38 | float: Maximum camera distance 39 | float: Foreground scale multiplier 40 | float: Scene radius 41 | int: Number of frames per camera 42 | int: Number of sequence frames 43 | float: Frames per second 44 | """ 45 | 46 | scene_path = dataset_path / scene_name 47 | splits = config["splits"] 48 | 49 | # Valid values for specific keys 50 | valid_values = { 51 | "subsample_factor": [1, 2], 52 | } 53 | 54 | # Validate specific keys 55 | for key, valid in valid_values.items(): 56 | if key in config and config[key] not in valid: 57 | raise ValueError(f"{key} {config[key]} must be a value in {valid}") 58 | 59 | # Debugging output 60 | if verbose: 61 | print("config:") 62 | for k, v in config.items(): 63 | print(f"\t{k}: {v}") 64 | 65 | # ------------------------------------------------------------------------- 66 | 67 | raise NotImplementedError("nerfies loader is not implemented yet") 68 | -------------------------------------------------------------------------------- /mvdatasets/loaders/dynamic/neu3d.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | import numpy as np 3 | from pathlib import Path 4 | import os 5 | import json 6 | from PIL import Image 7 | from tqdm import tqdm 8 | from mvdatasets.utils.images import image_to_numpy 9 | from mvdatasets import Camera 10 | from mvdatasets.geometry.primitives.point_cloud import PointCloud 11 | from mvdatasets.geometry.primitives.bounding_box import BoundingBox 12 | from mvdatasets.utils.printing import print_error, print_warning, print_success 13 | from mvdatasets.utils.loader_utils import rescale 14 | from mvdatasets.geometry.common import rot_euler_3d_deg 15 | from mvdatasets.geometry.quaternions import quats_to_rots 16 | 17 | 18 | def load( 19 | dataset_path: Path, 20 | scene_name: str, 21 | config: dict, 22 | verbose: bool = False, 23 | ): 24 | """neu3d data format loader. 25 | 26 | Args: 27 | dataset_path (Path): Path to the dataset folder. 28 | scene_name (str): Name of the scene / sequence to load. 29 | splits (list): Splits to load (e.g., ["train", "val"]). 30 | config (DatasetConfig): Dataset configuration parameters. 31 | verbose (bool, optional): Whether to print debug information. Defaults to False. 32 | 33 | Returns: 34 | dict: Dictionary of splits with lists of Camera objects. 35 | np.ndarray: Global transform (4, 4) 36 | str: Scene type 37 | List[PointCloud]: List of PointClouds 38 | float: Minimum camera distance 39 | float: Maximum camera distance 40 | float: Foreground scale multiplier 41 | float: Scene radius 42 | int: Number of frames per camera 43 | int: Number of sequence frames 44 | float: Frames per second 45 | """ 46 | 47 | scene_path = dataset_path / scene_name 48 | splits = config["splits"] 49 | 50 | # Valid values for specific keys 51 | valid_values = {} 52 | 53 | # Validate specific keys 54 | for key, valid in valid_values.items(): 55 | if key in config and config[key] not in valid: 56 | raise ValueError(f"{key} {config[key]} must be a value in {valid}") 57 | 58 | # Debugging output 59 | if verbose: 60 | print("config:") 61 | for k, v in config.items(): 62 | print(f"\t{k}: {v}") 63 | 64 | # ------------------------------------------------------------------------- 65 | 66 | # load poses 67 | # poses_bounds.npy 68 | poses_bounds_path = scene_path / "poses_bounds.npy" 69 | if not poses_bounds_path.exists(): 70 | raise ValueError(f"File {poses_bounds_path} does not exist.") 71 | 72 | poses_bounds = np.load(poses_bounds_path) 73 | print(poses_bounds) 74 | 75 | # camera poses are in OpenGL format 76 | # Poses are stored as 3x4 numpy arrays that represent camera-to-world 77 | # transformation matrices. The other data you will need is simple pinhole 78 | # camera intrinsics (hwf = [height, width, focal length]) and near/far scene bounds 79 | 80 | poses_list = [] 81 | intrinsics_list = [] 82 | for poses_bound in poses_bounds: 83 | c2w_ogl = poses_bound[: 3 * 4].reshape(3, 4) # :12 84 | height = poses_bound[3 * 4] # 12 85 | width = poses_bound[3 * 4 + 1] # 13 86 | focal_length = poses_bound[3 * 4 + 2] # 14 87 | near = poses_bound[3 * 4 + 3] # 15 88 | far = poses_bound[3 * 4 + 4] # 16 89 | print("c2w", c2w_ogl) 90 | print("height", height) 91 | print("width", width) 92 | print("focal_length", focal_length) 93 | print("near", near) 94 | print("far", far) 95 | # poses_list.append(poses_bound[:3, :4]) 96 | # intrinsics_list.append(poses_bound[3:]) 97 | 98 | # find all cam00.mp4 cameras feeds 99 | cam_videos = list(scene_path.glob("cam*.mp4")) 100 | cam_videos = sorted(cam_videos) 101 | 102 | # check if frames where already extracted 103 | for cam_video in cam_videos: 104 | video_path = scene_path / cam_video 105 | frames_folder = scene_path / cam_video.stem 106 | if not frames_folder.exists(): 107 | print_warning(f"Frames folder {frames_folder} does not exist.") 108 | # extract frames 109 | from mvdatasets.utils.video_utils import extract_frames 110 | 111 | extract_frames( 112 | video_path=video_path, 113 | subsample_factor=1, 114 | ext="jpg", 115 | ) 116 | print_success(f"Frames extracted to {frames_folder}") 117 | 118 | # TODO: complete implementation 119 | 120 | raise NotImplementedError("nerfies loader is not implemented yet") 121 | -------------------------------------------------------------------------------- /mvdatasets/loaders/static/dmsr.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | from pathlib import Path 3 | import os 4 | import json 5 | import numpy as np 6 | from PIL import Image 7 | from tqdm import tqdm 8 | from mvdatasets import Camera 9 | from mvdatasets.utils.images import image_to_numpy 10 | from mvdatasets.utils.loader_utils import rescale 11 | from mvdatasets.geometry.common import rot_euler_3d_deg 12 | from mvdatasets.utils.printing import print_error, print_warning, print_success 13 | 14 | 15 | def load( 16 | dataset_path: Path, 17 | scene_name: str, 18 | config: dict, 19 | verbose: bool = False, 20 | ): 21 | """DMSR data format loader. 22 | 23 | Args: 24 | dataset_path (Path): Path to the dataset folder. 25 | scene_name (str): Name of the scene / sequence to load. 26 | splits (list): Splits to load (e.g., ["train", "val"]). 27 | config (DatasetConfig): Dataset configuration parameters. 28 | verbose (bool, optional): Whether to print debug information. Defaults to False. 29 | 30 | Returns: 31 | dict: Dictionary of splits with lists of Camera objects. 32 | np.ndarray: Global transform (4, 4) 33 | str: Scene type 34 | List[PointCloud]: List of PointClouds 35 | float: Minimum camera distance 36 | float: Maximum camera distance 37 | float: Foreground scale multiplier 38 | float: Scene radius 39 | """ 40 | 41 | scene_path = dataset_path / scene_name 42 | splits = config["splits"] 43 | 44 | # Valid values for specific keys 45 | valid_values = {} 46 | 47 | # Validate specific keys 48 | for key, valid in valid_values.items(): 49 | if key in config and config[key] not in valid: 50 | raise ValueError(f"{key} {config[key]} must be a value in {valid}") 51 | 52 | # Debugging output 53 | if verbose: 54 | print("config:") 55 | for k, v in config.items(): 56 | print(f"\t{k}: {v}") 57 | 58 | # ------------------------------------------------------------------------- 59 | 60 | # read all poses 61 | poses_all = [] 62 | for split in ["train", "test"]: 63 | # load current split transforms 64 | with open(os.path.join(scene_path, split, "transforms.json"), "r") as fp: 65 | metas = json.load(fp) 66 | 67 | for frame in metas["frames"]: 68 | camera_pose = frame["transform_matrix"] 69 | poses_all.append(camera_pose) 70 | 71 | # rescale (optional) 72 | scene_radius_mult, min_camera_distance, max_camera_distance = rescale( 73 | poses_all, to_distance=config["max_cameras_distance"] 74 | ) 75 | 76 | scene_radius = max_camera_distance 77 | 78 | # global transform 79 | global_transform = np.eye(4) 80 | # rotate and scale 81 | rot = rot_euler_3d_deg( 82 | config["rotate_deg"][0], config["rotate_deg"][1], config["rotate_deg"][2] 83 | ) 84 | global_transform[:3, :3] = scene_radius_mult * rot 85 | 86 | # local transform 87 | local_transform = np.eye(4) 88 | local_transform[:3, :3] = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) 89 | 90 | # cameras objects 91 | height, width = None, None 92 | cameras_splits = {} 93 | for split in splits: 94 | cameras_splits[split] = [] 95 | 96 | # load current split transforms 97 | with open(os.path.join(scene_path, split, "transforms.json"), "r") as fp: 98 | metas = json.load(fp) 99 | 100 | camera_angle_x = metas["camera_angle_x"] 101 | 102 | # load images to cpu as numpy arrays 103 | frames_list = [] 104 | 105 | for frame in metas["frames"]: 106 | img_path = frame["file_path"].split("/")[-1] + ".png" 107 | camera_pose = frame["transform_matrix"] 108 | frames_list.append((img_path, camera_pose)) 109 | frames_list.sort(key=lambda x: int(x[0].split(".")[0].split("_")[-1])) 110 | 111 | if split == "test": 112 | # skip every test_skip images 113 | test_skip = config["test_skip"] 114 | frames_list = frames_list[::test_skip] 115 | 116 | # read H, W 117 | if height is None or width is None: 118 | frame = frames_list[0] 119 | im_name = frame[0] 120 | # load PIL image 121 | img_pil = Image.open(os.path.join(scene_path, f"{split}", "rgbs", im_name)) 122 | img_np = image_to_numpy(img_pil, use_uint8=True) 123 | height, width = img_np.shape[:2] 124 | 125 | # iterate over images and load them 126 | pbar = tqdm(frames_list, desc=split, ncols=100) 127 | for frame in pbar: 128 | # get image name 129 | im_name = frame[0] 130 | # camera_pose = frame[1] 131 | 132 | if config["pose_only"]: 133 | cam_imgs = None 134 | else: 135 | # load PIL image 136 | img_pil = Image.open( 137 | os.path.join(scene_path, f"{split}", "rgbs", im_name) 138 | ) 139 | img_np = image_to_numpy(img_pil, use_uint8=True) 140 | # remove alpha (it is always 1) 141 | img_np = img_np[:, :, :3] 142 | # get images 143 | cam_imgs = img_np[None, ...] 144 | # depth_imgs = depth_np[None, ...] 145 | 146 | # im_name = im_name.replace('r', 'd') 147 | # depth_pil = Image.open(os.path.join(scene_path, f"{split}", "depth", im_name)) 148 | # depth_np = image_to_numpy(depth_pil)[..., None] 149 | 150 | # get frame idx and pose 151 | idx = int(frame[0].split(".")[0].split("_")[-1]) 152 | 153 | pose = np.array(frame[1], dtype=np.float32) 154 | intrinsics = np.eye(3, dtype=np.float32) 155 | focal_length = 0.5 * width / np.tan(0.5 * camera_angle_x) 156 | intrinsics[0, 0] = focal_length 157 | intrinsics[1, 1] = focal_length 158 | intrinsics[0, 2] = width / 2.0 159 | intrinsics[1, 2] = height / 2.0 160 | 161 | camera = Camera( 162 | intrinsics=intrinsics, 163 | pose=pose, 164 | global_transform=global_transform, 165 | local_transform=local_transform, 166 | rgbs=cam_imgs, 167 | # depths=depth_imgs, 168 | masks=None, # dataset has no masks 169 | camera_label=str(idx), 170 | width=width, 171 | height=height, 172 | subsample_factor=int(config["subsample_factor"]), 173 | # verbose=verbose, 174 | ) 175 | 176 | cameras_splits[split].append(camera) 177 | 178 | return { 179 | "scene_type": config["scene_type"], 180 | "init_sphere_radius_mult": config["init_sphere_radius_mult"], 181 | "foreground_scale_mult": config["foreground_scale_mult"], 182 | "cameras_splits": cameras_splits, 183 | "global_transform": global_transform, 184 | "min_camera_distance": min_camera_distance, 185 | "max_camera_distance": max_camera_distance, 186 | "scene_radius": scene_radius, 187 | } 188 | -------------------------------------------------------------------------------- /mvdatasets/utils/.deprecated/data_converter.py: -------------------------------------------------------------------------------- 1 | def convert_to_ingp_format(mv_data, outdir_path): 2 | 3 | # "camera_angle_x" 4 | # "camera_angle_y" 5 | 6 | pass 7 | 8 | 9 | def convert_to_dtu_format(mv_data, outdir_path): 10 | 11 | # "camera_angle_x" 12 | # "camera_angle_y" 13 | 14 | pass 15 | -------------------------------------------------------------------------------- /mvdatasets/utils/.deprecated/loader.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import Sampler 3 | 4 | 5 | class DatasetSampler(Sampler): 6 | """Custom sampler for the dataset. 7 | Args: 8 | dataset (Dataset): dataset to sample from 9 | shuffle (bool): shuffle the data or not 10 | """ 11 | 12 | def __init__(self, dataset, shuffle=False): 13 | self.dataset = dataset 14 | self.nr_samples = len(dataset) 15 | self.shuffle = shuffle 16 | 17 | def __iter__(self): 18 | if self.shuffle: 19 | return iter(torch.randperm(self.nr_samples)) 20 | else: 21 | return iter(torch.arange(self.nr_samples)) 22 | 23 | def __len__(self): 24 | return self.nr_samples 25 | 26 | 27 | def custom_collate(batch): 28 | idx, rays_o, rays_d, gt_rgb, gt_mask, frame_idx = zip(*batch) 29 | return ( 30 | torch.stack(idx), 31 | torch.stack(rays_o), 32 | torch.stack(rays_d), 33 | torch.stack(gt_rgb), 34 | torch.stack(gt_mask), 35 | torch.stack(frame_idx), 36 | ) 37 | 38 | 39 | def get_next_batch(data_loader): 40 | """calls next on the data_loader and returns a batch of data""" 41 | idx, rays_o, rays_d, gt_rgb, gt_mask, frame_idx = next(iter(data_loader)) 42 | idx = idx.repeat_interleave(data_loader.dataset.per_camera_rays_batch_size) 43 | rays_o = rays_o.view(-1, 3) 44 | rays_d = rays_d.view(-1, 3) 45 | gt_rgb = gt_rgb.view(-1, 3) 46 | gt_mask = gt_mask.view(-1, 1) 47 | frame_idx = frame_idx.repeat_interleave( 48 | data_loader.dataset.per_camera_rays_batch_size 49 | ) 50 | 51 | # print("idx", idx.shape, idx.device) 52 | # print("rays_o", rays_o.shape, rays_o.device) 53 | # print("rays_d", rays_d.shape, rays_d.device) 54 | # print("gt_rgb", gt_rgb.shape, gt_rgb.device) 55 | # print("gt_mask", gt_mask.shape, gt_mask.device) 56 | # print("frame_idx", frame_idx.shape, frame_idx.device) 57 | 58 | if data_loader.sampler.shuffle: 59 | rand_perm = torch.randperm(idx.shape[0], device=rays_o.device) 60 | idx = idx[rand_perm] 61 | rays_o = rays_o[rand_perm] 62 | rays_d = rays_d[rand_perm] 63 | gt_rgb = gt_rgb[rand_perm] 64 | gt_mask = gt_mask[rand_perm] 65 | frame_idx = frame_idx[rand_perm] 66 | 67 | return idx, rays_o, rays_d, gt_rgb, gt_mask, frame_idx 68 | -------------------------------------------------------------------------------- /mvdatasets/utils/.deprecated/pycolmap.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from PIL import Image 4 | from mvdatasets.geometry.common import qvec2rotmat 5 | from mvdatasets.utils.images import image_to_numpy 6 | 7 | 8 | def read_points3D(reconstruction): 9 | point_cloud = [] 10 | for point3D_id, point3D in reconstruction.points3D.items(): 11 | point_cloud.append(point3D.xyz) 12 | point_cloud = np.array(point_cloud) 13 | return point_cloud 14 | 15 | 16 | def read_cameras_params(reconstruction): 17 | 18 | cameras_params = {} 19 | for camera_id, camera in reconstruction.cameras.items(): 20 | params = {} 21 | # PINHOLE 22 | if camera.model_id == 1: 23 | params["fx"] = camera.params[0] 24 | params["fy"] = camera.params[1] # fy 25 | params["cx"] = camera.params[2] # cx 26 | params["cy"] = camera.params[3] # cy 27 | # # SIMPLE_RADIAL 28 | # elif camera.model_id == 2: 29 | # intrinsics[0, 0] = camera.params[0] # fx 30 | # intrinsics[1, 1] = camera.params[0] # fy = fx 31 | # intrinsics[0, 2] = camera.params[1] # cx 32 | # intrinsics[1, 2] = camera.params[2] # cy 33 | # # camera.params[3] # k1 34 | else: 35 | raise ValueError(f"camera model {camera.model_id} not implemented.") 36 | cameras_params[str(camera_id)] = params 37 | return cameras_params 38 | 39 | 40 | def read_cameras(reconstruction, images_path): 41 | 42 | cameras_params = read_cameras_params(reconstruction) 43 | 44 | cameras_meta = [] 45 | for image_id, image in reconstruction.images.items(): 46 | 47 | rotation = qvec2rotmat(image.qvec) 48 | translation = image.tvec 49 | 50 | params = cameras_params[str(image.camera_id)] 51 | 52 | # load PIL image 53 | img_pil = Image.open(os.path.join(images_path, image.name)) 54 | img_np = image_to_numpy(img_pil, use_uint8=True) 55 | 56 | cameras_meta.append( 57 | { 58 | "id": image_id, 59 | "rotation": rotation, 60 | "translation": translation, 61 | "params": params, 62 | "img": img_np, 63 | } 64 | ) 65 | 66 | # order by "id" 67 | cameras_meta = sorted(cameras_meta, key=lambda x: x["id"]) 68 | 69 | return cameras_meta 70 | -------------------------------------------------------------------------------- /mvdatasets/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from mvdatasets.utils.profiler import Profiler 2 | -------------------------------------------------------------------------------- /mvdatasets/utils/camera_utils.py: -------------------------------------------------------------------------------- 1 | # FROM: gsplat 2 | # undistortion 3 | # self.mapx_dict = dict() 4 | # self.mapy_dict = dict() 5 | # self.roi_undist_dict = dict() 6 | # for camera_id in self.params_dict.keys(): 7 | # params = self.params_dict[camera_id] 8 | # if len(params) == 0: 9 | # continue # no distortion 10 | # assert camera_id in self.Ks_dict, f"Missing K for camera {camera_id}" 11 | # assert ( 12 | # camera_id in self.params_dict 13 | # ), f"Missing params for camera {camera_id}" 14 | # K = self.Ks_dict[camera_id] 15 | # width, height = self.imsize_dict[camera_id] 16 | 17 | # if camtype == "perspective": 18 | # K_undist, roi_undist = cv2.getOptimalNewCameraMatrix( 19 | # K, params, (width, height), 0 20 | # ) 21 | # mapx, mapy = cv2.initUndistortRectifyMap( 22 | # K, params, None, K_undist, (width, height), cv2.CV_32FC1 23 | # ) 24 | # mask = None 25 | # elif camtype == "fisheye": 26 | # fx = K[0, 0] 27 | # fy = K[1, 1] 28 | # cx = K[0, 2] 29 | # cy = K[1, 2] 30 | # grid_x, grid_y = np.meshgrid( 31 | # np.arange(width, dtype=np.float32), 32 | # np.arange(height, dtype=np.float32), 33 | # indexing="xy", 34 | # ) 35 | # x1 = (grid_x - cx) / fx 36 | # y1 = (grid_y - cy) / fy 37 | # theta = np.sqrt(x1**2 + y1**2) 38 | # r = ( 39 | # 1.0 40 | # + params[0] * theta**2 41 | # + params[1] * theta**4 42 | # + params[2] * theta**6 43 | # + params[3] * theta**8 44 | # ) 45 | # mapx = fx * x1 * r + width // 2 46 | # mapy = fy * y1 * r + height // 2 47 | 48 | # # Use mask to define ROI 49 | # mask = np.logical_and( 50 | # np.logical_and(mapx > 0, mapy > 0), 51 | # np.logical_and(mapx < width - 1, mapy < height - 1), 52 | # ) 53 | # y_indices, x_indices = np.nonzero(mask) 54 | # y_min, y_max = y_indices.min(), y_indices.max() + 1 55 | # x_min, x_max = x_indices.min(), x_indices.max() + 1 56 | # mask = mask[y_min:y_max, x_min:x_max] 57 | # K_undist = K.copy() 58 | # K_undist[0, 2] -= x_min 59 | # K_undist[1, 2] -= y_min 60 | # roi_undist = [x_min, y_min, x_max - x_min, y_max - y_min] 61 | # else: 62 | # assert_never(camtype) 63 | 64 | # self.mapx_dict[camera_id] = mapx 65 | # self.mapy_dict[camera_id] = mapy 66 | # self.Ks_dict[camera_id] = K_undist 67 | # self.roi_undist_dict[camera_id] = roi_undist 68 | # self.imsize_dict[camera_id] = (roi_undist[2], roi_undist[3]) 69 | # self.mask_dict[camera_id] = mask 70 | -------------------------------------------------------------------------------- /mvdatasets/utils/loader_utils.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple, Optional 2 | import numpy as np 3 | 4 | 5 | def get_min_max_cameras_distances(poses: list) -> tuple: 6 | """ 7 | return maximum pose distance from origin 8 | 9 | Args: 10 | poses (list): list of numpy (4, 4) poses 11 | 12 | Returns: 13 | min_dist (float): minumum camera distance from origin 14 | max_dist (float): maximum camera distance from origin 15 | """ 16 | if len(poses) == 0: 17 | raise ValueError("poses list empty") 18 | 19 | # get all camera centers 20 | camera_centers = np.stack(poses, 0)[:, :3, 3] 21 | camera_distances_from_origin = np.linalg.norm(camera_centers, axis=1) 22 | 23 | min_dist = np.min(camera_distances_from_origin) 24 | max_dist = np.max(camera_distances_from_origin) 25 | 26 | return min_dist, max_dist 27 | 28 | 29 | def rescale( 30 | all_poses: List[np.ndarray], to_distance: Optional[float] = None 31 | ) -> Tuple[float, float, float]: 32 | """returns a scaling factor for the scene such that the furthest camera is at target distance 33 | 34 | Args: 35 | all_poses (List[np.ndarray]): list of camera poses 36 | to_distance (float, optional): maximum distance of the furthest camera from the origin. Defaults to None. 37 | 38 | Returns: 39 | float: scaling factor 40 | float: distance of rescaled closest camera from the origin 41 | float: distance of rescaled furthest camera from the origin 42 | """ 43 | # init multiplier 44 | scene_radius_mult = 1.0 45 | 46 | # find scene radius 47 | min_camera_distance, max_camera_distance = get_min_max_cameras_distances(all_poses) 48 | 49 | if to_distance is None: 50 | return scene_radius_mult, min_camera_distance, max_camera_distance 51 | 52 | # scene scale such that furthest away camera is at target distance 53 | scene_radius_mult = to_distance / max_camera_distance 54 | 55 | # new scene scale 56 | min_camera_distance = min_camera_distance * scene_radius_mult 57 | max_camera_distance = max_camera_distance * scene_radius_mult 58 | 59 | return scene_radius_mult, min_camera_distance, max_camera_distance 60 | -------------------------------------------------------------------------------- /mvdatasets/utils/memory.py: -------------------------------------------------------------------------------- 1 | def bytes_to_gb(n_bytes: int) -> float: 2 | return n_bytes / 1e9 3 | 4 | 5 | def bytes_to_mb(n_bytes: int) -> float: 6 | return n_bytes / 1e6 7 | -------------------------------------------------------------------------------- /mvdatasets/utils/mesh.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import open3d as o3d 4 | from rich import print 5 | from mvdatasets.utils.texture import SHTexture 6 | from mvdatasets.utils.printing import print_log 7 | 8 | 9 | def triangle_to_vertices_uvs_conversion(triangle_uvs, faces, vertices): 10 | flat_faces = faces.flatten() 11 | unique_values, unique_indices = np.unique(flat_faces, return_index=True) 12 | vertices_uvs = np.zeros((vertices.shape[0], 2), dtype=np.float32) 13 | vertices_uvs[flat_faces[unique_indices]] = triangle_uvs[unique_indices] 14 | return vertices_uvs 15 | 16 | 17 | class Mesh: 18 | def __init__( 19 | self, 20 | o3d_mesh=None, 21 | mesh_meta=None, 22 | verbose=True, 23 | ): 24 | # only one of o3d_mesh and mesh_meta should be provided 25 | if o3d_mesh is not None and mesh_meta is not None: 26 | raise ValueError("only one of o3d_mesh and mesh_meta should be provided") 27 | 28 | # at least one of o3d_mesh and mesh_meta should be provided 29 | if o3d_mesh is None and mesh_meta is None: 30 | raise ValueError( 31 | "at least one of o3d_mesh and mesh_meta should be provided" 32 | ) 33 | 34 | if o3d_mesh is not None: 35 | 36 | mesh = o3d_mesh 37 | textures_meta = [] 38 | 39 | else: 40 | 41 | self.mesh_path = mesh_meta["mesh_path"] 42 | textures_meta = mesh_meta.get("textures", []) 43 | 44 | # make sure mesh_path exists 45 | assert os.path.exists( 46 | self.mesh_path 47 | ), f"mesh path {self.mesh_path} does not exist" 48 | 49 | # load mesh 50 | mesh = o3d.io.read_triangle_mesh(self.mesh_path, print_progress=False) 51 | 52 | mesh.compute_vertex_normals() 53 | 54 | # vertices 55 | self.vertices = np.asarray(mesh.vertices).astype(np.float32) 56 | if verbose: 57 | print(f"[bold blue]INFO[/bold blue]: mesh vertices: {self.vertices.shape}") 58 | 59 | # faces 60 | self.faces = np.asarray(mesh.triangles).astype(np.int32) 61 | if verbose: 62 | print(f"[bold blue]INFO[/bold blue]: mesh faces: {self.faces.shape}") 63 | 64 | # normals 65 | self.normals = np.asarray(mesh.vertex_normals).astype(np.float32) 66 | if verbose: 67 | print(f"[bold blue]INFO[/bold blue]: mesh normals: {self.normals.shape}") 68 | 69 | # uvs (if any) 70 | triangle_uvs = np.asarray(mesh.triangle_uvs).astype(np.float32) 71 | if triangle_uvs.shape[0] == 0: 72 | self.vertices_uvs = None 73 | self.has_uvs = False 74 | else: 75 | self.vertices_uvs = triangle_to_vertices_uvs_conversion( 76 | triangle_uvs, 77 | self.faces, 78 | self.vertices, 79 | ) 80 | self.has_uvs = True 81 | if verbose: 82 | print( 83 | f"[bold blue]INFO[/bold blue]: mesh vertices uvs: {self.vertices_uvs.shape if self.vertices_uvs is not None else None}" 84 | ) 85 | 86 | # load texture 87 | if len(textures_meta) > 0: 88 | assert self.has_uvs, "mesh must have uvs to load texture" 89 | # TODO: support RGBATexture 90 | # if len(textures_meta) == 1: 91 | # print_log("loading RGBATexture") 92 | # self.texture = RGBATexture(textures_meta[0], verbose=verbose) 93 | # else: 94 | print_log("loading SHTexture") 95 | self.texture = SHTexture(textures_meta, verbose=verbose) 96 | self.has_texture = True 97 | else: 98 | self.texture = None 99 | self.has_texture = False 100 | 101 | # sample a random color for each mesh face (for visualization) 102 | self.faces_color = np.random.rand(self.faces.shape[0], 3).astype(np.float32) 103 | 104 | @property 105 | def uv_idx(self): 106 | return self.faces 107 | 108 | @property 109 | def uv(self): 110 | return self.vertices_uvs 111 | -------------------------------------------------------------------------------- /mvdatasets/utils/open3d_rendering.py: -------------------------------------------------------------------------------- 1 | import open3d as o3d 2 | import numpy as np 3 | 4 | 5 | def render_o3d_mesh_normals(camera, o3d_mesh): 6 | return render_o3d_mesh(camera, o3d_mesh)["normals"] 7 | 8 | 9 | def render_o3d_mesh_depth(camera, o3d_mesh): 10 | return render_o3d_mesh(camera, o3d_mesh)["depth"] 11 | 12 | 13 | def render_o3d_mesh(camera, o3d_mesh): 14 | """render and open3d mesh with open3d raycasting from camera""" 15 | 16 | # gen rays 17 | rays_o, rays_d, _ = camera.get_rays() 18 | rays_o = rays_o.cpu().numpy() 19 | rays_d = rays_d.cpu().numpy() 20 | rays = o3d.core.Tensor( 21 | np.concatenate([rays_o, rays_d], axis=1), 22 | dtype=o3d.core.Dtype.Float32, 23 | ) 24 | 25 | # setup scene 26 | scene = o3d.t.geometry.RaycastingScene() 27 | scene.add_triangles(o3d.t.geometry.TriangleMesh.from_legacy(o3d_mesh)) 28 | 29 | ans = scene.cast_rays(rays) 30 | 31 | hits = np.logical_not(np.isinf(ans["t_hit"].numpy())) 32 | hits = hits.reshape(camera.height, camera.width)[..., None] 33 | 34 | depth = ans["t_hit"].numpy() 35 | depth[np.isinf(depth)] = 0.0 36 | depth = depth.reshape(camera.height, camera.width)[..., None] 37 | 38 | normals = ans["primitive_normals"].numpy() 39 | normals = (normals + 1.0) * 0.5 40 | normals = normals.reshape(camera.height, camera.width, 3) 41 | normals = normals * hits 42 | 43 | return {"hits": hits, "depth": depth, "normals": normals} 44 | 45 | 46 | def render_o3d_scene(camera, o3d_scene): 47 | """render and open3d mesh with open3d raycasting from camera""" 48 | 49 | # gen rays 50 | rays_o, rays_d, _ = camera.get_rays() 51 | rays_o = rays_o.cpu().numpy() 52 | rays_d = rays_d.cpu().numpy() 53 | rays = o3d.core.Tensor( 54 | np.concatenate([rays_o, rays_d], axis=1), 55 | dtype=o3d.core.Dtype.Float32, 56 | ) 57 | 58 | ans = o3d_scene.cast_rays(rays) 59 | 60 | hits = np.logical_not(np.isinf(ans["t_hit"].numpy())) 61 | hits = hits.reshape(camera.height, camera.width)[..., None] 62 | 63 | depth = ans["t_hit"].numpy() 64 | depth[np.isinf(depth)] = 0.0 65 | depth = depth.reshape(camera.height, camera.width)[..., None] 66 | 67 | normals = ans["primitive_normals"].numpy() 68 | normals = (normals + 1.0) * 0.5 69 | normals = normals.reshape(camera.height, camera.width, 3) 70 | normals = normals * hits 71 | 72 | geom_ids = ans["geometry_ids"].numpy() 73 | geom_ids = geom_ids.reshape(camera.height, camera.width, 1) 74 | 75 | return {"hits": hits, "depth": depth, "normals": normals, "geom_ids": geom_ids} 76 | -------------------------------------------------------------------------------- /mvdatasets/utils/point_clouds.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | import open3d as o3d 3 | import os 4 | import numpy as np 5 | from mvdatasets.geometry.primitives.point_cloud import PointCloud 6 | 7 | 8 | def load_point_cloud(point_cloud_path, max_nr_points=None, verbose=False): 9 | """Loads point cloud from file. 10 | If the file is a mesh, points are its vertices. 11 | 12 | Args: 13 | point_cloud_path: point cloud file path 14 | max_nr_points: maximum number of points to load 15 | 16 | Returns: 17 | points_3d (N, 3): numpy array 18 | """ 19 | 20 | # if exists, load it 21 | if os.path.exists(point_cloud_path): 22 | # if format is .ply or .obj 23 | if point_cloud_path.endswith(".ply") or point_cloud_path.endswith(".obj"): 24 | if verbose: 25 | print("loading point cloud from {}".format(point_cloud_path)) 26 | point_cloud = o3d.io.read_point_cloud(point_cloud_path) 27 | points_3d = np.asarray(point_cloud.points) 28 | if max_nr_points is not None and points_3d.shape[0] > max_nr_points: 29 | # downsample 30 | random_idx = np.random.choice( 31 | points_3d.shape[0], max_nr_points, replace=False 32 | ) 33 | points_3d = points_3d[random_idx] 34 | if verbose: 35 | print("loaded {} points".format(points_3d.shape[0])) 36 | return points_3d 37 | else: 38 | raise ValueError("unsupported point cloud format") 39 | else: 40 | raise ValueError("point cloud path {} does not exist".format(point_cloud_path)) 41 | 42 | 43 | def load_point_clouds(point_clouds_paths, max_nr_points=10000, verbose=False): 44 | """Loads point cloud from files. 45 | If the file is a mesh, points are its vertices. 46 | 47 | Args: 48 | point_clouds_paths: ordered list of point cloud file paths 49 | max_nr_points: maximum number of points to load 50 | 51 | Returns: 52 | point_clouds []: ordered list of (N, 3) numpy arrays 53 | """ 54 | 55 | point_clouds = [] 56 | for pc_path in point_clouds_paths: 57 | points_3d = load_point_cloud(pc_path, max_nr_points, verbose=verbose) 58 | point_cloud = PointCloud(points_3d) 59 | point_clouds.append(point_cloud) 60 | return point_clouds 61 | -------------------------------------------------------------------------------- /mvdatasets/utils/printing.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | import logging 3 | from rich.progress import ( 4 | BarColumn, 5 | MofNCompleteColumn, 6 | Progress, 7 | TextColumn, 8 | TimeElapsedColumn, 9 | TimeRemainingColumn, 10 | ProgressColumn, 11 | Task, 12 | filesize, 13 | ) 14 | from rich.text import Text 15 | import traceback 16 | 17 | logging.basicConfig(level=logging.INFO) 18 | 19 | 20 | def print_error( 21 | message, exc_type=None, exc_value=None, exc_traceback=None, terminate=True 22 | ): 23 | """ 24 | Print a detailed error message with a stack trace. 25 | 26 | :param message: The error message to display. 27 | :param exc_type: The type of the exception (optional). 28 | :param exc_value: The value of the exception (optional). 29 | :param exc_traceback: The traceback object (optional). 30 | :param terminate: Whether to terminate the execution by raising an exception (default: True). 31 | """ 32 | print(f"[bold red]ERROR:[/bold red] {message}") 33 | 34 | # Use traceback information if available 35 | if exc_type and exc_traceback: 36 | print("\n[bold blue]Stack Trace:[/bold blue]") 37 | detailed_traceback = "".join( 38 | traceback.format_exception(exc_type, exc_value, exc_traceback) 39 | ) 40 | print(f"[dim]{detailed_traceback}[/dim]") 41 | elif not exc_type: 42 | # Provide a default traceback if none is supplied 43 | print("[dim](No traceback available.)[/dim]") 44 | 45 | if terminate: 46 | # Raise a generic or specific exception to halt execution 47 | raise Exception(message) if not exc_type else exc_type(exc_value) 48 | 49 | 50 | def print_warning(message): 51 | print(f"[bold yellow]WARNING[/bold yellow] {message}") 52 | 53 | 54 | def print_info(message): 55 | print(f"[bold blue]INFO[/bold blue] {message}") 56 | 57 | 58 | def print_log(message): 59 | # logger = logging.getLogger(__name__) 60 | # logger.info(message) 61 | print(f"[bold purple]LOG[/bold purple] {message}") 62 | 63 | 64 | def print_success(message): 65 | print(f"[bold green]SUCCESS[/bold green] {message}") 66 | 67 | 68 | class RateColumn(ProgressColumn): 69 | """Renders human readable processing rate.""" 70 | 71 | def render(self, task: "Task") -> Text: 72 | """Render the speed in iterations per second.""" 73 | speed = task.finished_speed or task.speed 74 | if speed is None: 75 | return Text("", style="progress.percentage") 76 | unit, suffix = filesize.pick_unit_and_suffix( 77 | int(speed), 78 | ["", "×10³", "×10⁶", "×10⁹", "×10¹²"], 79 | 1000, 80 | ) 81 | data_speed = speed / unit 82 | return Text(f"{data_speed:.1f}{suffix} it/s", style="progress.percentage") 83 | 84 | 85 | def progress_bar(name): 86 | 87 | # Define custom progress bar 88 | progress_bar = Progress( 89 | # name of the task 90 | TextColumn(f"[bold blue]{name}"), 91 | # progress percentage 92 | TextColumn("•"), 93 | TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), 94 | # progress bar 95 | BarColumn(), 96 | # completed iterations 97 | MofNCompleteColumn(), 98 | # elapsed time 99 | TextColumn("•"), 100 | TimeElapsedColumn(), 101 | # remaining time 102 | TextColumn("•"), 103 | TimeRemainingColumn(), 104 | # show speed 105 | TextColumn("•"), 106 | RateColumn(), 107 | ) 108 | 109 | return progress_bar 110 | -------------------------------------------------------------------------------- /mvdatasets/utils/profiler.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class Profiler: 5 | """ 6 | Profiler class to measure time of different parts of the code. 7 | """ 8 | 9 | def __init__(self, verbose=False): 10 | self.active = {} # store active timers start time 11 | self.history = {} # keep avg time for each name 12 | self.verbose = verbose 13 | 14 | def reset(self): 15 | """ 16 | Resets the profiler. 17 | """ 18 | self.active = {} 19 | self.history = {} 20 | 21 | def start(self, name): 22 | """ 23 | Starts a timer with the given name. 24 | """ 25 | self.active[name] = time.time() 26 | 27 | def end(self, name): 28 | """ 29 | Ends a timer with the given name, updates the history. 30 | """ 31 | elapsed = time.time() - self.active[name] 32 | 33 | if name in self.history: 34 | # read previous state, update it and write it back 35 | state = self.history[name] 36 | state["count"] += 1 37 | state["last"] = elapsed 38 | state["sum"] += elapsed 39 | state["avg"] = state["sum"] / state["count"] 40 | self.history[name] = state 41 | else: 42 | # create new state 43 | self.history[name] = { 44 | "count": 1, 45 | "last": elapsed, 46 | "sum": elapsed, 47 | "avg": elapsed, 48 | } 49 | 50 | if self.verbose: 51 | print(f"{name} took {elapsed} seconds") 52 | 53 | self.active.pop(name) 54 | 55 | def get_last_time(self, name): 56 | """ 57 | Returns the last time for the given timer. 58 | """ 59 | last = -1.0 60 | if name in self.history: 61 | state = self.history[name] 62 | last = state["last"] 63 | return last 64 | 65 | def get_avg_time(self, name): 66 | """ 67 | Returns the average time for the given timer. 68 | """ 69 | avg = -1.0 70 | if name in self.history: 71 | state = self.history[name] 72 | avg = state["avg"] 73 | return avg 74 | 75 | def print_avg_times(self): 76 | """ 77 | Prints the average time for each timer. 78 | Thread timers are averaged over all threads. 79 | """ 80 | print("\nPROFILER AVG TIMES") 81 | processed = {} # threads aggregated 82 | for name, state in self.history.items(): 83 | # check if end of name contains a number 84 | if name.split("_")[-1].isdigit(): # thread id 85 | # remove the number from the end of the name 86 | key = name[: -(len(name.split("_")[-1]) + 1)] 87 | else: 88 | key = name 89 | 90 | # average over all threads 91 | if key in processed: 92 | # read processed state, update it and write it back 93 | prev_state = processed[key] 94 | prev_state["count"] += state["count"] 95 | prev_state["sum"] += state["sum"] 96 | prev_state["avg"] = prev_state["sum"] / prev_state["count"] 97 | processed[key] = state 98 | else: 99 | processed[key] = state 100 | 101 | for name, state in processed.items(): 102 | print(f"{name} took {state['avg'] * 1000} ms", f"{1/state['avg']:.2f} it/s") 103 | print("") 104 | -------------------------------------------------------------------------------- /mvdatasets/utils/tensor_mesh.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class TensorMesh: 5 | def __init__(self, mesh, device="cuda"): 6 | self.vertices = torch.from_numpy(mesh.vertices).to(device) 7 | self.faces = torch.from_numpy(mesh.faces).to(device) 8 | 9 | self.vertices_uvs = None 10 | if mesh.vertices_uvs is not None: 11 | self.vertices_uvs = torch.from_numpy(mesh.vertices_uvs).to(device) 12 | 13 | # self.texture = None 14 | # if mesh.texture is not None: 15 | # self.texture = torch.from_numpy(mesh.texture.image).to(device) 16 | 17 | # for visualization only 18 | 19 | self.faces_color = None 20 | if mesh.faces_color is not None: 21 | self.faces_color = torch.from_numpy(mesh.faces_color).to(device) 22 | 23 | self.mesh_color = torch.rand(1, 3).to(device) 24 | 25 | def get_faces_uvs(self): 26 | if self.vertices_uvs is None: 27 | return None 28 | return self.vertices_uvs[self.faces.flatten()].view(-1, 3, 2) 29 | -------------------------------------------------------------------------------- /mvdatasets/utils/tensor_texture.py: -------------------------------------------------------------------------------- 1 | from rich import print 2 | import numpy as np 3 | import torch 4 | import cv2 5 | from PIL import Image 6 | from mvdatasets.utils.images import ( 7 | uv_coords_to_pix, 8 | non_normalized_uv_coords_to_lerp_weights, 9 | non_normalize_uv_coord, 10 | non_normalized_uv_coords_to_interp_corners, 11 | ) 12 | 13 | 14 | class TensorTexture: 15 | def __init__( 16 | self, 17 | texture_np=None, 18 | texture_path=None, 19 | res=None, 20 | lerp=True, 21 | device="cuda", 22 | ): 23 | super(TensorTexture, self).__init__() 24 | self.device = device 25 | self.lerp = lerp 26 | 27 | if texture_np is None: 28 | if texture_path is None: 29 | raise ValueError("texture_np and texture_path cannot be both None") 30 | 31 | texture_pil = Image.open(texture_path).convert("RGBA") 32 | texture_np = np.array(texture_pil) / 255.0 33 | 34 | self.data = torch.from_numpy(texture_np).float().to(self.device) 35 | 36 | if res is not None: 37 | # use opencv to resize to res 38 | texture_cv = cv2.resize( 39 | texture_np, (res[1], res[0]), interpolation=cv2.INTER_LINEAR 40 | ) 41 | self.data = torch.from_numpy(texture_cv).float().to(self.device) 42 | 43 | # 44 | self.data = self.data.reshape(self.data.shape[0], self.data.shape[1], -1) 45 | self.shape = self.data.shape 46 | 47 | # height, width 48 | self.res = ( 49 | torch.tensor([self.shape[0], self.shape[1]]).long().to(self.device) 50 | ) # [height, width] 51 | print("self.res", self.res) 52 | 53 | if self.lerp: 54 | # pad with zeros 55 | data_shape = self.data.shape 56 | # to list to modify 57 | data_shape = list(data_shape) 58 | # add 2 to first two dimensions 59 | data_shape[0] += 2 60 | data_shape[1] += 2 61 | padded_data = torch.zeros(data_shape, device=self.device) 62 | padded_data[1:-1, 1:-1] = self.data 63 | self.data = padded_data 64 | 65 | print("self.data.shape", self.data.shape) 66 | 67 | def __call__(self, uv_coords): 68 | 69 | if self.lerp: 70 | # uv coords are width, height 71 | # res is height, width 72 | # flip=True 73 | # results are non normalized uv coordinates 74 | uv_coords_nn = non_normalize_uv_coord(uv_coords, self.res, flip=True) 75 | 76 | # results are non normalized uv coordinates of the corners 77 | uv_corners_coords_nn = non_normalized_uv_coords_to_interp_corners( 78 | uv_coords_nn 79 | ) # [N, 4, 2] 80 | 81 | # find lerp weights 82 | lerp_weights = non_normalized_uv_coords_to_lerp_weights( 83 | uv_coords_nn, uv_corners_coords_nn 84 | ) 85 | 86 | # convert to (padded) pixel coordinates 87 | uv_corners_pix = uv_corners_coords_nn.floor().long() + 1 88 | uv_corners_pix = uv_corners_pix.reshape(-1, 2) # [N*4, 2] 89 | # print("uv_corners_pix.shape", uv_corners_pix.shape) 90 | # query texture at pixels coordinates 91 | vertices_values = self.data[uv_corners_pix[:, 1], uv_corners_pix[:, 0]] 92 | # print("vertices_values.shape", vertices_values.shape) 93 | vertices_values = vertices_values.reshape(uv_coords.shape[0], 4, -1) 94 | 95 | # interpolate 96 | # print("lerp_weights.shape", lerp_weights.shape) 97 | # print("vertices_values.shape", vertices_values.shape) 98 | output = (vertices_values * lerp_weights).sum(dim=1) 99 | else: 100 | # anchor uv_coords [0, 1] to pixel coordinates 101 | # uv coords are width, height 102 | # res is height, width 103 | # flip=True 104 | # results are pixel coordinates 105 | uv_pix = uv_coords_to_pix(uv_coords, self.res, flip=True) # u, v 106 | 107 | # pixel coordinates are width, height 108 | # images is height, width 109 | # query texture at pixel coordinates 110 | output = self.data[uv_pix[:, 1], uv_pix[:, 0]] 111 | 112 | return output 113 | 114 | def to_numpy(self): 115 | return self.data[: self.shape[0], : self.shape[1]].detach().cpu().numpy() 116 | -------------------------------------------------------------------------------- /mvdatasets/utils/texture.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import math 4 | from rich import print 5 | from PIL import Image 6 | 7 | from mvdatasets.utils.images import image_to_numpy 8 | 9 | 10 | def sample_texture(image, uvs): 11 | """ 12 | Args: 13 | image: torch.Tensor, [H, W, C] or [H, W, C, F] 14 | uvs: torch.Tensor, [N, 2] in [0, 1] 15 | Returns: 16 | vals: torch.Tensor, [N, C] or [N, C, F] 17 | """ 18 | # get image dims 19 | height, width = image.shape[:2] 20 | assert height == width, "only square textures are supported" 21 | texture_res = height 22 | # convert to uv coordinates 23 | uvs = uvs * torch.tensor([texture_res, texture_res], device=uvs.device) 24 | # convert to pixel coordinates 25 | uvs[:, 0] = torch.clamp(uvs[:, 0], 0, texture_res - 1) 26 | uvs[:, 1] = torch.clamp(uvs[:, 1], 0, texture_res - 1) 27 | uvs = uvs.int() 28 | # sample 29 | vals = image[uvs[:, 0], uvs[:, 1]] 30 | return vals 31 | 32 | 33 | class RGBATexture: 34 | def __init__( 35 | self, 36 | texture_meta: dict, 37 | verbose=True, 38 | ): 39 | 40 | texture_path = texture_meta["texture_path"] 41 | # ignore texture scale as values should always be in [0, 1] 42 | assert texture_path.endswith( 43 | ".png" 44 | ), f"texture {texture_path} must be a .png file" 45 | if verbose: 46 | print( 47 | f"[bold blue]INFO[/bold blue]: loading texture {texture_path.split('/')[-1]}" 48 | ) 49 | texture = image_to_numpy(Image.open(texture_path)).astype(np.float32) 50 | 51 | if "texture_scale" not in texture_meta: 52 | texture_meta["texture_scale"] = [0, 1] 53 | texture_scale = texture_meta["texture_scale"] 54 | min_val = texture_scale[0] 55 | max_val = texture_scale[1] 56 | scale = max_val - min_val 57 | texture = texture * scale + min_val 58 | 59 | if verbose: 60 | print( 61 | f"[bold blue]INFO[/bold blue]: texture {texture_path.split('/')[-1]} loaded: {texture.shape}" 62 | ) 63 | self.image = texture 64 | self.height, self.width, self.nr_channels = self.image.shape 65 | 66 | 67 | class SHTexture: 68 | def __init__( 69 | self, 70 | textures_meta: list, 71 | verbose=True, 72 | ): 73 | sh_deg = math.sqrt(len(textures_meta)) - 1 74 | assert sh_deg.is_integer(), "number of textures must be a square number" 75 | sh_deg = int(sh_deg) 76 | print(f"[bold blue]INFO[/bold blue]: sh_deg {sh_deg}") 77 | 78 | all_sh_coeffs = [] 79 | channel_sh_coeffs = [] 80 | nr_sh_coeffs = (sh_deg + 1) ** 2 81 | nr_read_sh_coeffs = 0 82 | 83 | for texture_meta in textures_meta: 84 | 85 | texture_path = texture_meta["texture_path"] 86 | 87 | if "texture_scale" not in texture_meta: 88 | texture_meta["texture_scale"] = [0, 1] 89 | else: 90 | texture_scale = texture_meta["texture_scale"] 91 | min_val = texture_scale[0] 92 | max_val = texture_scale[1] 93 | 94 | assert texture_path.endswith( 95 | ".png" 96 | ), f"texture {texture_path} must be a .png file" 97 | if verbose: 98 | print( 99 | f"[bold blue]INFO[/bold blue]: loading texture {texture_path.split('/')[-1]}" 100 | ) 101 | sh_coeffs = image_to_numpy(Image.open(texture_path)).astype(np.float32) 102 | 103 | sh_coeffs = sh_coeffs * (max_val - min_val) + min_val 104 | 105 | channel_sh_coeffs.append(sh_coeffs) 106 | 107 | nr_read_sh_coeffs += 4 108 | if verbose: 109 | print( 110 | f"[bold blue]INFO[/bold blue]: texture {texture_path.split('/')[-1]} loaded: {sh_coeffs.shape}" 111 | ) 112 | 113 | if sh_deg == 0: 114 | all_sh_coeffs.append(sh_coeffs) 115 | break 116 | 117 | if nr_read_sh_coeffs == nr_sh_coeffs: 118 | image = np.concatenate(channel_sh_coeffs, axis=-1) 119 | all_sh_coeffs.append(image) 120 | channel_sh_coeffs = [] 121 | nr_read_sh_coeffs = 0 122 | 123 | if len(all_sh_coeffs) > 1: 124 | texture = np.stack(all_sh_coeffs, axis=2) 125 | else: 126 | texture = all_sh_coeffs[0] 127 | # unsqueeze last dim 128 | texture = np.expand_dims(texture, axis=-1) 129 | 130 | if verbose: 131 | print( 132 | f"[bold blue]INFO[/bold blue]: sh coeffs params {texture.shape} loaded" 133 | ) 134 | self.image = texture 135 | self.height, self.width, self.nr_channels, _ = self.image.shape 136 | -------------------------------------------------------------------------------- /mvdatasets/utils/video_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from typing import Optional 4 | from pathlib import Path 5 | 6 | 7 | def extract_frames( 8 | video_path: Path, 9 | output_path: Optional[Path] = None, 10 | subsample_factor: int = 1, 11 | ext: str = "png", 12 | skip_time: int = 1, 13 | start_time: str = "00:00:00", 14 | end_time: Optional[str] = None, 15 | ): 16 | # get video name 17 | video_name = os.path.basename(video_path) 18 | # remove extension 19 | video_name = os.path.splitext(video_name)[0] 20 | # if outout path is not given 21 | if output_path is None: 22 | # create output folder in same folder containing the video 23 | # whose name is the video name 24 | output_path = video_path.parent / video_name 25 | else: 26 | # create output folder in output path 27 | # whose name is the video name 28 | output_path = output_path / video_name 29 | # create output folder 30 | os.makedirs(output_path, exist_ok=True) 31 | to_str = f"-to {end_time}" if end_time else "" 32 | # get video height and width 33 | h_str = subprocess.check_output( 34 | f"ffprobe -v error -select_streams v:0 -show_entries stream=height -of csv=s=x:p=0 {video_path}", 35 | shell=True, 36 | ) 37 | h = int(h_str) 38 | height = h // subsample_factor 39 | command = f"ffmpeg -i {video_path} -vf \"select='not(mod(n,{skip_time}))',scale=-1:{height}\" -vsync vfr -ss {start_time} {to_str} {output_path}/%05d.{ext}" 40 | subprocess.call(command, shell=True) 41 | -------------------------------------------------------------------------------- /mvdatasets/utils/virtual_cameras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from typing import List 3 | from mvdatasets import Camera 4 | from mvdatasets.geometry.common import look_at, deg2rad 5 | 6 | 7 | def sample_cameras_on_hemisphere( 8 | intrinsics: np.ndarray, 9 | width: int, 10 | height: int, 11 | radius: float = 1.0, 12 | nr_cameras: int = 10, 13 | up: str = "z", 14 | center: np.ndarray = np.array([0, 0, 0]), 15 | ) -> List[Camera]: 16 | 17 | if up == "z": 18 | up = np.array([0, 0, 1], dtype=np.float32) 19 | elif up == "y": 20 | up = np.array([0, 1, 0], dtype=np.float32) 21 | else: 22 | raise ValueError(f"Invalid `up` value: {up}, must be 'z' or 'y'.") 23 | 24 | azimuth_deg = np.random.uniform(0, 360, nr_cameras) 25 | elevation_deg = np.random.uniform(-90, 90, nr_cameras) 26 | azimuth_rad = deg2rad(azimuth_deg) 27 | elevation_rad = deg2rad(elevation_deg) 28 | x = np.cos(azimuth_rad) * np.cos(elevation_rad) * radius 29 | y = np.sin(azimuth_rad) * np.cos(elevation_rad) * radius # y is up 30 | z = np.sin(elevation_rad) * radius # z is up 31 | # x = np.array(x) 32 | # y = np.array(y) 33 | # z = np.array(z) 34 | cameras_centers = np.column_stack((x, y, z)) 35 | 36 | cameras = [] 37 | for i in range(nr_cameras): 38 | 39 | # get rotation matrix from azimuth and elevation 40 | pose = np.eye(4) 41 | pose = look_at(cameras_centers[i], center, up) 42 | 43 | # # local transform 44 | # local_transform = np.eye(4) 45 | # local_transform[:3, :3] = np.array( 46 | # [[-1, 0, 0], [0, -1, 0], [0, 0, -1]], dtype=np.float32 47 | # ) 48 | 49 | camera = Camera( 50 | intrinsics, 51 | pose, 52 | width=width, 53 | height=height, 54 | # local_transform=local_transform, 55 | camera_label=i, 56 | ) 57 | cameras.append(camera) 58 | 59 | return cameras 60 | -------------------------------------------------------------------------------- /mvdatasets/visualization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/mvdatasets/visualization/__init__.py -------------------------------------------------------------------------------- /mvdatasets/visualization/colormaps.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # from https://github.com/epic-kitchens/VISOR-VIS/blob/main/vis.py 4 | davis_palette = np.repeat(np.expand_dims(np.arange(0, 256), 1), 3, 1).astype(np.uint8) 5 | davis_palette[:104, :] = [ 6 | [0, 0, 0], 7 | [200, 0, 0], 8 | [0, 200, 0], 9 | [200, 128, 0], 10 | [0, 0, 200], 11 | [200, 0, 200], 12 | [0, 200, 200], 13 | [200, 200, 200], 14 | [252, 93, 82], 15 | [160, 121, 99], 16 | [164, 188, 119], 17 | [0, 60, 29], 18 | [75, 237, 255], 19 | [148, 169, 183], 20 | [96, 74, 207], 21 | [255, 186, 255], 22 | [255, 218, 231], 23 | [136, 30, 23], 24 | [231, 181, 131], 25 | [219, 226, 216], 26 | [0, 196, 107], 27 | [0, 107, 119], 28 | [0, 125, 227], 29 | [153, 134, 227], 30 | [91, 0, 56], 31 | [86, 0, 7], 32 | [246, 207, 195], 33 | [87, 51, 0], 34 | [125, 131, 122], 35 | [187, 237, 218], 36 | [46, 57, 59], 37 | [164, 191, 255], 38 | [37, 29, 57], 39 | [144, 53, 104], 40 | [79, 53, 54], 41 | [255, 163, 128], 42 | [255, 233, 180], 43 | [68, 100, 62], 44 | [0, 231, 199], 45 | [0, 170, 233], 46 | [0, 20, 103], 47 | [195, 181, 219], 48 | [148, 122, 135], 49 | [200, 128, 129], 50 | [46, 20, 10], 51 | [86, 78, 24], 52 | [180, 255, 188], 53 | [0, 36, 33], 54 | [0, 101, 139], 55 | [50, 60, 111], 56 | [188, 81, 205], 57 | [168, 9, 70], 58 | [167, 91, 59], 59 | [35, 32, 0], 60 | [0, 124, 28], 61 | [0, 156, 145], 62 | [0, 36, 57], 63 | [0, 0, 152], 64 | [89, 12, 97], 65 | [249, 145, 183], 66 | [255, 153, 170], 67 | [255, 153, 229], 68 | [184, 143, 204], 69 | [208, 204, 255], 70 | [11, 0, 128], 71 | [69, 149, 230], 72 | [82, 204, 194], 73 | [77, 255, 136], 74 | [6, 26, 0], 75 | [92, 102, 41], 76 | [102, 85, 61], 77 | [76, 45, 0], 78 | [229, 69, 69], 79 | [127, 38, 53], 80 | [128, 51, 108], 81 | [41, 20, 51], 82 | [25, 16, 3], 83 | [102, 71, 71], 84 | [77, 54, 71], 85 | [143, 122, 153], 86 | [42, 41, 51], 87 | [4, 0, 51], 88 | [31, 54, 77], 89 | [204, 255, 251], 90 | [51, 128, 77], 91 | [61, 153, 31], 92 | [194, 204, 143], 93 | [255, 234, 204], 94 | [204, 119, 0], 95 | [204, 102, 102], 96 | [64, 0, 0], 97 | [191, 0, 0], 98 | [64, 128, 0], 99 | [191, 128, 0], 100 | [64, 0, 128], 101 | [191, 0, 128], 102 | [64, 128, 128], 103 | [191, 128, 128], 104 | [0, 64, 0], 105 | [128, 64, 0], 106 | [0, 191, 0], 107 | [128, 191, 0], 108 | [0, 64, 128], 109 | [128, 64, 128], 110 | ] # first 90 for the regular colors and the last 14 for objects having more than one segment 111 | -------------------------------------------------------------------------------- /mvdatasets/visualization/video_gen.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from tqdm import tqdm 4 | from copy import deepcopy 5 | from typing import Literal, List, Optional 6 | from pathlib import Path 7 | from mvdatasets.camera import Camera 8 | from mvdatasets.visualization.matplotlib import plot_camera_trajectory, plot_3d 9 | from mvdatasets.utils.printing import print_log 10 | from mvdatasets.geometry.primitives import PointCloud 11 | 12 | 13 | def make_video_camera_trajectory( 14 | cameras: List[Camera], 15 | save_path: Path, # e.g. Path("./trajectory.mp4"), 16 | dataset_name: Optional[str] = None, 17 | point_clouds: List[PointCloud] = None, 18 | nr_frames: int = -1, # -1 means all frames 19 | max_nr_points: int = 10000, 20 | fps: int = 10, 21 | remove_tmp_files: bool = True, 22 | azimuth_deg: float = 60.0, 23 | elevation_deg: float = 30.0, 24 | scene_radius: float = 1.0, 25 | up: Literal["z", "y"] = "z", 26 | draw_origin: bool = True, 27 | ) -> None: 28 | 29 | # check if save_path extension is mp4 30 | if save_path.suffix != ".mp4": 31 | raise ValueError("save_path extension must be mp4") 32 | 33 | # uniform sampling of sequence lenght 34 | sequence_len = len(cameras) 35 | if nr_frames == -1: 36 | nr_frames = sequence_len 37 | elif nr_frames > sequence_len or nr_frames <= 0: 38 | raise ValueError( 39 | f"nr_frames must be less than or equal to {sequence_len} and greater than 0" 40 | ) 41 | step_size = sequence_len // nr_frames 42 | frames_idxs = np.arange(0, sequence_len, step_size) 43 | 44 | # remove extension from save_path 45 | output_path = save_path.parent / save_path.stem 46 | 47 | # create output folder (e.g. ./trajectory) 48 | 49 | # if output_path exists, remove it 50 | if os.path.exists(output_path): 51 | print_log(f"overriding existing {output_path}") 52 | os.system(f"rm -rf {output_path}") 53 | os.makedirs(output_path) 54 | 55 | # downsample point cloud 56 | new_point_clouds = [] 57 | for point_cloud in point_clouds: 58 | new_point_cloud = deepcopy(point_cloud) 59 | new_point_cloud.downsample(max_nr_points) 60 | new_point_clouds.append(new_point_cloud) 61 | point_clouds = new_point_clouds 62 | 63 | # Visualize cameras 64 | pbar = tqdm(enumerate(frames_idxs), desc="frames", ncols=100) 65 | for _, last_frame_idx in pbar: 66 | 67 | # get camera 68 | camera = cameras[last_frame_idx] 69 | 70 | # get timestamp 71 | ts = camera.get_timestamps()[0] 72 | # round to 3 decimal places 73 | ts = round(ts, 3) 74 | 75 | # save plot as png in output_path 76 | plot_camera_trajectory( 77 | cameras=cameras, 78 | last_frame_idx=last_frame_idx, 79 | draw_every_n_cameras=1, 80 | point_clouds=point_clouds, 81 | azimuth_deg=azimuth_deg, 82 | elevation_deg=elevation_deg, 83 | max_nr_points=None, 84 | up=up, 85 | scene_radius=scene_radius, 86 | draw_rgb_frame=True, 87 | draw_all_cameras_frames=False, 88 | draw_image_planes=True, 89 | draw_cameras_frustums=True, 90 | draw_origin=draw_origin, 91 | figsize=(15, 15), 92 | title=f"{dataset_name} camera trajectory up to time {ts} [s]", 93 | show=False, 94 | save_path=os.path.join(output_path, f"{format(last_frame_idx, '09d')}.png"), 95 | ) 96 | 97 | # make video from plots in output_path 98 | os.system( 99 | f'ffmpeg -y -r {fps} -i {output_path}/%09d.png -vf scale="trunc(iw/2)*2:trunc(ih/2)*2" -vcodec libx264 -crf 25 -pix_fmt yuv420p {save_path}' 100 | ) 101 | print_log(f"video saved at {save_path}") 102 | 103 | # remove tmp files 104 | if remove_tmp_files: 105 | os.system(f"rm -rf {output_path}") 106 | print_log("removed temporary files") 107 | 108 | 109 | def make_video_depth_unproject( 110 | cameras: List[Camera], 111 | point_clouds: List[PointCloud], 112 | save_path: Path, # e.g. Path("./trajectory.mp4"), 113 | dataset_name: Optional[str] = None, 114 | max_nr_points: int = 10000, 115 | fps: int = 10, 116 | remove_tmp_files: bool = True, 117 | azimuth_deg: float = 60.0, 118 | elevation_deg: float = 30.0, 119 | scene_radius: float = 1.0, 120 | draw_bounding_cube: bool = True, 121 | draw_image_planes: bool = True, 122 | up: Literal["z", "y"] = "z", 123 | draw_origin: bool = True, 124 | ) -> None: 125 | 126 | # check if save_path extension is mp4 127 | if save_path.suffix != ".mp4": 128 | raise ValueError("save_path extension must be mp4") 129 | 130 | # assert len(cameras) == len(point_clouds) 131 | assert len(cameras) == len( 132 | point_clouds 133 | ), "cameras and point_clouds must have the same length" 134 | 135 | sequence_len = len(cameras) 136 | step_size = 1 137 | frames_idxs = np.arange(0, sequence_len, step_size) 138 | 139 | # remove extension from save_path 140 | output_path = save_path.parent / save_path.stem 141 | 142 | # create output folder (e.g. ./trajectory) 143 | 144 | # if output_path exists, remove it 145 | if os.path.exists(output_path): 146 | print_log(f"overriding existing {output_path}") 147 | os.system(f"rm -rf {output_path}") 148 | os.makedirs(output_path) 149 | 150 | # downsample point cloud 151 | new_point_clouds = [] 152 | for point_cloud in point_clouds: 153 | new_point_cloud = deepcopy(point_cloud) 154 | new_point_cloud.downsample(max_nr_points) 155 | new_point_clouds.append(new_point_cloud) 156 | point_clouds = new_point_clouds 157 | 158 | # Visualize cameras 159 | pbar = tqdm(enumerate(frames_idxs), desc="frames", ncols=100) 160 | for _, i in pbar: 161 | 162 | # get camera 163 | camera = cameras[i] 164 | pc = point_clouds[i] 165 | 166 | # plot point clouds and camera 167 | plot_3d( 168 | cameras=[camera], 169 | point_clouds=[pc], 170 | max_nr_points=None, 171 | azimuth_deg=azimuth_deg, 172 | elevation_deg=elevation_deg, 173 | up="z", 174 | scene_radius=scene_radius, 175 | draw_bounding_cube=draw_bounding_cube, 176 | draw_image_planes=draw_image_planes, 177 | draw_origin=draw_origin, 178 | draw_cameras_frustums=True, 179 | figsize=(15, 15), 180 | title=f"{dataset_name} camera {camera.get_camera_label()} depth unprojection", 181 | show=False, 182 | save_path=os.path.join(output_path, f"{format(i, '09d')}.png"), 183 | ) 184 | 185 | # make video from plots in output_path 186 | os.system( 187 | f'ffmpeg -y -r {fps} -i {output_path}/%09d.png -vf scale="trunc(iw/2)*2:trunc(ih/2)*2" -vcodec libx264 -crf 25 -pix_fmt yuv420p {save_path}' 188 | ) 189 | print_log(f"video saved at {save_path}") 190 | 191 | # remove tmp files 192 | if remove_tmp_files: 193 | os.system(f"rm -rf {output_path}") 194 | print_log("removed temporary files") 195 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=64.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.black] 6 | line-length = 88 -------------------------------------------------------------------------------- /scripts/download/dtu.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the dtu dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from google drive 7 | gdown 1maZGcJBFgMOsFCcKwLsw1od5Qm1ZQ2RU -O data/dtu.zip 8 | 9 | # unzip 10 | unzip data/dtu.zip -d data 11 | 12 | # remove zip file 13 | rm data/dtu.zip -------------------------------------------------------------------------------- /scripts/download/hypernerf.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the HyperNeRF dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from source 7 | 8 | # all scenes 9 | wget https://github.com/google/hypernerf/releases/download/v0.1/misc_americano.zip -O data/americano.zip 10 | wget https://github.com/google/hypernerf/releases/download/v0.1/misc_cross-hands.zip -O data/cross-hands.zip 11 | wget https://github.com/google/hypernerf/releases/download/v0.1/misc_espresso.zip -O data/espresso.zip 12 | wget https://github.com/google/hypernerf/releases/download/v0.1/misc_oven-mitts.zip -O data/oven-mitts.zip 13 | wget https://github.com/google/hypernerf/releases/download/v0.1/misc_split-cookie.zip -O data/split-cookie.zip 14 | wget https://github.com/google/hypernerf/releases/download/v0.1/misc_tamping.zip -O data/tamping.zip 15 | 16 | # unzip 17 | unzip data/americano.zip -d data/hypernerf 18 | unzip data/cross-hands.zip -d data/hypernerf 19 | unzip data/espresso.zip -d data/hypernerf 20 | unzip data/oven-mitts.zip -d data/hypernerf 21 | unzip data/split-cookie.zip -d data/hypernerf 22 | unzip data/tamping.zip -d data/hypernerf 23 | 24 | # rename split-cookie1 to split-cookie 25 | mv data/hypernerf/cross-hands1 data/hypernerf/cross-hands 26 | 27 | # remove zip file 28 | rm data/americano.zip 29 | rm data/cross-hands.zip 30 | rm data/espresso.zip 31 | rm data/oven-mitts.zip 32 | rm data/split-cookie.zip 33 | rm data/tamping.zip 34 | -------------------------------------------------------------------------------- /scripts/download/iphone.md: -------------------------------------------------------------------------------- 1 | # Download the iphone dataset 2 | 3 | 1. Download scenes folders from [here](https://drive.google.com/drive/folders/1cBw3CUKu2sWQfc_1LbFZGbpdQyTFzDEX?usp=drive_link). 4 | 2. Unzip them to `data\iphone` (e.g. `data\iphone\paper-windmill`). -------------------------------------------------------------------------------- /scripts/download/iphone_som.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the iphone (shape-of-motion) dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from google drive 7 | gdown 1QJQnVw_szoy_k5x9k_BAE2BWtf_BXqKn -O data/apple.zip 8 | gdown 1QihG5A7c_bpkse5b0OBdqqFThgX0kDyZ -O data/backpack.zip 9 | gdown 1b9Y-hUm9Cviuq-fl7gG-q7rUK7j0u2Rv -O data/block.zip 10 | gdown 1inkHp24an1TyWvBekBxu2wRIyLQ0gkhO -O data/creeper.zip 11 | gdown 1frv8miU24Dl7fqblYt7zkwj129ci-68U -O data/handwavy.zip 12 | gdown 1BmuxJXKi6dVaNOjmppuETQsaspAV9Wca -O data/haru-sit.zip 13 | gdown 1OpgF2ILf43jcN-226wQcxImjcfMAVOwA -O data/mochi-high-five.zip 14 | gdown 15PirJRqsT5lLjuGdLWALBDFMQanj8FTh -O data/paper-windmill.zip 15 | gdown 1Uc2BXpONnWhxKNs6tKMle0MiSVMVZsuB -O data/pillow.zip 16 | gdown 1055wcQk-ZfVWXa_g-dpQIRQy-kLBL_Lk -O data/spin.zip 17 | gdown 18sjQQMU6AijyXg4BoucLX82R959BYAzz -O data/sriracha-tree.zip 18 | gdown 1Mqm4C1Oitv4AsDM2n0Ojbt5pmF_qXVfI -O data/teddy.zip 19 | 20 | # unzip 21 | unzip -o data/apple.zip -d data/iphone_som 22 | unzip -o data/backpack.zip -d data/iphone_som 23 | unzip -o data/block.zip -d data/iphone_som 24 | unzip -o data/creeper.zip -d data/iphone_som 25 | unzip -o data/handwavy.zip -d data/iphone_som 26 | unzip -o data/haru-sit.zip -d data/iphone_som 27 | unzip -o data/mochi-high-five.zip -d data/iphone_som 28 | unzip -o data/paper-windmill.zip -d data/iphone_som 29 | unzip -o data/pillow.zip -d data/iphone_som 30 | unzip -o data/spin.zip -d data/iphone_som 31 | unzip -o data/sriracha-tree.zip -d data/iphone_som 32 | unzip -o data/teddy.zip -d data/iphone_som 33 | 34 | # remove zip file 35 | rm data/apple.zip 36 | rm data/backpack.zip 37 | rm data/block.zip 38 | rm data/creeper.zip 39 | rm data/handwavy.zip 40 | rm data/haru-sit.zip 41 | rm data/mochi-high-five.zip 42 | rm data/paper-windmill.zip 43 | rm data/pillow.zip 44 | rm data/spin.zip 45 | rm data/sriracha-tree.zip 46 | rm data/teddy.zip 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /scripts/download/mipnerf360.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the Mip-NeRF360 dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from source 7 | wget http://storage.googleapis.com/gresearch/refraw360/360_v2.zip -O data/mipnerf360.zip 8 | 9 | # unzip 10 | unzip data/mipnerf360.zip -d data/mipnerf360 11 | 12 | # remove zip file 13 | rm data/mipnerf360.zip -------------------------------------------------------------------------------- /scripts/download/nerf_furry.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the nerf_furry dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from google drive 7 | gdown 18W9aSIL4SnCDaHJ8uaGvWF4GgAze14lm -O data/nerf_furry.zip 8 | 9 | # unzip 10 | unzip data/nerf_furry.zip -d data 11 | 12 | # remove zip file 13 | rm data/nerf_furry.zip -------------------------------------------------------------------------------- /scripts/download/nerf_llff.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the nerf_llff dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from google drive 7 | gdown 11PhkBXZZNYTD2emdG1awALlhCnkq7aN- -O data/nerf_llff.zip 8 | 9 | # unzip 10 | unzip data/nerf_llff.zip -d data 11 | 12 | # rename nerf_llff to llff 13 | mv data/nerf_llff_data data/nerf_llff 14 | 15 | # remove zip file 16 | rm data/nerf_llff.zip -------------------------------------------------------------------------------- /scripts/download/nerf_synthetic.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the nerf_synthetic dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from google drive 7 | gdown 1OsiBs2udl32-1CqTXCitmov4NQCYdA9g -O data/nerf_synthetic.zip 8 | 9 | # unzip 10 | unzip data/nerf_synthetic.zip -d data 11 | 12 | # remove zip file 13 | rm data/nerf_synthetic.zip -------------------------------------------------------------------------------- /scripts/download/neu3d.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the Neu3D dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from source 7 | 8 | # all scenes 9 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/coffee_martini.zip -O data/coffee_martini.zip 10 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/cook_spinach.zip -O data/cook_spinach.zip 11 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/cut_roasted_beef.zip -O data/cut_roasted_beef.zip 12 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/flame_steak.zip -O data/flame_steak.zip 13 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/sear_steak.zip -O data/sear_steak.zip 14 | 15 | # flame salmon 16 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/flame_salmon_1_split.z01 -O data/flame_salmon_1_split.z01 17 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/flame_salmon_1_split.z02 -O data/flame_salmon_1_split.z02 18 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/flame_salmon_1_split.z03 -O data/flame_salmon_1_split.z03 19 | wget https://github.com/facebookresearch/Neural_3D_Video/releases/download/v1.0/flame_salmon_1_split.zip -O data/flame_salmon_1_split.zip 20 | zip -F data/flame_salmon_1_split.zip --out flame_salmon.zip 21 | 22 | # unzip 23 | unzip data/coffee_martini.zip -d data/neu3d 24 | unzip data/cook_spinach.zip -d data/neu3d 25 | unzip data/cut_roasted_beef.zip -d data/neu3d 26 | unzip data/flame_steak.zip -d data/neu3d 27 | unzip data/sear_steak.zip -d data/neu3d 28 | unzip flame_salmon.zip -d data/neu3d 29 | 30 | # rename flame_salmon_1 to flame_salmon 31 | mv data/neu3d/flame_salmon_1 data/neu3d/flame_salmon 32 | 33 | # remove zip file 34 | rm data/coffee_martini.zip 35 | rm data/cook_spinach.zip 36 | rm data/cut_roasted_beef.zip 37 | rm data/flame_steak.zip 38 | rm data/sear_steak.zip 39 | rm data/flame_salmon_1_split.z01 40 | rm data/flame_salmon_1_split.z02 41 | rm data/flame_salmon_1_split.z03 42 | rm data/flame_salmon_1_split.zip 43 | rm flame_salmon.zip 44 | -------------------------------------------------------------------------------- /scripts/download/panoptic_sports.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the panoptic-sports dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from source 7 | wget https://omnomnom.vision.rwth-aachen.de/data/Dynamic3DGaussians/data.zip -O data/pan_sports.zip 8 | 9 | # unzip 10 | unzip data/pan_sports.zip -d data/pan_sports 11 | 12 | # move files to data folder to panoptic-sports 13 | mv data/pan_sports/data data/panoptic-sports 14 | 15 | # remove folder 16 | rm -r data/pan_sports 17 | 18 | # remove zip file 19 | rm data/pan_sports.zip -------------------------------------------------------------------------------- /scripts/download/ref_nerf.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the Ref-NeRF dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from source 7 | wget https://storage.googleapis.com/gresearch/refraw360/ref.zip -O data/ref_nerf.zip 8 | 9 | # unzip 10 | unzip data/ref_nerf.zip -d data 11 | 12 | # remove zip file 13 | rm data/ref_nerf.zip -------------------------------------------------------------------------------- /scripts/download/shelly.sh: -------------------------------------------------------------------------------- 1 | # Downlaod the shelly dataset 2 | 3 | # create data folder if it doesn't exist 4 | mkdir -p data 5 | 6 | # download from google drive 7 | gdown 1Qyf_UMd49Pm-8xjSI4j0t-Np8JWeuwOk -O data/shelly.zip 8 | 9 | # unzip 10 | unzip data/shelly.zip -d data 11 | 12 | # remove zip file 13 | rm data/shelly.zip -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name="mvdatasets", 5 | version="1.0.0", 6 | description="Standardized DataLoaders for 3D Computer Vision", 7 | long_description=open("README.md").read(), 8 | long_description_content_type="text/markdown", 9 | url="https://github.com/autonomousvision/mvdatasets", 10 | author="Stefano Esposito", 11 | author_email="stefano.esposito@uni-tuebingen.de", 12 | packages=find_packages(), 13 | install_requires=[ 14 | # main 15 | "gdown>=5.2.0", 16 | "flake8>=7.1.1", 17 | "black>=24.8.0", 18 | "ffmpeg>=1.4.0", 19 | "pyyaml>=6.0", 20 | "numpy>=1.21.6", 21 | "tqdm>=4.67.1", 22 | "torch>=1.13.1", 23 | "pillow>=9.5.0", 24 | "torchvision>=0.14.1", 25 | "iopath>=0.1.10", 26 | "matplotlib>=3.5.3", 27 | "jupyter>=1.1.1", 28 | "opencv-python>=4.11.0.86", 29 | "open3d>=0.18.0", 30 | "rich>=13.8.1", 31 | "tyro>=0.9.11", 32 | "pycolmap@git+https://github.com/rmbrualla/pycolmap@cc7ea4b7301720ac29287dbe450952511b32125e", 33 | ], 34 | extras_require={ 35 | "tests": [ 36 | # tests 37 | "pytest" 38 | ], 39 | "docs": [ 40 | # docs 41 | "sphinx", 42 | "sphinx-rtd-theme", 43 | "sphinxcontrib-mermaid", 44 | "sphinxcontrib-bibtex", 45 | "sphinxcontrib-osexample", 46 | "myst-parser", 47 | "pre-commit", 48 | ], 49 | }, 50 | python_requires=">=3.8", 51 | ) 52 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomousvision/mvdatasets/a4a176c79675e885f1201e7255de93ba61a38fbb/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_projection_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import torch 4 | from mvdatasets.geometry.projections import ( 5 | local_perspective_projection, 6 | local_inv_perspective_projection, 7 | global_perspective_projection, 8 | global_inv_perspective_projection, 9 | ) 10 | 11 | 12 | class TestProjectionFunctions(unittest.TestCase): 13 | 14 | def test_local_perspective_projection(self): 15 | # Test with NumPy 16 | intrinsics = np.array( 17 | [[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=np.float32 18 | ) 19 | points_3d = np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]], dtype=np.float32) 20 | result = local_perspective_projection(intrinsics, points_3d) 21 | expected = np.array( 22 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=np.float32 23 | ) 24 | np.testing.assert_allclose(result, expected, atol=1e-4) 25 | 26 | # Test with PyTorch 27 | intrinsics = torch.tensor( 28 | [[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=torch.float32 29 | ) 30 | points_3d = torch.tensor([[1, 1, 1], [2, 2, 2], [3, 3, 3]], dtype=torch.float32) 31 | result = local_perspective_projection(intrinsics, points_3d) 32 | expected = torch.tensor( 33 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=torch.float32 34 | ) 35 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 36 | 37 | def test_local_inv_perspective_projection(self): 38 | # Test with NumPy 39 | intrinsics_inv = np.linalg.inv( 40 | np.array([[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=np.float32) 41 | ) 42 | points_2d = np.array( 43 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=np.float32 44 | ) 45 | result = local_inv_perspective_projection(intrinsics_inv, points_2d) 46 | expected = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32) 47 | np.testing.assert_allclose(result, expected, atol=1e-4) 48 | 49 | # Test with PyTorch 50 | intrinsics_inv = torch.inverse( 51 | torch.tensor( 52 | [[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=torch.float32 53 | ) 54 | ) 55 | points_2d = torch.tensor( 56 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=torch.float32 57 | ) 58 | result = local_inv_perspective_projection(intrinsics_inv, points_2d) 59 | expected = torch.tensor([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=torch.float32) 60 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 61 | 62 | def test_global_perspective_projection(self): 63 | # Test with NumPy 64 | intrinsics = np.array( 65 | [[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=np.float32 66 | ) 67 | c2w = np.eye(4, dtype=np.float32) 68 | points_3d_world = np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]], dtype=np.float32) 69 | result, mask = global_perspective_projection(intrinsics, c2w, points_3d_world) 70 | expected = np.array( 71 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=np.float32 72 | ) 73 | np.testing.assert_allclose(result, expected, atol=1e-4) 74 | np.testing.assert_array_equal(mask, [True, True, True]) 75 | 76 | # Test with PyTorch 77 | intrinsics = torch.tensor( 78 | [[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=torch.float32 79 | ) 80 | c2w = torch.eye(4, dtype=torch.float32) 81 | points_3d_world = torch.tensor( 82 | [[1, 1, 1], [2, 2, 2], [3, 3, 3]], dtype=torch.float32 83 | ) 84 | result, mask = global_perspective_projection(intrinsics, c2w, points_3d_world) 85 | expected = torch.tensor( 86 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=torch.float32 87 | ) 88 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 89 | self.assertTrue(torch.equal(mask, torch.tensor([True, True, True]))) 90 | 91 | def test_global_inv_perspective_projection(self): 92 | # Test with NumPy 93 | intrinsics_inv = np.linalg.inv( 94 | np.array([[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=np.float32) 95 | ) 96 | c2w = np.eye(4, dtype=np.float32) 97 | points_2d_screen = np.array( 98 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=np.float32 99 | ) 100 | depth = np.array([1, 2, 3], dtype=np.float32) 101 | result = global_inv_perspective_projection( 102 | intrinsics_inv, c2w, points_2d_screen, depth 103 | ) 104 | expected = np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]], dtype=np.float32) 105 | np.testing.assert_allclose(result, expected, atol=1e-4) 106 | 107 | # Test with PyTorch 108 | intrinsics_inv = torch.inverse( 109 | torch.tensor( 110 | [[1000, 0, 320], [0, 1000, 240], [0, 0, 1]], dtype=torch.float32 111 | ) 112 | ) 113 | c2w = torch.eye(4, dtype=torch.float32) 114 | points_2d_screen = torch.tensor( 115 | [[1320, 1240], [1320, 1240], [1320, 1240]], dtype=torch.float32 116 | ) 117 | depth = torch.tensor([1, 2, 3], dtype=torch.float32) 118 | result = global_inv_perspective_projection( 119 | intrinsics_inv, c2w, points_2d_screen, depth 120 | ) 121 | expected = torch.tensor([[1, 1, 1], [2, 2, 2], [3, 3, 3]], dtype=torch.float32) 122 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 123 | 124 | 125 | if __name__ == "__main__": 126 | unittest.main() 127 | -------------------------------------------------------------------------------- /tests/test_quaternion_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import torch 3 | import numpy as np 4 | from mvdatasets.geometry.quaternions import ( 5 | quat_multiply, 6 | quat_invert, 7 | make_quaternion_deg, 8 | make_quaternion_rad, 9 | rots_to_quats, 10 | angular_distance, 11 | quats_to_rots, 12 | ) 13 | 14 | 15 | class TestQuaternionFunctions(unittest.TestCase): 16 | 17 | def test_quat_multiply(self): 18 | a = torch.tensor([[1.0, 0.0, 0.0, 0.0]]) 19 | b = torch.tensor([[0.0, 1.0, 0.0, 0.0]]) 20 | result = quat_multiply(a, b) 21 | expected = torch.tensor([[0.0, 1.0, 0.0, 0.0]]) 22 | self.assertTrue(torch.allclose(result, expected)) 23 | 24 | # Test associativity of quaternion multiplication 25 | c = torch.tensor([[0.0, 0.0, 1.0, 0.0]]) 26 | result_abc = quat_multiply(quat_multiply(a, b), c) 27 | result_cba = quat_multiply(a, quat_multiply(b, c)) 28 | self.assertTrue(torch.allclose(result_abc, result_cba)) 29 | 30 | def test_quat_invert(self): 31 | q = torch.tensor([[0.7071, 0.7071, 0.0, 0.0]]) 32 | result = quat_invert(q) 33 | expected = torch.tensor([[0.7071, -0.7071, 0.0, 0.0]]) 34 | self.assertTrue(torch.allclose(result, expected)) 35 | 36 | # Verify double inversion gives the original quaternion 37 | double_inverted = quat_invert(result) 38 | self.assertTrue(torch.allclose(double_inverted, q)) 39 | 40 | def test_make_quaternion_deg(self): 41 | q = make_quaternion_deg(90, 0, 0) 42 | expected = torch.tensor([0.7071, 0.7071, 0.0, 0.0]) 43 | self.assertTrue(torch.allclose(q, expected, atol=1e-4)) 44 | 45 | def test_make_quaternion_rad(self): 46 | q = make_quaternion_rad(np.pi / 2, 0, 0) 47 | expected = torch.tensor([0.7071, 0.7071, 0.0, 0.0]) 48 | self.assertTrue(torch.allclose(q, expected, atol=1e-4)) 49 | 50 | def test_rots_to_quats(self): 51 | rots = torch.eye(3).unsqueeze(0) # Single identity matrix 52 | result = rots_to_quats(rots) 53 | expected = torch.tensor([[1.0, 0.0, 0.0, 0.0]]) 54 | self.assertTrue(torch.allclose(result, expected)) 55 | 56 | # Test rotation matrix for a 90-degree rotation around Z-axis 57 | rot_z = torch.tensor([[[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]]) 58 | result = rots_to_quats(rot_z) 59 | expected = torch.tensor([[0.7071, 0.0, 0.0, 0.7071]]) 60 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 61 | 62 | def test_angular_distance(self): 63 | q1 = torch.tensor([[1.0, 0.0, 0.0, 0.0]]) 64 | q2 = torch.tensor([[0.7071, 0.7071, 0.0, 0.0]]) 65 | result = angular_distance(q1, q2) 66 | expected = torch.tensor([0.2929]) # 1 - abs(dot_product) 67 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 68 | 69 | # Test angular distance between identical quaternions 70 | result = angular_distance(q1, q1) 71 | expected = torch.tensor([0.0]) 72 | self.assertTrue(torch.allclose(result, expected)) 73 | 74 | def test_quats_to_rots(self): 75 | quats = torch.tensor([[1.0, 0.0, 0.0, 0.0]]) 76 | result = quats_to_rots(quats) 77 | expected = torch.eye(3).unsqueeze(0) 78 | self.assertTrue(torch.allclose(result, expected)) 79 | 80 | # Test quaternion to rotation matrix for 90-degree rotation around Y-axis 81 | quats = torch.tensor([[0.7071, 0.0, 0.7071, 0.0]]) 82 | result = quats_to_rots(quats) 83 | expected = torch.tensor([[[0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [-1.0, 0.0, 0.0]]]) 84 | self.assertTrue(torch.allclose(result, expected, atol=1e-4)) 85 | 86 | 87 | if __name__ == "__main__": 88 | unittest.main() 89 | -------------------------------------------------------------------------------- /tests/test_rigid_geometry_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import torch 4 | from mvdatasets.geometry.rigid import ( 5 | apply_rotation_3d, 6 | apply_transformation_3d, 7 | pose_local_rotation, 8 | pose_global_rotation, 9 | ) 10 | 11 | 12 | class Test3DTransformations(unittest.TestCase): 13 | 14 | def test_apply_rotation_3d(self): 15 | # Test with NumPy 16 | points = np.array([[1, 0, 0]], dtype=np.float32) 17 | rot = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]], np.float32) 18 | result = apply_rotation_3d(points, rot) 19 | expected = np.array([[0, 1, 0]], dtype=np.float32) 20 | np.testing.assert_allclose(result, expected) 21 | 22 | # Test with batched rotation 23 | points = np.array([[1, 0, 0], [0, 1, 0]], dtype=np.float32) 24 | rot = np.array( 25 | [[[0, -1, 0], [1, 0, 0], [0, 0, 1]], [[0, 1, 0], [-1, 0, 0], [0, 0, 1]]], 26 | dtype=np.float32, 27 | ) 28 | result = apply_rotation_3d(points, rot) 29 | expected = np.array([[0, 1, 0], [1, 0, 0]], dtype=np.float32) 30 | np.testing.assert_allclose(result, expected) 31 | 32 | # Test with PyTorch 33 | points = torch.tensor([[1, 0, 0]], dtype=torch.float32) 34 | rot = torch.tensor([[0, -1, 0], [1, 0, 0], [0, 0, 1]], dtype=torch.float32) 35 | result = apply_rotation_3d(points, rot) 36 | expected = torch.tensor([[0, 1, 0]], dtype=torch.float32) 37 | self.assertTrue(torch.allclose(result, expected)) 38 | 39 | def test_apply_transformation_3d(self): 40 | # Test with NumPy 41 | points = np.array([[1, 0, 0]], dtype=np.float32) 42 | transform = np.array( 43 | [[0, -1, 0, 1], [1, 0, 0, 2], [0, 0, 1, 3], [0, 0, 0, 1]], dtype=np.float32 44 | ) 45 | result = apply_transformation_3d(points, transform) 46 | expected = np.array([[1, 3, 3]], dtype=np.float32) 47 | np.testing.assert_allclose(result, expected) 48 | 49 | # Test with PyTorch 50 | points = torch.tensor([[1, 0, 0]], dtype=torch.float32) 51 | transform = torch.tensor( 52 | [[0, -1, 0, 1], [1, 0, 0, 2], [0, 0, 1, 3], [0, 0, 0, 1]], 53 | dtype=torch.float32, 54 | ) 55 | result = apply_transformation_3d(points, transform) 56 | expected = torch.tensor([[1, 3, 3]], dtype=torch.float32) 57 | self.assertTrue(torch.allclose(result, expected)) 58 | 59 | def test_pose_local_rotation(self): 60 | # Test with NumPy 61 | pose = np.eye(4, dtype=np.float32) 62 | rotation = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]], dtype=np.float32) 63 | result = pose_local_rotation(pose, rotation) 64 | expected = np.array( 65 | [[0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=np.float32 66 | ) 67 | np.testing.assert_allclose(result, expected) 68 | 69 | # Test with PyTorch 70 | pose = torch.eye(4, dtype=torch.float32) 71 | rotation = torch.tensor([[0, -1, 0], [1, 0, 0], [0, 0, 1]], dtype=torch.float32) 72 | result = pose_local_rotation(pose, rotation) 73 | expected = torch.tensor( 74 | [[0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], 75 | dtype=torch.float32, 76 | ) 77 | self.assertTrue(torch.allclose(result, expected)) 78 | 79 | def test_pose_global_rotation(self): 80 | # Test with NumPy 81 | pose = np.eye(4, dtype=np.float32) 82 | rotation = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]], dtype=np.float32) 83 | result = pose_global_rotation(pose, rotation) 84 | expected = np.array( 85 | [[0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=np.float32 86 | ) 87 | np.testing.assert_allclose(result, expected) 88 | 89 | # Test with PyTorch 90 | pose = torch.eye(4, dtype=torch.float32) 91 | rotation = torch.tensor([[0, -1, 0], [1, 0, 0], [0, 0, 1]], dtype=torch.float32) 92 | result = pose_global_rotation(pose, rotation) 93 | expected = torch.tensor( 94 | [[0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], 95 | dtype=torch.float32, 96 | ) 97 | self.assertTrue(torch.allclose(result, expected)) 98 | 99 | 100 | if __name__ == "__main__": 101 | unittest.main() 102 | --------------------------------------------------------------------------------