├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── baselines └── third_party │ ├── evaluation │ ├── LICENSE │ ├── README.md │ ├── elpips │ │ ├── LICENSE │ │ ├── __init__.py │ │ ├── elpips.py │ │ ├── networks.py │ │ ├── pnetlin.py │ │ └── util.py │ ├── environment.yml │ ├── lpips_tf.py │ ├── run_evaluation.py │ ├── run_evaluation_nerf.sh │ ├── run_evaluation_ours.sh │ ├── run_evaluation_xfields.sh │ └── setup.py │ ├── nerf_tensorflow │ ├── LICENSE │ ├── README.facebook │ ├── README.md │ ├── configs │ │ ├── llff_config.txt │ │ ├── shiny_config.txt │ │ ├── shiny_config_dense.txt │ │ ├── stanford_config.txt │ │ └── tanks_config.txt │ ├── download_example_data.sh │ ├── download_example_weights.sh │ ├── environment.yml │ ├── extract_subdivision.py │ ├── load_blender.py │ ├── load_deepvoxels.py │ ├── load_llff.py │ ├── load_shiny.py │ ├── load_stanford.py │ ├── load_tanks.py │ ├── run_nerf.py │ └── run_nerf_helpers.py │ └── xfields │ ├── LICENSE │ ├── README.facebook │ ├── README.md │ ├── bilinear_sampler.py │ ├── blending.py │ ├── environment.yml │ ├── flow_vis.py │ ├── load_imgs.py │ ├── model.py │ ├── test.py │ ├── test_xfields.sh │ ├── train.py │ └── train_xfields.sh ├── conf ├── __init__.py ├── config.yaml └── experiment │ ├── blender_subdivided.yaml │ ├── dataset │ ├── blender.yaml │ ├── llff.yaml │ ├── llff_large.yaml │ ├── llff_undistort.yaml │ ├── llff_undistort_large.yaml │ ├── shiny.yaml │ ├── shiny_dense.yaml │ ├── shiny_dense_large.yaml │ ├── shiny_large.yaml │ ├── shiny_small.yaml │ ├── stanford.yaml │ ├── stanford_beans.yaml │ ├── stanford_bracelet.yaml │ ├── stanford_bulldozer.yaml │ ├── stanford_bunny.yaml │ ├── stanford_chess.yaml │ ├── stanford_flowers.yaml │ ├── stanford_gem.yaml │ ├── stanford_knights.yaml │ ├── stanford_tarot.yaml │ ├── stanford_tarot_small.yaml │ ├── stanford_treasure.yaml │ ├── stanford_truck.yaml │ ├── tamul.yaml │ ├── tanks.yaml │ ├── xfields_island.yaml │ └── xfields_splash.yaml │ ├── llff_subdivided.yaml │ ├── llff_subdivided_half.yaml │ ├── local.yaml │ ├── model │ ├── blender_subdivided.yaml │ ├── llff_affine.yaml │ ├── llff_feature.yaml │ ├── llff_half.yaml │ ├── llff_subdivided_affine.yaml │ ├── llff_subdivided_feature.yaml │ ├── llff_subdivided_half.yaml │ ├── llff_subdivided_no_embed.yaml │ ├── shiny_affine.yaml │ ├── shiny_subdivided_affine.yaml │ ├── shiny_subdivided_affine_dense.yaml │ ├── shiny_subdivided_pluecker.yaml │ ├── stanford_affine.yaml │ ├── stanford_affine_pluecker.yaml │ ├── stanford_feature.yaml │ ├── stanford_half.yaml │ ├── stanford_no_embed.yaml │ └── stanford_pluecker.yaml │ ├── params │ └── local.yaml │ ├── regularizers │ ├── all.yaml │ ├── blurry_teacher │ │ └── default.yaml │ ├── depth_classification │ │ └── default.yaml │ ├── fourier │ │ └── default.yaml │ ├── inverse_ray_depth │ │ └── default.yaml │ ├── multiple_ray_depth │ │ └── default.yaml │ ├── ray_bundle │ │ └── default.yaml │ ├── ray_depth │ │ └── default.yaml │ ├── ray_depth_blending │ │ └── default.yaml │ ├── ray_depth_occ_dir │ │ └── default.yaml │ ├── ray_interpolation │ │ └── default.yaml │ ├── teacher │ │ └── default.yaml │ ├── voxel_sparsity │ │ └── default.yaml │ ├── warp_level │ │ ├── lf.yaml │ │ └── subdivided.yaml │ └── weak_teacher │ │ ├── lf.yaml │ │ └── subdivided.yaml │ ├── shiny_lf.yaml │ ├── shiny_lf_dense.yaml │ ├── shiny_subdivided.yaml │ ├── shiny_subdivided_dense.yaml │ ├── stanford_lf.yaml │ ├── stanford_lf_half.yaml │ ├── training │ ├── blender_subdivided.yaml │ ├── default.yaml │ ├── llff_lf.yaml │ ├── llff_lf_large.yaml │ ├── llff_subdivided.yaml │ ├── llff_subdivided_half.yaml │ ├── shiny_lf.yaml │ ├── shiny_subdivided.yaml │ ├── shiny_subdivided_dense.yaml │ ├── stanford_lf.yaml │ ├── stanford_lf_half.yaml │ └── stanford_subdivided.yaml │ └── visualizers │ ├── all.yaml │ ├── closest_view │ └── default.yaml │ ├── epipolar │ └── default.yaml │ └── focus │ └── default.yaml ├── datasets ├── __init__.py ├── base.py ├── blender.py ├── fourier.py ├── lightfield.py ├── llff.py ├── random.py ├── shiny.py ├── stanford.py ├── tamul.py └── tanks.py ├── environment.yml ├── environment_win.yml ├── losses.py ├── main.py ├── metrics.py ├── nlf ├── __init__.py ├── activations.py ├── embedding.py ├── models.py ├── nets.py ├── regularizers │ ├── __init__.py │ ├── base.py │ ├── depth_classification.py │ ├── fourier.py │ ├── inverse_ray_depth.py │ ├── multiple_ray_depth.py │ ├── ray_blending.py │ ├── ray_bundle.py │ ├── ray_depth.py │ ├── ray_depth_depth_class.py │ ├── ray_depth_multi_depth.py │ ├── ray_depth_occ_dir.py │ ├── ray_depth_occ_weight.py │ ├── ray_interpolation.py │ ├── teacher.py │ ├── voxel_sparsity.py │ └── warp.py ├── rendering.py ├── subdivision.py └── visualizers │ ├── __init__.py │ ├── base.py │ ├── closest_view.py │ ├── epipolar.py │ └── focus.py ├── scripts ├── run_all.sh ├── run_all_llff.sh ├── run_all_paper.sh ├── run_all_shiny.sh ├── run_all_stanford.sh ├── run_one_lf.sh ├── run_one_lf_reg.sh ├── run_one_lf_teacher.sh ├── run_one_subdivided.sh ├── run_one_subdivided_few.sh ├── run_one_subdivided_reg.sh ├── run_one_subdivided_reg_small.sh ├── run_one_subdivided_small.sh ├── run_one_subdivided_teacher.sh └── run_one_subdivided_teacher_few.sh ├── setup.py ├── third_party └── intersect │ ├── README.facebook │ ├── __init__.py │ ├── binding.cpp │ ├── cuda_utils.h │ ├── cutil_math.h │ ├── intersect.cpp │ ├── intersect.h │ ├── intersect_gpu.cu │ ├── octree.cpp │ ├── octree.h │ ├── sample.cpp │ ├── sample.h │ ├── sample_gpu.cu │ ├── utils.h │ └── utils.py ├── utils ├── __init__.py ├── config_utils.py ├── intersect_utils.py ├── pose_utils.py ├── ray_utils.py ├── visualization.py └── warmup_scheduler.py └── web_teaser.svg /.gitignore: -------------------------------------------------------------------------------- 1 | # Neural Light Fields 2 | data/ 3 | checkpoints/ 4 | logs/ 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | outputs/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 111 | __pypackages__/ 112 | 113 | # Celery stuff 114 | celerybeat-schedule 115 | celerybeat.pid 116 | 117 | # SageMath parsed files 118 | *.sage.py 119 | 120 | # Environments 121 | .env 122 | .venv 123 | env/ 124 | venv/ 125 | ENV/ 126 | env.bak/ 127 | venv.bak/ 128 | 129 | # Spyder project settings 130 | .spyderproject 131 | .spyproject 132 | 133 | # Rope project settings 134 | .ropeproject 135 | 136 | # mkdocs documentation 137 | /site 138 | 139 | # mypy 140 | .mypy_cache/ 141 | .dmypy.json 142 | dmypy.json 143 | 144 | # Pyre type checker 145 | .pyre/ 146 | 147 | # pytype static type analyzer 148 | .pytype/ 149 | 150 | # Cython debug symbols 151 | cython_debug/ 152 | 153 | # PyCharm 154 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 155 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 156 | # and can be added to the global gitignore or merged into this file. For a more nuclear 157 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 158 | #.idea/ 159 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ============= 3 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | This Code of Conduct also applies outside the project spaces when there is a 56 | reasonable belief that an individual's behavior may have a negative impact on 57 | the project or its community. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported by contacting the project team at . All 63 | complaints will be reviewed and investigated and will result in a response that 64 | is deemed necessary and appropriate to the circumstances. The project team is 65 | obligated to maintain confidentiality with regard to the reporter of an incident. 66 | Further details of specific enforcement policies may be posted separately. 67 | 68 | Project maintainers who do not follow or enforce the Code of Conduct in good 69 | faith may face temporary or permanent repercussions as determined by other 70 | members of the project's leadership. 71 | 72 | ## Attribution 73 | 74 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 75 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 76 | 77 | [homepage]: https://www.contributor-covenant.org 78 | 79 | For answers to common questions about this code of conduct, see 80 | https://www.contributor-covenant.org/faq 81 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Neural Light Fields 2 | We want to make contributing to this project as easy and transparent as possible. 3 | 4 | ## Pull Requests 5 | We welcome pull requests. 6 | 7 | 1. Fork the repo and create your branch from `master`. 8 | 2. If you haven't already, complete the Contributor License Agreement ("CLA"). 9 | 10 | ## Contributor License Agreement ("CLA") 11 | In order to accept your pull request, we need you to submit a CLA. You only need 12 | to do this once to work on any of Facebook's open source projects. 13 | 14 | Complete your CLA here: 15 | 16 | ## Issues 17 | We use GitHub issues to track public bugs. Please ensure your description is 18 | clear and has sufficient instructions to be able to reproduce the issue. 19 | 20 | ## License 21 | By contributing to Neural Light Fields, you agree that your contributions will be licensed 22 | under the [LICENSE](./LICENSE) file in the root directory of this source tree. 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Meta Platforms, Inc. and affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2018, alexlee-gk 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/README.md: -------------------------------------------------------------------------------- 1 | # lpips-tensorflow 2 | Tensorflow port for the [PyTorch](https://github.com/richzhang/PerceptualSimilarity) implementation of the [Learned Perceptual Image Patch Similarity (LPIPS)](http://richzhang.github.io/PerceptualSimilarity/) metric. 3 | This is done by exporting the model from PyTorch to ONNX and then to TensorFlow. 4 | 5 | ## Getting started 6 | ### Installation 7 | - Clone this repo. 8 | ```bash 9 | git clone https://github.com/alexlee-gk/lpips-tensorflow.git 10 | cd lpips-tensorflow 11 | ``` 12 | - Install TensorFlow and dependencies from http://tensorflow.org/ 13 | - Install other dependencies. 14 | ```bash 15 | pip install -r requirements.txt 16 | ``` 17 | 18 | ### Using the LPIPS metric 19 | The `lpips` TensorFlow function works with individual images or batches of images. 20 | It also works with images of any spatial dimensions (but the dimensions should be at least the size of the network's receptive field). 21 | This example computes the LPIPS distance between batches of images. 22 | ```python 23 | import numpy as np 24 | import tensorflow as tf 25 | import lpips_tf 26 | 27 | batch_size = 32 28 | image_shape = (batch_size, 64, 64, 3) 29 | image0 = np.random.random(image_shape) 30 | image1 = np.random.random(image_shape) 31 | image0_ph = tf.placeholder(tf.float32) 32 | image1_ph = tf.placeholder(tf.float32) 33 | 34 | distance_t = lpips_tf.lpips(image0_ph, image1_ph, model='net-lin', net='alex') 35 | 36 | with tf.Session() as session: 37 | distance = session.run(distance_t, feed_dict={image0_ph: image0, image1_ph: image1}) 38 | ``` 39 | 40 | ## Exporting additional models 41 | ### Export PyTorch model to TensorFlow through ONNX 42 | - Clone the PerceptualSimilarity submodule and add it to the PYTHONPATH. 43 | ```bash 44 | git submodule update --init --recursive 45 | export PYTHONPATH=PerceptualSimilarity:$PYTHONPATH 46 | ``` 47 | - Install more dependencies. 48 | ```bash 49 | pip install -r requirements-dev.txt 50 | ``` 51 | - Export the model to ONNX *.onnx and TensorFlow *.pb files in the `models` directory. 52 | ```bash 53 | python export_to_tensorflow.py --model net-lin --net alex 54 | ``` 55 | 56 | ### Known issues 57 | - The SqueezeNet model cannot be exported since ONNX cannot export one of the operators. 58 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/elpips/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018-2019, Markus Kettunen, Seyoung Park 2 | Copyright (c) 2018, Richard Zhang, Phillip Isola, Alexei A. Efros, Eli Shechtman, Oliver Wang 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 19 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /baselines/third_party/evaluation/elpips/__init__.py: -------------------------------------------------------------------------------- 1 | from . import elpips 2 | import tensorflow as tf 3 | 4 | 5 | Config = elpips.Config 6 | 7 | def elpips_squeeze_maxpool(batch_size=1, n=1,dtype=tf.float32): 8 | '''E-LPIPS-SQUEEZENET-MAXPOOL configuration with all input transformations and dropout with p=0.99. Returns the average result over n samples. Does not use average pooling since SqueezeNet would require re-training for that. Experimental! 9 | Warning: Some versions of TensorFlow might have bugs that make n > 1 problematic due to the tf.while_loop used internally when n > 1. 10 | ''' 11 | config = Config() 12 | config.metric = 'squeeze_ensemble_maxpool' 13 | config.batch_size = batch_size 14 | config.average_over = n 15 | config.dtype = dtype 16 | 17 | return config 18 | 19 | def elpips_vgg(batch_size=1, n=1, dtype=tf.float32): 20 | '''E-LPIPS-VGG configuration with all input transformations and dropout with p=0.99. Returns the average result over n samples. 21 | Warning: Some versions of TensorFlow might have bugs that make n > 1 problematic due to the tf.while_loop used internally when n > 1.''' 22 | config = Config() 23 | config.metric = 'vgg_ensemble' 24 | config.batch_size = batch_size 25 | config.average_over = n 26 | config.dtype = dtype 27 | 28 | return config 29 | 30 | def lpips_squeeze(batch_size=1, dtype=tf.float32): 31 | '''Plain LPIPS-SQUEEZE configuration.''' 32 | config = Config() 33 | config.metric = 'squeeze' 34 | config.enable_dropout = False 35 | config.enable_offset = False 36 | config.enable_flip = False 37 | config.enable_swap = False 38 | config.enable_color_permutation = False 39 | config.enable_color_multiplication = False 40 | config.enable_scale = False 41 | config.batch_size = batch_size 42 | config.average_over = 1 43 | config.dtype = dtype 44 | return config 45 | 46 | def lpips_vgg(batch_size=1, dtype=tf.float32): 47 | '''No augmentations.''' 48 | config = Config() 49 | config.metric = 'vgg' 50 | config.enable_dropout = False 51 | config.enable_offset = False 52 | config.enable_flip = False 53 | config.enable_swap = False 54 | config.enable_color_permutation = False 55 | config.enable_color_multiplication = False 56 | config.enable_scale = False 57 | config.batch_size = batch_size 58 | config.average_over = 1 59 | config.dtype = dtype 60 | return config 61 | 62 | 63 | def get_config(config_name, batch_size=1, n=1, dtype=tf.float32): 64 | '''Returns a config name by string.''' 65 | 66 | if config_name == 'elpips_vgg': 67 | return elpips_vgg(batch_size=batch_size, n=n, dtype=dtype) 68 | elif config_name == 'elpips_squeeze_maxpool': 69 | config = elpips_squeeze_maxpool(batch_size=batch_size, n=n, dtype=dtype) 70 | return config 71 | elif config_name == 'lpips_squeeze': 72 | return lpips_squeeze(batch_size=batch_size, dtype=dtype) 73 | elif config_name == 'lpips_vgg': 74 | return lpips_vgg(batch_size=batch_size, dtype=dtype) 75 | else: 76 | raise Exception('Unknown config_name.') 77 | 78 | 79 | Metric = elpips.Metric -------------------------------------------------------------------------------- /baselines/third_party/evaluation/elpips/util.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | 5 | def switch_case_cond(cases, default_case): 6 | if cases: 7 | condition, effect = cases[0] 8 | return tf.cond(condition, effect, lambda: switch_case_cond(cases[1:], default_case)) 9 | return default_case() 10 | 11 | def switch_case_where(cases, default_case): 12 | if cases: 13 | condition, effect = cases[0] 14 | return tf.where(condition, effect, switch_case_where(cases[1:], default_case)) 15 | return default_case 16 | 17 | 18 | def np_dtype(tf_dtype): 19 | if tf_dtype == tf.float32: 20 | return np.float32 21 | if tf_dtype == tf.float64: 22 | return np.float64 23 | raise Exception('Unsupported dtype') 24 | 25 | def f32_to_dtype(x, dtype): 26 | if dtype == tf.float32: 27 | return x 28 | return tf.cast(x, dtype) 29 | 30 | 31 | def as_tuple(x): 32 | '''Formats x as a tuple. If x is already a tuple returns it as is, otherwise returns a 1-tuple containing x.''' 33 | if isinstance(x, tuple): 34 | return x 35 | else: 36 | return (x,) 37 | 38 | def for_each(x, func): 39 | '''Runs 'func' for x, or each item of x if x is a tuple. Returns the results in the same format.''' 40 | if isinstance(x, tuple): 41 | return tuple((func(s) for s in x)) 42 | else: 43 | return func(x) 44 | 45 | def for_tuple(x, func): 46 | '''Runs 'func' for as_tuple(x). Returns the results in the original format. Assumes 'func' returns tuple when given tuple.''' 47 | if isinstance(x, tuple): 48 | return func(x) 49 | else: 50 | return func(as_tuple(x))[0] 51 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/environment.yml: -------------------------------------------------------------------------------- 1 | # To run: conda env create -f environment.yml 2 | name: eval 3 | channels: 4 | - conda-forge 5 | - pytorch 6 | dependencies: 7 | - python==3.7 8 | - pip 9 | - cudatoolkit=10.0 10 | - imagemagick 11 | - pip: 12 | - tensorflow-gpu==1.14 13 | - numpy 14 | - opencv-python 15 | - imageio 16 | - scikit-image==0.18.0 17 | - numba 18 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/lpips_tf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import tensorflow as tf 5 | from six.moves import urllib 6 | 7 | _URL = 'http://rail.eecs.berkeley.edu/models/lpips' 8 | 9 | 10 | 11 | def lpips(input0, input1, model='net-lin', net='alex', version=0.1): 12 | """ 13 | Learned Perceptual Image Patch Similarity (LPIPS) metric. 14 | 15 | Args: 16 | input0: An image tensor of shape `[..., height, width, channels]`, 17 | with values in [0, 1]. 18 | input1: An image tensor of shape `[..., height, width, channels]`, 19 | with values in [0, 1]. 20 | 21 | Returns: 22 | The Learned Perceptual Image Patch Similarity (LPIPS) distance. 23 | 24 | Reference: 25 | Richard Zhang, Phillip Isola, Alexei A. Efros, Eli Shechtman, Oliver Wang. 26 | The Unreasonable Effectiveness of Deep Features as a Perceptual Metric. 27 | In CVPR, 2018. 28 | """ 29 | # flatten the leading dimensions 30 | batch_shape = tf.shape(input0)[:-3] 31 | input0 = tf.reshape(input0, tf.concat([[-1], tf.shape(input0)[-3:]], axis=0)) 32 | input1 = tf.reshape(input1, tf.concat([[-1], tf.shape(input1)[-3:]], axis=0)) 33 | # NHWC to NCHW 34 | input0 = tf.transpose(input0, [0, 3, 1, 2]) 35 | input1 = tf.transpose(input1, [0, 3, 1, 2]) 36 | # normalize to [-1, 1] 37 | input0 = input0 * 2.0 - 1.0 38 | input1 = input1 * 2.0 - 1.0 39 | 40 | input0_name, input1_name = '0:0', '1:0' 41 | 42 | default_graph = tf.get_default_graph() 43 | producer_version = default_graph.graph_def_versions.producer 44 | 45 | cache_dir = os.path.expanduser('.') 46 | os.makedirs(cache_dir, exist_ok=True) 47 | # files to try. try a specific producer version, but fallback to the version-less version (latest). 48 | pb_fnames = [ 49 | '%s_%s_v%s_%d.pb' % (model, net, version, producer_version), 50 | '%s_%s_v%s.pb' % (model, net, version), 51 | ] 52 | for pb_fname in pb_fnames: 53 | if os.path.isfile(os.path.join(cache_dir, pb_fname)): 54 | break 55 | 56 | with open(os.path.join(cache_dir, pb_fname), 'rb') as f: 57 | graph_def = tf.GraphDef() 58 | graph_def.ParseFromString(f.read()) 59 | _ = tf.import_graph_def(graph_def, 60 | input_map={input0_name: input0, input1_name: input1}) 61 | distance, = default_graph.get_operations()[-1].outputs 62 | 63 | if distance.shape.ndims == 4: 64 | distance = tf.squeeze(distance, axis=[-3, -2, -1]) 65 | # reshape the leading dimensions 66 | distance = tf.reshape(distance, batch_shape) 67 | return distance 68 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/run_evaluation_nerf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ~/miniconda3/bin/activate eval 4 | 5 | HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 \ 6 | CUDA_VISIBLE_DEVICES=$3 python run_evaluation.py \ 7 | --mode nerf \ 8 | --gt_dir ~/local/results/nerf_$1/testset_$2 \ 9 | --pred_dir ~/local/results/nerf_$1/testset_$2 \ 10 | --out_dir eval/eval_nerf_$1 \ 11 | --out_file metrics/nerf_$1.txt 12 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/run_evaluation_ours.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ~/miniconda3/bin/activate eval 4 | 5 | HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 \ 6 | CUDA_VISIBLE_DEVICES=$2 python run_evaluation.py \ 7 | --mode ours \ 8 | --gt_dir ~/local/results/ours_$1/ignore \ 9 | --pred_dir ~/local/results/ours_$1/ignore \ 10 | --out_dir eval/eval_ours_$1 \ 11 | --out_file metrics/ours_$1.txt 12 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/run_evaluation_xfields.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ~/miniconda3/bin/activate eval 4 | 5 | HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 \ 6 | CUDA_VISIBLE_DEVICES=$2 python run_evaluation.py \ 7 | --mode xfields \ 8 | --gt_dir ~/local/data/stanford_half/$1 \ 9 | --pred_dir ~/local/results/xfields_$3/rendered_images \ 10 | --out_dir eval/eval_xfields_$3 \ 11 | --out_file metrics/xfields_$3.txt 12 | -------------------------------------------------------------------------------- /baselines/third_party/evaluation/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup( 6 | name='lpips-tf', 7 | description='Tensorflow port for the Learned Perceptual Image Patch Similarity (LPIPS) metric', 8 | author='Alex Lee', 9 | url='https://github.com/alexlee-gk/lpips-tensorflow/', 10 | py_modules=['lpips_tf'] 11 | ) 12 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 bmild 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/README.facebook: -------------------------------------------------------------------------------- 1 | Project: NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis 2 | Version: https://github.com/bmild/nerf/tree/20a91e764a28816ee2234fcadb73bd59a613a44c 3 | Website: https://github.com/bmild/nerf 4 | License: MIT License 5 | Description: Official implementation of this paper: 6 | 7 | @inproceedings{mildenhall2020nerf, 8 | title={NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis}, 9 | author={Ben Mildenhall and Pratul P. Srinivasan and Matthew Tancik and Jonathan T. Barron and Ravi Ramamoorthi and Ren Ng}, 10 | year={2020}, 11 | booktitle={ECCV}, 12 | } 13 | 14 | Changes: Stanford Light Field Dataset loading 15 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/configs/llff_config.txt: -------------------------------------------------------------------------------- 1 | # This replicates the paper result for "Fern" 2 | # when trained to 200k iters. Settings are the 3 | # same for all other LLFF-style real scenes. 4 | 5 | expname = nerf_fern 6 | basedir = ~/local/logs 7 | datadir = ~/local/data/nerf_llff_data/fern 8 | dataset_type = llff 9 | factor = 8 10 | llffhold = 8 11 | 12 | no_batching = False 13 | 14 | use_viewdirs = True 15 | lrate_decay = 250 16 | raw_noise_std = 1.0 17 | 18 | N_samples = 64 19 | N_importance = 128 20 | N_rand = 1024 21 | 22 | i_weights = 20000 23 | i_video = 200000 24 | i_testset = 40000 25 | 26 | render_teacher = False 27 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/configs/shiny_config.txt: -------------------------------------------------------------------------------- 1 | # This replicates the paper result for "Fern" 2 | # when trained to 200k iters. Settings are the 3 | # same for all other LLFF-style real scenes. 4 | 5 | expname = nerf_cd 6 | basedir = ~/local/logs 7 | datadir = ~/local/data/shiny/cd 8 | dataset_type = shiny 9 | factor = 8 10 | llffhold = 8 11 | 12 | no_batching = False 13 | 14 | use_viewdirs = True 15 | lrate_decay = 250 16 | raw_noise_std = 1.0 17 | 18 | N_samples = 64 19 | N_importance = 128 20 | N_rand = 1024 21 | 22 | i_weights = 20000 23 | i_video = 200000 24 | i_testset = 40000 25 | 26 | render_teacher = True 27 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/configs/shiny_config_dense.txt: -------------------------------------------------------------------------------- 1 | # This replicates the paper result for "Fern" 2 | # when trained to 200k iters. Settings are the 3 | # same for all other LLFF-style real scenes. 4 | 5 | expname = nerf_cd 6 | basedir = ~/local/logs 7 | datadir = ~/local/data/shiny/cd 8 | dataset_type = shiny 9 | factor = 8 10 | llffhold = 8 11 | 12 | no_batching = False 13 | 14 | use_viewdirs = True 15 | lrate_decay = 250 16 | raw_noise_std = 1.0 17 | 18 | N_samples = 64 19 | N_importance = 128 20 | N_rand = 1024 21 | 22 | i_weights = 10000 23 | i_video = 100000 24 | i_testset = 10000 25 | 26 | render_teacher = True 27 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/configs/stanford_config.txt: -------------------------------------------------------------------------------- 1 | # This replicates the paper result for "Fern" 2 | # when trained to 200k iters. Settings are the 3 | # same for all other LLFF-style real scenes. 4 | 5 | expname = nerf_gem 6 | basedir = ~/local/logs 7 | datadir = ~/local/data/stanford_eval/gem 8 | dataset_type = stanford 9 | factor = 2 10 | 11 | no_batching = False 12 | 13 | use_viewdirs = True 14 | lrate_decay = 250 15 | raw_noise_std = 1.0 16 | 17 | N_samples = 64 18 | N_importance = 128 19 | N_rand = 1024 20 | 21 | i_weights = 10000 22 | i_video = 100000 23 | i_testset = 100000 24 | 25 | render_teacher = True 26 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/configs/tanks_config.txt: -------------------------------------------------------------------------------- 1 | # This replicates the paper result for "Fern" 2 | # when trained to 200k iters. Settings are the 3 | # same for all other LLFF-style real scenes. 4 | 5 | expname = tanks_paper_Truck 6 | basedir = ~/local/logs 7 | datadir = ~/local/data/tanks/Truck 8 | dataset_type = tanks 9 | factor = 4 10 | llffhold = 8 11 | 12 | no_batching = True 13 | 14 | use_viewdirs = True 15 | #lrate_decay = 250 16 | lrate_decay = 500 17 | #raw_noise_std = 1.0 18 | 19 | N_samples = 64 20 | N_importance = 128 21 | N_rand = 1024 22 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/download_example_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wget http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/tiny_nerf_data.npz 4 | mkdir -p data 5 | cd data 6 | wget http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/nerf_example_data.zip 7 | unzip nerf_example_data.zip 8 | cd .. 9 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/download_example_weights.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p logs 4 | cd logs 5 | wget http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/fern_example_weights.zip 6 | unzip fern_example_weights.zip 7 | wget http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/lego_example_weights.zip 8 | unzip lego_example_weights.zip 9 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/environment.yml: -------------------------------------------------------------------------------- 1 | # To run: conda env create -f environment.yml 2 | name: nerf_tensorflow 3 | channels: 4 | - conda-forge 5 | - pytorch 6 | dependencies: 7 | - python==3.7 8 | - pip 9 | - cudatoolkit=11.0 10 | - imagemagick 11 | - pip: 12 | - tensorflow-gpu==2.4 13 | - numpy 14 | - matplotlib 15 | - imageio 16 | - imageio-ffmpeg 17 | - configargparse 18 | - PyMCubes 19 | - scipy 20 | - trimesh 21 | - pyrender 22 | -------------------------------------------------------------------------------- /baselines/third_party/nerf_tensorflow/load_deepvoxels.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import imageio 4 | 5 | 6 | def load_dv_data(scene='cube', basedir='/data/deepvoxels', testskip=8): 7 | 8 | 9 | def parse_intrinsics(filepath, trgt_sidelength, invert_y=False): 10 | # Get camera intrinsics 11 | with open(filepath, 'r') as file: 12 | f, cx, cy = list(map(float, file.readline().split()))[:3] 13 | grid_barycenter = np.array(list(map(float, file.readline().split()))) 14 | near_plane = float(file.readline()) 15 | scale = float(file.readline()) 16 | height, width = map(float, file.readline().split()) 17 | 18 | try: 19 | world2cam_poses = int(file.readline()) 20 | except ValueError: 21 | world2cam_poses = None 22 | 23 | if world2cam_poses is None: 24 | world2cam_poses = False 25 | 26 | world2cam_poses = bool(world2cam_poses) 27 | 28 | print(cx,cy,f,height,width) 29 | 30 | cx = cx / width * trgt_sidelength 31 | cy = cy / height * trgt_sidelength 32 | f = trgt_sidelength / height * f 33 | 34 | fx = f 35 | if invert_y: 36 | fy = -f 37 | else: 38 | fy = f 39 | 40 | # Build the intrinsic matrices 41 | full_intrinsic = np.array([[fx, 0., cx, 0.], 42 | [0., fy, cy, 0], 43 | [0., 0, 1, 0], 44 | [0, 0, 0, 1]]) 45 | 46 | return full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses 47 | 48 | 49 | def load_pose(filename): 50 | assert os.path.isfile(filename) 51 | nums = open(filename).read().split() 52 | return np.array([float(x) for x in nums]).reshape([4,4]).astype(np.float32) 53 | 54 | 55 | H = 512 56 | W = 512 57 | deepvoxels_base = '{}/train/{}/'.format(basedir, scene) 58 | 59 | full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses = parse_intrinsics(os.path.join(deepvoxels_base, 'intrinsics.txt'), H) 60 | print(full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses) 61 | focal = full_intrinsic[0,0] 62 | print(H, W, focal) 63 | 64 | 65 | def dir2poses(posedir): 66 | poses = np.stack([load_pose(os.path.join(posedir, f)) for f in sorted(os.listdir(posedir)) if f.endswith('txt')], 0) 67 | transf = np.array([ 68 | [1,0,0,0], 69 | [0,-1,0,0], 70 | [0,0,-1,0], 71 | [0,0,0,1.], 72 | ]) 73 | poses = poses @ transf 74 | poses = poses[:,:3,:4].astype(np.float32) 75 | return poses 76 | 77 | posedir = os.path.join(deepvoxels_base, 'pose') 78 | poses = dir2poses(posedir) 79 | testposes = dir2poses('{}/test/{}/pose'.format(basedir, scene)) 80 | testposes = testposes[::testskip] 81 | valposes = dir2poses('{}/validation/{}/pose'.format(basedir, scene)) 82 | valposes = valposes[::testskip] 83 | 84 | imgfiles = [f for f in sorted(os.listdir(os.path.join(deepvoxels_base, 'rgb'))) if f.endswith('png')] 85 | imgs = np.stack([imageio.imread(os.path.join(deepvoxels_base, 'rgb', f))/255. for f in imgfiles], 0).astype(np.float32) 86 | 87 | 88 | testimgd = '{}/test/{}/rgb'.format(basedir, scene) 89 | imgfiles = [f for f in sorted(os.listdir(testimgd)) if f.endswith('png')] 90 | testimgs = np.stack([imageio.imread(os.path.join(testimgd, f))/255. for f in imgfiles[::testskip]], 0).astype(np.float32) 91 | 92 | valimgd = '{}/validation/{}/rgb'.format(basedir, scene) 93 | imgfiles = [f for f in sorted(os.listdir(valimgd)) if f.endswith('png')] 94 | valimgs = np.stack([imageio.imread(os.path.join(valimgd, f))/255. for f in imgfiles[::testskip]], 0).astype(np.float32) 95 | 96 | all_imgs = [imgs, valimgs, testimgs] 97 | counts = [0] + [x.shape[0] for x in all_imgs] 98 | counts = np.cumsum(counts) 99 | i_split = [np.arange(counts[i], counts[i+1]) for i in range(3)] 100 | 101 | imgs = np.concatenate(all_imgs, 0) 102 | poses = np.concatenate([poses, valposes, testposes], 0) 103 | 104 | render_poses = testposes 105 | 106 | print(poses.shape, imgs.shape) 107 | 108 | return imgs, poses, render_poses, [H,W,focal], i_split 109 | 110 | 111 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Mojtaba Bemana 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/README.facebook: -------------------------------------------------------------------------------- 1 | Project: X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation 2 | Version: 1.0 3 | Release date: 10-18-2020 4 | Website: https://github.com/m-bemana/xfields 5 | License: MIT 6 | Description: Official implementation of 7 | 8 | @article{Bemana2020xfields, 9 | author = {Mojtaba Bemana, Karol Myszkowski, Hans-Peter Seidel, Tobias Ritschel}, 10 | title = {X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation}, 11 | journal = {ACM Transactions on Graphics (Proc. SIGGRAPH Asia 2020)}, 12 | year = {2020}, 13 | volume = {39}, 14 | number = {6}, 15 | doi = {10.1145/3414685.3417827} 16 | } 17 | 18 | Changes: Modifications to allow for spiral rendering, stanford light field evaluation 19 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/environment.yml: -------------------------------------------------------------------------------- 1 | # To run: conda env create -f environment.yml 2 | name: xfields 3 | channels: 4 | - conda-forge 5 | - pytorch 6 | dependencies: 7 | - python==3.7 8 | - pip 9 | - cudatoolkit=10.0 10 | - imagemagick 11 | - pip: 12 | - tensorflow-gpu==1.14 13 | - tensorlayer 14 | - numpy 15 | - opencv-python 16 | - imageio 17 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/flow_vis.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Tom Runia 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to conditions. 11 | # 12 | # Author: Tom Runia 13 | # Date Created: 2018-08-03 14 | 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import numpy as np 20 | 21 | 22 | def make_colorwheel(): 23 | ''' 24 | Generates a color wheel for optical flow visualization as presented in: 25 | Baker et al. "A Database and Evaluation Methodology for Optical Flow" (ICCV, 2007) 26 | URL: http://vision.middlebury.edu/flow/flowEval-iccv07.pdf 27 | 28 | According to the C++ source code of Daniel Scharstein 29 | According to the Matlab source code of Deqing Sun 30 | ''' 31 | 32 | RY = 15 33 | YG = 6 34 | GC = 4 35 | CB = 11 36 | BM = 13 37 | MR = 6 38 | 39 | ncols = RY + YG + GC + CB + BM + MR 40 | colorwheel = np.zeros((ncols, 3)) 41 | col = 0 42 | 43 | # RY 44 | colorwheel[0:RY, 0] = 255 45 | colorwheel[0:RY, 1] = np.floor(255*np.arange(0,RY)/RY) 46 | col = col+RY 47 | # YG 48 | colorwheel[col:col+YG, 0] = 255 - np.floor(255*np.arange(0,YG)/YG) 49 | colorwheel[col:col+YG, 1] = 255 50 | col = col+YG 51 | # GC 52 | colorwheel[col:col+GC, 1] = 255 53 | colorwheel[col:col+GC, 2] = np.floor(255*np.arange(0,GC)/GC) 54 | col = col+GC 55 | # CB 56 | colorwheel[col:col+CB, 1] = 255 - np.floor(255*np.arange(CB)/CB) 57 | colorwheel[col:col+CB, 2] = 255 58 | col = col+CB 59 | # BM 60 | colorwheel[col:col+BM, 2] = 255 61 | colorwheel[col:col+BM, 0] = np.floor(255*np.arange(0,BM)/BM) 62 | col = col+BM 63 | # MR 64 | colorwheel[col:col+MR, 2] = 255 - np.floor(255*np.arange(MR)/MR) 65 | colorwheel[col:col+MR, 0] = 255 66 | return colorwheel 67 | 68 | 69 | def flow_compute_color(u, v, convert_to_bgr=False): 70 | ''' 71 | Applies the flow color wheel to (possibly clipped) flow components u and v. 72 | 73 | According to the C++ source code of Daniel Scharstein 74 | According to the Matlab source code of Deqing Sun 75 | 76 | :param u: np.ndarray, input horizontal flow 77 | :param v: np.ndarray, input vertical flow 78 | :param convert_to_bgr: bool, whether to change ordering and output BGR instead of RGB 79 | :return: 80 | ''' 81 | 82 | flow_image = np.zeros((u.shape[0], u.shape[1], 3), np.uint8) 83 | 84 | colorwheel = make_colorwheel() # shape [55x3] 85 | ncols = colorwheel.shape[0] 86 | 87 | rad = np.sqrt(np.square(u) + np.square(v)) 88 | a = np.arctan2(-v, -u)/np.pi 89 | 90 | fk = (a+1) / 2*(ncols-1) 91 | k0 = np.floor(fk).astype(np.int32) 92 | k1 = k0 + 1 93 | k1[k1 == ncols] = 0 94 | f = fk - k0 95 | 96 | for i in range(colorwheel.shape[1]): 97 | 98 | tmp = colorwheel[:,i] 99 | col0 = tmp[k0] / 255.0 100 | col1 = tmp[k1] / 255.0 101 | col = (1-f)*col0 + f*col1 102 | 103 | idx = (rad <= 1) 104 | col[idx] = 1 - rad[idx] * (1-col[idx]) 105 | col[~idx] = col[~idx] * 0.75 # out of range? 106 | 107 | # Note the 2-i => BGR instead of RGB 108 | ch_idx = 2-i if convert_to_bgr else i 109 | flow_image[:,:,ch_idx] = np.floor(255 * col) 110 | 111 | return flow_image 112 | 113 | 114 | def flow_to_color(flow_uv, clip_flow=None, convert_to_bgr=False): 115 | ''' 116 | Expects a two dimensional flow image of shape [H,W,2] 117 | 118 | According to the C++ source code of Daniel Scharstein 119 | According to the Matlab source code of Deqing Sun 120 | 121 | :param flow_uv: np.ndarray of shape [H,W,2] 122 | :param clip_flow: float, maximum clipping value for flow 123 | :return: 124 | ''' 125 | 126 | assert flow_uv.ndim == 3, 'input flow must have three dimensions' 127 | assert flow_uv.shape[2] == 2, 'input flow must have shape [H,W,2]' 128 | 129 | if clip_flow is not None: 130 | flow_uv = np.clip(flow_uv, 0, clip_flow) 131 | 132 | u = flow_uv[:,:,0] 133 | v = flow_uv[:,:,1] 134 | 135 | rad = np.sqrt(np.square(u) + np.square(v)) 136 | rad_max = np.max(rad) 137 | 138 | epsilon = 1e-5 139 | u = u / (rad_max + epsilon) 140 | v = v / (rad_max + epsilon) 141 | 142 | return flow_compute_color(u, v, convert_to_bgr) 143 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/load_imgs.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | import numpy as np 5 | import cv2,os 6 | 7 | 8 | def load_imgs(args): 9 | 10 | 11 | print("Dataset path: %s"%(args.dataset)) 12 | path_images = sorted(os.listdir(args.dataset)) 13 | 14 | 15 | img = cv2.imread(os.path.join(args.dataset, path_images[0])) 16 | h_res,w_res,_ = img.shape 17 | h_res = h_res//args.factor 18 | w_res = w_res//args.factor 19 | num_n = args.num_n 20 | 21 | print('Downsampling factor: %d'%(args.factor)) 22 | print('Output resolution: %d x %d'%(h_res,w_res)) 23 | dims = args.dim 24 | 25 | if len(path_images) != np.prod(dims): 26 | rows = 17 27 | cols = 17 28 | step = 4 29 | 30 | new_path_images = [] 31 | 32 | for t_idx in range(0, rows, 1): 33 | for s_idx in range(0, cols, 1): 34 | if ((t_idx % step) == 0) and ((s_idx % step) == 0): 35 | idx = t_idx * cols + s_idx 36 | new_path_images.append(path_images[idx]) 37 | else: 38 | continue 39 | 40 | path_images = new_path_images 41 | 42 | coordinates = [] 43 | all_pairs = [] 44 | images = [] 45 | if args.type == ['light','view','time']: 46 | 47 | for id in range(len(path_images)): 48 | 49 | print('\r Reading image %d out of %d '%(id+1,len(path_images)),end=" " ) 50 | 51 | img = cv2.imread(os.path.join(args.dataset, path_images[id])) 52 | img = cv2.resize(img,None,fx=1/args.factor,fy=1/args.factor, interpolation = cv2.INTER_AREA) 53 | 54 | img = np.float32(img)/255.0 55 | 56 | time = id //(dims[0]*dims[1]) 57 | rest = id % (dims[0]*dims[1]) 58 | view = rest % dims[1] 59 | light = rest // dims[1] 60 | 61 | coordinates.append(np.array([[[light,view,time]]])) 62 | images.append(img) 63 | 64 | 65 | pair = np.array([light,light-1,light+1,view,view,view,time,time,time]) 66 | pair = np.where(pair < 0, 2, pair) 67 | pair = np.where(pair > dims[0]-1 , dims[0]-3, pair) 68 | all_pairs.append(pair) 69 | 70 | 71 | pair = np.array([light,light,light,view,view-1,view+1,time,time,time]) 72 | pair = np.where(pair < 0, 2, pair) 73 | pair = np.where(pair > dims[1]-1 , dims[1]-3, pair) 74 | all_pairs.append(pair) 75 | 76 | 77 | pair = np.array([light,light,light,view,view,view,time,time-1,time+1]) 78 | pair = np.where(pair < 0, 2, pair) 79 | pair = np.where(pair > dims[2]-1 , dims[2]-3, pair) 80 | all_pairs.append(pair) 81 | 82 | 83 | stack = np.stack(all_pairs,0) 84 | img_index = stack[:,0:3]*dims[1] + stack[:,3:6] + stack[:,6:9]*dims[0]*dims[1] 85 | albedo_index = stack[:,3:4]*dims[1] + stack[:,6:7] 86 | training_pairs = np.concatenate((img_index,albedo_index),-1) 87 | images = np.stack(images,0) 88 | coordinates = np.stack(coordinates,0) 89 | 90 | 91 | 92 | if args.type == ['view'] or args.type == ['light']: 93 | 94 | if num_n > np.prod(dims): 95 | num_n = np.prod(dims)-1 96 | 97 | for id in range(len(path_images)): 98 | 99 | print('\r Reading image %d out of %d '%(id+1,len(path_images)),end=" " ) 100 | 101 | img = cv2.imread(os.path.join(args.dataset, path_images[id])) 102 | img = cv2.resize(img,None,fx=1/args.factor,fy=1/args.factor, interpolation = cv2.INTER_AREA) 103 | 104 | img = np.float32(img)/255.0 105 | 106 | 107 | 108 | cx = id % dims[1] 109 | cy = id // dims[1] 110 | 111 | coordinates.append(np.array([[[cx,args.br*cy]]])) 112 | images.append(img) 113 | 114 | 115 | images = np.stack(images,0) 116 | coordinates = np.stack(coordinates,0) 117 | 118 | for id in range(len(path_images)): 119 | 120 | dist = np.sum(np.square(coordinates[id,0,0,:] - coordinates[:,0,0,:]),-1) 121 | idx = np.argsort(dist)[:num_n+1] 122 | all_pairs.append(idx) 123 | 124 | training_pairs = np.stack(all_pairs,0) 125 | 126 | 127 | if args.type == ['time']: 128 | 129 | 130 | if num_n > np.prod(dims): 131 | num_n = np.prod(dims)-1 132 | 133 | for id in range(len(path_images)): 134 | 135 | print('\r Reading image %d out of %d '%(id+1,len(path_images)),end=" " ) 136 | 137 | img = cv2.imread(os.path.join(args.dataset, path_images[id])) 138 | img = cv2.resize(img,None,fx=1/args.factor,fy=1/args.factor, interpolation = cv2.INTER_AREA) 139 | 140 | img = np.float32(img)/255.0 141 | 142 | 143 | coordinates.append(np.array([[[id]]])) 144 | images.append(img) 145 | 146 | images = np.stack(images,0) 147 | coordinates = np.stack(coordinates,0) 148 | 149 | for id in range(len(path_images)): 150 | 151 | dist = np.abs(coordinates[id,0,0,0] - coordinates[:,0,0,0]) 152 | idx = np.argsort(dist)[:num_n+1] 153 | all_pairs.append(idx) 154 | 155 | training_pairs = np.stack(all_pairs,0) 156 | 157 | return images,coordinates,training_pairs,h_res,w_res 158 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/model.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorlayer.layers import PadLayer,Conv2d,UpSampling2dLayer,InputLayer,ConcatLayer 4 | import numpy as np 5 | 6 | def upsampling_factor_padding(h_res,w_res): 7 | 8 | res_temp = h_res 9 | py =[res_temp%2] 10 | while res_temp!=1: 11 | res_temp = res_temp//2 12 | py.append(res_temp%2) 13 | 14 | del py[-1] 15 | py = np.flip(py) 16 | 17 | res_temp = w_res 18 | px =[res_temp%2] 19 | while res_temp!=1: 20 | res_temp = res_temp//2 21 | px.append(res_temp%2) 22 | 23 | del px[-1] 24 | px = np.flip(px) 25 | 26 | lx = len(px) 27 | ly = len(py) 28 | up_x = 2*np.ones((lx)) 29 | up_y = 2*np.ones((ly)) 30 | 31 | if lx > ly: 32 | py = np.append(py,[0]*(lx-ly)) 33 | up_y = np.append(up_y,[1]*(lx-ly)) 34 | 35 | 36 | if ly > lx: 37 | px = np.append(px,[0]*(ly-lx)) 38 | up_x = np.append(up_x,[1]*(ly-lx)) 39 | 40 | return px,py,up_x,up_y 41 | 42 | 43 | def gen_flow(batch_input,num_out): 44 | padding_d = [[0,0],[1,1],[1,1],[0,0]] 45 | batch_input = PadLayer(batch_input,padding_d,"REFLECT") 46 | network = Conv2d(batch_input, n_filter=num_out, filter_size=(3, 3),strides=(1, 1), act = tf.tanh, padding='VALID',W_init=tf.random_normal_initializer(0, 0.02),b_init = tf.constant_initializer(value=0.0)) 47 | return network.outputs 48 | 49 | def conv_layer(batch_input, out_channels,padding_d,fs): 50 | batch_input = PadLayer(batch_input,padding_d,"REFLECT") 51 | network = Conv2d(batch_input, n_filter=out_channels, filter_size=(fs,fs),strides=(1, 1), act=tf.nn.leaky_relu, padding='VALID',W_init=tf.random_normal_initializer(0, 0.02),b_init = tf.constant_initializer(value=0.0)) 52 | return network 53 | 54 | 55 | def Flow(input_coordinates,h_res,w_res,num_out,ngf,min_,max_): 56 | 57 | 58 | # we calculated the amount of padding for each layer and 59 | # the total number of upsampling in each dimension to output the resolution h_res*w_res. 60 | padx,pady,up_x,up_y = upsampling_factor_padding(h_res,w_res) 61 | 62 | num_l = len(padx) 63 | layer_specs = [ngf*16, ngf*16 , ngf*16 , ngf*8 , ngf*8 , ngf*8 , ngf*4 ] 64 | layer_specs.extend([ngf*4]*(num_l-len(layer_specs))) 65 | 66 | 67 | # coordconv layer 68 | coordconv = tf.constant([[[[min_, min_], 69 | [max_, min_]], 70 | [[min_, max_], 71 | [max_, max_]]]],dtype=tf.float32) 72 | 73 | coordconv_tl = InputLayer(tf.tile(coordconv,[input_coordinates.shape[0],1,1,1])) 74 | output = InputLayer(input_coordinates) 75 | 76 | for num,num_filter in enumerate(layer_specs): 77 | 78 | with tf.variable_scope("layer_%d" % (num)): 79 | 80 | upsampled = UpSampling2dLayer(output,(up_y[num],up_x[num]),True,0,True) 81 | if num == 0: 82 | padding = [[0,0],[0,pady[num]],[0,padx[num]],[0,0]] 83 | output = conv_layer(upsampled,num_filter,padding,1) 84 | coordconv_tl = PadLayer(coordconv_tl,padding,"REFLECT") 85 | # concatenating the coordconv layer 86 | output = ConcatLayer([output,coordconv_tl],-1) 87 | else: 88 | padding = [[0,0],[1,1 + pady[num]],[1,1 + padx[num]],[0,0]] 89 | output = conv_layer(upsampled,num_filter,padding,3) 90 | 91 | with tf.variable_scope("outputs_flows"): 92 | flows = gen_flow(output,num_out) 93 | 94 | 95 | return flows 96 | 97 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/test_xfields.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ~/miniconda3/bin/activate xfields 4 | 5 | CUDA_VISIBLE_DEVICES=$2 \ 6 | python test.py \ 7 | --dataset ~/local/data/stanford_half/$1 \ 8 | --savepath ~/local/logs/xfields_$1 \ 9 | --epoch_end $3 \ 10 | --type view 11 | -------------------------------------------------------------------------------- /baselines/third_party/xfields/train_xfields.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ~/miniconda3/bin/activate xfields 4 | 5 | CUDA_VISIBLE_DEVICES=$2 \ 6 | python main.py \ 7 | --dataset ~/local/data/stanford_half/$1 \ 8 | --savepath ~/local/logs/xfields_$1 \ 9 | --type view 10 | -------------------------------------------------------------------------------- /conf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/neural-light-fields/1667d88ca902bed3cc4e2d0d51029cb38a2e7a26/conf/__init__.py -------------------------------------------------------------------------------- /conf/config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - experiment: local 3 | -------------------------------------------------------------------------------- /conf/experiment/blender_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: blender 6 | - model: blender_subdivided 7 | - training: blender_subdivided 8 | - regularizers: all 9 | - visualizers: all 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /conf/experiment/dataset/blender.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: blender 4 | collection: lego 5 | data_subdir: nerf_synthetic 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [400, 400] 9 | spherical_poses: True 10 | use_ndc: False 11 | 12 | precrop_frac: 0.5 13 | precrop_iters: 0 14 | 15 | val_num: 8 16 | val_skip: 1 17 | -------------------------------------------------------------------------------- /conf/experiment/dataset/llff.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: llff 4 | collection: fern 5 | data_subdir: nerf_llff_data 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [504, 378] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/llff_large.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: llff 4 | collection: fern 5 | data_subdir: nerf_llff_data 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [1008, 756] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/llff_undistort.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: fern_undistort 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [504, 378] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/llff_undistort_large.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: fern_undistort 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [1008, 756] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/shiny.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: food 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [504, 378] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/shiny_dense.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: cd 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [504, 284] 9 | use_ndc: True 10 | 11 | val_num: 1000 12 | val_skip: 8 13 | 14 | render_params: 15 | interpolate: False 16 | supersample: 4 17 | max_frames: 400 18 | crop: 0.8 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/shiny_dense_large.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: cd 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [1008, 567] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | max_frames: 400 19 | crop: 0.8 20 | -------------------------------------------------------------------------------- /conf/experiment/dataset/shiny_large.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: food 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [1008, 756] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/shiny_small.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: shiny 4 | collection: food 5 | data_subdir: shiny 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [252, 189] 9 | spherical_poses: False 10 | use_ndc: True 11 | 12 | val_num: 1000 13 | val_skip: 8 14 | 15 | render_params: 16 | interpolate: False 17 | supersample: 4 18 | crop: 1.0 19 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: gem 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: 9 | downsample: 1 10 | spherical_poses: False 11 | use_ndc: False 12 | 13 | val_pairs: [] 14 | val_num: 8 15 | val_skip: 1 16 | 17 | render_params: 18 | spiral: True 19 | far: False 20 | interpolate: False 21 | supersample: 4 22 | crop: 1.0 23 | 24 | spiral_rad: 0.5 25 | uv_downscale: 0.0 26 | 27 | lightfield: 28 | rows: 17 29 | cols: 17 30 | step: 4 31 | supersample: 2 32 | disp_row: 8 33 | 34 | use_file_coords: True 35 | st_scale: 0.25 36 | vis_st_scale: 37 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_beans.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: beans 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 256] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 17 23 | cols: 17 24 | step: 4 25 | supersample: 2 26 | disp_row: 8 27 | 28 | use_file_coords: True 29 | st_scale: 0.25 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_bracelet.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: bracelet 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 320] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 17 23 | cols: 17 24 | step: 4 25 | supersample: 2 26 | disp_row: 8 27 | 28 | use_file_coords: True 29 | st_scale: 0.1 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_bulldozer.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: bulldozer 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [768, 576] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | spiral: True 18 | far: False 19 | interpolate: False 20 | supersample: 4 21 | crop: 1.0 22 | 23 | spiral_rad: 0.35 24 | uv_downscale: 0.0 25 | 26 | lightfield: 27 | rows: 17 28 | cols: 17 29 | step: 4 30 | supersample: 2 31 | disp_row: 8 32 | 33 | use_file_coords: True 34 | st_scale: 0.5 35 | vis_st_scale: 36 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_bunny.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: bunny 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 512] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 17 23 | cols: 17 24 | step: 4 25 | supersample: 2 26 | disp_row: 8 27 | 28 | use_file_coords: True 29 | st_scale: 0.25 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_chess.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: chess 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [700, 400] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 17 23 | cols: 17 24 | step: 4 25 | supersample: 2 26 | disp_row: 8 27 | 28 | use_file_coords: True 29 | st_scale: 0.1 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_flowers.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: flowers 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [640, 768] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | spiral: True 18 | far: False 19 | interpolate: False 20 | supersample: 4 21 | crop: 1.0 22 | 23 | spiral_rad: 0.5 24 | uv_downscale: 0.25 25 | 26 | lightfield: 27 | rows: 17 28 | cols: 17 29 | step: 4 30 | supersample: 4 31 | disp_row: 8 32 | 33 | use_file_coords: True 34 | st_scale: 0.25 35 | vis_st_scale: 36 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_gem.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: gem 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [384, 512] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | spiral: True 18 | far: False 19 | interpolate: False 20 | supersample: 4 21 | crop: 1.0 22 | 23 | spiral_rad: 0.5 24 | uv_downscale: 0.0 25 | 26 | lightfield: 27 | rows: 17 28 | cols: 17 29 | step: 4 30 | supersample: 2 31 | disp_row: 8 32 | 33 | use_file_coords: True 34 | st_scale: 0.25 35 | vis_st_scale: 36 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_knights.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: knights 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 512] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | spiral: True 18 | far: False 19 | interpolate: False 20 | supersample: 4 21 | crop: 1.0 22 | 23 | spiral_rad: 0.65 24 | uv_downscale: 0.0 25 | 26 | lightfield: 27 | rows: 17 28 | cols: 17 29 | step: 4 30 | supersample: 4 31 | disp_row: 8 32 | 33 | use_file_coords: True 34 | st_scale: 0.5 35 | vis_st_scale: 36 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_tarot.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: tarot 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 512] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 17 23 | cols: 17 24 | step: 4 25 | supersample: 2 26 | disp_row: 8 27 | 28 | use_file_coords: True 29 | st_scale: 0.5 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_tarot_small.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: tarot_small 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 512] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | spiral: True 18 | far: False 19 | interpolate: False 20 | supersample: 4 21 | crop: 1.0 22 | 23 | spiral_rad: 0.5 24 | uv_downscale: 0.0 25 | 26 | lightfield: 27 | rows: 17 28 | cols: 17 29 | step: 4 30 | supersample: 4 31 | disp_row: 16 32 | 33 | use_file_coords: True 34 | st_scale: 0.25 35 | vis_st_scale: 36 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_treasure.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: treasure 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [768, 640] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | spiral: True 18 | far: False 19 | interpolate: False 20 | supersample: 4 21 | crop: 1.0 22 | 23 | spiral_rad: 0.75 24 | uv_downscale: 0.0 25 | 26 | lightfield: 27 | rows: 17 28 | cols: 17 29 | step: 4 30 | supersample: 2 31 | disp_row: 8 32 | 33 | use_file_coords: True 34 | st_scale: 0.25 35 | vis_st_scale: 36 | -------------------------------------------------------------------------------- /conf/experiment/dataset/stanford_truck.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: truck 5 | data_subdir: stanford_half 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [640, 480] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 17 23 | cols: 17 24 | step: 4 25 | supersample: 2 26 | disp_row: 8 27 | 28 | use_file_coords: True 29 | st_scale: 0.1 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/tamul.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: tamul_lightfield 4 | collection: cars_1 5 | data_subdir: tamul 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [540, 372] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_num: 200 13 | val_skip: 1 14 | val_all: False 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 8 23 | cols: 8 24 | step: 7 25 | supersample: 2 26 | disp_row: 3 27 | 28 | st_scale: 0.05 29 | vis_st_scale: 30 | -------------------------------------------------------------------------------- /conf/experiment/dataset/tanks.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: tanks 4 | collection: Truck 5 | data_subdir: tanks 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | img_wh: [480, 270] 8 | val_num: 8 9 | spherical_poses: True 10 | -------------------------------------------------------------------------------- /conf/experiment/dataset/xfields_island.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: island 5 | data_subdir: stanford 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [862, 862] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [2,2] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 5 23 | cols: 5 24 | step: 1 25 | supersample: 4 26 | disp_row: 2 27 | 28 | use_file_coords: False 29 | st_scale: 0.5 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/dataset/xfields_splash.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | name: stanford 4 | collection: splash 5 | data_subdir: stanford 6 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}/" 7 | 8 | img_wh: [512, 512] 9 | spherical_poses: False 10 | use_ndc: False 11 | 12 | val_pairs: [2,2] 13 | val_num: 8 14 | val_skip: 1 15 | 16 | render_params: 17 | interpolate: False 18 | supersample: 4 19 | crop: 1.0 20 | 21 | lightfield: 22 | rows: 5 23 | cols: 5 24 | step: 1 25 | supersample: 4 26 | disp_row: 2 27 | 28 | use_file_coords: False 29 | st_scale: 0.5 30 | vis_st_scale: 31 | -------------------------------------------------------------------------------- /conf/experiment/llff_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: llff 6 | - model: llff_subdivided_affine 7 | - training: llff_subdivided 8 | - regularizers: all 9 | - visualizers: all 10 | -------------------------------------------------------------------------------- /conf/experiment/llff_subdivided_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: llff 6 | - model: llff_subdivided_half 7 | - training: llff_subdivided_half 8 | - regularizers: all 9 | - visualizers: all 10 | -------------------------------------------------------------------------------- /conf/experiment/local.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: llff 6 | - model: llff_subdivided_affine 7 | - training: llff_subdivided 8 | - regularizers: all 9 | - visualizers: all 10 | -------------------------------------------------------------------------------- /conf/experiment/model/blender_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-1.1, -1.1, -1.1] 9 | max_point: [1.1, 1.1, 1.1] 10 | voxel_size: 11 | max_hits: 32 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | no_voxel: False 25 | use_bg: False 26 | white_background: True 27 | 28 | 29 | render: 30 | type: subdivided 31 | 32 | 33 | ray: 34 | type: lightfield 35 | 36 | param: 37 | n_dims: 6 38 | fn: pluecker 39 | voxel_size: 40 | 41 | latent_pe: 42 | type: select 43 | select_channels: 3 44 | freq_multiplier: 2.0 45 | n_freqs: 8 46 | wait_iters: 0 47 | max_freq_iter: 0 48 | exclude_identity: False 49 | 50 | # Embedding 51 | embedding_net: 52 | pe: 53 | type: windowed 54 | n_freqs: 0 55 | wait_iters: 0 56 | max_freq_iter: 0 57 | exclude_identity: False 58 | 59 | type: local_affine 60 | depth: 8 61 | hidden_channels: 256 62 | skips: [4] 63 | param_channels: 6 64 | tform_out_channels: 32 65 | tform_activation: 66 | type: row_l2_norm 67 | param_channels: 4 68 | bias_activation: tanh 69 | activation: identity 70 | 71 | # Color 72 | color_pe: 73 | type: windowed 74 | freq_multiplier: 2.0 75 | n_freqs: 8 76 | wait_iters: 0 77 | max_freq_iter: 25000 78 | exclude_identity: False 79 | 80 | color_net: 81 | type: base 82 | depth: 8 83 | hidden_channels: 256 84 | skips: [4] 85 | activation: sigmoid 86 | 87 | fuse: 88 | type: default 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_affine.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 0 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | pe: 21 | type: windowed 22 | n_freqs: 0 23 | wait_iters: 0 24 | max_freq_iter: 80000 25 | exclude_identity: False 26 | 27 | type: local_affine 28 | depth: 8 29 | hidden_channels: 256 30 | skips: [4] 31 | param_channels: 4 32 | tform_out_channels: 32 33 | tform_activation: row_l2_norm 34 | bias_activation: tanh 35 | activation: identity 36 | 37 | color_pe: 38 | type: windowed 39 | n_freqs: 10 40 | wait_iters: 0 41 | max_freq_iter: 80000 42 | exclude_identity: False 43 | 44 | color_net: 45 | type: base 46 | depth: 8 47 | hidden_channels: 256 48 | skips: [4] 49 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_feature.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 80000 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | type: feature 21 | depth: 8 22 | hidden_channels: 256 23 | skips: [4] 24 | out_channels: 32 25 | activation: l2_norm 26 | 27 | color_pe: 28 | type: windowed 29 | n_freqs: 10 30 | wait_iters: 0 31 | max_freq_iter: 80000 32 | exclude_identity: False 33 | 34 | color_net: 35 | type: base 36 | depth: 8 37 | hidden_channels: 256 38 | skips: [4] 39 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 0 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | pe: 21 | type: windowed 22 | n_freqs: 0 23 | wait_iters: 0 24 | max_freq_iter: 0 25 | exclude_identity: False 26 | 27 | type: local_affine 28 | depth: 6 29 | hidden_channels: 256 30 | skips: [3] 31 | param_channels: 4 32 | tform_out_channels: 32 33 | 34 | tform_activation: row_l2_norm 35 | bias_activation: tanh 36 | activation: identity 37 | 38 | layer_activation: 39 | type: leaky_relu 40 | sigma: 0.25 41 | 42 | color_pe: 43 | type: windowed 44 | freq_multiplier: 4.0 45 | n_freqs: 5 46 | wait_iters: 0 47 | max_freq_iter: 40000 48 | exclude_identity: False 49 | 50 | color_net: 51 | type: base 52 | depth: 6 53 | hidden_channels: 256 54 | skips: [3] 55 | 56 | layer_activation: 57 | type: leaky_relu 58 | sigma: 0.25 59 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_subdivided_affine.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-1.5, -1.5, -1] 9 | max_point: [1.5, 1.5, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | no_voxel: False 25 | use_bg: False 26 | 27 | 28 | render: 29 | type: subdivided 30 | 31 | 32 | ray: 33 | type: lightfield 34 | 35 | use_latent_color: True 36 | 37 | param: 38 | n_dims: 4 39 | fn: two_plane 40 | voxel_size: 41 | 42 | latent_pe: 43 | type: select 44 | select_channels: 3 45 | n_freqs: 8 46 | wait_iters: 0 47 | max_freq_iter: 0 48 | exclude_identity: False 49 | 50 | embedding_pe: 51 | type: windowed 52 | n_freqs: 0 53 | wait_iters: 0 54 | max_freq_iter: 50000 55 | exclude_identity: False 56 | 57 | embedding_net: 58 | pe: 59 | type: windowed 60 | n_freqs: 0 61 | wait_iters: 0 62 | max_freq_iter: 50000 63 | exclude_identity: False 64 | 65 | type: local_affine 66 | depth: 8 67 | hidden_channels: 256 68 | skips: [4] 69 | param_channels: 4 70 | tform_out_channels: 32 71 | tform_activation: row_l2_norm 72 | bias_activation: tanh 73 | activation: identity 74 | 75 | color_pe: 76 | type: windowed 77 | n_freqs: 8 78 | wait_iters: 0 79 | max_freq_iter: 50000 80 | exclude_identity: False 81 | 82 | color_net: 83 | type: base 84 | depth: 8 85 | hidden_channels: 256 86 | skips: [4] 87 | activation: sigmoid 88 | 89 | fuse: 90 | type: default 91 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_subdivided_feature.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-1.5, -1.5, -1] 9 | max_point: [1.5, 1.5, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | use_bg: False 25 | 26 | 27 | render: 28 | type: subdivided 29 | 30 | 31 | ray: 32 | type: lightfield 33 | 34 | use_latent_color: True 35 | 36 | param: 37 | n_dims: 4 38 | fn: two_plane 39 | voxel_size: 40 | 41 | latent_pe: 42 | type: select 43 | select_channels: 3 44 | n_freqs: 8 45 | wait_iters: 0 46 | max_freq_iter: 0 47 | exclude_identity: False 48 | 49 | embedding_pe: 50 | type: windowed 51 | n_freqs: 0 52 | wait_iters: 0 53 | max_freq_iter: 50000 54 | exclude_identity: False 55 | 56 | embedding_net: 57 | type: feature 58 | depth: 8 59 | hidden_channels: 256 60 | skips: [4] 61 | out_channels: 32 62 | activation: l2_norm 63 | 64 | color_pe: 65 | type: windowed 66 | n_freqs: 8 67 | wait_iters: 0 68 | max_freq_iter: 50000 69 | exclude_identity: False 70 | 71 | color_net: 72 | type: base 73 | depth: 8 74 | hidden_channels: 256 75 | skips: [4] 76 | activation: sigmoid 77 | 78 | 79 | fuse: 80 | type: default 81 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_subdivided_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-1.5, -1.5, -1] 9 | max_point: [1.5, 1.5, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: True 24 | no_voxel: False 25 | use_bg: False 26 | 27 | 28 | render: 29 | type: subdivided 30 | 31 | 32 | ray: 33 | type: lightfield 34 | 35 | use_latent_color: True 36 | 37 | param: 38 | n_dims: 4 39 | fn: two_plane 40 | voxel_size: 41 | 42 | latent_pe: 43 | type: select 44 | select_channels: 3 45 | freq_multiplier: 8.0 46 | n_freqs: 4 47 | wait_iters: 0 48 | max_freq_iter: 0 49 | exclude_identity: False 50 | 51 | embedding_pe: 52 | type: windowed 53 | n_freqs: 0 54 | wait_iters: 0 55 | max_freq_iter: 0 56 | exclude_identity: False 57 | 58 | embedding_net: 59 | pe: 60 | type: windowed 61 | n_freqs: 0 62 | wait_iters: 0 63 | max_freq_iter: 0 64 | exclude_identity: False 65 | 66 | type: local_affine 67 | depth: 6 68 | hidden_channels: 256 69 | skips: [3] 70 | param_channels: 4 71 | tform_out_channels: 32 72 | 73 | tform_activation: row_l2_norm 74 | bias_activation: tanh 75 | activation: identity 76 | 77 | layer_activation: 78 | type: leaky_relu 79 | sigma: 0.25 80 | 81 | color_pe: 82 | type: windowed 83 | freq_multiplier: 4.0 84 | n_freqs: 5 85 | wait_iters: 0 86 | max_freq_iter: 40000 87 | exclude_identity: False 88 | 89 | color_net: 90 | type: base 91 | depth: 6 92 | hidden_channels: 256 93 | skips: [3] 94 | 95 | layer_activation: 96 | type: leaky_relu 97 | sigma: 0.25 98 | 99 | fuse: 100 | type: default 101 | -------------------------------------------------------------------------------- /conf/experiment/model/llff_subdivided_no_embed.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-1.5, -1.5, -1] 9 | max_point: [1.5, 1.5, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | use_bg: False 25 | 26 | 27 | render: 28 | type: subdivided 29 | 30 | 31 | ray: 32 | type: lightfield 33 | 34 | use_latent_color: True 35 | 36 | param: 37 | n_dims: 4 38 | fn: two_plane 39 | voxel_size: 40 | 41 | latent_pe: 42 | type: select 43 | select_channels: 3 44 | n_freqs: 8 45 | wait_iters: 0 46 | max_freq_iter: 0 47 | exclude_identity: False 48 | 49 | embedding_pe: 50 | type: windowed 51 | n_freqs: 0 52 | wait_iters: 0 53 | max_freq_iter: 50000 54 | exclude_identity: False 55 | 56 | embedding_net: 57 | type: feature 58 | depth: 0 59 | hidden_channels: 256 60 | skips: [] 61 | out_channels: 32 62 | activation: identity 63 | 64 | color_pe: 65 | type: windowed 66 | n_freqs: 8 67 | wait_iters: 0 68 | max_freq_iter: 50000 69 | exclude_identity: False 70 | 71 | color_net: 72 | type: base 73 | depth: 8 74 | hidden_channels: 256 75 | skips: [4] 76 | activation: sigmoid 77 | 78 | 79 | fuse: 80 | type: default 81 | -------------------------------------------------------------------------------- /conf/experiment/model/shiny_affine.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 0 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | pe: 21 | type: windowed 22 | n_freqs: 0 23 | wait_iters: 0 24 | max_freq_iter: 0 25 | exclude_identity: False 26 | 27 | type: local_affine 28 | depth: 8 29 | hidden_channels: 256 30 | skips: [4] 31 | param_channels: 4 32 | tform_out_channels: 32 33 | tform_activation: row_l2_norm 34 | bias_activation: tanh 35 | activation: identity 36 | 37 | color_pe: 38 | type: windowed 39 | n_freqs: 8 40 | wait_iters: 0 41 | max_freq_iter: 25000 42 | exclude_identity: False 43 | 44 | color_net: 45 | type: base 46 | depth: 8 47 | hidden_channels: 256 48 | skips: [4] 49 | 50 | -------------------------------------------------------------------------------- /conf/experiment/model/shiny_subdivided_affine.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-2, -2, -1] 9 | max_point: [2, 2, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | no_voxel: False 25 | use_bg: False 26 | 27 | 28 | render: 29 | type: subdivided 30 | 31 | 32 | ray: 33 | type: lightfield 34 | 35 | use_latent_color: True 36 | 37 | param: 38 | n_dims: 4 39 | fn: two_plane 40 | voxel_size: 41 | 42 | latent_pe: 43 | type: select 44 | select_channels: 3 45 | n_freqs: 8 46 | wait_iters: 0 47 | max_freq_iter: 0 48 | exclude_identity: False 49 | 50 | embedding_pe: 51 | type: windowed 52 | n_freqs: 0 53 | wait_iters: 0 54 | max_freq_iter: 0 55 | exclude_identity: False 56 | 57 | embedding_net: 58 | pe: 59 | type: windowed 60 | n_freqs: 0 61 | wait_iters: 0 62 | max_freq_iter: 0 63 | exclude_identity: False 64 | 65 | type: local_affine 66 | depth: 8 67 | hidden_channels: 256 68 | skips: [4] 69 | param_channels: 4 70 | tform_out_channels: 32 71 | tform_activation: row_l2_norm 72 | bias_activation: tanh 73 | activation: identity 74 | 75 | color_pe: 76 | type: windowed 77 | n_freqs: 8 78 | wait_iters: 0 79 | max_freq_iter: 50000 80 | exclude_identity: False 81 | 82 | color_net: 83 | type: base 84 | depth: 8 85 | hidden_channels: 256 86 | skips: [4] 87 | activation: sigmoid 88 | 89 | fuse: 90 | type: default 91 | -------------------------------------------------------------------------------- /conf/experiment/model/shiny_subdivided_affine_dense.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-2, -2, -1] 9 | max_point: [2, 2, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | no_voxel: False 25 | use_bg: False 26 | 27 | 28 | render: 29 | type: subdivided 30 | 31 | 32 | ray: 33 | type: lightfield 34 | 35 | use_latent_color: True 36 | 37 | param: 38 | n_dims: 4 39 | fn: two_plane 40 | voxel_size: 41 | 42 | latent_pe: 43 | type: select 44 | select_channels: 3 45 | n_freqs: 8 46 | wait_iters: 0 47 | max_freq_iter: 0 48 | exclude_identity: False 49 | 50 | embedding_pe: 51 | type: windowed 52 | n_freqs: 0 53 | wait_iters: 0 54 | max_freq_iter: 0 55 | exclude_identity: False 56 | 57 | embedding_net: 58 | pe: 59 | type: windowed 60 | n_freqs: 0 61 | wait_iters: 0 62 | max_freq_iter: 0 63 | exclude_identity: False 64 | 65 | type: local_affine 66 | depth: 8 67 | hidden_channels: 256 68 | skips: [4] 69 | param_channels: 4 70 | tform_out_channels: 32 71 | tform_activation: row_l2_norm 72 | bias_activation: tanh 73 | activation: identity 74 | 75 | color_pe: 76 | type: windowed 77 | n_freqs: 8 78 | wait_iters: 0 79 | max_freq_iter: 25000 80 | exclude_identity: False 81 | 82 | color_net: 83 | type: base 84 | depth: 8 85 | hidden_channels: 256 86 | skips: [4] 87 | activation: sigmoid 88 | 89 | fuse: 90 | type: default 91 | -------------------------------------------------------------------------------- /conf/experiment/model/shiny_subdivided_pluecker.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: subdivided_lightfield 4 | 5 | subdivision: 6 | type: latent_voxel 7 | 8 | min_point: [-2, -2, -1] 9 | max_point: [2, 2, 1] 10 | voxel_size: 11 | max_hits: 4 12 | 13 | latent_dim: 0 14 | 15 | update_every: inf 16 | split_every: inf 17 | max_update_iters: 10 18 | threshold: 19 | type: linear 20 | start: 0.001 21 | end: 0.01 22 | 23 | forward_facing: False 24 | no_voxel: False 25 | use_bg: False 26 | 27 | 28 | render: 29 | type: subdivided 30 | 31 | 32 | ray: 33 | type: lightfield 34 | 35 | use_latent_color: True 36 | 37 | param: 38 | n_dims: 6 39 | fn: pluecker 40 | voxel_size: 41 | 42 | latent_pe: 43 | type: select 44 | select_channels: 3 45 | n_freqs: 8 46 | wait_iters: 0 47 | max_freq_iter: 0 48 | exclude_identity: False 49 | 50 | embedding_pe: 51 | type: windowed 52 | n_freqs: 0 53 | wait_iters: 0 54 | max_freq_iter: 0 55 | exclude_identity: False 56 | 57 | embedding_net: 58 | pe: 59 | type: windowed 60 | n_freqs: 0 61 | wait_iters: 0 62 | max_freq_iter: 0 63 | exclude_identity: False 64 | 65 | type: local_affine 66 | depth: 8 67 | hidden_channels: 256 68 | skips: [4] 69 | param_channels: 6 70 | tform_out_channels: 32 71 | tform_activation: identity 72 | bias_activation: tanh 73 | activation: 74 | type: row_l2_norm 75 | param_channels: 8 76 | fac: 8.0 77 | 78 | color_pe: 79 | type: windowed 80 | n_freqs: 8 81 | wait_iters: 0 82 | max_freq_iter: 25000 83 | exclude_identity: False 84 | 85 | color_net: 86 | type: base 87 | depth: 8 88 | hidden_channels: 256 89 | skips: [4] 90 | activation: sigmoid 91 | 92 | fuse: 93 | type: default 94 | 95 | -------------------------------------------------------------------------------- /conf/experiment/model/stanford_affine.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 0 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | pe: 21 | type: windowed 22 | n_freqs: 0 23 | wait_iters: 0 24 | max_freq_iter: 0 25 | exclude_identity: False 26 | 27 | type: local_affine 28 | depth: 8 29 | hidden_channels: 256 30 | skips: [4] 31 | param_channels: 4 32 | tform_out_channels: 32 33 | tform_activation: row_l2_norm 34 | bias_activation: tanh 35 | activation: identity 36 | 37 | color_pe: 38 | type: windowed 39 | n_freqs: 8 40 | wait_iters: 0 41 | max_freq_iter: 6000 42 | exclude_identity: False 43 | 44 | color_net: 45 | type: base 46 | depth: 8 47 | hidden_channels: 256 48 | skips: [4] 49 | -------------------------------------------------------------------------------- /conf/experiment/model/stanford_affine_pluecker.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 6 10 | fn: pluecker 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 0 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | pe: 21 | type: windowed 22 | n_freqs: 0 23 | wait_iters: 0 24 | max_freq_iter: 0 25 | exclude_identity: False 26 | 27 | type: local_affine 28 | depth: 8 29 | hidden_channels: 256 30 | skips: [4] 31 | param_channels: 6 32 | tform_out_channels: 32 33 | tform_activation: 34 | type: row_l2_norm 35 | param_channels: 6 36 | bias_activation: tanh 37 | activation: identity 38 | 39 | color_pe: 40 | type: windowed 41 | n_freqs: 8 42 | wait_iters: 0 43 | max_freq_iter: 6000 44 | exclude_identity: False 45 | 46 | color_net: 47 | type: base 48 | depth: 8 49 | hidden_channels: 256 50 | skips: [4] 51 | 52 | -------------------------------------------------------------------------------- /conf/experiment/model/stanford_feature.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 80000 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | type: feature 21 | depth: 8 22 | hidden_channels: 256 23 | skips: [4] 24 | out_channels: 32 25 | activation: l2_norm 26 | 27 | color_pe: 28 | type: windowed 29 | n_freqs: 8 30 | wait_iters: 0 31 | max_freq_iter: 6000 32 | exclude_identity: False 33 | 34 | color_net: 35 | type: base 36 | depth: 8 37 | hidden_channels: 256 38 | skips: [4] 39 | -------------------------------------------------------------------------------- /conf/experiment/model/stanford_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 0 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | type: local_affine 21 | 22 | param_channels: 4 23 | 24 | pe: 25 | type: windowed 26 | n_freqs: 0 27 | wait_iters: 0 28 | max_freq_iter: 0 29 | exclude_identity: False 30 | 31 | depth: 6 32 | hidden_channels: 256 33 | skips: [3] 34 | layer_activation: 35 | type: leaky_relu 36 | sigma: 0.25 37 | 38 | extra_in_channels: 0 39 | extra_tform_activation: 40 | type: row_l2_norm 41 | param_channels: 2 42 | extra_bias_activation: zero 43 | extra_activation: identity 44 | 45 | tform_out_channels: 32 46 | tform_activation: 47 | type: row_l2_norm 48 | param_channels: 4 49 | bias_activation: zero 50 | activation: identity 51 | 52 | color_pe: 53 | type: windowed 54 | freq_multiplier: 4.0 55 | n_freqs: 4 56 | wait_iters: 0 57 | max_freq_iter: 6000 58 | exclude_identity: False 59 | 60 | color_net: 61 | type: base 62 | depth: 6 63 | hidden_channels: 256 64 | skips: [3] 65 | 66 | layer_activation: 67 | type: leaky_relu 68 | sigma: 0.25 69 | -------------------------------------------------------------------------------- /conf/experiment/model/stanford_no_embed.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 4 10 | fn: two_plane 11 | 12 | embedding_pe: 13 | type: windowed 14 | n_freqs: 0 15 | wait_iters: 0 16 | max_freq_iter: 80000 17 | exclude_identity: False 18 | 19 | embedding_net: 20 | type: feature 21 | depth: 0 22 | hidden_channels: 256 23 | skips: [] 24 | out_channels: 32 25 | activation: identity 26 | 27 | color_pe: 28 | type: windowed 29 | n_freqs: 8 30 | wait_iters: 0 31 | max_freq_iter: 6000 32 | exclude_identity: False 33 | 34 | color_net: 35 | type: base 36 | depth: 8 37 | hidden_channels: 256 38 | skips: [4] 39 | -------------------------------------------------------------------------------- /conf/experiment/model/stanford_pluecker.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: lightfield 4 | 5 | render: 6 | type: lightfield 7 | 8 | param: 9 | n_dims: 6 10 | fn: pluecker 11 | multiplier: 1.0 12 | 13 | embedding_pe: 14 | type: windowed 15 | n_freqs: 0 16 | wait_iters: 0 17 | max_freq_iter: 0 18 | exclude_identity: False 19 | 20 | embedding_net: 21 | pe: 22 | type: windowed 23 | n_freqs: 0 24 | wait_iters: 0 25 | max_freq_iter: 0 26 | exclude_identity: False 27 | 28 | type: local_affine 29 | depth: 8 30 | hidden_channels: 256 31 | skips: [4] 32 | param_channels: 6 33 | tform_out_channels: 32 34 | tform_activation: identity 35 | bias_activation: tanh 36 | activation: 37 | type: row_l2_norm 38 | param_channels: 8 39 | fac: 8.0 40 | 41 | color_pe: 42 | type: windowed 43 | n_freqs: 8 44 | wait_iters: 0 45 | max_freq_iter: 6000 46 | exclude_identity: False 47 | 48 | color_net: 49 | type: base 50 | depth: 8 51 | hidden_channels: 256 52 | skips: [4] 53 | 54 | 55 | -------------------------------------------------------------------------------- /conf/experiment/params/local.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | ckpt_dir: checkpoints 4 | log_dir: logs 5 | data_dir: data 6 | name: "${experiment.dataset.name}_${experiment.dataset.collection}" 7 | 8 | print_loss: False 9 | save_results: True 10 | 11 | tensorboard: False 12 | log_images: False 13 | log_videos: False 14 | 15 | test_only: False 16 | render_only: False 17 | load_from_weights: False 18 | start_epoch: 0 19 | 20 | seed: 1 21 | 22 | save_video_dir: "${experiment.params.name}/val_videos" 23 | save_image_dir: "${experiment.params.name}/val_images" 24 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/all.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/blurry_teacher/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: blurry_teacher 4 | batch_size: 4096 5 | 6 | dataset: 7 | name: "dense_${experiment.dataset.name}" 8 | collection: "${experiment.dataset.collection}_dense_blurry" 9 | split: train 10 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}_dense_blurry/" 11 | 12 | num_rows: 10 13 | num_cols: 10 14 | train_row_skip: 1 15 | train_col_skip: 1 16 | size: 1000 17 | 18 | use_patches: True 19 | blur_radius: 0 20 | patch_width: 64 21 | 22 | blur_radius: 0 23 | 24 | weight: 25 | type: exponential_decay 26 | start: 0.5 27 | decay: 0.1 28 | stop_weight: 0.025 29 | num_epochs: 250 30 | 31 | loss: 32 | type: mse 33 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/depth_classification/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | 4 | type: depth_classification 5 | ray_chunk: 32768 6 | batch_size: 4096 7 | weight: 0.5 8 | use_color_embedding: True 9 | 10 | use_disparity: True 11 | offset: 0.0 12 | near: 1.0 13 | far: 16.0 14 | num_slices: 128 15 | 16 | dataset: 17 | name: random_view 18 | num_views: all 19 | 20 | lookup_loss: 21 | type: mae 22 | weight: 0.5 23 | warmup_iters: 1000 24 | 25 | angle_std: 10.0 26 | dist_std: 0.5 27 | 28 | color_loss: 29 | type: mae 30 | weight: 0.25 31 | 32 | angle_std: 5.0 33 | dist_std: 0.25 34 | 35 | depth_loss: 36 | type: mae 37 | weight: 0.25 38 | 39 | angle_std: 5.0 40 | dist_std: 0.25 41 | 42 | range: 43 | pos: 1.0 44 | dir: 1.0 45 | 46 | jitter: 47 | pos: 0.25 48 | dir: 0.25 49 | 50 | param: 51 | n_dims: 6 52 | fn: pluecker 53 | 54 | pe: 55 | n_freqs: 10 56 | warmup_iters: 0 57 | max_freq_iter: 0 58 | exclude_identity: False 59 | 60 | net: 61 | depth: 3 62 | hidden_channels: 128 63 | skips: [] 64 | activation: 'identity' 65 | 66 | optimizer: 67 | optimizer: adam 68 | lr: 0.001 69 | clip: False 70 | clip_amount: 1.0 71 | momentum: 0.9 72 | weight_decay: 0 73 | lr_scheduler: steplr 74 | warmup_multipler: 1.0 75 | warmup_epochs: 0 76 | 77 | decay_step: 500 78 | decay_gamma: 0.1 79 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/fourier/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: fourier 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | batch_size: 1 7 | use_inp_freq: inf 8 | wait_iters: 0 9 | 10 | weight: 11 | type: exponential_decay 12 | start: 1.92 13 | decay: 0.1 14 | num_epochs: 500 15 | 16 | dataset: 17 | name: fourier_lightfield 18 | num_views: all 19 | 20 | fourier_loss: 21 | type: mse 22 | weight: 1 23 | wait_iters: 0 24 | 25 | range: 26 | pos: 1.0 27 | dir: 1.0 28 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/inverse_ray_depth/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: inverse_ray_depth 4 | ray_chunk: 32768 5 | net_chunk: 32768 6 | batch_size: 4096 7 | 8 | wait_iters: 0 9 | warmup_iters: 0 10 | use_inp_freq: 2 11 | 12 | dataset: 13 | name: random_view 14 | num_views: all 15 | 16 | weight: 17 | type: exponential_decay 18 | start: 0.5 19 | decay: 0.5 20 | num_epochs: 500 21 | 22 | use_disparity: False 23 | num_samples: 8 24 | 25 | range: 26 | pos: 1.0 27 | dir: 1.0 28 | 29 | lookup_weight_map: 30 | angle_std: 5.0 31 | dist_std: 0.125 32 | rgb_std: 0.125 33 | 34 | embedding_lookup_loss: 35 | type: mae 36 | weight: 1.0 37 | wait_iters: 0 38 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/multiple_ray_depth/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: multiple_ray_depth 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | batch_size: 32768 7 | 8 | wait_iters: 1000 9 | warmup_iters: 10000 10 | use_inp_freq: 2 11 | 12 | use_disparity: False 13 | occlusion_aware: False 14 | num_slices: 8 15 | num_filler: 0 16 | 17 | dataset: 18 | name: random_view 19 | num_views: all 20 | 21 | weight: 22 | type: exponential_decay 23 | start: 0.5 24 | decay: 0.1 25 | num_epochs: 500 26 | 27 | range: 28 | pos: 1.0 29 | dir: 1.0 30 | 31 | jitter: 32 | pos: 0.25 33 | dir: 0.25 34 | bundle_size: 1 35 | 36 | lookup_weight_map: 37 | angle_std: 10.0 38 | dist_std: 0.1 39 | 40 | rgb_std: 41 | type: linear_decay 42 | num_epochs: 100 43 | start: 0.1 44 | end: 0.01 45 | 46 | color_weight_map: 47 | angle_std: 5.0 48 | dist_std: 0.05 49 | 50 | rgb_std: 51 | type: linear_decay 52 | num_epochs: 100 53 | start: 0.1 54 | end: 0.01 55 | 56 | depth_weight_map: 57 | angle_std: 10.0 58 | dist_std: 0.25 59 | 60 | color_lookup_loss: 61 | type: mae 62 | weight: 0.25 63 | wait_iters: 0 64 | 65 | color_loss: 66 | type: mae 67 | weight: 0.5 68 | wait_iters: 0 69 | 70 | depth_loss: 71 | type: mae 72 | weight: 0.25 73 | wait_iters: 0 74 | 75 | param: 76 | n_dims: 4 77 | fn: two_plane 78 | 79 | pe: 80 | n_freqs: 8 81 | wait_iters: 0 82 | max_freq_iter: 50000 83 | exclude_identity: False 84 | 85 | net: 86 | depth: 8 87 | hidden_channels: 256 88 | skips: [4] 89 | activation: identity 90 | 91 | optimizer: 92 | optimizer: adam 93 | lr: 0.001 94 | clip: False 95 | clip_amount: 1.0 96 | momentum: 0.9 97 | weight_decay: 0 98 | 99 | lr_scheduler: exp 100 | warmup_multiplier: 1.0 101 | warmup_epochs: 0 102 | 103 | decay_step: 500 104 | decay_gamma: 0.1 105 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/ray_bundle/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | 4 | type: ray_bundle 5 | ray_chunk: 131072 6 | net_chunk: 131072 7 | batch_size: 32768 8 | use_inp_freq: inf 9 | wait_iters: 0 10 | 11 | weight: 12 | type: exponential_decay 13 | start: 0.5 14 | decay: 0.1 15 | num_epochs: 500 16 | 17 | dataset: 18 | name: random_lightfield 19 | st_plane: -1.0 20 | uv_plane: 0.0 21 | num_views: all 22 | 23 | color_weight_map: 24 | angle_std: 5.0 25 | dist_std: 0.1 26 | 27 | color_loss: 28 | type: mse 29 | wait_iters: 0 30 | 31 | range: 32 | pos: 1.0 33 | dir: 1.0 34 | 35 | jitter: 36 | dir: 0.25 37 | bundle_size: 8 38 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/ray_depth/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: ray_depth 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | batch_size: 4096 7 | 8 | wait_iters: 1000 9 | warmup_iters: 1000 10 | use_inp_freq: 2 11 | 12 | occlusion_aware: False 13 | 14 | dataset: 15 | name: random_view 16 | num_views: all 17 | 18 | weight: 19 | type: exponential_decay 20 | start: 0.5 21 | decay: 0.1 22 | num_epochs: 500 23 | 24 | range: 25 | pos: 1.0 26 | dir: 1.0 27 | 28 | jitter: 29 | pos: 0.1 30 | dir: 0.1 31 | bundle_size: 1 32 | 33 | lookup_weight_map: 34 | angle_std: 10.0 35 | dist_std: 0.25 36 | 37 | rgb_std: 38 | type: linear_decay 39 | num_epochs: 100 40 | start: 0.1 41 | end: 0.01 42 | 43 | color_weight_map: 44 | angle_std: 10.0 45 | dist_std: 0.25 46 | 47 | rgb_std: 48 | type: linear_decay 49 | num_epochs: 100 50 | start: 0.1 51 | end: 0.01 52 | 53 | depth_weight_map: 54 | angle_std: 10.0 55 | dist_std: 0.25 56 | depth_std: 0.5 57 | 58 | color_lookup_loss: 59 | type: mae 60 | weight: 0.25 61 | wait_iters: 0 62 | 63 | color_loss: 64 | type: mae 65 | weight: 0.5 66 | wait_iters: 10000 67 | 68 | depth_loss: 69 | type: mae 70 | weight: 0.25 71 | wait_iters: 0 72 | 73 | param: 74 | n_dims: 6 75 | fn: pluecker 76 | 77 | pe: 78 | n_freqs: 8 79 | wait_iters: 0 80 | max_freq_iter: 50000 81 | exclude_identity: False 82 | 83 | net: 84 | depth: 8 85 | hidden_channels: 256 86 | skips: [4] 87 | activation: identity 88 | 89 | optimizer: 90 | optimizer: adam 91 | lr: 0.001 92 | clip: False 93 | clip_amount: 1.0 94 | momentum: 0.9 95 | weight_decay: 0 96 | 97 | lr_scheduler: exp 98 | warmup_multiplier: 1.0 99 | warmup_epochs: 0 100 | 101 | decay_step: 500 102 | decay_gamma: 0.1 103 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/ray_depth_blending/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | 4 | type: ray_depth_blending 5 | ray_chunk: 32768 6 | batch_size: 4096 7 | use_inp_freq: 0 8 | 9 | use_depth_embedding: False 10 | use_color_embedding: True 11 | 12 | dataset: 13 | name: random_view 14 | num_views: 8 15 | 16 | warmup_iters: 1000 17 | 18 | weight: 19 | type: exponential_decay 20 | start: 1.0 21 | decay: 0.5 22 | num_epochs: 500 23 | 24 | lookup: 25 | angle_std: 5.0 26 | dist_std: 0.1 27 | 28 | color_loss: 29 | type: mae 30 | weight: 0.5 31 | 32 | angle_std: 5.0 33 | dist_std: 0.05 34 | 35 | depth_loss: 36 | type: mae 37 | weight: 0.5 38 | 39 | angle_std: 5.0 40 | dist_std: 0.05 41 | 42 | range: 43 | pos: 1.0 44 | dir: 1.0 45 | 46 | jitter: 47 | pos: 0.1 48 | dir: 0.1 49 | 50 | param: 51 | n_dims: 6 52 | fn: pluecker 53 | 54 | pe: 55 | n_freqs: 10 56 | warmup_iters: 0 57 | max_freq_iter: 120000 58 | exclude_identity: False 59 | 60 | net: 61 | depth: 8 62 | hidden_channels: 256 63 | skips: [4] 64 | activation: 'identity' 65 | 66 | optimizer: 67 | optimizer: adam 68 | lr: 0.001 69 | clip: False 70 | clip_amount: 1.0 71 | momentum: 0.9 72 | weight_decay: 0 73 | 74 | lr_scheduler: exp 75 | warmup_multipler: 1.0 76 | warmup_epochs: 0 77 | 78 | decay_step: 500 79 | decay_gamma: 0.5 80 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/ray_depth_occ_dir/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: ray_depth_occ_dir 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | batch_size: 4096 7 | 8 | wait_iters: 1000 9 | warmup_iters: 1000 10 | use_inp_freq: 2 11 | 12 | dataset: 13 | name: random_view 14 | num_views: all 15 | 16 | weight: 17 | type: exponential_decay 18 | start: 0.25 19 | decay: 0.1 20 | num_epochs: 500 21 | 22 | use_disparity: False 23 | num_features: 128 24 | num_samples: 2 25 | 26 | range: 27 | pos: 1.0 28 | dir: 1.0 29 | 30 | jitter: 31 | pos: 0.25 32 | dir: 0.25 33 | bundle_size: 1 34 | 35 | lookup_weight_map: 36 | angle_std: 10.0 37 | dist_std: 0.25 38 | 39 | color_weight_map: 40 | angle_std: 5.0 41 | dist_std: 0.1 42 | 43 | depth_weight_map: 44 | angle_std: 5.0 45 | dist_std: 0.1 46 | 47 | color_lookup_loss: 48 | type: mae 49 | weight: 0.1 50 | wait_iters: 0 51 | 52 | color_loss: 53 | type: mae 54 | weight: 0.5 55 | wait_iters: 0 56 | 57 | depth_loss: 58 | type: mae 59 | weight: 0.25 60 | wait_iters: 0 61 | 62 | occ_loss: 63 | type: mae 64 | weight: 0.05 65 | wait_iters: 0 66 | 67 | param: 68 | n_dims: 6 69 | fn: pluecker 70 | 71 | pe: 72 | n_freqs: 4 73 | wait_iters: 0 74 | max_freq_iter: 0 75 | exclude_identity: False 76 | 77 | net: 78 | depth: 8 79 | hidden_channels: 256 80 | skips: [4] 81 | activation: identity 82 | 83 | optimizer: 84 | optimizer: adam 85 | lr: 0.001 86 | clip: False 87 | clip_amount: 1.0 88 | momentum: 0.9 89 | weight_decay: 0 90 | 91 | lr_scheduler: exp 92 | warmup_multiplier: 1.0 93 | warmup_epochs: 0 94 | 95 | decay_step: 500 96 | decay_gamma: 0.1 97 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/ray_interpolation/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: ray_interpolation 4 | ray_chunk: 32768 5 | net_chunk: 32768 6 | batch_size: 4096 7 | 8 | warmup_iters: 0 9 | use_inp_freq: 0 10 | 11 | dataset: 12 | name: random_view 13 | num_views: 8 14 | 15 | weight: 16 | type: exponential_decay 17 | start: 1.0 18 | decay: 1.0 19 | num_epochs: 500 20 | 21 | use_disparity: True 22 | points_per_view: 8 23 | 24 | range: 25 | pos: 1.0 26 | dir: 1.0 27 | 28 | color_loss: 29 | type: mse 30 | weight: 1.0 31 | warmup_iters: 0 32 | 33 | angle_std: 25.0 34 | dist_std: 0.5 35 | 36 | blending_net: 37 | type: base 38 | depth: 8 39 | hidden_channels: 256 40 | skips: [4] 41 | activation: sigmoid 42 | #activation: softmax 43 | #activation: identity 44 | 45 | optimizer: 46 | optimizer: adam 47 | lr: 0.001 48 | clip: False 49 | clip_amount: 1.0 50 | momentum: 0.9 51 | weight_decay: 0 52 | 53 | lr_scheduler: exp 54 | warmup_multipler: 1.0 55 | warmup_epochs: 0 56 | 57 | decay_step: 500 58 | decay_gamma: 0.5 59 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/teacher/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: teacher 4 | batch_size: 4096 5 | 6 | dataset: 7 | name: "dense_${experiment.dataset.name}" 8 | collection: "${experiment.dataset.collection}_teacher" 9 | split: train 10 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}_teacher/" 11 | 12 | num_rows: 10 13 | num_cols: 10 14 | train_row_skip: 1 15 | train_col_skip: 1 16 | size: 1000 17 | 18 | weight: 19 | type: exponential_decay 20 | start: 1.0 21 | decay: 1.0 22 | stop_weight: 0.025 23 | num_epochs: 500 24 | 25 | loss: 26 | type: mse 27 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/voxel_sparsity/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: voxel_sparsity 4 | ray_chunk: 32768 5 | batch_size: 32768 6 | use_inp_freq: inf 7 | 8 | weight: 9 | type: exponential_decay 10 | start: 0.01 11 | decay: 0.5 12 | num_epochs: 500 13 | 14 | loss: 15 | type: mse 16 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/warp_level/lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: warp_level 4 | ray_chunk: 32768 5 | batch_size: 8192 6 | use_inp_freq: inf 7 | wait_iters: 10000 8 | 9 | weight: 10 | type: exponential_decay 11 | start: 0.1 12 | decay: 1.0 13 | num_epochs: 500 14 | 15 | dataset: 16 | name: random_lightfield 17 | st_plane: -1.0 18 | uv_plane: 0.0 19 | st_scale: 20 | 21 | color_loss: 22 | type: mae 23 | weight: 0.25 24 | wait_iters: 0 25 | 26 | svd_loss: 27 | type: mae 28 | weight: 0.0 29 | wait_iters: 0 30 | 31 | level_loss: 32 | type: mae 33 | weight: 0.25 34 | wait_iters: 0 35 | 36 | range: 37 | pos: 1.0 38 | dir: 1.0 39 | 40 | jitter: 41 | pos: 0.1 42 | dir: 0.1 43 | bundle_size: 1 44 | 45 | param: 46 | n_dims: 4 47 | fn: two_plane 48 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/warp_level/subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: warp_level 4 | ray_chunk: 32768 5 | batch_size: 4096 6 | use_inp_freq: inf 7 | wait_iters: 10000 8 | 9 | weight: 10 | type: exponential_decay 11 | start: 0.1 12 | decay: 1.0 13 | num_epochs: 500 14 | 15 | dataset: 16 | name: random_lightfield 17 | st_plane: -1.0 18 | uv_plane: 0.0 19 | st_scale: 20 | 21 | color_loss: 22 | type: mae 23 | weight: 0.25 24 | wait_iters: 0 25 | 26 | svd_loss: 27 | type: mae 28 | weight: 0.0 29 | wait_iters: 0 30 | 31 | level_loss: 32 | type: mae 33 | weight: 0.25 34 | wait_iters: 0 35 | 36 | range: 37 | pos: 1.0 38 | dir: 1.0 39 | 40 | jitter: 41 | pos: 0.1 42 | dir: 0.1 43 | bundle_size: 1 44 | 45 | param: 46 | n_dims: 4 47 | fn: two_plane 48 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/weak_teacher/lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: teacher 4 | batch_size: 8192 5 | 6 | dataset: 7 | name: "dense_${experiment.dataset.name}" 8 | collection: "${experiment.dataset.collection}_dense" 9 | split: train 10 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}_teacher/" 11 | 12 | num_rows: 10 13 | num_cols: 10 14 | train_row_skip: 1 15 | train_col_skip: 1 16 | size: 1000 17 | 18 | weight: 19 | type: exponential_decay 20 | start: 1.0 21 | decay: 0.25 22 | stop_weight: 0.025 23 | num_epochs: 500 24 | 25 | loss: 26 | type: mse 27 | -------------------------------------------------------------------------------- /conf/experiment/regularizers/weak_teacher/subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: teacher 4 | batch_size: 4096 5 | 6 | dataset: 7 | name: "dense_${experiment.dataset.name}" 8 | collection: "${experiment.dataset.collection}_dense" 9 | split: train 10 | root_dir: "${experiment.params.data_dir}/${experiment.dataset.data_subdir}/${experiment.dataset.collection}_teacher/" 11 | 12 | num_rows: 10 13 | num_cols: 10 14 | train_row_skip: 1 15 | train_col_skip: 1 16 | size: 1000 17 | 18 | weight: 19 | type: exponential_decay 20 | start: 1.0 21 | decay: 0.25 22 | stop_weight: 0.025 23 | num_epochs: 500 24 | 25 | loss: 26 | type: mse 27 | -------------------------------------------------------------------------------- /conf/experiment/shiny_lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: shiny 6 | - model: shiny_affine 7 | - training: shiny_lf 8 | - regularizers: all 9 | - visualizers: all 10 | 11 | 12 | -------------------------------------------------------------------------------- /conf/experiment/shiny_lf_dense.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: shiny_dense 6 | - model: shiny_affine 7 | - training: shiny_lf 8 | - regularizers: all 9 | - visualizers: all 10 | 11 | -------------------------------------------------------------------------------- /conf/experiment/shiny_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: shiny 6 | - model: shiny_subdivided_affine 7 | - training: shiny_subdivided 8 | - regularizers: all 9 | - visualizers: all 10 | 11 | 12 | -------------------------------------------------------------------------------- /conf/experiment/shiny_subdivided_dense.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: shiny_dense 6 | - model: shiny_subdivided_affine_dense 7 | - training: shiny_subdivided_dense 8 | - regularizers: all 9 | - visualizers: all 10 | -------------------------------------------------------------------------------- /conf/experiment/stanford_lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: stanford_gem 6 | - model: stanford_affine 7 | - training: stanford_lf 8 | - regularizers: all 9 | - visualizers: all 10 | -------------------------------------------------------------------------------- /conf/experiment/stanford_lf_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - params: local 5 | - dataset: stanford 6 | - model: stanford_half 7 | - training: stanford_lf_half 8 | - regularizers: all 9 | - visualizers: all 10 | -------------------------------------------------------------------------------- /conf/experiment/training/blender_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 4096 4 | ray_chunk: 8192 5 | net_chunk: 8192 6 | 7 | update_data_every: 1 8 | reload_data_every: 100 9 | 10 | log_every: 1 11 | val_every: 50 12 | render_every: 100 13 | ckpt_every: 100 14 | test_every: 200 15 | flush_logs: 1000 16 | num_epochs: 1000 17 | 18 | num_workers: 16 19 | num_gpus: 1 20 | 21 | multiscale: False 22 | scales: [2, 1] 23 | scale_epochs: [0, 50] 24 | scale_batch_sizes: [4096, 32768] 25 | 26 | weight_init: 27 | type: none 28 | 29 | loss: 30 | type: mse 31 | 32 | color: 33 | optimizer: adam 34 | lr: 0.00075 35 | min_lr: 0.000075 36 | clip: False 37 | clip_amount: 1.0 38 | alpha: 0.95 39 | momentum: 0.0 40 | weight_decay: 0 41 | 42 | lr_scheduler: exp 43 | warmup_multiplier: 1.0 44 | warmup_epochs: 1 45 | 46 | decay_epoch: 150 47 | decay_gamma: 0.25 48 | 49 | embedding: 50 | optimizer: adam 51 | lr: 0.0005 52 | min_lr: 0.00005 53 | clip: False 54 | clip_amount: 1.0 55 | alpha: 0.95 56 | momentum: 0.0 57 | weight_decay: 0 58 | 59 | lr_scheduler: exp 60 | warmup_multiplier: 1.0 61 | warmup_epochs: 1 62 | 63 | decay_epoch: 150 64 | decay_gamma: 0.125 65 | -------------------------------------------------------------------------------- /conf/experiment/training/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 4096 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | 7 | update_data_every: 1 8 | reload_data_every: 100 9 | 10 | log_every: 1 11 | val_every: 25 12 | render_every: 100 13 | ckpt_every: 100 14 | test_every: 200 15 | flush_logs: 1000 16 | num_epochs: 5000 17 | 18 | num_workers: 8 19 | num_gpus: 1 20 | 21 | weight_init: 22 | type: none 23 | 24 | loss: 25 | type: mse 26 | 27 | color: 28 | optimizer: adam 29 | lr: 0.001 30 | clip: False 31 | clip_amount: 1.0 32 | momentum: 0.9 33 | weight_decay: 0 34 | 35 | lr_scheduler: steplr 36 | warmup_multiplier: 1.0 37 | warmup_epochs: 0 38 | 39 | decay_step: 2000 40 | decay_gamma: 0.5 41 | 42 | embedding: 43 | optimizer: adam 44 | lr: 0.0005 45 | clip: False 46 | clip_amount: 1.0 47 | momentum: 0.9 48 | weight_decay: 0 49 | 50 | lr_scheduler: exp 51 | warmup_multiplier: 1.0 52 | warmup_epochs: 0 53 | 54 | decay_step: 500 55 | decay_gamma: 0.1 56 | -------------------------------------------------------------------------------- /conf/experiment/training/llff_lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 8192 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | 7 | reload_data_every: 100 8 | update_data_every: 1 9 | 10 | log_every: 1 11 | val_every: 100 12 | render_every: 100 13 | ckpt_every: 50 14 | test_every: 200 15 | flush_logs: 1000 16 | num_epochs: 5000 17 | 18 | num_workers: 8 19 | num_gpus: 1 20 | 21 | multiscale: False 22 | scales: [2, 1] 23 | scale_epochs: [0, 50] 24 | scale_batch_sizes: [4096, 32768] 25 | 26 | weight_init: 27 | type: none 28 | 29 | loss: 30 | type: mse 31 | 32 | color: 33 | optimizer: adam 34 | lr: 0.00075 35 | clip: False 36 | clip_amount: 1.0 37 | momentum: 0.9 38 | weight_decay: 0 39 | 40 | lr_scheduler: exp 41 | warmup_multiplier: 1.0 42 | warmup_epochs: 1 43 | 44 | decay_step: 1000 45 | decay_gamma: 0.25 46 | 47 | embedding: 48 | optimizer: adam 49 | lr: 0.00035 50 | clip: False 51 | clip_amount: 1.0 52 | momentum: 0.9 53 | weight_decay: 0 54 | 55 | lr_scheduler: exp 56 | warmup_multiplier: 1.0 57 | warmup_epochs: 1 58 | 59 | decay_step: 1000 60 | decay_gamma: 0.1 61 | -------------------------------------------------------------------------------- /conf/experiment/training/llff_lf_large.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 32768 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | 7 | update_data_every: 1 8 | reload_data_every: 100 9 | 10 | log_every: 1 11 | val_every: 100 12 | render_every: 100 13 | ckpt_every: 50 14 | test_every: 200 15 | flush_logs: 1000 16 | num_epochs: 5000 17 | 18 | num_workers: 8 19 | num_gpus: 1 20 | 21 | multiscale: False 22 | scales: [2, 1] 23 | scale_epochs: [0, 50] 24 | scale_batch_sizes: [4096, 32768] 25 | 26 | weight_init: 27 | type: none 28 | 29 | loss: 30 | type: mse 31 | 32 | color: 33 | optimizer: adam 34 | lr: 0.0005 35 | clip: False 36 | clip_amount: 1.0 37 | momentum: 0.9 38 | weight_decay: 0 39 | 40 | lr_scheduler: exp 41 | warmup_multiplier: 1.0 42 | warmup_epochs: 1 43 | 44 | decay_step: 1000 45 | decay_gamma: 0.25 46 | 47 | embedding: 48 | optimizer: adam 49 | lr: 0.00025 50 | clip: False 51 | clip_amount: 1.0 52 | momentum: 0.9 53 | weight_decay: 0 54 | 55 | lr_scheduler: exp 56 | warmup_multiplier: 1.0 57 | warmup_epochs: 1 58 | 59 | decay_step: 1000 60 | decay_gamma: 0.1 61 | -------------------------------------------------------------------------------- /conf/experiment/training/llff_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 4096 4 | ray_chunk: 8192 5 | net_chunk: 8192 6 | 7 | update_data_every: 1 8 | reload_data_every: 100 9 | 10 | log_every: 1 11 | val_every: 50 12 | render_every: 200 13 | ckpt_every: 100 14 | test_every: 200 15 | flush_logs: 1000 16 | num_epochs: 5000 17 | 18 | num_workers: 8 19 | num_gpus: 1 20 | 21 | multiscale: False 22 | scales: [2, 1] 23 | scale_epochs: [0, 50] 24 | scale_batch_sizes: [4096, 32768] 25 | 26 | weight_init: 27 | type: none 28 | 29 | loss: 30 | type: mse 31 | 32 | color: 33 | optimizer: adam 34 | lr: 0.00075 35 | clip: False 36 | clip_amount: 1.0 37 | momentum: 0.9 38 | weight_decay: 0 39 | 40 | lr_scheduler: exp 41 | warmup_multiplier: 1.0 42 | warmup_epochs: 1 43 | 44 | decay_step: 500 45 | decay_gamma: 0.25 46 | 47 | embedding: 48 | optimizer: adam 49 | lr: 0.0005 50 | clip: False 51 | clip_amount: 1.0 52 | momentum: 0.9 53 | weight_decay: 0 54 | 55 | lr_scheduler: exp 56 | warmup_multiplier: 1.0 57 | warmup_epochs: 1 58 | 59 | decay_step: 500 60 | decay_gamma: 0.125 61 | -------------------------------------------------------------------------------- /conf/experiment/training/llff_subdivided_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 8192 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | 7 | update_data_every: 1 8 | reload_data_every: 100 9 | 10 | log_every: 1 11 | val_every: 50 12 | render_every: 200 13 | ckpt_every: 100 14 | test_every: 200 15 | flush_logs: 1000 16 | num_epochs: 5000 17 | 18 | num_workers: 8 19 | num_gpus: 1 20 | 21 | multiscale: False 22 | scales: [2, 1] 23 | scale_epochs: [0, 50] 24 | scale_batch_sizes: [4096, 32768] 25 | 26 | weight_init: 27 | type: none 28 | 29 | loss: 30 | type: mse 31 | 32 | color: 33 | optimizer: adam 34 | lr: 0.00075 35 | clip: False 36 | clip_amount: 1.0 37 | momentum: 0.9 38 | weight_decay: 0 39 | 40 | lr_scheduler: exp 41 | warmup_multiplier: 1.0 42 | warmup_epochs: 1 43 | 44 | decay_step: 1000 45 | decay_gamma: 0.25 46 | 47 | embedding: 48 | optimizer: adam 49 | lr: 0.00035 50 | clip: False 51 | clip_amount: 1.0 52 | momentum: 0.9 53 | weight_decay: 0 54 | 55 | lr_scheduler: exp 56 | warmup_multiplier: 1.0 57 | warmup_epochs: 1 58 | 59 | decay_step: 500 60 | decay_gamma: 0.1 61 | -------------------------------------------------------------------------------- /conf/experiment/training/shiny_lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 16384 4 | ray_chunk: 32768 5 | net_chunk: 32768 6 | 7 | 8 | update_data_every: 1 9 | reload_data_every: 100 10 | 11 | log_every: 1 12 | val_every: 100 13 | render_every: 1000 14 | ckpt_every: 100 15 | test_every: 200 16 | flush_logs: 1000 17 | num_epochs: 1000 18 | 19 | num_workers: 16 20 | num_gpus: 1 21 | 22 | multiscale: False 23 | scales: [2, 1] 24 | scale_epochs: [0, 50] 25 | scale_batch_sizes: [4096, 32768] 26 | 27 | weight_init: 28 | type: none 29 | 30 | loss: 31 | type: mse 32 | 33 | color: 34 | optimizer: adam 35 | lr: 0.001 36 | clip: False 37 | clip_amount: 1.0 38 | momentum: 0.9 39 | weight_decay: 0 40 | 41 | lr_scheduler: exp 42 | warmup_multiplier: 1.0 43 | warmup_epochs: 1 44 | 45 | decay_step: 400 46 | decay_gamma: 0.25 47 | 48 | embedding: 49 | optimizer: adam 50 | lr: 0.00075 51 | clip: False 52 | clip_amount: 1.0 53 | momentum: 0.9 54 | weight_decay: 0 55 | 56 | lr_scheduler: exp 57 | warmup_multiplier: 1.0 58 | warmup_epochs: 1 59 | 60 | decay_step: 200 61 | decay_gamma: 0.125 62 | -------------------------------------------------------------------------------- /conf/experiment/training/shiny_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 4096 4 | ray_chunk: 8192 5 | net_chunk: 8192 6 | 7 | 8 | update_data_every: 1 9 | reload_data_every: 100 10 | 11 | log_every: 1 12 | val_every: 100 13 | render_every: 1000 14 | ckpt_every: 100 15 | test_every: 200 16 | flush_logs: 1000 17 | num_epochs: 1000 18 | 19 | num_workers: 16 20 | num_gpus: 1 21 | 22 | multiscale: False 23 | scales: [2, 1] 24 | scale_epochs: [0, 50] 25 | scale_batch_sizes: [4096, 32768] 26 | 27 | weight_init: 28 | type: none 29 | 30 | loss: 31 | type: mse 32 | 33 | color: 34 | optimizer: adam 35 | lr: 0.00075 36 | clip: False 37 | clip_amount: 1.0 38 | momentum: 0.9 39 | weight_decay: 0 40 | 41 | lr_scheduler: exp 42 | warmup_multiplier: 1.0 43 | warmup_epochs: 1 44 | 45 | decay_step: 500 46 | decay_gamma: 0.25 47 | 48 | embedding: 49 | optimizer: adam 50 | lr: 0.0005 51 | clip: False 52 | clip_amount: 1.0 53 | momentum: 0.9 54 | weight_decay: 0 55 | 56 | lr_scheduler: exp 57 | warmup_multiplier: 1.0 58 | warmup_epochs: 1 59 | 60 | decay_step: 500 61 | decay_gamma: 0.125 62 | -------------------------------------------------------------------------------- /conf/experiment/training/shiny_subdivided_dense.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 8192 4 | ray_chunk: 8192 5 | net_chunk: 8192 6 | 7 | 8 | update_data_every: 1 9 | reload_data_every: 100 10 | 11 | log_every: 1 12 | val_every: 100 13 | render_every: 1000 14 | ckpt_every: 100 15 | test_every: 200 16 | flush_logs: 1000 17 | num_epochs: 1000 18 | 19 | num_workers: 16 20 | num_gpus: 1 21 | 22 | multiscale: False 23 | scales: [2, 1] 24 | scale_epochs: [0, 50] 25 | scale_batch_sizes: [4096, 32768] 26 | 27 | weight_init: 28 | type: none 29 | 30 | loss: 31 | type: mse 32 | 33 | color: 34 | optimizer: adam 35 | lr: 0.001 36 | clip: False 37 | clip_amount: 1.0 38 | momentum: 0.9 39 | weight_decay: 0 40 | 41 | lr_scheduler: exp 42 | warmup_multiplier: 1.0 43 | warmup_epochs: 1 44 | 45 | decay_step: 200 46 | decay_gamma: 0.25 47 | 48 | embedding: 49 | optimizer: adam 50 | lr: 0.00075 51 | clip: False 52 | clip_amount: 1.0 53 | momentum: 0.9 54 | weight_decay: 0 55 | 56 | lr_scheduler: exp 57 | warmup_multiplier: 1.0 58 | warmup_epochs: 1 59 | 60 | decay_step: 200 61 | decay_gamma: 0.125 62 | -------------------------------------------------------------------------------- /conf/experiment/training/stanford_lf.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 16384 4 | ray_chunk: 32768 5 | net_chunk: 32768 6 | 7 | 8 | update_data_every: 1 9 | reload_data_every: 100 10 | 11 | log_every: 1 12 | val_every: 100 13 | render_every: 1000 14 | ckpt_every: 100 15 | test_every: 200 16 | flush_logs: 1000 17 | num_epochs: 1000 18 | 19 | num_workers: 16 20 | num_gpus: 1 21 | 22 | multiscale: False 23 | scales: [2, 1] 24 | scale_epochs: [0, 50] 25 | scale_batch_sizes: [4096, 32768] 26 | 27 | weight_init: 28 | type: none 29 | 30 | loss: 31 | type: mse 32 | 33 | color: 34 | optimizer: adam 35 | lr: 0.001 36 | clip: False 37 | clip_amount: 1.0 38 | momentum: 0.9 39 | weight_decay: 0 40 | 41 | lr_scheduler: exp 42 | warmup_multiplier: 1.0 43 | warmup_epochs: 1 44 | 45 | decay_step: 400 46 | decay_gamma: 0.25 47 | 48 | embedding: 49 | optimizer: adam 50 | lr: 0.00075 51 | clip: False 52 | clip_amount: 1.0 53 | momentum: 0.9 54 | weight_decay: 0 55 | 56 | lr_scheduler: exp 57 | warmup_multiplier: 1.0 58 | warmup_epochs: 1 59 | 60 | decay_step: 200 61 | decay_gamma: 0.125 62 | -------------------------------------------------------------------------------- /conf/experiment/training/stanford_lf_half.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 16384 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | 7 | 8 | update_data_every: 1 9 | reload_data_every: 100 10 | 11 | log_every: 1 12 | val_every: 100 13 | render_every: 1000 14 | ckpt_every: 100 15 | test_every: 200 16 | flush_logs: 1000 17 | num_epochs: 1000 18 | 19 | num_workers: 16 20 | num_gpus: 1 21 | 22 | multiscale: False 23 | scales: [2, 1] 24 | scale_epochs: [0, 50] 25 | scale_batch_sizes: [4096, 32768] 26 | 27 | weight_init: 28 | type: none 29 | 30 | loss: 31 | type: mse 32 | 33 | color: 34 | optimizer: adam 35 | lr: 0.001 36 | clip: False 37 | clip_amount: 1.0 38 | momentum: 0.9 39 | weight_decay: 0 40 | 41 | lr_scheduler: exp 42 | warmup_multiplier: 1.0 43 | warmup_epochs: 1 44 | 45 | decay_step: 200 46 | decay_gamma: 0.25 47 | 48 | embedding: 49 | optimizer: adam 50 | lr: 0.001 51 | clip: False 52 | clip_amount: 1.0 53 | momentum: 0.9 54 | weight_decay: 0 55 | 56 | lr_scheduler: exp 57 | warmup_multiplier: 1.0 58 | warmup_epochs: 1 59 | 60 | decay_step: 100 61 | decay_gamma: 0.125 62 | 63 | -------------------------------------------------------------------------------- /conf/experiment/training/stanford_subdivided.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | batch_size: 4096 4 | ray_chunk: 131072 5 | net_chunk: 131072 6 | 7 | update_data_every: 1 8 | reload_data_every: 100 9 | 10 | log_every: 1 11 | val_every: 200 12 | render_every: 200 13 | ckpt_every: 50 14 | test_every: 1000 15 | flush_logs: 1000 16 | num_epochs: 5000 17 | 18 | num_workers: 8 19 | num_gpus: 1 20 | 21 | render_only: False 22 | test_only: False 23 | 24 | multiscale: False 25 | scales: [2, 1] 26 | scale_epochs: [0, 50] 27 | scale_batch_sizes: [4096, 32768] 28 | 29 | weight_init: 30 | type: none 31 | 32 | loss: 33 | type: mse 34 | 35 | color: 36 | optimizer: adam 37 | lr: 0.0005 38 | clip: False 39 | clip_amount: 1.0 40 | momentum: 0.9 41 | weight_decay: 0 42 | 43 | lr_scheduler: exp 44 | warmup_multiplier: 1.0 45 | warmup_epochs: 1 46 | 47 | decay_step: 1000 48 | decay_gamma: 0.25 49 | 50 | embedding: 51 | optimizer: adam 52 | lr: 0.00025 53 | clip: False 54 | clip_amount: 1.0 55 | momentum: 0.9 56 | weight_decay: 0 57 | 58 | lr_scheduler: exp 59 | warmup_multiplier: 1.0 60 | warmup_epochs: 1 61 | 62 | decay_step: 1000 63 | decay_gamma: 0.1 64 | -------------------------------------------------------------------------------- /conf/experiment/visualizers/all.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | defaults: 4 | - epipolar: default 5 | - closest_view: default 6 | -------------------------------------------------------------------------------- /conf/experiment/visualizers/closest_view/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: closest_view 4 | -------------------------------------------------------------------------------- /conf/experiment/visualizers/epipolar/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: epipolar 4 | t: 5 | v: 6 | H: 7 | 8 | st_scale: 9 | uv_scale: 10 | 11 | near: -1.0 12 | far: 0.0 13 | -------------------------------------------------------------------------------- /conf/experiment/visualizers/focus/default.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | 3 | type: focus 4 | 5 | s: 6 | t: 7 | 8 | ds: 200.0 9 | dt: 200.0 10 | 11 | st_scale: 12 | uv_scale: 13 | 14 | near: -1.0 15 | far: 0.0 16 | focal: 0.0 17 | -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from .blender import BlenderLightfieldDataset, BlenderDataset, DenseBlenderDataset 8 | from .fourier import FourierDataset, FourierLightfieldDataset 9 | from .llff import LLFFDataset, DenseLLFFDataset 10 | from .random import ( 11 | RandomRayDataset, 12 | RandomViewSubsetDataset, 13 | RandomRayLightfieldDataset, 14 | ) 15 | from .shiny import ShinyDataset, DenseShinyDataset 16 | from .stanford import StanfordLightfieldDataset 17 | from .tamul import TamulLightfieldDataset 18 | from .tanks import TanksDataset 19 | 20 | dataset_dict = { 21 | 'fourier': FourierDataset, 22 | 'fourier_lightfield': FourierLightfieldDataset, 23 | 'random_ray': RandomRayDataset, 24 | 'random_lightfield': RandomRayLightfieldDataset, 25 | 'random_view': RandomViewSubsetDataset, 26 | 'blender': BlenderDataset, 27 | 'dense_blender': DenseBlenderDataset, 28 | 'llff': LLFFDataset, 29 | 'dense_llff': DenseLLFFDataset, 30 | 'dense_shiny': DenseShinyDataset, 31 | 'shiny': ShinyDataset, 32 | 'tanks': TanksDataset, 33 | 'blender_lightfield': BlenderLightfieldDataset, 34 | 'stanford': StanfordLightfieldDataset, 35 | 'tamul_lightfield': TamulLightfieldDataset, 36 | } 37 | -------------------------------------------------------------------------------- /datasets/stanford.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import numpy as np 9 | 10 | from PIL import Image 11 | 12 | from datasets.lightfield import LightfieldDataset 13 | 14 | from utils.ray_utils import ( 15 | get_lightfield_rays 16 | ) 17 | 18 | 19 | class StanfordLightfieldDataset(LightfieldDataset): 20 | def __init__( 21 | self, 22 | cfg, 23 | split='train', 24 | **kwargs 25 | ): 26 | self.use_file_coords = cfg.dataset.lightfield.use_file_coords if 'use_file_coords' in cfg.dataset.lightfield else False 27 | 28 | super().__init__(cfg, split, **kwargs) 29 | 30 | if self.split == 'train': 31 | self.poses = [] 32 | 33 | for (s_idx, t_idx) in self.all_st_idx: 34 | idx = t_idx * self.cols + s_idx 35 | coord = self.normalize_coord(self.camera_coords[idx]) 36 | self.poses.append(coord) 37 | 38 | def read_meta(self): 39 | self.image_paths = sorted( 40 | self.pmgr.ls(self.root_dir) 41 | ) 42 | 43 | if self.img_wh is None: 44 | image_path = self.image_paths[0] 45 | 46 | with self.pmgr.open( 47 | os.path.join(self.root_dir, image_path), 48 | 'rb' 49 | ) as im_file: 50 | img = np.array(Image.open(im_file).convert('RGB')) 51 | 52 | self._img_wh = (img.shape[1] // self.downsample, img.shape[0] // self.downsample) 53 | self.img_wh = (img.shape[1] // self.downsample, img.shape[0] // self.downsample) 54 | self.aspect = (float(self.img_wh[0]) / self.img_wh[1]) 55 | 56 | self.camera_coords = [] 57 | 58 | if self.use_file_coords: 59 | for image_path in self.image_paths: 60 | if self.dataset_cfg.collection in ['beans', 'tarot', 'tarot_small', 'knights']: 61 | yx = image_path.split('_')[-2:] 62 | y = float(yx[0]) 63 | x = float(yx[1].split('.png')[0]) 64 | else: 65 | yx = image_path.split('_')[-3:-1] 66 | y, x = -float(yx[0]), float(yx[1]) 67 | 68 | self.camera_coords.append((x, y)) 69 | 70 | def get_camera_range(self): 71 | xs = [coord[0] for coord in self.camera_coords] 72 | ys = [coord[1] for coord in self.camera_coords] 73 | 74 | min_x, max_x = np.min(xs), np.max(xs) 75 | min_y, max_y = np.min(ys), np.max(ys) 76 | 77 | return (min_x, max_x), (min_y, max_y) 78 | 79 | def normalize_coord(self, coord): 80 | x_range, y_range = self.get_camera_range() 81 | aspect = (x_range[1] - x_range[0]) / (y_range[1] - y_range[0]) 82 | 83 | norm_x = ((coord[0] - x_range[0]) / (x_range[1] - x_range[0])) * 2 - 1 84 | norm_y = (((coord[1] - y_range[0]) / (y_range[1] - y_range[0])) * 2 - 1) / aspect 85 | 86 | return (norm_x, norm_y) 87 | 88 | def get_rays(self, s_idx, t_idx): 89 | if not self.use_file_coords: 90 | return super().get_rays(s_idx, t_idx) 91 | 92 | idx = t_idx * self.cols + s_idx 93 | coord = self.normalize_coord(self.camera_coords[idx]) 94 | 95 | if self.split == 'render': 96 | st_scale = self.vis_st_scale 97 | else: 98 | st_scale = self.st_scale 99 | 100 | return get_lightfield_rays( 101 | self.img_wh[0], self.img_wh[1], 102 | coord[0], coord[1], 103 | self.aspect, 104 | st_scale=st_scale 105 | ) 106 | 107 | def get_rgb(self, s_idx, t_idx): 108 | idx = t_idx * self.cols + s_idx 109 | image_path = self.image_paths[idx] 110 | 111 | with self.pmgr.open( 112 | os.path.join(self.root_dir, image_path), 113 | 'rb' 114 | ) as im_file: 115 | img = Image.open(im_file).convert('RGB') 116 | 117 | img = img.resize(self.img_wh, Image.LANCZOS) 118 | img = self.transform(img) 119 | img = img.view(3, -1).permute(1, 0) 120 | 121 | return img 122 | -------------------------------------------------------------------------------- /datasets/tamul.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import numpy as np 9 | 10 | from PIL import Image 11 | 12 | from datasets.lightfield import LightfieldDataset 13 | 14 | 15 | class TamulLightfieldDataset(LightfieldDataset): 16 | def __init__( 17 | self, 18 | cfg, 19 | split='train', 20 | **kwargs 21 | ): 22 | super().__init__(cfg, split, **kwargs) 23 | 24 | def read_meta(self): 25 | self.image_paths = sorted( 26 | self.pmgr.ls(self.root_dir) 27 | ) 28 | 29 | if self.img_wh is None: 30 | image_path = self.image_paths[0] 31 | 32 | with self.pmgr.open( 33 | os.path.join(self.root_dir, image_path), 34 | 'rb' 35 | ) as im_file: 36 | img = np.array(Image.open(im_file).convert('RGB')) 37 | 38 | self._img_wh = (img.shape[1] // self.downsample, img.shape[0] // self.downsample) 39 | self.img_wh = (img.shape[1] // self.downsample, img.shape[0] // self.downsample) 40 | self.aspect = (float(self.img_wh[0]) / self.img_wh[1]) 41 | 42 | def get_rgb(self, s_idx, t_idx): 43 | idx = t_idx * self.cols + s_idx 44 | image_path = self.image_paths[idx] 45 | 46 | with self.pmgr.open( 47 | os.path.join(self.root_dir, image_path), 48 | 'rb' 49 | ) as im_file: 50 | img = Image.open(im_file).convert('RGB') 51 | 52 | img = img.resize(self.img_wh, Image.LANCZOS) 53 | img = self.transform(img) 54 | img = img.view(3, -1).permute(1, 0) 55 | 56 | return img 57 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | name: neural-light-fields 8 | channels: 9 | - pyg 10 | - pytorch 11 | - conda-forge 12 | - defaults 13 | dependencies: 14 | - cudatoolkit=11.3.1 15 | - cudatoolkit-dev 16 | - kornia=0.6.1 17 | - pillow=8.4.0 18 | - python=3.8 19 | - pip 20 | - setuptools=59.5.0 21 | - pytorch=1.10.0 22 | - pytorch-lightning=1.5.5 23 | - pytorch-scatter=2.0.9 24 | - scikit-image=0.18.3 25 | - scipy=1.7.1 26 | - pip: 27 | - hydra-core==1.1.1 28 | - imageio==2.13.3 29 | - iopath==0.1.9 30 | - omegaconf==2.1.1 31 | - opencv-python 32 | - torchvision==0.11.1 33 | - gdown 34 | -------------------------------------------------------------------------------- /environment_win.yml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | name: neural-light-fields 8 | channels: 9 | - pyg 10 | - pytorch 11 | - conda-forge 12 | - defaults 13 | dependencies: 14 | - cudatoolkit=11.3.1 15 | - cudatoolkit-dev 16 | - kornia=0.6.1 17 | - pillow=8.4.0 18 | - python=3.8 19 | - pip 20 | - pytorch=1.10.0 21 | - pytorch-lightning=1.5.5 22 | - pytorch-scatter=2.0.9 23 | - pywin32 24 | - scikit-image=0.18.3 25 | - scipy=1.7.1 26 | - pip: 27 | - hydra-core==1.1.1 28 | - imageio==2.13.3 29 | - iopath==0.1.9 30 | - omegaconf==2.1.1 31 | - opencv-python 32 | - torchvision==0.11.1 33 | 34 | -------------------------------------------------------------------------------- /losses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | from torch import nn 9 | 10 | 11 | class MSELoss(nn.Module): 12 | def __init__(self, *args): 13 | super().__init__() 14 | self.loss = nn.MSELoss(reduction='mean') 15 | 16 | def forward(self, inputs, targets): 17 | loss = self.loss(inputs, targets) 18 | return loss 19 | 20 | 21 | class MAELoss(nn.Module): 22 | def __init__(self, *args): 23 | super().__init__() 24 | self.loss = nn.L1Loss(reduction='mean') 25 | 26 | def forward(self, inputs, targets): 27 | loss = self.loss(inputs, targets) 28 | return loss 29 | 30 | 31 | class ComplexMSELoss(nn.Module): 32 | def __init__(self, *args): 33 | super().__init__() 34 | self.loss = nn.MSELoss(reduction='mean') 35 | 36 | def forward(self, inputs, targets): 37 | loss = self.loss(torch.real(inputs), torch.real(targets)) 38 | loss += self.loss(torch.imag(inputs), torch.imag(targets)) 39 | return loss 40 | 41 | 42 | class ComplexMAELoss(nn.Module): 43 | def __init__(self, *args): 44 | super().__init__() 45 | self.loss = nn.L1Loss(reduction='mean') 46 | 47 | def forward(self, inputs, targets): 48 | loss = self.loss(torch.real(inputs), torch.real(targets)) 49 | loss += self.loss(torch.imag(inputs), torch.imag(targets)) 50 | return loss 51 | 52 | 53 | class MSETopN(nn.Module): 54 | def __init__(self, cfg): 55 | super().__init__() 56 | 57 | self.frac = cfg.frac 58 | self.loss = nn.MSELoss(reduction='mean') 59 | 60 | def forward(self, inputs, targets): 61 | diff = torch.abs(inputs - targets) 62 | n = int(self.frac * targets.shape[0]) 63 | 64 | idx = torch.argsort(diff, dim=0) 65 | 66 | targets_sorted = torch.gather(targets, 0, idx) 67 | targets_sorted = targets_sorted[:n] 68 | 69 | inputs_sorted = torch.gather(inputs, 0, idx) 70 | inputs_sorted = inputs_sorted[:n] 71 | 72 | loss = self.loss(inputs_sorted, targets_sorted) 73 | return loss 74 | 75 | 76 | class MAETopN(nn.Module): 77 | def __init__(self, cfg): 78 | super().__init__() 79 | 80 | self.frac = cfg.frac 81 | self.loss = nn.L1Loss(reduction='mean') 82 | 83 | def forward(self, inputs, targets): 84 | diff = torch.abs(inputs - targets) 85 | n = int(self.frac * targets.shape[0]) 86 | 87 | idx = torch.argsort(diff, dim=0) 88 | 89 | targets_sorted = torch.gather(targets, 0, idx) 90 | targets_sorted = targets_sorted[:n] 91 | 92 | inputs_sorted = torch.gather(inputs, 0, idx) 93 | inputs_sorted = inputs_sorted[:n] 94 | 95 | loss = self.loss(inputs_sorted, targets_sorted) 96 | return loss 97 | 98 | 99 | loss_dict = { 100 | 'mse': MSELoss, 101 | 'mae': MAELoss, 102 | 'complex_mse': ComplexMSELoss, 103 | 'complex_mae': ComplexMAELoss, 104 | 'mse_top_n': MSETopN, 105 | 'mae_top_n': MAETopN, 106 | } 107 | -------------------------------------------------------------------------------- /metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | import numpy as np 9 | from kornia.losses import ssim as dssim 10 | 11 | from skimage.metrics import structural_similarity, peak_signal_noise_ratio # @manual=fbsource//third-party/pythonlibs/native/scikit-image:scikit-image 12 | 13 | 14 | def mse(image_pred, image_gt, valid_mask=None, reduction='mean'): 15 | value = (image_pred-image_gt)**2 16 | 17 | if valid_mask is not None: 18 | value = value[valid_mask] 19 | if reduction == 'mean': 20 | return torch.mean(value) 21 | 22 | return value 23 | 24 | def psnr(image0, image1): 25 | return peak_signal_noise_ratio(np.array(image1), np.array(image0), data_range=1.0) 26 | 27 | def ssim(image0, image1): 28 | return structural_similarity(np.array(image1), np.array(image0), win_size=11, multichannel=True, gaussian_weights=True) 29 | 30 | def psnr_gpu(image_pred, image_gt, valid_mask=None, reduction='mean'): 31 | return -10*torch.log10(mse(image_pred, image_gt, valid_mask, reduction)) 32 | 33 | def ssim_gpu(image_pred, image_gt, reduction='mean'): 34 | """ 35 | image_pred and image_gt: (1, 3, H, W) 36 | """ 37 | dssim_ = dssim(image_pred, image_gt, 11, reduction) # dissimilarity in [0, 1] 38 | return 1-2*dssim_ # in [-1, 1] 39 | 40 | def get_mean_outputs(outputs, cpu=False): 41 | # Stack 42 | stacked = {} 43 | 44 | for x in outputs: 45 | for key, val in x.items(): 46 | if key not in stacked: 47 | stacked[key] = [] 48 | 49 | stacked[key].append(val) 50 | 51 | # Mean 52 | mean = {} 53 | 54 | for key in stacked: 55 | if cpu: 56 | mean_val = np.stack(stacked[key]).mean() 57 | else: 58 | mean_val = torch.stack(stacked[key]).mean() 59 | 60 | mean[key] = mean_val 61 | 62 | return mean 63 | -------------------------------------------------------------------------------- /nlf/nets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | from torch import nn 9 | 10 | from nlf.embedding import WindowedPE 11 | from nlf.activations import get_activation 12 | 13 | 14 | class BaseNet(nn.Module): 15 | def __init__( 16 | self, 17 | in_channels, 18 | out_channels, 19 | cfg, 20 | ): 21 | super().__init__() 22 | 23 | self.D = cfg.depth 24 | self.W = cfg.hidden_channels 25 | 26 | self.in_channels = in_channels 27 | self.out_channels = out_channels 28 | self.skips = cfg.skips if 'skips' in cfg else [] 29 | self.activation = cfg.activation if 'activation' in cfg else 'sigmoid' 30 | self.layer_activation = cfg.layer_activation if 'layer_activation' in cfg else 'leaky_relu' 31 | 32 | for i in range(self.D): 33 | if i == 0: 34 | layer = nn.Linear(self.in_channels, self.W) 35 | elif i in self.skips: 36 | layer = nn.Linear(self.W + self.in_channels, self.W) 37 | else: 38 | layer = nn.Linear(self.W, self.W) 39 | 40 | layer = nn.Sequential(layer, get_activation(self.layer_activation)) 41 | setattr(self, f'encoding{i+1}', layer) 42 | 43 | self.encoding_final = nn.Linear(self.W, self.W) 44 | 45 | # Output 46 | self.out_layer = nn.Sequential( 47 | nn.Linear(self.W, self.out_channels), 48 | get_activation(self.activation) 49 | ) 50 | 51 | def forward(self, x, sigma_only=False): 52 | input_x = x 53 | 54 | for i in range(self.D): 55 | if i in self.skips: 56 | x = torch.cat([input_x, x], -1) 57 | 58 | x = getattr(self, f'encoding{i+1}')(x) 59 | 60 | encoding_final = self.encoding_final(x) 61 | return self.out_layer(encoding_final) 62 | 63 | def set_iter(self, i): 64 | pass 65 | 66 | 67 | class NeRFNet(nn.Module): 68 | def __init__( 69 | self, 70 | in_channels, 71 | out_channels, 72 | cfg, 73 | ): 74 | super().__init__() 75 | 76 | self.D = cfg.depth 77 | self.W = cfg.hidden_channels 78 | 79 | self.in_channels = in_channels 80 | self.out_channels = out_channels 81 | self.skips = cfg.skips if 'skips' in cfg else [] 82 | self.activation = cfg.activation if 'activation' in cfg else 'sigmoid' 83 | 84 | self.pos_pe = WindowedPE( 85 | 3, 86 | cfg.pos_pe 87 | ) 88 | self.dir_pe = WindowedPE( 89 | 3, 90 | cfg.dir_pe 91 | ) 92 | 93 | for i in range(self.D): 94 | if i == 0: 95 | layer = nn.Linear(self.pos_pe.out_channels, self.W) 96 | elif i in self.skips: 97 | layer = nn.Linear(self.W + self.pos_pe.out_channels, self.W) 98 | else: 99 | layer = nn.Linear(self.W, self.W) 100 | 101 | layer = nn.Sequential(layer, nn.ReLU(True)) 102 | setattr(self, f'encoding{i+1}', layer) 103 | 104 | self.encoding_dir = nn.Sequential( 105 | nn.Linear(self.W + self.dir_pe.out_channels, self.W // 2), 106 | nn.ReLU(True) 107 | ) 108 | 109 | self.encoding_final = nn.Linear(self.W, self.W) 110 | 111 | # Output 112 | self.sigma = nn.Sequential( 113 | nn.Linear(self.W, 1), 114 | nn.Sigmoid(), 115 | ) 116 | 117 | self.rgb = nn.Sequential( 118 | nn.Linear(self.W // 2, 3), 119 | nn.Sigmoid(), 120 | ) 121 | 122 | def forward(self, x, sigma_only=False): 123 | input_pos, input_dir = torch.split(x, [3, 3], -1) 124 | 125 | input_pos = self.pos_pe(input_pos) 126 | input_dir = self.dir_pe(input_dir) 127 | 128 | pos = input_pos 129 | 130 | for i in range(self.D): 131 | if i in self.skips: 132 | pos = torch.cat([input_pos, pos], -1) 133 | 134 | pos = getattr(self, f'encoding{i+1}')(pos) 135 | 136 | encoding_final = self.encoding_final(x) 137 | return self.out_layer(encoding_final) 138 | 139 | def set_iter(self, i): 140 | pass 141 | 142 | net_dict = { 143 | 'base': BaseNet, 144 | 'nerf': NeRFNet, 145 | } 146 | -------------------------------------------------------------------------------- /nlf/regularizers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from .depth_classification import DepthClassificationRegularizer 8 | from .fourier import FourierRegularizer 9 | from .inverse_ray_depth import InverseRayDepthRegularizer 10 | from .multiple_ray_depth import MultipleRayDepthRegularizer 11 | from .ray_blending import RayDepthBlendingRegularizer 12 | from .ray_bundle import RayBundleRegularizer 13 | from .ray_depth import RayDepthRegularizer 14 | from .ray_depth_occ_dir import RayDepthOccDirRegularizer 15 | from .ray_interpolation import RayInterpolationRegularizer 16 | from .teacher import TeacherRegularizer, BlurryTeacherRegularizer 17 | from .voxel_sparsity import VoxelSparsityRegularizer 18 | from .warp import WarpRegularizer, WarpLevelSetRegularizer 19 | 20 | regularizer_dict = { 21 | 'ray_depth': RayDepthRegularizer, 22 | 'ray_depth_occ_dir': RayDepthOccDirRegularizer, 23 | 'multiple_ray_depth': MultipleRayDepthRegularizer, 24 | 'inverse_ray_depth': InverseRayDepthRegularizer, 25 | 'ray_depth_blending': RayDepthBlendingRegularizer, 26 | 'ray_interpolation': RayInterpolationRegularizer, 27 | 'depth_classification': DepthClassificationRegularizer, 28 | 'fourier': FourierRegularizer, 29 | 'ray_bundle': RayBundleRegularizer, 30 | 'teacher': TeacherRegularizer, 31 | 'blurry_teacher': BlurryTeacherRegularizer, 32 | 'voxel_sparsity': VoxelSparsityRegularizer, 33 | 'warp': WarpRegularizer, 34 | 'warp_level': WarpLevelSetRegularizer, 35 | } 36 | -------------------------------------------------------------------------------- /nlf/regularizers/fourier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | 9 | from nlf.regularizers.base import BaseRegularizer 10 | from datasets.fourier import fft_rgb 11 | 12 | 13 | class FourierRegularizer(BaseRegularizer): 14 | def __init__( 15 | self, 16 | system, 17 | cfg 18 | ): 19 | super().__init__(system, cfg) 20 | 21 | self.range = cfg.range 22 | self.use_absolute = 'complex' not in cfg.fourier_loss.type 23 | 24 | def loss(self, train_batch, batch_idx): 25 | system = self.get_system() 26 | dataset = self.get_dataset() 27 | 28 | ## Get rays 29 | all_rgb_fft = dataset.all_rgb_fft.to( 30 | train_batch['rays'].device 31 | ) 32 | 33 | ## Query 34 | rays = dataset.get_random_rays(self.cfg.range).type_as( 35 | train_batch['rays'] 36 | ) 37 | rgb = system(rays)['rgb'].view( 38 | 1, system.img_wh[1], system.img_wh[0], 3 39 | ) 40 | 41 | #### Losses #### 42 | 43 | all_losses = { 44 | loss: 0.0 for loss in self.loss_fns.keys() 45 | } 46 | 47 | if self._do_loss('fourier_loss'): 48 | rgb_fft = fft_rgb(rgb) 49 | 50 | if self.use_absolute: 51 | rgb_fft = torch.abs(rgb_fft) 52 | all_rgb_fft = torch.abs(all_rgb_fft) 53 | 54 | all_losses['fourier_loss'] = self._loss_fn( 55 | 'fourier_loss', 56 | rgb_fft, 57 | all_rgb_fft 58 | ) 59 | 60 | ## Total loss 61 | total_loss = 0.0 62 | 63 | for name in all_losses.keys(): 64 | print(name + ':', all_losses[name]) 65 | total_loss += all_losses[name] 66 | 67 | return total_loss 68 | -------------------------------------------------------------------------------- /nlf/regularizers/ray_bundle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from nlf.regularizers.base import BaseRegularizer 8 | 9 | 10 | class RayBundleRegularizer(BaseRegularizer): 11 | def __init__( 12 | self, 13 | system, 14 | cfg 15 | ): 16 | super().__init__(system, cfg) 17 | 18 | self.range = cfg.range 19 | self.jitter = cfg.jitter 20 | 21 | def _loss(self, train_batch, batch_idx): 22 | system = self.get_system() 23 | dataset = self.get_dataset() 24 | batch = self.get_batch(train_batch, batch_idx, apply_ndc=dataset.use_ndc) 25 | 26 | #### Forward #### 27 | 28 | if dataset.use_ndc: 29 | rays, jitter_rays = batch['rays_no_ndc'], batch['jitter_rays_no_ndc'] 30 | else: 31 | rays, jitter_rays = batch['rays'], batch['jitter_rays'] 32 | 33 | if 'rgb' in batch: 34 | rgb = batch['rgb'] 35 | else: 36 | rgb = system(rays, apply_ndc=dataset.use_ndc)['rgb'] 37 | 38 | #### Losses #### 39 | 40 | all_losses = { 41 | loss: 0.0 for loss in self.loss_fns.keys() 42 | } 43 | 44 | ## Compute weight 45 | rgb = rgb.unsqueeze(-2) 46 | rays = rays.unsqueeze(-2) 47 | 48 | if self._do_loss('color_loss'): 49 | color_weight_map = self.get_weight_map( 50 | rays, 51 | jitter_rays, 52 | 'color_weight_map' 53 | ) 54 | 55 | ## Color consistency loss 56 | if self._do_loss('color_loss'): 57 | sh = jitter_rays.shape 58 | jitter_rgb = system(jitter_rays.view(-1, jitter_rays.shape[-1]), apply_ndc=dataset.use_ndc)['rgb'] 59 | jitter_rgb = jitter_rgb.view(sh[0], sh[1], jitter_rgb.shape[-1]) 60 | 61 | all_losses['color_loss'] = self._loss_fn( 62 | 'color_loss', 63 | (rgb * color_weight_map).permute(1, 0, 2), 64 | (jitter_rgb * color_weight_map).permute(1, 0, 2), 65 | ) 66 | 67 | ## Total loss 68 | total_loss = 0.0 69 | 70 | for name in all_losses.keys(): 71 | print(name + ':', all_losses[name]) 72 | total_loss += all_losses[name] 73 | 74 | return total_loss 75 | -------------------------------------------------------------------------------- /nlf/regularizers/teacher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from losses import loss_dict 8 | 9 | from nlf.regularizers.base import BaseRegularizer 10 | 11 | from kornia.filters import gaussian_blur2d 12 | 13 | 14 | class TeacherRegularizer(BaseRegularizer): 15 | def __init__( 16 | self, 17 | system, 18 | cfg 19 | ): 20 | super().__init__(system, cfg) 21 | 22 | self.loss_fn = loss_dict[self.cfg.loss.type]() 23 | self.use_inp_freq = 'inf' 24 | 25 | def _loss(self, train_batch, batch_idx): 26 | system = self.get_system() 27 | 28 | if self.loss_weight() < self.cfg.weight.stop_weight: 29 | return 0.0 30 | 31 | # Get inputs 32 | dataset = self.get_dataset() 33 | batch = dataset.get_batch(batch_idx, self.batch_size) 34 | 35 | rays = batch['rays'].type_as(train_batch['rays']) 36 | rgb = batch['rgb'].type_as(train_batch['rgb']) 37 | 38 | # Loss 39 | pred_rgb = system(rays)['rgb'] 40 | 41 | return self.loss_fn( 42 | pred_rgb, 43 | rgb 44 | ) 45 | 46 | 47 | class BlurryTeacherRegularizer(BaseRegularizer): 48 | def __init__( 49 | self, 50 | system, 51 | cfg 52 | ): 53 | super().__init__(system, cfg) 54 | 55 | self.loss_fn = loss_dict[self.cfg.loss.type]() 56 | self.use_inp_freq = 'inf' 57 | 58 | self.patch_width = self.cfg.dataset.patch_width 59 | self.blur_radius = self.cfg.blur_radius 60 | self.batch_size = self.patch_width * self.patch_width 61 | 62 | def _loss(self, train_batch, batch_idx): 63 | system = self.get_system() 64 | 65 | if self.loss_weight() < self.cfg.weight.stop_weight: 66 | return 0.0 67 | 68 | # Get inputs 69 | dataset = self.get_dataset() 70 | batch = dataset.get_batch(batch_idx, self.batch_size) 71 | 72 | rays = batch['rays'].type_as(train_batch['rays']) 73 | rgb = batch['rgb'].type_as(train_batch['rgb']) 74 | 75 | # Run forward and blur 76 | pred_rgb = system(rays)['rgb'] 77 | pred_rgb = pred_rgb.view(-1, self.patch_width, self.patch_width, 3).permute(0, 3, 1, 2) 78 | rgb = rgb.view(-1, self.patch_width, self.patch_width, 3).permute(0, 3, 1, 2) 79 | 80 | if self.blur_radius > 0: 81 | blur_rgb = gaussian_blur2d( 82 | pred_rgb, 83 | (self.blur_radius * 2 + 1, self.blur_radius * 2 + 1), 84 | (self.blur_radius / 3.0, self.blur_radius / 3.0) 85 | ) 86 | blur_rgb = blur_rgb[ 87 | ..., 88 | self.blur_radius:-self.blur_radius, 89 | self.blur_radius:-self.blur_radius 90 | ] 91 | rgb = rgb[ 92 | ..., 93 | self.blur_radius:-self.blur_radius, 94 | self.blur_radius:-self.blur_radius 95 | ] 96 | else: 97 | blur_rgb = pred_rgb 98 | 99 | # Loss 100 | return self.loss_fn( 101 | blur_rgb, 102 | rgb 103 | ) 104 | -------------------------------------------------------------------------------- /nlf/regularizers/voxel_sparsity.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | 9 | from nlf.regularizers.base import BaseRegularizer 10 | 11 | 12 | class VoxelSparsityRegularizer(BaseRegularizer): 13 | def __init__( 14 | self, 15 | system, 16 | cfg 17 | ): 18 | super().__init__(system, cfg) 19 | 20 | def get_subdivision(self): 21 | system = self.get_system() 22 | return system.subdivision 23 | 24 | def loss(self, batch, batch_idx): 25 | system = self.get_system() 26 | pos_model = system.render_fn.model.pos_model 27 | chunk = system.cfg.training.ray_chunk 28 | subdivision = self.get_subdivision() 29 | 30 | points = subdivision.voxel_centers[0, :subdivision.num_voxels].cuda() 31 | sampled_xyz = points.unsqueeze(1) 32 | 33 | sh = sampled_xyz.shape[:-1] # noqa 34 | sampled_idx = torch.arange( 35 | points.size(0), device=points.device 36 | )[:, None].expand(*sampled_xyz.size()[:2]) 37 | sampled_xyz, sampled_idx = sampled_xyz.reshape(-1, 3), sampled_idx.reshape(-1) 38 | 39 | ## Evaluate 40 | B = sampled_xyz.shape[0] 41 | out_chunks = [] 42 | 43 | for i in range(0, B, chunk): 44 | # Get points, idx 45 | cur_pts = sampled_xyz[i:i+chunk].unsqueeze(1) 46 | cur_idx = sampled_idx[i:i+chunk].unsqueeze(1) 47 | 48 | cur_mask = cur_idx.eq(-1) 49 | cur_idx[cur_mask] = 0 50 | 51 | # Get codes 52 | cur_codes = subdivision.get_vertex_codes(cur_pts, cur_idx, cur_mask) 53 | 54 | # Combine inputs 55 | cur_inps = torch.cat([cur_pts, cur_codes], -1) 56 | cur_inps = cur_inps.view(-1, cur_inps.shape[-1]) 57 | out_chunks += [pos_model.pos_forward(cur_inps)] 58 | 59 | out = torch.cat(out_chunks, 0) 60 | out = out[..., -1].view(-1) 61 | 62 | loss = -(out * torch.log(out + 1e-8) + (1 - out) * torch.log(1 - out + 1e-8)).mean() 63 | print(out.max(), loss) 64 | return loss 65 | -------------------------------------------------------------------------------- /nlf/visualizers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from .closest_view import ClosestViewVisualizer 8 | from .epipolar import EPIVisualizer 9 | from .focus import FocusVisualizer 10 | 11 | visualizer_dict = { 12 | 'closest_view': ClosestViewVisualizer, 13 | 'epipolar': EPIVisualizer, 14 | 'focus': FocusVisualizer, 15 | } 16 | -------------------------------------------------------------------------------- /nlf/visualizers/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from torch import nn 8 | 9 | 10 | class BaseVisualizer(nn.Module): 11 | def __init__( 12 | self, 13 | system, 14 | cfg 15 | ): 16 | super().__init__() 17 | 18 | self.cfg = cfg 19 | 20 | ## (Hack) Prevent from storing system variables 21 | self.systems = [system] 22 | 23 | def get_system(self): 24 | return self.systems[0] 25 | 26 | def validation_video(self, batch): 27 | return {} 28 | 29 | def validation_image(self, batch, batch_idx): 30 | return {} 31 | -------------------------------------------------------------------------------- /nlf/visualizers/closest_view.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import numpy as np # noqa 8 | 9 | from nlf.visualizers.base import BaseVisualizer 10 | 11 | 12 | class ClosestViewVisualizer(BaseVisualizer): 13 | def __init__( 14 | self, 15 | system, 16 | cfg 17 | ): 18 | super().__init__(system, cfg) 19 | 20 | def validation(self, batch): 21 | system = self.get_system() 22 | 23 | pose = batch['pose'] 24 | rgb = system.trainer.datamodule.train_dataset.get_closest_rgb(pose).cpu() 25 | rgb = rgb.permute(2, 0, 1) 26 | 27 | return { 28 | 'rgb': rgb 29 | } 30 | 31 | def validation_video(self, batch): 32 | temp_outputs = self.validation(batch) 33 | outputs = {} 34 | 35 | for key in temp_outputs.keys(): 36 | outputs[f'videos/closest_{key}'] = temp_outputs[key] 37 | 38 | return outputs 39 | 40 | def validation_image(self, batch, batch_idx): 41 | return {} 42 | -------------------------------------------------------------------------------- /nlf/visualizers/epipolar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from nlf.visualizers.base import BaseVisualizer 8 | from utils.ray_utils import ( 9 | get_epi_rays, 10 | ) 11 | 12 | from utils.visualization import ( 13 | get_warp_dimensions, 14 | visualize_warp 15 | ) 16 | 17 | 18 | class EPIVisualizer(BaseVisualizer): 19 | def __init__( 20 | self, 21 | system, 22 | cfg 23 | ): 24 | super().__init__(system, cfg) 25 | 26 | self.v = cfg.v if 'v' in cfg else None 27 | self.t = cfg.v if 't' in cfg else None 28 | self.H = cfg.H if 'H' in cfg else None 29 | 30 | self.near = cfg.near if 'near' in cfg else -1.0 31 | self.far = cfg.far if 'far' in cfg else 0.0 32 | 33 | if 'st_scale' in cfg and cfg.st_scale is not None: 34 | self.st_scale = cfg.st_scale 35 | elif 'lightfield' in system.cfg.dataset and 'st_scale' in system.cfg.dataset.lightfield: 36 | self.st_scale = system.cfg.dataset.lightfield.st_scale 37 | else: 38 | self.st_scale = 1.0 39 | 40 | if 'uv_scale' in cfg and cfg.uv_scale is not None: 41 | self.uv_scale = cfg.uv_scale 42 | else: 43 | self.uv_scale = 1.0 44 | 45 | def validation(self, batch, batch_idx): 46 | if batch_idx > 0: 47 | return 48 | 49 | system = self.get_system() 50 | dataset = system.trainer.datamodule.train_dataset 51 | W = system.cur_wh[0] 52 | H = system.cur_wh[1] 53 | 54 | # Coordinates 55 | if self.t is not None: 56 | t = self.t 57 | else: 58 | t = 0 59 | 60 | if self.v is not None: 61 | v = self.v 62 | else: 63 | v = 0 64 | 65 | if self.H is not None: 66 | H = self.H 67 | 68 | ## Forward 69 | outputs = {} 70 | 71 | # Generate EPI rays 72 | rays = get_epi_rays( 73 | W, v, H, t, dataset.aspect, 74 | st_scale=self.st_scale, 75 | uv_scale=self.uv_scale, 76 | near=self.near, far=self.far 77 | ).type_as(batch['rays']) 78 | 79 | # RGB 80 | rgb = system(rays)['rgb'] 81 | rgb = rgb.view(H, W, 3).cpu() 82 | rgb = rgb.permute(2, 0, 1) 83 | 84 | outputs['rgb'] = rgb 85 | 86 | # Warp 87 | if not system.is_subdivided: 88 | embedding = system.embed(rays)['embedding'] 89 | params = system.embed_params(rays)['params'] 90 | elif system.render_fn.subdivision.max_hits < 4: 91 | embedding = system.render_fn.embed_vis(rays)['embedding'] 92 | params = system.render_fn.embed_params_vis(rays)['params'] 93 | 94 | if not system.is_subdivided or system.render_fn.subdivision.max_hits < 4: 95 | warp_dims = get_warp_dimensions( 96 | embedding, W, H, k=min(embedding.shape[-1], 3) 97 | ) 98 | warp = visualize_warp( 99 | embedding, warp_dims, W, H, k=min(embedding.shape[-1], 3) 100 | ) 101 | outputs['warp'] = warp 102 | 103 | warp_dims = get_warp_dimensions( 104 | params, W, H, k=min(embedding.shape[-1], 3) 105 | ) 106 | warp = visualize_warp( 107 | params, warp_dims, W, H, k=min(embedding.shape[-1], 3) 108 | ) 109 | outputs['tform'] = warp 110 | 111 | return outputs 112 | 113 | def validation_image(self, batch, batch_idx): 114 | if batch_idx > 0: 115 | return {} 116 | 117 | # Outputs 118 | temp_outputs = self.validation(batch, batch_idx) 119 | outputs = {} 120 | 121 | for key in temp_outputs.keys(): 122 | outputs[f'images/epi_{key}'] = temp_outputs[key] 123 | 124 | return outputs 125 | -------------------------------------------------------------------------------- /nlf/visualizers/focus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | 9 | from nlf.visualizers.base import BaseVisualizer 10 | from utils.ray_utils import get_lightfield_rays 11 | 12 | 13 | class FocusVisualizer(BaseVisualizer): 14 | def __init__( 15 | self, 16 | system, 17 | cfg 18 | ): 19 | super().__init__(system, cfg) 20 | 21 | self.s = cfg.s if 's' in cfg else None 22 | self.t = cfg.t if 't' in cfg else None 23 | 24 | self.ds = cfg.ds if 'ds' in cfg else None 25 | self.dt = cfg.dt if 'dt' in cfg else None 26 | 27 | self.near = cfg.near if 'near' in cfg else -1.0 28 | self.far = cfg.far if 'far' in cfg else 0.0 29 | self.focal = cfg.focal if 'focal' in cfg else 0.0 30 | 31 | self.n_freqs = system.cfg.model.color_pe.n_freqs 32 | self.freq_bands = 2 ** torch.linspace(0, self.n_freqs - 1, self.n_freqs) 33 | 34 | if 'st_scale' in cfg and cfg.st_scale is not None: 35 | self.st_scale = cfg.st_scale 36 | elif 'lightfield' in system.cfg.dataset and 'st_scale' in system.cfg.dataset.lightfield: 37 | self.st_scale = system.cfg.dataset.lightfield.st_scale 38 | else: 39 | self.st_scale = 1.0 40 | 41 | if 'uv_scale' in cfg and cfg.uv_scale is not None: 42 | self.uv_scale = cfg.uv_scale 43 | else: 44 | self.uv_scale = 1.0 45 | 46 | def validation(self, batch, batch_idx): 47 | if batch_idx > 0: 48 | return 49 | 50 | system = self.get_system() 51 | dataset = system.trainer.datamodule.train_dataset 52 | 53 | W = system.cur_wh[0] 54 | H = system.cur_wh[1] 55 | 56 | param_channels = 4 57 | 58 | ## Forward 59 | outputs = {} 60 | 61 | # Generate image rays 62 | if self.s is not None: 63 | s = self.s 64 | else: 65 | s = 0.0 66 | 67 | if self.t is not None: 68 | t = self.t 69 | else: 70 | t = 0.0 71 | 72 | rays = get_lightfield_rays( 73 | W, H, s, t, dataset.aspect, 74 | st_scale=self.st_scale, 75 | uv_scale=self.uv_scale, 76 | near=self.near, far=self.far 77 | ).type_as(batch['rays']) 78 | 79 | # Cone 80 | if self.ds is None: 81 | ds = 1.0 82 | else: 83 | ds = self.ds 84 | 85 | if self.dt is None: 86 | dt = 1.0 87 | else: 88 | dt = self.dt 89 | 90 | du = (self.focal - self.far) * ds / (self.far - self.near) 91 | dv = (self.focal - self.far) * dt / (self.far - self.near) 92 | 93 | ds_vec = torch.zeros((1, param_channels, 1)).type_as(rays) 94 | ds_vec[..., 0, :] = ds 95 | ds_vec[..., 2, :] = du 96 | 97 | dt_vec = torch.zeros((1, param_channels, 1)).type_as(rays) 98 | dt_vec[..., 1, :] = dt 99 | dt_vec[..., 3, :] = dv 100 | 101 | # Warp 102 | params = system.embed_params(rays)['params'] 103 | out_channels = (params.shape[-1] // (param_channels + 1)) 104 | tform = params[..., :-out_channels].reshape( 105 | -1, out_channels, param_channels 106 | ) 107 | tform = torch.nn.functional.normalize(tform, p=2.0, dim=-1) 108 | 109 | s_response = (tform @ ds_vec).squeeze(-1) / W 110 | t_response = (tform @ dt_vec).squeeze(-1) / W 111 | 112 | max_response = torch.maximum( 113 | torch.abs(s_response), 114 | torch.abs(t_response), 115 | ) 116 | max_freq = 1.0 / max_response 117 | 118 | # Calculate weights 119 | pe_weight = {} 120 | 121 | for j, freq in enumerate(self.freq_bands): 122 | #weight = (max_freq / freq - 0.5) * 2 123 | weight = max_freq / freq 124 | pe_weight[j] = torch.clamp(weight, torch.zeros_like(weight), torch.ones_like(weight)) 125 | 126 | print(pe_weight[0].shape, rays.shape) 127 | print(pe_weight[len(self.freq_bands) - 1][0]) 128 | 129 | # RGB out of focus 130 | rgb = system(rays, pe_weight=pe_weight, chunk_args=['pe_weight'])['rgb'] 131 | rgb = rgb.view(H, W, 3).cpu() 132 | rgb = rgb.permute(2, 0, 1) 133 | 134 | outputs['rgb_cone'] = rgb 135 | 136 | # RGB in focus 137 | rgb = system(rays)['rgb'] 138 | rgb = rgb.view(H, W, 3).cpu() 139 | rgb = rgb.permute(2, 0, 1) 140 | 141 | outputs['rgb_ray'] = rgb 142 | 143 | return outputs 144 | 145 | def validation_image(self, batch, batch_idx): 146 | if batch_idx > 0: 147 | return {} 148 | 149 | # Outputs 150 | temp_outputs = self.validation(batch, batch_idx) 151 | outputs = {} 152 | 153 | for key in temp_outputs.keys(): 154 | outputs[f'images/focus_{key}'] = temp_outputs[key] 155 | 156 | return outputs 157 | -------------------------------------------------------------------------------- /scripts/run_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### LLFF 4 | # 5 | ## LF 6 | #bash x3d/slf/scripts/run_all_llff.sh run_one_lf stanford_half 4 two_plane 32 "" llff_lf 7 | #bash x3d/slf/scripts/run_all_llff.sh run_one_lf_reg stanford_half 4 two_plane 32 "" llff_lf 8 | #bash x3d/slf/scripts/run_all_llff.sh run_one_lf_teacher stanford_half 4 two_plane 32 "" llff_lf 9 | # 10 | # 11 | ### Shiny 12 | # 13 | ## Subdivided 14 | #bash x3d/slf/scripts/run_all_shiny.sh run_one_subdivided llff_subdivided_half 4 two_plane 32 "" shiny_subdivided 15 | #bash x3d/slf/scripts/run_all_shiny.sh run_one_subdivided_reg llff_subdivided_half 4 two_plane 32 "" shiny_subdivided 16 | # 17 | #bash x3d/slf/scripts/run_all_shiny.sh run_one_subdivided llff_subdivided_half 4 two_plane 32 "" shiny_subdivided 18 | #bash x3d/slf/scripts/run_all_shiny.sh run_one_subdivided_reg llff_subdivided_half 4 two_plane 32 "" shiny_subdivided 19 | # 20 | # LF 21 | bash x3d/slf/scripts/run_all_shiny.sh run_one_lf stanford_half 4 two_plane 32 "" shiny_lf 22 | bash x3d/slf/scripts/run_all_shiny.sh run_one_lf_reg stanford_half 4 two_plane 32 "" shiny_lf 23 | # 24 | # 25 | ### Stanford 26 | # 27 | ## Subdivided 28 | #bash x3d/slf/scripts/run_all_stanford.sh run_one_subdivided_small llff_subdivided_half 4 two_plane 32 "" stanford_subdivided 29 | #bash x3d/slf/scripts/run_all_stanford.sh run_one_subdivided_reg_small llff_subdivided_half 4 two_plane 32 "" stanford_subdivided 30 | -------------------------------------------------------------------------------- /scripts/run_all_llff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fern 4 | bash x3d/slf/scripts/$1.sh llff$6 fern $2 $3 $4 $5 $7 5 | 6 | # room 7 | bash x3d/slf/scripts/$1.sh llff$6 room $2 $3 $4 $5 $7 8 | 9 | # horns 10 | bash x3d/slf/scripts/$1.sh llff$6 horns $2 $3 $4 $5 $7 11 | 12 | # trex 13 | bash x3d/slf/scripts/$1.sh llff$6 trex $2 $3 $4 $5 $7 14 | 15 | # orchids 16 | bash x3d/slf/scripts/$1.sh llff$6 orchids $2 $3 $4 $5 $7 17 | 18 | # leaves 19 | bash x3d/slf/scripts/$1.sh llff$6 leaves $2 $3 $4 $5 $7 20 | 21 | # flower 22 | bash x3d/slf/scripts/$1.sh llff$6 flower $2 $3 $4 $5 $7 23 | 24 | # fortress 25 | bash x3d/slf/scripts/$1.sh llff$6 fortress $2 $3 $4 $5 $7 26 | -------------------------------------------------------------------------------- /scripts/run_all_paper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | ## LLFF 5 | 6 | # Subdivided 7 | bash x3d/slf/scripts/run_all_llff.sh run_one_subdivided llff_subdivided_affine 4 two_plane 32 "" llff_subdivided 8 | bash x3d/slf/scripts/run_all_llff.sh run_one_subdivided llff_subdivided_half 4 two_plane 32 "" llff_subdivided_half 9 | 10 | bash x3d/slf/scripts/run_all_llff.sh run_one_subdivided_teacher_few llff_subdivided_affine 4 two_plane 32 "" llff_subdivided 11 | bash x3d/slf/scripts/run_all_llff.sh run_one_subdivided_teacher_few llff_subdivided_half 4 two_plane 32 "" llff_subdivided_half 12 | 13 | bash x3d/slf/scripts/run_all_llff.sh run_one_subdivided_few llff_subdivided_feature 4 two_plane 32 "" llff_subdivided 14 | bash x3d/slf/scripts/run_all_llff.sh run_one_subdivided_few llff_subdivided_no_embed 4 two_plane 32 "" llff_subdivided 15 | 16 | 17 | ## Stanford 18 | 19 | bash x3d/slf/scripts/run_all_stanford.sh run_one_lf stanford_affine 4 two_plane 32 "" stanford_lf 20 | bash x3d/slf/scripts/run_all_stanford.sh run_one_lf stanford_half 4 two_plane 32 "" stanford_lf_half 21 | 22 | bash x3d/slf/scripts/run_all_stanford.sh run_one_lf stanford_feature 4 two_plane 32 "" stanford_lf 23 | bash x3d/slf/scripts/run_all_stanford.sh run_one_lf stanford_no_embed 4 two_plane 32 "" stanford_lf 24 | -------------------------------------------------------------------------------- /scripts/run_all_shiny.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # cd 4 | bash x3d/slf/scripts/$1.sh shiny_cd$6 cd $2 $3 $4 $5 $7 5 | 6 | # lab 7 | bash x3d/slf/scripts/$1.sh shiny_cd$6 lab $2 $3 $4 $5 $7 8 | 9 | # food 10 | bash x3d/slf/scripts/$1.sh shiny$6 food $2 $3 $4 $5 $7 11 | 12 | # crest 13 | bash x3d/slf/scripts/$1.sh shiny$6 crest $2 $3 $4 $5 $7 14 | 15 | # giants 16 | bash x3d/slf/scripts/$1.sh shiny$6 giants $2 $3 $4 $5 $7 17 | 18 | # pasta 19 | bash x3d/slf/scripts/$1.sh shiny$6 pasta $2 $3 $4 $5 $7 20 | 21 | # seasoning 22 | bash x3d/slf/scripts/$1.sh shiny$6 seasoning $2 $3 $4 $5 $7 23 | 24 | # tools 25 | bash x3d/slf/scripts/$1.sh shiny$6 tools $2 $3 $4 $5 $7 26 | 27 | # glass 28 | bash x3d/slf/scripts/$1.sh shiny$6 glass $2 $3 $4 $5 $7 29 | -------------------------------------------------------------------------------- /scripts/run_all_stanford.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | # tarot small 5 | bash x3d/slf/scripts/$1.sh \ 6 | stanford_tarot_small tarot_small $2 \ 7 | $3 $4 \ 8 | $5 $7 9 | 10 | # tarot 11 | bash x3d/slf/scripts/$1.sh \ 12 | stanford_tarot tarot $2 \ 13 | $3 $4 \ 14 | $5 $7 15 | 16 | # gem 17 | bash x3d/slf/scripts/$1.sh \ 18 | stanford_gem gem $2 \ 19 | $3 $4 \ 20 | $5 $7 21 | 22 | # treasure 23 | bash x3d/slf/scripts/$1.sh \ 24 | stanford_treasure treasure $2 \ 25 | $3 $4 \ 26 | $5 $7 27 | 28 | # bulldozer 29 | bash x3d/slf/scripts/$1.sh \ 30 | stanford_bulldozer bulldozer $2 \ 31 | $3 $4 \ 32 | $5 $7 33 | 34 | # truck 35 | bash x3d/slf/scripts/$1.sh \ 36 | stanford_truck truck $2 \ 37 | $3 $4 \ 38 | $5 $7 39 | 40 | # flowers 41 | bash x3d/slf/scripts/$1.sh \ 42 | stanford_flowers flowers $2 \ 43 | $3 $4 \ 44 | $5 $7 45 | 46 | # bracelet 47 | bash x3d/slf/scripts/$1.sh \ 48 | stanford_bracelet bracelet $2 \ 49 | $3 $4 \ 50 | $5 $7 51 | 52 | # bunny 53 | bash x3d/slf/scripts/$1.sh \ 54 | stanford_bunny bunny $2 \ 55 | $3 $4 \ 56 | $5 $7 57 | 58 | # beans 59 | bash x3d/slf/scripts/$1.sh \ 60 | stanford_beans beans $2 \ 61 | $3 $4 \ 62 | $5 $7 63 | 64 | # chess 65 | bash x3d/slf/scripts/$1.sh \ 66 | stanford_chess chess $2 \ 67 | $3 $4 \ 68 | $5 $7 69 | 70 | # knights 71 | bash x3d/slf/scripts/$1.sh \ 72 | stanford_knights knights $2 \ 73 | $3 $4 \ 74 | $5 $7 75 | -------------------------------------------------------------------------------- /scripts/run_one_lf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.param.n_dims=$4 \ 9 | experiment.model.param.fn=$5 \ 10 | experiment.model.embedding_net.out_channels=$6 \ 11 | experiment/training=$7 12 | -------------------------------------------------------------------------------- /scripts/run_one_lf_reg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.param.n_dims=$4 \ 9 | experiment.model.param.fn=$5 \ 10 | experiment.model.embedding_net.out_channels=$6 \ 11 | +experiment/regularizers/warp_level=lf \ 12 | experiment/training=$7 13 | -------------------------------------------------------------------------------- /scripts/run_one_lf_teacher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.param.n_dims=$4 \ 9 | experiment.model.param.fn=$5 \ 10 | experiment.model.embedding_net.out_channels=$6 \ 11 | +experiment/regularizers/weak_teacher=lf \ 12 | experiment/training=$7 13 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=4 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | experiment/training=$7 13 | 14 | python main.py \ 15 | experiment=local \ 16 | experiment/dataset=$1 \ 17 | experiment.dataset.collection=$2 \ 18 | experiment/model=$3 \ 19 | experiment.model.subdivision.max_hits=8 \ 20 | experiment.model.ray.param.n_dims=$4 \ 21 | experiment.model.ray.param.fn=$5 \ 22 | experiment.model.ray.embedding_net.out_channels=$6 \ 23 | experiment/training=$7 24 | 25 | python main.py \ 26 | experiment=local \ 27 | experiment/dataset=$1 \ 28 | experiment.dataset.collection=$2 \ 29 | experiment/model=$3 \ 30 | experiment.model.subdivision.max_hits=16 \ 31 | experiment.model.ray.param.n_dims=$4 \ 32 | experiment.model.ray.param.fn=$5 \ 33 | experiment.model.ray.embedding_net.out_channels=$6 \ 34 | experiment/training=$7 35 | 36 | python main.py \ 37 | experiment=local \ 38 | experiment/dataset=$1 \ 39 | experiment.dataset.collection=$2 \ 40 | experiment/model=$3 \ 41 | experiment.model.subdivision.max_hits=32 \ 42 | experiment.model.ray.param.n_dims=$4 \ 43 | experiment.model.ray.param.fn=$5 \ 44 | experiment.model.ray.embedding_net.out_channels=$6 \ 45 | experiment/training=$7 46 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided_few.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=16 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | experiment/training=$7 13 | 14 | python main.py \ 15 | experiment=local \ 16 | experiment/dataset=$1 \ 17 | experiment.dataset.collection=$2 \ 18 | experiment/model=$3 \ 19 | experiment.model.subdivision.max_hits=32 \ 20 | experiment.model.ray.param.n_dims=$4 \ 21 | experiment.model.ray.param.fn=$5 \ 22 | experiment.model.ray.embedding_net.out_channels=$6 \ 23 | experiment/training=$7 24 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided_reg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=4 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | +experiment/regularizers/warp_level=subdivided \ 13 | experiment/training=$7 14 | 15 | python main.py \ 16 | experiment=local \ 17 | experiment/dataset=$1 \ 18 | experiment.dataset.collection=$2 \ 19 | experiment/model=$3 \ 20 | experiment.model.subdivision.max_hits=8 \ 21 | experiment.model.ray.param.n_dims=$4 \ 22 | experiment.model.ray.param.fn=$5 \ 23 | experiment.model.ray.embedding_net.out_channels=$6 \ 24 | +experiment/regularizers/warp_level=subdivided \ 25 | experiment/training=$7 26 | 27 | python main.py \ 28 | experiment=local \ 29 | experiment/dataset=$1 \ 30 | experiment.dataset.collection=$2 \ 31 | experiment/model=$3 \ 32 | experiment.model.subdivision.max_hits=16 \ 33 | experiment.model.ray.param.n_dims=$4 \ 34 | experiment.model.ray.param.fn=$5 \ 35 | experiment.model.ray.embedding_net.out_channels=$6 \ 36 | +experiment/regularizers/warp_level=subdivided \ 37 | experiment/training=$7 38 | 39 | python main.py \ 40 | experiment=local \ 41 | experiment/dataset=$1 \ 42 | experiment.dataset.collection=$2 \ 43 | experiment/model=$3 \ 44 | experiment.model.subdivision.max_hits=32 \ 45 | experiment.model.ray.param.n_dims=$4 \ 46 | experiment.model.ray.param.fn=$5 \ 47 | experiment.model.ray.embedding_net.out_channels=$6 \ 48 | +experiment/regularizers/warp_level=subdivided \ 49 | experiment/training=$7 50 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided_reg_small.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=2 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | +experiment/regularizers/warp_level=subdivided \ 13 | experiment/training=$7 14 | 15 | python main.py \ 16 | experiment=local \ 17 | experiment/dataset=$1 \ 18 | experiment.dataset.collection=$2 \ 19 | experiment/model=$3 \ 20 | experiment.model.subdivision.max_hits=4 \ 21 | experiment.model.ray.param.n_dims=$4 \ 22 | experiment.model.ray.param.fn=$5 \ 23 | experiment.model.ray.embedding_net.out_channels=$6 \ 24 | +experiment/regularizers/warp_level=subdivided \ 25 | experiment/training=$7 26 | 27 | python main.py \ 28 | experiment=local \ 29 | experiment/dataset=$1 \ 30 | experiment.dataset.collection=$2 \ 31 | experiment/model=$3 \ 32 | experiment.model.subdivision.max_hits=8 \ 33 | experiment.model.ray.param.n_dims=$4 \ 34 | experiment.model.ray.param.fn=$5 \ 35 | experiment.model.ray.embedding_net.out_channels=$6 \ 36 | +experiment/regularizers/warp_level=subdivided \ 37 | experiment/training=$7 38 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided_small.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=2 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | experiment/training=$7 13 | 14 | python main.py \ 15 | experiment=local \ 16 | experiment/dataset=$1 \ 17 | experiment.dataset.collection=$2 \ 18 | experiment/model=$3 \ 19 | experiment.model.subdivision.max_hits=4 \ 20 | experiment.model.ray.param.n_dims=$4 \ 21 | experiment.model.ray.param.fn=$5 \ 22 | experiment.model.ray.embedding_net.out_channels=$6 \ 23 | experiment/training=$7 24 | 25 | python main.py \ 26 | experiment=local \ 27 | experiment/dataset=$1 \ 28 | experiment.dataset.collection=$2 \ 29 | experiment/model=$3 \ 30 | experiment.model.subdivision.max_hits=8 \ 31 | experiment.model.ray.param.n_dims=$4 \ 32 | experiment.model.ray.param.fn=$5 \ 33 | experiment.model.ray.embedding_net.out_channels=$6 \ 34 | experiment/training=$7 35 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided_teacher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=2 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | +experiment/regularizers/weak_teacher=subdivided \ 13 | experiment/training=$7 14 | 15 | python main.py \ 16 | experiment=local \ 17 | experiment/dataset=$1 \ 18 | experiment.dataset.collection=$2 \ 19 | experiment/model=$3 \ 20 | experiment.model.subdivision.max_hits=4 \ 21 | experiment.model.ray.param.n_dims=$4 \ 22 | experiment.model.ray.param.fn=$5 \ 23 | experiment.model.ray.embedding_net.out_channels=$6 \ 24 | +experiment/regularizers/weak_teacher=subdivided \ 25 | experiment/training=$7 26 | 27 | python main.py \ 28 | experiment=local \ 29 | experiment/dataset=$1 \ 30 | experiment.dataset.collection=$2 \ 31 | experiment/model=$3 \ 32 | experiment.model.subdivision.max_hits=8 \ 33 | experiment.model.ray.param.n_dims=$4 \ 34 | experiment.model.ray.param.fn=$5 \ 35 | experiment.model.ray.embedding_net.out_channels=$6 \ 36 | +experiment/regularizers/weak_teacher=subdivided \ 37 | experiment/training=$7 38 | 39 | python main.py \ 40 | experiment=local \ 41 | experiment/dataset=$1 \ 42 | experiment.dataset.collection=$2 \ 43 | experiment/model=$3 \ 44 | experiment.model.subdivision.max_hits=16 \ 45 | experiment.model.ray.param.n_dims=$4 \ 46 | experiment.model.ray.param.fn=$5 \ 47 | experiment.model.ray.embedding_net.out_channels=$6 \ 48 | +experiment/regularizers/weak_teacher=subdivided \ 49 | experiment/training=$7 50 | 51 | python main.py \ 52 | experiment=local \ 53 | experiment/dataset=$1 \ 54 | experiment.dataset.collection=$2 \ 55 | experiment/model=$3 \ 56 | experiment.model.subdivision.max_hits=32 \ 57 | experiment.model.ray.param.n_dims=$4 \ 58 | experiment.model.ray.param.fn=$5 \ 59 | experiment.model.ray.embedding_net.out_channels=$6 \ 60 | +experiment/regularizers/weak_teacher=subdivided \ 61 | experiment/training=$7 62 | -------------------------------------------------------------------------------- /scripts/run_one_subdivided_teacher_few.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python main.py \ 4 | experiment=local \ 5 | experiment/dataset=$1 \ 6 | experiment.dataset.collection=$2 \ 7 | experiment/model=$3 \ 8 | experiment.model.subdivision.max_hits=16 \ 9 | experiment.model.ray.param.n_dims=$4 \ 10 | experiment.model.ray.param.fn=$5 \ 11 | experiment.model.ray.embedding_net.out_channels=$6 \ 12 | +experiment/regularizers/weak_teacher=subdivided \ 13 | experiment/training=$7 14 | 15 | python main.py \ 16 | experiment=local \ 17 | experiment/dataset=$1 \ 18 | experiment.dataset.collection=$2 \ 19 | experiment/model=$3 \ 20 | experiment.model.subdivision.max_hits=32 \ 21 | experiment.model.ray.param.n_dims=$4 \ 22 | experiment.model.ray.param.fn=$5 \ 23 | experiment.model.ray.embedding_net.out_channels=$6 \ 24 | +experiment/regularizers/weak_teacher=subdivided \ 25 | experiment/training=$7 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | os.environ["PATH"] += os.pathsep + os.path.join(os.path.expanduser('~'), 'anaconda3/envs/neural-light-fields/pkgs/cuda-toolkit') 9 | from setuptools import setup 10 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 11 | import glob 12 | 13 | # build clib 14 | _ext_src_root = "third_party/intersect" 15 | _ext_sources = glob.glob("{}/*.cpp".format(_ext_src_root)) + glob.glob( 16 | "{}/*.cu".format(_ext_src_root) 17 | ) 18 | _ext_headers = glob.glob("{}/*.h".format(_ext_src_root)) 19 | 20 | setup( 21 | name='third_party.intersect', 22 | ext_modules=[ 23 | CUDAExtension( 24 | name='third_party.intersect.cuda_lib', 25 | sources=_ext_sources, 26 | extra_compile_args={ 27 | "cxx": ["-O2", "-I{}".format("{}/include".format(_ext_src_root))], 28 | "nvcc": ["-O2", "-I{}".format("{}/include".format(_ext_src_root))], 29 | }, 30 | ) 31 | ], 32 | cmdclass={ 33 | 'build_ext': BuildExtension 34 | }, 35 | ) 36 | -------------------------------------------------------------------------------- /third_party/intersect/README.facebook: -------------------------------------------------------------------------------- 1 | Project: Neural Sparse Voxel Fields (NSVF) 2 | Version: 1.0 3 | Release date: 01-01-2021 4 | Website: https://lingjie0206.github.io/papers/NSVF/ 5 | License: MIT 6 | Description: High performance GPU voxel octree intersection code from the Neural Sparse Voxel Fields project (developed, in part by FAIR). 7 | Changes: Modifications to allow for intersections of front / back-faces of voxels only 8 | -------------------------------------------------------------------------------- /third_party/intersect/binding.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #include "intersect.h" 7 | #include "octree.h" 8 | #include "sample.h" 9 | 10 | 11 | PYBIND11_MODULE(cuda_lib, m) { 12 | m.def("ball_intersect", &ball_intersect); 13 | m.def("aabb_intersect", &aabb_intersect); 14 | m.def("svo_intersect", &svo_intersect); 15 | m.def("triangle_intersect", &triangle_intersect); 16 | 17 | m.def("uniform_ray_sampling", &uniform_ray_sampling); 18 | m.def("inverse_cdf_sampling", &inverse_cdf_sampling); 19 | 20 | m.def("build_octree", &build_octree); 21 | } 22 | -------------------------------------------------------------------------------- /third_party/intersect/cuda_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #ifndef _CUDA_UTILS_H 7 | #define _CUDA_UTILS_H 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #define TOTAL_THREADS 512 19 | 20 | inline int opt_n_threads(int work_size) { 21 | const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0); 22 | 23 | return max(min(1 << pow_2, TOTAL_THREADS), 1); 24 | } 25 | 26 | inline dim3 opt_block_config(int x, int y) { 27 | const int x_threads = opt_n_threads(x); 28 | const int y_threads = 29 | max(min(opt_n_threads(y), TOTAL_THREADS / x_threads), 1); 30 | dim3 block_config(x_threads, y_threads, 1); 31 | 32 | return block_config; 33 | } 34 | 35 | #define CUDA_CHECK_ERRORS() \ 36 | do { \ 37 | cudaError_t err = cudaGetLastError(); \ 38 | if (cudaSuccess != err) { \ 39 | fprintf(stderr, "CUDA kernel failed : %s\n%s at L:%d in %s\n", \ 40 | cudaGetErrorString(err), __PRETTY_FUNCTION__, __LINE__, \ 41 | __FILE__); \ 42 | exit(-1); \ 43 | } \ 44 | } while (0) 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /third_party/intersect/intersect.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #pragma once 7 | #include 8 | #include 9 | 10 | std::tuple ball_intersect(at::Tensor ray_start, at::Tensor ray_dir, at::Tensor points, 11 | const float radius, const int n_max); 12 | std::tuple aabb_intersect(at::Tensor ray_start, at::Tensor ray_dir, at::Tensor points, 13 | const float voxelsize, const int n_max); 14 | std::tuple svo_intersect(at::Tensor ray_start, at::Tensor ray_dir, at::Tensor points, at::Tensor children, 15 | const float voxelsize, const int n_max, const bool z_only); 16 | std::tuple< at::Tensor, at::Tensor, at::Tensor > triangle_intersect(at::Tensor ray_start, at::Tensor ray_dir, at::Tensor face_points, 17 | const float cagesize, const float blur, const int n_max); 18 | -------------------------------------------------------------------------------- /third_party/intersect/octree.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #include "octree.h" 7 | #include "utils.h" 8 | #include 9 | #include 10 | using namespace std::chrono; 11 | 12 | 13 | typedef struct OcTree 14 | { 15 | int depth; 16 | int index; 17 | at::Tensor center; 18 | struct OcTree *children[8]; 19 | void init(at::Tensor center, int d, int i) { 20 | this->center = center; 21 | this->depth = d; 22 | this->index = i; 23 | for (int i=0; i<8; i++) this->children[i] = nullptr; 24 | } 25 | }OcTree; 26 | 27 | class EasyOctree { 28 | public: 29 | OcTree *root; 30 | int total; 31 | int terminal; 32 | 33 | at::Tensor all_centers; 34 | at::Tensor all_children; 35 | 36 | EasyOctree(at::Tensor center, int depth) { 37 | root = new OcTree; 38 | root->init(center, depth, -1); 39 | total = -1; 40 | terminal = -1; 41 | } 42 | ~EasyOctree() { 43 | OcTree *p = root; 44 | destory(p); 45 | } 46 | void destory(OcTree * &p); 47 | void insert(OcTree * &p, at::Tensor point, int index); 48 | void finalize(); 49 | std::pair count(OcTree * &p); 50 | }; 51 | 52 | void EasyOctree::destory(OcTree * &p){ 53 | if (p != nullptr) { 54 | for (int i=0; i<8; i++) { 55 | if (p->children[i] != nullptr) destory(p->children[i]); 56 | } 57 | delete p; 58 | p = nullptr; 59 | } 60 | } 61 | 62 | void EasyOctree::insert(OcTree * &p, at::Tensor point, int index) { 63 | at::Tensor diff = (point > p->center).to(at::kInt); 64 | int idx = diff[0].item() + 2 * diff[1].item() + 4 * diff[2].item(); 65 | if (p->depth == 0) { 66 | p->children[idx] = new OcTree; 67 | p->children[idx]->init(point, -1, index); 68 | } else { 69 | if (p->children[idx] == nullptr) { 70 | int length = 1 << (p->depth - 1); 71 | at::Tensor new_center = p->center + (2 * diff - 1) * length; 72 | p->children[idx] = new OcTree; 73 | p->children[idx]->init(new_center, p->depth-1, -1); 74 | } 75 | insert(p->children[idx], point, index); 76 | } 77 | } 78 | 79 | std::pair EasyOctree::count(OcTree * &p) { 80 | int total = 0, terminal = 0; 81 | for (int i=0; i<8; i++) { 82 | if (p->children[i] != nullptr) { 83 | std::pair sub = count(p->children[i]); 84 | total += sub.first; 85 | terminal += sub.second; 86 | } 87 | } 88 | total += 1; 89 | if (p->depth == -1) terminal += 1; 90 | return std::make_pair(total, terminal); 91 | } 92 | 93 | void EasyOctree::finalize() { 94 | std::pair outs = count(root); 95 | total = outs.first; terminal = outs.second; 96 | 97 | all_centers = 98 | torch::zeros({outs.first, 3}, at::device(root->center.device()).dtype(at::ScalarType::Int)); 99 | all_children = 100 | -torch::ones({outs.first, 9}, at::device(root->center.device()).dtype(at::ScalarType::Int)); 101 | 102 | int node_idx = outs.first - 1; 103 | root->index = node_idx; 104 | 105 | std::queue all_leaves; all_leaves.push(root); 106 | while (!all_leaves.empty()) { 107 | OcTree* node_ptr = all_leaves.front(); 108 | all_leaves.pop(); 109 | for (int i=0; i<8; i++) { 110 | if (node_ptr->children[i] != nullptr) { 111 | if (node_ptr->children[i]->depth > -1) { 112 | node_idx--; 113 | node_ptr->children[i]->index = node_idx; 114 | } 115 | all_leaves.push(node_ptr->children[i]); 116 | all_children[node_ptr->index][i] = node_ptr->children[i]->index; 117 | } 118 | } 119 | all_children[node_ptr->index][8] = 1 << (node_ptr->depth + 1); 120 | all_centers[node_ptr->index] = node_ptr->center; 121 | } 122 | assert (node_idx == outs.second); 123 | }; 124 | 125 | std::tuple build_octree(at::Tensor center, at::Tensor points, int depth) { 126 | auto start = high_resolution_clock::now(); 127 | EasyOctree tree(center, depth); 128 | for (int k=0; k(stop - start); 133 | printf("Building EasyOctree done. total #nodes = %d, terminal #nodes = %d (time taken %f s)\n", 134 | tree.total, tree.terminal, float(duration.count()) / 1000000.); 135 | return std::make_tuple(tree.all_centers, tree.all_children); 136 | } -------------------------------------------------------------------------------- /third_party/intersect/octree.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #pragma once 7 | #include 8 | #include 9 | 10 | std::tuple build_octree(at::Tensor center, at::Tensor points, int depth); 11 | -------------------------------------------------------------------------------- /third_party/intersect/sample.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #include "sample.h" 7 | #include "utils.h" 8 | #include 9 | 10 | 11 | void uniform_ray_sampling_kernel_wrapper( 12 | int b, int num_rays, int max_hits, int max_steps, float step_size, 13 | const int *pts_idx, const float *min_depth, const float *max_depth, const float *uniform_noise, 14 | int *sampled_idx, float *sampled_depth, float *sampled_dists); 15 | 16 | void inverse_cdf_sampling_kernel_wrapper( 17 | int b, int num_rays, int max_hits, int max_steps, float fixed_step_size, 18 | const int *pts_idx, const float *min_depth, const float *max_depth, 19 | const float *uniform_noise, const float *probs, const float *steps, 20 | int *sampled_idx, float *sampled_depth, float *sampled_dists); 21 | 22 | 23 | std::tuple< at::Tensor, at::Tensor, at::Tensor> uniform_ray_sampling( 24 | at::Tensor pts_idx, at::Tensor min_depth, at::Tensor max_depth, at::Tensor uniform_noise, 25 | const float step_size, const int max_steps){ 26 | 27 | CHECK_CONTIGUOUS(pts_idx); 28 | CHECK_CONTIGUOUS(min_depth); 29 | CHECK_CONTIGUOUS(max_depth); 30 | CHECK_CONTIGUOUS(uniform_noise); 31 | CHECK_IS_FLOAT(min_depth); 32 | CHECK_IS_FLOAT(max_depth); 33 | CHECK_IS_FLOAT(uniform_noise); 34 | CHECK_IS_INT(pts_idx); 35 | CHECK_CUDA(pts_idx); 36 | CHECK_CUDA(min_depth); 37 | CHECK_CUDA(max_depth); 38 | CHECK_CUDA(uniform_noise); 39 | 40 | at::Tensor sampled_idx = 41 | -torch::ones({pts_idx.size(0), pts_idx.size(1), max_steps}, 42 | at::device(pts_idx.device()).dtype(at::ScalarType::Int)); 43 | at::Tensor sampled_depth = 44 | torch::zeros({min_depth.size(0), min_depth.size(1), max_steps}, 45 | at::device(min_depth.device()).dtype(at::ScalarType::Float)); 46 | at::Tensor sampled_dists = 47 | torch::zeros({min_depth.size(0), min_depth.size(1), max_steps}, 48 | at::device(min_depth.device()).dtype(at::ScalarType::Float)); 49 | uniform_ray_sampling_kernel_wrapper(min_depth.size(0), min_depth.size(1), min_depth.size(2), sampled_depth.size(2), 50 | step_size, 51 | pts_idx.data_ptr (), min_depth.data_ptr (), max_depth.data_ptr (), 52 | uniform_noise.data_ptr (), sampled_idx.data_ptr (), 53 | sampled_depth.data_ptr (), sampled_dists.data_ptr ()); 54 | return std::make_tuple(sampled_idx, sampled_depth, sampled_dists); 55 | } 56 | 57 | 58 | std::tuple inverse_cdf_sampling( 59 | at::Tensor pts_idx, at::Tensor min_depth, at::Tensor max_depth, at::Tensor uniform_noise, 60 | at::Tensor probs, at::Tensor steps, float fixed_step_size) { 61 | 62 | CHECK_CONTIGUOUS(pts_idx); 63 | CHECK_CONTIGUOUS(min_depth); 64 | CHECK_CONTIGUOUS(max_depth); 65 | CHECK_CONTIGUOUS(probs); 66 | CHECK_CONTIGUOUS(steps); 67 | CHECK_CONTIGUOUS(uniform_noise); 68 | CHECK_IS_FLOAT(min_depth); 69 | CHECK_IS_FLOAT(max_depth); 70 | CHECK_IS_FLOAT(uniform_noise); 71 | CHECK_IS_FLOAT(probs); 72 | CHECK_IS_FLOAT(steps); 73 | CHECK_IS_INT(pts_idx); 74 | CHECK_CUDA(pts_idx); 75 | CHECK_CUDA(min_depth); 76 | CHECK_CUDA(max_depth); 77 | CHECK_CUDA(uniform_noise); 78 | CHECK_CUDA(probs); 79 | CHECK_CUDA(steps); 80 | 81 | int max_steps = uniform_noise.size(-1); 82 | at::Tensor sampled_idx = 83 | -torch::ones({pts_idx.size(0), pts_idx.size(1), max_steps}, 84 | at::device(pts_idx.device()).dtype(at::ScalarType::Int)); 85 | at::Tensor sampled_depth = 86 | torch::zeros({min_depth.size(0), min_depth.size(1), max_steps}, 87 | at::device(min_depth.device()).dtype(at::ScalarType::Float)); 88 | at::Tensor sampled_dists = 89 | torch::zeros({min_depth.size(0), min_depth.size(1), max_steps}, 90 | at::device(min_depth.device()).dtype(at::ScalarType::Float)); 91 | inverse_cdf_sampling_kernel_wrapper(min_depth.size(0), min_depth.size(1), min_depth.size(2), sampled_depth.size(2), fixed_step_size, 92 | pts_idx.data_ptr (), min_depth.data_ptr (), max_depth.data_ptr (), 93 | uniform_noise.data_ptr (), probs.data_ptr (), steps.data_ptr (), 94 | sampled_idx.data_ptr (), sampled_depth.data_ptr (), sampled_dists.data_ptr ()); 95 | return std::make_tuple(sampled_idx, sampled_depth, sampled_dists); 96 | } 97 | -------------------------------------------------------------------------------- /third_party/intersect/sample.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #pragma once 7 | #include 8 | #include 9 | 10 | 11 | std::tuple uniform_ray_sampling( 12 | at::Tensor pts_idx, at::Tensor min_depth, at::Tensor max_depth, at::Tensor uniform_noise, 13 | const float step_size, const int max_steps); 14 | std::tuple inverse_cdf_sampling( 15 | at::Tensor pts_idx, at::Tensor min_depth, at::Tensor max_depth, at::Tensor uniform_noise, 16 | at::Tensor probs, at::Tensor steps, float fixed_step_size); -------------------------------------------------------------------------------- /third_party/intersect/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Facebook, Inc. and its affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | #pragma once 7 | #include 8 | #include 9 | 10 | #define CHECK_CUDA(x) \ 11 | do { \ 12 | TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor"); \ 13 | } while (0) 14 | 15 | #define CHECK_CONTIGUOUS(x) \ 16 | do { \ 17 | TORCH_CHECK(x.is_contiguous(), #x " must be a contiguous tensor"); \ 18 | } while (0) 19 | 20 | #define CHECK_IS_INT(x) \ 21 | do { \ 22 | TORCH_CHECK(x.scalar_type() == at::ScalarType::Int, \ 23 | #x " must be an int tensor"); \ 24 | } while (0) 25 | 26 | #define CHECK_IS_FLOAT(x) \ 27 | do { \ 28 | TORCH_CHECK(x.scalar_type() == at::ScalarType::Float, \ 29 | #x " must be a float tensor"); \ 30 | } while (0) 31 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | import numpy as np 9 | 10 | from functools import partial 11 | from torch.optim import SGD, Adam, RMSprop 12 | from torch.optim.lr_scheduler import ( 13 | CosineAnnealingLR, MultiStepLR, LambdaLR 14 | ) 15 | 16 | from .warmup_scheduler import GradualWarmupScheduler 17 | 18 | 19 | def no_init(cfg): 20 | def init(m): 21 | pass 22 | 23 | return init 24 | 25 | def uniform_weights_init(cfg): 26 | def init(m): 27 | if isinstance(m, torch.nn.Linear): 28 | torch.nn.init.uniform_(m.weight, a=-0.1, b=0.1) 29 | torch.nn.init.uniform_(m.bias, a=-0.1, b=0.1) 30 | 31 | return init 32 | 33 | def xavier_uniform_weights_init(cfg): 34 | def init(m): 35 | if isinstance(m, torch.nn.Linear): 36 | torch.nn.init.xavier_uniform_(m.weight) 37 | torch.nn.init.uniform_(m.bias, a=-0.01, b=0.01) 38 | 39 | return init 40 | 41 | weight_init_dict = { 42 | 'none': no_init, 43 | 'uniform': uniform_weights_init, 44 | 'xavier_uniform': xavier_uniform_weights_init, 45 | } 46 | 47 | def to8b(x): return (255*np.clip(x, 0, 1)).astype(np.uint8) 48 | 49 | def get_optimizer(hparams, models): 50 | eps = 1e-8 51 | parameters = [] 52 | 53 | for model in models: 54 | parameters += list(model.parameters()) 55 | if hparams.optimizer == 'sgd': 56 | optimizer = SGD( 57 | parameters, lr=hparams.lr, 58 | momentum=hparams.momentum, 59 | weight_decay=hparams.weight_decay 60 | ) 61 | elif hparams.optimizer == 'adam': 62 | optimizer = Adam( 63 | parameters, lr=hparams.lr, eps=eps, 64 | weight_decay=hparams.weight_decay 65 | ) 66 | elif hparams.optimizer == 'rmsprop': 67 | optimizer = RMSprop( 68 | parameters, alpha=hparams.alpha, momentum=hparams.momentum, 69 | lr=hparams.lr, eps=eps, 70 | weight_decay=hparams.weight_decay 71 | ) 72 | else: 73 | raise ValueError('optimizer not recognized!') 74 | 75 | return optimizer 76 | 77 | def exp_decay(decay_gamma, decay_step, epoch): 78 | return decay_gamma ** (epoch / decay_step) 79 | 80 | def poly_exp_decay(num_epochs, poly_exp, epoch): 81 | return (1 - epoch / num_epochs) ** poly_exp 82 | 83 | def get_scheduler(hparams, optimizer): 84 | eps = 1e-8 85 | 86 | if hparams.lr_scheduler == 'exp': 87 | scheduler = LambdaLR( 88 | optimizer, 89 | partial(exp_decay, hparams.decay_gamma, hparams.decay_step), 90 | ) 91 | elif hparams.lr_scheduler == 'steplr': 92 | scheduler = MultiStepLR( 93 | optimizer, milestones=[hparams.decay_step], 94 | gamma=hparams.decay_gamma 95 | ) 96 | elif hparams.lr_scheduler == 'cosine': 97 | scheduler = CosineAnnealingLR( 98 | optimizer, T_max=hparams.num_epochs, eta_min=eps 99 | ) 100 | elif hparams.lr_scheduler == 'poly': 101 | scheduler = LambdaLR( 102 | optimizer, 103 | partial(poly_exp_decay, hparams.num_epochs, hparams.poly_exp), 104 | ) 105 | else: 106 | raise ValueError('scheduler not recognized!') 107 | 108 | if hparams.warmup_epochs > 0: 109 | scheduler = GradualWarmupScheduler( 110 | optimizer, multiplier=hparams.warmup_multiplier, 111 | total_epoch=hparams.warmup_epochs, after_scheduler=scheduler 112 | ) 113 | 114 | return scheduler 115 | 116 | def get_learning_rate(optimizer): 117 | for param_group in optimizer.param_groups: 118 | return param_group['lr'] 119 | 120 | def extract_model_state_dict(ckpt_path, model_name='model', prefixes_to_ignore=[]): # noqa 121 | checkpoint = torch.load(ckpt_path, map_location=torch.device('cpu')) 122 | checkpoint_ = {} 123 | 124 | if 'state_dict' in checkpoint: # if it's a pytorch-lightning checkpoint 125 | checkpoint = checkpoint['state_dict'] 126 | 127 | for k, v in checkpoint.items(): 128 | if not k.startswith(model_name): 129 | continue 130 | 131 | k = k[len(model_name)+1:] 132 | 133 | for prefix in prefixes_to_ignore: 134 | if k.startswith(prefix): 135 | print('ignore', k) 136 | break 137 | else: 138 | checkpoint_[k] = v 139 | 140 | return checkpoint_ 141 | 142 | def load_ckpt(model, ckpt_path, model_name='model', prefixes_to_ignore=[]): # noqa 143 | model_dict = model.state_dict() 144 | checkpoint_ = extract_model_state_dict(ckpt_path, model_name, prefixes_to_ignore) 145 | model_dict.update(checkpoint_) 146 | model.load_state_dict(model_dict) 147 | -------------------------------------------------------------------------------- /utils/config_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # @manual //github/third-party/omry/omegaconf:omegaconf 8 | from omegaconf import DictConfig 9 | 10 | 11 | def format_config(cfg: DictConfig): 12 | format_config_helper(cfg, cfg) 13 | 14 | 15 | def format_config_helper(cfg, master_config: DictConfig): 16 | if isinstance(cfg, DictConfig): 17 | for key, _ in cfg.items(): 18 | if isinstance(cfg[key], str): 19 | cfg[key] = cfg[key].format(config=master_config) 20 | else: 21 | format_config_helper(cfg[key], master_config) 22 | 23 | 24 | def replace_config(cfg, **kwargs): 25 | if isinstance(cfg, DictConfig): 26 | for key, _ in cfg.items(): 27 | if key in kwargs.keys(): 28 | cfg[key] = kwargs[key] 29 | else: 30 | replace_config(cfg[key], **kwargs) 31 | -------------------------------------------------------------------------------- /utils/visualization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | import numpy as np 9 | 10 | 11 | def get_warp_dimensions(embedding, W, H, k=3): 12 | embedding_std = np.array(torch.std(embedding, 0, True).cpu()) 13 | return list(np.argsort(-embedding_std, axis=-1)[:k]) 14 | 15 | def visualize_warp(embedding, warp_dims, W, H, k=3): 16 | warp_vis = np.array(embedding[..., warp_dims].cpu()) 17 | 18 | warp_vis = np.abs(warp_vis) / np.max(np.abs(warp_vis)) 19 | warp_vis = np.clip(warp_vis, 0, 1) 20 | warp_vis = np.transpose( 21 | np.reshape(warp_vis, (H, W, k)), (2, 0, 1) 22 | ) 23 | 24 | return warp_vis 25 | -------------------------------------------------------------------------------- /utils/warmup_scheduler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from torch.optim.lr_scheduler import _LRScheduler 8 | 9 | 10 | class GradualWarmupScheduler(_LRScheduler): 11 | """ Gradually warm-up(increasing) learning rate in optimizer. 12 | Implementation: https://github.com/kwea123/nerf_pl 13 | Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'. 14 | Args: 15 | optimizer (Optimizer): Wrapped optimizer. 16 | multiplier: target learning rate = base lr * multiplier 17 | total_epoch: target learning rate is reached at total_epoch, gradually 18 | after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau) 19 | """ 20 | 21 | def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None): 22 | self.multiplier = multiplier 23 | self.total_epoch = total_epoch 24 | self.after_scheduler = after_scheduler 25 | self.finished = False 26 | 27 | super().__init__(optimizer) 28 | 29 | def get_lr(self): 30 | if self.last_epoch >= self.total_epoch: 31 | if self.after_scheduler: 32 | if not self.finished: 33 | self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] 34 | self.finished = True 35 | 36 | lrs = self.after_scheduler.get_lr() 37 | else: 38 | lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] 39 | else: 40 | lrs = [base_lr * ((self.last_epoch + 1) / (self.total_epoch + 1)) * self.multiplier for base_lr in self.base_lrs] 41 | 42 | return lrs 43 | 44 | def step(self, epoch=None, metrics=None): 45 | if self.finished and self.after_scheduler: 46 | self.get_lr() 47 | 48 | if epoch is None: 49 | self.after_scheduler.step(None) 50 | else: 51 | self.after_scheduler.step(epoch - self.total_epoch) 52 | else: 53 | return super(GradualWarmupScheduler, self).step(epoch) 54 | --------------------------------------------------------------------------------