├── .gitignore
├── .gitmodules
├── LICENSE.md
├── LICENSE_GS.md
├── README.md
├── arguments
└── __init__.py
├── assets
└── teaser.png
├── bash_scripts
├── m360_indoor
│ ├── eval_bonsai.sh
│ ├── eval_counter.sh
│ ├── eval_kitchen.sh
│ └── eval_room.sh
├── m360_outdoor
│ ├── eval_bicycle.sh
│ ├── eval_flowers.sh
│ ├── eval_garden.sh
│ ├── eval_stump.sh
│ └── eval_treehill.sh
├── run_all.sh
├── tandt
│ ├── eval_train.sh
│ └── eval_truck.sh
└── video_all.sh
├── compile.bat
├── compile.sh
├── convert.py
├── create_off.py
├── create_video.py
├── full_eval.py
├── lpipsPyTorch
├── __init__.py
└── modules
│ ├── lpips.py
│ ├── networks.py
│ └── utils.py
├── mesh.py
├── metrics.py
├── render.py
├── requirements.yaml
├── scene
├── __init__.py
├── cameras.py
├── colmap_loader.py
├── dataset_readers.py
└── triangle_model.py
├── scripts
├── dtu_eval.py
├── eval_dtu
│ ├── eval.py
│ ├── evaluate_single_scene.py
│ └── render_utils.py
├── eval_tnt
│ ├── README.md
│ ├── compute_bbox_for_mesh.py
│ ├── config.py
│ ├── cull_mesh.py
│ ├── evaluate_single_scene.py
│ ├── evaluation.py
│ ├── help_func.py
│ ├── plot.py
│ ├── registration.py
│ ├── requirements.txt
│ ├── run.py
│ ├── trajectory_io.py
│ └── util.py
└── tnt_eval.py
├── train.py
├── train_game_engine.py
├── triangle_renderer
└── __init__.py
└── utils
├── camera_utils.py
├── general_utils.py
├── graphics_utils.py
├── image_utils.py
├── loss_utils.py
├── mesh_utils.py
├── point_utils.py
├── render_utils.py
├── sh_utils.py
└── system_utils.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | */__pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
110 | .pdm.toml
111 | .pdm-python
112 | .pdm-build/
113 |
114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
115 | __pypackages__/
116 |
117 | # Celery stuff
118 | celerybeat-schedule
119 | celerybeat.pid
120 |
121 | # SageMath parsed files
122 | *.sage.py
123 |
124 | # Environments
125 | .env
126 | .venv
127 | env/
128 | venv/
129 | ENV/
130 | env.bak/
131 | venv.bak/
132 |
133 | # Spyder project settings
134 | .spyderproject
135 | .spyproject
136 |
137 | # Rope project settings
138 | .ropeproject
139 |
140 | # mkdocs documentation
141 | /site
142 |
143 | # mypy
144 | .mypy_cache/
145 | .dmypy.json
146 | dmypy.json
147 |
148 | # Pyre type checker
149 | .pyre/
150 |
151 | # pytype static type analyzer
152 | .pytype/
153 |
154 | # Cython debug symbols
155 | cython_debug/
156 |
157 | # PyCharm
158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
160 | # and can be added to the global gitignore or merged into this file. For a more nuclear
161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162 | #.idea/
--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "submodules/simple-knn"]
2 | path = submodules/simple-knn
3 | url = https://gitlab.inria.fr/bkerbl/simple-knn.git
4 | [submodule "submodules/diff-triangle-rasterization"]
5 | path = submodules/diff-triangle-rasterization
6 | url = https://github.com/trianglesplatting/diff-triangle-rasterization.git
7 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | Copyright (C) 2024, University of Liege, KAUST and University of Oxford
179 |
180 | Licensed under the Apache License, Version 2.0 (the "License");
181 | you may not use this file except in compliance with the License.
182 | You may obtain a copy of the License at
183 |
184 | http://www.apache.org/licenses/LICENSE-2.0
185 |
186 | Unless required by applicable law or agreed to in writing, software
187 | distributed under the License is distributed on an "AS IS" BASIS,
188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
189 | See the License for the specific language governing permissions and
190 | limitations under the License.
191 |
--------------------------------------------------------------------------------
/LICENSE_GS.md:
--------------------------------------------------------------------------------
1 | Gaussian-Splatting License
2 | ===========================
3 |
4 | **Inria** and **the Max Planck Institut for Informatik (MPII)** hold all the ownership rights on the *Software* named **gaussian-splatting**.
5 | The *Software* is in the process of being registered with the Agence pour la Protection des
6 | Programmes (APP).
7 |
8 | The *Software* is still being developed by the *Licensor*.
9 |
10 | *Licensor*'s goal is to allow the research community to use, test and evaluate
11 | the *Software*.
12 |
13 | ## 1. Definitions
14 |
15 | *Licensee* means any person or entity that uses the *Software* and distributes
16 | its *Work*.
17 |
18 | *Licensor* means the owners of the *Software*, i.e Inria and MPII
19 |
20 | *Software* means the original work of authorship made available under this
21 | License ie gaussian-splatting.
22 |
23 | *Work* means the *Software* and any additions to or derivative works of the
24 | *Software* that are made available under this License.
25 |
26 |
27 | ## 2. Purpose
28 | This license is intended to define the rights granted to the *Licensee* by
29 | Licensors under the *Software*.
30 |
31 | ## 3. Rights granted
32 |
33 | For the above reasons Licensors have decided to distribute the *Software*.
34 | Licensors grant non-exclusive rights to use the *Software* for research purposes
35 | to research users (both academic and industrial), free of charge, without right
36 | to sublicense.. The *Software* may be used "non-commercially", i.e., for research
37 | and/or evaluation purposes only.
38 |
39 | Subject to the terms and conditions of this License, you are granted a
40 | non-exclusive, royalty-free, license to reproduce, prepare derivative works of,
41 | publicly display, publicly perform and distribute its *Work* and any resulting
42 | derivative works in any form.
43 |
44 | ## 4. Limitations
45 |
46 | **4.1 Redistribution.** You may reproduce or distribute the *Work* only if (a) you do
47 | so under this License, (b) you include a complete copy of this License with
48 | your distribution, and (c) you retain without modification any copyright,
49 | patent, trademark, or attribution notices that are present in the *Work*.
50 |
51 | **4.2 Derivative Works.** You may specify that additional or different terms apply
52 | to the use, reproduction, and distribution of your derivative works of the *Work*
53 | ("Your Terms") only if (a) Your Terms provide that the use limitation in
54 | Section 2 applies to your derivative works, and (b) you identify the specific
55 | derivative works that are subject to Your Terms. Notwithstanding Your Terms,
56 | this License (including the redistribution requirements in Section 3.1) will
57 | continue to apply to the *Work* itself.
58 |
59 | **4.3** Any other use without of prior consent of Licensors is prohibited. Research
60 | users explicitly acknowledge having received from Licensors all information
61 | allowing to appreciate the adequacy between of the *Software* and their needs and
62 | to undertake all necessary precautions for its execution and use.
63 |
64 | **4.4** The *Software* is provided both as a compiled library file and as source
65 | code. In case of using the *Software* for a publication or other results obtained
66 | through the use of the *Software*, users are strongly encouraged to cite the
67 | corresponding publications as explained in the documentation of the *Software*.
68 |
69 | ## 5. Disclaimer
70 |
71 | THE USER CANNOT USE, EXPLOIT OR DISTRIBUTE THE *SOFTWARE* FOR COMMERCIAL PURPOSES
72 | WITHOUT PRIOR AND EXPLICIT CONSENT OF LICENSORS. YOU MUST CONTACT INRIA FOR ANY
73 | UNAUTHORIZED USE: stip-sophia.transfert@inria.fr . ANY SUCH ACTION WILL
74 | CONSTITUTE A FORGERY. THIS *SOFTWARE* IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES
75 | OF ANY NATURE AND ANY EXPRESS OR IMPLIED WARRANTIES, WITH REGARDS TO COMMERCIAL
76 | USE, PROFESSIONNAL USE, LEGAL OR NOT, OR OTHER, OR COMMERCIALISATION OR
77 | ADAPTATION. UNLESS EXPLICITLY PROVIDED BY LAW, IN NO EVENT, SHALL INRIA OR THE
78 | AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
79 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
80 | GOODS OR SERVICES, LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION)
81 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
82 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING FROM, OUT OF OR
83 | IN CONNECTION WITH THE *SOFTWARE* OR THE USE OR OTHER DEALINGS IN THE *SOFTWARE*.
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Triangle Splatting for Real-Time Radiance Field Rendering
2 |
3 |
7 |
8 |
9 |
10 | Jan Held*, Renaud Vandeghen*, Adrien Deliege, Abdullah Hamdi, Silvio Giancola, Anthony Cioppa, Andrea Vedaldi, Bernard Ghanem, Andrea Tagliasacchi, Marc Van Droogenbroeck
11 |
12 |
13 |
14 |
15 |
16 |
17 |
71 |
72 | ``PR_Ignatius_@d_th_0_0030.pdf`` (Precision and recall curves with a F-score)
73 |
74 | | 
| 
|
75 | |--|--|
76 | | ``Ignatius.precision.ply`` | ``Ignatius.recall.ply`` |
77 |
78 | (3D visualization of precision and recall. Each mesh is color coded using hot colormap)
79 |
80 | # Requirements
81 |
82 | - Python 3
83 | - open3d v0.9.0
84 | - matplotlib
85 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/compute_bbox_for_mesh.py:
--------------------------------------------------------------------------------
1 | # ----------------------------------------------------------------------------
2 | # - TanksAndTemples Website Toolbox -
3 | # - http://www.tanksandtemples.org -
4 | # ----------------------------------------------------------------------------
5 | # The MIT License (MIT)
6 | #
7 | # Copyright (c) 2017
8 | # Arno Knapitsch
9 | # Jaesik Park
10 | # Qian-Yi Zhou
11 | # Vladlen Koltun
12 | #
13 | # Permission is hereby granted, free of charge, to any person obtaining a copy
14 | # of this software and associated documentation files (the "Software"), to deal
15 | # in the Software without restriction, including without limitation the rights
16 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 | # copies of the Software, and to permit persons to whom the Software is
18 | # furnished to do so, subject to the following conditions:
19 | #
20 | # The above copyright notice and this permission notice shall be included in
21 | # all copies or substantial portions of the Software.
22 | #
23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 | # THE SOFTWARE.
30 | # ----------------------------------------------------------------------------
31 | #
32 | # This python script is for downloading dataset from www.tanksandtemples.org
33 | # The dataset has a different license, please refer to
34 | # https://tanksandtemples.org/license/
35 |
36 | # this script requires Open3D python binding
37 | # please follow the intructions in setup.py before running this script.
38 | import numpy as np
39 | import open3d as o3d
40 | import os
41 | import argparse
42 | import torch
43 |
44 | from config import scenes_tau_dict
45 | from registration import (
46 | trajectory_alignment,
47 | registration_vol_ds,
48 | registration_unif,
49 | read_trajectory,
50 | )
51 | from help_func import auto_orient_and_center_poses
52 | from trajectory_io import CameraPose
53 | from evaluation import EvaluateHisto
54 | from util import make_dir
55 | from plot import plot_graph
56 |
57 |
58 | def run_evaluation(dataset_dir, traj_path, ply_path, out_dir, view_crop):
59 | scene = os.path.basename(os.path.normpath(dataset_dir))
60 |
61 | if scene not in scenes_tau_dict:
62 | print(dataset_dir, scene)
63 | raise Exception("invalid dataset-dir, not in scenes_tau_dict")
64 |
65 | print("")
66 | print("===========================")
67 | print("Evaluating %s" % scene)
68 | print("===========================")
69 |
70 | dTau = scenes_tau_dict[scene]
71 | # put the crop-file, the GT file, the COLMAP SfM log file and
72 | # the alignment of the according scene in a folder of
73 | # the same scene name in the dataset_dir
74 | colmap_ref_logfile = os.path.join(dataset_dir, scene + "_COLMAP_SfM.log")
75 |
76 | # this is for groundtruth pointcloud, we can use it
77 | alignment = os.path.join(dataset_dir, scene + "_trans.txt")
78 | gt_filen = os.path.join(dataset_dir, scene + ".ply")
79 | # this crop file is also w.r.t the groundtruth pointcloud, we can use it.
80 | # Otherwise we have to crop the estimated pointcloud by ourself
81 | cropfile = os.path.join(dataset_dir, scene + ".json")
82 | # this is not so necessary
83 | map_file = os.path.join(dataset_dir, scene + "_mapping_reference.txt")
84 | if not os.path.isfile(map_file):
85 | map_file = None
86 | map_file = None
87 |
88 | make_dir(out_dir)
89 |
90 | # Load reconstruction and according GT
91 | print(ply_path)
92 | pcd = o3d.io.read_point_cloud(ply_path)
93 | print(gt_filen)
94 | gt_pcd = o3d.io.read_point_cloud(gt_filen)
95 |
96 | gt_trans = np.loadtxt(alignment)
97 | print(traj_path)
98 | traj_to_register = []
99 | if traj_path.endswith('.npy'):
100 | ld = np.load(traj_path)
101 | for i in range(len(ld)):
102 | traj_to_register.append(CameraPose(meta=None, mat=ld[i]))
103 | elif traj_path.endswith('.json'): # instant-npg or sdfstudio format
104 | import json
105 | with open(traj_path, encoding='UTF-8') as f:
106 | meta = json.load(f)
107 | poses_dict = {}
108 | for i, frame in enumerate(meta['frames']):
109 | filepath = frame['file_path']
110 | new_i = int(filepath[13:18]) - 1
111 | poses_dict[new_i] = np.array(frame['transform_matrix'])
112 | poses = []
113 | for i in range(len(poses_dict)):
114 | poses.append(poses_dict[i])
115 | poses = torch.from_numpy(np.array(poses).astype(np.float32))
116 | poses, _ = auto_orient_and_center_poses(poses, method='up', center_poses=True)
117 | scale_factor = 1.0 / float(torch.max(torch.abs(poses[:, :3, 3])))
118 | poses[:, :3, 3] *= scale_factor
119 | poses = poses.numpy()
120 | for i in range(len(poses)):
121 | traj_to_register.append(CameraPose(meta=None, mat=poses[i]))
122 |
123 | else:
124 | traj_to_register = read_trajectory(traj_path)
125 | print(colmap_ref_logfile)
126 | gt_traj_col = read_trajectory(colmap_ref_logfile)
127 |
128 | trajectory_transform = trajectory_alignment(map_file, traj_to_register,
129 | gt_traj_col, gt_trans, scene)
130 | inv_transform = np.linalg.inv(trajectory_transform)
131 | points = np.asarray(gt_pcd.points)
132 | points = points @ inv_transform[:3, :3].T + inv_transform[:3, 3:].T
133 | print(points.min(axis=0), points.max(axis=0))
134 | print(np.concatenate([points.min(axis=0), points.max(axis=0)]).reshape(-1).tolist())
135 |
136 |
137 | if __name__ == "__main__":
138 | parser = argparse.ArgumentParser()
139 | parser.add_argument(
140 | "--dataset-dir",
141 | type=str,
142 | required=True,
143 | help="path to a dataset/scene directory containing X.json, X.ply, ...",
144 | )
145 | parser.add_argument(
146 | "--traj-path",
147 | type=str,
148 | required=True,
149 | help=
150 | "path to trajectory file. See `convert_to_logfile.py` to create this file.",
151 | )
152 | parser.add_argument(
153 | "--ply-path",
154 | type=str,
155 | required=True,
156 | help="path to reconstruction ply file",
157 | )
158 | parser.add_argument(
159 | "--out-dir",
160 | type=str,
161 | default="",
162 | help=
163 | "output directory, default: an evaluation directory is created in the directory of the ply file",
164 | )
165 | parser.add_argument(
166 | "--view-crop",
167 | type=int,
168 | default=0,
169 | help="whether view the crop pointcloud after aligned",
170 | )
171 | args = parser.parse_args()
172 |
173 | args.view_crop = False # (args.view_crop > 0)
174 | if args.out_dir.strip() == "":
175 | args.out_dir = os.path.join(os.path.dirname(args.ply_path),
176 | "evaluation")
177 |
178 | run_evaluation(
179 | dataset_dir=args.dataset_dir,
180 | traj_path=args.traj_path,
181 | ply_path=args.ply_path,
182 | out_dir=args.out_dir,
183 | view_crop=args.view_crop
184 | )
185 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/config.py:
--------------------------------------------------------------------------------
1 | # ----------------------------------------------------------------------------
2 | # - TanksAndTemples Website Toolbox -
3 | # - http://www.tanksandtemples.org -
4 | # ----------------------------------------------------------------------------
5 | # The MIT License (MIT)
6 | #
7 | # Copyright (c) 2017
8 | # Arno Knapitsch
9 | # Jaesik Park
10 | # Qian-Yi Zhou
11 | # Vladlen Koltun
12 | #
13 | # Permission is hereby granted, free of charge, to any person obtaining a copy
14 | # of this software and associated documentation files (the "Software"), to deal
15 | # in the Software without restriction, including without limitation the rights
16 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 | # copies of the Software, and to permit persons to whom the Software is
18 | # furnished to do so, subject to the following conditions:
19 | #
20 | # The above copyright notice and this permission notice shall be included in
21 | # all copies or substantial portions of the Software.
22 | #
23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 | # THE SOFTWARE.
30 | # ----------------------------------------------------------------------------
31 |
32 | # some global parameters - do not modify
33 | scenes_tau_dict = {
34 | "Barn": 0.01,
35 | "Caterpillar": 0.005,
36 | "Church": 0.025,
37 | "Courthouse": 0.025,
38 | "Ignatius": 0.003,
39 | "Meetingroom": 0.01,
40 | "Truck": 0.005,
41 | }
42 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/evaluate_single_scene.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 | import cv2
5 | import numpy as np
6 | import os
7 | import glob
8 | from skimage.morphology import binary_dilation, disk
9 | import argparse
10 |
11 | import trimesh
12 | from pathlib import Path
13 | import subprocess
14 | import sys
15 | import json
16 |
17 |
18 | if __name__ == "__main__":
19 |
20 | parser = argparse.ArgumentParser(
21 | description='Arguments to evaluate the mesh.'
22 | )
23 |
24 | parser.add_argument('--input_mesh', type=str, help='path to the mesh to be evaluated')
25 | parser.add_argument('--scene', type=str, help='scan id of the input mesh')
26 | parser.add_argument('--output_dir', type=str, default='evaluation_results_single', help='path to the output folder')
27 | parser.add_argument('--TNT', type=str, default='Offical_DTU_Dataset', help='path to the GT DTU point clouds')
28 | args = parser.parse_args()
29 |
30 |
31 | TNT_Dataset = args.TNT
32 | out_dir = args.output_dir
33 | Path(out_dir).mkdir(parents=True, exist_ok=True)
34 | scene = args.scene
35 | ply_file = args.input_mesh
36 | result_mesh_file = os.path.join(out_dir, "culled_mesh.ply")
37 | # read scene.json
38 | f"python run.py --dataset-dir {ply_file} --traj-path {TNT_Dataset}/{scene}/{scene}_COLMAP_SfM.log --ply-path {TNT_Dataset}/{scene}/{scene}_COLMAP.ply"
--------------------------------------------------------------------------------
/scripts/eval_tnt/evaluation.py:
--------------------------------------------------------------------------------
1 | # ----------------------------------------------------------------------------
2 | # - TanksAndTemples Website Toolbox -
3 | # - http://www.tanksandtemples.org -
4 | # ----------------------------------------------------------------------------
5 | # The MIT License (MIT)
6 | #
7 | # Copyright (c) 2017
8 | # Arno Knapitsch
9 | # Jaesik Park
10 | # Qian-Yi Zhou
11 | # Vladlen Koltun
12 | #
13 | # Permission is hereby granted, free of charge, to any person obtaining a copy
14 | # of this software and associated documentation files (the "Software"), to deal
15 | # in the Software without restriction, including without limitation the rights
16 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 | # copies of the Software, and to permit persons to whom the Software is
18 | # furnished to do so, subject to the following conditions:
19 | #
20 | # The above copyright notice and this permission notice shall be included in
21 | # all copies or substantial portions of the Software.
22 | #
23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 | # THE SOFTWARE.
30 | # ----------------------------------------------------------------------------
31 | #
32 | # This python script is for downloading dataset from www.tanksandtemples.org
33 | # The dataset has a different license, please refer to
34 | # https://tanksandtemples.org/license/
35 |
36 | import json
37 | import copy
38 | import os
39 | import numpy as np
40 | import open3d as o3d
41 | import matplotlib.pyplot as plt
42 |
43 |
44 | def read_alignment_transformation(filename):
45 | with open(filename) as data_file:
46 | data = json.load(data_file)
47 | return np.asarray(data["transformation"]).reshape((4, 4)).transpose()
48 |
49 |
50 | def write_color_distances(path, pcd, distances, max_distance):
51 | o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug)
52 | # cmap = plt.get_cmap("afmhot")
53 | cmap = plt.get_cmap("hot_r")
54 | distances = np.array(distances)
55 | colors = cmap(np.minimum(distances, max_distance) / max_distance)[:, :3]
56 | pcd.colors = o3d.utility.Vector3dVector(colors)
57 | o3d.io.write_point_cloud(path, pcd)
58 |
59 |
60 | def EvaluateHisto(
61 | source,
62 | target,
63 | trans,
64 | crop_volume,
65 | voxel_size,
66 | threshold,
67 | filename_mvs,
68 | plot_stretch,
69 | scene_name,
70 | view_crop,
71 | verbose=True,
72 | ):
73 | print("[EvaluateHisto]")
74 | o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug)
75 | s = copy.deepcopy(source)
76 | s.transform(trans)
77 | if crop_volume is not None:
78 | s = crop_volume.crop_point_cloud(s)
79 | if view_crop:
80 | o3d.visualization.draw_geometries([s, ])
81 | else:
82 | print("No bounding box provided to crop estimated point cloud, leaving it as the loaded version!!")
83 | s = s.voxel_down_sample(voxel_size)
84 | s.estimate_normals(search_param=o3d.geometry.KDTreeSearchParamKNN(knn=20))
85 | print(filename_mvs + "/" + scene_name + ".precision.ply")
86 |
87 | t = copy.deepcopy(target)
88 | if crop_volume is not None:
89 | t = crop_volume.crop_point_cloud(t)
90 | else:
91 | print("No bounding box provided to crop groundtruth point cloud, leaving it as the loaded version!!")
92 |
93 | t = t.voxel_down_sample(voxel_size)
94 | t.estimate_normals(search_param=o3d.geometry.KDTreeSearchParamKNN(knn=20))
95 | print("[compute_point_cloud_to_point_cloud_distance]")
96 | distance1 = s.compute_point_cloud_distance(t)
97 | print("[compute_point_cloud_to_point_cloud_distance]")
98 | distance2 = t.compute_point_cloud_distance(s)
99 |
100 | # write the distances to bin files
101 | # np.array(distance1).astype("float64").tofile(
102 | # filename_mvs + "/" + scene_name + ".precision.bin"
103 | # )
104 | # np.array(distance2).astype("float64").tofile(
105 | # filename_mvs + "/" + scene_name + ".recall.bin"
106 | # )
107 |
108 | # Colorize the poincloud files prith the precision and recall values
109 | # o3d.io.write_point_cloud(
110 | # filename_mvs + "/" + scene_name + ".precision.ply", s
111 | # )
112 | # o3d.io.write_point_cloud(
113 | # filename_mvs + "/" + scene_name + ".precision.ncb.ply", s
114 | # )
115 | # o3d.io.write_point_cloud(filename_mvs + "/" + scene_name + ".recall.ply", t)
116 |
117 | source_n_fn = filename_mvs + "/" + scene_name + ".precision.ply"
118 | target_n_fn = filename_mvs + "/" + scene_name + ".recall.ply"
119 |
120 | print("[ViewDistances] Add color coding to visualize error")
121 | # eval_str_viewDT = (
122 | # OPEN3D_EXPERIMENTAL_BIN_PATH
123 | # + "ViewDistances "
124 | # + source_n_fn
125 | # + " --max_distance "
126 | # + str(threshold * 3)
127 | # + " --write_color_back --without_gui"
128 | # )
129 | # os.system(eval_str_viewDT)
130 | write_color_distances(source_n_fn, s, distance1, 3 * threshold)
131 |
132 | print("[ViewDistances] Add color coding to visualize error")
133 | # eval_str_viewDT = (
134 | # OPEN3D_EXPERIMENTAL_BIN_PATH
135 | # + "ViewDistances "
136 | # + target_n_fn
137 | # + " --max_distance "
138 | # + str(threshold * 3)
139 | # + " --write_color_back --without_gui"
140 | # )
141 | # os.system(eval_str_viewDT)
142 | write_color_distances(target_n_fn, t, distance2, 3 * threshold)
143 |
144 | # get histogram and f-score
145 | [
146 | precision,
147 | recall,
148 | fscore,
149 | edges_source,
150 | cum_source,
151 | edges_target,
152 | cum_target,
153 | ] = get_f1_score_histo2(threshold, filename_mvs, plot_stretch, distance1,
154 | distance2)
155 | np.savetxt(filename_mvs + "/" + scene_name + ".recall.txt", cum_target)
156 | np.savetxt(filename_mvs + "/" + scene_name + ".precision.txt", cum_source)
157 | np.savetxt(
158 | filename_mvs + "/" + scene_name + ".prf_tau_plotstr.txt",
159 | np.array([precision, recall, fscore, threshold, plot_stretch]),
160 | )
161 |
162 | return [
163 | precision,
164 | recall,
165 | fscore,
166 | edges_source,
167 | cum_source,
168 | edges_target,
169 | cum_target,
170 | ]
171 |
172 |
173 | def get_f1_score_histo2(threshold,
174 | filename_mvs,
175 | plot_stretch,
176 | distance1,
177 | distance2,
178 | verbose=True):
179 | print("[get_f1_score_histo2]")
180 | dist_threshold = threshold
181 | if len(distance1) and len(distance2):
182 |
183 | recall = float(sum(d < threshold for d in distance2)) / float(
184 | len(distance2))
185 | precision = float(sum(d < threshold for d in distance1)) / float(
186 | len(distance1))
187 | fscore = 2 * recall * precision / (recall + precision)
188 | num = len(distance1)
189 | bins = np.arange(0, dist_threshold * plot_stretch, dist_threshold / 100)
190 | hist, edges_source = np.histogram(distance1, bins)
191 | cum_source = np.cumsum(hist).astype(float) / num
192 |
193 | num = len(distance2)
194 | bins = np.arange(0, dist_threshold * plot_stretch, dist_threshold / 100)
195 | hist, edges_target = np.histogram(distance2, bins)
196 | cum_target = np.cumsum(hist).astype(float) / num
197 |
198 | else:
199 | precision = 0
200 | recall = 0
201 | fscore = 0
202 | edges_source = np.array([0])
203 | cum_source = np.array([0])
204 | edges_target = np.array([0])
205 | cum_target = np.array([0])
206 |
207 | return [
208 | precision,
209 | recall,
210 | fscore,
211 | edges_source,
212 | cum_source,
213 | edges_target,
214 | cum_target,
215 | ]
216 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/help_func.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # coding=utf-8
3 | import torch
4 |
5 | def rotation_matrix(a, b):
6 | """Compute the rotation matrix that rotates vector a to vector b.
7 |
8 | Args:
9 | a: The vector to rotate.
10 | b: The vector to rotate to.
11 | Returns:
12 | The rotation matrix.
13 | """
14 | a = a / torch.linalg.norm(a)
15 | b = b / torch.linalg.norm(b)
16 | v = torch.cross(a, b)
17 | c = torch.dot(a, b)
18 | # If vectors are exactly opposite, we add a little noise to one of them
19 | if c < -1 + 1e-8:
20 | eps = (torch.rand(3) - 0.5) * 0.01
21 | return rotation_matrix(a + eps, b)
22 | s = torch.linalg.norm(v)
23 | skew_sym_mat = torch.Tensor(
24 | [
25 | [0, -v[2], v[1]],
26 | [v[2], 0, -v[0]],
27 | [-v[1], v[0], 0],
28 | ]
29 | )
30 | return torch.eye(3) + skew_sym_mat + skew_sym_mat @ skew_sym_mat * ((1 - c) / (s**2 + 1e-8))
31 |
32 |
33 | def auto_orient_and_center_poses(
34 | poses, method="up", center_poses=True
35 | ):
36 | """Orients and centers the poses. We provide two methods for orientation: pca and up.
37 |
38 | pca: Orient the poses so that the principal component of the points is aligned with the axes.
39 | This method works well when all of the cameras are in the same plane.
40 | up: Orient the poses so that the average up vector is aligned with the z axis.
41 | This method works well when images are not at arbitrary angles.
42 |
43 |
44 | Args:
45 | poses: The poses to orient.
46 | method: The method to use for orientation.
47 | center_poses: If True, the poses are centered around the origin.
48 |
49 | Returns:
50 | The oriented poses.
51 | """
52 |
53 | translation = poses[..., :3, 3]
54 |
55 | mean_translation = torch.mean(translation, dim=0)
56 | translation_diff = translation - mean_translation
57 |
58 | if center_poses:
59 | translation = mean_translation
60 | else:
61 | translation = torch.zeros_like(mean_translation)
62 |
63 | if method == "pca":
64 | _, eigvec = torch.linalg.eigh(translation_diff.T @ translation_diff)
65 | eigvec = torch.flip(eigvec, dims=(-1,))
66 |
67 | if torch.linalg.det(eigvec) < 0:
68 | eigvec[:, 2] = -eigvec[:, 2]
69 |
70 | transform = torch.cat([eigvec, eigvec @ -translation[..., None]], dim=-1)
71 | oriented_poses = transform @ poses
72 |
73 | if oriented_poses.mean(axis=0)[2, 1] < 0:
74 | oriented_poses[:, 1:3] = -1 * oriented_poses[:, 1:3]
75 | elif method == "up":
76 | up = torch.mean(poses[:, :3, 1], dim=0)
77 | up = up / torch.linalg.norm(up)
78 |
79 | rotation = rotation_matrix(up, torch.Tensor([0, 0, 1]))
80 | transform = torch.cat([rotation, rotation @ -translation[..., None]], dim=-1)
81 | oriented_poses = transform @ poses
82 | elif method == "none":
83 | transform = torch.eye(4)
84 | transform[:3, 3] = -translation
85 | transform = transform[:3, :]
86 | oriented_poses = transform @ poses
87 |
88 | return oriented_poses, transform
89 |
90 |
91 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/plot.py:
--------------------------------------------------------------------------------
1 | # ----------------------------------------------------------------------------
2 | # - TanksAndTemples Website Toolbox -
3 | # - http://www.tanksandtemples.org -
4 | # ----------------------------------------------------------------------------
5 | # The MIT License (MIT)
6 | #
7 | # Copyright (c) 2017
8 | # Arno Knapitsch
9 | # Jaesik Park
10 | # Qian-Yi Zhou
11 | # Vladlen Koltun
12 | #
13 | # Permission is hereby granted, free of charge, to any person obtaining a copy
14 | # of this software and associated documentation files (the "Software"), to deal
15 | # in the Software without restriction, including without limitation the rights
16 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 | # copies of the Software, and to permit persons to whom the Software is
18 | # furnished to do so, subject to the following conditions:
19 | #
20 | # The above copyright notice and this permission notice shall be included in
21 | # all copies or substantial portions of the Software.
22 | #
23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 | # THE SOFTWARE.
30 | # ----------------------------------------------------------------------------
31 | #
32 | # This python script is for downloading dataset from www.tanksandtemples.org
33 | # The dataset has a different license, please refer to
34 | # https://tanksandtemples.org/license/
35 |
36 | import matplotlib.pyplot as plt
37 | from cycler import cycler
38 |
39 |
40 | def plot_graph(
41 | scene,
42 | fscore,
43 | dist_threshold,
44 | edges_source,
45 | cum_source,
46 | edges_target,
47 | cum_target,
48 | plot_stretch,
49 | mvs_outpath,
50 | show_figure=False,
51 | ):
52 | f = plt.figure()
53 | plt_size = [14, 7]
54 | pfontsize = "medium"
55 |
56 | ax = plt.subplot(111)
57 | label_str = "precision"
58 | ax.plot(
59 | edges_source[1::],
60 | cum_source * 100,
61 | c="red",
62 | label=label_str,
63 | linewidth=2.0,
64 | )
65 |
66 | label_str = "recall"
67 | ax.plot(
68 | edges_target[1::],
69 | cum_target * 100,
70 | c="blue",
71 | label=label_str,
72 | linewidth=2.0,
73 | )
74 |
75 | ax.grid(True)
76 | plt.rcParams["figure.figsize"] = plt_size
77 | plt.rc("axes", prop_cycle=cycler("color", ["r", "g", "b", "y"]))
78 | plt.title("Precision and Recall: " + scene + ", " + "%02.2f f-score" %
79 | (fscore * 100))
80 | plt.axvline(x=dist_threshold, c="black", ls="dashed", linewidth=2.0)
81 |
82 | plt.ylabel("# of points (%)", fontsize=15)
83 | plt.xlabel("Meters", fontsize=15)
84 | plt.axis([0, dist_threshold * plot_stretch, 0, 100])
85 | ax.legend(shadow=True, fancybox=True, fontsize=pfontsize)
86 | # plt.axis([0, dist_threshold*plot_stretch, 0, 100])
87 |
88 | plt.setp(ax.get_legend().get_texts(), fontsize=pfontsize)
89 |
90 | plt.legend(loc=2, borderaxespad=0.0, fontsize=pfontsize)
91 | plt.legend(loc=4)
92 | leg = plt.legend(loc="lower right")
93 |
94 | box = ax.get_position()
95 | ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
96 |
97 | # Put a legend to the right of the current axis
98 | ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
99 | plt.setp(ax.get_legend().get_texts(), fontsize=pfontsize)
100 | png_name = mvs_outpath + "/PR_{0}_@d_th_0_{1}.png".format(
101 | scene, "%04d" % (dist_threshold * 10000))
102 | pdf_name = mvs_outpath + "/PR_{0}_@d_th_0_{1}.pdf".format(
103 | scene, "%04d" % (dist_threshold * 10000))
104 |
105 | # save figure and display
106 | f.savefig(png_name, format="png", bbox_inches="tight")
107 | f.savefig(pdf_name, format="pdf", bbox_inches="tight")
108 | if show_figure:
109 | plt.show()
110 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/registration.py:
--------------------------------------------------------------------------------
1 | # ----------------------------------------------------------------------------
2 | # - TanksAndTemples Website Toolbox -
3 | # - http://www.tanksandtemples.org -
4 | # ----------------------------------------------------------------------------
5 | # The MIT License (MIT)
6 | #
7 | # Copyright (c) 2017
8 | # Arno Knapitsch
9 | # Jaesik Park
10 | # Qian-Yi Zhou
11 | # Vladlen Koltun
12 | #
13 | # Permission is hereby granted, free of charge, to any person obtaining a copy
14 | # of this software and associated documentation files (the "Software"), to deal
15 | # in the Software without restriction, including without limitation the rights
16 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 | # copies of the Software, and to permit persons to whom the Software is
18 | # furnished to do so, subject to the following conditions:
19 | #
20 | # The above copyright notice and this permission notice shall be included in
21 | # all copies or substantial portions of the Software.
22 | #
23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 | # THE SOFTWARE.
30 | # ----------------------------------------------------------------------------
31 | #
32 | # This python script is for downloading dataset from www.tanksandtemples.org
33 | # The dataset has a different license, please refer to
34 | # https://tanksandtemples.org/license/
35 |
36 | from trajectory_io import read_trajectory, convert_trajectory_to_pointcloud
37 | import copy
38 | import numpy as np
39 | import open3d as o3d
40 |
41 | MAX_POINT_NUMBER = 4e6
42 |
43 |
44 | def read_mapping(filename):
45 | mapping = []
46 | with open(filename, "r") as f:
47 | n_sampled_frames = int(f.readline())
48 | n_total_frames = int(f.readline())
49 | mapping = np.zeros(shape=(n_sampled_frames, 2))
50 | metastr = f.readline()
51 | for iter in range(n_sampled_frames):
52 | metadata = list(map(int, metastr.split()))
53 | mapping[iter, :] = metadata
54 | metastr = f.readline()
55 | return [n_sampled_frames, n_total_frames, mapping]
56 |
57 |
58 | def gen_sparse_trajectory(mapping, f_trajectory):
59 | sparse_traj = []
60 | for m in mapping:
61 | sparse_traj.append(f_trajectory[int(m[1] - 1)])
62 | return sparse_traj
63 |
64 |
65 | def trajectory_alignment(map_file, traj_to_register, gt_traj_col, gt_trans,
66 | scene):
67 | traj_pcd_col = convert_trajectory_to_pointcloud(gt_traj_col)
68 | if gt_trans is not None:
69 | traj_pcd_col.transform(gt_trans)
70 | corres = o3d.utility.Vector2iVector(
71 | np.asarray(list(map(lambda x: [x, x], range(len(gt_traj_col))))))
72 | rr = o3d.registration.RANSACConvergenceCriteria()
73 | rr.max_iteration = 100000
74 | rr.max_validation = 100000
75 |
76 | # in this case a log file was used which contains
77 | # every movie frame (see tutorial for details)
78 | if len(traj_to_register) > 1600 and map_file is not None:
79 | n_sampled_frames, n_total_frames, mapping = read_mapping(map_file)
80 | traj_col2 = gen_sparse_trajectory(mapping, traj_to_register)
81 | traj_to_register_pcd = convert_trajectory_to_pointcloud(traj_col2)
82 | else:
83 | print("Estimated trajectory will leave as it is, no sparsity op is performed!")
84 | traj_to_register_pcd = convert_trajectory_to_pointcloud(
85 | traj_to_register)
86 | randomvar = 0.0
87 | if randomvar < 1e-5:
88 | traj_to_register_pcd_rand = traj_to_register_pcd
89 | else:
90 | nr_of_cam_pos = len(traj_to_register_pcd.points)
91 | rand_number_added = np.asanyarray(traj_to_register_pcd.points) * (
92 | np.random.rand(nr_of_cam_pos, 3) * randomvar - randomvar / 2.0 + 1)
93 | list_rand = list(rand_number_added)
94 | traj_to_register_pcd_rand = o3d.geometry.PointCloud()
95 | for elem in list_rand:
96 | traj_to_register_pcd_rand.points.append(elem)
97 |
98 | # Rough registration based on aligned colmap SfM data
99 | reg = o3d.registration.registration_ransac_based_on_correspondence(
100 | traj_to_register_pcd_rand,
101 | traj_pcd_col,
102 | corres,
103 | 0.2,
104 | o3d.registration.TransformationEstimationPointToPoint(True),
105 | 6,
106 | rr,
107 | )
108 | return reg.transformation
109 |
110 |
111 | def crop_and_downsample(
112 | pcd,
113 | crop_volume,
114 | down_sample_method="voxel",
115 | voxel_size=0.01,
116 | trans=np.identity(4),
117 | ):
118 | pcd_copy = copy.deepcopy(pcd)
119 | pcd_copy.transform(trans)
120 | pcd_crop = crop_volume.crop_point_cloud(pcd_copy)
121 | if down_sample_method == "voxel":
122 | # return voxel_down_sample(pcd_crop, voxel_size)
123 | return pcd_crop.voxel_down_sample(voxel_size)
124 | elif down_sample_method == "uniform":
125 | n_points = len(pcd_crop.points)
126 | if n_points > MAX_POINT_NUMBER:
127 | ds_rate = int(round(n_points / float(MAX_POINT_NUMBER)))
128 | return pcd_crop.uniform_down_sample(ds_rate)
129 | return pcd_crop
130 |
131 |
132 | def registration_unif(
133 | source,
134 | gt_target,
135 | init_trans,
136 | crop_volume,
137 | threshold,
138 | max_itr,
139 | max_size=4 * MAX_POINT_NUMBER,
140 | verbose=True,
141 | ):
142 | if verbose:
143 | print("[Registration] threshold: %f" % threshold)
144 | o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug)
145 | s = crop_and_downsample(source,
146 | crop_volume,
147 | down_sample_method="uniform",
148 | trans=init_trans)
149 | t = crop_and_downsample(gt_target,
150 | crop_volume,
151 | down_sample_method="uniform")
152 | reg = o3d.registration.registration_icp(
153 | s,
154 | t,
155 | threshold,
156 | np.identity(4),
157 | o3d.registration.TransformationEstimationPointToPoint(True),
158 | o3d.registration.ICPConvergenceCriteria(1e-6, max_itr),
159 | )
160 | reg.transformation = np.matmul(reg.transformation, init_trans)
161 | return reg
162 |
163 |
164 | def registration_vol_ds(
165 | source,
166 | gt_target,
167 | init_trans,
168 | crop_volume,
169 | voxel_size,
170 | threshold,
171 | max_itr,
172 | verbose=True,
173 | ):
174 | if verbose:
175 | print("[Registration] voxel_size: %f, threshold: %f" %
176 | (voxel_size, threshold))
177 | o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug)
178 | s = crop_and_downsample(
179 | source,
180 | crop_volume,
181 | down_sample_method="voxel",
182 | voxel_size=voxel_size,
183 | trans=init_trans,
184 | )
185 | t = crop_and_downsample(
186 | gt_target,
187 | crop_volume,
188 | down_sample_method="voxel",
189 | voxel_size=voxel_size,
190 | )
191 | reg = o3d.registration.registration_icp(
192 | s,
193 | t,
194 | threshold,
195 | np.identity(4),
196 | o3d.registration.TransformationEstimationPointToPoint(True),
197 | o3d.registration.ICPConvergenceCriteria(1e-6, max_itr),
198 | )
199 | reg.transformation = np.matmul(reg.transformation, init_trans)
200 | return reg
201 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/requirements.txt:
--------------------------------------------------------------------------------
1 | matplotlib>=1.3
2 | open3d==0.10
3 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/run.py:
--------------------------------------------------------------------------------
1 | # ----------------------------------------------------------------------------
2 | # - TanksAndTemples Website Toolbox -
3 | # - http://www.tanksandtemples.org -
4 | # ----------------------------------------------------------------------------
5 | # The MIT License (MIT)
6 | #
7 | # Copyright (c) 2017
8 | # Arno Knapitsch
9 | # Jaesik Park
10 | # Qian-Yi Zhou
11 | # Vladlen Koltun
12 | #
13 | # Permission is hereby granted, free of charge, to any person obtaining a copy
14 | # of this software and associated documentation files (the "Software"), to deal
15 | # in the Software without restriction, including without limitation the rights
16 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17 | # copies of the Software, and to permit persons to whom the Software is
18 | # furnished to do so, subject to the following conditions:
19 | #
20 | # The above copyright notice and this permission notice shall be included in
21 | # all copies or substantial portions of the Software.
22 | #
23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 | # THE SOFTWARE.
30 | # ----------------------------------------------------------------------------
31 | #
32 | # This python script is for downloading dataset from www.tanksandtemples.org
33 | # The dataset has a different license, please refer to
34 | # https://tanksandtemples.org/license/
35 |
36 | # this script requires Open3D python binding
37 | # please follow the intructions in setup.py before running this script.
38 | import numpy as np
39 | import open3d as o3d
40 | import os
41 | import argparse
42 | # import torch
43 |
44 | from config import scenes_tau_dict
45 | from registration import (
46 | trajectory_alignment,
47 | registration_vol_ds,
48 | registration_unif,
49 | read_trajectory,
50 | )
51 | # from help_func import auto_orient_and_center_poses
52 | from trajectory_io import CameraPose
53 | from evaluation import EvaluateHisto
54 | from util import make_dir
55 | from plot import plot_graph
56 |
57 |
58 | def run_evaluation(dataset_dir, traj_path, ply_path, out_dir, view_crop):
59 | scene = os.path.basename(os.path.normpath(dataset_dir))
60 |
61 | if scene not in scenes_tau_dict:
62 | print(dataset_dir, scene)
63 | raise Exception("invalid dataset-dir, not in scenes_tau_dict")
64 |
65 | print("")
66 | print("===========================")
67 | print("Evaluating %s" % scene)
68 | print("===========================")
69 |
70 | dTau = scenes_tau_dict[scene]
71 | # put the crop-file, the GT file, the COLMAP SfM log file and
72 | # the alignment of the according scene in a folder of
73 | # the same scene name in the dataset_dir
74 | colmap_ref_logfile = os.path.join(dataset_dir, scene + "_COLMAP_SfM.log")
75 |
76 | # this is for groundtruth pointcloud, we can use it
77 | alignment = os.path.join(dataset_dir, scene + "_trans.txt")
78 | gt_filen = os.path.join(dataset_dir, scene + ".ply")
79 | # this crop file is also w.r.t the groundtruth pointcloud, we can use it.
80 | # Otherwise we have to crop the estimated pointcloud by ourself
81 | cropfile = os.path.join(dataset_dir, scene + ".json")
82 | # this is not so necessary
83 | map_file = os.path.join(dataset_dir, scene + "_mapping_reference.txt")
84 | if not os.path.isfile(map_file):
85 | map_file = None
86 | map_file = None
87 |
88 | make_dir(out_dir)
89 |
90 | # Load reconstruction and according GT
91 | print(ply_path)
92 | pcd = o3d.io.read_point_cloud(ply_path)
93 | # add center points
94 | import trimesh
95 | mesh = trimesh.load_mesh(ply_path)
96 | # add center points
97 | sampled_vertices = mesh.vertices[mesh.faces].mean(axis=1)
98 | # add 4 points based on the face vertices
99 | # face_vertices = mesh.vertices[mesh.faces]# .mean(axis=1)
100 | # weights = np.array([[3, 3, 3],
101 | # [4, 4, 1],
102 | # [4, 1, 4],
103 | # [1, 4, 4]],dtype=np.float32) / 9.0
104 | # sampled_vertices = np.sum(face_vertices.reshape(-1, 1, 3, 3) * weights.reshape(1, 4, 3, 1), axis=2).reshape(-1, 3)
105 |
106 | vertices = np.concatenate([mesh.vertices, sampled_vertices], axis=0)
107 | pcd = o3d.geometry.PointCloud()
108 | pcd.points = o3d.utility.Vector3dVector(vertices)
109 | ### end add center points
110 |
111 | print(gt_filen)
112 | gt_pcd = o3d.io.read_point_cloud(gt_filen)
113 |
114 | gt_trans = np.loadtxt(alignment)
115 | print(traj_path)
116 | traj_to_register = []
117 | if traj_path.endswith('.npy'):
118 | ld = np.load(traj_path)
119 | for i in range(len(ld)):
120 | traj_to_register.append(CameraPose(meta=None, mat=ld[i]))
121 | elif traj_path.endswith('.json'): # instant-npg or sdfstudio format
122 | import json
123 | with open(traj_path, encoding='UTF-8') as f:
124 | meta = json.load(f)
125 | poses_dict = {}
126 | for i, frame in enumerate(meta['frames']):
127 | filepath = frame['file_path']
128 | new_i = int(filepath[13:18]) - 1
129 | poses_dict[new_i] = np.array(frame['transform_matrix'])
130 | poses = []
131 | for i in range(len(poses_dict)):
132 | poses.append(poses_dict[i])
133 | poses = torch.from_numpy(np.array(poses).astype(np.float32))
134 | poses, _ = auto_orient_and_center_poses(poses, method='up', center_poses=True)
135 | scale_factor = 1.0 / float(torch.max(torch.abs(poses[:, :3, 3])))
136 | poses[:, :3, 3] *= scale_factor
137 | poses = poses.numpy()
138 | for i in range(len(poses)):
139 | traj_to_register.append(CameraPose(meta=None, mat=poses[i]))
140 |
141 | else:
142 | traj_to_register = read_trajectory(traj_path)
143 | print(colmap_ref_logfile)
144 | gt_traj_col = read_trajectory(colmap_ref_logfile)
145 |
146 | trajectory_transform = trajectory_alignment(map_file, traj_to_register,
147 | gt_traj_col, gt_trans, scene)
148 |
149 |
150 | # big pointclouds will be downlsampled to this number to speed up alignment
151 | dist_threshold = dTau
152 | # Refine alignment by using the actual GT and MVS pointclouds
153 | vol = o3d.visualization.read_selection_polygon_volume(cropfile)
154 |
155 | # Registration refinment in 3 iterations
156 | r2 = registration_vol_ds(pcd, gt_pcd, trajectory_transform, vol, dTau,
157 | dTau * 80, 20)
158 | r3 = registration_vol_ds(pcd, gt_pcd, r2.transformation, vol, dTau / 2.0,
159 | dTau * 20, 20)
160 | r = registration_unif(pcd, gt_pcd, r3.transformation, vol, 2 * dTau, 20)
161 | trajectory_transform = r.transformation
162 |
163 | # Histogramms and P/R/F1
164 | plot_stretch = 5
165 | [
166 | precision,
167 | recall,
168 | fscore,
169 | edges_source,
170 | cum_source,
171 | edges_target,
172 | cum_target,
173 | ] = EvaluateHisto(
174 | pcd,
175 | gt_pcd,
176 | trajectory_transform, # r.transformation,
177 | vol,
178 | dTau / 2.0,
179 | dTau,
180 | out_dir,
181 | plot_stretch,
182 | scene,
183 | view_crop
184 | )
185 | eva = [precision, recall, fscore]
186 | print("==============================")
187 | print("evaluation result : %s" % scene)
188 | print("==============================")
189 | print("distance tau : %.3f" % dTau)
190 | print("precision : %.4f" % eva[0])
191 | print("recall : %.4f" % eva[1])
192 | print("f-score : %.4f" % eva[2])
193 | print("==============================")
194 |
195 | # Plotting
196 | plot_graph(
197 | scene,
198 | fscore,
199 | dist_threshold,
200 | edges_source,
201 | cum_source,
202 | edges_target,
203 | cum_target,
204 | plot_stretch,
205 | out_dir,
206 | )
207 |
208 |
209 | if __name__ == "__main__":
210 | parser = argparse.ArgumentParser()
211 | parser.add_argument(
212 | "--dataset-dir",
213 | type=str,
214 | required=True,
215 | help="path to a dataset/scene directory containing X.json, X.ply, ...",
216 | )
217 | parser.add_argument(
218 | "--traj-path",
219 | type=str,
220 | required=True,
221 | help=
222 | "path to trajectory file. See `convert_to_logfile.py` to create this file.",
223 | )
224 | parser.add_argument(
225 | "--ply-path",
226 | type=str,
227 | required=True,
228 | help="path to reconstruction ply file",
229 | )
230 | parser.add_argument(
231 | "--out-dir",
232 | type=str,
233 | default="",
234 | help=
235 | "output directory, default: an evaluation directory is created in the directory of the ply file",
236 | )
237 | parser.add_argument(
238 | "--view-crop",
239 | type=int,
240 | default=0,
241 | help="whether view the crop pointcloud after aligned",
242 | )
243 | args = parser.parse_args()
244 |
245 | args.view_crop = False # (args.view_crop > 0)
246 | if args.out_dir.strip() == "":
247 | args.out_dir = os.path.join(os.path.dirname(args.ply_path),
248 | "evaluation")
249 |
250 | run_evaluation(
251 | dataset_dir=args.dataset_dir,
252 | traj_path=args.traj_path,
253 | ply_path=args.ply_path,
254 | out_dir=args.out_dir,
255 | view_crop=args.view_crop
256 | )
257 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/trajectory_io.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import open3d as o3d
3 |
4 |
5 | class CameraPose:
6 |
7 | def __init__(self, meta, mat):
8 | self.metadata = meta
9 | self.pose = mat
10 |
11 | def __str__(self):
12 | return ("Metadata : " + " ".join(map(str, self.metadata)) + "\n" +
13 | "Pose : " + "\n" + np.array_str(self.pose))
14 |
15 |
16 | def convert_trajectory_to_pointcloud(traj):
17 | pcd = o3d.geometry.PointCloud()
18 | for t in traj:
19 | pcd.points.append(t.pose[:3, 3])
20 | return pcd
21 |
22 |
23 | def read_trajectory(filename):
24 | traj = []
25 | with open(filename, "r") as f:
26 | metastr = f.readline()
27 | while metastr:
28 | metadata = map(int, metastr.split())
29 | mat = np.zeros(shape=(4, 4))
30 | for i in range(4):
31 | matstr = f.readline()
32 | mat[i, :] = np.fromstring(matstr, dtype=float, sep=" \t")
33 | traj.append(CameraPose(metadata, mat))
34 | metastr = f.readline()
35 | return traj
36 |
37 |
38 | def write_trajectory(traj, filename):
39 | with open(filename, "w") as f:
40 | for x in traj:
41 | p = x.pose.tolist()
42 | f.write(" ".join(map(str, x.metadata)) + "\n")
43 | f.write("\n".join(
44 | " ".join(map("{0:.12f}".format, p[i])) for i in range(4)))
45 | f.write("\n")
46 |
--------------------------------------------------------------------------------
/scripts/eval_tnt/util.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 |
4 | def make_dir(path):
5 | if not os.path.exists(path):
6 | os.makedirs(path)
7 |
--------------------------------------------------------------------------------
/scripts/tnt_eval.py:
--------------------------------------------------------------------------------
1 | import os
2 | from argparse import ArgumentParser
3 |
4 | tnt_360_scenes = ['Truck']
5 | tnt_large_scenes = []
6 |
7 | parser = ArgumentParser(description="Full evaluation script parameters")
8 | parser.add_argument("--skip_training", action="store_true")
9 | parser.add_argument("--skip_rendering", action="store_true")
10 | parser.add_argument("--skip_metrics", action="store_true")
11 | parser.add_argument("--output_path", default="./eval/tnt")
12 | parser.add_argument('--TNT_data', "-TNT_data", required=True, type=str)
13 | args, _ = parser.parse_known_args()
14 |
15 | if not args.skip_metrics:
16 | parser.add_argument('--TNT_GT', required=True, type=str)
17 | args = parser.parse_args()
18 |
19 |
20 | if not args.skip_training:
21 | common_args = " --quiet --test_iterations -1 --depth_ratio 1.0 -r 2 --eval --max_shapes 2600000 --lambda_normals 0.005" # --lambda_dist 1000"
22 |
23 | for scene in tnt_360_scenes:
24 | source = args.TNT_data + "/" + scene
25 | print("python train_db_mesh.py -s " + source + " -m " + args.output_path + "/" + scene + common_args + ' --lambda_dist 100')
26 | os.system("python train_db_mesh.py -s " + source + " -m " + args.output_path + "/" + scene + common_args)
27 |
28 | for scene in tnt_large_scenes:
29 | source = args.TNT_data + "/" + scene
30 | print("python train_db_mesh.py -s " + source + " -m " + args.output_path + "/" + scene + common_args+ ' --lambda_dist 10')
31 | os.system("python train_db_mesh.py -s " + source + " -m " + args.output_path + "/" + scene + common_args)
32 |
33 |
34 | if not args.skip_rendering:
35 | all_sources = []
36 | common_args = " --quiet --depth_ratio 1.0 "
37 |
38 | for scene in tnt_360_scenes:
39 | source = args.TNT_data + "/" + scene
40 | print("python render.py --iteration 30000 -s " + source + " -m " + args.output_path + "/" + scene + common_args + ' --num_cluster 1 --voxel_size 0.004 --sdf_trunc 0.016 --depth_trunc 3.0')
41 | os.system("python render.py --iteration 30000 -s " + source + " -m " + args.output_path + "/" + scene + common_args + ' --num_cluster 1 --voxel_size 0.004 --sdf_trunc 0.016 --depth_trunc 3.0')
42 |
43 | for scene in tnt_large_scenes:
44 | source = args.TNT_data + "/" + scene
45 | print("python render.py --iteration 30000 -s " + source + " -m " + args.output_path + "/" + scene + common_args + ' --num_cluster 1 --voxel_size 0.006 --sdf_trunc 0.024 --depth_trunc 4.5')
46 | os.system("python render.py --iteration 30000 -s " + source + " -m " + args.output_path + "/" + scene + common_args + ' --num_cluster 1 --voxel_size 0.006 --sdf_trunc 0.024 --depth_trunc 4.5')
47 |
48 | if not args.skip_metrics:
49 | script_dir = os.path.dirname(os.path.abspath(__file__))
50 | all_scenes = tnt_360_scenes + tnt_large_scenes
51 |
52 | for scene in all_scenes:
53 | ply_file = f"{args.output_path}/{scene}/train/ours_{30000}/fuse_post.ply"
54 | string = f"OMP_NUM_THREADS=4 python {script_dir}/eval_tnt/run.py " + \
55 | f"--dataset-dir {args.TNT_GT}/{scene} " + \
56 | f"--traj-path {args.TNT_data}/{scene}/{scene}_COLMAP_SfM.log " + \
57 | f"--ply-path {ply_file}"
58 | print(string)
59 | os.system(string)
60 |
61 |
62 | # python scripts/tnt_eval.py --TNT_data /gpfs/scratch/acad/telim/datasets/TNT_GOF/TrainingSet --TNT_GT /gpfs/scratch/acad/telim/datasets/tandt_gt
--------------------------------------------------------------------------------
/triangle_renderer/__init__.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2023, Inria
4 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
5 | # All rights reserved.
6 | #
7 | # This software is free for non-commercial, research and evaluation use
8 | # under the terms of the LICENSE_GS.md file.
9 | #
10 | # For inquiries contact george.drettakis@inria.fr
11 | #
12 | # The modifications of the code are under the following copyright:
13 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
14 | # TELIM research group, http://www.telecom.ulg.ac.be/
15 | # IVUL research group, https://ivul.kaust.edu.sa/
16 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
17 | # All rights reserved.
18 | # The modifications are under the LICENSE.md file.
19 | #
20 | # For inquiries contact jan.held@uliege.be
21 | #
22 |
23 | import torch
24 | import math
25 | from diff_triangle_rasterization import TriangleRasterizationSettings, TriangleRasterizer
26 | from scene.triangle_model import TriangleModel
27 | from utils.sh_utils import eval_sh
28 | from utils.point_utils import depth_to_normal
29 |
30 |
31 | def render(viewpoint_camera, pc : TriangleModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None):
32 | """
33 | Render the scene.
34 |
35 | Background tensor (bg_color) must be on GPU!
36 | """
37 |
38 | # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means
39 | screenspace_points = torch.zeros_like(pc.get_triangles_points[:,0,:].squeeze(), dtype=pc.get_triangles_points.dtype, requires_grad=True, device="cuda") + 0
40 | scaling = torch.zeros_like(pc.get_triangles_points[:,0,0].squeeze(), dtype=pc.get_triangles_points.dtype, requires_grad=True, device="cuda").detach()
41 | density_factor = torch.zeros_like(pc.get_triangles_points[:,0,0].squeeze(), dtype=pc.get_triangles_points.dtype, requires_grad=True, device="cuda").detach()
42 |
43 | try:
44 | screenspace_points.retain_grad()
45 | except:
46 | pass
47 |
48 | # Set up rasterization configuration
49 | tanfovx = math.tan(viewpoint_camera.FoVx * 0.5)
50 | tanfovy = math.tan(viewpoint_camera.FoVy * 0.5)
51 |
52 | raster_settings = TriangleRasterizationSettings(
53 | image_height=int(viewpoint_camera.image_height),
54 | image_width=int(viewpoint_camera.image_width),
55 | tanfovx=tanfovx,
56 | tanfovy=tanfovy,
57 | bg=bg_color,
58 | scale_modifier=scaling_modifier,
59 | viewmatrix=viewpoint_camera.world_view_transform,
60 | projmatrix=viewpoint_camera.full_proj_transform,
61 | sh_degree=pc.active_sh_degree,
62 | campos=viewpoint_camera.camera_center,
63 | prefiltered=False,
64 | debug=pipe.debug
65 | )
66 |
67 | rasterizer = TriangleRasterizer(raster_settings=raster_settings)
68 |
69 | opacity = pc.get_opacity
70 | triangles_points = pc.get_triangles_points_flatten
71 | sigma = pc.get_sigma
72 | num_points_per_triangle = pc.get_num_points_per_triangle
73 | cumsum_of_points_per_triangle = pc.get_cumsum_of_points_per_triangle
74 | number_of_points = pc.get_number_of_points
75 | means2D = screenspace_points
76 |
77 | # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors
78 | # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer.
79 | shs = None
80 | colors_precomp = None
81 | if override_color is None:
82 | if pipe.convert_SHs_python:
83 | shs_view = pc.get_features.transpose(1, 2).view(-1, 3, (pc.max_sh_degree+1)**2)
84 | dir_pp = (pc.get_xyz - viewpoint_camera.camera_center.repeat(pc.get_features.shape[0], 1))
85 | dir_pp_normalized = dir_pp/dir_pp.norm(dim=1, keepdim=True)
86 | sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized)
87 | colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0)
88 | else:
89 | shs = pc.get_features
90 |
91 | else:
92 | colors_precomp = override_color
93 |
94 |
95 |
96 | mask = ((torch.sigmoid(pc._mask) > 0.01).float()- torch.sigmoid(pc._mask)).detach() + torch.sigmoid(pc._mask)
97 | opacity = opacity * mask
98 |
99 | # Rasterize visible triangles to image, obtain their radii (on screen).
100 | rendered_image, radii, scaling, density_factor, allmap, max_blending = rasterizer(
101 | triangles_points=triangles_points,
102 | sigma=sigma,
103 | num_points_per_triangle = num_points_per_triangle,
104 | cumsum_of_points_per_triangle = cumsum_of_points_per_triangle,
105 | number_of_points = number_of_points,
106 | shs = shs,
107 | colors_precomp = colors_precomp,
108 | opacities = opacity,
109 | means2D = means2D,
110 | scaling = scaling,
111 | density_factor = density_factor
112 | )
113 |
114 | rets = {"render": rendered_image,
115 | "viewspace_points": screenspace_points,
116 | "visibility_filter" : radii > 0,
117 | "radii": radii,
118 | "scaling": scaling,
119 | "density_factor": density_factor,
120 | "max_blending": max_blending
121 | }
122 |
123 | # additional regularizations
124 | render_alpha = allmap[1:2]
125 |
126 | # get normal map
127 | # transform normal from view space to world space
128 | render_normal = allmap[2:5]
129 | render_normal = (render_normal.permute(1,2,0) @ (viewpoint_camera.world_view_transform[:3,:3].T)).permute(2,0,1)
130 |
131 | # get median depth map
132 | render_depth_median = allmap[5:6]
133 | render_depth_median = torch.nan_to_num(render_depth_median, 0, 0)
134 |
135 | # get expected depth map
136 | render_depth_expected = allmap[0:1]
137 | render_depth_expected = (render_depth_expected / render_alpha)
138 | render_depth_expected = torch.nan_to_num(render_depth_expected, 0, 0)
139 |
140 | # get depth distortion map
141 | render_dist = allmap[6:7]
142 |
143 | # psedo surface attributes
144 | # surf depth is either median or expected by setting depth_ratio to 1 or 0
145 | # for bounded scene, use median depth, i.e., depth_ratio = 1;
146 | # for unbounded scene, use expected depth, i.e., depth_ration = 0, to reduce disk anliasing.
147 | #depth_ratio=0
148 | surf_depth = render_depth_expected * (1-pipe.depth_ratio) + (pipe.depth_ratio) * render_depth_median
149 |
150 | # assume the depth points form the 'surface' and generate psudo surface normal for regularizations.
151 | surf_normal = depth_to_normal(viewpoint_camera, surf_depth)
152 | surf_normal = surf_normal.permute(2,0,1)
153 | # remember to multiply with accum_alpha since render_normal is unnormalized.
154 | surf_normal = surf_normal * (render_alpha).detach()
155 |
156 | rets.update({
157 | 'rend_alpha': render_alpha,
158 | 'rend_normal': render_normal,
159 | 'rend_dist': render_dist,
160 | 'surf_depth': surf_depth,
161 | 'surf_normal': surf_normal,
162 | })
163 |
164 | return rets
165 |
166 |
167 |
168 |
169 |
170 |
--------------------------------------------------------------------------------
/utils/camera_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright (C) 2023, Inria
3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
4 | # All rights reserved.
5 | #
6 | # This software is free for non-commercial, research and evaluation use
7 | # under the terms of the LICENSE.md file.
8 | #
9 | # For inquiries contact george.drettakis@inria.fr
10 | #
11 |
12 | from scene.cameras import Camera
13 | import numpy as np
14 | from utils.general_utils import PILtoTorch
15 | from utils.graphics_utils import fov2focal
16 |
17 | WARNED = False
18 |
19 | def loadCam(args, id, cam_info, resolution_scale):
20 | orig_w, orig_h = cam_info.image.size
21 |
22 | if args.resolution in [1, 2, 4, 8]:
23 | resolution = round(orig_w/(resolution_scale * args.resolution)), round(orig_h/(resolution_scale * args.resolution))
24 | else: # should be a type that converts to float
25 | if args.resolution == -1:
26 | if orig_w > 1600:
27 | global WARNED
28 | if not WARNED:
29 | print("[ INFO ] Encountered quite large input images (>1.6K pixels width), rescaling to 1.6K.\n "
30 | "If this is not desired, please explicitly specify '--resolution/-r' as 1")
31 | WARNED = True
32 | global_down = orig_w / 1600
33 | else:
34 | global_down = 1
35 | else:
36 | global_down = orig_w / args.resolution
37 |
38 | scale = float(global_down) * float(resolution_scale)
39 | resolution = (int(orig_w / scale), int(orig_h / scale))
40 |
41 | if len(cam_info.image.split()) > 3:
42 | import torch
43 | resized_image_rgb = torch.cat([PILtoTorch(im, resolution) for im in cam_info.image.split()[:3]], dim=0)
44 | loaded_mask = PILtoTorch(cam_info.image.split()[3], resolution)
45 | gt_image = resized_image_rgb
46 | else:
47 | resized_image_rgb = PILtoTorch(cam_info.image, resolution)
48 | loaded_mask = None
49 | gt_image = resized_image_rgb
50 |
51 | return Camera(colmap_id=cam_info.uid, R=cam_info.R, T=cam_info.T,
52 | FoVx=cam_info.FovX, FoVy=cam_info.FovY,
53 | image=gt_image, gt_alpha_mask=loaded_mask,
54 | image_name=cam_info.image_name, uid=id, data_device=args.data_device)
55 |
56 | def cameraList_from_camInfos(cam_infos, resolution_scale, args):
57 | camera_list = []
58 |
59 | for id, c in enumerate(cam_infos):
60 | camera_list.append(loadCam(args, id, c, resolution_scale))
61 |
62 | return camera_list
63 |
64 | def camera_to_JSON(id, camera : Camera):
65 | Rt = np.zeros((4, 4))
66 | Rt[:3, :3] = camera.R.transpose()
67 | Rt[:3, 3] = camera.T
68 | Rt[3, 3] = 1.0
69 |
70 | W2C = np.linalg.inv(Rt)
71 | pos = W2C[:3, 3]
72 | rot = W2C[:3, :3]
73 | serializable_array_2d = [x.tolist() for x in rot]
74 | camera_entry = {
75 | 'id' : id,
76 | 'img_name' : camera.image_name,
77 | 'width' : camera.width,
78 | 'height' : camera.height,
79 | 'position': pos.tolist(),
80 | 'rotation': serializable_array_2d,
81 | 'fy' : fov2focal(camera.FovY, camera.height),
82 | 'fx' : fov2focal(camera.FovX, camera.width)
83 | }
84 | return camera_entry
--------------------------------------------------------------------------------
/utils/general_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2023, Inria
4 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
5 | # All rights reserved.
6 | #
7 | # This software is free for non-commercial, research and evaluation use
8 | # under the terms of the LICENSE_GS.md file.
9 | #
10 | # For inquiries contact george.drettakis@inria.fr
11 | #
12 | # The modifications of the code are under the following copyright:
13 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
14 | # TELIM research group, http://www.telecom.ulg.ac.be/
15 | # IVUL research group, https://ivul.kaust.edu.sa/
16 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
17 | # All rights reserved.
18 | # The modifications are under the LICENSE.md file.
19 | #
20 | # For inquiries contact jan.held@uliege.be
21 | #
22 |
23 | import torch
24 | import sys
25 | from datetime import datetime
26 | import numpy as np
27 | import random
28 |
29 |
30 | def scaled_sigmoid(x):
31 | return 5 * torch.sigmoid(x)
32 |
33 | def inverse_sigmoid(x):
34 | return torch.log(x/(1-x))
35 |
36 | def inverse_sigmoid_10(x):
37 | return -torch.log((10 / x) - 1)
38 |
39 | def PILtoTorch(pil_image, resolution):
40 | resized_image_PIL = pil_image.resize(resolution)
41 | resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0
42 | if len(resized_image.shape) == 3:
43 | return resized_image.permute(2, 0, 1)
44 | else:
45 | return resized_image.unsqueeze(dim=-1).permute(2, 0, 1)
46 |
47 | def get_expon_lr_func(
48 | lr_init, lr_final, lr_delay_steps=0, lr_delay_mult=1.0, max_steps=1000000
49 | ):
50 | """
51 | Copied from Plenoxels
52 |
53 | Continuous learning rate decay function. Adapted from JaxNeRF
54 | The returned rate is lr_init when step=0 and lr_final when step=max_steps, and
55 | is log-linearly interpolated elsewhere (equivalent to exponential decay).
56 | If lr_delay_steps>0 then the learning rate will be scaled by some smooth
57 | function of lr_delay_mult, such that the initial learning rate is
58 | lr_init*lr_delay_mult at the beginning of optimization but will be eased back
59 | to the normal learning rate when steps>lr_delay_steps.
60 | :param conf: config subtree 'lr' or similar
61 | :param max_steps: int, the number of steps during optimization.
62 | :return HoF which takes step as input
63 | """
64 |
65 | def helper(step):
66 | if step < 0 or (lr_init == 0.0 and lr_final == 0.0):
67 | # Disable this parameter
68 | return 0.0
69 | if lr_delay_steps > 0:
70 | # A kind of reverse cosine decay.
71 | delay_rate = lr_delay_mult + (1 - lr_delay_mult) * np.sin(
72 | 0.5 * np.pi * np.clip(step / lr_delay_steps, 0, 1)
73 | )
74 | else:
75 | delay_rate = 1.0
76 | t = np.clip(step / max_steps, 0, 1)
77 | log_lerp = np.exp(np.log(lr_init) * (1 - t) + np.log(lr_final) * t)
78 | return delay_rate * log_lerp
79 |
80 | return helper
81 |
82 | def strip_lowerdiag(L):
83 | uncertainty = torch.zeros((L.shape[0], 6), dtype=torch.float, device="cuda")
84 |
85 | uncertainty[:, 0] = L[:, 0, 0]
86 | uncertainty[:, 1] = L[:, 0, 1]
87 | uncertainty[:, 2] = L[:, 0, 2]
88 | uncertainty[:, 3] = L[:, 1, 1]
89 | uncertainty[:, 4] = L[:, 1, 2]
90 | uncertainty[:, 5] = L[:, 2, 2]
91 | return uncertainty
92 |
93 | def strip_symmetric(sym):
94 | return strip_lowerdiag(sym)
95 |
96 | def build_rotation(r):
97 | norm = torch.sqrt(r[:,0]*r[:,0] + r[:,1]*r[:,1] + r[:,2]*r[:,2] + r[:,3]*r[:,3])
98 |
99 | q = r / norm[:, None]
100 |
101 | R = torch.zeros((q.size(0), 3, 3), device='cuda')
102 |
103 | r = q[:, 0]
104 | x = q[:, 1]
105 | y = q[:, 2]
106 | z = q[:, 3]
107 |
108 | R[:, 0, 0] = 1 - 2 * (y*y + z*z)
109 | R[:, 0, 1] = 2 * (x*y - r*z)
110 | R[:, 0, 2] = 2 * (x*z + r*y)
111 | R[:, 1, 0] = 2 * (x*y + r*z)
112 | R[:, 1, 1] = 1 - 2 * (x*x + z*z)
113 | R[:, 1, 2] = 2 * (y*z - r*x)
114 | R[:, 2, 0] = 2 * (x*z - r*y)
115 | R[:, 2, 1] = 2 * (y*z + r*x)
116 | R[:, 2, 2] = 1 - 2 * (x*x + y*y)
117 | return R
118 |
119 | def build_scaling_rotation(s, r):
120 | L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda")
121 | R = build_rotation(r)
122 |
123 | L[:,0,0] = s[:,0]
124 | L[:,1,1] = s[:,1]
125 | L[:,2,2] = s[:,2]
126 |
127 | L = R @ L
128 | return L
129 |
130 | def safe_state(silent):
131 | old_f = sys.stdout
132 | class F:
133 | def __init__(self, silent):
134 | self.silent = silent
135 |
136 | def write(self, x):
137 | if not self.silent:
138 | if x.endswith("\n"):
139 | old_f.write(x.replace("\n", " [{}]\n".format(str(datetime.now().strftime("%d/%m %H:%M:%S")))))
140 | else:
141 | old_f.write(x)
142 |
143 | def flush(self):
144 | old_f.flush()
145 |
146 | sys.stdout = F(silent)
147 |
148 | random.seed(0)
149 | np.random.seed(0)
150 | torch.manual_seed(0)
151 | torch.cuda.set_device(torch.device("cuda:0"))
152 |
--------------------------------------------------------------------------------
/utils/graphics_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2023, Inria
4 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
5 | # All rights reserved.
6 | #
7 | # This software is free for non-commercial, research and evaluation use
8 | # under the terms of the LICENSE_GS.md file.
9 | #
10 | # For inquiries contact george.drettakis@inria.fr
11 | #
12 | # The modifications of the code are under the following copyright:
13 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
14 | # TELIM research group, http://www.telecom.ulg.ac.be/
15 | # IVUL research group, https://ivul.kaust.edu.sa/
16 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
17 | # All rights reserved.
18 | # The modifications are under the LICENSE.md file.
19 | #
20 | # For inquiries contact jan.held@uliege.be
21 | #
22 |
23 | import torch
24 | import math
25 | import numpy as np
26 | from typing import NamedTuple
27 |
28 | class BasicPointCloud(NamedTuple):
29 | points : np.array
30 | colors : np.array
31 | normals : np.array
32 |
33 | def geom_transform_points(points, transf_matrix):
34 | P, _ = points.shape
35 | ones = torch.ones(P, 1, dtype=points.dtype, device=points.device)
36 | points_hom = torch.cat([points, ones], dim=1)
37 | points_out = torch.matmul(points_hom, transf_matrix.unsqueeze(0))
38 |
39 | denom = points_out[..., 3:] + 0.0000001
40 | return (points_out[..., :3] / denom).squeeze(dim=0)
41 |
42 | def getWorld2View(R, t):
43 | Rt = np.zeros((4, 4))
44 | Rt[:3, :3] = R.transpose()
45 | Rt[:3, 3] = t
46 | Rt[3, 3] = 1.0
47 | return np.float32(Rt)
48 |
49 | def getWorld2View2(R, t, translate=np.array([.0, .0, .0]), scale=1.0):
50 | Rt = np.zeros((4, 4))
51 | Rt[:3, :3] = R.transpose()
52 | Rt[:3, 3] = t
53 | Rt[3, 3] = 1.0
54 |
55 | C2W = np.linalg.inv(Rt)
56 | cam_center = C2W[:3, 3]
57 | cam_center = (cam_center + translate) * scale
58 | C2W[:3, 3] = cam_center
59 | Rt = np.linalg.inv(C2W)
60 | return np.float32(Rt)
61 |
62 | def getProjectionMatrix(znear, zfar, fovX, fovY):
63 | tanHalfFovY = math.tan((fovY / 2))
64 | tanHalfFovX = math.tan((fovX / 2))
65 |
66 | top = tanHalfFovY * znear
67 | bottom = -top
68 | right = tanHalfFovX * znear
69 | left = -right
70 |
71 | P = torch.zeros(4, 4)
72 |
73 | z_sign = 1.0
74 |
75 | P[0, 0] = 2.0 * znear / (right - left)
76 | P[1, 1] = 2.0 * znear / (top - bottom)
77 | P[0, 2] = (right + left) / (right - left)
78 | P[1, 2] = (top + bottom) / (top - bottom)
79 | P[3, 2] = z_sign
80 | P[2, 2] = z_sign * zfar / (zfar - znear)
81 | P[2, 3] = -(zfar * znear) / (zfar - znear)
82 | return P
83 |
84 | def fov2focal(fov, pixels):
85 | return pixels / (2 * math.tan(fov / 2))
86 |
87 | def focal2fov(focal, pixels):
88 | return 2*math.atan(pixels/(2*focal))
--------------------------------------------------------------------------------
/utils/image_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2023, Inria
4 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
5 | # All rights reserved.
6 | #
7 | # This software is free for non-commercial, research and evaluation use
8 | # under the terms of the LICENSE_GS.md file.
9 | #
10 | # For inquiries contact george.drettakis@inria.fr
11 | #
12 | # The modifications of the code are under the following copyright:
13 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
14 | # TELIM research group, http://www.telecom.ulg.ac.be/
15 | # IVUL research group, https://ivul.kaust.edu.sa/
16 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
17 | # All rights reserved.
18 | # The modifications are under the LICENSE.md file.
19 | #
20 | # For inquiries contact jan.held@uliege.be
21 | #
22 |
23 | import torch
24 | import torch.nn as nn
25 | import torch.nn.functional as F
26 |
27 | def mse(img1, img2):
28 | return (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
29 |
30 | def psnr(img1, img2):
31 | mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
32 | return 20 * torch.log10(1.0 / torch.sqrt(mse))
33 |
34 | class DoGFilter(nn.Module):
35 | def __init__(self, channels, sigma1):
36 | super(DoGFilter, self).__init__()
37 | self.channels = channels
38 | self.sigma1 = sigma1
39 | self.sigma2 = 2 * sigma1 # Ensure the 1:2 ratio
40 | self.kernel_size1 = int(2 * round(3 * self.sigma1) + 1)
41 | self.kernel_size2 = int(2 * round(3 * self.sigma2) + 1)
42 | self.padding1 = (self.kernel_size1 - 1) // 2
43 | self.padding2 = (self.kernel_size2 - 1) // 2
44 | self.weight1 = self.get_gaussian_kernel(self.kernel_size1, self.sigma1)
45 | self.weight2 = self.get_gaussian_kernel(self.kernel_size2, self.sigma2)
46 |
47 |
48 | def get_gaussian_kernel(self, kernel_size, sigma):
49 | x_cord = torch.arange(kernel_size)
50 | x_grid = x_cord.repeat(kernel_size).view(kernel_size, kernel_size)
51 | y_grid = x_grid.t()
52 | xy_grid = torch.stack([x_grid, y_grid], dim=-1).float()
53 |
54 | mean = (kernel_size - 1) / 2.
55 | variance = sigma**2.
56 |
57 | kernel = torch.exp(-(xy_grid - mean).pow(2).sum(dim=-1) / (2 * variance))
58 | kernel = kernel / kernel.sum() # Normalize the kernel
59 | kernel = kernel.repeat(self.channels, 1, 1, 1)
60 |
61 | return kernel
62 |
63 | @torch.no_grad()
64 | def forward(self, x):
65 | gaussian1 = F.conv2d(x, self.weight1.to(x.device), bias=None, stride=1, padding=self.padding1, groups=self.channels)
66 | gaussian2 = F.conv2d(x, self.weight2.to(x.device), bias=None, stride=1, padding=self.padding2, groups=self.channels)
67 | return gaussian1 - gaussian2
68 |
69 | def apply_dog_filter(batch, freq=50, scale_factor=0.5):
70 | """
71 | Apply a Difference of Gaussian filter to a batch of images.
72 |
73 | Args:
74 | batch: torch.Tensor, shape (B, C, H, W)
75 | freq: Control variable ranging from 0 to 100.
76 | - 0 means original image
77 | - 1.0 means smoother difference
78 | - 100 means sharpest difference
79 | scale_factor: Factor by which the image is downscaled before applying DoG.
80 |
81 | Returns:
82 | torch.Tensor: Processed image using DoG.
83 | """
84 | # Convert to grayscale if it's a color image
85 | if batch.size(1) == 3:
86 | batch = torch.mean(batch, dim=1, keepdim=True)
87 |
88 | # Downscale the image
89 | downscaled = F.interpolate(batch, scale_factor=scale_factor, mode='bilinear', align_corners=False)
90 |
91 | channels = downscaled.size(1)
92 |
93 | # Set sigma1 value based on freq parameter. sigma2 will be 2*sigma1.
94 | sigma1 = 0.1 + (100 - freq) * 0.1 if freq >=50 else 0.1 + freq * 0.1
95 |
96 | dog_filter = DoGFilter(channels, sigma1)
97 | mask = dog_filter(downscaled)
98 |
99 | # Upscale the mask back to original size
100 | upscaled_mask = F.interpolate(mask, size=batch.shape[-2:], mode='bilinear', align_corners=False)
101 |
102 | upscaled_mask = upscaled_mask - upscaled_mask.min()
103 | upscaled_mask = upscaled_mask / upscaled_mask.max() if freq >=50 else 1.0 - upscaled_mask / upscaled_mask.max()
104 |
105 | upscaled_mask = (upscaled_mask >=0.5).to(torch.float)
106 | return upscaled_mask[:,0,...]
--------------------------------------------------------------------------------
/utils/loss_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2023, Inria
4 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
5 | # All rights reserved.
6 | #
7 | # This software is free for non-commercial, research and evaluation use
8 | # under the terms of the LICENSE_GS.md file.
9 | #
10 | # For inquiries contact george.drettakis@inria.fr
11 | #
12 | # The modifications of the code are under the following copyright:
13 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
14 | # TELIM research group, http://www.telecom.ulg.ac.be/
15 | # IVUL research group, https://ivul.kaust.edu.sa/
16 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
17 | # All rights reserved.
18 | # The modifications are under the LICENSE.md file.
19 | #
20 | # For inquiries contact jan.held@uliege.be
21 | #
22 |
23 | import torch
24 | import torch.nn.functional as F
25 | from torch.autograd import Variable
26 | from math import exp
27 |
28 |
29 |
30 | def equilateral_regularizer(triangles):
31 |
32 | nan_mask = torch.isnan(triangles).any(dim=(1, 2))
33 | if nan_mask.any():
34 | print("NaN detected in triangle(s):")
35 |
36 | v0 = triangles[:, 1, :] - triangles[:, 0, :]
37 | v1 = triangles[:, 2, :] - triangles[:, 0, :]
38 | cross = torch.cross(v0, v1, dim=1)
39 | area = 0.5 * torch.norm(cross, dim=1)
40 |
41 | return area
42 |
43 |
44 | def l1_loss(network_output, gt):
45 | return torch.abs((network_output - gt)).mean()
46 |
47 | def l2_loss(network_output, gt):
48 | return ((network_output - gt) ** 2).mean()
49 |
50 | def lp_loss(pred, target, p=0.7, eps=1e-6):
51 | """
52 | Computes Lp loss with 0 < p < 1.
53 | Args:
54 | pred: (N, C, H, W) predicted image
55 | target: (N, C, H, W) groundtruth image
56 | p: norm degree < 1
57 | eps: small constant for numerical stability
58 | """
59 | diff = torch.abs(pred - target) + eps
60 | loss = torch.pow(diff, p).mean()
61 | return loss
62 |
63 | def gaussian(window_size, sigma):
64 | gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)])
65 | return gauss / gauss.sum()
66 |
67 | def create_window(window_size, channel):
68 | _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
69 | _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
70 | window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous())
71 | return window
72 |
73 | def ssim(img1, img2, window_size=11, size_average=True):
74 | channel = img1.size(-3)
75 | window = create_window(window_size, channel)
76 | if img1.is_cuda:
77 | window = window.cuda(img1.get_device())
78 | window = window.type_as(img1)
79 |
80 | return _ssim(img1, img2, window, window_size, channel, size_average)
81 |
82 | def _ssim(img1, img2, window, window_size, channel, size_average=True):
83 | mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel)
84 | mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel)
85 |
86 | mu1_sq = mu1.pow(2)
87 | mu2_sq = mu2.pow(2)
88 | mu1_mu2 = mu1 * mu2
89 |
90 | sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq
91 | sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq
92 | sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2
93 |
94 | C1 = 0.01 ** 2
95 | C2 = 0.03 ** 2
96 |
97 | ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2))
98 |
99 | if size_average:
100 | return ssim_map.mean()
101 | else:
102 | return ssim_map.mean(1).mean(1).mean(1)
103 |
104 |
--------------------------------------------------------------------------------
/utils/point_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 | import numpy as np
5 | import os, cv2
6 | import matplotlib.pyplot as plt
7 | import math
8 |
9 | def depths_to_points(view, depthmap):
10 | c2w = (view.world_view_transform.T).inverse()
11 | W, H = view.image_width, view.image_height
12 | ndc2pix = torch.tensor([
13 | [W / 2, 0, 0, (W) / 2],
14 | [0, H / 2, 0, (H) / 2],
15 | [0, 0, 0, 1]]).float().cuda().T
16 | projection_matrix = c2w.T @ view.full_proj_transform
17 | intrins = (projection_matrix @ ndc2pix)[:3,:3].T
18 |
19 | grid_x, grid_y = torch.meshgrid(torch.arange(W, device='cuda').float(), torch.arange(H, device='cuda').float(), indexing='xy')
20 | points = torch.stack([grid_x, grid_y, torch.ones_like(grid_x)], dim=-1).reshape(-1, 3)
21 | rays_d = points @ intrins.inverse().T @ c2w[:3,:3].T
22 | rays_o = c2w[:3,3]
23 | points = depthmap.reshape(-1, 1) * rays_d + rays_o
24 | return points
25 |
26 | def depth_to_normal(view, depth):
27 | """
28 | view: view camera
29 | depth: depthmap
30 | """
31 | points = depths_to_points(view, depth).reshape(*depth.shape[1:], 3)
32 | output = torch.zeros_like(points)
33 | dx = torch.cat([points[2:, 1:-1] - points[:-2, 1:-1]], dim=0)
34 | dy = torch.cat([points[1:-1, 2:] - points[1:-1, :-2]], dim=1)
35 | normal_map = torch.nn.functional.normalize(torch.cross(dx, dy, dim=-1), dim=-1)
36 | output[1:-1, 1:-1, :] = normal_map
37 | return output
--------------------------------------------------------------------------------
/utils/render_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2022, Google LLC
4 | # Licensed under the Apache License, Version 2.0
5 | #
6 | # The modifications of the code are under the following copyright:
7 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
8 | # TELIM research group, http://www.telecom.ulg.ac.be/
9 | # IVUL research group, https://ivul.kaust.edu.sa/
10 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
11 | # All rights reserved.
12 | # The modifications are under the LICENSE.md file.
13 | #
14 | # For inquiries contact jan.held@uliege.be
15 | #
16 |
17 | import numpy as np
18 | import os
19 | from typing import List, Mapping, Optional, Text, Tuple, Union
20 | import copy
21 | from PIL import Image
22 | import mediapy as media
23 | from matplotlib import cm
24 | from tqdm import tqdm
25 | from utils.graphics_utils import getProjectionMatrix
26 |
27 | import torch
28 |
29 | def normalize(x: np.ndarray) -> np.ndarray:
30 | """Normalization helper function."""
31 | return x / np.linalg.norm(x)
32 |
33 | def pad_poses(p: np.ndarray) -> np.ndarray:
34 | """Pad [..., 3, 4] pose matrices with a homogeneous bottom row [0,0,0,1]."""
35 | bottom = np.broadcast_to([0, 0, 0, 1.], p[..., :1, :4].shape)
36 | return np.concatenate([p[..., :3, :4], bottom], axis=-2)
37 |
38 |
39 | def unpad_poses(p: np.ndarray) -> np.ndarray:
40 | """Remove the homogeneous bottom row from [..., 4, 4] pose matrices."""
41 | return p[..., :3, :4]
42 |
43 |
44 | def recenter_poses(poses: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
45 | """Recenter poses around the origin."""
46 | cam2world = average_pose(poses)
47 | transform = np.linalg.inv(pad_poses(cam2world))
48 | poses = transform @ pad_poses(poses)
49 | return unpad_poses(poses), transform
50 |
51 |
52 | def average_pose(poses: np.ndarray) -> np.ndarray:
53 | """New pose using average position, z-axis, and up vector of input poses."""
54 | position = poses[:, :3, 3].mean(0)
55 | z_axis = poses[:, :3, 2].mean(0)
56 | up = poses[:, :3, 1].mean(0)
57 | cam2world = viewmatrix(z_axis, up, position)
58 | return cam2world
59 |
60 | def viewmatrix(lookdir: np.ndarray, up: np.ndarray,
61 | position: np.ndarray) -> np.ndarray:
62 | """Construct lookat view matrix."""
63 | vec2 = normalize(lookdir)
64 | vec0 = normalize(np.cross(up, vec2))
65 | vec1 = normalize(np.cross(vec2, vec0))
66 | m = np.stack([vec0, vec1, vec2, position], axis=1)
67 | return m
68 |
69 | def focus_point_fn(poses: np.ndarray) -> np.ndarray:
70 | """Calculate nearest point to all focal axes in poses."""
71 | directions, origins = poses[:, :3, 2:3], poses[:, :3, 3:4]
72 | m = np.eye(3) - directions * np.transpose(directions, [0, 2, 1])
73 | mt_m = np.transpose(m, [0, 2, 1]) @ m
74 | focus_pt = np.linalg.inv(mt_m.mean(0)) @ (mt_m @ origins).mean(0)[:, 0]
75 | return focus_pt
76 |
77 | def transform_poses_pca(poses: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
78 | """Transforms poses so principal components lie on XYZ axes.
79 |
80 | Args:
81 | poses: a (N, 3, 4) array containing the cameras' camera to world transforms.
82 |
83 | Returns:
84 | A tuple (poses, transform), with the transformed poses and the applied
85 | camera_to_world transforms.
86 | """
87 | t = poses[:, :3, 3]
88 | t_mean = t.mean(axis=0)
89 | t = t - t_mean
90 |
91 | eigval, eigvec = np.linalg.eig(t.T @ t)
92 | # Sort eigenvectors in order of largest to smallest eigenvalue.
93 | inds = np.argsort(eigval)[::-1]
94 | eigvec = eigvec[:, inds]
95 | rot = eigvec.T
96 | if np.linalg.det(rot) < 0:
97 | rot = np.diag(np.array([1, 1, -1])) @ rot
98 |
99 | transform = np.concatenate([rot, rot @ -t_mean[:, None]], -1)
100 | poses_recentered = unpad_poses(transform @ pad_poses(poses))
101 | transform = np.concatenate([transform, np.eye(4)[3:]], axis=0)
102 |
103 | # Flip coordinate system if z component of y-axis is negative
104 | if poses_recentered.mean(axis=0)[2, 1] < 0:
105 | poses_recentered = np.diag(np.array([1, -1, -1])) @ poses_recentered
106 | transform = np.diag(np.array([1, -1, -1, 1])) @ transform
107 |
108 | return poses_recentered, transform
109 | # points = np.random.rand(3,100)
110 | # points_h = np.concatenate((points,np.ones_like(points[:1])), axis=0)
111 | # (poses_recentered @ points_h)[0]
112 | # (transform @ pad_poses(poses) @ points_h)[0,:3]
113 | # import pdb; pdb.set_trace()
114 |
115 | # # Just make sure it's it in the [-1, 1]^3 cube
116 | # scale_factor = 1. / np.max(np.abs(poses_recentered[:, :3, 3]))
117 | # poses_recentered[:, :3, 3] *= scale_factor
118 | # transform = np.diag(np.array([scale_factor] * 3 + [1])) @ transform
119 |
120 | # return poses_recentered, transform
121 |
122 | def generate_ellipse_path(poses: np.ndarray,
123 | n_frames: int = 120,
124 | const_speed: bool = True,
125 | z_variation: float = 0.,
126 | z_phase: float = 0.) -> np.ndarray:
127 | """Generate an elliptical render path based on the given poses."""
128 | # Calculate the focal point for the path (cameras point toward this).
129 | center = focus_point_fn(poses)
130 | # Path height sits at z=0 (in middle of zero-mean capture pattern).
131 | offset = np.array([center[0], center[1], 0])
132 |
133 | # Calculate scaling for ellipse axes based on input camera positions.
134 | sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
135 | # Use ellipse that is symmetric about the focal point in xy.
136 | low = -sc + offset
137 | high = sc + offset
138 | # Optional height variation need not be symmetric
139 | z_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
140 | z_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
141 |
142 | def get_positions(theta):
143 | # Interpolate between bounds with trig functions to get ellipse in x-y.
144 | # Optionally also interpolate in z to change camera height along path.
145 | return np.stack([
146 | low[0] + (high - low)[0] * (np.cos(theta) * .5 + .5),
147 | low[1] + (high - low)[1] * (np.sin(theta) * .5 + .5),
148 | z_variation * (z_low[2] + (z_high - z_low)[2] *
149 | (np.cos(theta + 2 * np.pi * z_phase) * .5 + .5)),
150 | ], -1)
151 |
152 | theta = np.linspace(0, 2. * np.pi, n_frames + 1, endpoint=True)
153 | positions = get_positions(theta)
154 |
155 | #if const_speed:
156 |
157 | # # Resample theta angles so that the velocity is closer to constant.
158 | # lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
159 | # theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
160 | # positions = get_positions(theta)
161 |
162 | # Throw away duplicated last position.
163 | positions = positions[:-1]
164 |
165 | # Set path's up vector to axis closest to average of input pose up vectors.
166 | avg_up = poses[:, :3, 1].mean(0)
167 | avg_up = avg_up / np.linalg.norm(avg_up)
168 | ind_up = np.argmax(np.abs(avg_up))
169 | up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
170 |
171 | return np.stack([viewmatrix(p - center, up, p) for p in positions])
172 |
173 |
174 | def generate_path(viewpoint_cameras, n_frames=480):
175 | c2ws = np.array([np.linalg.inv(np.asarray((cam.world_view_transform.T).cpu().numpy())) for cam in viewpoint_cameras])
176 | pose = c2ws[:,:3,:] @ np.diag([1, -1, -1, 1])
177 | pose_recenter, colmap_to_world_transform = transform_poses_pca(pose)
178 |
179 | # generate new poses
180 | new_poses = generate_ellipse_path(poses=pose_recenter, n_frames=n_frames)
181 | # warp back to orignal scale
182 | new_poses = np.linalg.inv(colmap_to_world_transform) @ pad_poses(new_poses)
183 |
184 | traj = []
185 | for c2w in new_poses:
186 | c2w = c2w @ np.diag([1, -1, -1, 1])
187 | cam = copy.deepcopy(viewpoint_cameras[0])
188 | cam.image_height = int(cam.image_height / 2) * 2
189 | cam.image_width = int(cam.image_width / 2) * 2
190 | cam.world_view_transform = torch.from_numpy(np.linalg.inv(c2w).T).float().cuda()
191 | cam.full_proj_transform = (cam.world_view_transform.unsqueeze(0).bmm(cam.projection_matrix.unsqueeze(0))).squeeze(0)
192 | cam.camera_center = cam.world_view_transform.inverse()[3, :3]
193 | traj.append(cam)
194 |
195 | return traj
196 |
197 | def generate_zoom_trajectory(viewpoint_cameras, n_frames=480, zoom_start=0, zoom_duration=120, zoom_intensity=2.0):
198 | traj = generate_path(viewpoint_cameras, n_frames=n_frames)
199 |
200 | cam0 = viewpoint_cameras[0]
201 | orig_fovx = cam0.FoVx
202 | orig_fovy = cam0.FoVy
203 | orig_focalx = cam0.image_width / (2 * np.tan(orig_fovx / 2))
204 | orig_focaly = cam0.image_height / (2 * np.tan(orig_fovy / 2))
205 |
206 | for i, cam in enumerate(traj):
207 | cam = copy.deepcopy(cam)
208 |
209 | if zoom_start <= i < zoom_start + zoom_duration:
210 | t = (i - zoom_start) / max(zoom_duration - 1, 1)
211 | zoom_factor = 1 + t * (zoom_intensity - 1)
212 |
213 | elif zoom_start + zoom_duration <= i < zoom_start + 2 * zoom_duration:
214 | t = (i - (zoom_start + zoom_duration)) / max(zoom_duration - 1, 1)
215 | zoom_factor = zoom_intensity - t * (zoom_intensity - 1)
216 | else:
217 | zoom_factor = 1.0
218 |
219 | new_focalx = orig_focalx * zoom_factor
220 | new_focaly = orig_focaly * zoom_factor
221 | new_fovx = 2 * np.arctan(cam.image_width / (2 * new_focalx))
222 | new_fovy = 2 * np.arctan(cam.image_height / (2 * new_focaly))
223 | cam.FoVx = new_fovx
224 | cam.FoVy = new_fovy
225 |
226 | cam.projection_matrix = getProjectionMatrix(znear=cam.znear, zfar=cam.zfar, fovX=new_fovx, fovY=new_fovy).transpose(0,1).cuda()
227 | cam.full_proj_transform = (cam.world_view_transform.unsqueeze(0).bmm(cam.projection_matrix.unsqueeze(0))).squeeze(0)
228 | traj[i] = cam
229 | return traj
230 |
231 | def load_img(pth: str) -> np.ndarray:
232 | """Load an image and cast to float32."""
233 | with open(pth, 'rb') as f:
234 | image = np.array(Image.open(f), dtype=np.float32)
235 | return image
236 |
237 |
238 | def create_videos(base_dir, input_dir, out_name, num_frames=480):
239 | """Creates videos out of the images saved to disk."""
240 | # Last two parts of checkpoint path are experiment name and scene name.
241 | video_prefix = f'{out_name}'
242 |
243 | zpad = max(5, len(str(num_frames - 1)))
244 | idx_to_str = lambda idx: str(idx).zfill(zpad)
245 |
246 | os.makedirs(base_dir, exist_ok=True)
247 |
248 | img_file = os.path.join(input_dir, 'renders', f'{idx_to_str(0)}.png')
249 | img = load_img(img_file)
250 | shape = img.shape
251 |
252 | print(f'Video shape is {shape[:2]}')
253 |
254 | video_kwargs = {
255 | 'shape': shape[:2],
256 | 'codec': 'h264',
257 | 'fps': 120,
258 | 'crf': 18,
259 | }
260 |
261 | video_file = os.path.join(base_dir, f'{video_prefix}_color.mp4')
262 | input_format = 'rgb'
263 |
264 | file_ext = 'png'
265 | idx = 0
266 |
267 | file0 = os.path.join(input_dir, 'renders', f'{idx_to_str(0)}.{file_ext}')
268 |
269 | if not os.path.exists(file0):
270 | return
271 | print(f'Making video {video_file}...')
272 | with media.VideoWriter(video_file, **video_kwargs, input_format=input_format) as writer:
273 | for idx in tqdm(range(num_frames)):
274 |
275 | img_file = os.path.join(input_dir, 'renders', f'{idx_to_str(idx)}.{file_ext}')
276 |
277 | if not os.path.exists(img_file):
278 | ValueError(f'Image file {img_file} does not exist.')
279 | img = load_img(img_file)
280 | img = img / 255.
281 |
282 | frame = (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)
283 | writer.add_image(frame)
284 | idx += 1
285 |
286 | def save_img_u8(img, pth):
287 | """Save an image (probably RGB) in [0, 1] to disk as a uint8 PNG."""
288 | with open(pth, 'wb') as f:
289 | Image.fromarray(
290 | (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)).save(
291 | f, 'PNG')
292 |
293 | def save_img_f32(depthmap, pth):
294 | """Save an image (probably a depthmap) to disk as a float32 TIFF."""
295 | with open(pth, 'wb') as f:
296 | Image.fromarray(np.nan_to_num(depthmap).astype(np.float32)).save(f, 'TIFF')
297 |
--------------------------------------------------------------------------------
/utils/sh_utils.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The PlenOctree Authors.
2 | # Redistribution and use in source and binary forms, with or without
3 | # modification, are permitted provided that the following conditions are met:
4 | #
5 | # 1. Redistributions of source code must retain the above copyright notice,
6 | # this list of conditions and the following disclaimer.
7 | #
8 | # 2. Redistributions in binary form must reproduce the above copyright notice,
9 | # this list of conditions and the following disclaimer in the documentation
10 | # and/or other materials provided with the distribution.
11 | #
12 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
13 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
15 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
16 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
17 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
18 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
19 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
20 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
22 | # POSSIBILITY OF SUCH DAMAGE.
23 |
24 | import torch
25 |
26 | C0 = 0.28209479177387814
27 | C1 = 0.4886025119029199
28 | C2 = [
29 | 1.0925484305920792,
30 | -1.0925484305920792,
31 | 0.31539156525252005,
32 | -1.0925484305920792,
33 | 0.5462742152960396
34 | ]
35 | C3 = [
36 | -0.5900435899266435,
37 | 2.890611442640554,
38 | -0.4570457994644658,
39 | 0.3731763325901154,
40 | -0.4570457994644658,
41 | 1.445305721320277,
42 | -0.5900435899266435
43 | ]
44 | C4 = [
45 | 2.5033429417967046,
46 | -1.7701307697799304,
47 | 0.9461746957575601,
48 | -0.6690465435572892,
49 | 0.10578554691520431,
50 | -0.6690465435572892,
51 | 0.47308734787878004,
52 | -1.7701307697799304,
53 | 0.6258357354491761,
54 | ]
55 |
56 |
57 | def eval_sh(deg, sh, dirs):
58 | """
59 | Evaluate spherical harmonics at unit directions
60 | using hardcoded SH polynomials.
61 | Works with torch/np/jnp.
62 | ... Can be 0 or more batch dimensions.
63 | Args:
64 | deg: int SH deg. Currently, 0-3 supported
65 | sh: jnp.ndarray SH coeffs [..., C, (deg + 1) ** 2]
66 | dirs: jnp.ndarray unit directions [..., 3]
67 | Returns:
68 | [..., C]
69 | """
70 | assert deg <= 4 and deg >= 0
71 | coeff = (deg + 1) ** 2
72 | assert sh.shape[-1] >= coeff
73 |
74 | result = C0 * sh[..., 0]
75 | if deg > 0:
76 | x, y, z = dirs[..., 0:1], dirs[..., 1:2], dirs[..., 2:3]
77 | result = (result -
78 | C1 * y * sh[..., 1] +
79 | C1 * z * sh[..., 2] -
80 | C1 * x * sh[..., 3])
81 |
82 | if deg > 1:
83 | xx, yy, zz = x * x, y * y, z * z
84 | xy, yz, xz = x * y, y * z, x * z
85 | result = (result +
86 | C2[0] * xy * sh[..., 4] +
87 | C2[1] * yz * sh[..., 5] +
88 | C2[2] * (2.0 * zz - xx - yy) * sh[..., 6] +
89 | C2[3] * xz * sh[..., 7] +
90 | C2[4] * (xx - yy) * sh[..., 8])
91 |
92 | if deg > 2:
93 | result = (result +
94 | C3[0] * y * (3 * xx - yy) * sh[..., 9] +
95 | C3[1] * xy * z * sh[..., 10] +
96 | C3[2] * y * (4 * zz - xx - yy)* sh[..., 11] +
97 | C3[3] * z * (2 * zz - 3 * xx - 3 * yy) * sh[..., 12] +
98 | C3[4] * x * (4 * zz - xx - yy) * sh[..., 13] +
99 | C3[5] * z * (xx - yy) * sh[..., 14] +
100 | C3[6] * x * (xx - 3 * yy) * sh[..., 15])
101 |
102 | if deg > 3:
103 | result = (result + C4[0] * xy * (xx - yy) * sh[..., 16] +
104 | C4[1] * yz * (3 * xx - yy) * sh[..., 17] +
105 | C4[2] * xy * (7 * zz - 1) * sh[..., 18] +
106 | C4[3] * yz * (7 * zz - 3) * sh[..., 19] +
107 | C4[4] * (zz * (35 * zz - 30) + 3) * sh[..., 20] +
108 | C4[5] * xz * (7 * zz - 3) * sh[..., 21] +
109 | C4[6] * (xx - yy) * (7 * zz - 1) * sh[..., 22] +
110 | C4[7] * xz * (xx - 3 * yy) * sh[..., 23] +
111 | C4[8] * (xx * (xx - 3 * yy) - yy * (3 * xx - yy)) * sh[..., 24])
112 | return result
113 |
114 | def RGB2SH(rgb):
115 | return (rgb - 0.5) / C0
116 |
117 | def SH2RGB(sh):
118 | return sh * C0 + 0.5
--------------------------------------------------------------------------------
/utils/system_utils.py:
--------------------------------------------------------------------------------
1 | #
2 | # The original code is under the following copyright:
3 | # Copyright (C) 2023, Inria
4 | # GRAPHDECO research group, https://team.inria.fr/graphdeco
5 | # All rights reserved.
6 | #
7 | # This software is free for non-commercial, research and evaluation use
8 | # under the terms of the LICENSE_GS.md file.
9 | #
10 | # For inquiries contact george.drettakis@inria.fr
11 | #
12 | # The modifications of the code are under the following copyright:
13 | # Copyright (C) 2024, University of Liege, KAUST and University of Oxford
14 | # TELIM research group, http://www.telecom.ulg.ac.be/
15 | # IVUL research group, https://ivul.kaust.edu.sa/
16 | # VGG research group, https://www.robots.ox.ac.uk/~vgg/
17 | # All rights reserved.
18 | # The modifications are under the LICENSE.md file.
19 | #
20 | # For inquiries contact jan.held@uliege.be
21 | #
22 |
23 | from errno import EEXIST
24 | from os import makedirs, path
25 | import os
26 |
27 | def mkdir_p(folder_path):
28 | # Creates a directory. equivalent to using mkdir -p on the command line
29 | try:
30 | makedirs(folder_path)
31 | except OSError as exc: # Python >2.5
32 | if exc.errno == EEXIST and path.isdir(folder_path):
33 | pass
34 | else:
35 | raise
36 |
37 | def searchForMaxIteration(folder):
38 | saved_iters = [int(fname.split("_")[-1]) for fname in os.listdir(folder)]
39 | return max(saved_iters)
40 |
--------------------------------------------------------------------------------