├── .gitignore ├── LICENSE ├── README.md ├── assets └── bsdf_256_256.bin ├── blender_backend ├── blender_utils.py └── relight_backend.py ├── colmap ├── build.py ├── build_windows_app.py ├── bundler_to_ply.py ├── clang_format_code.py ├── crawl_camera_specs.py ├── database.py ├── export_inlier_matches.py ├── export_inlier_pairs.py ├── export_to_bundler.py ├── export_to_visualsfm.py ├── flickr_downloader.py ├── merge_ply_files.py ├── nvm_to_ply.py ├── plyfile.py ├── read_write_dense.py ├── read_write_fused_vis.py ├── read_write_model.py ├── test_read_write_dense.py ├── test_read_write_fused_vis.py ├── test_read_write_model.py └── visualize_model.py ├── configs ├── mat │ ├── custom │ │ ├── brassgourd.yaml │ │ ├── casserole.yaml │ │ ├── goldenqilin.yaml │ │ ├── luckycat.yaml │ │ └── shoe.yaml │ ├── orb │ │ ├── cactus.yaml │ │ ├── car.yaml │ │ ├── gnome.yaml │ │ ├── grogu.yaml │ │ └── teapot.yaml │ └── syn │ │ ├── FlightHelmet.yaml │ │ ├── armadillo.yaml │ │ ├── compressor.yaml │ │ ├── dragon.yaml │ │ ├── horse.yaml │ │ ├── lego.yaml │ │ ├── motor.yaml │ │ ├── robot.yaml │ │ └── rover.yaml ├── shape │ ├── custom │ │ ├── brassgourd.yaml │ │ ├── casserole.yaml │ │ ├── goldenqilin.yaml │ │ ├── luckycat.yaml │ │ └── shoe.yaml │ ├── nero │ │ ├── angel.yaml │ │ ├── bell.yaml │ │ ├── cat.yaml │ │ ├── horse.yaml │ │ ├── luyu.yaml │ │ ├── potion.yaml │ │ ├── tbell.yaml │ │ └── teapot.yaml │ ├── orb │ │ ├── cactus.yaml │ │ ├── car.yaml │ │ ├── gnome.yaml │ │ ├── grogu.yaml │ │ └── teapot.yaml │ └── syn │ │ ├── FlightHelmet.yaml │ │ ├── armadillo.yaml │ │ ├── compressor.yaml │ │ ├── dragon.yaml │ │ ├── lego.yaml │ │ ├── motor.yaml │ │ ├── robot.yaml │ │ └── rover.yaml └── synthetic_split_128.pkl ├── dataset ├── database.py ├── name2dataset.py └── train_dataset.py ├── eval_geo.py ├── eval_mat.py ├── eval_orb_relight.py ├── eval_orb_shape.py ├── extract_mesh.py ├── network ├── fields.py ├── invRenderer.py ├── loss.py ├── materialRenderer.py ├── metrics.py ├── other_field.py └── shapeRenderer.py ├── raytracing ├── __init__.py └── raytracer.py ├── relight_orb.py ├── requirements.txt ├── run_colmap.py ├── run_training.py ├── train ├── train_tools.py ├── train_valid.py └── trainer_inv.py ├── user-imgs └── teaser.png └── utils ├── base_utils.py ├── dataset_utils.py ├── draw_utils.py ├── network_utils.py ├── pose_utils.py ├── raw_utils.py ├── ref_utils.py └── rend_util.py /.gitignore: -------------------------------------------------------------------------------- 1 | */__pycache__/* 2 | /data/* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Riga2 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## TensoSDF: Roughness-aware Tensorial Representation for Robust Geometry and Material Reconstruction (SIGGRAPH 2024) 2 | 3 | ### [Paper](https://arxiv.org/abs/2402.02771) | [Project page](https://riga2.github.io/tensosdf/) 4 | 5 | ![Teaser](https://github.com/Riga2/TensoSDF/blob/main/user-imgs/teaser.png) 6 | 7 | The method is based on [NeRO](https://github.com/liuyuan-pal/NeRO), please refer to it to setup the environment. 8 | And then use pip to install the requirements.txt in this project. 9 | ``` 10 | cd TensoSDF 11 | pip install -r requirements.txt 12 | ``` 13 | 14 | ## Datasets 15 | Download the [TensoSDF synthetic dataset](https://drive.google.com/file/d/1JI2kMvi_79JIUBGbUBckxeCAgrWEW0kl/view?usp=drive_link) and the [ORB real dataset](https://stanfordorb.github.io/). For the ORB dataset, We use the *blender_LDR.tar.gz* for training and *ground_truth.tar.gz* for evaluation. 16 | 17 | ## TensoSDF synthetic dataset 18 | ### Geometry reconstruction 19 | 20 | Below take the "compressor" scene as an example: 21 | 22 | ``` 23 | # you need to modify the "dataset_dir" in configs/shape/syn/compressor.yaml first. 24 | 25 | # reconstruct the geometry 26 | python run_training.py --cfg configs/shape/syn/compressor.yaml 27 | 28 | # evaluate the geometry reconstruction results via normal MAE metric 29 | python eval_geo.py --cfg configs/shape/syn/compressor.yaml 30 | 31 | # extract mesh from the model 32 | python extract_mesh.py --cfg configs/shape/syn/compressor.yaml 33 | ``` 34 | 35 | Intermediate results will be saved at ```data/train_vis```. Models will be saved at ```data/model```. NVS results will be saved at ```data/nvs```. Extracted mesh will be saved at ```data/meshes```. 36 | 37 | ### Material reconstruction 38 | 39 | ``` 40 | # you need to modify the "dataset_dir" in configs/mat/syn/compressor.yaml first. 41 | 42 | # estimate the material 43 | python run_training.py --cfg configs/mat/syn/compressor.yaml 44 | 45 | # evaluate the relighting results using the estimated materials via PSNR, SSIM and LPIPS metrics 46 | python eval_mat.py --cfg configs/shape/syn/compressor.yaml --blender your_blender_path --env_dir your_environment_lights_dir 47 | ``` 48 | Intermediate results will be saved at ```data/train_vis```. Models will be saved at ```data/model```. Extracted materials will be saved at ```data/materials```. Relighting results will be saved at ```data/relight```. 49 | 50 | ## ORB real dataset 51 | ### Geometry reconstruction 52 | 53 | Below take the "teapot" scene as an example: 54 | 55 | ``` 56 | # you need to modify the "dataset_dir" in configs/shape/orb/teapot.yaml first. 57 | 58 | # reconstruct the geometry 59 | python run_training.py --cfg configs/shape/orb/teapot.yaml 60 | 61 | # extract mesh from the model 62 | python extract_mesh.py --cfg configs/shape/orb/teapot.yaml 63 | 64 | # evaluate the geometry reconstruction results via the CD metric 65 | python eval_orb_shape.py --out_mesh_path data/meshes/teapot_scene006_shape-180000.ply --target_mesh_path your_ORB_GT_mesh_path 66 | ``` 67 | 68 | Intermediate results will be saved at ```data/train_vis```. Models will be saved at ```data/model```. Extracted mesh will be saved at ```data/meshes```. 69 | 70 | ### Material reconstruction 71 | 72 | ``` 73 | # you need to modify the "dataset_dir" in configs/mat/orb/teapot.yaml first. 74 | 75 | # estimate the material 76 | python run_training.py --cfg configs/mat/orb/teapot.yaml 77 | 78 | # extract the materials and relight with new environment lights 79 | python eval_mat.py --cfg configs/mat/orb/teapot.yaml --blender your_blender_path --orb_relight_gt_dir your_ORB_GT_relighting_dir --orb_relight_env your_relighting_env_name --orb_blender_dir your_orb_dataset_dir 80 | 81 | # evaluate the relighting results via PSNR, SSIM and LPIPS metrics 82 | python eval_orb_relight.py --relight_dir your_relighting_results_dir --gt_dir your_GT_relighting_in_orb_dataset_dir 83 | ``` 84 | Intermediate results will be saved at ```data/train_vis```. Models will be saved at ```data/model```. Extracted materials will be saved at ```data/materials```. Relighting results will be saved at ```data/relight```. 85 | 86 | ## BibTeX 87 | ``` 88 | @article{Li:2024:TensoSDF, 89 | title={TensoSDF: Roughness-aware Tensorial Representation for Robust Geometry and Material Reconstruction}, 90 | author={Jia Li and Lu Wang and Lei Zhang and Beibei Wang}, 91 | journal ={ACM Transactions on Graphics (Proceedings of SIGGRAPH 2024)}, 92 | year = {2024}, 93 | volume = {43}, 94 | number = {4}, 95 | pages={150:1--13} 96 | } 97 | ``` 98 | -------------------------------------------------------------------------------- /assets/bsdf_256_256.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Riga2/TensoSDF/6074ab11af1bfe9c50a0d30c9630c4a083289b8b/assets/bsdf_256_256.bin -------------------------------------------------------------------------------- /colmap/build_windows_app.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import os 33 | import glob 34 | import shutil 35 | import argparse 36 | 37 | 38 | def parse_args(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--install_path", required=True, 41 | help="The installation prefix, e.g., build/__install__") 42 | parser.add_argument("--app_path", required=True, 43 | help="The application path, e.g., " 44 | "build/COLMAP-dev-windows") 45 | args = parser.parse_args() 46 | return args 47 | 48 | 49 | def mkdir_if_not_exists(path): 50 | assert os.path.exists(os.path.dirname(os.path.abspath(path))) 51 | if not os.path.exists(path): 52 | os.makedirs(path) 53 | 54 | 55 | def main(): 56 | args = parse_args() 57 | 58 | mkdir_if_not_exists(args.app_path) 59 | mkdir_if_not_exists(os.path.join(args.app_path, "bin")) 60 | mkdir_if_not_exists(os.path.join(args.app_path, "lib")) 61 | mkdir_if_not_exists(os.path.join(args.app_path, "lib/platforms")) 62 | 63 | # Copy batch scripts to app directory. 64 | shutil.copyfile( 65 | os.path.join(args.install_path, "COLMAP.bat"), 66 | os.path.join(args.app_path, "COLMAP.bat")) 67 | shutil.copyfile( 68 | os.path.join(args.install_path, "RUN_TESTS.bat"), 69 | os.path.join(args.app_path, "RUN_TESTS.bat")) 70 | 71 | # Copy executables to app directory. 72 | exe_files = glob.glob(os.path.join(args.install_path, "bin/*.exe")) 73 | for exe_file in exe_files: 74 | shutil.copyfile(exe_file, os.path.join(args.app_path, "bin", 75 | os.path.basename(exe_file))) 76 | 77 | # Copy shared libraries to app directory. 78 | dll_files = glob.glob(os.path.join(args.install_path, "lib/*.dll")) 79 | for dll_file in dll_files: 80 | shutil.copyfile(dll_file, os.path.join(args.app_path, "lib", 81 | os.path.basename(dll_file))) 82 | shutil.copyfile( 83 | os.path.join(args.install_path, "lib/platforms/qwindows.dll"), 84 | os.path.join(args.app_path, "lib/platforms/qwindows.dll")) 85 | 86 | # Create zip archive for deployment. 87 | shutil.make_archive(args.app_path, "zip", root_dir=args.app_path) 88 | 89 | 90 | if __name__ == "__main__": 91 | main() 92 | -------------------------------------------------------------------------------- /colmap/bundler_to_ply.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script converts a Bundler reconstruction file to a PLY point cloud. 33 | 34 | import argparse 35 | import numpy as np 36 | 37 | 38 | def parse_args(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--bundler_path", required=True) 41 | parser.add_argument("--ply_path", required=True) 42 | parser.add_argument("--normalize", type=bool, default=True) 43 | parser.add_argument("--normalize_p0", type=float, default=0.2) 44 | parser.add_argument("--normalize_p1", type=float, default=0.8) 45 | parser.add_argument("--min_track_length", type=int, default=3) 46 | args = parser.parse_args() 47 | return args 48 | 49 | 50 | def main(): 51 | args = parse_args() 52 | 53 | with open(args.bundler_path, "r") as fid: 54 | line = fid.readline() 55 | line = fid.readline() 56 | num_images, num_points = map(int, line.split()) 57 | 58 | for i in range(5 * num_images): 59 | fid.readline() 60 | 61 | xyz = np.zeros((num_points, 3), dtype=np.float64) 62 | rgb = np.zeros((num_points, 3), dtype=np.uint16) 63 | track_lengths = np.zeros((num_points,), dtype=np.uint32) 64 | 65 | for i in range(num_points): 66 | if i % 1000 == 0: 67 | print("Reading point", i, "/", num_points) 68 | xyz[i] = map(float, fid.readline().split()) 69 | rgb[i] = map(int, fid.readline().split()) 70 | track_lengths[i] = int(fid.readline().split()[0]) 71 | 72 | mask = track_lengths >= args.min_track_length 73 | xyz = xyz[mask] 74 | rgb = rgb[mask] 75 | 76 | if args.normalize: 77 | sorted_x = np.sort(xyz[:, 0]) 78 | sorted_y = np.sort(xyz[:, 1]) 79 | sorted_z = np.sort(xyz[:, 2]) 80 | 81 | num_coords = sorted_x.size 82 | min_coord = int(args.normalize_p0 * num_coords) 83 | max_coord = int(args.normalize_p1 * num_coords) 84 | mean_coords = xyz.mean(0) 85 | 86 | bbox_min = np.array([sorted_x[min_coord], sorted_y[min_coord], 87 | sorted_z[min_coord]]) 88 | bbox_max = np.array([sorted_x[max_coord], sorted_y[max_coord], 89 | sorted_z[max_coord]]) 90 | 91 | extent = np.linalg.norm(bbox_max - bbox_min) 92 | scale = 10.0 / extent 93 | 94 | xyz -= mean_coords 95 | xyz *= scale 96 | 97 | xyz[:, 2] *= -1 98 | 99 | with open(args.ply_path, "w") as fid: 100 | fid.write("ply\n") 101 | fid.write("format ascii 1.0\n") 102 | fid.write("element vertex %d\n" % xyz.shape[0]) 103 | fid.write("property float x\n") 104 | fid.write("property float y\n") 105 | fid.write("property float z\n") 106 | fid.write("property float nx\n") 107 | fid.write("property float ny\n") 108 | fid.write("property float nz\n") 109 | fid.write("property uchar diffuse_red\n") 110 | fid.write("property uchar diffuse_green\n") 111 | fid.write("property uchar diffuse_blue\n") 112 | fid.write("end_header\n") 113 | for i in range(xyz.shape[0]): 114 | if i % 1000 == 0: 115 | print("Writing point", i, "/", xyz.shape[0]) 116 | fid.write("%f %f %f 0 0 0 %d %d %d\n" % (xyz[i, 0], xyz[i, 1], 117 | xyz[i, 2], rgb[i, 0], 118 | rgb[i, 1], rgb[i, 2])) 119 | 120 | 121 | if __name__ == "__main__": 122 | main() 123 | -------------------------------------------------------------------------------- /colmap/clang_format_code.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import os 33 | import string 34 | import argparse 35 | import subprocess 36 | 37 | 38 | def parse_args(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--path", required=True) 41 | parser.add_argument("--exts", default=".h,.cc") 42 | parser.add_argument("--style", default="File") 43 | args = parser.parse_args() 44 | return args 45 | 46 | 47 | def main(): 48 | args = parse_args() 49 | 50 | exts = map(string.lower, args.exts.split(",")) 51 | 52 | for root, subdirs, files in os.walk(args.path): 53 | for f in files: 54 | name, ext = os.path.splitext(f) 55 | if ext.lower() in exts: 56 | file_path = os.path.join(root, f) 57 | proc = subprocess.Popen(["clang-format", "--style", 58 | args.style, file_path], 59 | stdout=subprocess.PIPE) 60 | 61 | text = "".join(proc.stdout) 62 | 63 | with open(file_path, "w") as fd: 64 | fd.write(text) 65 | 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /colmap/crawl_camera_specs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import re 33 | import argparse 34 | import requests 35 | from lxml.html import soupparser 36 | 37 | 38 | MAX_REQUEST_TRIALS = 10 39 | 40 | 41 | def parse_args(): 42 | parser = argparse.ArgumentParser() 43 | parser.add_argument("--lib_path", required=True) 44 | args = parser.parse_args() 45 | return args 46 | 47 | 48 | def request_trial(func, *args, **kwargs): 49 | for i in range(MAX_REQUEST_TRIALS): 50 | try: 51 | response = func(*args, **kwargs) 52 | except: 53 | continue 54 | else: 55 | return response 56 | 57 | raise SystemError 58 | 59 | 60 | def main(): 61 | args = parse_args() 62 | 63 | ########################################################################## 64 | # Header file 65 | ########################################################################## 66 | 67 | with open(args.lib_path + ".h", "w") as f: 68 | f.write("#include \n") 69 | f.write("#include \n") 70 | f.write("#include \n\n") 71 | f.write("// { make1 : ({ model1 : sensor-width in mm }, ...), ... }\n") 72 | f.write("typedef std::vector> make_specs_t;\n") 73 | f.write("typedef std::unordered_map camera_specs_t;;\n\n") 74 | f.write("camera_specs_t InitializeCameraSpecs();\n\n") 75 | 76 | ########################################################################## 77 | # Source file 78 | ########################################################################## 79 | 80 | makes_response = requests.get("http://www.digicamdb.com") 81 | makes_tree = soupparser.fromstring(makes_response.text) 82 | makes_node = makes_tree.find(".//select[@id=\"select_brand\"]") 83 | makes = [b.attrib["value"] for b in makes_node.iter("option")] 84 | 85 | with open(args.lib_path + ".cc", "w") as f: 86 | f.write("camera_specs_t InitializeCameraSpecs() {\n") 87 | f.write(" camera_specs_t specs;\n\n") 88 | for make in makes: 89 | f.write(" {\n") 90 | f.write(" auto& make_specs = specs[\"%s\"];\n" % make.lower().replace(" ", "")) 91 | 92 | models_response = request_trial( 93 | requests.post, 94 | "http://www.digicamdb.com/inc/ajax.php", 95 | data={"b": make, "role": "header_search"}) 96 | 97 | models_tree = soupparser.fromstring(models_response.text) 98 | models_code = "" 99 | num_models = 0 100 | for model_node in models_tree.iter("option"): 101 | model = model_node.attrib.get("value") 102 | model_name = model_node.text 103 | if model is None: 104 | continue 105 | 106 | url = "http://www.digicamdb.com/specs/{0}_{1}" \ 107 | .format(make, model) 108 | specs_response = request_trial(requests.get, url) 109 | 110 | specs_tree = soupparser.fromstring(specs_response.text) 111 | for spec in specs_tree.findall(".//td[@class=\"info_key\"]"): 112 | if spec.text.strip() == "Sensor:": 113 | sensor_text = spec.find("..").find("./td[@class=\"bold\"]") 114 | sensor_text = sensor_text.text.strip() 115 | m = re.match(".*?([\d.]+) x ([\d.]+).*?", sensor_text) 116 | sensor_width = m.group(1) 117 | data = (model_name.lower().replace(" ", ""), 118 | float(sensor_width.replace(" ", ""))) 119 | models_code += " make_specs.emplace_back(\"%s\", %.4ff);\n" % data 120 | 121 | print(make, model_name) 122 | print(" ", sensor_text) 123 | 124 | num_models += 1 125 | 126 | f.write(" make_specs.reserve(%d);\n" % num_models) 127 | f.write(models_code) 128 | f.write(" }\n\n") 129 | 130 | f.write(" return specs;\n") 131 | f.write("}\n") 132 | 133 | 134 | if __name__ == "__main__": 135 | main() 136 | -------------------------------------------------------------------------------- /colmap/export_inlier_matches.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script exports inlier matches from a COLMAP database to a text file. 33 | 34 | import os 35 | import argparse 36 | import sqlite3 37 | import numpy as np 38 | 39 | 40 | def parse_args(): 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--database_path", required=True) 43 | parser.add_argument("--output_path", required=True) 44 | parser.add_argument("--min_num_matches", type=int, default=15) 45 | args = parser.parse_args() 46 | return args 47 | 48 | 49 | def pair_id_to_image_ids(pair_id): 50 | image_id2 = pair_id % 2147483647 51 | image_id1 = (pair_id - image_id2) / 2147483647 52 | return image_id1, image_id2 53 | 54 | 55 | def main(): 56 | args = parse_args() 57 | 58 | connection = sqlite3.connect(args.database_path) 59 | cursor = connection.cursor() 60 | 61 | images = {} 62 | cursor.execute("SELECT image_id, camera_id, name FROM images;") 63 | for row in cursor: 64 | image_id = row[0] 65 | image_name = row[2] 66 | images[image_id] = image_name 67 | 68 | with open(os.path.join(args.output_path), "w") as fid: 69 | cursor.execute( 70 | "SELECT pair_id, data FROM two_view_geometries WHERE rows>=?;", 71 | (args.min_num_matches,)) 72 | for row in cursor: 73 | pair_id = row[0] 74 | inlier_matches = np.fromstring(row[1], 75 | dtype=np.uint32).reshape(-1, 2) 76 | image_id1, image_id2 = pair_id_to_image_ids(pair_id) 77 | image_name1 = images[image_id1] 78 | image_name2 = images[image_id2] 79 | fid.write("%s %s %d\n" % (image_name1, image_name2, 80 | inlier_matches.shape[0])) 81 | for i in range(inlier_matches.shape[0]): 82 | fid.write("%d %d\n" % tuple(inlier_matches[i])) 83 | 84 | cursor.close() 85 | connection.close() 86 | 87 | 88 | if __name__ == "__main__": 89 | main() 90 | -------------------------------------------------------------------------------- /colmap/export_inlier_pairs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script exports inlier image pairs from a COLMAP database to a text file. 33 | 34 | import sqlite3 35 | import argparse 36 | 37 | 38 | def parse_args(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--database_path", required=True) 41 | parser.add_argument("--match_list_path", required=True) 42 | parser.add_argument("--min_num_matches", type=int, default=15) 43 | args = parser.parse_args() 44 | return args 45 | 46 | 47 | def pair_id_to_image_ids(pair_id): 48 | image_id2 = pair_id % 2147483647 49 | image_id1 = (pair_id - image_id2) / 2147483647 50 | return image_id1, image_id2 51 | 52 | 53 | def main(): 54 | args = parse_args() 55 | 56 | connection = sqlite3.connect(args.database_path) 57 | cursor = connection.cursor() 58 | 59 | # Get a mapping between image ids and image names 60 | image_id_to_name = dict() 61 | cursor.execute('SELECT image_id, name FROM images;') 62 | for row in cursor: 63 | image_id = row[0] 64 | name = row[1] 65 | image_id_to_name[image_id] = name 66 | 67 | # Iterate over entries in the two_view_geometries table 68 | output = open(args.match_list_path, 'w') 69 | cursor.execute('SELECT pair_id, rows FROM two_view_geometries;') 70 | for row in cursor: 71 | pair_id = row[0] 72 | rows = row[1] 73 | 74 | if rows < args.min_num_matches: 75 | continue 76 | 77 | image_id1, image_id2 = pair_id_to_image_ids(pair_id) 78 | image_name1 = image_id_to_name[image_id1] 79 | image_name2 = image_id_to_name[image_id2] 80 | 81 | output.write("%s %s\n" % (image_name1, image_name2)) 82 | 83 | output.close() 84 | cursor.close() 85 | connection.close() 86 | 87 | 88 | if __name__ == "__main__": 89 | main() 90 | -------------------------------------------------------------------------------- /colmap/export_to_bundler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script exports a COLMAP database to the file structure to run Bundler. 33 | 34 | import os 35 | import argparse 36 | import sqlite3 37 | import shutil 38 | import gzip 39 | import numpy as np 40 | 41 | 42 | def parse_args(): 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument("--database_path", required=True) 45 | parser.add_argument("--image_path", required=True) 46 | parser.add_argument("--output_path", required=True) 47 | parser.add_argument("--min_num_matches", type=int, default=15) 48 | args = parser.parse_args() 49 | return args 50 | 51 | 52 | def pair_id_to_image_ids(pair_id): 53 | image_id2 = pair_id % 2147483647 54 | image_id1 = (pair_id - image_id2) / 2147483647 55 | return image_id1, image_id2 56 | 57 | 58 | def main(): 59 | args = parse_args() 60 | 61 | connection = sqlite3.connect(args.database_path) 62 | cursor = connection.cursor() 63 | 64 | try: 65 | os.makedirs(args.output_path) 66 | except: 67 | pass 68 | 69 | cameras = {} 70 | cursor.execute("SELECT camera_id, params FROM cameras;") 71 | for row in cursor: 72 | camera_id = row[0] 73 | params = np.fromstring(row[1], dtype=np.double) 74 | cameras[camera_id] = params 75 | 76 | images = {} 77 | with open(os.path.join(args.output_path, "list.txt"), "w") as fid: 78 | cursor.execute("SELECT image_id, camera_id, name FROM images;") 79 | for row in cursor: 80 | image_id = row[0] 81 | camera_id = row[1] 82 | image_name = row[2] 83 | print("Copying image", image_name) 84 | images[image_id] = (len(images), image_name) 85 | fid.write("./%s 0 %f\n" % (image_name, cameras[camera_id][0])) 86 | if not os.path.exists(os.path.join(args.output_path, image_name)): 87 | shutil.copyfile(os.path.join(args.image_path, image_name), 88 | os.path.join(args.output_path, image_name)) 89 | 90 | for image_id, (image_idx, image_name) in images.iteritems(): 91 | print("Exporting key file for", image_name) 92 | base_name, ext = os.path.splitext(image_name) 93 | key_file_name = os.path.join(args.output_path, base_name + ".key") 94 | key_file_name_gz = key_file_name + ".gz" 95 | if os.path.exists(key_file_name_gz): 96 | continue 97 | 98 | cursor.execute("SELECT data FROM keypoints WHERE image_id=?;", 99 | (image_id,)) 100 | row = next(cursor) 101 | if row[0] is None: 102 | keypoints = np.zeros((0, 6), dtype=np.float32) 103 | descriptors = np.zeros((0, 128), dtype=np.uint8) 104 | else: 105 | keypoints = np.fromstring(row[0], dtype=np.float32).reshape(-1, 6) 106 | cursor.execute("SELECT data FROM descriptors WHERE image_id=?;", 107 | (image_id,)) 108 | row = next(cursor) 109 | descriptors = np.fromstring(row[0], dtype=np.uint8).reshape(-1, 128) 110 | 111 | with open(key_file_name, "w") as fid: 112 | fid.write("%d %d\n" % (keypoints.shape[0], descriptors.shape[1])) 113 | for r in range(keypoints.shape[0]): 114 | fid.write("%f %f %f %f\n" % (keypoints[r, 1], keypoints[r, 0], 115 | keypoints[r, 2], keypoints[r, 3])) 116 | for i in range(0, 128, 20): 117 | desc_block = descriptors[r, i:i+20] 118 | fid.write(" ".join(map(str, desc_block.ravel().tolist()))) 119 | fid.write("\n") 120 | 121 | with open(key_file_name, "rb") as fid_in: 122 | with gzip.open(key_file_name + ".gz", "wb") as fid_out: 123 | fid_out.writelines(fid_in) 124 | 125 | os.remove(key_file_name) 126 | 127 | with open(os.path.join(args.output_path, "matches.init.txt"), "w") as fid: 128 | cursor.execute("SELECT pair_id, data FROM two_view_geometries " 129 | "WHERE rows>=?;", (args.min_num_matches,)) 130 | for row in cursor: 131 | pair_id = row[0] 132 | inlier_matches = np.fromstring(row[1], 133 | dtype=np.uint32).reshape(-1, 2) 134 | image_id1, image_id2 = pair_id_to_image_ids(pair_id) 135 | image_idx1 = images[image_id1][0] 136 | image_idx2 = images[image_id2][0] 137 | fid.write("%d %d\n%d\n" % (image_idx1, image_idx2, 138 | inlier_matches.shape[0])) 139 | for i in range(inlier_matches.shape[0]): 140 | fid.write("%d %d\n" % (inlier_matches[i, 0], 141 | inlier_matches[i, 1])) 142 | 143 | with open(os.path.join(args.output_path, "run_bundler.sh"), "w") as fid: 144 | fid.write("bin/Bundler list.txt \\\n") 145 | fid.write("--run_bundle \\\n") 146 | fid.write("--use_focal_estimate \\\n") 147 | fid.write("--output_all bundle_ \\\n") 148 | fid.write("--constrain_focal \\\n") 149 | fid.write("--estimate_distortion \\\n") 150 | fid.write("--match_table matches.init.txt \\\n") 151 | fid.write("--variable_focal_length \\\n") 152 | fid.write("--output_dir bundle \\\n") 153 | fid.write("--output bundle.out \\\n") 154 | fid.write("--constrain_focal_weight 0.0001 \\\n") 155 | 156 | cursor.close() 157 | connection.close() 158 | 159 | 160 | if __name__ == "__main__": 161 | main() 162 | -------------------------------------------------------------------------------- /colmap/export_to_visualsfm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script exports a COLMAP database to the file structure to run VisualSfM. 33 | 34 | import os 35 | import sys 36 | import argparse 37 | import sqlite3 38 | import shutil 39 | import gzip 40 | import numpy as np 41 | 42 | 43 | def parse_args(): 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument("--database_path", required=True) 46 | parser.add_argument("--image_path", required=True) 47 | parser.add_argument("--output_path", required=True) 48 | parser.add_argument("--min_num_matches", type=int, default=15) 49 | parser.add_argument("--binary_feature_files", type=bool, default=True) 50 | args = parser.parse_args() 51 | return args 52 | 53 | 54 | def pair_id_to_image_ids(pair_id): 55 | image_id2 = pair_id % 2147483647 56 | image_id1 = (pair_id - image_id2) / 2147483647 57 | return image_id1, image_id2 58 | 59 | 60 | def main(): 61 | args = parse_args() 62 | 63 | connection = sqlite3.connect(args.database_path) 64 | cursor = connection.cursor() 65 | 66 | try: 67 | os.makedirs(args.output_path) 68 | except: 69 | pass 70 | 71 | cameras = {} 72 | cursor.execute("SELECT camera_id, params FROM cameras;") 73 | for row in cursor: 74 | camera_id = row[0] 75 | params = np.fromstring(row[1], dtype=np.double) 76 | cameras[camera_id] = params 77 | 78 | images = {} 79 | cursor.execute("SELECT image_id, camera_id, name FROM images;") 80 | for row in cursor: 81 | image_id = row[0] 82 | camera_id = row[1] 83 | image_name = row[2] 84 | print("Copying image", image_name) 85 | images[image_id] = (len(images), image_name) 86 | if not os.path.exists(os.path.join(args.output_path, image_name)): 87 | shutil.copyfile(os.path.join(args.image_path, image_name), 88 | os.path.join(args.output_path, image_name)) 89 | 90 | # The magic numbers used in VisualSfM's binary file format for storing the 91 | # feature descriptors. 92 | sift_name = 1413892435 93 | sift_version_v4 = 808334422 94 | sift_eof_marker = 1179600383 95 | 96 | for image_id, (image_idx, image_name) in images.iteritems(): 97 | print("Exporting key file for", image_name) 98 | base_name, ext = os.path.splitext(image_name) 99 | key_file_name = os.path.join(args.output_path, base_name + ".sift") 100 | if os.path.exists(key_file_name): 101 | continue 102 | 103 | cursor.execute("SELECT data FROM keypoints WHERE image_id=?;", 104 | (image_id,)) 105 | row = next(cursor) 106 | if row[0] is None: 107 | keypoints = np.zeros((0, 6), dtype=np.float32) 108 | descriptors = np.zeros((0, 128), dtype=np.uint8) 109 | else: 110 | keypoints = np.fromstring(row[0], dtype=np.float32).reshape(-1, 6) 111 | cursor.execute("SELECT data FROM descriptors WHERE image_id=?;", 112 | (image_id,)) 113 | row = next(cursor) 114 | descriptors = np.fromstring(row[0], dtype=np.uint8).reshape(-1, 128) 115 | 116 | if args.binary_feature_files: 117 | with open(key_file_name, "wb") as fid: 118 | fid.write(struct.pack("i", sift_name)) 119 | fid.write(struct.pack("i", sift_version_v4)) 120 | fid.write(struct.pack("i", keypoints.shape[0])) 121 | fid.write(struct.pack("i", 4)) 122 | fid.write(struct.pack("i", 128)) 123 | keypoints[:, :4].astype(np.float32).tofile(fid) 124 | descriptors.astype(np.uint8).tofile(fid) 125 | fid.write(struct.pack("i", sift_eof_marker)) 126 | else: 127 | with open(key_file_name, "w") as fid: 128 | fid.write("%d %d\n" % (keypoints.shape[0], 129 | descriptors.shape[1])) 130 | for r in range(keypoints.shape[0]): 131 | fid.write("%f %f 0 0 " % (keypoints[r, 0], 132 | keypoints[r, 1])) 133 | fid.write(" ".join(map(str, 134 | descriptors[r].ravel().tolist()))) 135 | fid.write("\n") 136 | 137 | with open(os.path.join(args.output_path, "matches.txt"), "w") as fid: 138 | cursor.execute("SELECT pair_id, data FROM two_view_geometries " 139 | "WHERE rows>=?;", (args.min_num_matches,)) 140 | for row in cursor: 141 | pair_id = row[0] 142 | inlier_matches = np.fromstring(row[1], 143 | dtype=np.uint32).reshape(-1, 2) 144 | image_id1, image_id2 = pair_id_to_image_ids(pair_id) 145 | image_name1 = images[image_id1][1] 146 | image_name2 = images[image_id2][1] 147 | fid.write("%s %s %d\n" % (image_name1, image_name2, 148 | inlier_matches.shape[0])) 149 | line1 = "" 150 | line2 = "" 151 | for i in range(inlier_matches.shape[0]): 152 | line1 += "%d " % inlier_matches[i, 0] 153 | line2 += "%d " % inlier_matches[i, 1] 154 | fid.write(line1 + "\n") 155 | fid.write(line2 + "\n") 156 | 157 | cursor.close() 158 | connection.close() 159 | 160 | 161 | if __name__ == "__main__": 162 | main() 163 | -------------------------------------------------------------------------------- /colmap/flickr_downloader.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import os 33 | import time 34 | import datetime 35 | import urllib 36 | import urllib2 37 | import socket 38 | import argparse 39 | import multiprocessing 40 | import xml.etree.ElementTree as ElementTree 41 | 42 | 43 | PER_PAGE = 500 44 | SORT = "date-posted-desc" 45 | URL = "https://api.flickr.com/services/rest/?method=flickr.photos.search&" \ 46 | "api_key=%s&text=%s&sort=%s&per_page=%d&page=%d&min_upload_date=%s&" \ 47 | "max_upload_date=%s&format=rest&extras=url_o,url_l,url_c,url_z,url_n" 48 | MAX_PAGE_REQUESTS = 5 49 | MAX_PAGE_TIMEOUT = 20 50 | MAX_IMAGE_REQUESTS = 3 51 | TIME_SKIP = 24 * 60 * 60 52 | MAX_DATE = time.time() 53 | MIN_DATE = MAX_DATE - TIME_SKIP 54 | 55 | 56 | def parse_args(): 57 | parser = argparse.ArgumentParser() 58 | parser.add_argument("--search_text", required=True) 59 | parser.add_argument("--api_key", required=True) 60 | parser.add_argument("--image_path", required=True) 61 | parser.add_argument("--num_procs", type=int, default=10) 62 | parser.add_argument("--max_days_without_image", type=int, default=365) 63 | args = parser.parse_args() 64 | return args 65 | 66 | 67 | def compose_url(page, api_key, text, min_date, max_date): 68 | return URL % (api_key, text, SORT, PER_PAGE, page, 69 | str(min_date), str(max_date)) 70 | 71 | 72 | def parse_page(page, api_key, text, min_date, max_date): 73 | f = None 74 | for _ in range(MAX_PAGE_REQUESTS): 75 | try: 76 | f = urllib2.urlopen(compose_url(page, api_key, text, min_date, 77 | max_date), timeout=MAX_PAGE_TIMEOUT) 78 | except socket.timeout: 79 | continue 80 | else: 81 | break 82 | 83 | if f is None: 84 | return {'pages': '0', 85 | 'total': '0', 86 | 'page': '0', 87 | 'perpage': '0'}, tuple() 88 | 89 | response = f.read() 90 | root = ElementTree.fromstring(response) 91 | 92 | if root.attrib["stat"] != "ok": 93 | raise IOError 94 | 95 | photos = [] 96 | for photo in root.iter("photo"): 97 | photos.append(photo.attrib) 98 | 99 | return root.find("photos").attrib, photos 100 | 101 | 102 | class PhotoDownloader(object): 103 | 104 | def __init__(self, image_path): 105 | self.image_path = image_path 106 | 107 | def __call__(self, photo): 108 | image_name = "%s_%s.jpg" % (photo["id"], photo["secret"]) 109 | path = os.path.join(self.image_path, image_name) 110 | if not os.path.exists(path): 111 | url = None 112 | for url_suffix in ("o", "l", "k", "h", "b", "c", "z"): 113 | url_attr = "url_%s" % url_suffix 114 | if photo.get(url_attr) is not None: 115 | url = photo.get(url_attr) 116 | break 117 | if url is not None: 118 | print(url) 119 | for _ in range(MAX_IMAGE_REQUESTS): 120 | try: 121 | urllib.urlretrieve(url, path) 122 | except urllib.ContentTooShortError: 123 | continue 124 | else: 125 | break 126 | 127 | 128 | def main(): 129 | args = parse_args() 130 | 131 | downloader = PhotoDownloader(args.image_path) 132 | pool = multiprocessing.Pool(processes=args.num_procs) 133 | 134 | num_pages = float("inf") 135 | page = 0 136 | 137 | min_date = MIN_DATE 138 | max_date = MAX_DATE 139 | 140 | days_in_row = 0; 141 | 142 | search_text = args.search_text.replace(" ", "-") 143 | 144 | while num_pages > page: 145 | page += 1 146 | 147 | metadata, photos = parse_page(page, args.api_key, search_text, 148 | min_date, max_date) 149 | 150 | num_pages = int(metadata["pages"]) 151 | 152 | print(78 * "=") 153 | print("Page:\t\t", page, "of", num_pages) 154 | print("Min-Date:\t", datetime.datetime.fromtimestamp(min_date)) 155 | print("Max-Date:\t", datetime.datetime.fromtimestamp(max_date)) 156 | print("Num-Photos:\t", len(photos)) 157 | print(78 * "=") 158 | 159 | try: 160 | pool.map_async(downloader, photos).get(1e10) 161 | except KeyboardInterrupt: 162 | pool.wait() 163 | break 164 | 165 | if page >= num_pages: 166 | max_date -= TIME_SKIP 167 | min_date -= TIME_SKIP 168 | page = 0 169 | 170 | if num_pages == 0: 171 | days_in_row = days_in_row + 1 172 | num_pages = float("inf") 173 | 174 | print(" No images in", days_in_row, "days in a row") 175 | 176 | if days_in_row == args.max_days_without_image: 177 | break 178 | else: 179 | days_in_row = 0 180 | 181 | 182 | if __name__ == "__main__": 183 | main() 184 | -------------------------------------------------------------------------------- /colmap/merge_ply_files.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script merges multiple homogeneous PLY files into a single PLY file. 33 | 34 | import os 35 | import glob 36 | import argparse 37 | import numpy as np 38 | import plyfile 39 | 40 | 41 | def parse_args(): 42 | parser = argparse.ArgumentParser() 43 | parser.add_argument("--folder_path", required=True) 44 | parser.add_argument("--merged_path", required=True) 45 | args = parser.parse_args() 46 | return args 47 | 48 | 49 | def main(): 50 | args = parse_args() 51 | 52 | files = [] 53 | for file_name in os.listdir(args.folder_path): 54 | if len(file_name) < 4 or file_name[-4:].lower() != ".ply": 55 | continue 56 | 57 | print("Reading file", file_name) 58 | file = plyfile.PlyData.read(os.path.join(args.folder_path, file_name)) 59 | for element in file.elements: 60 | files.append(element.data) 61 | 62 | print("Merging files") 63 | merged_file = np.concatenate(files, -1) 64 | merged_el = plyfile.PlyElement.describe(merged_file, 'vertex') 65 | 66 | print("Writing merged file") 67 | plyfile.PlyData([merged_el]).write(args.merged_path) 68 | 69 | 70 | if __name__ == '__main__': 71 | main() 72 | -------------------------------------------------------------------------------- /colmap/nvm_to_ply.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | # This script converts a VisualSfM reconstruction file to a PLY point cloud. 33 | 34 | import os 35 | import argparse 36 | import numpy as np 37 | 38 | 39 | def parse_args(): 40 | parser = argparse.ArgumentParser() 41 | parser.add_argument("--nvm_path", required=True) 42 | parser.add_argument("--ply_path", required=True) 43 | parser.add_argument("--normalize", type=bool, default=True) 44 | parser.add_argument("--normalize_p0", type=float, default=0.2) 45 | parser.add_argument("--normalize_p1", type=float, default=0.8) 46 | parser.add_argument("--min_track_length", type=int, default=3) 47 | args = parser.parse_args() 48 | return args 49 | 50 | 51 | def main(): 52 | args = parse_args() 53 | 54 | with open(args.nvm_path, "r") as fid: 55 | line = fid.readline() 56 | line = fid.readline() 57 | num_images = int(fid.readline()) 58 | 59 | for i in range(num_images + 1): 60 | fid.readline() 61 | 62 | num_points = int(fid.readline()) 63 | 64 | xyz = np.zeros((num_points, 3), dtype=np.float64) 65 | rgb = np.zeros((num_points, 3), dtype=np.uint16) 66 | track_lengths = np.zeros((num_points,), dtype=np.uint32) 67 | 68 | for i in range(num_points): 69 | if i % 1000 == 0: 70 | print("Reading point", i, "/", num_points) 71 | elems = fid.readline().split() 72 | xyz[i] = map(float, elems[0:3]) 73 | rgb[i] = map(int, elems[3:6]) 74 | track_lengths[i] = int(elems[6]) 75 | 76 | mask = track_lengths >= args.min_track_length 77 | xyz = xyz[mask] 78 | rgb = rgb[mask] 79 | 80 | if args.normalize: 81 | sorted_x = np.sort(xyz[:, 0]) 82 | sorted_y = np.sort(xyz[:, 1]) 83 | sorted_z = np.sort(xyz[:, 2]) 84 | 85 | num_coords = sorted_x.size 86 | min_coord = int(args.normalize_p0 * num_coords) 87 | max_coord = int(args.normalize_p1 * num_coords) 88 | mean_coords = xyz.mean(0) 89 | 90 | bbox_min = np.array([sorted_x[min_coord], sorted_y[min_coord], 91 | sorted_z[min_coord]]) 92 | bbox_max = np.array([sorted_x[max_coord], sorted_y[max_coord], 93 | sorted_z[max_coord]]) 94 | 95 | extent = np.linalg.norm(bbox_max - bbox_min) 96 | scale = 10.0 / extent 97 | 98 | xyz -= mean_coords 99 | xyz *= scale 100 | 101 | with open(args.ply_path, "w") as fid: 102 | fid.write("ply\n") 103 | fid.write("format ascii 1.0\n") 104 | fid.write("element vertex %d\n" % xyz.shape[0]) 105 | fid.write("property float x\n") 106 | fid.write("property float y\n") 107 | fid.write("property float z\n") 108 | fid.write("property float nx\n") 109 | fid.write("property float ny\n") 110 | fid.write("property float nz\n") 111 | fid.write("property uchar diffuse_red\n") 112 | fid.write("property uchar diffuse_green\n") 113 | fid.write("property uchar diffuse_blue\n") 114 | fid.write("end_header\n") 115 | for i in range(xyz.shape[0]): 116 | if i % 1000 == 0: 117 | print("Writing point", i, "/", xyz.shape[0]) 118 | fid.write("%f %f %f 0 0 0 %d %d %d\n" % (xyz[i, 0], xyz[i, 1], 119 | xyz[i, 2], rgb[i, 0], 120 | rgb[i, 1], rgb[i, 2])) 121 | 122 | 123 | if __name__ == "__main__": 124 | main() 125 | -------------------------------------------------------------------------------- /colmap/read_write_dense.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 17 | # its contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | # 32 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 33 | 34 | import argparse 35 | import numpy as np 36 | import os 37 | import struct 38 | 39 | 40 | def read_array(path): 41 | with open(path, "rb") as fid: 42 | width, height, channels = np.genfromtxt(fid, delimiter="&", max_rows=1, 43 | usecols=(0, 1, 2), dtype=int) 44 | fid.seek(0) 45 | num_delimiter = 0 46 | byte = fid.read(1) 47 | while True: 48 | if byte == b"&": 49 | num_delimiter += 1 50 | if num_delimiter >= 3: 51 | break 52 | byte = fid.read(1) 53 | array = np.fromfile(fid, np.float32) 54 | array = array.reshape((width, height, channels), order="F") 55 | return np.transpose(array, (1, 0, 2)).squeeze() 56 | 57 | 58 | def write_array(array, path): 59 | """ 60 | see: src/mvs/mat.h 61 | void Mat::Write(const std::string& path) 62 | """ 63 | assert array.dtype == np.float32 64 | if len(array.shape) == 2: 65 | height, width = array.shape 66 | channels = 1 67 | elif len(array.shape) == 3: 68 | height, width, channels = array.shape 69 | else: 70 | assert False 71 | 72 | with open(path, "w") as fid: 73 | fid.write(str(width) + "&" + str(height) + "&" + str(channels) + "&") 74 | 75 | with open(path, "ab") as fid: 76 | if len(array.shape) == 2: 77 | array_trans = np.transpose(array, (1, 0)) 78 | elif len(array.shape) == 3: 79 | array_trans = np.transpose(array, (1, 0, 2)) 80 | else: 81 | assert False 82 | data_1d = array_trans.reshape(-1, order="F") 83 | data_list = data_1d.tolist() 84 | endian_character = "<" 85 | format_char_sequence = "".join(["f"] * len(data_list)) 86 | byte_data = struct.pack(endian_character + format_char_sequence, *data_list) 87 | fid.write(byte_data) 88 | 89 | 90 | def parse_args(): 91 | parser = argparse.ArgumentParser() 92 | parser.add_argument("-d", "--depth_map", 93 | help="path to depth map", type=str, required=True) 94 | parser.add_argument("-n", "--normal_map", 95 | help="path to normal map", type=str, required=True) 96 | parser.add_argument("--min_depth_percentile", 97 | help="minimum visualization depth percentile", 98 | type=float, default=5) 99 | parser.add_argument("--max_depth_percentile", 100 | help="maximum visualization depth percentile", 101 | type=float, default=95) 102 | args = parser.parse_args() 103 | return args 104 | 105 | 106 | def main(): 107 | args = parse_args() 108 | 109 | if args.min_depth_percentile > args.max_depth_percentile: 110 | raise ValueError("min_depth_percentile should be less than or equal " 111 | "to the max_depth_perceintile.") 112 | 113 | # Read depth and normal maps corresponding to the same image. 114 | if not os.path.exists(args.depth_map): 115 | raise FileNotFoundError("File not found: {}".format(args.depth_map)) 116 | 117 | if not os.path.exists(args.normal_map): 118 | raise FileNotFoundError("File not found: {}".format(args.normal_map)) 119 | 120 | depth_map = read_array(args.depth_map) 121 | normal_map = read_array(args.normal_map) 122 | 123 | min_depth, max_depth = np.percentile( 124 | depth_map, [args.min_depth_percentile, args.max_depth_percentile]) 125 | depth_map[depth_map < min_depth] = min_depth 126 | depth_map[depth_map > max_depth] = max_depth 127 | 128 | import pylab as plt 129 | 130 | # Visualize the depth map. 131 | plt.figure() 132 | plt.imshow(depth_map) 133 | plt.title("depth map") 134 | 135 | # Visualize the normal map. 136 | plt.figure() 137 | plt.imshow(normal_map) 138 | plt.title("normal map") 139 | 140 | plt.show() 141 | 142 | 143 | if __name__ == "__main__": 144 | main() 145 | -------------------------------------------------------------------------------- /colmap/read_write_fused_vis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 17 | # its contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | # 32 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 33 | 34 | import os 35 | import collections 36 | import numpy as np 37 | import pandas as pd 38 | from pyntcloud import PyntCloud 39 | 40 | from read_write_model import read_next_bytes, write_next_bytes 41 | 42 | 43 | MeshPoint = collections.namedtuple( 44 | "MeshingPoint", ["position", "color", "normal", "num_visible_images", "visible_image_idxs"]) 45 | 46 | 47 | def read_fused(path_to_fused_ply, path_to_fused_ply_vis): 48 | """ 49 | see: src/mvs/meshing.cc 50 | void ReadDenseReconstruction(const std::string& path 51 | """ 52 | assert os.path.isfile(path_to_fused_ply) 53 | assert os.path.isfile(path_to_fused_ply_vis) 54 | 55 | point_cloud = PyntCloud.from_file(path_to_fused_ply) 56 | xyz_arr = point_cloud.points.loc[:, ["x", "y", "z"]].to_numpy() 57 | normal_arr = point_cloud.points.loc[:, ["nx", "ny", "nz"]].to_numpy() 58 | color_arr = point_cloud.points.loc[:, ["red", "green", "blue"]].to_numpy() 59 | 60 | with open(path_to_fused_ply_vis, "rb") as fid: 61 | num_points = read_next_bytes(fid, 8, "Q")[0] 62 | mesh_points = [0] * num_points 63 | for i in range(num_points): 64 | num_visible_images = read_next_bytes(fid, 4, "I")[0] 65 | visible_image_idxs = read_next_bytes( 66 | fid, num_bytes=4*num_visible_images, 67 | format_char_sequence="I"*num_visible_images) 68 | visible_image_idxs = np.array(tuple(map(int, visible_image_idxs))) 69 | mesh_point = MeshPoint( 70 | position=xyz_arr[i], 71 | color=color_arr[i], 72 | normal=normal_arr[i], 73 | num_visible_images=num_visible_images, 74 | visible_image_idxs=visible_image_idxs) 75 | mesh_points[i] = mesh_point 76 | return mesh_points 77 | 78 | 79 | def write_fused_ply(mesh_points, path_to_fused_ply): 80 | columns = ["x", "y", "z", "nx", "ny", "nz", "red", "green", "blue"] 81 | points_data_frame = pd.DataFrame( 82 | np.zeros((len(mesh_points), len(columns))), 83 | columns=columns) 84 | 85 | positions = np.asarray([point.position for point in mesh_points]) 86 | normals = np.asarray([point.normal for point in mesh_points]) 87 | colors = np.asarray([point.color for point in mesh_points]) 88 | 89 | points_data_frame.loc[:, ["x", "y", "z"]] = positions 90 | points_data_frame.loc[:, ["nx", "ny", "nz"]] = normals 91 | points_data_frame.loc[:, ["red", "green", "blue"]] = colors 92 | 93 | points_data_frame = points_data_frame.astype({ 94 | "x": positions.dtype, "y": positions.dtype, "z": positions.dtype, 95 | "red": colors.dtype, "green": colors.dtype, "blue": colors.dtype, 96 | "nx": normals.dtype, "ny": normals.dtype, "nz": normals.dtype}) 97 | 98 | point_cloud = PyntCloud(points_data_frame) 99 | point_cloud.to_file(path_to_fused_ply) 100 | 101 | 102 | def write_fused_ply_vis(mesh_points, path_to_fused_ply_vis): 103 | """ 104 | see: src/mvs/fusion.cc 105 | void WritePointsVisibility(const std::string& path, const std::vector>& points_visibility) 106 | """ 107 | with open(path_to_fused_ply_vis, "wb") as fid: 108 | write_next_bytes(fid, len(mesh_points), "Q") 109 | for point in mesh_points: 110 | write_next_bytes(fid, point.num_visible_images, "I") 111 | format_char_sequence = "I"*point.num_visible_images 112 | write_next_bytes(fid, [*point.visible_image_idxs], format_char_sequence) 113 | 114 | 115 | def write_fused(points, path_to_fused_ply, path_to_fused_ply_vis): 116 | write_fused_ply(points, path_to_fused_ply) 117 | write_fused_ply_vis(points, path_to_fused_ply_vis) 118 | -------------------------------------------------------------------------------- /colmap/test_read_write_dense.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import numpy as np 33 | from read_write_dense import read_array, write_array 34 | 35 | 36 | def main(): 37 | import sys 38 | if len(sys.argv) != 3: 39 | print("Usage: python test_read_write_dense.py " 40 | "path/to/dense/input.bin path/to/dense/output.bin") 41 | return 42 | 43 | print("Checking consistency of reading and writing dense arrays " 44 | + "(depth maps / normal maps) ...") 45 | 46 | path_to_dense_input = sys.argv[1] 47 | path_to_dense_output = sys.argv[2] 48 | 49 | dense_input = read_array(path_to_dense_input) 50 | print("Input shape: " + str(dense_input.shape)) 51 | 52 | write_array(dense_input, path_to_dense_output) 53 | dense_output = read_array(path_to_dense_output) 54 | 55 | np.testing.assert_array_equal(dense_input, dense_output) 56 | 57 | print("... dense arrays are equal.") 58 | 59 | 60 | if __name__ == "__main__": 61 | main() 62 | -------------------------------------------------------------------------------- /colmap/test_read_write_fused_vis.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import filecmp 33 | from read_write_fused_vis import read_fused, write_fused 34 | 35 | 36 | def main(): 37 | import sys 38 | if len(sys.argv) != 5: 39 | print("Usage: python test_read_write_fused_vis.py " 40 | "path/to/input_fused.ply path/to/input_fused.ply.vis " 41 | "path/to/output_fused.ply path/to/output_fused.ply.vis") 42 | return 43 | 44 | print("Checking consistency of reading and writing fused.ply and fused.ply.vis files ...") 45 | 46 | path_to_fused_ply_input = sys.argv[1] 47 | path_to_fused_ply_vis_input = sys.argv[2] 48 | path_to_fused_ply_output = sys.argv[3] 49 | path_to_fused_ply_vis_output = sys.argv[4] 50 | 51 | mesh_points = read_fused(path_to_fused_ply_input, path_to_fused_ply_vis_input) 52 | write_fused(mesh_points, path_to_fused_ply_output, path_to_fused_ply_vis_output) 53 | 54 | assert filecmp.cmp(path_to_fused_ply_input, path_to_fused_ply_output) 55 | assert filecmp.cmp(path_to_fused_ply_vis_input, path_to_fused_ply_vis_output) 56 | 57 | print("... Results are equal.") 58 | 59 | 60 | if __name__ == "__main__": 61 | main() 62 | 63 | -------------------------------------------------------------------------------- /colmap/test_read_write_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | import numpy as np 33 | from read_write_model import read_model, write_model 34 | from tempfile import mkdtemp 35 | 36 | 37 | def compare_cameras(cameras1, cameras2): 38 | assert len(cameras1) == len(cameras2) 39 | for camera_id1 in cameras1: 40 | camera1 = cameras1[camera_id1] 41 | camera2 = cameras2[camera_id1] 42 | assert camera1.id == camera2.id 43 | assert camera1.width == camera2.width 44 | assert camera1.height == camera2.height 45 | assert np.allclose(camera1.params, camera2.params) 46 | 47 | 48 | def compare_images(images1, images2): 49 | assert len(images1) == len(images2) 50 | for image_id1 in images1: 51 | image1 = images1[image_id1] 52 | image2 = images2[image_id1] 53 | assert image1.id == image2.id 54 | assert np.allclose(image1.qvec, image2.qvec) 55 | assert np.allclose(image1.tvec, image2.tvec) 56 | assert image1.camera_id == image2.camera_id 57 | assert image1.name == image2.name 58 | assert np.allclose(image1.xys, image2.xys) 59 | assert np.array_equal(image1.point3D_ids, image2.point3D_ids) 60 | 61 | 62 | def compare_points(points3D1, points3D2): 63 | for point3D_id1 in points3D1: 64 | point3D1 = points3D1[point3D_id1] 65 | point3D2 = points3D2[point3D_id1] 66 | assert point3D1.id == point3D2.id 67 | assert np.allclose(point3D1.xyz, point3D2.xyz) 68 | assert np.array_equal(point3D1.rgb, point3D2.rgb) 69 | assert np.allclose(point3D1.error, point3D2.error) 70 | assert np.array_equal(point3D1.image_ids, point3D2.image_ids) 71 | assert np.array_equal(point3D1.point2D_idxs, point3D2.point2D_idxs) 72 | 73 | 74 | def main(): 75 | import sys 76 | if len(sys.argv) != 3: 77 | print("Usage: python read_model.py " 78 | "path/to/model/folder/txt path/to/model/folder/bin") 79 | return 80 | 81 | print("Comparing text and binary models ...") 82 | 83 | path_to_model_txt_folder = sys.argv[1] 84 | path_to_model_bin_folder = sys.argv[2] 85 | cameras_txt, images_txt, points3D_txt = \ 86 | read_model(path_to_model_txt_folder, ext=".txt") 87 | cameras_bin, images_bin, points3D_bin = \ 88 | read_model(path_to_model_bin_folder, ext=".bin") 89 | compare_cameras(cameras_txt, cameras_bin) 90 | compare_images(images_txt, images_bin) 91 | compare_points(points3D_txt, points3D_bin) 92 | 93 | print("... text and binary models are equal.") 94 | print("Saving text model and reloading it ...") 95 | 96 | tmpdir = mkdtemp() 97 | write_model(cameras_bin, images_bin, points3D_bin, tmpdir, ext='.txt') 98 | cameras_txt, images_txt, points3D_txt = \ 99 | read_model(tmpdir, ext=".txt") 100 | compare_cameras(cameras_txt, cameras_bin) 101 | compare_images(images_txt, images_bin) 102 | compare_points(points3D_txt, points3D_bin) 103 | 104 | print("... saved text and loaded models are equal.") 105 | print("Saving binary model and reloading it ...") 106 | 107 | write_model(cameras_bin, images_bin, points3D_bin, tmpdir, ext='.bin') 108 | cameras_bin, images_bin, points3D_bin = \ 109 | read_model(tmpdir, ext=".bin") 110 | compare_cameras(cameras_txt, cameras_bin) 111 | compare_images(images_txt, images_bin) 112 | compare_points(points3D_txt, points3D_bin) 113 | 114 | print("... saved binary and loaded models are equal.") 115 | 116 | 117 | if __name__ == "__main__": 118 | main() 119 | -------------------------------------------------------------------------------- /colmap/visualize_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | # its contributors may be used to endorse or promote products derived 16 | # from this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | 30 | import argparse 31 | import numpy as np 32 | import open3d 33 | 34 | from read_write_model import read_model, write_model, qvec2rotmat, rotmat2qvec 35 | 36 | 37 | class Model: 38 | def __init__(self): 39 | self.cameras = [] 40 | self.images = [] 41 | self.points3D = [] 42 | self.__vis = None 43 | 44 | def read_model(self, path, ext=""): 45 | self.cameras, self.images, self.points3D = read_model(path, ext) 46 | 47 | def add_points(self, min_track_len=3, remove_statistical_outlier=True): 48 | pcd = open3d.geometry.PointCloud() 49 | 50 | xyz = [] 51 | rgb = [] 52 | for point3D in self.points3D.values(): 53 | track_len = len(point3D.point2D_idxs) 54 | if track_len < min_track_len: 55 | continue 56 | xyz.append(point3D.xyz) 57 | rgb.append(point3D.rgb / 255) 58 | 59 | pcd.points = open3d.utility.Vector3dVector(xyz) 60 | pcd.colors = open3d.utility.Vector3dVector(rgb) 61 | 62 | # remove obvious outliers 63 | if remove_statistical_outlier: 64 | [pcd, _] = pcd.remove_statistical_outlier(nb_neighbors=20, 65 | std_ratio=2.0) 66 | 67 | # open3d.visualization.draw_geometries([pcd]) 68 | self.__vis.add_geometry(pcd) 69 | self.__vis.poll_events() 70 | self.__vis.update_renderer() 71 | 72 | def add_cameras(self, scale=1): 73 | frames = [] 74 | for img in self.images.values(): 75 | # rotation 76 | R = qvec2rotmat(img.qvec) 77 | 78 | # translation 79 | t = img.tvec 80 | 81 | # invert 82 | t = -R.T @ t 83 | R = R.T 84 | 85 | # intrinsics 86 | cam = self.cameras[img.camera_id] 87 | 88 | if cam.model in ("SIMPLE_PINHOLE", "SIMPLE_RADIAL", "RADIAL"): 89 | fx = fy = cam.params[0] 90 | cx = cam.params[1] 91 | cy = cam.params[2] 92 | elif cam.model in ("PINHOLE", "OPENCV", "OPENCV_FISHEYE"): 93 | fx = cam.params[0] 94 | fy = cam.params[1] 95 | cx = cam.params[2] 96 | cy = cam.params[3] 97 | else: 98 | raise Exception("Camera model not supported") 99 | 100 | # intrinsics 101 | K = np.identity(3) 102 | K[0, 0] = fx 103 | K[1, 1] = fy 104 | K[0, 2] = cx 105 | K[1, 2] = cy 106 | 107 | # create axis, plane and pyramed geometries that will be drawn 108 | cam_model = draw_camera(K, R, t, cam.width, cam.height, scale) 109 | frames.extend(cam_model) 110 | 111 | # add geometries to visualizer 112 | for i in frames: 113 | self.__vis.add_geometry(i) 114 | 115 | def create_window(self): 116 | self.__vis = open3d.visualization.Visualizer() 117 | self.__vis.create_window() 118 | 119 | def show(self): 120 | self.__vis.poll_events() 121 | self.__vis.update_renderer() 122 | self.__vis.run() 123 | self.__vis.destroy_window() 124 | 125 | 126 | def draw_camera(K, R, t, w, h, 127 | scale=1, color=[0.8, 0.2, 0.8]): 128 | """Create axis, plane and pyramed geometries in Open3D format. 129 | :param K: calibration matrix (camera intrinsics) 130 | :param R: rotation matrix 131 | :param t: translation 132 | :param w: image width 133 | :param h: image height 134 | :param scale: camera model scale 135 | :param color: color of the image plane and pyramid lines 136 | :return: camera model geometries (axis, plane and pyramid) 137 | """ 138 | 139 | # intrinsics 140 | K = K.copy() / scale 141 | Kinv = np.linalg.inv(K) 142 | 143 | # 4x4 transformation 144 | T = np.column_stack((R, t)) 145 | T = np.vstack((T, (0, 0, 0, 1))) 146 | 147 | # axis 148 | axis = open3d.geometry.TriangleMesh.create_coordinate_frame(size=0.5 * scale) 149 | axis.transform(T) 150 | 151 | # points in pixel 152 | points_pixel = [ 153 | [0, 0, 0], 154 | [0, 0, 1], 155 | [w, 0, 1], 156 | [0, h, 1], 157 | [w, h, 1], 158 | ] 159 | 160 | # pixel to camera coordinate system 161 | points = [Kinv @ p for p in points_pixel] 162 | 163 | # image plane 164 | width = abs(points[1][0]) + abs(points[3][0]) 165 | height = abs(points[1][1]) + abs(points[3][1]) 166 | plane = open3d.geometry.TriangleMesh.create_box(width, height, depth=1e-6) 167 | plane.paint_uniform_color(color) 168 | plane.translate([points[1][0], points[1][1], scale]) 169 | plane.transform(T) 170 | 171 | # pyramid 172 | points_in_world = [(R @ p + t) for p in points] 173 | lines = [ 174 | [0, 1], 175 | [0, 2], 176 | [0, 3], 177 | [0, 4], 178 | ] 179 | colors = [color for i in range(len(lines))] 180 | line_set = open3d.geometry.LineSet( 181 | points=open3d.utility.Vector3dVector(points_in_world), 182 | lines=open3d.utility.Vector2iVector(lines)) 183 | line_set.colors = open3d.utility.Vector3dVector(colors) 184 | 185 | # return as list in Open3D format 186 | return [axis, plane, line_set] 187 | 188 | 189 | def parse_args(): 190 | parser = argparse.ArgumentParser(description="Visualize COLMAP binary and text models") 191 | parser.add_argument("--input_model", required=True, help="path to input model folder") 192 | parser.add_argument("--input_format", choices=[".bin", ".txt"], 193 | help="input model format", default="") 194 | args = parser.parse_args() 195 | return args 196 | 197 | 198 | def main(): 199 | args = parse_args() 200 | 201 | # read COLMAP model 202 | model = Model() 203 | model.read_model(args.input_model, ext=args.input_format) 204 | 205 | print("num_cameras:", len(model.cameras)) 206 | print("num_images:", len(model.images)) 207 | print("num_points3D:", len(model.points3D)) 208 | 209 | # display using Open3D visualization tools 210 | model.create_window() 211 | model.add_points() 212 | model.add_cameras(scale=0.25) 213 | model.show() 214 | 215 | 216 | if __name__ == "__main__": 217 | main() 218 | -------------------------------------------------------------------------------- /configs/mat/custom/brassgourd.yaml: -------------------------------------------------------------------------------- 1 | name: brassgourd_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: custom/brassgourd/raw_1600 7 | mesh: data/custom_results/meshes/brassgourd_shape-180000-crop.ply 8 | geo_model_path: data/model/brassgourd_shape/model.pth 9 | 10 | reg_diffuse_light: true 11 | reg_diffuse_light_lambda: 0.1 12 | reg_mat: true 13 | shader_cfg: 14 | diffuse_sample_num: 512 15 | specular_sample_num: 256 16 | outer_light_version: sphere_direction 17 | light_exp_max: 5.0 18 | inner_light_exp_max: 5.0 19 | human_lights: true 20 | 21 | ######loss###### 22 | loss: ['nerf_render','mat_reg'] 23 | val_metric: ['mat_render'] 24 | key_metric_name: psnr 25 | 26 | ####dataset##### 27 | train_dataset_type: dummy 28 | dataset_dir: /home/riga/NeRF/nerf_data/customData 29 | train_dataset_cfg: 30 | database_name: custom/brassgourd/raw_1600 31 | val_set_list: 32 | - 33 | name: val 34 | type: dummy 35 | cfg: 36 | database_name: custom/brassgourd/raw_1600 37 | 38 | ####trainier#### 39 | optimizer_type: adam 40 | lr_type: warm_up_cos 41 | total_step: 100000 42 | val_interval: 5000 43 | save_interval: 500 44 | train_log_step: 10 45 | -------------------------------------------------------------------------------- /configs/mat/custom/casserole.yaml: -------------------------------------------------------------------------------- 1 | name: casserole_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: custom/casserole/raw_1600 7 | mesh: data/custom_results/meshes/casserole_shape-180000-crop.ply 8 | geo_model_path: data/model/casserole_shape/model.pth 9 | 10 | reg_diffuse_light: true 11 | reg_diffuse_light_lambda: 0.1 12 | reg_mat: true 13 | shader_cfg: 14 | diffuse_sample_num: 512 15 | specular_sample_num: 256 16 | outer_light_version: sphere_direction 17 | light_exp_max: 5.0 18 | inner_light_exp_max: 5.0 19 | human_lights: true 20 | 21 | ######loss###### 22 | loss: ['nerf_render','mat_reg'] 23 | val_metric: ['mat_render'] 24 | key_metric_name: psnr 25 | 26 | ####dataset##### 27 | train_dataset_type: dummy 28 | dataset_dir: /home/riga/NeRF/nerf_data/customData 29 | train_dataset_cfg: 30 | database_name: custom/casserole/raw_1600 31 | val_set_list: 32 | - 33 | name: val 34 | type: dummy 35 | cfg: 36 | database_name: custom/casserole/raw_1600 37 | 38 | ####trainier#### 39 | optimizer_type: adam 40 | lr_type: warm_up_cos 41 | total_step: 100000 42 | val_interval: 5000 43 | save_interval: 500 44 | train_log_step: 10 45 | -------------------------------------------------------------------------------- /configs/mat/custom/goldenqilin.yaml: -------------------------------------------------------------------------------- 1 | name: goldenqilin_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: custom/goldenqilin/raw_1600 7 | mesh: data/custom_results/meshes/goldenqilin_shape-180000-crop.ply 8 | geo_model_path: data/model/goldenqilin_shape/model.pth 9 | 10 | reg_diffuse_light: true 11 | reg_diffuse_light_lambda: 0.1 12 | reg_mat: true 13 | shader_cfg: 14 | diffuse_sample_num: 512 15 | specular_sample_num: 256 16 | outer_light_version: sphere_direction 17 | light_exp_max: 5.0 18 | inner_light_exp_max: 5.0 19 | human_lights: true 20 | 21 | ######loss###### 22 | loss: ['nerf_render','mat_reg'] 23 | val_metric: ['mat_render'] 24 | key_metric_name: psnr 25 | 26 | ####dataset##### 27 | train_dataset_type: dummy 28 | dataset_dir: /home/riga/NeRF/nerf_data/customData 29 | train_dataset_cfg: 30 | database_name: custom/goldenqilin/raw_1600 31 | val_set_list: 32 | - 33 | name: val 34 | type: dummy 35 | cfg: 36 | database_name: custom/goldenqilin/raw_1600 37 | 38 | ####trainier#### 39 | optimizer_type: adam 40 | lr_type: warm_up_cos 41 | total_step: 100000 42 | val_interval: 5000 43 | save_interval: 500 44 | train_log_step: 10 45 | -------------------------------------------------------------------------------- /configs/mat/custom/luckycat.yaml: -------------------------------------------------------------------------------- 1 | name: luckycat_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: custom/luckycat/raw_1600 7 | mesh: data/custom_results/meshes/luckycat_shape-180000-crop.ply 8 | geo_model_path: data/model/luckycat_shape/model.pth 9 | 10 | reg_diffuse_light: true 11 | reg_diffuse_light_lambda: 0.1 12 | reg_mat: true 13 | shader_cfg: 14 | diffuse_sample_num: 512 15 | specular_sample_num: 256 16 | outer_light_version: sphere_direction 17 | light_exp_max: 5.0 18 | inner_light_exp_max: 5.0 19 | human_lights: true 20 | 21 | ######loss###### 22 | loss: ['nerf_render','mat_reg'] 23 | val_metric: ['mat_render'] 24 | key_metric_name: psnr 25 | 26 | ####dataset##### 27 | train_dataset_type: dummy 28 | dataset_dir: /home/riga/NeRF/nerf_data/customData 29 | train_dataset_cfg: 30 | database_name: custom/luckycat/raw_1600 31 | val_set_list: 32 | - 33 | name: val 34 | type: dummy 35 | cfg: 36 | database_name: custom/luckycat/raw_1600 37 | 38 | ####trainier#### 39 | optimizer_type: adam 40 | lr_type: warm_up_cos 41 | total_step: 100000 42 | val_interval: 5000 43 | save_interval: 500 44 | train_log_step: 10 45 | -------------------------------------------------------------------------------- /configs/mat/custom/shoe.yaml: -------------------------------------------------------------------------------- 1 | name: shoe_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: custom/shoe/raw_1600 7 | mesh: data/custom_results/meshes/shoe_shape-180000-crop.ply 8 | geo_model_path: data/model/shoe_shape/model.pth 9 | 10 | reg_diffuse_light: true 11 | reg_diffuse_light_lambda: 0.1 12 | reg_mat: true 13 | shader_cfg: 14 | diffuse_sample_num: 512 15 | specular_sample_num: 256 16 | outer_light_version: sphere_direction 17 | light_exp_max: 5.0 18 | inner_light_exp_max: 5.0 19 | human_lights: true 20 | 21 | ######loss###### 22 | loss: ['nerf_render','mat_reg'] 23 | val_metric: ['mat_render'] 24 | key_metric_name: psnr 25 | 26 | ####dataset##### 27 | train_dataset_type: dummy 28 | dataset_dir: /home/riga/NeRF/nerf_data/customData 29 | train_dataset_cfg: 30 | database_name: custom/shoe/raw_1600 31 | val_set_list: 32 | - 33 | name: val 34 | type: dummy 35 | cfg: 36 | database_name: custom/shoe/raw_1600 37 | 38 | ####trainier#### 39 | optimizer_type: adam 40 | lr_type: warm_up_cos 41 | total_step: 100000 42 | val_interval: 5000 43 | save_interval: 500 44 | train_log_step: 10 45 | -------------------------------------------------------------------------------- /configs/mat/orb/cactus.yaml: -------------------------------------------------------------------------------- 1 | name: cactus_scene001_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: orb/cactus_scene001 7 | mesh: data/meshes/orb/cactus_scene001_shape-180000.ply 8 | geo_model_path: data/model/cactus_scene001_shape/model.pth 9 | # split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: orb/cactus_scene001 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: orb/cactus_scene001 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 0 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/orb/car.yaml: -------------------------------------------------------------------------------- 1 | name: car_scene004_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: orb/car_scene004 7 | mesh: data/meshes/orb/car_scene004_shape-180000.ply 8 | geo_model_path: data/model/car_scene004_shape/model.pth 9 | # split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: orb/car_scene004 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: orb/car_scene004 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 0 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/orb/gnome.yaml: -------------------------------------------------------------------------------- 1 | name: gnome_scene003_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: orb/gnome_scene003 7 | mesh: data/meshes/orb/gnome_scene003_shape-180000.ply 8 | geo_model_path: data/model/gnome_scene003_shape/model.pth 9 | # split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: orb/gnome_scene003 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: orb/gnome_scene003 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 0 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/orb/grogu.yaml: -------------------------------------------------------------------------------- 1 | name: grogu_scene001_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: orb/grogu_scene001 7 | mesh: data/meshes/orb/grogu_scene001_shape-180000.ply 8 | geo_model_path: data/model/grogu_scene001_shape/model.pth 9 | # split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: orb/grogu_scene001 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: orb/grogu_scene001 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 0 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/orb/teapot.yaml: -------------------------------------------------------------------------------- 1 | name: teapot_scene006_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: orb/teapot_scene006 7 | mesh: data/meshes/teapot_scene006_shape-180000.ply 8 | geo_model_path: data/model/teapot_scene006_shape/model.pth 9 | # split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/blender_LDR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: orb/teapot_scene006 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: orb/teapot_scene006 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 0 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/syn/FlightHelmet.yaml: -------------------------------------------------------------------------------- 1 | name: FlightHelmet_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoSDF/FlightHelmet 7 | mesh: data/meshes/FlightHelmet_shape-180000.ply 8 | geo_model_path: data/model/FlightHelmet_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoSDF/FlightHelmet 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoSDF/FlightHelmet 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 2 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/syn/armadillo.yaml: -------------------------------------------------------------------------------- 1 | name: armadillo_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoIR/armadillo 7 | mesh: data/meshes/armadillo_shape-180000.ply 8 | geo_model_path: data/model/armadillo_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoIR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoIR/armadillo 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoIR/armadillo 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 2 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false 51 | -------------------------------------------------------------------------------- /configs/mat/syn/compressor.yaml: -------------------------------------------------------------------------------- 1 | name: compressor_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoSDF/compressor 7 | mesh: data/meshes/compressor_shape-180000.ply 8 | geo_model_path: data/model/compressor_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoSDF/compressor 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoSDF/compressor 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 0 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/syn/dragon.yaml: -------------------------------------------------------------------------------- 1 | name: dragon_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoSDF/dragon 7 | mesh: data/meshes/dragon_shape-180000.ply 8 | geo_model_path: data/model/dragon_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoSDF/dragon 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoSDF/dragon 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 2 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/syn/horse.yaml: -------------------------------------------------------------------------------- 1 | name: horse_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: syn/horse 7 | mesh: data/meshes/horse_shape-180000.ply 8 | geo_model_path: data/model/horse_shape/model.pth 9 | 10 | reg_diffuse_light: true 11 | reg_diffuse_light_lambda: 0.1 12 | reg_mat: true 13 | shader_cfg: 14 | diffuse_sample_num: 512 15 | specular_sample_num: 256 16 | outer_light_version: direction 17 | light_exp_max: 5.0 18 | inner_light_exp_max: 5.0 19 | human_lights: false 20 | 21 | ######loss###### 22 | loss: ['nerf_render','mat_reg'] 23 | val_metric: ['mat_render'] 24 | key_metric_name: psnr 25 | 26 | ####dataset##### 27 | train_dataset_type: dummy 28 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 29 | train_dataset_cfg: 30 | database_name: syn/horse 31 | val_set_list: 32 | - 33 | name: val 34 | type: dummy 35 | cfg: 36 | database_name: syn/horse 37 | 38 | ####trainier#### 39 | optimizer_type: adam 40 | lr_type: warm_up_cos 41 | total_step: 100000 42 | val_interval: 5000 43 | save_interval: 500 44 | train_log_step: 10 45 | 46 | ####relighting settings#### 47 | trans: true -------------------------------------------------------------------------------- /configs/mat/syn/lego.yaml: -------------------------------------------------------------------------------- 1 | name: lego_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoIR/lego 7 | mesh: data/meshes/lego_shape-180000.ply 8 | geo_model_path: data/model/lego_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoIR # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoIR/lego 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoIR/lego 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 2 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false 51 | -------------------------------------------------------------------------------- /configs/mat/syn/motor.yaml: -------------------------------------------------------------------------------- 1 | name: motor_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoSDF/motor 7 | mesh: data/meshes/motor_shape-180000.ply 8 | geo_model_path: data/model/motor_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoSDF/motor 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoSDF/motor 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 1 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/syn/robot.yaml: -------------------------------------------------------------------------------- 1 | name: robot_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoSDF/robot 7 | mesh: data/meshes/robot_shape-180000.ply 8 | geo_model_path: data/model/robot_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoSDF/robot 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoSDF/robot 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 2 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/mat/syn/rover.yaml: -------------------------------------------------------------------------------- 1 | name: rover_mat 2 | isMaterial: true 3 | 4 | ####network##### 5 | network: material 6 | database_name: tensoSDF/rover 7 | mesh: data/meshes/rover_shape-180000.ply 8 | geo_model_path: data/model/rover_shape/model.pth 9 | split_manul: true 10 | nerfDataType: true 11 | 12 | reg_diffuse_light: true 13 | reg_diffuse_light_lambda: 0.1 14 | reg_mat: true 15 | shader_cfg: 16 | diffuse_sample_num: 512 17 | specular_sample_num: 256 18 | outer_light_version: direction 19 | light_exp_max: 5.0 20 | inner_light_exp_max: 5.0 21 | human_lights: false 22 | 23 | ######loss###### 24 | loss: ['nerf_render','mat_reg'] 25 | val_metric: ['mat_render'] 26 | key_metric_name: psnr 27 | 28 | ####dataset##### 29 | train_dataset_type: dummy 30 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 31 | train_dataset_cfg: 32 | database_name: tensoSDF/rover 33 | val_set_list: 34 | - 35 | name: val 36 | type: dummy 37 | cfg: 38 | database_name: tensoSDF/rover 39 | 40 | ####trainier#### 41 | optimizer_type: adam 42 | lr_type: warm_up_cos 43 | total_step: 100000 44 | val_interval: 5000 45 | save_interval: 500 46 | train_log_step: 10 47 | 48 | ####relighting settings#### 49 | albedoRescale: 2 # 0: don't rescale, 1: apply single rescaling, 2: apply three rescaling 50 | trans: false -------------------------------------------------------------------------------- /configs/shape/custom/brassgourd.yaml: -------------------------------------------------------------------------------- 1 | name: brassgourd_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: custom/brassgourd/raw_1600 # this means we run reconstruction on the image resolution of 1024 6 | shader_config: 7 | human_light: false # we model the reflection capturer 8 | apply_occ_loss: true # apply the occlusion loss 9 | occ_loss_step: 10000 # occlusion loss is applied after 20k steps 10 | clip_sample_variance: false # this affects how we sample points on the ray 11 | has_radiance_field: true 12 | radiance_field_step: 40000 13 | apply_gaussian_loss: false 14 | gaussianLoss_step: 40000 15 | predict_BG: true 16 | isBGWhite: false 17 | downsample_ratio: 0.5 18 | alphaMask_thres: 0.0001 19 | mul_length: 0 20 | 21 | ######loss###### 22 | # losses used in training 23 | loss: ['nerf_render','eikonal','std','init_sdf_reg','occ', 'Hessian', 'TV', 'Sparse'] 24 | val_metric: ['shape_render'] # this controls how we output images during training 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | freeze_inv_s_step: 8000 # we freeze the 1/s in the NeuS for 15k steps for initialization. 28 | hessian_weight: 0.0005 29 | sparse_weight: 0.05 30 | gaussian_weight: 0.00001 31 | 32 | ####dataset##### 33 | train_dataset_type: dummy 34 | dataset_dir: /home/riga/NeRF/nerf_data/customData 35 | train_dataset_cfg: 36 | database_name: custom/brassgourd/raw_1600 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: custom/brassgourd/raw_1600 43 | 44 | ####trainier#### 45 | optimizer_type: adam 46 | lr_type: warm_up_cos 47 | lr_cfg: {} 48 | total_step: 180000 # 300000 49 | val_interval: 2500 # 5000 50 | save_interval: 1000 51 | train_log_step: 20 52 | N_voxel_init: 2097153 # 128**3 + 1 53 | N_voxel_final: 134217729 # 512**3 + 1 54 | upsample_list: [40000, 80000] 55 | update_AlphaMask_lst: [40000, 80000] # update_AlphaMask_lst: [60000] 56 | hessian_ratio: [0.1, 0.05] 57 | 58 | ###SDF setting### 59 | sdf_n_comp: 36 60 | sdf_dim: 256 61 | app_dim: 128 62 | 63 | ###Mesh extraction### 64 | blend_ratio: 0.8 -------------------------------------------------------------------------------- /configs/shape/custom/casserole.yaml: -------------------------------------------------------------------------------- 1 | name: casserole_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: custom/casserole/raw_1600 # this means we run reconstruction on the image resolution of 1024 6 | shader_config: 7 | human_light: false # we model the reflection capturer 8 | apply_occ_loss: true # apply the occlusion loss 9 | occ_loss_step: 10000 # occlusion loss is applied after 20k steps 10 | clip_sample_variance: false # this affects how we sample points on the ray 11 | has_radiance_field: true 12 | radiance_field_step: 40000 13 | apply_gaussian_loss: false 14 | gaussianLoss_step: 40000 15 | predict_BG: true 16 | isBGWhite: false 17 | downsample_ratio: 0.5 18 | alphaMask_thres: 0.0001 19 | mul_length: 0 20 | 21 | ######loss###### 22 | # losses used in training 23 | loss: ['nerf_render','eikonal','std','init_sdf_reg','occ', 'Hessian', 'TV', 'Sparse'] 24 | val_metric: ['shape_render'] # this controls how we output images during training 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | freeze_inv_s_step: 8000 # we freeze the 1/s in the NeuS for 15k steps for initialization. 28 | hessian_weight: 0.0005 29 | sparse_weight: 0.05 30 | gaussian_weight: 0.00001 31 | 32 | ####dataset##### 33 | train_dataset_type: dummy 34 | dataset_dir: /home/riga/NeRF/nerf_data/customData 35 | train_dataset_cfg: 36 | database_name: custom/casserole/raw_1600 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: custom/casserole/raw_1600 43 | 44 | ####trainier#### 45 | optimizer_type: adam 46 | lr_type: warm_up_cos 47 | lr_cfg: {} 48 | total_step: 180000 # 300000 49 | val_interval: 2500 # 5000 50 | save_interval: 1000 51 | train_log_step: 20 52 | N_voxel_init: 2097153 # 128**3 + 1 53 | N_voxel_final: 134217729 # 512**3 + 1 54 | upsample_list: [40000, 80000] 55 | update_AlphaMask_lst: [40000, 80000] # update_AlphaMask_lst: [60000] 56 | hessian_ratio: [0.1, 0.05] 57 | 58 | ###SDF setting### 59 | sdf_n_comp: 36 60 | sdf_dim: 256 61 | app_dim: 128 62 | 63 | ###Mesh extraction### 64 | blend_ratio: 0.8 -------------------------------------------------------------------------------- /configs/shape/custom/goldenqilin.yaml: -------------------------------------------------------------------------------- 1 | name: goldenqilin_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: custom/goldenqilin/raw_1600 # this means we run reconstruction on the image resolution of 1024 6 | shader_config: 7 | human_light: false # we model the reflection capturer 8 | apply_occ_loss: true # apply the occlusion loss 9 | occ_loss_step: 10000 # occlusion loss is applied after 20k steps 10 | clip_sample_variance: false # this affects how we sample points on the ray 11 | has_radiance_field: true 12 | radiance_field_step: 40000 13 | apply_gaussian_loss: false 14 | gaussianLoss_step: 40000 15 | predict_BG: true 16 | isBGWhite: false 17 | downsample_ratio: 0.5 18 | alphaMask_thres: 0.0001 19 | mul_length: 0 20 | 21 | ######loss###### 22 | # losses used in training 23 | loss: ['nerf_render','eikonal','std','init_sdf_reg','occ', 'Hessian', 'TV', 'Sparse'] 24 | val_metric: ['shape_render'] # this controls how we output images during training 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | freeze_inv_s_step: 8000 # we freeze the 1/s in the NeuS for 15k steps for initialization. 28 | hessian_weight: 0.0005 29 | sparse_weight: 0.05 30 | gaussian_weight: 0.00001 31 | 32 | ####dataset##### 33 | train_dataset_type: dummy 34 | dataset_dir: /home/riga/NeRF/nerf_data/customData 35 | train_dataset_cfg: 36 | database_name: custom/goldenqilin/raw_1600 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: custom/goldenqilin/raw_1600 43 | 44 | ####trainier#### 45 | optimizer_type: adam 46 | lr_type: warm_up_cos 47 | lr_cfg: {} 48 | total_step: 180000 # 300000 49 | val_interval: 2500 # 5000 50 | save_interval: 1000 51 | train_log_step: 20 52 | N_voxel_init: 2097153 # 128**3 + 1 53 | N_voxel_final: 134217729 # 512**3 + 1 54 | upsample_list: [40000, 80000] 55 | update_AlphaMask_lst: [40000, 80000] # update_AlphaMask_lst: [60000] 56 | hessian_ratio: [0.1, 0.05] 57 | 58 | ###SDF setting### 59 | sdf_n_comp: 36 60 | sdf_dim: 256 61 | app_dim: 128 62 | 63 | ###Mesh extraction### 64 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/custom/luckycat.yaml: -------------------------------------------------------------------------------- 1 | name: luckycat_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: custom/luckycat/raw_1600 # this means we run reconstruction on the image resolution of 1024 6 | shader_config: 7 | human_light: false # we model the reflection capturer 8 | apply_occ_loss: true # apply the occlusion loss 9 | occ_loss_step: 10000 # occlusion loss is applied after 20k steps 10 | clip_sample_variance: false # this affects how we sample points on the ray 11 | has_radiance_field: true 12 | radiance_field_step: 40000 13 | apply_gaussian_loss: false 14 | gaussianLoss_step: 40000 15 | predict_BG: true 16 | isBGWhite: false 17 | downsample_ratio: 0.5 18 | alphaMask_thres: 0.0001 19 | mul_length: 0 20 | 21 | ######loss###### 22 | # losses used in training 23 | loss: ['nerf_render','eikonal','std','init_sdf_reg','occ', 'Hessian', 'TV', 'Sparse'] 24 | val_metric: ['shape_render'] # this controls how we output images during training 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | freeze_inv_s_step: 8000 # we freeze the 1/s in the NeuS for 15k steps for initialization. 28 | hessian_weight: 0.0005 29 | sparse_weight: 0.05 30 | gaussian_weight: 0.00001 31 | 32 | ####dataset##### 33 | train_dataset_type: dummy 34 | dataset_dir: /home/riga/NeRF/nerf_data/customData 35 | train_dataset_cfg: 36 | database_name: custom/luckycat/raw_1600 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: custom/luckycat/raw_1600 43 | 44 | ####trainier#### 45 | optimizer_type: adam 46 | lr_type: warm_up_cos 47 | lr_cfg: {} 48 | total_step: 180000 # 300000 49 | val_interval: 2500 # 5000 50 | save_interval: 1000 51 | train_log_step: 20 52 | N_voxel_init: 2097153 # 128**3 + 1 53 | N_voxel_final: 134217729 # 512**3 + 1 54 | upsample_list: [40000, 80000] 55 | update_AlphaMask_lst: [40000, 80000] # update_AlphaMask_lst: [60000] 56 | hessian_ratio: [0.1, 0.05] 57 | 58 | ###SDF setting### 59 | sdf_n_comp: 36 60 | sdf_dim: 256 61 | app_dim: 128 62 | 63 | ###Mesh extraction### 64 | blend_ratio: 0.2 -------------------------------------------------------------------------------- /configs/shape/custom/shoe.yaml: -------------------------------------------------------------------------------- 1 | name: shoe_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: custom/shoe/raw_1600 # this means we run reconstruction on the image resolution of 1024 6 | shader_config: 7 | human_light: false # we model the reflection capturer 8 | apply_occ_loss: true # apply the occlusion loss 9 | occ_loss_step: 10000 # occlusion loss is applied after 20k steps 10 | clip_sample_variance: false # this affects how we sample points on the ray 11 | has_radiance_field: true 12 | radiance_field_step: 40000 13 | apply_gaussian_loss: false 14 | gaussianLoss_step: 40000 15 | predict_BG: true 16 | isBGWhite: false 17 | downsample_ratio: 0.5 18 | alphaMask_thres: 0.0001 19 | mul_length: 0 20 | 21 | ######loss###### 22 | # losses used in training 23 | loss: ['nerf_render','eikonal','std','init_sdf_reg','occ', 'Hessian', 'TV', 'Sparse'] 24 | val_metric: ['shape_render'] # this controls how we output images during training 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | freeze_inv_s_step: 8000 # we freeze the 1/s in the NeuS for 15k steps for initialization. 28 | hessian_weight: 0.0005 29 | sparse_weight: 0.05 30 | gaussian_weight: 0.00001 31 | 32 | ####dataset##### 33 | train_dataset_type: dummy 34 | dataset_dir: /home/riga/NeRF/nerf_data/customData 35 | train_dataset_cfg: 36 | database_name: custom/shoe/raw_1600 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: custom/shoe/raw_1600 43 | 44 | ####trainier#### 45 | optimizer_type: adam 46 | lr_type: warm_up_cos 47 | lr_cfg: {} 48 | total_step: 180000 # 300000 49 | val_interval: 2500 # 5000 50 | save_interval: 1000 51 | train_log_step: 20 52 | N_voxel_init: 2097153 # 128**3 + 1 53 | N_voxel_final: 134217729 # 512**3 + 1 54 | upsample_list: [40000, 80000] 55 | update_AlphaMask_lst: [40000, 80000] # update_AlphaMask_lst: [60000] 56 | hessian_ratio: [0.1, 0.05] 57 | 58 | ###SDF setting### 59 | sdf_n_comp: 36 60 | sdf_dim: 256 61 | app_dim: 128 62 | 63 | ###Mesh extraction### 64 | blend_ratio: 0.6 -------------------------------------------------------------------------------- /configs/shape/nero/angel.yaml: -------------------------------------------------------------------------------- 1 | name: angel_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/angel 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: false 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.05 29 | 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/angel 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/angel 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/bell.yaml: -------------------------------------------------------------------------------- 1 | name: bell_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/bell 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'mask', 'Gaussain'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.05 29 | 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/bell 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/bell 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/cat.yaml: -------------------------------------------------------------------------------- 1 | name: cat_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/cat 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: false 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.01 29 | 30 | ####dataset##### 31 | train_dataset_type: dummy 32 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 33 | train_dataset_cfg: 34 | database_name: syn/cat 35 | split_manul: false 36 | val_set_list: 37 | - 38 | name: val 39 | type: dummy 40 | cfg: 41 | database_name: syn/cat 42 | split_manul: false 43 | 44 | ####trainier#### 45 | optimizer_type: adam 46 | lr_type: warm_up_cos 47 | lr_cfg: {} 48 | total_step: 180000 49 | val_interval: 2500 50 | save_interval: 1000 51 | train_log_step: 20 52 | N_voxel_init: 2097153 # 128**3 + 1 53 | N_voxel_final: 134217729 # 512**3 + 1 54 | upsample_list: [40000, 80000] 55 | update_AlphaMask_lst: [40000, 80000] 56 | hessian_ratio: [0.1, 0.05] 57 | 58 | ###SDF setting### 59 | sdf_n_comp: 36 60 | sdf_dim: 256 61 | app_dim: 128 62 | 63 | ###Mesh extraction### 64 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/horse.yaml: -------------------------------------------------------------------------------- 1 | name: horse_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/horse 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: false 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.05 29 | 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/horse 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/horse 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/luyu.yaml: -------------------------------------------------------------------------------- 1 | name: luyu_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/luyu 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: false 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.01 29 | 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/luyu 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/luyu 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/potion.yaml: -------------------------------------------------------------------------------- 1 | name: potion_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/potion 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'Gaussian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.01 29 | gaussian_weight: 0.00001 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/potion 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/potion 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/tbell.yaml: -------------------------------------------------------------------------------- 1 | name: tbell_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/tbell 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'Gaussian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.01 29 | gaussian_weight: 0.00001 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/tbell 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/tbell 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/nero/teapot.yaml: -------------------------------------------------------------------------------- 1 | name: teapot_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: syn/teapot 6 | split_manul: false 7 | nerfDataType: false 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: false 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.0001 20 | mul_length: 0 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'Hessian', 'Gaussian', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | hessian_weight: 0.0005 28 | sparse_weight: 0.01 29 | gaussian_weight: 0.00001 30 | 31 | ####dataset##### 32 | train_dataset_type: dummy 33 | dataset_dir: /home/riga/NeRF/nerf_data/GlossySynthetic # !!!! change your dataset dir !!!! 34 | train_dataset_cfg: 35 | database_name: syn/teapot 36 | split_manul: false 37 | val_set_list: 38 | - 39 | name: val 40 | type: dummy 41 | cfg: 42 | database_name: syn/teapot 43 | split_manul: false 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | upsample_list: [40000, 80000] 56 | update_AlphaMask_lst: [40000, 80000] 57 | hessian_ratio: [0.1, 0.05] 58 | 59 | ###SDF setting### 60 | sdf_n_comp: 36 61 | sdf_dim: 256 62 | app_dim: 128 63 | 64 | ###Mesh extraction### 65 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/orb/cactus.yaml: -------------------------------------------------------------------------------- 1 | name: cactus_scene001_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: orb/cactus_scene001 6 | nerfDataType: True 7 | apply_occ_loss: true 8 | occ_loss_step: 10000 # 20000 9 | apply_mask_loss: true 10 | clip_sample_variance: false 11 | has_radiance_field: true 12 | radiance_field_step: 20000 13 | apply_gaussian_loss: true 14 | gaussianLoss_step: 20000 15 | predict_BG: false 16 | isBGWhite: true 17 | downsample_ratio: 0.25 18 | alphaMask_thres: 0.00001 19 | mul_length: 15 20 | 21 | ######loss###### 22 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 23 | val_metric: ['shape_render'] 24 | key_metric_name: psnr 25 | eikonal_weight: 0.1 26 | mask_loss_weight: 0.5 27 | hessian_weight: 0.0005 28 | gaussian_weight: 0.00001 29 | sparse_weight: 0.1 30 | sparse_ratio: [0.2, 0.2] 31 | freeze_inv_s_step: 8000 # 15000 32 | 33 | ####dataset##### 34 | train_dataset_type: dummy 35 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 36 | train_dataset_cfg: 37 | database_name: orb/cactus_scene001 38 | val_set_list: 39 | - 40 | name: val 41 | type: dummy 42 | cfg: 43 | database_name: orb/cactus_scene001 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | step_ratio: 2.5 56 | upsample_list: [20000, 40000] 57 | update_AlphaMask_lst: [20000] 58 | hessian_ratio: [0.1, 0.05] 59 | 60 | ###SDF setting### 61 | sdf_n_comp: 36 62 | sdf_dim: 256 63 | app_dim: 128 64 | 65 | ###Mesh extraction### 66 | blend_ratio: 0.8 -------------------------------------------------------------------------------- /configs/shape/orb/car.yaml: -------------------------------------------------------------------------------- 1 | name: car_scene004_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: orb/car_scene004 6 | nerfDataType: True 7 | apply_occ_loss: true 8 | occ_loss_step: 10000 # 20000 9 | apply_mask_loss: true 10 | clip_sample_variance: false 11 | has_radiance_field: true 12 | radiance_field_step: 20000 13 | apply_gaussian_loss: false 14 | gaussianLoss_step: 20000 15 | predict_BG: false 16 | isBGWhite: true 17 | downsample_ratio: 0.25 18 | alphaMask_thres: 0.00001 19 | mul_length: 15 20 | 21 | ######loss###### 22 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask'] 23 | val_metric: ['shape_render'] 24 | key_metric_name: psnr 25 | eikonal_weight: 0.1 26 | mask_loss_weight: 0.5 27 | hessian_weight: 0.0005 28 | gaussian_weight: 0.00001 29 | sparse_weight: 0.1 30 | sparse_ratio: [0.2, 0.2] 31 | freeze_inv_s_step: 8000 # 15000 32 | 33 | ####dataset##### 34 | train_dataset_type: dummy 35 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 36 | train_dataset_cfg: 37 | database_name: orb/car_scene004 38 | val_set_list: 39 | - 40 | name: val 41 | type: dummy 42 | cfg: 43 | database_name: orb/car_scene004 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | step_ratio: 2.5 56 | upsample_list: [20000, 40000] 57 | update_AlphaMask_lst: [20000] 58 | hessian_ratio: [0.1, 0.05] 59 | 60 | ###SDF setting### 61 | sdf_n_comp: 36 62 | sdf_dim: 256 63 | app_dim: 128 64 | 65 | ###Mesh extraction### 66 | blend_ratio: 0.9 -------------------------------------------------------------------------------- /configs/shape/orb/gnome.yaml: -------------------------------------------------------------------------------- 1 | name: gnome_scene003_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: orb/gnome_scene003 6 | nerfDataType: True 7 | apply_occ_loss: true 8 | occ_loss_step: 10000 # 20000 9 | apply_mask_loss: true 10 | clip_sample_variance: false 11 | has_radiance_field: true 12 | radiance_field_step: 20000 13 | apply_gaussian_loss: true 14 | gaussianLoss_step: 20000 15 | predict_BG: false 16 | isBGWhite: true 17 | downsample_ratio: 0.25 18 | alphaMask_thres: 0.00001 19 | mul_length: 15 20 | 21 | ######loss###### 22 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 23 | val_metric: ['shape_render'] 24 | key_metric_name: psnr 25 | eikonal_weight: 0.1 26 | mask_loss_weight: 0.5 27 | hessian_weight: 0.0005 28 | gaussian_weight: 0.00001 29 | sparse_weight: 0.1 30 | sparse_ratio: [0.2, 0.2] 31 | freeze_inv_s_step: 8000 # 15000 32 | 33 | ####dataset##### 34 | train_dataset_type: dummy 35 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 36 | train_dataset_cfg: 37 | database_name: orb/gnome_scene003 38 | val_set_list: 39 | - 40 | name: val 41 | type: dummy 42 | cfg: 43 | database_name: orb/gnome_scene003 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | step_ratio: 2.5 56 | upsample_list: [20000, 40000] 57 | update_AlphaMask_lst: [20000] 58 | hessian_ratio: [0.1, 0.05] 59 | 60 | ###SDF setting### 61 | sdf_n_comp: 36 62 | sdf_dim: 256 63 | app_dim: 128 64 | 65 | ###Mesh extraction### 66 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/orb/grogu.yaml: -------------------------------------------------------------------------------- 1 | name: grogu_scene001_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: orb/grogu_scene001 6 | nerfDataType: True 7 | apply_occ_loss: true 8 | occ_loss_step: 10000 # 20000 9 | apply_mask_loss: true 10 | clip_sample_variance: false 11 | has_radiance_field: true 12 | radiance_field_step: 20000 13 | apply_gaussian_loss: true 14 | gaussianLoss_step: 20000 15 | predict_BG: false 16 | isBGWhite: true 17 | downsample_ratio: 0.25 18 | alphaMask_thres: 0.00001 19 | mul_length: 15 20 | 21 | ######loss###### 22 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 23 | val_metric: ['shape_render'] 24 | key_metric_name: psnr 25 | eikonal_weight: 0.1 26 | mask_loss_weight: 0.5 27 | hessian_weight: 0.0005 28 | gaussian_weight: 0.00001 29 | sparse_weight: 0.1 30 | sparse_ratio: [0.2, 0.2] 31 | freeze_inv_s_step: 8000 # 15000 32 | 33 | ####dataset##### 34 | train_dataset_type: dummy 35 | dataset_dir: /nerf_data/llff/blender_LDR # !!!! change your dataset dir !!!! 36 | train_dataset_cfg: 37 | database_name: orb/grogu_scene001 38 | val_set_list: 39 | - 40 | name: val 41 | type: dummy 42 | cfg: 43 | database_name: orb/grogu_scene001 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | step_ratio: 2.5 56 | upsample_list: [20000, 40000] 57 | update_AlphaMask_lst: [20000] 58 | hessian_ratio: [0.1, 0.05] 59 | 60 | ###SDF setting### 61 | sdf_n_comp: 36 62 | sdf_dim: 256 63 | app_dim: 128 64 | 65 | ###Mesh extraction### 66 | blend_ratio: 0.8 -------------------------------------------------------------------------------- /configs/shape/orb/teapot.yaml: -------------------------------------------------------------------------------- 1 | name: teapot_scene006_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: orb/teapot_scene006 6 | nerfDataType: True 7 | apply_occ_loss: true 8 | occ_loss_step: 10000 # 20000 9 | apply_mask_loss: true 10 | clip_sample_variance: false 11 | has_radiance_field: true 12 | radiance_field_step: 20000 13 | apply_gaussian_loss: true 14 | gaussianLoss_step: 20000 15 | predict_BG: false 16 | isBGWhite: true 17 | downsample_ratio: 0.25 18 | alphaMask_thres: 0.00001 19 | mul_length: 15 20 | 21 | ######loss###### 22 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 23 | val_metric: ['shape_render'] 24 | key_metric_name: psnr 25 | eikonal_weight: 0.1 26 | mask_loss_weight: 0.5 27 | hessian_weight: 0.0005 28 | gaussian_weight: 0.00001 29 | sparse_weight: 0.1 30 | sparse_ratio: [0.2, 0.2] 31 | freeze_inv_s_step: 8000 # 15000 32 | 33 | ####dataset##### 34 | train_dataset_type: dummy 35 | dataset_dir: /home/riga/NeRF/nerf_data/blender_LDR # !!!! change your dataset dir !!!! 36 | train_dataset_cfg: 37 | database_name: orb/teapot_scene006 38 | val_set_list: 39 | - 40 | name: val 41 | type: dummy 42 | cfg: 43 | database_name: orb/teapot_scene006 44 | 45 | ####trainier#### 46 | optimizer_type: adam 47 | lr_type: warm_up_cos 48 | lr_cfg: {} 49 | total_step: 180000 50 | val_interval: 2500 51 | save_interval: 1000 52 | train_log_step: 20 53 | N_voxel_init: 2097153 # 128**3 + 1 54 | N_voxel_final: 134217729 # 512**3 + 1 55 | step_ratio: 2.5 56 | upsample_list: [20000, 40000] 57 | update_AlphaMask_lst: [20000] 58 | hessian_ratio: [0.1, 0.05] 59 | 60 | ###SDF setting### 61 | sdf_n_comp: 36 62 | sdf_dim: 256 63 | app_dim: 128 64 | 65 | ###Mesh extraction### 66 | blend_ratio: 0.8 -------------------------------------------------------------------------------- /configs/shape/syn/FlightHelmet.yaml: -------------------------------------------------------------------------------- 1 | name: FlightHelmet_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoSDF/FlightHelmet 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoSDF/FlightHelmet 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoSDF/FlightHelmet 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 # 300000 53 | val_interval: 2500 # 5000 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.5 -------------------------------------------------------------------------------- /configs/shape/syn/armadillo.yaml: -------------------------------------------------------------------------------- 1 | name: armadillo_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoIR/armadillo 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoIR # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoIR/armadillo 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoIR/armadillo 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 53 | val_interval: 2500 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.0 -------------------------------------------------------------------------------- /configs/shape/syn/compressor.yaml: -------------------------------------------------------------------------------- 1 | name: compressor_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoSDF/compressor 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoSDF/compressor 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoSDF/compressor 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 53 | val_interval: 2500 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.2 -------------------------------------------------------------------------------- /configs/shape/syn/dragon.yaml: -------------------------------------------------------------------------------- 1 | name: dragon_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoSDF/dragon 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoSDF/dragon 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoSDF/dragon 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 # 300000 53 | val_interval: 2500 # 5000 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.0 -------------------------------------------------------------------------------- /configs/shape/syn/lego.yaml: -------------------------------------------------------------------------------- 1 | name: lego_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoIR/lego 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: false 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 20 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoIR # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoIR/lego 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoIR/lego 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 53 | val_interval: 2500 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.0 -------------------------------------------------------------------------------- /configs/shape/syn/motor.yaml: -------------------------------------------------------------------------------- 1 | name: motor_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoSDF/motor 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoSDF/motor 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoSDF/motor 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 53 | val_interval: 2500 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.2 -------------------------------------------------------------------------------- /configs/shape/syn/robot.yaml: -------------------------------------------------------------------------------- 1 | name: robot_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoSDF/robot 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoSDF/robot 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoSDF/robot 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 53 | val_interval: 2500 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.2 -------------------------------------------------------------------------------- /configs/shape/syn/rover.yaml: -------------------------------------------------------------------------------- 1 | name: rover_shape 2 | 3 | ####network##### 4 | network: shape 5 | database_name: tensoSDF/rover 6 | split_manul: true 7 | nerfDataType: True 8 | apply_occ_loss: true 9 | occ_loss_step: 10000 # 20000 10 | apply_mask_loss: true 11 | clip_sample_variance: false 12 | has_radiance_field: true 13 | radiance_field_step: 20000 14 | apply_gaussian_loss: true 15 | gaussianLoss_step: 20000 16 | predict_BG: false 17 | isBGWhite: true 18 | downsample_ratio: 0.5 19 | alphaMask_thres: 0.00001 20 | mul_length: 15 21 | 22 | ######loss###### 23 | loss: ['nerf_render','eikonal', 'std', 'init_sdf_reg','occ', 'Sparse', 'TV', 'mask', 'Gaussian'] 24 | val_metric: ['shape_render'] 25 | key_metric_name: psnr 26 | eikonal_weight: 0.1 27 | mask_loss_weight: 0.5 28 | hessian_weight: 0.0005 29 | gaussian_weight: 0.00001 30 | sparse_weight: 0.1 31 | sparse_ratio: [0.2, 0.2] 32 | freeze_inv_s_step: 8000 # 15000 33 | 34 | ####dataset##### 35 | train_dataset_type: dummy 36 | dataset_dir: /home/riga/NeRF/nerf_data/tensoSDF # !!!! change your dataset dir !!!! 37 | train_dataset_cfg: 38 | database_name: tensoSDF/rover 39 | split_manul: true 40 | val_set_list: 41 | - 42 | name: val 43 | type: dummy 44 | cfg: 45 | database_name: tensoSDF/rover 46 | split_manul: true 47 | 48 | ####trainier#### 49 | optimizer_type: adam 50 | lr_type: warm_up_cos 51 | lr_cfg: {} 52 | total_step: 180000 53 | val_interval: 2500 54 | save_interval: 1000 55 | train_log_step: 20 56 | N_voxel_init: 2097153 # 128**3 + 1 57 | N_voxel_final: 134217729 # 512**3 + 1 58 | step_ratio: 2.5 59 | upsample_list: [20000, 40000] 60 | update_AlphaMask_lst: [20000] 61 | hessian_ratio: [0.1, 0.05] 62 | 63 | ###SDF setting### 64 | sdf_n_comp: 36 65 | sdf_dim: 256 66 | app_dim: 128 67 | 68 | ###Mesh extraction### 69 | blend_ratio: 0.0 -------------------------------------------------------------------------------- /configs/synthetic_split_128.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Riga2/TensoSDF/6074ab11af1bfe9c50a0d30c9630c4a083289b8b/configs/synthetic_split_128.pkl -------------------------------------------------------------------------------- /dataset/name2dataset.py: -------------------------------------------------------------------------------- 1 | from dataset.train_dataset import DummyDataset 2 | 3 | name2dataset={ 4 | 'dummy': DummyDataset, 5 | } -------------------------------------------------------------------------------- /dataset/train_dataset.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data import Dataset 2 | 3 | from dataset.database import get_database_split, parse_database_name 4 | class DummyDataset(Dataset): 5 | default_cfg={ 6 | 'database_name': '', 7 | 'split_manul': False, 8 | } 9 | def __init__(self, cfg, is_train, dataset_dir): 10 | self.cfg={**self.default_cfg,**cfg} 11 | if not is_train: 12 | database = parse_database_name(self.cfg['database_name'], dataset_dir) 13 | split_manul = False 14 | if self.cfg['split_manul']: 15 | split_manul = True 16 | train_ids, test_ids = get_database_split(database, 'validation', split_manul) 17 | self.train_num = len(train_ids) 18 | self.test_num = len(test_ids) 19 | self.is_train = is_train 20 | 21 | def __getitem__(self, index): 22 | if self.is_train: 23 | return {} 24 | else: 25 | return {'index': index} 26 | 27 | def __len__(self): 28 | if self.is_train: 29 | return 99999999 30 | else: 31 | return self.test_num -------------------------------------------------------------------------------- /eval_geo.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" 3 | os.environ["CUDA_VISIBLE_DEVICES"] = '2' 4 | import random 5 | import argparse 6 | from utils.base_utils import load_cfg 7 | import torch 8 | import numpy as np 9 | from tqdm import tqdm 10 | 11 | from network.invRenderer import name2renderer 12 | from dataset.database import parse_database_name 13 | from skimage.metrics import peak_signal_noise_ratio as psnr, structural_similarity as ssim 14 | from utils.base_utils import color_map_backward, color_map_forward, rgb_lpips 15 | import cv2 16 | from torch.utils.data import DataLoader 17 | import matplotlib.pyplot as plt 18 | from network.other_field import GaussianBlur2D, GaussianBlur1D 19 | import torch.nn.functional as F 20 | from skimage.io import imread, imsave 21 | 22 | class ShapeTester: 23 | default_cfg={ 24 | "multi_gpus": False, 25 | "worker_num": 8, 26 | 'random_seed': 6033, 27 | 'isBGWhite': True, 28 | } 29 | 30 | def __init__(self, cfg): 31 | self.cfg={**self.default_cfg,**cfg} 32 | torch.manual_seed(self.cfg['random_seed']) 33 | np.random.seed(self.cfg['random_seed']) 34 | random.seed(self.cfg['random_seed']) 35 | self.model_name=cfg['name'] 36 | self.model_dir=os.path.join('data/model', cfg['name']) 37 | if not os.path.exists(self.model_dir): 38 | raise ImportError 39 | self.pth_fn=os.path.join(self.model_dir,'model.pth') 40 | self.best_pth_fn=os.path.join(self.model_dir,'model_best.pth') 41 | self.base_save_path = os.path.join('data/nvs', cfg['name']) 42 | os.makedirs(self.base_save_path, exist_ok=True) 43 | 44 | def _init_dataset(self): 45 | self.database = parse_database_name(self.cfg['database_name'], self.cfg['dataset_dir'], isTest=True, isWhiteBG=self.cfg['isBGWhite']) 46 | self.test_ids = self.database.get_img_ids() 47 | self.dataloader = DataLoader(self.test_ids, 1, False, num_workers=self.cfg['worker_num']) 48 | print(f'Test set len {len(self.test_ids)}') 49 | 50 | def _init_network(self): 51 | best_para,start_step=0,0 52 | if os.path.exists(self.pth_fn): 53 | checkpoint=torch.load(self.pth_fn) 54 | best_para = checkpoint['best_para'] 55 | start_step = checkpoint['step'] 56 | if 'kwargs' in checkpoint: 57 | kwargs = checkpoint['kwargs'] 58 | self.cfg.update(kwargs) 59 | self.network = name2renderer[self.cfg['network']](self.cfg).cuda() 60 | self.network.load_ckpt(checkpoint) 61 | print(f'==> resuming from step {start_step} best para {best_para}') 62 | else: 63 | raise NotImplementedError 64 | 65 | def run(self): 66 | self._init_dataset() 67 | self._init_network() 68 | 69 | num = len(self.test_ids) 70 | img_avg_psnr = 0 71 | img_avg_ssim = 0 72 | normal_avg_mae = 0 73 | def dir_maker(name): 74 | dir = os.path.join(self.base_save_path, name) 75 | os.makedirs(dir, exist_ok=True) 76 | return dir 77 | 78 | imgs_dir = dir_maker('imgs') 79 | normals_dir = dir_maker('normals') 80 | normals_vis_dir = dir_maker('normals_vis') 81 | albedo_dir = dir_maker('albedo') 82 | rough_dir = dir_maker('roughness') 83 | occ_dir = dir_maker('occ') 84 | diff_spec_color_dir = dir_maker('diff_spec_color') 85 | light_dir = dir_maker('light') 86 | radiance_dir = dir_maker('radiance') 87 | imgs_diff_dir = dir_maker('imgs_diff') 88 | 89 | save_func = lambda save_dir, index, im : cv2.imwrite(os.path.join(save_dir, str(int(index)) + '.png'), im[..., ::-1]) 90 | save_exr = lambda save_dir, index, im : cv2.imwrite(os.path.join(save_dir, str(int(index)) + '.exr'), im[..., ::-1]) 91 | per_res_msg = "" 92 | for _, ind in tqdm(enumerate(self.test_ids)): 93 | pose = self.database.get_pose(ind) 94 | K = self.database.get_K(ind) 95 | gt_imgs = self.database.get_image(ind) 96 | h, w = gt_imgs.shape[:2] 97 | with torch.no_grad(): 98 | outputs = self.network.nvs(pose, K, h, w) 99 | for k in outputs: 100 | if k != 'normal': 101 | outputs[k] = color_map_backward(outputs[k]) 102 | cur_psnr = psnr(gt_imgs, outputs['color']) 103 | cur_ssim = ssim(gt_imgs, outputs['color'], win_size=11, channel_axis=2, data_range=255) 104 | img_avg_psnr += cur_psnr 105 | img_avg_ssim += cur_ssim 106 | per_res_msg += f'{ind:03} psnr: {cur_psnr}, ssim: {cur_ssim}' 107 | 108 | gt_normals = self.database.get_normal(ind) 109 | gt_normals = normalize_numpy(gt_normals) 110 | cur_mae = np.mean(np.arccos(np.clip(np.sum(gt_normals * outputs['normal'], axis=-1), -1, 1)) * 180 / np.pi) 111 | normal_avg_mae += cur_mae 112 | per_res_msg += f', mae: {cur_mae}' 113 | 114 | normal_rgb = color_map_backward((outputs['normal'] + 1.0) * 0.5) 115 | gt_normal_rgb = color_map_backward((gt_normals + 1.0) * 0.5) 116 | normal_diff_map = np.repeat(color_map_backward(np.sum(np.power(outputs['normal'] - gt_normals, 2), axis=-1, keepdims=True)), 3, axis=-1) 117 | save_func(normals_dir, ind, np.concatenate([normal_rgb, gt_normal_rgb, normal_diff_map], axis=1)) 118 | 119 | per_res_msg += '\n' 120 | color_diff = np.repeat(np.clip(np.sum(np.abs(outputs['color'].astype(np.float32) - gt_imgs.astype(np.float32)), axis=-1, keepdims=True), a_min=0, a_max=255.0).astype(np.uint8), 3, axis=-1) 121 | save_func(imgs_diff_dir, ind, color_diff) 122 | save_func(imgs_dir, ind, outputs['color']) 123 | save_func(albedo_dir, ind, outputs['albedo']) 124 | save_func(rough_dir, ind, outputs['roughness']) 125 | save_func(radiance_dir, ind, outputs['radiance']) 126 | save_func(occ_dir, ind, np.concatenate((outputs['occ_predict'], outputs['occ_trace']), axis=1)) 127 | save_func(diff_spec_color_dir, ind, np.concatenate((outputs['diff_color'], outputs['spec_color']), axis=1)) 128 | save_func(light_dir, ind, np.concatenate((outputs['diff_light'], outputs['spec_light'], outputs['indirect_light']), axis=1)) 129 | save_func(normals_vis_dir, ind, outputs['normal_vis']) 130 | 131 | img_avg_psnr /= num 132 | img_avg_ssim /= num 133 | normal_avg_mae /= num 134 | 135 | saved_message = f'{self.model_name}: \n' \ 136 | + f'\tPSNR_nvs: {img_avg_psnr:.3f}, SSIM_nvs: {img_avg_ssim:.5f}' \ 137 | + f', Normal_MAE_nvs: {normal_avg_mae:.5f}' 138 | with open(f'{self.base_save_path}/metrics_record.txt', 'a') as f: 139 | f.write(saved_message) 140 | print(saved_message) 141 | with open(f'{self.base_save_path}/per_record.txt', 'a') as f: 142 | f.write(per_res_msg) 143 | 144 | 145 | def normalize_numpy(x, axis=-1, order=2): 146 | norm = np.linalg.norm(x, ord=order, axis=axis, keepdims=True) 147 | return x / np.maximum(norm, np.finfo(float).eps) 148 | 149 | 150 | if __name__ == '__main__': 151 | parser = argparse.ArgumentParser() 152 | parser.add_argument('--cfg', type=str, default='configs/shape/syn/motor.yaml') 153 | flags = parser.parse_args() 154 | ShapeTester(load_cfg(flags.cfg)).run() -------------------------------------------------------------------------------- /eval_orb_shape.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import trimesh 3 | from scipy.spatial import cKDTree as KDTree 4 | import logging 5 | import argparse 6 | import os 7 | from utils.base_utils import load_cfg 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | target_volume = 900 # roughly corresponds to a witdth of 10cm 12 | num_points_align = 1000 13 | max_iterations = 100 14 | cost_threshold = 0 15 | num_points_chamfer = 30000 16 | 17 | 18 | def sample_surface_point(mesh, num_points, even=False): 19 | if even: 20 | sample_points, indexes = trimesh.sample.sample_surface_even(mesh, count=num_points) 21 | while len(sample_points) < num_points: 22 | more_sample_points, indexes = trimesh.sample.sample_surface_even(mesh, count=num_points) 23 | sample_points = np.concatenate([sample_points, more_sample_points], axis=0) 24 | else: 25 | sample_points, indexes = trimesh.sample.sample_surface(mesh, count=num_points) 26 | return sample_points[:num_points] 27 | 28 | 29 | def load_mesh(fpath: str) -> trimesh.Trimesh: 30 | if fpath.endswith('.npz'): 31 | mesh_npz = np.load(fpath) 32 | verts = mesh_npz['verts'] 33 | faces = mesh_npz['faces'] 34 | faces = np.concatenate((faces, faces[:, list(reversed(range(faces.shape[-1])))]), axis=0) 35 | mesh = trimesh.Trimesh(vertices=verts, faces=faces) 36 | else: 37 | mesh = trimesh.load_mesh(fpath) 38 | return mesh 39 | 40 | 41 | # https://github.com/facebookresearch/DeepSDF/blob/main/deep_sdf/metrics/chamfer.py 42 | def compute_trimesh_chamfer(gt_points, gen_mesh, num_mesh_samples=30000): 43 | """ 44 | This function computes a symmetric chamfer distance, i.e. the sum of both chamfers. 45 | gt_points: trimesh.points.PointCloud of just poins, sampled from the surface (see 46 | compute_metrics.ply for more documentation) 47 | gen_mesh: trimesh.base.Trimesh of output mesh from whichever autoencoding reconstruction 48 | method (see compute_metrics.py for more) 49 | """ 50 | if gen_mesh is None: 51 | gen_points_sampled = np.zeros((num_mesh_samples, 3)) 52 | else: 53 | gen_points_sampled = trimesh.sample.sample_surface(gen_mesh, num_mesh_samples)[0] 54 | 55 | # only need numpy array of points 56 | gt_points_np = gt_points.vertices 57 | 58 | # one direction 59 | gen_points_kd_tree = KDTree(gen_points_sampled) 60 | one_distances, one_vertex_ids = gen_points_kd_tree.query(gt_points_np) 61 | gt_to_gen_chamfer = np.mean(np.square(one_distances)) 62 | 63 | # other direction 64 | gt_points_kd_tree = KDTree(gt_points_np) 65 | two_distances, two_vertex_ids = gt_points_kd_tree.query(gen_points_sampled) 66 | gen_to_gt_chamfer = np.mean(np.square(two_distances)) 67 | 68 | return gt_to_gen_chamfer, gen_to_gt_chamfer 69 | 70 | 71 | def compute_shape_score(output_mesh, target_mesh): 72 | if output_mesh is None: 73 | logger.error('output mesh not found') 74 | return {} 75 | try: 76 | mesh_result = load_mesh(output_mesh) 77 | except ValueError: 78 | import traceback; traceback.print_exc() 79 | mesh_result = None 80 | mesh_scan = load_mesh(target_mesh) 81 | gt_to_gen_chamfer, gen_to_gt_chamfer = compute_trimesh_chamfer(mesh_scan, mesh_result, num_points_chamfer) 82 | bidir_chamfer = (gt_to_gen_chamfer + gen_to_gt_chamfer) / 2. 83 | return {'bidir_chamfer': bidir_chamfer} 84 | 85 | 86 | def calculate_scale(mesh, target_volume, method='volume'): 87 | if method == 'bounding_box': 88 | width, height, length = mesh.extents 89 | bounding_box_volume = (width * height * length) 90 | scale = (target_volume / bounding_box_volume)**(1/3) 91 | elif method == 'volume': 92 | voxel_length = mesh.extents.min() /100 93 | voxel = mesh.voxelized(voxel_length).fill() 94 | voxel_volume = voxel.volume 95 | scale = (target_volume / voxel_volume)**(1/3) 96 | return scale 97 | 98 | def main(out_mesh_path, target_mesh_path): 99 | cd = compute_shape_score(out_mesh_path, target_mesh_path) 100 | orb_records = f'{out_mesh_path}: {cd}' 101 | print(cd) 102 | shape_metrics_dir = 'data/meshes/orb' 103 | os.makedirs(shape_metrics_dir, exist_ok=True) 104 | with open(f'{shape_metrics_dir}/orb_records.txt','a') as f: 105 | f.write(orb_records+'\n') 106 | 107 | if __name__ == '__main__': 108 | parser = argparse.ArgumentParser() 109 | parser.add_argument('--out_mesh_path', type=str, default='data/meshes/orb/cactus_scene001_shape-180000.ply') 110 | parser.add_argument('--target_mesh_path', type=str, default='/home/riga/NeRF/nerf_data/ground_truth/cactus_scene001/mesh_blender/mesh.obj') 111 | 112 | flags = parser.parse_args() 113 | 114 | main(flags.out_mesh_path, flags.target_mesh_path) -------------------------------------------------------------------------------- /extract_mesh.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | os.environ["CUDA_VISIBLE_DEVICES"] = '0' 4 | from pathlib import Path 5 | import sys 6 | 7 | import torch 8 | import trimesh 9 | from network.invRenderer import name2renderer 10 | from utils.base_utils import load_cfg 11 | from utils.network_utils import extract_geometry 12 | 13 | def main(): 14 | cfg = load_cfg(flags.cfg) 15 | 16 | if flags.model == 'best': 17 | ckpt = torch.load(f'data/model/{cfg["name"]}/model_best.pth') 18 | else: 19 | ckpt = torch.load(f'data/model/{cfg["name"]}/model.pth') 20 | step = ckpt['step'] 21 | kwargs = ckpt['kwargs'] 22 | cfg.update(kwargs) 23 | network = name2renderer[cfg['network']](cfg, training=False) 24 | network.load_ckpt(ckpt) 25 | network.eval().cuda() 26 | torch.set_default_tensor_type('torch.cuda.FloatTensor') 27 | print(f'successfully load {cfg["name"]} step {step}!') 28 | 29 | bbox_min = -torch.ones(3) 30 | bbox_max = torch.ones(3) 31 | ratio = cfg['blend_ratio'] 32 | print(f'Blend ratio: {ratio}') 33 | with torch.no_grad(): 34 | def func(x): 35 | level0 = torch.zeros(x.shape[:-1] + (1, )) 36 | level1 = torch.ones(x.shape[:-1] + (1, )) 37 | sdf = network.sdf_network.sdf(x, level0) * (1.0 - ratio) + network.sdf_network.sdf(x, level1) * ratio 38 | return sdf 39 | vertices, triangles = extract_geometry(bbox_min, bbox_max, flags.resolution, 0, func) 40 | 41 | # output geometry 42 | mesh = trimesh.Trimesh(vertices, triangles) 43 | output_dir = Path('data/meshes') 44 | output_dir.mkdir(exist_ok=True) 45 | mesh.export(str(output_dir/f'{cfg["name"]}-{step}.ply')) 46 | 47 | 48 | if __name__ == "__main__": 49 | parser = argparse.ArgumentParser() 50 | parser.add_argument('--cfg', type=str, default='configs/shape/syn/lego.yaml') 51 | parser.add_argument('--resolution', type=int, default=512) 52 | parser.add_argument('--model', type=str, choices=['latest', 'best'], default='latest') 53 | flags = parser.parse_args() 54 | main() -------------------------------------------------------------------------------- /network/invRenderer.py: -------------------------------------------------------------------------------- 1 | from network.shapeRenderer import ShapeRenderer 2 | from network.materialRenderer import MaterialRenderer 3 | 4 | name2renderer={ 5 | 'shape' : ShapeRenderer, 6 | 'material': MaterialRenderer, 7 | } 8 | -------------------------------------------------------------------------------- /network/metrics.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | from skimage.io import imsave 5 | 6 | from network.loss import Loss 7 | from utils.base_utils import color_map_backward 8 | from utils.draw_utils import concat_images_list 9 | from skimage.metrics import structural_similarity 10 | 11 | def compute_psnr(img_gt, img_pr): 12 | img_gt = img_gt.reshape([-1, 3]).astype(np.float32) 13 | img_pr = img_pr.reshape([-1, 3]).astype(np.float32) 14 | mse = np.mean((img_gt - img_pr) ** 2, 0) 15 | mse = np.mean(mse) 16 | psnr = 10 * np.log10(255 * 255 / mse) 17 | return psnr 18 | 19 | def process_key_img(data, h, w): 20 | img = color_map_backward(data.detach().cpu().numpy()) 21 | img = img.reshape([h, w, -1]) 22 | if img.shape[-1] == 1: img = np.repeat(img, 3, axis=-1) 23 | return img 24 | 25 | def get_key_images(data_pr, keys, h, w): 26 | results=[] 27 | for k in keys: 28 | if k in data_pr: results.append(process_key_img(data_pr[k], h, w)) 29 | return results 30 | 31 | def draw_materials(data_pr, h, w): 32 | keys=['diffuse_albedo', 'diffuse_light', 'diffuse_color', 33 | 'specular_albedo', 'specular_light', 'specular_color', 'specular_direct_light', 34 | 'metallic', 'roughness', 'occ_prob', 'indirect_light'] 35 | results = get_key_images(data_pr, keys, h, w) 36 | results = [concat_images_list(*results[0:3]),concat_images_list(*results[3:7]),concat_images_list(*results[7:])] 37 | return results 38 | 39 | class ShapeRenderMetrics(Loss): 40 | def __init__(self, cfg): 41 | pass 42 | 43 | def __call__(self, data_pr, data_gt, step, **kwargs): 44 | rgb_gt = color_map_backward(data_pr['gt_rgb'].detach().cpu().numpy()) # h,w,3 45 | imgs = [rgb_gt] 46 | 47 | # compute psnr 48 | rgb_pr = color_map_backward(data_pr['ray_rgb'].detach().cpu().numpy()) # h,w,3 49 | psnr = compute_psnr(rgb_gt, rgb_pr) 50 | ssim = structural_similarity(rgb_gt, rgb_pr, win_size=11, channel_axis=2, data_range=255) 51 | outputs={'psnr': np.asarray([psnr]),'ssim': np.asarray([ssim])} 52 | imgs.append(rgb_pr) 53 | 54 | # normal 55 | h, w, _ = rgb_pr.shape 56 | normal = color_map_backward(data_pr['normal_vis'].detach().cpu().numpy()) # h,w,3 57 | imgs.append(normal.reshape([h,w,3])) 58 | 59 | if 'human_light' in data_pr: 60 | imgs.append(process_key_img(data_pr['human_light'], h, w)) 61 | 62 | if 'radiance' in data_pr: 63 | imgs.append(color_map_backward(data_pr['radiance'].detach().cpu().numpy())) # h,w,3 64 | elif 'acc' in data_pr: 65 | imgs.append(process_key_img(data_pr['acc'], h, w)) 66 | 67 | imgs = [concat_images_list(*imgs)] 68 | 69 | imgs += draw_materials(data_pr, h, w) 70 | 71 | # output image 72 | data_index=kwargs['data_index'] 73 | model_name=kwargs['model_name'] 74 | output_path = Path(f'data/train_vis/{model_name}') 75 | output_path.mkdir(exist_ok=True, parents=True) 76 | imsave(f'{str(output_path)}/{step}-index-{data_index}.jpg', concat_images_list(*imgs, vert=True)) 77 | return outputs 78 | 79 | class MaterialRenderMetrics(Loss): 80 | def __init__(self, cfg): 81 | pass 82 | 83 | def __call__(self, data_pr, data_gt, step, *args, **kwargs): 84 | rgb_gt = color_map_backward(data_pr['rgb_gt'].detach().cpu().numpy()) # h,w,3 85 | rgb_pr = color_map_backward(data_pr['rgb_pr'].detach().cpu().numpy()) # h,w,3 86 | imgs = [rgb_gt, rgb_pr] 87 | 88 | # compute psnr 89 | psnr = compute_psnr(rgb_gt, rgb_pr) 90 | ssim = structural_similarity(rgb_gt, rgb_pr, win_size=11, channel_axis=2, data_range=255) 91 | outputs={'psnr': np.asarray([psnr]),'ssim': np.asarray([ssim])} 92 | 93 | additional_keys = ['albedo', 'metallic', 'roughness', 'specular_light', 'specular_color', 'diffuse_light', 'diffuse_color', 'occ_trace', 'indirect_light'] 94 | for k in additional_keys: 95 | img = color_map_backward(data_pr[k].detach().cpu().numpy()) 96 | if img.shape[-1] == 1: img = np.repeat(img, 3, axis=-1) 97 | imgs.append(img) 98 | 99 | output_imgs = [concat_images_list(*imgs[:5]),concat_images_list(*imgs[5:])] 100 | 101 | # output image 102 | data_index=kwargs['data_index'] 103 | model_name=kwargs['model_name'] 104 | output_path = Path(f'data/train_vis/{model_name}') 105 | output_path.mkdir(exist_ok=True, parents=True) 106 | imsave(f'{str(output_path)}/{step}-index-{data_index}.jpg', concat_images_list(*output_imgs, vert=True)) 107 | return outputs 108 | 109 | name2metrics={ 110 | 'shape_render': ShapeRenderMetrics, 111 | 'mat_render': MaterialRenderMetrics, 112 | } 113 | 114 | def psnr(results): 115 | return np.mean(results['psnr']) 116 | 117 | 118 | name2key_metrics={ 119 | 'psnr': psnr, 120 | } 121 | -------------------------------------------------------------------------------- /raytracing/__init__.py: -------------------------------------------------------------------------------- 1 | from .raytracer import RayTracer -------------------------------------------------------------------------------- /raytracing/raytracer.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import torch 4 | 5 | # CUDA extension 6 | import _raytracing as _backend 7 | 8 | class RayTracer(): 9 | def __init__(self, vertices, triangles): 10 | # vertices: np.ndarray, [N, 3] 11 | # triangles: np.ndarray, [M, 3] 12 | 13 | if torch.is_tensor(vertices): vertices = vertices.detach().cpu().numpy() 14 | if torch.is_tensor(triangles): triangles = triangles.detach().cpu().numpy() 15 | 16 | assert triangles.shape[0] > 8, "BVH needs at least 8 triangles." 17 | 18 | # implementation 19 | self.impl = _backend.create_raytracer(vertices, triangles) 20 | 21 | def trace(self, rays_o, rays_d, inplace=False): 22 | # rays_o: torch.Tensor, cuda, float, [N, 3] 23 | # rays_d: torch.Tensor, cuda, float, [N, 3] 24 | # inplace: write positions to rays_o, face_normals to rays_d 25 | 26 | rays_o = rays_o.float().contiguous() 27 | rays_d = rays_d.float().contiguous() 28 | 29 | if not rays_o.is_cuda: rays_o = rays_o.cuda() 30 | if not rays_d.is_cuda: rays_d = rays_d.cuda() 31 | 32 | prefix = rays_o.shape[:-1] 33 | rays_o = rays_o.view(-1, 3) 34 | rays_d = rays_d.view(-1, 3) 35 | 36 | N = rays_o.shape[0] 37 | 38 | if not inplace: 39 | # allocate 40 | positions = torch.empty_like(rays_o) 41 | face_normals = torch.empty_like(rays_d) 42 | else: 43 | positions = rays_o 44 | face_normals = rays_d 45 | 46 | depth = torch.empty_like(rays_o[:, 0]) 47 | 48 | # inplace write intersections back to rays_o 49 | self.impl.trace(rays_o, rays_d, positions, face_normals, depth) # [N, 3] 50 | 51 | positions = positions.view(*prefix, 3) 52 | face_normals = face_normals.view(*prefix, 3) 53 | depth = depth.view(*prefix) 54 | 55 | return positions, face_normals, depth -------------------------------------------------------------------------------- /relight_orb.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import subprocess 3 | 4 | 5 | if __name__ == "__main__": 6 | env_name = 'cactus_scene007' 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('--blender', type=str, default='/home/riga/blender-3.6.5-linux-x64/blender') 9 | parser.add_argument('--mesh', type=str, default='data/meshes/orb/cactus_scene001_shape-180000.ply') 10 | parser.add_argument('--material', type=str, default='data/materials/cactus_scene001_mat-100000') 11 | parser.add_argument('--env_name', type=str, default=env_name) 12 | parser.add_argument('--hdr', type=str, default=f'/home/riga/NeRF/nerf_data/ground_truth/{env_name}/env_map') 13 | parser.add_argument('--gt', type=str, default='/home/riga/NeRF/nerf_data/blender_LDR/cactus_scene001') 14 | parser.add_argument('--name', type=str, default=f'cactus_scene001_relighting_{env_name}') 15 | parser.add_argument('--trans', type=bool, default=False) 16 | parser.add_argument('--dataset', type=str, default='orb', choices=['nerf', 'nero', 'tensoSDF', 'orb']) 17 | args = parser.parse_args() 18 | 19 | cmds=[ 20 | args.blender, '--background', '--python', 'blender_backend/relight_backend.py', '--', 21 | '--output', f'data/relight/orb/noScale/{args.name}', 22 | '--mesh', args.mesh, 23 | '--material', args.material, 24 | '--env_fn', args.hdr, 25 | '--gt', args.gt, 26 | '--dataset', args.dataset, 27 | '--env_name', args.env_name, 28 | ] 29 | if args.trans: 30 | cmds.append('--trans') 31 | subprocess.run(cmds) 32 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorboardX 2 | # torch==1.11.0 3 | numpy 4 | open3d==0.16.0 5 | pymcubes 6 | plyfile 7 | h5py 8 | transforms3d 9 | tqdm 10 | opencv-python 11 | pyyaml 12 | scikit-image 13 | ghalton 14 | humanfriendly -------------------------------------------------------------------------------- /run_colmap.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import subprocess 3 | import os 4 | from pathlib import Path 5 | 6 | import numpy as np 7 | from skimage.io import imread 8 | 9 | from colmap.database import COLMAPDatabase 10 | from colmap.read_write_model import CAMERA_MODEL_NAMES 11 | 12 | def run_sfm(image_dir, project_dir, same_camera=False, colmap_path='$HOME/code/colmap/build/src/exe/colmap'): 13 | Path(project_dir).mkdir(exist_ok=True, parents=True) 14 | # create database for all images 15 | db = COLMAPDatabase.connect(f'{project_dir}/database.db') 16 | db.create_tables() 17 | 18 | # add images 19 | img_dir = Path(image_dir) 20 | img_fns = [] 21 | for pattern in ['*.jpg','*.png','*.PNG','*.JPG']: 22 | img_fns+=[fn for fn in img_dir.glob(pattern)] 23 | img_fns = sorted(img_fns) 24 | global_cam_id = None 25 | for k, img_fn in enumerate(img_fns): 26 | img = imread(img_fn) 27 | h, w, _ = img.shape 28 | focal = np.sqrt(h**2+w**2) # guess a focal here 29 | if same_camera: 30 | if k==0: global_cam_id = db.add_camera(CAMERA_MODEL_NAMES['SIMPLE_PINHOLE'].model_id, 31 | float(w), float(h), np.array([focal, w/2, h/2], np.float64), prior_focal_length=True) 32 | db.add_image(img_fn.name, global_cam_id) 33 | else: 34 | cam_id = db.add_camera(CAMERA_MODEL_NAMES['SIMPLE_PINHOLE'].model_id, 35 | float(w), float(h), np.array([focal, w/2,h/2],np.float64),prior_focal_length=True) 36 | db.add_image(img_fn.name, cam_id) 37 | 38 | db.commit() 39 | db.close() 40 | 41 | # feature extraction 42 | cmd=[colmap_path,'feature_extractor', 43 | '--database_path',f'{project_dir}/database.db', 44 | '--image_path',f'{image_dir}'] 45 | print(' '.join(cmd)) 46 | subprocess.run(cmd,check=True) 47 | 48 | # feature matching 49 | cmd=[colmap_path,'exhaustive_matcher', 50 | '--database_path',f'{project_dir}/database.db'] 51 | print(' '.join(cmd)) 52 | subprocess.run(cmd,check=True) 53 | 54 | # SfM 55 | Path(f'{project_dir}/sparse').mkdir(exist_ok=True,parents=True) 56 | cmd=[colmap_path,'mapper', 57 | '--database_path',f'{project_dir}/database.db', 58 | '--image_path',f'{image_dir}', 59 | '--output_path',f'{project_dir}/sparse'] 60 | print(' '.join(cmd)) 61 | subprocess.run(cmd,check=True) 62 | 63 | # dense reconstruction 64 | Path(f'{project_dir}/dense').mkdir(exist_ok=True,parents=True) 65 | cmd=[colmap_path,'image_undistorter', 66 | '--image_path',f'{image_dir}', 67 | '--input_path',f'{project_dir}/sparse/0', 68 | '--output_path',f'{project_dir}/dense'] 69 | print(' '.join(cmd)) 70 | subprocess.run(cmd,check=True) 71 | 72 | cmd=[colmap_path,'patch_match_stereo', 73 | '--workspace_path',f'{project_dir}/dense'] 74 | print(' '.join(cmd)) 75 | subprocess.run(cmd,check=True) 76 | 77 | cmd = [str(colmap_path), 'stereo_fusion', 78 | '--workspace_path', f'{project_dir}/dense', 79 | '--workspace_format', 'COLMAP', 80 | '--input_type', 'geometric', 81 | '--output_path', f'{project_dir}/points.ply', 82 | # '--StereoFusion.num_threads','14', 83 | '--StereoFusion.check_num_images','5', 84 | # '--StereoFusion.cache_size','14', 85 | # '--StereoFusion.use_cache','1', 86 | # '--StereoFusion.max_image_size','1024' 87 | ] 88 | print(' '.join(cmd)) 89 | subprocess.run(cmd, check=True) 90 | 91 | def main(): 92 | parser = argparse.ArgumentParser() 93 | parser.add_argument('--project_dir', type=str, default='/nerf_data/nero_real/casserole') 94 | parser.add_argument('--colmap', type=str, default='/home/czl/Desktop/colmap/build/src/colmap/exe/colmap') 95 | parser.add_argument('--same_camera', action='store_true', default=True, dest='same_camera') 96 | args = parser.parse_args() 97 | image_dir = f'{args.project_dir}/images' 98 | run_sfm(image_dir, args.project_dir, args.same_camera, colmap_path=args.colmap) 99 | 100 | if __name__=="__main__": 101 | main() 102 | -------------------------------------------------------------------------------- /run_training.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ["CUDA_VISIBLE_DEVICES"] = '0' 3 | import argparse 4 | from humanfriendly import format_timespan 5 | 6 | from train.trainer_inv import TrainerInv 7 | from utils.base_utils import load_cfg 8 | import os 9 | import torch 10 | import numpy as np 11 | import time 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('--cfg', type=str, default='configs/shape/syn/compressor.yaml') 15 | # parser.add_argument('--cfg', type=str, default='configs/mat/syn/compressor.yaml') 16 | flags = parser.parse_args() 17 | 18 | train_time_st = time.time() 19 | 20 | TrainerInv(load_cfg(flags.cfg), config_path=flags.cfg).run() 21 | 22 | print(f"Training done, costs {format_timespan(time.time() - train_time_st)}.") 23 | -------------------------------------------------------------------------------- /train/train_tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | from collections import OrderedDict 3 | 4 | import torch 5 | import numpy as np 6 | from tensorboardX import SummaryWriter 7 | import torch.nn as nn 8 | 9 | 10 | def load_model(model, optim, model_dir, epoch=-1): 11 | if not os.path.exists(model_dir): 12 | return 0 13 | 14 | pths = [int(pth.split('.')[0]) for pth in os.listdir(model_dir)] 15 | if len(pths) == 0: 16 | return 0 17 | if epoch == -1: 18 | pth = max(pths) 19 | else: 20 | pth = epoch 21 | 22 | pretrained_model = torch.load(os.path.join(model_dir, '{}.pth'.format(pth))) 23 | model.load_state_dict(pretrained_model['net']) 24 | optim.load_state_dict(pretrained_model['optim']) 25 | print('load {} epoch {}'.format(model_dir, pretrained_model['epoch'] + 1)) 26 | return pretrained_model['epoch'] + 1 27 | 28 | def adjust_learning_rate(optimizer, epoch, lr_decay_rate, lr_decay_epoch, min_lr=1e-5): 29 | if ((epoch + 1) % lr_decay_epoch) != 0: 30 | return 31 | 32 | for param_group in optimizer.param_groups: 33 | # print(param_group) 34 | lr_before = param_group['lr'] 35 | param_group['lr'] = param_group['lr'] * lr_decay_rate 36 | param_group['lr'] = max(param_group['lr'], min_lr) 37 | 38 | print('changing learning rate {:5f} to {:.5f}'.format(lr_before, max(param_group['lr'], min_lr))) 39 | 40 | def reset_learning_rate(optimizer, lr): 41 | for param_group in optimizer.param_groups: 42 | # print(param_group) 43 | # lr_before = param_group['lr'] 44 | param_group['lr'] = lr 45 | # print('changing learning rate {:5f} to {:.5f}'.format(lr_before,lr)) 46 | return lr 47 | 48 | def save_model(net, optim, epoch, model_dir): 49 | os.system('mkdir -p {}'.format(model_dir)) 50 | torch.save({ 51 | 'net': net.feats_state_dict(), 52 | 'optim': optim.feats_state_dict(), 53 | 'epoch': epoch 54 | }, os.path.join(model_dir, '{}.pth'.format(epoch))) 55 | 56 | class Recorder(object): 57 | def __init__(self, rec_dir, rec_fn): 58 | self.rec_dir = rec_dir 59 | self.rec_fn = rec_fn 60 | self.data = OrderedDict() 61 | self.writer = SummaryWriter(log_dir=rec_dir) 62 | 63 | def rec_loss(self, losses_batch, step, epoch, prefix='train', dump=False): 64 | for k, v in losses_batch.items(): 65 | name = '{}/{}'.format(prefix, k) 66 | if name in self.data: 67 | self.data[name].append(v) 68 | else: 69 | self.data[name] = [v] 70 | 71 | if dump: 72 | if prefix == 'train': 73 | msg = '{} epoch {} step {} '.format(prefix, epoch, step) 74 | else: 75 | msg = '{} epoch {} '.format(prefix, epoch) 76 | for k, v in self.data.items(): 77 | if not k.startswith(prefix): continue 78 | if len(v) > 0: 79 | msg += '{} {:.5f} '.format(k.split('/')[-1], np.mean(v)) 80 | self.writer.add_scalar(k, np.mean(v), step) 81 | self.data[k] = [] 82 | 83 | print(msg) 84 | with open(self.rec_fn, 'a') as f: 85 | f.write(msg + '\n') 86 | 87 | def rec_msg(self, msg): 88 | print(msg) 89 | with open(self.rec_fn, 'a') as f: 90 | f.write(msg + '\n') 91 | 92 | 93 | class Logger: 94 | def __init__(self, log_dir): 95 | self.log_dir=log_dir 96 | self.data = OrderedDict() 97 | self.writer = SummaryWriter(log_dir=log_dir) 98 | 99 | def log(self,data, prefix='train',step=None,verbose=False): 100 | msg=f'{prefix} ' 101 | for k, v in data.items(): 102 | msg += f'{k} {v:.5f} ' 103 | self.writer.add_scalar(f'{prefix}/{k}',v,step) 104 | 105 | if verbose: 106 | print(msg) 107 | with open(os.path.join(self.log_dir,f'{prefix}.txt'), 'a') as f: 108 | f.write(msg + '\n') 109 | 110 | def print_shape(obj): 111 | if type(obj) == list or type(obj) == tuple: 112 | shapes = [item.shape for item in obj] 113 | print(shapes) 114 | else: 115 | print(obj.shape) 116 | 117 | def overwrite_configs(cfg_base: dict, cfg: dict): 118 | keysNotinBase = [] 119 | for key in cfg.keys(): 120 | if key in cfg_base.keys(): 121 | cfg_base[key] = cfg[key] 122 | else: 123 | keysNotinBase.append(key) 124 | cfg_base.update({key: cfg[key]}) 125 | if len(keysNotinBase) != 0: 126 | print('==== WARNING: These keys are not set in DEFAULT_BASE_CONFIG... ====') 127 | print(keysNotinBase) 128 | return cfg_base 129 | 130 | def to_cuda(data): 131 | if type(data)==list: 132 | results = [] 133 | for i, item in enumerate(data): 134 | results.append(to_cuda(item)) 135 | return results 136 | elif type(data)==dict: 137 | results={} 138 | for k,v in data.items(): 139 | results[k]=to_cuda(v) 140 | return results 141 | elif type(data).__name__ == "Tensor": 142 | return data.cuda() 143 | else: 144 | return data 145 | 146 | def dim_extend(data_list): 147 | results = [] 148 | for i, tensor in enumerate(data_list): 149 | results.append(tensor[None,...]) 150 | return results 151 | 152 | class MultiGPUWrapper(nn.Module): 153 | def __init__(self,network,losses): 154 | super().__init__() 155 | self.network=network 156 | self.losses=losses 157 | 158 | def forward(self, data_gt): 159 | results={} 160 | data_pr=self.network(data_gt) 161 | results.update(data_pr) 162 | for loss in self.losses: 163 | results.update(loss(data_pr,data_gt,data_gt['step'])) 164 | return results 165 | 166 | class DummyLoss: 167 | def __init__(self,losses): 168 | self.keys=[] 169 | for loss in losses: 170 | self.keys+=loss.keys 171 | 172 | def __call__(self, data_pr, data_gt, step): 173 | return {key: data_pr[key] for key in self.keys} 174 | -------------------------------------------------------------------------------- /train/train_valid.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import torch 4 | import numpy as np 5 | from tqdm import tqdm 6 | 7 | from network.metrics import name2key_metrics 8 | from train.train_tools import to_cuda 9 | 10 | 11 | class ValidationEvaluator: 12 | default_cfg={} 13 | def __init__(self,cfg): 14 | self.cfg={**self.default_cfg,**cfg} 15 | self.key_metric_name=cfg['key_metric_name'] 16 | self.key_metric=name2key_metrics[self.key_metric_name] 17 | 18 | def __call__(self, model, losses, eval_dataset, step, model_name, val_set_name=None): 19 | if val_set_name is not None: model_name=f'{model_name}-{val_set_name}' 20 | model.eval() 21 | eval_results={} 22 | begin=time.time() 23 | for data_i, data in tqdm(enumerate(eval_dataset)): 24 | data = to_cuda(data) 25 | data['eval']=True 26 | data['step']=step 27 | with torch.no_grad(): 28 | outputs=model(data) 29 | 30 | for loss in losses: 31 | loss_results=loss(outputs, data, step, data_index=data_i, model_name=model_name) 32 | for k,v in loss_results.items(): 33 | if type(v)==torch.Tensor: 34 | v=v.detach().cpu().numpy() 35 | 36 | if k in eval_results: 37 | eval_results[k].append(v) 38 | else: 39 | eval_results[k]=[v] 40 | 41 | for k,v in eval_results.items(): 42 | eval_results[k]=np.concatenate(v,axis=0) 43 | 44 | # evaluate poses 45 | # with torch.no_grad(): 46 | # eval_results.update(model.evaluation()) 47 | 48 | key_metric_val=self.key_metric(eval_results) 49 | eval_results[self.key_metric_name]=key_metric_val 50 | print('eval cost {} s'.format(time.time()-begin)) 51 | torch.cuda.empty_cache() 52 | return eval_results, key_metric_val 53 | -------------------------------------------------------------------------------- /user-imgs/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Riga2/TensoSDF/6074ab11af1bfe9c50a0d30c9630c4a083289b8b/user-imgs/teaser.png -------------------------------------------------------------------------------- /utils/dataset_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import time 3 | import random 4 | import torch 5 | 6 | def dummy_collate_fn(data_list): 7 | return data_list[0] 8 | 9 | def simple_collate_fn(data_list): 10 | ks=data_list[0].keys() 11 | outputs={k:[] for k in ks} 12 | for k in ks: 13 | for data in data_list: 14 | outputs[k].append(data[k]) 15 | outputs[k]=torch.stack(outputs[k],0) 16 | return outputs 17 | 18 | def set_seed(index,is_train): 19 | if is_train: 20 | np.random.seed((index+int(time.time()))%(2**16)) 21 | random.seed((index+int(time.time()))%(2**16)+1) 22 | torch.random.manual_seed((index+int(time.time()))%(2**16)+1) 23 | else: 24 | np.random.seed(index % (2 ** 16)) 25 | random.seed(index % (2 ** 16) + 1) 26 | torch.random.manual_seed(index % (2 ** 16) + 1) -------------------------------------------------------------------------------- /utils/raw_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | def linear_to_srgb(linear): 5 | if isinstance(linear, torch.Tensor): 6 | """Assumes `linear` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB.""" 7 | eps = torch.finfo(torch.float32).eps 8 | srgb0 = 323 / 25 * linear 9 | srgb1 = (211 * torch.clamp(linear, min=eps)**(5 / 12) - 11) / 200 10 | return torch.where(linear <= 0.0031308, srgb0, srgb1) 11 | elif isinstance(linear, np.ndarray): 12 | eps = np.finfo(np.float32).eps 13 | srgb0 = 323 / 25 * linear 14 | srgb1 = (211 * np.maximum(eps, linear) ** (5 / 12) - 11) / 200 15 | return np.where(linear <= 0.0031308, srgb0, srgb1) 16 | else: 17 | raise NotImplementedError 18 | 19 | def srgb_to_linear(srgb): 20 | if isinstance(srgb, torch.Tensor): 21 | """Assumes `srgb` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB.""" 22 | eps = torch.finfo(torch.float32).eps 23 | linear0 = 25 / 323 * srgb 24 | linear1 = torch.clamp(((200 * srgb + 11) / (211)), min=eps)**(12 / 5) 25 | return torch.where(srgb <= 0.04045, linear0, linear1) 26 | elif isinstance(srgb, np.ndarray): 27 | """Assumes `srgb` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB.""" 28 | eps = np.finfo(np.float32).eps 29 | linear0 = 25 / 323 * srgb 30 | linear1 = np.maximum(((200 * srgb + 11) / (211)), eps)**(12 / 5) 31 | return np.where(srgb <= 0.04045, linear0, linear1) 32 | else: 33 | raise NotImplementedError 34 | 35 | def reshape_quads(*planes): 36 | """Reshape pixels from four input images to make tiled 2x2 quads.""" 37 | planes = np.stack(planes, -1) 38 | shape = planes.shape[:-1] 39 | # Create [2, 2] arrays out of 4 channels. 40 | zup = planes.reshape(shape + (2, 2,)) 41 | # Transpose so that x-axis dimensions come before y-axis dimensions. 42 | zup = np.transpose(zup, (0, 2, 1, 3)) 43 | # Reshape to 2D. 44 | zup = zup.reshape((shape[0] * 2, shape[1] * 2)) 45 | return zup 46 | 47 | def bilinear_upsample(z): 48 | """2x bilinear image upsample.""" 49 | # Using np.roll makes the right and bottom edges wrap around. The raw image 50 | # data has a few garbage columns/rows at the edges that must be discarded 51 | # anyway, so this does not matter in practice. 52 | # Horizontally interpolated values. 53 | zx = .5 * (z + np.roll(z, -1, axis=-1)) 54 | # Vertically interpolated values. 55 | zy = .5 * (z + np.roll(z, -1, axis=-2)) 56 | # Diagonally interpolated values. 57 | zxy = .5 * (zx + np.roll(zx, -1, axis=-2)) 58 | return reshape_quads(z, zx, zy, zxy) 59 | 60 | def upsample_green(g1, g2): 61 | """Special 2x upsample from the two green channels.""" 62 | z = np.zeros_like(g1) 63 | z = reshape_quads(z, g1, g2, z) 64 | alt = 0 65 | # Grab the 4 directly adjacent neighbors in a "cross" pattern. 66 | for i in range(4): 67 | axis = -1 - (i // 2) 68 | roll = -1 + 2 * (i % 2) 69 | alt = alt + .25 * np.roll(z, roll, axis=axis) 70 | # For observed pixels, alt = 0, and for unobserved pixels, alt = avg(cross), 71 | # so alt + z will have every pixel filled in. 72 | return alt + z 73 | 74 | def bilinear_demosaic_raw_nerf(bayer, mode='rggb'): 75 | if mode=='rggb': 76 | r, g1, g2, b = [bayer[(i // 2)::2, (i % 2)::2] for i in range(4)] 77 | elif mode=='bggr': 78 | b, g1, g2, r = [bayer[(i // 2)::2, (i % 2)::2] for i in range(4)] 79 | else: 80 | raise NotImplementedError 81 | r = bilinear_upsample(r) 82 | # Flip in x and y before and after calling upsample, as bilinear_upsample 83 | # assumes that the samples are at the top-left corner of the 2x2 sample. 84 | b = bilinear_upsample(b[::-1, ::-1])[::-1, ::-1] 85 | g = upsample_green(g1, g2) 86 | rgb = np.stack([r, g, b], -1) 87 | return rgb 88 | 89 | def bilinear_demosaic_simple(bayer, mode='rggb'): 90 | if mode=='rggb': 91 | r, g1, g2, b = [bayer[(i // 2)::2, (i % 2)::2] for i in range(4)] 92 | elif mode=='bggr': 93 | b, g1, g2, r = [bayer[(i // 2)::2, (i % 2)::2] for i in range(4)] 94 | else: 95 | raise NotImplementedError 96 | r = bilinear_upsample(r) 97 | b = bilinear_upsample(b[::-1, ::-1])[::-1, ::-1] 98 | g1 = bilinear_upsample(g1) 99 | g2 = bilinear_upsample(g2[::-1, ::-1])[::-1, ::-1] 100 | rgbg = np.stack([r, g1, b, g2], -1) 101 | return rgbg -------------------------------------------------------------------------------- /utils/ref_utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import numpy as np 5 | 6 | 7 | 8 | def generalized_binomial_coeff(a, k): 9 | """Compute generalized binomial coefficients.""" 10 | return np.prod(a - np.arange(k)) / np.math.factorial(k) 11 | 12 | 13 | def assoc_legendre_coeff(l, m, k): 14 | """Compute associated Legendre polynomial coefficients. 15 | 16 | Returns the coefficient of the cos^k(theta)*sin^m(theta) term in the 17 | (l, m)th associated Legendre polynomial, P_l^m(cos(theta)). 18 | 19 | Args: 20 | l: associated Legendre polynomial degree. 21 | m: associated Legendre polynomial order. 22 | k: power of cos(theta). 23 | 24 | Returns: 25 | A float, the coefficient of the term corresponding to the inputs. 26 | """ 27 | return ((-1)**m * 2**l * np.math.factorial(l) / np.math.factorial(k) / 28 | np.math.factorial(l - k - m) * 29 | generalized_binomial_coeff(0.5 * (l + k + m - 1.0), l)) 30 | 31 | 32 | def sph_harm_coeff(l, m, k): 33 | """Compute spherical harmonic coefficients.""" 34 | return (np.sqrt( 35 | (2.0 * l + 1.0) * np.math.factorial(l - m) / 36 | (4.0 * np.pi * np.math.factorial(l + m))) * assoc_legendre_coeff(l, m, k)) 37 | 38 | 39 | 40 | def get_ml_array(deg_view): 41 | """Create a list with all pairs of (l, m) values to use in the encoding.""" 42 | ml_list = [] 43 | for i in range(deg_view): 44 | l = 2**i 45 | # Only use nonnegative m values, later splitting real and imaginary parts. 46 | for m in range(l + 1): 47 | ml_list.append((m, l)) 48 | 49 | # Convert list into a numpy array. 50 | ml_array = np.array(ml_list).T 51 | return ml_array 52 | 53 | def generate_ide_fn(deg_view): 54 | """Generate integrated directional encoding (IDE) function. 55 | 56 | This function returns a function that computes the integrated directional 57 | encoding from Equations 6-8 of arxiv.org/abs/2112.03907. 58 | 59 | Args: 60 | deg_view: number of spherical harmonics degrees to use. 61 | 62 | Returns: 63 | A function for evaluating integrated directional encoding. 64 | 65 | Raises: 66 | ValueError: if deg_view is larger than 5. 67 | """ 68 | if deg_view > 5: 69 | raise ValueError('Only deg_view of at most 5 is numerically stable.') 70 | 71 | ml_array = get_ml_array(deg_view) 72 | l_max = 2**(deg_view - 1) 73 | 74 | # Create a matrix corresponding to ml_array holding all coefficients, which, 75 | # when multiplied (from the right) by the z coordinate Vandermonde matrix, 76 | # results in the z component of the encoding. 77 | mat = np.zeros((l_max + 1, ml_array.shape[1])) 78 | for i, (m, l) in enumerate(ml_array.T): 79 | for k in range(l - m + 1): 80 | mat[k, i] = sph_harm_coeff(l, m, k) 81 | 82 | mat = torch.from_numpy(mat.astype(np.float32)).cuda() 83 | ml_array = torch.from_numpy(ml_array.astype(np.float32)).cuda() 84 | 85 | def integrated_dir_enc_fn(xyz, kappa_inv): 86 | """Function returning integrated directional encoding (IDE). 87 | 88 | Args: 89 | xyz: [..., 3] array of Cartesian coordinates of directions to evaluate at. 90 | kappa_inv: [..., 1] reciprocal of the concentration parameter of the von 91 | Mises-Fisher distribution. 92 | 93 | Returns: 94 | An array with the resulting IDE. 95 | """ 96 | x = xyz[..., 0:1] 97 | y = xyz[..., 1:2] 98 | z = xyz[..., 2:3] 99 | 100 | # Compute z Vandermonde matrix. 101 | vmz = torch.concat([z**i for i in range(mat.shape[0])], dim=-1) 102 | 103 | # Compute x+iy Vandermonde matrix. 104 | vmxy = torch.concat([(x + 1j * y)**m for m in ml_array[0, :]], dim=-1) 105 | 106 | # Get spherical harmonics. 107 | sph_harms = vmxy * torch.matmul(vmz, mat) 108 | 109 | # Apply attenuation function using the von Mises-Fisher distribution 110 | # concentration parameter, kappa. 111 | sigma = 0.5 * ml_array[1, :] * (ml_array[1, :] + 1) 112 | ide = sph_harms * torch.exp(-sigma * kappa_inv) 113 | 114 | # Split into real and imaginary parts and return 115 | return torch.concat([torch.real(ide), torch.imag(ide)], dim=-1) 116 | 117 | return integrated_dir_enc_fn 118 | 119 | def get_lat_long(): 120 | res = (1080, 1080*3) 121 | gy, gx = torch.meshgrid(torch.linspace(0.0 + 1.0 / res[0], 1.0 - 1.0 / res[0], res[0], device='cuda'), 122 | torch.linspace(-1.0 + 1.0 / res[1], 1.0 - 1.0 / res[1], res[1], device='cuda'), 123 | indexing='ij') # [h,w] 124 | 125 | sintheta, costheta = torch.sin(gy * np.pi), torch.cos(gy * np.pi) 126 | sinphi, cosphi = torch.sin(gx * np.pi), torch.cos(gx * np.pi) 127 | reflvec = torch.stack((sintheta * sinphi, costheta, -sintheta * cosphi), dim=-1) 128 | return reflvec 129 | 130 | def components_from_spherical_harmonics(levels, directions): 131 | """ 132 | levels: int 133 | directions : Tensor(batch, 3) 134 | Returns value for each component of spherical harmonics. 135 | 136 | Args: 137 | levels: Number of spherical harmonic levels to compute. 138 | directions: Spherical harmonic coefficients 139 | """ 140 | num_components = levels**2 141 | components = torch.zeros((*directions.shape[:-1], num_components), device=directions.device) 142 | 143 | assert 1 <= levels <= 5, f"SH levels must be in [1,4], got {levels}" 144 | assert directions.shape[-1] == 3, f"Direction input should have three dimensions. Got {directions.shape[-1]}" 145 | 146 | x = directions[..., 0] 147 | y = directions[..., 1] 148 | z = directions[..., 2] 149 | 150 | xx = x**2 151 | yy = y**2 152 | zz = z**2 153 | 154 | # l0 155 | components[..., 0] = 0.28209479177387814 156 | 157 | # l1 158 | if levels > 1: 159 | components[..., 1] = 0.4886025119029199 * y 160 | components[..., 2] = 0.4886025119029199 * z 161 | components[..., 3] = 0.4886025119029199 * x 162 | 163 | # l2 164 | if levels > 2: 165 | components[..., 4] = 1.0925484305920792 * x * y 166 | components[..., 5] = 1.0925484305920792 * y * z 167 | components[..., 6] = 0.9461746957575601 * zz - 0.31539156525251999 168 | components[..., 7] = 1.0925484305920792 * x * z 169 | components[..., 8] = 0.5462742152960396 * (xx - yy) 170 | 171 | # l3 172 | if levels > 3: 173 | components[..., 9] = 0.5900435899266435 * y * (3 * xx - yy) 174 | components[..., 10] = 2.890611442640554 * x * y * z 175 | components[..., 11] = 0.4570457994644658 * y * (5 * zz - 1) 176 | components[..., 12] = 0.3731763325901154 * z * (5 * zz - 3) 177 | components[..., 13] = 0.4570457994644658 * x * (5 * zz - 1) 178 | components[..., 14] = 1.445305721320277 * z * (xx - yy) 179 | components[..., 15] = 0.5900435899266435 * x * (xx - 3 * yy) 180 | 181 | # l4 182 | if levels > 4: 183 | components[..., 16] = 2.5033429417967046 * x * y * (xx - yy) 184 | components[..., 17] = 1.7701307697799304 * y * z * (3 * xx - yy) 185 | components[..., 18] = 0.9461746957575601 * x * y * (7 * zz - 1) 186 | components[..., 19] = 0.6690465435572892 * y * z * (7 * zz - 3) 187 | components[..., 20] = 0.10578554691520431 * (35 * zz * zz - 30 * zz + 3) 188 | components[..., 21] = 0.6690465435572892 * x * z * (7 * zz - 3) 189 | components[..., 22] = 0.47308734787878004 * (xx - yy) * (7 * zz - 1) 190 | components[..., 23] = 1.7701307697799304 * x * z * (xx - 3 * yy) 191 | components[..., 24] = 0.6258357354491761 * (xx * (xx - 3 * yy) - yy * (3 * xx - yy)) 192 | 193 | return components -------------------------------------------------------------------------------- /utils/rend_util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import imageio 3 | import skimage 4 | import cv2 5 | import torch 6 | from torch.nn import functional as F 7 | 8 | def load_rgb(path): 9 | img = imageio.imread(path) 10 | img = skimage.img_as_float32(img) 11 | 12 | # pixel values between [-1,1] 13 | img -= 0.5 14 | img *= 2. 15 | img = img.transpose(2, 0, 1) 16 | return img 17 | 18 | def load_mask(path): 19 | alpha = imageio.imread(path, as_gray=True) 20 | alpha = skimage.img_as_float32(alpha) 21 | object_mask = alpha > 127.5 22 | 23 | return object_mask 24 | 25 | def load_K_Rt_from_P(filename, P=None): 26 | if P is None: 27 | lines = open(filename).read().splitlines() 28 | if len(lines) == 4: 29 | lines = lines[1:] 30 | lines = [[x[0], x[1], x[2], x[3]] for x in (x.split(" ") for x in lines)] 31 | P = np.asarray(lines).astype(np.float32).squeeze() 32 | 33 | out = cv2.decomposeProjectionMatrix(P) 34 | K = out[0] 35 | R = out[1] 36 | t = out[2] 37 | 38 | K = K/K[2,2] 39 | intrinsics = np.eye(4) 40 | intrinsics[:3, :3] = K 41 | 42 | pose = np.eye(4, dtype=np.float32) 43 | pose[:3, :3] = R.transpose() 44 | pose[:3,3] = (t[:3] / t[3])[:,0] 45 | 46 | return intrinsics, pose 47 | 48 | def get_camera_params(uv, pose, intrinsics): 49 | if pose.shape[1] == 7: #In case of quaternion vector representation 50 | cam_loc = pose[:, 4:] 51 | R = quat_to_rot(pose[:,:4]) 52 | p = torch.eye(4).repeat(pose.shape[0],1,1).cuda().float() 53 | p[:, :3, :3] = R 54 | p[:, :3, 3] = cam_loc 55 | else: # In case of pose matrix representation 56 | cam_loc = pose[:, :3, 3] 57 | p = pose 58 | 59 | batch_size, num_samples, _ = uv.shape 60 | 61 | depth = torch.ones((batch_size, num_samples)).cuda() 62 | x_cam = uv[:, :, 0].view(batch_size, -1) 63 | y_cam = uv[:, :, 1].view(batch_size, -1) 64 | z_cam = depth.view(batch_size, -1) 65 | 66 | pixel_points_cam = lift(x_cam, y_cam, z_cam, intrinsics=intrinsics) 67 | 68 | # permute for batch matrix product 69 | pixel_points_cam = pixel_points_cam.permute(0, 2, 1) 70 | 71 | world_coords = torch.bmm(p, pixel_points_cam).permute(0, 2, 1)[:, :, :3] 72 | ray_dirs = world_coords - cam_loc[:, None, :] 73 | ray_dirs = F.normalize(ray_dirs, dim=2) 74 | 75 | return ray_dirs, cam_loc 76 | 77 | def get_camera_for_plot(pose): 78 | if pose.shape[1] == 7: #In case of quaternion vector representation 79 | cam_loc = pose[:, 4:].detach() 80 | R = quat_to_rot(pose[:,:4].detach()) 81 | else: # In case of pose matrix representation 82 | cam_loc = pose[:, :3, 3] 83 | R = pose[:, :3, :3] 84 | cam_dir = R[:, :3, 2] 85 | return cam_loc, cam_dir 86 | 87 | def lift(x, y, z, intrinsics): 88 | # parse intrinsics 89 | intrinsics = intrinsics.cuda() 90 | fx = intrinsics[:, 0, 0] 91 | fy = intrinsics[:, 1, 1] 92 | cx = intrinsics[:, 0, 2] 93 | cy = intrinsics[:, 1, 2] 94 | sk = intrinsics[:, 0, 1] 95 | 96 | x_lift = (x - cx.unsqueeze(-1) + cy.unsqueeze(-1)*sk.unsqueeze(-1)/fy.unsqueeze(-1) - sk.unsqueeze(-1)*y/fy.unsqueeze(-1)) / fx.unsqueeze(-1) * z 97 | y_lift = (y - cy.unsqueeze(-1)) / fy.unsqueeze(-1) * z 98 | 99 | # homogeneous 100 | return torch.stack((x_lift, y_lift, z, torch.ones_like(z).cuda()), dim=-1) 101 | 102 | def quat_to_rot(q): 103 | batch_size, _ = q.shape 104 | q = F.normalize(q, dim=1) 105 | R = torch.ones((batch_size, 3,3)).cuda() 106 | qr=q[:,0] 107 | qi = q[:, 1] 108 | qj = q[:, 2] 109 | qk = q[:, 3] 110 | R[:, 0, 0]=1-2 * (qj**2 + qk**2) 111 | R[:, 0, 1] = 2 * (qj *qi -qk*qr) 112 | R[:, 0, 2] = 2 * (qi * qk + qr * qj) 113 | R[:, 1, 0] = 2 * (qj * qi + qk * qr) 114 | R[:, 1, 1] = 1-2 * (qi**2 + qk**2) 115 | R[:, 1, 2] = 2*(qj*qk - qi*qr) 116 | R[:, 2, 0] = 2 * (qk * qi-qj * qr) 117 | R[:, 2, 1] = 2 * (qj*qk + qi*qr) 118 | R[:, 2, 2] = 1-2 * (qi**2 + qj**2) 119 | return R 120 | 121 | def rot_to_quat(R): 122 | batch_size, _,_ = R.shape 123 | q = torch.ones((batch_size, 4)).cuda() 124 | 125 | R00 = R[:, 0,0] 126 | R01 = R[:, 0, 1] 127 | R02 = R[:, 0, 2] 128 | R10 = R[:, 1, 0] 129 | R11 = R[:, 1, 1] 130 | R12 = R[:, 1, 2] 131 | R20 = R[:, 2, 0] 132 | R21 = R[:, 2, 1] 133 | R22 = R[:, 2, 2] 134 | 135 | q[:,0]=torch.sqrt(1.0+R00+R11+R22)/2 136 | q[:, 1]=(R21-R12)/(4*q[:,0]) 137 | q[:, 2] = (R02 - R20) / (4 * q[:, 0]) 138 | q[:, 3] = (R10 - R01) / (4 * q[:, 0]) 139 | return q 140 | 141 | def get_sphere_intersection(cam_loc, ray_directions, r = 1.0): 142 | # Input: n_images x 4 x 4 ; n_images x n_rays x 3 143 | # Output: n_images * n_rays x 2 (close and far) ; n_images * n_rays 144 | 145 | n_imgs, n_pix, _ = ray_directions.shape 146 | 147 | cam_loc = cam_loc.unsqueeze(-1) 148 | ray_cam_dot = torch.bmm(ray_directions, cam_loc).squeeze() 149 | under_sqrt = ray_cam_dot ** 2 - (cam_loc.norm(2,1) ** 2 - r ** 2) 150 | 151 | under_sqrt = under_sqrt.reshape(-1) 152 | mask_intersect = under_sqrt > 0 153 | 154 | sphere_intersections = torch.zeros(n_imgs * n_pix, 2).cuda().float() 155 | sphere_intersections[mask_intersect] = torch.sqrt(under_sqrt[mask_intersect]).unsqueeze(-1) * torch.Tensor([-1, 1]).cuda().float() 156 | sphere_intersections[mask_intersect] -= ray_cam_dot.reshape(-1)[mask_intersect].unsqueeze(-1) 157 | 158 | sphere_intersections = sphere_intersections.reshape(n_imgs, n_pix, 2) 159 | sphere_intersections = sphere_intersections.clamp_min(0.0) 160 | mask_intersect = mask_intersect.reshape(n_imgs, n_pix) 161 | 162 | return sphere_intersections, mask_intersect 163 | 164 | def get_depth(points, pose): 165 | ''' Retruns depth from 3D points according to camera pose ''' 166 | batch_size, num_samples, _ = points.shape 167 | if pose.shape[1] == 7: # In case of quaternion vector representation 168 | cam_loc = pose[:, 4:] 169 | R = quat_to_rot(pose[:, :4]) 170 | pose = torch.eye(4).unsqueeze(0).repeat(batch_size, 1, 1).cuda().float() 171 | pose[:, :3, 3] = cam_loc 172 | pose[:, :3, :3] = R 173 | 174 | points_hom = torch.cat((points, torch.ones((batch_size, num_samples, 1)).cuda()), dim=2) 175 | 176 | # permute for batch matrix product 177 | points_hom = points_hom.permute(0, 2, 1) 178 | 179 | points_cam = torch.inverse(pose).bmm(points_hom) 180 | depth = points_cam[:, 2, :][:, :, None] 181 | return depth 182 | 183 | --------------------------------------------------------------------------------