├── LICENSE ├── README.md ├── __pycache__ ├── cam_utils.cpython-310.pyc ├── helper_train.cpython-310.pyc └── train_gui_utils.cpython-310.pyc ├── arguments ├── N3DV │ ├── coffee_martini.py │ ├── cook_spinach.py │ ├── cut_roasted_beef.py │ ├── flame_salmon_1.py │ ├── flame_steak.py │ └── sear_steak.py ├── Technicolor │ ├── Birthday.py │ ├── Fabien.py │ ├── Fatma.py │ ├── Hands.py │ ├── Painter.py │ ├── Theater.py │ └── Train.py ├── __init__.py └── __pycache__ │ └── __init__.cpython-310.pyc ├── cam_utils.py ├── convert.py ├── data_tools ├── colmap2nerf.py ├── interactive_invoke.py └── phone_catch.py ├── experimental_logs ├── README.md ├── arxiv_logs.txt └── asset │ ├── coffee_martini_node.zip │ ├── cook_spinach_node.zip │ ├── cut_roasted_beef_node.zip │ ├── flame_salmon_1_node.zip │ ├── flame_steak_node.zip │ ├── img │ ├── arg_cook_spinach.jpg │ └── arg_cut_roasted_beef.jpg │ └── sear_steak_node.zip ├── gaussian_renderer ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── __init__.cpython-38.pyc │ ├── network_gui.cpython-310.pyc │ └── network_gui.cpython-38.pyc └── network_gui.py ├── imgs └── Framework.jpg ├── install_env.sh ├── lap_deform.py ├── lpipsPyTorch ├── __init__.py ├── __pycache__ │ └── __init__.cpython-310.pyc └── modules │ ├── __pycache__ │ ├── lpips.cpython-310.pyc │ ├── networks.cpython-310.pyc │ └── utils.cpython-310.pyc │ ├── lpips.py │ ├── networks.py │ └── utils.py ├── main.py ├── metrics.py ├── render.py ├── requirements.txt ├── scene ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── __init__.cpython-37.pyc │ ├── __init__.cpython-38.pyc │ ├── cameras.cpython-310.pyc │ ├── cameras.cpython-38.pyc │ ├── cameras_st.cpython-310.pyc │ ├── colmap_loader.cpython-310.pyc │ ├── colmap_loader.cpython-37.pyc │ ├── colmap_loader.cpython-38.pyc │ ├── dataset_readers.cpython-310.pyc │ ├── dataset_readers.cpython-37.pyc │ ├── dataset_readers.cpython-38.pyc │ ├── deform_model.cpython-310.pyc │ ├── deform_model.cpython-38.pyc │ ├── gaussian_model.cpython-310.pyc │ └── gaussian_model.cpython-38.pyc ├── cameras.py ├── cameras_st.py ├── colmap_loader.py ├── dataset_readers.py ├── deform_model.py └── gaussian_model.py ├── script ├── Readme.md ├── __pycache__ │ └── pre_immersive_distorted.cpython-310.pyc ├── n3d_process.py ├── pre_immersive_distorted.py ├── technicolor_process.py └── utils_pre.py ├── start_order.sh ├── submodules ├── diff-gaussian-rasterization │ ├── .gitignore │ ├── .gitmodules │ ├── CMakeLists.txt │ ├── LICENSE.md │ ├── README.md │ ├── cuda_rasterizer │ │ ├── auxiliary.h │ │ ├── backward.cu │ │ ├── backward.h │ │ ├── config.h │ │ ├── forward.cu │ │ ├── forward.h │ │ ├── rasterizer.h │ │ ├── rasterizer_impl.cu │ │ └── rasterizer_impl.h │ ├── diff_gaussian_rasterization │ │ └── __init__.py │ ├── ext.cpp │ ├── rasterize_points.cu │ ├── rasterize_points.h │ ├── setup.py │ └── third_party │ │ └── stbi_image_write.h └── simple-knn │ ├── LICENSE.md │ ├── build │ ├── lib.linux-x86_64-cpython-310 │ │ └── simple_knn │ │ │ └── _C.cpython-310-x86_64-linux-gnu.so │ └── temp.linux-x86_64-cpython-310 │ │ ├── .ninja_deps │ │ ├── .ninja_log │ │ ├── build.ninja │ │ ├── ext.o │ │ ├── simple_knn.o │ │ └── spatial.o │ ├── ext.cpp │ ├── setup.py │ ├── simple_knn.cu │ ├── simple_knn.egg-info │ ├── PKG-INFO │ ├── SOURCES.txt │ ├── dependency_links.txt │ └── top_level.txt │ ├── simple_knn.h │ ├── simple_knn │ └── .gitkeep │ ├── spatial.cu │ └── spatial.h ├── technicolor.sh ├── test.sh ├── thirdparty ├── colmap │ └── pre_colmap.py └── gaussian_splatting │ ├── helper3dg.py │ └── utils │ └── my_utils.py ├── train.py ├── train_gui_utils.py └── utils ├── __pycache__ ├── camera_utils.cpython-310.pyc ├── camera_utils_st.cpython-310.pyc ├── deform_utils.cpython-310.pyc ├── depth_utils.cpython-310.pyc ├── general_utils.cpython-310.pyc ├── graphics_utils.cpython-310.pyc ├── graphics_utils.cpython-37.pyc ├── image_utils.cpython-310.pyc ├── logger_hook.cpython-310.pyc ├── loss_utils.cpython-310.pyc ├── loss_utils.cpython-37.pyc ├── pose_utils.cpython-310.pyc ├── rigid_utils.cpython-310.pyc ├── sh_utils.cpython-310.pyc ├── system_utils.cpython-310.pyc ├── system_utils.cpython-37.pyc └── time_utils.cpython-310.pyc ├── arap_deform.py ├── bezier.py ├── camera_utils.py ├── camera_utils_st.py ├── deform_utils.py ├── dual_quaternion.py ├── general_utils.py ├── graphics_utils.py ├── image_utils.py ├── interactive_utils.py ├── logger_hook.py ├── loss_utils.py ├── other_utils.py ├── pickle_utils.py ├── pose_utils.py ├── preprocess.py ├── rigid_utils.py ├── sh_utils.py ├── system_utils.py ├── time_utils.py └── vis_utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Ellison King 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /__pycache__/cam_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/__pycache__/cam_utils.cpython-310.pyc -------------------------------------------------------------------------------- /__pycache__/helper_train.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/__pycache__/helper_train.cpython-310.pyc -------------------------------------------------------------------------------- /__pycache__/train_gui_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/__pycache__/train_gui_utils.cpython-310.pyc -------------------------------------------------------------------------------- /arguments/N3DV/coffee_martini.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type=None, # N3DV 3 | # densify_grad_threshold = 0.0002, # 0.0002 4 | #scaling_lr = 0.003, # 0.001 5 | # position_lr_init = 0.00016/5, #1.6e-5, #0.00016/5,#0.00016/5, #1.6e-5 #0.00016, 6 | # rotation_lr = 1e-4, #0.001, 7 | # movelr = 3.5/14, 8 | position_lr_init = 0.00016, 9 | flag=0 10 | ) 11 | 12 | OptimizationParams = dict( 13 | 14 | ) 15 | PipelineParams = dict() 16 | -------------------------------------------------------------------------------- /arguments/N3DV/cook_spinach.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type=None, # N3DV 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | # densify_grad_threshold=0.0002, # 0.0002 8 | # scaling_lr=0.001, # 0.001 9 | # position_lr_init=1.6e-5/10, #1.6e-5 # 0.00016/5, 10 | 11 | densify_grad_threshold=0.0002, # 0.0002 12 | # scaling_lr=0.005, # 0.001 13 | position_lr_init=1.6e-4, #1.6e-5 # 0.00016/5, 14 | flag=0 15 | ) 16 | PipelineParams = dict() 17 | -------------------------------------------------------------------------------- /arguments/N3DV/cut_roasted_beef.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type=None, # N3DV 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | 8 | #scaling_lr=0.0001, 9 | 10 | #rotation_lr=1e-4, 11 | #movelr=3.5 / 14, 12 | flag=0 13 | ) 14 | PipelineParams = dict() 15 | -------------------------------------------------------------------------------- /arguments/N3DV/flame_salmon_1.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type=None, # N3DV 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | # densify_grad_threshold=0.0002, # 0.0002 8 | # scaling_lr=0.001, # 0.001 9 | # position_lr_init=0.00016/5, # 1.6e-5 # 10 | # densify_grad_threshold=0.0005, 11 | 12 | #scaling_lr=0.001, 13 | #position_lr_init=1.6e-4, 14 | #densify=2 15 | flag=0 16 | ) 17 | PipelineParams = dict() 18 | -------------------------------------------------------------------------------- /arguments/N3DV/flame_steak.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type=None, # N3DV 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | 8 | #scaling_lr=0.0001, # 0.001 9 | #rotation_lr=1e-4, # 0.001, 10 | #movelr=3.5 / 14, 11 | #position_lr_init=1.6e-5, 12 | flag=0 13 | ) 14 | PipelineParams = dict() 15 | -------------------------------------------------------------------------------- /arguments/N3DV/sear_steak.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type=None, # N3DV 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | # densify_grad_threshold=0.0002, # 0.0002 8 | # scaling_lr=0.001, # 0.001 9 | # position_lr_init=0.00016/5, #0.00016/5, 10 | flag=0 11 | ) 12 | PipelineParams = dict() 13 | -------------------------------------------------------------------------------- /arguments/Technicolor/Birthday.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | densify_grad_threshold=0.0002, # 0.0002 8 | scaling_lr=0.001, # 0.001 9 | position_lr_init=0.00016, # 1.6e-5 # 10 | flag=0 11 | ) 12 | PipelineParams = dict() 13 | -------------------------------------------------------------------------------- /arguments/Technicolor/Fabien.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | # densify_grad_threshold=0.0002, # 0.0002 8 | # scaling_lr=0.001, # 0.001 9 | # position_lr_init=1.6e-5/5, #0.00016, # 1.6e-5 # 10 | densify_grad_threshold=0.0005, # 0.0002 11 | scaling_lr=0.001/10, # 0.001 12 | position_lr_init=1.6e-5 / 10, # 0.00016, # 1.6e-5 # 13 | flag=0 14 | ) 15 | PipelineParams = dict() 16 | -------------------------------------------------------------------------------- /arguments/Technicolor/Fatma.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | densify_grad_threshold=0.0002, # 0.0002 8 | scaling_lr=0.001, # 0.001 9 | position_lr_init=1.6e-5/5, #0.00016, # 1.6e-5 # 10 | flag=0 11 | ) 12 | PipelineParams = dict() 13 | -------------------------------------------------------------------------------- /arguments/Technicolor/Hands.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | densify_grad_threshold=0.0002, # 0.0002 8 | scaling_lr=0.001, # 0.001 9 | position_lr_init=1.6e-5/5, #0.00016, # 1.6e-5 # 10 | flag=0 11 | ) 12 | PipelineParams = dict() 13 | -------------------------------------------------------------------------------- /arguments/Technicolor/Painter.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | densify_grad_threshold=0.0002, # 0.0002 8 | scaling_lr=0.001, # 0.001 9 | position_lr_init=0.00016, # 1.6e-5 # 10 | flag=0 11 | ) 12 | PipelineParams = dict() 13 | -------------------------------------------------------------------------------- /arguments/Technicolor/Theater.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | densify_grad_threshold=0.0005, # 0.0002 8 | scaling_lr=0.001/10, # 0.001 9 | position_lr_init=1.6e-5/10, # 1.6e-5 # 10 | 11 | feature_lr = 0.0025/5, 12 | opacity_lr = 0.05/5, 13 | rotation_lr = 0.001/5, 14 | 15 | # densify_grad_threshold=0.0005, # 0.0002 16 | # position_lr_init=1.6e-5 / 5, # 0.00016, # 1.6e-5 # 17 | flag=0 18 | 19 | ) 20 | PipelineParams = dict() 21 | -------------------------------------------------------------------------------- /arguments/Technicolor/Train.py: -------------------------------------------------------------------------------- 1 | ModelParams = dict( 2 | dataset_type='technicolor', 3 | 4 | ) 5 | 6 | OptimizationParams = dict( 7 | densify_grad_threshold=0.0002, # 0.0002 8 | scaling_lr=0.001, # 0.001 9 | position_lr_init=1.6e-5/5, #0.00016, # 1.6e-5 # 10 | flag=0 11 | ) 12 | PipelineParams = dict() 13 | -------------------------------------------------------------------------------- /arguments/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/arguments/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /cam_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.spatial.transform import Rotation as R 3 | 4 | import torch 5 | 6 | def dot(x, y): 7 | if isinstance(x, np.ndarray): 8 | return np.sum(x * y, -1, keepdims=True) 9 | else: 10 | return torch.sum(x * y, -1, keepdim=True) 11 | 12 | 13 | def length(x, eps=1e-20): 14 | if isinstance(x, np.ndarray): 15 | return np.sqrt(np.maximum(np.sum(x * x, axis=-1, keepdims=True), eps)) 16 | else: 17 | return torch.sqrt(torch.clamp(dot(x, x), min=eps)) 18 | 19 | 20 | def safe_normalize(x, eps=1e-20): 21 | return x / length(x, eps) 22 | 23 | 24 | def look_at(campos, target, opengl=True): 25 | # campos: [N, 3], camera/eye position 26 | # target: [N, 3], object to look at 27 | # return: [N, 3, 3], rotation matrix 28 | if not opengl: 29 | # camera forward aligns with -z 30 | forward_vector = safe_normalize(target - campos) 31 | up_vector = np.array([0, 1, 0], dtype=np.float32) 32 | right_vector = safe_normalize(np.cross(forward_vector, up_vector)) 33 | up_vector = safe_normalize(np.cross(right_vector, forward_vector)) 34 | else: 35 | # camera forward aligns with +z 36 | forward_vector = safe_normalize(campos - target) 37 | up_vector = np.array([0, 1, 0], dtype=np.float32) 38 | right_vector = safe_normalize(np.cross(up_vector, forward_vector)) 39 | up_vector = safe_normalize(np.cross(forward_vector, right_vector)) 40 | R = np.stack([right_vector, up_vector, forward_vector], axis=1) 41 | return R 42 | 43 | 44 | # elevation & azimuth to pose (cam2world) matrix 45 | def orbit_camera(elevation, azimuth, radius=1, is_degree=True, target=None, opengl=True): 46 | # radius: scalar 47 | # elevation: scalar, in (-90, 90), from +y to -y is (-90, 90) 48 | # azimuth: scalar, in (-180, 180), from +z to +x is (0, 90) 49 | # return: [4, 4], camera pose matrix 50 | if is_degree: 51 | elevation = np.deg2rad(elevation) 52 | azimuth = np.deg2rad(azimuth) 53 | x = radius * np.cos(elevation) * np.sin(azimuth) 54 | y = - radius * np.sin(elevation) 55 | z = radius * np.cos(elevation) * np.cos(azimuth) 56 | if target is None: 57 | target = np.zeros([3], dtype=np.float32) 58 | campos = np.array([x, y, z]) + target # [3] 59 | T = np.eye(4, dtype=np.float32) 60 | T[:3, :3] = look_at(campos, target, opengl) 61 | T[:3, 3] = campos 62 | return T 63 | 64 | 65 | class OrbitCamera: 66 | def __init__(self, W, H, r=2, fovy=60, near=0.01, far=100): 67 | self.W = W 68 | self.H = H 69 | self.radius = r # camera distance from center 70 | self.fovy = np.deg2rad(fovy) # deg 2 rad 71 | self.near = near 72 | self.far = far 73 | self.center = np.array([0, 0, 0], dtype=np.float32) # look at this point 74 | # self.rot = R.from_matrix(np.eye(3)) 75 | self.rot = R.from_matrix(np.array([[1., 0., 0.,], 76 | [0., 0., -1.], 77 | [0., 1., 0.]])) 78 | self.up = np.array([0, 1, 0], dtype=np.float32) # need to be normalized! 79 | self.side = np.array([1, 0, 0], dtype=np.float32) 80 | 81 | @property 82 | def fovx(self): 83 | return 2 * np.arctan(np.tan(self.fovy / 2) * self.W / self.H) 84 | 85 | @property 86 | def campos(self): 87 | return self.pose[:3, 3] 88 | 89 | # pose (c2w) 90 | @property 91 | def pose(self): 92 | # first move camera to radius 93 | res = np.eye(4, dtype=np.float32) 94 | res[2, 3] = self.radius # opengl convention... 95 | # rotate 96 | rot = np.eye(4, dtype=np.float32) 97 | rot[:3, :3] = self.rot.as_matrix() 98 | res = rot @ res 99 | # translate 100 | res[:3, 3] -= self.center 101 | return res 102 | 103 | # view (w2c) 104 | @property 105 | def view(self): 106 | return np.linalg.inv(self.pose) 107 | 108 | # projection (perspective) 109 | @property 110 | def perspective(self): 111 | y = np.tan(self.fovy / 2) 112 | aspect = self.W / self.H 113 | return np.array( 114 | [ 115 | [1 / (y * aspect), 0, 0, 0], 116 | [0, -1 / y, 0, 0], 117 | [ 118 | 0, 119 | 0, 120 | -(self.far + self.near) / (self.far - self.near), 121 | -(2 * self.far * self.near) / (self.far - self.near), 122 | ], 123 | [0, 0, -1, 0], 124 | ], 125 | dtype=np.float32, 126 | ) 127 | 128 | # intrinsics 129 | @property 130 | def intrinsics(self): 131 | focal = self.H / (2 * np.tan(self.fovy / 2)) 132 | return np.array([focal, focal, self.W // 2, self.H // 2], dtype=np.float32) 133 | 134 | @property 135 | def mvp(self): 136 | return self.perspective @ np.linalg.inv(self.pose) # [4, 4] 137 | 138 | def orbit(self, dx, dy): 139 | # rotate along camera up/side axis! 140 | side = self.rot.as_matrix()[:3, 0] 141 | up = self.rot.as_matrix()[:3, 1] 142 | rotvec_x = up * np.radians(-0.05 * dx) 143 | rotvec_y = side * np.radians(-0.05 * dy) 144 | self.rot = R.from_rotvec(rotvec_x) * R.from_rotvec(rotvec_y) * self.rot 145 | 146 | def scale(self, delta): 147 | self.radius *= 1.1 ** (-delta) 148 | 149 | def pan(self, dx, dy, dz=0): 150 | # pan in camera coordinate system (careful on the sensitivity!) 151 | self.center += 0.0001 * self.rot.as_matrix()[:3, :3] @ np.array([-dx, -dy, dz]) 152 | -------------------------------------------------------------------------------- /convert.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | from argparse import ArgumentParser 14 | import shutil 15 | 16 | # This Python script is based on the shell converter script provided in the MipNerF 360 repository. 17 | parser = ArgumentParser("Colmap converter") 18 | parser.add_argument("--no_gpu", action='store_true') 19 | parser.add_argument("--skip_matching", action='store_true') 20 | parser.add_argument("--source_path", "-s", required=True, type=str) 21 | parser.add_argument("--camera", default="OPENCV", type=str) 22 | parser.add_argument("--colmap_executable", default="", type=str) 23 | parser.add_argument("--resize", action="store_true") 24 | parser.add_argument("--magick_executable", default="", type=str) 25 | args = parser.parse_args() 26 | colmap_command = '"{}"'.format(args.colmap_executable) if len(args.colmap_executable) > 0 else "colmap" 27 | magick_command = '"{}"'.format(args.magick_executable) if len(args.magick_executable) > 0 else "magick" 28 | use_gpu = 1 if not args.no_gpu else 0 29 | 30 | if not args.skip_matching: 31 | os.makedirs(args.source_path + "/distorted/sparse", exist_ok=True) 32 | 33 | ## Feature extraction 34 | os.system(colmap_command + " feature_extractor "\ 35 | "--database_path " + args.source_path + "/distorted/database.db \ 36 | --image_path " + args.source_path + "/input \ 37 | --ImageReader.single_camera 1 \ 38 | --ImageReader.camera_model " + args.camera + " \ 39 | --SiftExtraction.use_gpu " + str(use_gpu)) 40 | 41 | ## Feature matching 42 | os.system(colmap_command + " exhaustive_matcher \ 43 | --database_path " + args.source_path + "/distorted/database.db \ 44 | --SiftMatching.use_gpu " + str(use_gpu)) 45 | 46 | ### Bundle adjustment 47 | # The default Mapper tolerance is unnecessarily large, 48 | # decreasing it speeds up bundle adjustment steps. 49 | os.system(colmap_command + " mapper \ 50 | --database_path " + args.source_path + "/distorted/database.db \ 51 | --image_path " + args.source_path + "/input \ 52 | --output_path " + args.source_path + "/distorted/sparse \ 53 | --Mapper.ba_global_function_tolerance=0.000001") 54 | 55 | ### Image undistortion 56 | ## We need to undistort our images into ideal pinhole intrinsics. 57 | os.system(colmap_command + " image_undistorter \ 58 | --image_path " + args.source_path + "/input \ 59 | --input_path " + args.source_path + "/distorted/sparse/0 \ 60 | --output_path " + args.source_path + "\ 61 | --output_type COLMAP") 62 | 63 | files = os.listdir(args.source_path + "/sparse") 64 | os.makedirs(args.source_path + "/sparse/0", exist_ok=True) 65 | # Copy each file from the source directory to the destination directory 66 | for file in files: 67 | if file == '0': 68 | continue 69 | source_file = os.path.join(args.source_path, "sparse", file) 70 | destination_file = os.path.join(args.source_path, "sparse", "0", file) 71 | shutil.move(source_file, destination_file) 72 | 73 | if(args.resize): 74 | print("Copying and resizing...") 75 | 76 | # Resize images. 77 | os.makedirs(args.source_path + "/images_2", exist_ok=True) 78 | os.makedirs(args.source_path + "/images_4", exist_ok=True) 79 | os.makedirs(args.source_path + "/images_8", exist_ok=True) 80 | # Get the list of files in the source directory 81 | files = os.listdir(args.source_path + "/images") 82 | # Copy each file from the source directory to the destination directory 83 | for file in files: 84 | source_file = os.path.join(args.source_path, "images", file) 85 | 86 | destination_file = os.path.join(args.source_path, "images_2", file) 87 | shutil.copy2(source_file, destination_file) 88 | os.system(magick_command + " mogrify -resize 50% " + destination_file) 89 | 90 | destination_file = os.path.join(args.source_path, "images_4", file) 91 | shutil.copy2(source_file, destination_file) 92 | os.system(magick_command + " mogrify -resize 25% " + destination_file) 93 | 94 | destination_file = os.path.join(args.source_path, "images_8", file) 95 | shutil.copy2(source_file, destination_file) 96 | os.system(magick_command + " mogrify -resize 12.5% " + destination_file) 97 | 98 | print("Done.") -------------------------------------------------------------------------------- /data_tools/phone_catch.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import sys 4 | import glob 5 | import torch 6 | import shutil 7 | import numpy as np 8 | from PIL import Image 9 | from scipy import optimize 10 | import matplotlib.pyplot as plt 11 | import matplotlib 12 | matplotlib.use('Agg') 13 | 14 | 15 | ''' 16 | MiVOS: https://github.com/hkchengrex/MiVOS 17 | ''' 18 | MIVOS_PATH='YOUR/PATH/TO/MiVOS/' 19 | sys.path.append(MIVOS_PATH) 20 | from interactive_invoke import seg_video 21 | 22 | from colmap2nerf import colmap2nerf_invoke 23 | 24 | 25 | def Laplacian(img): 26 | return cv2.Laplacian(img, cv2.CV_64F).var() 27 | 28 | 29 | def cal_ambiguity(path): 30 | imgs = sorted(glob.glob(path + '/*.png')) 31 | laplace = np.zeros(len(imgs), np.float32) 32 | laplace_dict = {} 33 | for i in range(len(imgs)): 34 | laplace[i] = Laplacian(cv2.cvtColor(cv2.imread(imgs[i]), cv2.COLOR_BGR2GRAY)) 35 | laplace_dict[imgs[i]] = laplace[i] 36 | fig = plt.figure() 37 | fig.add_subplot(1, 2, 1) 38 | plt.hist(laplace) 39 | fig.add_subplot(1, 2, 2) 40 | plt.plot(np.arange(len(laplace)), laplace) 41 | if not os.path.exists(path + '/../noise/'): 42 | os.makedirs(path + '/../noise/') 43 | elif os.path.exists(path + '../noise/'): 44 | return None, None 45 | else: 46 | return None, None 47 | plt.savefig(path+'/../noise/laplace.png') 48 | return laplace, laplace_dict 49 | 50 | 51 | def select_ambiguity(path, nb=10, threshold=0.8, mv_files=False): 52 | if mv_files and os.path.exists(path + '/../noise/'): 53 | print('No need to select. Already done.') 54 | return None, None 55 | def linear(x, a, b): 56 | return a * x + b 57 | laplace, laplace_dic = cal_ambiguity(path) 58 | if laplace is None: 59 | return None, None 60 | imgs = list(laplace_dic.keys()) 61 | amb_img = [] 62 | amb_lap = [] 63 | for i in range(len(laplace)): 64 | i1 = max(0, int(i - nb / 2)) 65 | i2 = min(len(laplace), int(i + nb / 2)) 66 | lap = laplace[i1: i2] 67 | para, _ = optimize.curve_fit(linear, np.arange(i1, i2), lap) 68 | lapi_ = i * para[0] + para[1] 69 | if laplace[i] / lapi_ < threshold: 70 | amb_img.append(imgs[i]) 71 | amb_lap.append(laplace[i]) 72 | if mv_files: 73 | if not os.path.exists(path + '/../noise/'): 74 | os.makedirs(path + '/../noise/') 75 | file_name = amb_img[-1].split('/')[-1].split('\\')[-1] 76 | shutil.move(amb_img[-1], path + '/../noise/' + file_name) 77 | return amb_img, amb_lap 78 | 79 | 80 | def mask_images(img_path, msk_path, sv_path=None, no_mask=False): 81 | image_names = sorted(os.listdir(img_path)) 82 | image_names = [img for img in image_names if img.endswith('.png') or img.endswith('.jpg')] 83 | msk_names = sorted(os.listdir(msk_path)) 84 | msk_names = [img for img in msk_names if img.endswith('.png') or img.endswith('.jpg')] 85 | 86 | if sv_path is None: 87 | if img_path.endswith('/'): 88 | img_path = img_path[:-1] 89 | sv_path = '/'.join(img_path.split('/')[:-1]) + '/masked_images/' 90 | if not os.path.exists(sv_path) and not os.path.exists(sv_path + '../unmasked_images/'): 91 | os.makedirs(sv_path) 92 | else: 93 | return sv_path 94 | 95 | for i in range(len(image_names)): 96 | image_name, msk_name = image_names[i], msk_names[i] 97 | mask = np.array(Image.open(msk_path + '/' + image_name)) 98 | image = np.array(Image.open(img_path + '/' + image_name)) 99 | mask = cv2.resize(mask, (image.shape[1], image.shape[0])) 100 | if no_mask: 101 | mask = np.ones_like(mask) 102 | if mask.max() == 1: 103 | mask = mask * 255 104 | # image[mask==0] = 0 105 | masked_image = np.concatenate([image, mask[..., np.newaxis]], axis=-1) 106 | Image.fromarray(masked_image).save(sv_path + image_name) 107 | return sv_path 108 | 109 | 110 | def extract_frames_mp4(path, gap=None, frame_num=300, sv_path=None): 111 | if sv_path is None: 112 | sv_path = '/'.join(path.split('/')[:-1]) + '/images/' 113 | if not os.path.exists(sv_path): 114 | os.makedirs(sv_path) 115 | else: 116 | return sv_path 117 | if not os.path.exists(path): 118 | raise NotADirectoryError(path + ' does not exists.') 119 | vidcap = cv2.VideoCapture(path) 120 | if gap is None: 121 | total_frame_num = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT)) 122 | gap = int(total_frame_num / frame_num) 123 | gap = max(gap, 1) 124 | 125 | success, image = vidcap.read() 126 | cv2.imwrite(sv_path + "/%05d.png" % 0, image) 127 | count = 1 128 | image_count = 1 129 | while success: 130 | success, image = vidcap.read() 131 | if count % gap == 0 and success: 132 | cv2.imwrite(sv_path + "/%05d.png" % image_count, image) 133 | image_count += 1 134 | count += 1 135 | return sv_path 136 | 137 | 138 | def rename_images(path): 139 | image_names = sorted(os.listdir(path)) 140 | image_names = [img for img in image_names if img.endswith('.png') or img.endswith('.jpg')] 141 | for i in range(len(image_names)): 142 | shutil.move(path + '/' + image_names[i], path + '/%05d.png' % i) 143 | 144 | 145 | if __name__ == '__main__': 146 | gap = None 147 | no_mask = False 148 | dataset_name = 'DATA_NAME' 149 | video_path = f'YOUR/PATH/TO/{dataset_name}/{dataset_name}.mp4' 150 | print('Extracting frames from video: ', video_path, ' with gap: ', gap) 151 | img_path = extract_frames_mp4(video_path, gap=gap) 152 | 153 | # print('Removing Blurry Images') 154 | # laplace, _ = select_ambiguity(img_path, nb=10, threshold=0.8, mv_files=True) 155 | # if laplace is not None: 156 | # rename_images(img_path) 157 | if not no_mask: 158 | print('Segmenting images with MiVOS ...') 159 | msk_path = seg_video(img_path=img_path) 160 | torch.cuda.empty_cache() 161 | print('Masking images with masks ...') 162 | msked_path = mask_images(img_path, msk_path, no_mask=no_mask) 163 | 164 | 165 | print('Running COLMAP ...') 166 | colmap2nerf_invoke(img_path) 167 | if img_path.endswith('/'): 168 | img_path = img_path[:-1] 169 | unmsk_path = '/'.join(img_path.split('/')[:-1]) + '/unmasked_images/' 170 | print('Rename masked and unmasked pathes.') 171 | if not no_mask: 172 | os.rename(img_path, unmsk_path) 173 | os.rename(msked_path, img_path) 174 | 175 | 176 | def red2mask(img_dir): 177 | img_paths = glob.glob(os.path.join(img_dir, "*.png")) 178 | imgs = [cv2.cv2.cvtColor(cv2.imread(x) , cv2.COLOR_BGR2GRAY) for x in img_paths] 179 | save_dir = os.path.join(os.path.dirname(img_dir, "white_mask")) 180 | os.makedirs(save_dir, exist_ok=True) 181 | for idx, img_path in enumerate(img_paths): 182 | save_path = os.path.join(save_dir, os.path.basename(img_path)) 183 | cv2.imwrite(save_path, imgs[idx]) -------------------------------------------------------------------------------- /experimental_logs/README.md: -------------------------------------------------------------------------------- 1 | # Introduction of Experimental Logs 2 | 3 | The source code of our [latest arXiv paper](https://arxiv.org/abs/2412.04282v2) has been released in this repository. Please feel free to test it if you are interested. 4 | For the false setting problem in the Table 1 of our [first arXiv paper](https://arxiv.org/pdf/2412.04282v1), we provide the detailed training settings and experimental logs for each sequence below: 5 | 6 | 7 | | Scene | Cook Spinach | Sear Steak | Flame Steak | Flame Salmon | Coffee Martini | Cut Roast Beef | 8 | |------------------|:------------:|:----------:|:-----------:|:------------:|:--------------:|:--------------:| 9 | | Training Setting | correct | wrong | wrong | wrong | wrong | wrong | 10 | | Parameter Setting | `eval=True` | `eval=False` | `eval=False` | `eval=False` | `eval=False` | `eval=False` | 11 | | Timestamp of Files | 2024.11.05 | 2024.11.05 | 2024.11.05 | 2024.11.04 | 2024.11.04 | 2024.11.05 | 12 | | Config Args | [cfg_args](/experimental_logs/asset/cook_spinach_node.zip) | [cfg_args](/experimental_logs/asset/sear_steak_node.zip) | [cfg_args](/experimental_logs/asset/flame_steak_node.zip) | [cfg_args](/experimental_logs/asset/flame_salmon_1_node.zip) | [cfg_args](/experimental_logs/asset/coffee_martini_node.zip) | [cfg_args](/experimental_logs/asset/cut_roasted_beef_node.zip) | 13 | 14 | 15 | ### ✅ Notes: 16 | 17 | **1.Config Args** 18 | 19 |

20 | 21 | Cook Spinach 22 | 23 | 24 | Cut Roasted Beef 25 | 26 |

27 | 28 | **2.Timestamp of Files** 29 | These files were generated during the previous experiments. You can easily retrieve the timestamp of each file using the following commands: 30 | 31 | - **On Ubuntu**: 32 | ``` 33 | cd asset 34 | unzip cook_spinach_node.zip 35 | cd cook_spinach_node 36 | ls -l --time-style=long-iso 37 | ``` 38 | where the date is the timestamp, like this: 39 | ``` 40 | 2024-11-05 14:21 cfg_args 41 | 2024-11-05 20:40 test_43000_runtimeresults.json 42 | ``` 43 | 44 | - **On Windows**: 45 | just right-click → Properties → check "Modified" time. -------------------------------------------------------------------------------- /experimental_logs/asset/coffee_martini_node.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/coffee_martini_node.zip -------------------------------------------------------------------------------- /experimental_logs/asset/cook_spinach_node.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/cook_spinach_node.zip -------------------------------------------------------------------------------- /experimental_logs/asset/cut_roasted_beef_node.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/cut_roasted_beef_node.zip -------------------------------------------------------------------------------- /experimental_logs/asset/flame_salmon_1_node.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/flame_salmon_1_node.zip -------------------------------------------------------------------------------- /experimental_logs/asset/flame_steak_node.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/flame_steak_node.zip -------------------------------------------------------------------------------- /experimental_logs/asset/img/arg_cook_spinach.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/img/arg_cook_spinach.jpg -------------------------------------------------------------------------------- /experimental_logs/asset/img/arg_cut_roasted_beef.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/img/arg_cut_roasted_beef.jpg -------------------------------------------------------------------------------- /experimental_logs/asset/sear_steak_node.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/experimental_logs/asset/sear_steak_node.zip -------------------------------------------------------------------------------- /gaussian_renderer/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/gaussian_renderer/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /gaussian_renderer/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/gaussian_renderer/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /gaussian_renderer/__pycache__/network_gui.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/gaussian_renderer/__pycache__/network_gui.cpython-310.pyc -------------------------------------------------------------------------------- /gaussian_renderer/__pycache__/network_gui.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/gaussian_renderer/__pycache__/network_gui.cpython-38.pyc -------------------------------------------------------------------------------- /gaussian_renderer/network_gui.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import traceback 14 | import socket 15 | import json 16 | from scene.cameras import MiniCam 17 | 18 | host = "127.0.0.1" 19 | port = 6009 20 | 21 | conn = None 22 | addr = None 23 | 24 | listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 25 | 26 | 27 | def init(wish_host, wish_port): 28 | global host, port, listener 29 | host = wish_host 30 | port = wish_port 31 | listener.bind((host, port)) 32 | listener.listen() 33 | listener.settimeout(0) 34 | 35 | 36 | def try_connect(): 37 | global conn, addr, listener 38 | try: 39 | conn, addr = listener.accept() 40 | print(f"\nConnected by {addr}") 41 | conn.settimeout(None) 42 | except Exception as inst: 43 | pass 44 | 45 | 46 | def read(): 47 | global conn 48 | messageLength = conn.recv(4) 49 | messageLength = int.from_bytes(messageLength, 'little') 50 | message = conn.recv(messageLength) 51 | return json.loads(message.decode("utf-8")) 52 | 53 | 54 | def send(message_bytes, verify): 55 | global conn 56 | if message_bytes != None: 57 | conn.sendall(message_bytes) 58 | conn.sendall(len(verify).to_bytes(4, 'little')) 59 | conn.sendall(bytes(verify, 'ascii')) 60 | 61 | 62 | def receive(): 63 | message = read() 64 | 65 | width = message["resolution_x"] 66 | height = message["resolution_y"] 67 | 68 | if width != 0 and height != 0: 69 | try: 70 | do_training = bool(message["train"]) 71 | fovy = message["fov_y"] 72 | fovx = message["fov_x"] 73 | znear = message["z_near"] 74 | zfar = message["z_far"] 75 | do_shs_python = bool(message["shs_python"]) 76 | do_rot_scale_python = bool(message["rot_scale_python"]) 77 | keep_alive = bool(message["keep_alive"]) 78 | scaling_modifier = message["scaling_modifier"] 79 | world_view_transform = torch.reshape(torch.tensor(message["view_matrix"]), (4, 4)).cuda() 80 | world_view_transform[:, 1] = -world_view_transform[:, 1] 81 | world_view_transform[:, 2] = -world_view_transform[:, 2] 82 | full_proj_transform = torch.reshape(torch.tensor(message["view_projection_matrix"]), (4, 4)).cuda() 83 | full_proj_transform[:, 1] = -full_proj_transform[:, 1] 84 | custom_cam = MiniCam(width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform) 85 | except Exception as e: 86 | print("") 87 | traceback.print_exc() 88 | raise e 89 | return custom_cam, do_training, do_shs_python, do_rot_scale_python, keep_alive, scaling_modifier 90 | else: 91 | return None, None, None, None, None, None 92 | -------------------------------------------------------------------------------- /imgs/Framework.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/imgs/Framework.jpg -------------------------------------------------------------------------------- /install_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | conda create -n taylorgaussian python=3.10 4 | conda activate taylorgaussian 5 | 6 | pip install -r requirements.txt 7 | pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117 8 | pip install setuptools==69.5.1 9 | pip install numpy==1.23.5 10 | pip install Cmake 11 | 12 | pip install git+https://github.com/Po-Hsun-Su/pytorch-ssim.git 13 | pip install git+https://github.com/facebookresearch/pytorch3d.git 14 | 15 | 16 | cd submodules 17 | git clone https://github.com/ashawkey/diff-gaussian-rasterization.git 18 | cd ./diff-gaussian-rasterization/ 19 | pip install -e . 20 | cd .. 21 | git clone https://github.com/camenduru/simple-knn.git 22 | cd ./simple-knn/ 23 | pip install -e . -------------------------------------------------------------------------------- /lpipsPyTorch/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .modules.lpips import LPIPS 4 | 5 | 6 | def lpips(x: torch.Tensor, 7 | y: torch.Tensor, 8 | net_type: str = 'alex', 9 | version: str = '0.1'): 10 | r"""Function that measures 11 | Learned Perceptual Image Patch Similarity (LPIPS). 12 | 13 | Arguments: 14 | x, y (torch.Tensor): the input tensors to compare. 15 | net_type (str): the network type to compare the features: 16 | 'alex' | 'squeeze' | 'vgg'. Default: 'alex'. 17 | version (str): the version of LPIPS. Default: 0.1. 18 | """ 19 | device = x.device 20 | criterion = LPIPS(net_type, version).to(device) 21 | return criterion(x, y) 22 | -------------------------------------------------------------------------------- /lpipsPyTorch/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/lpipsPyTorch/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /lpipsPyTorch/modules/__pycache__/lpips.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/lpipsPyTorch/modules/__pycache__/lpips.cpython-310.pyc -------------------------------------------------------------------------------- /lpipsPyTorch/modules/__pycache__/networks.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/lpipsPyTorch/modules/__pycache__/networks.cpython-310.pyc -------------------------------------------------------------------------------- /lpipsPyTorch/modules/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/lpipsPyTorch/modules/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /lpipsPyTorch/modules/lpips.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from .networks import get_network, LinLayers 5 | from .utils import get_state_dict 6 | 7 | 8 | class LPIPS(nn.Module): 9 | r"""Creates a criterion that measures 10 | Learned Perceptual Image Patch Similarity (LPIPS). 11 | 12 | Arguments: 13 | net_type (str): the network type to compare the features: 14 | 'alex' | 'squeeze' | 'vgg'. Default: 'alex'. 15 | version (str): the version of LPIPS. Default: 0.1. 16 | """ 17 | 18 | def __init__(self, net_type: str = 'alex', version: str = '0.1'): 19 | assert version in ['0.1'], 'v0.1 is only supported now' 20 | 21 | super(LPIPS, self).__init__() 22 | 23 | # pretrained network 24 | self.net = get_network(net_type) 25 | 26 | # linear layers 27 | self.lin = LinLayers(self.net.n_channels_list) 28 | self.lin.load_state_dict(get_state_dict(net_type, version)) 29 | 30 | def forward(self, x: torch.Tensor, y: torch.Tensor): 31 | feat_x, feat_y = self.net(x), self.net(y) 32 | 33 | diff = [(fx - fy) ** 2 for fx, fy in zip(feat_x, feat_y)] 34 | res = [l(d).mean((2, 3), True) for d, l in zip(diff, self.lin)] 35 | 36 | return torch.sum(torch.cat(res, 0), 0, True) 37 | -------------------------------------------------------------------------------- /lpipsPyTorch/modules/networks.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | from itertools import chain 4 | 5 | import torch 6 | import torch.nn as nn 7 | from torchvision import models 8 | 9 | from .utils import normalize_activation 10 | 11 | 12 | def get_network(net_type: str): 13 | if net_type == 'alex': 14 | return AlexNet() 15 | elif net_type == 'squeeze': 16 | return SqueezeNet() 17 | elif net_type == 'vgg': 18 | return VGG16() 19 | else: 20 | raise NotImplementedError('choose net_type from [alex, squeeze, vgg].') 21 | 22 | 23 | class LinLayers(nn.ModuleList): 24 | def __init__(self, n_channels_list: Sequence[int]): 25 | super(LinLayers, self).__init__([ 26 | nn.Sequential( 27 | nn.Identity(), 28 | nn.Conv2d(nc, 1, 1, 1, 0, bias=False) 29 | ) for nc in n_channels_list 30 | ]) 31 | 32 | for param in self.parameters(): 33 | param.requires_grad = False 34 | 35 | 36 | class BaseNet(nn.Module): 37 | def __init__(self): 38 | super(BaseNet, self).__init__() 39 | 40 | # register buffer 41 | self.register_buffer( 42 | 'mean', torch.Tensor([-.030, -.088, -.188])[None, :, None, None]) 43 | self.register_buffer( 44 | 'std', torch.Tensor([.458, .448, .450])[None, :, None, None]) 45 | 46 | def set_requires_grad(self, state: bool): 47 | for param in chain(self.parameters(), self.buffers()): 48 | param.requires_grad = state 49 | 50 | def z_score(self, x: torch.Tensor): 51 | return (x - self.mean) / self.std 52 | 53 | def forward(self, x: torch.Tensor): 54 | x = self.z_score(x) 55 | 56 | output = [] 57 | for i, (_, layer) in enumerate(self.layers._modules.items(), 1): 58 | x = layer(x) 59 | if i in self.target_layers: 60 | output.append(normalize_activation(x)) 61 | if len(output) == len(self.target_layers): 62 | break 63 | return output 64 | 65 | 66 | class SqueezeNet(BaseNet): 67 | def __init__(self): 68 | super(SqueezeNet, self).__init__() 69 | 70 | self.layers = models.squeezenet1_1(True).features 71 | self.target_layers = [2, 5, 8, 10, 11, 12, 13] 72 | self.n_channels_list = [64, 128, 256, 384, 384, 512, 512] 73 | 74 | self.set_requires_grad(False) 75 | 76 | 77 | class AlexNet(BaseNet): 78 | def __init__(self): 79 | super(AlexNet, self).__init__() 80 | 81 | self.layers = models.alexnet(True).features 82 | self.target_layers = [2, 5, 8, 10, 12] 83 | self.n_channels_list = [64, 192, 384, 256, 256] 84 | 85 | self.set_requires_grad(False) 86 | 87 | 88 | class VGG16(BaseNet): 89 | def __init__(self): 90 | super(VGG16, self).__init__() 91 | 92 | self.layers = models.vgg16(weights=models.VGG16_Weights.IMAGENET1K_V1).features 93 | self.target_layers = [4, 9, 16, 23, 30] 94 | self.n_channels_list = [64, 128, 256, 512, 512] 95 | 96 | self.set_requires_grad(False) 97 | -------------------------------------------------------------------------------- /lpipsPyTorch/modules/utils.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import torch 4 | 5 | 6 | def normalize_activation(x, eps=1e-10): 7 | norm_factor = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) 8 | return x / (norm_factor + eps) 9 | 10 | 11 | def get_state_dict(net_type: str = 'alex', version: str = '0.1'): 12 | # build url 13 | url = 'https://raw.githubusercontent.com/richzhang/PerceptualSimilarity/' \ 14 | + f'master/lpips/weights/v{version}/{net_type}.pth' 15 | 16 | # download 17 | old_state_dict = torch.hub.load_state_dict_from_url( 18 | url, progress=True, 19 | map_location=None if torch.cuda.is_available() else torch.device('cpu') 20 | ) 21 | 22 | # rename keys 23 | new_state_dict = OrderedDict() 24 | for key, val in old_state_dict.items(): 25 | new_key = key 26 | new_key = new_key.replace('lin', '') 27 | new_key = new_key.replace('model.', '') 28 | new_state_dict[new_key] = val 29 | 30 | return new_state_dict 31 | -------------------------------------------------------------------------------- /metrics.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from pathlib import Path 13 | import os 14 | from PIL import Image 15 | import torch 16 | import torchvision.transforms.functional as tf 17 | from utils.loss_utils import ssim 18 | from lpipsPyTorch import lpips 19 | import json 20 | from tqdm import tqdm 21 | from utils.image_utils import psnr 22 | from argparse import ArgumentParser 23 | 24 | 25 | def readImages(renders_dir, gt_dir): 26 | renders = [] 27 | gts = [] 28 | image_names = [] 29 | for fname in os.listdir(renders_dir): 30 | render = Image.open(renders_dir / fname) 31 | gt = Image.open(gt_dir / fname) 32 | renders.append(tf.to_tensor(render).unsqueeze(0)[:, :3, :, :].cuda()) 33 | gts.append(tf.to_tensor(gt).unsqueeze(0)[:, :3, :, :].cuda()) 34 | image_names.append(fname) 35 | return renders, gts, image_names 36 | 37 | 38 | def evaluate(model_paths): 39 | full_dict = {} 40 | per_view_dict = {} 41 | full_dict_polytopeonly = {} 42 | per_view_dict_polytopeonly = {} 43 | print("") 44 | 45 | for scene_dir in model_paths: 46 | try: 47 | print("Scene:", scene_dir) 48 | full_dict[scene_dir] = {} 49 | per_view_dict[scene_dir] = {} 50 | full_dict_polytopeonly[scene_dir] = {} 51 | per_view_dict_polytopeonly[scene_dir] = {} 52 | 53 | test_dir = Path(scene_dir) / "test" 54 | 55 | for method in os.listdir(test_dir): 56 | if not method.startswith("ours"): 57 | continue 58 | print("Method:", method) 59 | 60 | full_dict[scene_dir][method] = {} 61 | per_view_dict[scene_dir][method] = {} 62 | full_dict_polytopeonly[scene_dir][method] = {} 63 | per_view_dict_polytopeonly[scene_dir][method] = {} 64 | 65 | method_dir = test_dir / method 66 | gt_dir = method_dir / "gt" 67 | renders_dir = method_dir / "renders" 68 | renders, gts, image_names = readImages(renders_dir, gt_dir) 69 | 70 | ssims = [] 71 | psnrs = [] 72 | lpipss = [] 73 | 74 | for idx in tqdm(range(len(renders)), desc="Metric evaluation progress"): 75 | ssims.append(ssim(renders[idx], gts[idx])) 76 | psnrs.append(psnr(renders[idx], gts[idx])) 77 | lpipss.append(lpips(renders[idx], gts[idx], net_type='vgg')) 78 | 79 | print(" SSIM : {:>12.7f}".format(torch.tensor(ssims).mean(), ".5")) 80 | print(" PSNR : {:>12.7f}".format(torch.tensor(psnrs).mean(), ".5")) 81 | print(" LPIPS: {:>12.7f}".format(torch.tensor(lpipss).mean(), ".5")) 82 | print("") 83 | 84 | full_dict[scene_dir][method].update({"SSIM": torch.tensor(ssims).mean().item(), 85 | "PSNR": torch.tensor(psnrs).mean().item(), 86 | "LPIPS": torch.tensor(lpipss).mean().item()}) 87 | per_view_dict[scene_dir][method].update( 88 | {"SSIM": {name: ssim for ssim, name in zip(torch.tensor(ssims).tolist(), image_names)}, 89 | "PSNR": {name: psnr for psnr, name in zip(torch.tensor(psnrs).tolist(), image_names)}, 90 | "LPIPS": {name: lp for lp, name in zip(torch.tensor(lpipss).tolist(), image_names)}}) 91 | 92 | with open(scene_dir + "/results.json", 'w') as fp: 93 | json.dump(full_dict[scene_dir], fp, indent=True) 94 | with open(scene_dir + "/per_view.json", 'w') as fp: 95 | json.dump(per_view_dict[scene_dir], fp, indent=True) 96 | except: 97 | print("Unable to compute metrics for model", scene_dir) 98 | 99 | 100 | if __name__ == "__main__": 101 | device = torch.device("cuda:0") 102 | torch.cuda.set_device(device) 103 | 104 | # Set up command line argument parser 105 | parser = ArgumentParser(description="Training script parameters") 106 | parser.add_argument('--model_paths', '-m', required=True, nargs="+", type=str, default=[]) 107 | args = parser.parse_args() 108 | evaluate(args.model_paths) 109 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.22.0 2 | opencv-python==4.5.5.62 3 | PyYAML==6.0 4 | scipy==1.10.1 5 | tensorboard==2.14.0 6 | tqdm==4.66.1 7 | imageio 8 | plyfile 9 | piq 10 | dearpygui 11 | lpips 12 | pytorch_msssim 13 | matplotlib 14 | scikit-image 15 | git+https://github.com/Po-Hsun-Su/pytorch-ssim.git 16 | git+https://github.com/facebookresearch/pytorch3d.git 17 | -------------------------------------------------------------------------------- /scene/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /scene/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /scene/__pycache__/cameras.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/cameras.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/cameras.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/cameras.cpython-38.pyc -------------------------------------------------------------------------------- /scene/__pycache__/cameras_st.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/cameras_st.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/colmap_loader.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/colmap_loader.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/colmap_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/colmap_loader.cpython-37.pyc -------------------------------------------------------------------------------- /scene/__pycache__/colmap_loader.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/colmap_loader.cpython-38.pyc -------------------------------------------------------------------------------- /scene/__pycache__/dataset_readers.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/dataset_readers.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/dataset_readers.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/dataset_readers.cpython-37.pyc -------------------------------------------------------------------------------- /scene/__pycache__/dataset_readers.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/dataset_readers.cpython-38.pyc -------------------------------------------------------------------------------- /scene/__pycache__/deform_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/deform_model.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/deform_model.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/deform_model.cpython-38.pyc -------------------------------------------------------------------------------- /scene/__pycache__/gaussian_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/gaussian_model.cpython-310.pyc -------------------------------------------------------------------------------- /scene/__pycache__/gaussian_model.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/scene/__pycache__/gaussian_model.cpython-38.pyc -------------------------------------------------------------------------------- /scene/cameras.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | from torch import nn 14 | import numpy as np 15 | from utils.graphics_utils import getWorld2View2, getProjectionMatrix 16 | 17 | 18 | class Camera(nn.Module): 19 | def __init__(self, colmap_id, R, T, FoVx, FoVy, image, gt_alpha_mask, image_name, uid, 20 | trans=np.array([0.0, 0.0, 0.0]), scale=1.0, data_device="cuda", fid=None, depth=None, flow_dirs=[]): 21 | super(Camera, self).__init__() 22 | 23 | self.uid = uid 24 | self.colmap_id = colmap_id 25 | self.R = R 26 | self.T = T 27 | self.FoVx = FoVx 28 | self.FoVy = FoVy 29 | self.image_name = image_name 30 | self.flow_dirs = flow_dirs 31 | 32 | try: 33 | self.data_device = torch.device(data_device) 34 | except Exception as e: 35 | print(e) 36 | print(f"[Warning] Custom device {data_device} failed, fallback to default cuda device") 37 | self.data_device = torch.device("cuda") 38 | 39 | self.original_image = image.clamp(0.0, 1.0).to(self.data_device) #.to(self.data_device) 40 | self.fid = torch.Tensor(np.array([fid])).to(self.data_device) 41 | self.image_width = self.original_image.shape[2] 42 | self.image_height = self.original_image.shape[1] 43 | self.depth = torch.Tensor(depth).to(self.data_device) if depth is not None else None 44 | self.gt_alpha_mask = gt_alpha_mask 45 | 46 | if gt_alpha_mask is not None: 47 | self.gt_alpha_mask = self.gt_alpha_mask.to(self.data_device) 48 | # self.original_image *= gt_alpha_mask.to(self.data_device) 49 | 50 | self.zfar = 100.0 51 | self.znear = 0.01 52 | 53 | self.trans = trans 54 | self.scale = scale 55 | 56 | self.world_view_transform = torch.tensor(getWorld2View2(R, T, trans, scale)).transpose(0, 1).to(self.data_device) 57 | self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0, 1).to(self.data_device) 58 | self.full_proj_transform = ( 59 | self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) 60 | self.camera_center = self.world_view_transform.inverse()[3, :3] 61 | 62 | def reset_extrinsic(self, R, T): 63 | self.world_view_transform = torch.tensor(getWorld2View2(R, T, self.trans, self.scale)).transpose(0, 1).cuda() 64 | self.full_proj_transform = ( 65 | self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) 66 | self.camera_center = self.world_view_transform.inverse()[3, :3] 67 | 68 | def load2device(self, data_device='cuda'): 69 | self.original_image = self.original_image.to(data_device) 70 | self.world_view_transform = self.world_view_transform.to(data_device) 71 | self.projection_matrix = self.projection_matrix.to(data_device) 72 | self.full_proj_transform = self.full_proj_transform.to(data_device) 73 | self.camera_center = self.camera_center.to(data_device) 74 | self.fid = self.fid.to(data_device) 75 | 76 | 77 | class MiniCam: 78 | def __init__(self, width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform): 79 | self.image_width = width 80 | self.image_height = height 81 | self.FoVy = fovy 82 | self.FoVx = fovx 83 | self.znear = znear 84 | self.zfar = zfar 85 | self.world_view_transform = world_view_transform 86 | self.full_proj_transform = full_proj_transform 87 | view_inv = torch.inverse(self.world_view_transform) 88 | self.camera_center = view_inv[3][:3] 89 | 90 | def reset_extrinsic(self, R, T): 91 | self.world_view_transform = torch.tensor(getWorld2View2(R, T)).transpose(0, 1).cuda() 92 | self.full_proj_transform = ( 93 | self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) 94 | self.camera_center = self.world_view_transform.inverse()[3, :3] 95 | -------------------------------------------------------------------------------- /scene/deform_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from utils.time_utils import DeformNetwork, ControlNodeWarp, StaticNetwork 5 | import os 6 | from utils.system_utils import searchForMaxIteration 7 | from utils.general_utils import get_expon_lr_func 8 | 9 | 10 | model_dict = {'mlp': DeformNetwork, 'node': ControlNodeWarp, 'static': StaticNetwork} 11 | 12 | 13 | class DeformModel: 14 | def __init__(self, deform_type='node', is_blender=False, d_rot_as_res=True, **kwargs): 15 | self.deform = model_dict[deform_type](is_blender=is_blender, d_rot_as_res=d_rot_as_res, **kwargs).cuda() 16 | self.name = self.deform.name 17 | self.optimizer = None 18 | self.spatial_lr_scale = 5 19 | self.d_rot_as_res = d_rot_as_res 20 | 21 | @property 22 | def reg_loss(self): 23 | return self.deform.reg_loss 24 | 25 | def step(self, xyz, time_emb, iteration=0, **kwargs): 26 | return self.deform(xyz, time_emb, iteration=iteration, **kwargs) 27 | 28 | def train_setting(self, training_args): 29 | l = [ 30 | {'params': group['params'], 31 | 'lr': training_args.position_lr_init * self.spatial_lr_scale * training_args.deform_lr_scale, 32 | "name": group['name']} 33 | for group in self.deform.trainable_parameters() 34 | ] 35 | self.optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15) 36 | 37 | self.deform_scheduler_args = get_expon_lr_func(lr_init=training_args.position_lr_init * self.spatial_lr_scale * training_args.deform_lr_scale, lr_final=training_args.position_lr_final * training_args.deform_lr_scale, lr_delay_mult=training_args.position_lr_delay_mult, max_steps=training_args.deform_lr_max_steps) 38 | if self.name == 'node': 39 | 40 | self.deform.as_gaussians.training_setup(training_args) 41 | 42 | def save_weights(self, model_path, iteration): 43 | out_weights_path = os.path.join(model_path, "deform/iteration_{}".format(iteration)) 44 | os.makedirs(out_weights_path, exist_ok=True) 45 | torch.save(self.deform.state_dict(), os.path.join(out_weights_path, 'deform.pth')) 46 | 47 | def load_weights(self, model_path, iteration=-1): 48 | if iteration == -1: 49 | loaded_iter = searchForMaxIteration(os.path.join(model_path, "deform")) 50 | else: 51 | loaded_iter = iteration 52 | weights_path = os.path.join(model_path, "deform/iteration_{}/deform.pth".format(loaded_iter)) 53 | if os.path.exists(weights_path): 54 | self.deform.load_state_dict(torch.load(weights_path)) 55 | return True 56 | else: 57 | return False 58 | 59 | def update_learning_rate(self, iteration): 60 | for param_group in self.optimizer.param_groups: 61 | if param_group["name"] == "deform" or param_group["name"] == "mlp" or 'node' in param_group['name']: 62 | lr = self.deform_scheduler_args(iteration) 63 | param_group['lr'] = lr 64 | return lr 65 | 66 | def densify(self, max_grad, x, x_grad, **kwargs): 67 | if self.name == 'node': 68 | self.deform.densify(max_grad=max_grad, optimizer=self.optimizer, x=x, x_grad=x_grad, **kwargs) 69 | else: 70 | return 71 | 72 | def update(self, iteration): 73 | self.deform.update(iteration) 74 | -------------------------------------------------------------------------------- /script/Readme.md: -------------------------------------------------------------------------------- 1 | # Environment Setup 2 | 3 | ``` 4 | conda deactivate 5 | conda create -n colmap python=3.8 6 | conda activate colmap 7 | pip install opencv-python-headless 8 | pip install tqdm 9 | pip install natsort 10 | pip install Pillow 11 | conda install colmap -c conda-forge 12 | conda install pytorch==1.13.0 torchvision==0.14.0 torchaudio==0.13.0 pytorch-cuda=11.7 -c pytorch -c nvidia 13 | ``` -------------------------------------------------------------------------------- /script/__pycache__/pre_immersive_distorted.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/script/__pycache__/pre_immersive_distorted.cpython-310.pyc -------------------------------------------------------------------------------- /script/n3d_process.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | # Copyright (c) 2023 OPPO 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import os 24 | from pathlib import Path 25 | 26 | import cv2 27 | import glob 28 | import tqdm 29 | import numpy as np 30 | import shutil 31 | import pickle 32 | import sys 33 | import argparse 34 | 35 | 36 | sys.path.append(".") 37 | from thirdparty.gaussian_splatting.utils.my_utils import posetow2c_matrcs, rotmat2qvec 38 | from thirdparty.colmap.pre_colmap import * 39 | from thirdparty.gaussian_splatting.helper3dg import getcolmapsinglen3d 40 | from script.utils_pre import write_colmap 41 | 42 | 43 | 44 | 45 | def extractframes(videopath: Path, startframe=0, endframe=300, downscale=1, save_subdir = '', ext='png'): 46 | output_dir = videopath.parent / save_subdir / videopath.stem 47 | 48 | if all((output_dir / f"{i}.{ext}").exists() for i in range(startframe, endframe)): 49 | print(f"Already extracted all the frames in {output_dir}") 50 | return 51 | 52 | cam = cv2.VideoCapture(str(videopath)) 53 | cam.set(cv2.CAP_PROP_POS_FRAMES, startframe) 54 | 55 | output_dir.mkdir(parents=True, exist_ok=True) 56 | 57 | for i in range(startframe, endframe): 58 | success, frame = cam.read() 59 | if not success: 60 | print(f"Error reading frame {i}") 61 | break 62 | 63 | if downscale > 1: 64 | new_width, new_height = int(frame.shape[1] / downscale), int(frame.shape[0] / downscale) 65 | frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) 66 | 67 | cv2.imwrite(str(output_dir / f"{i}.{ext}"), frame) 68 | 69 | cam.release() 70 | 71 | 72 | def preparecolmapdynerf(folder, offset=0): 73 | folderlist = sorted(folder.glob("cam??/")) 74 | 75 | savedir = folder / f"colmap_{offset}" / "input" 76 | savedir.mkdir(exist_ok=True, parents=True) 77 | 78 | for folder in folderlist: 79 | imagepath = folder / f"{offset}.png" 80 | imagesavepath = savedir / f"{folder.name}.png" 81 | 82 | if (imagesavepath.exists()): 83 | continue 84 | print('imagepath:',imagepath) 85 | assert imagepath.exists 86 | print('imagesavepath:',imagesavepath) 87 | # shutil.copy(imagepath, imagesavepath) 88 | imagesavepath.symlink_to(imagepath.resolve()) 89 | 90 | 91 | def convertdynerftocolmapdb(path, offset=0, downscale=1): 92 | originnumpy = path / "poses_bounds.npy" 93 | video_paths = sorted(path.glob('cam*.mp4')) 94 | 95 | with open(originnumpy, 'rb') as numpy_file: 96 | poses_bounds = np.load(numpy_file) 97 | poses = poses_bounds[:, :15].reshape(-1, 3, 5) 98 | 99 | llffposes = poses.copy().transpose(1, 2, 0) 100 | w2c_matriclist = posetow2c_matrcs(llffposes) 101 | assert (type(w2c_matriclist) == list) 102 | 103 | cameras = [] 104 | for i in range(len(poses)): 105 | cameraname = video_paths[i].stem 106 | m = w2c_matriclist[i] 107 | colmapR = m[:3, :3] 108 | T = m[:3, 3] 109 | 110 | H, W, focal = poses[i, :, -1] / downscale 111 | 112 | colmapQ = rotmat2qvec(colmapR) 113 | 114 | camera = { 115 | 'id': i + 1, 116 | 'filename': f"{cameraname}.png", 117 | 'w': W, 118 | 'h': H, 119 | 'fx': focal, 120 | 'fy': focal, 121 | 'cx': W // 2, 122 | 'cy': H // 2, 123 | 'q': colmapQ, 124 | 't': T, 125 | } 126 | cameras.append(camera) 127 | 128 | write_colmap(path, cameras, offset) 129 | 130 | 131 | 132 | if __name__ == "__main__" : 133 | parser = argparse.ArgumentParser() 134 | 135 | parser.add_argument("--videopath", default="", type=str) 136 | parser.add_argument("--startframe", default=0, type=int) 137 | parser.add_argument("--endframe", default=50, type=int) 138 | parser.add_argument("--downscale", default=1, type=int) 139 | 140 | args = parser.parse_args() 141 | videopath = Path(args.videopath) 142 | 143 | startframe = args.startframe 144 | endframe = args.endframe 145 | downscale = args.downscale 146 | 147 | print(f"params: startframe={startframe} - endframe={endframe} - downscale={downscale} - videopath={videopath}") 148 | 149 | if startframe >= endframe: 150 | print("start frame must smaller than end frame") 151 | quit() 152 | if startframe < 0 or endframe > 300: 153 | print("frame must in range 0-300") 154 | quit() 155 | if not videopath.exists(): 156 | print("path not exist") 157 | quit() 158 | ##### step1 159 | print("start extracting 300 frames from videos") 160 | videoslist = sorted(videopath.glob("*.mp4")) 161 | for v in tqdm.tqdm(videoslist, desc="Extract frames from videos"): 162 | extractframes(v, downscale=downscale) 163 | 164 | 165 | 166 | # # ## step2 prepare colmap input 167 | print("start preparing colmap image input") 168 | for offset in range(startframe, endframe): 169 | preparecolmapdynerf(videopath, offset) 170 | 171 | 172 | print("start preparing colmap database input") 173 | # # ## step 3 prepare colmap db file 174 | for offset in tqdm.tqdm(range(startframe, endframe), desc="convertdynerftocolmapdb"): 175 | convertdynerftocolmapdb(videopath, offset, downscale) 176 | 177 | 178 | # ## step 4 run colmap, per frame, if error, reinstall opencv-headless 179 | for offset in range(startframe, endframe): 180 | getcolmapsinglen3d(videopath, offset) 181 | 182 | -------------------------------------------------------------------------------- /script/utils_pre.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append(".") 3 | from thirdparty.colmap.pre_colmap import * 4 | from pathlib import Path 5 | 6 | def write_colmap(path, cameras, offset=0): 7 | projectfolder = path / f"colmap_{offset}" 8 | manualfolder = projectfolder / "manual" 9 | manualfolder.mkdir(exist_ok=True) 10 | 11 | savetxt = manualfolder / "images.txt" 12 | savecamera = manualfolder / "cameras.txt" 13 | savepoints = manualfolder / "points3D.txt" 14 | 15 | imagetxtlist = [] 16 | cameratxtlist = [] 17 | 18 | db_file = projectfolder / "input.db" 19 | if db_file.exists(): 20 | db_file.unlink() 21 | 22 | db = COLMAPDatabase.connect(db_file) 23 | 24 | db.create_tables() 25 | 26 | 27 | for cam in cameras: 28 | id = cam['id'] 29 | filename = cam['filename'] 30 | 31 | # intrinsics 32 | w = cam['w'] 33 | h = cam['h'] 34 | fx = cam['fx'] 35 | fy = cam['fy'] 36 | cx = cam['cx'] 37 | cy = cam['cy'] 38 | 39 | # extrinsics 40 | colmapQ = cam['q'] 41 | T = cam['t'] 42 | 43 | # check that cx is almost w /2, idem for cy 44 | # assert abs(cx - w / 2) / cx < 0.10, f"cx is not close to w/2: {cx}, w: {w}" 45 | # assert abs(cy - h / 2) / cy < 0.10, f"cy is not close to h/2: {cy}, h: {h}" 46 | 47 | line = f"{id} " + " ".join(map(str, colmapQ)) + " " + " ".join(map(str, T)) + f" {id} {filename}\n" 48 | imagetxtlist.append(line) 49 | imagetxtlist.append("\n") 50 | 51 | params = np.array((fx , fy, cx, cy,)) 52 | 53 | camera_id = db.add_camera(1, w, h, params) 54 | cameraline = f"{id} PINHOLE {w} {h} {fx} {fy} {cx} {cy}\n" 55 | cameratxtlist.append(cameraline) 56 | image_id = db.add_image(filename, camera_id, prior_q=colmapQ, prior_t=T, image_id=id) 57 | db.commit() 58 | db.close() 59 | 60 | savetxt.write_text("".join(imagetxtlist)) 61 | savecamera.write_text("".join(cameratxtlist)) 62 | savepoints.write_text("") # Creating an empty points3D.txt file 63 | -------------------------------------------------------------------------------- /start_order.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | GPU_ID=$1 3 | 4 | 5 | run_pipeline() { 6 | # train 7 | CUDA_VISIBLE_DEVICES="$GPU_ID" python main.py \ 8 | --source_path data/Neural3D/"$data_name"/colmap_0 \ 9 | --model_path logs/N3DV/"$data_name"_test \ 10 | --deform_type node \ 11 | --node_num 1024 \ 12 | --hyper_dim 8 \ 13 | --is_blender \ 14 | --gt_alpha_mask_as_dynamic_mask \ 15 | --gs_with_motion_mask \ 16 | --eval \ 17 | --local_frame \ 18 | --resolution 2 \ 19 | --W 800 \ 20 | --H 800 \ 21 | --config arguments/N3DV/"$data_name".py 22 | # render 23 | CUDA_VISIBLE_DEVICES="$GPU_ID" python render.py \ 24 | --source_path data/Neural3D/"$data_name"/colmap_0 \ 25 | --model_path logs/N3DV/"$data_name"_test \ 26 | --deform_type node \ 27 | --node_num 1024 \ 28 | --hyper_dim 8 \ 29 | --is_blender \ 30 | --gt_alpha_mask_as_dynamic_mask \ 31 | --gs_with_motion_mask \ 32 | --eval \ 33 | --skip_train \ 34 | --local_frame \ 35 | --resolution 2 \ 36 | --W 800 \ 37 | --H 800 \ 38 | --dataset_type colmap 39 | } 40 | 41 | Neural3D_DA=("cut_roasted_beef" "sear_steak" "flame_steak") 42 | Neural3D_DA=( "cook_spinach" "flame_salmon_1" "coffee_martini" ) 43 | # run_pipeline 44 | for data_name in "${Neural3D_DA[@]}"; 45 | do 46 | echo "Dataset: Neural3D_DA/${data_name}" 47 | run_pipeline "$GPU_ID" "$data_name" 48 | done 49 | -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | diff_gaussian_rasterization.egg-info/ 3 | dist/ 4 | 5 | __pycache__ 6 | 7 | *.so -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/glm"] 2 | path = third_party/glm 3 | url = https://github.com/g-truc/glm.git 4 | -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | cmake_minimum_required(VERSION 3.20) 13 | 14 | project(DiffRast LANGUAGES CUDA CXX) 15 | 16 | set(CMAKE_CXX_STANDARD 17) 17 | set(CMAKE_CXX_EXTENSIONS OFF) 18 | set(CMAKE_CUDA_STANDARD 17) 19 | 20 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") 21 | 22 | add_library(CudaRasterizer 23 | cuda_rasterizer/backward.h 24 | cuda_rasterizer/backward.cu 25 | cuda_rasterizer/forward.h 26 | cuda_rasterizer/forward.cu 27 | cuda_rasterizer/auxiliary.h 28 | cuda_rasterizer/rasterizer_impl.cu 29 | cuda_rasterizer/rasterizer_impl.h 30 | cuda_rasterizer/rasterizer.h 31 | ) 32 | 33 | set_target_properties(CudaRasterizer PROPERTIES CUDA_ARCHITECTURES "75;86") 34 | 35 | target_include_directories(CudaRasterizer PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/cuda_rasterizer) 36 | target_include_directories(CudaRasterizer PRIVATE third_party/glm ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) 37 | -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/LICENSE.md: -------------------------------------------------------------------------------- 1 | Gaussian-Splatting License 2 | =========================== 3 | 4 | **Inria** and **the Max Planck Institut for Informatik (MPII)** hold all the ownership rights on the *Software* named **gaussian-splatting**. 5 | The *Software* is in the process of being registered with the Agence pour la Protection des 6 | Programmes (APP). 7 | 8 | The *Software* is still being developed by the *Licensor*. 9 | 10 | *Licensor*'s goal is to allow the research community to use, test and evaluate 11 | the *Software*. 12 | 13 | ## 1. Definitions 14 | 15 | *Licensee* means any person or entity that uses the *Software* and distributes 16 | its *Work*. 17 | 18 | *Licensor* means the owners of the *Software*, i.e Inria and MPII 19 | 20 | *Software* means the original work of authorship made available under this 21 | License ie gaussian-splatting. 22 | 23 | *Work* means the *Software* and any additions to or derivative works of the 24 | *Software* that are made available under this License. 25 | 26 | 27 | ## 2. Purpose 28 | This license is intended to define the rights granted to the *Licensee* by 29 | Licensors under the *Software*. 30 | 31 | ## 3. Rights granted 32 | 33 | For the above reasons Licensors have decided to distribute the *Software*. 34 | Licensors grant non-exclusive rights to use the *Software* for research purposes 35 | to research users (both academic and industrial), free of charge, without right 36 | to sublicense.. The *Software* may be used "non-commercially", i.e., for research 37 | and/or evaluation purposes only. 38 | 39 | Subject to the terms and conditions of this License, you are granted a 40 | non-exclusive, royalty-free, license to reproduce, prepare derivative works of, 41 | publicly display, publicly perform and distribute its *Work* and any resulting 42 | derivative works in any form. 43 | 44 | ## 4. Limitations 45 | 46 | **4.1 Redistribution.** You may reproduce or distribute the *Work* only if (a) you do 47 | so under this License, (b) you include a complete copy of this License with 48 | your distribution, and (c) you retain without modification any copyright, 49 | patent, trademark, or attribution notices that are present in the *Work*. 50 | 51 | **4.2 Derivative Works.** You may specify that additional or different terms apply 52 | to the use, reproduction, and distribution of your derivative works of the *Work* 53 | ("Your Terms") only if (a) Your Terms provide that the use limitation in 54 | Section 2 applies to your derivative works, and (b) you identify the specific 55 | derivative works that are subject to Your Terms. Notwithstanding Your Terms, 56 | this License (including the redistribution requirements in Section 3.1) will 57 | continue to apply to the *Work* itself. 58 | 59 | **4.3** Any other use without of prior consent of Licensors is prohibited. Research 60 | users explicitly acknowledge having received from Licensors all information 61 | allowing to appreciate the adequacy between of the *Software* and their needs and 62 | to undertake all necessary precautions for its execution and use. 63 | 64 | **4.4** The *Software* is provided both as a compiled library file and as source 65 | code. In case of using the *Software* for a publication or other results obtained 66 | through the use of the *Software*, users are strongly encouraged to cite the 67 | corresponding publications as explained in the documentation of the *Software*. 68 | 69 | ## 5. Disclaimer 70 | 71 | THE USER CANNOT USE, EXPLOIT OR DISTRIBUTE THE *SOFTWARE* FOR COMMERCIAL PURPOSES 72 | WITHOUT PRIOR AND EXPLICIT CONSENT OF LICENSORS. YOU MUST CONTACT INRIA FOR ANY 73 | UNAUTHORIZED USE: stip-sophia.transfert@inria.fr . ANY SUCH ACTION WILL 74 | CONSTITUTE A FORGERY. THIS *SOFTWARE* IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES 75 | OF ANY NATURE AND ANY EXPRESS OR IMPLIED WARRANTIES, WITH REGARDS TO COMMERCIAL 76 | USE, PROFESSIONNAL USE, LEGAL OR NOT, OR OTHER, OR COMMERCIALISATION OR 77 | ADAPTATION. UNLESS EXPLICITLY PROVIDED BY LAW, IN NO EVENT, SHALL INRIA OR THE 78 | AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 79 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 80 | GOODS OR SERVICES, LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION) 81 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 82 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING FROM, OUT OF OR 83 | IN CONNECTION WITH THE *SOFTWARE* OR THE USE OR OTHER DEALINGS IN THE *SOFTWARE*. 84 | -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/README.md: -------------------------------------------------------------------------------- 1 | # Differential Gaussian Rasterization 2 | 3 | **NOTE**: this is a modified version to support depth & alpha rendering (both forward and backward) from the [original repository](https://github.com/graphdeco-inria/diff-gaussian-rasterization). 4 | 5 | ```python 6 | rendered_image, radii, rendered_depth, rendered_alpha = rasterizer( 7 | means3D=means3D, 8 | means2D=means2D, 9 | shs=shs, 10 | colors_precomp=colors_precomp, 11 | opacities=opacity, 12 | scales=scales, 13 | rotations=rotations, 14 | cov3D_precomp=cov3D_precomp, 15 | ) 16 | ``` 17 | 18 | 19 | Used as the rasterization engine for the paper "3D Gaussian Splatting for Real-Time Rendering of Radiance Fields". If you can make use of it in your own research, please be so kind to cite us. 20 | 21 |
22 |
23 |

BibTeX

24 | 
@Article{kerbl3Dgaussians,
25 |       author       = {Kerbl, Bernhard and Kopanas, Georgios and Leimk{\"u}hler, Thomas and Drettakis, George},
26 |       title        = {3D Gaussian Splatting for Real-Time Radiance Field Rendering},
27 |       journal      = {ACM Transactions on Graphics},
28 |       number       = {4},
29 |       volume       = {42},
30 |       month        = {July},
31 |       year         = {2023},
32 |       url          = {https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/}
33 | }
34 |
35 |
36 | -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/cuda_rasterizer/auxiliary.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #ifndef CUDA_RASTERIZER_AUXILIARY_H_INCLUDED 13 | #define CUDA_RASTERIZER_AUXILIARY_H_INCLUDED 14 | 15 | #include "config.h" 16 | #include "stdio.h" 17 | 18 | #define BLOCK_SIZE (BLOCK_X * BLOCK_Y) 19 | #define NUM_WARPS (BLOCK_SIZE/32) 20 | 21 | // Spherical harmonics coefficients 22 | __device__ const float SH_C0 = 0.28209479177387814f; 23 | __device__ const float SH_C1 = 0.4886025119029199f; 24 | __device__ const float SH_C2[] = { 25 | 1.0925484305920792f, 26 | -1.0925484305920792f, 27 | 0.31539156525252005f, 28 | -1.0925484305920792f, 29 | 0.5462742152960396f 30 | }; 31 | __device__ const float SH_C3[] = { 32 | -0.5900435899266435f, 33 | 2.890611442640554f, 34 | -0.4570457994644658f, 35 | 0.3731763325901154f, 36 | -0.4570457994644658f, 37 | 1.445305721320277f, 38 | -0.5900435899266435f 39 | }; 40 | 41 | __forceinline__ __device__ float ndc2Pix(float v, int S) 42 | { 43 | return ((v + 1.0) * S - 1.0) * 0.5; 44 | } 45 | 46 | __forceinline__ __device__ void getRect(const float2 p, int max_radius, uint2& rect_min, uint2& rect_max, dim3 grid) 47 | { 48 | rect_min = { 49 | min(grid.x, max((int)0, (int)((p.x - max_radius) / BLOCK_X))), 50 | min(grid.y, max((int)0, (int)((p.y - max_radius) / BLOCK_Y))) 51 | }; 52 | rect_max = { 53 | min(grid.x, max((int)0, (int)((p.x + max_radius + BLOCK_X - 1) / BLOCK_X))), 54 | min(grid.y, max((int)0, (int)((p.y + max_radius + BLOCK_Y - 1) / BLOCK_Y))) 55 | }; 56 | } 57 | 58 | __forceinline__ __device__ float3 transformPoint4x3(const float3& p, const float* matrix) 59 | { 60 | float3 transformed = { 61 | matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z + matrix[12], 62 | matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z + matrix[13], 63 | matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z + matrix[14], 64 | }; 65 | return transformed; 66 | } 67 | 68 | __forceinline__ __device__ float4 transformPoint4x4(const float3& p, const float* matrix) 69 | { 70 | float4 transformed = { 71 | matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z + matrix[12], 72 | matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z + matrix[13], 73 | matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z + matrix[14], 74 | matrix[3] * p.x + matrix[7] * p.y + matrix[11] * p.z + matrix[15] 75 | }; 76 | return transformed; 77 | } 78 | 79 | __forceinline__ __device__ float3 transformVec4x3(const float3& p, const float* matrix) 80 | { 81 | float3 transformed = { 82 | matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z, 83 | matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z, 84 | matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z, 85 | }; 86 | return transformed; 87 | } 88 | 89 | __forceinline__ __device__ float3 transformVec4x3Transpose(const float3& p, const float* matrix) 90 | { 91 | float3 transformed = { 92 | matrix[0] * p.x + matrix[1] * p.y + matrix[2] * p.z, 93 | matrix[4] * p.x + matrix[5] * p.y + matrix[6] * p.z, 94 | matrix[8] * p.x + matrix[9] * p.y + matrix[10] * p.z, 95 | }; 96 | return transformed; 97 | } 98 | 99 | __forceinline__ __device__ float dnormvdz(float3 v, float3 dv) 100 | { 101 | float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; 102 | float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); 103 | float dnormvdz = (-v.x * v.z * dv.x - v.y * v.z * dv.y + (sum2 - v.z * v.z) * dv.z) * invsum32; 104 | return dnormvdz; 105 | } 106 | 107 | __forceinline__ __device__ float3 dnormvdv(float3 v, float3 dv) 108 | { 109 | float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; 110 | float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); 111 | 112 | float3 dnormvdv; 113 | dnormvdv.x = ((+sum2 - v.x * v.x) * dv.x - v.y * v.x * dv.y - v.z * v.x * dv.z) * invsum32; 114 | dnormvdv.y = (-v.x * v.y * dv.x + (sum2 - v.y * v.y) * dv.y - v.z * v.y * dv.z) * invsum32; 115 | dnormvdv.z = (-v.x * v.z * dv.x - v.y * v.z * dv.y + (sum2 - v.z * v.z) * dv.z) * invsum32; 116 | return dnormvdv; 117 | } 118 | 119 | __forceinline__ __device__ float4 dnormvdv(float4 v, float4 dv) 120 | { 121 | float sum2 = v.x * v.x + v.y * v.y + v.z * v.z + v.w * v.w; 122 | float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); 123 | 124 | float4 vdv = { v.x * dv.x, v.y * dv.y, v.z * dv.z, v.w * dv.w }; 125 | float vdv_sum = vdv.x + vdv.y + vdv.z + vdv.w; 126 | float4 dnormvdv; 127 | dnormvdv.x = ((sum2 - v.x * v.x) * dv.x - v.x * (vdv_sum - vdv.x)) * invsum32; 128 | dnormvdv.y = ((sum2 - v.y * v.y) * dv.y - v.y * (vdv_sum - vdv.y)) * invsum32; 129 | dnormvdv.z = ((sum2 - v.z * v.z) * dv.z - v.z * (vdv_sum - vdv.z)) * invsum32; 130 | dnormvdv.w = ((sum2 - v.w * v.w) * dv.w - v.w * (vdv_sum - vdv.w)) * invsum32; 131 | return dnormvdv; 132 | } 133 | 134 | __forceinline__ __device__ float sigmoid(float x) 135 | { 136 | return 1.0f / (1.0f + expf(-x)); 137 | } 138 | 139 | __forceinline__ __device__ bool in_frustum(int idx, 140 | const float* orig_points, 141 | const float* viewmatrix, 142 | const float* projmatrix, 143 | bool prefiltered, 144 | float3& p_view) 145 | { 146 | float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; 147 | 148 | // Bring points to screen space 149 | float4 p_hom = transformPoint4x4(p_orig, projmatrix); 150 | float p_w = 1.0f / (p_hom.w + 0.0000001f); 151 | float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; 152 | p_view = transformPoint4x3(p_orig, viewmatrix); 153 | 154 | if (p_view.z <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) 155 | { 156 | if (prefiltered) 157 | { 158 | printf("Point is filtered although prefiltered is set. This shouldn't happen!"); 159 | __trap(); 160 | } 161 | return false; 162 | } 163 | return true; 164 | } 165 | 166 | #define CHECK_CUDA(A, debug) \ 167 | A; if(debug) { \ 168 | auto ret = cudaDeviceSynchronize(); \ 169 | if (ret != cudaSuccess) { \ 170 | std::cerr << "\n[CUDA ERROR] in " << __FILE__ << "\nLine " << __LINE__ << ": " << cudaGetErrorString(ret); \ 171 | throw std::runtime_error(cudaGetErrorString(ret)); \ 172 | } \ 173 | } 174 | 175 | #endif -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #ifndef CUDA_RASTERIZER_BACKWARD_H_INCLUDED 13 | #define CUDA_RASTERIZER_BACKWARD_H_INCLUDED 14 | 15 | #include 16 | #include "cuda_runtime.h" 17 | #include "device_launch_parameters.h" 18 | #define GLM_FORCE_CUDA 19 | #include 20 | 21 | namespace BACKWARD 22 | { 23 | void render( 24 | const dim3 grid, dim3 block, 25 | const uint2* ranges, 26 | const uint32_t* point_list, 27 | int W, int H, 28 | const float* bg_color, 29 | const float2* means2D, 30 | const float4* conic_opacity, 31 | const float* colors, 32 | const float* depths, 33 | const float* alphas, 34 | const uint32_t* n_contrib, 35 | const float* dL_dpixels, 36 | const float* dL_dpixel_depths, 37 | const float* dL_dalphas, 38 | float3* dL_dmean2D, 39 | float4* dL_dconic2D, 40 | float* dL_dopacity, 41 | float* dL_dcolors, 42 | float* dL_ddepths); 43 | 44 | void preprocess( 45 | int P, int D, int M, 46 | const float3* means, 47 | const int* radii, 48 | const float* shs, 49 | const bool* clamped, 50 | const glm::vec3* scales, 51 | const glm::vec4* rotations, 52 | const float scale_modifier, 53 | const float* cov3Ds, 54 | const float* view, 55 | const float* proj, 56 | const float focal_x, float focal_y, 57 | const float tan_fovx, float tan_fovy, 58 | const glm::vec3* campos, 59 | const float3* dL_dmean2D, 60 | const float* dL_dconics, 61 | glm::vec3* dL_dmeans, 62 | float* dL_dcolor, 63 | float* dL_ddepth, 64 | float* dL_dcov3D, 65 | float* dL_dsh, 66 | glm::vec3* dL_dscale, 67 | glm::vec4* dL_drot); 68 | } 69 | 70 | #endif -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/cuda_rasterizer/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #ifndef CUDA_RASTERIZER_CONFIG_H_INCLUDED 13 | #define CUDA_RASTERIZER_CONFIG_H_INCLUDED 14 | 15 | #define NUM_CHANNELS 3 // Default 3, RGB 16 | #define BLOCK_X 16 17 | #define BLOCK_Y 16 18 | 19 | #endif -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #ifndef CUDA_RASTERIZER_FORWARD_H_INCLUDED 13 | #define CUDA_RASTERIZER_FORWARD_H_INCLUDED 14 | 15 | #include 16 | #include "cuda_runtime.h" 17 | #include "device_launch_parameters.h" 18 | #define GLM_FORCE_CUDA 19 | #include 20 | 21 | namespace FORWARD 22 | { 23 | // Perform initial steps for each Gaussian prior to rasterization. 24 | void preprocess(int P, int D, int M, 25 | const float* orig_points, 26 | const glm::vec3* scales, 27 | const float scale_modifier, 28 | const glm::vec4* rotations, 29 | const float* opacities, 30 | const float* shs, 31 | bool* clamped, 32 | const float* cov3D_precomp, 33 | const float* colors_precomp, 34 | const float* viewmatrix, 35 | const float* projmatrix, 36 | const glm::vec3* cam_pos, 37 | const int W, int H, 38 | const float focal_x, float focal_y, 39 | const float tan_fovx, float tan_fovy, 40 | int* radii, 41 | float2* points_xy_image, 42 | float* depths, 43 | float* cov3Ds, 44 | float* colors, 45 | float4* conic_opacity, 46 | const dim3 grid, 47 | uint32_t* tiles_touched, 48 | bool prefiltered); 49 | 50 | // Main rasterization method. 51 | void render( 52 | const dim3 grid, dim3 block, 53 | const uint2* ranges, 54 | const uint32_t* point_list, 55 | int W, int H, 56 | const float2* points_xy_image, 57 | const float* features, 58 | const float* depths, 59 | const float4* conic_opacity, 60 | float* out_alpha, 61 | uint32_t* n_contrib, 62 | const float* bg_color, 63 | float* out_color, 64 | float* out_depth); 65 | } 66 | 67 | 68 | #endif -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #ifndef CUDA_RASTERIZER_H_INCLUDED 13 | #define CUDA_RASTERIZER_H_INCLUDED 14 | 15 | #include 16 | #include 17 | 18 | namespace CudaRasterizer 19 | { 20 | class Rasterizer 21 | { 22 | public: 23 | 24 | static void markVisible( 25 | int P, 26 | float* means3D, 27 | float* viewmatrix, 28 | float* projmatrix, 29 | bool* present); 30 | 31 | static int forward( 32 | std::function geometryBuffer, 33 | std::function binningBuffer, 34 | std::function imageBuffer, 35 | const int P, int D, int M, 36 | const float* background, 37 | const int width, int height, 38 | const float* means3D, 39 | const float* shs, 40 | const float* colors_precomp, 41 | const float* opacities, 42 | const float* scales, 43 | const float scale_modifier, 44 | const float* rotations, 45 | const float* cov3D_precomp, 46 | const float* viewmatrix, 47 | const float* projmatrix, 48 | const float* cam_pos, 49 | const float tan_fovx, float tan_fovy, 50 | const bool prefiltered, 51 | float* out_color, 52 | float* out_depth, 53 | float* out_alpha, 54 | int* radii = nullptr, 55 | bool debug = false); 56 | 57 | static void backward( 58 | const int P, int D, int M, int R, 59 | const float* background, 60 | const int width, int height, 61 | const float* means3D, 62 | const float* shs, 63 | const float* colors_precomp, 64 | const float* alphas, 65 | const float* scales, 66 | const float scale_modifier, 67 | const float* rotations, 68 | const float* cov3D_precomp, 69 | const float* viewmatrix, 70 | const float* projmatrix, 71 | const float* campos, 72 | const float tan_fovx, float tan_fovy, 73 | const int* radii, 74 | char* geom_buffer, 75 | char* binning_buffer, 76 | char* image_buffer, 77 | const float* dL_dpix, 78 | const float* dL_dpix_depth, 79 | const float* dL_dalphas, 80 | float* dL_dmean2D, 81 | float* dL_dconic, 82 | float* dL_dopacity, 83 | float* dL_dcolor, 84 | float* dL_ddepth, 85 | float* dL_dmean3D, 86 | float* dL_dcov3D, 87 | float* dL_dsh, 88 | float* dL_dscale, 89 | float* dL_drot, 90 | bool debug); 91 | }; 92 | }; 93 | 94 | #endif -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #pragma once 13 | 14 | #include 15 | #include 16 | #include "rasterizer.h" 17 | #include 18 | 19 | namespace CudaRasterizer 20 | { 21 | template 22 | static void obtain(char*& chunk, T*& ptr, std::size_t count, std::size_t alignment) 23 | { 24 | std::size_t offset = (reinterpret_cast(chunk) + alignment - 1) & ~(alignment - 1); 25 | ptr = reinterpret_cast(offset); 26 | chunk = reinterpret_cast(ptr + count); 27 | } 28 | 29 | struct GeometryState 30 | { 31 | size_t scan_size; 32 | float* depths; 33 | char* scanning_space; 34 | bool* clamped; 35 | int* internal_radii; 36 | float2* means2D; 37 | float* cov3D; 38 | float4* conic_opacity; 39 | float* rgb; 40 | uint32_t* point_offsets; 41 | uint32_t* tiles_touched; 42 | 43 | static GeometryState fromChunk(char*& chunk, size_t P); 44 | }; 45 | 46 | struct ImageState 47 | { 48 | uint2* ranges; 49 | uint32_t* n_contrib; 50 | 51 | static ImageState fromChunk(char*& chunk, size_t N); 52 | }; 53 | 54 | struct BinningState 55 | { 56 | size_t sorting_size; 57 | uint64_t* point_list_keys_unsorted; 58 | uint64_t* point_list_keys; 59 | uint32_t* point_list_unsorted; 60 | uint32_t* point_list; 61 | char* list_sorting_space; 62 | 63 | static BinningState fromChunk(char*& chunk, size_t P); 64 | }; 65 | 66 | template 67 | size_t required(size_t P) 68 | { 69 | char* size = nullptr; 70 | T::fromChunk(size, P); 71 | return ((size_t)size) + 128; 72 | } 73 | }; -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from typing import NamedTuple 13 | import torch.nn as nn 14 | import torch 15 | from . import _C 16 | 17 | def cpu_deep_copy_tuple(input_tuple): 18 | copied_tensors = [item.cpu().clone() if isinstance(item, torch.Tensor) else item for item in input_tuple] 19 | return tuple(copied_tensors) 20 | 21 | def rasterize_gaussians( 22 | means3D, 23 | means2D, 24 | sh, 25 | colors_precomp, 26 | opacities, 27 | scales, 28 | rotations, 29 | cov3Ds_precomp, 30 | raster_settings, 31 | ): 32 | return _RasterizeGaussians.apply( 33 | means3D, 34 | means2D, 35 | sh, 36 | colors_precomp, 37 | opacities, 38 | scales, 39 | rotations, 40 | cov3Ds_precomp, 41 | raster_settings, 42 | ) 43 | 44 | class _RasterizeGaussians(torch.autograd.Function): 45 | @staticmethod 46 | def forward( 47 | ctx, 48 | means3D, 49 | means2D, 50 | sh, 51 | colors_precomp, 52 | opacities, 53 | scales, 54 | rotations, 55 | cov3Ds_precomp, 56 | raster_settings, 57 | ): 58 | 59 | # Restructure arguments the way that the C++ lib expects them 60 | args = ( 61 | raster_settings.bg, 62 | means3D, 63 | colors_precomp, 64 | opacities, 65 | scales, 66 | rotations, 67 | raster_settings.scale_modifier, 68 | cov3Ds_precomp, 69 | raster_settings.viewmatrix, 70 | raster_settings.projmatrix, 71 | raster_settings.tanfovx, 72 | raster_settings.tanfovy, 73 | raster_settings.image_height, 74 | raster_settings.image_width, 75 | sh, 76 | raster_settings.sh_degree, 77 | raster_settings.campos, 78 | raster_settings.prefiltered, 79 | raster_settings.debug 80 | ) 81 | 82 | # Invoke C++/CUDA rasterizer 83 | if raster_settings.debug: 84 | cpu_args = cpu_deep_copy_tuple(args) # Copy them before they can be corrupted 85 | try: 86 | num_rendered, color, depth, alpha, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args) 87 | except Exception as ex: 88 | torch.save(cpu_args, "snapshot_fw.dump") 89 | print("\nAn error occured in forward. Please forward snapshot_fw.dump for debugging.") 90 | raise ex 91 | else: 92 | num_rendered, color, depth, alpha, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args) 93 | 94 | # Keep relevant tensors for backward 95 | ctx.raster_settings = raster_settings 96 | ctx.num_rendered = num_rendered 97 | ctx.save_for_backward(colors_precomp, means3D, scales, rotations, cov3Ds_precomp, radii, sh, geomBuffer, binningBuffer, imgBuffer, alpha) 98 | return color, radii, depth, alpha 99 | 100 | @staticmethod 101 | def backward(ctx, grad_color, grad_radii, grad_depth, grad_alpha): 102 | 103 | # Restore necessary values from context 104 | num_rendered = ctx.num_rendered 105 | raster_settings = ctx.raster_settings 106 | colors_precomp, means3D, scales, rotations, cov3Ds_precomp, radii, sh, geomBuffer, binningBuffer, imgBuffer, alpha = ctx.saved_tensors 107 | 108 | # Restructure args as C++ method expects them 109 | args = (raster_settings.bg, 110 | means3D, 111 | radii, 112 | colors_precomp, 113 | scales, 114 | rotations, 115 | raster_settings.scale_modifier, 116 | cov3Ds_precomp, 117 | raster_settings.viewmatrix, 118 | raster_settings.projmatrix, 119 | raster_settings.tanfovx, 120 | raster_settings.tanfovy, 121 | grad_color, 122 | grad_depth, 123 | grad_alpha, 124 | sh, 125 | raster_settings.sh_degree, 126 | raster_settings.campos, 127 | geomBuffer, 128 | num_rendered, 129 | binningBuffer, 130 | imgBuffer, 131 | alpha, 132 | raster_settings.debug) 133 | 134 | # Compute gradients for relevant tensors by invoking backward method 135 | if raster_settings.debug: 136 | cpu_args = cpu_deep_copy_tuple(args) # Copy them before they can be corrupted 137 | try: 138 | grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_gaussians_backward(*args) 139 | except Exception as ex: 140 | torch.save(cpu_args, "snapshot_bw.dump") 141 | print("\nAn error occured in backward. Writing snapshot_bw.dump for debugging.\n") 142 | raise ex 143 | else: 144 | grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_gaussians_backward(*args) 145 | 146 | grads = ( 147 | grad_means3D, 148 | grad_means2D, 149 | grad_sh, 150 | grad_colors_precomp, 151 | grad_opacities, 152 | grad_scales, 153 | grad_rotations, 154 | grad_cov3Ds_precomp, 155 | None, 156 | ) 157 | 158 | return grads 159 | 160 | class GaussianRasterizationSettings(NamedTuple): 161 | image_height: int 162 | image_width: int 163 | tanfovx : float 164 | tanfovy : float 165 | bg : torch.Tensor 166 | scale_modifier : float 167 | viewmatrix : torch.Tensor 168 | projmatrix : torch.Tensor 169 | sh_degree : int 170 | campos : torch.Tensor 171 | prefiltered : bool 172 | debug : bool 173 | 174 | class GaussianRasterizer(nn.Module): 175 | def __init__(self, raster_settings): 176 | super().__init__() 177 | self.raster_settings = raster_settings 178 | 179 | def markVisible(self, positions): 180 | # Mark visible points (based on frustum culling for camera) with a boolean 181 | with torch.no_grad(): 182 | raster_settings = self.raster_settings 183 | visible = _C.mark_visible( 184 | positions, 185 | raster_settings.viewmatrix, 186 | raster_settings.projmatrix) 187 | 188 | return visible 189 | 190 | def forward(self, means3D, means2D, opacities, shs = None, colors_precomp = None, scales = None, rotations = None, cov3D_precomp = None): 191 | 192 | raster_settings = self.raster_settings 193 | 194 | if (shs is None and colors_precomp is None) or (shs is not None and colors_precomp is not None): 195 | raise Exception('Please provide excatly one of either SHs or precomputed colors!') 196 | 197 | if ((scales is None or rotations is None) and cov3D_precomp is None) or ((scales is not None or rotations is not None) and cov3D_precomp is not None): 198 | raise Exception('Please provide exactly one of either scale/rotation pair or precomputed 3D covariance!') 199 | 200 | if shs is None: 201 | shs = torch.Tensor([]) 202 | if colors_precomp is None: 203 | colors_precomp = torch.Tensor([]) 204 | 205 | if scales is None: 206 | scales = torch.Tensor([]) 207 | if rotations is None: 208 | rotations = torch.Tensor([]) 209 | if cov3D_precomp is None: 210 | cov3D_precomp = torch.Tensor([]) 211 | 212 | # Invoke C++/CUDA rasterization routine 213 | return rasterize_gaussians( 214 | means3D, 215 | means2D, 216 | shs, 217 | colors_precomp, 218 | opacities, 219 | scales, 220 | rotations, 221 | cov3D_precomp, 222 | raster_settings, 223 | ) 224 | 225 | -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/ext.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #include 13 | #include "rasterize_points.h" 14 | 15 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 16 | m.def("rasterize_gaussians", &RasterizeGaussiansCUDA); 17 | m.def("rasterize_gaussians_backward", &RasterizeGaussiansBackwardCUDA); 18 | m.def("mark_visible", &markVisible); 19 | } -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/rasterize_points.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "cuda_rasterizer/config.h" 22 | #include "cuda_rasterizer/rasterizer.h" 23 | #include 24 | #include 25 | #include 26 | 27 | std::function resizeFunctional(torch::Tensor& t) { 28 | auto lambda = [&t](size_t N) { 29 | t.resize_({(long long)N}); 30 | return reinterpret_cast(t.contiguous().data_ptr()); 31 | }; 32 | return lambda; 33 | } 34 | 35 | std::tuple 36 | RasterizeGaussiansCUDA( 37 | const torch::Tensor& background, 38 | const torch::Tensor& means3D, 39 | const torch::Tensor& colors, 40 | const torch::Tensor& opacity, 41 | const torch::Tensor& scales, 42 | const torch::Tensor& rotations, 43 | const float scale_modifier, 44 | const torch::Tensor& cov3D_precomp, 45 | const torch::Tensor& viewmatrix, 46 | const torch::Tensor& projmatrix, 47 | const float tan_fovx, 48 | const float tan_fovy, 49 | const int image_height, 50 | const int image_width, 51 | const torch::Tensor& sh, 52 | const int degree, 53 | const torch::Tensor& campos, 54 | const bool prefiltered, 55 | const bool debug) 56 | { 57 | if (means3D.ndimension() != 2 || means3D.size(1) != 3) { 58 | AT_ERROR("means3D must have dimensions (num_points, 3)"); 59 | } 60 | 61 | const int P = means3D.size(0); 62 | const int H = image_height; 63 | const int W = image_width; 64 | 65 | auto int_opts = means3D.options().dtype(torch::kInt32); 66 | auto float_opts = means3D.options().dtype(torch::kFloat32); 67 | 68 | torch::Tensor out_color = torch::full({NUM_CHANNELS, H, W}, 0.0, float_opts); 69 | torch::Tensor out_depth = torch::full({1, H, W}, 0.0, float_opts); 70 | torch::Tensor out_alpha = torch::full({1, H, W}, 0.0, float_opts); 71 | torch::Tensor radii = torch::full({P}, 0, means3D.options().dtype(torch::kInt32)); 72 | 73 | torch::Device device(torch::kCUDA); 74 | torch::TensorOptions options(torch::kByte); 75 | torch::Tensor geomBuffer = torch::empty({0}, options.device(device)); 76 | torch::Tensor binningBuffer = torch::empty({0}, options.device(device)); 77 | torch::Tensor imgBuffer = torch::empty({0}, options.device(device)); 78 | std::function geomFunc = resizeFunctional(geomBuffer); 79 | std::function binningFunc = resizeFunctional(binningBuffer); 80 | std::function imgFunc = resizeFunctional(imgBuffer); 81 | 82 | int rendered = 0; 83 | if(P != 0) 84 | { 85 | int M = 0; 86 | if(sh.size(0) != 0) 87 | { 88 | M = sh.size(1); 89 | } 90 | 91 | rendered = CudaRasterizer::Rasterizer::forward( 92 | geomFunc, 93 | binningFunc, 94 | imgFunc, 95 | P, degree, M, 96 | background.contiguous().data(), 97 | W, H, 98 | means3D.contiguous().data(), 99 | sh.contiguous().data_ptr(), 100 | colors.contiguous().data(), 101 | opacity.contiguous().data(), 102 | scales.contiguous().data_ptr(), 103 | scale_modifier, 104 | rotations.contiguous().data_ptr(), 105 | cov3D_precomp.contiguous().data(), 106 | viewmatrix.contiguous().data(), 107 | projmatrix.contiguous().data(), 108 | campos.contiguous().data(), 109 | tan_fovx, 110 | tan_fovy, 111 | prefiltered, 112 | out_color.contiguous().data(), 113 | out_depth.contiguous().data(), 114 | out_alpha.contiguous().data(), 115 | radii.contiguous().data(), 116 | debug); 117 | } 118 | return std::make_tuple(rendered, out_color, out_depth, out_alpha, radii, geomBuffer, binningBuffer, imgBuffer); 119 | } 120 | 121 | std::tuple 122 | RasterizeGaussiansBackwardCUDA( 123 | const torch::Tensor& background, 124 | const torch::Tensor& means3D, 125 | const torch::Tensor& radii, 126 | const torch::Tensor& colors, 127 | const torch::Tensor& scales, 128 | const torch::Tensor& rotations, 129 | const float scale_modifier, 130 | const torch::Tensor& cov3D_precomp, 131 | const torch::Tensor& viewmatrix, 132 | const torch::Tensor& projmatrix, 133 | const float tan_fovx, 134 | const float tan_fovy, 135 | const torch::Tensor& dL_dout_color, 136 | const torch::Tensor& dL_dout_depth, 137 | const torch::Tensor& dL_dout_alpha, 138 | const torch::Tensor& sh, 139 | const int degree, 140 | const torch::Tensor& campos, 141 | const torch::Tensor& geomBuffer, 142 | const int R, 143 | const torch::Tensor& binningBuffer, 144 | const torch::Tensor& imageBuffer, 145 | const torch::Tensor& alphas, 146 | const bool debug) 147 | { 148 | const int P = means3D.size(0); 149 | const int H = dL_dout_color.size(1); 150 | const int W = dL_dout_color.size(2); 151 | 152 | int M = 0; 153 | if(sh.size(0) != 0) 154 | { 155 | M = sh.size(1); 156 | } 157 | 158 | torch::Tensor dL_dmeans3D = torch::zeros({P, 3}, means3D.options()); 159 | torch::Tensor dL_dmeans2D = torch::zeros({P, 3}, means3D.options()); 160 | torch::Tensor dL_dcolors = torch::zeros({P, NUM_CHANNELS}, means3D.options()); 161 | torch::Tensor dL_ddepths = torch::zeros({P, 1}, means3D.options()); 162 | torch::Tensor dL_dconic = torch::zeros({P, 2, 2}, means3D.options()); 163 | torch::Tensor dL_dopacity = torch::zeros({P, 1}, means3D.options()); 164 | torch::Tensor dL_dcov3D = torch::zeros({P, 6}, means3D.options()); 165 | torch::Tensor dL_dsh = torch::zeros({P, M, 3}, means3D.options()); 166 | torch::Tensor dL_dscales = torch::zeros({P, 3}, means3D.options()); 167 | torch::Tensor dL_drotations = torch::zeros({P, 4}, means3D.options()); 168 | 169 | if(P != 0) 170 | { 171 | CudaRasterizer::Rasterizer::backward(P, degree, M, R, 172 | background.contiguous().data(), 173 | W, H, 174 | means3D.contiguous().data(), 175 | sh.contiguous().data(), 176 | colors.contiguous().data(), 177 | alphas.contiguous().data(), 178 | scales.data_ptr(), 179 | scale_modifier, 180 | rotations.data_ptr(), 181 | cov3D_precomp.contiguous().data(), 182 | viewmatrix.contiguous().data(), 183 | projmatrix.contiguous().data(), 184 | campos.contiguous().data(), 185 | tan_fovx, 186 | tan_fovy, 187 | radii.contiguous().data(), 188 | reinterpret_cast(geomBuffer.contiguous().data_ptr()), 189 | reinterpret_cast(binningBuffer.contiguous().data_ptr()), 190 | reinterpret_cast(imageBuffer.contiguous().data_ptr()), 191 | dL_dout_color.contiguous().data(), 192 | dL_dout_depth.contiguous().data(), 193 | dL_dout_alpha.contiguous().data(), 194 | dL_dmeans2D.contiguous().data(), 195 | dL_dconic.contiguous().data(), 196 | dL_dopacity.contiguous().data(), 197 | dL_dcolors.contiguous().data(), 198 | dL_ddepths.contiguous().data(), 199 | dL_dmeans3D.contiguous().data(), 200 | dL_dcov3D.contiguous().data(), 201 | dL_dsh.contiguous().data(), 202 | dL_dscales.contiguous().data(), 203 | dL_drotations.contiguous().data(), 204 | debug); 205 | } 206 | 207 | return std::make_tuple(dL_dmeans2D, dL_dcolors, dL_dopacity, dL_dmeans3D, dL_dcov3D, dL_dsh, dL_dscales, dL_drotations); 208 | } 209 | 210 | torch::Tensor markVisible( 211 | torch::Tensor& means3D, 212 | torch::Tensor& viewmatrix, 213 | torch::Tensor& projmatrix) 214 | { 215 | const int P = means3D.size(0); 216 | 217 | torch::Tensor present = torch::full({P}, false, means3D.options().dtype(at::kBool)); 218 | 219 | if(P != 0) 220 | { 221 | CudaRasterizer::Rasterizer::markVisible(P, 222 | means3D.contiguous().data(), 223 | viewmatrix.contiguous().data(), 224 | projmatrix.contiguous().data(), 225 | present.contiguous().data()); 226 | } 227 | 228 | return present; 229 | } -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/rasterize_points.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #pragma once 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | std::tuple 19 | RasterizeGaussiansCUDA( 20 | const torch::Tensor& background, 21 | const torch::Tensor& means3D, 22 | const torch::Tensor& colors, 23 | const torch::Tensor& opacity, 24 | const torch::Tensor& scales, 25 | const torch::Tensor& rotations, 26 | const float scale_modifier, 27 | const torch::Tensor& cov3D_precomp, 28 | const torch::Tensor& viewmatrix, 29 | const torch::Tensor& projmatrix, 30 | const float tan_fovx, 31 | const float tan_fovy, 32 | const int image_height, 33 | const int image_width, 34 | const torch::Tensor& sh, 35 | const int degree, 36 | const torch::Tensor& campos, 37 | const bool prefiltered, 38 | const bool debug); 39 | 40 | std::tuple 41 | RasterizeGaussiansBackwardCUDA( 42 | const torch::Tensor& background, 43 | const torch::Tensor& means3D, 44 | const torch::Tensor& radii, 45 | const torch::Tensor& colors, 46 | const torch::Tensor& scales, 47 | const torch::Tensor& rotations, 48 | const float scale_modifier, 49 | const torch::Tensor& cov3D_precomp, 50 | const torch::Tensor& viewmatrix, 51 | const torch::Tensor& projmatrix, 52 | const float tan_fovx, 53 | const float tan_fovy, 54 | const torch::Tensor& dL_dout_color, 55 | const torch::Tensor& dL_dout_depth, 56 | const torch::Tensor& dL_dout_alpha, 57 | const torch::Tensor& sh, 58 | const int degree, 59 | const torch::Tensor& campos, 60 | const torch::Tensor& geomBuffer, 61 | const int R, 62 | const torch::Tensor& binningBuffer, 63 | const torch::Tensor& imageBuffer, 64 | const torch::Tensor& alpha, 65 | const bool debug); 66 | 67 | torch::Tensor markVisible( 68 | torch::Tensor& means3D, 69 | torch::Tensor& viewmatrix, 70 | torch::Tensor& projmatrix); -------------------------------------------------------------------------------- /submodules/diff-gaussian-rasterization/setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from setuptools import setup 13 | from torch.utils.cpp_extension import CUDAExtension, BuildExtension 14 | import os 15 | os.path.dirname(os.path.abspath(__file__)) 16 | 17 | setup( 18 | name="diff_gaussian_rasterization", 19 | packages=['diff_gaussian_rasterization'], 20 | ext_modules=[ 21 | CUDAExtension( 22 | name="diff_gaussian_rasterization._C", 23 | sources=[ 24 | "cuda_rasterizer/rasterizer_impl.cu", 25 | "cuda_rasterizer/forward.cu", 26 | "cuda_rasterizer/backward.cu", 27 | "rasterize_points.cu", 28 | "ext.cpp"], 29 | extra_compile_args={"nvcc": ["-I" + os.path.join(os.path.dirname(os.path.abspath(__file__)), "third_party/glm/")]}) 30 | ], 31 | cmdclass={ 32 | 'build_ext': BuildExtension 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /submodules/simple-knn/LICENSE.md: -------------------------------------------------------------------------------- 1 | Gaussian-Splatting License 2 | =========================== 3 | 4 | **Inria** and **the Max Planck Institut for Informatik (MPII)** hold all the ownership rights on the *Software* named **gaussian-splatting**. 5 | The *Software* is in the process of being registered with the Agence pour la Protection des 6 | Programmes (APP). 7 | 8 | The *Software* is still being developed by the *Licensor*. 9 | 10 | *Licensor*'s goal is to allow the research community to use, test and evaluate 11 | the *Software*. 12 | 13 | ## 1. Definitions 14 | 15 | *Licensee* means any person or entity that uses the *Software* and distributes 16 | its *Work*. 17 | 18 | *Licensor* means the owners of the *Software*, i.e Inria and MPII 19 | 20 | *Software* means the original work of authorship made available under this 21 | License ie gaussian-splatting. 22 | 23 | *Work* means the *Software* and any additions to or derivative works of the 24 | *Software* that are made available under this License. 25 | 26 | 27 | ## 2. Purpose 28 | This license is intended to define the rights granted to the *Licensee* by 29 | Licensors under the *Software*. 30 | 31 | ## 3. Rights granted 32 | 33 | For the above reasons Licensors have decided to distribute the *Software*. 34 | Licensors grant non-exclusive rights to use the *Software* for research purposes 35 | to research users (both academic and industrial), free of charge, without right 36 | to sublicense.. The *Software* may be used "non-commercially", i.e., for research 37 | and/or evaluation purposes only. 38 | 39 | Subject to the terms and conditions of this License, you are granted a 40 | non-exclusive, royalty-free, license to reproduce, prepare derivative works of, 41 | publicly display, publicly perform and distribute its *Work* and any resulting 42 | derivative works in any form. 43 | 44 | ## 4. Limitations 45 | 46 | **4.1 Redistribution.** You may reproduce or distribute the *Work* only if (a) you do 47 | so under this License, (b) you include a complete copy of this License with 48 | your distribution, and (c) you retain without modification any copyright, 49 | patent, trademark, or attribution notices that are present in the *Work*. 50 | 51 | **4.2 Derivative Works.** You may specify that additional or different terms apply 52 | to the use, reproduction, and distribution of your derivative works of the *Work* 53 | ("Your Terms") only if (a) Your Terms provide that the use limitation in 54 | Section 2 applies to your derivative works, and (b) you identify the specific 55 | derivative works that are subject to Your Terms. Notwithstanding Your Terms, 56 | this License (including the redistribution requirements in Section 3.1) will 57 | continue to apply to the *Work* itself. 58 | 59 | **4.3** Any other use without of prior consent of Licensors is prohibited. Research 60 | users explicitly acknowledge having received from Licensors all information 61 | allowing to appreciate the adequacy between of the *Software* and their needs and 62 | to undertake all necessary precautions for its execution and use. 63 | 64 | **4.4** The *Software* is provided both as a compiled library file and as source 65 | code. In case of using the *Software* for a publication or other results obtained 66 | through the use of the *Software*, users are strongly encouraged to cite the 67 | corresponding publications as explained in the documentation of the *Software*. 68 | 69 | ## 5. Disclaimer 70 | 71 | THE USER CANNOT USE, EXPLOIT OR DISTRIBUTE THE *SOFTWARE* FOR COMMERCIAL PURPOSES 72 | WITHOUT PRIOR AND EXPLICIT CONSENT OF LICENSORS. YOU MUST CONTACT INRIA FOR ANY 73 | UNAUTHORIZED USE: stip-sophia.transfert@inria.fr . ANY SUCH ACTION WILL 74 | CONSTITUTE A FORGERY. THIS *SOFTWARE* IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES 75 | OF ANY NATURE AND ANY EXPRESS OR IMPLIED WARRANTIES, WITH REGARDS TO COMMERCIAL 76 | USE, PROFESSIONNAL USE, LEGAL OR NOT, OR OTHER, OR COMMERCIALISATION OR 77 | ADAPTATION. UNLESS EXPLICITLY PROVIDED BY LAW, IN NO EVENT, SHALL INRIA OR THE 78 | AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 79 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 80 | GOODS OR SERVICES, LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION) 81 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 82 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING FROM, OUT OF OR 83 | IN CONNECTION WITH THE *SOFTWARE* OR THE USE OR OTHER DEALINGS IN THE *SOFTWARE*. 84 | 85 | ## 6. Files subject to permissive licenses 86 | The contents of the file ```utils/loss_utils.py``` are based on publicly available code authored by Evan Su, which falls under the permissive MIT license. 87 | 88 | Title: pytorch-ssim\ 89 | Project code: https://github.com/Po-Hsun-Su/pytorch-ssim\ 90 | Copyright Evan Su, 2017\ 91 | License: https://github.com/Po-Hsun-Su/pytorch-ssim/blob/master/LICENSE.txt (MIT) -------------------------------------------------------------------------------- /submodules/simple-knn/build/lib.linux-x86_64-cpython-310/simple_knn/_C.cpython-310-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/submodules/simple-knn/build/lib.linux-x86_64-cpython-310/simple_knn/_C.cpython-310-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /submodules/simple-knn/build/temp.linux-x86_64-cpython-310/.ninja_deps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/.ninja_deps -------------------------------------------------------------------------------- /submodules/simple-knn/build/temp.linux-x86_64-cpython-310/.ninja_log: -------------------------------------------------------------------------------- 1 | # ninja log v5 2 | 0 12982 1740924351317050518 /data/hbb/nlp/SC-GS/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/simple_knn.o 9805073580aaa04f 3 | 0 21594 1740924359921146916 /data/hbb/nlp/SC-GS/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/ext.o f43c02b0bed76cf4 4 | 0 53399 1740924391737491202 /data/hbb/nlp/SC-GS/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/spatial.o f12d5314fda66397 5 | -------------------------------------------------------------------------------- /submodules/simple-knn/build/temp.linux-x86_64-cpython-310/build.ninja: -------------------------------------------------------------------------------- 1 | ninja_required_version = 1.3 2 | cxx = c++ 3 | nvcc = /home/hbb/data/cuda-11.7/bin/nvcc 4 | 5 | cflags = -pthread -B /data/hbb/anaconda3/envs/SC-GS/compiler_compat -Wno-unused-result -Wsign-compare -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /data/hbb/anaconda3/envs/SC-GS/include -fPIC -O2 -isystem /data/hbb/anaconda3/envs/SC-GS/include -fPIC -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include/TH -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include/THC -I/home/hbb/data/cuda-11.7/include -I/data/hbb/anaconda3/envs/SC-GS/include/python3.10 -c 6 | post_cflags = -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14 7 | cuda_cflags = -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include/TH -I/data/hbb/anaconda3/envs/SC-GS/lib/python3.10/site-packages/torch/include/THC -I/home/hbb/data/cuda-11.7/include -I/data/hbb/anaconda3/envs/SC-GS/include/python3.10 -c 8 | cuda_post_cflags = -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 -std=c++14 9 | cuda_dlink_post_cflags = 10 | ldflags = 11 | 12 | rule compile 13 | command = $cxx -MMD -MF $out.d $cflags -c $in -o $out $post_cflags 14 | depfile = $out.d 15 | deps = gcc 16 | 17 | rule cuda_compile 18 | depfile = $out.d 19 | deps = gcc 20 | command = $nvcc $cuda_cflags -c $in -o $out $cuda_post_cflags 21 | 22 | 23 | 24 | 25 | 26 | build /data/hbb/nlp/SC-GS/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/ext.o: compile /data/hbb/nlp/SC-GS/submodules/simple-knn/ext.cpp 27 | build /data/hbb/nlp/SC-GS/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/simple_knn.o: cuda_compile /data/hbb/nlp/SC-GS/submodules/simple-knn/simple_knn.cu 28 | build /data/hbb/nlp/SC-GS/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/spatial.o: cuda_compile /data/hbb/nlp/SC-GS/submodules/simple-knn/spatial.cu 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /submodules/simple-knn/build/temp.linux-x86_64-cpython-310/ext.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/ext.o -------------------------------------------------------------------------------- /submodules/simple-knn/build/temp.linux-x86_64-cpython-310/simple_knn.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/simple_knn.o -------------------------------------------------------------------------------- /submodules/simple-knn/build/temp.linux-x86_64-cpython-310/spatial.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/submodules/simple-knn/build/temp.linux-x86_64-cpython-310/spatial.o -------------------------------------------------------------------------------- /submodules/simple-knn/ext.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #include 13 | #include "spatial.h" 14 | 15 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 16 | m.def("distCUDA2", &distCUDA2); 17 | } 18 | -------------------------------------------------------------------------------- /submodules/simple-knn/setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from setuptools import setup 13 | from torch.utils.cpp_extension import CUDAExtension, BuildExtension 14 | import os 15 | 16 | cxx_compiler_flags = [] 17 | 18 | if os.name == 'nt': 19 | cxx_compiler_flags.append("/wd4624") 20 | 21 | setup( 22 | name="simple_knn", 23 | ext_modules=[ 24 | CUDAExtension( 25 | name="simple_knn._C", 26 | sources=[ 27 | "spatial.cu", 28 | "simple_knn.cu", 29 | "ext.cpp"], 30 | extra_compile_args={"nvcc": [], "cxx": cxx_compiler_flags}) 31 | ], 32 | cmdclass={ 33 | 'build_ext': BuildExtension 34 | } 35 | ) 36 | -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #define BOX_SIZE 1024 13 | 14 | #include "cuda_runtime.h" 15 | #include "device_launch_parameters.h" 16 | #include "simple_knn.h" 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #define __CUDACC__ 25 | #include 26 | #include 27 | 28 | namespace cg = cooperative_groups; 29 | 30 | struct CustomMin 31 | { 32 | __device__ __forceinline__ 33 | float3 operator()(const float3& a, const float3& b) const { 34 | return { min(a.x, b.x), min(a.y, b.y), min(a.z, b.z) }; 35 | } 36 | }; 37 | 38 | struct CustomMax 39 | { 40 | __device__ __forceinline__ 41 | float3 operator()(const float3& a, const float3& b) const { 42 | return { max(a.x, b.x), max(a.y, b.y), max(a.z, b.z) }; 43 | } 44 | }; 45 | 46 | __host__ __device__ uint32_t prepMorton(uint32_t x) 47 | { 48 | x = (x | (x << 16)) & 0x030000FF; 49 | x = (x | (x << 8)) & 0x0300F00F; 50 | x = (x | (x << 4)) & 0x030C30C3; 51 | x = (x | (x << 2)) & 0x09249249; 52 | return x; 53 | } 54 | 55 | __host__ __device__ uint32_t coord2Morton(float3 coord, float3 minn, float3 maxx) 56 | { 57 | uint32_t x = prepMorton(((coord.x - minn.x) / (maxx.x - minn.x)) * ((1 << 10) - 1)); 58 | uint32_t y = prepMorton(((coord.y - minn.y) / (maxx.y - minn.y)) * ((1 << 10) - 1)); 59 | uint32_t z = prepMorton(((coord.z - minn.z) / (maxx.z - minn.z)) * ((1 << 10) - 1)); 60 | 61 | return x | (y << 1) | (z << 2); 62 | } 63 | 64 | __global__ void coord2Morton(int P, const float3* points, float3 minn, float3 maxx, uint32_t* codes) 65 | { 66 | auto idx = cg::this_grid().thread_rank(); 67 | if (idx >= P) 68 | return; 69 | 70 | codes[idx] = coord2Morton(points[idx], minn, maxx); 71 | } 72 | 73 | struct MinMax 74 | { 75 | float3 minn; 76 | float3 maxx; 77 | }; 78 | 79 | __global__ void boxMinMax(uint32_t P, float3* points, uint32_t* indices, MinMax* boxes) 80 | { 81 | auto idx = cg::this_grid().thread_rank(); 82 | 83 | MinMax me; 84 | if (idx < P) 85 | { 86 | me.minn = points[indices[idx]]; 87 | me.maxx = points[indices[idx]]; 88 | } 89 | else 90 | { 91 | me.minn = { FLT_MAX, FLT_MAX, FLT_MAX }; 92 | me.maxx = { -FLT_MAX,-FLT_MAX,-FLT_MAX }; 93 | } 94 | 95 | __shared__ MinMax redResult[BOX_SIZE]; 96 | 97 | for (int off = BOX_SIZE / 2; off >= 1; off /= 2) 98 | { 99 | if (threadIdx.x < 2 * off) 100 | redResult[threadIdx.x] = me; 101 | __syncthreads(); 102 | 103 | if (threadIdx.x < off) 104 | { 105 | MinMax other = redResult[threadIdx.x + off]; 106 | me.minn.x = min(me.minn.x, other.minn.x); 107 | me.minn.y = min(me.minn.y, other.minn.y); 108 | me.minn.z = min(me.minn.z, other.minn.z); 109 | me.maxx.x = max(me.maxx.x, other.maxx.x); 110 | me.maxx.y = max(me.maxx.y, other.maxx.y); 111 | me.maxx.z = max(me.maxx.z, other.maxx.z); 112 | } 113 | __syncthreads(); 114 | } 115 | 116 | if (threadIdx.x == 0) 117 | boxes[blockIdx.x] = me; 118 | } 119 | 120 | __device__ __host__ float distBoxPoint(const MinMax& box, const float3& p) 121 | { 122 | float3 diff = { 0, 0, 0 }; 123 | if (p.x < box.minn.x || p.x > box.maxx.x) 124 | diff.x = min(abs(p.x - box.minn.x), abs(p.x - box.maxx.x)); 125 | if (p.y < box.minn.y || p.y > box.maxx.y) 126 | diff.y = min(abs(p.y - box.minn.y), abs(p.y - box.maxx.y)); 127 | if (p.z < box.minn.z || p.z > box.maxx.z) 128 | diff.z = min(abs(p.z - box.minn.z), abs(p.z - box.maxx.z)); 129 | return diff.x * diff.x + diff.y * diff.y + diff.z * diff.z; 130 | } 131 | 132 | template 133 | __device__ void updateKBest(const float3& ref, const float3& point, float* knn) 134 | { 135 | float3 d = { point.x - ref.x, point.y - ref.y, point.z - ref.z }; 136 | float dist = d.x * d.x + d.y * d.y + d.z * d.z; 137 | for (int j = 0; j < K; j++) 138 | { 139 | if (knn[j] > dist) 140 | { 141 | float t = knn[j]; 142 | knn[j] = dist; 143 | dist = t; 144 | } 145 | } 146 | } 147 | 148 | __global__ void boxMeanDist(uint32_t P, float3* points, uint32_t* indices, MinMax* boxes, float* dists) 149 | { 150 | int idx = cg::this_grid().thread_rank(); 151 | if (idx >= P) 152 | return; 153 | 154 | float3 point = points[indices[idx]]; 155 | float best[3] = { FLT_MAX, FLT_MAX, FLT_MAX }; 156 | 157 | for (int i = max(0, idx - 3); i <= min(P - 1, idx + 3); i++) 158 | { 159 | if (i == idx) 160 | continue; 161 | updateKBest<3>(point, points[indices[i]], best); 162 | } 163 | 164 | float reject = best[2]; 165 | best[0] = FLT_MAX; 166 | best[1] = FLT_MAX; 167 | best[2] = FLT_MAX; 168 | 169 | for (int b = 0; b < (P + BOX_SIZE - 1) / BOX_SIZE; b++) 170 | { 171 | MinMax box = boxes[b]; 172 | float dist = distBoxPoint(box, point); 173 | if (dist > reject || dist > best[2]) 174 | continue; 175 | 176 | for (int i = b * BOX_SIZE; i < min(P, (b + 1) * BOX_SIZE); i++) 177 | { 178 | if (i == idx) 179 | continue; 180 | updateKBest<3>(point, points[indices[i]], best); 181 | } 182 | } 183 | dists[indices[idx]] = (best[0] + best[1] + best[2]) / 3.0f; 184 | } 185 | 186 | void SimpleKNN::knn(int P, float3* points, float* meanDists) 187 | { 188 | float3* result; 189 | cudaMalloc(&result, sizeof(float3)); 190 | size_t temp_storage_bytes; 191 | 192 | float3 init = { 0, 0, 0 }, minn, maxx; 193 | 194 | cub::DeviceReduce::Reduce(nullptr, temp_storage_bytes, points, result, P, CustomMin(), init); 195 | thrust::device_vector temp_storage(temp_storage_bytes); 196 | 197 | cub::DeviceReduce::Reduce(temp_storage.data().get(), temp_storage_bytes, points, result, P, CustomMin(), init); 198 | cudaMemcpy(&minn, result, sizeof(float3), cudaMemcpyDeviceToHost); 199 | 200 | cub::DeviceReduce::Reduce(temp_storage.data().get(), temp_storage_bytes, points, result, P, CustomMax(), init); 201 | cudaMemcpy(&maxx, result, sizeof(float3), cudaMemcpyDeviceToHost); 202 | 203 | thrust::device_vector morton(P); 204 | thrust::device_vector morton_sorted(P); 205 | coord2Morton << <(P + 255) / 256, 256 >> > (P, points, minn, maxx, morton.data().get()); 206 | 207 | thrust::device_vector indices(P); 208 | thrust::sequence(indices.begin(), indices.end()); 209 | thrust::device_vector indices_sorted(P); 210 | 211 | cub::DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, morton.data().get(), morton_sorted.data().get(), indices.data().get(), indices_sorted.data().get(), P); 212 | temp_storage.resize(temp_storage_bytes); 213 | 214 | cub::DeviceRadixSort::SortPairs(temp_storage.data().get(), temp_storage_bytes, morton.data().get(), morton_sorted.data().get(), indices.data().get(), indices_sorted.data().get(), P); 215 | 216 | uint32_t num_boxes = (P + BOX_SIZE - 1) / BOX_SIZE; 217 | thrust::device_vector boxes(num_boxes); 218 | boxMinMax << > > (P, points, indices_sorted.data().get(), boxes.data().get()); 219 | boxMeanDist << > > (P, points, indices_sorted.data().get(), boxes.data().get(), meanDists); 220 | 221 | cudaFree(result); 222 | } -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.2 2 | Name: simple_knn 3 | Version: 0.0.0 4 | License-File: LICENSE.md 5 | -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | LICENSE.md 2 | ext.cpp 3 | setup.py 4 | simple_knn.cu 5 | spatial.cu 6 | simple_knn.egg-info/PKG-INFO 7 | simple_knn.egg-info/SOURCES.txt 8 | simple_knn.egg-info/dependency_links.txt 9 | simple_knn.egg-info/top_level.txt -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | simple_knn 2 | -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #ifndef SIMPLEKNN_H_INCLUDED 13 | #define SIMPLEKNN_H_INCLUDED 14 | 15 | class SimpleKNN 16 | { 17 | public: 18 | static void knn(int P, float3* points, float* meanDists); 19 | }; 20 | 21 | #endif -------------------------------------------------------------------------------- /submodules/simple-knn/simple_knn/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/submodules/simple-knn/simple_knn/.gitkeep -------------------------------------------------------------------------------- /submodules/simple-knn/spatial.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #include "spatial.h" 13 | #include "simple_knn.h" 14 | 15 | torch::Tensor 16 | distCUDA2(const torch::Tensor& points) 17 | { 18 | const int P = points.size(0); 19 | 20 | auto float_opts = points.options().dtype(torch::kFloat32); 21 | torch::Tensor means = torch::full({P}, 0.0, float_opts); 22 | 23 | SimpleKNN::knn(P, (float3*)points.contiguous().data(), means.contiguous().data()); 24 | 25 | return means; 26 | } -------------------------------------------------------------------------------- /submodules/simple-knn/spatial.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2023, Inria 3 | * GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | * All rights reserved. 5 | * 6 | * This software is free for non-commercial, research and evaluation use 7 | * under the terms of the LICENSE.md file. 8 | * 9 | * For inquiries contact george.drettakis@inria.fr 10 | */ 11 | 12 | #include 13 | 14 | torch::Tensor distCUDA2(const torch::Tensor& points); -------------------------------------------------------------------------------- /technicolor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | GPU_ID=$1 3 | 4 | run_pipeline() { 5 | 6 | # train 7 | CUDA_VISIBLE_DEVICES="$GPU_ID" python main.py \ 8 | --source_path data/"$data_name_path"/colmap_0 \ 9 | --model_path logs/technicolor/"$data_name" \ 10 | --deform_type node \ 11 | --node_num 1024 \ 12 | --hyper_dim 8 \ 13 | --is_blender \ 14 | --eval \ 15 | --gt_alpha_mask_as_scene_mask \ 16 | --local_frame \ 17 | --resolution 2 \ 18 | --W 800 \ 19 | --H 800 \ 20 | --config arguments/"$data_name_path".py 21 | 22 | 23 | 24 | 25 | # render 26 | CUDA_VISIBLE_DEVICES="$GPU_ID" python render.py \ 27 | --source_path data/"$data_name_path"/colmap_0 \ 28 | --model_path logs/technicolor/"$data_name" \ 29 | --deform_type node \ 30 | --node_num 1024 \ 31 | --hyper_dim 8 \ 32 | --is_blender \ 33 | --eval \ 34 | --gt_alpha_mask_as_scene_mask \ 35 | --skip_train \ 36 | --local_frame \ 37 | --resolution 2 \ 38 | --W 800 \ 39 | --H 800 \ 40 | --dataset_type $valloader 41 | } 42 | 43 | 44 | Neural3D_DA=("Birthday" "Train" "Painter" "Fatma") 45 | valloader="technicolorvalid" 46 | # run_pipeline 47 | for data_name in "${Neural3D_DA[@]}"; 48 | do 49 | echo "Dataset: Technicolor/${data_name}" 50 | data_name_path="Technicolor/${data_name}" 51 | run_pipeline "$GPU_ID" "$data_name_path" "$data_name" "$valloader" 52 | done 53 | 54 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | GPU_ID=$1 3 | 4 | 5 | run_pipeline() { 6 | 7 | # train 8 | CUDA_VISIBLE_DEVICES="$GPU_ID" python main.py \ 9 | --source_path data_spacetime/Neural3D/"$data_name"/colmap_0 \ 10 | --model_path logs/N3DV/"$data_name"\ 11 | --deform_type node \ 12 | --node_num 4096 \ 13 | --hyper_dim 8 \ 14 | --is_blender \ 15 | --gt_alpha_mask_as_dynamic_mask \ 16 | --gs_with_motion_mask \ 17 | --eval \ 18 | --local_frame \ 19 | --resolution 2 \ 20 | --W 800 \ 21 | --H 800 \ 22 | --config arguments/N3DV/"$data_name".py 23 | 24 | 25 | 26 | 27 | # render 28 | CUDA_VISIBLE_DEVICES="$GPU_ID" python render.py \ 29 | --source_path data_spacetime/Neural3D/"$data_name"/colmap_0 \ 30 | --model_path output/N3DV/"$data_name" \ 31 | --deform_type node \ 32 | --node_num 4096 \ 33 | --hyper_dim 8 \ 34 | --is_blender \ 35 | --gt_alpha_mask_as_dynamic_mask \ 36 | --gs_with_motion_mask \ 37 | --eval \ 38 | --skip_train \ 39 | --local_frame \ 40 | --resolution 2 \ 41 | --W 800 \ 42 | --H 800 \ 43 | --dataset_type colmap 44 | } 45 | 46 | 47 | Neural3D_DA=("cut_roasted_beef" "sear_steak" "flame_steak" "cook_spinach" "flame_salmon_1" "coffee_martini" ) 48 | 49 | # run_pipeline 50 | for data_name in "${Neural3D_DA[@]}"; 51 | do 52 | echo "Dataset: Neural3D_DA/${data_name}" 53 | run_pipeline "$GPU_ID" "$data_name" 54 | done 55 | 56 | 57 | #run_pipeline() { 58 | # # render 59 | # CUDA_VISIBLE_DEVICES="$GPU_ID" python render.py \ 60 | # --source_path data_spacetime/"$data_name_path"/colmap_0 \ 61 | # --model_path output/render_4_b/"$data_name" \ 62 | # --deform_type node \ 63 | # --node_num 1024 \ 64 | # --hyper_dim 8 \ 65 | # --is_blender \ 66 | # --eval \ 67 | # --gt_alpha_mask_as_scene_mask \ 68 | # --skip_train \ 69 | # --local_frame \ 70 | # --resolution 2 \ 71 | # --W 800 \ 72 | # --H 800 \ 73 | # --dataset_type $valloader 74 | #} 75 | # 76 | # 77 | #Neural3D_DA=("Birthday" "Train" "Painter" "Fatma") 78 | #valloader="technicolorvalid" 79 | ## run_pipeline 80 | #for data_name in "${Neural3D_DA[@]}"; 81 | #do 82 | # echo "Dataset: Technicolor/${data_name}" 83 | # data_name_path="Technicolor/${data_name}" 84 | # run_pipeline "$GPU_ID" "$data_name_path" "$data_name" "$valloader" 85 | #done 86 | -------------------------------------------------------------------------------- /thirdparty/colmap/pre_colmap.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import sqlite3 3 | import numpy as np 4 | 5 | # source: https://github.com/colmap/colmap/blob/dev/scripts/python/database.py 6 | IS_PYTHON3 = sys.version_info[0] >= 3 7 | 8 | MAX_IMAGE_ID = 2**31 - 1 9 | 10 | CREATE_CAMERAS_TABLE = """CREATE TABLE IF NOT EXISTS cameras ( 11 | camera_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 12 | model INTEGER NOT NULL, 13 | width INTEGER NOT NULL, 14 | height INTEGER NOT NULL, 15 | params BLOB, 16 | prior_focal_length INTEGER NOT NULL)""" 17 | 18 | CREATE_DESCRIPTORS_TABLE = """CREATE TABLE IF NOT EXISTS descriptors ( 19 | image_id INTEGER PRIMARY KEY NOT NULL, 20 | rows INTEGER NOT NULL, 21 | cols INTEGER NOT NULL, 22 | data BLOB, 23 | FOREIGN KEY(image_id) REFERENCES images(image_id) ON DELETE CASCADE)""" 24 | 25 | CREATE_IMAGES_TABLE = """CREATE TABLE IF NOT EXISTS images ( 26 | image_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 27 | name TEXT NOT NULL UNIQUE, 28 | camera_id INTEGER NOT NULL, 29 | prior_qw REAL, 30 | prior_qx REAL, 31 | prior_qy REAL, 32 | prior_qz REAL, 33 | prior_tx REAL, 34 | prior_ty REAL, 35 | prior_tz REAL, 36 | CONSTRAINT image_id_check CHECK(image_id >= 0 and image_id < {}), 37 | FOREIGN KEY(camera_id) REFERENCES cameras(camera_id)) 38 | """.format(MAX_IMAGE_ID) 39 | 40 | CREATE_TWO_VIEW_GEOMETRIES_TABLE = """ 41 | CREATE TABLE IF NOT EXISTS two_view_geometries ( 42 | pair_id INTEGER PRIMARY KEY NOT NULL, 43 | rows INTEGER NOT NULL, 44 | cols INTEGER NOT NULL, 45 | data BLOB, 46 | config INTEGER NOT NULL, 47 | F BLOB, 48 | E BLOB, 49 | H BLOB, 50 | qvec BLOB, 51 | tvec BLOB) 52 | """ 53 | 54 | CREATE_KEYPOINTS_TABLE = """CREATE TABLE IF NOT EXISTS keypoints ( 55 | image_id INTEGER PRIMARY KEY NOT NULL, 56 | rows INTEGER NOT NULL, 57 | cols INTEGER NOT NULL, 58 | data BLOB, 59 | FOREIGN KEY(image_id) REFERENCES images(image_id) ON DELETE CASCADE) 60 | """ 61 | 62 | CREATE_MATCHES_TABLE = """CREATE TABLE IF NOT EXISTS matches ( 63 | pair_id INTEGER PRIMARY KEY NOT NULL, 64 | rows INTEGER NOT NULL, 65 | cols INTEGER NOT NULL, 66 | data BLOB)""" 67 | 68 | CREATE_NAME_INDEX = \ 69 | "CREATE UNIQUE INDEX IF NOT EXISTS index_name ON images(name)" 70 | 71 | CREATE_ALL = "; ".join([ 72 | CREATE_CAMERAS_TABLE, 73 | CREATE_IMAGES_TABLE, 74 | CREATE_KEYPOINTS_TABLE, 75 | CREATE_DESCRIPTORS_TABLE, 76 | CREATE_MATCHES_TABLE, 77 | CREATE_TWO_VIEW_GEOMETRIES_TABLE, 78 | CREATE_NAME_INDEX 79 | ]) 80 | 81 | 82 | def image_ids_to_pair_id(image_id1, image_id2): 83 | if image_id1 > image_id2: 84 | image_id1, image_id2 = image_id2, image_id1 85 | return image_id1 * MAX_IMAGE_ID + image_id2 86 | 87 | 88 | def pair_id_to_image_ids(pair_id): 89 | image_id2 = pair_id % MAX_IMAGE_ID 90 | image_id1 = (pair_id - image_id2) / MAX_IMAGE_ID 91 | return image_id1, image_id2 92 | 93 | 94 | def array_to_blob(array): 95 | if IS_PYTHON3: 96 | return array.tostring() 97 | else: 98 | return np.getbuffer(array) 99 | 100 | 101 | def blob_to_array(blob, dtype, shape=(-1,)): 102 | if IS_PYTHON3: 103 | return np.fromstring(blob, dtype=dtype).reshape(*shape) 104 | else: 105 | return np.frombuffer(blob, dtype=dtype).reshape(*shape) 106 | 107 | 108 | class COLMAPDatabase(sqlite3.Connection): 109 | 110 | @staticmethod 111 | def connect(database_path): 112 | return sqlite3.connect(database_path, factory=COLMAPDatabase) 113 | 114 | 115 | def __init__(self, *args, **kwargs): 116 | super(COLMAPDatabase, self).__init__(*args, **kwargs) 117 | 118 | self.create_tables = lambda: self.executescript(CREATE_ALL) 119 | self.create_cameras_table = \ 120 | lambda: self.executescript(CREATE_CAMERAS_TABLE) 121 | self.create_descriptors_table = \ 122 | lambda: self.executescript(CREATE_DESCRIPTORS_TABLE) 123 | self.create_images_table = \ 124 | lambda: self.executescript(CREATE_IMAGES_TABLE) 125 | self.create_two_view_geometries_table = \ 126 | lambda: self.executescript(CREATE_TWO_VIEW_GEOMETRIES_TABLE) 127 | self.create_keypoints_table = \ 128 | lambda: self.executescript(CREATE_KEYPOINTS_TABLE) 129 | self.create_matches_table = \ 130 | lambda: self.executescript(CREATE_MATCHES_TABLE) 131 | self.create_name_index = lambda: self.executescript(CREATE_NAME_INDEX) 132 | 133 | def add_camera(self, model, width, height, params, 134 | prior_focal_length=False, camera_id=None): 135 | params = np.asarray(params, np.float64) 136 | cursor = self.execute( 137 | "INSERT INTO cameras VALUES (?, ?, ?, ?, ?, ?)", 138 | (camera_id, model, width, height, array_to_blob(params), 139 | prior_focal_length)) 140 | return cursor.lastrowid 141 | 142 | def add_image(self, name, camera_id, 143 | prior_q=np.full(4, np.NaN), prior_t=np.full(3, np.NaN), image_id=None): 144 | cursor = self.execute( 145 | "INSERT INTO images VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 146 | (image_id, name, camera_id, prior_q[0], prior_q[1], prior_q[2], 147 | prior_q[3], prior_t[0], prior_t[1], prior_t[2])) 148 | return cursor.lastrowid 149 | 150 | def add_keypoints(self, image_id, keypoints): 151 | assert(len(keypoints.shape) == 2) 152 | assert(keypoints.shape[1] in [2, 4, 6]) 153 | 154 | keypoints = np.asarray(keypoints, np.float32) 155 | self.execute( 156 | "INSERT INTO keypoints VALUES (?, ?, ?, ?)", 157 | (image_id,) + keypoints.shape + (array_to_blob(keypoints),)) 158 | 159 | def add_descriptors(self, image_id, descriptors): 160 | descriptors = np.ascontiguousarray(descriptors, np.uint8) 161 | self.execute( 162 | "INSERT INTO descriptors VALUES (?, ?, ?, ?)", 163 | (image_id,) + descriptors.shape + (array_to_blob(descriptors),)) 164 | 165 | def add_matches(self, image_id1, image_id2, matches): 166 | assert(len(matches.shape) == 2) 167 | assert(matches.shape[1] == 2) 168 | 169 | if image_id1 > image_id2: 170 | matches = matches[:,::-1] 171 | 172 | pair_id = image_ids_to_pair_id(image_id1, image_id2) 173 | matches = np.asarray(matches, np.uint32) 174 | self.execute( 175 | "INSERT INTO matches VALUES (?, ?, ?, ?)", 176 | (pair_id,) + matches.shape + (array_to_blob(matches),)) 177 | 178 | def add_two_view_geometry(self, image_id1, image_id2, matches, 179 | F=np.eye(3), E=np.eye(3), H=np.eye(3), 180 | qvec=np.array([1.0, 0.0, 0.0, 0.0]), 181 | tvec=np.zeros(3), config=2): 182 | assert(len(matches.shape) == 2) 183 | assert(matches.shape[1] == 2) 184 | 185 | if image_id1 > image_id2: 186 | matches = matches[:,::-1] 187 | 188 | pair_id = image_ids_to_pair_id(image_id1, image_id2) 189 | matches = np.asarray(matches, np.uint32) 190 | F = np.asarray(F, dtype=np.float64) 191 | E = np.asarray(E, dtype=np.float64) 192 | H = np.asarray(H, dtype=np.float64) 193 | qvec = np.asarray(qvec, dtype=np.float64) 194 | tvec = np.asarray(tvec, dtype=np.float64) 195 | self.execute( 196 | "INSERT INTO two_view_geometries VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 197 | (pair_id,) + matches.shape + (array_to_blob(matches), config, 198 | array_to_blob(F), array_to_blob(E), array_to_blob(H), 199 | array_to_blob(qvec), array_to_blob(tvec))) 200 | 201 | 202 | -------------------------------------------------------------------------------- /thirdparty/gaussian_splatting/utils/my_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | 5 | def normalize(x): 6 | return x / np.linalg.norm(x) 7 | 8 | def viewmatrix(z, up, pos): 9 | vec2 = normalize(z) 10 | vec1_avg = up 11 | vec0 = normalize(np.cross(vec1_avg, vec2)) 12 | vec1 = normalize(np.cross(vec2, vec0)) 13 | m = np.stack([vec0, vec1, vec2, pos], 1) 14 | return m 15 | 16 | 17 | def poses_avg(poses): 18 | 19 | hwf = poses[0, :3, -1:] 20 | 21 | center = poses[:, :3, 3].mean(0) 22 | vec2 = normalize(poses[:, :3, 2].sum(0)) 23 | up = poses[:, :3, 1].sum(0) 24 | c2w = np.concatenate([viewmatrix(vec2, up, center), hwf], 1) 25 | 26 | return c2w 27 | 28 | 29 | 30 | 31 | def posetow2c_matrcs(poses): 32 | tmp = inversestep4(inversestep3(inversestep2(inversestep1(poses)))) 33 | N = tmp.shape[0] 34 | ret = [] 35 | for i in range(N): 36 | ret.append(tmp[i]) 37 | return ret 38 | 39 | 40 | def getRTfromPose(w2c_mats): 41 | for m in w2c_mats: 42 | R = m[:3, :3] 43 | t = m[:3, 3] 44 | #print(R, t) 45 | 46 | def tolist(w2c_mats): 47 | return w2c_mats.tolist() 48 | def inversestep4(c2w_mats): 49 | return np.linalg.inv(c2w_mats) 50 | def inversestep3(newposes): 51 | tmp = newposes.transpose([2, 0, 1]) # 20, 3, 4 52 | N, _, __ = tmp.shape 53 | zeros = np.zeros((N, 1, 4)) 54 | zeros[:, 0, 3] = 1 55 | c2w_mats = np.concatenate([tmp, zeros], axis=1) 56 | return c2w_mats 57 | 58 | def inversestep2(newposes): 59 | return newposes[:,0:4, :] 60 | def inversestep1(newposes): 61 | poses = np.concatenate([newposes[:, 1:2, :], newposes[:, 0:1, :], -newposes[:, 2:3, :], newposes[:, 3:4, :], newposes[:, 4:5, :]], axis=1) 62 | return poses 63 | 64 | 65 | 66 | 67 | 68 | def rotmat2qvec(R): 69 | Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat 70 | K = np.array([ 71 | [Rxx - Ryy - Rzz, 0, 0, 0], 72 | [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0], 73 | [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0], 74 | [Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0 75 | eigvals, eigvecs = np.linalg.eigh(K) 76 | qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)] 77 | if qvec[0] < 0: 78 | qvec *= -1 79 | return qvec 80 | 81 | 82 | 83 | def qvec2rotmat(qvec): 84 | return np.array([ 85 | [1 - 2 * qvec[2]**2 - 2 * qvec[3]**2, 86 | 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], 87 | 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2]], 88 | [2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], 89 | 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2, 90 | 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1]], 91 | [2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], 92 | 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], 93 | 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2]]) -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | import torch 14 | from scene import Scene 15 | import uuid 16 | from utils.image_utils import psnr #, lpips, alex_lpips 17 | 18 | from utils.image_utils import ssim as ssim_func 19 | # from piq import LPIPS # # 20 | # lpips = LPIPS() # # 21 | from argparse import Namespace 22 | from pytorch_msssim import ms_ssim 23 | from torch.utils.data import DataLoader 24 | 25 | from lpipsPyTorch import lpips # add 26 | 27 | try: 28 | from torch.utils.tensorboard import SummaryWriter 29 | 30 | TENSORBOARD_FOUND = True 31 | except ImportError: 32 | TENSORBOARD_FOUND = False 33 | 34 | 35 | def prepare_output_and_logger(args): 36 | if not args.model_path: 37 | if os.getenv('OAR_JOB_ID'): 38 | unique_str = os.getenv('OAR_JOB_ID') 39 | else: 40 | unique_str = str(uuid.uuid4()) 41 | args.model_path = os.path.join("./output/", unique_str[0:10]) 42 | 43 | # Set up output folder 44 | print("Output folder: {}".format(args.model_path)) 45 | os.makedirs(args.model_path, exist_ok=True) 46 | with open(os.path.join(args.model_path, "cfg_args"), 'w') as cfg_log_f: 47 | cfg_log_f.write(str(Namespace(**vars(args)))) 48 | 49 | # Create Tensorboard writer 50 | tb_writer = None 51 | if TENSORBOARD_FOUND: 52 | tb_writer = SummaryWriter(args.model_path) 53 | else: 54 | print("Tensorboard not available: not logging progress") 55 | return tb_writer 56 | 57 | 58 | def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_iterations, scene: Scene, renderFunc, renderArgs, deform, load2gpu_on_the_fly, progress_bar=None): 59 | if tb_writer: 60 | tb_writer.add_scalar('train_loss_patches/l1_loss', Ll1.item(), iteration) 61 | tb_writer.add_scalar('train_loss_patches/total_loss', loss.item(), iteration) 62 | tb_writer.add_scalar('iter_time', elapsed, iteration) 63 | 64 | test_psnr = 0.0 65 | test_ssim = 0.0 66 | test_lpips = 1e10 67 | test_ms_ssim = 0.0 68 | test_alex_lpips = 1e10 69 | # Report test and samples of training set 70 | if iteration in testing_iterations: 71 | torch.cuda.empty_cache() 72 | validation_configs = ({'name': 'test', 'cameras': scene.getTestCameras()}, 73 | {'name': 'train', 74 | 'cameras': [scene.getTrainCameras()[idx % len(scene.getTrainCameras())] for idx in range(5, 30, 5)]}) 75 | for config in validation_configs: 76 | if config['cameras'] and len(config['cameras']) > 0: 77 | # images = torch.tensor([], device="cuda") 78 | # gts = torch.tensor([], device="cuda") 79 | psnr_list, ssim_list, lpips_list, l1_list = [], [], [], [] 80 | ms_ssim_list, alex_lpips_list = [], [] 81 | for idx, viewpoint in enumerate(config['cameras']): 82 | if load2gpu_on_the_fly: 83 | viewpoint.load2device() 84 | fid = viewpoint.fid 85 | xyz = scene.gaussians.get_xyz 86 | 87 | if deform.name == 'mlp': 88 | time_input = fid.unsqueeze(0).expand(xyz.shape[0], -1) 89 | elif deform.name == 'node': 90 | time_input = deform.deform.expand_time(fid) 91 | else: 92 | time_input = 0 93 | 94 | d_values = deform.step(xyz.detach(), time_input, feature=scene.gaussians.feature, is_training=False, motion_mask=scene.gaussians.motion_mask, camera_center=viewpoint.camera_center) 95 | d_xyz, d_rotation, d_scaling, d_opacity, d_color = d_values['d_xyz'], d_values['d_rotation'], d_values['d_scaling'], d_values['d_opacity'], d_values['d_color'] 96 | 97 | image = torch.clamp(renderFunc(viewpoint, scene.gaussians, *renderArgs, d_xyz=d_xyz, d_rotation=d_rotation, d_scaling=d_scaling, d_opacity=d_opacity, d_color=d_color, d_rot_as_res=deform.d_rot_as_res)["render"], 0.0, 1.0) 98 | gt_image = torch.clamp(viewpoint.original_image.to("cuda"), 0.0, 1.0) 99 | 100 | l1_list.append(l1_loss(image[None], gt_image[None]).mean()) 101 | psnr_list.append(psnr(image[None], gt_image[None]).mean()) 102 | ssim_list.append(ssim_func(image[None], gt_image[None], data_range=1.).mean()) 103 | # lpips_list.append(lpips(image[None], gt_image[None]).mean()) # # 104 | ms_ssim_list.append(ms_ssim(image[None], gt_image[None], data_range=1.).mean()) 105 | # alex_lpips_list.append(alex_lpips(image[None], gt_image[None]).mean()) # # 106 | 107 | alex_lpips_list.append(lpips(image[None], gt_image[None], net_type='alex')) # ADD 108 | lpips_list.append(lpips(image[None], gt_image[None], net_type='vgg')) # ADD 109 | 110 | # images = torch.cat((images, image.unsqueeze(0)), dim=0) 111 | # gts = torch.cat((gts, gt_image.unsqueeze(0)), dim=0) 112 | 113 | if load2gpu_on_the_fly: 114 | viewpoint.load2device('cpu') 115 | if tb_writer and (idx < 5): 116 | tb_writer.add_images(config['name'] + "_view_{}/render".format(viewpoint.image_name), image[None], global_step=iteration) 117 | if iteration == testing_iterations[0]: 118 | tb_writer.add_images(config['name'] + "_view_{}/ground_truth".format(viewpoint.image_name), gt_image[None], global_step=iteration) 119 | 120 | l1_test = torch.stack(l1_list).mean() 121 | psnr_test = torch.stack(psnr_list).mean() 122 | ssim_test = torch.stack(ssim_list).mean() 123 | lpips_test = torch.stack(lpips_list).mean() 124 | ms_ssim_test = torch.stack(ms_ssim_list).mean() 125 | alex_lpips_test = torch.stack(alex_lpips_list).mean() 126 | if config['name'] == 'test' or len(validation_configs[0]['cameras']) == 0: 127 | test_psnr = psnr_test 128 | test_ssim = ssim_test 129 | test_lpips = lpips_test 130 | test_ms_ssim = ms_ssim_test 131 | test_alex_lpips = alex_lpips_test 132 | if tb_writer: 133 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - l1_loss', l1_test, iteration) 134 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - psnr', psnr_test, iteration) 135 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - ssim', test_ssim, iteration) 136 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - lpips', test_lpips, iteration) 137 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - ms-ssim', test_ms_ssim, iteration) 138 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - alex-lpips', test_alex_lpips, iteration) 139 | 140 | if tb_writer: 141 | tb_writer.add_histogram("scene/opacity_histogram", scene.gaussians.get_opacity, iteration) 142 | tb_writer.add_scalar('total_points', scene.gaussians.get_xyz.shape[0], iteration) 143 | torch.cuda.empty_cache() 144 | 145 | return test_psnr, test_ssim, test_lpips, test_ms_ssim, test_alex_lpips 146 | -------------------------------------------------------------------------------- /train_gui_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | class DeformKeypoints: 6 | def __init__(self) -> None: 7 | self.keypoints3d_list = [] # list of keypoints group 8 | self.keypoints_idx_list = [] # keypoints index 9 | self.keypoints3d_delta_list = [] 10 | self.selective_keypoints_idx_list = [] # keypoints index 11 | self.idx2group = {} 12 | 13 | self.selective_rotation_keypoints_idx_list = [] 14 | # self.rotation_idx2group = {} 15 | 16 | def get_kpt_idx(self,): 17 | return self.keypoints_idx_list 18 | 19 | def get_kpt(self,): 20 | return self.keypoints3d_list 21 | 22 | def get_kpt_delta(self,): 23 | return self.keypoints3d_delta_list 24 | 25 | def get_deformed_kpt_np(self, rate=1.): 26 | return np.array(self.keypoints3d_list) + np.array(self.keypoints3d_delta_list) * rate 27 | 28 | def add_kpts(self, keypoints_coord, keypoints_idx, expand=False): 29 | # keypoints3d: [N, 3], keypoints_idx: [N,], torch.tensor 30 | # self.selective_keypoints_idx_list.clear() 31 | selective_keypoints_idx_list = [] if not expand else self.selective_keypoints_idx_list 32 | for idx in range(len(keypoints_idx)): 33 | if not self.contain_kpt(keypoints_idx[idx].item()): 34 | selective_keypoints_idx_list.append(len(self.keypoints_idx_list)) 35 | self.keypoints_idx_list.append(keypoints_idx[idx].item()) 36 | self.keypoints3d_list.append(keypoints_coord[idx].cpu().numpy()) 37 | self.keypoints3d_delta_list.append(np.zeros_like(self.keypoints3d_list[-1])) 38 | 39 | for kpt_idx in keypoints_idx: 40 | self.idx2group[kpt_idx.item()] = selective_keypoints_idx_list 41 | 42 | self.selective_keypoints_idx_list = selective_keypoints_idx_list 43 | 44 | def contain_kpt(self, idx): 45 | # idx: int 46 | if idx in self.keypoints_idx_list: 47 | return True 48 | else: 49 | return False 50 | 51 | def select_kpt(self, idx): 52 | # idx: int 53 | # output: idx list of this group 54 | if idx in self.keypoints_idx_list: 55 | self.selective_keypoints_idx_list = self.idx2group[idx] 56 | 57 | def select_rotation_kpt(self, idx): 58 | if idx in self.keypoints_idx_list: 59 | self.selective_rotation_keypoints_idx_list = self.idx2group[idx] 60 | 61 | def get_rotation_center(self,): 62 | selected_rotation_points = self.get_deformed_kpt_np()[self.selective_rotation_keypoints_idx_list] 63 | return selected_rotation_points.mean(axis=0) 64 | 65 | def get_selective_center(self,): 66 | selected_points = self.get_deformed_kpt_np()[self.selective_keypoints_idx_list] 67 | return selected_points.mean(axis=0) 68 | 69 | def delete_kpt(self, idx): 70 | for kidx in self.selective_keypoints_idx_list: 71 | list_idx = self.idx2group.pop(kidx) 72 | self.keypoints3d_delta_list.pop(list_idx) 73 | self.keypoints3d_list.pop(list_idx) 74 | self.keypoints_idx_list.pop(list_idx) 75 | 76 | def delete_batch_ktps(self, batch_idx): 77 | pass 78 | 79 | def update_delta(self, delta): 80 | # delta: [3,], np.array 81 | for idx in self.selective_keypoints_idx_list: 82 | self.keypoints3d_delta_list[idx] += delta 83 | 84 | def set_delta(self, delta): 85 | # delta: [N, 3], np.array 86 | for id, idx in enumerate(self.selective_keypoints_idx_list): 87 | self.keypoints3d_delta_list[idx] = delta[id] 88 | 89 | 90 | def set_rotation_delta(self, rot_mat): 91 | kpts3d = self.get_deformed_kpt_np()[self.selective_keypoints_idx_list] 92 | kpts3d_mean = kpts3d.mean(axis=0) 93 | kpts3d = (kpts3d - kpts3d_mean) @ rot_mat.T + kpts3d_mean 94 | delta = kpts3d - np.array(self.keypoints3d_list)[self.selective_keypoints_idx_list] 95 | for id, idx in enumerate(self.selective_keypoints_idx_list): 96 | self.keypoints3d_delta_list[idx] = delta[id] 97 | -------------------------------------------------------------------------------- /utils/__pycache__/camera_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/camera_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/camera_utils_st.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/camera_utils_st.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/deform_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/deform_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/depth_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/depth_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/general_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/general_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/graphics_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/graphics_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/graphics_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/graphics_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/image_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/image_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/logger_hook.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/logger_hook.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/loss_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/loss_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/loss_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/loss_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/pose_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/pose_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/rigid_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/rigid_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/sh_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/sh_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/system_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/system_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/__pycache__/system_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/system_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/time_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EllisonKing/taylor_gaussian/f95da60669041de65bfec4b91086cb7ceb18ada6/utils/__pycache__/time_utils.cpython-310.pyc -------------------------------------------------------------------------------- /utils/bezier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class BezierCurve: 5 | def __init__(self, points: np.ndarray) -> None: 6 | if points.ndim == 2: 7 | points = points[None] 8 | self.points = points # N, T, D 9 | self.T = points.shape[1] 10 | 11 | def __call__(self, t: float): 12 | assert 0 <= t <= 1, f't: {t} out of range [0, 1]!' 13 | return self.interpolate(t, self.points) 14 | 15 | def interpolate(self, t, points): 16 | if points.shape[1] < 2: 17 | raise ValueError(f"points shape error: {points.shape}") 18 | elif points.shape[1] == 2: 19 | point0, point1 = points[:, 0], points[:, 1] 20 | else: 21 | point0 = self.interpolate(t, points[:, :-1]) 22 | point1 = self.interpolate(t, points[:, 1:]) 23 | return (1 - t) * point0 + t * point1 24 | 25 | 26 | class PieceWiseLinear: 27 | def __init__(self, points: np.ndarray) -> None: 28 | if points.ndim == 2: 29 | points = points[None] 30 | self.points = points # N, T, D 31 | self.T = points.shape[1] 32 | 33 | def __call__(self, t: float): 34 | assert 0 <= t <= 1, f't: {t} out of range [0, 1]!' 35 | return self.interpolate(t, self.points) 36 | 37 | def interpolate(self, t, points): 38 | if points.shape[1] < 2: 39 | raise ValueError(f"points shape error: {points.shape}") 40 | else: 41 | t_scaled = t * (self.T - 1) 42 | t_floor = min(self.T - 2, max(0, int(np.floor(t_scaled)))) 43 | t_ceil = t_floor + 1 44 | point0, point1 = points[:, t_floor], points[:, t_ceil] 45 | return (t_ceil - t_scaled) * point0 + (t_scaled - t_floor) * point1 46 | -------------------------------------------------------------------------------- /utils/camera_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from scene.cameras import Camera 13 | import numpy as np 14 | from utils.general_utils import PILtoTorch, ArrayToTorch 15 | from utils.graphics_utils import fov2focal 16 | import json 17 | 18 | WARNED = False 19 | 20 | 21 | def loadCam(args, id, cam_info, resolution_scale, flow_dirs): 22 | orig_w, orig_h = cam_info.image.size 23 | 24 | if args.resolution in [1, 2, 4, 8]: 25 | resolution = round(orig_w / (resolution_scale * args.resolution)), round( 26 | orig_h / (resolution_scale * args.resolution)) 27 | else: # should be a type that converts to float 28 | if args.resolution == -1: 29 | if orig_w > 1600: 30 | global WARNED 31 | if not WARNED: 32 | print("[ INFO ] Encountered quite large input images (>1.6K pixels width), rescaling to 1.6K.\n " 33 | "If this is not desired, please explicitly specify '--resolution/-r' as 1") 34 | WARNED = True 35 | global_down = orig_w / 1600 36 | else: 37 | global_down = 1 38 | else: 39 | global_down = orig_w / args.resolution 40 | 41 | scale = float(global_down) * float(resolution_scale) 42 | resolution = (int(orig_w / scale), int(orig_h / scale)) 43 | 44 | resized_image_rgb = PILtoTorch(cam_info.image, resolution) 45 | 46 | gt_image = resized_image_rgb[:3, ...] 47 | loaded_mask = None 48 | 49 | if resized_image_rgb.shape[0] == 4: 50 | loaded_mask = resized_image_rgb[3:4, ...] 51 | 52 | return Camera(colmap_id=cam_info.uid, R=cam_info.R, T=cam_info.T, 53 | FoVx=cam_info.FovX, FoVy=cam_info.FovY, 54 | image=gt_image, gt_alpha_mask=loaded_mask, 55 | image_name=cam_info.image_name, uid=id, 56 | data_device=args.data_device if not args.load2gpu_on_the_fly else 'cpu', fid=cam_info.fid, 57 | depth=cam_info.depth, flow_dirs=flow_dirs) 58 | 59 | 60 | def cameraList_from_camInfos(cam_infos, resolution_scale, args, flow_dirs_list=None): 61 | camera_list = [] 62 | 63 | for id, c in enumerate(cam_infos): 64 | camera_list.append(loadCam(args, id, c, resolution_scale, [] if flow_dirs_list is None else flow_dirs_list[id])) 65 | 66 | return camera_list 67 | 68 | 69 | def camera_to_JSON(id, camera: Camera): 70 | Rt = np.zeros((4, 4)) 71 | Rt[:3, :3] = camera.R.transpose() 72 | Rt[:3, 3] = camera.T 73 | Rt[3, 3] = 1.0 74 | 75 | W2C = np.linalg.inv(Rt) 76 | pos = W2C[:3, 3] 77 | rot = W2C[:3, :3] 78 | serializable_array_2d = [x.tolist() for x in rot] 79 | camera_entry = { 80 | 'id': id, 81 | 'img_name': camera.image_name, 82 | 'width': camera.width, 83 | 'height': camera.height, 84 | 'position': pos.tolist(), 85 | 'rotation': serializable_array_2d, 86 | 'fy': fov2focal(camera.FovY, camera.height), 87 | 'fx': fov2focal(camera.FovX, camera.width) 88 | } 89 | return camera_entry 90 | 91 | 92 | def camera_nerfies_from_JSON(path, scale): 93 | """Loads a JSON camera into memory.""" 94 | with open(path, 'r') as fp: 95 | camera_json = json.load(fp) 96 | 97 | # Fix old camera JSON. 98 | if 'tangential' in camera_json: 99 | camera_json['tangential_distortion'] = camera_json['tangential'] 100 | 101 | return dict( 102 | orientation=np.array(camera_json['orientation']), 103 | position=np.array(camera_json['position']), 104 | focal_length=camera_json['focal_length'] * scale, 105 | principal_point=np.array(camera_json['principal_point']) * scale, 106 | skew=camera_json['skew'], 107 | pixel_aspect_ratio=camera_json['pixel_aspect_ratio'], 108 | radial_distortion=np.array(camera_json['radial_distortion']), 109 | tangential_distortion=np.array(camera_json['tangential_distortion']), 110 | image_size=np.array((int(round(camera_json['image_size'][0] * scale)), 111 | int(round(camera_json['image_size'][1] * scale)))), 112 | ) 113 | -------------------------------------------------------------------------------- /utils/dual_quaternion.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: 5 | """ 6 | Returns torch.sqrt(torch.max(0, x)) 7 | but with a zero subgradient where x is 0. 8 | """ 9 | ret = torch.zeros_like(x) 10 | positive_mask = x > 0 11 | ret[positive_mask] = torch.sqrt(x[positive_mask]) 12 | return ret 13 | 14 | 15 | def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor: 16 | """ 17 | Convert rotations given as rotation matrices to quaternions. 18 | 19 | Args: 20 | matrix: Rotation matrices as tensor of shape (..., 3, 3). 21 | 22 | Returns: 23 | quaternions with real part first, as tensor of shape (..., 4). 24 | """ 25 | if matrix.size(-1) != 3 or matrix.size(-2) != 3: 26 | raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") 27 | 28 | batch_dim = matrix.shape[:-2] 29 | m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( 30 | matrix.reshape(batch_dim + (9,)), dim=-1 31 | ) 32 | 33 | q_abs = _sqrt_positive_part( 34 | torch.stack( 35 | [ 36 | 1.0 + m00 + m11 + m22, 37 | 1.0 + m00 - m11 - m22, 38 | 1.0 - m00 + m11 - m22, 39 | 1.0 - m00 - m11 + m22, 40 | ], 41 | dim=-1, 42 | ) 43 | ) 44 | 45 | # we produce the desired quaternion multiplied by each of r, i, j, k 46 | quat_by_rijk = torch.stack( 47 | [ 48 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 49 | # `int`. 50 | torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), 51 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 52 | # `int`. 53 | torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), 54 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 55 | # `int`. 56 | torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), 57 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 58 | # `int`. 59 | torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), 60 | ], 61 | dim=-2, 62 | ) 63 | 64 | # We floor here at 0.1 but the exact level is not important; if q_abs is small, 65 | # the candidate won't be picked. 66 | flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) 67 | quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) 68 | 69 | # if not for numerical problems, quat_candidates[i] should be same (up to a sign), 70 | # forall i; we pick the best-conditioned one (with the largest denominator) 71 | 72 | return quat_candidates[ 73 | torch.nn.functional.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, : 74 | ].reshape(batch_dim + (4,)) 75 | 76 | 77 | def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor: 78 | r, i, j, k = torch.unbind(quaternions, -1) 79 | two_s = 2.0 / (quaternions * quaternions).sum(-1) 80 | o = torch.stack( 81 | ( 82 | 1 - two_s * (j * j + k * k), 83 | two_s * (i * j - k * r), 84 | two_s * (i * k + j * r), 85 | two_s * (i * j + k * r), 86 | 1 - two_s * (i * i + k * k), 87 | two_s * (j * k - i * r), 88 | two_s * (i * k - j * r), 89 | two_s * (j * k + i * r), 90 | 1 - two_s * (i * i + j * j), 91 | ), 92 | -1, 93 | ) 94 | return o.reshape(quaternions.shape[:-1] + (3, 3)) 95 | 96 | 97 | def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor: 98 | return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions) 99 | 100 | 101 | def quaternion_raw_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: 102 | aw, ax, ay, az = torch.unbind(a, -1) 103 | bw, bx, by, bz = torch.unbind(b, -1) 104 | ow = aw * bw - ax * bx - ay * by - az * bz 105 | ox = aw * bx + ax * bw + ay * bz - az * by 106 | oy = aw * by - ax * bz + ay * bw + az * bx 107 | oz = aw * bz + ax * by - ay * bx + az * bw 108 | return torch.stack((ow, ox, oy, oz), -1) 109 | 110 | 111 | def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: 112 | ab = quaternion_raw_multiply(a, b) 113 | return standardize_quaternion(ab) 114 | 115 | 116 | def dualquaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: 117 | a_real, b_real = a[..., :4], b[..., :4] 118 | a_imag, b_imag = a[..., 4:], b[..., 4:] 119 | o_real = quaternion_multiply(a_real, b_real) 120 | o_imag = quaternion_multiply(a_imag, b_real) + quaternion_multiply(a_real, b_imag) 121 | o = torch.cat([o_real, o_imag], dim=-1) 122 | return o 123 | 124 | 125 | def conjugation(q): 126 | if q.shape[-1] == 4: 127 | q = torch.cat([q[..., :1], -q[..., 1:]], dim=-1) 128 | elif q.shape[-1] == 8: 129 | q = torch.cat([q[..., :1], -q[..., 1:4], q[..., 4:5], -q[..., 5:]], dim=-1) 130 | else: 131 | raise TypeError(f'q should be of [..., 4] or [..., 8] but got {q.shape}!') 132 | return q 133 | 134 | 135 | def QT2DQ(q, t, rot_as_q=True): 136 | if not rot_as_q: 137 | q = matrix_to_quaternion(q) 138 | q = torch.nn.functional.normalize(q) 139 | real = q 140 | t = torch.cat([torch.zeros_like(t[..., :1]), t], dim=-1) 141 | image = quaternion_multiply(t, q) / 2 142 | dq = torch.cat([real, image], dim=-1) 143 | return dq 144 | 145 | 146 | def DQ2QT(dq, rot_as_q=False): 147 | real = dq[..., :4] 148 | imag = dq[..., 4:] 149 | real_norm = real.norm(dim=-1, keepdim=True).clamp(min=1e-8) 150 | real, imag = real / real_norm, imag / real_norm 151 | 152 | w0, x0, y0, z0 = torch.unbind(real, -1) 153 | w1, x1, y1, z1 = torch.unbind(imag, -1) 154 | 155 | t = 2* torch.stack([- w1*x0 + x1*w0 - y1*z0 + z1*y0, 156 | - w1*y0 + x1*z0 + y1*w0 - z1*x0, 157 | - w1*z0 - x1*y0 + y1*x0 + z1*w0], dim=-1) 158 | R = torch.stack([1-2*y0**2-2*z0**2, 2*x0*y0-2*w0*z0, 2*x0*z0+2*w0*y0, 159 | 2*x0*y0+2*w0*z0, 1-2*x0**2-2*z0**2, 2*y0*z0-2*w0*x0, 160 | 2*x0*z0-2*w0*y0, 2*y0*z0+2*w0*x0, 1-2*x0**2-2*y0**2], dim=-1).reshape([*w0.shape, 3, 3]) 161 | if rot_as_q: 162 | q = matrix_to_quaternion(R) 163 | return q, t 164 | else: 165 | return R, t 166 | 167 | 168 | def DQBlending(q, t, weights, rot_as_q=True): 169 | ''' 170 | Input: 171 | q: [..., k, 4]; t: [..., k, 3]; weights: [..., k] 172 | Output: 173 | q_: [..., 4]; t_: [..., 3] 174 | ''' 175 | dq = QT2DQ(q=q, t=t) 176 | dq_avg = (dq * weights[..., None]).sum(dim=-2) 177 | q_, t_ = DQ2QT(dq_avg, rot_as_q=rot_as_q) 178 | return q_, t_ 179 | 180 | 181 | def interpolate(q0, t0, q1, t1, weight, rot_as_q=True): 182 | dq0 = QT2DQ(q=q0, t=t0) 183 | dq1 = QT2DQ(q=q1, t=t1) 184 | dq_avg = dq0 * weight + dq1 * (1 - weight) 185 | q, t = DQ2QT(dq=dq_avg, rot_as_q=rot_as_q) 186 | return q, t 187 | 188 | 189 | def transformation_blending(transformations, weights): 190 | Rs, Ts = transformations[:, :3, :3], transformations[:, :3, 3] 191 | qs = matrix_to_quaternion(Rs) 192 | q, T = DQBlending(qs[None], Ts[None], weights) 193 | R = quaternion_to_matrix(q) 194 | transformation = torch.eye(4).to(transformations.device)[None].expand(weights.shape[0], 4, 4).clone() 195 | transformation[:, :3, :3] = R 196 | transformation[:, :3, 3] = T 197 | return transformation 198 | -------------------------------------------------------------------------------- /utils/graphics_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import math 14 | import numpy as np 15 | from typing import NamedTuple 16 | 17 | 18 | class BasicPointCloud(NamedTuple): 19 | points: np.array 20 | colors: np.array 21 | normals: np.array 22 | 23 | 24 | class BasicPointCloud_ST(NamedTuple): 25 | points: np.array 26 | colors: np.array 27 | normals: np.array 28 | times: np.array # 29 | 30 | def geom_transform_points(points, transf_matrix): 31 | P, _ = points.shape 32 | ones = torch.ones(P, 1, dtype=points.dtype, device=points.device) 33 | points_hom = torch.cat([points, ones], dim=1) 34 | points_out = torch.matmul(points_hom, transf_matrix.unsqueeze(0)) 35 | 36 | denom = points_out[..., 3:] + 0.0000001 37 | return (points_out[..., :3] / denom).squeeze(dim=0) 38 | 39 | 40 | def getWorld2View(R, t): 41 | Rt = np.zeros((4, 4)) 42 | Rt[:3, :3] = R.transpose() 43 | Rt[:3, 3] = t 44 | Rt[3, 3] = 1.0 45 | return np.float32(Rt) 46 | 47 | 48 | def getWorld2View2(R, t, translate=np.array([.0, .0, .0]), scale=1.0): 49 | Rt = np.zeros((4, 4)) 50 | Rt[:3, :3] = R.transpose() 51 | Rt[:3, 3] = t 52 | Rt[3, 3] = 1.0 53 | 54 | C2W = np.linalg.inv(Rt) 55 | cam_center = C2W[:3, 3] 56 | cam_center = (cam_center + translate) * scale 57 | C2W[:3, 3] = cam_center 58 | Rt = np.linalg.inv(C2W) 59 | return np.float32(Rt) 60 | 61 | 62 | def getProjectionMatrix(znear, zfar, fovX, fovY): 63 | tanHalfFovY = math.tan((fovY / 2)) 64 | tanHalfFovX = math.tan((fovX / 2)) 65 | 66 | top = tanHalfFovY * znear 67 | bottom = -top 68 | right = tanHalfFovX * znear 69 | left = -right 70 | 71 | P = torch.zeros(4, 4) 72 | 73 | z_sign = 1.0 74 | 75 | P[0, 0] = 2.0 * znear / (right - left) 76 | P[1, 1] = 2.0 * znear / (top - bottom) 77 | P[0, 2] = (right + left) / (right - left) 78 | P[1, 2] = (top + bottom) / (top - bottom) 79 | P[3, 2] = z_sign 80 | P[2, 2] = z_sign * zfar / (zfar - znear) 81 | P[2, 3] = -(zfar * znear) / (zfar - znear) 82 | return P 83 | 84 | 85 | def getProjectionMatrixCV(znear, zfar, fovX, fovY, cx=0.0, cy=0.0): 86 | ''' 87 | cx and cy range is -0.5 to 0.5 88 | we use cx cy range -0.5 * 0.5 89 | 90 | ''' 91 | tanHalfFovY = math.tan(fovY / 2) 92 | tanHalfFovX = math.tan(fovX / 2) 93 | 94 | top = tanHalfFovY * znear 95 | bottom = -top 96 | right = tanHalfFovX * znear 97 | left = -right 98 | 99 | # Adjust for off-center projection 100 | deltax = (2 * tanHalfFovX * znear) * cx # 101 | deltay = (2 * tanHalfFovY * znear) * cy # 102 | 103 | left += deltax 104 | right += deltax 105 | top += deltay 106 | bottom += deltay 107 | 108 | # left -= deltax 109 | # right -= deltax 110 | # top -= deltay 111 | # bottom -= deltay 112 | 113 | # left = left(1 +cx)* znear 114 | # right += cx * znear 115 | # top += cy * znear 116 | # bottom += cy * znear 117 | 118 | P = torch.zeros(4, 4) 119 | 120 | z_sign = 1.0 121 | 122 | P[0, 0] = 2.0 * znear / (right - left) 123 | P[1, 1] = 2.0 * znear / (top - bottom) 124 | P[0, 2] = (right + left) / (right - left) 125 | P[1, 2] = (top + bottom) / (top - bottom) 126 | P[3, 2] = z_sign 127 | # P[2, 2] = z_sign * zfar / (zfar - znear) 128 | P[2, 2] = z_sign * (zfar + znear) / (zfar - znear) 129 | 130 | P[2, 3] = -(zfar * znear) / (zfar - znear) 131 | return P 132 | 133 | 134 | def fov2focal(fov, pixels): 135 | return pixels / (2 * math.tan(fov / 2)) 136 | 137 | 138 | def focal2fov(focal, pixels): 139 | return 2 * math.atan(pixels / (2 * focal)) 140 | -------------------------------------------------------------------------------- /utils/image_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | 14 | # NeRF-DS Alex LPIPS 15 | import lpips as lpips_lib 16 | loss_fn_alex = lpips_lib.LPIPS(net='alex') 17 | loss_fn_alex.net.cuda() 18 | loss_fn_alex.scaling_layer.cuda() 19 | loss_fn_alex.lins.cuda() 20 | def alex_lpips(image1, image2): 21 | image1 = image1 * 2 - 1 22 | image2 = image2 * 2 - 1 23 | lpips = loss_fn_alex(image1, image2) 24 | return lpips 25 | 26 | 27 | def mse(img1, img2): 28 | return (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True) 29 | 30 | 31 | def psnr(img1, img2): 32 | mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True) 33 | return 20 * torch.log10(1.0 / torch.sqrt(mse)) 34 | 35 | 36 | from piq import ssim, LPIPS 37 | lpips = LPIPS() 38 | -------------------------------------------------------------------------------- /utils/interactive_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | class DeformKeypoints: 6 | def __init__(self) -> None: 7 | self.keypoints3d_list = [] # list of keypoints group 8 | self.keypoints_idx_list = [] # keypoints index 9 | self.keypoints3d_delta_list = [] 10 | self.selective_keypoints_idx_list = [] # keypoints index 11 | self.idx2group = {} 12 | 13 | self.selective_rotation_keypoints_idx_list = [] 14 | # self.rotation_idx2group = {} 15 | 16 | def get_kpt_idx(self,): 17 | return self.keypoints_idx_list 18 | 19 | def get_kpt(self,): 20 | return self.keypoints3d_list 21 | 22 | def get_kpt_delta(self,): 23 | return self.keypoints3d_delta_list 24 | 25 | def get_deformed_kpt_np(self, rate=1.): 26 | return np.array(self.keypoints3d_list) + np.array(self.keypoints3d_delta_list) * rate 27 | 28 | def add_kpts(self, keypoints_coord, keypoints_idx, expand=False): 29 | # keypoints3d: [N, 3], keypoints_idx: [N,], torch.tensor 30 | # self.selective_keypoints_idx_list.clear() 31 | selective_keypoints_idx_list = [] if not expand else self.selective_keypoints_idx_list 32 | for idx in range(len(keypoints_idx)): 33 | if not self.contain_kpt(keypoints_idx[idx].item()): 34 | selective_keypoints_idx_list.append(len(self.keypoints_idx_list)) 35 | self.keypoints_idx_list.append(keypoints_idx[idx].item()) 36 | self.keypoints3d_list.append(keypoints_coord[idx].cpu().numpy()) 37 | self.keypoints3d_delta_list.append(np.zeros_like(self.keypoints3d_list[-1])) 38 | 39 | for kpt_idx in keypoints_idx: 40 | self.idx2group[kpt_idx.item()] = selective_keypoints_idx_list 41 | 42 | self.selective_keypoints_idx_list = selective_keypoints_idx_list 43 | 44 | def contain_kpt(self, idx): 45 | # idx: int 46 | if idx in self.keypoints_idx_list: 47 | return True 48 | else: 49 | return False 50 | 51 | def select_kpt(self, idx): 52 | # idx: int 53 | # output: idx list of this group 54 | if idx in self.keypoints_idx_list: 55 | self.selective_keypoints_idx_list = self.idx2group[idx] 56 | 57 | def select_rotation_kpt(self, idx): 58 | if idx in self.keypoints_idx_list: 59 | self.selective_rotation_keypoints_idx_list = self.idx2group[idx] 60 | 61 | def get_rotation_center(self,): 62 | selected_rotation_points = self.get_deformed_kpt_np()[self.selective_rotation_keypoints_idx_list] 63 | return selected_rotation_points.mean(axis=0) 64 | 65 | def get_selective_center(self,): 66 | selected_points = self.get_deformed_kpt_np()[self.selective_keypoints_idx_list] 67 | return selected_points.mean(axis=0) 68 | 69 | def delete_kpt(self, idx): 70 | pass 71 | 72 | def delete_batch_ktps(self, batch_idx): 73 | pass 74 | 75 | def update_delta(self, delta): 76 | # delta: [3,], np.array 77 | for idx in self.selective_keypoints_idx_list: 78 | self.keypoints3d_delta_list[idx] += delta 79 | 80 | def set_delta(self, delta): 81 | # delta: [N, 3], np.array 82 | for id, idx in enumerate(self.selective_keypoints_idx_list): 83 | self.keypoints3d_delta_list[idx] = delta[id] 84 | 85 | 86 | def set_rotation_delta(self, rot_mat): 87 | kpts3d = self.get_deformed_kpt_np()[self.selective_keypoints_idx_list] 88 | kpts3d_mean = self.get_rotation_center() 89 | kpts3d = (kpts3d - kpts3d_mean) @ rot_mat.T + kpts3d_mean 90 | delta = kpts3d - np.array(self.keypoints3d_list)[self.selective_keypoints_idx_list] 91 | for id, idx in enumerate(self.selective_keypoints_idx_list): 92 | self.keypoints3d_delta_list[idx] = delta[id] 93 | -------------------------------------------------------------------------------- /utils/logger_hook.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from mmengine.hooks import LoggerHook 3 | 4 | 5 | def setup_logger(log_file='default.log'): 6 | logger = logging.getLogger('mmdet_logger') 7 | logger.setLevel(logging.INFO) 8 | 9 | # Create file handler 10 | fh = logging.FileHandler(log_file) 11 | fh.setLevel(logging.INFO) 12 | 13 | # Create formatter and add it to the handlers 14 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 15 | fh.setFormatter(formatter) 16 | 17 | # Add the handlers to the logger 18 | logger.addHandler(fh) 19 | 20 | # Initialize LoggerHook 21 | logger_hook = LoggerHook(out_dir=log_file) 22 | return logger_hook 23 | 24 | 25 | # 使用示例 26 | # logger_hook = setup_logger('my_log.log') 27 | # logger_hook.log('This is a log message.') 28 | -------------------------------------------------------------------------------- /utils/other_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | 5 | def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: 6 | """ 7 | Returns torch.sqrt(torch.max(0, x)) 8 | but with a zero subgradient where x is 0. 9 | """ 10 | ret = torch.zeros_like(x) 11 | positive_mask = x > 0 12 | ret[positive_mask] = torch.sqrt(x[positive_mask]) 13 | return ret 14 | 15 | 16 | def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor: 17 | """ 18 | Convert rotations given as rotation matrices to quaternions. 19 | 20 | Args: 21 | matrix: Rotation matrices as tensor of shape (..., 3, 3). 22 | 23 | Returns: 24 | quaternions with real part first, as tensor of shape (..., 4). 25 | """ 26 | if matrix.size(-1) != 3 or matrix.size(-2) != 3: 27 | raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") 28 | 29 | batch_dim = matrix.shape[:-2] 30 | m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( 31 | matrix.reshape(batch_dim + (9,)), dim=-1 32 | ) 33 | 34 | q_abs = _sqrt_positive_part( 35 | torch.stack( 36 | [ 37 | 1.0 + m00 + m11 + m22, 38 | 1.0 + m00 - m11 - m22, 39 | 1.0 - m00 + m11 - m22, 40 | 1.0 - m00 - m11 + m22, 41 | ], 42 | dim=-1, 43 | ) 44 | ) 45 | 46 | # we produce the desired quaternion multiplied by each of r, i, j, k 47 | quat_by_rijk = torch.stack( 48 | [ 49 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 50 | # `int`. 51 | torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), 52 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 53 | # `int`. 54 | torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), 55 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 56 | # `int`. 57 | torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), 58 | # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and 59 | # `int`. 60 | torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), 61 | ], 62 | dim=-2, 63 | ) 64 | 65 | # We floor here at 0.1 but the exact level is not important; if q_abs is small, 66 | # the candidate won't be picked. 67 | flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) 68 | quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) 69 | 70 | # if not for numerical problems, quat_candidates[i] should be same (up to a sign), 71 | # forall i; we pick the best-conditioned one (with the largest denominator) 72 | 73 | return quat_candidates[ 74 | torch.nn.functional.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, : 75 | ].reshape(batch_dim + (4,)) 76 | 77 | 78 | def depth2normal(depth:torch.Tensor, focal:float=None): 79 | if depth.dim() == 2: 80 | depth = depth[None, None] 81 | elif depth.dim() == 3: 82 | depth = depth.squeeze()[None, None] 83 | if focal is None: 84 | focal = depth.shape[-1] / 2 / np.tan(torch.pi/6) 85 | depth = torch.cat([depth[:, :, :1], depth, depth[:, :, -1:]], dim=2) 86 | depth = torch.cat([depth[..., :1], depth, depth[..., -1:]], dim=3) 87 | kernel = torch.tensor([[[ 0, 0, 0], 88 | [-.5, 0, .5], 89 | [ 0, 0, 0]], 90 | [[ 0, -.5, 0], 91 | [ 0, 0, 0], 92 | [ 0, .5, 0]]], device=depth.device, dtype=depth.dtype)[:, None] 93 | normal = torch.nn.functional.conv2d(depth, kernel, padding='valid')[0].permute(1, 2, 0) 94 | normal = normal / (depth[0, 0, 1:-1, 1:-1, None] + 1e-10) * focal 95 | normal = torch.cat([normal, torch.ones_like(normal[..., :1])], dim=-1) 96 | normal = normal / normal.norm(dim=-1, keepdim=True) 97 | return normal.permute(2, 0, 1) 98 | -------------------------------------------------------------------------------- /utils/pickle_utils.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | 4 | def save_obj(path, obj): 5 | file = open(path, 'wb') 6 | obj_str = pickle.dumps(obj) 7 | file.write(obj_str) 8 | file.close() 9 | 10 | 11 | def load_obj(path): 12 | file = open(path, 'rb') 13 | obj = pickle.loads(file.read()) 14 | file.close() 15 | return obj 16 | -------------------------------------------------------------------------------- /utils/pose_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from utils.graphics_utils import fov2focal 4 | 5 | trans_t = lambda t: torch.Tensor([ 6 | [1, 0, 0, 0], 7 | [0, 1, 0, 0], 8 | [0, 0, 1, t], 9 | [0, 0, 0, 1]]).float() 10 | 11 | rot_phi = lambda phi: torch.Tensor([ 12 | [1, 0, 0, 0], 13 | [0, np.cos(phi), -np.sin(phi), 0], 14 | [0, np.sin(phi), np.cos(phi), 0], 15 | [0, 0, 0, 1]]).float() 16 | 17 | rot_theta = lambda th: torch.Tensor([ 18 | [np.cos(th), 0, -np.sin(th), 0], 19 | [0, 1, 0, 0], 20 | [np.sin(th), 0, np.cos(th), 0], 21 | [0, 0, 0, 1]]).float() 22 | 23 | 24 | def rodrigues_mat_to_rot(R): 25 | eps = 1e-16 26 | trc = np.trace(R) 27 | trc2 = (trc - 1.) / 2. 28 | # sinacostrc2 = np.sqrt(1 - trc2 * trc2) 29 | s = np.array([R[2, 1] - R[1, 2], R[0, 2] - R[2, 0], R[1, 0] - R[0, 1]]) 30 | if (1 - trc2 * trc2) >= eps: 31 | tHeta = np.arccos(trc2) 32 | tHetaf = tHeta / (2 * (np.sin(tHeta))) 33 | else: 34 | tHeta = np.real(np.arccos(trc2)) 35 | tHetaf = 0.5 / (1 - tHeta / 6) 36 | omega = tHetaf * s 37 | return omega 38 | 39 | 40 | def rodrigues_rot_to_mat(r): 41 | wx, wy, wz = r 42 | theta = np.sqrt(wx * wx + wy * wy + wz * wz) 43 | a = np.cos(theta) 44 | b = (1 - np.cos(theta)) / (theta * theta) 45 | c = np.sin(theta) / theta 46 | R = np.zeros([3, 3]) 47 | R[0, 0] = a + b * (wx * wx) 48 | R[0, 1] = b * wx * wy - c * wz 49 | R[0, 2] = b * wx * wz + c * wy 50 | R[1, 0] = b * wx * wy + c * wz 51 | R[1, 1] = a + b * (wy * wy) 52 | R[1, 2] = b * wy * wz - c * wx 53 | R[2, 0] = b * wx * wz - c * wy 54 | R[2, 1] = b * wz * wy + c * wx 55 | R[2, 2] = a + b * (wz * wz) 56 | return R 57 | 58 | 59 | def normalize(x): 60 | return x / np.linalg.norm(x) 61 | 62 | 63 | def pose_spherical(theta, phi, radius): 64 | c2w = trans_t(radius) 65 | c2w = rot_phi(phi / 180. * np.pi) @ c2w 66 | c2w = rot_theta(theta / 180. * np.pi) @ c2w 67 | c2w = torch.Tensor(np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]])) @ c2w 68 | return c2w 69 | 70 | def viewmatrix(z, up, pos): 71 | vec2 = normalize(z) 72 | vec1_avg = up 73 | vec0 = normalize(np.cross(vec1_avg, vec2)) 74 | vec1 = normalize(np.cross(vec2, vec0)) 75 | m = np.stack([vec0, vec1, vec2, pos], 1) 76 | return m 77 | 78 | def poses_avg(poses): 79 | center = poses[:, :3, 3].mean(0) 80 | vec2 = normalize(poses[:, :3, 2].sum(0)) 81 | up = poses[:, :3, 1].sum(0) 82 | c2w = viewmatrix(vec2, up, center) 83 | return c2w 84 | 85 | def render_path_spiral(c2ws, focal, zrate=.1, rots=3, N=300): 86 | c2w = poses_avg(c2ws) 87 | up = normalize(c2ws[:, :3, 1].sum(0)) 88 | tt = c2ws[:,:3,3] 89 | rads = np.percentile(np.abs(tt), 90, 0) 90 | rads[:] = rads.max() * .05 91 | 92 | render_poses = [] 93 | rads = np.array(list(rads) + [1.]) 94 | for theta in np.linspace(0., 2. * np.pi * rots, N+1)[:-1]: 95 | c = np.dot(c2w[:3,:4], np.array([np.cos(theta), -np.sin(theta), -np.sin(theta*zrate), 1.]) * rads) 96 | z = normalize(c - np.dot(c2w[:3,:4], np.array([0,0,-focal, 1.]))) 97 | # c = np.dot(c2w[:3,:4], np.array([np.cos(theta), -np.sin(theta), -np.sin(theta*zrate), 1.]) * rads) 98 | # z = normalize(c2w[:3, 2]) 99 | render_poses.append(viewmatrix(z, up, c)) 100 | render_poses = np.stack(render_poses, axis=0) 101 | render_poses = np.concatenate([render_poses, np.zeros_like(render_poses[..., :1, :])], axis=1) 102 | render_poses[..., 3, 3] = 1 103 | render_poses = np.array(render_poses, dtype=np.float32) 104 | return render_poses 105 | 106 | def render_wander_path(view): 107 | focal_length = fov2focal(view.FoVy, view.image_height) 108 | R = view.R 109 | R[:, 1] = -R[:, 1] 110 | R[:, 2] = -R[:, 2] 111 | T = -view.T.reshape(-1, 1) 112 | pose = np.concatenate([R, T], -1) 113 | 114 | num_frames = 60 115 | max_disp = 5000.0 # 64 , 48 116 | 117 | max_trans = max_disp / focal_length # Maximum camera translation to satisfy max_disp parameter 118 | output_poses = [] 119 | 120 | for i in range(num_frames): 121 | x_trans = max_trans * np.sin(2.0 * np.pi * float(i) / float(num_frames)) 122 | y_trans = max_trans * np.cos(2.0 * np.pi * float(i) / float(num_frames)) / 3.0 # * 3.0 / 4.0 123 | z_trans = max_trans * np.cos(2.0 * np.pi * float(i) / float(num_frames)) / 3.0 124 | 125 | i_pose = np.concatenate([ 126 | np.concatenate( 127 | [np.eye(3), np.array([x_trans, y_trans, z_trans])[:, np.newaxis]], axis=1), 128 | np.array([0.0, 0.0, 0.0, 1.0])[np.newaxis, :] 129 | ], axis=0) # [np.newaxis, :, :] 130 | 131 | i_pose = np.linalg.inv(i_pose) # torch.tensor(np.linalg.inv(i_pose)).float() 132 | 133 | ref_pose = np.concatenate([pose, np.array([0.0, 0.0, 0.0, 1.0])[np.newaxis, :]], axis=0) 134 | 135 | render_pose = np.dot(ref_pose, i_pose) 136 | output_poses.append(torch.Tensor(render_pose)) 137 | 138 | return output_poses 139 | -------------------------------------------------------------------------------- /utils/preprocess.py: -------------------------------------------------------------------------------- 1 | # @title Configure dataset directories 2 | import os 3 | from pathlib import Path 4 | 5 | # @markdown The base directory for all captures. This can be anything if you're running this notebook on your own Jupyter runtime. 6 | save_dir = '/data00/yzy/Git_Project/data/dynamic/mine/' # @param {type: 'string'} 7 | capture_name = 'lemon' # @param {type: 'string'} 8 | # The root directory for this capture. 9 | root_dir = Path(save_dir, capture_name) 10 | # Where to save RGB images. 11 | rgb_dir = root_dir / 'rgb' 12 | rgb_raw_dir = root_dir / 'rgb-raw' 13 | # Where to save the COLMAP outputs. 14 | colmap_dir = root_dir / 'colmap' 15 | colmap_db_path = colmap_dir / 'database.db' 16 | colmap_out_path = colmap_dir / 'sparse' 17 | 18 | colmap_out_path.mkdir(exist_ok=True, parents=True) 19 | rgb_raw_dir.mkdir(exist_ok=True, parents=True) 20 | 21 | print(f"""Directories configured: 22 | root_dir = {root_dir} 23 | rgb_raw_dir = {rgb_raw_dir} 24 | rgb_dir = {rgb_dir} 25 | colmap_dir = {colmap_dir} 26 | """) 27 | 28 | # ==================== colmap ========================= 29 | # @title Extract features. 30 | # @markdown Computes SIFT features and saves them to the COLMAP DB. 31 | share_intrinsics = True # @param {type: 'boolean'} 32 | assume_upright_cameras = True # @param {type: 'boolean'} 33 | 34 | # @markdown This sets the scale at which we will run COLMAP. A scale of 1 will be more accurate but will be slow. 35 | colmap_image_scale = 4 # @param {type: 'number'} 36 | colmap_rgb_dir = rgb_dir / f'{colmap_image_scale}x' 37 | 38 | # @markdown Check this if you want to re-process SfM. 39 | overwrite = False # @param {type: 'boolean'} 40 | 41 | if overwrite and colmap_db_path.exists(): 42 | colmap_db_path.unlink() 43 | 44 | os.system('colmap feature_extractor \ 45 | --SiftExtraction.use_gpu 0 \ 46 | --SiftExtraction.upright {int(assume_upright_cameras)} \ 47 | --ImageReader.camera_model OPENCV \ 48 | --ImageReader.single_camera {int(share_intrinsics)} \ 49 | --database_path "{str(colmap_db_path)}" \ 50 | --image_path "{str(colmap_rgb_dir)}"') 51 | 52 | # @title Match features. 53 | # @markdown Match the SIFT features between images. Use `exhaustive` if you only have a few images and use `vocab_tree` if you have a lot of images. 54 | 55 | match_method = 'exhaustive' # @param ["exhaustive", "vocab_tree"] 56 | 57 | if match_method == 'exhaustive': 58 | os.system('colmap exhaustive_matcher \ 59 | --SiftMatching.use_gpu 0 \ 60 | --database_path "{str(colmap_db_path)}"') 61 | 62 | # @title Reconstruction. 63 | # @markdown Run structure-from-motion to compute camera parameters. 64 | 65 | refine_principal_point = True # @param {type:"boolean"} 66 | min_num_matches = 32 # @param {type: 'number'} 67 | filter_max_reproj_error = 2 # @param {type: 'number'} 68 | tri_complete_max_reproj_error = 2 # @param {type: 'number'} 69 | 70 | os.system('colmap mapper \ 71 | --Mapper.ba_refine_principal_point {int(refine_principal_point)} \ 72 | --Mapper.filter_max_reproj_error $filter_max_reproj_error \ 73 | --Mapper.tri_complete_max_reproj_error $tri_complete_max_reproj_error \ 74 | --Mapper.min_num_matches $min_num_matches \ 75 | --database_path "{str(colmap_db_path)}" \ 76 | --image_path "{str(colmap_rgb_dir)}" \ 77 | --export_path "{str(colmap_out_path)}"') 78 | 79 | print("debug") 80 | -------------------------------------------------------------------------------- /utils/rigid_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def skew(w: torch.Tensor) -> torch.Tensor: 5 | """Build a skew matrix ("cross product matrix") for vector w. 6 | 7 | Modern Robotics Eqn 3.30. 8 | 9 | Args: 10 | w: (N, 3) A 3-vector 11 | 12 | Returns: 13 | W: (N, 3, 3) A skew matrix such that W @ v == w x v 14 | """ 15 | zeros = torch.zeros(w.shape[0], device=w.device) 16 | w_skew_list = [zeros, -w[:, 2], w[:, 1], 17 | w[:, 2], zeros, -w[:, 0], 18 | -w[:, 1], w[:, 0], zeros] 19 | w_skew = torch.stack(w_skew_list, dim=-1).reshape(-1, 3, 3) 20 | return w_skew 21 | 22 | 23 | def rp_to_se3(R: torch.Tensor, p: torch.Tensor) -> torch.Tensor: 24 | """Rotation and translation to homogeneous transform. 25 | 26 | Args: 27 | R: (3, 3) An orthonormal rotation matrix. 28 | p: (3,) A 3-vector representing an offset. 29 | 30 | Returns: 31 | X: (4, 4) The homogeneous transformation matrix described by rotating by R 32 | and translating by p. 33 | """ 34 | bottom_row = torch.tensor([[0.0, 0.0, 0.0, 1.0]], device=R.device).repeat(R.shape[0], 1, 1) 35 | transform = torch.cat([torch.cat([R, p], dim=-1), bottom_row], dim=1) 36 | 37 | return transform 38 | 39 | 40 | def exp_so3(w: torch.Tensor, theta: float) -> torch.Tensor: 41 | """Exponential map from Lie algebra so3 to Lie group SO3. 42 | 43 | Modern Robotics Eqn 3.51, a.k.a. Rodrigues' formula. 44 | 45 | Args: 46 | w: (3,) An axis of rotation. 47 | theta: An angle of rotation. 48 | 49 | Returns: 50 | R: (3, 3) An orthonormal rotation matrix representing a rotation of 51 | magnitude theta about axis w. 52 | """ 53 | W = skew(w) 54 | identity = torch.eye(3).unsqueeze(0).repeat(W.shape[0], 1, 1).to(W.device) 55 | W_sqr = torch.bmm(W, W) # batch matrix multiplication 56 | R = identity + torch.sin(theta.unsqueeze(-1)) * W + (1.0 - torch.cos(theta.unsqueeze(-1))) * W_sqr 57 | return R 58 | 59 | 60 | def exp_se3(S: torch.Tensor, theta: float) -> torch.Tensor: 61 | """Exponential map from Lie algebra so3 to Lie group SO3. 62 | 63 | Modern Robotics Eqn 3.88. 64 | 65 | Args: 66 | S: (6,) A screw axis of motion. 67 | theta: Magnitude of motion. 68 | 69 | Returns: 70 | a_X_b: (4, 4) The homogeneous transformation matrix attained by integrating 71 | motion of magnitude theta about S for one second. 72 | """ 73 | w, v = torch.split(S, 3, dim=-1) 74 | W = skew(w) 75 | R = exp_so3(w, theta) 76 | 77 | identity = torch.eye(3).unsqueeze(0).repeat(W.shape[0], 1, 1).to(W.device) 78 | W_sqr = torch.bmm(W, W) 79 | theta = theta.view(-1, 1, 1) 80 | 81 | p = torch.bmm((theta * identity + (1.0 - torch.cos(theta)) * W + (theta - torch.sin(theta)) * W_sqr), 82 | v.unsqueeze(-1)) 83 | return rp_to_se3(R, p) 84 | 85 | 86 | def to_homogenous(v: torch.Tensor) -> torch.Tensor: 87 | """Converts a vector to a homogeneous coordinate vector by appending a 1. 88 | 89 | Args: 90 | v: A tensor representing a vector or batch of vectors. 91 | 92 | Returns: 93 | A tensor with an additional dimension set to 1. 94 | """ 95 | return torch.cat([v, torch.ones_like(v[..., :1])], dim=-1) 96 | 97 | 98 | def from_homogenous(v: torch.Tensor) -> torch.Tensor: 99 | """Converts a homogeneous coordinate vector to a standard vector by dividing by the last element. 100 | 101 | Args: 102 | v: A tensor representing a homogeneous coordinate vector or batch of homogeneous coordinate vectors. 103 | 104 | Returns: 105 | A tensor with the last dimension removed. 106 | """ 107 | return v[..., :3] / v[..., -1:] 108 | -------------------------------------------------------------------------------- /utils/sh_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The PlenOctree Authors. 2 | # Redistribution and use in source and binary forms, with or without 3 | # modification, are permitted provided that the following conditions are met: 4 | # 5 | # 1. Redistributions of source code must retain the above copyright notice, 6 | # this list of conditions and the following disclaimer. 7 | # 8 | # 2. Redistributions in binary form must reproduce the above copyright notice, 9 | # this list of conditions and the following disclaimer in the documentation 10 | # and/or other materials provided with the distribution. 11 | # 12 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 13 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 14 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 15 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 16 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 17 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 18 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 19 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 20 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 21 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 22 | # POSSIBILITY OF SUCH DAMAGE. 23 | 24 | import torch 25 | 26 | C0 = 0.28209479177387814 27 | C1 = 0.4886025119029199 28 | C2 = [ 29 | 1.0925484305920792, 30 | -1.0925484305920792, 31 | 0.31539156525252005, 32 | -1.0925484305920792, 33 | 0.5462742152960396 34 | ] 35 | C3 = [ 36 | -0.5900435899266435, 37 | 2.890611442640554, 38 | -0.4570457994644658, 39 | 0.3731763325901154, 40 | -0.4570457994644658, 41 | 1.445305721320277, 42 | -0.5900435899266435 43 | ] 44 | C4 = [ 45 | 2.5033429417967046, 46 | -1.7701307697799304, 47 | 0.9461746957575601, 48 | -0.6690465435572892, 49 | 0.10578554691520431, 50 | -0.6690465435572892, 51 | 0.47308734787878004, 52 | -1.7701307697799304, 53 | 0.6258357354491761, 54 | ] 55 | 56 | 57 | def eval_sh(deg, sh, dirs): 58 | """ 59 | Evaluate spherical harmonics at unit directions 60 | using hardcoded SH polynomials. 61 | Works with torch/np/jnp. 62 | ... Can be 0 or more batch dimensions. 63 | Args: 64 | deg: int SH deg. Currently, 0-3 supported 65 | sh: jnp.ndarray SH coeffs [..., C, (deg + 1) ** 2] 66 | dirs: jnp.ndarray unit directions [..., 3] 67 | Returns: 68 | [..., C] 69 | """ 70 | assert deg <= 4 and deg >= 0 71 | coeff = (deg + 1) ** 2 72 | assert sh.shape[-1] >= coeff 73 | 74 | result = C0 * sh[..., 0] 75 | if deg > 0: 76 | x, y, z = dirs[..., 0:1], dirs[..., 1:2], dirs[..., 2:3] 77 | result = (result - 78 | C1 * y * sh[..., 1] + 79 | C1 * z * sh[..., 2] - 80 | C1 * x * sh[..., 3]) 81 | 82 | if deg > 1: 83 | xx, yy, zz = x * x, y * y, z * z 84 | xy, yz, xz = x * y, y * z, x * z 85 | result = (result + 86 | C2[0] * xy * sh[..., 4] + 87 | C2[1] * yz * sh[..., 5] + 88 | C2[2] * (2.0 * zz - xx - yy) * sh[..., 6] + 89 | C2[3] * xz * sh[..., 7] + 90 | C2[4] * (xx - yy) * sh[..., 8]) 91 | 92 | if deg > 2: 93 | result = (result + 94 | C3[0] * y * (3 * xx - yy) * sh[..., 9] + 95 | C3[1] * xy * z * sh[..., 10] + 96 | C3[2] * y * (4 * zz - xx - yy) * sh[..., 11] + 97 | C3[3] * z * (2 * zz - 3 * xx - 3 * yy) * sh[..., 12] + 98 | C3[4] * x * (4 * zz - xx - yy) * sh[..., 13] + 99 | C3[5] * z * (xx - yy) * sh[..., 14] + 100 | C3[6] * x * (xx - 3 * yy) * sh[..., 15]) 101 | 102 | if deg > 3: 103 | result = (result + C4[0] * xy * (xx - yy) * sh[..., 16] + 104 | C4[1] * yz * (3 * xx - yy) * sh[..., 17] + 105 | C4[2] * xy * (7 * zz - 1) * sh[..., 18] + 106 | C4[3] * yz * (7 * zz - 3) * sh[..., 19] + 107 | C4[4] * (zz * (35 * zz - 30) + 3) * sh[..., 20] + 108 | C4[5] * xz * (7 * zz - 3) * sh[..., 21] + 109 | C4[6] * (xx - yy) * (7 * zz - 1) * sh[..., 22] + 110 | C4[7] * xz * (xx - 3 * yy) * sh[..., 23] + 111 | C4[8] * (xx * (xx - 3 * yy) - yy * (3 * xx - yy)) * sh[..., 24]) 112 | return result 113 | 114 | 115 | def RGB2SH(rgb): 116 | return (rgb - 0.5) / C0 117 | 118 | 119 | def SH2RGB(sh): 120 | return sh * C0 + 0.5 121 | -------------------------------------------------------------------------------- /utils/system_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from errno import EEXIST 13 | from os import makedirs, path 14 | import os 15 | 16 | 17 | def mkdir_p(folder_path): 18 | # Creates a directory. equivalent to using mkdir -p on the command line 19 | try: 20 | makedirs(folder_path) 21 | except OSError as exc: # Python >2.5 22 | if exc.errno == EEXIST and path.isdir(folder_path): 23 | pass 24 | else: 25 | raise 26 | 27 | 28 | def searchForMaxIteration(folder): 29 | if not os.path.exists(folder): 30 | return None 31 | saved_iters = [int(fname.split("_")[-1]) for fname in os.listdir(folder) if "_" in fname] 32 | return max(saved_iters) if saved_iters != [] else None 33 | -------------------------------------------------------------------------------- /utils/vis_utils.py: -------------------------------------------------------------------------------- 1 | from gaussian_renderer import render 2 | 3 | 4 | def render_cur_cam(self, cur_cam): 5 | fid = cur_cam.fid 6 | if self.deform.name == 'node': 7 | if 'Node' in self.visualization_mode: 8 | gaussians = self.deform.deform.as_gaussians # if self.iteration_node_rendering < self.opt.iterations_node_rendering else self.deform.deform.as_gaussians_visualization 9 | time_input = fid.unsqueeze(0).expand(gaussians.get_xyz.shape[0], -1) 10 | d_values = self.deform.deform.query_network(x=gaussians.get_xyz.detach(), t=time_input) 11 | if self.motion_animation_d_values is not None: 12 | for key in self.motion_animation_d_values: 13 | d_values[key] = self.motion_animation_d_values[key] 14 | d_xyz, d_opacity, d_color = d_values['d_xyz'] * gaussians.motion_mask, d_values['d_opacity'] * gaussians.motion_mask if d_values['d_opacity'] is not None else None, d_values['d_color'] * gaussians.motion_mask if d_values['d_color'] is not None else None 15 | d_rotation, d_scaling = 0., 0. 16 | if self.animation_trans_bias is not None: 17 | d_xyz = d_xyz + self.animation_trans_bias 18 | gs_rot_bias = None 19 | vis_scale_const = self.vis_scale_const 20 | else: 21 | time_input = self.deform.deform.expand_time(fid) 22 | d_values = self.deform.step(self.gaussians.get_xyz.detach(), time_input, feature=self.gaussians.feature, is_training=False, node_trans_bias=self.animation_trans_bias, node_rot_bias=self.animation_rot_bias, motion_mask=self.gaussians.motion_mask, camera_center=cur_cam.camera_center, animation_d_values=self.motion_animation_d_values) 23 | gaussians = self.gaussians 24 | d_xyz, d_rotation, d_scaling, d_opacity, d_color = d_values['d_xyz'], d_values['d_rotation'], d_values['d_scaling'], d_values['d_opacity'], d_values['d_color'] 25 | gs_rot_bias = d_values['gs_rot_bias'] # GS rotation bias 26 | vis_scale_const = None 27 | else: 28 | vis_scale_const = None 29 | if self.iteration < self.opt.warm_up: 30 | d_xyz, d_rotation, d_scaling, d_opacity, d_color = 0.0, 0.0, 0.0, 0.0, 0.0 31 | gaussians = self.gaussians 32 | else: 33 | N = self.gaussians.get_xyz.shape[0] 34 | time_input = fid.unsqueeze(0).expand(N, -1) 35 | gaussians = self.gaussians 36 | d_values = self.deform.step(self.gaussians.get_xyz.detach(), time_input, feature=self.gaussians.feature, camera_center=cur_cam.camera_center) 37 | d_xyz, d_rotation, d_scaling, d_opacity, d_color = d_values['d_xyz'], d_values['d_rotation'], d_values['d_scaling'], d_values['d_opacity'], d_values['d_color'] 38 | gs_rot_bias = None 39 | 40 | render_motion = "Motion" in self.visualization_mode 41 | if render_motion: 42 | vis_scale_const = self.vis_scale_const 43 | if type(d_rotation) is not float and gaussians._rotation.shape[0] != d_rotation.shape[0]: 44 | d_xyz, d_rotation, d_scaling = 0, 0, 0 45 | print('Async in Gaussian Switching') 46 | out = render(viewpoint_camera=cur_cam, pc=gaussians, pipe=self.pipe, bg_color=self.background, d_xyz=d_xyz, d_rotation=d_rotation, d_scaling=d_scaling, render_motion=render_motion, d_opacity=d_opacity, d_color=d_color, d_rot_as_res=self.deform.d_rot_as_res, gs_rot_bias=gs_rot_bias, scale_const=vis_scale_const) 47 | 48 | buffer_image = out[self.mode] # [3, H, W] 49 | return buffer_image 50 | --------------------------------------------------------------------------------