├── assets ├── cmu-logo.jpg ├── pku-logo.jpg ├── huawei-logo.jpg └── tugraz-logo.jpg ├── .gitignore ├── environment_cuda12.yml ├── environment.yml ├── .gitmodules ├── configs ├── kbuffer.json ├── vanilla.json └── hierarchical.json ├── utils ├── image_utils.py ├── system_utils.py ├── graphics_utils.py ├── loss_utils.py ├── camera_utils.py ├── general_utils.py └── sh_utils.py ├── lpipsPyTorch ├── __init__.py └── modules │ ├── utils.py │ ├── lpips.py │ └── networks.py ├── gaussian_renderer ├── network_gui.py └── __init__.py ├── scene ├── cameras.py ├── __init__.py ├── dataset_readers.py ├── colmap_loader.py └── gaussian_model.py ├── full_eval.py ├── render.py ├── LICENSE.md ├── metrics.py ├── convert.py ├── README.md ├── arguments └── __init__.py └── train.py /assets/cmu-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cekavis/VRSplat/HEAD/assets/cmu-logo.jpg -------------------------------------------------------------------------------- /assets/pku-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cekavis/VRSplat/HEAD/assets/pku-logo.jpg -------------------------------------------------------------------------------- /assets/huawei-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cekavis/VRSplat/HEAD/assets/huawei-logo.jpg -------------------------------------------------------------------------------- /assets/tugraz-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cekavis/VRSplat/HEAD/assets/tugraz-logo.jpg -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .vscode 3 | output 4 | build 5 | diff_rasterization/diff_rast.egg-info 6 | diff_rasterization/dist 7 | tensorboard_3d 8 | screenshots 9 | __pycache__ 10 | -------------------------------------------------------------------------------- /environment_cuda12.yml: -------------------------------------------------------------------------------- 1 | name: vrsplat 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | - nvidia 7 | dependencies: 8 | - plyfile 9 | - python=3.10 10 | - pip 11 | - tqdm 12 | - dacite 13 | - pytorch>=2.1 14 | - pytorch-cuda=12.1 15 | - torchvision 16 | - pip: 17 | - submodules/simple-knn 18 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: vrsplat 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - cudatoolkit=11.6 8 | - plyfile 9 | - python=3.7.13 10 | - pip=22.3.1 11 | - pytorch=1.12.1 12 | - torchaudio=0.12.1 13 | - torchvision=0.13.1 14 | - tqdm 15 | - dacite 16 | - pip: 17 | - submodules/simple-knn 18 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "submodules/simple-knn"] 2 | path = submodules/simple-knn 3 | url = https://gitlab.inria.fr/bkerbl/simple-knn.git 4 | [submodule "SIBR_viewers"] 5 | path = SIBR_viewers 6 | url = https://github.com/Cekavis/SIBR_VRSplat 7 | [submodule "submodules/diff-gaussian-rasterization"] 8 | path = submodules/diff-gaussian-rasterization 9 | url = https://github.com/Cekavis/VRSplat-rasterization 10 | -------------------------------------------------------------------------------- /configs/kbuffer.json: -------------------------------------------------------------------------------- 1 | { 2 | "culling_settings": { 3 | "hierarchical_4x4_culling": false, 4 | "rect_bounding": true, 5 | "tight_opacity_bounding": true, 6 | "tile_based_culling": false 7 | }, 8 | "load_balancing": true, 9 | "proper_ewa_scaling": false, 10 | "sort_settings": { 11 | "queue_sizes": { 12 | "per_pixel": 16, 13 | "tile_2x2": 0, 14 | "tile_4x4": 0 15 | }, 16 | "sort_mode": 2, 17 | "sort_order": 3 18 | } 19 | } -------------------------------------------------------------------------------- /configs/vanilla.json: -------------------------------------------------------------------------------- 1 | { 2 | "culling_settings": { 3 | "hierarchical_4x4_culling": false, 4 | "rect_bounding": true, 5 | "tight_opacity_bounding": true, 6 | "tile_based_culling": false 7 | }, 8 | "load_balancing": true, 9 | "proper_ewa_scaling": false, 10 | "sort_settings": { 11 | "queue_sizes": { 12 | "per_pixel": 4, 13 | "tile_2x2": 8, 14 | "tile_4x4": 64 15 | }, 16 | "sort_mode": 0, 17 | "sort_order": 0 18 | } 19 | } -------------------------------------------------------------------------------- /configs/hierarchical.json: -------------------------------------------------------------------------------- 1 | { 2 | "culling_settings": { 3 | "hierarchical_4x4_culling": true, 4 | "rect_bounding": true, 5 | "tight_opacity_bounding": true, 6 | "tile_based_culling": false 7 | }, 8 | "load_balancing": true, 9 | "proper_ewa_scaling": false, 10 | "sort_settings": { 11 | "queue_sizes": { 12 | "per_pixel": 4, 13 | "tile_2x2": 8, 14 | "tile_4x4": 64 15 | }, 16 | "sort_mode": 3, 17 | "sort_order": 3 18 | } 19 | } -------------------------------------------------------------------------------- /utils/image_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | 14 | def mse(img1, img2): 15 | return (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True) 16 | 17 | def psnr(img1, img2): 18 | mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True) 19 | return 20 * torch.log10(1.0 / torch.sqrt(mse)) 20 | -------------------------------------------------------------------------------- /lpipsPyTorch/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .modules.lpips import LPIPS 4 | 5 | class LPIPSEval: 6 | r"""Class for measuring lpips between two images 7 | 8 | Arguments: 9 | net_type (str): the network type to compare the features: 10 | 'alex' | 'squeeze' | 'vgg'. Default: 'alex'. 11 | version (str): the version of LPIPS. Default: 0.1. 12 | """ 13 | criterion = None 14 | 15 | def __init__(self, device, net_type: str = 'alex', version: str = '0.1'): 16 | self.criterion = LPIPS(net_type, version).to(device) 17 | 18 | r"""Compute lpips for two images 19 | 20 | Arguments: 21 | x, y (torch.Tensor): the input tensors to compare. 22 | """ 23 | def eval(self, x: torch.Tensor, y: torch.Tensor): 24 | return self.criterion(x, y) -------------------------------------------------------------------------------- /utils/system_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from errno import EEXIST 13 | from os import makedirs, path 14 | import os 15 | 16 | def mkdir_p(folder_path): 17 | # Creates a directory. equivalent to using mkdir -p on the command line 18 | try: 19 | makedirs(folder_path) 20 | except OSError as exc: # Python >2.5 21 | if exc.errno == EEXIST and path.isdir(folder_path): 22 | pass 23 | else: 24 | raise 25 | 26 | def searchForMaxIteration(folder): 27 | saved_iters = [int(fname.split("_")[-1]) for fname in os.listdir(folder)] 28 | return max(saved_iters) 29 | -------------------------------------------------------------------------------- /lpipsPyTorch/modules/utils.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import torch 4 | 5 | 6 | def normalize_activation(x, eps=1e-10): 7 | norm_factor = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) 8 | return x / (norm_factor + eps) 9 | 10 | 11 | def get_state_dict(net_type: str = 'alex', version: str = '0.1'): 12 | # build url 13 | url = 'https://raw.githubusercontent.com/richzhang/PerceptualSimilarity/' \ 14 | + f'master/lpips/weights/v{version}/{net_type}.pth' 15 | 16 | # download 17 | old_state_dict = torch.hub.load_state_dict_from_url( 18 | url, progress=True, 19 | map_location=None if torch.cuda.is_available() else torch.device('cpu') 20 | ) 21 | 22 | # rename keys 23 | new_state_dict = OrderedDict() 24 | for key, val in old_state_dict.items(): 25 | new_key = key 26 | new_key = new_key.replace('lin', '') 27 | new_key = new_key.replace('model.', '') 28 | new_state_dict[new_key] = val 29 | 30 | return new_state_dict 31 | -------------------------------------------------------------------------------- /lpipsPyTorch/modules/lpips.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from .networks import get_network, LinLayers 5 | from .utils import get_state_dict 6 | 7 | 8 | class LPIPS(nn.Module): 9 | r"""Creates a criterion that measures 10 | Learned Perceptual Image Patch Similarity (LPIPS). 11 | 12 | Arguments: 13 | net_type (str): the network type to compare the features: 14 | 'alex' | 'squeeze' | 'vgg'. Default: 'alex'. 15 | version (str): the version of LPIPS. Default: 0.1. 16 | """ 17 | def __init__(self, net_type: str = 'alex', version: str = '0.1'): 18 | 19 | assert version in ['0.1'], 'v0.1 is only supported now' 20 | 21 | super(LPIPS, self).__init__() 22 | 23 | # pretrained network 24 | self.net = get_network(net_type) 25 | 26 | # linear layers 27 | self.lin = LinLayers(self.net.n_channels_list) 28 | self.lin.load_state_dict(get_state_dict(net_type, version)) 29 | 30 | def forward(self, x: torch.Tensor, y: torch.Tensor): 31 | feat_x, feat_y = self.net(x), self.net(y) 32 | 33 | diff = [(fx - fy) ** 2 for fx, fy in zip(feat_x, feat_y)] 34 | res = [l(d).mean((2, 3), True) for d, l in zip(diff, self.lin)] 35 | 36 | return torch.sum(torch.cat(res, 0), 0, True) 37 | -------------------------------------------------------------------------------- /utils/graphics_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import math 14 | import numpy as np 15 | from typing import NamedTuple 16 | 17 | class BasicPointCloud(NamedTuple): 18 | points : np.array 19 | colors : np.array 20 | normals : np.array 21 | 22 | def geom_transform_points(points, transf_matrix): 23 | P, _ = points.shape 24 | ones = torch.ones(P, 1, dtype=points.dtype, device=points.device) 25 | points_hom = torch.cat([points, ones], dim=1) 26 | points_out = torch.matmul(points_hom, transf_matrix.unsqueeze(0)) 27 | 28 | denom = points_out[..., 3:] + 0.0000001 29 | return (points_out[..., :3] / denom).squeeze(dim=0) 30 | 31 | def getWorld2View(R, t): 32 | Rt = np.zeros((4, 4)) 33 | Rt[:3, :3] = R.transpose() 34 | Rt[:3, 3] = t 35 | Rt[3, 3] = 1.0 36 | return np.float32(Rt) 37 | 38 | def getWorld2View2(R, t, translate=np.array([.0, .0, .0]), scale=1.0): 39 | Rt = np.zeros((4, 4)) 40 | Rt[:3, :3] = R.transpose() 41 | Rt[:3, 3] = t 42 | Rt[3, 3] = 1.0 43 | 44 | C2W = np.linalg.inv(Rt) 45 | cam_center = C2W[:3, 3] 46 | cam_center = (cam_center + translate) * scale 47 | C2W[:3, 3] = cam_center 48 | Rt = np.linalg.inv(C2W) 49 | return np.float32(Rt) 50 | 51 | def getProjectionMatrix(znear, zfar, fovX, fovY): 52 | tanHalfFovY = math.tan((fovY / 2)) 53 | tanHalfFovX = math.tan((fovX / 2)) 54 | 55 | top = tanHalfFovY * znear 56 | bottom = -top 57 | right = tanHalfFovX * znear 58 | left = -right 59 | 60 | P = torch.zeros(4, 4) 61 | 62 | z_sign = 1.0 63 | 64 | P[0, 0] = 2.0 * znear / (right - left) 65 | P[1, 1] = 2.0 * znear / (top - bottom) 66 | P[0, 2] = (right + left) / (right - left) 67 | P[1, 2] = (top + bottom) / (top - bottom) 68 | P[3, 2] = z_sign 69 | P[2, 2] = z_sign * zfar / (zfar - znear) 70 | P[2, 3] = -(zfar * znear) / (zfar - znear) 71 | return P 72 | 73 | def fov2focal(fov, pixels): 74 | return pixels / (2 * math.tan(fov / 2)) 75 | 76 | def focal2fov(focal, pixels): 77 | return 2*math.atan(pixels/(2*focal)) -------------------------------------------------------------------------------- /utils/loss_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import torch.nn.functional as F 14 | from torch.autograd import Variable 15 | from math import exp 16 | 17 | def l1_loss(network_output, gt): 18 | return torch.abs((network_output - gt)).mean() 19 | 20 | def l2_loss(network_output, gt): 21 | return ((network_output - gt) ** 2).mean() 22 | 23 | def gaussian(window_size, sigma): 24 | gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)]) 25 | return gauss / gauss.sum() 26 | 27 | def create_window(window_size, channel): 28 | _1D_window = gaussian(window_size, 1.5).unsqueeze(1) 29 | _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0) 30 | window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous()) 31 | return window 32 | 33 | def ssim(img1, img2, window_size=11, size_average=True): 34 | channel = img1.size(-3) 35 | window = create_window(window_size, channel) 36 | 37 | if img1.is_cuda: 38 | window = window.cuda(img1.get_device()) 39 | window = window.type_as(img1) 40 | 41 | return _ssim(img1, img2, window, window_size, channel, size_average) 42 | 43 | def _ssim(img1, img2, window, window_size, channel, size_average=True): 44 | mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel) 45 | mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel) 46 | 47 | mu1_sq = mu1.pow(2) 48 | mu2_sq = mu2.pow(2) 49 | mu1_mu2 = mu1 * mu2 50 | 51 | sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq 52 | sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq 53 | sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2 54 | 55 | C1 = 0.01 ** 2 56 | C2 = 0.03 ** 2 57 | 58 | ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)) 59 | 60 | if size_average: 61 | return ssim_map.mean() 62 | else: 63 | return ssim_map.mean(1).mean(1).mean(1) 64 | 65 | -------------------------------------------------------------------------------- /gaussian_renderer/network_gui.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import traceback 14 | import socket 15 | import json 16 | from scene.cameras import MiniCam 17 | 18 | host = "127.0.0.1" 19 | port = 6009 20 | 21 | conn = None 22 | addr = None 23 | 24 | listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 25 | 26 | def init(wish_host, wish_port): 27 | global host, port, listener 28 | host = wish_host 29 | port = wish_port 30 | listener.bind((host, port)) 31 | listener.listen() 32 | listener.settimeout(0) 33 | 34 | def try_connect(): 35 | global conn, addr, listener 36 | try: 37 | conn, addr = listener.accept() 38 | print(f"\nConnected by {addr}") 39 | conn.settimeout(None) 40 | except Exception as inst: 41 | pass 42 | 43 | def read(): 44 | global conn 45 | messageLength = conn.recv(4) 46 | messageLength = int.from_bytes(messageLength, 'little') 47 | message = conn.recv(messageLength) 48 | return json.loads(message.decode("utf-8")) 49 | 50 | def send(message_bytes, verify): 51 | global conn 52 | if message_bytes != None: 53 | conn.sendall(message_bytes) 54 | conn.sendall(len(verify).to_bytes(4, 'little')) 55 | conn.sendall(bytes(verify, 'ascii')) 56 | 57 | def receive(): 58 | message = read() 59 | 60 | width = message["resolution_x"] 61 | height = message["resolution_y"] 62 | 63 | if width != 0 and height != 0: 64 | try: 65 | do_training = bool(message["train"]) 66 | fovy = message["fov_y"] 67 | fovx = message["fov_x"] 68 | znear = message["z_near"] 69 | zfar = message["z_far"] 70 | do_shs_python = bool(message["shs_python"]) 71 | do_rot_scale_python = bool(message["rot_scale_python"]) 72 | keep_alive = bool(message["keep_alive"]) 73 | scaling_modifier = message["scaling_modifier"] 74 | world_view_transform = torch.reshape(torch.tensor(message["view_matrix"]), (4, 4)).cuda() 75 | world_view_transform[:,1] = -world_view_transform[:,1] 76 | world_view_transform[:,2] = -world_view_transform[:,2] 77 | full_proj_transform = torch.reshape(torch.tensor(message["view_projection_matrix"]), (4, 4)).cuda() 78 | full_proj_transform[:,1] = -full_proj_transform[:,1] 79 | custom_cam = MiniCam(width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform) 80 | except Exception as e: 81 | print("") 82 | traceback.print_exc() 83 | raise e 84 | return custom_cam, do_training, do_shs_python, do_rot_scale_python, keep_alive, scaling_modifier 85 | else: 86 | return None, None, None, None, None, None -------------------------------------------------------------------------------- /utils/camera_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from scene.cameras import Camera 13 | import numpy as np 14 | from utils.general_utils import PILtoTorch 15 | from utils.graphics_utils import fov2focal 16 | 17 | WARNED = False 18 | 19 | def loadCam(args, id, cam_info, resolution_scale): 20 | orig_w, orig_h = cam_info.image.size 21 | 22 | if args.resolution in [1, 2, 4, 8]: 23 | resolution = round(orig_w/(resolution_scale * args.resolution)), round(orig_h/(resolution_scale * args.resolution)) 24 | else: # should be a type that converts to float 25 | if args.resolution == -1: 26 | if orig_w > 1600: 27 | global WARNED 28 | if not WARNED: 29 | print("[ INFO ] Encountered quite large input images (>1.6K pixels width), rescaling to 1.6K.\n " 30 | "If this is not desired, please explicitly specify '--resolution/-r' as 1") 31 | WARNED = True 32 | global_down = orig_w / 1600 33 | else: 34 | global_down = 1 35 | else: 36 | global_down = orig_w / args.resolution 37 | 38 | scale = float(global_down) * float(resolution_scale) 39 | resolution = (int(orig_w / scale), int(orig_h / scale)) 40 | 41 | resized_image_rgb = PILtoTorch(cam_info.image, resolution) 42 | 43 | gt_image = resized_image_rgb[:3, ...] 44 | loaded_mask = None 45 | 46 | if resized_image_rgb.shape[1] == 4: 47 | loaded_mask = resized_image_rgb[3:4, ...] 48 | 49 | return Camera(colmap_id=cam_info.uid, R=cam_info.R, T=cam_info.T, 50 | FoVx=cam_info.FovX, FoVy=cam_info.FovY, 51 | image=gt_image, gt_alpha_mask=loaded_mask, 52 | image_name=cam_info.image_name, uid=id, data_device=args.data_device) 53 | 54 | def cameraList_from_camInfos(cam_infos, resolution_scale, args): 55 | camera_list = [] 56 | 57 | for id, c in enumerate(cam_infos): 58 | camera_list.append(loadCam(args, id, c, resolution_scale)) 59 | 60 | return camera_list 61 | 62 | def camera_to_JSON(id, camera : Camera): 63 | Rt = np.zeros((4, 4)) 64 | Rt[:3, :3] = camera.R.transpose() 65 | Rt[:3, 3] = camera.T 66 | Rt[3, 3] = 1.0 67 | 68 | W2C = np.linalg.inv(Rt) 69 | pos = W2C[:3, 3] 70 | rot = W2C[:3, :3] 71 | serializable_array_2d = [x.tolist() for x in rot] 72 | camera_entry = { 73 | 'id' : id, 74 | 'img_name' : camera.image_name, 75 | 'width' : camera.width, 76 | 'height' : camera.height, 77 | 'position': pos.tolist(), 78 | 'rotation': serializable_array_2d, 79 | 'fy' : fov2focal(camera.FovY, camera.height), 80 | 'fx' : fov2focal(camera.FovX, camera.width) 81 | } 82 | return camera_entry 83 | -------------------------------------------------------------------------------- /lpipsPyTorch/modules/networks.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | from itertools import chain 4 | 5 | import torch 6 | import torch.nn as nn 7 | from torchvision import models 8 | 9 | from .utils import normalize_activation 10 | 11 | 12 | def get_network(net_type: str): 13 | if net_type == 'alex': 14 | return AlexNet() 15 | elif net_type == 'squeeze': 16 | return SqueezeNet() 17 | elif net_type == 'vgg': 18 | return VGG16() 19 | else: 20 | raise NotImplementedError('choose net_type from [alex, squeeze, vgg].') 21 | 22 | 23 | class LinLayers(nn.ModuleList): 24 | def __init__(self, n_channels_list: Sequence[int]): 25 | super(LinLayers, self).__init__([ 26 | nn.Sequential( 27 | nn.Identity(), 28 | nn.Conv2d(nc, 1, 1, 1, 0, bias=False) 29 | ) for nc in n_channels_list 30 | ]) 31 | 32 | for param in self.parameters(): 33 | param.requires_grad = False 34 | 35 | 36 | class BaseNet(nn.Module): 37 | def __init__(self): 38 | super(BaseNet, self).__init__() 39 | 40 | # register buffer 41 | self.register_buffer( 42 | 'mean', torch.Tensor([-.030, -.088, -.188])[None, :, None, None]) 43 | self.register_buffer( 44 | 'std', torch.Tensor([.458, .448, .450])[None, :, None, None]) 45 | 46 | def set_requires_grad(self, state: bool): 47 | for param in chain(self.parameters(), self.buffers()): 48 | param.requires_grad = state 49 | 50 | def z_score(self, x: torch.Tensor): 51 | return (x - self.mean) / self.std 52 | 53 | def forward(self, x: torch.Tensor): 54 | x = self.z_score(x) 55 | 56 | output = [] 57 | for i, (_, layer) in enumerate(self.layers._modules.items(), 1): 58 | x = layer(x) 59 | if i in self.target_layers: 60 | output.append(normalize_activation(x)) 61 | if len(output) == len(self.target_layers): 62 | break 63 | return output 64 | 65 | 66 | class SqueezeNet(BaseNet): 67 | def __init__(self): 68 | super(SqueezeNet, self).__init__() 69 | 70 | self.layers = models.squeezenet1_1(True).features 71 | self.target_layers = [2, 5, 8, 10, 11, 12, 13] 72 | self.n_channels_list = [64, 128, 256, 384, 384, 512, 512] 73 | 74 | self.set_requires_grad(False) 75 | 76 | 77 | class AlexNet(BaseNet): 78 | def __init__(self): 79 | super(AlexNet, self).__init__() 80 | 81 | self.layers = models.alexnet(True).features 82 | self.target_layers = [2, 5, 8, 10, 12] 83 | self.n_channels_list = [64, 192, 384, 256, 256] 84 | 85 | self.set_requires_grad(False) 86 | 87 | 88 | class VGG16(BaseNet): 89 | def __init__(self): 90 | super(VGG16, self).__init__() 91 | 92 | self.layers = models.vgg16(weights=models.VGG16_Weights.IMAGENET1K_V1).features 93 | self.target_layers = [4, 9, 16, 23, 30] 94 | self.n_channels_list = [64, 128, 256, 512, 512] 95 | 96 | self.set_requires_grad(False) 97 | -------------------------------------------------------------------------------- /scene/cameras.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | from torch import nn 14 | import numpy as np 15 | from utils.graphics_utils import getWorld2View2, getProjectionMatrix 16 | 17 | class Camera(nn.Module): 18 | def __init__(self, colmap_id, R, T, FoVx, FoVy, image, gt_alpha_mask, 19 | image_name, uid, 20 | trans=np.array([0.0, 0.0, 0.0]), scale=1.0, data_device = "cuda" 21 | ): 22 | super(Camera, self).__init__() 23 | 24 | self.uid = uid 25 | self.colmap_id = colmap_id 26 | self.R = R 27 | self.T = T 28 | self.FoVx = FoVx 29 | self.FoVy = FoVy 30 | self.image_name = image_name 31 | 32 | try: 33 | self.data_device = torch.device(data_device) 34 | except Exception as e: 35 | print(e) 36 | print(f"[Warning] Custom device {data_device} failed, fallback to default cuda device" ) 37 | self.data_device = torch.device("cuda") 38 | 39 | self.original_image = image.clamp(0.0, 1.0).to(self.data_device) 40 | self.image_width = self.original_image.shape[2] 41 | self.image_height = self.original_image.shape[1] 42 | 43 | if gt_alpha_mask is not None: 44 | self.original_image *= gt_alpha_mask.to(self.data_device) 45 | else: 46 | self.original_image *= torch.ones((1, self.image_height, self.image_width), device=self.data_device) 47 | 48 | self.zfar = 100.0 49 | self.znear = 0.01 50 | 51 | self.trans = trans 52 | self.scale = scale 53 | 54 | self.world_view_transform = torch.tensor(getWorld2View2(R, T, trans, scale)).transpose(0, 1).cuda() 55 | self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).cuda() 56 | self.full_proj_transform = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) 57 | self.full_proj_transform_inverse = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0).inverse() 58 | self.camera_center = self.world_view_transform.inverse()[3, :3] 59 | 60 | class MiniCam: 61 | def __init__(self, width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform): 62 | self.image_width = width 63 | self.image_height = height 64 | self.FoVy = fovy 65 | self.FoVx = fovx 66 | self.znear = znear 67 | self.zfar = zfar 68 | self.world_view_transform = world_view_transform 69 | self.full_proj_transform = full_proj_transform 70 | view_inv = torch.inverse(self.world_view_transform) 71 | self.camera_center = view_inv[3][:3] 72 | self.full_proj_transform_inverse = torch.inverse(self.full_proj_transform) 73 | 74 | -------------------------------------------------------------------------------- /full_eval.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | from argparse import ArgumentParser 14 | 15 | mipnerf360_outdoor_scenes = ["bicycle", "flowers", "garden", "stump", "treehill"] 16 | mipnerf360_indoor_scenes = ["room", "counter", "kitchen", "bonsai"] 17 | tanks_and_temples_scenes = ["truck", "train"] 18 | deep_blending_scenes = ["drjohnson", "playroom"] 19 | 20 | parser = ArgumentParser(description="Full evaluation script parameters") 21 | parser.add_argument("--skip_training", action="store_true") 22 | parser.add_argument("--skip_rendering", action="store_true") 23 | parser.add_argument("--skip_metrics", action="store_true") 24 | parser.add_argument("--output_path", default="./eval") 25 | args, _ = parser.parse_known_args() 26 | 27 | all_scenes = [] 28 | all_scenes.extend(mipnerf360_outdoor_scenes) 29 | all_scenes.extend(mipnerf360_indoor_scenes) 30 | all_scenes.extend(tanks_and_temples_scenes) 31 | all_scenes.extend(deep_blending_scenes) 32 | 33 | if not args.skip_training or not args.skip_rendering: 34 | parser.add_argument('--mipnerf360', "-m360", required=True, type=str) 35 | parser.add_argument("--tanksandtemples", "-tat", required=True, type=str) 36 | parser.add_argument("--deepblending", "-db", required=True, type=str) 37 | parser.add_argument('--config', required=True, type=str) 38 | parser.add_argument("--opacity_decay", type=float, default=0) 39 | args = parser.parse_args() 40 | 41 | if not args.skip_training: 42 | common_args = f"--splatting_config=\"{args.config}\" --quiet --eval --test_iterations -1" 43 | if args.opacity_decay > 0: 44 | common_args += f' --opacity_decay={args.opacity_decay} ' 45 | for scene in mipnerf360_outdoor_scenes: 46 | source = args.mipnerf360 + "/" + scene 47 | os.system(f"python train.py -s {source} -i images_4 -m {args.output_path}/{scene} {common_args}") 48 | for scene in mipnerf360_indoor_scenes: 49 | source = args.mipnerf360 + "/" + scene 50 | os.system(f"python train.py -s {source} -i images_2 -m {args.output_path}/{scene} {common_args}") 51 | for scene in tanks_and_temples_scenes: 52 | source = args.tanksandtemples + "/" + scene 53 | os.system(f"python train.py -s {source} -m {args.output_path}/{scene} {common_args}") 54 | for scene in deep_blending_scenes: 55 | source = args.deepblending + "/" + scene 56 | os.system(f"python train.py -s {source} -m {args.output_path}/{scene} {common_args}") 57 | 58 | if not args.skip_rendering: 59 | all_sources = [] 60 | for scene in mipnerf360_outdoor_scenes: 61 | all_sources.append(args.mipnerf360 + "/" + scene) 62 | for scene in mipnerf360_indoor_scenes: 63 | all_sources.append(args.mipnerf360 + "/" + scene) 64 | for scene in tanks_and_temples_scenes: 65 | all_sources.append(args.tanksandtemples + "/" + scene) 66 | for scene in deep_blending_scenes: 67 | all_sources.append(args.deepblending + "/" + scene) 68 | 69 | common_args = " --quiet --eval --skip_train" 70 | for scene, source in zip(all_scenes, all_sources): 71 | for it in [7000, 30000]: 72 | os.system(f"python render.py --iteration {it} -s {source} -m {args.output_path}/{scene} {common_args}") 73 | 74 | if not args.skip_metrics: 75 | for scene in all_scenes: 76 | scenes_string = "\"" + args.output_path + "/" + scene + "\"" 77 | 78 | os.system("python metrics.py -m " + scenes_string) -------------------------------------------------------------------------------- /render.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | from scene import Scene 14 | import os 15 | from tqdm import tqdm 16 | from os import makedirs 17 | from gaussian_renderer import render 18 | import torchvision 19 | import json 20 | from utils.general_utils import safe_state 21 | from argparse import ArgumentParser 22 | from arguments import ModelParams, PipelineParams, get_combined_args 23 | from arguments import ModelParams, PipelineParams, SplattingSettings 24 | from diff_gaussian_rasterization import ExtendedSettings 25 | from gaussian_renderer import GaussianModel 26 | 27 | def render_set(model_path, name, iteration, views, gaussians, pipeline, background, splat_args: ExtendedSettings, render_depth: bool): 28 | render_path = os.path.join(model_path, name, "ours_{}".format(iteration), "renders" if not render_depth else "depth") 29 | gts_path = os.path.join(model_path, name, "ours_{}".format(iteration), "gt") 30 | 31 | makedirs(render_path, exist_ok=True) 32 | makedirs(gts_path, exist_ok=True) 33 | 34 | for idx, view in enumerate(tqdm(views, desc="Rendering progress")): 35 | rendering = render(view, gaussians, pipeline, background, splat_args=splat_args, render_depth=render_depth)["render"] 36 | gt = view.original_image[0:3, :, :] 37 | torchvision.utils.save_image(rendering, os.path.join(render_path, '{0:05d}'.format(idx) + ".png")) 38 | torchvision.utils.save_image(gt, os.path.join(gts_path, '{0:05d}'.format(idx) + ".png")) 39 | 40 | def render_sets(dataset : ModelParams, iteration : int, pipeline : PipelineParams, skip_train : bool, skip_test : bool, splat_args: ExtendedSettings, render_depth: bool): 41 | with torch.no_grad(): 42 | gaussians = GaussianModel(dataset.sh_degree) 43 | scene = Scene(dataset, gaussians, load_iteration=iteration, shuffle=False, skip_test=args.skip_test, skip_train=args.skip_train) 44 | 45 | bg_color = [1,1,1] if dataset.white_background else [0, 0, 0] 46 | background = torch.tensor(bg_color, dtype=torch.float32, device="cuda") 47 | 48 | if not skip_train: 49 | render_set(dataset.model_path, "train", scene.loaded_iter, scene.getTrainCameras(), gaussians, pipeline, background, splat_args, render_depth) 50 | 51 | if not skip_test: 52 | render_set(dataset.model_path, "test", scene.loaded_iter, scene.getTestCameras(), gaussians, pipeline, background, splat_args, render_depth) 53 | 54 | # write number of gaussians too 55 | num_gaussians = scene.gaussians.get_xyz.shape[0] 56 | with open(os.path.join(dataset.model_path, "point_cloud", f'iteration_{scene.loaded_iter}', 'num_gaussians.json'), 'w') as fp: 57 | json.dump(obj={ 58 | "num_gaussians": num_gaussians, 59 | }, fp=fp, indent=2) 60 | 61 | if __name__ == "__main__": 62 | # Set up command line argument parser 63 | parser = ArgumentParser(description="Testing script parameters") 64 | model = ModelParams(parser, sentinel=True) 65 | pipeline = PipelineParams(parser) 66 | ss = SplattingSettings(parser, render=True) 67 | parser.add_argument("--iteration", default=-1, type=int) 68 | parser.add_argument("--skip_train", action="store_true") 69 | parser.add_argument("--skip_test", action="store_true") 70 | parser.add_argument("--quiet", action="store_true") 71 | parser.add_argument("--render_depth", action="store_true") 72 | args = get_combined_args(parser) 73 | print("Rendering " + args.model_path) 74 | 75 | splat_args = ss.get_settings(args) 76 | 77 | # Initialize system state (RNG) 78 | safe_state(args.quiet) 79 | 80 | render_sets(model.extract(args), args.iteration, pipeline.extract(args), args.skip_train, args.skip_test, splat_args, args.render_depth) -------------------------------------------------------------------------------- /gaussian_renderer/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import json 13 | import math 14 | import torch 15 | from diff_gaussian_rasterization import GaussianRasterizationSettings, ExtendedSettings, GaussianRasterizer 16 | from scene.gaussian_model import GaussianModel 17 | from utils.sh_utils import eval_sh 18 | 19 | def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None, splat_args: ExtendedSettings = None, render_depth: bool = False): 20 | """ 21 | Render the scene. 22 | 23 | Background tensor (bg_color) must be on GPU! 24 | """ 25 | 26 | # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means 27 | screenspace_points = torch.zeros_like(pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda") + 0 28 | try: 29 | screenspace_points.retain_grad() 30 | except: 31 | pass 32 | 33 | # Set up rasterization configuration 34 | tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) 35 | tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) 36 | 37 | raster_settings = GaussianRasterizationSettings( 38 | image_height=int(viewpoint_camera.image_height), 39 | image_width=int(viewpoint_camera.image_width), 40 | tanfovx=tanfovx, 41 | tanfovy=tanfovy, 42 | bg=bg_color, 43 | scale_modifier=scaling_modifier, 44 | viewmatrix=viewpoint_camera.world_view_transform, 45 | projmatrix=viewpoint_camera.full_proj_transform, 46 | inv_viewprojmatrix=viewpoint_camera.full_proj_transform_inverse, 47 | sh_degree=pc.active_sh_degree, 48 | campos=viewpoint_camera.camera_center, 49 | prefiltered=False, 50 | settings=splat_args, 51 | render_depth=render_depth, 52 | debug=pipe.debug 53 | ) 54 | 55 | rasterizer = GaussianRasterizer(raster_settings=raster_settings) 56 | 57 | means3D = pc.get_xyz 58 | means2D = screenspace_points 59 | opacity = pc.get_opacity 60 | 61 | # If precomputed 3d covariance is provided, use it. If not, then it will be computed from 62 | # scaling / rotation by the rasterizer. 63 | scales = None 64 | rotations = None 65 | cov3D_precomp = None 66 | if pipe.compute_cov3D_python: 67 | cov3D_precomp = pc.get_covariance(scaling_modifier) 68 | else: 69 | scales = pc.get_scaling 70 | rotations = pc.get_rotation 71 | 72 | # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors 73 | # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. 74 | shs = None 75 | colors_precomp = None 76 | if override_color is None: 77 | if pipe.convert_SHs_python: 78 | shs_view = pc.get_features.transpose(1, 2).view(-1, 3, (pc.max_sh_degree+1)**2) 79 | dir_pp = (pc.get_xyz - viewpoint_camera.camera_center.repeat(pc.get_features.shape[0], 1)) 80 | dir_pp_normalized = dir_pp/dir_pp.norm(dim=1, keepdim=True) 81 | sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized) 82 | colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0) 83 | else: 84 | shs = pc.get_features 85 | else: 86 | colors_precomp = override_color 87 | 88 | # Rasterize visible Gaussians to image, obtain their radii (on screen). 89 | rendered_image, radii = rasterizer( 90 | means3D = means3D, 91 | means2D = means2D, 92 | shs = shs, 93 | colors_precomp = colors_precomp, 94 | opacities = opacity, 95 | scales = scales, 96 | rotations = rotations, 97 | cov3D_precomp = cov3D_precomp) 98 | 99 | # Those Gaussians that were frustum culled or had a radius of 0 were not visible. 100 | # They will be excluded from value updates used in the splitting criteria. 101 | return {"render": rendered_image, 102 | "viewspace_points": screenspace_points, 103 | "visibility_filter" : radii > 0, 104 | "radii": radii} 105 | -------------------------------------------------------------------------------- /scene/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | import random 14 | import json 15 | from utils.system_utils import searchForMaxIteration 16 | from scene.dataset_readers import sceneLoadTypeCallbacks 17 | from scene.gaussian_model import GaussianModel 18 | from arguments import ModelParams 19 | from utils.camera_utils import cameraList_from_camInfos, camera_to_JSON 20 | 21 | class Scene: 22 | 23 | gaussians : GaussianModel 24 | 25 | def __init__(self, args : ModelParams, gaussians : GaussianModel, load_iteration=None, shuffle=True, resolution_scales=[1.0], skip_train=False, skip_test=False): 26 | """b 27 | :param path: Path to colmap scene main folder. 28 | """ 29 | self.model_path = args.model_path 30 | self.loaded_iter = None 31 | self.gaussians = gaussians 32 | 33 | if load_iteration: 34 | if load_iteration == -1: 35 | self.loaded_iter = searchForMaxIteration(os.path.join(self.model_path, "point_cloud")) 36 | else: 37 | self.loaded_iter = load_iteration 38 | print("Loading trained model at iteration {}".format(self.loaded_iter)) 39 | 40 | self.train_cameras = {} 41 | self.test_cameras = {} 42 | 43 | if os.path.exists(os.path.join(args.source_path, "sparse")): 44 | scene_info = sceneLoadTypeCallbacks["Colmap"](args.source_path, args.images, args.eval) 45 | elif os.path.exists(os.path.join(args.source_path, "transforms_train.json")): 46 | print("Found transforms_train.json file, assuming Blender data set!") 47 | scene_info = sceneLoadTypeCallbacks["Blender"](args.source_path, args.white_background, args.eval) 48 | else: 49 | assert False, "Could not recognize scene type!" 50 | 51 | if not self.loaded_iter: 52 | with open(scene_info.ply_path, 'rb') as src_file, open(os.path.join(self.model_path, "input.ply") , 'wb') as dest_file: 53 | dest_file.write(src_file.read()) 54 | json_cams = [] 55 | camlist = [] 56 | if scene_info.test_cameras: 57 | camlist.extend(scene_info.test_cameras) 58 | if scene_info.train_cameras: 59 | camlist.extend(scene_info.train_cameras) 60 | for id, cam in enumerate(camlist): 61 | json_cams.append(camera_to_JSON(id, cam)) 62 | with open(os.path.join(self.model_path, "cameras.json"), 'w') as file: 63 | json.dump(json_cams, file) 64 | 65 | if shuffle: 66 | random.shuffle(scene_info.train_cameras) # Multi-res consistent random shuffling 67 | random.shuffle(scene_info.test_cameras) # Multi-res consistent random shuffling 68 | 69 | self.cameras_extent = scene_info.nerf_normalization["radius"] 70 | 71 | for resolution_scale in resolution_scales: 72 | if not skip_train: 73 | print("Loading Training Cameras") 74 | self.train_cameras[resolution_scale] = cameraList_from_camInfos(scene_info.train_cameras, resolution_scale, args) 75 | if not skip_test: 76 | print("Loading Test Cameras") 77 | self.test_cameras[resolution_scale] = cameraList_from_camInfos(scene_info.test_cameras, resolution_scale, args) 78 | 79 | if self.loaded_iter: 80 | self.gaussians.load_ply(os.path.join(self.model_path, 81 | "point_cloud", 82 | "iteration_" + str(self.loaded_iter), 83 | "point_cloud.ply")) 84 | else: 85 | self.gaussians.create_from_pcd(scene_info.point_cloud, self.cameras_extent) 86 | 87 | def save(self, iteration): 88 | point_cloud_path = os.path.join(self.model_path, "point_cloud/iteration_{}".format(iteration)) 89 | self.gaussians.save_ply(os.path.join(point_cloud_path, "point_cloud.ply")) 90 | 91 | def getTrainCameras(self, scale=1.0): 92 | return self.train_cameras[scale] 93 | 94 | def getTestCameras(self, scale=1.0): 95 | return self.test_cameras[scale] -------------------------------------------------------------------------------- /utils/general_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import sys 14 | from datetime import datetime 15 | import numpy as np 16 | import random 17 | 18 | def inverse_sigmoid(x): 19 | return torch.log(x/(1-x)) 20 | 21 | def PILtoTorch(pil_image, resolution): 22 | resized_image_PIL = pil_image.resize(resolution) 23 | resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0 24 | if len(resized_image.shape) == 3: 25 | return resized_image.permute(2, 0, 1) 26 | else: 27 | return resized_image.unsqueeze(dim=-1).permute(2, 0, 1) 28 | 29 | def get_expon_lr_func( 30 | lr_init, lr_final, lr_delay_steps=0, lr_delay_mult=1.0, max_steps=1000000 31 | ): 32 | """ 33 | Copied from Plenoxels 34 | 35 | Continuous learning rate decay function. Adapted from JaxNeRF 36 | The returned rate is lr_init when step=0 and lr_final when step=max_steps, and 37 | is log-linearly interpolated elsewhere (equivalent to exponential decay). 38 | If lr_delay_steps>0 then the learning rate will be scaled by some smooth 39 | function of lr_delay_mult, such that the initial learning rate is 40 | lr_init*lr_delay_mult at the beginning of optimization but will be eased back 41 | to the normal learning rate when steps>lr_delay_steps. 42 | :param conf: config subtree 'lr' or similar 43 | :param max_steps: int, the number of steps during optimization. 44 | :return HoF which takes step as input 45 | """ 46 | 47 | def helper(step): 48 | if step < 0 or (lr_init == 0.0 and lr_final == 0.0): 49 | # Disable this parameter 50 | return 0.0 51 | if lr_delay_steps > 0: 52 | # A kind of reverse cosine decay. 53 | delay_rate = lr_delay_mult + (1 - lr_delay_mult) * np.sin( 54 | 0.5 * np.pi * np.clip(step / lr_delay_steps, 0, 1) 55 | ) 56 | else: 57 | delay_rate = 1.0 58 | t = np.clip(step / max_steps, 0, 1) 59 | log_lerp = np.exp(np.log(lr_init) * (1 - t) + np.log(lr_final) * t) 60 | return delay_rate * log_lerp 61 | 62 | return helper 63 | 64 | def strip_lowerdiag(L): 65 | uncertainty = torch.zeros((L.shape[0], 6), dtype=torch.float, device="cuda") 66 | 67 | uncertainty[:, 0] = L[:, 0, 0] 68 | uncertainty[:, 1] = L[:, 0, 1] 69 | uncertainty[:, 2] = L[:, 0, 2] 70 | uncertainty[:, 3] = L[:, 1, 1] 71 | uncertainty[:, 4] = L[:, 1, 2] 72 | uncertainty[:, 5] = L[:, 2, 2] 73 | return uncertainty 74 | 75 | def strip_symmetric(sym): 76 | return strip_lowerdiag(sym) 77 | 78 | def build_rotation(r): 79 | norm = torch.sqrt(r[:,0]*r[:,0] + r[:,1]*r[:,1] + r[:,2]*r[:,2] + r[:,3]*r[:,3]) 80 | 81 | q = r / norm[:, None] 82 | 83 | R = torch.zeros((q.size(0), 3, 3), device='cuda') 84 | 85 | r = q[:, 0] 86 | x = q[:, 1] 87 | y = q[:, 2] 88 | z = q[:, 3] 89 | 90 | R[:, 0, 0] = 1 - 2 * (y*y + z*z) 91 | R[:, 0, 1] = 2 * (x*y - r*z) 92 | R[:, 0, 2] = 2 * (x*z + r*y) 93 | R[:, 1, 0] = 2 * (x*y + r*z) 94 | R[:, 1, 1] = 1 - 2 * (x*x + z*z) 95 | R[:, 1, 2] = 2 * (y*z - r*x) 96 | R[:, 2, 0] = 2 * (x*z - r*y) 97 | R[:, 2, 1] = 2 * (y*z + r*x) 98 | R[:, 2, 2] = 1 - 2 * (x*x + y*y) 99 | return R 100 | 101 | def build_scaling_rotation(s, r): 102 | L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda") 103 | R = build_rotation(r) 104 | 105 | L[:,0,0] = s[:,0] 106 | L[:,1,1] = s[:,1] 107 | L[:,2,2] = s[:,2] 108 | 109 | L = R @ L 110 | return L 111 | 112 | def safe_state(silent): 113 | old_f = sys.stdout 114 | class F: 115 | def __init__(self, silent): 116 | self.silent = silent 117 | 118 | def write(self, x): 119 | if not self.silent: 120 | if x.endswith("\n"): 121 | old_f.write(x.replace("\n", " [{}]\n".format(str(datetime.now().strftime("%d/%m %H:%M:%S"))))) 122 | else: 123 | old_f.write(x) 124 | 125 | def flush(self): 126 | old_f.flush() 127 | 128 | sys.stdout = F(silent) 129 | 130 | random.seed(0) 131 | np.random.seed(0) 132 | torch.manual_seed(0) 133 | torch.cuda.set_device(torch.device("cuda:0")) 134 | -------------------------------------------------------------------------------- /utils/sh_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The PlenOctree Authors. 2 | # Redistribution and use in source and binary forms, with or without 3 | # modification, are permitted provided that the following conditions are met: 4 | # 5 | # 1. Redistributions of source code must retain the above copyright notice, 6 | # this list of conditions and the following disclaimer. 7 | # 8 | # 2. Redistributions in binary form must reproduce the above copyright notice, 9 | # this list of conditions and the following disclaimer in the documentation 10 | # and/or other materials provided with the distribution. 11 | # 12 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 13 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 14 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 15 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 16 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 17 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 18 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 19 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 20 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 21 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 22 | # POSSIBILITY OF SUCH DAMAGE. 23 | 24 | import torch 25 | 26 | C0 = 0.28209479177387814 27 | C1 = 0.4886025119029199 28 | C2 = [ 29 | 1.0925484305920792, 30 | -1.0925484305920792, 31 | 0.31539156525252005, 32 | -1.0925484305920792, 33 | 0.5462742152960396 34 | ] 35 | C3 = [ 36 | -0.5900435899266435, 37 | 2.890611442640554, 38 | -0.4570457994644658, 39 | 0.3731763325901154, 40 | -0.4570457994644658, 41 | 1.445305721320277, 42 | -0.5900435899266435 43 | ] 44 | C4 = [ 45 | 2.5033429417967046, 46 | -1.7701307697799304, 47 | 0.9461746957575601, 48 | -0.6690465435572892, 49 | 0.10578554691520431, 50 | -0.6690465435572892, 51 | 0.47308734787878004, 52 | -1.7701307697799304, 53 | 0.6258357354491761, 54 | ] 55 | 56 | 57 | def eval_sh(deg, sh, dirs): 58 | """ 59 | Evaluate spherical harmonics at unit directions 60 | using hardcoded SH polynomials. 61 | Works with torch/np/jnp. 62 | ... Can be 0 or more batch dimensions. 63 | Args: 64 | deg: int SH deg. Currently, 0-3 supported 65 | sh: jnp.ndarray SH coeffs [..., C, (deg + 1) ** 2] 66 | dirs: jnp.ndarray unit directions [..., 3] 67 | Returns: 68 | [..., C] 69 | """ 70 | assert deg <= 4 and deg >= 0 71 | coeff = (deg + 1) ** 2 72 | assert sh.shape[-1] >= coeff 73 | 74 | result = C0 * sh[..., 0] 75 | if deg > 0: 76 | x, y, z = dirs[..., 0:1], dirs[..., 1:2], dirs[..., 2:3] 77 | result = (result - 78 | C1 * y * sh[..., 1] + 79 | C1 * z * sh[..., 2] - 80 | C1 * x * sh[..., 3]) 81 | 82 | if deg > 1: 83 | xx, yy, zz = x * x, y * y, z * z 84 | xy, yz, xz = x * y, y * z, x * z 85 | result = (result + 86 | C2[0] * xy * sh[..., 4] + 87 | C2[1] * yz * sh[..., 5] + 88 | C2[2] * (2.0 * zz - xx - yy) * sh[..., 6] + 89 | C2[3] * xz * sh[..., 7] + 90 | C2[4] * (xx - yy) * sh[..., 8]) 91 | 92 | if deg > 2: 93 | result = (result + 94 | C3[0] * y * (3 * xx - yy) * sh[..., 9] + 95 | C3[1] * xy * z * sh[..., 10] + 96 | C3[2] * y * (4 * zz - xx - yy)* sh[..., 11] + 97 | C3[3] * z * (2 * zz - 3 * xx - 3 * yy) * sh[..., 12] + 98 | C3[4] * x * (4 * zz - xx - yy) * sh[..., 13] + 99 | C3[5] * z * (xx - yy) * sh[..., 14] + 100 | C3[6] * x * (xx - 3 * yy) * sh[..., 15]) 101 | 102 | if deg > 3: 103 | result = (result + C4[0] * xy * (xx - yy) * sh[..., 16] + 104 | C4[1] * yz * (3 * xx - yy) * sh[..., 17] + 105 | C4[2] * xy * (7 * zz - 1) * sh[..., 18] + 106 | C4[3] * yz * (7 * zz - 3) * sh[..., 19] + 107 | C4[4] * (zz * (35 * zz - 30) + 3) * sh[..., 20] + 108 | C4[5] * xz * (7 * zz - 3) * sh[..., 21] + 109 | C4[6] * (xx - yy) * (7 * zz - 1) * sh[..., 22] + 110 | C4[7] * xz * (xx - 3 * yy) * sh[..., 23] + 111 | C4[8] * (xx * (xx - 3 * yy) - yy * (3 * xx - yy)) * sh[..., 24]) 112 | return result 113 | 114 | def RGB2SH(rgb): 115 | return (rgb - 0.5) / C0 116 | 117 | def SH2RGB(sh): 118 | return sh * C0 + 0.5 -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Gaussian-Splatting License 2 | =========================== 3 | 4 | **Inria** and **the Max Planck Institut for Informatik (MPII)** hold all the ownership rights on the *Software* named **gaussian-splatting**. 5 | The *Software* is in the process of being registered with the Agence pour la Protection des 6 | Programmes (APP). 7 | 8 | The *Software* is still being developed by the *Licensor*. 9 | 10 | *Licensor*'s goal is to allow the research community to use, test and evaluate 11 | the *Software*. 12 | 13 | ## 1. Definitions 14 | 15 | *Licensee* means any person or entity that uses the *Software* and distributes 16 | its *Work*. 17 | 18 | *Licensor* means the owners of the *Software*, i.e Inria and MPII 19 | 20 | *Software* means the original work of authorship made available under this 21 | License ie gaussian-splatting. 22 | 23 | *Work* means the *Software* and any additions to or derivative works of the 24 | *Software* that are made available under this License. 25 | 26 | 27 | ## 2. Purpose 28 | This license is intended to define the rights granted to the *Licensee* by 29 | Licensors under the *Software*. 30 | 31 | ## 3. Rights granted 32 | 33 | For the above reasons Licensors have decided to distribute the *Software*. 34 | Licensors grant non-exclusive rights to use the *Software* for research purposes 35 | to research users (both academic and industrial), free of charge, without right 36 | to sublicense.. The *Software* may be used "non-commercially", i.e., for research 37 | and/or evaluation purposes only. 38 | 39 | Subject to the terms and conditions of this License, you are granted a 40 | non-exclusive, royalty-free, license to reproduce, prepare derivative works of, 41 | publicly display, publicly perform and distribute its *Work* and any resulting 42 | derivative works in any form. 43 | 44 | ## 4. Limitations 45 | 46 | **4.1 Redistribution.** You may reproduce or distribute the *Work* only if (a) you do 47 | so under this License, (b) you include a complete copy of this License with 48 | your distribution, and (c) you retain without modification any copyright, 49 | patent, trademark, or attribution notices that are present in the *Work*. 50 | 51 | **4.2 Derivative Works.** You may specify that additional or different terms apply 52 | to the use, reproduction, and distribution of your derivative works of the *Work* 53 | ("Your Terms") only if (a) Your Terms provide that the use limitation in 54 | Section 2 applies to your derivative works, and (b) you identify the specific 55 | derivative works that are subject to Your Terms. Notwithstanding Your Terms, 56 | this License (including the redistribution requirements in Section 3.1) will 57 | continue to apply to the *Work* itself. 58 | 59 | **4.3** Any other use without of prior consent of Licensors is prohibited. Research 60 | users explicitly acknowledge having received from Licensors all information 61 | allowing to appreciate the adequacy between of the *Software* and their needs and 62 | to undertake all necessary precautions for its execution and use. 63 | 64 | **4.4** The *Software* is provided both as a compiled library file and as source 65 | code. In case of using the *Software* for a publication or other results obtained 66 | through the use of the *Software*, users are strongly encouraged to cite the 67 | corresponding publications as explained in the documentation of the *Software*. 68 | 69 | ## 5. Disclaimer 70 | 71 | THE USER CANNOT USE, EXPLOIT OR DISTRIBUTE THE *SOFTWARE* FOR COMMERCIAL PURPOSES 72 | WITHOUT PRIOR AND EXPLICIT CONSENT OF LICENSORS. YOU MUST CONTACT INRIA FOR ANY 73 | UNAUTHORIZED USE: stip-sophia.transfert@inria.fr . ANY SUCH ACTION WILL 74 | CONSTITUTE A FORGERY. THIS *SOFTWARE* IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES 75 | OF ANY NATURE AND ANY EXPRESS OR IMPLIED WARRANTIES, WITH REGARDS TO COMMERCIAL 76 | USE, PROFESSIONNAL USE, LEGAL OR NOT, OR OTHER, OR COMMERCIALISATION OR 77 | ADAPTATION. UNLESS EXPLICITLY PROVIDED BY LAW, IN NO EVENT, SHALL INRIA OR THE 78 | AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 79 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 80 | GOODS OR SERVICES, LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION) 81 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 82 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING FROM, OUT OF OR 83 | IN CONNECTION WITH THE *SOFTWARE* OR THE USE OR OTHER DEALINGS IN THE *SOFTWARE*. 84 | 85 | ## 6. Files subject to permissive licenses 86 | The contents of the file ```utils/loss_utils.py``` are based on publicly available code authored by Evan Su, which falls under the permissive MIT license. 87 | 88 | Title: pytorch-ssim\ 89 | Project code: https://github.com/Po-Hsun-Su/pytorch-ssim\ 90 | Copyright Evan Su, 2017\ 91 | License: https://github.com/Po-Hsun-Su/pytorch-ssim/blob/master/LICENSE.txt (MIT) -------------------------------------------------------------------------------- /metrics.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from pathlib import Path 13 | import os 14 | from PIL import Image 15 | import torch 16 | import torchvision.transforms.functional as tf 17 | from utils.loss_utils import ssim 18 | from lpipsPyTorch import LPIPSEval 19 | import json 20 | from tqdm import tqdm 21 | from utils.image_utils import psnr 22 | from argparse import ArgumentParser 23 | from utils.flip import LDRFLIPLoss 24 | 25 | def readImages(renders_dir, gt_dir): 26 | renders = [] 27 | gts = [] 28 | image_names = [] 29 | for fname in os.listdir(renders_dir): 30 | render = Image.open(renders_dir / fname) 31 | gt = Image.open(gt_dir / fname) 32 | renders.append(tf.to_tensor(render).unsqueeze(0)[:, :3, :, :].cuda()) 33 | gts.append(tf.to_tensor(gt).unsqueeze(0)[:, :3, :, :].cuda()) 34 | image_names.append(fname) 35 | return renders, gts, image_names 36 | 37 | def evaluate(model_paths): 38 | 39 | full_dict = {} 40 | per_view_dict = {} 41 | full_dict_polytopeonly = {} 42 | per_view_dict_polytopeonly = {} 43 | print("") 44 | flip = LDRFLIPLoss() 45 | lpips = LPIPSEval(net_type='vgg', device='cuda') 46 | 47 | for scene_dir in model_paths: 48 | try: 49 | print("Scene:", scene_dir) 50 | full_dict[scene_dir] = {} 51 | per_view_dict[scene_dir] = {} 52 | full_dict_polytopeonly[scene_dir] = {} 53 | per_view_dict_polytopeonly[scene_dir] = {} 54 | 55 | test_dir = Path(scene_dir) / "test" 56 | pointcloud_dir = Path(scene_dir) / "point_cloud" 57 | 58 | for method in os.listdir(test_dir): 59 | print("Method:", method) 60 | 61 | full_dict[scene_dir][method] = {} 62 | per_view_dict[scene_dir][method] = {} 63 | full_dict_polytopeonly[scene_dir][method] = {} 64 | per_view_dict_polytopeonly[scene_dir][method] = {} 65 | 66 | method_dir = test_dir / method 67 | gt_dir = method_dir / "gt" 68 | renders_dir = method_dir / "renders" 69 | renders, gts, image_names = readImages(renders_dir, gt_dir) 70 | 71 | ssims = [] 72 | psnrs = [] 73 | lpipss = [] 74 | flips = [] 75 | 76 | for idx in tqdm(range(len(renders)), desc="Metric evaluation progress"): 77 | ssims.append(ssim(renders[idx], gts[idx])) 78 | psnrs.append(psnr(renders[idx], gts[idx])) 79 | lpipss.append(lpips.criterion(renders[idx], gts[idx])) 80 | flips.append(flip(renders[idx], gts[idx]).mean().item()) 81 | 82 | # load number of gaussians 83 | with open(os.path.join(pointcloud_dir, f"iteration_{method.split('_')[-1]}", "num_gaussians.json"), 'r') as fp: 84 | num_gaussians = json.load(fp)["num_gaussians"] 85 | 86 | print(" SSIM : {:>12.7f}".format(torch.tensor(ssims).mean(), ".5")) 87 | print(" PSNR : {:>12.7f}".format(torch.tensor(psnrs).mean(), ".5")) 88 | print(" LPIPS: {:>12.7f}".format(torch.tensor(lpipss).mean(), ".5")) 89 | print(" FLIP : {:>12.7f}".format(torch.tensor(flips).mean(), ".5")) 90 | print(" NUM : {:>12}".format(num_gaussians)) 91 | print("") 92 | 93 | full_dict[scene_dir][method].update({"SSIM": torch.tensor(ssims).mean().item(), 94 | "PSNR": torch.tensor(psnrs).mean().item(), 95 | "LPIPS": torch.tensor(lpipss).mean().item(), 96 | "FLIPS": torch.tensor(flips).mean().item(), 97 | "NUM": num_gaussians}) 98 | per_view_dict[scene_dir][method].update({"SSIM": {name: ssim for ssim, name in zip(torch.tensor(ssims).tolist(), image_names)}, 99 | "PSNR": {name: psnr for psnr, name in zip(torch.tensor(psnrs).tolist(), image_names)}, 100 | "LPIPS": {name: lp for lp, name in zip(torch.tensor(lpipss).tolist(), image_names)}, 101 | "FLIPS": {name: fl for fl, name in zip(torch.tensor(flips).tolist(), image_names)}}) 102 | 103 | with open(scene_dir + "/results.json", 'w') as fp: 104 | json.dump(full_dict[scene_dir], fp, indent=True) 105 | with open(scene_dir + "/per_view.json", 'w') as fp: 106 | json.dump(per_view_dict[scene_dir], fp, indent=True) 107 | except: 108 | print("Unable to compute metrics for model", scene_dir) 109 | 110 | if __name__ == "__main__": 111 | device = torch.device("cuda:0") 112 | torch.cuda.set_device(device) 113 | 114 | # Set up command line argument parser 115 | parser = ArgumentParser(description="Training script parameters") 116 | parser.add_argument('--model_paths', '-m', required=True, nargs="+", type=str, default=[]) 117 | args = parser.parse_args() 118 | evaluate(args.model_paths) 119 | -------------------------------------------------------------------------------- /convert.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | import logging 14 | from argparse import ArgumentParser 15 | import shutil 16 | 17 | # This Python script is based on the shell converter script provided in the MipNerF 360 repository. 18 | parser = ArgumentParser("Colmap converter") 19 | parser.add_argument("--no_gpu", action='store_true') 20 | parser.add_argument("--skip_matching", action='store_true') 21 | parser.add_argument("--source_path", "-s", required=True, type=str) 22 | parser.add_argument("--camera", default="OPENCV", type=str) 23 | parser.add_argument("--colmap_executable", default="", type=str) 24 | parser.add_argument("--resize", action="store_true") 25 | parser.add_argument("--magick_executable", default="", type=str) 26 | args = parser.parse_args() 27 | colmap_command = '"{}"'.format(args.colmap_executable) if len(args.colmap_executable) > 0 else "colmap" 28 | magick_command = '"{}"'.format(args.magick_executable) if len(args.magick_executable) > 0 else "magick" 29 | use_gpu = 1 if not args.no_gpu else 0 30 | 31 | if not args.skip_matching: 32 | os.makedirs(args.source_path + "/distorted/sparse", exist_ok=True) 33 | 34 | ## Feature extraction 35 | feat_extracton_cmd = colmap_command + " feature_extractor "\ 36 | "--database_path " + args.source_path + "/distorted/database.db \ 37 | --image_path " + args.source_path + "/input \ 38 | --ImageReader.single_camera 1 \ 39 | --ImageReader.camera_model " + args.camera + " \ 40 | --SiftExtraction.use_gpu " + str(use_gpu) 41 | exit_code = os.system(feat_extracton_cmd) 42 | if exit_code != 0: 43 | logging.error(f"Feature extraction failed with code {exit_code}. Exiting.") 44 | exit(exit_code) 45 | 46 | ## Feature matching 47 | feat_matching_cmd = colmap_command + " exhaustive_matcher \ 48 | --database_path " + args.source_path + "/distorted/database.db \ 49 | --SiftMatching.use_gpu " + str(use_gpu) 50 | exit_code = os.system(feat_matching_cmd) 51 | if exit_code != 0: 52 | logging.error(f"Feature matching failed with code {exit_code}. Exiting.") 53 | exit(exit_code) 54 | 55 | ### Bundle adjustment 56 | # The default Mapper tolerance is unnecessarily large, 57 | # decreasing it speeds up bundle adjustment steps. 58 | mapper_cmd = (colmap_command + " mapper \ 59 | --database_path " + args.source_path + "/distorted/database.db \ 60 | --image_path " + args.source_path + "/input \ 61 | --output_path " + args.source_path + "/distorted/sparse \ 62 | --Mapper.ba_global_function_tolerance=0.000001") 63 | exit_code = os.system(mapper_cmd) 64 | if exit_code != 0: 65 | logging.error(f"Mapper failed with code {exit_code}. Exiting.") 66 | exit(exit_code) 67 | 68 | ### Image undistortion 69 | ## We need to undistort our images into ideal pinhole intrinsics. 70 | img_undist_cmd = (colmap_command + " image_undistorter \ 71 | --image_path " + args.source_path + "/input \ 72 | --input_path " + args.source_path + "/distorted/sparse/0 \ 73 | --output_path " + args.source_path + "\ 74 | --output_type COLMAP") 75 | exit_code = os.system(img_undist_cmd) 76 | if exit_code != 0: 77 | logging.error(f"Mapper failed with code {exit_code}. Exiting.") 78 | exit(exit_code) 79 | 80 | files = os.listdir(args.source_path + "/sparse") 81 | os.makedirs(args.source_path + "/sparse/0", exist_ok=True) 82 | # Copy each file from the source directory to the destination directory 83 | for file in files: 84 | if file == '0': 85 | continue 86 | source_file = os.path.join(args.source_path, "sparse", file) 87 | destination_file = os.path.join(args.source_path, "sparse", "0", file) 88 | shutil.move(source_file, destination_file) 89 | 90 | if(args.resize): 91 | print("Copying and resizing...") 92 | 93 | # Resize images. 94 | os.makedirs(args.source_path + "/images_2", exist_ok=True) 95 | os.makedirs(args.source_path + "/images_4", exist_ok=True) 96 | os.makedirs(args.source_path + "/images_8", exist_ok=True) 97 | # Get the list of files in the source directory 98 | files = os.listdir(args.source_path + "/images") 99 | # Copy each file from the source directory to the destination directory 100 | for file in files: 101 | source_file = os.path.join(args.source_path, "images", file) 102 | 103 | destination_file = os.path.join(args.source_path, "images_2", file) 104 | shutil.copy2(source_file, destination_file) 105 | exit_code = os.system(magick_command + " mogrify -resize 50% " + destination_file) 106 | if exit_code != 0: 107 | logging.error(f"50% resize failed with code {exit_code}. Exiting.") 108 | exit(exit_code) 109 | 110 | destination_file = os.path.join(args.source_path, "images_4", file) 111 | shutil.copy2(source_file, destination_file) 112 | exit_code = os.system(magick_command + " mogrify -resize 25% " + destination_file) 113 | if exit_code != 0: 114 | logging.error(f"25% resize failed with code {exit_code}. Exiting.") 115 | exit(exit_code) 116 | 117 | destination_file = os.path.join(args.source_path, "images_8", file) 118 | shutil.copy2(source_file, destination_file) 119 | exit_code = os.system(magick_command + " mogrify -resize 12.5% " + destination_file) 120 | if exit_code != 0: 121 | logging.error(f"12.5% resize failed with code {exit_code}. Exiting.") 122 | exit(exit_code) 123 | 124 | print("Done.") 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VRSplat 2 | 3 | ## Fast and Robust Gaussian Splatting for Virtual Reality 4 | 5 | [Xuechang Tu](https://scholar.google.com/citations?user=LGmQ3bUAAAAJ) 6 | [Lukas Radl](https://r4dl.github.io/), 7 | [Michael Steiner](https://steimich96.github.io), 8 | [Markus Steinberger](https://www.markussteinberger.net/), 9 | [Bernhard Kerbl](https://snosixtyboo.github.io/), 10 | [Fernando de la Torre]() 11 | 12 | | [Webpage](https://cekavis.github.io/VRSplat/) 13 | | [Full Paper](https://r4dl.github.io/data/vrsplat_authorversion.pdf) 14 | | [Pre-trained Models (1.04 GB)](https://drive.google.com/file/d/1RSJjlAwUs_3LTxWX7l221HsVLsmOFfE0/view?usp=sharing) 15 |
16 | 17 | This repository contains the official authors implementation associated with the paper "VRSplat: Fast and Robust Gaussian Splatting for Virtual Reality". 18 | 19 | 　　　 20 | 21 | Abstract: *3D Gaussian Splatting (3DGS) has rapidly become a leading technique for novel-view synthesis, providing exceptional performance through efficient software-based GPU rasterization. Its versatility enables real-time applications, including on mobile and lower-powered devices. However, 3DGS faces key challenges in virtual reality (VR): (1) temporal artifacts, such as popping during head movements, (2) projection-based distortions that result in disturbing and view-inconsistent floaters, and (3) reduced framerates when rendering large numbers of Gaussians, falling below the critical threshold for VR. Compared to desktop environments, these issues are drastically amplified by large field-of-view, constant head movements, and high resolution of head-mounted displays (HMDs). In this work, we introduce VRSplat: we combine and extend several recent advancements in 3DGS to address challenges of VR holistically. We show how the ideas of Mini-Splatting, StopThePop, and Optimal Projection can complement each other, by modifying the individual techniques and core 3DGS rasterizer. Additionally, we propose an efficient foveated rasterizer that handles focus and peripheral areas in a single GPU launch, avoiding redundant computations and improving GPU utilization. Our method also incorporates a fine-tuning step that optimizes Gaussian parameters based on StopThePop depth evaluations and Optimal Projection. We validate our method through a controlled user study with 25 participants, showing a strong preference for VRSplat over other configurations of Mini-Splatting. VRSplat is the first, systematically evaluated 3DGS approach capable of supporting modern VR applications, achieving 72+ FPS while eliminating popping and stereo-disrupting floaters.* 22 | 23 |
24 |
25 |

BibTeX

26 | 
@article{Tu2025VRSplat,
 27 |   author    = {Tu, Xuechang and Radl, Lukas and Steiner, Michael and Steinberger, Markus and Kerbl, Bernhard and de la Torre, Fornando},
 28 |   title     = {{VRsplat: Fast and Robust Gaussian Splatting for Virtual Reality}},
 29 |   journal   = {Proc. ACM Comput. Graph. Interact. Tech.},
 30 |   volume    = {8},
 31 |   number    = {1},
 32 |   articleno = {1},
 33 |   year      = {2025},
 34 | }
35 |
36 |
37 | 38 | ## Overview 39 | Our repository is built on [3D Gaussian Splatting](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/): 40 | For a full breakdown on how to get the code running, please consider [3DGS's Readme](https://github.com/graphdeco-inria/gaussian-splatting/blob/main/README.md). 41 | 42 | The project is split into submodules, each maintained in a separate github repository: 43 | 44 | * [VRSplat-Rasterization](https://github.com/Cekavis/VRSplat-Rasterization): A clone of the original [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization) that contains our CUDA rasterizer implementation 45 | * [SIBR_VRSplat](https://github.com/Cekavis/SIBR_VRSplat): A clone of the [SIBR Core](https://gitlab.inria.fr/sibr/sibr_core) project, containing an adapted VR viewer with our additional settings and functionalities 46 | 47 | ## Licensing 48 | 49 | The majority of the projects is licensed under the ["Gaussian-Splatting License"](LICENSE.md), with the exception of: 50 | 51 | * [StopThePop header files](submodules/diff-gaussian-rasterization/cuda_rasterizer/stopthepop): MIT License 52 | * [FLIP](utils/flip.py): BSD-3 license 53 | 54 | There are also several changes in the original source code. 55 | If you use any of our additional functionalities, please cite our paper and link to this repository. 56 | 57 | ## Cloning the Repository 58 | 59 | The repository contains submodules, thus please check it out with 60 | ```shell 61 | git clone https://github.com/Cekavis/VRSplat --recursive 62 | ``` 63 | 64 | ## Setup 65 | 66 | ### Local Setup 67 | 68 | Our default, provided install method is based on Conda package and environment management: 69 | ```shell 70 | SET DISTUTILS_USE_SDK=1 # Windows only 71 | conda env create --file environment.yml 72 | conda activate vrsplat 73 | ``` 74 | 75 | > **Note:** This process assumes that you have CUDA SDK **11** installed. 76 | Optionally, you can use CUDA **12** and Pytorch **2.1**, by using `environment_cuda12.yml` instead of `environment.yml`. 77 | 78 | Subsequently, install the CUDA rasterizer: 79 | ```shell 80 | pip install submodules/diff-gaussian-rasterization 81 | ``` 82 | 83 | > **Note:** This can take several minutes. If you experience unreasonably long build times, consider using [StopThePop's fast build mode](https://github.com/r4dl/StopThePop#stopthepop_fastbuild). 84 | 85 | ### Running 86 | 87 | To train a model from scratch, run: 88 | 89 | ```shell 90 | python train.py --splatting_config configs/hierarchical.json -s 91 | ``` 92 | 93 | To fine-tune the output of another 3DGS training method (e.g. [MiniSplatting](https://github.com/fatPeter/mini-splatting)), first ensure that a compatible checkpoint is available (e.g. trained with `--checkpoint_iterations 30000`). Then, run: 94 | ```shell 95 | python train.py --splatting_config configs/hierarchical.json -s --start_checkpoint --iterations 35000 96 | ``` 97 | 98 | For a full explanation of the training configuration, please refer to [StopThePop](https://github.com/r4dl/StopThePop). 99 | 100 | ## Interactive VR Viewer 101 | 102 | See [SIBR_VRSplat](https://github.com/Cekavis/SIBR_VRSplat). 103 | 104 | ## FAQ 105 | 106 | Please refer to 3DGS's FAQ, contained in [their README](https://github.com/graphdeco-inria/gaussian-splatting/blob/main/README.md). In addition, several issues are also covered on [3DGS's issues page](https://github.com/graphdeco-inria/gaussian-splatting/issues). 107 | -------------------------------------------------------------------------------- /arguments/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | from argparse import ArgumentParser, Namespace 13 | from diff_gaussian_rasterization import ExtendedSettings, GlobalSortOrder, SortMode 14 | import json 15 | import sys 16 | import os 17 | from distutils.util import strtobool 18 | 19 | class GroupParams: 20 | pass 21 | 22 | class ParamGroup: 23 | def __init__(self, parser: ArgumentParser, name : str, fill_none = False): 24 | group = parser.add_argument_group(name) 25 | for key, value in vars(self).items(): 26 | shorthand = False 27 | if key.startswith("_"): 28 | shorthand = True 29 | key = key[1:] 30 | t = type(value) 31 | value = value if not fill_none else None 32 | if shorthand: 33 | if t == bool: 34 | group.add_argument("--" + key, ("-" + key[0:1]), default=value, action="store_true") 35 | else: 36 | group.add_argument("--" + key, ("-" + key[0:1]), default=value, type=t) 37 | else: 38 | if t == bool: 39 | group.add_argument("--" + key, default=value, action="store_true") 40 | else: 41 | group.add_argument("--" + key, default=value, type=t) 42 | 43 | def extract(self, args): 44 | group = GroupParams() 45 | for arg in vars(args).items(): 46 | if arg[0] in vars(self) or ("_" + arg[0]) in vars(self): 47 | setattr(group, arg[0], arg[1]) 48 | return group 49 | 50 | class ModelParams(ParamGroup): 51 | def __init__(self, parser, sentinel=False): 52 | self.sh_degree = 3 53 | self._source_path = "" 54 | self._model_path = "" 55 | self._images = "images" 56 | self._resolution = -1 57 | self._white_background = False 58 | self.data_device = "cuda" 59 | self.eval = False 60 | super().__init__(parser, "Loading Parameters", sentinel) 61 | 62 | def extract(self, args): 63 | g = super().extract(args) 64 | g.source_path = os.path.abspath(g.source_path) 65 | return g 66 | 67 | class PipelineParams(ParamGroup): 68 | def __init__(self, parser): 69 | self.convert_SHs_python = False 70 | self.compute_cov3D_python = False 71 | self.debug = False 72 | super().__init__(parser, "Pipeline Parameters") 73 | 74 | class SplattingSettings(): 75 | 76 | group_config = None 77 | group_settings = None 78 | settings = ExtendedSettings() 79 | parser = None 80 | render = False 81 | 82 | def __init__(self, parser, render=False): 83 | self.parser = parser 84 | self.render = render 85 | if not render: 86 | self.group_config = parser.add_argument_group("Splatting Config") 87 | self.group_config.add_argument("--splatting_config", type=str) 88 | 89 | bool_ = lambda x: bool(strtobool(x)) 90 | 91 | self.group_settings = parser.add_argument_group("Splatting Settings") 92 | self.group_settings.add_argument("--sort_mode", type=lambda sortmode: SortMode[sortmode], choices=list(SortMode)) 93 | self.group_settings.add_argument("--sort_order", type=lambda sortorder: GlobalSortOrder[sortorder], choices=list(GlobalSortOrder)) 94 | self.group_settings.add_argument("--tile_4x4", type=int, choices=[64], help='only needed if using sort_mode HIER') 95 | self.group_settings.add_argument("--tile_2x2", type=int, choices=[8,12,20], help='only needed if using sort_mode HIER') 96 | self.group_settings.add_argument("--per_pixel", type=int, choices=[1,2,4,8,12,16,20,24], help='if using sort_mode HIER, only {4,8,16} are valid') 97 | self.group_settings.add_argument("--rect_bounding", type=bool_, choices=[True, False], help="Bound 2D Gaussians with a rectangle instead of a circle") 98 | self.group_settings.add_argument("--tight_opacity_bounding", type=bool_, choices=[True, False], help="Bound 2D Gaussians by considering their opacity") 99 | self.group_settings.add_argument("--tile_based_culling", type=bool_, choices=[True, False], help="Cull complete tiles based on opacity") 100 | self.group_settings.add_argument("--hierarchical_4x4_culling", type=bool_, choices=[True, False], help="Cull Gaussians for 4x4 subtiles, only when using sort_mode HIER") 101 | self.group_settings.add_argument("--new_culling", type=bool_, choices=[True, False]) 102 | self.group_settings.add_argument("--load_balancing", type=bool_, choices=[True, False], help="Perform per-tile computations cooperatively (e.g. duplication)") 103 | self.group_settings.add_argument("--proper_ewa_scaling", type=bool_, choices=[True, False], help='Dilation of 2D Gaussians as proposed by Yu et al. ("Mip-Splatting")') 104 | 105 | def get_settings(self, arguments): 106 | # get valid choices from configargparse 107 | config = None 108 | 109 | # load default dict, if passed 110 | if self.render: 111 | cmdlne_string = sys.argv[1:] 112 | args_cmdline = self.parser.parse_args(cmdlne_string) 113 | cfgfilepath = os.path.join(args_cmdline.model_path, "config.json") 114 | print("Looking for splatting config file in", cfgfilepath) 115 | if os.path.exists(cfgfilepath): 116 | print("Config file found: {}".format(cfgfilepath)) 117 | self.settings = ExtendedSettings.from_json(cfgfilepath) 118 | else: 119 | print("No config file found, assuming default values") 120 | else: 121 | for arg in vars(arguments).items(): 122 | if any([arg[0] in z.option_strings[0] for z in self.group_config._group_actions]): 123 | # json passed, load it 124 | if arg[1] is None: 125 | continue 126 | with open(arg[1], 'r') as json_file: 127 | config = json.load(json_file) 128 | self.settings = ExtendedSettings.from_dict(config) 129 | 130 | for arg in vars(arguments).items(): 131 | if any([arg[0] in z.option_strings[0] for z in self.group_settings._group_actions]): 132 | # pass any options which were not given 133 | if arg[1] is None: 134 | continue 135 | self.settings.set_value(arg[0], arg[1]) 136 | 137 | return self.settings 138 | 139 | class OptimizationParams(ParamGroup): 140 | def __init__(self, parser): 141 | self.iterations = 30_000 142 | self.position_lr_init = 0.00016 143 | self.position_lr_final = 0.0000016 144 | self.position_lr_delay_mult = 0.01 145 | self.position_lr_max_steps = 30_000 146 | self.feature_lr = 0.0025 147 | self.opacity_lr = 0.05 148 | self.scaling_lr = 0.005 149 | self.rotation_lr = 0.001 150 | self.percent_dense = 0.01 151 | self.lambda_dssim = 0.2 152 | self.densification_interval = 100 153 | self.opacity_reset_interval = 3000 154 | self.densify_from_iter = 500 155 | self.densify_until_iter = 15_000 156 | self.densify_grad_threshold = 0.0002 157 | self.random_background = False 158 | super().__init__(parser, "Optimization Parameters") 159 | 160 | def get_combined_args(parser : ArgumentParser): 161 | cmdlne_string = sys.argv[1:] 162 | cfgfile_string = "Namespace()" 163 | args_cmdline = parser.parse_args(cmdlne_string) 164 | 165 | try: 166 | cfgfilepath = os.path.join(args_cmdline.model_path, "cfg_args") 167 | print("Looking for config file in", cfgfilepath) 168 | with open(cfgfilepath) as cfg_file: 169 | print("Config file found: {}".format(cfgfilepath)) 170 | cfgfile_string = cfg_file.read() 171 | except TypeError: 172 | print("Config file not found at") 173 | pass 174 | args_cfgfile = eval(cfgfile_string) 175 | 176 | merged_dict = vars(args_cfgfile).copy() 177 | for k,v in vars(args_cmdline).items(): 178 | if v != None: 179 | merged_dict[k] = v 180 | return Namespace(**merged_dict) 181 | -------------------------------------------------------------------------------- /scene/dataset_readers.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | import sys 14 | from PIL import Image 15 | from typing import NamedTuple 16 | from scene.colmap_loader import read_extrinsics_text, read_intrinsics_text, qvec2rotmat, \ 17 | read_extrinsics_binary, read_intrinsics_binary, read_points3D_binary, read_points3D_text 18 | from utils.graphics_utils import getWorld2View2, focal2fov, fov2focal 19 | import numpy as np 20 | import json 21 | from pathlib import Path 22 | from plyfile import PlyData, PlyElement 23 | from utils.sh_utils import SH2RGB 24 | from scene.gaussian_model import BasicPointCloud 25 | 26 | class CameraInfo(NamedTuple): 27 | uid: int 28 | R: np.array 29 | T: np.array 30 | FovY: np.array 31 | FovX: np.array 32 | image: np.array 33 | image_path: str 34 | image_name: str 35 | width: int 36 | height: int 37 | 38 | class SceneInfo(NamedTuple): 39 | point_cloud: BasicPointCloud 40 | train_cameras: list 41 | test_cameras: list 42 | nerf_normalization: dict 43 | ply_path: str 44 | 45 | def getNerfppNorm(cam_info): 46 | def get_center_and_diag(cam_centers): 47 | cam_centers = np.hstack(cam_centers) 48 | avg_cam_center = np.mean(cam_centers, axis=1, keepdims=True) 49 | center = avg_cam_center 50 | dist = np.linalg.norm(cam_centers - center, axis=0, keepdims=True) 51 | diagonal = np.max(dist) 52 | return center.flatten(), diagonal 53 | 54 | cam_centers = [] 55 | 56 | for cam in cam_info: 57 | W2C = getWorld2View2(cam.R, cam.T) 58 | C2W = np.linalg.inv(W2C) 59 | cam_centers.append(C2W[:3, 3:4]) 60 | 61 | center, diagonal = get_center_and_diag(cam_centers) 62 | radius = diagonal * 1.1 63 | 64 | translate = -center 65 | 66 | return {"translate": translate, "radius": radius} 67 | 68 | def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder): 69 | cam_infos = [] 70 | for idx, key in enumerate(cam_extrinsics): 71 | sys.stdout.write('\r') 72 | # the exact output you're looking for: 73 | sys.stdout.write("Reading camera {}/{}".format(idx+1, len(cam_extrinsics))) 74 | sys.stdout.flush() 75 | 76 | extr = cam_extrinsics[key] 77 | intr = cam_intrinsics[extr.camera_id] 78 | height = intr.height 79 | width = intr.width 80 | 81 | uid = intr.id 82 | R = np.transpose(qvec2rotmat(extr.qvec)) 83 | T = np.array(extr.tvec) 84 | 85 | if intr.model=="SIMPLE_PINHOLE": 86 | focal_length_x = intr.params[0] 87 | FovY = focal2fov(focal_length_x, height) 88 | FovX = focal2fov(focal_length_x, width) 89 | elif intr.model=="PINHOLE": 90 | focal_length_x = intr.params[0] 91 | focal_length_y = intr.params[1] 92 | FovY = focal2fov(focal_length_y, height) 93 | FovX = focal2fov(focal_length_x, width) 94 | else: 95 | assert False, "Colmap camera model not handled: only undistorted datasets (PINHOLE or SIMPLE_PINHOLE cameras) supported!" 96 | 97 | image_path = os.path.join(images_folder, os.path.basename(extr.name)) 98 | image_name = os.path.basename(image_path).split(".")[0] 99 | image = Image.open(image_path) 100 | 101 | cam_info = CameraInfo(uid=uid, R=R, T=T, FovY=FovY, FovX=FovX, image=image, 102 | image_path=image_path, image_name=image_name, width=width, height=height) 103 | cam_infos.append(cam_info) 104 | sys.stdout.write('\n') 105 | return cam_infos 106 | 107 | def fetchPly(path): 108 | plydata = PlyData.read(path) 109 | vertices = plydata['vertex'] 110 | positions = np.vstack([vertices['x'], vertices['y'], vertices['z']]).T 111 | colors = np.vstack([vertices['red'], vertices['green'], vertices['blue']]).T / 255.0 112 | normals = np.vstack([vertices['nx'], vertices['ny'], vertices['nz']]).T 113 | return BasicPointCloud(points=positions, colors=colors, normals=normals) 114 | 115 | def storePly(path, xyz, rgb): 116 | # Define the dtype for the structured array 117 | dtype = [('x', 'f4'), ('y', 'f4'), ('z', 'f4'), 118 | ('nx', 'f4'), ('ny', 'f4'), ('nz', 'f4'), 119 | ('red', 'u1'), ('green', 'u1'), ('blue', 'u1')] 120 | 121 | normals = np.zeros_like(xyz) 122 | 123 | elements = np.empty(xyz.shape[0], dtype=dtype) 124 | attributes = np.concatenate((xyz, normals, rgb), axis=1) 125 | elements[:] = list(map(tuple, attributes)) 126 | 127 | # Create the PlyData object and write to file 128 | vertex_element = PlyElement.describe(elements, 'vertex') 129 | ply_data = PlyData([vertex_element]) 130 | ply_data.write(path) 131 | 132 | def readColmapSceneInfo(path, images, eval, llffhold=8): 133 | try: 134 | cameras_extrinsic_file = os.path.join(path, "sparse/0", "images.bin") 135 | cameras_intrinsic_file = os.path.join(path, "sparse/0", "cameras.bin") 136 | cam_extrinsics = read_extrinsics_binary(cameras_extrinsic_file) 137 | cam_intrinsics = read_intrinsics_binary(cameras_intrinsic_file) 138 | except: 139 | cameras_extrinsic_file = os.path.join(path, "sparse/0", "images.txt") 140 | cameras_intrinsic_file = os.path.join(path, "sparse/0", "cameras.txt") 141 | cam_extrinsics = read_extrinsics_text(cameras_extrinsic_file) 142 | cam_intrinsics = read_intrinsics_text(cameras_intrinsic_file) 143 | 144 | reading_dir = "images" if images == None else images 145 | cam_infos_unsorted = readColmapCameras(cam_extrinsics=cam_extrinsics, cam_intrinsics=cam_intrinsics, images_folder=os.path.join(path, reading_dir)) 146 | cam_infos = sorted(cam_infos_unsorted.copy(), key = lambda x : x.image_name) 147 | 148 | if eval: 149 | train_cam_infos = [c for idx, c in enumerate(cam_infos) if idx % llffhold != 0] 150 | test_cam_infos = [c for idx, c in enumerate(cam_infos) if idx % llffhold == 0] 151 | else: 152 | train_cam_infos = cam_infos 153 | test_cam_infos = [] 154 | 155 | nerf_normalization = getNerfppNorm(train_cam_infos) 156 | 157 | ply_path = os.path.join(path, "sparse/0/points3D.ply") 158 | bin_path = os.path.join(path, "sparse/0/points3D.bin") 159 | txt_path = os.path.join(path, "sparse/0/points3D.txt") 160 | if not os.path.exists(ply_path): 161 | print("Converting point3d.bin to .ply, will happen only the first time you open the scene.") 162 | try: 163 | xyz, rgb, _ = read_points3D_binary(bin_path) 164 | except: 165 | xyz, rgb, _ = read_points3D_text(txt_path) 166 | storePly(ply_path, xyz, rgb) 167 | try: 168 | pcd = fetchPly(ply_path) 169 | except: 170 | pcd = None 171 | 172 | scene_info = SceneInfo(point_cloud=pcd, 173 | train_cameras=train_cam_infos, 174 | test_cameras=test_cam_infos, 175 | nerf_normalization=nerf_normalization, 176 | ply_path=ply_path) 177 | return scene_info 178 | 179 | def readCamerasFromTransforms(path, transformsfile, white_background, extension=".png"): 180 | cam_infos = [] 181 | 182 | with open(os.path.join(path, transformsfile)) as json_file: 183 | contents = json.load(json_file) 184 | fovx = contents["camera_angle_x"] 185 | 186 | frames = contents["frames"] 187 | for idx, frame in enumerate(frames): 188 | cam_name = os.path.join(path, frame["file_path"] + extension) 189 | 190 | # NeRF 'transform_matrix' is a camera-to-world transform 191 | c2w = np.array(frame["transform_matrix"]) 192 | # change from OpenGL/Blender camera axes (Y up, Z back) to COLMAP (Y down, Z forward) 193 | c2w[:3, 1:3] *= -1 194 | 195 | # get the world-to-camera transform and set R, T 196 | w2c = np.linalg.inv(c2w) 197 | R = np.transpose(w2c[:3,:3]) # R is stored transposed due to 'glm' in CUDA code 198 | T = w2c[:3, 3] 199 | 200 | image_path = os.path.join(path, cam_name) 201 | image_name = Path(cam_name).stem 202 | image = Image.open(image_path) 203 | 204 | im_data = np.array(image.convert("RGBA")) 205 | 206 | bg = np.array([1,1,1]) if white_background else np.array([0, 0, 0]) 207 | 208 | norm_data = im_data / 255.0 209 | arr = norm_data[:,:,:3] * norm_data[:, :, 3:4] + bg * (1 - norm_data[:, :, 3:4]) 210 | image = Image.fromarray(np.array(arr*255.0, dtype=np.byte), "RGB") 211 | 212 | fovy = focal2fov(fov2focal(fovx, image.size[0]), image.size[1]) 213 | FovY = fovy 214 | FovX = fovx 215 | 216 | cam_infos.append(CameraInfo(uid=idx, R=R, T=T, FovY=FovY, FovX=FovX, image=image, 217 | image_path=image_path, image_name=image_name, width=image.size[0], height=image.size[1])) 218 | 219 | return cam_infos 220 | 221 | def readNerfSyntheticInfo(path, white_background, eval, extension=".png"): 222 | print("Reading Training Transforms") 223 | train_cam_infos = readCamerasFromTransforms(path, "transforms_train.json", white_background, extension) 224 | print("Reading Test Transforms") 225 | test_cam_infos = readCamerasFromTransforms(path, "transforms_test.json", white_background, extension) 226 | 227 | if not eval: 228 | train_cam_infos.extend(test_cam_infos) 229 | test_cam_infos = [] 230 | 231 | nerf_normalization = getNerfppNorm(train_cam_infos) 232 | 233 | ply_path = os.path.join(path, "points3d.ply") 234 | if not os.path.exists(ply_path): 235 | # Since this data set has no colmap data, we start with random points 236 | num_pts = 100_000 237 | print(f"Generating random point cloud ({num_pts})...") 238 | 239 | # We create random points inside the bounds of the synthetic Blender scenes 240 | xyz = np.random.random((num_pts, 3)) * 2.6 - 1.3 241 | shs = np.random.random((num_pts, 3)) / 255.0 242 | pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((num_pts, 3))) 243 | 244 | storePly(ply_path, xyz, SH2RGB(shs) * 255) 245 | try: 246 | pcd = fetchPly(ply_path) 247 | except: 248 | pcd = None 249 | 250 | scene_info = SceneInfo(point_cloud=pcd, 251 | train_cameras=train_cam_infos, 252 | test_cameras=test_cam_infos, 253 | nerf_normalization=nerf_normalization, 254 | ply_path=ply_path) 255 | return scene_info 256 | 257 | sceneLoadTypeCallbacks = { 258 | "Colmap": readColmapSceneInfo, 259 | "Blender" : readNerfSyntheticInfo 260 | } -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import os 13 | import torch 14 | import json 15 | from random import randint 16 | from utils.loss_utils import l1_loss, ssim 17 | from gaussian_renderer import render, network_gui 18 | import sys 19 | from scene import Scene, GaussianModel 20 | from utils.general_utils import safe_state 21 | import uuid 22 | from tqdm import tqdm 23 | from utils.image_utils import psnr 24 | from argparse import ArgumentParser, Namespace 25 | from arguments import ModelParams, PipelineParams, OptimizationParams, SplattingSettings 26 | from diff_gaussian_rasterization import ExtendedSettings 27 | try: 28 | from torch.utils.tensorboard import SummaryWriter 29 | TENSORBOARD_FOUND = True 30 | except ImportError: 31 | TENSORBOARD_FOUND = False 32 | 33 | def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoint_iterations, checkpoint, debug_from, splat_args: ExtendedSettings, opacity_decay: float = 0): 34 | first_iter = 0 35 | tb_writer = prepare_output_and_logger(dataset, splat_args) 36 | gaussians = GaussianModel(dataset.sh_degree) 37 | scene = Scene(dataset, gaussians) 38 | gaussians.training_setup(opt) 39 | if checkpoint: 40 | (model_params, first_iter) = torch.load(checkpoint) 41 | gaussians.restore(model_params, opt) 42 | 43 | bg_color = [1, 1, 1] if dataset.white_background else [0, 0, 0] 44 | background = torch.tensor(bg_color, dtype=torch.float32, device="cuda") 45 | 46 | iter_start = torch.cuda.Event(enable_timing = True) 47 | iter_end = torch.cuda.Event(enable_timing = True) 48 | 49 | viewpoint_stack = None 50 | ema_loss_for_log = 0.0 51 | progress_bar = tqdm(range(first_iter, opt.iterations), desc="Training progress") 52 | first_iter += 1 53 | for iteration in range(first_iter, opt.iterations + 1): 54 | if network_gui.conn == None: 55 | network_gui.try_connect() 56 | while network_gui.conn != None: 57 | try: 58 | net_image_bytes = None 59 | custom_cam, do_training, pipe.convert_SHs_python, pipe.compute_cov3D_python, keep_alive, scaling_modifer = network_gui.receive() 60 | if custom_cam != None: 61 | net_image = render(custom_cam, gaussians, pipe, background, scaling_modifer, splat_args=splat_args)["render"] 62 | net_image_bytes = memoryview((torch.clamp(net_image, min=0, max=1.0) * 255).byte().permute(1, 2, 0).contiguous().cpu().numpy()) 63 | network_gui.send(net_image_bytes, dataset.source_path) 64 | if do_training and ((iteration < int(opt.iterations)) or not keep_alive): 65 | break 66 | except Exception as e: 67 | network_gui.conn = None 68 | 69 | iter_start.record() 70 | 71 | gaussians.update_learning_rate(iteration) 72 | 73 | # Every 1000 its we increase the levels of SH up to a maximum degree 74 | if iteration % 1000 == 0: 75 | gaussians.oneupSHdegree() 76 | 77 | # Pick a random Camera 78 | if not viewpoint_stack: 79 | viewpoint_stack = scene.getTrainCameras().copy() 80 | viewpoint_cam = viewpoint_stack.pop(randint(0, len(viewpoint_stack)-1)) 81 | 82 | # Render 83 | if (iteration - 1) == debug_from: 84 | pipe.debug = True 85 | 86 | bg = torch.rand((3), device="cuda") if opt.random_background else background 87 | 88 | render_pkg = render(viewpoint_cam, gaussians, pipe, bg, splat_args=splat_args) 89 | image, viewspace_point_tensor, visibility_filter, radii = render_pkg["render"], render_pkg["viewspace_points"], render_pkg["visibility_filter"], render_pkg["radii"] 90 | 91 | # Loss 92 | gt_image = viewpoint_cam.original_image.cuda() 93 | Ll1 = l1_loss(image, gt_image) 94 | loss = (1.0 - opt.lambda_dssim) * Ll1 + opt.lambda_dssim * (1.0 - ssim(image, gt_image)) 95 | loss.backward() 96 | 97 | iter_end.record() 98 | 99 | with torch.no_grad(): 100 | # Progress bar 101 | ema_loss_for_log = 0.4 * loss.item() + 0.6 * ema_loss_for_log 102 | if iteration % 10 == 0: 103 | progress_bar.set_postfix({"Loss": f"{ema_loss_for_log:.{7}f}"}) 104 | progress_bar.update(10) 105 | if iteration == opt.iterations: 106 | progress_bar.close() 107 | 108 | # Log and save 109 | training_report(tb_writer, iteration, Ll1, loss, l1_loss, iter_start.elapsed_time(iter_end), testing_iterations, scene, render, (pipe, background, 1, None, splat_args)) 110 | if (iteration in saving_iterations): 111 | print("\n[ITER {}] Saving Gaussians".format(iteration)) 112 | scene.save(iteration) 113 | 114 | # Densification 115 | if iteration < opt.densify_until_iter: 116 | # Keep track of max radii in image-space for pruning 117 | gaussians.max_radii2D[visibility_filter] = torch.max(gaussians.max_radii2D[visibility_filter], radii[visibility_filter]) 118 | gaussians.add_densification_stats(viewspace_point_tensor, visibility_filter) 119 | 120 | if iteration > opt.densify_from_iter and iteration % opt.densification_interval == 0: 121 | size_threshold = 20 if iteration > opt.opacity_reset_interval else None 122 | gaussians.densify_and_prune(opt.densify_grad_threshold, 0.005, scene.cameras_extent, size_threshold) 123 | 124 | if opacity_decay == 0 and iteration % opt.opacity_reset_interval == 0 or (dataset.white_background and iteration == opt.densify_from_iter): 125 | gaussians.reset_opacity() 126 | 127 | if opacity_decay != 0 and iteration % 50 == 0 and iteration > opt.densify_from_iter: 128 | gaussians.decay_opacity(opacity_decay) 129 | 130 | # Optimizer step 131 | if iteration < opt.iterations: 132 | gaussians.optimizer.step() 133 | gaussians.optimizer.zero_grad(set_to_none = True) 134 | 135 | if (iteration in checkpoint_iterations): 136 | print("\n[ITER {}] Saving Checkpoint".format(iteration)) 137 | torch.save((gaussians.capture(), iteration), scene.model_path + "/chkpnt" + str(iteration) + ".pth") 138 | 139 | def prepare_output_and_logger(args, settings: ExtendedSettings): 140 | if not args.model_path: 141 | if os.getenv('OAR_JOB_ID'): 142 | unique_str=os.getenv('OAR_JOB_ID') 143 | else: 144 | unique_str = str(uuid.uuid4()) 145 | args.model_path = os.path.join("./output/", unique_str[0:10]) 146 | 147 | # Set up output folder 148 | print("Output folder: {}".format(args.model_path)) 149 | os.makedirs(args.model_path, exist_ok = True) 150 | with open(os.path.join(args.model_path, "cfg_args"), 'w') as cfg_log_f: 151 | cfg_log_f.write(str(Namespace(**vars(args)))) 152 | 153 | # write config file 154 | with open(os.path.join(args.model_path, "config.json"), 'w') as config_json: 155 | json.dump(settings.to_dict(), config_json) 156 | 157 | # Create Tensorboard writer 158 | tb_writer = None 159 | if TENSORBOARD_FOUND: 160 | tb_writer = SummaryWriter(args.model_path) 161 | else: 162 | print("Tensorboard not available: not logging progress") 163 | return tb_writer 164 | 165 | def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_iterations, scene : Scene, renderFunc, renderArgs): 166 | if tb_writer: 167 | tb_writer.add_scalar('train_loss_patches/l1_loss', Ll1.item(), iteration) 168 | tb_writer.add_scalar('train_loss_patches/total_loss', loss.item(), iteration) 169 | tb_writer.add_scalar('iter_time', elapsed, iteration) 170 | 171 | # Report test and samples of training set 172 | if iteration in testing_iterations: 173 | torch.cuda.empty_cache() 174 | validation_configs = ({'name': 'test', 'cameras' : scene.getTestCameras()}, 175 | {'name': 'train', 'cameras' : [scene.getTrainCameras()[idx % len(scene.getTrainCameras())] for idx in range(5, 30, 5)]}) 176 | 177 | for config in validation_configs: 178 | if config['cameras'] and len(config['cameras']) > 0: 179 | l1_test = 0.0 180 | psnr_test = 0.0 181 | for idx, viewpoint in enumerate(config['cameras']): 182 | image = torch.clamp(renderFunc(viewpoint, scene.gaussians, *renderArgs)["render"], 0.0, 1.0) 183 | gt_image = torch.clamp(viewpoint.original_image.to("cuda"), 0.0, 1.0) 184 | if tb_writer and (idx < 5): 185 | tb_writer.add_images(config['name'] + "_view_{}/render".format(viewpoint.image_name), image[None], global_step=iteration) 186 | if iteration == testing_iterations[0]: 187 | tb_writer.add_images(config['name'] + "_view_{}/ground_truth".format(viewpoint.image_name), gt_image[None], global_step=iteration) 188 | l1_test += l1_loss(image, gt_image).mean().double() 189 | psnr_test += psnr(image, gt_image).mean().double() 190 | psnr_test /= len(config['cameras']) 191 | l1_test /= len(config['cameras']) 192 | print("\n[ITER {}] Evaluating {}: L1 {} PSNR {}".format(iteration, config['name'], l1_test, psnr_test)) 193 | if tb_writer: 194 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - l1_loss', l1_test, iteration) 195 | tb_writer.add_scalar(config['name'] + '/loss_viewpoint - psnr', psnr_test, iteration) 196 | 197 | if tb_writer: 198 | tb_writer.add_histogram("scene/opacity_histogram", scene.gaussians.get_opacity, iteration) 199 | tb_writer.add_histogram("scene/scale_histogram_mean", scene.gaussians.get_scaling.mean(-1, keepdim=True), iteration) 200 | tb_writer.add_histogram("scene/scale_histogram_max", scene.gaussians.get_scaling.max(-1, keepdim=True).values, iteration) 201 | tb_writer.add_histogram("scene/max_radii_2d", scene.gaussians.max_radii2D[..., None], iteration) 202 | tb_writer.add_scalar('total_points', scene.gaussians.get_xyz.shape[0], iteration) 203 | torch.cuda.empty_cache() 204 | 205 | if __name__ == "__main__": 206 | # Set up command line argument parser 207 | parser = ArgumentParser(description="Training script parameters") 208 | lp = ModelParams(parser) 209 | op = OptimizationParams(parser) 210 | pp = PipelineParams(parser) 211 | ss = SplattingSettings(parser) 212 | parser.add_argument('--ip', type=str, default="127.0.0.1") 213 | parser.add_argument('--port', type=int, default=6009) 214 | parser.add_argument('--debug_from', type=int, default=-1) 215 | parser.add_argument('--detect_anomaly', action='store_true', default=False) 216 | parser.add_argument("--test_iterations", nargs="+", type=int, default=[7_000, 12_000, 17_000, 22_000, 30_000]) 217 | parser.add_argument("--save_iterations", nargs="+", type=int, default=[7_000, 30_000]) 218 | parser.add_argument("--quiet", action="store_true") 219 | parser.add_argument("--checkpoint_iterations", nargs="+", type=int, default=[]) 220 | parser.add_argument("--start_checkpoint", type=str, default = None) 221 | parser.add_argument("--opacity_decay", type=float, default=0) 222 | args = parser.parse_args(sys.argv[1:]) 223 | args.save_iterations.append(args.iterations) 224 | 225 | print("Optimizing " + args.model_path) 226 | 227 | # Initialize system state (RNG) 228 | safe_state(args.quiet) 229 | 230 | # Start GUI server, configure and run training 231 | network_gui.init(args.ip, args.port) 232 | torch.autograd.set_detect_anomaly(args.detect_anomaly) 233 | 234 | splat_args = ss.get_settings(args) 235 | 236 | training(lp.extract(args), op.extract(args), pp.extract(args), args.test_iterations, args.save_iterations, args.checkpoint_iterations, args.start_checkpoint, args.debug_from, splat_args, args.opacity_decay) 237 | 238 | # All done 239 | print("\nTraining complete.") 240 | -------------------------------------------------------------------------------- /scene/colmap_loader.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import numpy as np 13 | import collections 14 | import struct 15 | 16 | CameraModel = collections.namedtuple( 17 | "CameraModel", ["model_id", "model_name", "num_params"]) 18 | Camera = collections.namedtuple( 19 | "Camera", ["id", "model", "width", "height", "params"]) 20 | BaseImage = collections.namedtuple( 21 | "Image", ["id", "qvec", "tvec", "camera_id", "name", "xys", "point3D_ids"]) 22 | Point3D = collections.namedtuple( 23 | "Point3D", ["id", "xyz", "rgb", "error", "image_ids", "point2D_idxs"]) 24 | CAMERA_MODELS = { 25 | CameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3), 26 | CameraModel(model_id=1, model_name="PINHOLE", num_params=4), 27 | CameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4), 28 | CameraModel(model_id=3, model_name="RADIAL", num_params=5), 29 | CameraModel(model_id=4, model_name="OPENCV", num_params=8), 30 | CameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8), 31 | CameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12), 32 | CameraModel(model_id=7, model_name="FOV", num_params=5), 33 | CameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4), 34 | CameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5), 35 | CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12) 36 | } 37 | CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) 38 | for camera_model in CAMERA_MODELS]) 39 | CAMERA_MODEL_NAMES = dict([(camera_model.model_name, camera_model) 40 | for camera_model in CAMERA_MODELS]) 41 | 42 | 43 | def qvec2rotmat(qvec): 44 | return np.array([ 45 | [1 - 2 * qvec[2]**2 - 2 * qvec[3]**2, 46 | 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], 47 | 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2]], 48 | [2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], 49 | 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2, 50 | 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1]], 51 | [2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], 52 | 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], 53 | 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2]]) 54 | 55 | def rotmat2qvec(R): 56 | Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat 57 | K = np.array([ 58 | [Rxx - Ryy - Rzz, 0, 0, 0], 59 | [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0], 60 | [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0], 61 | [Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0 62 | eigvals, eigvecs = np.linalg.eigh(K) 63 | qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)] 64 | if qvec[0] < 0: 65 | qvec *= -1 66 | return qvec 67 | 68 | class Image(BaseImage): 69 | def qvec2rotmat(self): 70 | return qvec2rotmat(self.qvec) 71 | 72 | def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"): 73 | """Read and unpack the next bytes from a binary file. 74 | :param fid: 75 | :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. 76 | :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. 77 | :param endian_character: Any of {@, =, <, >, !} 78 | :return: Tuple of read and unpacked values. 79 | """ 80 | data = fid.read(num_bytes) 81 | return struct.unpack(endian_character + format_char_sequence, data) 82 | 83 | def read_points3D_text(path): 84 | """ 85 | see: src/base/reconstruction.cc 86 | void Reconstruction::ReadPoints3DText(const std::string& path) 87 | void Reconstruction::WritePoints3DText(const std::string& path) 88 | """ 89 | xyzs = None 90 | rgbs = None 91 | errors = None 92 | num_points = 0 93 | with open(path, "r") as fid: 94 | while True: 95 | line = fid.readline() 96 | if not line: 97 | break 98 | line = line.strip() 99 | if len(line) > 0 and line[0] != "#": 100 | num_points += 1 101 | 102 | 103 | xyzs = np.empty((num_points, 3)) 104 | rgbs = np.empty((num_points, 3)) 105 | errors = np.empty((num_points, 1)) 106 | count = 0 107 | with open(path, "r") as fid: 108 | while True: 109 | line = fid.readline() 110 | if not line: 111 | break 112 | line = line.strip() 113 | if len(line) > 0 and line[0] != "#": 114 | elems = line.split() 115 | xyz = np.array(tuple(map(float, elems[1:4]))) 116 | rgb = np.array(tuple(map(int, elems[4:7]))) 117 | error = np.array(float(elems[7])) 118 | xyzs[count] = xyz 119 | rgbs[count] = rgb 120 | errors[count] = error 121 | count += 1 122 | 123 | return xyzs, rgbs, errors 124 | 125 | def read_points3D_binary(path_to_model_file): 126 | """ 127 | see: src/base/reconstruction.cc 128 | void Reconstruction::ReadPoints3DBinary(const std::string& path) 129 | void Reconstruction::WritePoints3DBinary(const std::string& path) 130 | """ 131 | 132 | 133 | with open(path_to_model_file, "rb") as fid: 134 | num_points = read_next_bytes(fid, 8, "Q")[0] 135 | 136 | xyzs = np.empty((num_points, 3)) 137 | rgbs = np.empty((num_points, 3)) 138 | errors = np.empty((num_points, 1)) 139 | 140 | for p_id in range(num_points): 141 | binary_point_line_properties = read_next_bytes( 142 | fid, num_bytes=43, format_char_sequence="QdddBBBd") 143 | xyz = np.array(binary_point_line_properties[1:4]) 144 | rgb = np.array(binary_point_line_properties[4:7]) 145 | error = np.array(binary_point_line_properties[7]) 146 | track_length = read_next_bytes( 147 | fid, num_bytes=8, format_char_sequence="Q")[0] 148 | track_elems = read_next_bytes( 149 | fid, num_bytes=8*track_length, 150 | format_char_sequence="ii"*track_length) 151 | xyzs[p_id] = xyz 152 | rgbs[p_id] = rgb 153 | errors[p_id] = error 154 | return xyzs, rgbs, errors 155 | 156 | def read_intrinsics_text(path): 157 | """ 158 | Taken from https://github.com/colmap/colmap/blob/dev/scripts/python/read_write_model.py 159 | """ 160 | cameras = {} 161 | with open(path, "r") as fid: 162 | while True: 163 | line = fid.readline() 164 | if not line: 165 | break 166 | line = line.strip() 167 | if len(line) > 0 and line[0] != "#": 168 | elems = line.split() 169 | camera_id = int(elems[0]) 170 | model = elems[1] 171 | assert model == "PINHOLE", "While the loader support other types, the rest of the code assumes PINHOLE" 172 | width = int(elems[2]) 173 | height = int(elems[3]) 174 | params = np.array(tuple(map(float, elems[4:]))) 175 | cameras[camera_id] = Camera(id=camera_id, model=model, 176 | width=width, height=height, 177 | params=params) 178 | return cameras 179 | 180 | def read_extrinsics_binary(path_to_model_file): 181 | """ 182 | see: src/base/reconstruction.cc 183 | void Reconstruction::ReadImagesBinary(const std::string& path) 184 | void Reconstruction::WriteImagesBinary(const std::string& path) 185 | """ 186 | images = {} 187 | with open(path_to_model_file, "rb") as fid: 188 | num_reg_images = read_next_bytes(fid, 8, "Q")[0] 189 | for _ in range(num_reg_images): 190 | binary_image_properties = read_next_bytes( 191 | fid, num_bytes=64, format_char_sequence="idddddddi") 192 | image_id = binary_image_properties[0] 193 | qvec = np.array(binary_image_properties[1:5]) 194 | tvec = np.array(binary_image_properties[5:8]) 195 | camera_id = binary_image_properties[8] 196 | image_name = "" 197 | current_char = read_next_bytes(fid, 1, "c")[0] 198 | while current_char != b"\x00": # look for the ASCII 0 entry 199 | image_name += current_char.decode("utf-8") 200 | current_char = read_next_bytes(fid, 1, "c")[0] 201 | num_points2D = read_next_bytes(fid, num_bytes=8, 202 | format_char_sequence="Q")[0] 203 | x_y_id_s = read_next_bytes(fid, num_bytes=24*num_points2D, 204 | format_char_sequence="ddq"*num_points2D) 205 | xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])), 206 | tuple(map(float, x_y_id_s[1::3]))]) 207 | point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3]))) 208 | images[image_id] = Image( 209 | id=image_id, qvec=qvec, tvec=tvec, 210 | camera_id=camera_id, name=image_name, 211 | xys=xys, point3D_ids=point3D_ids) 212 | return images 213 | 214 | 215 | def read_intrinsics_binary(path_to_model_file): 216 | """ 217 | see: src/base/reconstruction.cc 218 | void Reconstruction::WriteCamerasBinary(const std::string& path) 219 | void Reconstruction::ReadCamerasBinary(const std::string& path) 220 | """ 221 | cameras = {} 222 | with open(path_to_model_file, "rb") as fid: 223 | num_cameras = read_next_bytes(fid, 8, "Q")[0] 224 | for _ in range(num_cameras): 225 | camera_properties = read_next_bytes( 226 | fid, num_bytes=24, format_char_sequence="iiQQ") 227 | camera_id = camera_properties[0] 228 | model_id = camera_properties[1] 229 | model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name 230 | width = camera_properties[2] 231 | height = camera_properties[3] 232 | num_params = CAMERA_MODEL_IDS[model_id].num_params 233 | params = read_next_bytes(fid, num_bytes=8*num_params, 234 | format_char_sequence="d"*num_params) 235 | cameras[camera_id] = Camera(id=camera_id, 236 | model=model_name, 237 | width=width, 238 | height=height, 239 | params=np.array(params)) 240 | assert len(cameras) == num_cameras 241 | return cameras 242 | 243 | 244 | def read_extrinsics_text(path): 245 | """ 246 | Taken from https://github.com/colmap/colmap/blob/dev/scripts/python/read_write_model.py 247 | """ 248 | images = {} 249 | with open(path, "r") as fid: 250 | while True: 251 | line = fid.readline() 252 | if not line: 253 | break 254 | line = line.strip() 255 | if len(line) > 0 and line[0] != "#": 256 | elems = line.split() 257 | image_id = int(elems[0]) 258 | qvec = np.array(tuple(map(float, elems[1:5]))) 259 | tvec = np.array(tuple(map(float, elems[5:8]))) 260 | camera_id = int(elems[8]) 261 | image_name = elems[9] 262 | elems = fid.readline().split() 263 | xys = np.column_stack([tuple(map(float, elems[0::3])), 264 | tuple(map(float, elems[1::3]))]) 265 | point3D_ids = np.array(tuple(map(int, elems[2::3]))) 266 | images[image_id] = Image( 267 | id=image_id, qvec=qvec, tvec=tvec, 268 | camera_id=camera_id, name=image_name, 269 | xys=xys, point3D_ids=point3D_ids) 270 | return images 271 | 272 | 273 | def read_colmap_bin_array(path): 274 | """ 275 | Taken from https://github.com/colmap/colmap/blob/dev/scripts/python/read_dense.py 276 | 277 | :param path: path to the colmap binary file. 278 | :return: nd array with the floating point values in the value 279 | """ 280 | with open(path, "rb") as fid: 281 | width, height, channels = np.genfromtxt(fid, delimiter="&", max_rows=1, 282 | usecols=(0, 1, 2), dtype=int) 283 | fid.seek(0) 284 | num_delimiter = 0 285 | byte = fid.read(1) 286 | while True: 287 | if byte == b"&": 288 | num_delimiter += 1 289 | if num_delimiter >= 3: 290 | break 291 | byte = fid.read(1) 292 | array = np.fromfile(fid, np.float32) 293 | array = array.reshape((width, height, channels), order="F") 294 | return np.transpose(array, (1, 0, 2)).squeeze() 295 | -------------------------------------------------------------------------------- /scene/gaussian_model.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2023, Inria 3 | # GRAPHDECO research group, https://team.inria.fr/graphdeco 4 | # All rights reserved. 5 | # 6 | # This software is free for non-commercial, research and evaluation use 7 | # under the terms of the LICENSE.md file. 8 | # 9 | # For inquiries contact george.drettakis@inria.fr 10 | # 11 | 12 | import torch 13 | import numpy as np 14 | from utils.general_utils import inverse_sigmoid, get_expon_lr_func, build_rotation 15 | from torch import nn 16 | import os 17 | from utils.system_utils import mkdir_p 18 | from plyfile import PlyData, PlyElement 19 | from utils.sh_utils import RGB2SH 20 | from simple_knn._C import distCUDA2 21 | from utils.graphics_utils import BasicPointCloud 22 | from utils.general_utils import strip_symmetric, build_scaling_rotation 23 | 24 | class GaussianModel: 25 | 26 | def setup_functions(self): 27 | def build_covariance_from_scaling_rotation(scaling, scaling_modifier, rotation): 28 | L = build_scaling_rotation(scaling_modifier * scaling, rotation) 29 | actual_covariance = L @ L.transpose(1, 2) 30 | symm = strip_symmetric(actual_covariance) 31 | return symm 32 | 33 | self.scaling_activation = torch.exp 34 | self.scaling_inverse_activation = torch.log 35 | 36 | self.covariance_activation = build_covariance_from_scaling_rotation 37 | 38 | self.opacity_activation = torch.sigmoid 39 | self.inverse_opacity_activation = inverse_sigmoid 40 | 41 | self.rotation_activation = torch.nn.functional.normalize 42 | 43 | 44 | def __init__(self, sh_degree : int): 45 | self.active_sh_degree = 0 46 | self.max_sh_degree = sh_degree 47 | self._xyz = torch.empty(0) 48 | self._features_dc = torch.empty(0) 49 | self._features_rest = torch.empty(0) 50 | self._scaling = torch.empty(0) 51 | self._rotation = torch.empty(0) 52 | self._opacity = torch.empty(0) 53 | self.max_radii2D = torch.empty(0) 54 | self.xyz_gradient_accum = torch.empty(0) 55 | self.denom = torch.empty(0) 56 | self.optimizer = None 57 | self.percent_dense = 0 58 | self.spatial_lr_scale = 0 59 | self.setup_functions() 60 | 61 | def capture(self): 62 | return ( 63 | self.active_sh_degree, 64 | self._xyz, 65 | self._features_dc, 66 | self._features_rest, 67 | self._scaling, 68 | self._rotation, 69 | self._opacity, 70 | self.max_radii2D, 71 | self.xyz_gradient_accum, 72 | self.denom, 73 | self.optimizer.state_dict(), 74 | self.spatial_lr_scale, 75 | ) 76 | 77 | def restore(self, model_args, training_args): 78 | (self.active_sh_degree, 79 | self._xyz, 80 | self._features_dc, 81 | self._features_rest, 82 | self._scaling, 83 | self._rotation, 84 | self._opacity, 85 | self.max_radii2D, 86 | xyz_gradient_accum, 87 | denom, 88 | opt_dict, 89 | self.spatial_lr_scale) = model_args 90 | self.training_setup(training_args) 91 | self.xyz_gradient_accum = xyz_gradient_accum 92 | self.denom = denom 93 | self.optimizer.load_state_dict(opt_dict) 94 | 95 | @property 96 | def get_scaling(self): 97 | return self.scaling_activation(self._scaling) 98 | 99 | @property 100 | def get_rotation(self): 101 | return self.rotation_activation(self._rotation) 102 | 103 | @property 104 | def get_xyz(self): 105 | return self._xyz 106 | 107 | @property 108 | def get_features(self): 109 | features_dc = self._features_dc 110 | features_rest = self._features_rest 111 | return torch.cat((features_dc, features_rest), dim=1) 112 | 113 | @property 114 | def get_opacity(self): 115 | return self.opacity_activation(self._opacity) 116 | 117 | def get_covariance(self, scaling_modifier = 1): 118 | return self.covariance_activation(self.get_scaling, scaling_modifier, self._rotation) 119 | 120 | def oneupSHdegree(self): 121 | if self.active_sh_degree < self.max_sh_degree: 122 | self.active_sh_degree += 1 123 | 124 | def create_from_pcd(self, pcd : BasicPointCloud, spatial_lr_scale : float): 125 | self.spatial_lr_scale = spatial_lr_scale 126 | fused_point_cloud = torch.tensor(np.asarray(pcd.points)).float().cuda() 127 | fused_color = RGB2SH(torch.tensor(np.asarray(pcd.colors)).float().cuda()) 128 | features = torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)).float().cuda() 129 | features[:, :3, 0 ] = fused_color 130 | features[:, 3:, 1:] = 0.0 131 | 132 | print("Number of points at initialisation : ", fused_point_cloud.shape[0]) 133 | 134 | dist2 = torch.clamp_min(distCUDA2(torch.from_numpy(np.asarray(pcd.points)).float().cuda()), 0.0000001) 135 | scales = torch.log(torch.sqrt(dist2))[...,None].repeat(1, 3) 136 | rots = torch.zeros((fused_point_cloud.shape[0], 4), device="cuda") 137 | rots[:, 0] = 1 138 | 139 | opacities = inverse_sigmoid(0.1 * torch.ones((fused_point_cloud.shape[0], 1), dtype=torch.float, device="cuda")) 140 | 141 | self._xyz = nn.Parameter(fused_point_cloud.requires_grad_(True)) 142 | self._features_dc = nn.Parameter(features[:,:,0:1].transpose(1, 2).contiguous().requires_grad_(True)) 143 | self._features_rest = nn.Parameter(features[:,:,1:].transpose(1, 2).contiguous().requires_grad_(True)) 144 | self._scaling = nn.Parameter(scales.requires_grad_(True)) 145 | self._rotation = nn.Parameter(rots.requires_grad_(True)) 146 | self._opacity = nn.Parameter(opacities.requires_grad_(True)) 147 | self.max_radii2D = torch.zeros((self.get_xyz.shape[0]), device="cuda") 148 | 149 | def training_setup(self, training_args): 150 | self.percent_dense = training_args.percent_dense 151 | self.xyz_gradient_accum = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") 152 | self.denom = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") 153 | 154 | l = [ 155 | {'params': [self._xyz], 'lr': training_args.position_lr_init * self.spatial_lr_scale, "name": "xyz"}, 156 | {'params': [self._features_dc], 'lr': training_args.feature_lr, "name": "f_dc"}, 157 | {'params': [self._features_rest], 'lr': training_args.feature_lr / 20.0, "name": "f_rest"}, 158 | {'params': [self._opacity], 'lr': training_args.opacity_lr, "name": "opacity"}, 159 | {'params': [self._scaling], 'lr': training_args.scaling_lr, "name": "scaling"}, 160 | {'params': [self._rotation], 'lr': training_args.rotation_lr, "name": "rotation"} 161 | ] 162 | 163 | self.optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15) 164 | self.xyz_scheduler_args = get_expon_lr_func(lr_init=training_args.position_lr_init*self.spatial_lr_scale, 165 | lr_final=training_args.position_lr_final*self.spatial_lr_scale, 166 | lr_delay_mult=training_args.position_lr_delay_mult, 167 | max_steps=training_args.position_lr_max_steps) 168 | 169 | def update_learning_rate(self, iteration): 170 | ''' Learning rate scheduling per step ''' 171 | for param_group in self.optimizer.param_groups: 172 | if param_group["name"] == "xyz": 173 | lr = self.xyz_scheduler_args(iteration) 174 | param_group['lr'] = lr 175 | return lr 176 | 177 | def construct_list_of_attributes(self): 178 | l = ['x', 'y', 'z', 'nx', 'ny', 'nz'] 179 | # All channels except the 3 DC 180 | for i in range(self._features_dc.shape[1]*self._features_dc.shape[2]): 181 | l.append('f_dc_{}'.format(i)) 182 | for i in range(self._features_rest.shape[1]*self._features_rest.shape[2]): 183 | l.append('f_rest_{}'.format(i)) 184 | l.append('opacity') 185 | for i in range(self._scaling.shape[1]): 186 | l.append('scale_{}'.format(i)) 187 | for i in range(self._rotation.shape[1]): 188 | l.append('rot_{}'.format(i)) 189 | return l 190 | 191 | def save_ply(self, path): 192 | mkdir_p(os.path.dirname(path)) 193 | 194 | xyz = self._xyz.detach().cpu().numpy() 195 | normals = np.zeros_like(xyz) 196 | f_dc = self._features_dc.detach().transpose(1, 2).flatten(start_dim=1).contiguous().cpu().numpy() 197 | f_rest = self._features_rest.detach().transpose(1, 2).flatten(start_dim=1).contiguous().cpu().numpy() 198 | opacities = self._opacity.detach().cpu().numpy() 199 | scale = self._scaling.detach().cpu().numpy() 200 | rotation = self._rotation.detach().cpu().numpy() 201 | 202 | dtype_full = [(attribute, 'f4') for attribute in self.construct_list_of_attributes()] 203 | 204 | elements = np.empty(xyz.shape[0], dtype=dtype_full) 205 | attributes = np.concatenate((xyz, normals, f_dc, f_rest, opacities, scale, rotation), axis=1) 206 | elements[:] = list(map(tuple, attributes)) 207 | el = PlyElement.describe(elements, 'vertex') 208 | PlyData([el]).write(path) 209 | 210 | def decay_opacity(self, val=0.999): 211 | opacities_new = inverse_sigmoid(self.get_opacity * val) 212 | optimizable_tensors = self.replace_tensor_to_optimizer(opacities_new, "opacity") 213 | self._opacity = optimizable_tensors["opacity"] 214 | 215 | def reset_opacity(self): 216 | opacities_new = inverse_sigmoid(torch.min(self.get_opacity, torch.ones_like(self.get_opacity)*0.01)) 217 | optimizable_tensors = self.replace_tensor_to_optimizer(opacities_new, "opacity") 218 | self._opacity = optimizable_tensors["opacity"] 219 | 220 | def load_ply(self, path): 221 | plydata = PlyData.read(path) 222 | 223 | xyz = np.stack((np.asarray(plydata.elements[0]["x"]), 224 | np.asarray(plydata.elements[0]["y"]), 225 | np.asarray(plydata.elements[0]["z"])), axis=1) 226 | opacities = np.asarray(plydata.elements[0]["opacity"])[..., np.newaxis] 227 | 228 | features_dc = np.zeros((xyz.shape[0], 3, 1)) 229 | features_dc[:, 0, 0] = np.asarray(plydata.elements[0]["f_dc_0"]) 230 | features_dc[:, 1, 0] = np.asarray(plydata.elements[0]["f_dc_1"]) 231 | features_dc[:, 2, 0] = np.asarray(plydata.elements[0]["f_dc_2"]) 232 | 233 | extra_f_names = [p.name for p in plydata.elements[0].properties if p.name.startswith("f_rest_")] 234 | extra_f_names = sorted(extra_f_names, key = lambda x: int(x.split('_')[-1])) 235 | assert len(extra_f_names)==3*(self.max_sh_degree + 1) ** 2 - 3 236 | features_extra = np.zeros((xyz.shape[0], len(extra_f_names))) 237 | for idx, attr_name in enumerate(extra_f_names): 238 | features_extra[:, idx] = np.asarray(plydata.elements[0][attr_name]) 239 | # Reshape (P,F*SH_coeffs) to (P, F, SH_coeffs except DC) 240 | features_extra = features_extra.reshape((features_extra.shape[0], 3, (self.max_sh_degree + 1) ** 2 - 1)) 241 | 242 | scale_names = [p.name for p in plydata.elements[0].properties if p.name.startswith("scale_")] 243 | scale_names = sorted(scale_names, key = lambda x: int(x.split('_')[-1])) 244 | scales = np.zeros((xyz.shape[0], len(scale_names))) 245 | for idx, attr_name in enumerate(scale_names): 246 | scales[:, idx] = np.asarray(plydata.elements[0][attr_name]) 247 | 248 | rot_names = [p.name for p in plydata.elements[0].properties if p.name.startswith("rot")] 249 | rot_names = sorted(rot_names, key = lambda x: int(x.split('_')[-1])) 250 | rots = np.zeros((xyz.shape[0], len(rot_names))) 251 | for idx, attr_name in enumerate(rot_names): 252 | rots[:, idx] = np.asarray(plydata.elements[0][attr_name]) 253 | 254 | self._xyz = nn.Parameter(torch.tensor(xyz, dtype=torch.float, device="cuda").requires_grad_(True)) 255 | self._features_dc = nn.Parameter(torch.tensor(features_dc, dtype=torch.float, device="cuda").transpose(1, 2).contiguous().requires_grad_(True)) 256 | self._features_rest = nn.Parameter(torch.tensor(features_extra, dtype=torch.float, device="cuda").transpose(1, 2).contiguous().requires_grad_(True)) 257 | self._opacity = nn.Parameter(torch.tensor(opacities, dtype=torch.float, device="cuda").requires_grad_(True)) 258 | self._scaling = nn.Parameter(torch.tensor(scales, dtype=torch.float, device="cuda").requires_grad_(True)) 259 | self._rotation = nn.Parameter(torch.tensor(rots, dtype=torch.float, device="cuda").requires_grad_(True)) 260 | 261 | self.active_sh_degree = self.max_sh_degree 262 | 263 | def replace_tensor_to_optimizer(self, tensor, name): 264 | optimizable_tensors = {} 265 | for group in self.optimizer.param_groups: 266 | if group["name"] == name: 267 | stored_state = self.optimizer.state.get(group['params'][0], None) 268 | stored_state["exp_avg"] = torch.zeros_like(tensor) 269 | stored_state["exp_avg_sq"] = torch.zeros_like(tensor) 270 | 271 | del self.optimizer.state[group['params'][0]] 272 | group["params"][0] = nn.Parameter(tensor.requires_grad_(True)) 273 | self.optimizer.state[group['params'][0]] = stored_state 274 | 275 | optimizable_tensors[group["name"]] = group["params"][0] 276 | return optimizable_tensors 277 | 278 | def _prune_optimizer(self, mask): 279 | optimizable_tensors = {} 280 | for group in self.optimizer.param_groups: 281 | stored_state = self.optimizer.state.get(group['params'][0], None) 282 | if stored_state is not None: 283 | stored_state["exp_avg"] = stored_state["exp_avg"][mask] 284 | stored_state["exp_avg_sq"] = stored_state["exp_avg_sq"][mask] 285 | 286 | del self.optimizer.state[group['params'][0]] 287 | group["params"][0] = nn.Parameter((group["params"][0][mask].requires_grad_(True))) 288 | self.optimizer.state[group['params'][0]] = stored_state 289 | 290 | optimizable_tensors[group["name"]] = group["params"][0] 291 | else: 292 | group["params"][0] = nn.Parameter(group["params"][0][mask].requires_grad_(True)) 293 | optimizable_tensors[group["name"]] = group["params"][0] 294 | return optimizable_tensors 295 | 296 | def prune_points(self, mask): 297 | valid_points_mask = ~mask 298 | optimizable_tensors = self._prune_optimizer(valid_points_mask) 299 | 300 | self._xyz = optimizable_tensors["xyz"] 301 | self._features_dc = optimizable_tensors["f_dc"] 302 | self._features_rest = optimizable_tensors["f_rest"] 303 | self._opacity = optimizable_tensors["opacity"] 304 | self._scaling = optimizable_tensors["scaling"] 305 | self._rotation = optimizable_tensors["rotation"] 306 | 307 | self.xyz_gradient_accum = self.xyz_gradient_accum[valid_points_mask] 308 | 309 | self.denom = self.denom[valid_points_mask] 310 | self.max_radii2D = self.max_radii2D[valid_points_mask] 311 | 312 | def cat_tensors_to_optimizer(self, tensors_dict): 313 | optimizable_tensors = {} 314 | for group in self.optimizer.param_groups: 315 | assert len(group["params"]) == 1 316 | extension_tensor = tensors_dict[group["name"]] 317 | stored_state = self.optimizer.state.get(group['params'][0], None) 318 | if stored_state is not None: 319 | 320 | stored_state["exp_avg"] = torch.cat((stored_state["exp_avg"], torch.zeros_like(extension_tensor)), dim=0) 321 | stored_state["exp_avg_sq"] = torch.cat((stored_state["exp_avg_sq"], torch.zeros_like(extension_tensor)), dim=0) 322 | 323 | del self.optimizer.state[group['params'][0]] 324 | group["params"][0] = nn.Parameter(torch.cat((group["params"][0], extension_tensor), dim=0).requires_grad_(True)) 325 | self.optimizer.state[group['params'][0]] = stored_state 326 | 327 | optimizable_tensors[group["name"]] = group["params"][0] 328 | else: 329 | group["params"][0] = nn.Parameter(torch.cat((group["params"][0], extension_tensor), dim=0).requires_grad_(True)) 330 | optimizable_tensors[group["name"]] = group["params"][0] 331 | 332 | return optimizable_tensors 333 | 334 | def densification_postfix(self, new_xyz, new_features_dc, new_features_rest, new_opacities, new_scaling, new_rotation): 335 | d = {"xyz": new_xyz, 336 | "f_dc": new_features_dc, 337 | "f_rest": new_features_rest, 338 | "opacity": new_opacities, 339 | "scaling" : new_scaling, 340 | "rotation" : new_rotation} 341 | 342 | optimizable_tensors = self.cat_tensors_to_optimizer(d) 343 | self._xyz = optimizable_tensors["xyz"] 344 | self._features_dc = optimizable_tensors["f_dc"] 345 | self._features_rest = optimizable_tensors["f_rest"] 346 | self._opacity = optimizable_tensors["opacity"] 347 | self._scaling = optimizable_tensors["scaling"] 348 | self._rotation = optimizable_tensors["rotation"] 349 | 350 | self.xyz_gradient_accum = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") 351 | self.denom = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") 352 | self.max_radii2D = torch.zeros((self.get_xyz.shape[0]), device="cuda") 353 | 354 | def densify_and_split(self, grads, grad_threshold, scene_extent, N=2): 355 | n_init_points = self.get_xyz.shape[0] 356 | # Extract points that satisfy the gradient condition 357 | padded_grad = torch.zeros((n_init_points), device="cuda") 358 | padded_grad[:grads.shape[0]] = grads.squeeze() 359 | selected_pts_mask = torch.where(padded_grad >= grad_threshold, True, False) 360 | selected_pts_mask = torch.logical_and(selected_pts_mask, 361 | torch.max(self.get_scaling, dim=1).values > self.percent_dense*scene_extent) 362 | 363 | stds = self.get_scaling[selected_pts_mask].repeat(N,1) 364 | means =torch.zeros((stds.size(0), 3),device="cuda") 365 | samples = torch.normal(mean=means, std=stds) 366 | rots = build_rotation(self._rotation[selected_pts_mask]).repeat(N,1,1) 367 | new_xyz = torch.bmm(rots, samples.unsqueeze(-1)).squeeze(-1) + self.get_xyz[selected_pts_mask].repeat(N, 1) 368 | new_scaling = self.scaling_inverse_activation(self.get_scaling[selected_pts_mask].repeat(N,1) / (0.8*N)) 369 | new_rotation = self._rotation[selected_pts_mask].repeat(N,1) 370 | new_features_dc = self._features_dc[selected_pts_mask].repeat(N,1,1) 371 | new_features_rest = self._features_rest[selected_pts_mask].repeat(N,1,1) 372 | new_opacity = self._opacity[selected_pts_mask].repeat(N,1) 373 | 374 | self.densification_postfix(new_xyz, new_features_dc, new_features_rest, new_opacity, new_scaling, new_rotation) 375 | 376 | prune_filter = torch.cat((selected_pts_mask, torch.zeros(N * selected_pts_mask.sum(), device="cuda", dtype=bool))) 377 | self.prune_points(prune_filter) 378 | 379 | def densify_and_clone(self, grads, grad_threshold, scene_extent): 380 | # Extract points that satisfy the gradient condition 381 | selected_pts_mask = torch.where(torch.norm(grads, dim=-1) >= grad_threshold, True, False) 382 | selected_pts_mask = torch.logical_and(selected_pts_mask, 383 | torch.max(self.get_scaling, dim=1).values <= self.percent_dense*scene_extent) 384 | 385 | new_xyz = self._xyz[selected_pts_mask] 386 | new_features_dc = self._features_dc[selected_pts_mask] 387 | new_features_rest = self._features_rest[selected_pts_mask] 388 | new_opacities = self._opacity[selected_pts_mask] 389 | new_scaling = self._scaling[selected_pts_mask] 390 | new_rotation = self._rotation[selected_pts_mask] 391 | 392 | self.densification_postfix(new_xyz, new_features_dc, new_features_rest, new_opacities, new_scaling, new_rotation) 393 | 394 | def densify_and_prune(self, max_grad, min_opacity, extent, max_screen_size): 395 | grads = self.xyz_gradient_accum / self.denom 396 | grads[grads.isnan()] = 0.0 397 | 398 | self.densify_and_clone(grads, max_grad, extent) 399 | self.densify_and_split(grads, max_grad, extent) 400 | 401 | prune_mask = (self.get_opacity < min_opacity).squeeze() 402 | if max_screen_size: 403 | big_points_vs = self.max_radii2D > max_screen_size 404 | big_points_ws = self.get_scaling.max(dim=1).values > 0.1 * extent 405 | prune_mask = torch.logical_or(torch.logical_or(prune_mask, big_points_vs), big_points_ws) 406 | self.prune_points(prune_mask) 407 | 408 | torch.cuda.empty_cache() 409 | 410 | def add_densification_stats(self, viewspace_point_tensor, update_filter): 411 | self.xyz_gradient_accum[update_filter] += torch.norm(viewspace_point_tensor.grad[update_filter,:2], dim=-1, keepdim=True) 412 | self.denom[update_filter] += 1 --------------------------------------------------------------------------------