├── .gitignore ├── CAM_1D ├── 1D function.ipynb ├── metrics.py └── network.py ├── CAM_ffnerv ├── LICENSE.txt ├── README.md ├── datasets │ └── README.md ├── main.py ├── model.py ├── requirements.txt └── utils.py ├── CAM_image.py ├── CAM_mipnerf ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── configs │ ├── blender.gin │ ├── blender_noextras.gin │ ├── blender_noipe.gin │ ├── llff.gin │ ├── multiblender.gin │ ├── multiblender_noextras.gin │ ├── multiblender_noipe.gin │ └── multiblender_noloss.gin ├── eval.py ├── internal │ ├── datasets.py │ ├── math.py │ ├── math_test.py │ ├── mip.py │ ├── mip_test.py │ ├── models.py │ ├── utils.py │ └── vis.py ├── requirements.txt ├── scripts │ ├── convert_blender_data.py │ ├── eval_blender.sh │ ├── eval_llff.sh │ ├── eval_multiblender.sh │ ├── summarize.ipynb │ ├── train_blender.sh │ ├── train_llff.sh │ └── train_multiblender.sh └── train.py ├── CAM_multinerf-mipnerf360 ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── configs │ ├── .ipynb_checkpoints │ │ ├── 360-checkpoint.gin │ │ ├── 360_glo4-checkpoint.gin │ │ └── render_config-checkpoint.gin │ ├── 360.gin │ ├── 360_glo4.gin │ ├── blender_256.gin │ ├── blender_512.gin │ ├── blender_refnerf.gin │ ├── debug.gin │ ├── llff_256.gin │ ├── llff_512.gin │ ├── llff_raw.gin │ ├── llff_raw_test.gin │ ├── render_config.gin │ └── tat.gin ├── eval.py ├── internal │ ├── camera_utils.py │ ├── configs.py │ ├── coord.py │ ├── datasets.py │ ├── geopoly.py │ ├── image.py │ ├── math.py │ ├── models.py │ ├── raw_utils.py │ ├── ref_utils.py │ ├── render.py │ ├── stepfun.py │ ├── train_utils.py │ ├── utils.py │ └── vis.py ├── render.py ├── requirements.txt ├── scripts │ ├── eval_360.sh │ ├── eval_blender.sh │ ├── eval_llff.sh │ ├── eval_raw.sh │ ├── eval_shinyblender.sh │ ├── generate_tables.ipynb │ ├── local_colmap_and_resize.sh │ ├── render_360.sh │ ├── render_llff.sh │ ├── render_raw.sh │ ├── run_all_unit_tests.sh │ ├── train_360.sh │ ├── train_blender.sh │ ├── train_llff.sh │ ├── train_raw.sh │ └── train_shinyblender.sh ├── tests │ ├── camera_utils_test.py │ ├── coord_test.py │ ├── datasets_test.py │ ├── geopoly_test.py │ ├── image_test.py │ ├── math_test.py │ ├── ref_utils_test.py │ ├── render_test.py │ ├── stepfun_test.py │ └── utils_test.py └── train.py ├── CAM_nerfacc ├── .github │ └── workflows │ │ ├── aws │ │ └── update_index.py │ │ ├── building.yml │ │ ├── code_checks.yml │ │ ├── cuda │ │ ├── Linux-env.sh │ │ ├── Linux.sh │ │ ├── Windows-env.sh │ │ └── Windows.sh │ │ └── publish.yml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGELOG.md ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── examples │ ├── datasets │ │ ├── __init__.py │ │ ├── dnerf_synthetic.py │ │ ├── nerf_360_v2.py │ │ ├── nerf_synthetic.py │ │ └── utils.py │ ├── pycolmap │ │ ├── .gitignore │ │ ├── LICENSE.txt │ │ ├── README.md │ │ ├── pycolmap │ │ │ ├── __init__.py │ │ │ ├── camera.py │ │ │ ├── database.py │ │ │ ├── image.py │ │ │ ├── rotation.py │ │ │ └── scene_manager.py │ │ └── tools │ │ │ ├── colmap_to_nvm.py │ │ │ ├── delete_images.py │ │ │ ├── impute_missing_cameras.py │ │ │ ├── save_cameras_as_ply.py │ │ │ ├── transform_model.py │ │ │ ├── write_camera_track_to_bundler.py │ │ │ └── write_depthmap_to_ply.py │ ├── radiance_fields │ │ ├── __init__.py │ │ └── mlp.py │ ├── requirements.txt │ ├── train_mlp_nerf.py │ ├── train_mlp_tnerf.py │ └── utils.py ├── nerfacc │ ├── __init__.py │ ├── cameras.py │ ├── cameras2.py │ ├── cuda │ │ ├── __init__.py │ │ ├── _backend.py │ │ └── csrc │ │ │ ├── camera.cu │ │ │ ├── grid.cu │ │ │ ├── include │ │ │ ├── data_spec.hpp │ │ │ ├── data_spec_packed.cuh │ │ │ ├── utils_camera.cuh │ │ │ ├── utils_contraction.cuh │ │ │ ├── utils_cuda.cuh │ │ │ ├── utils_grid.cuh │ │ │ ├── utils_math.cuh │ │ │ └── utils_scan.cuh │ │ │ ├── nerfacc.cpp │ │ │ ├── pdf.cu │ │ │ └── scan.cu │ ├── data_specs.py │ ├── estimators │ │ ├── __init__.py │ │ ├── base.py │ │ ├── occ_grid.py │ │ └── prop_net.py │ ├── grid.py │ ├── pack.py │ ├── pdf.py │ ├── scan.py │ ├── version.py │ └── volrend.py ├── scripts │ ├── run_aws_listing.py │ ├── run_dev_checks.py │ └── run_profiler.py ├── setup.cfg └── setup.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.DS_Store* 3 | -------------------------------------------------------------------------------- /CAM_1D/metrics.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | def _binarize(y_data, threshold): 7 | """ 8 | args: 9 | y_data : [float] 4-d tensor in [batch_size, channels, img_rows, img_cols] 10 | threshold : [float] [0.0, 1.0] 11 | return 4-d binarized y_data 12 | """ 13 | y_data[y_data < threshold] = 0.0 14 | y_data[y_data >= threshold] = 1.0 15 | return y_data 16 | 17 | class MSE(object): 18 | def __init__(self, des="Mean Square Error"): 19 | self.des = des 20 | 21 | def __repr__(self): 22 | return "MSE" 23 | 24 | def __call__(self, y_pred, y_true, dim=1, threshold=None): 25 | """ 26 | args: 27 | y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols] 28 | y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols] 29 | threshold : [0.0, 1.0] 30 | return mean_squared_error, smaller the better 31 | """ 32 | if threshold: 33 | y_pred = _binarize(y_pred, threshold) 34 | return torch.mean((y_pred - y_true) ** 2) 35 | 36 | 37 | class PSNR(object): 38 | def __init__(self, des="Peak Signal to Noise Ratio"): 39 | self.des = des 40 | 41 | def __repr__(self): 42 | return "PSNR" 43 | 44 | def __call__(self, y_pred, y_true, dim=1, threshold=None): 45 | """ 46 | args: 47 | y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols] 48 | y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols] 49 | threshold : [0.0, 1.0] 50 | return PSNR, larger the better 51 | """ 52 | if threshold: 53 | y_pred = _binarize(y_pred, threshold) 54 | mse = torch.mean((y_pred - y_true) ** 2) 55 | return 10 * torch.log10(1 / mse) 56 | 57 | 58 | class SSIM(object): 59 | ''' 60 | modified from https://github.com/jorge-pessoa/pytorch-msssim 61 | ''' 62 | def __init__(self, des="structural similarity index"): 63 | self.des = des 64 | 65 | def __repr__(self): 66 | return "SSIM" 67 | 68 | def gaussian(self, w_size, sigma): 69 | gauss = torch.Tensor([math.exp(-(x - w_size//2)**2/float(2*sigma**2)) for x in range(w_size)]) 70 | return gauss/gauss.sum() 71 | 72 | def create_window(self, w_size, channel=1): 73 | _1D_window = self.gaussian(w_size, 1.5).unsqueeze(1) 74 | _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0) 75 | window = _2D_window.expand(channel, 1, w_size, w_size).contiguous() 76 | return window 77 | 78 | def __call__(self, y_pred, y_true, w_size=11, size_average=True, full=False): 79 | """ 80 | args: 81 | y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols] 82 | y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols] 83 | w_size : int, default 11 84 | size_average : boolean, default True 85 | full : boolean, default False 86 | return ssim, larger the better 87 | """ 88 | # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh). 89 | if torch.max(y_pred) > 128: 90 | max_val = 255 91 | else: 92 | max_val = 1 93 | 94 | if torch.min(y_pred) < -0.5: 95 | min_val = -1 96 | else: 97 | min_val = 0 98 | L = max_val - min_val 99 | 100 | padd = 0 101 | (_, channel, height, width) = y_pred.size() 102 | window = self.create_window(w_size, channel=channel).to(y_pred.device) 103 | 104 | mu1 = F.conv2d(y_pred, window, padding=padd, groups=channel) 105 | mu2 = F.conv2d(y_true, window, padding=padd, groups=channel) 106 | 107 | mu1_sq = mu1.pow(2) 108 | mu2_sq = mu2.pow(2) 109 | mu1_mu2 = mu1 * mu2 110 | 111 | sigma1_sq = F.conv2d(y_pred * y_pred, window, padding=padd, groups=channel) - mu1_sq 112 | sigma2_sq = F.conv2d(y_true * y_true, window, padding=padd, groups=channel) - mu2_sq 113 | sigma12 = F.conv2d(y_pred * y_true, window, padding=padd, groups=channel) - mu1_mu2 114 | 115 | C1 = (0.01 * L) ** 2 116 | C2 = (0.03 * L) ** 2 117 | 118 | v1 = 2.0 * sigma12 + C2 119 | v2 = sigma1_sq + sigma2_sq + C2 120 | cs = torch.mean(v1 / v2) # contrast sensitivity 121 | 122 | ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2) 123 | 124 | if size_average: 125 | ret = ssim_map.mean() 126 | else: 127 | ret = ssim_map.mean(1).mean(1).mean(1) 128 | 129 | if full: 130 | return ret, cs 131 | return ret -------------------------------------------------------------------------------- /CAM_1D/network.py: -------------------------------------------------------------------------------- 1 | # import re 2 | import numpy as np 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import math 8 | 9 | class SinusoidalEncoder(nn.Module): 10 | """Sinusoidal Positional Encoder used in Nerf.""" 11 | 12 | def __init__(self, x_dim, min_deg, max_deg, use_identity: bool = True): 13 | super().__init__() 14 | self.x_dim = x_dim 15 | self.min_deg = min_deg 16 | self.max_deg = max_deg 17 | self.use_identity = use_identity 18 | self.register_buffer( 19 | "scales", torch.tensor([2**i for i in range(min_deg, max_deg)]) 20 | ) 21 | 22 | @property 23 | def latent_dim(self) -> int: 24 | return ( 25 | int(self.use_identity) + (self.max_deg - self.min_deg) * 2 26 | ) * self.x_dim 27 | 28 | def forward(self, x: torch.Tensor) -> torch.Tensor: 29 | """ 30 | Args: 31 | x: [..., x_dim] 32 | Returns: 33 | latent: [..., latent_dim] 34 | """ 35 | if self.max_deg == self.min_deg: 36 | return x 37 | xb = torch.reshape( 38 | (x[Ellipsis, None, :] * self.scales[:, None]), 39 | list(x.shape[:-1]) + [(self.max_deg - self.min_deg) * self.x_dim], 40 | ) 41 | latent = torch.sin(torch.cat([xb, xb + 0.5 * math.pi], dim=-1)) 42 | if self.use_identity: 43 | latent = torch.cat([x] + [latent], dim=-1) 44 | return latent 45 | 46 | class CAM_PE(nn.Module): 47 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats): 48 | super(CAM_PE, self).__init__() 49 | 50 | self.time_encoder = SinusoidalEncoder(1, 0, 16, True) 51 | self.net = [nn.Linear(self.time_encoder.latent_dim, hidden_feats)] + \ 52 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)] 53 | self.net_last = nn.Linear(hidden_feats, out_feats) 54 | 55 | self.net = nn.ModuleList(self.net) 56 | 57 | self.affine = nn.ParameterList() 58 | for i in range(n_hidden_layers+1): 59 | self.affine.append(torch.nn.Parameter(torch.cat([torch.ones((1, 1, 64, 1)), torch.zeros((1, 1, 64, 1))],dim=1))) 60 | 61 | 62 | def forward(self, x): 63 | coord = x.clone()*2-1.0 64 | coord = torch.cat((torch.zeros_like(coord),coord), dim=-1) 65 | x = self.time_encoder(x) 66 | for i, l in enumerate(self.net): 67 | x = l(x) 68 | affine = F.grid_sample(self.affine[i], coord.view(1,-1,1,2), align_corners=True).view(2,-1,1) 69 | x = affine[0]*x + affine[1] 70 | x = self.net_last(x) 71 | return x 72 | 73 | class CAM(nn.Module): 74 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats): 75 | super(CAM, self).__init__() 76 | 77 | self.net = [nn.Linear(1, hidden_feats)] + \ 78 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)] 79 | self.net_last = nn.Linear(hidden_feats, out_feats) 80 | self.net = nn.ModuleList(self.net) 81 | 82 | self.affine = nn.ParameterList() 83 | for i in range(n_hidden_layers+1): 84 | self.affine.append(torch.nn.Parameter(torch.cat([torch.ones((1, 1, 64, 1)), torch.zeros((1, 1, 64, 1))],dim=1))) 85 | 86 | 87 | def forward(self, x): 88 | coord = x.clone()*2-1.0 89 | coord = torch.cat((torch.zeros_like(coord),coord), dim=-1) 90 | for i, l in enumerate(self.net): 91 | x = l(x) 92 | affine = F.grid_sample(self.affine[i], coord.view(1,-1,1,2), align_corners=True).view(2,-1,1) 93 | x = affine[0]*x + affine[1] 94 | x = self.net_last(x) 95 | return x 96 | 97 | class PE(nn.Module): 98 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats): 99 | super(PE, self).__init__() 100 | 101 | self.time_encoder = SinusoidalEncoder(1, 0, 16, True) 102 | self.net = [nn.Linear(self.time_encoder.latent_dim, hidden_feats)] + \ 103 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)] 104 | self.net_last = nn.Linear(hidden_feats, out_feats) 105 | self.net = nn.ModuleList(self.net) 106 | 107 | 108 | def forward(self, x): 109 | x = self.time_encoder(x) 110 | for i, l in enumerate(self.net): 111 | x = l(x) 112 | x = self.net_last(x) 113 | return x 114 | 115 | class NoEnc(nn.Module): 116 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats): 117 | super(NoEnc, self).__init__() 118 | 119 | self.net = [nn.Linear(1, hidden_feats)] + \ 120 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)] 121 | self.net_last = nn.Linear(hidden_feats, out_feats) 122 | self.net = nn.ModuleList(self.net) 123 | 124 | 125 | def forward(self, x): 126 | for i, l in enumerate(self.net): 127 | x = l(x) 128 | x = self.net_last(x) 129 | return x -------------------------------------------------------------------------------- /CAM_ffnerv/LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Joo Chan Lee 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /CAM_ffnerv/README.md: -------------------------------------------------------------------------------- 1 | # FFNeRV: Flow-Guided Frame-Wise Neural Representations for Videos 2 | #### Joo Chan Lee, Daniel Rho, Jong Hwan Ko†, and Eunbyung Park† 3 | 4 | ### [[Project Page](https://maincold2.github.io/ffnerv/)] [[Paper(arxiv)](https://arxiv.org/abs/2212.12294)] 5 | 6 | Our code is based on [NeRV](https://github.com/haochen-rye/NeRV) 7 | 8 | ## Method overview 9 | 10 | 11 | ## 0. Requirements 12 | ``` 13 | pip install -r requirements.txt 14 | ``` 15 | (Update) Highly recommend upgrading the Pytorch version (>= 1.13.0) for faster training and inference. 16 | 17 | Dataset link: [UVG](https://ultravideo.fi/#testsequences) 18 | 19 | ## 1. Training (Encoding) 20 | ### For representation 21 | ```bash 22 | python main.py -e 300 --lower-width 96 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_156 --expansion 1 --loss Fusion6 --strides 5 2 2 2 2 --conv-type conv -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --t-dim 64 128 256 512 23 | ``` 24 | ### For compression 25 | ```bash 26 | python main.py -e 600 --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_48 --expansion 8 --loss Fusion6 --strides 5 3 2 2 2 --conv-type compact -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --wbit 8 --t-dim 300 600 --resol 1920 1080 27 | ``` 28 | - "--conv-type": the type of convolution blocks 29 | - "--agg-ind": relative video indices for aggregation 30 | - "--wbit": for n-bit quantization (QAT) 31 | - "--t-dim": multi-channels of temporal grids 32 | 33 | More details can be found in "main.py" 34 | 35 | 36 | ## 2. Evaluation (+ Entropy coding) 37 | ```bash 38 | python main.py --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_48 --expansion 8 --strides 5 3 2 2 2 --conv-type compact -b 1 --agg-ind -2 -1 1 2 --wbit 8 --t-dim 300 600 --resol 1920 1080 --eval-only 39 | ``` 40 | 41 | ## 3. Decoding 42 | ```bash 43 | python main.py --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_48 --expansion 8 --strides 5 3 2 2 2 --conv-type compact -b 1 --agg-ind -2 -1 1 2 --wbit 8 --t-dim 300 600 --resol 1920 1080 --eval-only --weight [weight_path] --dump-images 44 | ``` 45 | -------------------------------------------------------------------------------- /CAM_ffnerv/datasets/README.md: -------------------------------------------------------------------------------- 1 | Video directory 2 | 3 | -------------------------------------------------------------------------------- /CAM_ffnerv/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.13.0 2 | numpy==1.19.2 3 | pytorch_msssim==0.2.1 4 | tqdm==4.59.0 5 | torchvision==0.9 6 | dahuffman==0.4.1 7 | Pillow==8.4.0 8 | yaspin==2.1.0 9 | tensorboard 10 | -------------------------------------------------------------------------------- /CAM_mipnerf/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code Reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /CAM_mipnerf/README.md: -------------------------------------------------------------------------------- 1 | # mip-NeRF 2 | 3 | This repository contains the code release for 4 | [Mip-NeRF: A Multiscale Representation for Anti-Aliasing Neural Radiance Fields](https://jonbarron.info/mipnerf/). 5 | This implementation is written in [JAX](https://github.com/google/jax), and 6 | is a fork of Google's [JaxNeRF implementation](https://github.com/google-research/google-research/tree/master/jaxnerf). 7 | Contact [Jon Barron](https://jonbarron.info/) if you encounter any issues. 8 | 9 | ![rays](https://user-images.githubusercontent.com/3310961/118305131-6ce86700-b49c-11eb-99b8-adcf276e9fe9.jpg) 10 | 11 | ## Abstract 12 | 13 | The rendering procedure used by neural radiance fields (NeRF) samples a scene 14 | with a single ray per pixel and may therefore produce renderings that are 15 | excessively blurred or aliased when training or testing images observe scene 16 | content at different resolutions. The straightforward solution of supersampling 17 | by rendering with multiple rays per pixel is impractical for NeRF, because 18 | rendering each ray requires querying a multilayer perceptron hundreds of times. 19 | Our solution, which we call "mip-NeRF" (à la "mipmap"), extends NeRF to 20 | represent the scene at a continuously-valued scale. By efficiently rendering 21 | anti-aliased conical frustums instead of rays, mip-NeRF reduces objectionable 22 | aliasing artifacts and significantly improves NeRF's ability to represent 23 | fine details, while also being 7% faster than NeRF and half the size. Compared 24 | to NeRF, mip-NeRF reduces average error rates by 17% on the dataset presented 25 | with NeRF and by 60% on a challenging multiscale variant of that dataset that 26 | we present. mip-NeRF is also able to match the accuracy of a brute-force 27 | supersampled NeRF on our multiscale dataset while being 22x faster. 28 | 29 | 30 | ## Installation 31 | We recommend using [Anaconda](https://www.anaconda.com/products/individual) to set 32 | up the environment. Run the following commands: 33 | 34 | ``` 35 | # Clone the repo 36 | git clone https://github.com/google/mipnerf.git; cd mipnerf 37 | # Create a conda environment, note you can use python 3.6-3.8 as 38 | # one of the dependencies (TensorFlow) hasn't supported python 3.9 yet. 39 | conda create --name mipnerf python=3.6.13; conda activate mipnerf 40 | # Prepare pip 41 | conda install pip; pip install --upgrade pip 42 | # Install requirements 43 | pip install -r requirements.txt 44 | ``` 45 | 46 | [Optional] Install GPU and TPU support for Jax 47 | ``` 48 | # Remember to change cuda101 to your CUDA version, e.g. cuda110 for CUDA 11.0. 49 | pip install --upgrade jax jaxlib==0.1.65+cuda101 -f https://storage.googleapis.com/jax-releases/jax_releases.html 50 | ``` 51 | 52 | ## Data 53 | 54 | Then, you'll need to download the datasets 55 | from the [NeRF official Google Drive](https://drive.google.com/drive/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1). 56 | Please download and unzip `nerf_synthetic.zip` and `nerf_llff_data.zip`. 57 | 58 | ### Generate multiscale dataset 59 | You can generate the multiscale dataset used in the paper by running the following command, 60 | ``` 61 | python scripts/convert_blender_data.py --blenderdir /nerf_synthetic --outdir /multiscale 62 | ``` 63 | 64 | ## Running 65 | 66 | Example scripts for training mip-NeRF on individual scenes from the three 67 | datasets used in the paper can be found in `scripts/`. You'll need to change 68 | the paths to point to wherever the datasets are located. 69 | [Gin](https://github.com/google/gin-config) configuration files for our model 70 | and some ablations can be found in `configs/`. 71 | An example script for evaluating on the test set of each scene can be found 72 | in `scripts/`, after which you can use `scripts/summarize.ipynb` to produce 73 | error metrics across all scenes in the same format as was used in tables in the 74 | paper. 75 | 76 | ### OOM errors 77 | You may need to reduce the batch size to avoid out of memory errors. For example the model can be run on a NVIDIA 3080 (10Gb) using the following flag. 78 | ``` 79 | --gin_param="Config.batch_size = 1024" 80 | ``` 81 | 82 | ## Citation 83 | If you use this software package, please cite our paper: 84 | 85 | ``` 86 | @misc{barron2021mipnerf, 87 | title={Mip-NeRF: A Multiscale Representation for Anti-Aliasing Neural Radiance Fields}, 88 | author={Jonathan T. Barron and Ben Mildenhall and Matthew Tancik and Peter Hedman and Ricardo Martin-Brualla and Pratul P. Srinivasan}, 89 | year={2021}, 90 | eprint={2103.13415}, 91 | archivePrefix={arXiv}, 92 | primaryClass={cs.CV} 93 | } 94 | ``` 95 | 96 | ## Acknowledgements 97 | Thanks to [Boyang Deng](https://boyangdeng.com/) for JaxNeRF. 98 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/blender.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'blender' 2 | Config.batching = 'single_image' 3 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/blender_noextras.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'blender' 2 | Config.batching = 'single_image' 3 | MipNerfModel.density_activation = @flax.nn.relu 4 | MipNerfModel.density_bias = 0.0 5 | MipNerfModel.rgb_padding = 0.0 6 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/blender_noipe.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'blender' 2 | Config.batching = 'single_image' 3 | MipNerfModel.disable_integration = True 4 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/llff.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.white_bkgd = False 3 | Config.randomized = True 4 | Config.near = 0. 5 | Config.far = 1. 6 | Config.factor = 4 7 | Config.llffhold = 8 8 | MipNerfModel.use_viewdirs = True 9 | MipNerfModel.ray_shape = 'cylinder' 10 | MipNerfModel.density_noise = 1. 11 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/multiblender.gin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_mipnerf/configs/multiblender.gin -------------------------------------------------------------------------------- /CAM_mipnerf/configs/multiblender_noextras.gin: -------------------------------------------------------------------------------- 1 | MipNerfModel.density_activation = @flax.nn.relu 2 | MipNerfModel.density_bias = 0.0 3 | MipNerfModel.rgb_padding = 0.0 4 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/multiblender_noipe.gin: -------------------------------------------------------------------------------- 1 | MipNerfModel.disable_integration = True 2 | -------------------------------------------------------------------------------- /CAM_mipnerf/configs/multiblender_noloss.gin: -------------------------------------------------------------------------------- 1 | Config.disable_multiscale_loss = True 2 | -------------------------------------------------------------------------------- /CAM_mipnerf/eval.py: -------------------------------------------------------------------------------- 1 | """Evaluation script for mip-NeRF.""" 2 | import functools 3 | from os import path 4 | 5 | from absl import app 6 | from absl import flags 7 | import flax 8 | from flax.metrics import tensorboard 9 | from flax.training import checkpoints 10 | import jax 11 | from jax import random 12 | import numpy as np 13 | 14 | from internal import datasets 15 | from internal import math 16 | from internal import models 17 | from internal import utils 18 | from internal import vis 19 | 20 | FLAGS = flags.FLAGS 21 | utils.define_common_flags() 22 | flags.DEFINE_bool( 23 | 'eval_once', True, 24 | 'If True, evaluate the model only once, otherwise keeping evaluating new' 25 | 'checkpoints if any exist.') 26 | flags.DEFINE_bool('save_output', True, 27 | 'If True, save predicted images to disk.') 28 | 29 | 30 | def main(unused_argv): 31 | config = utils.load_config() 32 | 33 | dataset = datasets.get_dataset('test', FLAGS.data_dir, config) 34 | model, init_variables = models.construct_mipnerf( 35 | random.PRNGKey(20200823), dataset.peek()) 36 | optimizer = flax.optim.Adam(config.lr_init).create(init_variables) 37 | state = utils.TrainState(optimizer=optimizer) 38 | del optimizer, init_variables 39 | 40 | # Rendering is forced to be deterministic even if training was randomized, as 41 | # this eliminates 'speckle' artifacts. 42 | def render_eval_fn(variables, _, rays): 43 | return jax.lax.all_gather( 44 | model.apply( 45 | variables, 46 | random.PRNGKey(0), # Unused. 47 | rays, 48 | randomized=False, 49 | white_bkgd=config.white_bkgd), 50 | axis_name='batch') 51 | 52 | # pmap over only the data input. 53 | render_eval_pfn = jax.pmap( 54 | render_eval_fn, 55 | in_axes=(None, None, 0), 56 | donate_argnums=2, 57 | axis_name='batch', 58 | ) 59 | 60 | ssim_fn = jax.jit(functools.partial(math.compute_ssim, max_val=1.)) 61 | 62 | last_step = 0 63 | out_dir = path.join(FLAGS.train_dir, 64 | 'path_renders' if config.render_path else 'test_preds') 65 | if not FLAGS.eval_once: 66 | summary_writer = tensorboard.SummaryWriter( 67 | path.join(FLAGS.train_dir, 'eval')) 68 | while True: 69 | state = checkpoints.restore_checkpoint(FLAGS.train_dir, state) 70 | step = int(state.optimizer.state.step) 71 | if step <= last_step: 72 | continue 73 | if FLAGS.save_output and (not utils.isdir(out_dir)): 74 | utils.makedirs(out_dir) 75 | psnr_values = [] 76 | ssim_values = [] 77 | avg_values = [] 78 | if not FLAGS.eval_once: 79 | showcase_index = random.randint(random.PRNGKey(step), (), 0, dataset.size) 80 | for idx in range(dataset.size): 81 | print(f'Evaluating {idx+1}/{dataset.size}') 82 | batch = next(dataset) 83 | pred_color, pred_distance, pred_acc = models.render_image( 84 | functools.partial(render_eval_pfn, state.optimizer.target), 85 | batch['rays'], 86 | None, 87 | chunk=FLAGS.chunk) 88 | 89 | vis_suite = vis.visualize_suite(pred_distance, pred_acc) 90 | 91 | if jax.host_id() != 0: # Only record via host 0. 92 | continue 93 | if not FLAGS.eval_once and idx == showcase_index: 94 | showcase_color = pred_color 95 | showcase_acc = pred_acc 96 | showcase_vis_suite = vis_suite 97 | if not config.render_path: 98 | showcase_gt = batch['pixels'] 99 | if not config.render_path: 100 | psnr = float( 101 | math.mse_to_psnr(((pred_color - batch['pixels'])**2).mean())) 102 | ssim = float(ssim_fn(pred_color, batch['pixels'])) 103 | print(f'PSNR={psnr:.4f} SSIM={ssim:.4f}') 104 | psnr_values.append(psnr) 105 | ssim_values.append(ssim) 106 | if FLAGS.save_output and (config.test_render_interval > 0): 107 | if (idx % config.test_render_interval) == 0: 108 | utils.save_img_uint8( 109 | pred_color, path.join(out_dir, 'color_{:03d}.png'.format(idx))) 110 | utils.save_img_float32( 111 | pred_distance, 112 | path.join(out_dir, 'distance_{:03d}.tiff'.format(idx))) 113 | utils.save_img_float32( 114 | pred_acc, path.join(out_dir, 'acc_{:03d}.tiff'.format(idx))) 115 | for k, v in vis_suite.items(): 116 | utils.save_img_uint8( 117 | v, path.join(out_dir, k + '_{:03d}.png'.format(idx))) 118 | if (not FLAGS.eval_once) and (jax.host_id() == 0): 119 | summary_writer.image('pred_color', showcase_color, step) 120 | summary_writer.image('pred_acc', showcase_acc, step) 121 | for k, v in showcase_vis_suite.items(): 122 | summary_writer.image('pred_' + k, v, step) 123 | if not config.render_path: 124 | summary_writer.scalar('psnr', np.mean(np.array(psnr_values)), step) 125 | summary_writer.scalar('ssim', np.mean(np.array(ssim_values)), step) 126 | summary_writer.image('target', showcase_gt, step) 127 | if FLAGS.save_output and (not config.render_path) and (jax.host_id() == 0): 128 | with utils.open_file(path.join(out_dir, f'psnrs_{step}.txt'), 'w') as f: 129 | f.write(' '.join([str(v) for v in psnr_values])) 130 | f.write('\n'+str(np.mean(np.array(psnr_values)))) 131 | with utils.open_file(path.join(out_dir, f'ssims_{step}.txt'), 'w') as f: 132 | f.write(' '.join([str(v) for v in ssim_values])) 133 | f.write('\n'+str(np.mean(np.array(ssim_values)))) 134 | if FLAGS.eval_once: 135 | break 136 | if int(step) >= config.max_steps: 137 | break 138 | last_step = step 139 | 140 | 141 | if __name__ == '__main__': 142 | app.run(main) 143 | -------------------------------------------------------------------------------- /CAM_mipnerf/internal/vis.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Lint as: python3 16 | """Helper functions for visualizing things.""" 17 | import jax 18 | import jax.numpy as jnp 19 | import jax.scipy as jsp 20 | import matplotlib.cm as cm 21 | 22 | 23 | def sinebow(h): 24 | """A cyclic and uniform colormap, see http://basecase.org/env/on-rainbows.""" 25 | f = lambda x: jnp.sin(jnp.pi * x)**2 26 | return jnp.stack([f(3 / 6 - h), f(5 / 6 - h), f(7 / 6 - h)], -1) 27 | 28 | 29 | def convolve2d(z, f): 30 | return jsp.signal.convolve2d( 31 | z, f, mode='same', precision=jax.lax.Precision.HIGHEST) 32 | 33 | 34 | def depth_to_normals(depth): 35 | """Assuming `depth` is orthographic, linearize it to a set of normals.""" 36 | f_blur = jnp.array([1, 2, 1]) / 4 37 | f_edge = jnp.array([-1, 0, 1]) / 2 38 | dy = convolve2d(depth, f_blur[None, :] * f_edge[:, None]) 39 | dx = convolve2d(depth, f_blur[:, None] * f_edge[None, :]) 40 | inv_denom = 1 / jnp.sqrt(1 + dx**2 + dy**2) 41 | normals = jnp.stack([dx * inv_denom, dy * inv_denom, inv_denom], -1) 42 | return normals 43 | 44 | 45 | def visualize_depth(depth, 46 | acc=None, 47 | near=None, 48 | far=None, 49 | ignore_frac=0, 50 | curve_fn=lambda x: -jnp.log(x + jnp.finfo(jnp.float32).eps), 51 | modulus=0, 52 | colormap=None): 53 | """Visualize a depth map. 54 | 55 | Args: 56 | depth: A depth map. 57 | acc: An accumulation map, in [0, 1]. 58 | near: The depth of the near plane, if None then just use the min(). 59 | far: The depth of the far plane, if None then just use the max(). 60 | ignore_frac: What fraction of the depth map to ignore when automatically 61 | generating `near` and `far`. Depends on `acc` as well as `depth'. 62 | curve_fn: A curve function that gets applied to `depth`, `near`, and `far` 63 | before the rest of visualization. Good choices: x, 1/(x+eps), log(x+eps). 64 | Note that the default choice will flip the sign of depths, so that the 65 | default colormap (turbo) renders "near" as red and "far" as blue. 66 | modulus: If > 0, mod the normalized depth by `modulus`. Use (0, 1]. 67 | colormap: A colormap function. If None (default), will be set to 68 | matplotlib's turbo if modulus==0, sinebow otherwise. 69 | 70 | Returns: 71 | An RGB visualization of `depth`. 72 | """ 73 | if acc is None: 74 | acc = jnp.ones_like(depth) 75 | acc = jnp.where(jnp.isnan(depth), jnp.zeros_like(acc), acc) 76 | 77 | # Sort `depth` and `acc` according to `depth`, then identify the depth values 78 | # that span the middle of `acc`, ignoring `ignore_frac` fraction of `acc`. 79 | sortidx = jnp.argsort(depth.reshape([-1])) 80 | depth_sorted = depth.reshape([-1])[sortidx] 81 | acc_sorted = acc.reshape([-1])[sortidx] 82 | cum_acc_sorted = jnp.cumsum(acc_sorted) 83 | mask = ((cum_acc_sorted >= cum_acc_sorted[-1] * ignore_frac) & 84 | (cum_acc_sorted <= cum_acc_sorted[-1] * (1 - ignore_frac))) 85 | depth_keep = depth_sorted[mask] 86 | 87 | # If `near` or `far` are None, use the highest and lowest non-NaN values in 88 | # `depth_keep` as automatic near/far planes. 89 | eps = jnp.finfo(jnp.float32).eps 90 | near = near or depth_keep[0] - eps 91 | far = far or depth_keep[-1] + eps 92 | 93 | # Curve all values. 94 | depth, near, far = [curve_fn(x) for x in [depth, near, far]] 95 | 96 | # Wrap the values around if requested. 97 | if modulus > 0: 98 | value = jnp.mod(depth, modulus) / modulus 99 | colormap = colormap or sinebow 100 | else: 101 | # Scale to [0, 1]. 102 | value = jnp.nan_to_num( 103 | jnp.clip((depth - jnp.minimum(near, far)) / jnp.abs(far - near), 0, 1)) 104 | colormap = colormap or cm.get_cmap('turbo') 105 | 106 | vis = colormap(value)[:, :, :3] 107 | 108 | # Set non-accumulated pixels to white. 109 | vis = vis * acc[:, :, None] + (1 - acc)[:, :, None] 110 | 111 | return vis 112 | 113 | 114 | def visualize_normals(depth, acc, scaling=None): 115 | """Visualize fake normals of `depth` (optionally scaled to be isotropic).""" 116 | if scaling is None: 117 | mask = ~jnp.isnan(depth) 118 | x, y = jnp.meshgrid( 119 | jnp.arange(depth.shape[1]), jnp.arange(depth.shape[0]), indexing='xy') 120 | xy_var = (jnp.var(x[mask]) + jnp.var(y[mask])) / 2 121 | z_var = jnp.var(depth[mask]) 122 | scaling = jnp.sqrt(xy_var / z_var) 123 | 124 | scaled_depth = scaling * depth 125 | normals = depth_to_normals(scaled_depth) 126 | vis = jnp.isnan(normals) + jnp.nan_to_num((normals + 1) / 2, 0) 127 | 128 | # Set non-accumulated pixels to white. 129 | if acc is not None: 130 | vis = vis * acc[:, :, None] + (1 - acc)[:, :, None] 131 | 132 | return vis 133 | 134 | 135 | def visualize_suite(depth, acc): 136 | """A wrapper around other visualizations for easy integration.""" 137 | vis = { 138 | 'depth': visualize_depth(depth, acc), 139 | 'depth_mod': visualize_depth(depth, acc, modulus=0.1), 140 | 'depth_normals': visualize_normals(depth, acc) 141 | } 142 | return vis 143 | -------------------------------------------------------------------------------- /CAM_mipnerf/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.16.4,<1.19.0 2 | jax>=0.2.12 3 | jaxlib>=0.1.65 4 | flax>=0.2.2 5 | opencv-python>=4.4.0 6 | Pillow>=7.2.0 7 | tensorboard>=2.4.0 8 | tensorflow>=2.3.1 9 | gin-config 10 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/convert_blender_data.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from os import path 4 | 5 | from absl import app 6 | from absl import flags 7 | import jax 8 | import numpy as np 9 | from PIL import Image 10 | 11 | FLAGS = flags.FLAGS 12 | 13 | flags.DEFINE_string('blenderdir', None, 14 | 'Base directory for all Blender data.') 15 | flags.DEFINE_string('outdir', None, 16 | 'Where to save multiscale data.') 17 | flags.DEFINE_integer('n_down', 4, 18 | 'How many levels of downscaling to use.') 19 | 20 | jax.config.parse_flags_with_absl() 21 | 22 | 23 | def load_renderings(data_dir, split): 24 | """Load images and metadata from disk.""" 25 | f = 'transforms_{}.json'.format(split) 26 | with open(path.join(data_dir, f), 'r') as fp: 27 | meta = json.load(fp) 28 | images = [] 29 | cams = [] 30 | print('Loading imgs') 31 | for frame in meta['frames']: 32 | fname = os.path.join(data_dir, frame['file_path'] + '.png') 33 | with open(fname, 'rb') as imgin: 34 | image = np.array(Image.open(imgin), dtype=np.float32) / 255. 35 | cams.append(frame['transform_matrix']) 36 | images.append(image) 37 | ret = {} 38 | ret['images'] = np.stack(images, axis=0) 39 | print('Loaded all images, shape is', ret['images'].shape) 40 | ret['camtoworlds'] = np.stack(cams, axis=0) 41 | w = ret['images'].shape[2] 42 | camera_angle_x = float(meta['camera_angle_x']) 43 | ret['focal'] = .5 * w / np.tan(.5 * camera_angle_x) 44 | return ret 45 | 46 | 47 | def down2(img): 48 | sh = img.shape 49 | return np.mean(np.reshape(img, [sh[0] // 2, 2, sh[1] // 2, 2, -1]), (1, 3)) 50 | 51 | 52 | def convert_to_nerfdata(basedir, newdir, n_down): 53 | """Convert Blender data to multiscale.""" 54 | if not os.path.exists(newdir): 55 | os.makedirs(newdir) 56 | splits = ['train', 'val', 'test'] 57 | bigmeta = {} 58 | # Foreach split in the dataset 59 | for split in splits: 60 | print('Split', split) 61 | # Load everything 62 | data = load_renderings(basedir, split) 63 | 64 | # Save out all the images 65 | imgdir = 'images_{}'.format(split) 66 | os.makedirs(os.path.join(newdir, imgdir), exist_ok=True) 67 | fnames = [] 68 | widths = [] 69 | heights = [] 70 | focals = [] 71 | cam2worlds = [] 72 | lossmults = [] 73 | labels = [] 74 | nears, fars = [], [] 75 | f = data['focal'] 76 | print('Saving images') 77 | for i, img in enumerate(data['images']): 78 | for j in range(n_down): 79 | fname = '{}/{:03d}_d{}.png'.format(imgdir, i, j) 80 | fnames.append(fname) 81 | fname = os.path.join(newdir, fname) 82 | with open(fname, 'wb') as imgout: 83 | img8 = Image.fromarray(np.uint8(img * 255)) 84 | img8.save(imgout) 85 | widths.append(img.shape[1]) 86 | heights.append(img.shape[0]) 87 | focals.append(f / 2**j) 88 | cam2worlds.append(data['camtoworlds'][i].tolist()) 89 | lossmults.append(4.**j) 90 | labels.append(j) 91 | nears.append(2.) 92 | fars.append(6.) 93 | img = down2(img) 94 | 95 | # Create metadata 96 | meta = {} 97 | meta['file_path'] = fnames 98 | meta['cam2world'] = cam2worlds 99 | meta['width'] = widths 100 | meta['height'] = heights 101 | meta['focal'] = focals 102 | meta['label'] = labels 103 | meta['near'] = nears 104 | meta['far'] = fars 105 | meta['lossmult'] = lossmults 106 | 107 | fx = np.array(focals) 108 | fy = np.array(focals) 109 | cx = np.array(meta['width']) * .5 110 | cy = np.array(meta['height']) * .5 111 | arr0 = np.zeros_like(cx) 112 | arr1 = np.ones_like(cx) 113 | k_inv = np.array([ 114 | [arr1 / fx, arr0, -cx / fx], 115 | [arr0, -arr1 / fy, cy / fy], 116 | [arr0, arr0, -arr1], 117 | ]) 118 | k_inv = np.moveaxis(k_inv, -1, 0) 119 | meta['pix2cam'] = k_inv.tolist() 120 | 121 | bigmeta[split] = meta 122 | 123 | for k in bigmeta: 124 | for j in bigmeta[k]: 125 | print(k, j, type(bigmeta[k][j]), np.array(bigmeta[k][j]).shape) 126 | 127 | jsonfile = os.path.join(newdir, 'metadata.json') 128 | with open(jsonfile, 'w') as f: 129 | json.dump(bigmeta, f, ensure_ascii=False, indent=4) 130 | 131 | 132 | def main(unused_argv): 133 | 134 | blenderdir = FLAGS.blenderdir 135 | outdir = FLAGS.outdir 136 | n_down = FLAGS.n_down 137 | if not os.path.exists(outdir): 138 | os.makedirs(outdir) 139 | 140 | dirs = [os.path.join(blenderdir, f) for f in os.listdir(blenderdir)] 141 | dirs = [d for d in dirs if os.path.isdir(d)] 142 | print(dirs) 143 | for basedir in dirs: 144 | print() 145 | newdir = os.path.join(outdir, os.path.basename(basedir)) 146 | print('Converting from', basedir, 'to', newdir) 147 | convert_to_nerfdata(basedir, newdir, n_down) 148 | 149 | 150 | if __name__ == '__main__': 151 | app.run(main) 152 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/eval_blender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Script for evaluating on the Blender dataset. 17 | 18 | SCENE=lego 19 | EXPERIMENT=debug 20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE 21 | DATA_DIR=/Users/barron/data/nerf_synthetic/$SCENE 22 | 23 | python -m eval \ 24 | --data_dir=$DATA_DIR \ 25 | --train_dir=$TRAIN_DIR \ 26 | --chunk=3076 \ 27 | --gin_file=configs/blender.gin \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/eval_llff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Script for evaluating on the LLFF dataset. 17 | 18 | SCENE=trex 19 | EXPERIMENT=debug 20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE 21 | DATA_DIR=/Users/barron/data/nerf_llff_data/$SCENE 22 | 23 | python -m eval \ 24 | --data_dir=$DATA_DIR \ 25 | --train_dir=$TRAIN_DIR \ 26 | --chunk=3076 \ 27 | --gin_file=configs/llff.gin \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/eval_multiblender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Script for evaluating on the multiscale Blender dataset. 17 | 18 | SCENE=lego 19 | EXPERIMENT=debug 20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE 21 | DATA_DIR=/Users/barron/data/down4/$SCENE 22 | 23 | python -m eval \ 24 | --data_dir=$DATA_DIR \ 25 | --train_dir=$TRAIN_DIR \ 26 | --chunk=3076 \ 27 | --gin_file=configs/multiblender.gin \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/train_blender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Script for training on the Blender dataset. 17 | 18 | SCENE=lego 19 | EXPERIMENT=debug 20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE 21 | DATA_DIR=/Users/barron/data/nerf_synthetic/$SCENE 22 | 23 | rm $TRAIN_DIR/* 24 | python -m train \ 25 | --data_dir=$DATA_DIR \ 26 | --train_dir=$TRAIN_DIR \ 27 | --gin_file=configs/blender.gin \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/train_llff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Script for training on the LLFF dataset. 17 | 18 | SCENE=trex 19 | EXPERIMENT=debug 20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE 21 | DATA_DIR=/Users/barron/data/nerf_llff_data/$SCENE 22 | 23 | rm $TRAIN_DIR/* 24 | python -m train \ 25 | --data_dir=$DATA_DIR \ 26 | --train_dir=$TRAIN_DIR \ 27 | --gin_file=configs/llff.gin \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_mipnerf/scripts/train_multiblender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Script for training on the multiscale Blender dataset. 17 | 18 | SCENE=lego 19 | EXPERIMENT=debug 20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE 21 | DATA_DIR=/Users/barron/data/down4/$SCENE 22 | 23 | rm $TRAIN_DIR/* 24 | python -m train \ 25 | --data_dir=$DATA_DIR \ 26 | --train_dir=$TRAIN_DIR \ 27 | --gin_file=configs/multiblender.gin \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/.gitignore: -------------------------------------------------------------------------------- 1 | internal/pycolmap 2 | __pycache__/ 3 | interal/__pycache__/ 4 | tests/__pycache__/ 5 | .DS_Store 6 | .vscode/ 7 | .idea/ 8 | __MACOSX/ 9 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement (CLA). You (or your employer) retain the copyright to your 10 | contribution; this simply gives us permission to use and redistribute your 11 | contributions as part of the project. Head over to 12 | to see your current agreements on file or 13 | to sign a new one. 14 | 15 | You generally only need to submit a CLA once, so if you've already submitted one 16 | (even if it was for a different project), you probably don't need to do it 17 | again. 18 | 19 | ## Code Reviews 20 | 21 | All submissions, including submissions by project members, require review. We 22 | use GitHub pull requests for this purpose. Consult 23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 24 | information on using pull requests. 25 | 26 | ## Community Guidelines 27 | 28 | This project follows 29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 30 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/.ipynb_checkpoints/360-checkpoint.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.near = 0.2 3 | Config.far = 1e6 4 | Config.factor = 4 5 | 6 | Model.raydist_fn = @jnp.reciprocal 7 | Model.opaque_background = True 8 | 9 | PropMLP.warp_fn = @coord.contract 10 | PropMLP.net_depth = 4 11 | PropMLP.net_width = 256 12 | PropMLP.disable_density_normals = True 13 | PropMLP.disable_rgb = True 14 | 15 | NerfMLP.warp_fn = @coord.contract 16 | NerfMLP.net_depth = 8 17 | NerfMLP.net_width = 1024 18 | NerfMLP.disable_density_normals = True 19 | 20 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/.ipynb_checkpoints/360_glo4-checkpoint.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.near = 0.2 3 | Config.far = 1e6 4 | Config.factor = 4 5 | 6 | Model.raydist_fn = @jnp.reciprocal 7 | Model.num_glo_features = 4 8 | Model.opaque_background = True 9 | 10 | PropMLP.warp_fn = @coord.contract 11 | PropMLP.net_depth = 4 12 | PropMLP.net_width = 256 13 | PropMLP.disable_density_normals = True 14 | PropMLP.disable_rgb = True 15 | 16 | NerfMLP.warp_fn = @coord.contract 17 | NerfMLP.net_depth = 8 18 | NerfMLP.net_width = 1024 19 | NerfMLP.disable_density_normals = True 20 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/.ipynb_checkpoints/render_config-checkpoint.gin: -------------------------------------------------------------------------------- 1 | Config.render_path = True 2 | Config.render_path_frames = 480 3 | Config.render_video_fps = 60 4 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/360.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.near = 0.2 3 | Config.far = 1e6 4 | Config.factor = 4 5 | 6 | Model.raydist_fn = @jnp.reciprocal 7 | Model.opaque_background = True 8 | 9 | PropMLP.warp_fn = @coord.contract 10 | PropMLP.net_depth = 4 11 | PropMLP.net_width = 256 12 | PropMLP.disable_density_normals = True 13 | PropMLP.disable_rgb = True 14 | 15 | NerfMLP.warp_fn = @coord.contract 16 | NerfMLP.net_depth = 8 17 | NerfMLP.net_width = 1024 18 | NerfMLP.disable_density_normals = True 19 | 20 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/360_glo4.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.near = 0.2 3 | Config.far = 1e6 4 | Config.factor = 4 5 | 6 | Model.raydist_fn = @jnp.reciprocal 7 | Model.num_glo_features = 4 8 | Model.opaque_background = True 9 | 10 | PropMLP.warp_fn = @coord.contract 11 | PropMLP.net_depth = 4 12 | PropMLP.net_width = 256 13 | PropMLP.disable_density_normals = True 14 | PropMLP.disable_rgb = True 15 | 16 | NerfMLP.warp_fn = @coord.contract 17 | NerfMLP.net_depth = 8 18 | NerfMLP.net_width = 1024 19 | NerfMLP.disable_density_normals = True 20 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/blender_256.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'blender' 2 | Config.batching = 'single_image' 3 | Config.near = 2 4 | Config.far = 6 5 | Config.eval_render_interval = 5 6 | Config.data_loss_type = 'mse' 7 | Config.adam_eps = 1e-8 8 | 9 | Model.num_levels = 2 10 | Model.num_prop_samples = 128 11 | Model.num_nerf_samples = 32 12 | 13 | PropMLP.net_depth = 4 14 | PropMLP.net_width = 256 15 | PropMLP.basis_shape = 'octahedron' 16 | PropMLP.basis_subdivisions = 1 17 | PropMLP.disable_density_normals = True 18 | PropMLP.disable_rgb = True 19 | 20 | NerfMLP.net_depth = 8 21 | NerfMLP.net_width = 256 22 | NerfMLP.basis_shape = 'octahedron' 23 | NerfMLP.basis_subdivisions = 1 24 | NerfMLP.disable_density_normals = True 25 | 26 | Config.distortion_loss_mult = 0. 27 | 28 | NerfMLP.max_deg_point = 16 29 | PropMLP.max_deg_point = 16 30 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/blender_512.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'blender' 2 | Config.batching = 'single_image' 3 | Config.near = 2 4 | Config.far = 6 5 | Config.eval_render_interval = 5 6 | Config.data_loss_type = 'mse' 7 | Config.adam_eps = 1e-8 8 | 9 | Model.num_levels = 2 10 | Model.num_prop_samples = 128 11 | Model.num_nerf_samples = 32 12 | 13 | PropMLP.net_depth = 4 14 | PropMLP.net_width = 256 15 | PropMLP.disable_density_normals = True 16 | PropMLP.disable_rgb = True 17 | 18 | NerfMLP.net_depth = 8 19 | NerfMLP.net_width = 512 20 | NerfMLP.disable_density_normals = True 21 | 22 | Config.distortion_loss_mult = 0. 23 | 24 | NerfMLP.max_deg_point = 16 25 | PropMLP.max_deg_point = 16 26 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/blender_refnerf.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'blender' 2 | Config.batching = 'single_image' 3 | Config.near = 2 4 | Config.far = 6 5 | Config.eval_render_interval = 5 6 | Config.compute_normal_metrics = True 7 | Config.data_loss_type = 'mse' 8 | Config.distortion_loss_mult = 0.0 9 | Config.orientation_loss_mult = 0.1 10 | Config.orientation_loss_target = 'normals_pred' 11 | Config.predicted_normal_loss_mult = 3e-4 12 | Config.orientation_coarse_loss_mult = 0.01 13 | Config.predicted_normal_coarse_loss_mult = 3e-5 14 | Config.interlevel_loss_mult = 0.0 15 | Config.data_coarse_loss_mult = 0.1 16 | Config.adam_eps = 1e-8 17 | 18 | Model.num_levels = 2 19 | Model.single_mlp = True 20 | Model.num_prop_samples = 128 # This needs to be set despite single_mlp = True. 21 | Model.num_nerf_samples = 128 22 | Model.anneal_slope = 0. 23 | Model.dilation_multiplier = 0. 24 | Model.dilation_bias = 0. 25 | Model.single_jitter = False 26 | Model.resample_padding = 0.01 27 | 28 | NerfMLP.net_depth = 8 29 | NerfMLP.net_width = 256 30 | NerfMLP.net_depth_viewdirs = 8 31 | NerfMLP.basis_shape = 'octahedron' 32 | NerfMLP.basis_subdivisions = 1 33 | NerfMLP.disable_density_normals = False 34 | NerfMLP.enable_pred_normals = True 35 | NerfMLP.use_directional_enc = True 36 | NerfMLP.use_reflections = True 37 | NerfMLP.deg_view = 5 38 | NerfMLP.enable_pred_roughness = True 39 | NerfMLP.use_diffuse_color = True 40 | NerfMLP.use_specular_tint = True 41 | NerfMLP.use_n_dot_v = True 42 | NerfMLP.bottleneck_width = 128 43 | NerfMLP.density_bias = 0.5 44 | NerfMLP.max_deg_point = 16 45 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/debug.gin: -------------------------------------------------------------------------------- 1 | # A short training schedule with no "warm up", useful for debugging. 2 | Config.checkpoint_every = 1000 3 | Config.print_every = 100 4 | Config.train_render_every = 1000 5 | Config.lr_delay_mult = 0.1 6 | Config.lr_delay_steps = 500 7 | Config.batch_size = 2048 8 | Config.render_chunk_size = 2048 9 | Config.lr_init = 5e-4 10 | Config.lr_final = 5e-6 11 | Config.factor = 4 12 | Config.early_exit_steps = 3000 13 | 14 | PropMLP.net_depth = 2 15 | PropMLP.net_width = 64 16 | 17 | NerfMLP.net_depth = 4 18 | NerfMLP.net_width = 128 19 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/llff_256.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.near = 0. 3 | Config.far = 1. 4 | Config.factor = 4 5 | Config.forward_facing = True 6 | Config.adam_eps = 1e-8 7 | 8 | Model.ray_shape = 'cylinder' 9 | Model.opaque_background = True 10 | Model.num_levels = 2 11 | Model.num_prop_samples = 128 12 | Model.num_nerf_samples = 32 13 | 14 | PropMLP.net_depth = 4 15 | PropMLP.net_width = 256 16 | PropMLP.basis_shape = 'octahedron' 17 | PropMLP.basis_subdivisions = 1 18 | PropMLP.disable_density_normals = True 19 | PropMLP.disable_rgb = True 20 | 21 | NerfMLP.net_depth = 8 22 | NerfMLP.net_width = 256 23 | NerfMLP.basis_shape = 'octahedron' 24 | NerfMLP.basis_subdivisions = 1 25 | NerfMLP.disable_density_normals = True 26 | 27 | NerfMLP.max_deg_point = 16 28 | PropMLP.max_deg_point = 16 29 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/llff_512.gin: -------------------------------------------------------------------------------- 1 | Config.dataset_loader = 'llff' 2 | Config.near = 0. 3 | Config.far = 1. 4 | Config.factor = 4 5 | Config.forward_facing = True 6 | Config.adam_eps = 1e-8 7 | 8 | Model.ray_shape = 'cylinder' 9 | Model.opaque_background = True 10 | Model.num_levels = 2 11 | Model.num_prop_samples = 128 12 | Model.num_nerf_samples = 32 13 | 14 | PropMLP.net_depth = 4 15 | PropMLP.net_width = 256 16 | PropMLP.disable_density_normals = True 17 | PropMLP.disable_rgb = True 18 | 19 | NerfMLP.net_depth = 8 20 | NerfMLP.net_width = 512 21 | NerfMLP.disable_density_normals = True 22 | 23 | NerfMLP.max_deg_point = 16 24 | PropMLP.max_deg_point = 16 25 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/llff_raw.gin: -------------------------------------------------------------------------------- 1 | # General LLFF settings 2 | 3 | Config.dataset_loader = 'llff' 4 | Config.near = 0. 5 | Config.far = 1. 6 | Config.factor = 4 7 | Config.forward_facing = True 8 | 9 | Model.ray_shape = 'cylinder' 10 | 11 | PropMLP.net_depth = 4 12 | PropMLP.net_width = 256 13 | PropMLP.basis_shape = 'octahedron' 14 | PropMLP.basis_subdivisions = 1 15 | PropMLP.disable_density_normals = True # Turn this off if using orientation loss. 16 | PropMLP.disable_rgb = True 17 | 18 | NerfMLP.net_depth = 8 19 | NerfMLP.net_width = 256 20 | NerfMLP.basis_shape = 'octahedron' 21 | NerfMLP.basis_subdivisions = 1 22 | NerfMLP.disable_density_normals = True # Turn this off if using orientation loss. 23 | 24 | NerfMLP.max_deg_point = 16 25 | PropMLP.max_deg_point = 16 26 | 27 | Config.train_render_every = 5000 28 | 29 | 30 | ########################## RawNeRF specific settings ########################## 31 | 32 | Config.rawnerf_mode = True 33 | Config.data_loss_type = 'rawnerf' 34 | Config.apply_bayer_mask = True 35 | Model.learned_exposure_scaling = True 36 | 37 | Model.num_levels = 2 38 | Model.num_prop_samples = 128 # Using extra samples for now because of noise instability. 39 | Model.num_nerf_samples = 128 40 | Model.opaque_background = True 41 | 42 | # RGB activation we use for linear color outputs is exp(x - 5). 43 | NerfMLP.rgb_padding = 0. 44 | NerfMLP.rgb_activation = @math.safe_exp 45 | NerfMLP.rgb_bias = -5. 46 | PropMLP.rgb_padding = 0. 47 | PropMLP.rgb_activation = @math.safe_exp 48 | PropMLP.rgb_bias = -5. 49 | 50 | ## Experimenting with the various regularizers and losses: 51 | Config.interlevel_loss_mult = .0 # Turning off interlevel for now (default = 1.). 52 | Config.distortion_loss_mult = .01 # Distortion loss helps with floaters (default = .01). 53 | Config.orientation_loss_mult = 0. # Orientation loss also not great (try .01). 54 | Config.data_coarse_loss_mult = 0.1 # Setting this to match old MipNeRF. 55 | 56 | ## Density noise used in original NeRF: 57 | NerfMLP.density_noise = 1. 58 | PropMLP.density_noise = 1. 59 | 60 | ## Use a single MLP for all rounds of sampling: 61 | Model.single_mlp = True 62 | 63 | ## Some algorithmic settings to match the paper: 64 | Model.anneal_slope = 0. 65 | Model.dilation_multiplier = 0. 66 | Model.dilation_bias = 0. 67 | Model.single_jitter = False 68 | NerfMLP.weight_init = 'glorot_uniform' 69 | PropMLP.weight_init = 'glorot_uniform' 70 | 71 | ## Training hyperparameters used in the paper: 72 | Config.batch_size = 16384 73 | Config.render_chunk_size = 16384 74 | Config.lr_init = 1e-3 75 | Config.lr_final = 1e-5 76 | Config.max_steps = 500000 77 | Config.checkpoint_every = 25000 78 | Config.lr_delay_steps = 2500 79 | Config.lr_delay_mult = 0.01 80 | Config.grad_max_norm = 0.1 81 | Config.grad_max_val = 0.1 82 | Config.adam_eps = 1e-8 83 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/llff_raw_test.gin: -------------------------------------------------------------------------------- 1 | include 'experimental/users/barron/mipnerf360/configs/llff_raw.gin' 2 | 3 | Config.factor = 0 4 | Config.eval_raw_affine_cc = True 5 | Config.eval_crop_borders = 16 6 | Config.vis_decimate = 4 7 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/render_config.gin: -------------------------------------------------------------------------------- 1 | Config.render_path = True 2 | Config.render_path_frames = 480 3 | Config.render_video_fps = 60 4 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/configs/tat.gin: -------------------------------------------------------------------------------- 1 | # This config is meant to be run while overriding a 360*.gin config. 2 | 3 | Config.dataset_loader = 'tat_nerfpp' 4 | Config.near = 0.1 5 | Config.far = 1e6 6 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/internal/coord.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tools for manipulating coordinate spaces and distances along rays.""" 15 | 16 | from internal import math 17 | import jax 18 | import jax.numpy as jnp 19 | 20 | 21 | def contract(x): 22 | """Contracts points towards the origin (Eq 10 of arxiv.org/abs/2111.12077).""" 23 | eps = jnp.finfo(jnp.float32).eps 24 | # Clamping to eps prevents non-finite gradients when x == 0. 25 | x_mag_sq = jnp.maximum(eps, jnp.sum(x**2, axis=-1, keepdims=True)) 26 | z = jnp.where(x_mag_sq <= 1, x, ((2 * jnp.sqrt(x_mag_sq) - 1) / x_mag_sq) * x) 27 | return z 28 | 29 | 30 | def inv_contract(z): 31 | """The inverse of contract().""" 32 | eps = jnp.finfo(jnp.float32).eps 33 | # Clamping to eps prevents non-finite gradients when z == 0. 34 | z_mag_sq = jnp.maximum(eps, jnp.sum(z**2, axis=-1, keepdims=True)) 35 | x = jnp.where(z_mag_sq <= 1, z, z / (2 * jnp.sqrt(z_mag_sq) - z_mag_sq)) 36 | return x 37 | 38 | 39 | def track_linearize(fn, mean, cov): 40 | """Apply function `fn` to a set of means and covariances, ala a Kalman filter. 41 | 42 | We can analytically transform a Gaussian parameterized by `mean` and `cov` 43 | with a function `fn` by linearizing `fn` around `mean`, and taking advantage 44 | of the fact that Covar[Ax + y] = A(Covar[x])A^T (see 45 | https://cs.nyu.edu/~roweis/notes/gaussid.pdf for details). 46 | 47 | Args: 48 | fn: the function applied to the Gaussians parameterized by (mean, cov). 49 | mean: a tensor of means, where the last axis is the dimension. 50 | cov: a tensor of covariances, where the last two axes are the dimensions. 51 | 52 | Returns: 53 | fn_mean: the transformed means. 54 | fn_cov: the transformed covariances. 55 | """ 56 | if (len(mean.shape) + 1) != len(cov.shape): 57 | raise ValueError('cov must be non-diagonal') 58 | fn_mean, lin_fn = jax.linearize(fn, mean) 59 | fn_cov = jax.vmap(lin_fn, -1, -2)(jax.vmap(lin_fn, -1, -2)(cov)) 60 | return fn_mean, fn_cov 61 | 62 | 63 | def construct_ray_warps(fn, t_near, t_far): 64 | """Construct a bijection between metric distances and normalized distances. 65 | 66 | See the text around Equation 11 in https://arxiv.org/abs/2111.12077 for a 67 | detailed explanation. 68 | 69 | Args: 70 | fn: the function to ray distances. 71 | t_near: a tensor of near-plane distances. 72 | t_far: a tensor of far-plane distances. 73 | 74 | Returns: 75 | t_to_s: a function that maps distances to normalized distances in [0, 1]. 76 | s_to_t: the inverse of t_to_s. 77 | """ 78 | if fn is None: 79 | fn_fwd = lambda x: x 80 | fn_inv = lambda x: x 81 | elif fn == 'piecewise': 82 | # Piecewise spacing combining identity and 1/x functions to allow t_near=0. 83 | fn_fwd = lambda x: jnp.where(x < 1, .5 * x, 1 - .5 / x) 84 | fn_inv = lambda x: jnp.where(x < .5, 2 * x, .5 / (1 - x)) 85 | else: 86 | inv_mapping = { 87 | 'reciprocal': jnp.reciprocal, 88 | 'log': jnp.exp, 89 | 'exp': jnp.log, 90 | 'sqrt': jnp.square, 91 | 'square': jnp.sqrt 92 | } 93 | fn_fwd = fn 94 | fn_inv = inv_mapping[fn.__name__] 95 | 96 | s_near, s_far = [fn_fwd(x) for x in (t_near, t_far)] 97 | t_to_s = lambda t: (fn_fwd(t) - s_near) / (s_far - s_near) 98 | s_to_t = lambda s: fn_inv(s * s_far + (1 - s) * s_near) 99 | return t_to_s, s_to_t 100 | 101 | 102 | def expected_sin(mean, var): 103 | """Compute the mean of sin(x), x ~ N(mean, var).""" 104 | return jnp.exp(-0.5 * var) * math.safe_sin(mean) # large var -> small value. 105 | 106 | 107 | def integrated_pos_enc(mean, var, min_deg, max_deg): 108 | """Encode `x` with sinusoids scaled by 2^[min_deg, max_deg). 109 | 110 | Args: 111 | mean: tensor, the mean coordinates to be encoded 112 | var: tensor, the variance of the coordinates to be encoded. 113 | min_deg: int, the min degree of the encoding. 114 | max_deg: int, the max degree of the encoding. 115 | 116 | Returns: 117 | encoded: jnp.ndarray, encoded variables. 118 | """ 119 | scales = 2**jnp.arange(min_deg, max_deg) 120 | shape = mean.shape[:-1] + (-1,) 121 | scaled_mean = jnp.reshape(mean[..., None, :] * scales[:, None], shape) 122 | scaled_var = jnp.reshape(var[..., None, :] * scales[:, None]**2, shape) 123 | 124 | return expected_sin( 125 | jnp.concatenate([scaled_mean, scaled_mean + 0.5 * jnp.pi], axis=-1), 126 | jnp.concatenate([scaled_var] * 2, axis=-1)) 127 | 128 | 129 | def lift_and_diagonalize(mean, cov, basis): 130 | """Project `mean` and `cov` onto basis and diagonalize the projected cov.""" 131 | fn_mean = math.matmul(mean, basis) 132 | fn_cov_diag = jnp.sum(basis * math.matmul(cov, basis), axis=-2) 133 | return fn_mean, fn_cov_diag 134 | 135 | 136 | def pos_enc(x, min_deg, max_deg, append_identity=True): 137 | """The positional encoding used by the original NeRF paper.""" 138 | scales = 2**jnp.arange(min_deg, max_deg) 139 | shape = x.shape[:-1] + (-1,) 140 | scaled_x = jnp.reshape((x[..., None, :] * scales[:, None]), shape) 141 | # Note that we're not using safe_sin, unlike IPE. 142 | four_feat = jnp.sin( 143 | jnp.concatenate([scaled_x, scaled_x + 0.5 * jnp.pi], axis=-1)) 144 | if append_identity: 145 | return jnp.concatenate([x] + [four_feat], axis=-1) 146 | else: 147 | return four_feat 148 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/internal/geopoly.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tools for constructing geodesic polyhedron, which are used as a basis.""" 16 | 17 | import itertools 18 | import numpy as np 19 | 20 | 21 | def compute_sq_dist(mat0, mat1=None): 22 | """Compute the squared Euclidean distance between all pairs of columns.""" 23 | if mat1 is None: 24 | mat1 = mat0 25 | # Use the fact that ||x - y||^2 == ||x||^2 + ||y||^2 - 2 x^T y. 26 | sq_norm0 = np.sum(mat0**2, 0) 27 | sq_norm1 = np.sum(mat1**2, 0) 28 | sq_dist = sq_norm0[:, None] + sq_norm1[None, :] - 2 * mat0.T @ mat1 29 | sq_dist = np.maximum(0, sq_dist) # Negative values must be numerical errors. 30 | return sq_dist 31 | 32 | 33 | def compute_tesselation_weights(v): 34 | """Tesselate the vertices of a triangle by a factor of `v`.""" 35 | if v < 1: 36 | raise ValueError(f'v {v} must be >= 1') 37 | int_weights = [] 38 | for i in range(v + 1): 39 | for j in range(v + 1 - i): 40 | int_weights.append((i, j, v - (i + j))) 41 | int_weights = np.array(int_weights) 42 | weights = int_weights / v # Barycentric weights. 43 | return weights 44 | 45 | 46 | def tesselate_geodesic(base_verts, base_faces, v, eps=1e-4): 47 | """Tesselate the vertices of a geodesic polyhedron. 48 | 49 | Args: 50 | base_verts: tensor of floats, the vertex coordinates of the geodesic. 51 | base_faces: tensor of ints, the indices of the vertices of base_verts that 52 | constitute eachface of the polyhedra. 53 | v: int, the factor of the tesselation (v==1 is a no-op). 54 | eps: float, a small value used to determine if two vertices are the same. 55 | 56 | Returns: 57 | verts: a tensor of floats, the coordinates of the tesselated vertices. 58 | """ 59 | if not isinstance(v, int): 60 | raise ValueError(f'v {v} must an integer') 61 | tri_weights = compute_tesselation_weights(v) 62 | 63 | verts = [] 64 | for base_face in base_faces: 65 | new_verts = np.matmul(tri_weights, base_verts[base_face, :]) 66 | new_verts /= np.sqrt(np.sum(new_verts**2, 1, keepdims=True)) 67 | verts.append(new_verts) 68 | verts = np.concatenate(verts, 0) 69 | 70 | sq_dist = compute_sq_dist(verts.T) 71 | assignment = np.array([np.min(np.argwhere(d <= eps)) for d in sq_dist]) 72 | unique = np.unique(assignment) 73 | verts = verts[unique, :] 74 | 75 | return verts 76 | 77 | 78 | def generate_basis(base_shape, 79 | angular_tesselation, 80 | remove_symmetries=True, 81 | eps=1e-4): 82 | """Generates a 3D basis by tesselating a geometric polyhedron. 83 | 84 | Args: 85 | base_shape: string, the name of the starting polyhedron, must be either 86 | 'icosahedron' or 'octahedron'. 87 | angular_tesselation: int, the number of times to tesselate the polyhedron, 88 | must be >= 1 (a value of 1 is a no-op to the polyhedron). 89 | remove_symmetries: bool, if True then remove the symmetric basis columns, 90 | which is usually a good idea because otherwise projections onto the basis 91 | will have redundant negative copies of each other. 92 | eps: float, a small number used to determine symmetries. 93 | 94 | Returns: 95 | basis: a matrix with shape [3, n]. 96 | """ 97 | if base_shape == 'icosahedron': 98 | a = (np.sqrt(5) + 1) / 2 99 | verts = np.array([(-1, 0, a), (1, 0, a), (-1, 0, -a), (1, 0, -a), (0, a, 1), 100 | (0, a, -1), (0, -a, 1), (0, -a, -1), (a, 1, 0), 101 | (-a, 1, 0), (a, -1, 0), (-a, -1, 0)]) / np.sqrt(a + 2) 102 | faces = np.array([(0, 4, 1), (0, 9, 4), (9, 5, 4), (4, 5, 8), (4, 8, 1), 103 | (8, 10, 1), (8, 3, 10), (5, 3, 8), (5, 2, 3), (2, 7, 3), 104 | (7, 10, 3), (7, 6, 10), (7, 11, 6), (11, 0, 6), (0, 1, 6), 105 | (6, 1, 10), (9, 0, 11), (9, 11, 2), (9, 2, 5), 106 | (7, 2, 11)]) 107 | verts = tesselate_geodesic(verts, faces, angular_tesselation) 108 | elif base_shape == 'octahedron': 109 | verts = np.array([(0, 0, -1), (0, 0, 1), (0, -1, 0), (0, 1, 0), (-1, 0, 0), 110 | (1, 0, 0)]) 111 | corners = np.array(list(itertools.product([-1, 1], repeat=3))) 112 | pairs = np.argwhere(compute_sq_dist(corners.T, verts.T) == 2) 113 | faces = np.sort(np.reshape(pairs[:, 1], [3, -1]).T, 1) 114 | verts = tesselate_geodesic(verts, faces, angular_tesselation) 115 | else: 116 | raise ValueError(f'base_shape {base_shape} not supported') 117 | 118 | if remove_symmetries: 119 | # Remove elements of `verts` that are reflections of each other. 120 | match = compute_sq_dist(verts.T, -verts.T) < eps 121 | verts = verts[np.any(np.triu(match), 1), :] 122 | 123 | basis = verts[:, ::-1] 124 | return basis 125 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/internal/image.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Functions for processing images.""" 16 | 17 | import types 18 | from typing import Optional, Union 19 | 20 | import dm_pix 21 | import jax 22 | import jax.numpy as jnp 23 | import numpy as np 24 | 25 | _Array = Union[np.ndarray, jnp.ndarray] 26 | 27 | 28 | def mse_to_psnr(mse): 29 | """Compute PSNR given an MSE (we assume the maximum pixel value is 1).""" 30 | return -10. / jnp.log(10.) * jnp.log(mse) 31 | 32 | 33 | def psnr_to_mse(psnr): 34 | """Compute MSE given a PSNR (we assume the maximum pixel value is 1).""" 35 | return jnp.exp(-0.1 * jnp.log(10.) * psnr) 36 | 37 | 38 | def ssim_to_dssim(ssim): 39 | """Compute DSSIM given an SSIM.""" 40 | return (1 - ssim) / 2 41 | 42 | 43 | def dssim_to_ssim(dssim): 44 | """Compute DSSIM given an SSIM.""" 45 | return 1 - 2 * dssim 46 | 47 | 48 | def linear_to_srgb(linear: _Array, 49 | eps: Optional[float] = None, 50 | xnp: types.ModuleType = jnp) -> _Array: 51 | """Assumes `linear` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB.""" 52 | if eps is None: 53 | eps = xnp.finfo(xnp.float32).eps 54 | srgb0 = 323 / 25 * linear 55 | srgb1 = (211 * xnp.maximum(eps, linear)**(5 / 12) - 11) / 200 56 | return xnp.where(linear <= 0.0031308, srgb0, srgb1) 57 | 58 | 59 | def srgb_to_linear(srgb: _Array, 60 | eps: Optional[float] = None, 61 | xnp: types.ModuleType = jnp) -> _Array: 62 | """Assumes `srgb` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB.""" 63 | if eps is None: 64 | eps = xnp.finfo(xnp.float32).eps 65 | linear0 = 25 / 323 * srgb 66 | linear1 = xnp.maximum(eps, ((200 * srgb + 11) / (211)))**(12 / 5) 67 | return xnp.where(srgb <= 0.04045, linear0, linear1) 68 | 69 | 70 | def downsample(img, factor): 71 | """Area downsample img (factor must evenly divide img height and width).""" 72 | sh = img.shape 73 | if not (sh[0] % factor == 0 and sh[1] % factor == 0): 74 | raise ValueError(f'Downsampling factor {factor} does not ' 75 | f'evenly divide image shape {sh[:2]}') 76 | img = img.reshape((sh[0] // factor, factor, sh[1] // factor, factor) + sh[2:]) 77 | img = img.mean((1, 3)) 78 | return img 79 | 80 | 81 | def color_correct(img, ref, num_iters=5, eps=0.5 / 255): 82 | """Warp `img` to match the colors in `ref_img`.""" 83 | if img.shape[-1] != ref.shape[-1]: 84 | raise ValueError( 85 | f'img\'s {img.shape[-1]} and ref\'s {ref.shape[-1]} channels must match' 86 | ) 87 | num_channels = img.shape[-1] 88 | img_mat = img.reshape([-1, num_channels]) 89 | ref_mat = ref.reshape([-1, num_channels]) 90 | is_unclipped = lambda z: (z >= eps) & (z <= (1 - eps)) # z \in [eps, 1-eps]. 91 | mask0 = is_unclipped(img_mat) 92 | # Because the set of saturated pixels may change after solving for a 93 | # transformation, we repeatedly solve a system `num_iters` times and update 94 | # our estimate of which pixels are saturated. 95 | for _ in range(num_iters): 96 | # Construct the left hand side of a linear system that contains a quadratic 97 | # expansion of each pixel of `img`. 98 | a_mat = [] 99 | for c in range(num_channels): 100 | a_mat.append(img_mat[:, c:(c + 1)] * img_mat[:, c:]) # Quadratic term. 101 | a_mat.append(img_mat) # Linear term. 102 | a_mat.append(jnp.ones_like(img_mat[:, :1])) # Bias term. 103 | a_mat = jnp.concatenate(a_mat, axis=-1) 104 | warp = [] 105 | for c in range(num_channels): 106 | # Construct the right hand side of a linear system containing each color 107 | # of `ref`. 108 | b = ref_mat[:, c] 109 | # Ignore rows of the linear system that were saturated in the input or are 110 | # saturated in the current corrected color estimate. 111 | mask = mask0[:, c] & is_unclipped(img_mat[:, c]) & is_unclipped(b) 112 | ma_mat = jnp.where(mask[:, None], a_mat, 0) 113 | mb = jnp.where(mask, b, 0) 114 | # Solve the linear system. We're using the np.lstsq instead of jnp because 115 | # it's significantly more stable in this case, for some reason. 116 | w = np.linalg.lstsq(ma_mat, mb, rcond=-1)[0] 117 | assert jnp.all(jnp.isfinite(w)) 118 | warp.append(w) 119 | warp = jnp.stack(warp, axis=-1) 120 | # Apply the warp to update img_mat. 121 | img_mat = jnp.clip( 122 | jnp.matmul(a_mat, warp, precision=jax.lax.Precision.HIGHEST), 0, 1) 123 | corrected_img = jnp.reshape(img_mat, img.shape) 124 | return corrected_img 125 | 126 | 127 | class MetricHarness: 128 | """A helper class for evaluating several error metrics.""" 129 | 130 | def __init__(self): 131 | self.ssim_fn = jax.jit(dm_pix.ssim) 132 | 133 | def __call__(self, rgb_pred, rgb_gt, name_fn=lambda s: s): 134 | """Evaluate the error between a predicted rgb image and the true image.""" 135 | psnr = float(mse_to_psnr(((rgb_pred - rgb_gt)**2).mean())) 136 | ssim = float(self.ssim_fn(rgb_pred, rgb_gt)) 137 | 138 | return { 139 | name_fn('psnr'): psnr, 140 | name_fn('ssim'): ssim, 141 | } 142 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/internal/math.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Mathy utility functions.""" 16 | 17 | import jax 18 | import jax.numpy as jnp 19 | 20 | 21 | def matmul(a, b): 22 | """jnp.matmul defaults to bfloat16, but this helper function doesn't.""" 23 | return jnp.matmul(a, b, precision=jax.lax.Precision.HIGHEST) 24 | 25 | 26 | def safe_trig_helper(x, fn, t=100 * jnp.pi): 27 | """Helper function used by safe_cos/safe_sin: mods x before sin()/cos().""" 28 | return fn(jnp.where(jnp.abs(x) < t, x, x % t)) 29 | 30 | 31 | def safe_cos(x): 32 | """jnp.cos() on a TPU may NaN out for large values.""" 33 | return safe_trig_helper(x, jnp.cos) 34 | 35 | 36 | def safe_sin(x): 37 | """jnp.sin() on a TPU may NaN out for large values.""" 38 | return safe_trig_helper(x, jnp.sin) 39 | 40 | 41 | @jax.custom_jvp 42 | def safe_exp(x): 43 | """jnp.exp() but with finite output and gradients for large inputs.""" 44 | return jnp.exp(jnp.minimum(x, 88.)) # jnp.exp(89) is infinity. 45 | 46 | 47 | @safe_exp.defjvp 48 | def safe_exp_jvp(primals, tangents): 49 | """Override safe_exp()'s gradient so that it's large when inputs are large.""" 50 | x, = primals 51 | x_dot, = tangents 52 | exp_x = safe_exp(x) 53 | exp_x_dot = exp_x * x_dot 54 | return exp_x, exp_x_dot 55 | 56 | 57 | def log_lerp(t, v0, v1): 58 | """Interpolate log-linearly from `v0` (t=0) to `v1` (t=1).""" 59 | if v0 <= 0 or v1 <= 0: 60 | raise ValueError(f'Interpolants {v0} and {v1} must be positive.') 61 | lv0 = jnp.log(v0) 62 | lv1 = jnp.log(v1) 63 | return jnp.exp(jnp.clip(t, 0, 1) * (lv1 - lv0) + lv0) 64 | 65 | 66 | def learning_rate_decay(step, 67 | lr_init, 68 | lr_final, 69 | max_steps, 70 | lr_delay_steps=0, 71 | lr_delay_mult=1): 72 | """Continuous learning rate decay function. 73 | 74 | The returned rate is lr_init when step=0 and lr_final when step=max_steps, and 75 | is log-linearly interpolated elsewhere (equivalent to exponential decay). 76 | If lr_delay_steps>0 then the learning rate will be scaled by some smooth 77 | function of lr_delay_mult, such that the initial learning rate is 78 | lr_init*lr_delay_mult at the beginning of optimization but will be eased back 79 | to the normal learning rate when steps>lr_delay_steps. 80 | 81 | Args: 82 | step: int, the current optimization step. 83 | lr_init: float, the initial learning rate. 84 | lr_final: float, the final learning rate. 85 | max_steps: int, the number of steps during optimization. 86 | lr_delay_steps: int, the number of steps to delay the full learning rate. 87 | lr_delay_mult: float, the multiplier on the rate when delaying it. 88 | 89 | Returns: 90 | lr: the learning for current step 'step'. 91 | """ 92 | if lr_delay_steps > 0: 93 | # A kind of reverse cosine decay. 94 | delay_rate = lr_delay_mult + (1 - lr_delay_mult) * jnp.sin( 95 | 0.5 * jnp.pi * jnp.clip(step / lr_delay_steps, 0, 1)) 96 | else: 97 | delay_rate = 1. 98 | return delay_rate * log_lerp(step / max_steps, lr_init, lr_final) 99 | 100 | 101 | def interp(*args): 102 | """A gather-based (GPU-friendly) vectorized replacement for jnp.interp().""" 103 | args_flat = [x.reshape([-1, x.shape[-1]]) for x in args] 104 | ret = jax.vmap(jnp.interp)(*args_flat).reshape(args[0].shape) 105 | return ret 106 | 107 | 108 | def sorted_interp(x, xp, fp): 109 | """A TPU-friendly version of interp(), where xp and fp must be sorted.""" 110 | 111 | # Identify the location in `xp` that corresponds to each `x`. 112 | # The final `True` index in `mask` is the start of the matching interval. 113 | mask = x[..., None, :] >= xp[..., :, None] 114 | 115 | def find_interval(x): 116 | # Grab the value where `mask` switches from True to False, and vice versa. 117 | # This approach takes advantage of the fact that `x` is sorted. 118 | x0 = jnp.max(jnp.where(mask, x[..., None], x[..., :1, None]), -2) 119 | x1 = jnp.min(jnp.where(~mask, x[..., None], x[..., -1:, None]), -2) 120 | return x0, x1 121 | 122 | fp0, fp1 = find_interval(fp) 123 | xp0, xp1 = find_interval(xp) 124 | 125 | offset = jnp.clip(jnp.nan_to_num((x - xp0) / (xp1 - xp0), 0), 0, 1) 126 | ret = fp0 + offset * (fp1 - fp0) 127 | return ret 128 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/internal/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utility functions.""" 16 | 17 | import enum 18 | import os 19 | from typing import Any, Dict, Optional, Union 20 | 21 | import flax 22 | import jax 23 | import jax.numpy as jnp 24 | import numpy as np 25 | from PIL import ExifTags 26 | from PIL import Image 27 | 28 | _Array = Union[np.ndarray, jnp.ndarray] 29 | 30 | 31 | @flax.struct.dataclass 32 | class Pixels: 33 | """All tensors must have the same num_dims and first n-1 dims must match.""" 34 | pix_x_int: _Array 35 | pix_y_int: _Array 36 | lossmult: _Array 37 | near: _Array 38 | far: _Array 39 | cam_idx: _Array 40 | exposure_idx: Optional[_Array] = None 41 | exposure_values: Optional[_Array] = None 42 | 43 | 44 | @flax.struct.dataclass 45 | class Rays: 46 | """All tensors must have the same num_dims and first n-1 dims must match.""" 47 | origins: _Array 48 | directions: _Array 49 | viewdirs: _Array 50 | radii: _Array 51 | imageplane: _Array 52 | lossmult: _Array 53 | near: _Array 54 | far: _Array 55 | cam_idx: _Array 56 | exposure_idx: Optional[_Array] = None 57 | exposure_values: Optional[_Array] = None 58 | 59 | 60 | # Dummy Rays object that can be used to initialize NeRF model. 61 | def dummy_rays(include_exposure_idx: bool = False, 62 | include_exposure_values: bool = False) -> Rays: 63 | data_fn = lambda n: jnp.zeros((1, n)) 64 | exposure_kwargs = {} 65 | if include_exposure_idx: 66 | exposure_kwargs['exposure_idx'] = data_fn(1).astype(jnp.int32) 67 | if include_exposure_values: 68 | exposure_kwargs['exposure_values'] = data_fn(1) 69 | return Rays( 70 | origins=data_fn(3), 71 | directions=data_fn(3), 72 | viewdirs=data_fn(3), 73 | radii=data_fn(1), 74 | imageplane=data_fn(2), 75 | lossmult=data_fn(1), 76 | near=data_fn(1), 77 | far=data_fn(1), 78 | cam_idx=data_fn(1).astype(jnp.int32), 79 | **exposure_kwargs) 80 | 81 | 82 | @flax.struct.dataclass 83 | class Batch: 84 | """Data batch for NeRF training or testing.""" 85 | rays: Union[Pixels, Rays] 86 | rgb: Optional[_Array] = None 87 | disps: Optional[_Array] = None 88 | normals: Optional[_Array] = None 89 | alphas: Optional[_Array] = None 90 | 91 | 92 | class DataSplit(enum.Enum): 93 | """Dataset split.""" 94 | TRAIN = 'train' 95 | TEST = 'test' 96 | 97 | 98 | class BatchingMethod(enum.Enum): 99 | """Draw rays randomly from a single image or all images, in each batch.""" 100 | ALL_IMAGES = 'all_images' 101 | SINGLE_IMAGE = 'single_image' 102 | 103 | 104 | def open_file(pth, mode='r'): 105 | return open(pth, mode=mode) 106 | 107 | 108 | def file_exists(pth): 109 | return os.path.exists(pth) 110 | 111 | 112 | def listdir(pth): 113 | return os.listdir(pth) 114 | 115 | 116 | def isdir(pth): 117 | return os.path.isdir(pth) 118 | 119 | 120 | def makedirs(pth): 121 | if not file_exists(pth): 122 | os.makedirs(pth) 123 | 124 | 125 | def shard(xs): 126 | """Split data into shards for multiple devices along the first dimension.""" 127 | return jax.tree_util.tree_map( 128 | lambda x: x.reshape((jax.local_device_count(), -1) + x.shape[1:]), xs) 129 | 130 | 131 | def unshard(x, padding=0): 132 | """Collect the sharded tensor to the shape before sharding.""" 133 | y = x.reshape([x.shape[0] * x.shape[1]] + list(x.shape[2:])) 134 | if padding > 0: 135 | y = y[:-padding] 136 | return y 137 | 138 | 139 | def load_img(pth: str) -> np.ndarray: 140 | """Load an image and cast to float32.""" 141 | with open_file(pth, 'rb') as f: 142 | image = np.array(Image.open(f), dtype=np.float32) 143 | return image 144 | 145 | 146 | def load_exif(pth: str) -> Dict[str, Any]: 147 | """Load EXIF data for an image.""" 148 | with open_file(pth, 'rb') as f: 149 | image_pil = Image.open(f) 150 | exif_pil = image_pil._getexif() # pylint: disable=protected-access 151 | if exif_pil is not None: 152 | exif = { 153 | ExifTags.TAGS[k]: v for k, v in exif_pil.items() if k in ExifTags.TAGS 154 | } 155 | else: 156 | exif = {} 157 | return exif 158 | 159 | 160 | def save_img_u8(img, pth): 161 | """Save an image (probably RGB) in [0, 1] to disk as a uint8 PNG.""" 162 | with open_file(pth, 'wb') as f: 163 | Image.fromarray( 164 | (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)).save( 165 | f, 'PNG') 166 | 167 | 168 | def save_img_f32(depthmap, pth): 169 | """Save an image (probably a depthmap) to disk as a float32 TIFF.""" 170 | with open_file(pth, 'wb') as f: 171 | Image.fromarray(np.nan_to_num(depthmap).astype(np.float32)).save(f, 'TIFF') 172 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | jax 3 | jaxlib 4 | flax 5 | opencv-python 6 | Pillow 7 | tensorboard 8 | tensorflow 9 | gin-config 10 | dm_pix 11 | rawpy 12 | mediapy 13 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/eval_360.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=gardenvase 19 | EXPERIMENT=360 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_real_360 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m eval \ 24 | --gin_configs=configs/360.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --logtostderr 28 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/eval_blender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=ficus 19 | EXPERIMENT=blender 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/dors_nerf_synthetic 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m eval \ 24 | --gin_configs=configs/blender_256.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --logtostderr 28 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/eval_llff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=flower 19 | EXPERIMENT=llff 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_llff_data 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m eval \ 24 | --gin_configs=configs/llff_256.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --logtostderr 28 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/eval_raw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=nightpiano 19 | EXPERIMENT=raw 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/rawnerf/scenes 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m eval \ 24 | --gin_configs=configs/llff_raw.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --logtostderr 28 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/eval_shinyblender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=toaster 19 | EXPERIMENT=shinyblender 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/dors_nerf_synthetic 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m eval \ 24 | --gin_configs=configs/blender_refnerf.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --logtostderr 28 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/local_colmap_and_resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Set to 0 if you do not have a GPU. 18 | USE_GPU=1 19 | # Path to a directory `base/` with images in `base/images/`. 20 | DATASET_PATH=$1 21 | # Recommended CAMERA values: OPENCV for perspective, OPENCV_FISHEYE for fisheye. 22 | CAMERA=${2:-OPENCV} 23 | 24 | 25 | # Run COLMAP. 26 | 27 | ### Feature extraction 28 | 29 | colmap feature_extractor \ 30 | --database_path "$DATASET_PATH"/database.db \ 31 | --image_path "$DATASET_PATH"/images \ 32 | --ImageReader.single_camera 1 \ 33 | --ImageReader.camera_model "$CAMERA" \ 34 | --SiftExtraction.use_gpu "$USE_GPU" 35 | 36 | 37 | ### Feature matching 38 | 39 | colmap exhaustive_matcher \ 40 | --database_path "$DATASET_PATH"/database.db \ 41 | --SiftMatching.use_gpu "$USE_GPU" 42 | 43 | ## Use if your scene has > 500 images 44 | ## Replace this path with your own local copy of the file. 45 | ## Download from: https://demuc.de/colmap/#download 46 | # VOCABTREE_PATH=/usr/local/google/home/bmild/vocab_tree_flickr100K_words32K.bin 47 | # colmap vocab_tree_matcher \ 48 | # --database_path "$DATASET_PATH"/database.db \ 49 | # --VocabTreeMatching.vocab_tree_path $VOCABTREE_PATH \ 50 | # --SiftMatching.use_gpu "$USE_GPU" 51 | 52 | 53 | ### Bundle adjustment 54 | 55 | # The default Mapper tolerance is unnecessarily large, 56 | # decreasing it speeds up bundle adjustment steps. 57 | mkdir -p "$DATASET_PATH"/sparse 58 | colmap mapper \ 59 | --database_path "$DATASET_PATH"/database.db \ 60 | --image_path "$DATASET_PATH"/images \ 61 | --output_path "$DATASET_PATH"/sparse \ 62 | --Mapper.ba_global_function_tolerance=0.000001 63 | 64 | 65 | ### Image undistortion 66 | 67 | ## Use this if you want to undistort your images into ideal pinhole intrinsics. 68 | # mkdir -p "$DATASET_PATH"/dense 69 | # colmap image_undistorter \ 70 | # --image_path "$DATASET_PATH"/images \ 71 | # --input_path "$DATASET_PATH"/sparse/0 \ 72 | # --output_path "$DATASET_PATH"/dense \ 73 | # --output_type COLMAP 74 | 75 | # Resize images. 76 | 77 | cp -r "$DATASET_PATH"/images "$DATASET_PATH"/images_2 78 | 79 | pushd "$DATASET_PATH"/images_2 80 | ls | xargs -P 8 -I {} mogrify -resize 50% {} 81 | popd 82 | 83 | cp -r "$DATASET_PATH"/images "$DATASET_PATH"/images_4 84 | 85 | pushd "$DATASET_PATH"/images_4 86 | ls | xargs -P 8 -I {} mogrify -resize 25% {} 87 | popd 88 | 89 | cp -r "$DATASET_PATH"/images "$DATASET_PATH"/images_8 90 | 91 | pushd "$DATASET_PATH"/images_8 92 | ls | xargs -P 8 -I {} mogrify -resize 12.5% {} 93 | popd 94 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/render_360.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=gardenvase 19 | EXPERIMENT=360 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_real_360 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m render \ 24 | --gin_configs=configs/360.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --gin_bindings="Config.render_path = True" \ 28 | --gin_bindings="Config.render_path_frames = 10" \ 29 | --gin_bindings="Config.render_dir = '${CHECKPOINT_DIR}/render/'" \ 30 | --gin_bindings="Config.render_video_fps = 2" \ 31 | --logtostderr 32 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/render_llff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=flower 19 | EXPERIMENT=llff 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_llff_data 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m render \ 24 | --gin_configs=configs/llff_256.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --gin_bindings="Config.render_path = True" \ 28 | --gin_bindings="Config.render_path_frames = 10" \ 29 | --gin_bindings="Config.render_dir = '${CHECKPOINT_DIR}/render/'" \ 30 | --gin_bindings="Config.render_video_fps = 2" \ 31 | --logtostderr 32 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/render_raw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=nightpiano 19 | EXPERIMENT=raw 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/rawnerf/scenes 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | python -m render \ 24 | --gin_configs=configs/llff_raw.gin \ 25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 27 | --gin_bindings="Config.render_path = True" \ 28 | --gin_bindings="Config.render_path_frames = 10" \ 29 | --gin_bindings="Config.render_dir = '${CHECKPOINT_DIR}/render/'" \ 30 | --gin_bindings="Config.render_video_fps = 2" \ 31 | --logtostderr 32 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/run_all_unit_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | python -m unittest tests.camera_utils_test 18 | python -m unittest tests.geopoly_test 19 | python -m unittest tests.stepfun_test 20 | python -m unittest tests.coord_test 21 | python -m unittest tests.image_test 22 | python -m unittest tests.ref_utils_test 23 | python -m unittest tests.utils_test 24 | python -m unittest tests.datasets_test 25 | python -m unittest tests.math_test 26 | python -m unittest tests.render_test 27 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/train_360.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=gardenvase 19 | EXPERIMENT=360 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_real_360 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | # If running one of the indoor scenes, add 24 | # --gin_bindings="Config.factor = 2" 25 | 26 | rm "$CHECKPOINT_DIR"/* 27 | python -m train \ 28 | --gin_configs=configs/360.gin \ 29 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 30 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 31 | --logtostderr 32 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/train_blender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=ficus 19 | EXPERIMENT=blender 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_synthetic 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | rm "$CHECKPOINT_DIR"/* 24 | python -m train \ 25 | --gin_configs=configs/blender_256.gin \ 26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/train_llff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=flower 19 | EXPERIMENT=llff 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_llff_data 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | rm "$CHECKPOINT_DIR"/* 24 | python -m train \ 25 | --gin_configs=configs/llff_256.gin \ 26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/train_raw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=nightpiano 19 | EXPERIMENT=raw 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/rawnerf/scenes 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | rm "$CHECKPOINT_DIR"/* 24 | python -m train \ 25 | --gin_configs=configs/llff_raw.gin \ 26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/scripts/train_shinyblender.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | export CUDA_VISIBLE_DEVICES=0 17 | 18 | SCENE=toaster 19 | EXPERIMENT=shinyblender 20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/dors_nerf_synthetic 21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE" 22 | 23 | rm "$CHECKPOINT_DIR"/* 24 | python -m train \ 25 | --gin_configs=configs/blender_refnerf.gin \ 26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \ 27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \ 28 | --logtostderr 29 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/tests/camera_utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for camera_utils.""" 16 | 17 | from absl.testing import absltest 18 | from absl.testing import parameterized 19 | from internal import camera_utils 20 | from jax import random 21 | import jax.numpy as jnp 22 | import numpy as np 23 | 24 | 25 | class CameraUtilsTest(parameterized.TestCase): 26 | 27 | def test_convert_to_ndc(self): 28 | rng = random.PRNGKey(0) 29 | for _ in range(10): 30 | # Random pinhole camera intrinsics. 31 | key, rng = random.split(rng) 32 | focal, width, height = random.uniform(key, (3,), minval=100., maxval=200.) 33 | camtopix = camera_utils.intrinsic_matrix(focal, focal, width / 2., 34 | height / 2.) 35 | pixtocam = np.linalg.inv(camtopix) 36 | near = 1. 37 | 38 | # Random rays, pointing forward (negative z direction). 39 | num_rays = 1000 40 | key, rng = random.split(rng) 41 | origins = jnp.array([0., 0., 1.]) 42 | origins += random.uniform(key, (num_rays, 3), minval=-1., maxval=1.) 43 | directions = jnp.array([0., 0., -1.]) 44 | directions += random.uniform(key, (num_rays, 3), minval=-.5, maxval=.5) 45 | 46 | # Project world-space points along each ray into NDC space. 47 | t = jnp.linspace(0., 1., 10) 48 | pts_world = origins + t[:, None, None] * directions 49 | pts_ndc = jnp.stack([ 50 | -focal / (.5 * width) * pts_world[..., 0] / pts_world[..., 2], 51 | -focal / (.5 * height) * pts_world[..., 1] / pts_world[..., 2], 52 | 1. + 2. * near / pts_world[..., 2], 53 | ], 54 | axis=-1) 55 | 56 | # Get NDC space rays. 57 | origins_ndc, directions_ndc = camera_utils.convert_to_ndc( 58 | origins, directions, pixtocam, near) 59 | 60 | # Ensure that the NDC space points lie on the calculated rays. 61 | directions_ndc_norm = jnp.linalg.norm( 62 | directions_ndc, axis=-1, keepdims=True) 63 | directions_ndc_unit = directions_ndc / directions_ndc_norm 64 | projection = ((pts_ndc - origins_ndc) * directions_ndc_unit).sum(axis=-1) 65 | pts_ndc_proj = origins_ndc + directions_ndc_unit * projection[..., None] 66 | 67 | # pts_ndc should be close to their projections pts_ndc_proj onto the rays. 68 | np.testing.assert_allclose(pts_ndc, pts_ndc_proj, atol=1e-5, rtol=1e-5) 69 | 70 | 71 | if __name__ == '__main__': 72 | absltest.main() 73 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/tests/datasets_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for datasets.""" 16 | 17 | from absl.testing import absltest 18 | from internal import camera_utils 19 | from internal import configs 20 | from internal import datasets 21 | from jax import random 22 | import numpy as np 23 | 24 | 25 | class DummyDataset(datasets.Dataset): 26 | 27 | def _load_renderings(self, config): 28 | """Generates dummy image and pose data.""" 29 | self._n_examples = 2 30 | self.height = 3 31 | self.width = 4 32 | self._resolution = self.height * self.width 33 | self.focal = 5. 34 | self.pixtocams = np.linalg.inv( 35 | camera_utils.intrinsic_matrix(self.focal, self.focal, self.width * 0.5, 36 | self.height * 0.5)) 37 | 38 | rng = random.PRNGKey(0) 39 | 40 | key, rng = random.split(rng) 41 | images_shape = (self._n_examples, self.height, self.width, 3) 42 | self.images = random.uniform(key, images_shape) 43 | 44 | key, rng = random.split(rng) 45 | self.camtoworlds = np.stack([ 46 | camera_utils.viewmatrix(*random.normal(k, (3, 3))) 47 | for k in random.split(key, self._n_examples) 48 | ], 49 | axis=0) 50 | 51 | 52 | class DatasetsTest(absltest.TestCase): 53 | 54 | def test_dataset_batch_creation(self): 55 | np.random.seed(0) 56 | config = configs.Config(batch_size=8) 57 | 58 | # Check shapes are consistent across all ray attributes. 59 | for split in ['train', 'test']: 60 | dummy_dataset = DummyDataset(split, '', config) 61 | rays = dummy_dataset.peek().rays 62 | sh_gt = rays.origins.shape[:-1] 63 | for z in rays.__dict__.values(): 64 | if z is not None: 65 | self.assertEqual(z.shape[:-1], sh_gt) 66 | 67 | # Check test batch generation matches golden data. 68 | dummy_dataset = DummyDataset('test', '', config) 69 | batch = dummy_dataset.peek() 70 | 71 | rgb = batch.rgb.ravel() 72 | rgb_gt = np.array([ 73 | 0.5289556, 0.28869557, 0.24527192, 0.12083626, 0.8904066, 0.6259936, 74 | 0.57573485, 0.09355974, 0.8017353, 0.538651, 0.4998169, 0.42061496, 75 | 0.5591258, 0.00577283, 0.6804651, 0.9139203, 0.00444758, 0.96962905, 76 | 0.52956843, 0.38282406, 0.28777933, 0.6640035, 0.39736128, 0.99495006, 77 | 0.13100398, 0.7597165, 0.8532667, 0.67468107, 0.6804743, 0.26873016, 78 | 0.60699487, 0.5722265, 0.44482303, 0.6511061, 0.54807067, 0.09894073 79 | ]) 80 | np.testing.assert_allclose(rgb, rgb_gt, atol=1e-4, rtol=1e-4) 81 | 82 | ray_origins = batch.rays.origins.ravel() 83 | ray_origins_gt = np.array([ 84 | -0.20050469, -0.6451472, -0.8818224, -0.20050469, -0.6451472, 85 | -0.8818224, -0.20050469, -0.6451472, -0.8818224, -0.20050469, 86 | -0.6451472, -0.8818224, -0.20050469, -0.6451472, -0.8818224, 87 | -0.20050469, -0.6451472, -0.8818224, -0.20050469, -0.6451472, 88 | -0.8818224, -0.20050469, -0.6451472, -0.8818224, -0.20050469, 89 | -0.6451472, -0.8818224, -0.20050469, -0.6451472, -0.8818224, 90 | -0.20050469, -0.6451472, -0.8818224, -0.20050469, -0.6451472, -0.8818224 91 | ]) 92 | np.testing.assert_allclose( 93 | ray_origins, ray_origins_gt, atol=1e-4, rtol=1e-4) 94 | 95 | ray_dirs = batch.rays.directions.ravel() 96 | ray_dirs_gt = np.array([ 97 | 0.24370372, 0.89296186, -0.5227117, 0.05601424, 0.8468699, -0.57417226, 98 | -0.13167524, 0.8007779, -0.62563276, -0.31936473, 0.75468594, 99 | -0.67709327, 0.17780769, 0.96766925, -0.34928587, -0.0098818, 0.9215773, 100 | -0.4007464, -0.19757128, 0.87548524, -0.4522069, -0.38526076, 101 | 0.82939327, -0.5036674, 0.11191163, 1.0423766, -0.17586003, -0.07577785, 102 | 0.9962846, -0.22732055, -0.26346734, 0.95019263, -0.2787811, 103 | -0.45115682, 0.90410066, -0.3302416 104 | ]) 105 | np.testing.assert_allclose(ray_dirs, ray_dirs_gt, atol=1e-4, rtol=1e-4) 106 | 107 | 108 | if __name__ == '__main__': 109 | absltest.main() 110 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/tests/ref_utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for ref_utils.""" 16 | 17 | from absl.testing import absltest 18 | from internal import ref_utils 19 | from jax import random 20 | import jax.numpy as jnp 21 | import numpy as np 22 | import scipy 23 | 24 | 25 | def generate_dir_enc_fn_scipy(deg_view): 26 | """Return spherical harmonics using scipy.special.sph_harm.""" 27 | ml_array = ref_utils.get_ml_array(deg_view) 28 | 29 | def dir_enc_fn(theta, phi): 30 | de = [scipy.special.sph_harm(m, l, phi, theta) for m, l in ml_array.T] 31 | de = np.stack(de, axis=-1) 32 | # Split into real and imaginary parts. 33 | return np.concatenate([np.real(de), np.imag(de)], axis=-1) 34 | 35 | return dir_enc_fn 36 | 37 | 38 | class RefUtilsTest(absltest.TestCase): 39 | 40 | def test_reflection(self): 41 | """Make sure reflected vectors have the same angle from normals as input.""" 42 | rng = random.PRNGKey(0) 43 | for shape in [(45, 3), (4, 7, 3)]: 44 | key, rng = random.split(rng) 45 | normals = random.normal(key, shape) 46 | key, rng = random.split(rng) 47 | directions = random.normal(key, shape) 48 | 49 | # Normalize normal vectors. 50 | normals = normals / ( 51 | jnp.linalg.norm(normals, axis=-1, keepdims=True) + 1e-10) 52 | 53 | reflected_directions = ref_utils.reflect(directions, normals) 54 | 55 | cos_angle_original = jnp.sum(directions * normals, axis=-1) 56 | cos_angle_reflected = jnp.sum(reflected_directions * normals, axis=-1) 57 | 58 | np.testing.assert_allclose( 59 | cos_angle_original, cos_angle_reflected, atol=1E-5, rtol=1E-5) 60 | 61 | def test_spherical_harmonics(self): 62 | """Make sure the fast spherical harmonics are accurate.""" 63 | shape = (12, 11, 13) 64 | 65 | # Generate random points on sphere. 66 | rng = random.PRNGKey(0) 67 | key1, key2 = random.split(rng) 68 | theta = random.uniform(key1, shape, minval=0.0, maxval=jnp.pi) 69 | phi = random.uniform(key2, shape, minval=0.0, maxval=2.0*jnp.pi) 70 | 71 | # Convert to Cartesian coordinates. 72 | x = jnp.sin(theta) * jnp.cos(phi) 73 | y = jnp.sin(theta) * jnp.sin(phi) 74 | z = jnp.cos(theta) 75 | xyz = jnp.stack([x, y, z], axis=-1) 76 | 77 | deg_view = 5 78 | de = ref_utils.generate_dir_enc_fn(deg_view)(xyz) 79 | de_scipy = generate_dir_enc_fn_scipy(deg_view)(theta, phi) 80 | 81 | np.testing.assert_allclose( 82 | de, de_scipy, atol=0.02, rtol=1e6) # Only use atol. 83 | self.assertFalse(jnp.any(jnp.isnan(de))) 84 | 85 | 86 | if __name__ == '__main__': 87 | absltest.main() 88 | -------------------------------------------------------------------------------- /CAM_multinerf-mipnerf360/tests/utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for utils.""" 16 | 17 | from absl.testing import absltest 18 | 19 | from internal import utils 20 | 21 | 22 | class UtilsTest(absltest.TestCase): 23 | 24 | def test_dummy_rays(self): 25 | """Ensures that the dummy Rays object is correctly initialized.""" 26 | rays = utils.dummy_rays() 27 | self.assertEqual(rays.origins.shape[-1], 3) 28 | 29 | 30 | if __name__ == '__main__': 31 | absltest.main() 32 | -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/aws/update_index.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | import boto3 4 | 5 | ROOT_URL = 'https://nerfacc-bucket.s3.us-west-2.amazonaws.com/whl' 6 | html = '\n\n\n{}\n\n' 7 | href = ' {}
' 8 | args = { 9 | 'ContentType': 'text/html', 10 | 'CacheControl': 'max-age=300', 11 | 'ACL': 'public-read', 12 | } 13 | 14 | bucket = boto3.resource('s3').Bucket(name='nerfacc-bucket') 15 | 16 | wheels_dict = defaultdict(list) 17 | for obj in bucket.objects.filter(Prefix='whl'): 18 | if obj.key[-3:] != 'whl': 19 | continue 20 | torch_version, wheel = obj.key.split('/')[-2:] 21 | wheel = f'{torch_version}/{wheel}' 22 | wheels_dict[torch_version].append(wheel) 23 | 24 | index_html = html.format('\n'.join([ 25 | href.format(f'{torch_version}.html'.replace('+', '%2B'), version) 26 | for version in wheels_dict 27 | ])) 28 | 29 | with open('index.html', 'w') as f: 30 | f.write(index_html) 31 | bucket.Object('whl/index.html').upload_file('index.html', args) 32 | 33 | for torch_version, wheel_names in wheels_dict.items(): 34 | torch_version_html = html.format('\n'.join([ 35 | href.format(f'{ROOT_URL}/{wheel_name}'.replace('+', '%2B'), wheel_name) 36 | for wheel_name in wheel_names 37 | ])) 38 | 39 | with open(f'{torch_version}.html', 'w') as f: 40 | f.write(torch_version_html) 41 | bucket.Object(f'whl/{torch_version}.html').upload_file( 42 | f'{torch_version}.html', args) -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/code_checks.yml: -------------------------------------------------------------------------------- 1 | name: Core Tests. 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Set up Python 3.8.12 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: "3.8.12" 22 | - name: Install dependencies 23 | run: | 24 | pip install isort==5.10.1 black[jupyter]==22.3.0 25 | - name: Run isort 26 | run: isort docs/ nerfacc/ scripts/ examples/ tests/ --profile black --skip examples/pycolmap --line-length 80 --check 27 | - name: Run Black 28 | run: black docs/ nerfacc/ scripts/ examples/ tests/ --exclude examples/pycolmap --line-length 80 --check 29 | # - name: Python Pylint 30 | # run: | 31 | # pylint nerfacc/ tests/ scripts/ examples/ 32 | -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/cuda/Linux-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Took from https://github.com/pyg-team/pyg-lib/ 4 | 5 | case ${1} in 6 | cu118) 7 | CUDA_HOME=/usr/local/cuda-11.8 8 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} 9 | PATH=${CUDA_HOME}/bin:${PATH} 10 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" 11 | ;; 12 | cu117) 13 | CUDA_HOME=/usr/local/cuda-11.7 14 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} 15 | PATH=${CUDA_HOME}/bin:${PATH} 16 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" 17 | ;; 18 | cu116) 19 | CUDA_HOME=/usr/local/cuda-11.6 20 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} 21 | PATH=${CUDA_HOME}/bin:${PATH} 22 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" 23 | ;; 24 | cu115) 25 | CUDA_HOME=/usr/local/cuda-11.5 26 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} 27 | PATH=${CUDA_HOME}/bin:${PATH} 28 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" 29 | ;; 30 | cu113) 31 | CUDA_HOME=/usr/local/cuda-11.3 32 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} 33 | PATH=${CUDA_HOME}/bin:${PATH} 34 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" 35 | ;; 36 | cu102) 37 | CUDA_HOME=/usr/local/cuda-10.2 38 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} 39 | PATH=${CUDA_HOME}/bin:${PATH} 40 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5" 41 | ;; 42 | *) 43 | ;; 44 | esac -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/cuda/Linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Took from https://github.com/pyg-team/pyg-lib/ 4 | 5 | OS=ubuntu2004 6 | 7 | case ${1} in 8 | cu118) 9 | CUDA=11.8 10 | APT_KEY=${OS}-${CUDA/./-}-local 11 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.0-520.61.05-1_amd64.deb 12 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.0/local_installers 13 | ;; 14 | cu117) 15 | CUDA=11.7 16 | APT_KEY=${OS}-${CUDA/./-}-local 17 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.1-515.65.01-1_amd64.deb 18 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.1/local_installers 19 | ;; 20 | cu116) 21 | CUDA=11.6 22 | APT_KEY=${OS}-${CUDA/./-}-local 23 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.2-510.47.03-1_amd64.deb 24 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.2/local_installers 25 | ;; 26 | cu115) 27 | CUDA=11.5 28 | APT_KEY=${OS}-${CUDA/./-}-local 29 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.2-495.29.05-1_amd64.deb 30 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.2/local_installers 31 | ;; 32 | cu113) 33 | CUDA=11.3 34 | APT_KEY=${OS}-${CUDA/./-}-local 35 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.0-465.19.01-1_amd64.deb 36 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.0/local_installers 37 | ;; 38 | cu102) 39 | CUDA=10.2 40 | APT_KEY=${CUDA/./-}-local-${CUDA}.89-440.33.01 41 | FILENAME=cuda-repo-${OS}-${APT_KEY}_1.0-1_amd64.deb 42 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}/Prod/local_installers 43 | ;; 44 | *) 45 | echo "Unrecognized CUDA_VERSION=${1}" 46 | exit 1 47 | ;; 48 | esac 49 | 50 | wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin 51 | sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600 52 | wget -nv ${URL}/${FILENAME} 53 | sudo dpkg -i ${FILENAME} 54 | 55 | if [ "${1}" = "cu117" ] || [ "${1}" = "cu118" ]; then 56 | sudo cp /var/cuda-repo-${APT_KEY}/cuda-*-keyring.gpg /usr/share/keyrings/ 57 | else 58 | sudo apt-key add /var/cuda-repo-${APT_KEY}/7fa2af80.pub 59 | fi 60 | 61 | sudo apt-get update 62 | sudo apt-get -y install cuda 63 | 64 | rm -f ${FILENAME} -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/cuda/Windows-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Took from https://github.com/pyg-team/pyg-lib/ 4 | 5 | case ${1} in 6 | cu118) 7 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.8 8 | PATH=${CUDA_HOME}/bin:$PATH 9 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH 10 | export TORCH_CUDA_ARCH_LIST="6.0+PTX" 11 | ;; 12 | cu117) 13 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.7 14 | PATH=${CUDA_HOME}/bin:$PATH 15 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH 16 | export TORCH_CUDA_ARCH_LIST="6.0+PTX" 17 | ;; 18 | cu116) 19 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.6 20 | PATH=${CUDA_HOME}/bin:$PATH 21 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH 22 | export TORCH_CUDA_ARCH_LIST="6.0+PTX" 23 | ;; 24 | cu115) 25 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.5 26 | PATH=${CUDA_HOME}/bin:$PATH 27 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH 28 | export TORCH_CUDA_ARCH_LIST="6.0+PTX" 29 | ;; 30 | cu113) 31 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3 32 | PATH=${CUDA_HOME}/bin:$PATH 33 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH 34 | export TORCH_CUDA_ARCH_LIST="6.0+PTX" 35 | ;; 36 | *) 37 | ;; 38 | esac -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/cuda/Windows.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Took from https://github.com/pyg-team/pyg-lib/ 4 | 5 | # Install NVIDIA drivers, see: 6 | # https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102 7 | curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip" 8 | 7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32" 9 | 10 | case ${1} in 11 | cu118) 12 | CUDA_SHORT=11.8 13 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers 14 | CUDA_FILE=cuda_${CUDA_SHORT}.0_522.06_windows.exe 15 | ;; 16 | cu117) 17 | CUDA_SHORT=11.7 18 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.1/local_installers 19 | CUDA_FILE=cuda_${CUDA_SHORT}.1_516.94_windows.exe 20 | ;; 21 | cu116) 22 | CUDA_SHORT=11.3 23 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers 24 | CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe 25 | ;; 26 | cu115) 27 | CUDA_SHORT=11.3 28 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers 29 | CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe 30 | ;; 31 | cu113) 32 | CUDA_SHORT=11.3 33 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers 34 | CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe 35 | ;; 36 | *) 37 | echo "Unrecognized CUDA_VERSION=${1}" 38 | exit 1 39 | ;; 40 | esac 41 | 42 | curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}" 43 | echo "" 44 | echo "Installing from ${CUDA_FILE}..." 45 | PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow" 46 | echo "Done!" 47 | rm -f "${CUDA_FILE}" -------------------------------------------------------------------------------- /CAM_nerfacc/.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | branches: [master] 10 | 11 | jobs: 12 | deploy: 13 | runs-on: ubuntu-latest 14 | environment: production 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: '3.7' 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install build twine 25 | - name: Strip unsupported tags in README 26 | run: | 27 | sed -i '//,//d' README.md 28 | - name: Build and publish 29 | env: 30 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 31 | run: | 32 | BUILD_NO_CUDA=1 python -m build 33 | twine upload --username __token__ --password $PYPI_TOKEN dist/* -------------------------------------------------------------------------------- /CAM_nerfacc/.gitignore: -------------------------------------------------------------------------------- 1 | # Visual Studio Code configs. 2 | .vscode/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | # lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | .DS_Store 110 | 111 | # Direnv config. 112 | .envrc 113 | 114 | # line_profiler 115 | *.lprof 116 | 117 | # vscode 118 | .vsocde 119 | 120 | outputs/ 121 | data 122 | 123 | benchmarks/* 124 | !benchmarks/barf 125 | !benchmarks/kplanes 126 | !benchmarks/tensorf 127 | !benchmarks/tineuvox -------------------------------------------------------------------------------- /CAM_nerfacc/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "examples/pycolmap"] 2 | path = examples/pycolmap 3 | url = https://github.com/rmbrualla/pycolmap.git 4 | 5 | [submodule "benchmarks/tensorf"] 6 | path = benchmarks/tensorf 7 | url = https://github.com/liruilong940607/tensorf.git 8 | branch = nerfacc 9 | 10 | [submodule "benchmarks/barf"] 11 | path = benchmarks/barf 12 | url = https://github.com/liruilong940607/barf.git 13 | branch = nerfacc 14 | 15 | [submodule "benchmarks/tineuvox"] 16 | path = benchmarks/tineuvox 17 | url = https://github.com/liruilong940607/tineuvox.git 18 | branch = nerfacc 19 | 20 | [submodule "benchmarks/kplanes"] 21 | path = benchmarks/kplanes 22 | url = https://github.com/liruilong940607/kplanes.git 23 | branch = nerfacc 24 | -------------------------------------------------------------------------------- /CAM_nerfacc/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.3.0 4 | hooks: 5 | - id: end-of-file-fixer 6 | - id: trailing-whitespace 7 | - id: check-yaml 8 | - id: check-merge-conflict 9 | - id: requirements-txt-fixer 10 | - repo: https://github.com/psf/black 11 | rev: 22.10.0 12 | hooks: 13 | - id: black 14 | language_version: python3.8.12 15 | args: # arguments to configure black 16 | - --line-length=80 17 | 18 | - repo: https://github.com/pycqa/isort 19 | rev: 5.10.1 20 | hooks: 21 | - id: isort 22 | -------------------------------------------------------------------------------- /CAM_nerfacc/.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-20.04 5 | tools: 6 | python: "3.9" 7 | 8 | sphinx: 9 | fail_on_warning: true 10 | configuration: docs/source/conf.py 11 | 12 | python: 13 | install: 14 | # Equivalent to 'pip install .' 15 | - method: pip 16 | path: . 17 | # Equivalent to 'pip install -r docs/requirements.txt' 18 | - requirements: docs/requirements.txt -------------------------------------------------------------------------------- /CAM_nerfacc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | # Change Log 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/). 7 | 8 | ## [0.5.0] - 2023-04-04 9 | 10 | This is a major upgrade in which 90% of the code has been rewritten. In this version 11 | we achieves: 12 | 13 | ![Teaser](/docs/source/_static/images/teaser.jpg?raw=true) 14 | 15 | Links: 16 | - Documentation: https://www.nerfacc.com/en/v0.5.0/ 17 | - ArXiv Report: Coming Soon. 18 | 19 | Methodologies: 20 | - Upgrade Occupancy Grid to support multiple levels. 21 | - Support Proposal Network from Mip-NeRF 360. 22 | - Update examples on unbounded scenes to use Multi-level Occupancy Grid or Proposal Network. 23 | - Contraction for Occupancy Grid is no longer supported due to it's inefficiency for ray traversal. 24 | 25 | API Changes: 26 | - [Changed] `OccupancyGrid()` -> `OccGridEstimator()`. 27 | - [Added] Argument `levels=1` for multi-level support. 28 | - [Added] Function `self.sampling()` that does basically the same thing with the old `nerfacc.ray_marching`. 29 | - [Renamed] Function `self.every_n_step()` -> `self.update_every_n_steps()` 30 | - [Added] `PropNetEstimator()`. With functions `self.sampling()`, `self.update_every_n_steps()` 31 | and `self.compute_loss()`. 32 | - [Removed] `ray_marching()`. Ray marching is now implemented through calling `sampling()` of 33 | the `OccGridEstimator()` / `PropNetEstimator()`. 34 | - [Changed] `ray_aabb_intersect()` now supports multiple aabb, and supports new argument `near_plane`, `far_plane`, `miss_value`. 35 | - [Changed] `render_*_from_*()`. The input shape changes from `(all_samples, 1)` to `(all_samples)`. And the function will returns all intermediate results so it might be a tuple. 36 | - [Changed] `rendering()`. The input shape changes from `(all_samples, 1)` to `(all_samples)`, including the shape assumption for the `rgb_sigma_fn` and `rgb_alpha_fn`. Be aware of this shape change. 37 | - [Changed] `accumulate_along_rays()`. The shape of the `weights` in the inputs should be `(all_samples)` now. 38 | - [Removed] `unpack_info()`, `pack_data()`, `unpack_data()` are temporally removed due to in-compatibility 39 | with the new backend implementation. Will add them back later. 40 | - [Added] Some basic functions that support both batched tensor and flattened tensor: `inclusive_prod()`, `inclusive_sum()`, `exclusive_prod()`, `exclusive_sum()`, `importance_sampling()`, `searchsorted()`. 41 | 42 | Examples & Benchmarks: 43 | - More benchmarks and examples. See folder `examples/` and `benchmarks/`. 44 | 45 | ## [0.3.5] - 2023-02-23 46 | 47 | A stable version that achieves: 48 | - The vanilla Nerf model with 8-layer MLPs can be trained to better quality (+0.5 PNSR) in 1 hour rather than days as in the paper. 49 | - The Instant-NGP Nerf model can be trained to equal quality in 4.5 minutes, comparing to the official pure-CUDA implementation. 50 | - The D-Nerf model for dynamic objects can also be trained in 1 hour rather than 2 days as in the paper, and with better quality (+~2.5 PSNR). 51 | - Both bounded and unbounded scenes are supported. 52 | 53 | Links: 54 | - Documentation: https://www.nerfacc.com/en/v0.3.5/ 55 | - ArXiv Report: https://arxiv.org/abs/2210.04847v2/ 56 | 57 | Methodologies: 58 | - Single resolution `nerfacc.OccupancyGrid` for synthetic scenes. 59 | - Contraction methods `nerfacc.ContractionType` for unbounded scenes. 60 | -------------------------------------------------------------------------------- /CAM_nerfacc/LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Ruilong Li 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | THIS SOFTWARE AND/OR DATA WAS DEPOSITED IN THE BAIR OPEN RESEARCH COMMONS 24 | REPOSITORY ON 05/08/2023. 25 | -------------------------------------------------------------------------------- /CAM_nerfacc/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include nerfacc/cuda/csrc/include/* 2 | include nerfacc/cuda/csrc/* 3 | 4 | include nerfacc/_cuda/csrc/include/* 5 | include nerfacc/_cuda/csrc/* 6 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_nerfacc/examples/datasets/__init__.py -------------------------------------------------------------------------------- /CAM_nerfacc/examples/datasets/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | """ 4 | 5 | import collections 6 | 7 | Rays = collections.namedtuple("Rays", ("origins", "viewdirs")) 8 | 9 | 10 | def namedtuple_map(fn, tup): 11 | """Apply `fn` to each element of `tup` and cast to `tup`'s namedtuple.""" 12 | return type(tup)(*(None if x is None else fn(x) for x in tup)) 13 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.sw* 3 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 True Price, UNC Chapel Hill 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/README.md: -------------------------------------------------------------------------------- 1 | # pycolmap 2 | Python interface for COLMAP reconstructions, plus some convenient scripts for loading/modifying/converting reconstructions. 3 | 4 | This code does not, however, run reconstruction -- it only provides a convenient interface for handling COLMAP's output. 5 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/pycolmap/__init__.py: -------------------------------------------------------------------------------- 1 | from camera import Camera 2 | from database import COLMAPDatabase 3 | from image import Image 4 | from scene_manager import SceneManager 5 | from rotation import Quaternion, DualQuaternion 6 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/pycolmap/image.py: -------------------------------------------------------------------------------- 1 | # Author: True Price 2 | 3 | import numpy as np 4 | 5 | #------------------------------------------------------------------------------- 6 | # 7 | # Image 8 | # 9 | #------------------------------------------------------------------------------- 10 | 11 | class Image: 12 | def __init__(self, name_, camera_id_, q_, tvec_): 13 | self.name = name_ 14 | self.camera_id = camera_id_ 15 | self.q = q_ 16 | self.tvec = tvec_ 17 | 18 | self.points2D = np.empty((0, 2), dtype=np.float64) 19 | self.point3D_ids = np.empty((0,), dtype=np.uint64) 20 | 21 | #--------------------------------------------------------------------------- 22 | 23 | def R(self): 24 | return self.q.ToR() 25 | 26 | #--------------------------------------------------------------------------- 27 | 28 | def C(self): 29 | return -self.R().T.dot(self.tvec) 30 | 31 | #--------------------------------------------------------------------------- 32 | 33 | @property 34 | def t(self): 35 | return self.tvec 36 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/tools/colmap_to_nvm.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import sys 3 | sys.path.append("..") 4 | 5 | import numpy as np 6 | 7 | from pycolmap import Quaternion, SceneManager 8 | 9 | 10 | #------------------------------------------------------------------------------- 11 | 12 | def main(args): 13 | scene_manager = SceneManager(args.input_folder) 14 | scene_manager.load() 15 | 16 | with open(args.output_file, "w") as fid: 17 | fid.write("NVM_V3\n \n{:d}\n".format(len(scene_manager.images))) 18 | 19 | image_fmt_str = " {:.3f} " + 7 * "{:.7f} " 20 | for image_id, image in scene_manager.images.iteritems(): 21 | camera = scene_manager.cameras[image.camera_id] 22 | f = 0.5 * (camera.fx + camera.fy) 23 | fid.write(args.image_name_prefix + image.name) 24 | fid.write(image_fmt_str.format( 25 | *((f,) + tuple(image.q.q) + tuple(image.C())))) 26 | if camera.distortion_func is None: 27 | fid.write("0 0\n") 28 | else: 29 | fid.write("{:.7f} 0\n".format(-camera.k1)) 30 | 31 | image_id_to_idx = dict( 32 | (image_id, i) for i, image_id in enumerate(scene_manager.images)) 33 | 34 | fid.write("{:d}\n".format(len(scene_manager.points3D))) 35 | for i, point3D_id in enumerate(scene_manager.point3D_ids): 36 | fid.write( 37 | "{:.7f} {:.7f} {:.7f} ".format(*scene_manager.points3D[i])) 38 | fid.write( 39 | "{:d} {:d} {:d} ".format(*scene_manager.point3D_colors[i])) 40 | keypoints = [ 41 | (image_id_to_idx[image_id], kp_idx) + 42 | tuple(scene_manager.images[image_id].points2D[kp_idx]) 43 | for image_id, kp_idx in 44 | scene_manager.point3D_id_to_images[point3D_id]] 45 | fid.write("{:d}".format(len(keypoints))) 46 | fid.write( 47 | (len(keypoints) * " {:d} {:d} {:.3f} {:.3f}" + "\n").format( 48 | *itertools.chain(*keypoints))) 49 | 50 | 51 | #------------------------------------------------------------------------------- 52 | 53 | if __name__ == "__main__": 54 | import argparse 55 | 56 | parser = argparse.ArgumentParser( 57 | description="Save a COLMAP reconstruction in the NVM format " 58 | "(http://ccwu.me/vsfm/doc.html#nvm).", 59 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 60 | 61 | parser.add_argument("input_folder") 62 | parser.add_argument("output_file") 63 | 64 | parser.add_argument("--image_name_prefix", type=str, default="", 65 | help="prefix image names with this string (e.g., 'images/')") 66 | 67 | args = parser.parse_args() 68 | 69 | main(args) 70 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/tools/delete_images.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | 4 | import numpy as np 5 | 6 | from pycolmap import DualQuaternion, Image, SceneManager 7 | 8 | 9 | #------------------------------------------------------------------------------- 10 | 11 | def main(args): 12 | scene_manager = SceneManager(args.input_folder) 13 | scene_manager.load() 14 | 15 | image_ids = map(scene_manager.get_image_from_name, 16 | iter(lambda: sys.stdin.readline().strip(), "")) 17 | scene_manager.delete_images(image_ids) 18 | 19 | scene_manager.save(args.output_folder) 20 | 21 | 22 | #------------------------------------------------------------------------------- 23 | 24 | if __name__ == "__main__": 25 | import argparse 26 | 27 | parser = argparse.ArgumentParser( 28 | description="Deletes images (filenames read from stdin) from a model.", 29 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 30 | 31 | parser.add_argument("input_folder") 32 | parser.add_argument("output_folder") 33 | 34 | args = parser.parse_args() 35 | 36 | main(args) 37 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/tools/save_cameras_as_ply.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | 4 | import numpy as np 5 | import os 6 | 7 | from pycolmap import SceneManager 8 | 9 | 10 | #------------------------------------------------------------------------------- 11 | 12 | # Saves the cameras as a mesh 13 | # 14 | # inputs: 15 | # - ply_file: output file 16 | # - images: ordered array of pycolmap Image objects 17 | # - color: color string for the camera 18 | # - scale: amount to shrink/grow the camera model 19 | def save_camera_ply(ply_file, images, scale): 20 | points3D = scale * np.array(( 21 | (0., 0., 0.), 22 | (-1., -1., 1.), 23 | (-1., 1., 1.), 24 | (1., -1., 1.), 25 | (1., 1., 1.))) 26 | 27 | faces = np.array(((0, 2, 1), 28 | (0, 4, 2), 29 | (0, 3, 4), 30 | (0, 1, 3), 31 | (1, 2, 4), 32 | (1, 4, 3))) 33 | 34 | r = np.linspace(0, 255, len(images), dtype=np.uint8) 35 | g = 255 - r 36 | b = r - np.linspace(0, 128, len(images), dtype=np.uint8) 37 | color = np.column_stack((r, g, b)) 38 | 39 | with open(ply_file, "w") as fid: 40 | print>>fid, "ply" 41 | print>>fid, "format ascii 1.0" 42 | print>>fid, "element vertex", len(points3D) * len(images) 43 | print>>fid, "property float x" 44 | print>>fid, "property float y" 45 | print>>fid, "property float z" 46 | print>>fid, "property uchar red" 47 | print>>fid, "property uchar green" 48 | print>>fid, "property uchar blue" 49 | print>>fid, "element face", len(faces) * len(images) 50 | print>>fid, "property list uchar int vertex_index" 51 | print>>fid, "end_header" 52 | 53 | for image, c in zip(images, color): 54 | for p3D in (points3D.dot(image.R()) + image.C()): 55 | print>>fid, p3D[0], p3D[1], p3D[2], c[0], c[1], c[2] 56 | 57 | for i in xrange(len(images)): 58 | for f in (faces + len(points3D) * i): 59 | print>>fid, "3 {} {} {}".format(*f) 60 | 61 | 62 | #------------------------------------------------------------------------------- 63 | 64 | def main(args): 65 | scene_manager = SceneManager(args.input_folder) 66 | scene_manager.load_images() 67 | 68 | images = sorted(scene_manager.images.itervalues(), 69 | key=lambda image: image.name) 70 | 71 | save_camera_ply(args.output_file, images, args.scale) 72 | 73 | 74 | #------------------------------------------------------------------------------- 75 | 76 | if __name__ == "__main__": 77 | import argparse 78 | 79 | parser = argparse.ArgumentParser( 80 | description="Saves camera positions to a PLY for easy viewing outside " 81 | "of COLMAP. Currently, camera FoV is not reflected in the output.", 82 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 83 | 84 | parser.add_argument("input_folder") 85 | parser.add_argument("output_file") 86 | 87 | parser.add_argument("--scale", type=float, default=1., 88 | help="Scaling factor for the camera mesh.") 89 | 90 | args = parser.parse_args() 91 | 92 | main(args) 93 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/tools/transform_model.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | 4 | import numpy as np 5 | 6 | from pycolmap import Quaternion, SceneManager 7 | 8 | 9 | #------------------------------------------------------------------------------- 10 | 11 | def main(args): 12 | scene_manager = SceneManager(args.input_folder) 13 | scene_manager.load() 14 | 15 | # expect each line of input corresponds to one row 16 | P = np.array([ 17 | map(float, sys.stdin.readline().strip().split()) for _ in xrange(3)]) 18 | 19 | scene_manager.points3D[:] = scene_manager.points3D.dot(P[:,:3].T) + P[:,3] 20 | 21 | # get rotation without any global scaling (assuming isotropic scaling) 22 | scale = np.cbrt(np.linalg.det(P[:,:3])) 23 | q_old_from_new = ~Quaternion.FromR(P[:,:3] / scale) 24 | 25 | for image in scene_manager.images.itervalues(): 26 | image.q *= q_old_from_new 27 | image.tvec = scale * image.tvec - image.R().dot(P[:,3]) 28 | 29 | scene_manager.save(args.output_folder) 30 | 31 | 32 | #------------------------------------------------------------------------------- 33 | 34 | if __name__ == "__main__": 35 | import argparse 36 | 37 | parser = argparse.ArgumentParser( 38 | description="Apply a 3x4 transformation matrix to a COLMAP model and " 39 | "save the result as a new model. Row-major input can be piped in from " 40 | "a file or entered via the command line.", 41 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 42 | 43 | parser.add_argument("input_folder") 44 | parser.add_argument("output_folder") 45 | 46 | args = parser.parse_args() 47 | 48 | main(args) 49 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/tools/write_camera_track_to_bundler.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | 4 | import numpy as np 5 | 6 | from pycolmap import SceneManager 7 | 8 | 9 | #------------------------------------------------------------------------------- 10 | 11 | def main(args): 12 | scene_manager = SceneManager(args.input_folder) 13 | scene_manager.load_cameras() 14 | scene_manager.load_images() 15 | 16 | if args.sort: 17 | images = sorted( 18 | scene_manager.images.itervalues(), key=lambda im: im.name) 19 | else: 20 | images = scene_manager.images.values() 21 | 22 | fid = open(args.output_file, "w") 23 | fid_filenames = open(args.output_file + ".list.txt", "w") 24 | 25 | print>>fid, "# Bundle file v0.3" 26 | print>>fid, len(images), 0 27 | 28 | for image in images: 29 | print>>fid_filenames, image.name 30 | camera = scene_manager.cameras[image.camera_id] 31 | print>>fid, 0.5 * (camera.fx + camera.fy), 0, 0 32 | R, t = image.R(), image.t 33 | print>>fid, R[0, 0], R[0, 1], R[0, 2] 34 | print>>fid, -R[1, 0], -R[1, 1], -R[1, 2] 35 | print>>fid, -R[2, 0], -R[2, 1], -R[2, 2] 36 | print>>fid, t[0], -t[1], -t[2] 37 | 38 | fid.close() 39 | fid_filenames.close() 40 | 41 | 42 | #------------------------------------------------------------------------------- 43 | 44 | if __name__ == "__main__": 45 | import argparse 46 | 47 | parser = argparse.ArgumentParser( 48 | description="Saves the camera positions in the Bundler format. Note " 49 | "that 3D points are not saved.", 50 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 51 | 52 | parser.add_argument("input_folder") 53 | parser.add_argument("output_file") 54 | 55 | parser.add_argument("--sort", default=False, action="store_true", 56 | help="sort the images by their filename") 57 | 58 | args = parser.parse_args() 59 | 60 | main(args) 61 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/pycolmap/tools/write_depthmap_to_ply.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | 4 | import imageio 5 | import numpy as np 6 | import os 7 | 8 | from plyfile import PlyData, PlyElement 9 | from pycolmap import SceneManager 10 | from scipy.ndimage.interpolation import zoom 11 | 12 | 13 | #------------------------------------------------------------------------------- 14 | 15 | def main(args): 16 | suffix = ".photometric.bin" if args.photometric else ".geometric.bin" 17 | 18 | image_file = os.path.join(args.dense_folder, "images", args.image_filename) 19 | depth_file = os.path.join( 20 | args.dense_folder, args.stereo_folder, "depth_maps", 21 | args.image_filename + suffix) 22 | if args.save_normals: 23 | normals_file = os.path.join( 24 | args.dense_folder, args.stereo_folder, "normal_maps", 25 | args.image_filename + suffix) 26 | 27 | # load camera intrinsics from the COLMAP reconstruction 28 | scene_manager = SceneManager(os.path.join(args.dense_folder, "sparse")) 29 | scene_manager.load_cameras() 30 | scene_manager.load_images() 31 | 32 | image_id, image = scene_manager.get_image_from_name(args.image_filename) 33 | camera = scene_manager.cameras[image.camera_id] 34 | rotation_camera_from_world = image.R() 35 | camera_center = image.C() 36 | 37 | # load image, depth map, and normal map 38 | image = imageio.imread(image_file) 39 | 40 | with open(depth_file, "rb") as fid: 41 | w = int("".join(iter(lambda: fid.read(1), "&"))) 42 | h = int("".join(iter(lambda: fid.read(1), "&"))) 43 | c = int("".join(iter(lambda: fid.read(1), "&"))) 44 | depth_map = np.fromfile(fid, np.float32).reshape(h, w) 45 | if (h, w) != image.shape[:2]: 46 | depth_map = zoom( 47 | depth_map, 48 | (float(image.shape[0]) / h, float(image.shape[1]) / w), 49 | order=0) 50 | 51 | if args.save_normals: 52 | with open(normals_file, "rb") as fid: 53 | w = int("".join(iter(lambda: fid.read(1), "&"))) 54 | h = int("".join(iter(lambda: fid.read(1), "&"))) 55 | c = int("".join(iter(lambda: fid.read(1), "&"))) 56 | normals = np.fromfile( 57 | fid, np.float32).reshape(c, h, w).transpose([1, 2, 0]) 58 | if (h, w) != image.shape[:2]: 59 | normals = zoom( 60 | normals, 61 | (float(image.shape[0]) / h, float(image.shape[1]) / w, 1.), 62 | order=0) 63 | 64 | if args.min_depth is not None: 65 | depth_map[depth_map < args.min_depth] = 0. 66 | if args.max_depth is not None: 67 | depth_map[depth_map > args.max_depth] = 0. 68 | 69 | # create 3D points 70 | #depth_map = np.minimum(depth_map, 100.) 71 | points3D = np.dstack(camera.get_image_grid() + [depth_map]) 72 | points3D[:,:,:2] *= depth_map[:,:,np.newaxis] 73 | 74 | # save 75 | points3D = points3D.astype(np.float32).reshape(-1, 3) 76 | if args.save_normals: 77 | normals = normals.astype(np.float32).reshape(-1, 3) 78 | image = image.reshape(-1, 3) 79 | if image.dtype != np.uint8: 80 | if image.max() <= 1: 81 | image = (image * 255.).astype(np.uint8) 82 | else: 83 | image = image.astype(np.uint8) 84 | 85 | if args.world_space: 86 | points3D = points3D.dot(rotation_camera_from_world) + camera_center 87 | if args.save_normals: 88 | normals = normals.dot(rotation_camera_from_world) 89 | 90 | if args.save_normals: 91 | vertices = np.rec.fromarrays( 92 | tuple(points3D.T) + tuple(normals.T) + tuple(image.T), 93 | names="x,y,z,nx,ny,nz,red,green,blue") 94 | else: 95 | vertices = np.rec.fromarrays( 96 | tuple(points3D.T) + tuple(image.T), names="x,y,z,red,green,blue") 97 | vertices = PlyElement.describe(vertices, "vertex") 98 | PlyData([vertices]).write(args.output_filename) 99 | 100 | 101 | #------------------------------------------------------------------------------- 102 | 103 | if __name__ == "__main__": 104 | import argparse 105 | 106 | parser = argparse.ArgumentParser( 107 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 108 | 109 | parser.add_argument("dense_folder", type=str) 110 | parser.add_argument("image_filename", type=str) 111 | parser.add_argument("output_filename", type=str) 112 | 113 | parser.add_argument( 114 | "--photometric", default=False, action="store_true", 115 | help="use photometric depthmap instead of geometric") 116 | 117 | parser.add_argument( 118 | "--world_space", default=False, action="store_true", 119 | help="apply the camera->world extrinsic transformation to the result") 120 | 121 | parser.add_argument( 122 | "--save_normals", default=False, action="store_true", 123 | help="load the estimated normal map and save as part of the PLY") 124 | 125 | parser.add_argument( 126 | "--stereo_folder", type=str, default="stereo", 127 | help="folder in the dense workspace containing depth and normal maps") 128 | 129 | parser.add_argument( 130 | "--min_depth", type=float, default=None, 131 | help="set pixels with depth less than this value to zero depth") 132 | 133 | parser.add_argument( 134 | "--max_depth", type=float, default=None, 135 | help="set pixels with depth greater than this value to zero depth") 136 | 137 | args = parser.parse_args() 138 | 139 | main(args) 140 | -------------------------------------------------------------------------------- /CAM_nerfacc/examples/radiance_fields/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_nerfacc/examples/radiance_fields/__init__.py -------------------------------------------------------------------------------- /CAM_nerfacc/examples/requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch 2 | opencv-python 3 | imageio 4 | numpy 5 | tqdm 6 | scipy 7 | lpips -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | """ 4 | from .data_specs import RayIntervals, RaySamples 5 | from .estimators.occ_grid import OccGridEstimator 6 | from .estimators.prop_net import PropNetEstimator 7 | from .grid import ray_aabb_intersect, traverse_grids 8 | from .pack import pack_info 9 | from .pdf import importance_sampling, searchsorted 10 | from .scan import exclusive_prod, exclusive_sum, inclusive_prod, inclusive_sum 11 | from .version import __version__ 12 | from .volrend import ( 13 | accumulate_along_rays, 14 | render_transmittance_from_alpha, 15 | render_transmittance_from_density, 16 | render_visibility_from_alpha, 17 | render_visibility_from_density, 18 | render_weight_from_alpha, 19 | render_weight_from_density, 20 | rendering, 21 | ) 22 | 23 | __all__ = [ 24 | "__version__", 25 | "inclusive_prod", 26 | "exclusive_prod", 27 | "inclusive_sum", 28 | "exclusive_sum", 29 | "pack_info", 30 | "render_visibility_from_alpha", 31 | "render_visibility_from_density", 32 | "render_weight_from_alpha", 33 | "render_weight_from_density", 34 | "render_transmittance_from_alpha", 35 | "render_transmittance_from_density", 36 | "accumulate_along_rays", 37 | "rendering", 38 | "importance_sampling", 39 | "searchsorted", 40 | "RayIntervals", 41 | "RaySamples", 42 | "ray_aabb_intersect", 43 | "traverse_grids", 44 | "OccGridEstimator", 45 | "PropNetEstimator", 46 | ] 47 | -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cameras2.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | 4 | Seems like both colmap and nerfstudio are based on OpenCV's camera model. 5 | 6 | References: 7 | - nerfstudio: https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/cameras/cameras.py 8 | - opencv: 9 | - https://docs.opencv.org/3.4/da/d54/group__imgproc__transform.html#ga69f2545a8b62a6b0fc2ee060dc30559d 10 | - https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html 11 | - https://docs.opencv.org/4.x/db/d58/group__calib3d__fisheye.html 12 | - https://github.com/opencv/opencv/blob/master/modules/calib3d/src/fisheye.cpp#L321 13 | - https://github.com/opencv/opencv/blob/17234f82d025e3bbfbf611089637e5aa2038e7b8/modules/calib3d/src/distortion_model.hpp 14 | - https://github.com/opencv/opencv/blob/8d0fbc6a1e9f20c822921e8076551a01e58cd632/modules/calib3d/src/undistort.dispatch.cpp#L578 15 | - colmap: https://github.com/colmap/colmap/blob/dev/src/base/camera_models.h 16 | - calcam: https://euratom-software.github.io/calcam/html/intro_theory.html 17 | - blender: 18 | - https://docs.blender.org/manual/en/latest/render/cycles/object_settings/cameras.html#fisheye-lens-polynomial 19 | - https://github.com/blender/blender/blob/03cc3b94c94c38767802bccac4e9384ab704065a/intern/cycles/kernel/kernel_projection.h 20 | - lensfun: https://lensfun.github.io/manual/v0.3.2/annotated.html 21 | 22 | - OpenCV and Blender has different fisheye camera models 23 | - https://stackoverflow.com/questions/73270140/pipeline-for-fisheye-distortion-and-undistortion-with-blender-and-opencv 24 | """ 25 | from typing import Literal, Optional, Tuple 26 | 27 | import torch 28 | import torch.nn.functional as F 29 | from torch import Tensor 30 | 31 | from . import cuda as _C 32 | 33 | 34 | def ray_directions_from_uvs( 35 | uvs: Tensor, # [..., 2] 36 | Ks: Tensor, # [..., 3, 3] 37 | params: Optional[Tensor] = None, # [..., M] 38 | ) -> Tensor: 39 | """Create ray directions from uvs and camera parameters in OpenCV format. 40 | 41 | Args: 42 | uvs: UV coordinates on image plane. (In pixel unit) 43 | Ks: Camera intrinsics. 44 | params: Camera distortion parameters. See `opencv.undistortPoints` for details. 45 | 46 | Returns: 47 | Normalized ray directions in camera space. 48 | """ 49 | u, v = torch.unbind(uvs + 0.5, dim=-1) 50 | fx, fy = Ks[..., 0, 0], Ks[..., 1, 1] 51 | cx, cy = Ks[..., 0, 2], Ks[..., 1, 2] 52 | 53 | # undo intrinsics 54 | xys = torch.stack([(u - cx) / fx, (v - cy) / fy], dim=-1) # [..., 2] 55 | 56 | # undo lens distortion 57 | if params is not None: 58 | M = params.shape[-1] 59 | 60 | if M == 14: # undo tilt projection 61 | R, R_inv = opencv_tilt_projection_matrix(params[..., -2:]) 62 | xys_homo = F.pad(xys, (0, 1), value=1.0) # [..., 3] 63 | xys_homo = torch.einsum( 64 | "...ij,...j->...i", R_inv, xys_homo 65 | ) # [..., 3] 66 | xys = xys_homo[..., :2] 67 | homo = xys_homo[..., 2:] 68 | xys /= torch.where(homo != 0.0, homo, torch.ones_like(homo)) 69 | 70 | xys = opencv_lens_undistortion(xys, params) # [..., 2] 71 | 72 | # normalized homogeneous coordinates 73 | dirs = F.pad(xys, (0, 1), value=1.0) # [..., 3] 74 | dirs = F.normalize(dirs, dim=-1) # [..., 3] 75 | return dirs 76 | 77 | 78 | def opencv_lens_undistortion( 79 | uv: Tensor, params: Tensor, eps: float = 1e-6, iters: int = 10 80 | ) -> Tensor: 81 | """Undistort the opencv distortion of {k1, k2, k3, k4, p1, p2}. 82 | 83 | Note: 84 | This function is not differentiable to any inputs. 85 | 86 | Args: 87 | uv: (..., 2) UV coordinates. 88 | params: (..., 6) or (6) OpenCV distortion parameters. 89 | 90 | Returns: 91 | (..., 2) undistorted UV coordinates. 92 | """ 93 | assert uv.shape[-1] == 2 94 | assert params.shape[-1] == 6 95 | batch_shape = uv.shape[:-1] 96 | params = torch.broadcast_to(params, batch_shape + (6,)) 97 | 98 | return _C.opencv_lens_undistortion( 99 | uv.contiguous(), params.contiguous(), eps, iters 100 | ) 101 | 102 | 103 | def opencv_tilt_projection_matrix(tau: Tensor) -> Tensor: 104 | """Create a tilt projection matrix. 105 | 106 | Reference: 107 | https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html 108 | 109 | Args: 110 | tau: (..., 2) tilt angles. 111 | 112 | Returns: 113 | (..., 3, 3) tilt projection matrix. 114 | """ 115 | 116 | cosx, cosy = torch.unbind(torch.cos(tau), -1) 117 | sinx, siny = torch.unbind(torch.sin(tau), -1) 118 | one = torch.ones_like(tau) 119 | zero = torch.zeros_like(tau) 120 | 121 | Rx = torch.stack( 122 | [one, zero, zero, zero, cosx, sinx, zero, -sinx, cosx], -1 123 | ).reshape(*tau.shape[:-1], 3, 3) 124 | Ry = torch.stack( 125 | [cosy, zero, -siny, zero, one, zero, siny, zero, cosy], -1 126 | ).reshape(*tau.shape[:-1], 3, 3) 127 | Rxy = torch.matmul(Ry, Rx) 128 | Rz = torch.stack( 129 | [ 130 | Rxy[..., 2, 2], 131 | zero, 132 | -Rxy[..., 0, 2], 133 | zero, 134 | Rxy[..., 2, 2], 135 | -Rxy[..., 1, 2], 136 | zero, 137 | zero, 138 | one, 139 | ], 140 | -1, 141 | ).reshape(*tau.shape[:-1], 3, 3) 142 | R = torch.matmul(Rz, Rxy) 143 | 144 | inv = 1.0 / Rxy[..., 2, 2] 145 | Rz_inv = torch.stack( 146 | [ 147 | inv, 148 | zero, 149 | inv * Rxy[..., 0, 2], 150 | zero, 151 | inv, 152 | inv * Rxy[..., 1, 2], 153 | zero, 154 | zero, 155 | one, 156 | ], 157 | -1, 158 | ).reshape(*tau.shape[:-1], 3, 3) 159 | R_inv = torch.matmul(Rxy.transpose(-1, -2), Rz_inv) 160 | return R, R_inv 161 | -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | """ 4 | 5 | from typing import Any, Callable 6 | 7 | 8 | def _make_lazy_cuda_func(name: str) -> Callable: 9 | def call_cuda(*args, **kwargs): 10 | # pylint: disable=import-outside-toplevel 11 | from ._backend import _C 12 | 13 | return getattr(_C, name)(*args, **kwargs) 14 | 15 | return call_cuda 16 | 17 | 18 | is_cub_available = _make_lazy_cuda_func("is_cub_available") 19 | 20 | # data specs 21 | MultiScaleGridSpec = _make_lazy_cuda_func("MultiScaleGridSpec") 22 | RaysSpec = _make_lazy_cuda_func("RaysSpec") 23 | RaySegmentsSpec = _make_lazy_cuda_func("RaySegmentsSpec") 24 | 25 | # grid 26 | ray_aabb_intersect = _make_lazy_cuda_func("ray_aabb_intersect") 27 | traverse_grids = _make_lazy_cuda_func("traverse_grids") 28 | 29 | # scan 30 | exclusive_sum_by_key = _make_lazy_cuda_func("exclusive_sum_by_key") 31 | inclusive_sum = _make_lazy_cuda_func("inclusive_sum") 32 | exclusive_sum = _make_lazy_cuda_func("exclusive_sum") 33 | inclusive_prod_forward = _make_lazy_cuda_func("inclusive_prod_forward") 34 | inclusive_prod_backward = _make_lazy_cuda_func("inclusive_prod_backward") 35 | exclusive_prod_forward = _make_lazy_cuda_func("exclusive_prod_forward") 36 | exclusive_prod_backward = _make_lazy_cuda_func("exclusive_prod_backward") 37 | 38 | # pdf 39 | importance_sampling = _make_lazy_cuda_func("importance_sampling") 40 | searchsorted = _make_lazy_cuda_func("searchsorted") 41 | 42 | # camera 43 | opencv_lens_undistortion = _make_lazy_cuda_func("opencv_lens_undistortion") 44 | opencv_lens_undistortion_fisheye = _make_lazy_cuda_func( 45 | "opencv_lens_undistortion_fisheye" 46 | ) 47 | -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/_backend.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | """ 4 | 5 | import glob 6 | import json 7 | import os 8 | import shutil 9 | from subprocess import DEVNULL, call 10 | 11 | from rich.console import Console 12 | from torch.utils.cpp_extension import _get_build_directory, load 13 | 14 | PATH = os.path.dirname(os.path.abspath(__file__)) 15 | 16 | 17 | def cuda_toolkit_available(): 18 | """Check if the nvcc is avaiable on the machine.""" 19 | try: 20 | call(["nvcc"], stdout=DEVNULL, stderr=DEVNULL) 21 | return True 22 | except FileNotFoundError: 23 | return False 24 | 25 | 26 | def cuda_toolkit_version(): 27 | """Get the cuda toolkit version.""" 28 | cuda_home = os.path.join(os.path.dirname(shutil.which("nvcc")), "..") 29 | if os.path.exists(os.path.join(cuda_home, "version.txt")): 30 | with open(os.path.join(cuda_home, "version.txt")) as f: 31 | cuda_version = f.read().strip().split()[-1] 32 | elif os.path.exists(os.path.join(cuda_home, "version.json")): 33 | with open(os.path.join(cuda_home, "version.json")) as f: 34 | cuda_version = json.load(f)["cuda"]["version"] 35 | else: 36 | raise RuntimeError("Cannot find the cuda version.") 37 | return cuda_version 38 | 39 | 40 | name = "nerfacc_cuda" 41 | build_dir = _get_build_directory(name, verbose=False) 42 | extra_include_paths = [] 43 | extra_cflags = ["-O3"] 44 | extra_cuda_cflags = ["-O3"] 45 | 46 | _C = None 47 | sources = list(glob.glob(os.path.join(PATH, "csrc/*.cu"))) + list( 48 | glob.glob(os.path.join(PATH, "csrc/*.cpp")) 49 | ) 50 | 51 | try: 52 | # try to import the compiled module (via setup.py) 53 | from nerfacc import csrc as _C 54 | except ImportError: 55 | # if failed, try with JIT compilation 56 | if cuda_toolkit_available(): 57 | if os.listdir(build_dir) != []: 58 | # If the build exists, we assume the extension has been built 59 | # and we can load it. 60 | 61 | _C = load( 62 | name=name, 63 | sources=sources, 64 | extra_cflags=extra_cflags, 65 | extra_cuda_cflags=extra_cuda_cflags, 66 | extra_include_paths=extra_include_paths, 67 | ) 68 | else: 69 | # Build from scratch. Remove the build directory just to be safe: pytorch jit might stuck 70 | # if the build directory exists. 71 | shutil.rmtree(build_dir) 72 | with Console().status( 73 | "[bold yellow]NerfAcc: Setting up CUDA (This may take a few minutes the first time)", 74 | spinner="bouncingBall", 75 | ): 76 | _C = load( 77 | name=name, 78 | sources=sources, 79 | extra_cflags=extra_cflags, 80 | extra_cuda_cflags=extra_cuda_cflags, 81 | extra_include_paths=extra_include_paths, 82 | ) 83 | else: 84 | Console().print( 85 | "[yellow]NerfAcc: No CUDA toolkit found. NerfAcc will be disabled.[/yellow]" 86 | ) 87 | 88 | 89 | __all__ = ["_C"] 90 | -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/csrc/include/data_spec.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "utils_cuda.cuh" 5 | 6 | struct MultiScaleGridSpec { 7 | torch::Tensor data; // [levels, resx, resy, resz] 8 | torch::Tensor occupied; // [levels, resx, resy, resz] 9 | torch::Tensor base_aabb; // [6,] 10 | 11 | inline void check() { 12 | CHECK_INPUT(data); 13 | CHECK_INPUT(occupied); 14 | CHECK_INPUT(base_aabb); 15 | 16 | TORCH_CHECK(data.ndimension() == 4); 17 | TORCH_CHECK(occupied.ndimension() == 4); 18 | TORCH_CHECK(base_aabb.ndimension() == 1); 19 | 20 | TORCH_CHECK(data.numel() == occupied.numel()); 21 | TORCH_CHECK(base_aabb.numel() == 6); 22 | } 23 | }; 24 | 25 | struct RaysSpec { 26 | torch::Tensor origins; // [n_rays, 3] 27 | torch::Tensor dirs; // [n_rays, 3] 28 | 29 | inline void check() { 30 | CHECK_INPUT(origins); 31 | CHECK_INPUT(dirs); 32 | 33 | TORCH_CHECK(origins.ndimension() == 2); 34 | TORCH_CHECK(dirs.ndimension() == 2); 35 | 36 | TORCH_CHECK(origins.numel() == dirs.numel()); 37 | 38 | TORCH_CHECK(origins.size(1) == 3); 39 | TORCH_CHECK(dirs.size(1) == 3); 40 | } 41 | }; 42 | 43 | 44 | struct RaySegmentsSpec { 45 | torch::Tensor vals; // [n_edges] or [n_rays, n_edges_per_ray] 46 | // for flattened tensor 47 | torch::Tensor chunk_starts; // [n_rays] 48 | torch::Tensor chunk_cnts; // [n_rays] 49 | torch::Tensor ray_indices; // [n_edges] 50 | torch::Tensor is_left; // [n_edges] have n_bins true values 51 | torch::Tensor is_right; // [n_edges] have n_bins true values 52 | 53 | inline void check() { 54 | CHECK_INPUT(vals); 55 | TORCH_CHECK(vals.defined()); 56 | 57 | // batched tensor [..., n_edges_per_ray] 58 | if (vals.ndimension() > 1) return; 59 | 60 | // flattend tensor [n_edges] 61 | CHECK_INPUT(chunk_starts); 62 | CHECK_INPUT(chunk_cnts); 63 | TORCH_CHECK(chunk_starts.defined()); 64 | TORCH_CHECK(chunk_cnts.defined()); 65 | TORCH_CHECK(chunk_starts.ndimension() == 1); 66 | TORCH_CHECK(chunk_cnts.ndimension() == 1); 67 | TORCH_CHECK(chunk_starts.numel() == chunk_cnts.numel()); 68 | if (ray_indices.defined()) { 69 | CHECK_INPUT(ray_indices); 70 | TORCH_CHECK(ray_indices.ndimension() == 1); 71 | TORCH_CHECK(vals.numel() == ray_indices.numel()); 72 | } 73 | if (is_left.defined()) { 74 | CHECK_INPUT(is_left); 75 | TORCH_CHECK(is_left.ndimension() == 1); 76 | TORCH_CHECK(vals.numel() == is_left.numel()); 77 | } 78 | if (is_right.defined()) { 79 | CHECK_INPUT(is_right); 80 | TORCH_CHECK(is_right.ndimension() == 1); 81 | TORCH_CHECK(vals.numel() == is_right.numel()); 82 | } 83 | } 84 | 85 | inline void memalloc_cnts(int32_t n_rays, at::TensorOptions options, bool zero_init = true) { 86 | TORCH_CHECK(!chunk_cnts.defined()); 87 | if (zero_init) { 88 | chunk_cnts = torch::zeros({n_rays}, options.dtype(torch::kLong)); 89 | } else { 90 | chunk_cnts = torch::empty({n_rays}, options.dtype(torch::kLong)); 91 | } 92 | } 93 | 94 | inline int64_t memalloc_data(bool alloc_masks = true, bool zero_init = true) { 95 | TORCH_CHECK(chunk_cnts.defined()); 96 | TORCH_CHECK(!chunk_starts.defined()); 97 | TORCH_CHECK(!vals.defined()); 98 | 99 | torch::Tensor cumsum = torch::cumsum(chunk_cnts, 0, chunk_cnts.scalar_type()); 100 | int64_t n_edges = cumsum[-1].item(); 101 | 102 | chunk_starts = cumsum - chunk_cnts; 103 | if (zero_init) { 104 | vals = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kFloat32)); 105 | ray_indices = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kLong)); 106 | if (alloc_masks) { 107 | is_left = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kBool)); 108 | is_right = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kBool)); 109 | } 110 | } else { 111 | vals = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kFloat32)); 112 | ray_indices = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kLong)); 113 | if (alloc_masks) { 114 | is_left = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kBool)); 115 | is_right = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kBool)); 116 | } 117 | } 118 | return 1; 119 | } 120 | }; -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/csrc/include/data_spec_packed.cuh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "data_spec.hpp" 6 | 7 | namespace { 8 | namespace device { 9 | 10 | struct PackedRaySegmentsSpec { 11 | PackedRaySegmentsSpec(RaySegmentsSpec& spec) : 12 | vals(spec.vals.defined() ? spec.vals.data_ptr() : nullptr), 13 | is_batched(spec.vals.defined() ? spec.vals.dim() > 1 : false), 14 | // for flattened tensor 15 | chunk_starts(spec.chunk_starts.defined() ? spec.chunk_starts.data_ptr() : nullptr), 16 | chunk_cnts(spec.chunk_cnts.defined() ? spec.chunk_cnts.data_ptr(): nullptr), 17 | ray_indices(spec.ray_indices.defined() ? spec.ray_indices.data_ptr() : nullptr), 18 | is_left(spec.is_left.defined() ? spec.is_left.data_ptr() : nullptr), 19 | is_right(spec.is_right.defined() ? spec.is_right.data_ptr() : nullptr), 20 | // for dimensions 21 | n_edges(spec.vals.defined() ? spec.vals.numel() : 0), 22 | n_rays(spec.chunk_cnts.defined() ? spec.chunk_cnts.size(0) : 0), // for flattened tensor 23 | n_edges_per_ray(spec.vals.defined() ? spec.vals.size(-1) : 0) // for batched tensor 24 | { } 25 | 26 | float* vals; 27 | bool is_batched; 28 | 29 | int64_t* chunk_starts; 30 | int64_t* chunk_cnts; 31 | int64_t* ray_indices; 32 | bool* is_left; 33 | bool* is_right; 34 | 35 | int64_t n_edges; 36 | int32_t n_rays; 37 | int32_t n_edges_per_ray; 38 | }; 39 | 40 | struct PackedMultiScaleGridSpec { 41 | PackedMultiScaleGridSpec(MultiScaleGridSpec& spec) : 42 | data(spec.data.data_ptr()), 43 | occupied(spec.occupied.data_ptr()), 44 | base_aabb(spec.base_aabb.data_ptr()), 45 | levels(spec.data.size(0)), 46 | resolution{ 47 | (int32_t)spec.data.size(1), 48 | (int32_t)spec.data.size(2), 49 | (int32_t)spec.data.size(3)} 50 | { } 51 | float* data; 52 | bool* occupied; 53 | float* base_aabb; 54 | int32_t levels; 55 | int3 resolution; 56 | }; 57 | 58 | struct PackedRaysSpec { 59 | PackedRaysSpec(RaysSpec& spec) : 60 | origins(spec.origins.data_ptr()), 61 | dirs(spec.dirs.data_ptr()), 62 | N(spec.origins.size(0)) 63 | { } 64 | float *origins; 65 | float *dirs; 66 | int32_t N; 67 | }; 68 | 69 | struct SingleRaySpec { 70 | // TODO: check inv_dir if dir is zero. 71 | __device__ SingleRaySpec( 72 | float *rays_o, float *rays_d, float tmin, float tmax) : 73 | origin{rays_o[0], rays_o[1], rays_o[2]}, 74 | dir{rays_d[0], rays_d[1], rays_d[2]}, 75 | inv_dir{1.0f/rays_d[0], 1.0f/rays_d[1], 1.0f/rays_d[2]}, 76 | tmin{tmin}, 77 | tmax{tmax} 78 | { } 79 | 80 | __device__ SingleRaySpec( 81 | PackedRaysSpec& rays, int32_t id, float tmin, float tmax) : 82 | origin{ 83 | rays.origins[id * 3], 84 | rays.origins[id * 3 + 1], 85 | rays.origins[id * 3 + 2]}, 86 | dir{ 87 | rays.dirs[id * 3], 88 | rays.dirs[id * 3 + 1], 89 | rays.dirs[id * 3 + 2]}, 90 | inv_dir{ 91 | 1.0f / rays.dirs[id * 3], 92 | 1.0f / rays.dirs[id * 3 + 1], 93 | 1.0f / rays.dirs[id * 3 + 2]}, 94 | tmin{tmin}, 95 | tmax{tmax} 96 | { } 97 | float3 origin; 98 | float3 dir; 99 | float3 inv_dir; 100 | float tmin; 101 | float tmax; 102 | }; 103 | 104 | struct AABBSpec { 105 | __device__ AABBSpec(float *aabb) : 106 | min{aabb[0], aabb[1], aabb[2]}, 107 | max{aabb[3], aabb[4], aabb[5]} 108 | { } 109 | __device__ AABBSpec(float3 min, float3 max) : 110 | min{min.x, min.y, min.z}, 111 | max{max.x, max.y, max.z} 112 | { } 113 | float3 min; 114 | float3 max; 115 | }; 116 | 117 | 118 | } // namespace device 119 | } // namespace -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/csrc/include/utils_contraction.cuh: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | */ 4 | 5 | #pragma once 6 | 7 | #include "utils_math.cuh" 8 | 9 | namespace { 10 | namespace device { 11 | 12 | enum ContractionType 13 | { 14 | AABB = 0, 15 | UN_BOUNDED_TANH = 1, 16 | UN_BOUNDED_SPHERE = 2, 17 | }; 18 | 19 | inline __device__ __host__ float3 roi_to_unit( 20 | const float3 xyz, const float3 roi_min, const float3 roi_max) 21 | { 22 | // roi -> [0, 1]^3 23 | return (xyz - roi_min) / (roi_max - roi_min); 24 | } 25 | 26 | inline __device__ __host__ float3 unit_to_roi( 27 | const float3 xyz, const float3 roi_min, const float3 roi_max) 28 | { 29 | // [0, 1]^3 -> roi 30 | return xyz * (roi_max - roi_min) + roi_min; 31 | } 32 | 33 | inline __device__ __host__ float3 inf_to_unit_tanh( 34 | const float3 xyz, float3 roi_min, const float3 roi_max) 35 | { 36 | /** 37 | [-inf, inf]^3 -> [0, 1]^3 38 | roi -> cube of [0.25, 0.75]^3 39 | **/ 40 | float3 xyz_unit = roi_to_unit(xyz, roi_min, roi_max); // roi -> [0, 1]^3 41 | xyz_unit = xyz_unit - 0.5f; // roi -> [-0.5, 0.5]^3 42 | return make_float3(tanhf(xyz_unit.x), tanhf(xyz_unit.y), tanhf(xyz_unit.z)) * 0.5f + 0.5f; 43 | } 44 | 45 | inline __device__ __host__ float3 unit_to_inf_tanh( 46 | const float3 xyz, float3 roi_min, const float3 roi_max) 47 | { 48 | /** 49 | [0, 1]^3 -> [-inf, inf]^3 50 | cube of [0.25, 0.75]^3 -> roi 51 | **/ 52 | float3 xyz_unit = clamp( 53 | make_float3( 54 | atanhf(xyz.x * 2.0f - 1.0f), 55 | atanhf(xyz.y * 2.0f - 1.0f), 56 | atanhf(xyz.z * 2.0f - 1.0f)), 57 | -1e10f, 58 | 1e10f); 59 | xyz_unit = xyz_unit + 0.5f; 60 | xyz_unit = unit_to_roi(xyz_unit, roi_min, roi_max); 61 | return xyz_unit; 62 | } 63 | 64 | inline __device__ __host__ float3 inf_to_unit_sphere( 65 | const float3 xyz, const float3 roi_min, const float3 roi_max) 66 | { 67 | /** From MipNeRF360 68 | [-inf, inf]^3 -> sphere of [0, 1]^3; 69 | roi -> sphere of [0.25, 0.75]^3 70 | **/ 71 | float3 xyz_unit = roi_to_unit(xyz, roi_min, roi_max); // roi -> [0, 1]^3 72 | xyz_unit = xyz_unit * 2.0f - 1.0f; // roi -> [-1, 1]^3 73 | 74 | float norm_sq = dot(xyz_unit, xyz_unit); 75 | float norm = sqrt(norm_sq); 76 | if (norm > 1.0f) 77 | { 78 | xyz_unit = (2.0f - 1.0f / norm) * (xyz_unit / norm); 79 | } 80 | xyz_unit = xyz_unit * 0.25f + 0.5f; // [-1, 1]^3 -> [0.25, 0.75]^3 81 | return xyz_unit; 82 | } 83 | 84 | inline __device__ __host__ float3 unit_sphere_to_inf( 85 | const float3 xyz, const float3 roi_min, const float3 roi_max) 86 | { 87 | /** From MipNeRF360 88 | sphere of [0, 1]^3 -> [-inf, inf]^3; 89 | sphere of [0.25, 0.75]^3 -> roi 90 | **/ 91 | float3 xyz_unit = (xyz - 0.5f) * 4.0f; // [0.25, 0.75]^3 -> [-1, 1]^3 92 | 93 | float norm_sq = dot(xyz_unit, xyz_unit); 94 | float norm = sqrt(norm_sq); 95 | if (norm > 1.0f) 96 | { 97 | xyz_unit = xyz_unit / fmaxf((2.0f * norm - 1.0f * norm_sq), 1e-10f); 98 | } 99 | xyz_unit = xyz_unit * 0.5f + 0.5f; // [-1, 1]^3 -> [0, 1]^3 100 | xyz_unit = unit_to_roi(xyz_unit, roi_min, roi_max); // [0, 1]^3 -> roi 101 | return xyz_unit; 102 | } 103 | 104 | inline __device__ __host__ float3 apply_contraction( 105 | const float3 xyz, const float3 roi_min, const float3 roi_max, 106 | const ContractionType type) 107 | { 108 | switch (type) 109 | { 110 | case AABB: 111 | return roi_to_unit(xyz, roi_min, roi_max); 112 | case UN_BOUNDED_TANH: 113 | return inf_to_unit_tanh(xyz, roi_min, roi_max); 114 | case UN_BOUNDED_SPHERE: 115 | return inf_to_unit_sphere(xyz, roi_min, roi_max); 116 | } 117 | } 118 | 119 | inline __device__ __host__ float3 apply_contraction_inv( 120 | const float3 xyz, const float3 roi_min, const float3 roi_max, 121 | const ContractionType type) 122 | { 123 | switch (type) 124 | { 125 | case AABB: 126 | return unit_to_roi(xyz, roi_min, roi_max); 127 | case UN_BOUNDED_TANH: 128 | return unit_to_inf_tanh(xyz, roi_min, roi_max); 129 | case UN_BOUNDED_SPHERE: 130 | return unit_sphere_to_inf(xyz, roi_min, roi_max); 131 | } 132 | } 133 | 134 | } // namespace device 135 | } // namespace -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/csrc/include/utils_cuda.cuh: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | */ 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | // #include 11 | 12 | // cub support for scan by key is added to cub 1.15 13 | // in https://github.com/NVIDIA/cub/pull/376 14 | #if CUB_VERSION >= 101500 15 | #define CUB_SUPPORTS_SCAN_BY_KEY() 1 16 | #else 17 | #define CUB_SUPPORTS_SCAN_BY_KEY() 0 18 | #endif 19 | 20 | #define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") 21 | #define CHECK_CONTIGUOUS(x) \ 22 | TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") 23 | #define CHECK_INPUT(x) \ 24 | CHECK_CUDA(x); \ 25 | CHECK_CONTIGUOUS(x) 26 | #define CUDA_GET_THREAD_ID(tid, Q) \ 27 | const int tid = blockIdx.x * blockDim.x + threadIdx.x; \ 28 | if (tid >= Q) \ 29 | return 30 | #define CUDA_N_BLOCKS_NEEDED(Q, CUDA_N_THREADS) ((Q - 1) / CUDA_N_THREADS + 1) 31 | #define DEVICE_GUARD(_ten) \ 32 | const at::cuda::OptionalCUDAGuard device_guard(device_of(_ten)); 33 | 34 | // https://github.com/pytorch/pytorch/blob/233305a852e1cd7f319b15b5137074c9eac455f6/aten/src/ATen/cuda/cub.cuh#L38-L46 35 | #define CUB_WRAPPER(func, ...) do { \ 36 | size_t temp_storage_bytes = 0; \ 37 | func(nullptr, temp_storage_bytes, __VA_ARGS__); \ 38 | auto& caching_allocator = *::c10::cuda::CUDACachingAllocator::get(); \ 39 | auto temp_storage = caching_allocator.allocate(temp_storage_bytes); \ 40 | func(temp_storage.get(), temp_storage_bytes, __VA_ARGS__); \ 41 | AT_CUDA_CHECK(cudaGetLastError()); \ 42 | } while (false) 43 | 44 | template 45 | inline __device__ __host__ scalar_t ceil_div(scalar_t a, scalar_t b) 46 | { 47 | return (a + b - 1) / b; 48 | } -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/csrc/include/utils_grid.cuh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "data_spec_packed.cuh" 4 | #include "utils_contraction.cuh" 5 | #include "utils_math.cuh" 6 | 7 | namespace { 8 | namespace device { 9 | 10 | inline __device__ bool ray_aabb_intersect( 11 | SingleRaySpec ray, AABBSpec aabb, 12 | // outputs 13 | float& tmin, float& tmax) 14 | { 15 | float tmin_temp{}; 16 | float tmax_temp{}; 17 | 18 | if (ray.inv_dir.x >= 0) { 19 | tmin = (aabb.min.x - ray.origin.x) * ray.inv_dir.x; 20 | tmax = (aabb.max.x - ray.origin.x) * ray.inv_dir.x; 21 | } else { 22 | tmin = (aabb.max.x - ray.origin.x) * ray.inv_dir.x; 23 | tmax = (aabb.min.x - ray.origin.x) * ray.inv_dir.x; 24 | } 25 | 26 | if (ray.inv_dir.y >= 0) { 27 | tmin_temp = (aabb.min.y - ray.origin.y) * ray.inv_dir.y; 28 | tmax_temp = (aabb.max.y - ray.origin.y) * ray.inv_dir.y; 29 | } else { 30 | tmin_temp = (aabb.max.y - ray.origin.y) * ray.inv_dir.y; 31 | tmax_temp = (aabb.min.y - ray.origin.y) * ray.inv_dir.y; 32 | } 33 | 34 | if (tmin > tmax_temp || tmin_temp > tmax) return false; 35 | if (tmin_temp > tmin) tmin = tmin_temp; 36 | if (tmax_temp < tmax) tmax = tmax_temp; 37 | 38 | if (ray.inv_dir.z >= 0) { 39 | tmin_temp = (aabb.min.z - ray.origin.z) * ray.inv_dir.z; 40 | tmax_temp = (aabb.max.z - ray.origin.z) * ray.inv_dir.z; 41 | } else { 42 | tmin_temp = (aabb.max.z - ray.origin.z) * ray.inv_dir.z; 43 | tmax_temp = (aabb.min.z - ray.origin.z) * ray.inv_dir.z; 44 | } 45 | 46 | if (tmin > tmax_temp || tmin_temp > tmax) return false; 47 | if (tmin_temp > tmin) tmin = tmin_temp; 48 | if (tmax_temp < tmax) tmax = tmax_temp; 49 | 50 | if (tmax <= 0) return false; 51 | 52 | tmin = fmaxf(tmin, ray.tmin); 53 | tmax = fminf(tmax, ray.tmax); 54 | return true; 55 | } 56 | 57 | 58 | inline __device__ void setup_traversal( 59 | SingleRaySpec ray, float tmin, float tmax, float eps, 60 | AABBSpec aabb, int3 resolution, 61 | // outputs 62 | float3 &delta, float3 &tdist, 63 | int3 &step_index, int3 ¤t_index, int3 &final_index) 64 | { 65 | const float3 res = make_float3(resolution); 66 | const float3 voxel_size = (aabb.max - aabb.min) / res; 67 | const float3 ray_start = ray.origin + ray.dir * (tmin + eps); 68 | const float3 ray_end = ray.origin + ray.dir * (tmax - eps); 69 | 70 | // get voxel index of start and end within grid 71 | // TODO: check float error here! 72 | current_index = make_int3( 73 | apply_contraction(ray_start, aabb.min, aabb.max, ContractionType::AABB) 74 | * res 75 | ); 76 | current_index = clamp(current_index, make_int3(0, 0, 0), resolution - 1); 77 | 78 | final_index = make_int3( 79 | apply_contraction(ray_end, aabb.min, aabb.max, ContractionType::AABB) 80 | * res 81 | ); 82 | final_index = clamp(final_index, make_int3(0, 0, 0), resolution - 1); 83 | 84 | // 85 | const int3 index_delta = make_int3( 86 | ray.dir.x > 0 ? 1 : 0, ray.dir.y > 0 ? 1 : 0, ray.dir.z > 0 ? 1 : 0 87 | ); 88 | const int3 start_index = current_index + index_delta; 89 | const float3 tmax_xyz = ((aabb.min + 90 | ((make_float3(start_index) * voxel_size) - ray_start)) * ray.inv_dir) + tmin; 91 | 92 | tdist = make_float3( 93 | (ray.dir.x == 0.0f) ? tmax : tmax_xyz.x, 94 | (ray.dir.y == 0.0f) ? tmax : tmax_xyz.y, 95 | (ray.dir.z == 0.0f) ? tmax : tmax_xyz.z 96 | ); 97 | // printf("tdist: %f %f %f\n", tdist.x, tdist.y, tdist.z); 98 | 99 | const float3 step_float = make_float3( 100 | (ray.dir.x == 0.0f) ? 0.0f : (ray.dir.x > 0.0f ? 1.0f : -1.0f), 101 | (ray.dir.y == 0.0f) ? 0.0f : (ray.dir.y > 0.0f ? 1.0f : -1.0f), 102 | (ray.dir.z == 0.0f) ? 0.0f : (ray.dir.z > 0.0f ? 1.0f : -1.0f) 103 | ); 104 | step_index = make_int3(step_float); 105 | // printf("step_index: %d %d %d\n", step_index.x, step_index.y, step_index.z); 106 | 107 | const float3 delta_temp = voxel_size * ray.inv_dir * step_float; 108 | delta = make_float3( 109 | (ray.dir.x == 0.0f) ? tmax : delta_temp.x, 110 | (ray.dir.y == 0.0f) ? tmax : delta_temp.y, 111 | (ray.dir.z == 0.0f) ? tmax : delta_temp.z 112 | ); 113 | // printf("delta: %f %f %f\n", delta.x, delta.y, delta.z); 114 | } 115 | 116 | inline __device__ bool single_traversal( 117 | float3& tdist, int3& current_index, 118 | const int3 overflow_index, const int3 step_index, const float3 delta) { 119 | if ((tdist.x < tdist.y) && (tdist.x < tdist.z)) { 120 | // X-axis traversal. 121 | current_index.x += step_index.x; 122 | tdist.x += delta.x; 123 | if (current_index.x == overflow_index.x) { 124 | return false; 125 | } 126 | } else if (tdist.y < tdist.z) { 127 | // Y-axis traversal. 128 | current_index.y += step_index.y; 129 | tdist.y += delta.y; 130 | if (current_index.y == overflow_index.y) { 131 | return false; 132 | } 133 | } else { 134 | // Z-axis traversal. 135 | current_index.z += step_index.z; 136 | tdist.z += delta.z; 137 | if (current_index.z == overflow_index.z) { 138 | return false; 139 | } 140 | } 141 | return true; 142 | } 143 | 144 | 145 | } // namespace device 146 | } // namespace -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/cuda/csrc/nerfacc.cpp: -------------------------------------------------------------------------------- 1 | // This file contains only Python bindings 2 | #include "include/data_spec.hpp" 3 | 4 | #include 5 | 6 | bool is_cub_available() { 7 | // FIXME: why return false? 8 | return (bool) CUB_SUPPORTS_SCAN_BY_KEY(); 9 | } 10 | 11 | // scan 12 | torch::Tensor exclusive_sum_by_key( 13 | torch::Tensor indices, 14 | torch::Tensor inputs, 15 | bool backward); 16 | torch::Tensor inclusive_sum( 17 | torch::Tensor chunk_starts, 18 | torch::Tensor chunk_cnts, 19 | torch::Tensor inputs, 20 | bool normalize, 21 | bool backward); 22 | torch::Tensor exclusive_sum( 23 | torch::Tensor chunk_starts, 24 | torch::Tensor chunk_cnts, 25 | torch::Tensor inputs, 26 | bool normalize, 27 | bool backward); 28 | torch::Tensor inclusive_prod_forward( 29 | torch::Tensor chunk_starts, 30 | torch::Tensor chunk_cnts, 31 | torch::Tensor inputs); 32 | torch::Tensor inclusive_prod_backward( 33 | torch::Tensor chunk_starts, 34 | torch::Tensor chunk_cnts, 35 | torch::Tensor inputs, 36 | torch::Tensor outputs, 37 | torch::Tensor grad_outputs); 38 | torch::Tensor exclusive_prod_forward( 39 | torch::Tensor chunk_starts, 40 | torch::Tensor chunk_cnts, 41 | torch::Tensor inputs); 42 | torch::Tensor exclusive_prod_backward( 43 | torch::Tensor chunk_starts, 44 | torch::Tensor chunk_cnts, 45 | torch::Tensor inputs, 46 | torch::Tensor outputs, 47 | torch::Tensor grad_outputs); 48 | 49 | // grid 50 | std::vector ray_aabb_intersect( 51 | const torch::Tensor rays_o, // [n_rays, 3] 52 | const torch::Tensor rays_d, // [n_rays, 3] 53 | const torch::Tensor aabbs, // [n_aabbs, 6] 54 | const float near_plane, 55 | const float far_plane, 56 | const float miss_value); 57 | std::vector traverse_grids( 58 | // rays 59 | const torch::Tensor rays_o, // [n_rays, 3] 60 | const torch::Tensor rays_d, // [n_rays, 3] 61 | // grids 62 | const torch::Tensor binaries, // [n_grids, resx, resy, resz] 63 | const torch::Tensor aabbs, // [n_grids, 6] 64 | // intersections 65 | const torch::Tensor t_mins, // [n_rays, n_grids] 66 | const torch::Tensor t_maxs, // [n_rays, n_grids] 67 | const torch::Tensor hits, // [n_rays, n_grids] 68 | // options 69 | const torch::Tensor near_planes, 70 | const torch::Tensor far_planes, 71 | const float step_size, 72 | const float cone_angle, 73 | const bool compute_intervals, 74 | const bool compute_samples); 75 | 76 | // pdf 77 | std::vector importance_sampling( 78 | RaySegmentsSpec ray_segments, 79 | torch::Tensor cdfs, 80 | torch::Tensor n_intervels_per_ray, 81 | bool stratified); 82 | std::vector importance_sampling( 83 | RaySegmentsSpec ray_segments, 84 | torch::Tensor cdfs, 85 | int64_t n_intervels_per_ray, 86 | bool stratified); 87 | std::vector searchsorted( 88 | RaySegmentsSpec query, 89 | RaySegmentsSpec key); 90 | 91 | // cameras 92 | torch::Tensor opencv_lens_undistortion( 93 | const torch::Tensor& uv, // [..., 2] 94 | const torch::Tensor& params, // [..., 6] 95 | const float eps, 96 | const int max_iterations); 97 | torch::Tensor opencv_lens_undistortion_fisheye( 98 | const torch::Tensor& uv, // [..., 2] 99 | const torch::Tensor& params, // [..., 4] 100 | const float criteria_eps, 101 | const int criteria_iters); 102 | 103 | 104 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 105 | #define _REG_FUNC(funname) m.def(#funname, &funname) 106 | _REG_FUNC(is_cub_available); // TODO: check this function 107 | 108 | _REG_FUNC(exclusive_sum_by_key); 109 | _REG_FUNC(inclusive_sum); 110 | _REG_FUNC(exclusive_sum); 111 | _REG_FUNC(inclusive_prod_forward); 112 | _REG_FUNC(inclusive_prod_backward); 113 | _REG_FUNC(exclusive_prod_forward); 114 | _REG_FUNC(exclusive_prod_backward); 115 | 116 | _REG_FUNC(ray_aabb_intersect); 117 | _REG_FUNC(traverse_grids); 118 | _REG_FUNC(searchsorted); 119 | 120 | _REG_FUNC(opencv_lens_undistortion); 121 | _REG_FUNC(opencv_lens_undistortion_fisheye); 122 | #undef _REG_FUNC 123 | 124 | m.def("importance_sampling", py::overload_cast(&importance_sampling)); 125 | m.def("importance_sampling", py::overload_cast(&importance_sampling)); 126 | 127 | py::class_(m, "MultiScaleGridSpec") 128 | .def(py::init<>()) 129 | .def_readwrite("data", &MultiScaleGridSpec::data) 130 | .def_readwrite("occupied", &MultiScaleGridSpec::occupied) 131 | .def_readwrite("base_aabb", &MultiScaleGridSpec::base_aabb); 132 | 133 | py::class_(m, "RaysSpec") 134 | .def(py::init<>()) 135 | .def_readwrite("origins", &RaysSpec::origins) 136 | .def_readwrite("dirs", &RaysSpec::dirs); 137 | 138 | py::class_(m, "RaySegmentsSpec") 139 | .def(py::init<>()) 140 | .def_readwrite("vals", &RaySegmentsSpec::vals) 141 | .def_readwrite("is_left", &RaySegmentsSpec::is_left) 142 | .def_readwrite("is_right", &RaySegmentsSpec::is_right) 143 | .def_readwrite("chunk_starts", &RaySegmentsSpec::chunk_starts) 144 | .def_readwrite("chunk_cnts", &RaySegmentsSpec::chunk_cnts) 145 | .def_readwrite("ray_indices", &RaySegmentsSpec::ray_indices); 146 | } -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/estimators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_nerfacc/nerfacc/estimators/__init__.py -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/estimators/base.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | class AbstractEstimator(nn.Module): 8 | """An abstract Transmittance Estimator class for Sampling.""" 9 | 10 | def __init__(self) -> None: 11 | super().__init__() 12 | self.register_buffer("_dummy", torch.empty(0), persistent=False) 13 | 14 | @property 15 | def device(self) -> torch.device: 16 | return self._dummy.device 17 | 18 | def sampling(self, *args, **kwargs) -> Any: 19 | raise NotImplementedError 20 | 21 | def update_every_n_steps(self, *args, **kwargs) -> None: 22 | raise NotImplementedError 23 | -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/pack.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | """ 4 | from typing import Optional 5 | 6 | import torch 7 | from torch import Tensor 8 | 9 | 10 | @torch.no_grad() 11 | def pack_info(ray_indices: Tensor, n_rays: Optional[int] = None) -> Tensor: 12 | """Pack `ray_indices` to `packed_info`. Useful for converting per sample data to per ray data. 13 | 14 | Note: 15 | this function is not differentiable to any inputs. 16 | 17 | Args: 18 | ray_indices: Ray indices of the samples. LongTensor with shape (n_sample). 19 | n_rays: Number of rays. If None, it is inferred from `ray_indices`. Default is None. 20 | 21 | Returns: 22 | A LongTensor of shape (n_rays, 2) that specifies the start and count 23 | of each chunk in the flattened input tensor, with in total n_rays chunks. 24 | 25 | Example: 26 | 27 | .. code-block:: python 28 | 29 | >>> ray_indices = torch.tensor([0, 0, 1, 1, 1, 2, 2, 2, 2], device="cuda") 30 | >>> packed_info = pack_info(ray_indices, n_rays=3) 31 | >>> packed_info 32 | tensor([[0, 2], [2, 3], [5, 4]], device='cuda:0') 33 | 34 | """ 35 | assert ( 36 | ray_indices.dim() == 1 37 | ), "ray_indices must be a 1D tensor with shape (n_samples)." 38 | if ray_indices.is_cuda: 39 | device = ray_indices.device 40 | dtype = ray_indices.dtype 41 | if n_rays is None: 42 | n_rays = ray_indices.max().item() + 1 43 | chunk_cnts = torch.zeros((n_rays,), device=device, dtype=dtype) 44 | chunk_cnts.index_add_(0, ray_indices, torch.ones_like(ray_indices)) 45 | chunk_starts = chunk_cnts.cumsum(dim=0, dtype=dtype) - chunk_cnts 46 | packed_info = torch.stack([chunk_starts, chunk_cnts], dim=-1) 47 | else: 48 | raise NotImplementedError("Only support cuda inputs.") 49 | return packed_info 50 | -------------------------------------------------------------------------------- /CAM_nerfacc/nerfacc/version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2022 Ruilong Li, UC Berkeley. 3 | """ 4 | 5 | __version__ = "0.5.1" 6 | -------------------------------------------------------------------------------- /CAM_nerfacc/scripts/run_aws_listing.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from collections import defaultdict 3 | 4 | from boto3 import client 5 | 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument("--access_key_id", type=str, required=True) 8 | parser.add_argument("--secret_access_key", type=str, required=True) 9 | parser.add_argument("--bucket", type=str, required=True) 10 | parser.add_argument("--region", type=str, required=True) 11 | args = parser.parse_args() 12 | 13 | ROOT_URL = f"https://{args.bucket}.s3.{args.region}.amazonaws.com/whl" 14 | html = "\n\n\n{}\n\n" 15 | href = ' {}
' 16 | html_args = { 17 | "ContentType": "text/html", 18 | "CacheControl": "max-age=300", 19 | "ACL": "public-read", 20 | } 21 | 22 | s3 = client( 23 | "s3", 24 | aws_access_key_id=args.access_key_id, 25 | aws_secret_access_key=args.secret_access_key, 26 | ) 27 | 28 | bucket = s3.Bucket(name="nerfacc-bucket") 29 | 30 | wheels_dict = defaultdict(list) 31 | for obj in bucket.objects.filter(Prefix="whl"): 32 | if obj.key[-3:] != "whl": 33 | continue 34 | torch_version, wheel = obj.key.split("/")[-2:] 35 | wheel = f"{torch_version}/{wheel}" 36 | wheels_dict[torch_version].append(wheel) 37 | 38 | index_html = html.format( 39 | "\n".join( 40 | [ 41 | href.format(f"{torch_version}.html".replace("+", "%2B"), version) 42 | for version in wheels_dict 43 | ] 44 | ) 45 | ) 46 | 47 | with open("index.html", "w") as f: 48 | f.write(index_html) 49 | bucket.Object("whl/index.html").upload_file("index.html", html_args) 50 | 51 | for torch_version, wheel_names in wheels_dict.items(): 52 | torch_version_html = html.format( 53 | "\n".join( 54 | [ 55 | href.format( 56 | f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"), wheel_name 57 | ) 58 | for wheel_name in wheel_names 59 | ] 60 | ) 61 | ) 62 | 63 | with open(f"{torch_version}.html", "w") as f: 64 | f.write(torch_version_html) 65 | bucket.Object(f"whl/{torch_version}.html").upload_file( 66 | f"{torch_version}.html", args 67 | ) 68 | -------------------------------------------------------------------------------- /CAM_nerfacc/scripts/run_dev_checks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Simple yaml debugger""" 3 | import subprocess 4 | 5 | import yaml 6 | from rich.console import Console 7 | from rich.style import Style 8 | 9 | console = Console(width=120) 10 | 11 | LOCAL_TESTS = [ 12 | "Run license checks", 13 | "Run isort", 14 | "Run Black", 15 | "Python Pylint", 16 | "Test with pytest", 17 | ] 18 | 19 | 20 | def run_command(command: str) -> bool: 21 | """Run a command kill actions if it fails 22 | 23 | Args: 24 | command: command to run 25 | continue_on_fail: whether to continue running commands if the current one fails. 26 | """ 27 | ret_code = subprocess.call(command, shell=True) 28 | if ret_code != 0: 29 | console.print(f"[bold red]Error: `{command}` failed.") 30 | return ret_code == 0 31 | 32 | 33 | def run_github_actions_file(filename: str): 34 | """Run a github actions file locally. 35 | 36 | Args: 37 | filename: Which yml github actions file to run. 38 | """ 39 | with open(filename, "rb") as f: 40 | my_dict = yaml.safe_load(f) 41 | steps = my_dict["jobs"]["build"]["steps"] 42 | 43 | success = True 44 | 45 | for step in steps: 46 | if "name" in step and step["name"] in LOCAL_TESTS: 47 | compressed = step["run"].replace("\n", ";").replace("\\", "") 48 | compressed = compressed.replace("--check", "") 49 | curr_command = f"{compressed}" 50 | 51 | console.line() 52 | console.rule(f"[bold green]Running: {curr_command}") 53 | success = success and run_command(curr_command) 54 | else: 55 | skip_name = step["name"] if "name" in step else step["uses"] 56 | console.print(f"Skipping {skip_name}") 57 | 58 | # Code Testing 59 | console.line() 60 | console.rule("[bold green]Running pytest") 61 | success = success and run_command("pytest") 62 | 63 | # Add checks for building documentation 64 | console.line() 65 | console.rule("[bold green]Building Documentation") 66 | success = success and run_command( 67 | "cd docs/; make clean; make html SPHINXOPTS='-W;'" 68 | ) 69 | 70 | if success: 71 | console.line() 72 | console.rule(characters="=") 73 | console.print( 74 | "[bold green]:TADA: :TADA: :TADA: ALL CHECKS PASSED :TADA: :TADA: :TADA:", 75 | justify="center", 76 | ) 77 | console.rule(characters="=") 78 | else: 79 | console.line() 80 | console.rule(characters="=", style=Style(color="red")) 81 | console.print( 82 | "[bold red]:skull: :skull: :skull: ERRORS FOUND :skull: :skull: :skull:", 83 | justify="center", 84 | ) 85 | console.rule(characters="=", style=Style(color="red")) 86 | 87 | 88 | if __name__ == "__main__": 89 | run_github_actions_file(filename=".github/workflows/code_checks.yml") 90 | -------------------------------------------------------------------------------- /CAM_nerfacc/scripts/run_profiler.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | import torch 4 | import tqdm 5 | 6 | import nerfacc 7 | 8 | # timing 9 | # https://github.com/pytorch/pytorch/commit/d2784c233bfc57a1d836d961694bcc8ec4ed45e4 10 | 11 | 12 | class Profiler: 13 | def __init__(self, warmup=10, repeat=1000): 14 | self.warmup = warmup 15 | self.repeat = repeat 16 | 17 | def __call__(self, func: Callable): 18 | # warmup 19 | for _ in range(self.warmup): 20 | func() 21 | torch.cuda.synchronize() 22 | 23 | # profile 24 | with torch.profiler.profile( 25 | activities=[ 26 | torch.profiler.ProfilerActivity.CPU, 27 | torch.profiler.ProfilerActivity.CUDA, 28 | ], 29 | profile_memory=True, 30 | ) as prof: 31 | for _ in range(self.repeat): 32 | func() 33 | torch.cuda.synchronize() 34 | 35 | # return 36 | events = prof.key_averages() 37 | # print(events.table(sort_by="self_cpu_time_total", row_limit=10)) 38 | self_cpu_time_total = ( 39 | sum([event.self_cpu_time_total for event in events]) / self.repeat 40 | ) 41 | self_cuda_time_total = ( 42 | sum([event.self_cuda_time_total for event in events]) / self.repeat 43 | ) 44 | self_cuda_memory_usage = max( 45 | [event.self_cuda_memory_usage for event in events] 46 | ) 47 | return ( 48 | self_cpu_time_total, # in us 49 | self_cuda_time_total, # in us 50 | self_cuda_memory_usage, # in bytes 51 | ) 52 | 53 | 54 | def main(): 55 | device = "cuda:0" 56 | torch.manual_seed(42) 57 | profiler = Profiler(warmup=10, repeat=100) 58 | 59 | # # contract 60 | # print("* contract") 61 | # x = torch.rand([1024, 3], device=device) 62 | # roi = torch.tensor([0, 0, 0, 1, 1, 1], dtype=torch.float32, device=device) 63 | # fn = lambda: nerfacc.contract( 64 | # x, roi=roi, type=nerfacc.ContractionType.UN_BOUNDED_TANH 65 | # ) 66 | # cpu_t, cuda_t, cuda_bytes = profiler(fn) 67 | # print(f"{cpu_t:.2f} us, {cuda_t:.2f} us, {cuda_bytes / 1024 / 1024:.2f} MB") 68 | 69 | # rendering 70 | print("* rendering") 71 | batch_size = 81920 72 | rays_o = torch.rand((batch_size, 3), device=device) 73 | rays_d = torch.randn((batch_size, 3), device=device) 74 | rays_d = rays_d / rays_d.norm(dim=-1, keepdim=True) 75 | 76 | ray_indices, t_starts, t_ends = nerfacc._ray_marching( 77 | rays_o, 78 | rays_d, 79 | near_plane=0.1, 80 | far_plane=1.0, 81 | render_step_size=1e-1, 82 | ) 83 | sigmas = torch.randn_like(t_starts, requires_grad=True) 84 | fn = ( 85 | lambda: nerfacc.render_weight_from_density( 86 | ray_indices, t_starts, t_ends, sigmas 87 | ) 88 | .sum() 89 | .backward() 90 | ) 91 | fn() 92 | torch.cuda.synchronize() 93 | for _ in tqdm.tqdm(range(100)): 94 | fn() 95 | torch.cuda.synchronize() 96 | 97 | cpu_t, cuda_t, cuda_bytes = profiler(fn) 98 | print(f"{cpu_t:.2f} us, {cuda_t:.2f} us, {cuda_bytes / 1024 / 1024:.2f} MB") 99 | 100 | packed_info = nerfacc.pack_info(ray_indices, n_rays=batch_size) 101 | fn = ( 102 | lambda: nerfacc._vol_rendering._RenderingDensity.apply( 103 | packed_info, t_starts, t_ends, sigmas, 0 104 | ) 105 | .sum() 106 | .backward() 107 | ) 108 | fn() 109 | torch.cuda.synchronize() 110 | for _ in tqdm.tqdm(range(100)): 111 | fn() 112 | torch.cuda.synchronize() 113 | cpu_t, cuda_t, cuda_bytes = profiler(fn) 114 | print(f"{cpu_t:.2f} us, {cuda_t:.2f} us, {cuda_bytes / 1024 / 1024:.2f} MB") 115 | 116 | 117 | if __name__ == "__main__": 118 | main() 119 | -------------------------------------------------------------------------------- /CAM_nerfacc/setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | multi_line_output = 3 3 | line_length = 80 4 | include_trailing_comma = true 5 | skip=./examples/pycolmap,./benchmarks -------------------------------------------------------------------------------- /CAM_nerfacc/setup.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import os.path as osp 4 | import platform 5 | import sys 6 | 7 | from setuptools import find_packages, setup 8 | 9 | __version__ = None 10 | exec(open("nerfacc/version.py", "r").read()) 11 | 12 | URL = "https://github.com/KAIR-BAIR/nerfacc" 13 | 14 | BUILD_NO_CUDA = os.getenv("BUILD_NO_CUDA", "0") == "1" 15 | WITH_SYMBOLS = os.getenv("WITH_SYMBOLS", "0") == "1" 16 | 17 | 18 | def get_ext(): 19 | from torch.utils.cpp_extension import BuildExtension 20 | 21 | return BuildExtension.with_options( 22 | no_python_abi_suffix=True, use_ninja=False 23 | ) 24 | 25 | 26 | def get_extensions(): 27 | import torch 28 | from torch.__config__ import parallel_info 29 | from torch.utils.cpp_extension import CUDAExtension 30 | 31 | extensions_dir = osp.join("nerfacc", "cuda", "csrc") 32 | sources = glob.glob(osp.join(extensions_dir, "*.cu")) + glob.glob( 33 | osp.join(extensions_dir, "*.cpp") 34 | ) 35 | # remove generated 'hip' files, in case of rebuilds 36 | sources = [path for path in sources if "hip" not in path] 37 | 38 | undef_macros = [] 39 | define_macros = [] 40 | 41 | if sys.platform == "win32": 42 | define_macros += [("nerfacc_EXPORTS", None)] 43 | 44 | extra_compile_args = {"cxx": ["-O3"]} 45 | if not os.name == "nt": # Not on Windows: 46 | extra_compile_args["cxx"] += ["-Wno-sign-compare"] 47 | extra_link_args = [] if WITH_SYMBOLS else ["-s"] 48 | 49 | info = parallel_info() 50 | if ( 51 | "backend: OpenMP" in info 52 | and "OpenMP not found" not in info 53 | and sys.platform != "darwin" 54 | ): 55 | extra_compile_args["cxx"] += ["-DAT_PARALLEL_OPENMP"] 56 | if sys.platform == "win32": 57 | extra_compile_args["cxx"] += ["/openmp"] 58 | else: 59 | extra_compile_args["cxx"] += ["-fopenmp"] 60 | else: 61 | print("Compiling without OpenMP...") 62 | 63 | # Compile for mac arm64 64 | if sys.platform == "darwin" and platform.machine() == "arm64": 65 | extra_compile_args["cxx"] += ["-arch", "arm64"] 66 | extra_link_args += ["-arch", "arm64"] 67 | 68 | nvcc_flags = os.getenv("NVCC_FLAGS", "") 69 | nvcc_flags = [] if nvcc_flags == "" else nvcc_flags.split(" ") 70 | nvcc_flags += ["-O3"] 71 | if torch.version.hip: 72 | # USE_ROCM was added to later versions of PyTorch. 73 | # Define here to support older PyTorch versions as well: 74 | define_macros += [("USE_ROCM", None)] 75 | undef_macros += ["__HIP_NO_HALF_CONVERSIONS__"] 76 | else: 77 | nvcc_flags += ["--expt-relaxed-constexpr"] 78 | extra_compile_args["nvcc"] = nvcc_flags 79 | 80 | extension = CUDAExtension( 81 | f"nerfacc.csrc", 82 | sources, 83 | include_dirs=[osp.join(extensions_dir, "include")], 84 | define_macros=define_macros, 85 | undef_macros=undef_macros, 86 | extra_compile_args=extra_compile_args, 87 | extra_link_args=extra_link_args, 88 | ) 89 | 90 | return [extension] 91 | 92 | 93 | # work-around hipify abs paths 94 | include_package_data = True 95 | # if torch.cuda.is_available() and torch.version.hip: 96 | # include_package_data = False 97 | 98 | setup( 99 | name="nerfacc", 100 | version=__version__, 101 | description="A General NeRF Acceleration Toolbox", 102 | author="Ruilong", 103 | author_email="ruilongli94@gmail.com", 104 | url=URL, 105 | download_url=f"{URL}/archive/{__version__}.tar.gz", 106 | keywords=[], 107 | python_requires=">=3.7", 108 | install_requires=["rich>=12", "torch", "typing_extensions; python_version<'3.8'"], 109 | extras_require={ 110 | # dev dependencies. Install them by `pip install nerfacc[dev]` 111 | "dev": [ 112 | "black[jupyter]==22.3.0", 113 | "isort==5.10.1", 114 | "pylint==2.13.4", 115 | "pytest==7.1.2", 116 | "pytest-xdist==2.5.0", 117 | "typeguard>=2.13.3", 118 | "pyyaml==6.0", 119 | "build", 120 | "twine", 121 | ], 122 | }, 123 | ext_modules=get_extensions() if not BUILD_NO_CUDA else [], 124 | cmdclass={"build_ext": get_ext()} if not BUILD_NO_CUDA else {}, 125 | packages=find_packages(), 126 | include_package_data=include_package_data, 127 | ) 128 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Coordinate-Aware Modulation for Neural Fields (ICLR 2024 Spotlight) 2 | ### Joo Chan Lee, Daniel Rho, Seungtae Nam, Jong Hwan Ko, and Eunbyung Park 3 | 4 | ### [[Project Page](https://maincold2.github.io/cam/)] [[Paper(arxiv)](https://arxiv.org/abs/2311.14993)] 5 | 6 | ## Method Overview 7 | 8 | 9 | In this work, we propose a novel way for exploiting both MLPs and grid representations in neural fields. Unlike the prevalent methods that combine them sequentially (extract features from the grids first and feed them to the MLP), we inject spectral bias-free grid representations into the intermediate features in the MLP. More specifically, we suggest a Coordinate-Aware Modulation (CAM), which modulates the intermediate features using scale and shift parameters extracted from the grid representations. This can maintain the strengths of MLPs while mitigating any remaining potential biases, facilitating the rapid learning of high-frequency components. 10 | 11 | ## Setup 12 | 13 | CAM is applied to various methods in a plug-and-play manner. We recommend following the original configurations of each method ([FFNeRV](https://github.com/maincold2/FFNeRV), [Mip-NeRF](https://github.com/google/mipnerf), [Mip-NeRF 360](https://github.com/google-research/multinerf), and [NerfAcc@ebeb5dd](https://github.com/nerfstudio-project/nerfacc/tree/ebeb5ddf733c04b425d5070efae9c3d23f64b078)). 14 | 15 | ## Image representation 16 | ### FFN 17 | We implement CAM on the FFN pytorch implementation in [FINN](https://github.com/yixin26/FINN). 18 | 19 | ```shell 20 | python CAM_image.py -g [GPU ID] --data [image or image_dir] --model [CAM or FFN] --exp [work_dir] 21 | ``` 22 | 23 | #### --downsample 24 | Downsample factor of training images, where 1 means image regression, 2 by default 25 | 26 | ## Video representation 27 | ### FFNeRV 28 | 29 | ```bash 30 | cd CAM_ffnerv 31 | 32 | # For representation 33 | python main.py -e 300 --lower-width 96 --num-blocks 1 --dataset [data_dir] --outf [work_dir] --fc-hw-dim 9_16_156 --expansion 1 --loss Fusion6 --strides 5 2 2 2 2 --conv-type conv -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --t-dim 64 128 256 512 --mod-dim 60 34 | 35 | # For compression 36 | python main.py -e 600 --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [work_dir] --fc-hw-dim 9_16_48 --expansion 8 --loss Fusion6 --strides 5 3 2 2 2 --conv-type compact -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --wbit 6 --t-dim 300 600 --resol 1920 1080 --mod-dim 30 37 | ``` 38 | 39 | #### --mod-dim 40 | Temporal resolution of modulation grids, where 0 means the original FFNeRV, 60 by default 41 | 42 | ## NeRF 43 | ### Mip-NeRF 44 | ```shell 45 | cd CAM_mipnerf 46 | 47 | python -m train --data_dir=[nerf_synthetic_dir] --train_dir=[work_dir] --gin_file=configs/blender.gin --logtostderr 48 | 49 | python -m eval --data_dir=[nerf_synthetic_dir] --train_dir=[work_dir] --gin_file=configs/blender.gin --logtostderr 50 | ``` 51 | ### Mip-NeRF 360 52 | ```shell 53 | cd CAM_multinerf-mipnerf360 54 | 55 | python -m train --gin_configs=configs/360.gin --gin_bindings="Config.data_dir = '[360 data_dir]'" --gin_bindings="Config.checkpoint_dir = '[work_dir]'" --logtostderr 56 | 57 | python -m eval --gin_configs=configs/360.gin --gin_bindings="Config.data_dir = '[360 data_dir]'" --gin_bindings="Config.checkpoint_dir = '[work_dir]'" --logtostderr 58 | ``` 59 | ### NerfAcc 60 | 61 | ```shell 62 | cd CAM_nerfacc 63 | 64 | PYTHONPATH=./ python examples/train_mlp_nerf.py --scene [scene_name] --data_root [nerfsyn_dataset_dir] --exp [work_dir] 65 | ``` 66 | 67 | ## Dynamic NeRF 68 | 69 | ### NerfAcc 70 | 71 | ```shell 72 | cd CAM_nerfacc 73 | 74 | PYTHONPATH=./ python examples/train_mlp_tnerf.py --scene [scene_name] --data_root [dnerf_dataset_dir] --exp [work_dir] 75 | ``` 76 | 77 | ## BibTeX 78 | ``` 79 | @inproceedings{ 80 | lee2024cam, 81 | title={Coordinate-Aware Modulation for Neural Fields}, 82 | author={Joo Chan Lee and Daniel Rho and Seungtae Nam and Jong Hwan Ko and Eunbyung Park}, 83 | booktitle={The Twelfth International Conference on Learning Representations}, 84 | year={2024}, 85 | url={https://openreview.net/forum?id=4UiLqimGm5} 86 | } 87 | ``` 88 | --------------------------------------------------------------------------------