├── .gitignore
├── CAM_1D
├── 1D function.ipynb
├── metrics.py
└── network.py
├── CAM_ffnerv
├── LICENSE.txt
├── README.md
├── datasets
│ └── README.md
├── main.py
├── model.py
├── requirements.txt
└── utils.py
├── CAM_image.py
├── CAM_mipnerf
├── CONTRIBUTING.md
├── LICENSE.txt
├── README.md
├── configs
│ ├── blender.gin
│ ├── blender_noextras.gin
│ ├── blender_noipe.gin
│ ├── llff.gin
│ ├── multiblender.gin
│ ├── multiblender_noextras.gin
│ ├── multiblender_noipe.gin
│ └── multiblender_noloss.gin
├── eval.py
├── internal
│ ├── datasets.py
│ ├── math.py
│ ├── math_test.py
│ ├── mip.py
│ ├── mip_test.py
│ ├── models.py
│ ├── utils.py
│ └── vis.py
├── requirements.txt
├── scripts
│ ├── convert_blender_data.py
│ ├── eval_blender.sh
│ ├── eval_llff.sh
│ ├── eval_multiblender.sh
│ ├── summarize.ipynb
│ ├── train_blender.sh
│ ├── train_llff.sh
│ └── train_multiblender.sh
└── train.py
├── CAM_multinerf-mipnerf360
├── .gitignore
├── CONTRIBUTING.md
├── LICENSE.txt
├── README.md
├── configs
│ ├── .ipynb_checkpoints
│ │ ├── 360-checkpoint.gin
│ │ ├── 360_glo4-checkpoint.gin
│ │ └── render_config-checkpoint.gin
│ ├── 360.gin
│ ├── 360_glo4.gin
│ ├── blender_256.gin
│ ├── blender_512.gin
│ ├── blender_refnerf.gin
│ ├── debug.gin
│ ├── llff_256.gin
│ ├── llff_512.gin
│ ├── llff_raw.gin
│ ├── llff_raw_test.gin
│ ├── render_config.gin
│ └── tat.gin
├── eval.py
├── internal
│ ├── camera_utils.py
│ ├── configs.py
│ ├── coord.py
│ ├── datasets.py
│ ├── geopoly.py
│ ├── image.py
│ ├── math.py
│ ├── models.py
│ ├── raw_utils.py
│ ├── ref_utils.py
│ ├── render.py
│ ├── stepfun.py
│ ├── train_utils.py
│ ├── utils.py
│ └── vis.py
├── render.py
├── requirements.txt
├── scripts
│ ├── eval_360.sh
│ ├── eval_blender.sh
│ ├── eval_llff.sh
│ ├── eval_raw.sh
│ ├── eval_shinyblender.sh
│ ├── generate_tables.ipynb
│ ├── local_colmap_and_resize.sh
│ ├── render_360.sh
│ ├── render_llff.sh
│ ├── render_raw.sh
│ ├── run_all_unit_tests.sh
│ ├── train_360.sh
│ ├── train_blender.sh
│ ├── train_llff.sh
│ ├── train_raw.sh
│ └── train_shinyblender.sh
├── tests
│ ├── camera_utils_test.py
│ ├── coord_test.py
│ ├── datasets_test.py
│ ├── geopoly_test.py
│ ├── image_test.py
│ ├── math_test.py
│ ├── ref_utils_test.py
│ ├── render_test.py
│ ├── stepfun_test.py
│ └── utils_test.py
└── train.py
├── CAM_nerfacc
├── .github
│ └── workflows
│ │ ├── aws
│ │ └── update_index.py
│ │ ├── building.yml
│ │ ├── code_checks.yml
│ │ ├── cuda
│ │ ├── Linux-env.sh
│ │ ├── Linux.sh
│ │ ├── Windows-env.sh
│ │ └── Windows.sh
│ │ └── publish.yml
├── .gitignore
├── .gitmodules
├── .pre-commit-config.yaml
├── .readthedocs.yaml
├── CHANGELOG.md
├── LICENSE.txt
├── MANIFEST.in
├── README.md
├── examples
│ ├── datasets
│ │ ├── __init__.py
│ │ ├── dnerf_synthetic.py
│ │ ├── nerf_360_v2.py
│ │ ├── nerf_synthetic.py
│ │ └── utils.py
│ ├── pycolmap
│ │ ├── .gitignore
│ │ ├── LICENSE.txt
│ │ ├── README.md
│ │ ├── pycolmap
│ │ │ ├── __init__.py
│ │ │ ├── camera.py
│ │ │ ├── database.py
│ │ │ ├── image.py
│ │ │ ├── rotation.py
│ │ │ └── scene_manager.py
│ │ └── tools
│ │ │ ├── colmap_to_nvm.py
│ │ │ ├── delete_images.py
│ │ │ ├── impute_missing_cameras.py
│ │ │ ├── save_cameras_as_ply.py
│ │ │ ├── transform_model.py
│ │ │ ├── write_camera_track_to_bundler.py
│ │ │ └── write_depthmap_to_ply.py
│ ├── radiance_fields
│ │ ├── __init__.py
│ │ └── mlp.py
│ ├── requirements.txt
│ ├── train_mlp_nerf.py
│ ├── train_mlp_tnerf.py
│ └── utils.py
├── nerfacc
│ ├── __init__.py
│ ├── cameras.py
│ ├── cameras2.py
│ ├── cuda
│ │ ├── __init__.py
│ │ ├── _backend.py
│ │ └── csrc
│ │ │ ├── camera.cu
│ │ │ ├── grid.cu
│ │ │ ├── include
│ │ │ ├── data_spec.hpp
│ │ │ ├── data_spec_packed.cuh
│ │ │ ├── utils_camera.cuh
│ │ │ ├── utils_contraction.cuh
│ │ │ ├── utils_cuda.cuh
│ │ │ ├── utils_grid.cuh
│ │ │ ├── utils_math.cuh
│ │ │ └── utils_scan.cuh
│ │ │ ├── nerfacc.cpp
│ │ │ ├── pdf.cu
│ │ │ └── scan.cu
│ ├── data_specs.py
│ ├── estimators
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── occ_grid.py
│ │ └── prop_net.py
│ ├── grid.py
│ ├── pack.py
│ ├── pdf.py
│ ├── scan.py
│ ├── version.py
│ └── volrend.py
├── scripts
│ ├── run_aws_listing.py
│ ├── run_dev_checks.py
│ └── run_profiler.py
├── setup.cfg
└── setup.py
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | *.DS_Store*
3 |
--------------------------------------------------------------------------------
/CAM_1D/metrics.py:
--------------------------------------------------------------------------------
1 | import math
2 | import torch
3 | import torch.nn as nn
4 | import torch.nn.functional as F
5 |
6 | def _binarize(y_data, threshold):
7 | """
8 | args:
9 | y_data : [float] 4-d tensor in [batch_size, channels, img_rows, img_cols]
10 | threshold : [float] [0.0, 1.0]
11 | return 4-d binarized y_data
12 | """
13 | y_data[y_data < threshold] = 0.0
14 | y_data[y_data >= threshold] = 1.0
15 | return y_data
16 |
17 | class MSE(object):
18 | def __init__(self, des="Mean Square Error"):
19 | self.des = des
20 |
21 | def __repr__(self):
22 | return "MSE"
23 |
24 | def __call__(self, y_pred, y_true, dim=1, threshold=None):
25 | """
26 | args:
27 | y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
28 | y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
29 | threshold : [0.0, 1.0]
30 | return mean_squared_error, smaller the better
31 | """
32 | if threshold:
33 | y_pred = _binarize(y_pred, threshold)
34 | return torch.mean((y_pred - y_true) ** 2)
35 |
36 |
37 | class PSNR(object):
38 | def __init__(self, des="Peak Signal to Noise Ratio"):
39 | self.des = des
40 |
41 | def __repr__(self):
42 | return "PSNR"
43 |
44 | def __call__(self, y_pred, y_true, dim=1, threshold=None):
45 | """
46 | args:
47 | y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
48 | y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
49 | threshold : [0.0, 1.0]
50 | return PSNR, larger the better
51 | """
52 | if threshold:
53 | y_pred = _binarize(y_pred, threshold)
54 | mse = torch.mean((y_pred - y_true) ** 2)
55 | return 10 * torch.log10(1 / mse)
56 |
57 |
58 | class SSIM(object):
59 | '''
60 | modified from https://github.com/jorge-pessoa/pytorch-msssim
61 | '''
62 | def __init__(self, des="structural similarity index"):
63 | self.des = des
64 |
65 | def __repr__(self):
66 | return "SSIM"
67 |
68 | def gaussian(self, w_size, sigma):
69 | gauss = torch.Tensor([math.exp(-(x - w_size//2)**2/float(2*sigma**2)) for x in range(w_size)])
70 | return gauss/gauss.sum()
71 |
72 | def create_window(self, w_size, channel=1):
73 | _1D_window = self.gaussian(w_size, 1.5).unsqueeze(1)
74 | _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
75 | window = _2D_window.expand(channel, 1, w_size, w_size).contiguous()
76 | return window
77 |
78 | def __call__(self, y_pred, y_true, w_size=11, size_average=True, full=False):
79 | """
80 | args:
81 | y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
82 | y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
83 | w_size : int, default 11
84 | size_average : boolean, default True
85 | full : boolean, default False
86 | return ssim, larger the better
87 | """
88 | # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
89 | if torch.max(y_pred) > 128:
90 | max_val = 255
91 | else:
92 | max_val = 1
93 |
94 | if torch.min(y_pred) < -0.5:
95 | min_val = -1
96 | else:
97 | min_val = 0
98 | L = max_val - min_val
99 |
100 | padd = 0
101 | (_, channel, height, width) = y_pred.size()
102 | window = self.create_window(w_size, channel=channel).to(y_pred.device)
103 |
104 | mu1 = F.conv2d(y_pred, window, padding=padd, groups=channel)
105 | mu2 = F.conv2d(y_true, window, padding=padd, groups=channel)
106 |
107 | mu1_sq = mu1.pow(2)
108 | mu2_sq = mu2.pow(2)
109 | mu1_mu2 = mu1 * mu2
110 |
111 | sigma1_sq = F.conv2d(y_pred * y_pred, window, padding=padd, groups=channel) - mu1_sq
112 | sigma2_sq = F.conv2d(y_true * y_true, window, padding=padd, groups=channel) - mu2_sq
113 | sigma12 = F.conv2d(y_pred * y_true, window, padding=padd, groups=channel) - mu1_mu2
114 |
115 | C1 = (0.01 * L) ** 2
116 | C2 = (0.03 * L) ** 2
117 |
118 | v1 = 2.0 * sigma12 + C2
119 | v2 = sigma1_sq + sigma2_sq + C2
120 | cs = torch.mean(v1 / v2) # contrast sensitivity
121 |
122 | ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
123 |
124 | if size_average:
125 | ret = ssim_map.mean()
126 | else:
127 | ret = ssim_map.mean(1).mean(1).mean(1)
128 |
129 | if full:
130 | return ret, cs
131 | return ret
--------------------------------------------------------------------------------
/CAM_1D/network.py:
--------------------------------------------------------------------------------
1 | # import re
2 | import numpy as np
3 |
4 | import torch
5 | import torch.nn as nn
6 | import torch.nn.functional as F
7 | import math
8 |
9 | class SinusoidalEncoder(nn.Module):
10 | """Sinusoidal Positional Encoder used in Nerf."""
11 |
12 | def __init__(self, x_dim, min_deg, max_deg, use_identity: bool = True):
13 | super().__init__()
14 | self.x_dim = x_dim
15 | self.min_deg = min_deg
16 | self.max_deg = max_deg
17 | self.use_identity = use_identity
18 | self.register_buffer(
19 | "scales", torch.tensor([2**i for i in range(min_deg, max_deg)])
20 | )
21 |
22 | @property
23 | def latent_dim(self) -> int:
24 | return (
25 | int(self.use_identity) + (self.max_deg - self.min_deg) * 2
26 | ) * self.x_dim
27 |
28 | def forward(self, x: torch.Tensor) -> torch.Tensor:
29 | """
30 | Args:
31 | x: [..., x_dim]
32 | Returns:
33 | latent: [..., latent_dim]
34 | """
35 | if self.max_deg == self.min_deg:
36 | return x
37 | xb = torch.reshape(
38 | (x[Ellipsis, None, :] * self.scales[:, None]),
39 | list(x.shape[:-1]) + [(self.max_deg - self.min_deg) * self.x_dim],
40 | )
41 | latent = torch.sin(torch.cat([xb, xb + 0.5 * math.pi], dim=-1))
42 | if self.use_identity:
43 | latent = torch.cat([x] + [latent], dim=-1)
44 | return latent
45 |
46 | class CAM_PE(nn.Module):
47 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats):
48 | super(CAM_PE, self).__init__()
49 |
50 | self.time_encoder = SinusoidalEncoder(1, 0, 16, True)
51 | self.net = [nn.Linear(self.time_encoder.latent_dim, hidden_feats)] + \
52 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)]
53 | self.net_last = nn.Linear(hidden_feats, out_feats)
54 |
55 | self.net = nn.ModuleList(self.net)
56 |
57 | self.affine = nn.ParameterList()
58 | for i in range(n_hidden_layers+1):
59 | self.affine.append(torch.nn.Parameter(torch.cat([torch.ones((1, 1, 64, 1)), torch.zeros((1, 1, 64, 1))],dim=1)))
60 |
61 |
62 | def forward(self, x):
63 | coord = x.clone()*2-1.0
64 | coord = torch.cat((torch.zeros_like(coord),coord), dim=-1)
65 | x = self.time_encoder(x)
66 | for i, l in enumerate(self.net):
67 | x = l(x)
68 | affine = F.grid_sample(self.affine[i], coord.view(1,-1,1,2), align_corners=True).view(2,-1,1)
69 | x = affine[0]*x + affine[1]
70 | x = self.net_last(x)
71 | return x
72 |
73 | class CAM(nn.Module):
74 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats):
75 | super(CAM, self).__init__()
76 |
77 | self.net = [nn.Linear(1, hidden_feats)] + \
78 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)]
79 | self.net_last = nn.Linear(hidden_feats, out_feats)
80 | self.net = nn.ModuleList(self.net)
81 |
82 | self.affine = nn.ParameterList()
83 | for i in range(n_hidden_layers+1):
84 | self.affine.append(torch.nn.Parameter(torch.cat([torch.ones((1, 1, 64, 1)), torch.zeros((1, 1, 64, 1))],dim=1)))
85 |
86 |
87 | def forward(self, x):
88 | coord = x.clone()*2-1.0
89 | coord = torch.cat((torch.zeros_like(coord),coord), dim=-1)
90 | for i, l in enumerate(self.net):
91 | x = l(x)
92 | affine = F.grid_sample(self.affine[i], coord.view(1,-1,1,2), align_corners=True).view(2,-1,1)
93 | x = affine[0]*x + affine[1]
94 | x = self.net_last(x)
95 | return x
96 |
97 | class PE(nn.Module):
98 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats):
99 | super(PE, self).__init__()
100 |
101 | self.time_encoder = SinusoidalEncoder(1, 0, 16, True)
102 | self.net = [nn.Linear(self.time_encoder.latent_dim, hidden_feats)] + \
103 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)]
104 | self.net_last = nn.Linear(hidden_feats, out_feats)
105 | self.net = nn.ModuleList(self.net)
106 |
107 |
108 | def forward(self, x):
109 | x = self.time_encoder(x)
110 | for i, l in enumerate(self.net):
111 | x = l(x)
112 | x = self.net_last(x)
113 | return x
114 |
115 | class NoEnc(nn.Module):
116 | def __init__(self, in_feats, hidden_feats, n_hidden_layers, out_feats):
117 | super(NoEnc, self).__init__()
118 |
119 | self.net = [nn.Linear(1, hidden_feats)] + \
120 | [nn.Linear(hidden_feats, hidden_feats) for _ in range(n_hidden_layers)]
121 | self.net_last = nn.Linear(hidden_feats, out_feats)
122 | self.net = nn.ModuleList(self.net)
123 |
124 |
125 | def forward(self, x):
126 | for i, l in enumerate(self.net):
127 | x = l(x)
128 | x = self.net_last(x)
129 | return x
--------------------------------------------------------------------------------
/CAM_ffnerv/LICENSE.txt:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Joo Chan Lee
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/CAM_ffnerv/README.md:
--------------------------------------------------------------------------------
1 | # FFNeRV: Flow-Guided Frame-Wise Neural Representations for Videos
2 | #### Joo Chan Lee, Daniel Rho, Jong Hwan Ko†, and Eunbyung Park†
3 |
4 | ### [[Project Page](https://maincold2.github.io/ffnerv/)] [[Paper(arxiv)](https://arxiv.org/abs/2212.12294)]
5 |
6 | Our code is based on [NeRV](https://github.com/haochen-rye/NeRV)
7 |
8 | ## Method overview
9 |
10 |
11 | ## 0. Requirements
12 | ```
13 | pip install -r requirements.txt
14 | ```
15 | (Update) Highly recommend upgrading the Pytorch version (>= 1.13.0) for faster training and inference.
16 |
17 | Dataset link: [UVG](https://ultravideo.fi/#testsequences)
18 |
19 | ## 1. Training (Encoding)
20 | ### For representation
21 | ```bash
22 | python main.py -e 300 --lower-width 96 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_156 --expansion 1 --loss Fusion6 --strides 5 2 2 2 2 --conv-type conv -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --t-dim 64 128 256 512
23 | ```
24 | ### For compression
25 | ```bash
26 | python main.py -e 600 --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_48 --expansion 8 --loss Fusion6 --strides 5 3 2 2 2 --conv-type compact -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --wbit 8 --t-dim 300 600 --resol 1920 1080
27 | ```
28 | - "--conv-type": the type of convolution blocks
29 | - "--agg-ind": relative video indices for aggregation
30 | - "--wbit": for n-bit quantization (QAT)
31 | - "--t-dim": multi-channels of temporal grids
32 |
33 | More details can be found in "main.py"
34 |
35 |
36 | ## 2. Evaluation (+ Entropy coding)
37 | ```bash
38 | python main.py --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_48 --expansion 8 --strides 5 3 2 2 2 --conv-type compact -b 1 --agg-ind -2 -1 1 2 --wbit 8 --t-dim 300 600 --resol 1920 1080 --eval-only
39 | ```
40 |
41 | ## 3. Decoding
42 | ```bash
43 | python main.py --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [out_dir] --fc-hw-dim 9_16_48 --expansion 8 --strides 5 3 2 2 2 --conv-type compact -b 1 --agg-ind -2 -1 1 2 --wbit 8 --t-dim 300 600 --resol 1920 1080 --eval-only --weight [weight_path] --dump-images
44 | ```
45 |
--------------------------------------------------------------------------------
/CAM_ffnerv/datasets/README.md:
--------------------------------------------------------------------------------
1 | Video directory
2 |
3 |
--------------------------------------------------------------------------------
/CAM_ffnerv/requirements.txt:
--------------------------------------------------------------------------------
1 | torch==1.13.0
2 | numpy==1.19.2
3 | pytorch_msssim==0.2.1
4 | tqdm==4.59.0
5 | torchvision==0.9
6 | dahuffman==0.4.1
7 | Pillow==8.4.0
8 | yaspin==2.1.0
9 | tensorboard
10 |
--------------------------------------------------------------------------------
/CAM_mipnerf/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code Reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google/conduct/).
29 |
--------------------------------------------------------------------------------
/CAM_mipnerf/README.md:
--------------------------------------------------------------------------------
1 | # mip-NeRF
2 |
3 | This repository contains the code release for
4 | [Mip-NeRF: A Multiscale Representation for Anti-Aliasing Neural Radiance Fields](https://jonbarron.info/mipnerf/).
5 | This implementation is written in [JAX](https://github.com/google/jax), and
6 | is a fork of Google's [JaxNeRF implementation](https://github.com/google-research/google-research/tree/master/jaxnerf).
7 | Contact [Jon Barron](https://jonbarron.info/) if you encounter any issues.
8 |
9 | 
10 |
11 | ## Abstract
12 |
13 | The rendering procedure used by neural radiance fields (NeRF) samples a scene
14 | with a single ray per pixel and may therefore produce renderings that are
15 | excessively blurred or aliased when training or testing images observe scene
16 | content at different resolutions. The straightforward solution of supersampling
17 | by rendering with multiple rays per pixel is impractical for NeRF, because
18 | rendering each ray requires querying a multilayer perceptron hundreds of times.
19 | Our solution, which we call "mip-NeRF" (à la "mipmap"), extends NeRF to
20 | represent the scene at a continuously-valued scale. By efficiently rendering
21 | anti-aliased conical frustums instead of rays, mip-NeRF reduces objectionable
22 | aliasing artifacts and significantly improves NeRF's ability to represent
23 | fine details, while also being 7% faster than NeRF and half the size. Compared
24 | to NeRF, mip-NeRF reduces average error rates by 17% on the dataset presented
25 | with NeRF and by 60% on a challenging multiscale variant of that dataset that
26 | we present. mip-NeRF is also able to match the accuracy of a brute-force
27 | supersampled NeRF on our multiscale dataset while being 22x faster.
28 |
29 |
30 | ## Installation
31 | We recommend using [Anaconda](https://www.anaconda.com/products/individual) to set
32 | up the environment. Run the following commands:
33 |
34 | ```
35 | # Clone the repo
36 | git clone https://github.com/google/mipnerf.git; cd mipnerf
37 | # Create a conda environment, note you can use python 3.6-3.8 as
38 | # one of the dependencies (TensorFlow) hasn't supported python 3.9 yet.
39 | conda create --name mipnerf python=3.6.13; conda activate mipnerf
40 | # Prepare pip
41 | conda install pip; pip install --upgrade pip
42 | # Install requirements
43 | pip install -r requirements.txt
44 | ```
45 |
46 | [Optional] Install GPU and TPU support for Jax
47 | ```
48 | # Remember to change cuda101 to your CUDA version, e.g. cuda110 for CUDA 11.0.
49 | pip install --upgrade jax jaxlib==0.1.65+cuda101 -f https://storage.googleapis.com/jax-releases/jax_releases.html
50 | ```
51 |
52 | ## Data
53 |
54 | Then, you'll need to download the datasets
55 | from the [NeRF official Google Drive](https://drive.google.com/drive/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1).
56 | Please download and unzip `nerf_synthetic.zip` and `nerf_llff_data.zip`.
57 |
58 | ### Generate multiscale dataset
59 | You can generate the multiscale dataset used in the paper by running the following command,
60 | ```
61 | python scripts/convert_blender_data.py --blenderdir /nerf_synthetic --outdir /multiscale
62 | ```
63 |
64 | ## Running
65 |
66 | Example scripts for training mip-NeRF on individual scenes from the three
67 | datasets used in the paper can be found in `scripts/`. You'll need to change
68 | the paths to point to wherever the datasets are located.
69 | [Gin](https://github.com/google/gin-config) configuration files for our model
70 | and some ablations can be found in `configs/`.
71 | An example script for evaluating on the test set of each scene can be found
72 | in `scripts/`, after which you can use `scripts/summarize.ipynb` to produce
73 | error metrics across all scenes in the same format as was used in tables in the
74 | paper.
75 |
76 | ### OOM errors
77 | You may need to reduce the batch size to avoid out of memory errors. For example the model can be run on a NVIDIA 3080 (10Gb) using the following flag.
78 | ```
79 | --gin_param="Config.batch_size = 1024"
80 | ```
81 |
82 | ## Citation
83 | If you use this software package, please cite our paper:
84 |
85 | ```
86 | @misc{barron2021mipnerf,
87 | title={Mip-NeRF: A Multiscale Representation for Anti-Aliasing Neural Radiance Fields},
88 | author={Jonathan T. Barron and Ben Mildenhall and Matthew Tancik and Peter Hedman and Ricardo Martin-Brualla and Pratul P. Srinivasan},
89 | year={2021},
90 | eprint={2103.13415},
91 | archivePrefix={arXiv},
92 | primaryClass={cs.CV}
93 | }
94 | ```
95 |
96 | ## Acknowledgements
97 | Thanks to [Boyang Deng](https://boyangdeng.com/) for JaxNeRF.
98 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/blender.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'blender'
2 | Config.batching = 'single_image'
3 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/blender_noextras.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'blender'
2 | Config.batching = 'single_image'
3 | MipNerfModel.density_activation = @flax.nn.relu
4 | MipNerfModel.density_bias = 0.0
5 | MipNerfModel.rgb_padding = 0.0
6 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/blender_noipe.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'blender'
2 | Config.batching = 'single_image'
3 | MipNerfModel.disable_integration = True
4 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/llff.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.white_bkgd = False
3 | Config.randomized = True
4 | Config.near = 0.
5 | Config.far = 1.
6 | Config.factor = 4
7 | Config.llffhold = 8
8 | MipNerfModel.use_viewdirs = True
9 | MipNerfModel.ray_shape = 'cylinder'
10 | MipNerfModel.density_noise = 1.
11 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/multiblender.gin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_mipnerf/configs/multiblender.gin
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/multiblender_noextras.gin:
--------------------------------------------------------------------------------
1 | MipNerfModel.density_activation = @flax.nn.relu
2 | MipNerfModel.density_bias = 0.0
3 | MipNerfModel.rgb_padding = 0.0
4 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/multiblender_noipe.gin:
--------------------------------------------------------------------------------
1 | MipNerfModel.disable_integration = True
2 |
--------------------------------------------------------------------------------
/CAM_mipnerf/configs/multiblender_noloss.gin:
--------------------------------------------------------------------------------
1 | Config.disable_multiscale_loss = True
2 |
--------------------------------------------------------------------------------
/CAM_mipnerf/eval.py:
--------------------------------------------------------------------------------
1 | """Evaluation script for mip-NeRF."""
2 | import functools
3 | from os import path
4 |
5 | from absl import app
6 | from absl import flags
7 | import flax
8 | from flax.metrics import tensorboard
9 | from flax.training import checkpoints
10 | import jax
11 | from jax import random
12 | import numpy as np
13 |
14 | from internal import datasets
15 | from internal import math
16 | from internal import models
17 | from internal import utils
18 | from internal import vis
19 |
20 | FLAGS = flags.FLAGS
21 | utils.define_common_flags()
22 | flags.DEFINE_bool(
23 | 'eval_once', True,
24 | 'If True, evaluate the model only once, otherwise keeping evaluating new'
25 | 'checkpoints if any exist.')
26 | flags.DEFINE_bool('save_output', True,
27 | 'If True, save predicted images to disk.')
28 |
29 |
30 | def main(unused_argv):
31 | config = utils.load_config()
32 |
33 | dataset = datasets.get_dataset('test', FLAGS.data_dir, config)
34 | model, init_variables = models.construct_mipnerf(
35 | random.PRNGKey(20200823), dataset.peek())
36 | optimizer = flax.optim.Adam(config.lr_init).create(init_variables)
37 | state = utils.TrainState(optimizer=optimizer)
38 | del optimizer, init_variables
39 |
40 | # Rendering is forced to be deterministic even if training was randomized, as
41 | # this eliminates 'speckle' artifacts.
42 | def render_eval_fn(variables, _, rays):
43 | return jax.lax.all_gather(
44 | model.apply(
45 | variables,
46 | random.PRNGKey(0), # Unused.
47 | rays,
48 | randomized=False,
49 | white_bkgd=config.white_bkgd),
50 | axis_name='batch')
51 |
52 | # pmap over only the data input.
53 | render_eval_pfn = jax.pmap(
54 | render_eval_fn,
55 | in_axes=(None, None, 0),
56 | donate_argnums=2,
57 | axis_name='batch',
58 | )
59 |
60 | ssim_fn = jax.jit(functools.partial(math.compute_ssim, max_val=1.))
61 |
62 | last_step = 0
63 | out_dir = path.join(FLAGS.train_dir,
64 | 'path_renders' if config.render_path else 'test_preds')
65 | if not FLAGS.eval_once:
66 | summary_writer = tensorboard.SummaryWriter(
67 | path.join(FLAGS.train_dir, 'eval'))
68 | while True:
69 | state = checkpoints.restore_checkpoint(FLAGS.train_dir, state)
70 | step = int(state.optimizer.state.step)
71 | if step <= last_step:
72 | continue
73 | if FLAGS.save_output and (not utils.isdir(out_dir)):
74 | utils.makedirs(out_dir)
75 | psnr_values = []
76 | ssim_values = []
77 | avg_values = []
78 | if not FLAGS.eval_once:
79 | showcase_index = random.randint(random.PRNGKey(step), (), 0, dataset.size)
80 | for idx in range(dataset.size):
81 | print(f'Evaluating {idx+1}/{dataset.size}')
82 | batch = next(dataset)
83 | pred_color, pred_distance, pred_acc = models.render_image(
84 | functools.partial(render_eval_pfn, state.optimizer.target),
85 | batch['rays'],
86 | None,
87 | chunk=FLAGS.chunk)
88 |
89 | vis_suite = vis.visualize_suite(pred_distance, pred_acc)
90 |
91 | if jax.host_id() != 0: # Only record via host 0.
92 | continue
93 | if not FLAGS.eval_once and idx == showcase_index:
94 | showcase_color = pred_color
95 | showcase_acc = pred_acc
96 | showcase_vis_suite = vis_suite
97 | if not config.render_path:
98 | showcase_gt = batch['pixels']
99 | if not config.render_path:
100 | psnr = float(
101 | math.mse_to_psnr(((pred_color - batch['pixels'])**2).mean()))
102 | ssim = float(ssim_fn(pred_color, batch['pixels']))
103 | print(f'PSNR={psnr:.4f} SSIM={ssim:.4f}')
104 | psnr_values.append(psnr)
105 | ssim_values.append(ssim)
106 | if FLAGS.save_output and (config.test_render_interval > 0):
107 | if (idx % config.test_render_interval) == 0:
108 | utils.save_img_uint8(
109 | pred_color, path.join(out_dir, 'color_{:03d}.png'.format(idx)))
110 | utils.save_img_float32(
111 | pred_distance,
112 | path.join(out_dir, 'distance_{:03d}.tiff'.format(idx)))
113 | utils.save_img_float32(
114 | pred_acc, path.join(out_dir, 'acc_{:03d}.tiff'.format(idx)))
115 | for k, v in vis_suite.items():
116 | utils.save_img_uint8(
117 | v, path.join(out_dir, k + '_{:03d}.png'.format(idx)))
118 | if (not FLAGS.eval_once) and (jax.host_id() == 0):
119 | summary_writer.image('pred_color', showcase_color, step)
120 | summary_writer.image('pred_acc', showcase_acc, step)
121 | for k, v in showcase_vis_suite.items():
122 | summary_writer.image('pred_' + k, v, step)
123 | if not config.render_path:
124 | summary_writer.scalar('psnr', np.mean(np.array(psnr_values)), step)
125 | summary_writer.scalar('ssim', np.mean(np.array(ssim_values)), step)
126 | summary_writer.image('target', showcase_gt, step)
127 | if FLAGS.save_output and (not config.render_path) and (jax.host_id() == 0):
128 | with utils.open_file(path.join(out_dir, f'psnrs_{step}.txt'), 'w') as f:
129 | f.write(' '.join([str(v) for v in psnr_values]))
130 | f.write('\n'+str(np.mean(np.array(psnr_values))))
131 | with utils.open_file(path.join(out_dir, f'ssims_{step}.txt'), 'w') as f:
132 | f.write(' '.join([str(v) for v in ssim_values]))
133 | f.write('\n'+str(np.mean(np.array(ssim_values))))
134 | if FLAGS.eval_once:
135 | break
136 | if int(step) >= config.max_steps:
137 | break
138 | last_step = step
139 |
140 |
141 | if __name__ == '__main__':
142 | app.run(main)
143 |
--------------------------------------------------------------------------------
/CAM_mipnerf/internal/vis.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | # Lint as: python3
16 | """Helper functions for visualizing things."""
17 | import jax
18 | import jax.numpy as jnp
19 | import jax.scipy as jsp
20 | import matplotlib.cm as cm
21 |
22 |
23 | def sinebow(h):
24 | """A cyclic and uniform colormap, see http://basecase.org/env/on-rainbows."""
25 | f = lambda x: jnp.sin(jnp.pi * x)**2
26 | return jnp.stack([f(3 / 6 - h), f(5 / 6 - h), f(7 / 6 - h)], -1)
27 |
28 |
29 | def convolve2d(z, f):
30 | return jsp.signal.convolve2d(
31 | z, f, mode='same', precision=jax.lax.Precision.HIGHEST)
32 |
33 |
34 | def depth_to_normals(depth):
35 | """Assuming `depth` is orthographic, linearize it to a set of normals."""
36 | f_blur = jnp.array([1, 2, 1]) / 4
37 | f_edge = jnp.array([-1, 0, 1]) / 2
38 | dy = convolve2d(depth, f_blur[None, :] * f_edge[:, None])
39 | dx = convolve2d(depth, f_blur[:, None] * f_edge[None, :])
40 | inv_denom = 1 / jnp.sqrt(1 + dx**2 + dy**2)
41 | normals = jnp.stack([dx * inv_denom, dy * inv_denom, inv_denom], -1)
42 | return normals
43 |
44 |
45 | def visualize_depth(depth,
46 | acc=None,
47 | near=None,
48 | far=None,
49 | ignore_frac=0,
50 | curve_fn=lambda x: -jnp.log(x + jnp.finfo(jnp.float32).eps),
51 | modulus=0,
52 | colormap=None):
53 | """Visualize a depth map.
54 |
55 | Args:
56 | depth: A depth map.
57 | acc: An accumulation map, in [0, 1].
58 | near: The depth of the near plane, if None then just use the min().
59 | far: The depth of the far plane, if None then just use the max().
60 | ignore_frac: What fraction of the depth map to ignore when automatically
61 | generating `near` and `far`. Depends on `acc` as well as `depth'.
62 | curve_fn: A curve function that gets applied to `depth`, `near`, and `far`
63 | before the rest of visualization. Good choices: x, 1/(x+eps), log(x+eps).
64 | Note that the default choice will flip the sign of depths, so that the
65 | default colormap (turbo) renders "near" as red and "far" as blue.
66 | modulus: If > 0, mod the normalized depth by `modulus`. Use (0, 1].
67 | colormap: A colormap function. If None (default), will be set to
68 | matplotlib's turbo if modulus==0, sinebow otherwise.
69 |
70 | Returns:
71 | An RGB visualization of `depth`.
72 | """
73 | if acc is None:
74 | acc = jnp.ones_like(depth)
75 | acc = jnp.where(jnp.isnan(depth), jnp.zeros_like(acc), acc)
76 |
77 | # Sort `depth` and `acc` according to `depth`, then identify the depth values
78 | # that span the middle of `acc`, ignoring `ignore_frac` fraction of `acc`.
79 | sortidx = jnp.argsort(depth.reshape([-1]))
80 | depth_sorted = depth.reshape([-1])[sortidx]
81 | acc_sorted = acc.reshape([-1])[sortidx]
82 | cum_acc_sorted = jnp.cumsum(acc_sorted)
83 | mask = ((cum_acc_sorted >= cum_acc_sorted[-1] * ignore_frac) &
84 | (cum_acc_sorted <= cum_acc_sorted[-1] * (1 - ignore_frac)))
85 | depth_keep = depth_sorted[mask]
86 |
87 | # If `near` or `far` are None, use the highest and lowest non-NaN values in
88 | # `depth_keep` as automatic near/far planes.
89 | eps = jnp.finfo(jnp.float32).eps
90 | near = near or depth_keep[0] - eps
91 | far = far or depth_keep[-1] + eps
92 |
93 | # Curve all values.
94 | depth, near, far = [curve_fn(x) for x in [depth, near, far]]
95 |
96 | # Wrap the values around if requested.
97 | if modulus > 0:
98 | value = jnp.mod(depth, modulus) / modulus
99 | colormap = colormap or sinebow
100 | else:
101 | # Scale to [0, 1].
102 | value = jnp.nan_to_num(
103 | jnp.clip((depth - jnp.minimum(near, far)) / jnp.abs(far - near), 0, 1))
104 | colormap = colormap or cm.get_cmap('turbo')
105 |
106 | vis = colormap(value)[:, :, :3]
107 |
108 | # Set non-accumulated pixels to white.
109 | vis = vis * acc[:, :, None] + (1 - acc)[:, :, None]
110 |
111 | return vis
112 |
113 |
114 | def visualize_normals(depth, acc, scaling=None):
115 | """Visualize fake normals of `depth` (optionally scaled to be isotropic)."""
116 | if scaling is None:
117 | mask = ~jnp.isnan(depth)
118 | x, y = jnp.meshgrid(
119 | jnp.arange(depth.shape[1]), jnp.arange(depth.shape[0]), indexing='xy')
120 | xy_var = (jnp.var(x[mask]) + jnp.var(y[mask])) / 2
121 | z_var = jnp.var(depth[mask])
122 | scaling = jnp.sqrt(xy_var / z_var)
123 |
124 | scaled_depth = scaling * depth
125 | normals = depth_to_normals(scaled_depth)
126 | vis = jnp.isnan(normals) + jnp.nan_to_num((normals + 1) / 2, 0)
127 |
128 | # Set non-accumulated pixels to white.
129 | if acc is not None:
130 | vis = vis * acc[:, :, None] + (1 - acc)[:, :, None]
131 |
132 | return vis
133 |
134 |
135 | def visualize_suite(depth, acc):
136 | """A wrapper around other visualizations for easy integration."""
137 | vis = {
138 | 'depth': visualize_depth(depth, acc),
139 | 'depth_mod': visualize_depth(depth, acc, modulus=0.1),
140 | 'depth_normals': visualize_normals(depth, acc)
141 | }
142 | return vis
143 |
--------------------------------------------------------------------------------
/CAM_mipnerf/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy>=1.16.4,<1.19.0
2 | jax>=0.2.12
3 | jaxlib>=0.1.65
4 | flax>=0.2.2
5 | opencv-python>=4.4.0
6 | Pillow>=7.2.0
7 | tensorboard>=2.4.0
8 | tensorflow>=2.3.1
9 | gin-config
10 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/convert_blender_data.py:
--------------------------------------------------------------------------------
1 | import json
2 | import os
3 | from os import path
4 |
5 | from absl import app
6 | from absl import flags
7 | import jax
8 | import numpy as np
9 | from PIL import Image
10 |
11 | FLAGS = flags.FLAGS
12 |
13 | flags.DEFINE_string('blenderdir', None,
14 | 'Base directory for all Blender data.')
15 | flags.DEFINE_string('outdir', None,
16 | 'Where to save multiscale data.')
17 | flags.DEFINE_integer('n_down', 4,
18 | 'How many levels of downscaling to use.')
19 |
20 | jax.config.parse_flags_with_absl()
21 |
22 |
23 | def load_renderings(data_dir, split):
24 | """Load images and metadata from disk."""
25 | f = 'transforms_{}.json'.format(split)
26 | with open(path.join(data_dir, f), 'r') as fp:
27 | meta = json.load(fp)
28 | images = []
29 | cams = []
30 | print('Loading imgs')
31 | for frame in meta['frames']:
32 | fname = os.path.join(data_dir, frame['file_path'] + '.png')
33 | with open(fname, 'rb') as imgin:
34 | image = np.array(Image.open(imgin), dtype=np.float32) / 255.
35 | cams.append(frame['transform_matrix'])
36 | images.append(image)
37 | ret = {}
38 | ret['images'] = np.stack(images, axis=0)
39 | print('Loaded all images, shape is', ret['images'].shape)
40 | ret['camtoworlds'] = np.stack(cams, axis=0)
41 | w = ret['images'].shape[2]
42 | camera_angle_x = float(meta['camera_angle_x'])
43 | ret['focal'] = .5 * w / np.tan(.5 * camera_angle_x)
44 | return ret
45 |
46 |
47 | def down2(img):
48 | sh = img.shape
49 | return np.mean(np.reshape(img, [sh[0] // 2, 2, sh[1] // 2, 2, -1]), (1, 3))
50 |
51 |
52 | def convert_to_nerfdata(basedir, newdir, n_down):
53 | """Convert Blender data to multiscale."""
54 | if not os.path.exists(newdir):
55 | os.makedirs(newdir)
56 | splits = ['train', 'val', 'test']
57 | bigmeta = {}
58 | # Foreach split in the dataset
59 | for split in splits:
60 | print('Split', split)
61 | # Load everything
62 | data = load_renderings(basedir, split)
63 |
64 | # Save out all the images
65 | imgdir = 'images_{}'.format(split)
66 | os.makedirs(os.path.join(newdir, imgdir), exist_ok=True)
67 | fnames = []
68 | widths = []
69 | heights = []
70 | focals = []
71 | cam2worlds = []
72 | lossmults = []
73 | labels = []
74 | nears, fars = [], []
75 | f = data['focal']
76 | print('Saving images')
77 | for i, img in enumerate(data['images']):
78 | for j in range(n_down):
79 | fname = '{}/{:03d}_d{}.png'.format(imgdir, i, j)
80 | fnames.append(fname)
81 | fname = os.path.join(newdir, fname)
82 | with open(fname, 'wb') as imgout:
83 | img8 = Image.fromarray(np.uint8(img * 255))
84 | img8.save(imgout)
85 | widths.append(img.shape[1])
86 | heights.append(img.shape[0])
87 | focals.append(f / 2**j)
88 | cam2worlds.append(data['camtoworlds'][i].tolist())
89 | lossmults.append(4.**j)
90 | labels.append(j)
91 | nears.append(2.)
92 | fars.append(6.)
93 | img = down2(img)
94 |
95 | # Create metadata
96 | meta = {}
97 | meta['file_path'] = fnames
98 | meta['cam2world'] = cam2worlds
99 | meta['width'] = widths
100 | meta['height'] = heights
101 | meta['focal'] = focals
102 | meta['label'] = labels
103 | meta['near'] = nears
104 | meta['far'] = fars
105 | meta['lossmult'] = lossmults
106 |
107 | fx = np.array(focals)
108 | fy = np.array(focals)
109 | cx = np.array(meta['width']) * .5
110 | cy = np.array(meta['height']) * .5
111 | arr0 = np.zeros_like(cx)
112 | arr1 = np.ones_like(cx)
113 | k_inv = np.array([
114 | [arr1 / fx, arr0, -cx / fx],
115 | [arr0, -arr1 / fy, cy / fy],
116 | [arr0, arr0, -arr1],
117 | ])
118 | k_inv = np.moveaxis(k_inv, -1, 0)
119 | meta['pix2cam'] = k_inv.tolist()
120 |
121 | bigmeta[split] = meta
122 |
123 | for k in bigmeta:
124 | for j in bigmeta[k]:
125 | print(k, j, type(bigmeta[k][j]), np.array(bigmeta[k][j]).shape)
126 |
127 | jsonfile = os.path.join(newdir, 'metadata.json')
128 | with open(jsonfile, 'w') as f:
129 | json.dump(bigmeta, f, ensure_ascii=False, indent=4)
130 |
131 |
132 | def main(unused_argv):
133 |
134 | blenderdir = FLAGS.blenderdir
135 | outdir = FLAGS.outdir
136 | n_down = FLAGS.n_down
137 | if not os.path.exists(outdir):
138 | os.makedirs(outdir)
139 |
140 | dirs = [os.path.join(blenderdir, f) for f in os.listdir(blenderdir)]
141 | dirs = [d for d in dirs if os.path.isdir(d)]
142 | print(dirs)
143 | for basedir in dirs:
144 | print()
145 | newdir = os.path.join(outdir, os.path.basename(basedir))
146 | print('Converting from', basedir, 'to', newdir)
147 | convert_to_nerfdata(basedir, newdir, n_down)
148 |
149 |
150 | if __name__ == '__main__':
151 | app.run(main)
152 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/eval_blender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Script for evaluating on the Blender dataset.
17 |
18 | SCENE=lego
19 | EXPERIMENT=debug
20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE
21 | DATA_DIR=/Users/barron/data/nerf_synthetic/$SCENE
22 |
23 | python -m eval \
24 | --data_dir=$DATA_DIR \
25 | --train_dir=$TRAIN_DIR \
26 | --chunk=3076 \
27 | --gin_file=configs/blender.gin \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/eval_llff.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Script for evaluating on the LLFF dataset.
17 |
18 | SCENE=trex
19 | EXPERIMENT=debug
20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE
21 | DATA_DIR=/Users/barron/data/nerf_llff_data/$SCENE
22 |
23 | python -m eval \
24 | --data_dir=$DATA_DIR \
25 | --train_dir=$TRAIN_DIR \
26 | --chunk=3076 \
27 | --gin_file=configs/llff.gin \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/eval_multiblender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Script for evaluating on the multiscale Blender dataset.
17 |
18 | SCENE=lego
19 | EXPERIMENT=debug
20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE
21 | DATA_DIR=/Users/barron/data/down4/$SCENE
22 |
23 | python -m eval \
24 | --data_dir=$DATA_DIR \
25 | --train_dir=$TRAIN_DIR \
26 | --chunk=3076 \
27 | --gin_file=configs/multiblender.gin \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/train_blender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Script for training on the Blender dataset.
17 |
18 | SCENE=lego
19 | EXPERIMENT=debug
20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE
21 | DATA_DIR=/Users/barron/data/nerf_synthetic/$SCENE
22 |
23 | rm $TRAIN_DIR/*
24 | python -m train \
25 | --data_dir=$DATA_DIR \
26 | --train_dir=$TRAIN_DIR \
27 | --gin_file=configs/blender.gin \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/train_llff.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Script for training on the LLFF dataset.
17 |
18 | SCENE=trex
19 | EXPERIMENT=debug
20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE
21 | DATA_DIR=/Users/barron/data/nerf_llff_data/$SCENE
22 |
23 | rm $TRAIN_DIR/*
24 | python -m train \
25 | --data_dir=$DATA_DIR \
26 | --train_dir=$TRAIN_DIR \
27 | --gin_file=configs/llff.gin \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_mipnerf/scripts/train_multiblender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # Script for training on the multiscale Blender dataset.
17 |
18 | SCENE=lego
19 | EXPERIMENT=debug
20 | TRAIN_DIR=/Users/barron/tmp/nerf_results/$EXPERIMENT/$SCENE
21 | DATA_DIR=/Users/barron/data/down4/$SCENE
22 |
23 | rm $TRAIN_DIR/*
24 | python -m train \
25 | --data_dir=$DATA_DIR \
26 | --train_dir=$TRAIN_DIR \
27 | --gin_file=configs/multiblender.gin \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/.gitignore:
--------------------------------------------------------------------------------
1 | internal/pycolmap
2 | __pycache__/
3 | interal/__pycache__/
4 | tests/__pycache__/
5 | .DS_Store
6 | .vscode/
7 | .idea/
8 | __MACOSX/
9 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement (CLA). You (or your employer) retain the copyright to your
10 | contribution; this simply gives us permission to use and redistribute your
11 | contributions as part of the project. Head over to
12 | to see your current agreements on file or
13 | to sign a new one.
14 |
15 | You generally only need to submit a CLA once, so if you've already submitted one
16 | (even if it was for a different project), you probably don't need to do it
17 | again.
18 |
19 | ## Code Reviews
20 |
21 | All submissions, including submissions by project members, require review. We
22 | use GitHub pull requests for this purpose. Consult
23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
24 | information on using pull requests.
25 |
26 | ## Community Guidelines
27 |
28 | This project follows
29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/).
30 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/.ipynb_checkpoints/360-checkpoint.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.near = 0.2
3 | Config.far = 1e6
4 | Config.factor = 4
5 |
6 | Model.raydist_fn = @jnp.reciprocal
7 | Model.opaque_background = True
8 |
9 | PropMLP.warp_fn = @coord.contract
10 | PropMLP.net_depth = 4
11 | PropMLP.net_width = 256
12 | PropMLP.disable_density_normals = True
13 | PropMLP.disable_rgb = True
14 |
15 | NerfMLP.warp_fn = @coord.contract
16 | NerfMLP.net_depth = 8
17 | NerfMLP.net_width = 1024
18 | NerfMLP.disable_density_normals = True
19 |
20 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/.ipynb_checkpoints/360_glo4-checkpoint.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.near = 0.2
3 | Config.far = 1e6
4 | Config.factor = 4
5 |
6 | Model.raydist_fn = @jnp.reciprocal
7 | Model.num_glo_features = 4
8 | Model.opaque_background = True
9 |
10 | PropMLP.warp_fn = @coord.contract
11 | PropMLP.net_depth = 4
12 | PropMLP.net_width = 256
13 | PropMLP.disable_density_normals = True
14 | PropMLP.disable_rgb = True
15 |
16 | NerfMLP.warp_fn = @coord.contract
17 | NerfMLP.net_depth = 8
18 | NerfMLP.net_width = 1024
19 | NerfMLP.disable_density_normals = True
20 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/.ipynb_checkpoints/render_config-checkpoint.gin:
--------------------------------------------------------------------------------
1 | Config.render_path = True
2 | Config.render_path_frames = 480
3 | Config.render_video_fps = 60
4 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/360.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.near = 0.2
3 | Config.far = 1e6
4 | Config.factor = 4
5 |
6 | Model.raydist_fn = @jnp.reciprocal
7 | Model.opaque_background = True
8 |
9 | PropMLP.warp_fn = @coord.contract
10 | PropMLP.net_depth = 4
11 | PropMLP.net_width = 256
12 | PropMLP.disable_density_normals = True
13 | PropMLP.disable_rgb = True
14 |
15 | NerfMLP.warp_fn = @coord.contract
16 | NerfMLP.net_depth = 8
17 | NerfMLP.net_width = 1024
18 | NerfMLP.disable_density_normals = True
19 |
20 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/360_glo4.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.near = 0.2
3 | Config.far = 1e6
4 | Config.factor = 4
5 |
6 | Model.raydist_fn = @jnp.reciprocal
7 | Model.num_glo_features = 4
8 | Model.opaque_background = True
9 |
10 | PropMLP.warp_fn = @coord.contract
11 | PropMLP.net_depth = 4
12 | PropMLP.net_width = 256
13 | PropMLP.disable_density_normals = True
14 | PropMLP.disable_rgb = True
15 |
16 | NerfMLP.warp_fn = @coord.contract
17 | NerfMLP.net_depth = 8
18 | NerfMLP.net_width = 1024
19 | NerfMLP.disable_density_normals = True
20 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/blender_256.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'blender'
2 | Config.batching = 'single_image'
3 | Config.near = 2
4 | Config.far = 6
5 | Config.eval_render_interval = 5
6 | Config.data_loss_type = 'mse'
7 | Config.adam_eps = 1e-8
8 |
9 | Model.num_levels = 2
10 | Model.num_prop_samples = 128
11 | Model.num_nerf_samples = 32
12 |
13 | PropMLP.net_depth = 4
14 | PropMLP.net_width = 256
15 | PropMLP.basis_shape = 'octahedron'
16 | PropMLP.basis_subdivisions = 1
17 | PropMLP.disable_density_normals = True
18 | PropMLP.disable_rgb = True
19 |
20 | NerfMLP.net_depth = 8
21 | NerfMLP.net_width = 256
22 | NerfMLP.basis_shape = 'octahedron'
23 | NerfMLP.basis_subdivisions = 1
24 | NerfMLP.disable_density_normals = True
25 |
26 | Config.distortion_loss_mult = 0.
27 |
28 | NerfMLP.max_deg_point = 16
29 | PropMLP.max_deg_point = 16
30 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/blender_512.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'blender'
2 | Config.batching = 'single_image'
3 | Config.near = 2
4 | Config.far = 6
5 | Config.eval_render_interval = 5
6 | Config.data_loss_type = 'mse'
7 | Config.adam_eps = 1e-8
8 |
9 | Model.num_levels = 2
10 | Model.num_prop_samples = 128
11 | Model.num_nerf_samples = 32
12 |
13 | PropMLP.net_depth = 4
14 | PropMLP.net_width = 256
15 | PropMLP.disable_density_normals = True
16 | PropMLP.disable_rgb = True
17 |
18 | NerfMLP.net_depth = 8
19 | NerfMLP.net_width = 512
20 | NerfMLP.disable_density_normals = True
21 |
22 | Config.distortion_loss_mult = 0.
23 |
24 | NerfMLP.max_deg_point = 16
25 | PropMLP.max_deg_point = 16
26 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/blender_refnerf.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'blender'
2 | Config.batching = 'single_image'
3 | Config.near = 2
4 | Config.far = 6
5 | Config.eval_render_interval = 5
6 | Config.compute_normal_metrics = True
7 | Config.data_loss_type = 'mse'
8 | Config.distortion_loss_mult = 0.0
9 | Config.orientation_loss_mult = 0.1
10 | Config.orientation_loss_target = 'normals_pred'
11 | Config.predicted_normal_loss_mult = 3e-4
12 | Config.orientation_coarse_loss_mult = 0.01
13 | Config.predicted_normal_coarse_loss_mult = 3e-5
14 | Config.interlevel_loss_mult = 0.0
15 | Config.data_coarse_loss_mult = 0.1
16 | Config.adam_eps = 1e-8
17 |
18 | Model.num_levels = 2
19 | Model.single_mlp = True
20 | Model.num_prop_samples = 128 # This needs to be set despite single_mlp = True.
21 | Model.num_nerf_samples = 128
22 | Model.anneal_slope = 0.
23 | Model.dilation_multiplier = 0.
24 | Model.dilation_bias = 0.
25 | Model.single_jitter = False
26 | Model.resample_padding = 0.01
27 |
28 | NerfMLP.net_depth = 8
29 | NerfMLP.net_width = 256
30 | NerfMLP.net_depth_viewdirs = 8
31 | NerfMLP.basis_shape = 'octahedron'
32 | NerfMLP.basis_subdivisions = 1
33 | NerfMLP.disable_density_normals = False
34 | NerfMLP.enable_pred_normals = True
35 | NerfMLP.use_directional_enc = True
36 | NerfMLP.use_reflections = True
37 | NerfMLP.deg_view = 5
38 | NerfMLP.enable_pred_roughness = True
39 | NerfMLP.use_diffuse_color = True
40 | NerfMLP.use_specular_tint = True
41 | NerfMLP.use_n_dot_v = True
42 | NerfMLP.bottleneck_width = 128
43 | NerfMLP.density_bias = 0.5
44 | NerfMLP.max_deg_point = 16
45 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/debug.gin:
--------------------------------------------------------------------------------
1 | # A short training schedule with no "warm up", useful for debugging.
2 | Config.checkpoint_every = 1000
3 | Config.print_every = 100
4 | Config.train_render_every = 1000
5 | Config.lr_delay_mult = 0.1
6 | Config.lr_delay_steps = 500
7 | Config.batch_size = 2048
8 | Config.render_chunk_size = 2048
9 | Config.lr_init = 5e-4
10 | Config.lr_final = 5e-6
11 | Config.factor = 4
12 | Config.early_exit_steps = 3000
13 |
14 | PropMLP.net_depth = 2
15 | PropMLP.net_width = 64
16 |
17 | NerfMLP.net_depth = 4
18 | NerfMLP.net_width = 128
19 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/llff_256.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.near = 0.
3 | Config.far = 1.
4 | Config.factor = 4
5 | Config.forward_facing = True
6 | Config.adam_eps = 1e-8
7 |
8 | Model.ray_shape = 'cylinder'
9 | Model.opaque_background = True
10 | Model.num_levels = 2
11 | Model.num_prop_samples = 128
12 | Model.num_nerf_samples = 32
13 |
14 | PropMLP.net_depth = 4
15 | PropMLP.net_width = 256
16 | PropMLP.basis_shape = 'octahedron'
17 | PropMLP.basis_subdivisions = 1
18 | PropMLP.disable_density_normals = True
19 | PropMLP.disable_rgb = True
20 |
21 | NerfMLP.net_depth = 8
22 | NerfMLP.net_width = 256
23 | NerfMLP.basis_shape = 'octahedron'
24 | NerfMLP.basis_subdivisions = 1
25 | NerfMLP.disable_density_normals = True
26 |
27 | NerfMLP.max_deg_point = 16
28 | PropMLP.max_deg_point = 16
29 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/llff_512.gin:
--------------------------------------------------------------------------------
1 | Config.dataset_loader = 'llff'
2 | Config.near = 0.
3 | Config.far = 1.
4 | Config.factor = 4
5 | Config.forward_facing = True
6 | Config.adam_eps = 1e-8
7 |
8 | Model.ray_shape = 'cylinder'
9 | Model.opaque_background = True
10 | Model.num_levels = 2
11 | Model.num_prop_samples = 128
12 | Model.num_nerf_samples = 32
13 |
14 | PropMLP.net_depth = 4
15 | PropMLP.net_width = 256
16 | PropMLP.disable_density_normals = True
17 | PropMLP.disable_rgb = True
18 |
19 | NerfMLP.net_depth = 8
20 | NerfMLP.net_width = 512
21 | NerfMLP.disable_density_normals = True
22 |
23 | NerfMLP.max_deg_point = 16
24 | PropMLP.max_deg_point = 16
25 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/llff_raw.gin:
--------------------------------------------------------------------------------
1 | # General LLFF settings
2 |
3 | Config.dataset_loader = 'llff'
4 | Config.near = 0.
5 | Config.far = 1.
6 | Config.factor = 4
7 | Config.forward_facing = True
8 |
9 | Model.ray_shape = 'cylinder'
10 |
11 | PropMLP.net_depth = 4
12 | PropMLP.net_width = 256
13 | PropMLP.basis_shape = 'octahedron'
14 | PropMLP.basis_subdivisions = 1
15 | PropMLP.disable_density_normals = True # Turn this off if using orientation loss.
16 | PropMLP.disable_rgb = True
17 |
18 | NerfMLP.net_depth = 8
19 | NerfMLP.net_width = 256
20 | NerfMLP.basis_shape = 'octahedron'
21 | NerfMLP.basis_subdivisions = 1
22 | NerfMLP.disable_density_normals = True # Turn this off if using orientation loss.
23 |
24 | NerfMLP.max_deg_point = 16
25 | PropMLP.max_deg_point = 16
26 |
27 | Config.train_render_every = 5000
28 |
29 |
30 | ########################## RawNeRF specific settings ##########################
31 |
32 | Config.rawnerf_mode = True
33 | Config.data_loss_type = 'rawnerf'
34 | Config.apply_bayer_mask = True
35 | Model.learned_exposure_scaling = True
36 |
37 | Model.num_levels = 2
38 | Model.num_prop_samples = 128 # Using extra samples for now because of noise instability.
39 | Model.num_nerf_samples = 128
40 | Model.opaque_background = True
41 |
42 | # RGB activation we use for linear color outputs is exp(x - 5).
43 | NerfMLP.rgb_padding = 0.
44 | NerfMLP.rgb_activation = @math.safe_exp
45 | NerfMLP.rgb_bias = -5.
46 | PropMLP.rgb_padding = 0.
47 | PropMLP.rgb_activation = @math.safe_exp
48 | PropMLP.rgb_bias = -5.
49 |
50 | ## Experimenting with the various regularizers and losses:
51 | Config.interlevel_loss_mult = .0 # Turning off interlevel for now (default = 1.).
52 | Config.distortion_loss_mult = .01 # Distortion loss helps with floaters (default = .01).
53 | Config.orientation_loss_mult = 0. # Orientation loss also not great (try .01).
54 | Config.data_coarse_loss_mult = 0.1 # Setting this to match old MipNeRF.
55 |
56 | ## Density noise used in original NeRF:
57 | NerfMLP.density_noise = 1.
58 | PropMLP.density_noise = 1.
59 |
60 | ## Use a single MLP for all rounds of sampling:
61 | Model.single_mlp = True
62 |
63 | ## Some algorithmic settings to match the paper:
64 | Model.anneal_slope = 0.
65 | Model.dilation_multiplier = 0.
66 | Model.dilation_bias = 0.
67 | Model.single_jitter = False
68 | NerfMLP.weight_init = 'glorot_uniform'
69 | PropMLP.weight_init = 'glorot_uniform'
70 |
71 | ## Training hyperparameters used in the paper:
72 | Config.batch_size = 16384
73 | Config.render_chunk_size = 16384
74 | Config.lr_init = 1e-3
75 | Config.lr_final = 1e-5
76 | Config.max_steps = 500000
77 | Config.checkpoint_every = 25000
78 | Config.lr_delay_steps = 2500
79 | Config.lr_delay_mult = 0.01
80 | Config.grad_max_norm = 0.1
81 | Config.grad_max_val = 0.1
82 | Config.adam_eps = 1e-8
83 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/llff_raw_test.gin:
--------------------------------------------------------------------------------
1 | include 'experimental/users/barron/mipnerf360/configs/llff_raw.gin'
2 |
3 | Config.factor = 0
4 | Config.eval_raw_affine_cc = True
5 | Config.eval_crop_borders = 16
6 | Config.vis_decimate = 4
7 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/render_config.gin:
--------------------------------------------------------------------------------
1 | Config.render_path = True
2 | Config.render_path_frames = 480
3 | Config.render_video_fps = 60
4 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/configs/tat.gin:
--------------------------------------------------------------------------------
1 | # This config is meant to be run while overriding a 360*.gin config.
2 |
3 | Config.dataset_loader = 'tat_nerfpp'
4 | Config.near = 0.1
5 | Config.far = 1e6
6 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/internal/coord.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | """Tools for manipulating coordinate spaces and distances along rays."""
15 |
16 | from internal import math
17 | import jax
18 | import jax.numpy as jnp
19 |
20 |
21 | def contract(x):
22 | """Contracts points towards the origin (Eq 10 of arxiv.org/abs/2111.12077)."""
23 | eps = jnp.finfo(jnp.float32).eps
24 | # Clamping to eps prevents non-finite gradients when x == 0.
25 | x_mag_sq = jnp.maximum(eps, jnp.sum(x**2, axis=-1, keepdims=True))
26 | z = jnp.where(x_mag_sq <= 1, x, ((2 * jnp.sqrt(x_mag_sq) - 1) / x_mag_sq) * x)
27 | return z
28 |
29 |
30 | def inv_contract(z):
31 | """The inverse of contract()."""
32 | eps = jnp.finfo(jnp.float32).eps
33 | # Clamping to eps prevents non-finite gradients when z == 0.
34 | z_mag_sq = jnp.maximum(eps, jnp.sum(z**2, axis=-1, keepdims=True))
35 | x = jnp.where(z_mag_sq <= 1, z, z / (2 * jnp.sqrt(z_mag_sq) - z_mag_sq))
36 | return x
37 |
38 |
39 | def track_linearize(fn, mean, cov):
40 | """Apply function `fn` to a set of means and covariances, ala a Kalman filter.
41 |
42 | We can analytically transform a Gaussian parameterized by `mean` and `cov`
43 | with a function `fn` by linearizing `fn` around `mean`, and taking advantage
44 | of the fact that Covar[Ax + y] = A(Covar[x])A^T (see
45 | https://cs.nyu.edu/~roweis/notes/gaussid.pdf for details).
46 |
47 | Args:
48 | fn: the function applied to the Gaussians parameterized by (mean, cov).
49 | mean: a tensor of means, where the last axis is the dimension.
50 | cov: a tensor of covariances, where the last two axes are the dimensions.
51 |
52 | Returns:
53 | fn_mean: the transformed means.
54 | fn_cov: the transformed covariances.
55 | """
56 | if (len(mean.shape) + 1) != len(cov.shape):
57 | raise ValueError('cov must be non-diagonal')
58 | fn_mean, lin_fn = jax.linearize(fn, mean)
59 | fn_cov = jax.vmap(lin_fn, -1, -2)(jax.vmap(lin_fn, -1, -2)(cov))
60 | return fn_mean, fn_cov
61 |
62 |
63 | def construct_ray_warps(fn, t_near, t_far):
64 | """Construct a bijection between metric distances and normalized distances.
65 |
66 | See the text around Equation 11 in https://arxiv.org/abs/2111.12077 for a
67 | detailed explanation.
68 |
69 | Args:
70 | fn: the function to ray distances.
71 | t_near: a tensor of near-plane distances.
72 | t_far: a tensor of far-plane distances.
73 |
74 | Returns:
75 | t_to_s: a function that maps distances to normalized distances in [0, 1].
76 | s_to_t: the inverse of t_to_s.
77 | """
78 | if fn is None:
79 | fn_fwd = lambda x: x
80 | fn_inv = lambda x: x
81 | elif fn == 'piecewise':
82 | # Piecewise spacing combining identity and 1/x functions to allow t_near=0.
83 | fn_fwd = lambda x: jnp.where(x < 1, .5 * x, 1 - .5 / x)
84 | fn_inv = lambda x: jnp.where(x < .5, 2 * x, .5 / (1 - x))
85 | else:
86 | inv_mapping = {
87 | 'reciprocal': jnp.reciprocal,
88 | 'log': jnp.exp,
89 | 'exp': jnp.log,
90 | 'sqrt': jnp.square,
91 | 'square': jnp.sqrt
92 | }
93 | fn_fwd = fn
94 | fn_inv = inv_mapping[fn.__name__]
95 |
96 | s_near, s_far = [fn_fwd(x) for x in (t_near, t_far)]
97 | t_to_s = lambda t: (fn_fwd(t) - s_near) / (s_far - s_near)
98 | s_to_t = lambda s: fn_inv(s * s_far + (1 - s) * s_near)
99 | return t_to_s, s_to_t
100 |
101 |
102 | def expected_sin(mean, var):
103 | """Compute the mean of sin(x), x ~ N(mean, var)."""
104 | return jnp.exp(-0.5 * var) * math.safe_sin(mean) # large var -> small value.
105 |
106 |
107 | def integrated_pos_enc(mean, var, min_deg, max_deg):
108 | """Encode `x` with sinusoids scaled by 2^[min_deg, max_deg).
109 |
110 | Args:
111 | mean: tensor, the mean coordinates to be encoded
112 | var: tensor, the variance of the coordinates to be encoded.
113 | min_deg: int, the min degree of the encoding.
114 | max_deg: int, the max degree of the encoding.
115 |
116 | Returns:
117 | encoded: jnp.ndarray, encoded variables.
118 | """
119 | scales = 2**jnp.arange(min_deg, max_deg)
120 | shape = mean.shape[:-1] + (-1,)
121 | scaled_mean = jnp.reshape(mean[..., None, :] * scales[:, None], shape)
122 | scaled_var = jnp.reshape(var[..., None, :] * scales[:, None]**2, shape)
123 |
124 | return expected_sin(
125 | jnp.concatenate([scaled_mean, scaled_mean + 0.5 * jnp.pi], axis=-1),
126 | jnp.concatenate([scaled_var] * 2, axis=-1))
127 |
128 |
129 | def lift_and_diagonalize(mean, cov, basis):
130 | """Project `mean` and `cov` onto basis and diagonalize the projected cov."""
131 | fn_mean = math.matmul(mean, basis)
132 | fn_cov_diag = jnp.sum(basis * math.matmul(cov, basis), axis=-2)
133 | return fn_mean, fn_cov_diag
134 |
135 |
136 | def pos_enc(x, min_deg, max_deg, append_identity=True):
137 | """The positional encoding used by the original NeRF paper."""
138 | scales = 2**jnp.arange(min_deg, max_deg)
139 | shape = x.shape[:-1] + (-1,)
140 | scaled_x = jnp.reshape((x[..., None, :] * scales[:, None]), shape)
141 | # Note that we're not using safe_sin, unlike IPE.
142 | four_feat = jnp.sin(
143 | jnp.concatenate([scaled_x, scaled_x + 0.5 * jnp.pi], axis=-1))
144 | if append_identity:
145 | return jnp.concatenate([x] + [four_feat], axis=-1)
146 | else:
147 | return four_feat
148 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/internal/geopoly.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Tools for constructing geodesic polyhedron, which are used as a basis."""
16 |
17 | import itertools
18 | import numpy as np
19 |
20 |
21 | def compute_sq_dist(mat0, mat1=None):
22 | """Compute the squared Euclidean distance between all pairs of columns."""
23 | if mat1 is None:
24 | mat1 = mat0
25 | # Use the fact that ||x - y||^2 == ||x||^2 + ||y||^2 - 2 x^T y.
26 | sq_norm0 = np.sum(mat0**2, 0)
27 | sq_norm1 = np.sum(mat1**2, 0)
28 | sq_dist = sq_norm0[:, None] + sq_norm1[None, :] - 2 * mat0.T @ mat1
29 | sq_dist = np.maximum(0, sq_dist) # Negative values must be numerical errors.
30 | return sq_dist
31 |
32 |
33 | def compute_tesselation_weights(v):
34 | """Tesselate the vertices of a triangle by a factor of `v`."""
35 | if v < 1:
36 | raise ValueError(f'v {v} must be >= 1')
37 | int_weights = []
38 | for i in range(v + 1):
39 | for j in range(v + 1 - i):
40 | int_weights.append((i, j, v - (i + j)))
41 | int_weights = np.array(int_weights)
42 | weights = int_weights / v # Barycentric weights.
43 | return weights
44 |
45 |
46 | def tesselate_geodesic(base_verts, base_faces, v, eps=1e-4):
47 | """Tesselate the vertices of a geodesic polyhedron.
48 |
49 | Args:
50 | base_verts: tensor of floats, the vertex coordinates of the geodesic.
51 | base_faces: tensor of ints, the indices of the vertices of base_verts that
52 | constitute eachface of the polyhedra.
53 | v: int, the factor of the tesselation (v==1 is a no-op).
54 | eps: float, a small value used to determine if two vertices are the same.
55 |
56 | Returns:
57 | verts: a tensor of floats, the coordinates of the tesselated vertices.
58 | """
59 | if not isinstance(v, int):
60 | raise ValueError(f'v {v} must an integer')
61 | tri_weights = compute_tesselation_weights(v)
62 |
63 | verts = []
64 | for base_face in base_faces:
65 | new_verts = np.matmul(tri_weights, base_verts[base_face, :])
66 | new_verts /= np.sqrt(np.sum(new_verts**2, 1, keepdims=True))
67 | verts.append(new_verts)
68 | verts = np.concatenate(verts, 0)
69 |
70 | sq_dist = compute_sq_dist(verts.T)
71 | assignment = np.array([np.min(np.argwhere(d <= eps)) for d in sq_dist])
72 | unique = np.unique(assignment)
73 | verts = verts[unique, :]
74 |
75 | return verts
76 |
77 |
78 | def generate_basis(base_shape,
79 | angular_tesselation,
80 | remove_symmetries=True,
81 | eps=1e-4):
82 | """Generates a 3D basis by tesselating a geometric polyhedron.
83 |
84 | Args:
85 | base_shape: string, the name of the starting polyhedron, must be either
86 | 'icosahedron' or 'octahedron'.
87 | angular_tesselation: int, the number of times to tesselate the polyhedron,
88 | must be >= 1 (a value of 1 is a no-op to the polyhedron).
89 | remove_symmetries: bool, if True then remove the symmetric basis columns,
90 | which is usually a good idea because otherwise projections onto the basis
91 | will have redundant negative copies of each other.
92 | eps: float, a small number used to determine symmetries.
93 |
94 | Returns:
95 | basis: a matrix with shape [3, n].
96 | """
97 | if base_shape == 'icosahedron':
98 | a = (np.sqrt(5) + 1) / 2
99 | verts = np.array([(-1, 0, a), (1, 0, a), (-1, 0, -a), (1, 0, -a), (0, a, 1),
100 | (0, a, -1), (0, -a, 1), (0, -a, -1), (a, 1, 0),
101 | (-a, 1, 0), (a, -1, 0), (-a, -1, 0)]) / np.sqrt(a + 2)
102 | faces = np.array([(0, 4, 1), (0, 9, 4), (9, 5, 4), (4, 5, 8), (4, 8, 1),
103 | (8, 10, 1), (8, 3, 10), (5, 3, 8), (5, 2, 3), (2, 7, 3),
104 | (7, 10, 3), (7, 6, 10), (7, 11, 6), (11, 0, 6), (0, 1, 6),
105 | (6, 1, 10), (9, 0, 11), (9, 11, 2), (9, 2, 5),
106 | (7, 2, 11)])
107 | verts = tesselate_geodesic(verts, faces, angular_tesselation)
108 | elif base_shape == 'octahedron':
109 | verts = np.array([(0, 0, -1), (0, 0, 1), (0, -1, 0), (0, 1, 0), (-1, 0, 0),
110 | (1, 0, 0)])
111 | corners = np.array(list(itertools.product([-1, 1], repeat=3)))
112 | pairs = np.argwhere(compute_sq_dist(corners.T, verts.T) == 2)
113 | faces = np.sort(np.reshape(pairs[:, 1], [3, -1]).T, 1)
114 | verts = tesselate_geodesic(verts, faces, angular_tesselation)
115 | else:
116 | raise ValueError(f'base_shape {base_shape} not supported')
117 |
118 | if remove_symmetries:
119 | # Remove elements of `verts` that are reflections of each other.
120 | match = compute_sq_dist(verts.T, -verts.T) < eps
121 | verts = verts[np.any(np.triu(match), 1), :]
122 |
123 | basis = verts[:, ::-1]
124 | return basis
125 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/internal/image.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Functions for processing images."""
16 |
17 | import types
18 | from typing import Optional, Union
19 |
20 | import dm_pix
21 | import jax
22 | import jax.numpy as jnp
23 | import numpy as np
24 |
25 | _Array = Union[np.ndarray, jnp.ndarray]
26 |
27 |
28 | def mse_to_psnr(mse):
29 | """Compute PSNR given an MSE (we assume the maximum pixel value is 1)."""
30 | return -10. / jnp.log(10.) * jnp.log(mse)
31 |
32 |
33 | def psnr_to_mse(psnr):
34 | """Compute MSE given a PSNR (we assume the maximum pixel value is 1)."""
35 | return jnp.exp(-0.1 * jnp.log(10.) * psnr)
36 |
37 |
38 | def ssim_to_dssim(ssim):
39 | """Compute DSSIM given an SSIM."""
40 | return (1 - ssim) / 2
41 |
42 |
43 | def dssim_to_ssim(dssim):
44 | """Compute DSSIM given an SSIM."""
45 | return 1 - 2 * dssim
46 |
47 |
48 | def linear_to_srgb(linear: _Array,
49 | eps: Optional[float] = None,
50 | xnp: types.ModuleType = jnp) -> _Array:
51 | """Assumes `linear` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB."""
52 | if eps is None:
53 | eps = xnp.finfo(xnp.float32).eps
54 | srgb0 = 323 / 25 * linear
55 | srgb1 = (211 * xnp.maximum(eps, linear)**(5 / 12) - 11) / 200
56 | return xnp.where(linear <= 0.0031308, srgb0, srgb1)
57 |
58 |
59 | def srgb_to_linear(srgb: _Array,
60 | eps: Optional[float] = None,
61 | xnp: types.ModuleType = jnp) -> _Array:
62 | """Assumes `srgb` is in [0, 1], see https://en.wikipedia.org/wiki/SRGB."""
63 | if eps is None:
64 | eps = xnp.finfo(xnp.float32).eps
65 | linear0 = 25 / 323 * srgb
66 | linear1 = xnp.maximum(eps, ((200 * srgb + 11) / (211)))**(12 / 5)
67 | return xnp.where(srgb <= 0.04045, linear0, linear1)
68 |
69 |
70 | def downsample(img, factor):
71 | """Area downsample img (factor must evenly divide img height and width)."""
72 | sh = img.shape
73 | if not (sh[0] % factor == 0 and sh[1] % factor == 0):
74 | raise ValueError(f'Downsampling factor {factor} does not '
75 | f'evenly divide image shape {sh[:2]}')
76 | img = img.reshape((sh[0] // factor, factor, sh[1] // factor, factor) + sh[2:])
77 | img = img.mean((1, 3))
78 | return img
79 |
80 |
81 | def color_correct(img, ref, num_iters=5, eps=0.5 / 255):
82 | """Warp `img` to match the colors in `ref_img`."""
83 | if img.shape[-1] != ref.shape[-1]:
84 | raise ValueError(
85 | f'img\'s {img.shape[-1]} and ref\'s {ref.shape[-1]} channels must match'
86 | )
87 | num_channels = img.shape[-1]
88 | img_mat = img.reshape([-1, num_channels])
89 | ref_mat = ref.reshape([-1, num_channels])
90 | is_unclipped = lambda z: (z >= eps) & (z <= (1 - eps)) # z \in [eps, 1-eps].
91 | mask0 = is_unclipped(img_mat)
92 | # Because the set of saturated pixels may change after solving for a
93 | # transformation, we repeatedly solve a system `num_iters` times and update
94 | # our estimate of which pixels are saturated.
95 | for _ in range(num_iters):
96 | # Construct the left hand side of a linear system that contains a quadratic
97 | # expansion of each pixel of `img`.
98 | a_mat = []
99 | for c in range(num_channels):
100 | a_mat.append(img_mat[:, c:(c + 1)] * img_mat[:, c:]) # Quadratic term.
101 | a_mat.append(img_mat) # Linear term.
102 | a_mat.append(jnp.ones_like(img_mat[:, :1])) # Bias term.
103 | a_mat = jnp.concatenate(a_mat, axis=-1)
104 | warp = []
105 | for c in range(num_channels):
106 | # Construct the right hand side of a linear system containing each color
107 | # of `ref`.
108 | b = ref_mat[:, c]
109 | # Ignore rows of the linear system that were saturated in the input or are
110 | # saturated in the current corrected color estimate.
111 | mask = mask0[:, c] & is_unclipped(img_mat[:, c]) & is_unclipped(b)
112 | ma_mat = jnp.where(mask[:, None], a_mat, 0)
113 | mb = jnp.where(mask, b, 0)
114 | # Solve the linear system. We're using the np.lstsq instead of jnp because
115 | # it's significantly more stable in this case, for some reason.
116 | w = np.linalg.lstsq(ma_mat, mb, rcond=-1)[0]
117 | assert jnp.all(jnp.isfinite(w))
118 | warp.append(w)
119 | warp = jnp.stack(warp, axis=-1)
120 | # Apply the warp to update img_mat.
121 | img_mat = jnp.clip(
122 | jnp.matmul(a_mat, warp, precision=jax.lax.Precision.HIGHEST), 0, 1)
123 | corrected_img = jnp.reshape(img_mat, img.shape)
124 | return corrected_img
125 |
126 |
127 | class MetricHarness:
128 | """A helper class for evaluating several error metrics."""
129 |
130 | def __init__(self):
131 | self.ssim_fn = jax.jit(dm_pix.ssim)
132 |
133 | def __call__(self, rgb_pred, rgb_gt, name_fn=lambda s: s):
134 | """Evaluate the error between a predicted rgb image and the true image."""
135 | psnr = float(mse_to_psnr(((rgb_pred - rgb_gt)**2).mean()))
136 | ssim = float(self.ssim_fn(rgb_pred, rgb_gt))
137 |
138 | return {
139 | name_fn('psnr'): psnr,
140 | name_fn('ssim'): ssim,
141 | }
142 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/internal/math.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Mathy utility functions."""
16 |
17 | import jax
18 | import jax.numpy as jnp
19 |
20 |
21 | def matmul(a, b):
22 | """jnp.matmul defaults to bfloat16, but this helper function doesn't."""
23 | return jnp.matmul(a, b, precision=jax.lax.Precision.HIGHEST)
24 |
25 |
26 | def safe_trig_helper(x, fn, t=100 * jnp.pi):
27 | """Helper function used by safe_cos/safe_sin: mods x before sin()/cos()."""
28 | return fn(jnp.where(jnp.abs(x) < t, x, x % t))
29 |
30 |
31 | def safe_cos(x):
32 | """jnp.cos() on a TPU may NaN out for large values."""
33 | return safe_trig_helper(x, jnp.cos)
34 |
35 |
36 | def safe_sin(x):
37 | """jnp.sin() on a TPU may NaN out for large values."""
38 | return safe_trig_helper(x, jnp.sin)
39 |
40 |
41 | @jax.custom_jvp
42 | def safe_exp(x):
43 | """jnp.exp() but with finite output and gradients for large inputs."""
44 | return jnp.exp(jnp.minimum(x, 88.)) # jnp.exp(89) is infinity.
45 |
46 |
47 | @safe_exp.defjvp
48 | def safe_exp_jvp(primals, tangents):
49 | """Override safe_exp()'s gradient so that it's large when inputs are large."""
50 | x, = primals
51 | x_dot, = tangents
52 | exp_x = safe_exp(x)
53 | exp_x_dot = exp_x * x_dot
54 | return exp_x, exp_x_dot
55 |
56 |
57 | def log_lerp(t, v0, v1):
58 | """Interpolate log-linearly from `v0` (t=0) to `v1` (t=1)."""
59 | if v0 <= 0 or v1 <= 0:
60 | raise ValueError(f'Interpolants {v0} and {v1} must be positive.')
61 | lv0 = jnp.log(v0)
62 | lv1 = jnp.log(v1)
63 | return jnp.exp(jnp.clip(t, 0, 1) * (lv1 - lv0) + lv0)
64 |
65 |
66 | def learning_rate_decay(step,
67 | lr_init,
68 | lr_final,
69 | max_steps,
70 | lr_delay_steps=0,
71 | lr_delay_mult=1):
72 | """Continuous learning rate decay function.
73 |
74 | The returned rate is lr_init when step=0 and lr_final when step=max_steps, and
75 | is log-linearly interpolated elsewhere (equivalent to exponential decay).
76 | If lr_delay_steps>0 then the learning rate will be scaled by some smooth
77 | function of lr_delay_mult, such that the initial learning rate is
78 | lr_init*lr_delay_mult at the beginning of optimization but will be eased back
79 | to the normal learning rate when steps>lr_delay_steps.
80 |
81 | Args:
82 | step: int, the current optimization step.
83 | lr_init: float, the initial learning rate.
84 | lr_final: float, the final learning rate.
85 | max_steps: int, the number of steps during optimization.
86 | lr_delay_steps: int, the number of steps to delay the full learning rate.
87 | lr_delay_mult: float, the multiplier on the rate when delaying it.
88 |
89 | Returns:
90 | lr: the learning for current step 'step'.
91 | """
92 | if lr_delay_steps > 0:
93 | # A kind of reverse cosine decay.
94 | delay_rate = lr_delay_mult + (1 - lr_delay_mult) * jnp.sin(
95 | 0.5 * jnp.pi * jnp.clip(step / lr_delay_steps, 0, 1))
96 | else:
97 | delay_rate = 1.
98 | return delay_rate * log_lerp(step / max_steps, lr_init, lr_final)
99 |
100 |
101 | def interp(*args):
102 | """A gather-based (GPU-friendly) vectorized replacement for jnp.interp()."""
103 | args_flat = [x.reshape([-1, x.shape[-1]]) for x in args]
104 | ret = jax.vmap(jnp.interp)(*args_flat).reshape(args[0].shape)
105 | return ret
106 |
107 |
108 | def sorted_interp(x, xp, fp):
109 | """A TPU-friendly version of interp(), where xp and fp must be sorted."""
110 |
111 | # Identify the location in `xp` that corresponds to each `x`.
112 | # The final `True` index in `mask` is the start of the matching interval.
113 | mask = x[..., None, :] >= xp[..., :, None]
114 |
115 | def find_interval(x):
116 | # Grab the value where `mask` switches from True to False, and vice versa.
117 | # This approach takes advantage of the fact that `x` is sorted.
118 | x0 = jnp.max(jnp.where(mask, x[..., None], x[..., :1, None]), -2)
119 | x1 = jnp.min(jnp.where(~mask, x[..., None], x[..., -1:, None]), -2)
120 | return x0, x1
121 |
122 | fp0, fp1 = find_interval(fp)
123 | xp0, xp1 = find_interval(xp)
124 |
125 | offset = jnp.clip(jnp.nan_to_num((x - xp0) / (xp1 - xp0), 0), 0, 1)
126 | ret = fp0 + offset * (fp1 - fp0)
127 | return ret
128 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/internal/utils.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Utility functions."""
16 |
17 | import enum
18 | import os
19 | from typing import Any, Dict, Optional, Union
20 |
21 | import flax
22 | import jax
23 | import jax.numpy as jnp
24 | import numpy as np
25 | from PIL import ExifTags
26 | from PIL import Image
27 |
28 | _Array = Union[np.ndarray, jnp.ndarray]
29 |
30 |
31 | @flax.struct.dataclass
32 | class Pixels:
33 | """All tensors must have the same num_dims and first n-1 dims must match."""
34 | pix_x_int: _Array
35 | pix_y_int: _Array
36 | lossmult: _Array
37 | near: _Array
38 | far: _Array
39 | cam_idx: _Array
40 | exposure_idx: Optional[_Array] = None
41 | exposure_values: Optional[_Array] = None
42 |
43 |
44 | @flax.struct.dataclass
45 | class Rays:
46 | """All tensors must have the same num_dims and first n-1 dims must match."""
47 | origins: _Array
48 | directions: _Array
49 | viewdirs: _Array
50 | radii: _Array
51 | imageplane: _Array
52 | lossmult: _Array
53 | near: _Array
54 | far: _Array
55 | cam_idx: _Array
56 | exposure_idx: Optional[_Array] = None
57 | exposure_values: Optional[_Array] = None
58 |
59 |
60 | # Dummy Rays object that can be used to initialize NeRF model.
61 | def dummy_rays(include_exposure_idx: bool = False,
62 | include_exposure_values: bool = False) -> Rays:
63 | data_fn = lambda n: jnp.zeros((1, n))
64 | exposure_kwargs = {}
65 | if include_exposure_idx:
66 | exposure_kwargs['exposure_idx'] = data_fn(1).astype(jnp.int32)
67 | if include_exposure_values:
68 | exposure_kwargs['exposure_values'] = data_fn(1)
69 | return Rays(
70 | origins=data_fn(3),
71 | directions=data_fn(3),
72 | viewdirs=data_fn(3),
73 | radii=data_fn(1),
74 | imageplane=data_fn(2),
75 | lossmult=data_fn(1),
76 | near=data_fn(1),
77 | far=data_fn(1),
78 | cam_idx=data_fn(1).astype(jnp.int32),
79 | **exposure_kwargs)
80 |
81 |
82 | @flax.struct.dataclass
83 | class Batch:
84 | """Data batch for NeRF training or testing."""
85 | rays: Union[Pixels, Rays]
86 | rgb: Optional[_Array] = None
87 | disps: Optional[_Array] = None
88 | normals: Optional[_Array] = None
89 | alphas: Optional[_Array] = None
90 |
91 |
92 | class DataSplit(enum.Enum):
93 | """Dataset split."""
94 | TRAIN = 'train'
95 | TEST = 'test'
96 |
97 |
98 | class BatchingMethod(enum.Enum):
99 | """Draw rays randomly from a single image or all images, in each batch."""
100 | ALL_IMAGES = 'all_images'
101 | SINGLE_IMAGE = 'single_image'
102 |
103 |
104 | def open_file(pth, mode='r'):
105 | return open(pth, mode=mode)
106 |
107 |
108 | def file_exists(pth):
109 | return os.path.exists(pth)
110 |
111 |
112 | def listdir(pth):
113 | return os.listdir(pth)
114 |
115 |
116 | def isdir(pth):
117 | return os.path.isdir(pth)
118 |
119 |
120 | def makedirs(pth):
121 | if not file_exists(pth):
122 | os.makedirs(pth)
123 |
124 |
125 | def shard(xs):
126 | """Split data into shards for multiple devices along the first dimension."""
127 | return jax.tree_util.tree_map(
128 | lambda x: x.reshape((jax.local_device_count(), -1) + x.shape[1:]), xs)
129 |
130 |
131 | def unshard(x, padding=0):
132 | """Collect the sharded tensor to the shape before sharding."""
133 | y = x.reshape([x.shape[0] * x.shape[1]] + list(x.shape[2:]))
134 | if padding > 0:
135 | y = y[:-padding]
136 | return y
137 |
138 |
139 | def load_img(pth: str) -> np.ndarray:
140 | """Load an image and cast to float32."""
141 | with open_file(pth, 'rb') as f:
142 | image = np.array(Image.open(f), dtype=np.float32)
143 | return image
144 |
145 |
146 | def load_exif(pth: str) -> Dict[str, Any]:
147 | """Load EXIF data for an image."""
148 | with open_file(pth, 'rb') as f:
149 | image_pil = Image.open(f)
150 | exif_pil = image_pil._getexif() # pylint: disable=protected-access
151 | if exif_pil is not None:
152 | exif = {
153 | ExifTags.TAGS[k]: v for k, v in exif_pil.items() if k in ExifTags.TAGS
154 | }
155 | else:
156 | exif = {}
157 | return exif
158 |
159 |
160 | def save_img_u8(img, pth):
161 | """Save an image (probably RGB) in [0, 1] to disk as a uint8 PNG."""
162 | with open_file(pth, 'wb') as f:
163 | Image.fromarray(
164 | (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)).save(
165 | f, 'PNG')
166 |
167 |
168 | def save_img_f32(depthmap, pth):
169 | """Save an image (probably a depthmap) to disk as a float32 TIFF."""
170 | with open_file(pth, 'wb') as f:
171 | Image.fromarray(np.nan_to_num(depthmap).astype(np.float32)).save(f, 'TIFF')
172 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | jax
3 | jaxlib
4 | flax
5 | opencv-python
6 | Pillow
7 | tensorboard
8 | tensorflow
9 | gin-config
10 | dm_pix
11 | rawpy
12 | mediapy
13 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/eval_360.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=gardenvase
19 | EXPERIMENT=360
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_real_360
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m eval \
24 | --gin_configs=configs/360.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --logtostderr
28 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/eval_blender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=ficus
19 | EXPERIMENT=blender
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/dors_nerf_synthetic
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m eval \
24 | --gin_configs=configs/blender_256.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --logtostderr
28 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/eval_llff.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=flower
19 | EXPERIMENT=llff
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_llff_data
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m eval \
24 | --gin_configs=configs/llff_256.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --logtostderr
28 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/eval_raw.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=nightpiano
19 | EXPERIMENT=raw
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/rawnerf/scenes
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m eval \
24 | --gin_configs=configs/llff_raw.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --logtostderr
28 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/eval_shinyblender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=toaster
19 | EXPERIMENT=shinyblender
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/dors_nerf_synthetic
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m eval \
24 | --gin_configs=configs/blender_refnerf.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --logtostderr
28 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/local_colmap_and_resize.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 |
17 | # Set to 0 if you do not have a GPU.
18 | USE_GPU=1
19 | # Path to a directory `base/` with images in `base/images/`.
20 | DATASET_PATH=$1
21 | # Recommended CAMERA values: OPENCV for perspective, OPENCV_FISHEYE for fisheye.
22 | CAMERA=${2:-OPENCV}
23 |
24 |
25 | # Run COLMAP.
26 |
27 | ### Feature extraction
28 |
29 | colmap feature_extractor \
30 | --database_path "$DATASET_PATH"/database.db \
31 | --image_path "$DATASET_PATH"/images \
32 | --ImageReader.single_camera 1 \
33 | --ImageReader.camera_model "$CAMERA" \
34 | --SiftExtraction.use_gpu "$USE_GPU"
35 |
36 |
37 | ### Feature matching
38 |
39 | colmap exhaustive_matcher \
40 | --database_path "$DATASET_PATH"/database.db \
41 | --SiftMatching.use_gpu "$USE_GPU"
42 |
43 | ## Use if your scene has > 500 images
44 | ## Replace this path with your own local copy of the file.
45 | ## Download from: https://demuc.de/colmap/#download
46 | # VOCABTREE_PATH=/usr/local/google/home/bmild/vocab_tree_flickr100K_words32K.bin
47 | # colmap vocab_tree_matcher \
48 | # --database_path "$DATASET_PATH"/database.db \
49 | # --VocabTreeMatching.vocab_tree_path $VOCABTREE_PATH \
50 | # --SiftMatching.use_gpu "$USE_GPU"
51 |
52 |
53 | ### Bundle adjustment
54 |
55 | # The default Mapper tolerance is unnecessarily large,
56 | # decreasing it speeds up bundle adjustment steps.
57 | mkdir -p "$DATASET_PATH"/sparse
58 | colmap mapper \
59 | --database_path "$DATASET_PATH"/database.db \
60 | --image_path "$DATASET_PATH"/images \
61 | --output_path "$DATASET_PATH"/sparse \
62 | --Mapper.ba_global_function_tolerance=0.000001
63 |
64 |
65 | ### Image undistortion
66 |
67 | ## Use this if you want to undistort your images into ideal pinhole intrinsics.
68 | # mkdir -p "$DATASET_PATH"/dense
69 | # colmap image_undistorter \
70 | # --image_path "$DATASET_PATH"/images \
71 | # --input_path "$DATASET_PATH"/sparse/0 \
72 | # --output_path "$DATASET_PATH"/dense \
73 | # --output_type COLMAP
74 |
75 | # Resize images.
76 |
77 | cp -r "$DATASET_PATH"/images "$DATASET_PATH"/images_2
78 |
79 | pushd "$DATASET_PATH"/images_2
80 | ls | xargs -P 8 -I {} mogrify -resize 50% {}
81 | popd
82 |
83 | cp -r "$DATASET_PATH"/images "$DATASET_PATH"/images_4
84 |
85 | pushd "$DATASET_PATH"/images_4
86 | ls | xargs -P 8 -I {} mogrify -resize 25% {}
87 | popd
88 |
89 | cp -r "$DATASET_PATH"/images "$DATASET_PATH"/images_8
90 |
91 | pushd "$DATASET_PATH"/images_8
92 | ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
93 | popd
94 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/render_360.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=gardenvase
19 | EXPERIMENT=360
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_real_360
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m render \
24 | --gin_configs=configs/360.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --gin_bindings="Config.render_path = True" \
28 | --gin_bindings="Config.render_path_frames = 10" \
29 | --gin_bindings="Config.render_dir = '${CHECKPOINT_DIR}/render/'" \
30 | --gin_bindings="Config.render_video_fps = 2" \
31 | --logtostderr
32 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/render_llff.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=flower
19 | EXPERIMENT=llff
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_llff_data
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m render \
24 | --gin_configs=configs/llff_256.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --gin_bindings="Config.render_path = True" \
28 | --gin_bindings="Config.render_path_frames = 10" \
29 | --gin_bindings="Config.render_dir = '${CHECKPOINT_DIR}/render/'" \
30 | --gin_bindings="Config.render_video_fps = 2" \
31 | --logtostderr
32 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/render_raw.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=nightpiano
19 | EXPERIMENT=raw
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/rawnerf/scenes
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | python -m render \
24 | --gin_configs=configs/llff_raw.gin \
25 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
26 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
27 | --gin_bindings="Config.render_path = True" \
28 | --gin_bindings="Config.render_path_frames = 10" \
29 | --gin_bindings="Config.render_dir = '${CHECKPOINT_DIR}/render/'" \
30 | --gin_bindings="Config.render_video_fps = 2" \
31 | --logtostderr
32 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/run_all_unit_tests.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 |
17 | python -m unittest tests.camera_utils_test
18 | python -m unittest tests.geopoly_test
19 | python -m unittest tests.stepfun_test
20 | python -m unittest tests.coord_test
21 | python -m unittest tests.image_test
22 | python -m unittest tests.ref_utils_test
23 | python -m unittest tests.utils_test
24 | python -m unittest tests.datasets_test
25 | python -m unittest tests.math_test
26 | python -m unittest tests.render_test
27 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/train_360.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=gardenvase
19 | EXPERIMENT=360
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_real_360
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | # If running one of the indoor scenes, add
24 | # --gin_bindings="Config.factor = 2"
25 |
26 | rm "$CHECKPOINT_DIR"/*
27 | python -m train \
28 | --gin_configs=configs/360.gin \
29 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
30 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
31 | --logtostderr
32 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/train_blender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=ficus
19 | EXPERIMENT=blender
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_synthetic
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | rm "$CHECKPOINT_DIR"/*
24 | python -m train \
25 | --gin_configs=configs/blender_256.gin \
26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/train_llff.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=flower
19 | EXPERIMENT=llff
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/nerf_llff_data
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | rm "$CHECKPOINT_DIR"/*
24 | python -m train \
25 | --gin_configs=configs/llff_256.gin \
26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/train_raw.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=nightpiano
19 | EXPERIMENT=raw
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/rawnerf/scenes
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | rm "$CHECKPOINT_DIR"/*
24 | python -m train \
25 | --gin_configs=configs/llff_raw.gin \
26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/scripts/train_shinyblender.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2022 Google LLC
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | export CUDA_VISIBLE_DEVICES=0
17 |
18 | SCENE=toaster
19 | EXPERIMENT=shinyblender
20 | DATA_DIR=/usr/local/google/home/barron/tmp/nerf_data/dors_nerf_synthetic
21 | CHECKPOINT_DIR=/usr/local/google/home/barron/tmp/nerf_results/"$EXPERIMENT"/"$SCENE"
22 |
23 | rm "$CHECKPOINT_DIR"/*
24 | python -m train \
25 | --gin_configs=configs/blender_refnerf.gin \
26 | --gin_bindings="Config.data_dir = '${DATA_DIR}/${SCENE}'" \
27 | --gin_bindings="Config.checkpoint_dir = '${CHECKPOINT_DIR}'" \
28 | --logtostderr
29 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/tests/camera_utils_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Tests for camera_utils."""
16 |
17 | from absl.testing import absltest
18 | from absl.testing import parameterized
19 | from internal import camera_utils
20 | from jax import random
21 | import jax.numpy as jnp
22 | import numpy as np
23 |
24 |
25 | class CameraUtilsTest(parameterized.TestCase):
26 |
27 | def test_convert_to_ndc(self):
28 | rng = random.PRNGKey(0)
29 | for _ in range(10):
30 | # Random pinhole camera intrinsics.
31 | key, rng = random.split(rng)
32 | focal, width, height = random.uniform(key, (3,), minval=100., maxval=200.)
33 | camtopix = camera_utils.intrinsic_matrix(focal, focal, width / 2.,
34 | height / 2.)
35 | pixtocam = np.linalg.inv(camtopix)
36 | near = 1.
37 |
38 | # Random rays, pointing forward (negative z direction).
39 | num_rays = 1000
40 | key, rng = random.split(rng)
41 | origins = jnp.array([0., 0., 1.])
42 | origins += random.uniform(key, (num_rays, 3), minval=-1., maxval=1.)
43 | directions = jnp.array([0., 0., -1.])
44 | directions += random.uniform(key, (num_rays, 3), minval=-.5, maxval=.5)
45 |
46 | # Project world-space points along each ray into NDC space.
47 | t = jnp.linspace(0., 1., 10)
48 | pts_world = origins + t[:, None, None] * directions
49 | pts_ndc = jnp.stack([
50 | -focal / (.5 * width) * pts_world[..., 0] / pts_world[..., 2],
51 | -focal / (.5 * height) * pts_world[..., 1] / pts_world[..., 2],
52 | 1. + 2. * near / pts_world[..., 2],
53 | ],
54 | axis=-1)
55 |
56 | # Get NDC space rays.
57 | origins_ndc, directions_ndc = camera_utils.convert_to_ndc(
58 | origins, directions, pixtocam, near)
59 |
60 | # Ensure that the NDC space points lie on the calculated rays.
61 | directions_ndc_norm = jnp.linalg.norm(
62 | directions_ndc, axis=-1, keepdims=True)
63 | directions_ndc_unit = directions_ndc / directions_ndc_norm
64 | projection = ((pts_ndc - origins_ndc) * directions_ndc_unit).sum(axis=-1)
65 | pts_ndc_proj = origins_ndc + directions_ndc_unit * projection[..., None]
66 |
67 | # pts_ndc should be close to their projections pts_ndc_proj onto the rays.
68 | np.testing.assert_allclose(pts_ndc, pts_ndc_proj, atol=1e-5, rtol=1e-5)
69 |
70 |
71 | if __name__ == '__main__':
72 | absltest.main()
73 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/tests/datasets_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Tests for datasets."""
16 |
17 | from absl.testing import absltest
18 | from internal import camera_utils
19 | from internal import configs
20 | from internal import datasets
21 | from jax import random
22 | import numpy as np
23 |
24 |
25 | class DummyDataset(datasets.Dataset):
26 |
27 | def _load_renderings(self, config):
28 | """Generates dummy image and pose data."""
29 | self._n_examples = 2
30 | self.height = 3
31 | self.width = 4
32 | self._resolution = self.height * self.width
33 | self.focal = 5.
34 | self.pixtocams = np.linalg.inv(
35 | camera_utils.intrinsic_matrix(self.focal, self.focal, self.width * 0.5,
36 | self.height * 0.5))
37 |
38 | rng = random.PRNGKey(0)
39 |
40 | key, rng = random.split(rng)
41 | images_shape = (self._n_examples, self.height, self.width, 3)
42 | self.images = random.uniform(key, images_shape)
43 |
44 | key, rng = random.split(rng)
45 | self.camtoworlds = np.stack([
46 | camera_utils.viewmatrix(*random.normal(k, (3, 3)))
47 | for k in random.split(key, self._n_examples)
48 | ],
49 | axis=0)
50 |
51 |
52 | class DatasetsTest(absltest.TestCase):
53 |
54 | def test_dataset_batch_creation(self):
55 | np.random.seed(0)
56 | config = configs.Config(batch_size=8)
57 |
58 | # Check shapes are consistent across all ray attributes.
59 | for split in ['train', 'test']:
60 | dummy_dataset = DummyDataset(split, '', config)
61 | rays = dummy_dataset.peek().rays
62 | sh_gt = rays.origins.shape[:-1]
63 | for z in rays.__dict__.values():
64 | if z is not None:
65 | self.assertEqual(z.shape[:-1], sh_gt)
66 |
67 | # Check test batch generation matches golden data.
68 | dummy_dataset = DummyDataset('test', '', config)
69 | batch = dummy_dataset.peek()
70 |
71 | rgb = batch.rgb.ravel()
72 | rgb_gt = np.array([
73 | 0.5289556, 0.28869557, 0.24527192, 0.12083626, 0.8904066, 0.6259936,
74 | 0.57573485, 0.09355974, 0.8017353, 0.538651, 0.4998169, 0.42061496,
75 | 0.5591258, 0.00577283, 0.6804651, 0.9139203, 0.00444758, 0.96962905,
76 | 0.52956843, 0.38282406, 0.28777933, 0.6640035, 0.39736128, 0.99495006,
77 | 0.13100398, 0.7597165, 0.8532667, 0.67468107, 0.6804743, 0.26873016,
78 | 0.60699487, 0.5722265, 0.44482303, 0.6511061, 0.54807067, 0.09894073
79 | ])
80 | np.testing.assert_allclose(rgb, rgb_gt, atol=1e-4, rtol=1e-4)
81 |
82 | ray_origins = batch.rays.origins.ravel()
83 | ray_origins_gt = np.array([
84 | -0.20050469, -0.6451472, -0.8818224, -0.20050469, -0.6451472,
85 | -0.8818224, -0.20050469, -0.6451472, -0.8818224, -0.20050469,
86 | -0.6451472, -0.8818224, -0.20050469, -0.6451472, -0.8818224,
87 | -0.20050469, -0.6451472, -0.8818224, -0.20050469, -0.6451472,
88 | -0.8818224, -0.20050469, -0.6451472, -0.8818224, -0.20050469,
89 | -0.6451472, -0.8818224, -0.20050469, -0.6451472, -0.8818224,
90 | -0.20050469, -0.6451472, -0.8818224, -0.20050469, -0.6451472, -0.8818224
91 | ])
92 | np.testing.assert_allclose(
93 | ray_origins, ray_origins_gt, atol=1e-4, rtol=1e-4)
94 |
95 | ray_dirs = batch.rays.directions.ravel()
96 | ray_dirs_gt = np.array([
97 | 0.24370372, 0.89296186, -0.5227117, 0.05601424, 0.8468699, -0.57417226,
98 | -0.13167524, 0.8007779, -0.62563276, -0.31936473, 0.75468594,
99 | -0.67709327, 0.17780769, 0.96766925, -0.34928587, -0.0098818, 0.9215773,
100 | -0.4007464, -0.19757128, 0.87548524, -0.4522069, -0.38526076,
101 | 0.82939327, -0.5036674, 0.11191163, 1.0423766, -0.17586003, -0.07577785,
102 | 0.9962846, -0.22732055, -0.26346734, 0.95019263, -0.2787811,
103 | -0.45115682, 0.90410066, -0.3302416
104 | ])
105 | np.testing.assert_allclose(ray_dirs, ray_dirs_gt, atol=1e-4, rtol=1e-4)
106 |
107 |
108 | if __name__ == '__main__':
109 | absltest.main()
110 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/tests/ref_utils_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Tests for ref_utils."""
16 |
17 | from absl.testing import absltest
18 | from internal import ref_utils
19 | from jax import random
20 | import jax.numpy as jnp
21 | import numpy as np
22 | import scipy
23 |
24 |
25 | def generate_dir_enc_fn_scipy(deg_view):
26 | """Return spherical harmonics using scipy.special.sph_harm."""
27 | ml_array = ref_utils.get_ml_array(deg_view)
28 |
29 | def dir_enc_fn(theta, phi):
30 | de = [scipy.special.sph_harm(m, l, phi, theta) for m, l in ml_array.T]
31 | de = np.stack(de, axis=-1)
32 | # Split into real and imaginary parts.
33 | return np.concatenate([np.real(de), np.imag(de)], axis=-1)
34 |
35 | return dir_enc_fn
36 |
37 |
38 | class RefUtilsTest(absltest.TestCase):
39 |
40 | def test_reflection(self):
41 | """Make sure reflected vectors have the same angle from normals as input."""
42 | rng = random.PRNGKey(0)
43 | for shape in [(45, 3), (4, 7, 3)]:
44 | key, rng = random.split(rng)
45 | normals = random.normal(key, shape)
46 | key, rng = random.split(rng)
47 | directions = random.normal(key, shape)
48 |
49 | # Normalize normal vectors.
50 | normals = normals / (
51 | jnp.linalg.norm(normals, axis=-1, keepdims=True) + 1e-10)
52 |
53 | reflected_directions = ref_utils.reflect(directions, normals)
54 |
55 | cos_angle_original = jnp.sum(directions * normals, axis=-1)
56 | cos_angle_reflected = jnp.sum(reflected_directions * normals, axis=-1)
57 |
58 | np.testing.assert_allclose(
59 | cos_angle_original, cos_angle_reflected, atol=1E-5, rtol=1E-5)
60 |
61 | def test_spherical_harmonics(self):
62 | """Make sure the fast spherical harmonics are accurate."""
63 | shape = (12, 11, 13)
64 |
65 | # Generate random points on sphere.
66 | rng = random.PRNGKey(0)
67 | key1, key2 = random.split(rng)
68 | theta = random.uniform(key1, shape, minval=0.0, maxval=jnp.pi)
69 | phi = random.uniform(key2, shape, minval=0.0, maxval=2.0*jnp.pi)
70 |
71 | # Convert to Cartesian coordinates.
72 | x = jnp.sin(theta) * jnp.cos(phi)
73 | y = jnp.sin(theta) * jnp.sin(phi)
74 | z = jnp.cos(theta)
75 | xyz = jnp.stack([x, y, z], axis=-1)
76 |
77 | deg_view = 5
78 | de = ref_utils.generate_dir_enc_fn(deg_view)(xyz)
79 | de_scipy = generate_dir_enc_fn_scipy(deg_view)(theta, phi)
80 |
81 | np.testing.assert_allclose(
82 | de, de_scipy, atol=0.02, rtol=1e6) # Only use atol.
83 | self.assertFalse(jnp.any(jnp.isnan(de)))
84 |
85 |
86 | if __name__ == '__main__':
87 | absltest.main()
88 |
--------------------------------------------------------------------------------
/CAM_multinerf-mipnerf360/tests/utils_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # https://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Tests for utils."""
16 |
17 | from absl.testing import absltest
18 |
19 | from internal import utils
20 |
21 |
22 | class UtilsTest(absltest.TestCase):
23 |
24 | def test_dummy_rays(self):
25 | """Ensures that the dummy Rays object is correctly initialized."""
26 | rays = utils.dummy_rays()
27 | self.assertEqual(rays.origins.shape[-1], 3)
28 |
29 |
30 | if __name__ == '__main__':
31 | absltest.main()
32 |
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/aws/update_index.py:
--------------------------------------------------------------------------------
1 | from collections import defaultdict
2 |
3 | import boto3
4 |
5 | ROOT_URL = 'https://nerfacc-bucket.s3.us-west-2.amazonaws.com/whl'
6 | html = '\n\n
\n{}\n\n'
7 | href = ' {}
'
8 | args = {
9 | 'ContentType': 'text/html',
10 | 'CacheControl': 'max-age=300',
11 | 'ACL': 'public-read',
12 | }
13 |
14 | bucket = boto3.resource('s3').Bucket(name='nerfacc-bucket')
15 |
16 | wheels_dict = defaultdict(list)
17 | for obj in bucket.objects.filter(Prefix='whl'):
18 | if obj.key[-3:] != 'whl':
19 | continue
20 | torch_version, wheel = obj.key.split('/')[-2:]
21 | wheel = f'{torch_version}/{wheel}'
22 | wheels_dict[torch_version].append(wheel)
23 |
24 | index_html = html.format('\n'.join([
25 | href.format(f'{torch_version}.html'.replace('+', '%2B'), version)
26 | for version in wheels_dict
27 | ]))
28 |
29 | with open('index.html', 'w') as f:
30 | f.write(index_html)
31 | bucket.Object('whl/index.html').upload_file('index.html', args)
32 |
33 | for torch_version, wheel_names in wheels_dict.items():
34 | torch_version_html = html.format('\n'.join([
35 | href.format(f'{ROOT_URL}/{wheel_name}'.replace('+', '%2B'), wheel_name)
36 | for wheel_name in wheel_names
37 | ]))
38 |
39 | with open(f'{torch_version}.html', 'w') as f:
40 | f.write(torch_version_html)
41 | bucket.Object(f'whl/{torch_version}.html').upload_file(
42 | f'{torch_version}.html', args)
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/code_checks.yml:
--------------------------------------------------------------------------------
1 | name: Core Tests.
2 |
3 | on:
4 | push:
5 | branches: [master]
6 | pull_request:
7 | branches: [master]
8 |
9 | permissions:
10 | contents: read
11 |
12 | jobs:
13 | build:
14 | runs-on: ubuntu-latest
15 |
16 | steps:
17 | - uses: actions/checkout@v3
18 | - name: Set up Python 3.8.12
19 | uses: actions/setup-python@v4
20 | with:
21 | python-version: "3.8.12"
22 | - name: Install dependencies
23 | run: |
24 | pip install isort==5.10.1 black[jupyter]==22.3.0
25 | - name: Run isort
26 | run: isort docs/ nerfacc/ scripts/ examples/ tests/ --profile black --skip examples/pycolmap --line-length 80 --check
27 | - name: Run Black
28 | run: black docs/ nerfacc/ scripts/ examples/ tests/ --exclude examples/pycolmap --line-length 80 --check
29 | # - name: Python Pylint
30 | # run: |
31 | # pylint nerfacc/ tests/ scripts/ examples/
32 |
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/cuda/Linux-env.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Took from https://github.com/pyg-team/pyg-lib/
4 |
5 | case ${1} in
6 | cu118)
7 | CUDA_HOME=/usr/local/cuda-11.8
8 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
9 | PATH=${CUDA_HOME}/bin:${PATH}
10 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
11 | ;;
12 | cu117)
13 | CUDA_HOME=/usr/local/cuda-11.7
14 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
15 | PATH=${CUDA_HOME}/bin:${PATH}
16 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
17 | ;;
18 | cu116)
19 | CUDA_HOME=/usr/local/cuda-11.6
20 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
21 | PATH=${CUDA_HOME}/bin:${PATH}
22 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
23 | ;;
24 | cu115)
25 | CUDA_HOME=/usr/local/cuda-11.5
26 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
27 | PATH=${CUDA_HOME}/bin:${PATH}
28 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
29 | ;;
30 | cu113)
31 | CUDA_HOME=/usr/local/cuda-11.3
32 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
33 | PATH=${CUDA_HOME}/bin:${PATH}
34 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
35 | ;;
36 | cu102)
37 | CUDA_HOME=/usr/local/cuda-10.2
38 | LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
39 | PATH=${CUDA_HOME}/bin:${PATH}
40 | export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
41 | ;;
42 | *)
43 | ;;
44 | esac
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/cuda/Linux.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Took from https://github.com/pyg-team/pyg-lib/
4 |
5 | OS=ubuntu2004
6 |
7 | case ${1} in
8 | cu118)
9 | CUDA=11.8
10 | APT_KEY=${OS}-${CUDA/./-}-local
11 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.0-520.61.05-1_amd64.deb
12 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.0/local_installers
13 | ;;
14 | cu117)
15 | CUDA=11.7
16 | APT_KEY=${OS}-${CUDA/./-}-local
17 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.1-515.65.01-1_amd64.deb
18 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.1/local_installers
19 | ;;
20 | cu116)
21 | CUDA=11.6
22 | APT_KEY=${OS}-${CUDA/./-}-local
23 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.2-510.47.03-1_amd64.deb
24 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.2/local_installers
25 | ;;
26 | cu115)
27 | CUDA=11.5
28 | APT_KEY=${OS}-${CUDA/./-}-local
29 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.2-495.29.05-1_amd64.deb
30 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.2/local_installers
31 | ;;
32 | cu113)
33 | CUDA=11.3
34 | APT_KEY=${OS}-${CUDA/./-}-local
35 | FILENAME=cuda-repo-${APT_KEY}_${CUDA}.0-465.19.01-1_amd64.deb
36 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}.0/local_installers
37 | ;;
38 | cu102)
39 | CUDA=10.2
40 | APT_KEY=${CUDA/./-}-local-${CUDA}.89-440.33.01
41 | FILENAME=cuda-repo-${OS}-${APT_KEY}_1.0-1_amd64.deb
42 | URL=https://developer.download.nvidia.com/compute/cuda/${CUDA}/Prod/local_installers
43 | ;;
44 | *)
45 | echo "Unrecognized CUDA_VERSION=${1}"
46 | exit 1
47 | ;;
48 | esac
49 |
50 | wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
51 | sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
52 | wget -nv ${URL}/${FILENAME}
53 | sudo dpkg -i ${FILENAME}
54 |
55 | if [ "${1}" = "cu117" ] || [ "${1}" = "cu118" ]; then
56 | sudo cp /var/cuda-repo-${APT_KEY}/cuda-*-keyring.gpg /usr/share/keyrings/
57 | else
58 | sudo apt-key add /var/cuda-repo-${APT_KEY}/7fa2af80.pub
59 | fi
60 |
61 | sudo apt-get update
62 | sudo apt-get -y install cuda
63 |
64 | rm -f ${FILENAME}
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/cuda/Windows-env.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Took from https://github.com/pyg-team/pyg-lib/
4 |
5 | case ${1} in
6 | cu118)
7 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.8
8 | PATH=${CUDA_HOME}/bin:$PATH
9 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
10 | export TORCH_CUDA_ARCH_LIST="6.0+PTX"
11 | ;;
12 | cu117)
13 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.7
14 | PATH=${CUDA_HOME}/bin:$PATH
15 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
16 | export TORCH_CUDA_ARCH_LIST="6.0+PTX"
17 | ;;
18 | cu116)
19 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.6
20 | PATH=${CUDA_HOME}/bin:$PATH
21 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
22 | export TORCH_CUDA_ARCH_LIST="6.0+PTX"
23 | ;;
24 | cu115)
25 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.5
26 | PATH=${CUDA_HOME}/bin:$PATH
27 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
28 | export TORCH_CUDA_ARCH_LIST="6.0+PTX"
29 | ;;
30 | cu113)
31 | CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
32 | PATH=${CUDA_HOME}/bin:$PATH
33 | PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
34 | export TORCH_CUDA_ARCH_LIST="6.0+PTX"
35 | ;;
36 | *)
37 | ;;
38 | esac
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/cuda/Windows.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Took from https://github.com/pyg-team/pyg-lib/
4 |
5 | # Install NVIDIA drivers, see:
6 | # https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
7 | curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
8 | 7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
9 |
10 | case ${1} in
11 | cu118)
12 | CUDA_SHORT=11.8
13 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
14 | CUDA_FILE=cuda_${CUDA_SHORT}.0_522.06_windows.exe
15 | ;;
16 | cu117)
17 | CUDA_SHORT=11.7
18 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.1/local_installers
19 | CUDA_FILE=cuda_${CUDA_SHORT}.1_516.94_windows.exe
20 | ;;
21 | cu116)
22 | CUDA_SHORT=11.3
23 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
24 | CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
25 | ;;
26 | cu115)
27 | CUDA_SHORT=11.3
28 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
29 | CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
30 | ;;
31 | cu113)
32 | CUDA_SHORT=11.3
33 | CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
34 | CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
35 | ;;
36 | *)
37 | echo "Unrecognized CUDA_VERSION=${1}"
38 | exit 1
39 | ;;
40 | esac
41 |
42 | curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
43 | echo ""
44 | echo "Installing from ${CUDA_FILE}..."
45 | PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
46 | echo "Done!"
47 | rm -f "${CUDA_FILE}"
--------------------------------------------------------------------------------
/CAM_nerfacc/.github/workflows/publish.yml:
--------------------------------------------------------------------------------
1 | # This workflows will upload a Python Package using twine when a release is created
2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
3 |
4 | name: Upload Python Package
5 |
6 | on:
7 | release:
8 | types: [created]
9 | branches: [master]
10 |
11 | jobs:
12 | deploy:
13 | runs-on: ubuntu-latest
14 | environment: production
15 |
16 | steps:
17 | - uses: actions/checkout@v3
18 | - name: Set up Python
19 | uses: actions/setup-python@v4
20 | with:
21 | python-version: '3.7'
22 | - name: Install dependencies
23 | run: |
24 | python -m pip install build twine
25 | - name: Strip unsupported tags in README
26 | run: |
27 | sed -i '//,//d' README.md
28 | - name: Build and publish
29 | env:
30 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
31 | run: |
32 | BUILD_NO_CUDA=1 python -m build
33 | twine upload --username __token__ --password $PYPI_TOKEN dist/*
--------------------------------------------------------------------------------
/CAM_nerfacc/.gitignore:
--------------------------------------------------------------------------------
1 | # Visual Studio Code configs.
2 | .vscode/
3 |
4 | # Byte-compiled / optimized / DLL files
5 | __pycache__/
6 | *.py[cod]
7 | *$py.class
8 |
9 | # C extensions
10 | *.so
11 |
12 | # Distribution / packaging
13 | .Python
14 | build/
15 | develop-eggs/
16 | dist/
17 | downloads/
18 | eggs/
19 | .eggs/
20 | # lib/
21 | lib64/
22 | parts/
23 | sdist/
24 | var/
25 | wheels/
26 | *.egg-info/
27 | .installed.cfg
28 | *.egg
29 | MANIFEST
30 |
31 | # PyInstaller
32 | # Usually these files are written by a python script from a template
33 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
34 | *.manifest
35 | *.spec
36 |
37 | # Installer logs
38 | pip-log.txt
39 | pip-delete-this-directory.txt
40 |
41 | # Unit test / coverage reports
42 | htmlcov/
43 | .tox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | .hypothesis/
51 | .pytest_cache/
52 |
53 | # Translations
54 | *.mo
55 | *.pot
56 |
57 | # Django stuff:
58 | *.log
59 | local_settings.py
60 | db.sqlite3
61 |
62 | # Flask stuff:
63 | instance/
64 | .webassets-cache
65 |
66 | # Scrapy stuff:
67 | .scrapy
68 |
69 | # Sphinx documentation
70 | docs/_build/
71 |
72 | # PyBuilder
73 | target/
74 |
75 | # Jupyter Notebook
76 | .ipynb_checkpoints
77 |
78 | # pyenv
79 | .python-version
80 |
81 | # celery beat schedule file
82 | celerybeat-schedule
83 |
84 | # SageMath parsed files
85 | *.sage.py
86 |
87 | # Environments
88 | .env
89 | .venv
90 | env/
91 | venv/
92 | ENV/
93 | env.bak/
94 | venv.bak/
95 |
96 | # Spyder project settings
97 | .spyderproject
98 | .spyproject
99 |
100 | # Rope project settings
101 | .ropeproject
102 |
103 | # mkdocs documentation
104 | /site
105 |
106 | # mypy
107 | .mypy_cache/
108 |
109 | .DS_Store
110 |
111 | # Direnv config.
112 | .envrc
113 |
114 | # line_profiler
115 | *.lprof
116 |
117 | # vscode
118 | .vsocde
119 |
120 | outputs/
121 | data
122 |
123 | benchmarks/*
124 | !benchmarks/barf
125 | !benchmarks/kplanes
126 | !benchmarks/tensorf
127 | !benchmarks/tineuvox
--------------------------------------------------------------------------------
/CAM_nerfacc/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "examples/pycolmap"]
2 | path = examples/pycolmap
3 | url = https://github.com/rmbrualla/pycolmap.git
4 |
5 | [submodule "benchmarks/tensorf"]
6 | path = benchmarks/tensorf
7 | url = https://github.com/liruilong940607/tensorf.git
8 | branch = nerfacc
9 |
10 | [submodule "benchmarks/barf"]
11 | path = benchmarks/barf
12 | url = https://github.com/liruilong940607/barf.git
13 | branch = nerfacc
14 |
15 | [submodule "benchmarks/tineuvox"]
16 | path = benchmarks/tineuvox
17 | url = https://github.com/liruilong940607/tineuvox.git
18 | branch = nerfacc
19 |
20 | [submodule "benchmarks/kplanes"]
21 | path = benchmarks/kplanes
22 | url = https://github.com/liruilong940607/kplanes.git
23 | branch = nerfacc
24 |
--------------------------------------------------------------------------------
/CAM_nerfacc/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | repos:
2 | - repo: https://github.com/pre-commit/pre-commit-hooks
3 | rev: v2.3.0
4 | hooks:
5 | - id: end-of-file-fixer
6 | - id: trailing-whitespace
7 | - id: check-yaml
8 | - id: check-merge-conflict
9 | - id: requirements-txt-fixer
10 | - repo: https://github.com/psf/black
11 | rev: 22.10.0
12 | hooks:
13 | - id: black
14 | language_version: python3.8.12
15 | args: # arguments to configure black
16 | - --line-length=80
17 |
18 | - repo: https://github.com/pycqa/isort
19 | rev: 5.10.1
20 | hooks:
21 | - id: isort
22 |
--------------------------------------------------------------------------------
/CAM_nerfacc/.readthedocs.yaml:
--------------------------------------------------------------------------------
1 | version: 2
2 |
3 | build:
4 | os: ubuntu-20.04
5 | tools:
6 | python: "3.9"
7 |
8 | sphinx:
9 | fail_on_warning: true
10 | configuration: docs/source/conf.py
11 |
12 | python:
13 | install:
14 | # Equivalent to 'pip install .'
15 | - method: pip
16 | path: .
17 | # Equivalent to 'pip install -r docs/requirements.txt'
18 | - requirements: docs/requirements.txt
--------------------------------------------------------------------------------
/CAM_nerfacc/CHANGELOG.md:
--------------------------------------------------------------------------------
1 |
2 | # Change Log
3 | All notable changes to this project will be documented in this file.
4 |
5 | The format is based on [Keep a Changelog](http://keepachangelog.com/)
6 | and this project adheres to [Semantic Versioning](http://semver.org/).
7 |
8 | ## [0.5.0] - 2023-04-04
9 |
10 | This is a major upgrade in which 90% of the code has been rewritten. In this version
11 | we achieves:
12 |
13 | 
14 |
15 | Links:
16 | - Documentation: https://www.nerfacc.com/en/v0.5.0/
17 | - ArXiv Report: Coming Soon.
18 |
19 | Methodologies:
20 | - Upgrade Occupancy Grid to support multiple levels.
21 | - Support Proposal Network from Mip-NeRF 360.
22 | - Update examples on unbounded scenes to use Multi-level Occupancy Grid or Proposal Network.
23 | - Contraction for Occupancy Grid is no longer supported due to it's inefficiency for ray traversal.
24 |
25 | API Changes:
26 | - [Changed] `OccupancyGrid()` -> `OccGridEstimator()`.
27 | - [Added] Argument `levels=1` for multi-level support.
28 | - [Added] Function `self.sampling()` that does basically the same thing with the old `nerfacc.ray_marching`.
29 | - [Renamed] Function `self.every_n_step()` -> `self.update_every_n_steps()`
30 | - [Added] `PropNetEstimator()`. With functions `self.sampling()`, `self.update_every_n_steps()`
31 | and `self.compute_loss()`.
32 | - [Removed] `ray_marching()`. Ray marching is now implemented through calling `sampling()` of
33 | the `OccGridEstimator()` / `PropNetEstimator()`.
34 | - [Changed] `ray_aabb_intersect()` now supports multiple aabb, and supports new argument `near_plane`, `far_plane`, `miss_value`.
35 | - [Changed] `render_*_from_*()`. The input shape changes from `(all_samples, 1)` to `(all_samples)`. And the function will returns all intermediate results so it might be a tuple.
36 | - [Changed] `rendering()`. The input shape changes from `(all_samples, 1)` to `(all_samples)`, including the shape assumption for the `rgb_sigma_fn` and `rgb_alpha_fn`. Be aware of this shape change.
37 | - [Changed] `accumulate_along_rays()`. The shape of the `weights` in the inputs should be `(all_samples)` now.
38 | - [Removed] `unpack_info()`, `pack_data()`, `unpack_data()` are temporally removed due to in-compatibility
39 | with the new backend implementation. Will add them back later.
40 | - [Added] Some basic functions that support both batched tensor and flattened tensor: `inclusive_prod()`, `inclusive_sum()`, `exclusive_prod()`, `exclusive_sum()`, `importance_sampling()`, `searchsorted()`.
41 |
42 | Examples & Benchmarks:
43 | - More benchmarks and examples. See folder `examples/` and `benchmarks/`.
44 |
45 | ## [0.3.5] - 2023-02-23
46 |
47 | A stable version that achieves:
48 | - The vanilla Nerf model with 8-layer MLPs can be trained to better quality (+0.5 PNSR) in 1 hour rather than days as in the paper.
49 | - The Instant-NGP Nerf model can be trained to equal quality in 4.5 minutes, comparing to the official pure-CUDA implementation.
50 | - The D-Nerf model for dynamic objects can also be trained in 1 hour rather than 2 days as in the paper, and with better quality (+~2.5 PSNR).
51 | - Both bounded and unbounded scenes are supported.
52 |
53 | Links:
54 | - Documentation: https://www.nerfacc.com/en/v0.3.5/
55 | - ArXiv Report: https://arxiv.org/abs/2210.04847v2/
56 |
57 | Methodologies:
58 | - Single resolution `nerfacc.OccupancyGrid` for synthetic scenes.
59 | - Contraction methods `nerfacc.ContractionType` for unbounded scenes.
60 |
--------------------------------------------------------------------------------
/CAM_nerfacc/LICENSE.txt:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 Ruilong Li
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 | THIS SOFTWARE AND/OR DATA WAS DEPOSITED IN THE BAIR OPEN RESEARCH COMMONS
24 | REPOSITORY ON 05/08/2023.
25 |
--------------------------------------------------------------------------------
/CAM_nerfacc/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include nerfacc/cuda/csrc/include/*
2 | include nerfacc/cuda/csrc/*
3 |
4 | include nerfacc/_cuda/csrc/include/*
5 | include nerfacc/_cuda/csrc/*
6 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/datasets/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_nerfacc/examples/datasets/__init__.py
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/datasets/utils.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | """
4 |
5 | import collections
6 |
7 | Rays = collections.namedtuple("Rays", ("origins", "viewdirs"))
8 |
9 |
10 | def namedtuple_map(fn, tup):
11 | """Apply `fn` to each element of `tup` and cast to `tup`'s namedtuple."""
12 | return type(tup)(*(None if x is None else fn(x) for x in tup))
13 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.sw*
3 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/LICENSE.txt:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 True Price, UNC Chapel Hill
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/README.md:
--------------------------------------------------------------------------------
1 | # pycolmap
2 | Python interface for COLMAP reconstructions, plus some convenient scripts for loading/modifying/converting reconstructions.
3 |
4 | This code does not, however, run reconstruction -- it only provides a convenient interface for handling COLMAP's output.
5 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/pycolmap/__init__.py:
--------------------------------------------------------------------------------
1 | from camera import Camera
2 | from database import COLMAPDatabase
3 | from image import Image
4 | from scene_manager import SceneManager
5 | from rotation import Quaternion, DualQuaternion
6 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/pycolmap/image.py:
--------------------------------------------------------------------------------
1 | # Author: True Price
2 |
3 | import numpy as np
4 |
5 | #-------------------------------------------------------------------------------
6 | #
7 | # Image
8 | #
9 | #-------------------------------------------------------------------------------
10 |
11 | class Image:
12 | def __init__(self, name_, camera_id_, q_, tvec_):
13 | self.name = name_
14 | self.camera_id = camera_id_
15 | self.q = q_
16 | self.tvec = tvec_
17 |
18 | self.points2D = np.empty((0, 2), dtype=np.float64)
19 | self.point3D_ids = np.empty((0,), dtype=np.uint64)
20 |
21 | #---------------------------------------------------------------------------
22 |
23 | def R(self):
24 | return self.q.ToR()
25 |
26 | #---------------------------------------------------------------------------
27 |
28 | def C(self):
29 | return -self.R().T.dot(self.tvec)
30 |
31 | #---------------------------------------------------------------------------
32 |
33 | @property
34 | def t(self):
35 | return self.tvec
36 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/tools/colmap_to_nvm.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | import sys
3 | sys.path.append("..")
4 |
5 | import numpy as np
6 |
7 | from pycolmap import Quaternion, SceneManager
8 |
9 |
10 | #-------------------------------------------------------------------------------
11 |
12 | def main(args):
13 | scene_manager = SceneManager(args.input_folder)
14 | scene_manager.load()
15 |
16 | with open(args.output_file, "w") as fid:
17 | fid.write("NVM_V3\n \n{:d}\n".format(len(scene_manager.images)))
18 |
19 | image_fmt_str = " {:.3f} " + 7 * "{:.7f} "
20 | for image_id, image in scene_manager.images.iteritems():
21 | camera = scene_manager.cameras[image.camera_id]
22 | f = 0.5 * (camera.fx + camera.fy)
23 | fid.write(args.image_name_prefix + image.name)
24 | fid.write(image_fmt_str.format(
25 | *((f,) + tuple(image.q.q) + tuple(image.C()))))
26 | if camera.distortion_func is None:
27 | fid.write("0 0\n")
28 | else:
29 | fid.write("{:.7f} 0\n".format(-camera.k1))
30 |
31 | image_id_to_idx = dict(
32 | (image_id, i) for i, image_id in enumerate(scene_manager.images))
33 |
34 | fid.write("{:d}\n".format(len(scene_manager.points3D)))
35 | for i, point3D_id in enumerate(scene_manager.point3D_ids):
36 | fid.write(
37 | "{:.7f} {:.7f} {:.7f} ".format(*scene_manager.points3D[i]))
38 | fid.write(
39 | "{:d} {:d} {:d} ".format(*scene_manager.point3D_colors[i]))
40 | keypoints = [
41 | (image_id_to_idx[image_id], kp_idx) +
42 | tuple(scene_manager.images[image_id].points2D[kp_idx])
43 | for image_id, kp_idx in
44 | scene_manager.point3D_id_to_images[point3D_id]]
45 | fid.write("{:d}".format(len(keypoints)))
46 | fid.write(
47 | (len(keypoints) * " {:d} {:d} {:.3f} {:.3f}" + "\n").format(
48 | *itertools.chain(*keypoints)))
49 |
50 |
51 | #-------------------------------------------------------------------------------
52 |
53 | if __name__ == "__main__":
54 | import argparse
55 |
56 | parser = argparse.ArgumentParser(
57 | description="Save a COLMAP reconstruction in the NVM format "
58 | "(http://ccwu.me/vsfm/doc.html#nvm).",
59 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
60 |
61 | parser.add_argument("input_folder")
62 | parser.add_argument("output_file")
63 |
64 | parser.add_argument("--image_name_prefix", type=str, default="",
65 | help="prefix image names with this string (e.g., 'images/')")
66 |
67 | args = parser.parse_args()
68 |
69 | main(args)
70 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/tools/delete_images.py:
--------------------------------------------------------------------------------
1 | import sys
2 | sys.path.append("..")
3 |
4 | import numpy as np
5 |
6 | from pycolmap import DualQuaternion, Image, SceneManager
7 |
8 |
9 | #-------------------------------------------------------------------------------
10 |
11 | def main(args):
12 | scene_manager = SceneManager(args.input_folder)
13 | scene_manager.load()
14 |
15 | image_ids = map(scene_manager.get_image_from_name,
16 | iter(lambda: sys.stdin.readline().strip(), ""))
17 | scene_manager.delete_images(image_ids)
18 |
19 | scene_manager.save(args.output_folder)
20 |
21 |
22 | #-------------------------------------------------------------------------------
23 |
24 | if __name__ == "__main__":
25 | import argparse
26 |
27 | parser = argparse.ArgumentParser(
28 | description="Deletes images (filenames read from stdin) from a model.",
29 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
30 |
31 | parser.add_argument("input_folder")
32 | parser.add_argument("output_folder")
33 |
34 | args = parser.parse_args()
35 |
36 | main(args)
37 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/tools/save_cameras_as_ply.py:
--------------------------------------------------------------------------------
1 | import sys
2 | sys.path.append("..")
3 |
4 | import numpy as np
5 | import os
6 |
7 | from pycolmap import SceneManager
8 |
9 |
10 | #-------------------------------------------------------------------------------
11 |
12 | # Saves the cameras as a mesh
13 | #
14 | # inputs:
15 | # - ply_file: output file
16 | # - images: ordered array of pycolmap Image objects
17 | # - color: color string for the camera
18 | # - scale: amount to shrink/grow the camera model
19 | def save_camera_ply(ply_file, images, scale):
20 | points3D = scale * np.array((
21 | (0., 0., 0.),
22 | (-1., -1., 1.),
23 | (-1., 1., 1.),
24 | (1., -1., 1.),
25 | (1., 1., 1.)))
26 |
27 | faces = np.array(((0, 2, 1),
28 | (0, 4, 2),
29 | (0, 3, 4),
30 | (0, 1, 3),
31 | (1, 2, 4),
32 | (1, 4, 3)))
33 |
34 | r = np.linspace(0, 255, len(images), dtype=np.uint8)
35 | g = 255 - r
36 | b = r - np.linspace(0, 128, len(images), dtype=np.uint8)
37 | color = np.column_stack((r, g, b))
38 |
39 | with open(ply_file, "w") as fid:
40 | print>>fid, "ply"
41 | print>>fid, "format ascii 1.0"
42 | print>>fid, "element vertex", len(points3D) * len(images)
43 | print>>fid, "property float x"
44 | print>>fid, "property float y"
45 | print>>fid, "property float z"
46 | print>>fid, "property uchar red"
47 | print>>fid, "property uchar green"
48 | print>>fid, "property uchar blue"
49 | print>>fid, "element face", len(faces) * len(images)
50 | print>>fid, "property list uchar int vertex_index"
51 | print>>fid, "end_header"
52 |
53 | for image, c in zip(images, color):
54 | for p3D in (points3D.dot(image.R()) + image.C()):
55 | print>>fid, p3D[0], p3D[1], p3D[2], c[0], c[1], c[2]
56 |
57 | for i in xrange(len(images)):
58 | for f in (faces + len(points3D) * i):
59 | print>>fid, "3 {} {} {}".format(*f)
60 |
61 |
62 | #-------------------------------------------------------------------------------
63 |
64 | def main(args):
65 | scene_manager = SceneManager(args.input_folder)
66 | scene_manager.load_images()
67 |
68 | images = sorted(scene_manager.images.itervalues(),
69 | key=lambda image: image.name)
70 |
71 | save_camera_ply(args.output_file, images, args.scale)
72 |
73 |
74 | #-------------------------------------------------------------------------------
75 |
76 | if __name__ == "__main__":
77 | import argparse
78 |
79 | parser = argparse.ArgumentParser(
80 | description="Saves camera positions to a PLY for easy viewing outside "
81 | "of COLMAP. Currently, camera FoV is not reflected in the output.",
82 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
83 |
84 | parser.add_argument("input_folder")
85 | parser.add_argument("output_file")
86 |
87 | parser.add_argument("--scale", type=float, default=1.,
88 | help="Scaling factor for the camera mesh.")
89 |
90 | args = parser.parse_args()
91 |
92 | main(args)
93 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/tools/transform_model.py:
--------------------------------------------------------------------------------
1 | import sys
2 | sys.path.append("..")
3 |
4 | import numpy as np
5 |
6 | from pycolmap import Quaternion, SceneManager
7 |
8 |
9 | #-------------------------------------------------------------------------------
10 |
11 | def main(args):
12 | scene_manager = SceneManager(args.input_folder)
13 | scene_manager.load()
14 |
15 | # expect each line of input corresponds to one row
16 | P = np.array([
17 | map(float, sys.stdin.readline().strip().split()) for _ in xrange(3)])
18 |
19 | scene_manager.points3D[:] = scene_manager.points3D.dot(P[:,:3].T) + P[:,3]
20 |
21 | # get rotation without any global scaling (assuming isotropic scaling)
22 | scale = np.cbrt(np.linalg.det(P[:,:3]))
23 | q_old_from_new = ~Quaternion.FromR(P[:,:3] / scale)
24 |
25 | for image in scene_manager.images.itervalues():
26 | image.q *= q_old_from_new
27 | image.tvec = scale * image.tvec - image.R().dot(P[:,3])
28 |
29 | scene_manager.save(args.output_folder)
30 |
31 |
32 | #-------------------------------------------------------------------------------
33 |
34 | if __name__ == "__main__":
35 | import argparse
36 |
37 | parser = argparse.ArgumentParser(
38 | description="Apply a 3x4 transformation matrix to a COLMAP model and "
39 | "save the result as a new model. Row-major input can be piped in from "
40 | "a file or entered via the command line.",
41 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
42 |
43 | parser.add_argument("input_folder")
44 | parser.add_argument("output_folder")
45 |
46 | args = parser.parse_args()
47 |
48 | main(args)
49 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/tools/write_camera_track_to_bundler.py:
--------------------------------------------------------------------------------
1 | import sys
2 | sys.path.append("..")
3 |
4 | import numpy as np
5 |
6 | from pycolmap import SceneManager
7 |
8 |
9 | #-------------------------------------------------------------------------------
10 |
11 | def main(args):
12 | scene_manager = SceneManager(args.input_folder)
13 | scene_manager.load_cameras()
14 | scene_manager.load_images()
15 |
16 | if args.sort:
17 | images = sorted(
18 | scene_manager.images.itervalues(), key=lambda im: im.name)
19 | else:
20 | images = scene_manager.images.values()
21 |
22 | fid = open(args.output_file, "w")
23 | fid_filenames = open(args.output_file + ".list.txt", "w")
24 |
25 | print>>fid, "# Bundle file v0.3"
26 | print>>fid, len(images), 0
27 |
28 | for image in images:
29 | print>>fid_filenames, image.name
30 | camera = scene_manager.cameras[image.camera_id]
31 | print>>fid, 0.5 * (camera.fx + camera.fy), 0, 0
32 | R, t = image.R(), image.t
33 | print>>fid, R[0, 0], R[0, 1], R[0, 2]
34 | print>>fid, -R[1, 0], -R[1, 1], -R[1, 2]
35 | print>>fid, -R[2, 0], -R[2, 1], -R[2, 2]
36 | print>>fid, t[0], -t[1], -t[2]
37 |
38 | fid.close()
39 | fid_filenames.close()
40 |
41 |
42 | #-------------------------------------------------------------------------------
43 |
44 | if __name__ == "__main__":
45 | import argparse
46 |
47 | parser = argparse.ArgumentParser(
48 | description="Saves the camera positions in the Bundler format. Note "
49 | "that 3D points are not saved.",
50 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
51 |
52 | parser.add_argument("input_folder")
53 | parser.add_argument("output_file")
54 |
55 | parser.add_argument("--sort", default=False, action="store_true",
56 | help="sort the images by their filename")
57 |
58 | args = parser.parse_args()
59 |
60 | main(args)
61 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/pycolmap/tools/write_depthmap_to_ply.py:
--------------------------------------------------------------------------------
1 | import sys
2 | sys.path.append("..")
3 |
4 | import imageio
5 | import numpy as np
6 | import os
7 |
8 | from plyfile import PlyData, PlyElement
9 | from pycolmap import SceneManager
10 | from scipy.ndimage.interpolation import zoom
11 |
12 |
13 | #-------------------------------------------------------------------------------
14 |
15 | def main(args):
16 | suffix = ".photometric.bin" if args.photometric else ".geometric.bin"
17 |
18 | image_file = os.path.join(args.dense_folder, "images", args.image_filename)
19 | depth_file = os.path.join(
20 | args.dense_folder, args.stereo_folder, "depth_maps",
21 | args.image_filename + suffix)
22 | if args.save_normals:
23 | normals_file = os.path.join(
24 | args.dense_folder, args.stereo_folder, "normal_maps",
25 | args.image_filename + suffix)
26 |
27 | # load camera intrinsics from the COLMAP reconstruction
28 | scene_manager = SceneManager(os.path.join(args.dense_folder, "sparse"))
29 | scene_manager.load_cameras()
30 | scene_manager.load_images()
31 |
32 | image_id, image = scene_manager.get_image_from_name(args.image_filename)
33 | camera = scene_manager.cameras[image.camera_id]
34 | rotation_camera_from_world = image.R()
35 | camera_center = image.C()
36 |
37 | # load image, depth map, and normal map
38 | image = imageio.imread(image_file)
39 |
40 | with open(depth_file, "rb") as fid:
41 | w = int("".join(iter(lambda: fid.read(1), "&")))
42 | h = int("".join(iter(lambda: fid.read(1), "&")))
43 | c = int("".join(iter(lambda: fid.read(1), "&")))
44 | depth_map = np.fromfile(fid, np.float32).reshape(h, w)
45 | if (h, w) != image.shape[:2]:
46 | depth_map = zoom(
47 | depth_map,
48 | (float(image.shape[0]) / h, float(image.shape[1]) / w),
49 | order=0)
50 |
51 | if args.save_normals:
52 | with open(normals_file, "rb") as fid:
53 | w = int("".join(iter(lambda: fid.read(1), "&")))
54 | h = int("".join(iter(lambda: fid.read(1), "&")))
55 | c = int("".join(iter(lambda: fid.read(1), "&")))
56 | normals = np.fromfile(
57 | fid, np.float32).reshape(c, h, w).transpose([1, 2, 0])
58 | if (h, w) != image.shape[:2]:
59 | normals = zoom(
60 | normals,
61 | (float(image.shape[0]) / h, float(image.shape[1]) / w, 1.),
62 | order=0)
63 |
64 | if args.min_depth is not None:
65 | depth_map[depth_map < args.min_depth] = 0.
66 | if args.max_depth is not None:
67 | depth_map[depth_map > args.max_depth] = 0.
68 |
69 | # create 3D points
70 | #depth_map = np.minimum(depth_map, 100.)
71 | points3D = np.dstack(camera.get_image_grid() + [depth_map])
72 | points3D[:,:,:2] *= depth_map[:,:,np.newaxis]
73 |
74 | # save
75 | points3D = points3D.astype(np.float32).reshape(-1, 3)
76 | if args.save_normals:
77 | normals = normals.astype(np.float32).reshape(-1, 3)
78 | image = image.reshape(-1, 3)
79 | if image.dtype != np.uint8:
80 | if image.max() <= 1:
81 | image = (image * 255.).astype(np.uint8)
82 | else:
83 | image = image.astype(np.uint8)
84 |
85 | if args.world_space:
86 | points3D = points3D.dot(rotation_camera_from_world) + camera_center
87 | if args.save_normals:
88 | normals = normals.dot(rotation_camera_from_world)
89 |
90 | if args.save_normals:
91 | vertices = np.rec.fromarrays(
92 | tuple(points3D.T) + tuple(normals.T) + tuple(image.T),
93 | names="x,y,z,nx,ny,nz,red,green,blue")
94 | else:
95 | vertices = np.rec.fromarrays(
96 | tuple(points3D.T) + tuple(image.T), names="x,y,z,red,green,blue")
97 | vertices = PlyElement.describe(vertices, "vertex")
98 | PlyData([vertices]).write(args.output_filename)
99 |
100 |
101 | #-------------------------------------------------------------------------------
102 |
103 | if __name__ == "__main__":
104 | import argparse
105 |
106 | parser = argparse.ArgumentParser(
107 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
108 |
109 | parser.add_argument("dense_folder", type=str)
110 | parser.add_argument("image_filename", type=str)
111 | parser.add_argument("output_filename", type=str)
112 |
113 | parser.add_argument(
114 | "--photometric", default=False, action="store_true",
115 | help="use photometric depthmap instead of geometric")
116 |
117 | parser.add_argument(
118 | "--world_space", default=False, action="store_true",
119 | help="apply the camera->world extrinsic transformation to the result")
120 |
121 | parser.add_argument(
122 | "--save_normals", default=False, action="store_true",
123 | help="load the estimated normal map and save as part of the PLY")
124 |
125 | parser.add_argument(
126 | "--stereo_folder", type=str, default="stereo",
127 | help="folder in the dense workspace containing depth and normal maps")
128 |
129 | parser.add_argument(
130 | "--min_depth", type=float, default=None,
131 | help="set pixels with depth less than this value to zero depth")
132 |
133 | parser.add_argument(
134 | "--max_depth", type=float, default=None,
135 | help="set pixels with depth greater than this value to zero depth")
136 |
137 | args = parser.parse_args()
138 |
139 | main(args)
140 |
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/radiance_fields/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_nerfacc/examples/radiance_fields/__init__.py
--------------------------------------------------------------------------------
/CAM_nerfacc/examples/requirements.txt:
--------------------------------------------------------------------------------
1 | git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
2 | opencv-python
3 | imageio
4 | numpy
5 | tqdm
6 | scipy
7 | lpips
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | """
4 | from .data_specs import RayIntervals, RaySamples
5 | from .estimators.occ_grid import OccGridEstimator
6 | from .estimators.prop_net import PropNetEstimator
7 | from .grid import ray_aabb_intersect, traverse_grids
8 | from .pack import pack_info
9 | from .pdf import importance_sampling, searchsorted
10 | from .scan import exclusive_prod, exclusive_sum, inclusive_prod, inclusive_sum
11 | from .version import __version__
12 | from .volrend import (
13 | accumulate_along_rays,
14 | render_transmittance_from_alpha,
15 | render_transmittance_from_density,
16 | render_visibility_from_alpha,
17 | render_visibility_from_density,
18 | render_weight_from_alpha,
19 | render_weight_from_density,
20 | rendering,
21 | )
22 |
23 | __all__ = [
24 | "__version__",
25 | "inclusive_prod",
26 | "exclusive_prod",
27 | "inclusive_sum",
28 | "exclusive_sum",
29 | "pack_info",
30 | "render_visibility_from_alpha",
31 | "render_visibility_from_density",
32 | "render_weight_from_alpha",
33 | "render_weight_from_density",
34 | "render_transmittance_from_alpha",
35 | "render_transmittance_from_density",
36 | "accumulate_along_rays",
37 | "rendering",
38 | "importance_sampling",
39 | "searchsorted",
40 | "RayIntervals",
41 | "RaySamples",
42 | "ray_aabb_intersect",
43 | "traverse_grids",
44 | "OccGridEstimator",
45 | "PropNetEstimator",
46 | ]
47 |
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cameras2.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 |
4 | Seems like both colmap and nerfstudio are based on OpenCV's camera model.
5 |
6 | References:
7 | - nerfstudio: https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/cameras/cameras.py
8 | - opencv:
9 | - https://docs.opencv.org/3.4/da/d54/group__imgproc__transform.html#ga69f2545a8b62a6b0fc2ee060dc30559d
10 | - https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html
11 | - https://docs.opencv.org/4.x/db/d58/group__calib3d__fisheye.html
12 | - https://github.com/opencv/opencv/blob/master/modules/calib3d/src/fisheye.cpp#L321
13 | - https://github.com/opencv/opencv/blob/17234f82d025e3bbfbf611089637e5aa2038e7b8/modules/calib3d/src/distortion_model.hpp
14 | - https://github.com/opencv/opencv/blob/8d0fbc6a1e9f20c822921e8076551a01e58cd632/modules/calib3d/src/undistort.dispatch.cpp#L578
15 | - colmap: https://github.com/colmap/colmap/blob/dev/src/base/camera_models.h
16 | - calcam: https://euratom-software.github.io/calcam/html/intro_theory.html
17 | - blender:
18 | - https://docs.blender.org/manual/en/latest/render/cycles/object_settings/cameras.html#fisheye-lens-polynomial
19 | - https://github.com/blender/blender/blob/03cc3b94c94c38767802bccac4e9384ab704065a/intern/cycles/kernel/kernel_projection.h
20 | - lensfun: https://lensfun.github.io/manual/v0.3.2/annotated.html
21 |
22 | - OpenCV and Blender has different fisheye camera models
23 | - https://stackoverflow.com/questions/73270140/pipeline-for-fisheye-distortion-and-undistortion-with-blender-and-opencv
24 | """
25 | from typing import Literal, Optional, Tuple
26 |
27 | import torch
28 | import torch.nn.functional as F
29 | from torch import Tensor
30 |
31 | from . import cuda as _C
32 |
33 |
34 | def ray_directions_from_uvs(
35 | uvs: Tensor, # [..., 2]
36 | Ks: Tensor, # [..., 3, 3]
37 | params: Optional[Tensor] = None, # [..., M]
38 | ) -> Tensor:
39 | """Create ray directions from uvs and camera parameters in OpenCV format.
40 |
41 | Args:
42 | uvs: UV coordinates on image plane. (In pixel unit)
43 | Ks: Camera intrinsics.
44 | params: Camera distortion parameters. See `opencv.undistortPoints` for details.
45 |
46 | Returns:
47 | Normalized ray directions in camera space.
48 | """
49 | u, v = torch.unbind(uvs + 0.5, dim=-1)
50 | fx, fy = Ks[..., 0, 0], Ks[..., 1, 1]
51 | cx, cy = Ks[..., 0, 2], Ks[..., 1, 2]
52 |
53 | # undo intrinsics
54 | xys = torch.stack([(u - cx) / fx, (v - cy) / fy], dim=-1) # [..., 2]
55 |
56 | # undo lens distortion
57 | if params is not None:
58 | M = params.shape[-1]
59 |
60 | if M == 14: # undo tilt projection
61 | R, R_inv = opencv_tilt_projection_matrix(params[..., -2:])
62 | xys_homo = F.pad(xys, (0, 1), value=1.0) # [..., 3]
63 | xys_homo = torch.einsum(
64 | "...ij,...j->...i", R_inv, xys_homo
65 | ) # [..., 3]
66 | xys = xys_homo[..., :2]
67 | homo = xys_homo[..., 2:]
68 | xys /= torch.where(homo != 0.0, homo, torch.ones_like(homo))
69 |
70 | xys = opencv_lens_undistortion(xys, params) # [..., 2]
71 |
72 | # normalized homogeneous coordinates
73 | dirs = F.pad(xys, (0, 1), value=1.0) # [..., 3]
74 | dirs = F.normalize(dirs, dim=-1) # [..., 3]
75 | return dirs
76 |
77 |
78 | def opencv_lens_undistortion(
79 | uv: Tensor, params: Tensor, eps: float = 1e-6, iters: int = 10
80 | ) -> Tensor:
81 | """Undistort the opencv distortion of {k1, k2, k3, k4, p1, p2}.
82 |
83 | Note:
84 | This function is not differentiable to any inputs.
85 |
86 | Args:
87 | uv: (..., 2) UV coordinates.
88 | params: (..., 6) or (6) OpenCV distortion parameters.
89 |
90 | Returns:
91 | (..., 2) undistorted UV coordinates.
92 | """
93 | assert uv.shape[-1] == 2
94 | assert params.shape[-1] == 6
95 | batch_shape = uv.shape[:-1]
96 | params = torch.broadcast_to(params, batch_shape + (6,))
97 |
98 | return _C.opencv_lens_undistortion(
99 | uv.contiguous(), params.contiguous(), eps, iters
100 | )
101 |
102 |
103 | def opencv_tilt_projection_matrix(tau: Tensor) -> Tensor:
104 | """Create a tilt projection matrix.
105 |
106 | Reference:
107 | https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html
108 |
109 | Args:
110 | tau: (..., 2) tilt angles.
111 |
112 | Returns:
113 | (..., 3, 3) tilt projection matrix.
114 | """
115 |
116 | cosx, cosy = torch.unbind(torch.cos(tau), -1)
117 | sinx, siny = torch.unbind(torch.sin(tau), -1)
118 | one = torch.ones_like(tau)
119 | zero = torch.zeros_like(tau)
120 |
121 | Rx = torch.stack(
122 | [one, zero, zero, zero, cosx, sinx, zero, -sinx, cosx], -1
123 | ).reshape(*tau.shape[:-1], 3, 3)
124 | Ry = torch.stack(
125 | [cosy, zero, -siny, zero, one, zero, siny, zero, cosy], -1
126 | ).reshape(*tau.shape[:-1], 3, 3)
127 | Rxy = torch.matmul(Ry, Rx)
128 | Rz = torch.stack(
129 | [
130 | Rxy[..., 2, 2],
131 | zero,
132 | -Rxy[..., 0, 2],
133 | zero,
134 | Rxy[..., 2, 2],
135 | -Rxy[..., 1, 2],
136 | zero,
137 | zero,
138 | one,
139 | ],
140 | -1,
141 | ).reshape(*tau.shape[:-1], 3, 3)
142 | R = torch.matmul(Rz, Rxy)
143 |
144 | inv = 1.0 / Rxy[..., 2, 2]
145 | Rz_inv = torch.stack(
146 | [
147 | inv,
148 | zero,
149 | inv * Rxy[..., 0, 2],
150 | zero,
151 | inv,
152 | inv * Rxy[..., 1, 2],
153 | zero,
154 | zero,
155 | one,
156 | ],
157 | -1,
158 | ).reshape(*tau.shape[:-1], 3, 3)
159 | R_inv = torch.matmul(Rxy.transpose(-1, -2), Rz_inv)
160 | return R, R_inv
161 |
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | """
4 |
5 | from typing import Any, Callable
6 |
7 |
8 | def _make_lazy_cuda_func(name: str) -> Callable:
9 | def call_cuda(*args, **kwargs):
10 | # pylint: disable=import-outside-toplevel
11 | from ._backend import _C
12 |
13 | return getattr(_C, name)(*args, **kwargs)
14 |
15 | return call_cuda
16 |
17 |
18 | is_cub_available = _make_lazy_cuda_func("is_cub_available")
19 |
20 | # data specs
21 | MultiScaleGridSpec = _make_lazy_cuda_func("MultiScaleGridSpec")
22 | RaysSpec = _make_lazy_cuda_func("RaysSpec")
23 | RaySegmentsSpec = _make_lazy_cuda_func("RaySegmentsSpec")
24 |
25 | # grid
26 | ray_aabb_intersect = _make_lazy_cuda_func("ray_aabb_intersect")
27 | traverse_grids = _make_lazy_cuda_func("traverse_grids")
28 |
29 | # scan
30 | exclusive_sum_by_key = _make_lazy_cuda_func("exclusive_sum_by_key")
31 | inclusive_sum = _make_lazy_cuda_func("inclusive_sum")
32 | exclusive_sum = _make_lazy_cuda_func("exclusive_sum")
33 | inclusive_prod_forward = _make_lazy_cuda_func("inclusive_prod_forward")
34 | inclusive_prod_backward = _make_lazy_cuda_func("inclusive_prod_backward")
35 | exclusive_prod_forward = _make_lazy_cuda_func("exclusive_prod_forward")
36 | exclusive_prod_backward = _make_lazy_cuda_func("exclusive_prod_backward")
37 |
38 | # pdf
39 | importance_sampling = _make_lazy_cuda_func("importance_sampling")
40 | searchsorted = _make_lazy_cuda_func("searchsorted")
41 |
42 | # camera
43 | opencv_lens_undistortion = _make_lazy_cuda_func("opencv_lens_undistortion")
44 | opencv_lens_undistortion_fisheye = _make_lazy_cuda_func(
45 | "opencv_lens_undistortion_fisheye"
46 | )
47 |
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/_backend.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | """
4 |
5 | import glob
6 | import json
7 | import os
8 | import shutil
9 | from subprocess import DEVNULL, call
10 |
11 | from rich.console import Console
12 | from torch.utils.cpp_extension import _get_build_directory, load
13 |
14 | PATH = os.path.dirname(os.path.abspath(__file__))
15 |
16 |
17 | def cuda_toolkit_available():
18 | """Check if the nvcc is avaiable on the machine."""
19 | try:
20 | call(["nvcc"], stdout=DEVNULL, stderr=DEVNULL)
21 | return True
22 | except FileNotFoundError:
23 | return False
24 |
25 |
26 | def cuda_toolkit_version():
27 | """Get the cuda toolkit version."""
28 | cuda_home = os.path.join(os.path.dirname(shutil.which("nvcc")), "..")
29 | if os.path.exists(os.path.join(cuda_home, "version.txt")):
30 | with open(os.path.join(cuda_home, "version.txt")) as f:
31 | cuda_version = f.read().strip().split()[-1]
32 | elif os.path.exists(os.path.join(cuda_home, "version.json")):
33 | with open(os.path.join(cuda_home, "version.json")) as f:
34 | cuda_version = json.load(f)["cuda"]["version"]
35 | else:
36 | raise RuntimeError("Cannot find the cuda version.")
37 | return cuda_version
38 |
39 |
40 | name = "nerfacc_cuda"
41 | build_dir = _get_build_directory(name, verbose=False)
42 | extra_include_paths = []
43 | extra_cflags = ["-O3"]
44 | extra_cuda_cflags = ["-O3"]
45 |
46 | _C = None
47 | sources = list(glob.glob(os.path.join(PATH, "csrc/*.cu"))) + list(
48 | glob.glob(os.path.join(PATH, "csrc/*.cpp"))
49 | )
50 |
51 | try:
52 | # try to import the compiled module (via setup.py)
53 | from nerfacc import csrc as _C
54 | except ImportError:
55 | # if failed, try with JIT compilation
56 | if cuda_toolkit_available():
57 | if os.listdir(build_dir) != []:
58 | # If the build exists, we assume the extension has been built
59 | # and we can load it.
60 |
61 | _C = load(
62 | name=name,
63 | sources=sources,
64 | extra_cflags=extra_cflags,
65 | extra_cuda_cflags=extra_cuda_cflags,
66 | extra_include_paths=extra_include_paths,
67 | )
68 | else:
69 | # Build from scratch. Remove the build directory just to be safe: pytorch jit might stuck
70 | # if the build directory exists.
71 | shutil.rmtree(build_dir)
72 | with Console().status(
73 | "[bold yellow]NerfAcc: Setting up CUDA (This may take a few minutes the first time)",
74 | spinner="bouncingBall",
75 | ):
76 | _C = load(
77 | name=name,
78 | sources=sources,
79 | extra_cflags=extra_cflags,
80 | extra_cuda_cflags=extra_cuda_cflags,
81 | extra_include_paths=extra_include_paths,
82 | )
83 | else:
84 | Console().print(
85 | "[yellow]NerfAcc: No CUDA toolkit found. NerfAcc will be disabled.[/yellow]"
86 | )
87 |
88 |
89 | __all__ = ["_C"]
90 |
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/csrc/include/data_spec.hpp:
--------------------------------------------------------------------------------
1 | #pragma once
2 |
3 | #include
4 | #include "utils_cuda.cuh"
5 |
6 | struct MultiScaleGridSpec {
7 | torch::Tensor data; // [levels, resx, resy, resz]
8 | torch::Tensor occupied; // [levels, resx, resy, resz]
9 | torch::Tensor base_aabb; // [6,]
10 |
11 | inline void check() {
12 | CHECK_INPUT(data);
13 | CHECK_INPUT(occupied);
14 | CHECK_INPUT(base_aabb);
15 |
16 | TORCH_CHECK(data.ndimension() == 4);
17 | TORCH_CHECK(occupied.ndimension() == 4);
18 | TORCH_CHECK(base_aabb.ndimension() == 1);
19 |
20 | TORCH_CHECK(data.numel() == occupied.numel());
21 | TORCH_CHECK(base_aabb.numel() == 6);
22 | }
23 | };
24 |
25 | struct RaysSpec {
26 | torch::Tensor origins; // [n_rays, 3]
27 | torch::Tensor dirs; // [n_rays, 3]
28 |
29 | inline void check() {
30 | CHECK_INPUT(origins);
31 | CHECK_INPUT(dirs);
32 |
33 | TORCH_CHECK(origins.ndimension() == 2);
34 | TORCH_CHECK(dirs.ndimension() == 2);
35 |
36 | TORCH_CHECK(origins.numel() == dirs.numel());
37 |
38 | TORCH_CHECK(origins.size(1) == 3);
39 | TORCH_CHECK(dirs.size(1) == 3);
40 | }
41 | };
42 |
43 |
44 | struct RaySegmentsSpec {
45 | torch::Tensor vals; // [n_edges] or [n_rays, n_edges_per_ray]
46 | // for flattened tensor
47 | torch::Tensor chunk_starts; // [n_rays]
48 | torch::Tensor chunk_cnts; // [n_rays]
49 | torch::Tensor ray_indices; // [n_edges]
50 | torch::Tensor is_left; // [n_edges] have n_bins true values
51 | torch::Tensor is_right; // [n_edges] have n_bins true values
52 |
53 | inline void check() {
54 | CHECK_INPUT(vals);
55 | TORCH_CHECK(vals.defined());
56 |
57 | // batched tensor [..., n_edges_per_ray]
58 | if (vals.ndimension() > 1) return;
59 |
60 | // flattend tensor [n_edges]
61 | CHECK_INPUT(chunk_starts);
62 | CHECK_INPUT(chunk_cnts);
63 | TORCH_CHECK(chunk_starts.defined());
64 | TORCH_CHECK(chunk_cnts.defined());
65 | TORCH_CHECK(chunk_starts.ndimension() == 1);
66 | TORCH_CHECK(chunk_cnts.ndimension() == 1);
67 | TORCH_CHECK(chunk_starts.numel() == chunk_cnts.numel());
68 | if (ray_indices.defined()) {
69 | CHECK_INPUT(ray_indices);
70 | TORCH_CHECK(ray_indices.ndimension() == 1);
71 | TORCH_CHECK(vals.numel() == ray_indices.numel());
72 | }
73 | if (is_left.defined()) {
74 | CHECK_INPUT(is_left);
75 | TORCH_CHECK(is_left.ndimension() == 1);
76 | TORCH_CHECK(vals.numel() == is_left.numel());
77 | }
78 | if (is_right.defined()) {
79 | CHECK_INPUT(is_right);
80 | TORCH_CHECK(is_right.ndimension() == 1);
81 | TORCH_CHECK(vals.numel() == is_right.numel());
82 | }
83 | }
84 |
85 | inline void memalloc_cnts(int32_t n_rays, at::TensorOptions options, bool zero_init = true) {
86 | TORCH_CHECK(!chunk_cnts.defined());
87 | if (zero_init) {
88 | chunk_cnts = torch::zeros({n_rays}, options.dtype(torch::kLong));
89 | } else {
90 | chunk_cnts = torch::empty({n_rays}, options.dtype(torch::kLong));
91 | }
92 | }
93 |
94 | inline int64_t memalloc_data(bool alloc_masks = true, bool zero_init = true) {
95 | TORCH_CHECK(chunk_cnts.defined());
96 | TORCH_CHECK(!chunk_starts.defined());
97 | TORCH_CHECK(!vals.defined());
98 |
99 | torch::Tensor cumsum = torch::cumsum(chunk_cnts, 0, chunk_cnts.scalar_type());
100 | int64_t n_edges = cumsum[-1].item();
101 |
102 | chunk_starts = cumsum - chunk_cnts;
103 | if (zero_init) {
104 | vals = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kFloat32));
105 | ray_indices = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kLong));
106 | if (alloc_masks) {
107 | is_left = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kBool));
108 | is_right = torch::zeros({n_edges}, chunk_cnts.options().dtype(torch::kBool));
109 | }
110 | } else {
111 | vals = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kFloat32));
112 | ray_indices = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kLong));
113 | if (alloc_masks) {
114 | is_left = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kBool));
115 | is_right = torch::empty({n_edges}, chunk_cnts.options().dtype(torch::kBool));
116 | }
117 | }
118 | return 1;
119 | }
120 | };
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/csrc/include/data_spec_packed.cuh:
--------------------------------------------------------------------------------
1 | #pragma once
2 |
3 | #include
4 |
5 | #include "data_spec.hpp"
6 |
7 | namespace {
8 | namespace device {
9 |
10 | struct PackedRaySegmentsSpec {
11 | PackedRaySegmentsSpec(RaySegmentsSpec& spec) :
12 | vals(spec.vals.defined() ? spec.vals.data_ptr() : nullptr),
13 | is_batched(spec.vals.defined() ? spec.vals.dim() > 1 : false),
14 | // for flattened tensor
15 | chunk_starts(spec.chunk_starts.defined() ? spec.chunk_starts.data_ptr() : nullptr),
16 | chunk_cnts(spec.chunk_cnts.defined() ? spec.chunk_cnts.data_ptr(): nullptr),
17 | ray_indices(spec.ray_indices.defined() ? spec.ray_indices.data_ptr() : nullptr),
18 | is_left(spec.is_left.defined() ? spec.is_left.data_ptr() : nullptr),
19 | is_right(spec.is_right.defined() ? spec.is_right.data_ptr() : nullptr),
20 | // for dimensions
21 | n_edges(spec.vals.defined() ? spec.vals.numel() : 0),
22 | n_rays(spec.chunk_cnts.defined() ? spec.chunk_cnts.size(0) : 0), // for flattened tensor
23 | n_edges_per_ray(spec.vals.defined() ? spec.vals.size(-1) : 0) // for batched tensor
24 | { }
25 |
26 | float* vals;
27 | bool is_batched;
28 |
29 | int64_t* chunk_starts;
30 | int64_t* chunk_cnts;
31 | int64_t* ray_indices;
32 | bool* is_left;
33 | bool* is_right;
34 |
35 | int64_t n_edges;
36 | int32_t n_rays;
37 | int32_t n_edges_per_ray;
38 | };
39 |
40 | struct PackedMultiScaleGridSpec {
41 | PackedMultiScaleGridSpec(MultiScaleGridSpec& spec) :
42 | data(spec.data.data_ptr()),
43 | occupied(spec.occupied.data_ptr()),
44 | base_aabb(spec.base_aabb.data_ptr()),
45 | levels(spec.data.size(0)),
46 | resolution{
47 | (int32_t)spec.data.size(1),
48 | (int32_t)spec.data.size(2),
49 | (int32_t)spec.data.size(3)}
50 | { }
51 | float* data;
52 | bool* occupied;
53 | float* base_aabb;
54 | int32_t levels;
55 | int3 resolution;
56 | };
57 |
58 | struct PackedRaysSpec {
59 | PackedRaysSpec(RaysSpec& spec) :
60 | origins(spec.origins.data_ptr()),
61 | dirs(spec.dirs.data_ptr()),
62 | N(spec.origins.size(0))
63 | { }
64 | float *origins;
65 | float *dirs;
66 | int32_t N;
67 | };
68 |
69 | struct SingleRaySpec {
70 | // TODO: check inv_dir if dir is zero.
71 | __device__ SingleRaySpec(
72 | float *rays_o, float *rays_d, float tmin, float tmax) :
73 | origin{rays_o[0], rays_o[1], rays_o[2]},
74 | dir{rays_d[0], rays_d[1], rays_d[2]},
75 | inv_dir{1.0f/rays_d[0], 1.0f/rays_d[1], 1.0f/rays_d[2]},
76 | tmin{tmin},
77 | tmax{tmax}
78 | { }
79 |
80 | __device__ SingleRaySpec(
81 | PackedRaysSpec& rays, int32_t id, float tmin, float tmax) :
82 | origin{
83 | rays.origins[id * 3],
84 | rays.origins[id * 3 + 1],
85 | rays.origins[id * 3 + 2]},
86 | dir{
87 | rays.dirs[id * 3],
88 | rays.dirs[id * 3 + 1],
89 | rays.dirs[id * 3 + 2]},
90 | inv_dir{
91 | 1.0f / rays.dirs[id * 3],
92 | 1.0f / rays.dirs[id * 3 + 1],
93 | 1.0f / rays.dirs[id * 3 + 2]},
94 | tmin{tmin},
95 | tmax{tmax}
96 | { }
97 | float3 origin;
98 | float3 dir;
99 | float3 inv_dir;
100 | float tmin;
101 | float tmax;
102 | };
103 |
104 | struct AABBSpec {
105 | __device__ AABBSpec(float *aabb) :
106 | min{aabb[0], aabb[1], aabb[2]},
107 | max{aabb[3], aabb[4], aabb[5]}
108 | { }
109 | __device__ AABBSpec(float3 min, float3 max) :
110 | min{min.x, min.y, min.z},
111 | max{max.x, max.y, max.z}
112 | { }
113 | float3 min;
114 | float3 max;
115 | };
116 |
117 |
118 | } // namespace device
119 | } // namespace
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/csrc/include/utils_contraction.cuh:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | */
4 |
5 | #pragma once
6 |
7 | #include "utils_math.cuh"
8 |
9 | namespace {
10 | namespace device {
11 |
12 | enum ContractionType
13 | {
14 | AABB = 0,
15 | UN_BOUNDED_TANH = 1,
16 | UN_BOUNDED_SPHERE = 2,
17 | };
18 |
19 | inline __device__ __host__ float3 roi_to_unit(
20 | const float3 xyz, const float3 roi_min, const float3 roi_max)
21 | {
22 | // roi -> [0, 1]^3
23 | return (xyz - roi_min) / (roi_max - roi_min);
24 | }
25 |
26 | inline __device__ __host__ float3 unit_to_roi(
27 | const float3 xyz, const float3 roi_min, const float3 roi_max)
28 | {
29 | // [0, 1]^3 -> roi
30 | return xyz * (roi_max - roi_min) + roi_min;
31 | }
32 |
33 | inline __device__ __host__ float3 inf_to_unit_tanh(
34 | const float3 xyz, float3 roi_min, const float3 roi_max)
35 | {
36 | /**
37 | [-inf, inf]^3 -> [0, 1]^3
38 | roi -> cube of [0.25, 0.75]^3
39 | **/
40 | float3 xyz_unit = roi_to_unit(xyz, roi_min, roi_max); // roi -> [0, 1]^3
41 | xyz_unit = xyz_unit - 0.5f; // roi -> [-0.5, 0.5]^3
42 | return make_float3(tanhf(xyz_unit.x), tanhf(xyz_unit.y), tanhf(xyz_unit.z)) * 0.5f + 0.5f;
43 | }
44 |
45 | inline __device__ __host__ float3 unit_to_inf_tanh(
46 | const float3 xyz, float3 roi_min, const float3 roi_max)
47 | {
48 | /**
49 | [0, 1]^3 -> [-inf, inf]^3
50 | cube of [0.25, 0.75]^3 -> roi
51 | **/
52 | float3 xyz_unit = clamp(
53 | make_float3(
54 | atanhf(xyz.x * 2.0f - 1.0f),
55 | atanhf(xyz.y * 2.0f - 1.0f),
56 | atanhf(xyz.z * 2.0f - 1.0f)),
57 | -1e10f,
58 | 1e10f);
59 | xyz_unit = xyz_unit + 0.5f;
60 | xyz_unit = unit_to_roi(xyz_unit, roi_min, roi_max);
61 | return xyz_unit;
62 | }
63 |
64 | inline __device__ __host__ float3 inf_to_unit_sphere(
65 | const float3 xyz, const float3 roi_min, const float3 roi_max)
66 | {
67 | /** From MipNeRF360
68 | [-inf, inf]^3 -> sphere of [0, 1]^3;
69 | roi -> sphere of [0.25, 0.75]^3
70 | **/
71 | float3 xyz_unit = roi_to_unit(xyz, roi_min, roi_max); // roi -> [0, 1]^3
72 | xyz_unit = xyz_unit * 2.0f - 1.0f; // roi -> [-1, 1]^3
73 |
74 | float norm_sq = dot(xyz_unit, xyz_unit);
75 | float norm = sqrt(norm_sq);
76 | if (norm > 1.0f)
77 | {
78 | xyz_unit = (2.0f - 1.0f / norm) * (xyz_unit / norm);
79 | }
80 | xyz_unit = xyz_unit * 0.25f + 0.5f; // [-1, 1]^3 -> [0.25, 0.75]^3
81 | return xyz_unit;
82 | }
83 |
84 | inline __device__ __host__ float3 unit_sphere_to_inf(
85 | const float3 xyz, const float3 roi_min, const float3 roi_max)
86 | {
87 | /** From MipNeRF360
88 | sphere of [0, 1]^3 -> [-inf, inf]^3;
89 | sphere of [0.25, 0.75]^3 -> roi
90 | **/
91 | float3 xyz_unit = (xyz - 0.5f) * 4.0f; // [0.25, 0.75]^3 -> [-1, 1]^3
92 |
93 | float norm_sq = dot(xyz_unit, xyz_unit);
94 | float norm = sqrt(norm_sq);
95 | if (norm > 1.0f)
96 | {
97 | xyz_unit = xyz_unit / fmaxf((2.0f * norm - 1.0f * norm_sq), 1e-10f);
98 | }
99 | xyz_unit = xyz_unit * 0.5f + 0.5f; // [-1, 1]^3 -> [0, 1]^3
100 | xyz_unit = unit_to_roi(xyz_unit, roi_min, roi_max); // [0, 1]^3 -> roi
101 | return xyz_unit;
102 | }
103 |
104 | inline __device__ __host__ float3 apply_contraction(
105 | const float3 xyz, const float3 roi_min, const float3 roi_max,
106 | const ContractionType type)
107 | {
108 | switch (type)
109 | {
110 | case AABB:
111 | return roi_to_unit(xyz, roi_min, roi_max);
112 | case UN_BOUNDED_TANH:
113 | return inf_to_unit_tanh(xyz, roi_min, roi_max);
114 | case UN_BOUNDED_SPHERE:
115 | return inf_to_unit_sphere(xyz, roi_min, roi_max);
116 | }
117 | }
118 |
119 | inline __device__ __host__ float3 apply_contraction_inv(
120 | const float3 xyz, const float3 roi_min, const float3 roi_max,
121 | const ContractionType type)
122 | {
123 | switch (type)
124 | {
125 | case AABB:
126 | return unit_to_roi(xyz, roi_min, roi_max);
127 | case UN_BOUNDED_TANH:
128 | return unit_to_inf_tanh(xyz, roi_min, roi_max);
129 | case UN_BOUNDED_SPHERE:
130 | return unit_sphere_to_inf(xyz, roi_min, roi_max);
131 | }
132 | }
133 |
134 | } // namespace device
135 | } // namespace
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/csrc/include/utils_cuda.cuh:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | */
4 |
5 | #pragma once
6 |
7 | #include
8 | #include
9 | #include
10 | // #include
11 |
12 | // cub support for scan by key is added to cub 1.15
13 | // in https://github.com/NVIDIA/cub/pull/376
14 | #if CUB_VERSION >= 101500
15 | #define CUB_SUPPORTS_SCAN_BY_KEY() 1
16 | #else
17 | #define CUB_SUPPORTS_SCAN_BY_KEY() 0
18 | #endif
19 |
20 | #define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
21 | #define CHECK_CONTIGUOUS(x) \
22 | TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
23 | #define CHECK_INPUT(x) \
24 | CHECK_CUDA(x); \
25 | CHECK_CONTIGUOUS(x)
26 | #define CUDA_GET_THREAD_ID(tid, Q) \
27 | const int tid = blockIdx.x * blockDim.x + threadIdx.x; \
28 | if (tid >= Q) \
29 | return
30 | #define CUDA_N_BLOCKS_NEEDED(Q, CUDA_N_THREADS) ((Q - 1) / CUDA_N_THREADS + 1)
31 | #define DEVICE_GUARD(_ten) \
32 | const at::cuda::OptionalCUDAGuard device_guard(device_of(_ten));
33 |
34 | // https://github.com/pytorch/pytorch/blob/233305a852e1cd7f319b15b5137074c9eac455f6/aten/src/ATen/cuda/cub.cuh#L38-L46
35 | #define CUB_WRAPPER(func, ...) do { \
36 | size_t temp_storage_bytes = 0; \
37 | func(nullptr, temp_storage_bytes, __VA_ARGS__); \
38 | auto& caching_allocator = *::c10::cuda::CUDACachingAllocator::get(); \
39 | auto temp_storage = caching_allocator.allocate(temp_storage_bytes); \
40 | func(temp_storage.get(), temp_storage_bytes, __VA_ARGS__); \
41 | AT_CUDA_CHECK(cudaGetLastError()); \
42 | } while (false)
43 |
44 | template
45 | inline __device__ __host__ scalar_t ceil_div(scalar_t a, scalar_t b)
46 | {
47 | return (a + b - 1) / b;
48 | }
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/csrc/include/utils_grid.cuh:
--------------------------------------------------------------------------------
1 | #pragma once
2 |
3 | #include "data_spec_packed.cuh"
4 | #include "utils_contraction.cuh"
5 | #include "utils_math.cuh"
6 |
7 | namespace {
8 | namespace device {
9 |
10 | inline __device__ bool ray_aabb_intersect(
11 | SingleRaySpec ray, AABBSpec aabb,
12 | // outputs
13 | float& tmin, float& tmax)
14 | {
15 | float tmin_temp{};
16 | float tmax_temp{};
17 |
18 | if (ray.inv_dir.x >= 0) {
19 | tmin = (aabb.min.x - ray.origin.x) * ray.inv_dir.x;
20 | tmax = (aabb.max.x - ray.origin.x) * ray.inv_dir.x;
21 | } else {
22 | tmin = (aabb.max.x - ray.origin.x) * ray.inv_dir.x;
23 | tmax = (aabb.min.x - ray.origin.x) * ray.inv_dir.x;
24 | }
25 |
26 | if (ray.inv_dir.y >= 0) {
27 | tmin_temp = (aabb.min.y - ray.origin.y) * ray.inv_dir.y;
28 | tmax_temp = (aabb.max.y - ray.origin.y) * ray.inv_dir.y;
29 | } else {
30 | tmin_temp = (aabb.max.y - ray.origin.y) * ray.inv_dir.y;
31 | tmax_temp = (aabb.min.y - ray.origin.y) * ray.inv_dir.y;
32 | }
33 |
34 | if (tmin > tmax_temp || tmin_temp > tmax) return false;
35 | if (tmin_temp > tmin) tmin = tmin_temp;
36 | if (tmax_temp < tmax) tmax = tmax_temp;
37 |
38 | if (ray.inv_dir.z >= 0) {
39 | tmin_temp = (aabb.min.z - ray.origin.z) * ray.inv_dir.z;
40 | tmax_temp = (aabb.max.z - ray.origin.z) * ray.inv_dir.z;
41 | } else {
42 | tmin_temp = (aabb.max.z - ray.origin.z) * ray.inv_dir.z;
43 | tmax_temp = (aabb.min.z - ray.origin.z) * ray.inv_dir.z;
44 | }
45 |
46 | if (tmin > tmax_temp || tmin_temp > tmax) return false;
47 | if (tmin_temp > tmin) tmin = tmin_temp;
48 | if (tmax_temp < tmax) tmax = tmax_temp;
49 |
50 | if (tmax <= 0) return false;
51 |
52 | tmin = fmaxf(tmin, ray.tmin);
53 | tmax = fminf(tmax, ray.tmax);
54 | return true;
55 | }
56 |
57 |
58 | inline __device__ void setup_traversal(
59 | SingleRaySpec ray, float tmin, float tmax, float eps,
60 | AABBSpec aabb, int3 resolution,
61 | // outputs
62 | float3 &delta, float3 &tdist,
63 | int3 &step_index, int3 ¤t_index, int3 &final_index)
64 | {
65 | const float3 res = make_float3(resolution);
66 | const float3 voxel_size = (aabb.max - aabb.min) / res;
67 | const float3 ray_start = ray.origin + ray.dir * (tmin + eps);
68 | const float3 ray_end = ray.origin + ray.dir * (tmax - eps);
69 |
70 | // get voxel index of start and end within grid
71 | // TODO: check float error here!
72 | current_index = make_int3(
73 | apply_contraction(ray_start, aabb.min, aabb.max, ContractionType::AABB)
74 | * res
75 | );
76 | current_index = clamp(current_index, make_int3(0, 0, 0), resolution - 1);
77 |
78 | final_index = make_int3(
79 | apply_contraction(ray_end, aabb.min, aabb.max, ContractionType::AABB)
80 | * res
81 | );
82 | final_index = clamp(final_index, make_int3(0, 0, 0), resolution - 1);
83 |
84 | //
85 | const int3 index_delta = make_int3(
86 | ray.dir.x > 0 ? 1 : 0, ray.dir.y > 0 ? 1 : 0, ray.dir.z > 0 ? 1 : 0
87 | );
88 | const int3 start_index = current_index + index_delta;
89 | const float3 tmax_xyz = ((aabb.min +
90 | ((make_float3(start_index) * voxel_size) - ray_start)) * ray.inv_dir) + tmin;
91 |
92 | tdist = make_float3(
93 | (ray.dir.x == 0.0f) ? tmax : tmax_xyz.x,
94 | (ray.dir.y == 0.0f) ? tmax : tmax_xyz.y,
95 | (ray.dir.z == 0.0f) ? tmax : tmax_xyz.z
96 | );
97 | // printf("tdist: %f %f %f\n", tdist.x, tdist.y, tdist.z);
98 |
99 | const float3 step_float = make_float3(
100 | (ray.dir.x == 0.0f) ? 0.0f : (ray.dir.x > 0.0f ? 1.0f : -1.0f),
101 | (ray.dir.y == 0.0f) ? 0.0f : (ray.dir.y > 0.0f ? 1.0f : -1.0f),
102 | (ray.dir.z == 0.0f) ? 0.0f : (ray.dir.z > 0.0f ? 1.0f : -1.0f)
103 | );
104 | step_index = make_int3(step_float);
105 | // printf("step_index: %d %d %d\n", step_index.x, step_index.y, step_index.z);
106 |
107 | const float3 delta_temp = voxel_size * ray.inv_dir * step_float;
108 | delta = make_float3(
109 | (ray.dir.x == 0.0f) ? tmax : delta_temp.x,
110 | (ray.dir.y == 0.0f) ? tmax : delta_temp.y,
111 | (ray.dir.z == 0.0f) ? tmax : delta_temp.z
112 | );
113 | // printf("delta: %f %f %f\n", delta.x, delta.y, delta.z);
114 | }
115 |
116 | inline __device__ bool single_traversal(
117 | float3& tdist, int3& current_index,
118 | const int3 overflow_index, const int3 step_index, const float3 delta) {
119 | if ((tdist.x < tdist.y) && (tdist.x < tdist.z)) {
120 | // X-axis traversal.
121 | current_index.x += step_index.x;
122 | tdist.x += delta.x;
123 | if (current_index.x == overflow_index.x) {
124 | return false;
125 | }
126 | } else if (tdist.y < tdist.z) {
127 | // Y-axis traversal.
128 | current_index.y += step_index.y;
129 | tdist.y += delta.y;
130 | if (current_index.y == overflow_index.y) {
131 | return false;
132 | }
133 | } else {
134 | // Z-axis traversal.
135 | current_index.z += step_index.z;
136 | tdist.z += delta.z;
137 | if (current_index.z == overflow_index.z) {
138 | return false;
139 | }
140 | }
141 | return true;
142 | }
143 |
144 |
145 | } // namespace device
146 | } // namespace
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/cuda/csrc/nerfacc.cpp:
--------------------------------------------------------------------------------
1 | // This file contains only Python bindings
2 | #include "include/data_spec.hpp"
3 |
4 | #include
5 |
6 | bool is_cub_available() {
7 | // FIXME: why return false?
8 | return (bool) CUB_SUPPORTS_SCAN_BY_KEY();
9 | }
10 |
11 | // scan
12 | torch::Tensor exclusive_sum_by_key(
13 | torch::Tensor indices,
14 | torch::Tensor inputs,
15 | bool backward);
16 | torch::Tensor inclusive_sum(
17 | torch::Tensor chunk_starts,
18 | torch::Tensor chunk_cnts,
19 | torch::Tensor inputs,
20 | bool normalize,
21 | bool backward);
22 | torch::Tensor exclusive_sum(
23 | torch::Tensor chunk_starts,
24 | torch::Tensor chunk_cnts,
25 | torch::Tensor inputs,
26 | bool normalize,
27 | bool backward);
28 | torch::Tensor inclusive_prod_forward(
29 | torch::Tensor chunk_starts,
30 | torch::Tensor chunk_cnts,
31 | torch::Tensor inputs);
32 | torch::Tensor inclusive_prod_backward(
33 | torch::Tensor chunk_starts,
34 | torch::Tensor chunk_cnts,
35 | torch::Tensor inputs,
36 | torch::Tensor outputs,
37 | torch::Tensor grad_outputs);
38 | torch::Tensor exclusive_prod_forward(
39 | torch::Tensor chunk_starts,
40 | torch::Tensor chunk_cnts,
41 | torch::Tensor inputs);
42 | torch::Tensor exclusive_prod_backward(
43 | torch::Tensor chunk_starts,
44 | torch::Tensor chunk_cnts,
45 | torch::Tensor inputs,
46 | torch::Tensor outputs,
47 | torch::Tensor grad_outputs);
48 |
49 | // grid
50 | std::vector ray_aabb_intersect(
51 | const torch::Tensor rays_o, // [n_rays, 3]
52 | const torch::Tensor rays_d, // [n_rays, 3]
53 | const torch::Tensor aabbs, // [n_aabbs, 6]
54 | const float near_plane,
55 | const float far_plane,
56 | const float miss_value);
57 | std::vector traverse_grids(
58 | // rays
59 | const torch::Tensor rays_o, // [n_rays, 3]
60 | const torch::Tensor rays_d, // [n_rays, 3]
61 | // grids
62 | const torch::Tensor binaries, // [n_grids, resx, resy, resz]
63 | const torch::Tensor aabbs, // [n_grids, 6]
64 | // intersections
65 | const torch::Tensor t_mins, // [n_rays, n_grids]
66 | const torch::Tensor t_maxs, // [n_rays, n_grids]
67 | const torch::Tensor hits, // [n_rays, n_grids]
68 | // options
69 | const torch::Tensor near_planes,
70 | const torch::Tensor far_planes,
71 | const float step_size,
72 | const float cone_angle,
73 | const bool compute_intervals,
74 | const bool compute_samples);
75 |
76 | // pdf
77 | std::vector importance_sampling(
78 | RaySegmentsSpec ray_segments,
79 | torch::Tensor cdfs,
80 | torch::Tensor n_intervels_per_ray,
81 | bool stratified);
82 | std::vector importance_sampling(
83 | RaySegmentsSpec ray_segments,
84 | torch::Tensor cdfs,
85 | int64_t n_intervels_per_ray,
86 | bool stratified);
87 | std::vector searchsorted(
88 | RaySegmentsSpec query,
89 | RaySegmentsSpec key);
90 |
91 | // cameras
92 | torch::Tensor opencv_lens_undistortion(
93 | const torch::Tensor& uv, // [..., 2]
94 | const torch::Tensor& params, // [..., 6]
95 | const float eps,
96 | const int max_iterations);
97 | torch::Tensor opencv_lens_undistortion_fisheye(
98 | const torch::Tensor& uv, // [..., 2]
99 | const torch::Tensor& params, // [..., 4]
100 | const float criteria_eps,
101 | const int criteria_iters);
102 |
103 |
104 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
105 | #define _REG_FUNC(funname) m.def(#funname, &funname)
106 | _REG_FUNC(is_cub_available); // TODO: check this function
107 |
108 | _REG_FUNC(exclusive_sum_by_key);
109 | _REG_FUNC(inclusive_sum);
110 | _REG_FUNC(exclusive_sum);
111 | _REG_FUNC(inclusive_prod_forward);
112 | _REG_FUNC(inclusive_prod_backward);
113 | _REG_FUNC(exclusive_prod_forward);
114 | _REG_FUNC(exclusive_prod_backward);
115 |
116 | _REG_FUNC(ray_aabb_intersect);
117 | _REG_FUNC(traverse_grids);
118 | _REG_FUNC(searchsorted);
119 |
120 | _REG_FUNC(opencv_lens_undistortion);
121 | _REG_FUNC(opencv_lens_undistortion_fisheye);
122 | #undef _REG_FUNC
123 |
124 | m.def("importance_sampling", py::overload_cast(&importance_sampling));
125 | m.def("importance_sampling", py::overload_cast(&importance_sampling));
126 |
127 | py::class_(m, "MultiScaleGridSpec")
128 | .def(py::init<>())
129 | .def_readwrite("data", &MultiScaleGridSpec::data)
130 | .def_readwrite("occupied", &MultiScaleGridSpec::occupied)
131 | .def_readwrite("base_aabb", &MultiScaleGridSpec::base_aabb);
132 |
133 | py::class_(m, "RaysSpec")
134 | .def(py::init<>())
135 | .def_readwrite("origins", &RaysSpec::origins)
136 | .def_readwrite("dirs", &RaysSpec::dirs);
137 |
138 | py::class_(m, "RaySegmentsSpec")
139 | .def(py::init<>())
140 | .def_readwrite("vals", &RaySegmentsSpec::vals)
141 | .def_readwrite("is_left", &RaySegmentsSpec::is_left)
142 | .def_readwrite("is_right", &RaySegmentsSpec::is_right)
143 | .def_readwrite("chunk_starts", &RaySegmentsSpec::chunk_starts)
144 | .def_readwrite("chunk_cnts", &RaySegmentsSpec::chunk_cnts)
145 | .def_readwrite("ray_indices", &RaySegmentsSpec::ray_indices);
146 | }
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/estimators/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/maincold2/CAM/a639d61147affc71f6aec0bf382fce3f9c47ccb2/CAM_nerfacc/nerfacc/estimators/__init__.py
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/estimators/base.py:
--------------------------------------------------------------------------------
1 | from typing import Any
2 |
3 | import torch
4 | import torch.nn as nn
5 |
6 |
7 | class AbstractEstimator(nn.Module):
8 | """An abstract Transmittance Estimator class for Sampling."""
9 |
10 | def __init__(self) -> None:
11 | super().__init__()
12 | self.register_buffer("_dummy", torch.empty(0), persistent=False)
13 |
14 | @property
15 | def device(self) -> torch.device:
16 | return self._dummy.device
17 |
18 | def sampling(self, *args, **kwargs) -> Any:
19 | raise NotImplementedError
20 |
21 | def update_every_n_steps(self, *args, **kwargs) -> None:
22 | raise NotImplementedError
23 |
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/pack.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | """
4 | from typing import Optional
5 |
6 | import torch
7 | from torch import Tensor
8 |
9 |
10 | @torch.no_grad()
11 | def pack_info(ray_indices: Tensor, n_rays: Optional[int] = None) -> Tensor:
12 | """Pack `ray_indices` to `packed_info`. Useful for converting per sample data to per ray data.
13 |
14 | Note:
15 | this function is not differentiable to any inputs.
16 |
17 | Args:
18 | ray_indices: Ray indices of the samples. LongTensor with shape (n_sample).
19 | n_rays: Number of rays. If None, it is inferred from `ray_indices`. Default is None.
20 |
21 | Returns:
22 | A LongTensor of shape (n_rays, 2) that specifies the start and count
23 | of each chunk in the flattened input tensor, with in total n_rays chunks.
24 |
25 | Example:
26 |
27 | .. code-block:: python
28 |
29 | >>> ray_indices = torch.tensor([0, 0, 1, 1, 1, 2, 2, 2, 2], device="cuda")
30 | >>> packed_info = pack_info(ray_indices, n_rays=3)
31 | >>> packed_info
32 | tensor([[0, 2], [2, 3], [5, 4]], device='cuda:0')
33 |
34 | """
35 | assert (
36 | ray_indices.dim() == 1
37 | ), "ray_indices must be a 1D tensor with shape (n_samples)."
38 | if ray_indices.is_cuda:
39 | device = ray_indices.device
40 | dtype = ray_indices.dtype
41 | if n_rays is None:
42 | n_rays = ray_indices.max().item() + 1
43 | chunk_cnts = torch.zeros((n_rays,), device=device, dtype=dtype)
44 | chunk_cnts.index_add_(0, ray_indices, torch.ones_like(ray_indices))
45 | chunk_starts = chunk_cnts.cumsum(dim=0, dtype=dtype) - chunk_cnts
46 | packed_info = torch.stack([chunk_starts, chunk_cnts], dim=-1)
47 | else:
48 | raise NotImplementedError("Only support cuda inputs.")
49 | return packed_info
50 |
--------------------------------------------------------------------------------
/CAM_nerfacc/nerfacc/version.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright (c) 2022 Ruilong Li, UC Berkeley.
3 | """
4 |
5 | __version__ = "0.5.1"
6 |
--------------------------------------------------------------------------------
/CAM_nerfacc/scripts/run_aws_listing.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | from collections import defaultdict
3 |
4 | from boto3 import client
5 |
6 | parser = argparse.ArgumentParser()
7 | parser.add_argument("--access_key_id", type=str, required=True)
8 | parser.add_argument("--secret_access_key", type=str, required=True)
9 | parser.add_argument("--bucket", type=str, required=True)
10 | parser.add_argument("--region", type=str, required=True)
11 | args = parser.parse_args()
12 |
13 | ROOT_URL = f"https://{args.bucket}.s3.{args.region}.amazonaws.com/whl"
14 | html = "\n\n\n{}\n\n"
15 | href = ' {}
'
16 | html_args = {
17 | "ContentType": "text/html",
18 | "CacheControl": "max-age=300",
19 | "ACL": "public-read",
20 | }
21 |
22 | s3 = client(
23 | "s3",
24 | aws_access_key_id=args.access_key_id,
25 | aws_secret_access_key=args.secret_access_key,
26 | )
27 |
28 | bucket = s3.Bucket(name="nerfacc-bucket")
29 |
30 | wheels_dict = defaultdict(list)
31 | for obj in bucket.objects.filter(Prefix="whl"):
32 | if obj.key[-3:] != "whl":
33 | continue
34 | torch_version, wheel = obj.key.split("/")[-2:]
35 | wheel = f"{torch_version}/{wheel}"
36 | wheels_dict[torch_version].append(wheel)
37 |
38 | index_html = html.format(
39 | "\n".join(
40 | [
41 | href.format(f"{torch_version}.html".replace("+", "%2B"), version)
42 | for version in wheels_dict
43 | ]
44 | )
45 | )
46 |
47 | with open("index.html", "w") as f:
48 | f.write(index_html)
49 | bucket.Object("whl/index.html").upload_file("index.html", html_args)
50 |
51 | for torch_version, wheel_names in wheels_dict.items():
52 | torch_version_html = html.format(
53 | "\n".join(
54 | [
55 | href.format(
56 | f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"), wheel_name
57 | )
58 | for wheel_name in wheel_names
59 | ]
60 | )
61 | )
62 |
63 | with open(f"{torch_version}.html", "w") as f:
64 | f.write(torch_version_html)
65 | bucket.Object(f"whl/{torch_version}.html").upload_file(
66 | f"{torch_version}.html", args
67 | )
68 |
--------------------------------------------------------------------------------
/CAM_nerfacc/scripts/run_dev_checks.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """Simple yaml debugger"""
3 | import subprocess
4 |
5 | import yaml
6 | from rich.console import Console
7 | from rich.style import Style
8 |
9 | console = Console(width=120)
10 |
11 | LOCAL_TESTS = [
12 | "Run license checks",
13 | "Run isort",
14 | "Run Black",
15 | "Python Pylint",
16 | "Test with pytest",
17 | ]
18 |
19 |
20 | def run_command(command: str) -> bool:
21 | """Run a command kill actions if it fails
22 |
23 | Args:
24 | command: command to run
25 | continue_on_fail: whether to continue running commands if the current one fails.
26 | """
27 | ret_code = subprocess.call(command, shell=True)
28 | if ret_code != 0:
29 | console.print(f"[bold red]Error: `{command}` failed.")
30 | return ret_code == 0
31 |
32 |
33 | def run_github_actions_file(filename: str):
34 | """Run a github actions file locally.
35 |
36 | Args:
37 | filename: Which yml github actions file to run.
38 | """
39 | with open(filename, "rb") as f:
40 | my_dict = yaml.safe_load(f)
41 | steps = my_dict["jobs"]["build"]["steps"]
42 |
43 | success = True
44 |
45 | for step in steps:
46 | if "name" in step and step["name"] in LOCAL_TESTS:
47 | compressed = step["run"].replace("\n", ";").replace("\\", "")
48 | compressed = compressed.replace("--check", "")
49 | curr_command = f"{compressed}"
50 |
51 | console.line()
52 | console.rule(f"[bold green]Running: {curr_command}")
53 | success = success and run_command(curr_command)
54 | else:
55 | skip_name = step["name"] if "name" in step else step["uses"]
56 | console.print(f"Skipping {skip_name}")
57 |
58 | # Code Testing
59 | console.line()
60 | console.rule("[bold green]Running pytest")
61 | success = success and run_command("pytest")
62 |
63 | # Add checks for building documentation
64 | console.line()
65 | console.rule("[bold green]Building Documentation")
66 | success = success and run_command(
67 | "cd docs/; make clean; make html SPHINXOPTS='-W;'"
68 | )
69 |
70 | if success:
71 | console.line()
72 | console.rule(characters="=")
73 | console.print(
74 | "[bold green]:TADA: :TADA: :TADA: ALL CHECKS PASSED :TADA: :TADA: :TADA:",
75 | justify="center",
76 | )
77 | console.rule(characters="=")
78 | else:
79 | console.line()
80 | console.rule(characters="=", style=Style(color="red"))
81 | console.print(
82 | "[bold red]:skull: :skull: :skull: ERRORS FOUND :skull: :skull: :skull:",
83 | justify="center",
84 | )
85 | console.rule(characters="=", style=Style(color="red"))
86 |
87 |
88 | if __name__ == "__main__":
89 | run_github_actions_file(filename=".github/workflows/code_checks.yml")
90 |
--------------------------------------------------------------------------------
/CAM_nerfacc/scripts/run_profiler.py:
--------------------------------------------------------------------------------
1 | from typing import Callable
2 |
3 | import torch
4 | import tqdm
5 |
6 | import nerfacc
7 |
8 | # timing
9 | # https://github.com/pytorch/pytorch/commit/d2784c233bfc57a1d836d961694bcc8ec4ed45e4
10 |
11 |
12 | class Profiler:
13 | def __init__(self, warmup=10, repeat=1000):
14 | self.warmup = warmup
15 | self.repeat = repeat
16 |
17 | def __call__(self, func: Callable):
18 | # warmup
19 | for _ in range(self.warmup):
20 | func()
21 | torch.cuda.synchronize()
22 |
23 | # profile
24 | with torch.profiler.profile(
25 | activities=[
26 | torch.profiler.ProfilerActivity.CPU,
27 | torch.profiler.ProfilerActivity.CUDA,
28 | ],
29 | profile_memory=True,
30 | ) as prof:
31 | for _ in range(self.repeat):
32 | func()
33 | torch.cuda.synchronize()
34 |
35 | # return
36 | events = prof.key_averages()
37 | # print(events.table(sort_by="self_cpu_time_total", row_limit=10))
38 | self_cpu_time_total = (
39 | sum([event.self_cpu_time_total for event in events]) / self.repeat
40 | )
41 | self_cuda_time_total = (
42 | sum([event.self_cuda_time_total for event in events]) / self.repeat
43 | )
44 | self_cuda_memory_usage = max(
45 | [event.self_cuda_memory_usage for event in events]
46 | )
47 | return (
48 | self_cpu_time_total, # in us
49 | self_cuda_time_total, # in us
50 | self_cuda_memory_usage, # in bytes
51 | )
52 |
53 |
54 | def main():
55 | device = "cuda:0"
56 | torch.manual_seed(42)
57 | profiler = Profiler(warmup=10, repeat=100)
58 |
59 | # # contract
60 | # print("* contract")
61 | # x = torch.rand([1024, 3], device=device)
62 | # roi = torch.tensor([0, 0, 0, 1, 1, 1], dtype=torch.float32, device=device)
63 | # fn = lambda: nerfacc.contract(
64 | # x, roi=roi, type=nerfacc.ContractionType.UN_BOUNDED_TANH
65 | # )
66 | # cpu_t, cuda_t, cuda_bytes = profiler(fn)
67 | # print(f"{cpu_t:.2f} us, {cuda_t:.2f} us, {cuda_bytes / 1024 / 1024:.2f} MB")
68 |
69 | # rendering
70 | print("* rendering")
71 | batch_size = 81920
72 | rays_o = torch.rand((batch_size, 3), device=device)
73 | rays_d = torch.randn((batch_size, 3), device=device)
74 | rays_d = rays_d / rays_d.norm(dim=-1, keepdim=True)
75 |
76 | ray_indices, t_starts, t_ends = nerfacc._ray_marching(
77 | rays_o,
78 | rays_d,
79 | near_plane=0.1,
80 | far_plane=1.0,
81 | render_step_size=1e-1,
82 | )
83 | sigmas = torch.randn_like(t_starts, requires_grad=True)
84 | fn = (
85 | lambda: nerfacc.render_weight_from_density(
86 | ray_indices, t_starts, t_ends, sigmas
87 | )
88 | .sum()
89 | .backward()
90 | )
91 | fn()
92 | torch.cuda.synchronize()
93 | for _ in tqdm.tqdm(range(100)):
94 | fn()
95 | torch.cuda.synchronize()
96 |
97 | cpu_t, cuda_t, cuda_bytes = profiler(fn)
98 | print(f"{cpu_t:.2f} us, {cuda_t:.2f} us, {cuda_bytes / 1024 / 1024:.2f} MB")
99 |
100 | packed_info = nerfacc.pack_info(ray_indices, n_rays=batch_size)
101 | fn = (
102 | lambda: nerfacc._vol_rendering._RenderingDensity.apply(
103 | packed_info, t_starts, t_ends, sigmas, 0
104 | )
105 | .sum()
106 | .backward()
107 | )
108 | fn()
109 | torch.cuda.synchronize()
110 | for _ in tqdm.tqdm(range(100)):
111 | fn()
112 | torch.cuda.synchronize()
113 | cpu_t, cuda_t, cuda_bytes = profiler(fn)
114 | print(f"{cpu_t:.2f} us, {cuda_t:.2f} us, {cuda_bytes / 1024 / 1024:.2f} MB")
115 |
116 |
117 | if __name__ == "__main__":
118 | main()
119 |
--------------------------------------------------------------------------------
/CAM_nerfacc/setup.cfg:
--------------------------------------------------------------------------------
1 | [isort]
2 | multi_line_output = 3
3 | line_length = 80
4 | include_trailing_comma = true
5 | skip=./examples/pycolmap,./benchmarks
--------------------------------------------------------------------------------
/CAM_nerfacc/setup.py:
--------------------------------------------------------------------------------
1 | import glob
2 | import os
3 | import os.path as osp
4 | import platform
5 | import sys
6 |
7 | from setuptools import find_packages, setup
8 |
9 | __version__ = None
10 | exec(open("nerfacc/version.py", "r").read())
11 |
12 | URL = "https://github.com/KAIR-BAIR/nerfacc"
13 |
14 | BUILD_NO_CUDA = os.getenv("BUILD_NO_CUDA", "0") == "1"
15 | WITH_SYMBOLS = os.getenv("WITH_SYMBOLS", "0") == "1"
16 |
17 |
18 | def get_ext():
19 | from torch.utils.cpp_extension import BuildExtension
20 |
21 | return BuildExtension.with_options(
22 | no_python_abi_suffix=True, use_ninja=False
23 | )
24 |
25 |
26 | def get_extensions():
27 | import torch
28 | from torch.__config__ import parallel_info
29 | from torch.utils.cpp_extension import CUDAExtension
30 |
31 | extensions_dir = osp.join("nerfacc", "cuda", "csrc")
32 | sources = glob.glob(osp.join(extensions_dir, "*.cu")) + glob.glob(
33 | osp.join(extensions_dir, "*.cpp")
34 | )
35 | # remove generated 'hip' files, in case of rebuilds
36 | sources = [path for path in sources if "hip" not in path]
37 |
38 | undef_macros = []
39 | define_macros = []
40 |
41 | if sys.platform == "win32":
42 | define_macros += [("nerfacc_EXPORTS", None)]
43 |
44 | extra_compile_args = {"cxx": ["-O3"]}
45 | if not os.name == "nt": # Not on Windows:
46 | extra_compile_args["cxx"] += ["-Wno-sign-compare"]
47 | extra_link_args = [] if WITH_SYMBOLS else ["-s"]
48 |
49 | info = parallel_info()
50 | if (
51 | "backend: OpenMP" in info
52 | and "OpenMP not found" not in info
53 | and sys.platform != "darwin"
54 | ):
55 | extra_compile_args["cxx"] += ["-DAT_PARALLEL_OPENMP"]
56 | if sys.platform == "win32":
57 | extra_compile_args["cxx"] += ["/openmp"]
58 | else:
59 | extra_compile_args["cxx"] += ["-fopenmp"]
60 | else:
61 | print("Compiling without OpenMP...")
62 |
63 | # Compile for mac arm64
64 | if sys.platform == "darwin" and platform.machine() == "arm64":
65 | extra_compile_args["cxx"] += ["-arch", "arm64"]
66 | extra_link_args += ["-arch", "arm64"]
67 |
68 | nvcc_flags = os.getenv("NVCC_FLAGS", "")
69 | nvcc_flags = [] if nvcc_flags == "" else nvcc_flags.split(" ")
70 | nvcc_flags += ["-O3"]
71 | if torch.version.hip:
72 | # USE_ROCM was added to later versions of PyTorch.
73 | # Define here to support older PyTorch versions as well:
74 | define_macros += [("USE_ROCM", None)]
75 | undef_macros += ["__HIP_NO_HALF_CONVERSIONS__"]
76 | else:
77 | nvcc_flags += ["--expt-relaxed-constexpr"]
78 | extra_compile_args["nvcc"] = nvcc_flags
79 |
80 | extension = CUDAExtension(
81 | f"nerfacc.csrc",
82 | sources,
83 | include_dirs=[osp.join(extensions_dir, "include")],
84 | define_macros=define_macros,
85 | undef_macros=undef_macros,
86 | extra_compile_args=extra_compile_args,
87 | extra_link_args=extra_link_args,
88 | )
89 |
90 | return [extension]
91 |
92 |
93 | # work-around hipify abs paths
94 | include_package_data = True
95 | # if torch.cuda.is_available() and torch.version.hip:
96 | # include_package_data = False
97 |
98 | setup(
99 | name="nerfacc",
100 | version=__version__,
101 | description="A General NeRF Acceleration Toolbox",
102 | author="Ruilong",
103 | author_email="ruilongli94@gmail.com",
104 | url=URL,
105 | download_url=f"{URL}/archive/{__version__}.tar.gz",
106 | keywords=[],
107 | python_requires=">=3.7",
108 | install_requires=["rich>=12", "torch", "typing_extensions; python_version<'3.8'"],
109 | extras_require={
110 | # dev dependencies. Install them by `pip install nerfacc[dev]`
111 | "dev": [
112 | "black[jupyter]==22.3.0",
113 | "isort==5.10.1",
114 | "pylint==2.13.4",
115 | "pytest==7.1.2",
116 | "pytest-xdist==2.5.0",
117 | "typeguard>=2.13.3",
118 | "pyyaml==6.0",
119 | "build",
120 | "twine",
121 | ],
122 | },
123 | ext_modules=get_extensions() if not BUILD_NO_CUDA else [],
124 | cmdclass={"build_ext": get_ext()} if not BUILD_NO_CUDA else {},
125 | packages=find_packages(),
126 | include_package_data=include_package_data,
127 | )
128 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Coordinate-Aware Modulation for Neural Fields (ICLR 2024 Spotlight)
2 | ### Joo Chan Lee, Daniel Rho, Seungtae Nam, Jong Hwan Ko, and Eunbyung Park
3 |
4 | ### [[Project Page](https://maincold2.github.io/cam/)] [[Paper(arxiv)](https://arxiv.org/abs/2311.14993)]
5 |
6 | ## Method Overview
7 |
8 |
9 | In this work, we propose a novel way for exploiting both MLPs and grid representations in neural fields. Unlike the prevalent methods that combine them sequentially (extract features from the grids first and feed them to the MLP), we inject spectral bias-free grid representations into the intermediate features in the MLP. More specifically, we suggest a Coordinate-Aware Modulation (CAM), which modulates the intermediate features using scale and shift parameters extracted from the grid representations. This can maintain the strengths of MLPs while mitigating any remaining potential biases, facilitating the rapid learning of high-frequency components.
10 |
11 | ## Setup
12 |
13 | CAM is applied to various methods in a plug-and-play manner. We recommend following the original configurations of each method ([FFNeRV](https://github.com/maincold2/FFNeRV), [Mip-NeRF](https://github.com/google/mipnerf), [Mip-NeRF 360](https://github.com/google-research/multinerf), and [NerfAcc@ebeb5dd](https://github.com/nerfstudio-project/nerfacc/tree/ebeb5ddf733c04b425d5070efae9c3d23f64b078)).
14 |
15 | ## Image representation
16 | ### FFN
17 | We implement CAM on the FFN pytorch implementation in [FINN](https://github.com/yixin26/FINN).
18 |
19 | ```shell
20 | python CAM_image.py -g [GPU ID] --data [image or image_dir] --model [CAM or FFN] --exp [work_dir]
21 | ```
22 |
23 | #### --downsample
24 | Downsample factor of training images, where 1 means image regression, 2 by default
25 |
26 | ## Video representation
27 | ### FFNeRV
28 |
29 | ```bash
30 | cd CAM_ffnerv
31 |
32 | # For representation
33 | python main.py -e 300 --lower-width 96 --num-blocks 1 --dataset [data_dir] --outf [work_dir] --fc-hw-dim 9_16_156 --expansion 1 --loss Fusion6 --strides 5 2 2 2 2 --conv-type conv -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --t-dim 64 128 256 512 --mod-dim 60
34 |
35 | # For compression
36 | python main.py -e 600 --lower-width 24 --num-blocks 1 --dataset [data_dir] --outf [work_dir] --fc-hw-dim 9_16_48 --expansion 8 --loss Fusion6 --strides 5 3 2 2 2 --conv-type compact -b 1 --lr 0.0005 --agg-ind -2 -1 1 2 --lw 0.1 --wbit 6 --t-dim 300 600 --resol 1920 1080 --mod-dim 30
37 | ```
38 |
39 | #### --mod-dim
40 | Temporal resolution of modulation grids, where 0 means the original FFNeRV, 60 by default
41 |
42 | ## NeRF
43 | ### Mip-NeRF
44 | ```shell
45 | cd CAM_mipnerf
46 |
47 | python -m train --data_dir=[nerf_synthetic_dir] --train_dir=[work_dir] --gin_file=configs/blender.gin --logtostderr
48 |
49 | python -m eval --data_dir=[nerf_synthetic_dir] --train_dir=[work_dir] --gin_file=configs/blender.gin --logtostderr
50 | ```
51 | ### Mip-NeRF 360
52 | ```shell
53 | cd CAM_multinerf-mipnerf360
54 |
55 | python -m train --gin_configs=configs/360.gin --gin_bindings="Config.data_dir = '[360 data_dir]'" --gin_bindings="Config.checkpoint_dir = '[work_dir]'" --logtostderr
56 |
57 | python -m eval --gin_configs=configs/360.gin --gin_bindings="Config.data_dir = '[360 data_dir]'" --gin_bindings="Config.checkpoint_dir = '[work_dir]'" --logtostderr
58 | ```
59 | ### NerfAcc
60 |
61 | ```shell
62 | cd CAM_nerfacc
63 |
64 | PYTHONPATH=./ python examples/train_mlp_nerf.py --scene [scene_name] --data_root [nerfsyn_dataset_dir] --exp [work_dir]
65 | ```
66 |
67 | ## Dynamic NeRF
68 |
69 | ### NerfAcc
70 |
71 | ```shell
72 | cd CAM_nerfacc
73 |
74 | PYTHONPATH=./ python examples/train_mlp_tnerf.py --scene [scene_name] --data_root [dnerf_dataset_dir] --exp [work_dir]
75 | ```
76 |
77 | ## BibTeX
78 | ```
79 | @inproceedings{
80 | lee2024cam,
81 | title={Coordinate-Aware Modulation for Neural Fields},
82 | author={Joo Chan Lee and Daniel Rho and Seungtae Nam and Jong Hwan Ko and Eunbyung Park},
83 | booktitle={The Twelfth International Conference on Learning Representations},
84 | year={2024},
85 | url={https://openreview.net/forum?id=4UiLqimGm5}
86 | }
87 | ```
88 |
--------------------------------------------------------------------------------