├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── .gitignore ├── README.md ├── mat_utils.py ├── rc2nerf.py ├── requirements.txt └── utils.py /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: joreeves 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **CSV** 24 | Attach or link to the CSV. 25 | 26 | **Screenshots** 27 | If applicable, add screenshots to help explain your problem. 28 | 29 | **Additional context** 30 | Add any other context about the problem here. 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __test__/ 2 | __pycache__/ 3 | 4 | .conda 5 | .ipynb_checkpoints 6 | .vscode 7 | 8 | *.pyc 9 | *.json 10 | *.lock 11 | *.log* 12 | *.csv 13 | *.json 14 | *.bat -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Reality Capture camera location to NERF conversion tool with per camera intrinsics 2 | This tool is for use with https://github.com/NVlabs/instant-ngp and allows the use of Reality Capture camera locations. 3 | 4 | ## Installation 5 | Copy rc2nerf.py file into instant-ngp\scripts folder 6 | 7 | ## Usage 8 | Use Reality Capture to align cameras. 9 | 10 | Export cameras alignment 11 | ``` 12 | Alignment -> Export -> Registration -> Internal/External camera parameters... 13 | ``` 14 | 15 | Save the CSV file exported from Reality Capture into the directory that contains your /images folder. 16 | 17 | Open a shell (CMD, Powershell, Bash, etc.) and navigate to the directory with your CSV file and /images folder: 18 | 19 | cd [PATH TO FOLDER] 20 | 21 | Run the rc2nerf.py on this CSV file using the following command, replacing the text in brackets […] with the file names and paths on your machine: 22 | 23 | ## Commands 24 | Example: 25 | ``` 26 | python "[PATH TO iNGP]\rc2nerf.py" --csv_in "[NAME_OF_CSV_FILE].csv" --imgfolder .\images 27 | ``` 28 | The quotes are only required if you have spaces in any of the folder or file names. 29 | 30 | ## Additional command examples 31 | Scale the scene down by 0.01 32 | ``` 33 | python "[PATH TO iNGP]\rc2nerf.py" --csv_in "[NAME_OF_CSV_FILE].csv" --imgfolder .\images --scale 0.01 34 | ``` 35 | 36 | Display the cameras in 3d and set the camera size (for debugging only, requires installing `matplotlib` and `pytransform3d`) 37 | ``` 38 | python "[PATH TO iNGP]\rc2nerf.py" --csv_in "[NAME_OF_CSV_FILE].csv" --imgfolder .\images --plot --camera_size 1 39 | ``` 40 | 41 | Arguments: 42 | 43 | | Argument | Default Value | Description | 44 | |------------------------|-----------------|----------------------------------------------| 45 | | --csv_in | None | specify csv file location | 46 | | --out | transforms.json | specify output file path | 47 | | --imgfolder | .\images | location of image folder | 48 | | --imgtype | jpg | ex.: jpg, png, ... | 49 | | --aabb_scale | 16 | sets the aabb scale | 50 | | --no_scene_orientation | False | disable the Reality Capture orientation | 51 | | --no_scale | False | disable the Reality Capture scale | 52 | | --no_center | False | disable the scene centering | 53 | | --plot | False | display the camera positions | 54 | | --camera_size | 0.1 | the size of the displayed cameras | 55 | | --debug_ignore_images | False | ignores the input images, for debugging only | 56 | | --threads | 8 | number of threads to use when reading images | 57 | -------------------------------------------------------------------------------- /mat_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | """ 5 | Code from pytransform3d.rotations 6 | """ 7 | 8 | def active_matrix_from_angle(basis, angle): 9 | r"""Compute active rotation matrix from rotation about basis vector. 10 | 11 | With the angle :math:`\alpha` and :math:`s = \sin{\alpha}, c=\cos{\alpha}`, 12 | we construct rotation matrices about the basis vectors as follows: 13 | 14 | .. math:: 15 | 16 | \boldsymbol{R}_x(\alpha) = 17 | \left( 18 | \begin{array}{ccc} 19 | 1 & 0 & 0\\ 20 | 0 & c & -s\\ 21 | 0 & s & c 22 | \end{array} 23 | \right) 24 | 25 | .. math:: 26 | 27 | \boldsymbol{R}_y(\alpha) = 28 | \left( 29 | \begin{array}{ccc} 30 | c & 0 & s\\ 31 | 0 & 1 & 0\\ 32 | -s & 0 & c 33 | \end{array} 34 | \right) 35 | 36 | .. math:: 37 | 38 | \boldsymbol{R}_z(\alpha) = 39 | \left( 40 | \begin{array}{ccc} 41 | c & -s & 0\\ 42 | s & c & 0\\ 43 | 0 & 0 & 1 44 | \end{array} 45 | \right) 46 | 47 | Parameters 48 | ---------- 49 | basis : int from [0, 1, 2] 50 | The rotation axis (0: x, 1: y, 2: z) 51 | 52 | angle : float 53 | Rotation angle 54 | 55 | Returns 56 | ------- 57 | R : array, shape (3, 3) 58 | Rotation matrix 59 | 60 | Raises 61 | ------ 62 | ValueError 63 | If basis is invalid 64 | """ 65 | c = np.cos(angle) 66 | s = np.sin(angle) 67 | 68 | if basis == 0: 69 | R = np.array([[1.0, 0.0, 0.0], 70 | [0.0, c, -s], 71 | [0.0, s, c]]) 72 | elif basis == 1: 73 | R = np.array([[c, 0.0, s], 74 | [0.0, 1.0, 0.0], 75 | [-s, 0.0, c]]) 76 | elif basis == 2: 77 | R = np.array([[c, -s, 0.0], 78 | [s, c, 0.0], 79 | [0.0, 0.0, 1.0]]) 80 | else: 81 | raise ValueError("Basis must be in [0, 1, 2]") 82 | 83 | return R 84 | 85 | 86 | def active_matrix_from_extrinsic_euler(e, order=[0,1,2]): 87 | """Compute active rotation matrix from extrinsic Cardan angles. 88 | 89 | Parameters 90 | ---------- 91 | e : array-like, shape (3,) 92 | Angles for rotation around x-, y-, and z-axes (extrinsic rotations) 93 | 94 | Returns 95 | ------- 96 | R : array, shape (3, 3) 97 | Rotation matrix 98 | """ 99 | alpha, beta, gamma = e 100 | R = active_matrix_from_angle(order[2], gamma).dot( 101 | active_matrix_from_angle(order[1], beta)).dot( 102 | active_matrix_from_angle(order[0], alpha)) 103 | return R 104 | 105 | 106 | def matrix_from_euler(e, rotation_order='xyz', degrees=False): 107 | assert len(rotation_order) == 3 108 | assert all([r in 'xyz' for r in rotation_order.lower()]) 109 | 110 | if degrees: 111 | e = np.radians(e) 112 | 113 | angles = {'x':0,'y':1,'z':2} 114 | order = [angles[r] for r in rotation_order.lower()] 115 | return active_matrix_from_extrinsic_euler(e, order) -------------------------------------------------------------------------------- /rc2nerf.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.config 3 | 4 | logging.config.dictConfig({ 5 | 'version': 1, 6 | 'formatters': { 7 | 'console': { 8 | 'format': '%(asctime)s | %(levelname)s | %(filename)s : %(lineno)s | >>> %(message)s', 9 | 'datefmt': '%Y-%m-%d %H:%M:%S' 10 | }, 11 | 'file': { 12 | 'format': '%(asctime)s | %(levelname)s | %(filename)s : %(lineno)s | >>> %(message)s', 13 | 'datefmt': '%Y-%m-%d %H:%M:%S' 14 | } 15 | }, 16 | 'handlers': { 17 | 'console': { 18 | 'class': 'logging.StreamHandler', 19 | 'formatter': 'console', 20 | 'level': 'INFO', 21 | 'stream': 'ext://sys.stdout' 22 | }, 23 | 'file': { 24 | 'class': 'logging.handlers.RotatingFileHandler', 25 | 'formatter': 'file', 26 | 'level': 'DEBUG', 27 | 'filename': 'rc2nerf.log', 28 | 'mode': 'a', 29 | 'maxBytes': 0, 30 | 'backupCount': 3 31 | } 32 | }, 33 | 'loggers': { 34 | '': { 35 | 'handlers': ['console', 'file'], 36 | 'level': 'DEBUG', 37 | 'propagate': True 38 | } 39 | } 40 | }) 41 | 42 | LOGGER = logging.getLogger(__name__) 43 | 44 | import argparse 45 | import csv 46 | import json 47 | import math 48 | import numpy as np 49 | import pandas as pd 50 | import os 51 | import cv2 52 | from copy import deepcopy as dc 53 | 54 | from tqdm import tqdm 55 | from pathlib import Path 56 | 57 | from utils import sharpness, Mat2Nerf, central_point, plot, _PLT, reflect 58 | from mat_utils import matrix_from_euler 59 | 60 | from concurrent.futures import ThreadPoolExecutor 61 | 62 | 63 | def parse_args(): 64 | parser = argparse.ArgumentParser(description="convert Reality Capture csv export to nerf format transforms.json") 65 | 66 | parser.add_argument("--csv_in", help="specify csv file location") #TODO: Chang to positional argument 67 | parser.add_argument("--out", dest="path", default="transforms.json", help="output path") 68 | parser.add_argument("--imgfolder", default="./images/", help="location of folder with images") 69 | parser.add_argument("--imgtype", default="jpg", help="type of images (ex. jpg, png, ...)") 70 | parser.add_argument("--aabb_scale", default=16, type=int, help="size of the aabb, default is 16") 71 | parser.add_argument("--plot", action="store_true", help="plot the cameras and the bounding region in 3D") 72 | parser.add_argument("--scale", default=1.0, type=float, help="scale the scene by a factor") 73 | parser.add_argument("--no_scale", action="store_true", help="DISABLES the scaling of the cameras to the bounding region") 74 | parser.add_argument("--no_center", action="store_true", help="DISABLES the centering of the cameras around the computed central point") 75 | parser.add_argument("--camera_size", default=0.1, type=float, help="size of the camera in the 3D plot. Does not affect the output.") 76 | parser.add_argument("--debug", action="store_true", help="enables debug mode") 77 | 78 | parser.add_argument("--debug_ignore_images", action="store_true", help="IGNORES the images in the xml file. For debugging purposes only.") 79 | 80 | parser.add_argument("--threads", default=8, type=int, help="number of threads to use for processing") 81 | 82 | args = parser.parse_args() 83 | return args 84 | 85 | 86 | def build_sensor(resolution, focal_length, intrinsics:dict): 87 | out = dict() 88 | 89 | width, height = resolution 90 | 91 | out["w"] = width 92 | out["h"] = height 93 | out["fl_x"] = focal_length 94 | out["fl_y"] = focal_length 95 | 96 | # # Given the w, h, pixel_width, pixel_height, and focal_length 97 | # # Calculate the focal length in pixels 98 | # fl_pxl = (w * focal_length) / (w * pixel_width) 99 | 100 | camera_angle_x = math.atan(float(width) / (float(focal_length) * 2)) * 2 101 | camera_angle_y = math.atan(float(height) / (float(focal_length) * 2)) * 2 102 | 103 | out["camera_angle_x"] = camera_angle_x 104 | out["camera_angle_y"] = camera_angle_y 105 | 106 | intrinsics_keys = ['cx', 'cy', 'b1', 'b2', 107 | 'k1', 'k2', 'k3', 'k4', 108 | 'p1', 'p2', 'p3', 'p4'] 109 | 110 | for intrinsic in intrinsics_keys: 111 | if intrinsic not in intrinsics.keys(): 112 | continue 113 | 114 | out[intrinsic] = intrinsics[intrinsic] 115 | 116 | return out 117 | 118 | 119 | def init_logging(args): 120 | # Get handlers from logging config 121 | handlers = logging.getLogger().handlers 122 | 123 | if args.debug: 124 | for log in handlers: 125 | log.setLevel(logging.DEBUG) 126 | 127 | # Get log path from config 128 | log_path = Path(handlers[1].baseFilename) 129 | 130 | if log_path.is_file(): 131 | handlers[1].doRollover() 132 | 133 | 134 | if __name__ == "__main__": 135 | args = parse_args() 136 | 137 | init_logging(args) 138 | 139 | CSV_PATH = args.csv_in 140 | IMGTYPE = args.imgtype 141 | IMGFOLDER = args.imgfolder 142 | 143 | IMGFOLDER = Path(IMGFOLDER) 144 | files = list(IMGFOLDER.glob('*.{}'.format(IMGTYPE))) 145 | stems = list([f.stem for f in files]) 146 | 147 | # Check if the files path has images in it 148 | if(len(files)==0) & (args.debug_ignore_images==False): 149 | LOGGER.error('No images found in folder: {}'.format(IMGFOLDER)) 150 | exit() 151 | 152 | out = dict() 153 | out['aabb_scale'] = args.aabb_scale 154 | 155 | def read_img(row): 156 | i, row = row 157 | 158 | if args.debug_ignore_images: 159 | return i, row, None 160 | 161 | img_file_path = IMGFOLDER / row['#name'] 162 | if img_file_path.exists(): 163 | img = cv2.imread(str(img_file_path)) 164 | else: 165 | img = None 166 | return i, row, img 167 | 168 | frames = [] 169 | 170 | df = pd.read_csv(CSV_PATH, sep=',') 171 | 172 | pbar = tqdm(total=len(df), desc='Processing reality capture csv') 173 | 174 | with ThreadPoolExecutor(max_workers=args.threads) as executor: 175 | for i, row, img in executor.map(read_img, df.iterrows()): 176 | pbar.update(1) 177 | 178 | if (img is None) and (args.debug_ignore_images==False): 179 | LOGGER.warning('Image not found: {}'.format(row['#name'])) 180 | continue 181 | 182 | LOGGER.debug('Processing image: {}'.format(row['#name'])) 183 | LOGGER.debug('Processing row: {}'.format(row)) 184 | 185 | # f, px, py, k1, k2, k3, k4, t1, t2 186 | 187 | height, width, *_ = img.shape 188 | 189 | focal = row['f'] * np.maximum(width, height) / 36 190 | 191 | intrinsics = dict( 192 | cx=row['px'] / 36.0 + width / 2.0, 193 | cy=row['py'] / 36.0 + height / 2.0, 194 | k1=row['k1'], 195 | k2=row['k2'], 196 | k3=row['k3'], 197 | k4=row['k4'], 198 | p1=row['t1'], 199 | p2=row['t2'], 200 | ) 201 | 202 | camera = build_sensor((width, height), focal, intrinsics) 203 | 204 | # See here for more on RC orientation: 205 | # https://forums.unrealengine.com/t/different-rotation-of-cameras-in-xmp-and-csv/710449/5 206 | # https://forums.unrealengine.com/t/realitycapture-xmp-camera-math/682564 207 | # https://forums.unrealengine.com/t/camera-export-and-file-formats/706644/4 208 | # https://forums.unrealengine.com/t/camera-coordinate-system-explanation/712595/2 209 | # https://forums.unrealengine.com/t/please-help-us-understand-the-internal-external-camera-parameters-export/712503 210 | 211 | mat = np.eye(4) 212 | 213 | mat[:3, :3] = matrix_from_euler([row['roll'], row['pitch'], -row['heading']], 'yxz', True) 214 | 215 | mat[:3,3] = np.array([row['x'], row['y'], row['alt']]) * float(args.scale) 216 | 217 | mat = mat[[2,0,1,3],:] # <<< This is the magic sauce 218 | 219 | camera['transform_matrix'] = mat #Mat2Nerf(mat) 220 | 221 | camera["file_path"] = str(IMGFOLDER / row['#name']) 222 | 223 | camera['sharpness'] = 1 if args.debug_ignore_images else sharpness(img) 224 | 225 | LOGGER.debug(f'Camera {i:03d} info:') 226 | for k,v in camera.items(): 227 | LOGGER.debug('{}: {}'.format(k, v)) 228 | LOGGER.debug('Finished processing {i:03d}\n') 229 | 230 | frames.append(camera) 231 | 232 | out['frames'] = frames 233 | 234 | if args.no_center: 235 | center = np.zeros(3) 236 | else: 237 | # Compute the center of attention 238 | center = central_point(out) 239 | 240 | # Set the offset and convert to list 241 | for f in out["frames"]: 242 | f["transform_matrix"][0:3,3] -= center 243 | f["transform_matrix"] = f["transform_matrix"].tolist() 244 | 245 | with open(args.path, "w") as f: 246 | json.dump(out, f, indent=4) 247 | 248 | if _PLT & args.plot: 249 | plot(out, center, args.camera_size) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | opencv-python 3 | pandas 4 | tqdm -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | LOGGER = logging.getLogger(__name__) 4 | 5 | import numpy as np 6 | import cv2 7 | 8 | 9 | # Check if plot libraries are installed 10 | try: 11 | from pytransform3d.plot_utils import plot_box 12 | from pytransform3d.transform_manager import TransformManager 13 | 14 | import pytransform3d.camera as pc 15 | import pytransform3d.transformations as pytr 16 | 17 | import matplotlib.pyplot as plt 18 | from matplotlib.widgets import Slider, Button 19 | _PLT = True 20 | except ImportError as e: 21 | LOGGER.info("Plot libraries not installed. Skipping plot functions. Install `pytransform3d` and `matplotlib` to enable plot functions.") 22 | _PLT = False 23 | 24 | 25 | ############################################################################### 26 | # START 27 | # code taken from https://github.com/NVlabs/instant-ngp 28 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 29 | 30 | def closest_point_2_lines(oa, da, ob, db): # returns point closest to both rays of form o+t*d, and a weight factor that goes to 0 if the lines are parallel 31 | da = da / np.linalg.norm(da) 32 | db = db / np.linalg.norm(db) 33 | c = np.cross(da, db) 34 | denom = np.linalg.norm(c)**2 35 | t = ob - oa 36 | ta = np.linalg.det([t, db, c]) / (denom + 1e-10) 37 | tb = np.linalg.det([t, da, c]) / (denom + 1e-10) 38 | if ta > 0: 39 | ta = 0 40 | if tb > 0: 41 | tb = 0 42 | return (oa+ta*da+ob+tb*db) * 0.5, denom 43 | 44 | def central_point(out): 45 | # find a central point they are all looking at 46 | # print("computing center of attention...") 47 | LOGGER.info("computing center of attention...") 48 | totw = 0.0 49 | totp = np.array([0.0, 0.0, 0.0]) 50 | for f in out["frames"]: 51 | mf = np.array(f["transform_matrix"])[0:3,:] 52 | for g in out["frames"]: 53 | mg = g["transform_matrix"][0:3,:] 54 | p, w = closest_point_2_lines(mf[:,3], mf[:,2], mg[:,3], mg[:,2]) 55 | if w > 0.01: 56 | totp += p*w 57 | totw += w 58 | 59 | if len(out["frames"]) == 0: 60 | LOGGER.error("No frames found when computing center of attention") 61 | return totp 62 | 63 | if (totw == 0) and (not totp.any()): 64 | LOGGER.error("Center of attention is zero") 65 | return totp 66 | 67 | totp /= totw 68 | # print("The center of attention is: {}".format(totp)) # the cameras are looking at totp 69 | LOGGER.info("The center of attention is: {}".format(totp)) 70 | 71 | return totp 72 | 73 | def sharpness(image): 74 | gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 75 | fm = cv2.Laplacian(gray, cv2.CV_64F).var() 76 | return fm 77 | 78 | #END 79 | ############################################################################### 80 | 81 | 82 | def reflect(axis, size=4): 83 | _diag = np.ones(size) 84 | _diag[axis] = -1 85 | refl = np.diag(_diag) 86 | return refl 87 | 88 | 89 | def Mat2Nerf(mat): 90 | M = np.array(mat) 91 | M = ((M @ reflect(2)) @ reflect(1)) 92 | return M 93 | 94 | 95 | def draw_cameras(ax, out, camera_size): 96 | # Plot the camera positions 97 | for f in out['frames']: 98 | sensor_size = np.array([f["w"], f["h"]]) 99 | 100 | intrinsic = np.eye(3) 101 | intrinsic[0,0] = f["fl_x"] 102 | intrinsic[1,1] = f["fl_y"] 103 | intrinsic[0,2] = f["cx"] if "cx" in f else sensor_size[0] / 2.0 104 | intrinsic[1,2] = f["cy"] if "cy" in f else sensor_size[1] / 2.0 105 | 106 | cam_mat = np.array(f["transform_matrix"]) 107 | 108 | # Scale the camera position 109 | # cam_mat[0:3,3] *= scale 110 | 111 | # Reflect the camera back for plotting 112 | cam_mat = cam_mat @ reflect(1) @ reflect(2) 113 | 114 | pytr.plot_transform(ax, A2B=cam_mat, s=camera_size) 115 | pc.plot_camera(ax, cam2world=cam_mat, M=intrinsic, 116 | sensor_size=sensor_size, 117 | virtual_image_distance=camera_size) 118 | 119 | 120 | def plot(out, origin, camera_size=0.1): 121 | 122 | # 3D plot the points and display them 123 | fig = plt.figure() 124 | 125 | ax = plt.axes(projection='3d') 126 | 127 | ax.set_xlabel('x') 128 | ax.set_ylabel('z') 129 | ax.set_zlabel('y') 130 | 131 | fig.subplots_adjust(left=0, right=1, bottom=0, top=1) 132 | 133 | # Find the scene scale 134 | P = [np.array(f["transform_matrix"])[0:3,3] for f in out['frames']] 135 | pos_min = np.min(P, axis=0) 136 | pos_max = np.max(P, axis=0) 137 | # print("Scene size:", pos_max - pos_min) 138 | center = (pos_max + pos_min) / 2.0 139 | max_half_extent = max(pos_max - pos_min) / 2.0 140 | # print("Max half extent:", max_half_extent) 141 | 142 | # Plot the camera positions 143 | draw_cameras(ax, out, camera_size) 144 | 145 | # Plot the origin for reference 146 | pytr.plot_transform(ax, A2B=np.eye(4), s=1) 147 | 148 | # if region is not None: 149 | # # Plot the bounding box 150 | # bbox_mat = region['transform_matrix'] 151 | # bbox_mat = bbox_mat @ reflect(1) @ reflect(2) 152 | # bbox_mat[0:3,3] -= origin # Translate the bbox to match the center 153 | # plot_box(ax, size=region['size'], A2B=bbox_mat, color='r', alpha=0.5) 154 | 155 | # Set the limits 156 | ax.set_xlim((center[0] - max_half_extent, center[0] + max_half_extent)) 157 | ax.set_ylim((center[1] - max_half_extent, center[1] + max_half_extent)) 158 | ax.set_zlim((center[2] - max_half_extent, center[2] + max_half_extent)) 159 | 160 | # Create sliders to adjust scale of the scene 161 | # slider_scale = Slider(plt.axes([0.25, 0.05, 0.65, 0.03]), 'Scale', 0.01, 10.0, valinit=1.0) 162 | # plt.axes([0.25, 0.05, 0.65, 0.03]) 163 | # def update(val): 164 | # scale = slider_scale.val 165 | # draw_cameras(ax, out, scale) 166 | # fig.canvas.draw_idle() 167 | 168 | # slider_scale.on_changed(update) 169 | 170 | plt.show() --------------------------------------------------------------------------------