├── .gitignore ├── LICENSE ├── README.md └── polyform ├── __init__.py ├── convert.py ├── convertors ├── README.md ├── __init__.py ├── convertor_interface.py └── instant_ngp.py ├── core ├── __init__.py ├── bbox.py └── capture_folder.py └── utils ├── __init__.py └── logging.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 PolyCam 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # polyform 2 | 3 | Polycam is an application for 3D scanning that can be downloaded for free on the App Store [here](https://apps.apple.com/us/app/polycam-lidar-3d-scanner/id1532482376). While the main purpose of Polycam is to reconstruct 3D meshes, along the way we collect and optimize images, camera intrinsic and extrinsic data as well as depth maps. We have heard from numerous members of the community that this raw data could be useful for them, for applications such as training NeRFs, improving indoor scene reconstruction or doing more analysis than what can be done in Polycam alone. 4 | 5 | This project, polyform, includes tools for converting from Polycam's raw data format into other data formats, such as those used to train NeRFs. As discussed in more detail below, the camera poses are globally optimized and do not require any further optimization (i.e. COLMAP is not required) to be used as input to a NeRF. 6 | 7 | We hope that by exposing a free raw data export option, as well as documentation and tools for working with this data, we can help to enable researchers working in the field of 3D computer vision, as well as creative technologists working on the forefront of 3D creation. If you make something cool and post it online please tag us @polycam3d. 8 | 9 | ## Requirements & preliminary remarks 10 | 11 | ### An iOS device with LiDAR is required 12 | 13 | Raw data with images, camera poses and depth maps are only available when running Polycam on an iOS device with LiDAR (i.e. 2020+ iPad Pro, or iPhone 12+ Pro), and only available for sessions captured in LiDAR mode or ROOM mode. 14 | 15 | ### Developer mode must be turned on 16 | 17 | Open the settings screen in the Polycam iOS app, scroll down to 'Developer mode' and turn it on. This will expose a 'raw data' export option in the data export menu for all lidar captures. 18 | 19 | **NOTE:** Because developer mode turns on data caching of intermediate artifacts, it does not apply retroactively. You will need to either re-process existing datasets or else collect new datasets to generate the full raw data needed for export. 20 | 21 | ### Installation and setup 22 | 23 | Python 3.7+ is recommended. 24 | 25 | ``` 26 | pip3 install numpy fire Pillow # install dependencies 27 | git clone https://github.com/PolyCam/polyform 28 | ``` 29 | 30 | ### Polycam poses are globally optimized 31 | 32 | Unlike most other 3D scanning apps, we globally optimize the ARKit camera poses before reconstructing the mesh, and these optimized camera poses should be as good or better than what you would get from an SfM software like COLMAP, particularly for complex indoor scenes where most SfM pipelines will fail. This means you do not need to worry about drift while scanning, and can loop back and scan the same area multiple times in a single session. 33 | 34 | Given that the pose optimization can take some time to run on very lage scenes, there are a few limitations: (i) by default we only run pose optimization on scenes with less than 700 frames but (ii) if you open up the CUSTOM processing panel you can turn on pose optimization (aka Loop Closure) for scenes up to 1400 frames. If the 'Loop Closure' option is disabled that means the scene is too large to optimize on device, but you can still export the ARKit poses. 35 | 36 | ## Converting from Polycam's data format to other formats 37 | 38 | 39 | ### Nerfstudio 40 | 41 | [Nerfstudio](https://github.com/nerfstudio-project/nerfstudio) offers a suite of tools for training, visualizing and working with NeRFS. They offer a tool to directly import Polycam's data format, and we recommend using their importer if you wish to use nerfstudio. See [their docs](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) for more info. 42 | 43 | ### Instant-ngp 44 | 45 | [Instant-NGP](https://github.com/NVlabs/instant-ngp) is a popular open source project for training and visualizing NeRFs. You can convert from Polycam's data format to the the instant-ngp format by running the following command from the root of the `polyform` repo: 46 | 47 | ``` 48 | python3 -m polyform.convert --format ingp 49 | ``` 50 | 51 | Note: you may need to tweak the `scale` parameter in the transforms.json file to get the best results. 52 | 53 | ### Adding additional convertors: 54 | 55 | If you would like to add an additional export format you can do so by consulting Polycam's data specification below, and using `polyform/convertors/instant_ngp.py` as an example. 56 | 57 | ## Polycam's data format 58 | 59 | The raw data exported from Polycam has the following form: 60 | 61 | ``` 62 | capture-folder/ 63 | raw.glb 64 | thumbnail.jpg 65 | polycam.mp4 66 | mesh_info.json 67 | keyframes/ 68 | images/ 69 | corrected_images/ 70 | cameras/ 71 | corrected_cameras/ 72 | depth/ 73 | confidence/ 74 | ``` 75 | 76 | we will describe each of these artifacts below. 77 | 78 | ### raw.glb, thumbnail, video, mesh_info 79 | 80 | The `raw.glb` file contains the reconstructed mesh in binary gLTF form. This is put into an axis-aligned coordinate system, and if you wish to put it into the same coorindate system as the raw data you will need to apply the inverse of the `alignmentTransform` contained in the `mesh_info.json` (stored in column-major format). 81 | 82 | The `thumbnail.jpg` and `polycam.mp4` are visual metadata which are included for convenience, and the `mesh_info.json` contains some metadata about the raw.glb mesh such as the number of faces and vertices. 83 | 84 | ### images 85 | 86 | This directory contains the original images, where the name of the file is the timestamp in microseconds, and will match the filenames for the other corresponding files (camera, depth map etc). Note that these images have some distortion, so you will probably want to use the `corrected_images` instead. 87 | 88 | ### corrected_images 89 | 90 | This directory contains the undistorted images that are computed during the pose optimization step. For best results you will want to use these images, although you may need to mask out the black pixels around the border that are a result of the undistortion process. If this directory does not exist, it means the session was too large for the pose optimization step to run on device. 91 | 92 | ### cameras 93 | 94 | This directory includes the `camera.json` files that hold the camera intrinsics and extrinsics, as well as the image `width`, `height` and a `blur_score` for the image, where a lower score means more blur. 95 | 96 | The `t_ij` element of the transform matrix is in row-major order, so `i` is the row and `j` is the column. We omit the last row becuase it is simply `[0, 0, 0, 1]`. 97 | 98 | **NOTE:** The coordinate system of the camera poses follows the ARKit gravity aligned convention which you can read about [here](https://developer.apple.com/documentation/arkit/arconfiguration/worldalignment/gravity). In short, this is a right-handed coordinate system where the `+Y axis` points up, and `-Z axis` points in the direction that the camera was pointing when the session started, and perpendicular to gravity. Note: depending on the coordinate system convention of your downstream application you might have to apply an additional rotation to the camera transform to get the conventions to match. 99 | 100 | ### corrected_cameras 101 | 102 | This directory includes the improved camera poses that have been globally optimized. The data is formatted in the same way as the cameras described above, and you should almost always use these files intead of the original camera files. If this directory does not exist, it means the session was too large for the pose optimization step to run on device, but you can still fall use the unoptimized ARKit camera poses. 103 | 104 | ### depth 105 | 106 | This directory contains the depth maps acquired during the scanning session (see [docs](https://developer.apple.com/documentation/arkit/arframe/3566299-scenedepth)). These depth maps are generated by ARKit using the LiDAR sensor. A couple things to note: 107 | 108 | - Because the resolution is lower than the image resolution, you will have to scale the camera intrinsics to match the depth map resolution if you wish to use them. 109 | - The depth maps are not perfect -- because the lidar sensor is low resolution, and the ML-based upresolution process can introduce artifacts, they may not be useful for some applications. In general, resolving geometric detail less than 1-2 cm is not possible, and sometimes they can contain gross errors. 110 | - The max range of the lidar sensor is 5M, so the depth map for any part of the scene larger than that is a total guess. 111 | 112 | That being said, they are pretty good for indoor scenes or other scenes that do not require fine geometric detail. 113 | 114 | The depth data is stored in lossless .png format as a single channel image where depth is encoded as a 16-bit integer with units of millimeters, so a value of 1250 would correspond to 1.25 meters. 115 | 116 | ### confidence 117 | 118 | This folder includes the pixel-wise confidence maps associated with the depth maps (see [docs](https://developer.apple.com/documentation/arkit/ardepthdata/3566295-confidencemap)). ARKit only provides three confidence values -- low, medium and high. We encode these as a single-channel 8-bit integer map, where 0 = low, 127 = medium, and 255 = high. The low confidence values are quite unreliable, and for most applications it would be best to filter them out. 119 | 120 | 121 | ## Tips on dataset collection 122 | 123 | To get the best datasets, for training NeRFS or just in general, it's important to (i) get great coverage of your subject and (ii) reduce blur as much as possible. To reduce blur it's important to have good lighting and move the camera slowly if using auto mode, but to get the clearest images, particularly for lower light conditions such as indoors, it's recommended to use the manual shutter mode to collect still images -- you will just need to be sure to collect a lot of them. 124 | 125 | ## Working at Polycam 126 | 127 | If you are an engineer or researcher working in the field of 3D computer vision and want to work on a product that's shipping 3D reconstruction to millions of users, please visit: https://poly.cam/careers. 128 | -------------------------------------------------------------------------------- /polyform/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolyCam/polyform/845389c5cc4ddc116925162467d1c9bfea829e4d/polyform/__init__.py -------------------------------------------------------------------------------- /polyform/convert.py: -------------------------------------------------------------------------------- 1 | ''' 2 | File: convert.py 3 | Polycam Inc. 4 | 5 | Created by Chris Heinrich on Tuesday, 1st November 2022 6 | Copyright © 2022 Polycam Inc. All rights reserved. 7 | ''' 8 | import fire 9 | from polyform.utils.logging import logger 10 | from polyform.core.capture_folder import CaptureFolder 11 | from polyform.convertors.instant_ngp import InstantNGPConvertor 12 | 13 | 14 | def convert(data_folder_path: str, format: str = "ingp"): 15 | """ 16 | Main entry point for the command line convertor 17 | Args: 18 | data_folder_path: path to the unzipped Polycam data folder 19 | format: Output format time. Supported values are [ingp] 20 | """ 21 | folder = CaptureFolder(data_folder_path) 22 | if format.lower() == "ingp" or format.lower() == "instant-ngp": 23 | convertor = InstantNGPConvertor() 24 | else: 25 | logger.error("Format {} is not curently supported. Consider adding a convertor for it".format(format)) 26 | exit(1) 27 | convertor.convert(folder) 28 | 29 | if __name__ == '__main__': 30 | fire.Fire(convert) -------------------------------------------------------------------------------- /polyform/convertors/README.md: -------------------------------------------------------------------------------- 1 | # Convertors 2 | 3 | Convertors, which convert from Polycam's data format into another data format, belong in this directory. 4 | 5 | Your convertor should implement the ConvertorInterface -- see instant_ngp.py for an example -------------------------------------------------------------------------------- /polyform/convertors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolyCam/polyform/845389c5cc4ddc116925162467d1c9bfea829e4d/polyform/convertors/__init__.py -------------------------------------------------------------------------------- /polyform/convertors/convertor_interface.py: -------------------------------------------------------------------------------- 1 | ''' 2 | File: convertor_interface.py 3 | Polycam Inc. 4 | 5 | Created by Chris Heinrich on Tuesday, 1st November 2022 6 | Copyright © 2022 Polycam Inc. All rights reserved. 7 | ''' 8 | from abc import ABC, abstractmethod 9 | from polyform.core.capture_folder import CaptureFolder 10 | 11 | 12 | class ConvertorInterface(ABC): 13 | @abstractmethod 14 | def convert(self, folder: CaptureFolder, output_path: str = ""): 15 | """ 16 | All data convertors must implement the covnert method 17 | """ 18 | pass 19 | -------------------------------------------------------------------------------- /polyform/convertors/instant_ngp.py: -------------------------------------------------------------------------------- 1 | ''' 2 | File: instant_ngp.py 3 | Polycam Inc. 4 | 5 | Created by Chris Heinrich on Tuesday, 1st November 2022 6 | Copyright © 2022 Polycam Inc. All rights reserved. 7 | ''' 8 | import os 9 | import json 10 | import numpy as np 11 | from PIL import Image 12 | from polyform.utils.logging import logger 13 | from polyform.core.capture_folder import * 14 | from polyform.convertors.convertor_interface import ConvertorInterface 15 | 16 | class InstantNGPConvertor(ConvertorInterface): 17 | """Converts Polycam data to the format expected by Instant-NGP""" 18 | def __init__(self, corrected_image_padding: int = 5): 19 | self.corrected_image_padding = corrected_image_padding 20 | 21 | def convert(self, folder: CaptureFolder, output_path: str = ""): 22 | """ 23 | Converts a Polycam CaptureFolder into an instant-ngp dataset by writing out the 24 | transforms.json file 25 | 26 | Args: 27 | folder: the capture folder to convert 28 | output_path: the path to write the transforms.json file. By default this is 29 | written at the root of the CaptureFolder's data directory. 30 | """ 31 | data = {} 32 | keyframes = folder.get_keyframes(rotate=True) 33 | if len(keyframes) == 0: 34 | logger.error("Capture folder does not have any data! Aborting conversion to Instant NGP") 35 | return 36 | """ 37 | NOTE 38 | We use the intrinsics from the first camera to populate the camera data in the transforms.json file. 39 | However the intrinsics do vary slightly (a pixel or two) frame-to-frame. If you are working with a NeRF implementation that does then 40 | it would be better to use the frame-dependent intrinsics. 41 | TODO 42 | Add another convertor that writes out multiple json files, with one camera per frame. Instant-NGP seems to support this, but nerfstudio may not. 43 | """ 44 | cam = keyframes[0].camera 45 | data["fl_x"] = cam.fx 46 | data["fl_y"] = cam.fy 47 | if folder.has_optimized_poses(): 48 | # For optimized data we apply a crop of the image data, and we need to update the intrinsics here to match 49 | data["cx"] = cam.cx - self.corrected_image_padding 50 | data["cy"] = cam.cy - self.corrected_image_padding 51 | data["w"] = cam.width - 2 * self.corrected_image_padding 52 | data["h"] = cam.height - 2 * self.corrected_image_padding 53 | else: 54 | data["cx"] = cam.cx 55 | data["cy"] = cam.cy 56 | data["w"] = cam.width 57 | data["h"] = cam.height 58 | 59 | bbox = CaptureFolder.camera_bbox(keyframes) 60 | print(bbox) 61 | ## Use camera bbox to compute scale, and offset 62 | ## See https://github.com/NVlabs/instant-ngp/blob/master/docs/nerf_dataset_tips.md#scaling-existing-datasets 63 | max_size = np.max(bbox.size()) * 0.75 64 | data["scale"] = float(1.0 / max_size) 65 | offset = -bbox.center() / max_size 66 | data["offset"] = [float(offset[0]) + 0.5, float(offset[1]) + 0.5, float(offset[2]) + 0.5] 67 | data["aabb_scale"] = 2 68 | 69 | ## add the frames 70 | frames = [] 71 | for keyframe in keyframes: 72 | frames.append(self._convert_keyframe(keyframe, folder)) 73 | data["frames"] = frames 74 | 75 | # Write the output file to json 76 | output_file_path = os.path.join(folder.root, "transforms.json") 77 | with open(output_file_path, "w") as f: 78 | json.dump(data, f, indent=2) 79 | logger.info("Successfuly wrote the data to {}".format(output_file_path)) 80 | 81 | def _convert_keyframe(self, keyframe: Keyframe, folder: CaptureFolder) -> dict: 82 | """ Converts Polycam keyframe into a dictionary to be serialized as json """ 83 | frame = {} 84 | # add the image 85 | if folder.has_optimized_poses(): 86 | # For the corrected images we apply an image crop to remove the black strip around the boundary that is a result of the undistortion process 87 | # Since this may not be handled by nerf software 88 | full_path = os.path.join(folder.root, CaptureArtifact.CORRECTED_IMAGES.value, "{}.jpg".format(keyframe.timestamp)) 89 | full_path_crop = os.path.join(folder.root, CaptureArtifact.CORRECTED_IMAGES.value, "{}_crop.jpg".format(keyframe.timestamp)) 90 | im = Image.open(full_path) 91 | im = im.crop((self.corrected_image_padding, self.corrected_image_padding, keyframe.camera.width - 2 * self.corrected_image_padding, keyframe.camera.height - 2 * self.corrected_image_padding)) 92 | im.save(full_path_crop) 93 | frame["file_path"] = "./{}/{}_crop.jpg".format(CaptureArtifact.CORRECTED_IMAGES.value, keyframe.timestamp) 94 | else: 95 | frame["file_path"] = "./{}/{}.jpg".format(CaptureArtifact.IMAGES.value, keyframe.timestamp) 96 | if keyframe.camera.blur_score: 97 | frame["sharpness"] = keyframe.camera.blur_score 98 | frame["transform_matrix"] = keyframe.camera.transform_rows 99 | return frame -------------------------------------------------------------------------------- /polyform/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolyCam/polyform/845389c5cc4ddc116925162467d1c9bfea829e4d/polyform/core/__init__.py -------------------------------------------------------------------------------- /polyform/core/bbox.py: -------------------------------------------------------------------------------- 1 | ''' 2 | File: bbox.py 3 | Polycam Inc. 4 | 5 | Created by Chris Heinrich on Saturday, 5th November 2022 6 | Copyright © 2022 Polycam Inc. All rights reserved. 7 | ''' 8 | 9 | import numpy as np 10 | from typing import List 11 | 12 | class BBox3D: 13 | def __init__(self, bbox_min: np.array, bbox_max: np.array): 14 | """ 15 | Initializes a 3D axis-aligned bbox from the min and max points 16 | 17 | Args: 18 | bbox_min: the minimum point, expected to be a 19 | """ 20 | assert bbox_min.shape == np.shape([0,0,0]), "bbox_min must be a numpy array of shape (3,)" 21 | assert bbox_max.shape == np.shape([0,0,0]), "bbox_max must be a numpy array of shape (3,)" 22 | self.min = bbox_min 23 | self.max = bbox_max 24 | 25 | def center(self) -> np.array: 26 | return (self.min + self.max) / 2.0 27 | 28 | def size(self) -> np.array: 29 | return (self.max - self.min) 30 | 31 | def __str__(self): 32 | return "\n *** BBox3D *** \nmin: {}\nmax: {}\nsize: {}\ncenter: {}\n".format(self.min, self.max, self.size(), self.center()) 33 | 34 | 35 | def bbox_from_points(points: List[np.array]) -> BBox3D: 36 | """ 37 | Generates a bounding box from a list of points (i.e. 3D numpy arrays) 38 | """ 39 | finfo = np.finfo(np.float32) 40 | bbox_min = np.asarray([finfo.max, finfo.max, finfo.max]) 41 | bbox_max = np.asarray([finfo.min, finfo.min, finfo.min]) 42 | for point in points: 43 | # x 44 | if point[0] < bbox_min[0]: 45 | bbox_min[0] = point[0] 46 | if point[0] > bbox_max[0]: 47 | bbox_max[0] = point[0] 48 | # y 49 | if point[1] < bbox_min[1]: 50 | bbox_min[1] = point[1] 51 | if point[1] > bbox_max[1]: 52 | bbox_max[1] = point[1] 53 | #z 54 | if point[2] < bbox_min[2]: 55 | bbox_min[2] = point[2] 56 | if point[2] > bbox_max[2]: 57 | bbox_max[2] = point[2] 58 | 59 | return BBox3D(bbox_min, bbox_max) -------------------------------------------------------------------------------- /polyform/core/capture_folder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | File: capture_folder.py 3 | Polycam Inc. 4 | 5 | Created by Chris Heinrich on Tuesday, 1st November 2022 6 | Copyright © 2022 Polycam Inc. All rights reserved. 7 | ''' 8 | 9 | 10 | """ 11 | The CaptureFolder and associated types are the main wrappers around Polycam data as it is laid out on the filesystem. 12 | 13 | Initializing a CapureFolder from the absolute path of the data folder on the filesystem is usually the first step to 14 | working with a raw Polycam dataset. 15 | """ 16 | 17 | import os 18 | import json 19 | import numpy as np 20 | from polyform.core.bbox import BBox3D, bbox_from_points 21 | from polyform.utils.logging import logger 22 | from enum import Enum 23 | from typing import List 24 | 25 | 26 | class CaptureArtifact(Enum): 27 | IMAGES = "keyframes/images" 28 | CORRECTED_IMAGES = "keyframes/corrected_images" 29 | CAMERAS = "keyframes/cameras" 30 | CORRECTED_CAMERAS = "keyframes/corrected_cameras" 31 | DEPTH_MAPS = "keyframes/depth" 32 | CONFIDENCE_MAPS = "keyframes/confidence" 33 | MESH_INFO = "mesh_info.json" 34 | ANCHORS = "anchors.json" 35 | PREVIEW_MESH = "mesh.obj" 36 | 37 | 38 | class Camera: 39 | def __init__(self, j: dict, rotate: bool = False): 40 | """ Initializes a Camera object from the Polycam camera json format 41 | Args: 42 | j: json representation of a camera object 43 | rotate: rotates transform data to use the instant-ngp/nerftsudio convention (default) 44 | """ 45 | self.fx = j["fx"] 46 | self.fy = j["fy"] 47 | self.cx = j["cx"] 48 | self.cy = j["cy"] 49 | self.width = j["width"] 50 | self.height = j["height"] 51 | self.blur_score = j["blur_score"] 52 | if rotate: 53 | self.transform_rows = [ 54 | [j["t_20"], j["t_21"], j["t_22"], j["t_23"]], 55 | [j["t_00"], j["t_01"], j["t_02"], j["t_03"]], 56 | [j["t_10"], j["t_11"], j["t_12"], j["t_13"]], 57 | [0.0,0.0,0.0,1.0]] 58 | else: 59 | self.transform_rows = [ 60 | [j["t_00"], j["t_01"], j["t_02"], j["t_03"]], 61 | [j["t_10"], j["t_11"], j["t_12"], j["t_13"]], 62 | [j["t_20"], j["t_21"], j["t_22"], j["t_23"]], 63 | [0.0,0.0,0.0,1.0]] 64 | self.transform = np.asarray(self.transform_rows, dtype=np.float32) 65 | 66 | class Keyframe: 67 | """ 68 | A Keyframe includes the camera information (extrinsics and intrinsics) as well as the 69 | path to the associated data on disk (images, depth map, confidence) 70 | """ 71 | def __init__(self, folder: str, timestamp: int, rotate: bool): 72 | self.folder = folder 73 | self.timestamp = timestamp 74 | self.image_path = os.path.join( 75 | folder, "{}/{}.jpg".format(CaptureArtifact.IMAGES.value, timestamp)) 76 | self.corrected_image_path = os.path.join( 77 | folder, "{}/{}.jpg".format(CaptureArtifact.CORRECTED_IMAGES.value, timestamp)) 78 | self.camera_path = os.path.join( 79 | folder, "{}/{}.json".format(CaptureArtifact.CAMERAS.value, timestamp)) 80 | self.corrected_camera_path = os.path.join( 81 | folder, "{}/{}.json".format(CaptureArtifact.CORRECTED_CAMERAS.value, timestamp)) 82 | self.depth_path = os.path.join( 83 | folder, "{}/{}.png".format(CaptureArtifact.DEPTH_MAPS.value, timestamp)) 84 | self.camera = Camera(self.get_best_camera_json(), rotate) 85 | 86 | def is_valid(self) -> bool: 87 | if not os.path.isfile(self.camera_path): 88 | return False 89 | if not os.path.isfile(self.image_path): 90 | return False 91 | if not os.path.isfile(self.depth_path): 92 | return False 93 | return True 94 | 95 | def is_optimized(self) -> bool: 96 | if not os.path.isfile(self.corrected_camera_path): 97 | return False 98 | if not os.path.isfile(self.corrected_image_path): 99 | return False 100 | return True 101 | 102 | def get_best_camera_json(self) -> dict: 103 | """ Returns the camera json for the optimized camera if it exists, othewise returns the ARKit camera """ 104 | if self.is_optimized(): 105 | return CaptureFolder.load_json(self.corrected_camera_path) 106 | else: 107 | return CaptureFolder.load_json(self.camera_path) 108 | 109 | def __str__(self): 110 | return "keyframe:{}".format(self.timestamp) 111 | 112 | 113 | class CaptureFolder: 114 | def __init__(self, root: str): 115 | self.root = root 116 | self.id = os.path.basename(os.path.normpath(root)) 117 | if not self.has_optimized_poses(): 118 | logger.warning("Camera poses have not been optimized, the ARKit poses will be used as a fallback") 119 | 120 | def get_artifact_path(self, artifact: CaptureArtifact) -> str: 121 | return os.path.join(self.root, artifact.value) 122 | 123 | def get_artifact_paths(self, folder_artifact: CaptureArtifact, 124 | file_extension: str) -> List[str]: 125 | """ 126 | Returns all of the artifacts located in the folder_artifact with the provided file_extension 127 | """ 128 | paths = [] 129 | folder_path = self.get_artifact_path(folder_artifact) 130 | if not os.path.isdir(folder_path): 131 | return paths 132 | return [os.path.join(folder_path, path) for path in sorted(os.listdir(folder_path)) if path.endswith(file_extension)] 133 | 134 | def has_artifact(self, artifact: CaptureArtifact) -> bool: 135 | return os.path.exists(self.get_artifact_path(artifact)) 136 | 137 | def has_optimized_poses(self) -> bool: 138 | return (self.has_artifact(CaptureArtifact.CORRECTED_CAMERAS) and self.has_artifact(CaptureArtifact.CORRECTED_IMAGES)) 139 | 140 | def get_image_paths(self) -> List[str]: 141 | return self.get_artifact_paths(CaptureArtifact.IMAGES, "jpg") 142 | 143 | def get_camera_paths(self) -> List[str]: 144 | return self.get_artifact_paths(CaptureArtifact.CAMERAS, "json") 145 | 146 | def get_depth_paths(self) -> List[str]: 147 | return self.get_artifact_paths(CaptureArtifact.DEPTH_IMAGES, "png") 148 | 149 | def get_keyframe_timestamps(self) -> List[int]: 150 | timestamps = [] 151 | folder_path = self.get_artifact_path(CaptureArtifact.CAMERAS) 152 | if not os.path.isdir(folder_path): 153 | return timestamps 154 | return [int(path.replace(".json", "")) for path in sorted(os.listdir(folder_path)) if path.endswith("json")] 155 | 156 | def get_keyframes(self, rotate: bool = False) -> List[Keyframe]: 157 | """ 158 | Returns all valid keyframes associated with this dataset 159 | """ 160 | keyframes = [] 161 | for ts in self.get_keyframe_timestamps(): 162 | keyframe = Keyframe(self.root, ts, rotate) 163 | if keyframe.is_valid(): 164 | keyframes.append(keyframe) 165 | return keyframes 166 | 167 | @staticmethod 168 | def load_json(path) -> dict: 169 | name, ext = os.path.splitext(path) 170 | if not os.path.exists(path) or ext.lower() != ".json": 171 | print("File at path {} did not exist or was not a json file. Returning empty dict".format(path)) 172 | return {} 173 | else: 174 | with open(path) as f: 175 | return json.load(f) 176 | 177 | def load_json_artifact(self, folder_artifact: CaptureArtifact) -> dict: 178 | path = self.get_artifact_path(folder_artifact) 179 | return CaptureFolder.load_json(path) 180 | 181 | @staticmethod 182 | def camera_bbox(keyframes: List[Keyframe]) -> BBox3D: 183 | """ 184 | Returns the bounding box that contains all the cameras 185 | """ 186 | positions = [] 187 | for keyframe in keyframes: 188 | positions.append(keyframe.camera.transform[0:3,3]) 189 | return bbox_from_points(points=positions) 190 | 191 | 192 | -------------------------------------------------------------------------------- /polyform/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolyCam/polyform/845389c5cc4ddc116925162467d1c9bfea829e4d/polyform/utils/__init__.py -------------------------------------------------------------------------------- /polyform/utils/logging.py: -------------------------------------------------------------------------------- 1 | ''' 2 | File: logging.py 3 | Polycam Inc. 4 | 5 | Created by Chris Heinrich on Tuesday, 1st November 2022 6 | Copyright © 2022 Polycam Inc. All rights reserved. 7 | ''' 8 | import os 9 | import logging 10 | 11 | """ 12 | Helper functions to make it faster to work with Python's standard logging library 13 | """ 14 | 15 | def set_log_level(logger, default_level=logging.INFO): 16 | if "LOG_LEVEL" in os.environ: 17 | level = os.environ["LOG_LEVEL"].upper() 18 | exec("logger.setLevel(logging.{})".format(level)) 19 | else: 20 | logger.setLevel(default_level) 21 | 22 | 23 | def setup_logger(name): 24 | logger = logging.getLogger(name) 25 | set_log_level(logger) 26 | logging.basicConfig() 27 | return logger 28 | 29 | 30 | def class_logger(obj): 31 | """ initializes a logger with type name of obj """ 32 | return setup_logger(type(obj).__name__) 33 | 34 | 35 | def file_logger(file): 36 | this_file = os.path.splitext(os.path.basename(file))[0] 37 | logger = setup_logger(' ' + this_file + ' ') 38 | return logger 39 | 40 | 41 | # A common logger instance that can be used when a dedicated 42 | # named logger is not needed 43 | logger = setup_logger('polyform') --------------------------------------------------------------------------------