├── .gitignore ├── LICENSE ├── README.md ├── area_1 ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── normal │ │ └── .gitkeep │ └── pose │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep ├── area_2 ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── depth │ │ └── .gitkeep │ ├── global_xyz │ │ └── .gitkeep │ ├── normal │ │ └── .gitkeep │ ├── pose │ │ └── .gitkeep │ ├── rgb │ │ └── .gitkeep │ ├── semantic │ │ └── .gitkeep │ └── semantic_pretty │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep ├── area_3 ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── depth │ │ └── .gitkeep │ ├── global_xyz │ │ └── .gitkeep │ ├── normal │ │ └── .gitkeep │ ├── pose │ │ └── .gitkeep │ ├── rgb │ │ └── .gitkeep │ ├── semantic │ │ └── .gitkeep │ └── semantic_pretty │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep ├── area_4 ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── depth │ │ └── .gitkeep │ ├── global_xyz │ │ └── .gitkeep │ ├── normal │ │ └── .gitkeep │ ├── pose │ │ └── .gitkeep │ ├── rgb │ │ └── .gitkeep │ ├── semantic │ │ └── .gitkeep │ └── semantic_pretty │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep ├── area_5a ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── depth │ │ └── .gitkeep │ ├── global_xyz │ │ └── .gitkeep │ ├── normal │ │ └── .gitkeep │ ├── pose │ │ └── .gitkeep │ ├── rgb │ │ └── .gitkeep │ ├── semantic │ │ └── .gitkeep │ └── semantic_pretty │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep ├── area_5b ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── depth │ │ └── .gitkeep │ ├── global_xyz │ │ └── .gitkeep │ ├── normal │ │ └── .gitkeep │ ├── pose │ │ └── .gitkeep │ ├── rgb │ │ └── .gitkeep │ ├── semantic │ │ └── .gitkeep │ └── semantic_pretty │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep ├── area_6 ├── 3d │ └── .gitkeep ├── LICENSE.pdf ├── data │ ├── depth │ │ └── .gitkeep │ ├── global_xyz │ │ └── .gitkeep │ ├── normal │ │ └── .gitkeep │ ├── pose │ │ └── .gitkeep │ ├── rgb │ │ └── .gitkeep │ ├── semantic │ │ └── .gitkeep │ └── semantic_pretty │ │ └── .gitkeep └── pano │ ├── depth │ └── .gitkeep │ ├── global_xyz │ └── .gitkeep │ ├── normal │ └── .gitkeep │ ├── pose │ └── .gitkeep │ ├── rgb │ └── .gitkeep │ ├── semantic │ └── .gitkeep │ └── semantic_pretty │ └── .gitkeep └── assets ├── figure_1.jpg ├── pcl_utils.h ├── semantic_labels.json └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Images 2 | *.png 3 | *.exr 4 | *.jpg 5 | *.jpeg 6 | 7 | # Pose information 8 | *.json 9 | *.csv 10 | 11 | # Annotations 12 | *.mat 13 | *.txt 14 | 15 | # Meshes 16 | *.obj 17 | *.mtl 18 | 19 | # Byte-compiled / optimized / DLL files 20 | __pycache__/ 21 | *.py[cod] 22 | *$py.class 23 | 24 | # C extensions 25 | *.so 26 | 27 | # Distribution / packaging 28 | .Python 29 | env/ 30 | build/ 31 | develop-eggs/ 32 | dist/ 33 | downloads/ 34 | eggs/ 35 | .eggs/ 36 | lib/ 37 | lib64/ 38 | parts/ 39 | sdist/ 40 | var/ 41 | *.egg-info/ 42 | .installed.cfg 43 | *.egg 44 | 45 | # PyInstaller 46 | # Usually these files are written by a python script from a template 47 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 48 | *.manifest 49 | *.spec 50 | 51 | # Installer logs 52 | pip-log.txt 53 | pip-delete-this-directory.txt 54 | 55 | # Unit test / coverage reports 56 | htmlcov/ 57 | .tox/ 58 | .coverage 59 | .coverage.* 60 | .cache 61 | nosetests.xml 62 | coverage.xml 63 | *,cover 64 | .hypothesis/ 65 | 66 | # Translations 67 | *.mo 68 | *.pot 69 | 70 | # Django stuff: 71 | *.log 72 | local_settings.py 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # IPython Notebook 88 | .ipynb_checkpoints 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # celery beat schedule file 94 | celerybeat-schedule 95 | 96 | # dotenv 97 | .env 98 | 99 | # virtualenv 100 | venv/ 101 | ENV/ 102 | 103 | # Spyder project settings 104 | .spyderproject 105 | 106 | # Rope project settings 107 | .ropeproject 108 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 2D-3D-Semantics Data 2 | 3 | ![Figure showing multiple modalities]( assets/figure_1.jpg?raw=true ) 4 | 5 | 6 | ## Overview: 7 | The 2D-3D-S dataset provides a variety of mutually registered modalities from 2D, 2.5D and 3D domains, with instance-level semantic and geometric annotations. It covers over 6,000 m2 collected in 6 large-scale indoor areas that originate from 3 different buildings. It contains over 70,000 RGB images, along with the corresponding depths, surface normals, semantic annotations, global XYZ images (all in forms of both regular and 360° equirectangular images) as well as camera information. It also includes registered raw and semantically annotated 3D meshes and point clouds. The dataset enables development of joint and cross-modal learning models and potentially unsupervised approaches utilizing the regularities present in large-scale indoor spaces. 8 | 9 | This repo provides a [set of tools](https://github.com/alexsax/2D-3D-Semantics/blob/master/README.md#tools) for interacting with the dataset as well as technical information not necessarily included on the project site. For more information on the dataset, visit the [[project site]](http://3Dsemantics.stanford.edu) or the [[dataset wiki]](https://github.com/alexsax/2D-3D-Semantics/wiki). 10 | 11 | 12 | ## Download 13 | The link will first take you to a license agreement, and then to the data. 14 | 15 | ### [[ Download the full 2D-3D-S Dataset ]](https://goo.gl/forms/2YSPaO2UKmn5Td5m2) [[ checksums ]](https://github.com/alexsax/2D-3D-Semantics/wiki/Checksum-Values-for-Data) 16 | 17 | The full dataset is very large at 766G. Therefore, we have split the data by area to accomodate a la carte data selection. The dataset also comes in two flavors: with global_xyz images (766G) and without (110G). Once you click on the download link and accept the license agreement, the variants are accessible from in the `withXYZ` and `noXYZ` folders, respectively. 18 | 19 | 20 | #### License Note: The dataset license is included in the above link. The license in this repository covers only the provided software. 21 | 22 | 23 | 24 | ### Citations 25 | If you use this dataset please cite: 26 | ``` 27 | @article{armeni2017joint, 28 | title={Joint 2d-3d-semantic data for indoor scene understanding}, 29 | author={Armeni, Iro and Sax, Sasha and Zamir, Amir R and Savarese, Silvio}, 30 | journal={arXiv preprint arXiv:1702.01105}, 31 | year={2017} 32 | } 33 | ``` 34 | 35 | 36 | ## Dataset Modalities 37 | Each physical area in the dataset has its own folder in the dataset. All the modalities and metadata for each area are contained in that folder. In addition, `/assets` contains some global metadata and utilities in Python. The utilities are in `/assets/utils.py` and contain functions for reading EXRs and parsing labels. 38 | 39 | ``` 40 | README.md 41 | /assets 42 | semantic_labels.json 43 | utils.py 44 | /area_1 45 | /3d 46 | pointcloud.mat 47 | rgb.obj # The raw 3d mesh with rgb textures 48 | rgb.mtl # The textures for the raw 3d mesh 49 | semantic.obj # Semantically-tagged 3d mesh 50 | semantic.mtl # Tectures for semantic.obj 51 | /rgb_textures 52 | {uuid_{i}}.jpg # Texture images for the rgb 3d mesh 53 | /data # all of the generated data 54 | /pose 55 | camera_{uuid}__{room}_{i}_frame_{j}_domain__pose.json 56 | /rgb 57 | camera_{uuid}__{room}_{i}_frame_{j}_domain__rgb.png 58 | /depth 59 | /global_xyz 60 | /normal 61 | /semantic 62 | /semantic_pretty 63 | /pano # equirectangular projections 64 | /pose 65 | camera_{uuid}__{room}_{i}_frame_equirectangular_domain__pose.json 66 | /rgb 67 | /depth 68 | /global_xyz 69 | /normal 70 | /semantic 71 | /semantic_pretty 72 | /raw # Raw data from Matterport 73 | {uuid}_pose_{pitch_level}_{yaw_position}.txt # RT matrix for raw sensor 74 | {uuid}_intrinsics_{pitch_level}.txt # Camera calibration for sensor at {pitch_level} 75 | {uuid}_i{pitch_level}_{yaw_position}.jpg # Raw RGB image from sensor 76 | {uuid}_d{pitch_level}_{yaw_position}.jpg # Raw depth image form sensor 77 | /area_2 78 | /area_3 79 | /area_4 80 | /area_5a 81 | /area_5b 82 | /area_6 83 | ``` 84 | 85 | ### 3D modalities: 86 | The dataset contains colored point clouds and textured meshes for each scanned area. 87 | 88 | **3D Point Cloud:** The raw colored 3D point clouds along with both object and scene instance-level annotations per point, (tightest) axis-aligned bounding boxes and voxels with binary occupancy and point correpsondence are stored in the Area_#_PointCloud.mat file. The variables are stored in the form of nested structs: 89 | ``` 90 | - Area: --> name: the area name as: Area_# 91 | --> Disjoint-Space: struct with information on the individual spaces in the buidling. 92 | ``` 93 | 94 | In each *Disjoint_Space* struct: 95 | ``` 96 | - Disjoint_Space: --> name: the name of that space, with per area global index (e.g. conferenceRoom_1, offie_13, etc.) 97 | -->AlignmentAngle: rotation angle around Z axis, to align spaces based on the CVPR2016 paper *3D Semantic Parsing of Large-Scale Indoor Spaces*. 98 | --> color: a unique RGB color value [0,1] for that room, mainly for visualization purposes 99 | --> object: a struct that contains all objects in that space. 100 | ``` 101 | 102 | In each *object* struct: 103 | ``` 104 | - object: --> name: the name of the object, wiith per space indexing* (e.g. chair-1, wall_3, clutter_13, etc.) 105 | --> points: the X,Y,Z coordinates of the 3D points that comprise this object 106 | --> RGB_color: the raw RGB color value [0,255] associated with each point. 107 | --> global_name: the name of the object, with per area global index** 108 | --> Bbox: [Xmin Ymin Zmin Xmax Ymax Zmax] of the object's boudning box 109 | --> Voxels: [Xmin Ymin Zmin Xmax Ymax Zmax] for each voxel in a 6x6x6 grid 110 | --> Voxel_Occupancy: binary occupancy per voxel (0: empty, 1: contains points) 111 | --> Points_per_Voxel: the object points that correspond to each voxel (in XYZ coordinates) 112 | ``` 113 | 114 | **3D Semantic mesh:** 115 | The 3D semantic mesh is labeled with instance-level per-face annotations. The mesh is stored in `semantic.obj` and `semantic.mtl` where face labels are stored as face's material's name. In Blender, for example, the material label can be retrieved with: 116 | ```python 117 | mesh_material_idx = mesh.data.polygons[ face_idx ].material_index 118 | material = mesh.data.materials[ mesh_material_idx ] 119 | label = material.name # final label! 120 | ``` 121 | The labels follow the convention `class_instanceNum_roomType_roomNum_areaNum`. 122 | 123 | When using the mesh's color, the material's color should be remapped for the task at hand as the default color is designed for visualizations. One way to encode the label in the color is to set 124 | ``` 125 | material.diffuse_color = get_color( labels.index( material.name ) ) 126 | ``` 127 | by using `labels` from `/assets/semantic_labels.json` and `get_color( color )` from `/assets/utils.py`. 128 | 129 | ### 2D modalities: 130 | The dataset contains densely sampled RGB images per scan location. We also provide depth and surface normal images, 2D semantic annotations (instance-level, per-pixel) and 3D coordinate encoded images where each pixel encodes the X, Y, Z location of the point in the world coordinate system. All images in the dataset are stored in full high-definition at 1080x1080 resolution. We provide the camera metadata for each generated image. Last, an equirectangular projection is also provided per scan location and modality. 131 | 132 | **Pose:** 133 | The pose files contain camera metadata for each image and are given in the `/pose` subdirectories. They have filenames which are globally unique due to the fact that camera uuids are not shared between areas. They are stored in json files, and contain 134 | ```python 135 | { 136 | "camera_k_matrix": # The 3x3 camera K matrix. Stored as a list-of-lists, 137 | "field_of_view_rads": # The Camera's field of view, in radians, 138 | "camera_original_rotation": # The camera's initial XYZ-Euler rotation in the .obj, 139 | "rotation_from_original_to_point": 140 | # Apply this to the original rotation in order to orient the camera for the corresponding picture, 141 | "point_uuid": # alias for camera_uuid, 142 | "camera_location": # XYZ location of the camera, 143 | "frame_num": # The frame_num in the filename, 144 | "camera_rt_matrix": # The 4x3 camera RT matrix, stored as a list-of-lists, 145 | "final_camera_rotation": # The camera Euler in the corresponding picture, 146 | "camera_uuid": # The globally unique identifier for the camera location, 147 | "room": # The room that this camera is in. Stored as roomType_roomNum_areaNum 148 | } 149 | ``` 150 | Note that in `equirectangular` images the FOV will be 90 degrees, but the image is actually panoramic and therefore a 360-degree view. 151 | 152 | **RGB:** 153 | RGB images are in the `/rgb` folder and contain synthesized but accurate RGB images of the scene. 154 | 155 | **Depth:** 156 | Depth images are stored as 16-bit PNGs and have a maximum depth of 128m and a sensitivity of 1/512m. Missing values are encoded with the value 2^16 - 1. Note that while depth is defined relative to the plane of the camera in the data (z-depth), it is defined as the distance from the point-center of the camera in the panoramics. 157 | 158 | **Global XYZ:** 159 | Global XYZ images contain the ground-truth location of each pixel in the mesh. They are stored as 16-bit 3-channel OpenEXR files and a convenience readin function is provided in `/assets/utils.py`. These can be used for generating point correspondences, e.g. for optical flow. Missing values are encoded as #000000. 160 | 161 | **Normal:** 162 | Normals are 127.5-centered per-channel surface normal images. For panoramic images, these normals are relative to the global corodinate system. Since the global coordinate system is impossible to determine from a sampled image, the `normal` images in `/data` have their normals defined relative to the direction the camera is facing. The normals axis-color convention is the same one used by NYU RGB-D. Areas where the mesh is missing have pixel color #808080. 163 | 164 | 165 | **Semantic:** 166 | Semantic images come in two variants, `semantic` and `semantic_pretty`. They both include information from the point cloud annotations, but only the `semantic` version should be used for learning. The `semantic` images have RGB images which are direct 24-bit base-256 integers which contain an index into `/assets/semantic_labels.json`. Pixels where the data is missing are encoded with the color #0D0D0D which is larger than the `len( labels )`. 167 | 168 | ### Raw Sensor data: 169 | The raw data from the Matterport sensors is also included in the `raw` folder. The Matterport camera contains 3 sensors at different pitches, and rotates 60 degrees at a time to complete a 360-degree sweep. We make available both the raw RGB images and also the raw depth images which can be stitched together to make a mesh, or else used independently. 170 | 171 | ## Tools 172 | This repository provides some basic tools for interacting with the dataset. Pull requests are welcome! 173 | 174 | The tools are located in the `assets/` folder and reside in two files. 175 | 176 | **utils.py (Python)** 177 | This file contains Python tools, and in general it is useful for working with images. It provides convenience functions for loading semantic labels, and reading the ground-truth label from an image pixel. It also provides a toolkit to work with .exr files. 178 | 179 | **pcl_utils.h (C++)** 180 | This file contains C++ tools, and in general it is useful for working with the .obj files and point clouds. It provides convenience functions for loading 6DOF camera poses. -------------------------------------------------------------------------------- /area_1/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/3d/.gitkeep -------------------------------------------------------------------------------- /area_1/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/LICENSE.pdf -------------------------------------------------------------------------------- /area_1/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_1/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_1/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_1/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_2/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/3d/.gitkeep -------------------------------------------------------------------------------- /area_2/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/LICENSE.pdf -------------------------------------------------------------------------------- /area_2/data/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/depth/.gitkeep -------------------------------------------------------------------------------- /area_2/data/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_2/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_2/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_2/data/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/rgb/.gitkeep -------------------------------------------------------------------------------- /area_2/data/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/semantic/.gitkeep -------------------------------------------------------------------------------- /area_2/data/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/data/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_2/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_2/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_3/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/3d/.gitkeep -------------------------------------------------------------------------------- /area_3/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/LICENSE.pdf -------------------------------------------------------------------------------- /area_3/data/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/depth/.gitkeep -------------------------------------------------------------------------------- /area_3/data/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_3/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_3/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_3/data/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/rgb/.gitkeep -------------------------------------------------------------------------------- /area_3/data/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/semantic/.gitkeep -------------------------------------------------------------------------------- /area_3/data/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/data/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_3/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_3/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_4/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/3d/.gitkeep -------------------------------------------------------------------------------- /area_4/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/LICENSE.pdf -------------------------------------------------------------------------------- /area_4/data/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/depth/.gitkeep -------------------------------------------------------------------------------- /area_4/data/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_4/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_4/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_4/data/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/rgb/.gitkeep -------------------------------------------------------------------------------- /area_4/data/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/semantic/.gitkeep -------------------------------------------------------------------------------- /area_4/data/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/data/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_4/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_4/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_5a/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/3d/.gitkeep -------------------------------------------------------------------------------- /area_5a/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/LICENSE.pdf -------------------------------------------------------------------------------- /area_5a/data/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/depth/.gitkeep -------------------------------------------------------------------------------- /area_5a/data/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_5a/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_5a/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_5a/data/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/rgb/.gitkeep -------------------------------------------------------------------------------- /area_5a/data/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/semantic/.gitkeep -------------------------------------------------------------------------------- /area_5a/data/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/data/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_5a/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5a/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_5b/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/3d/.gitkeep -------------------------------------------------------------------------------- /area_5b/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/LICENSE.pdf -------------------------------------------------------------------------------- /area_5b/data/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/depth/.gitkeep -------------------------------------------------------------------------------- /area_5b/data/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_5b/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_5b/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_5b/data/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/rgb/.gitkeep -------------------------------------------------------------------------------- /area_5b/data/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/semantic/.gitkeep -------------------------------------------------------------------------------- /area_5b/data/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/data/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_5b/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_5b/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_6/3d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/3d/.gitkeep -------------------------------------------------------------------------------- /area_6/LICENSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/LICENSE.pdf -------------------------------------------------------------------------------- /area_6/data/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/depth/.gitkeep -------------------------------------------------------------------------------- /area_6/data/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_6/data/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/normal/.gitkeep -------------------------------------------------------------------------------- /area_6/data/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/pose/.gitkeep -------------------------------------------------------------------------------- /area_6/data/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/rgb/.gitkeep -------------------------------------------------------------------------------- /area_6/data/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/semantic/.gitkeep -------------------------------------------------------------------------------- /area_6/data/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/data/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/depth/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/depth/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/global_xyz/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/global_xyz/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/normal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/normal/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/pose/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/pose/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/rgb/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/rgb/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/semantic/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/semantic/.gitkeep -------------------------------------------------------------------------------- /area_6/pano/semantic_pretty/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/area_6/pano/semantic_pretty/.gitkeep -------------------------------------------------------------------------------- /assets/figure_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexsax/2D-3D-Semantics/54a532959b20203ea4c3fcc26f5c8bf678d6fdb4/assets/figure_1.jpg -------------------------------------------------------------------------------- /assets/pcl_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * pcl_utils.cpp - Some convenience functions for using data from 3 | * Joint 2D-3D-Semantic Data for Indoor Scene Understanding 4 | * in conjunction with Point Cloud Library (PCL) 5 | * 6 | * Website: 3dsemantics.stanford.edu 7 | * Paper: https://arxiv.org/pdf/1702.01105.pdf 8 | * 9 | * Usage: Copy or include the code 10 | */ 11 | #ifndef SEMANTIC2D3D_UTILS_H 12 | #define SEMANTIC2D3D_UTILS_H 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace utils 20 | { 21 | 22 | /** Gets the camera translation from a pose file ptree and returns it**/ 23 | inline Eigen::Affine3f 24 | getCameraTranslation( const boost::property_tree::ptree &pt ){ 25 | // Read in camera translation 26 | float tr[3]; 27 | int i = 0; 28 | boost::property_tree::ptree::const_iterator end = pt.get_child("camera_location").end(); 29 | for (boost::property_tree::ptree::const_iterator it = pt.get_child("camera_location").begin(); it != end; ++it, ++i) { 30 | tr[i] = it->second.get_value(); 31 | } 32 | Eigen::Affine3f translation(Eigen::Translation3f(tr[0], tr[1], tr[2])); 33 | return translation; 34 | } 35 | 36 | /** Gets the camera rotation from a pose file ptree and returns it**/ 37 | inline Eigen::Affine3f 38 | getCameraRotation( const boost::property_tree::ptree &pt ){ 39 | // Read in the camera euler angles 40 | int i = 0; 41 | float euler_angles[3]; 42 | boost::property_tree::ptree::const_iterator end = pt.get_child("camera_rotation_final").end(); 43 | for (boost::property_tree::ptree::const_iterator it = pt.get_child("camera_rotation_final").begin(); it != end; ++it, ++i) { 44 | euler_angles[i] = it->second.get_value(); 45 | std::cout << "Angle " << i << ": " << euler_angles[i] << std::endl; 46 | } 47 | //Roll pitch and yaw in Radians 48 | float roll = euler_angles[2], pitch = euler_angles[1], yaw = euler_angles[0]; 49 | std::cout << roll << ", " << pitch << ", " << yaw << std::endl; 50 | Eigen::Quaternionf q; 51 | q = Eigen::AngleAxisf(roll, Eigen::Vector3f::UnitZ()) 52 | * Eigen::AngleAxisf(pitch, Eigen::Vector3f::UnitY()) 53 | * Eigen::AngleAxisf(yaw, Eigen::Vector3f::UnitX()); 54 | Eigen::Affine3f rotation( q ); 55 | return rotation; 56 | } 57 | 58 | /** Load in the json pose files into a boost::ptree **/ 59 | inline boost::property_tree::ptree 60 | loadPoseFile( const std::string &json_filename ) { 61 | // Read in view_dict 62 | boost::property_tree::ptree pt; 63 | std::ifstream in(json_filename); 64 | std::stringstream buffer; 65 | buffer << in.rdbuf(); 66 | read_json(buffer, pt); 67 | return pt; 68 | } 69 | 70 | /** Debugging function to print out a boost::ptree **/ 71 | void print(boost::property_tree::ptree const& pt){ 72 | using boost::property_tree::ptree; 73 | ptree::const_iterator end = pt.end(); 74 | for (ptree::const_iterator it = pt.begin(); it != end; ++it) { 75 | std::cout << it->first << ": " << it->second.get_value() << std::endl; 76 | print(it->second); 77 | } 78 | } 79 | } // END namespace 80 | 81 | #endif // #ifndef SEMANTIC2D3D_UTILS_H 82 | -------------------------------------------------------------------------------- /assets/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | utils.py - Some convenience functions for using data from 3 | Joint 2D-3D-Semantic Data for Indoor Scene Understanding 4 | I. Armeni*, A. Sax*, A. Zamir, S. Savarese 5 | Website: 3dsemantics.stanford.edu 6 | Paper: https://arxiv.org/pdf/1702.01105.pdf 7 | 8 | Code Author: Alexander Sax 9 | 10 | Usage: For import only. i.e. 'import utils.py' 11 | Dependencies include scipy, OpenEXR 12 | """ 13 | 14 | import array 15 | import Imath 16 | import json 17 | import numpy 18 | import OpenEXR 19 | from scipy.ndimage import imread 20 | 21 | """ Semantics """ 22 | def get_index( color ): 23 | ''' Parse a color as a base-256 number and returns the index 24 | Args: 25 | color: A 3-tuple in RGB-order where each element \in [0, 255] 26 | Returns: 27 | index: an int containing the indec specified in 'color' 28 | ''' 29 | return color[0] * 256 * 256 + color[1] * 256 + color[2] 30 | 31 | def get_color( i ): 32 | ''' Parse a 24-bit integer as a RGB color. I.e. Convert to base 256 33 | Args: 34 | index: An int. The first 24 bits will be interpreted as a color. 35 | Negative values will not work properly. 36 | Returns: 37 | color: A color s.t. get_index( get_color( i ) ) = i 38 | ''' 39 | b = ( i ) % 256 # least significant byte 40 | g = ( i >> 8 ) % 256 41 | r = ( i >> 16 ) % 256 # most significant byte 42 | return r,g,b 43 | 44 | """ Label functions """ 45 | def load_labels( label_file ): 46 | """ Convenience function for loading JSON labels """ 47 | with open( label_file ) as f: 48 | return json.load( f ) 49 | 50 | def parse_label( label ): 51 | """ Parses a label into a dict """ 52 | res = {} 53 | clazz, instance_num, room_type, room_num, area_num = label.split( "_" ) 54 | res[ 'instance_class' ] = clazz 55 | res[ 'instance_num' ] = int( instance_num ) 56 | res[ 'room_type' ] = room_type 57 | res[ 'room_num' ] = int( room_num ) 58 | res[ 'area_num' ] = int( area_num ) 59 | return res 60 | 61 | 62 | """ EXR Functions """ 63 | def normalize_array_for_matplotlib( arr_to_rescale ): 64 | ''' Rescales an array to be between [0, 1] 65 | Args: 66 | arr_to_rescale: 67 | Returns: 68 | An array in [0,1] with f(0) = 0.5 69 | ''' 70 | return ( arr_to_rescale / np.abs( arr_to_rescale ).max() ) / 2 + 0.5 71 | 72 | def read_exr( image_fpath ): 73 | """ Reads an openEXR file into an RGB matrix with floats """ 74 | f = OpenEXR.InputFile( image_fpath ) 75 | dw = f.header()['dataWindow'] 76 | w, h = (dw.max.x - dw.min.x + 1, dw.max.y - dw.min.y + 1) 77 | im = np.empty( (h, w, 3) ) 78 | 79 | # Read in the EXR 80 | FLOAT = Imath.PixelType(Imath.PixelType.FLOAT) 81 | channels = f.channels( ["R", "G", "B"], FLOAT ) 82 | for i, channel in enumerate( channels ): 83 | im[:,:,i] = np.reshape( array.array( 'f', channel ), (h, w) ) 84 | return im 85 | 86 | 87 | --------------------------------------------------------------------------------