├── .gitignore ├── Configuration └── SampleCode.xcconfig ├── LICENSE └── LICENSE.txt ├── README.md ├── SceneDepthPointCloud.xcodeproj ├── .xcodesamplecode.plist ├── project.pbxproj ├── project.xcworkspace │ └── xcshareddata │ │ └── WorkspaceSettings.xcsettings └── xcshareddata │ └── xcschemes │ └── SceneDepthPointCloud.xcscheme └── SceneDepthPointCloud ├── AppDelegate.swift ├── Assets.xcassets ├── AppIcon.appiconset │ ├── AR_icon_basic copy-1.png │ ├── AR_icon_basic copy-10.png │ ├── AR_icon_basic copy-11.png │ ├── AR_icon_basic copy-12.png │ ├── AR_icon_basic copy-13.png │ ├── AR_icon_basic copy-14.png │ ├── AR_icon_basic copy-15.png │ ├── AR_icon_basic copy-16.png │ ├── AR_icon_basic copy-2.png │ ├── AR_icon_basic copy-3.png │ ├── AR_icon_basic copy-4.png │ ├── AR_icon_basic copy-5.png │ ├── AR_icon_basic copy-6.png │ ├── AR_icon_basic copy-7.png │ ├── AR_icon_basic copy-8.png │ ├── AR_icon_basic copy-9.png │ ├── AR_icon_basic copy.png │ ├── AR_icon_basic.png │ └── Contents.json └── Contents.json ├── Base.lproj ├── LaunchScreen.storyboard └── Main.storyboard ├── Helpers.swift ├── Info.plist ├── MetalBuffer.swift ├── Renderer.swift ├── SceneDepthPointCloud-Bridging-Header.h ├── ShaderTypes.h ├── Shaders.metal └── ViewController.swift /.gitignore: -------------------------------------------------------------------------------- 1 | # See LICENSE folder for this sample’s licensing information. 2 | # 3 | # Apple sample code gitignore configuration. 4 | 5 | # Finder 6 | .DS_Store 7 | 8 | # Xcode - User files 9 | xcuserdata/ 10 | 11 | **/*.xcodeproj/project.xcworkspace/* 12 | !**/*.xcodeproj/project.xcworkspace/xcshareddata 13 | 14 | **/*.xcodeproj/project.xcworkspace/xcshareddata/* 15 | !**/*.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings 16 | 17 | **/*.playground/playground.xcworkspace/* 18 | !**/*.playground/playground.xcworkspace/xcshareddata 19 | 20 | **/*.playground/playground.xcworkspace/xcshareddata/* 21 | !**/*.playground/playground.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings 22 | -------------------------------------------------------------------------------- /Configuration/SampleCode.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // See LICENSE folder for this sample’s licensing information. 3 | // 4 | // SampleCode.xcconfig 5 | // 6 | 7 | // The `SAMPLE_CODE_DISAMBIGUATOR` configuration is to make it easier to build 8 | // and run a sample code project. Once you set your project's development team, 9 | // you'll have a unique bundle identifier. This is because the bundle identifier 10 | // is derived based on the 'SAMPLE_CODE_DISAMBIGUATOR' value. Do not use this 11 | // approach in your own projects—it's only useful for sample code projects because 12 | // they are frequently downloaded and don't have a development team set. 13 | SAMPLE_CODE_DISAMBIGUATOR=${DEVELOPMENT_TEAM} 14 | -------------------------------------------------------------------------------- /LICENSE/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright © 2020 Apple Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Visualizing a Point Cloud Using Scene Depth 2 | 3 | Place points in the real-world using the scene's depth data to visualize the shape of the physical environment. 4 | 5 | ## Overview 6 | 7 | - Note: This sample code project is associated with WWDC20 session [10611: Explore ARKit 4](https://developer.apple.com/wwdc20/10611/). 8 | 9 | ## Lidar Required 10 | 11 | You will need an Iphone or Ipad with a LIDAR sensor. 12 | 13 | ## Unity 14 | This is a native IOS app, if you are looking for a Unity implementation with AR Foundation, check out this link: 15 | https://github.com/cdmvision/arfoundation-densepointcloud 16 | 17 | --- 18 | 19 | Click Below to see a video sample of the project working on the Iphone 12 Pro LIDAR: 20 | 21 | ## Test 1 - Vegetation 22 | 23 | [![Point Cloud Slam](https://img.youtube.com/vi/Mybgpf3F6aI/0.jpg)](https://youtu.be/Mybgpf3F6aI "Ios Lidar Slam") 24 | 25 | 26 | ## Test 2 - Vehicle 27 | 28 | [![Point Cloud Slam](https://img.youtube.com/vi/xepcIAzD8zo/0.jpg)](https://youtu.be/xepcIAzD8zo "Ios Lidar Slam") 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /SceneDepthPointCloud.xcodeproj/.xcodesamplecode.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /SceneDepthPointCloud.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 50; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | E73DD13124636FEF00D77039 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = E73DD13024636FEF00D77039 /* AppDelegate.swift */; }; 11 | E73DD13324636FEF00D77039 /* Shaders.metal in Sources */ = {isa = PBXBuildFile; fileRef = E73DD13224636FEF00D77039 /* Shaders.metal */; }; 12 | E73DD13624636FEF00D77039 /* Renderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = E73DD13524636FEF00D77039 /* Renderer.swift */; }; 13 | E73DD13824636FEF00D77039 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = E73DD13724636FEF00D77039 /* ViewController.swift */; }; 14 | E73DD13B24636FEF00D77039 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = E73DD13924636FEF00D77039 /* Main.storyboard */; }; 15 | E73DD13D24636FF100D77039 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = E73DD13C24636FF100D77039 /* Assets.xcassets */; }; 16 | E73DD14024636FF100D77039 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = E73DD13E24636FF100D77039 /* LaunchScreen.storyboard */; }; 17 | E7B7223B24807ACA002E43E8 /* MetalBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7B7223A24807ACA002E43E8 /* MetalBuffer.swift */; }; 18 | E7B7223D24809D1E002E43E8 /* Helpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7B7223C24809D1E002E43E8 /* Helpers.swift */; }; 19 | /* End PBXBuildFile section */ 20 | 21 | /* Begin PBXFileReference section */ 22 | 948146554F46E56B015690CC /* SampleCode.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = SampleCode.xcconfig; path = Configuration/SampleCode.xcconfig; sourceTree = ""; }; 23 | B57D60B9736905755DECF7CE /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 24 | E73DD12D24636FEF00D77039 /* Point Cloud.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Point Cloud.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 25 | E73DD13024636FEF00D77039 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 26 | E73DD13224636FEF00D77039 /* Shaders.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = Shaders.metal; sourceTree = ""; }; 27 | E73DD13424636FEF00D77039 /* ShaderTypes.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ShaderTypes.h; sourceTree = ""; }; 28 | E73DD13524636FEF00D77039 /* Renderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Renderer.swift; sourceTree = ""; }; 29 | E73DD13724636FEF00D77039 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; 30 | E73DD13A24636FEF00D77039 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 31 | E73DD13C24636FF100D77039 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 32 | E73DD13F24636FF100D77039 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 33 | E73DD14124636FF100D77039 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 34 | E73DD14224636FF100D77039 /* SceneDepthPointCloud-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "SceneDepthPointCloud-Bridging-Header.h"; sourceTree = ""; }; 35 | E7B7223A24807ACA002E43E8 /* MetalBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MetalBuffer.swift; sourceTree = ""; }; 36 | E7B7223C24809D1E002E43E8 /* Helpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Helpers.swift; sourceTree = ""; }; 37 | E7E2ED6F1473F1FD1E9543FA /* LICENSE.txt */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text; path = LICENSE.txt; sourceTree = ""; }; 38 | /* End PBXFileReference section */ 39 | 40 | /* Begin PBXFrameworksBuildPhase section */ 41 | E73DD12A24636FEF00D77039 /* Frameworks */ = { 42 | isa = PBXFrameworksBuildPhase; 43 | buildActionMask = 2147483647; 44 | files = ( 45 | ); 46 | runOnlyForDeploymentPostprocessing = 0; 47 | }; 48 | /* End PBXFrameworksBuildPhase section */ 49 | 50 | /* Begin PBXGroup section */ 51 | A6415A999BE1C768F18B2649 /* Configuration */ = { 52 | isa = PBXGroup; 53 | children = ( 54 | 948146554F46E56B015690CC /* SampleCode.xcconfig */, 55 | ); 56 | name = Configuration; 57 | sourceTree = ""; 58 | }; 59 | BF2DD93DB9836723DA6BED14 /* LICENSE */ = { 60 | isa = PBXGroup; 61 | children = ( 62 | E7E2ED6F1473F1FD1E9543FA /* LICENSE.txt */, 63 | ); 64 | path = LICENSE; 65 | sourceTree = ""; 66 | }; 67 | E73DD12424636FEF00D77039 = { 68 | isa = PBXGroup; 69 | children = ( 70 | B57D60B9736905755DECF7CE /* README.md */, 71 | E73DD12F24636FEF00D77039 /* SceneDepthPointCloud */, 72 | E73DD12E24636FEF00D77039 /* Products */, 73 | A6415A999BE1C768F18B2649 /* Configuration */, 74 | BF2DD93DB9836723DA6BED14 /* LICENSE */, 75 | ); 76 | sourceTree = ""; 77 | }; 78 | E73DD12E24636FEF00D77039 /* Products */ = { 79 | isa = PBXGroup; 80 | children = ( 81 | E73DD12D24636FEF00D77039 /* Point Cloud.app */, 82 | ); 83 | name = Products; 84 | sourceTree = ""; 85 | }; 86 | E73DD12F24636FEF00D77039 /* SceneDepthPointCloud */ = { 87 | isa = PBXGroup; 88 | children = ( 89 | E73DD13024636FEF00D77039 /* AppDelegate.swift */, 90 | E73DD13724636FEF00D77039 /* ViewController.swift */, 91 | E73DD13224636FEF00D77039 /* Shaders.metal */, 92 | E73DD13424636FEF00D77039 /* ShaderTypes.h */, 93 | E7B7223A24807ACA002E43E8 /* MetalBuffer.swift */, 94 | E73DD13524636FEF00D77039 /* Renderer.swift */, 95 | E7B7223C24809D1E002E43E8 /* Helpers.swift */, 96 | E73DD13924636FEF00D77039 /* Main.storyboard */, 97 | E73DD13C24636FF100D77039 /* Assets.xcassets */, 98 | E73DD13E24636FF100D77039 /* LaunchScreen.storyboard */, 99 | E73DD14124636FF100D77039 /* Info.plist */, 100 | E73DD14224636FF100D77039 /* SceneDepthPointCloud-Bridging-Header.h */, 101 | ); 102 | path = SceneDepthPointCloud; 103 | sourceTree = ""; 104 | }; 105 | /* End PBXGroup section */ 106 | 107 | /* Begin PBXNativeTarget section */ 108 | E73DD12C24636FEF00D77039 /* SceneDepthPointCloud */ = { 109 | isa = PBXNativeTarget; 110 | buildConfigurationList = E73DD14524636FF100D77039 /* Build configuration list for PBXNativeTarget "SceneDepthPointCloud" */; 111 | buildPhases = ( 112 | E73DD12924636FEF00D77039 /* Sources */, 113 | E73DD12A24636FEF00D77039 /* Frameworks */, 114 | E73DD12B24636FEF00D77039 /* Resources */, 115 | ); 116 | buildRules = ( 117 | ); 118 | dependencies = ( 119 | ); 120 | name = SceneDepthPointCloud; 121 | productName = SceneDepthDemo; 122 | productReference = E73DD12D24636FEF00D77039 /* Point Cloud.app */; 123 | productType = "com.apple.product-type.application"; 124 | }; 125 | /* End PBXNativeTarget section */ 126 | 127 | /* Begin PBXProject section */ 128 | E73DD12524636FEF00D77039 /* Project object */ = { 129 | isa = PBXProject; 130 | attributes = { 131 | LastSwiftUpdateCheck = 1140; 132 | LastUpgradeCheck = 1200; 133 | ORGANIZATIONNAME = Apple; 134 | TargetAttributes = { 135 | E73DD12C24636FEF00D77039 = { 136 | CreatedOnToolsVersion = 11.4; 137 | }; 138 | }; 139 | }; 140 | buildConfigurationList = E73DD12824636FEF00D77039 /* Build configuration list for PBXProject "SceneDepthPointCloud" */; 141 | compatibilityVersion = "Xcode 9.3"; 142 | developmentRegion = en; 143 | hasScannedForEncodings = 0; 144 | knownRegions = ( 145 | en, 146 | Base, 147 | ); 148 | mainGroup = E73DD12424636FEF00D77039; 149 | productRefGroup = E73DD12E24636FEF00D77039 /* Products */; 150 | projectDirPath = ""; 151 | projectRoot = ""; 152 | targets = ( 153 | E73DD12C24636FEF00D77039 /* SceneDepthPointCloud */, 154 | ); 155 | }; 156 | /* End PBXProject section */ 157 | 158 | /* Begin PBXResourcesBuildPhase section */ 159 | E73DD12B24636FEF00D77039 /* Resources */ = { 160 | isa = PBXResourcesBuildPhase; 161 | buildActionMask = 2147483647; 162 | files = ( 163 | E73DD14024636FF100D77039 /* LaunchScreen.storyboard in Resources */, 164 | E73DD13D24636FF100D77039 /* Assets.xcassets in Resources */, 165 | E73DD13B24636FEF00D77039 /* Main.storyboard in Resources */, 166 | ); 167 | runOnlyForDeploymentPostprocessing = 0; 168 | }; 169 | /* End PBXResourcesBuildPhase section */ 170 | 171 | /* Begin PBXSourcesBuildPhase section */ 172 | E73DD12924636FEF00D77039 /* Sources */ = { 173 | isa = PBXSourcesBuildPhase; 174 | buildActionMask = 2147483647; 175 | files = ( 176 | E73DD13324636FEF00D77039 /* Shaders.metal in Sources */, 177 | E7B7223D24809D1E002E43E8 /* Helpers.swift in Sources */, 178 | E7B7223B24807ACA002E43E8 /* MetalBuffer.swift in Sources */, 179 | E73DD13624636FEF00D77039 /* Renderer.swift in Sources */, 180 | E73DD13824636FEF00D77039 /* ViewController.swift in Sources */, 181 | E73DD13124636FEF00D77039 /* AppDelegate.swift in Sources */, 182 | ); 183 | runOnlyForDeploymentPostprocessing = 0; 184 | }; 185 | /* End PBXSourcesBuildPhase section */ 186 | 187 | /* Begin PBXVariantGroup section */ 188 | E73DD13924636FEF00D77039 /* Main.storyboard */ = { 189 | isa = PBXVariantGroup; 190 | children = ( 191 | E73DD13A24636FEF00D77039 /* Base */, 192 | ); 193 | name = Main.storyboard; 194 | sourceTree = ""; 195 | }; 196 | E73DD13E24636FF100D77039 /* LaunchScreen.storyboard */ = { 197 | isa = PBXVariantGroup; 198 | children = ( 199 | E73DD13F24636FF100D77039 /* Base */, 200 | ); 201 | name = LaunchScreen.storyboard; 202 | sourceTree = ""; 203 | }; 204 | /* End PBXVariantGroup section */ 205 | 206 | /* Begin XCBuildConfiguration section */ 207 | E73DD14324636FF100D77039 /* Debug */ = { 208 | isa = XCBuildConfiguration; 209 | baseConfigurationReference = 948146554F46E56B015690CC /* SampleCode.xcconfig */; 210 | buildSettings = { 211 | ALWAYS_SEARCH_USER_PATHS = NO; 212 | CLANG_ANALYZER_NONNULL = YES; 213 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 214 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 215 | CLANG_CXX_LIBRARY = "libc++"; 216 | CLANG_ENABLE_MODULES = YES; 217 | CLANG_ENABLE_OBJC_ARC = YES; 218 | CLANG_ENABLE_OBJC_WEAK = YES; 219 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 220 | CLANG_WARN_BOOL_CONVERSION = YES; 221 | CLANG_WARN_COMMA = YES; 222 | CLANG_WARN_CONSTANT_CONVERSION = YES; 223 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 224 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 225 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 226 | CLANG_WARN_EMPTY_BODY = YES; 227 | CLANG_WARN_ENUM_CONVERSION = YES; 228 | CLANG_WARN_INFINITE_RECURSION = YES; 229 | CLANG_WARN_INT_CONVERSION = YES; 230 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 231 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 232 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 233 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 234 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 235 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 236 | CLANG_WARN_STRICT_PROTOTYPES = YES; 237 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 238 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 239 | CLANG_WARN_UNREACHABLE_CODE = YES; 240 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 241 | CODE_SIGN_STYLE = Automatic; 242 | COPY_PHASE_STRIP = NO; 243 | DEBUG_INFORMATION_FORMAT = dwarf; 244 | ENABLE_STRICT_OBJC_MSGSEND = YES; 245 | ENABLE_TESTABILITY = YES; 246 | GCC_C_LANGUAGE_STANDARD = gnu11; 247 | GCC_DYNAMIC_NO_PIC = NO; 248 | GCC_NO_COMMON_BLOCKS = YES; 249 | GCC_OPTIMIZATION_LEVEL = 0; 250 | GCC_PREPROCESSOR_DEFINITIONS = ( 251 | "DEBUG=1", 252 | "$(inherited)", 253 | ); 254 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 255 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 256 | GCC_WARN_UNDECLARED_SELECTOR = YES; 257 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 258 | GCC_WARN_UNUSED_FUNCTION = YES; 259 | GCC_WARN_UNUSED_VARIABLE = YES; 260 | IPHONEOS_DEPLOYMENT_TARGET = 14.0; 261 | MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; 262 | MTL_FAST_MATH = YES; 263 | ONLY_ACTIVE_ARCH = YES; 264 | SDKROOT = iphoneos.internal; 265 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; 266 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 267 | }; 268 | name = Debug; 269 | }; 270 | E73DD14424636FF100D77039 /* Release */ = { 271 | isa = XCBuildConfiguration; 272 | baseConfigurationReference = 948146554F46E56B015690CC /* SampleCode.xcconfig */; 273 | buildSettings = { 274 | ALWAYS_SEARCH_USER_PATHS = NO; 275 | CLANG_ANALYZER_NONNULL = YES; 276 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 277 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 278 | CLANG_CXX_LIBRARY = "libc++"; 279 | CLANG_ENABLE_MODULES = YES; 280 | CLANG_ENABLE_OBJC_ARC = YES; 281 | CLANG_ENABLE_OBJC_WEAK = YES; 282 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 283 | CLANG_WARN_BOOL_CONVERSION = YES; 284 | CLANG_WARN_COMMA = YES; 285 | CLANG_WARN_CONSTANT_CONVERSION = YES; 286 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 287 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 288 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 289 | CLANG_WARN_EMPTY_BODY = YES; 290 | CLANG_WARN_ENUM_CONVERSION = YES; 291 | CLANG_WARN_INFINITE_RECURSION = YES; 292 | CLANG_WARN_INT_CONVERSION = YES; 293 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 294 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 295 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 296 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 297 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 298 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 299 | CLANG_WARN_STRICT_PROTOTYPES = YES; 300 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 301 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 302 | CLANG_WARN_UNREACHABLE_CODE = YES; 303 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 304 | CODE_SIGN_STYLE = Automatic; 305 | COPY_PHASE_STRIP = NO; 306 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 307 | ENABLE_NS_ASSERTIONS = NO; 308 | ENABLE_STRICT_OBJC_MSGSEND = YES; 309 | GCC_C_LANGUAGE_STANDARD = gnu11; 310 | GCC_NO_COMMON_BLOCKS = YES; 311 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 312 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 313 | GCC_WARN_UNDECLARED_SELECTOR = YES; 314 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 315 | GCC_WARN_UNUSED_FUNCTION = YES; 316 | GCC_WARN_UNUSED_VARIABLE = YES; 317 | IPHONEOS_DEPLOYMENT_TARGET = 14.0; 318 | MTL_ENABLE_DEBUG_INFO = NO; 319 | MTL_FAST_MATH = YES; 320 | SDKROOT = iphoneos.internal; 321 | SWIFT_COMPILATION_MODE = wholemodule; 322 | SWIFT_OPTIMIZATION_LEVEL = "-O"; 323 | VALIDATE_PRODUCT = YES; 324 | }; 325 | name = Release; 326 | }; 327 | E73DD14624636FF100D77039 /* Debug */ = { 328 | isa = XCBuildConfiguration; 329 | baseConfigurationReference = 948146554F46E56B015690CC /* SampleCode.xcconfig */; 330 | buildSettings = { 331 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 332 | CODE_SIGN_ENTITLEMENTS = ""; 333 | CODE_SIGN_IDENTITY = "iPhone Developer"; 334 | CODE_SIGN_STYLE = Automatic; 335 | CURRENT_PROJECT_VERSION = 1; 336 | DEVELOPMENT_TEAM = T7TF583JL4; 337 | INFOPLIST_FILE = SceneDepthPointCloud/Info.plist; 338 | LD_RUNPATH_SEARCH_PATHS = ( 339 | "$(inherited)", 340 | "@executable_path/Frameworks", 341 | ); 342 | MARKETING_VERSION = 1.0; 343 | PRODUCT_BUNDLE_IDENTIFIER = "com.example.apple-samplecode.SceneDepthPointCloud${SAMPLE_CODE_DISAMBIGUATOR}"; 344 | PRODUCT_NAME = "Point Cloud"; 345 | PROVISIONING_PROFILE_SPECIFIER = ""; 346 | SDKROOT = iphoneos; 347 | SWIFT_OBJC_BRIDGING_HEADER = "SceneDepthPointCloud/SceneDepthPointCloud-Bridging-Header.h"; 348 | SWIFT_VERSION = 5.0; 349 | TARGETED_DEVICE_FAMILY = "1,2"; 350 | }; 351 | name = Debug; 352 | }; 353 | E73DD14724636FF100D77039 /* Release */ = { 354 | isa = XCBuildConfiguration; 355 | baseConfigurationReference = 948146554F46E56B015690CC /* SampleCode.xcconfig */; 356 | buildSettings = { 357 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 358 | CODE_SIGN_ENTITLEMENTS = ""; 359 | CODE_SIGN_IDENTITY = "iPhone Developer"; 360 | CODE_SIGN_STYLE = Automatic; 361 | CURRENT_PROJECT_VERSION = 1; 362 | DEVELOPMENT_TEAM = T7TF583JL4; 363 | INFOPLIST_FILE = SceneDepthPointCloud/Info.plist; 364 | LD_RUNPATH_SEARCH_PATHS = ( 365 | "$(inherited)", 366 | "@executable_path/Frameworks", 367 | ); 368 | MARKETING_VERSION = 1.0; 369 | PRODUCT_BUNDLE_IDENTIFIER = "com.example.apple-samplecode.SceneDepthPointCloud${SAMPLE_CODE_DISAMBIGUATOR}"; 370 | PRODUCT_NAME = "Point Cloud"; 371 | PROVISIONING_PROFILE_SPECIFIER = ""; 372 | SDKROOT = iphoneos; 373 | SWIFT_OBJC_BRIDGING_HEADER = "SceneDepthPointCloud/SceneDepthPointCloud-Bridging-Header.h"; 374 | SWIFT_VERSION = 5.0; 375 | TARGETED_DEVICE_FAMILY = "1,2"; 376 | }; 377 | name = Release; 378 | }; 379 | /* End XCBuildConfiguration section */ 380 | 381 | /* Begin XCConfigurationList section */ 382 | E73DD12824636FEF00D77039 /* Build configuration list for PBXProject "SceneDepthPointCloud" */ = { 383 | isa = XCConfigurationList; 384 | buildConfigurations = ( 385 | E73DD14324636FF100D77039 /* Debug */, 386 | E73DD14424636FF100D77039 /* Release */, 387 | ); 388 | defaultConfigurationIsVisible = 0; 389 | defaultConfigurationName = Release; 390 | }; 391 | E73DD14524636FF100D77039 /* Build configuration list for PBXNativeTarget "SceneDepthPointCloud" */ = { 392 | isa = XCConfigurationList; 393 | buildConfigurations = ( 394 | E73DD14624636FF100D77039 /* Debug */, 395 | E73DD14724636FF100D77039 /* Release */, 396 | ); 397 | defaultConfigurationIsVisible = 0; 398 | defaultConfigurationName = Release; 399 | }; 400 | /* End XCConfigurationList section */ 401 | }; 402 | rootObject = E73DD12524636FEF00D77039 /* Project object */; 403 | } 404 | -------------------------------------------------------------------------------- /SceneDepthPointCloud.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | BuildSystemType 6 | Latest 7 | 8 | 9 | -------------------------------------------------------------------------------- /SceneDepthPointCloud.xcodeproj/xcshareddata/xcschemes/SceneDepthPointCloud.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 24 | 25 | 30 | 31 | 32 | 33 | 43 | 45 | 51 | 52 | 53 | 54 | 60 | 62 | 68 | 69 | 70 | 71 | 73 | 74 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Contains the application's delegate. 6 | */ 7 | 8 | import UIKit 9 | import ARKit 10 | 11 | @UIApplicationMain 12 | class AppDelegate: UIResponder, UIApplicationDelegate { 13 | 14 | var window: UIWindow? 15 | 16 | func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { 17 | if !ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) { 18 | // Ensure that the device supports scene depth and present 19 | // an error-message view controller, if not. 20 | let storyboard = UIStoryboard(name: "Main", bundle: nil) 21 | window?.rootViewController = storyboard.instantiateViewController(withIdentifier: "unsupportedDeviceMessage") 22 | } 23 | return true 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-1.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-10.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-11.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-12.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-13.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-14.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-15.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-16.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-2.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-3.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-4.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-5.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-6.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-7.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-8.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy-9.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic copy.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/isakdiaz/arkit-scenedepth-pointcloud/e3f46e6ab065a11816b40cc4c2814e59641c6196/SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/AR_icon_basic.png -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "filename" : "AR_icon_basic copy-6.png", 5 | "idiom" : "iphone", 6 | "scale" : "2x", 7 | "size" : "20x20" 8 | }, 9 | { 10 | "filename" : "AR_icon_basic copy-16.png", 11 | "idiom" : "iphone", 12 | "scale" : "3x", 13 | "size" : "20x20" 14 | }, 15 | { 16 | "filename" : "AR_icon_basic copy-9.png", 17 | "idiom" : "iphone", 18 | "scale" : "2x", 19 | "size" : "29x29" 20 | }, 21 | { 22 | "filename" : "AR_icon_basic copy-10.png", 23 | "idiom" : "iphone", 24 | "scale" : "3x", 25 | "size" : "29x29" 26 | }, 27 | { 28 | "filename" : "AR_icon_basic copy-4.png", 29 | "idiom" : "iphone", 30 | "scale" : "2x", 31 | "size" : "40x40" 32 | }, 33 | { 34 | "filename" : "AR_icon_basic copy-12.png", 35 | "idiom" : "iphone", 36 | "scale" : "3x", 37 | "size" : "40x40" 38 | }, 39 | { 40 | "filename" : "AR_icon_basic copy-13.png", 41 | "idiom" : "iphone", 42 | "scale" : "2x", 43 | "size" : "60x60" 44 | }, 45 | { 46 | "filename" : "AR_icon_basic copy-14.png", 47 | "idiom" : "iphone", 48 | "scale" : "3x", 49 | "size" : "60x60" 50 | }, 51 | { 52 | "filename" : "AR_icon_basic copy-11.png", 53 | "idiom" : "ipad", 54 | "scale" : "1x", 55 | "size" : "20x20" 56 | }, 57 | { 58 | "filename" : "AR_icon_basic copy-7.png", 59 | "idiom" : "ipad", 60 | "scale" : "2x", 61 | "size" : "20x20" 62 | }, 63 | { 64 | "filename" : "AR_icon_basic copy-15.png", 65 | "idiom" : "ipad", 66 | "scale" : "1x", 67 | "size" : "29x29" 68 | }, 69 | { 70 | "filename" : "AR_icon_basic copy-8.png", 71 | "idiom" : "ipad", 72 | "scale" : "2x", 73 | "size" : "29x29" 74 | }, 75 | { 76 | "filename" : "AR_icon_basic copy-5.png", 77 | "idiom" : "ipad", 78 | "scale" : "1x", 79 | "size" : "40x40" 80 | }, 81 | { 82 | "filename" : "AR_icon_basic copy-3.png", 83 | "idiom" : "ipad", 84 | "scale" : "2x", 85 | "size" : "40x40" 86 | }, 87 | { 88 | "filename" : "AR_icon_basic copy-2.png", 89 | "idiom" : "ipad", 90 | "scale" : "1x", 91 | "size" : "76x76" 92 | }, 93 | { 94 | "filename" : "AR_icon_basic copy-1.png", 95 | "idiom" : "ipad", 96 | "scale" : "2x", 97 | "size" : "76x76" 98 | }, 99 | { 100 | "filename" : "AR_icon_basic copy.png", 101 | "idiom" : "ipad", 102 | "scale" : "2x", 103 | "size" : "83.5x83.5" 104 | }, 105 | { 106 | "filename" : "AR_icon_basic.png", 107 | "idiom" : "ios-marketing", 108 | "scale" : "1x", 109 | "size" : "1024x1024" 110 | } 111 | ], 112 | "info" : { 113 | "author" : "xcode", 114 | "version" : 1 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Base.lproj/LaunchScreen.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Base.lproj/Main.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 38 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Helpers.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | General Helper methods and properties 6 | */ 7 | 8 | import ARKit 9 | 10 | typealias Float2 = SIMD2 11 | typealias Float3 = SIMD3 12 | 13 | extension Float { 14 | static let degreesToRadian = Float.pi / 180 15 | } 16 | 17 | extension matrix_float3x3 { 18 | mutating func copy(from affine: CGAffineTransform) { 19 | columns.0 = Float3(Float(affine.a), Float(affine.c), Float(affine.tx)) 20 | columns.1 = Float3(Float(affine.b), Float(affine.d), Float(affine.ty)) 21 | columns.2 = Float3(0, 0, 1) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | $(PRODUCT_BUNDLE_PACKAGE_TYPE) 17 | CFBundleShortVersionString 18 | $(MARKETING_VERSION) 19 | CFBundleVersion 20 | $(CURRENT_PROJECT_VERSION) 21 | LSRequiresIPhoneOS 22 | 23 | NSCameraUsageDescription 24 | The camera is used for augmenting reality. 25 | UILaunchStoryboardName 26 | LaunchScreen 27 | UIMainStoryboardFile 28 | Main 29 | UIRequiredDeviceCapabilities 30 | 31 | armv7 32 | arkit 33 | metal 34 | 35 | UIRequiresFullScreen 36 | 37 | UIStatusBarHidden 38 | 39 | UISupportedInterfaceOrientations 40 | 41 | UIInterfaceOrientationLandscapeRight 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/MetalBuffer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Type-safe utility for working with MTLBuffers. 6 | */ 7 | 8 | import MetalKit 9 | 10 | protocol Resource { 11 | associatedtype Element 12 | } 13 | 14 | /// A wrapper around MTLBuffer which provides type safe access and assignment to the underlying MTLBuffer's contents. 15 | 16 | struct MetalBuffer: Resource { 17 | 18 | /// The underlying MTLBuffer. 19 | fileprivate let buffer: MTLBuffer 20 | 21 | /// The index that the buffer should be bound to during encoding. 22 | /// Should correspond with the index that the buffer is expected to be at in Metal shaders. 23 | fileprivate let index: Int 24 | 25 | /// The number of elements of T the buffer can hold. 26 | let count: Int 27 | var stride: Int { 28 | MemoryLayout.stride 29 | } 30 | 31 | /// Initializes the buffer with zeros, the buffer is given an appropriate length based on the provided element count. 32 | init(device: MTLDevice, count: Int, index: UInt32, label: String? = nil, options: MTLResourceOptions = []) { 33 | 34 | guard let buffer = device.makeBuffer(length: MemoryLayout.stride * count, options: options) else { 35 | fatalError("Failed to create MTLBuffer.") 36 | } 37 | self.buffer = buffer 38 | self.buffer.label = label 39 | self.count = count 40 | self.index = Int(index) 41 | } 42 | 43 | /// Initializes the buffer with the contents of the provided array. 44 | init(device: MTLDevice, array: [Element], index: UInt32, options: MTLResourceOptions = []) { 45 | 46 | guard let buffer = device.makeBuffer(bytes: array, length: MemoryLayout.stride * array.count, options: .storageModeShared) else { 47 | fatalError("Failed to create MTLBuffer") 48 | } 49 | self.buffer = buffer 50 | self.count = array.count 51 | self.index = Int(index) 52 | } 53 | 54 | /// Replaces the buffer's memory at the specified element index with the provided value. 55 | func assign(_ value: T, at index: Int = 0) { 56 | precondition(index <= count - 1, "Index \(index) is greater than maximum allowable index of \(count - 1) for this buffer.") 57 | withUnsafePointer(to: value) { 58 | buffer.contents().advanced(by: index * stride).copyMemory(from: $0, byteCount: stride) 59 | } 60 | } 61 | 62 | /// Replaces the buffer's memory with the values in the array. 63 | func assign(with array: [Element]) { 64 | let byteCount = array.count * stride 65 | precondition(byteCount == buffer.length, "Mismatch between the byte count of the array's contents and the MTLBuffer length.") 66 | buffer.contents().copyMemory(from: array, byteCount: byteCount) 67 | } 68 | 69 | /// Returns a copy of the value at the specified element index in the buffer. 70 | subscript(index: Int) -> Element { 71 | get { 72 | precondition(stride * index <= buffer.length - stride, "This buffer is not large enough to have an element at the index: \(index)") 73 | return buffer.contents().advanced(by: index * stride).load(as: Element.self) 74 | } 75 | 76 | set { 77 | assign(newValue, at: index) 78 | } 79 | } 80 | 81 | } 82 | 83 | // Note: This extension is in this file because access to Buffer.buffer is fileprivate. 84 | // Access to Buffer.buffer was made fileprivate to ensure that only this file can touch the underlying MTLBuffer. 85 | extension MTLRenderCommandEncoder { 86 | func setVertexBuffer(_ vertexBuffer: MetalBuffer, offset: Int = 0) { 87 | setVertexBuffer(vertexBuffer.buffer, offset: offset, index: vertexBuffer.index) 88 | } 89 | 90 | func setFragmentBuffer(_ fragmentBuffer: MetalBuffer, offset: Int = 0) { 91 | setFragmentBuffer(fragmentBuffer.buffer, offset: offset, index: fragmentBuffer.index) 92 | } 93 | 94 | func setVertexResource(_ resource: R) { 95 | if let buffer = resource as? MetalBuffer { 96 | setVertexBuffer(buffer) 97 | } 98 | 99 | if let texture = resource as? Texture { 100 | setVertexTexture(texture.texture, index: texture.index) 101 | } 102 | } 103 | 104 | func setFragmentResource(_ resource: R) { 105 | if let buffer = resource as? MetalBuffer { 106 | setFragmentBuffer(buffer) 107 | } 108 | 109 | if let texture = resource as? Texture { 110 | setFragmentTexture(texture.texture, index: texture.index) 111 | } 112 | } 113 | } 114 | 115 | struct Texture: Resource { 116 | typealias Element = Any 117 | 118 | let texture: MTLTexture 119 | let index: Int 120 | } 121 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Renderer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | The host app renderer. 6 | */ 7 | 8 | import Metal 9 | import MetalKit 10 | import ARKit 11 | 12 | final class Renderer { 13 | // Maximum number of points we store in the point cloud 14 | private let maxPoints = 2500_000 15 | // Number of sample points on the grid 16 | private let numGridPoints = 3500 17 | // Particle's size in pixels 18 | private let particleSize: Float = 30 19 | // We only use landscape orientation in this app 20 | private let orientation = UIInterfaceOrientation.landscapeRight 21 | // Camera's threshold values for detecting when the camera moves so that we can accumulate the points 22 | private let cameraRotationThreshold = cos(2 * .degreesToRadian) 23 | private let cameraTranslationThreshold: Float = pow(0.02, 2) // (meter-squared) 24 | // The max number of command buffers in flight 25 | private let maxInFlightBuffers = 3 26 | 27 | private lazy var rotateToARCamera = Self.makeRotateToARCameraMatrix(orientation: orientation) 28 | private let session: ARSession 29 | 30 | // Metal objects and textures 31 | private let device: MTLDevice 32 | private let library: MTLLibrary 33 | private let renderDestination: RenderDestinationProvider 34 | private let relaxedStencilState: MTLDepthStencilState 35 | private let depthStencilState: MTLDepthStencilState 36 | private let commandQueue: MTLCommandQueue 37 | private lazy var unprojectPipelineState = makeUnprojectionPipelineState()! 38 | private lazy var rgbPipelineState = makeRGBPipelineState()! 39 | private lazy var particlePipelineState = makeParticlePipelineState()! 40 | // texture cache for captured image 41 | private lazy var textureCache = makeTextureCache() 42 | private var capturedImageTextureY: CVMetalTexture? 43 | private var capturedImageTextureCbCr: CVMetalTexture? 44 | private var depthTexture: CVMetalTexture? 45 | private var confidenceTexture: CVMetalTexture? 46 | 47 | // Multi-buffer rendering pipeline 48 | private let inFlightSemaphore: DispatchSemaphore 49 | private var currentBufferIndex = 0 50 | 51 | // The current viewport size 52 | private var viewportSize = CGSize() 53 | // The grid of sample points 54 | private lazy var gridPointsBuffer = MetalBuffer(device: device, 55 | array: makeGridPoints(), 56 | index: kGridPoints.rawValue, options: []) 57 | 58 | // RGB buffer 59 | private lazy var rgbUniforms: RGBUniforms = { 60 | var uniforms = RGBUniforms() 61 | uniforms.radius = rgbRadius 62 | uniforms.viewToCamera.copy(from: viewToCamera) 63 | uniforms.viewRatio = Float(viewportSize.width / viewportSize.height) 64 | return uniforms 65 | }() 66 | private var rgbUniformsBuffers = [MetalBuffer]() 67 | // Point Cloud buffer 68 | private lazy var pointCloudUniforms: PointCloudUniforms = { 69 | var uniforms = PointCloudUniforms() 70 | uniforms.maxPoints = Int32(maxPoints) 71 | uniforms.confidenceThreshold = Int32(confidenceThreshold) 72 | uniforms.particleSize = particleSize 73 | uniforms.cameraResolution = cameraResolution 74 | return uniforms 75 | }() 76 | private var pointCloudUniformsBuffers = [MetalBuffer]() 77 | // Particles buffer 78 | private var particlesBuffer: MetalBuffer 79 | private var currentPointIndex = 0 80 | private var currentPointCount = 0 81 | 82 | // Camera data 83 | private var sampleFrame: ARFrame { session.currentFrame! } 84 | private lazy var cameraResolution = Float2(Float(sampleFrame.camera.imageResolution.width), Float(sampleFrame.camera.imageResolution.height)) 85 | private lazy var viewToCamera = sampleFrame.displayTransform(for: orientation, viewportSize: viewportSize).inverted() 86 | private lazy var lastCameraTransform = sampleFrame.camera.transform 87 | 88 | // interfaces 89 | var confidenceThreshold = 1 { 90 | didSet { 91 | // apply the change for the shader 92 | pointCloudUniforms.confidenceThreshold = Int32(confidenceThreshold) 93 | } 94 | } 95 | 96 | var rgbRadius: Float = 0 { 97 | didSet { 98 | // apply the change for the shader 99 | rgbUniforms.radius = rgbRadius 100 | } 101 | } 102 | 103 | init(session: ARSession, metalDevice device: MTLDevice, renderDestination: RenderDestinationProvider) { 104 | self.session = session 105 | self.device = device 106 | self.renderDestination = renderDestination 107 | 108 | library = device.makeDefaultLibrary()! 109 | commandQueue = device.makeCommandQueue()! 110 | 111 | // initialize our buffers 112 | for _ in 0 ..< maxInFlightBuffers { 113 | rgbUniformsBuffers.append(.init(device: device, count: 1, index: 0)) 114 | pointCloudUniformsBuffers.append(.init(device: device, count: 1, index: kPointCloudUniforms.rawValue)) 115 | } 116 | particlesBuffer = .init(device: device, count: maxPoints, index: kParticleUniforms.rawValue) 117 | 118 | // rbg does not need to read/write depth 119 | let relaxedStateDescriptor = MTLDepthStencilDescriptor() 120 | relaxedStencilState = device.makeDepthStencilState(descriptor: relaxedStateDescriptor)! 121 | 122 | // setup depth test for point cloud 123 | let depthStateDescriptor = MTLDepthStencilDescriptor() 124 | depthStateDescriptor.depthCompareFunction = .lessEqual 125 | depthStateDescriptor.isDepthWriteEnabled = true 126 | depthStencilState = device.makeDepthStencilState(descriptor: depthStateDescriptor)! 127 | 128 | inFlightSemaphore = DispatchSemaphore(value: maxInFlightBuffers) 129 | } 130 | 131 | func drawRectResized(size: CGSize) { 132 | viewportSize = size 133 | } 134 | 135 | private func updateCapturedImageTextures(frame: ARFrame) { 136 | // Create two textures (Y and CbCr) from the provided frame's captured image 137 | let pixelBuffer = frame.capturedImage 138 | guard CVPixelBufferGetPlaneCount(pixelBuffer) >= 2 else { 139 | return 140 | } 141 | 142 | capturedImageTextureY = makeTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .r8Unorm, planeIndex: 0) 143 | capturedImageTextureCbCr = makeTexture(fromPixelBuffer: pixelBuffer, pixelFormat: .rg8Unorm, planeIndex: 1) 144 | } 145 | 146 | private func updateDepthTextures(frame: ARFrame) -> Bool { 147 | guard let depthMap = frame.sceneDepth?.depthMap, 148 | let confidenceMap = frame.sceneDepth?.confidenceMap else { 149 | return false 150 | } 151 | 152 | depthTexture = makeTexture(fromPixelBuffer: depthMap, pixelFormat: .r32Float, planeIndex: 0) 153 | confidenceTexture = makeTexture(fromPixelBuffer: confidenceMap, pixelFormat: .r8Uint, planeIndex: 0) 154 | 155 | return true 156 | } 157 | 158 | private func update(frame: ARFrame) { 159 | // frame dependent info 160 | let camera = frame.camera 161 | let cameraIntrinsicsInversed = camera.intrinsics.inverse 162 | let viewMatrix = camera.viewMatrix(for: orientation) 163 | let viewMatrixInversed = viewMatrix.inverse 164 | let projectionMatrix = camera.projectionMatrix(for: orientation, viewportSize: viewportSize, zNear: 0.001, zFar: 0) 165 | pointCloudUniforms.viewProjectionMatrix = projectionMatrix * viewMatrix 166 | pointCloudUniforms.localToWorld = viewMatrixInversed * rotateToARCamera 167 | pointCloudUniforms.cameraIntrinsicsInversed = cameraIntrinsicsInversed 168 | } 169 | 170 | func draw() { 171 | guard let currentFrame = session.currentFrame, 172 | let renderDescriptor = renderDestination.currentRenderPassDescriptor, 173 | let commandBuffer = commandQueue.makeCommandBuffer(), 174 | let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderDescriptor) else { 175 | return 176 | } 177 | 178 | _ = inFlightSemaphore.wait(timeout: DispatchTime.distantFuture) 179 | commandBuffer.addCompletedHandler { [weak self] commandBuffer in 180 | if let self = self { 181 | self.inFlightSemaphore.signal() 182 | } 183 | } 184 | 185 | // update frame data 186 | update(frame: currentFrame) 187 | updateCapturedImageTextures(frame: currentFrame) 188 | 189 | // handle buffer rotating 190 | currentBufferIndex = (currentBufferIndex + 1) % maxInFlightBuffers 191 | pointCloudUniformsBuffers[currentBufferIndex][0] = pointCloudUniforms 192 | 193 | if shouldAccumulate(frame: currentFrame), updateDepthTextures(frame: currentFrame) { 194 | accumulatePoints(frame: currentFrame, commandBuffer: commandBuffer, renderEncoder: renderEncoder) 195 | } 196 | 197 | // check and render rgb camera image 198 | if rgbUniforms.radius > 0 { 199 | var retainingTextures = [capturedImageTextureY, capturedImageTextureCbCr] 200 | commandBuffer.addCompletedHandler { buffer in 201 | retainingTextures.removeAll() 202 | } 203 | rgbUniformsBuffers[currentBufferIndex][0] = rgbUniforms 204 | 205 | renderEncoder.setDepthStencilState(relaxedStencilState) 206 | renderEncoder.setRenderPipelineState(rgbPipelineState) 207 | renderEncoder.setVertexBuffer(rgbUniformsBuffers[currentBufferIndex]) 208 | renderEncoder.setFragmentBuffer(rgbUniformsBuffers[currentBufferIndex]) 209 | renderEncoder.setFragmentTexture(CVMetalTextureGetTexture(capturedImageTextureY!), index: Int(kTextureY.rawValue)) 210 | renderEncoder.setFragmentTexture(CVMetalTextureGetTexture(capturedImageTextureCbCr!), index: Int(kTextureCbCr.rawValue)) 211 | renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) 212 | } 213 | 214 | // render particles 215 | renderEncoder.setDepthStencilState(depthStencilState) 216 | renderEncoder.setRenderPipelineState(particlePipelineState) 217 | renderEncoder.setVertexBuffer(pointCloudUniformsBuffers[currentBufferIndex]) 218 | renderEncoder.setVertexBuffer(particlesBuffer) 219 | renderEncoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: currentPointCount) 220 | renderEncoder.endEncoding() 221 | 222 | commandBuffer.present(renderDestination.currentDrawable!) 223 | commandBuffer.commit() 224 | } 225 | 226 | private func shouldAccumulate(frame: ARFrame) -> Bool { 227 | let cameraTransform = frame.camera.transform 228 | return currentPointCount == 0 229 | || dot(cameraTransform.columns.2, lastCameraTransform.columns.2) <= cameraRotationThreshold 230 | || distance_squared(cameraTransform.columns.3, lastCameraTransform.columns.3) >= cameraTranslationThreshold 231 | } 232 | 233 | private func accumulatePoints(frame: ARFrame, commandBuffer: MTLCommandBuffer, renderEncoder: MTLRenderCommandEncoder) { 234 | pointCloudUniforms.pointCloudCurrentIndex = Int32(currentPointIndex) 235 | 236 | var retainingTextures = [capturedImageTextureY, capturedImageTextureCbCr, depthTexture, confidenceTexture] 237 | commandBuffer.addCompletedHandler { buffer in 238 | retainingTextures.removeAll() 239 | } 240 | 241 | renderEncoder.setDepthStencilState(relaxedStencilState) 242 | renderEncoder.setRenderPipelineState(unprojectPipelineState) 243 | renderEncoder.setVertexBuffer(pointCloudUniformsBuffers[currentBufferIndex]) 244 | renderEncoder.setVertexBuffer(particlesBuffer) 245 | renderEncoder.setVertexBuffer(gridPointsBuffer) 246 | renderEncoder.setVertexTexture(CVMetalTextureGetTexture(capturedImageTextureY!), index: Int(kTextureY.rawValue)) 247 | renderEncoder.setVertexTexture(CVMetalTextureGetTexture(capturedImageTextureCbCr!), index: Int(kTextureCbCr.rawValue)) 248 | renderEncoder.setVertexTexture(CVMetalTextureGetTexture(depthTexture!), index: Int(kTextureDepth.rawValue)) 249 | renderEncoder.setVertexTexture(CVMetalTextureGetTexture(confidenceTexture!), index: Int(kTextureConfidence.rawValue)) 250 | renderEncoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: gridPointsBuffer.count) 251 | 252 | currentPointIndex = (currentPointIndex + gridPointsBuffer.count) % maxPoints 253 | currentPointCount = min(currentPointCount + gridPointsBuffer.count, maxPoints) 254 | lastCameraTransform = frame.camera.transform 255 | } 256 | } 257 | 258 | // MARK: - Metal Helpers 259 | 260 | private extension Renderer { 261 | func makeUnprojectionPipelineState() -> MTLRenderPipelineState? { 262 | guard let vertexFunction = library.makeFunction(name: "unprojectVertex") else { 263 | return nil 264 | } 265 | 266 | let descriptor = MTLRenderPipelineDescriptor() 267 | descriptor.vertexFunction = vertexFunction 268 | descriptor.isRasterizationEnabled = false 269 | descriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat 270 | descriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat 271 | 272 | return try? device.makeRenderPipelineState(descriptor: descriptor) 273 | } 274 | 275 | func makeRGBPipelineState() -> MTLRenderPipelineState? { 276 | guard let vertexFunction = library.makeFunction(name: "rgbVertex"), 277 | let fragmentFunction = library.makeFunction(name: "rgbFragment") else { 278 | return nil 279 | } 280 | 281 | let descriptor = MTLRenderPipelineDescriptor() 282 | descriptor.vertexFunction = vertexFunction 283 | descriptor.fragmentFunction = fragmentFunction 284 | descriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat 285 | descriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat 286 | 287 | return try? device.makeRenderPipelineState(descriptor: descriptor) 288 | } 289 | 290 | func makeParticlePipelineState() -> MTLRenderPipelineState? { 291 | guard let vertexFunction = library.makeFunction(name: "particleVertex"), 292 | let fragmentFunction = library.makeFunction(name: "particleFragment") else { 293 | return nil 294 | } 295 | 296 | let descriptor = MTLRenderPipelineDescriptor() 297 | descriptor.vertexFunction = vertexFunction 298 | descriptor.fragmentFunction = fragmentFunction 299 | descriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat 300 | descriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat 301 | descriptor.colorAttachments[0].isBlendingEnabled = true 302 | descriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha 303 | descriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha 304 | descriptor.colorAttachments[0].destinationAlphaBlendFactor = .oneMinusSourceAlpha 305 | 306 | return try? device.makeRenderPipelineState(descriptor: descriptor) 307 | } 308 | 309 | /// Makes sample points on camera image, also precompute the anchor point for animation 310 | func makeGridPoints() -> [Float2] { 311 | let gridArea = cameraResolution.x * cameraResolution.y 312 | let spacing = sqrt(gridArea / Float(numGridPoints)) 313 | let deltaX = Int(round(cameraResolution.x / spacing)) 314 | let deltaY = Int(round(cameraResolution.y / spacing)) 315 | 316 | var points = [Float2]() 317 | for gridY in 0 ..< deltaY { 318 | let alternatingOffsetX = Float(gridY % 2) * spacing / 2 319 | for gridX in 0 ..< deltaX { 320 | let cameraPoint = Float2(alternatingOffsetX + (Float(gridX) + 0.5) * spacing, (Float(gridY) + 0.5) * spacing) 321 | 322 | points.append(cameraPoint) 323 | } 324 | } 325 | 326 | return points 327 | } 328 | 329 | func makeTextureCache() -> CVMetalTextureCache { 330 | // Create captured image texture cache 331 | var cache: CVMetalTextureCache! 332 | CVMetalTextureCacheCreate(nil, nil, device, nil, &cache) 333 | 334 | return cache 335 | } 336 | 337 | func makeTexture(fromPixelBuffer pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, planeIndex: Int) -> CVMetalTexture? { 338 | let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex) 339 | let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex) 340 | 341 | var texture: CVMetalTexture? = nil 342 | let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, pixelFormat, width, height, planeIndex, &texture) 343 | 344 | if status != kCVReturnSuccess { 345 | texture = nil 346 | } 347 | 348 | return texture 349 | } 350 | 351 | static func cameraToDisplayRotation(orientation: UIInterfaceOrientation) -> Int { 352 | switch orientation { 353 | case .landscapeLeft: 354 | return 180 355 | case .portrait: 356 | return 90 357 | case .portraitUpsideDown: 358 | return -90 359 | default: 360 | return 0 361 | } 362 | } 363 | 364 | static func makeRotateToARCameraMatrix(orientation: UIInterfaceOrientation) -> matrix_float4x4 { 365 | // flip to ARKit Camera's coordinate 366 | let flipYZ = matrix_float4x4( 367 | [1, 0, 0, 0], 368 | [0, -1, 0, 0], 369 | [0, 0, -1, 0], 370 | [0, 0, 0, 1] ) 371 | 372 | let rotationAngle = Float(cameraToDisplayRotation(orientation: orientation)) * .degreesToRadian 373 | return flipYZ * matrix_float4x4(simd_quaternion(rotationAngle, Float3(0, 0, 1))) 374 | } 375 | } 376 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/SceneDepthPointCloud-Bridging-Header.h: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Bridging header for shared shader types. 6 | */ 7 | 8 | #import "ShaderTypes.h" 9 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/ShaderTypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Types and enums that are shared between shaders and the host app code. 6 | */ 7 | 8 | #ifndef ShaderTypes_h 9 | #define ShaderTypes_h 10 | 11 | #include 12 | 13 | enum TextureIndices { 14 | kTextureY = 0, 15 | kTextureCbCr = 1, 16 | kTextureDepth = 2, 17 | kTextureConfidence = 3 18 | }; 19 | 20 | enum BufferIndices { 21 | kPointCloudUniforms = 0, 22 | kParticleUniforms = 1, 23 | kGridPoints = 2, 24 | }; 25 | 26 | struct RGBUniforms { 27 | matrix_float3x3 viewToCamera; 28 | float viewRatio; 29 | float radius; 30 | }; 31 | 32 | struct PointCloudUniforms { 33 | matrix_float4x4 viewProjectionMatrix; 34 | matrix_float4x4 localToWorld; 35 | matrix_float3x3 cameraIntrinsicsInversed; 36 | simd_float2 cameraResolution; 37 | 38 | float particleSize; 39 | int maxPoints; 40 | int pointCloudCurrentIndex; 41 | int confidenceThreshold; 42 | }; 43 | 44 | struct ParticleUniforms { 45 | simd_float3 position; 46 | simd_float3 color; 47 | float confidence; 48 | }; 49 | 50 | #endif /* ShaderTypes_h */ 51 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/Shaders.metal: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | The sample app's shaders. 6 | */ 7 | 8 | #include 9 | #include 10 | #import "ShaderTypes.h" 11 | 12 | using namespace metal; 13 | 14 | // Camera's RGB vertex shader outputs 15 | struct RGBVertexOut { 16 | float4 position [[position]]; 17 | float2 texCoord; 18 | }; 19 | 20 | // Particle vertex shader outputs and fragment shader inputs 21 | struct ParticleVertexOut { 22 | float4 position [[position]]; 23 | float pointSize [[point_size]]; 24 | float4 color; 25 | }; 26 | 27 | constexpr sampler colorSampler(mip_filter::linear, mag_filter::linear, min_filter::linear); 28 | constant auto yCbCrToRGB = float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f), 29 | float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f), 30 | float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f), 31 | float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f)); 32 | constant float2 viewVertices[] = { float2(-1, 1), float2(-1, -1), float2(1, 1), float2(1, -1) }; 33 | constant float2 viewTexCoords[] = { float2(0, 0), float2(0, 1), float2(1, 0), float2(1, 1) }; 34 | 35 | /// Retrieves the world position of a specified camera point with depth 36 | static simd_float4 worldPoint(simd_float2 cameraPoint, float depth, matrix_float3x3 cameraIntrinsicsInversed, matrix_float4x4 localToWorld) { 37 | const auto localPoint = cameraIntrinsicsInversed * simd_float3(cameraPoint, 1) * depth; 38 | const auto worldPoint = localToWorld * simd_float4(localPoint, 1); 39 | 40 | return worldPoint / worldPoint.w; 41 | } 42 | 43 | /// Vertex shader that takes in a 2D grid-point and infers its 3D position in world-space, along with RGB and confidence 44 | vertex void unprojectVertex(uint vertexID [[vertex_id]], 45 | constant PointCloudUniforms &uniforms [[buffer(kPointCloudUniforms)]], 46 | device ParticleUniforms *particleUniforms [[buffer(kParticleUniforms)]], 47 | constant float2 *gridPoints [[buffer(kGridPoints)]], 48 | texture2d capturedImageTextureY [[texture(kTextureY)]], 49 | texture2d capturedImageTextureCbCr [[texture(kTextureCbCr)]], 50 | texture2d depthTexture [[texture(kTextureDepth)]], 51 | texture2d confidenceTexture [[texture(kTextureConfidence)]]) { 52 | 53 | const auto gridPoint = gridPoints[vertexID]; 54 | const auto currentPointIndex = (uniforms.pointCloudCurrentIndex + vertexID) % uniforms.maxPoints; 55 | const auto texCoord = gridPoint / uniforms.cameraResolution; 56 | // Sample the depth map to get the depth value 57 | const auto depth = depthTexture.sample(colorSampler, texCoord).r; 58 | // With a 2D point plus depth, we can now get its 3D position 59 | const auto position = worldPoint(gridPoint, depth, uniforms.cameraIntrinsicsInversed, uniforms.localToWorld); 60 | 61 | // Sample Y and CbCr textures to get the YCbCr color at the given texture coordinate 62 | const auto ycbcr = float4(capturedImageTextureY.sample(colorSampler, texCoord).r, capturedImageTextureCbCr.sample(colorSampler, texCoord.xy).rg, 1); 63 | const auto sampledColor = (yCbCrToRGB * ycbcr).rgb; 64 | // Sample the confidence map to get the confidence value 65 | const auto confidence = confidenceTexture.sample(colorSampler, texCoord).r; 66 | 67 | // Write the data to the buffer 68 | particleUniforms[currentPointIndex].position = position.xyz; 69 | particleUniforms[currentPointIndex].color = sampledColor; 70 | particleUniforms[currentPointIndex].confidence = confidence; 71 | } 72 | 73 | vertex RGBVertexOut rgbVertex(uint vertexID [[vertex_id]], 74 | constant RGBUniforms &uniforms [[buffer(0)]]) { 75 | const float3 texCoord = float3(viewTexCoords[vertexID], 1) * uniforms.viewToCamera; 76 | 77 | RGBVertexOut out; 78 | out.position = float4(viewVertices[vertexID], 0, 1); 79 | out.texCoord = texCoord.xy; 80 | 81 | return out; 82 | } 83 | 84 | fragment float4 rgbFragment(RGBVertexOut in [[stage_in]], 85 | constant RGBUniforms &uniforms [[buffer(0)]], 86 | texture2d capturedImageTextureY [[texture(kTextureY)]], 87 | texture2d capturedImageTextureCbCr [[texture(kTextureCbCr)]]) { 88 | 89 | const float2 offset = (in.texCoord - 0.5) * float2(1, 1 / uniforms.viewRatio) * 2; 90 | const float visibility = saturate(uniforms.radius * uniforms.radius - length_squared(offset)); 91 | const float4 ycbcr = float4(capturedImageTextureY.sample(colorSampler, in.texCoord.xy).r, capturedImageTextureCbCr.sample(colorSampler, in.texCoord.xy).rg, 1); 92 | 93 | // convert and save the color back to the buffer 94 | const float3 sampledColor = (yCbCrToRGB * ycbcr).rgb; 95 | return float4(sampledColor, 1) * visibility; 96 | } 97 | 98 | vertex ParticleVertexOut particleVertex(uint vertexID [[vertex_id]], 99 | constant PointCloudUniforms &uniforms [[buffer(kPointCloudUniforms)]], 100 | constant ParticleUniforms *particleUniforms [[buffer(kParticleUniforms)]]) { 101 | 102 | // get point data 103 | const auto particleData = particleUniforms[vertexID]; 104 | const auto position = particleData.position; 105 | const auto confidence = particleData.confidence; 106 | const auto sampledColor = particleData.color; 107 | const auto visibility = confidence >= uniforms.confidenceThreshold; 108 | 109 | // animate and project the point 110 | float4 projectedPosition = uniforms.viewProjectionMatrix * float4(position, 1.0); 111 | const float pointSize = max(uniforms.particleSize / max(1.0, projectedPosition.z), 2.0); 112 | projectedPosition /= projectedPosition.w; 113 | 114 | // prepare for output 115 | ParticleVertexOut out; 116 | out.position = projectedPosition; 117 | out.pointSize = pointSize; 118 | out.color = float4(sampledColor, visibility); 119 | 120 | return out; 121 | } 122 | 123 | fragment float4 particleFragment(ParticleVertexOut in [[stage_in]], 124 | const float2 coords [[point_coord]]) { 125 | // we draw within a circle 126 | const float distSquared = length_squared(coords - float2(0.5)); 127 | if (in.color.a == 0 || distSquared > 0.25) { 128 | discard_fragment(); 129 | } 130 | 131 | return in.color; 132 | } 133 | -------------------------------------------------------------------------------- /SceneDepthPointCloud/ViewController.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Main view controller for the AR experience. 6 | */ 7 | 8 | import UIKit 9 | import Metal 10 | import MetalKit 11 | import ARKit 12 | 13 | final class ViewController: UIViewController, ARSessionDelegate { 14 | private let isUIEnabled = true 15 | private let confidenceControl = UISegmentedControl(items: ["Low", "Medium", "High"]) 16 | private let rgbRadiusSlider = UISlider() 17 | 18 | private let session = ARSession() 19 | private var renderer: Renderer! 20 | 21 | override func viewDidLoad() { 22 | super.viewDidLoad() 23 | 24 | guard let device = MTLCreateSystemDefaultDevice() else { 25 | print("Metal is not supported on this device") 26 | return 27 | } 28 | 29 | session.delegate = self 30 | 31 | // Set the view to use the default device 32 | if let view = view as? MTKView { 33 | view.device = device 34 | 35 | view.backgroundColor = UIColor.clear 36 | // we need this to enable depth test 37 | view.depthStencilPixelFormat = .depth32Float 38 | view.contentScaleFactor = 1 39 | view.delegate = self 40 | 41 | // Configure the renderer to draw to the view 42 | renderer = Renderer(session: session, metalDevice: device, renderDestination: view) 43 | renderer.drawRectResized(size: view.bounds.size) 44 | } 45 | 46 | // Confidence control 47 | confidenceControl.backgroundColor = .white 48 | confidenceControl.selectedSegmentIndex = renderer.confidenceThreshold 49 | confidenceControl.addTarget(self, action: #selector(viewValueChanged), for: .valueChanged) 50 | 51 | // RGB Radius control 52 | rgbRadiusSlider.minimumValue = 0 53 | rgbRadiusSlider.maximumValue = 1.5 54 | rgbRadiusSlider.isContinuous = true 55 | rgbRadiusSlider.value = renderer.rgbRadius 56 | rgbRadiusSlider.addTarget(self, action: #selector(viewValueChanged), for: .valueChanged) 57 | 58 | let stackView = UIStackView(arrangedSubviews: [confidenceControl, rgbRadiusSlider]) 59 | stackView.isHidden = !isUIEnabled 60 | stackView.translatesAutoresizingMaskIntoConstraints = false 61 | stackView.axis = .vertical 62 | stackView.spacing = 20 63 | view.addSubview(stackView) 64 | NSLayoutConstraint.activate([ 65 | stackView.centerXAnchor.constraint(equalTo: view.centerXAnchor), 66 | stackView.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -50) 67 | ]) 68 | } 69 | 70 | override func viewWillAppear(_ animated: Bool) { 71 | super.viewWillAppear(animated) 72 | 73 | // Create a world-tracking configuration, and 74 | // enable the scene depth frame-semantic. 75 | let configuration = ARWorldTrackingConfiguration() 76 | configuration.frameSemantics = .sceneDepth 77 | 78 | // Run the view's session 79 | session.run(configuration) 80 | 81 | // The screen shouldn't dim during AR experiences. 82 | UIApplication.shared.isIdleTimerDisabled = true 83 | } 84 | 85 | @objc 86 | private func viewValueChanged(view: UIView) { 87 | switch view { 88 | 89 | case confidenceControl: 90 | renderer.confidenceThreshold = confidenceControl.selectedSegmentIndex 91 | 92 | case rgbRadiusSlider: 93 | renderer.rgbRadius = rgbRadiusSlider.value 94 | 95 | default: 96 | break 97 | } 98 | } 99 | 100 | // Auto-hide the home indicator to maximize immersion in AR experiences. 101 | override var prefersHomeIndicatorAutoHidden: Bool { 102 | return true 103 | } 104 | 105 | // Hide the status bar to maximize immersion in AR experiences. 106 | override var prefersStatusBarHidden: Bool { 107 | return true 108 | } 109 | 110 | func session(_ session: ARSession, didFailWithError error: Error) { 111 | // Present an error message to the user. 112 | guard error is ARError else { return } 113 | let errorWithInfo = error as NSError 114 | let messages = [ 115 | errorWithInfo.localizedDescription, 116 | errorWithInfo.localizedFailureReason, 117 | errorWithInfo.localizedRecoverySuggestion 118 | ] 119 | let errorMessage = messages.compactMap({ $0 }).joined(separator: "\n") 120 | DispatchQueue.main.async { 121 | // Present an alert informing about the error that has occurred. 122 | let alertController = UIAlertController(title: "The AR session failed.", message: errorMessage, preferredStyle: .alert) 123 | let restartAction = UIAlertAction(title: "Restart Session", style: .default) { _ in 124 | alertController.dismiss(animated: true, completion: nil) 125 | if let configuration = self.session.configuration { 126 | self.session.run(configuration, options: .resetSceneReconstruction) 127 | } 128 | } 129 | alertController.addAction(restartAction) 130 | self.present(alertController, animated: true, completion: nil) 131 | } 132 | } 133 | } 134 | 135 | // MARK: - MTKViewDelegate 136 | 137 | extension ViewController: MTKViewDelegate { 138 | // Called whenever view changes orientation or layout is changed 139 | func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { 140 | renderer.drawRectResized(size: size) 141 | } 142 | 143 | // Called whenever the view needs to render 144 | func draw(in view: MTKView) { 145 | renderer.draw() 146 | } 147 | } 148 | 149 | // MARK: - RenderDestinationProvider 150 | 151 | protocol RenderDestinationProvider { 152 | var currentRenderPassDescriptor: MTLRenderPassDescriptor? { get } 153 | var currentDrawable: CAMetalDrawable? { get } 154 | var colorPixelFormat: MTLPixelFormat { get set } 155 | var depthStencilPixelFormat: MTLPixelFormat { get set } 156 | var sampleCount: Int { get set } 157 | } 158 | 159 | extension MTKView: RenderDestinationProvider { 160 | 161 | } 162 | --------------------------------------------------------------------------------