├── .gitignore ├── README.md ├── RealTimeRectangleDetection.xcodeproj └── project.pbxproj ├── RealTimeRectangleDetection ├── AppDelegate.swift ├── Assets.xcassets │ ├── AppIcon.appiconset │ │ └── Contents.json │ └── Contents.json ├── Base.lproj │ ├── LaunchScreen.storyboard │ └── Main.storyboard ├── Info.plist └── ViewController.swift ├── landscape.png └── portrait.png /.gitignore: -------------------------------------------------------------------------------- 1 | ### Xcode ### 2 | # Xcode 3 | # 4 | # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore 5 | 6 | ## User settings 7 | xcuserdata/ 8 | 9 | ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) 10 | *.xcscmblueprint 11 | *.xccheckout 12 | 13 | ## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) 14 | build/ 15 | DerivedData/ 16 | *.moved-aside 17 | *.pbxuser 18 | !default.pbxuser 19 | *.mode1v3 20 | !default.mode1v3 21 | *.mode2v3 22 | !default.mode2v3 23 | *.perspectivev3 24 | !default.perspectivev3 25 | 26 | ### Xcode Patch ### 27 | *.xcodeproj/* 28 | !*.xcodeproj/project.pbxproj 29 | !*.xcodeproj/xcshareddata/ 30 | !*.xcworkspace/contents.xcworkspacedata 31 | /*.gcno 32 | **/xcshareddata/WorkspaceSettings.xcsettings 33 | 34 | ### Mac OS X 35 | *.DS_Store 36 | 37 | ### Backup files 38 | *~.nib 39 | \#*# 40 | .#* 41 | *.swp 42 | *.un~ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ARKit + Vision for Real Time Rectangle Detection 2 | 3 | This project demonstrates how to use Apple's Vision library to identify rectangles with ARKit. 4 | 5 | ## Screen Shot 6 | ### Landscape 7 | ![LandScape](landscape.png) 8 | ### Portrait 9 | ![Portrait](portrait.png) 10 | 11 | ## Background 12 | This sample is based on ARKit and Vision. 13 | If you're not already familiar with the [ARKit](https://developer.apple.com/documentation/arkit/understanding_augmented_reality) and [Vision](https://developer.apple.com/documentation/vision) libraries. 14 | 15 | ## How it Works 16 | - The `@IBAction func doubleTapped(_ sender: Any)` method inside of `ViewController.swift` triggers to detect rectangle in the current frame. 17 | 18 | - The `func findRectangle(frame currentFrame: ARFrame)` method gets an image in ARFrame and then kicks VNDetectRectangle. Be careful orientation of the image because video frame orientation is always landscape unless physical device orientation is portrait. 19 | 20 | - The `func draw(rectangles: [VNRectangleObservation], onImageLayer drawlayer: CALayer)` method adds layer to draw a rectangle path. 21 | 22 | - The `func shapeLayerRect(color: UIColor, observation: VNRectangleObservation) -> CAShapeLayer` method draws detected rectangle path. Detected rectangle 4 points are not suitable for showing on ARSCNView because world is not same. We have to change it by affine transform but it is complicated. It's a good way to get affine transform by [displayTransform()](https://developer.apple.com/documentation/arkit/arframe/2923543-displaytransform). 23 | 24 | ## To Run 25 | 1. Prerquisites: You must have a device running iOS 12 with an A9 or later processor and XCode 10.1. Both of these are currently in beta but are available via the [Apple Developer portal](https://developer.apple.com/download/). 26 | 2. Download the source code for this project and open the project in XCode 10.1. 27 | 3. Change the Bundle Identifier and Team to your own unique identifier and team. Note that this project does not require a developer license to run on a phone, so you can use a personal team. 28 | 4. Run in XCode on your device. 29 | -------------------------------------------------------------------------------- /RealTimeRectangleDetection.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 50; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | C8D1E68A21E35AFE009AFCB0 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8D1E68921E35AFE009AFCB0 /* AppDelegate.swift */; }; 11 | C8D1E68C21E35AFE009AFCB0 /* art.scnassets in Resources */ = {isa = PBXBuildFile; fileRef = C8D1E68B21E35AFE009AFCB0 /* art.scnassets */; }; 12 | C8D1E68E21E35AFE009AFCB0 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8D1E68D21E35AFE009AFCB0 /* ViewController.swift */; }; 13 | C8D1E69121E35AFE009AFCB0 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = C8D1E68F21E35AFE009AFCB0 /* Main.storyboard */; }; 14 | C8D1E69321E35B01009AFCB0 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C8D1E69221E35B01009AFCB0 /* Assets.xcassets */; }; 15 | C8D1E69621E35B01009AFCB0 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = C8D1E69421E35B01009AFCB0 /* LaunchScreen.storyboard */; }; 16 | /* End PBXBuildFile section */ 17 | 18 | /* Begin PBXFileReference section */ 19 | C8D1E68621E35AFE009AFCB0 /* RealTimeRectangleDetection.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = RealTimeRectangleDetection.app; sourceTree = BUILT_PRODUCTS_DIR; }; 20 | C8D1E68921E35AFE009AFCB0 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 21 | C8D1E68B21E35AFE009AFCB0 /* art.scnassets */ = {isa = PBXFileReference; lastKnownFileType = wrapper.scnassets; path = art.scnassets; sourceTree = ""; }; 22 | C8D1E68D21E35AFE009AFCB0 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; 23 | C8D1E69021E35AFE009AFCB0 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 24 | C8D1E69221E35B01009AFCB0 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 25 | C8D1E69521E35B01009AFCB0 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 26 | C8D1E69721E35B01009AFCB0 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 27 | /* End PBXFileReference section */ 28 | 29 | /* Begin PBXFrameworksBuildPhase section */ 30 | C8D1E68321E35AFE009AFCB0 /* Frameworks */ = { 31 | isa = PBXFrameworksBuildPhase; 32 | buildActionMask = 2147483647; 33 | files = ( 34 | ); 35 | runOnlyForDeploymentPostprocessing = 0; 36 | }; 37 | /* End PBXFrameworksBuildPhase section */ 38 | 39 | /* Begin PBXGroup section */ 40 | C8D1E67D21E35AFE009AFCB0 = { 41 | isa = PBXGroup; 42 | children = ( 43 | C8D1E68821E35AFE009AFCB0 /* RealTimeRectangleDetection */, 44 | C8D1E68721E35AFE009AFCB0 /* Products */, 45 | ); 46 | sourceTree = ""; 47 | }; 48 | C8D1E68721E35AFE009AFCB0 /* Products */ = { 49 | isa = PBXGroup; 50 | children = ( 51 | C8D1E68621E35AFE009AFCB0 /* RealTimeRectangleDetection.app */, 52 | ); 53 | name = Products; 54 | sourceTree = ""; 55 | }; 56 | C8D1E68821E35AFE009AFCB0 /* RealTimeRectangleDetection */ = { 57 | isa = PBXGroup; 58 | children = ( 59 | C8D1E68921E35AFE009AFCB0 /* AppDelegate.swift */, 60 | C8D1E68B21E35AFE009AFCB0 /* art.scnassets */, 61 | C8D1E68D21E35AFE009AFCB0 /* ViewController.swift */, 62 | C8D1E68F21E35AFE009AFCB0 /* Main.storyboard */, 63 | C8D1E69221E35B01009AFCB0 /* Assets.xcassets */, 64 | C8D1E69421E35B01009AFCB0 /* LaunchScreen.storyboard */, 65 | C8D1E69721E35B01009AFCB0 /* Info.plist */, 66 | ); 67 | path = RealTimeRectangleDetection; 68 | sourceTree = ""; 69 | }; 70 | /* End PBXGroup section */ 71 | 72 | /* Begin PBXNativeTarget section */ 73 | C8D1E68521E35AFE009AFCB0 /* RealTimeRectangleDetection */ = { 74 | isa = PBXNativeTarget; 75 | buildConfigurationList = C8D1E69A21E35B01009AFCB0 /* Build configuration list for PBXNativeTarget "RealTimeRectangleDetection" */; 76 | buildPhases = ( 77 | C8D1E68221E35AFE009AFCB0 /* Sources */, 78 | C8D1E68321E35AFE009AFCB0 /* Frameworks */, 79 | C8D1E68421E35AFE009AFCB0 /* Resources */, 80 | ); 81 | buildRules = ( 82 | ); 83 | dependencies = ( 84 | ); 85 | name = RealTimeRectangleDetection; 86 | productName = RealTimeRectangleDetection; 87 | productReference = C8D1E68621E35AFE009AFCB0 /* RealTimeRectangleDetection.app */; 88 | productType = "com.apple.product-type.application"; 89 | }; 90 | /* End PBXNativeTarget section */ 91 | 92 | /* Begin PBXProject section */ 93 | C8D1E67E21E35AFE009AFCB0 /* Project object */ = { 94 | isa = PBXProject; 95 | attributes = { 96 | LastSwiftUpdateCheck = 1010; 97 | LastUpgradeCheck = 1010; 98 | ORGANIZATIONNAME = Nob; 99 | TargetAttributes = { 100 | C8D1E68521E35AFE009AFCB0 = { 101 | CreatedOnToolsVersion = 10.1; 102 | }; 103 | }; 104 | }; 105 | buildConfigurationList = C8D1E68121E35AFE009AFCB0 /* Build configuration list for PBXProject "RealTimeRectangleDetection" */; 106 | compatibilityVersion = "Xcode 9.3"; 107 | developmentRegion = en; 108 | hasScannedForEncodings = 0; 109 | knownRegions = ( 110 | en, 111 | Base, 112 | ); 113 | mainGroup = C8D1E67D21E35AFE009AFCB0; 114 | productRefGroup = C8D1E68721E35AFE009AFCB0 /* Products */; 115 | projectDirPath = ""; 116 | projectRoot = ""; 117 | targets = ( 118 | C8D1E68521E35AFE009AFCB0 /* RealTimeRectangleDetection */, 119 | ); 120 | }; 121 | /* End PBXProject section */ 122 | 123 | /* Begin PBXResourcesBuildPhase section */ 124 | C8D1E68421E35AFE009AFCB0 /* Resources */ = { 125 | isa = PBXResourcesBuildPhase; 126 | buildActionMask = 2147483647; 127 | files = ( 128 | C8D1E68C21E35AFE009AFCB0 /* art.scnassets in Resources */, 129 | C8D1E69621E35B01009AFCB0 /* LaunchScreen.storyboard in Resources */, 130 | C8D1E69321E35B01009AFCB0 /* Assets.xcassets in Resources */, 131 | C8D1E69121E35AFE009AFCB0 /* Main.storyboard in Resources */, 132 | ); 133 | runOnlyForDeploymentPostprocessing = 0; 134 | }; 135 | /* End PBXResourcesBuildPhase section */ 136 | 137 | /* Begin PBXSourcesBuildPhase section */ 138 | C8D1E68221E35AFE009AFCB0 /* Sources */ = { 139 | isa = PBXSourcesBuildPhase; 140 | buildActionMask = 2147483647; 141 | files = ( 142 | C8D1E68E21E35AFE009AFCB0 /* ViewController.swift in Sources */, 143 | C8D1E68A21E35AFE009AFCB0 /* AppDelegate.swift in Sources */, 144 | ); 145 | runOnlyForDeploymentPostprocessing = 0; 146 | }; 147 | /* End PBXSourcesBuildPhase section */ 148 | 149 | /* Begin PBXVariantGroup section */ 150 | C8D1E68F21E35AFE009AFCB0 /* Main.storyboard */ = { 151 | isa = PBXVariantGroup; 152 | children = ( 153 | C8D1E69021E35AFE009AFCB0 /* Base */, 154 | ); 155 | name = Main.storyboard; 156 | sourceTree = ""; 157 | }; 158 | C8D1E69421E35B01009AFCB0 /* LaunchScreen.storyboard */ = { 159 | isa = PBXVariantGroup; 160 | children = ( 161 | C8D1E69521E35B01009AFCB0 /* Base */, 162 | ); 163 | name = LaunchScreen.storyboard; 164 | sourceTree = ""; 165 | }; 166 | /* End PBXVariantGroup section */ 167 | 168 | /* Begin XCBuildConfiguration section */ 169 | C8D1E69821E35B01009AFCB0 /* Debug */ = { 170 | isa = XCBuildConfiguration; 171 | buildSettings = { 172 | ALWAYS_SEARCH_USER_PATHS = NO; 173 | CLANG_ANALYZER_NONNULL = YES; 174 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 175 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 176 | CLANG_CXX_LIBRARY = "libc++"; 177 | CLANG_ENABLE_MODULES = YES; 178 | CLANG_ENABLE_OBJC_ARC = YES; 179 | CLANG_ENABLE_OBJC_WEAK = YES; 180 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 181 | CLANG_WARN_BOOL_CONVERSION = YES; 182 | CLANG_WARN_COMMA = YES; 183 | CLANG_WARN_CONSTANT_CONVERSION = YES; 184 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 185 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 186 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 187 | CLANG_WARN_EMPTY_BODY = YES; 188 | CLANG_WARN_ENUM_CONVERSION = YES; 189 | CLANG_WARN_INFINITE_RECURSION = YES; 190 | CLANG_WARN_INT_CONVERSION = YES; 191 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 192 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 193 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 194 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 195 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 196 | CLANG_WARN_STRICT_PROTOTYPES = YES; 197 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 198 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 199 | CLANG_WARN_UNREACHABLE_CODE = YES; 200 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 201 | CODE_SIGN_IDENTITY = "iPhone Developer"; 202 | COPY_PHASE_STRIP = NO; 203 | DEBUG_INFORMATION_FORMAT = dwarf; 204 | ENABLE_STRICT_OBJC_MSGSEND = YES; 205 | ENABLE_TESTABILITY = YES; 206 | GCC_C_LANGUAGE_STANDARD = gnu11; 207 | GCC_DYNAMIC_NO_PIC = NO; 208 | GCC_NO_COMMON_BLOCKS = YES; 209 | GCC_OPTIMIZATION_LEVEL = 0; 210 | GCC_PREPROCESSOR_DEFINITIONS = ( 211 | "DEBUG=1", 212 | "$(inherited)", 213 | ); 214 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 215 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 216 | GCC_WARN_UNDECLARED_SELECTOR = YES; 217 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 218 | GCC_WARN_UNUSED_FUNCTION = YES; 219 | GCC_WARN_UNUSED_VARIABLE = YES; 220 | IPHONEOS_DEPLOYMENT_TARGET = 12.1; 221 | MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; 222 | MTL_FAST_MATH = YES; 223 | ONLY_ACTIVE_ARCH = YES; 224 | SDKROOT = iphoneos; 225 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; 226 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 227 | }; 228 | name = Debug; 229 | }; 230 | C8D1E69921E35B01009AFCB0 /* Release */ = { 231 | isa = XCBuildConfiguration; 232 | buildSettings = { 233 | ALWAYS_SEARCH_USER_PATHS = NO; 234 | CLANG_ANALYZER_NONNULL = YES; 235 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 236 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 237 | CLANG_CXX_LIBRARY = "libc++"; 238 | CLANG_ENABLE_MODULES = YES; 239 | CLANG_ENABLE_OBJC_ARC = YES; 240 | CLANG_ENABLE_OBJC_WEAK = YES; 241 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 242 | CLANG_WARN_BOOL_CONVERSION = YES; 243 | CLANG_WARN_COMMA = YES; 244 | CLANG_WARN_CONSTANT_CONVERSION = YES; 245 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 246 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 247 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 248 | CLANG_WARN_EMPTY_BODY = YES; 249 | CLANG_WARN_ENUM_CONVERSION = YES; 250 | CLANG_WARN_INFINITE_RECURSION = YES; 251 | CLANG_WARN_INT_CONVERSION = YES; 252 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 253 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 254 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 255 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 256 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 257 | CLANG_WARN_STRICT_PROTOTYPES = YES; 258 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 259 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 260 | CLANG_WARN_UNREACHABLE_CODE = YES; 261 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 262 | CODE_SIGN_IDENTITY = "iPhone Developer"; 263 | COPY_PHASE_STRIP = NO; 264 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 265 | ENABLE_NS_ASSERTIONS = NO; 266 | ENABLE_STRICT_OBJC_MSGSEND = YES; 267 | GCC_C_LANGUAGE_STANDARD = gnu11; 268 | GCC_NO_COMMON_BLOCKS = YES; 269 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 270 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 271 | GCC_WARN_UNDECLARED_SELECTOR = YES; 272 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 273 | GCC_WARN_UNUSED_FUNCTION = YES; 274 | GCC_WARN_UNUSED_VARIABLE = YES; 275 | IPHONEOS_DEPLOYMENT_TARGET = 12.1; 276 | MTL_ENABLE_DEBUG_INFO = NO; 277 | MTL_FAST_MATH = YES; 278 | SDKROOT = iphoneos; 279 | SWIFT_COMPILATION_MODE = wholemodule; 280 | SWIFT_OPTIMIZATION_LEVEL = "-O"; 281 | VALIDATE_PRODUCT = YES; 282 | }; 283 | name = Release; 284 | }; 285 | C8D1E69B21E35B01009AFCB0 /* Debug */ = { 286 | isa = XCBuildConfiguration; 287 | buildSettings = { 288 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 289 | CODE_SIGN_STYLE = Automatic; 290 | DEVELOPMENT_TEAM = 4EEDAR4AHC; 291 | INFOPLIST_FILE = RealTimeRectangleDetection/Info.plist; 292 | LD_RUNPATH_SEARCH_PATHS = ( 293 | "$(inherited)", 294 | "@executable_path/Frameworks", 295 | ); 296 | PRODUCT_BUNDLE_IDENTIFIER = nsunrise.com.RealTimeRectangleDetection; 297 | PRODUCT_NAME = "$(TARGET_NAME)"; 298 | SWIFT_VERSION = 4.2; 299 | TARGETED_DEVICE_FAMILY = "1,2"; 300 | }; 301 | name = Debug; 302 | }; 303 | C8D1E69C21E35B01009AFCB0 /* Release */ = { 304 | isa = XCBuildConfiguration; 305 | buildSettings = { 306 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 307 | CODE_SIGN_STYLE = Automatic; 308 | DEVELOPMENT_TEAM = 4EEDAR4AHC; 309 | INFOPLIST_FILE = RealTimeRectangleDetection/Info.plist; 310 | LD_RUNPATH_SEARCH_PATHS = ( 311 | "$(inherited)", 312 | "@executable_path/Frameworks", 313 | ); 314 | PRODUCT_BUNDLE_IDENTIFIER = nsunrise.com.RealTimeRectangleDetection; 315 | PRODUCT_NAME = "$(TARGET_NAME)"; 316 | SWIFT_VERSION = 4.2; 317 | TARGETED_DEVICE_FAMILY = "1,2"; 318 | }; 319 | name = Release; 320 | }; 321 | /* End XCBuildConfiguration section */ 322 | 323 | /* Begin XCConfigurationList section */ 324 | C8D1E68121E35AFE009AFCB0 /* Build configuration list for PBXProject "RealTimeRectangleDetection" */ = { 325 | isa = XCConfigurationList; 326 | buildConfigurations = ( 327 | C8D1E69821E35B01009AFCB0 /* Debug */, 328 | C8D1E69921E35B01009AFCB0 /* Release */, 329 | ); 330 | defaultConfigurationIsVisible = 0; 331 | defaultConfigurationName = Release; 332 | }; 333 | C8D1E69A21E35B01009AFCB0 /* Build configuration list for PBXNativeTarget "RealTimeRectangleDetection" */ = { 334 | isa = XCConfigurationList; 335 | buildConfigurations = ( 336 | C8D1E69B21E35B01009AFCB0 /* Debug */, 337 | C8D1E69C21E35B01009AFCB0 /* Release */, 338 | ); 339 | defaultConfigurationIsVisible = 0; 340 | defaultConfigurationName = Release; 341 | }; 342 | /* End XCConfigurationList section */ 343 | }; 344 | rootObject = C8D1E67E21E35AFE009AFCB0 /* Project object */; 345 | } 346 | -------------------------------------------------------------------------------- /RealTimeRectangleDetection/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AppDelegate.swift 3 | // RealTimeRectangleDetection 4 | // 5 | // Created by Nob on 2019/01/07. 6 | // Copyright © 2019 Nob. All rights reserved. 7 | // 8 | 9 | import UIKit 10 | 11 | @UIApplicationMain 12 | class AppDelegate: UIResponder, UIApplicationDelegate { 13 | 14 | var window: UIWindow? 15 | 16 | 17 | func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { 18 | // Override point for customization after application launch. 19 | return true 20 | } 21 | 22 | func applicationWillResignActive(_ application: UIApplication) { 23 | // Sent when the application is about to move from active to inactive state. This can occur for certain types of temporary interruptions (such as an incoming phone call or SMS message) or when the user quits the application and it begins the transition to the background state. 24 | // Use this method to pause ongoing tasks, disable timers, and invalidate graphics rendering callbacks. Games should use this method to pause the game. 25 | } 26 | 27 | func applicationDidEnterBackground(_ application: UIApplication) { 28 | // Use this method to release shared resources, save user data, invalidate timers, and store enough application state information to restore your application to its current state in case it is terminated later. 29 | // If your application supports background execution, this method is called instead of applicationWillTerminate: when the user quits. 30 | } 31 | 32 | func applicationWillEnterForeground(_ application: UIApplication) { 33 | // Called as part of the transition from the background to the active state; here you can undo many of the changes made on entering the background. 34 | } 35 | 36 | func applicationDidBecomeActive(_ application: UIApplication) { 37 | // Restart any tasks that were paused (or not yet started) while the application was inactive. If the application was previously in the background, optionally refresh the user interface. 38 | } 39 | 40 | func applicationWillTerminate(_ application: UIApplication) { 41 | // Called when the application is about to terminate. Save data if appropriate. See also applicationDidEnterBackground:. 42 | } 43 | 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /RealTimeRectangleDetection/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "iphone", 5 | "size" : "20x20", 6 | "scale" : "2x" 7 | }, 8 | { 9 | "idiom" : "iphone", 10 | "size" : "20x20", 11 | "scale" : "3x" 12 | }, 13 | { 14 | "idiom" : "iphone", 15 | "size" : "29x29", 16 | "scale" : "2x" 17 | }, 18 | { 19 | "idiom" : "iphone", 20 | "size" : "29x29", 21 | "scale" : "3x" 22 | }, 23 | { 24 | "idiom" : "iphone", 25 | "size" : "40x40", 26 | "scale" : "2x" 27 | }, 28 | { 29 | "idiom" : "iphone", 30 | "size" : "40x40", 31 | "scale" : "3x" 32 | }, 33 | { 34 | "idiom" : "iphone", 35 | "size" : "60x60", 36 | "scale" : "2x" 37 | }, 38 | { 39 | "idiom" : "iphone", 40 | "size" : "60x60", 41 | "scale" : "3x" 42 | }, 43 | { 44 | "idiom" : "ipad", 45 | "size" : "20x20", 46 | "scale" : "1x" 47 | }, 48 | { 49 | "idiom" : "ipad", 50 | "size" : "20x20", 51 | "scale" : "2x" 52 | }, 53 | { 54 | "idiom" : "ipad", 55 | "size" : "29x29", 56 | "scale" : "1x" 57 | }, 58 | { 59 | "idiom" : "ipad", 60 | "size" : "29x29", 61 | "scale" : "2x" 62 | }, 63 | { 64 | "idiom" : "ipad", 65 | "size" : "40x40", 66 | "scale" : "1x" 67 | }, 68 | { 69 | "idiom" : "ipad", 70 | "size" : "40x40", 71 | "scale" : "2x" 72 | }, 73 | { 74 | "idiom" : "ipad", 75 | "size" : "76x76", 76 | "scale" : "1x" 77 | }, 78 | { 79 | "idiom" : "ipad", 80 | "size" : "76x76", 81 | "scale" : "2x" 82 | }, 83 | { 84 | "idiom" : "ipad", 85 | "size" : "83.5x83.5", 86 | "scale" : "2x" 87 | }, 88 | { 89 | "idiom" : "ios-marketing", 90 | "size" : "1024x1024", 91 | "scale" : "1x" 92 | } 93 | ], 94 | "info" : { 95 | "version" : 1, 96 | "author" : "xcode" 97 | } 98 | } -------------------------------------------------------------------------------- /RealTimeRectangleDetection/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "version" : 1, 4 | "author" : "xcode" 5 | } 6 | } -------------------------------------------------------------------------------- /RealTimeRectangleDetection/Base.lproj/LaunchScreen.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /RealTimeRectangleDetection/Base.lproj/Main.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /RealTimeRectangleDetection/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | APPL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleVersion 20 | 1 21 | LSRequiresIPhoneOS 22 | 23 | NSCameraUsageDescription 24 | 25 | UILaunchStoryboardName 26 | LaunchScreen 27 | UIMainStoryboardFile 28 | Main 29 | UIRequiredDeviceCapabilities 30 | 31 | armv7 32 | arkit 33 | 34 | UIStatusBarHidden 35 | 36 | UISupportedInterfaceOrientations 37 | 38 | UIInterfaceOrientationPortrait 39 | UIInterfaceOrientationLandscapeLeft 40 | UIInterfaceOrientationLandscapeRight 41 | UIInterfaceOrientationPortraitUpsideDown 42 | 43 | UISupportedInterfaceOrientations~ipad 44 | 45 | UIInterfaceOrientationPortrait 46 | UIInterfaceOrientationPortraitUpsideDown 47 | UIInterfaceOrientationLandscapeLeft 48 | UIInterfaceOrientationLandscapeRight 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /RealTimeRectangleDetection/ViewController.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ViewController.swift 3 | // RealTimeRectangleDetection 4 | // 5 | // Created by Nob on 2019/01/07. 6 | // Copyright © 2019 Nob. All rights reserved. 7 | // 8 | 9 | import UIKit 10 | import SceneKit 11 | import ARKit 12 | import Vision 13 | 14 | class ViewController: UIViewController, ARSCNViewDelegate { 15 | 16 | @IBOutlet var sceneView: ARSCNView! 17 | 18 | // Layer into which to draw bounding box paths. 19 | var drawLayer: CALayer? 20 | 21 | override func viewDidLoad() { 22 | super.viewDidLoad() 23 | 24 | // Set the view's delegate 25 | sceneView.delegate = self 26 | 27 | // Show statistics such as fps and timing information 28 | sceneView.showsStatistics = true 29 | 30 | // Create a new scene 31 | let scene = SCNScene() 32 | 33 | // Set the scene to the view 34 | sceneView.scene = scene 35 | } 36 | 37 | override func viewWillAppear(_ animated: Bool) { 38 | super.viewWillAppear(animated) 39 | 40 | // Create a session configuration 41 | let configuration = ARWorldTrackingConfiguration() 42 | configuration.planeDetection = [.horizontal] 43 | 44 | sceneView.debugOptions = [ARSCNDebugOptions.showFeaturePoints] 45 | 46 | // Run the view's session 47 | sceneView.session.run(configuration) 48 | } 49 | 50 | override func viewWillDisappear(_ animated: Bool) { 51 | super.viewWillDisappear(animated) 52 | 53 | // Pause the view's session 54 | sceneView.session.pause() 55 | } 56 | 57 | // MARK: - ARSCNViewDelegate 58 | 59 | func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) { 60 | print("ARAnchor Added") 61 | 62 | sceneView.debugOptions = [] 63 | } 64 | 65 | @IBAction func doubleTapped(_ sender: Any) { 66 | let tapRecognizer = sender as! UITapGestureRecognizer 67 | 68 | if tapRecognizer.state == .recognized { 69 | print("tapped") 70 | 71 | if let drawLayer = self.drawLayer { 72 | drawLayer.sublayers?.forEach({ layer in 73 | layer.removeFromSuperlayer() 74 | }) 75 | } 76 | 77 | let layer = CALayer() 78 | layer.bounds = sceneView.bounds 79 | layer.anchorPoint = CGPoint.zero 80 | layer.opacity = 0.5 81 | self.view.layer.addSublayer(layer) 82 | drawLayer = layer 83 | 84 | if let frame = self.sceneView.session.currentFrame { 85 | findRectangle(frame: frame) 86 | } 87 | } 88 | } 89 | 90 | // Updates selectedRectangleObservation with the the rectangle found in the given ARFrame at the given location 91 | fileprivate func findRectangle(frame currentFrame: ARFrame) { 92 | // Perform request on background thread 93 | DispatchQueue.global(qos: .background).async { 94 | let request = VNDetectRectanglesRequest(completionHandler: self.handleDetectedRectangles) 95 | request.maximumObservations = 1 96 | 97 | let handler = VNImageRequestHandler(cvPixelBuffer: currentFrame.capturedImage, orientation: .downMirrored, options: [:]) 98 | do { 99 | try handler.perform([request]) 100 | } catch let error as NSError { 101 | print("Failed to perform image request: \(error)") 102 | return 103 | } 104 | } 105 | } 106 | 107 | fileprivate func handleDetectedRectangles(request: VNRequest?, error: Error?) { 108 | if let nsError = error as NSError? { 109 | print("Rectangle Detection Error \(nsError)") 110 | return 111 | } 112 | // Since handlers are executing on a background thread, explicitly send draw calls to the main thread. 113 | DispatchQueue.main.async { 114 | guard let results = request?.results as? [VNRectangleObservation] else { return } 115 | if let drawLayer = self.drawLayer { 116 | self.draw(rectangles: results, onImageLayer: drawLayer) 117 | drawLayer.setNeedsDisplay() 118 | } 119 | } 120 | } 121 | 122 | fileprivate func draw(rectangles: [VNRectangleObservation], onImageLayer drawlayer: CALayer) { 123 | CATransaction.begin() 124 | 125 | for observation in rectangles { 126 | let rectLayer = shapeLayerRect(color: .blue, observation: observation) 127 | 128 | // Add to pathLayer on top of image. 129 | drawlayer.addSublayer(rectLayer) 130 | } 131 | 132 | CATransaction.commit() 133 | } 134 | 135 | fileprivate func shapeLayerRect(color: UIColor, observation: VNRectangleObservation) -> CAShapeLayer { 136 | // Create a new layer. 137 | let layer = CAShapeLayer() 138 | 139 | guard let drawBounds = self.drawLayer?.bounds else { return layer } 140 | let orientation = UIApplication.shared.statusBarOrientation 141 | guard let arTransform = self.sceneView.session.currentFrame?.displayTransform(for: orientation, viewportSize: drawBounds.size) else { return layer } 142 | let t = CGAffineTransform(scaleX: drawBounds.width, y: drawBounds.height) 143 | 144 | let convertedTopLeft = observation.topLeft.applying(arTransform).applying(t) 145 | let convertedTopRight = observation.topRight.applying(arTransform).applying(t) 146 | let convertedBottomLeft = observation.bottomLeft.applying(arTransform).applying(t) 147 | let convertedBottomRight = observation.bottomRight.applying(arTransform).applying(t) 148 | 149 | let linePath = UIBezierPath() 150 | linePath.move(to: convertedTopLeft) 151 | linePath.addLine(to: convertedTopRight) 152 | linePath.addLine(to: convertedBottomRight) 153 | linePath.addLine(to: convertedBottomLeft) 154 | linePath.addLine(to: convertedTopLeft) 155 | linePath.close() 156 | layer.strokeColor = color.cgColor 157 | layer.lineWidth = 2 158 | layer.path = linePath.cgPath 159 | 160 | return layer 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /landscape.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nob3rise/RealTimeRectangleDetection/ba26ec6dc6aa145179e174ed11cca2332bb863dd/landscape.png -------------------------------------------------------------------------------- /portrait.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nob3rise/RealTimeRectangleDetection/ba26ec6dc6aa145179e174ed11cca2332bb863dd/portrait.png --------------------------------------------------------------------------------