├── .gitignore ├── .gitmodules ├── App ├── App.swift ├── Assets.xcassets │ ├── AccentColor.colorset │ │ └── Contents.json │ ├── AppIcon.appiconset │ │ └── Contents.json │ └── Contents.json ├── Entitlements.entitlements ├── SpatialDemo-Info.plist └── SpatialScene.swift ├── Engine ├── ARKitAnchorSupport.swift ├── ARKitMeshSupport.swift ├── ARSessionManager.swift ├── Anchor.swift ├── Bridging.h ├── BufferUtilities.swift ├── Camera.swift ├── Entity.swift ├── Light.swift ├── LockingQueue.swift ├── Material.swift ├── Materials.metal ├── Math.swift ├── Mesh.swift ├── MetalContext.swift ├── Model.swift ├── ModelIOSupport.swift ├── Physics │ ├── JoltPhysics.h │ ├── JoltPhysics.mm │ ├── JoltPhysics.swift │ └── PhysicsBridge.swift ├── PhysicsBody.swift ├── SceneRenderer.swift ├── ShaderTypes.h ├── Skinning.metal ├── SpatialEvent.swift ├── SpatialRenderLoop.swift ├── TextureLoader.swift └── TextureResource.swift ├── LICENSE ├── README.md ├── Resources ├── Blocks.usdz ├── Hand_Left.usdz ├── Hand_Right.usdz ├── Hand_Right_backup.usdz ├── LICENSE └── Placement_Reticle.usdz ├── SpatialRendering.xcodeproj ├── project.pbxproj ├── project.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ │ └── IDEWorkspaceChecks.plist └── xcshareddata │ └── xcschemes │ └── SpatialDemo.xcscheme ├── SpatialRendering.xcworkspace └── contents.xcworkspacedata ├── ThirdParty └── JoltPhysics.xcodeproj │ ├── project.pbxproj │ ├── project.xcworkspace │ └── xcshareddata │ │ └── WorkspaceSettings.xcsettings │ └── xcshareddata │ └── xcschemes │ └── Jolt.xcscheme └── images └── title-slide.png /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | xcuserdata 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ThirdParty/JoltPhysics"] 2 | path = ThirdParty/JoltPhysics 3 | url = https://github.com/jrouwe/JoltPhysics 4 | -------------------------------------------------------------------------------- /App/App.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import SwiftUI 18 | 19 | #if os(visionOS) 20 | import CompositorServices 21 | 22 | extension RenderLayout { 23 | init (_ layout: LayerRenderer.Layout) { 24 | switch layout { 25 | case .shared: 26 | self = .shared 27 | case .layered: 28 | self = .layered 29 | case .dedicated: 30 | fallthrough 31 | default: 32 | self = .dedicated 33 | } 34 | } 35 | } 36 | 37 | // A type that can be used to customize the creation of a LayerRenderer 38 | struct LayerConfiguration : CompositorLayerConfiguration { 39 | let context: MetalContext 40 | 41 | func makeConfiguration(capabilities: LayerRenderer.Capabilities, 42 | configuration: inout LayerRenderer.Configuration) 43 | { 44 | // Enable foveated rendering if available 45 | let supportsFoveation = capabilities.supportsFoveation 46 | configuration.isFoveationEnabled = supportsFoveation 47 | 48 | // Select a layout based on device capabilities 49 | let canUseAmplification = context.device.supportsVertexAmplificationCount(2) 50 | let supportedLayouts = capabilities.supportedLayouts(options: supportsFoveation ? .foveationEnabled : []) 51 | if supportedLayouts.contains(.layered) { 52 | // Layered rendering isn't supported everywhere, but when available, it is the most efficient option. 53 | configuration.layout = .layered 54 | print("Selected layered renderer layout") 55 | } else if canUseAmplification && !configuration.isFoveationEnabled { 56 | // On platforms where layered rendering isn't supported, we can use a 57 | // shared layout, provided we don't use foeveated rendering. 58 | configuration.layout = .shared 59 | print("Selected shared renderer layout") 60 | } else { 61 | // If all else fails, we can use a dedicated layout, but this requires one pass per view. 62 | configuration.layout = .dedicated 63 | print("Selected dedicated renderer layout") 64 | } 65 | 66 | // Use the global preferred pixel formats as our drawable formats 67 | configuration.colorFormat = context.preferredColorPixelFormat 68 | configuration.depthFormat = context.preferredDepthPixelFormat 69 | } 70 | } 71 | 72 | @main 73 | struct SpatialApp: App { 74 | @State var context = MetalContext.shared 75 | @State var selectedImmersionStyle: (any ImmersionStyle) = .mixed 76 | 77 | init() { 78 | SRJPhysicsWorld.initializeJoltPhysics() 79 | } 80 | 81 | var body: some Scene { 82 | ImmersiveSpace(id: "Immersive Space") { 83 | // The content of this immersive space is a compositor layer, which 84 | // allows us to render mixed and fully immersive 3D content via a 85 | // layer renderer 86 | CompositorLayer(configuration: LayerConfiguration(context: context)) { layerRenderer in 87 | // Spin up a high-priority task so updates and rendering happen off the main thread 88 | Task.detached(priority: .high) { 89 | await runRenderLoop(layerRenderer) 90 | } 91 | } 92 | } 93 | .immersionStyle(selection: $selectedImmersionStyle, in: .mixed, .full) 94 | // Prevent hands from occluding rendered content 95 | .upperLimbVisibility(.hidden) 96 | // Prevent home button from interfering with interaction/gestures 97 | .persistentSystemOverlays(.hidden) 98 | } 99 | 100 | func runRenderLoop(_ layerRenderer: LayerRenderer) async { 101 | do { 102 | // We need an ARKit session to be running in order to determine where 103 | // spatial content should be drawn, so get that going first. 104 | let sessionManager = ARSessionManager() 105 | try await sessionManager.start(options: [.planeDetection, .sceneReconstruction, .handTracking, .lightEstimation]) 106 | 107 | // Load scene content 108 | let scene = try SpatialScene(sessionManager: sessionManager, context: context) 109 | 110 | // Create a renderer 111 | let renderer = try SceneRenderer(context: context, layout: RenderLayout(layerRenderer.configuration.layout)) 112 | 113 | // Subscribe to world-sensing updates 114 | sessionManager.onWorldAnchorUpdate = { worldAnchor, event in 115 | scene.enqueueEvent(.worldAnchor(worldAnchor, event)) 116 | } 117 | sessionManager.onPlaneAnchorUpdate = { planeAnchor, event in 118 | scene.enqueueEvent(.planeAnchor(planeAnchor, event)) 119 | } 120 | sessionManager.onMeshAnchorUpdate = { meshAnchor, event in 121 | scene.enqueueEvent(.meshAnchor(meshAnchor, event)) 122 | } 123 | sessionManager.onEnvironmentLightAnchorUpdate = { lightAnchor, event in 124 | scene.enqueueEvent(.environmentLightAnchor(lightAnchor, event)) 125 | } 126 | 127 | // Subscribe to spatial events from the layer renderer 128 | layerRenderer.onSpatialEvent = { events in 129 | for event in events { 130 | scene.enqueueEvent(.spatialInput(SpatialInputEvent(event))) 131 | } 132 | } 133 | 134 | // Create a render loop to draw our immersive content on a cadence determined by the system 135 | let renderLoop = SpatialRenderLoop(scene: scene, renderer: renderer, sessionManager: sessionManager) 136 | await renderLoop.run(layerRenderer) 137 | } catch { 138 | print("Failed to start render loop: \(error.localizedDescription)") 139 | } 140 | print("Exiting render loop...") 141 | } 142 | } 143 | 144 | #endif 145 | 146 | -------------------------------------------------------------------------------- /App/Assets.xcassets/AccentColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "idiom" : "universal" 5 | } 6 | ], 7 | "info" : { 8 | "author" : "xcode", 9 | "version" : 1 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /App/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "universal", 5 | "platform" : "ios", 6 | "size" : "1024x1024" 7 | }, 8 | { 9 | "idiom" : "mac", 10 | "scale" : "1x", 11 | "size" : "16x16" 12 | }, 13 | { 14 | "idiom" : "mac", 15 | "scale" : "2x", 16 | "size" : "16x16" 17 | }, 18 | { 19 | "idiom" : "mac", 20 | "scale" : "1x", 21 | "size" : "32x32" 22 | }, 23 | { 24 | "idiom" : "mac", 25 | "scale" : "2x", 26 | "size" : "32x32" 27 | }, 28 | { 29 | "idiom" : "mac", 30 | "scale" : "1x", 31 | "size" : "128x128" 32 | }, 33 | { 34 | "idiom" : "mac", 35 | "scale" : "2x", 36 | "size" : "128x128" 37 | }, 38 | { 39 | "idiom" : "mac", 40 | "scale" : "1x", 41 | "size" : "256x256" 42 | }, 43 | { 44 | "idiom" : "mac", 45 | "scale" : "2x", 46 | "size" : "256x256" 47 | }, 48 | { 49 | "idiom" : "mac", 50 | "scale" : "1x", 51 | "size" : "512x512" 52 | }, 53 | { 54 | "idiom" : "mac", 55 | "scale" : "2x", 56 | "size" : "512x512" 57 | } 58 | ], 59 | "info" : { 60 | "author" : "xcode", 61 | "version" : 1 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /App/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /App/Entitlements.entitlements: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.apple.security.app-sandbox 6 | 7 | com.apple.security.files.user-selected.read-only 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /App/SpatialDemo-Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | NSWorldSensingUsageDescription 6 | This app uses world sensing to enable interaction between virtual content and the real world. It does not collect or store any personally identifiable information. 7 | NSHandsTrackingUsageDescription 8 | This app uses hand tracking to enable interaction between virtual content and the user's hands. 9 | UIApplicationSceneManifest 10 | 11 | UIApplicationSupportsMultipleScenes 12 | 13 | UIApplicationPreferredDefaultSceneSessionRole 14 | CPSceneSessionRoleImmersiveSpaceApplication 15 | UISceneConfigurations 16 | 17 | CPSceneSessionRoleImmersiveSpaceApplication 18 | 19 | 20 | UISceneInitialImmersionStyle 21 | UIImmersionStyleMixed 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /App/SpatialScene.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import SwiftUI 19 | import Metal 20 | import MetalKit 21 | import ModelIO 22 | 23 | class SpatialScene : Entity, SceneContent { 24 | enum ScenePhase { 25 | case selectingPlacement 26 | case playing 27 | } 28 | var scenePhase = ScenePhase.selectingPlacement 29 | 30 | let sessionManager: ARSessionManager 31 | let context: MetalContext 32 | let rootEntity: Entity 33 | var anchoredContentRoot: AnchorEntity? 34 | 35 | var entities: [Entity] { 36 | return [Entity](flatEntities) 37 | } 38 | var lights: [Light] = [] 39 | 40 | var environmentLight: EnvironmentLight? 41 | 42 | private let physicsBridge: PhysicsBridge 43 | private let eventQueue = LockingQueue() 44 | private var flatEntities = Set() 45 | private var anchorEntities: [UUID : AnchorEntity] = [:] 46 | private var meshEntities: [UUID : Entity] = [:] 47 | private var planeEntities: [UUID : Entity] = [:] 48 | private var handEntities: [Handedness : HandEntity] = [:] 49 | private var environmentMeshMaterial: Material! 50 | private var reticleEntity: Entity? 51 | private var candidatePlanes = Set() 52 | private var selectedPlane: Entity? 53 | private var blockPrototypes = [Entity]() 54 | 55 | #if targetEnvironment(simulator) 56 | let floorLevel: Float = -0.5 57 | #else 58 | let floorLevel: Float = 0.0 59 | #endif 60 | 61 | init(sessionManager: ARSessionManager, context: MetalContext) throws { 62 | self.sessionManager = sessionManager 63 | self.context = context 64 | self.rootEntity = Entity() 65 | self.physicsBridge = JoltPhysicsBridge() 66 | 67 | super.init() 68 | 69 | self.addChild(rootEntity) 70 | try makeScene() 71 | } 72 | 73 | func makeScene() throws { 74 | environmentMeshMaterial = OcclusionMaterial() 75 | environmentMeshMaterial.name = "Occlusion" 76 | 77 | let sunLight = Light(type: .directional, color: simd_float3(1, 1, 1), intensity: 1) 78 | sunLight.transform.look(at: simd_float3(0, 0, 0), from: simd_float3(0.1, 1, 0.1), up: simd_float3(0, 1, 0)) 79 | lights.append(sunLight) 80 | 81 | #if targetEnvironment(simulator) 82 | let accentTop = Light(type: .point, color: simd_float3(1, 1, 1), intensity: 25) 83 | accentTop.transform.look(at: simd_float3(0, 0, 0), from: simd_float3(-3, 3, 0), up: simd_float3(0, 1, 0)) 84 | lights.append(accentTop) 85 | 86 | let accentBottom = Light(type: .point, color: simd_float3(1, 1, 1), intensity: 25) 87 | accentBottom.transform.look(at: simd_float3(0, 0, 0), from: simd_float3(-3, -3, 0), up: simd_float3(0, 1, 0)) 88 | lights.append(accentBottom) 89 | #endif 90 | 91 | if let handURL = Bundle.main.url(forResource: "Hand_Left", withExtension: "usdz") { 92 | let model = try Model(fileURL: handURL, context: context) 93 | if let handArmature = model.rootEntities.first { 94 | let handEntity = HandEntity(handedness: .left, context: context) 95 | handEntity.addChild(handArmature) 96 | rootEntity.addChild(handEntity) 97 | handEntities[.left] = handEntity 98 | handEntity.worldTransform = Transform(position: simd_float3(-0.15, floorLevel + 0.8, -0.15)) 99 | if let handMaterial = handEntity.child(named: "Mesh")?.mesh?.materials.first as? PhysicallyBasedMaterial { 100 | handMaterial.blendMode = .sourceOverPremultiplied 101 | handMaterial.isDoubleSided = false 102 | handMaterial.baseColor.baseColorFactor.w = 0.1 103 | } 104 | } 105 | } 106 | if let handURL = Bundle.main.url(forResource: "Hand_Right", withExtension: "usdz") { 107 | let model = try Model(fileURL: handURL, context: context) 108 | if let handArmature = model.rootEntities.first { 109 | let handEntity = HandEntity(handedness: .right, context: context) 110 | handEntity.addChild(handArmature) 111 | rootEntity.addChild(handEntity) 112 | handEntities[.right] = handEntity 113 | handEntity.worldTransform = Transform(position: simd_float3(0.15, floorLevel + 0.8, -0.15)) 114 | if let handMaterial = handEntity.child(named: "Mesh")?.mesh?.materials.first as? PhysicallyBasedMaterial { 115 | handMaterial.blendMode = .sourceOverPremultiplied 116 | handMaterial.isDoubleSided = false 117 | handMaterial.baseColor.baseColorFactor.w = 0.1 118 | } 119 | } 120 | } 121 | 122 | if let reticleURL = Bundle.main.url(forResource: "Placement_Reticle", withExtension: "usdz") { 123 | let model = try Model(fileURL: reticleURL, context: context) 124 | // USD tends to create unnecessarily deep prim hierarchies, so just reach in and grab the mesh node. 125 | if let meshEntity = model.rootEntities.first?.child(named: "mesh") { 126 | let reticleTransform = meshEntity.worldTransform 127 | meshEntity.removeFromParent() 128 | meshEntity.modelTransform = reticleTransform 129 | reticleEntity = meshEntity 130 | } 131 | } 132 | 133 | if let modelURL = Bundle.main.url(forResource: "Blocks", withExtension: "usdz") { 134 | let model = try Model(fileURL: modelURL, context: context) 135 | let blockEntities = model.rootEntities.first?.childEntities(matching: { $0.mesh != nil }) ?? [] 136 | blockEntities.forEach { entity in 137 | // Since we want to apply physics to each object in the model, we move it to the scene root 138 | // by de-parenting it and assigning its initial world transform as its model transform. 139 | // This allows us to handle object scaling correctly in the rendering and physics systems. 140 | let placementTransform = entity.worldTransform 141 | entity.removeFromParent() 142 | entity.modelTransform = placementTransform 143 | entity.generateCollisionShapes(recursive: false, bodyMode: .dynamic) 144 | } 145 | self.blockPrototypes = blockEntities 146 | } 147 | 148 | // Create a large ground plane so that even if we don't have plane detection 149 | // or scene reconstruction enabled, virtual objects don't fall infinitely far 150 | let floorExtent = simd_float3(10, 0.02, 10) 151 | let groundPlane = Entity() 152 | groundPlane.name = "Ground Plane" 153 | groundPlane.isHidden = true 154 | groundPlane.modelTransform.position = simd_float3(0, floorLevel - floorExtent.y * 0.5, 0) 155 | let groundShape = PhysicsShape(boxWithExtents: floorExtent) 156 | groundPlane.physicsBody = PhysicsBody(mode: .static, shape: groundShape) 157 | rootEntity.addChild(groundPlane) 158 | 159 | #if targetEnvironment(simulator) || !os(visionOS) 160 | selectPlaneForPlacement(groundPlane, pose: Transform()) 161 | #endif 162 | } 163 | 164 | func enqueueEvent(_ event: Event) { 165 | eventQueue.enqueue(event) 166 | } 167 | 168 | override func update(_ timestep: TimeInterval) { 169 | for event in eventQueue.popAll() { 170 | switch event { 171 | case .worldAnchor(let anchor, let anchorEvent): 172 | handleWorldAnchorEvent(anchor, event: anchorEvent) 173 | case .meshAnchor(let anchor, let anchorEvent): 174 | handleMeshAnchorEvent(anchor, event: anchorEvent) 175 | case .planeAnchor(let anchor, let anchorEvent): 176 | handlePlaneAnchorEvent(anchor, event: anchorEvent) 177 | case .handAnchor(let anchor, let anchorEvent): 178 | handleHandAnchorEvent(anchor, event: anchorEvent) 179 | case .spatialInput(let spatialEvents): 180 | handleSpatialInput(spatialEvents) 181 | case .environmentLightAnchor(let anchor, let anchorEvent): 182 | handleEnvironmentLightEvent(anchor, event: anchorEvent) 183 | } 184 | } 185 | 186 | for entity in entities { 187 | entity.update(timestep) 188 | } 189 | 190 | physicsBridge.update(entities: entities, timestep: timestep) 191 | } 192 | 193 | override func didAddChild(_ entity: Entity) { 194 | let flattenedSubtree = flattenedHierarchy(entity) 195 | for element in flattenedSubtree { 196 | flatEntities.insert(element) 197 | physicsBridge.addEntity(element) 198 | } 199 | super.didAddChild(entity) 200 | } 201 | 202 | override func didRemoveChild(_ entity: Entity) { 203 | let flattenedSubtree = flattenedHierarchy(entity) 204 | for element in flattenedSubtree { 205 | flatEntities.remove(element) 206 | physicsBridge.removeEntity(element) 207 | } 208 | super.didRemoveChild(entity) 209 | } 210 | 211 | private func updatePhysicsShapeForStaticEntity(_ entity: Entity) { 212 | physicsBridge.removeEntity(entity) 213 | if let mesh = entity.mesh { 214 | let shape = PhysicsShape(concaveMeshFromMesh: mesh) 215 | entity.physicsBody = PhysicsBody(mode: .static, shape: shape) 216 | } else { 217 | entity.physicsBody = nil 218 | } 219 | physicsBridge.addEntity(entity) 220 | } 221 | 222 | func handlePlaneAnchorEvent(_ anchor: PlaneAnchor, event: AnchorEvent) { 223 | let desiredPlacements: [PlaneAnchor.Classification] = [.table] 224 | switch event { 225 | case .added: 226 | let planeEntity = Entity() 227 | planeEntity.name = "Plane \(anchor.id)" 228 | planeEntity.worldTransform = Transform(anchor.originFromAnchorTransform) 229 | planeEntity.mesh = anchor.mesh 230 | planeEntity.mesh?.materials = [environmentMeshMaterial] 231 | planeEntities[anchor.id] = planeEntity 232 | if scenePhase == .selectingPlacement { 233 | if desiredPlacements.contains(anchor.classification) { 234 | candidatePlanes.insert(planeEntity) 235 | if let planeReticle = reticleEntity?.clone(recursively: false) { 236 | let reticlePivot = Entity(transform: Transform(position: simd_float3(0, 0.005, 0))) 237 | reticlePivot.addChild(planeReticle) 238 | planeEntity.addChild(reticlePivot) 239 | } 240 | } 241 | } 242 | rootEntity.addChild(planeEntity) 243 | case .updated: 244 | if let planeEntity = planeEntities[anchor.id] { 245 | planeEntity.worldTransform = Transform(anchor.originFromAnchorTransform) 246 | planeEntity.mesh = anchor.mesh 247 | planeEntity.mesh?.materials = [environmentMeshMaterial] 248 | updatePhysicsShapeForStaticEntity(planeEntity) 249 | } 250 | case .removed: 251 | if let planeEntity = planeEntities[anchor.id] { 252 | planeEntities.removeValue(forKey: anchor.id) 253 | candidatePlanes.remove(planeEntity) 254 | planeEntity.removeFromParent() 255 | } 256 | } 257 | } 258 | 259 | func handleMeshAnchorEvent(_ anchor: MeshAnchor, event: AnchorEvent) { 260 | switch event { 261 | case .added: 262 | let meshEntity = Entity() 263 | meshEntity.name = "World Mesh \(anchor.id)" 264 | meshEntity.worldTransform = Transform(anchor.originFromAnchorTransform) 265 | meshEntity.mesh = anchor.mesh 266 | meshEntity.mesh?.materials = [environmentMeshMaterial] 267 | meshEntities[anchor.id] = meshEntity 268 | rootEntity.addChild(meshEntity) 269 | case .updated: 270 | if let meshEntity = meshEntities[anchor.id] { 271 | meshEntity.worldTransform = Transform(anchor.originFromAnchorTransform) 272 | meshEntity.mesh = anchor.mesh 273 | meshEntity.mesh?.materials = [environmentMeshMaterial] 274 | updatePhysicsShapeForStaticEntity(meshEntity) 275 | } 276 | case .removed: 277 | if let meshEntity = meshEntities[anchor.id] { 278 | meshEntities.removeValue(forKey: anchor.id) 279 | meshEntity.removeFromParent() 280 | } 281 | } 282 | } 283 | 284 | func handleHandAnchorEvent(_ anchor: HandAnchor, event: AnchorEvent) { 285 | switch event { 286 | case .added: 287 | if let handEntity = handEntities[anchor.handedness] { 288 | handEntity.isHidden = false 289 | handEntity.updatePose(from: anchor) 290 | } 291 | case .updated: 292 | if let handEntity = handEntities[anchor.handedness] { 293 | handEntity.updatePose(from: anchor) 294 | } 295 | case .removed: 296 | if let handEntity = handEntities[anchor.handedness] { 297 | handEntity.isHidden = true 298 | } 299 | } 300 | } 301 | 302 | func handleWorldAnchorEvent(_ anchor: WorldAnchor, event: AnchorEvent) { 303 | switch event { 304 | case .added: 305 | if let anchorEntity = anchorEntities[anchor.id] { 306 | anchorEntity.updatePose(from: anchor) 307 | } else { 308 | let anchorEntity = AnchorEntity(transform: Transform(anchor.originFromAnchorTransform)) 309 | anchorEntity.name = "Anchor \(anchor.id)" 310 | anchorEntities[anchor.id] = anchorEntity 311 | } 312 | case .updated: 313 | if let anchorEntity = anchorEntities[anchor.id] { 314 | anchorEntity.updatePose(from: anchor) 315 | } 316 | case .removed: 317 | anchorEntities.removeValue(forKey: anchor.id) 318 | } 319 | } 320 | 321 | private func handleEnvironmentLightEvent(_ anchor: EnvironmentLightAnchor, event: AnchorEvent) { 322 | switch event { 323 | case .added: 324 | fallthrough 325 | case .updated: 326 | // Only bother updating environmental lighting once we have an environment cube map 327 | if anchor.light.environmentTexture != nil { 328 | self.environmentLight = anchor.light 329 | } 330 | case .removed: 331 | // Although it might be more correct to remove environment lights when their 332 | // corresponding anchors go away, we prefer to use environment lighting even 333 | // if it's stale, so we retain the most recent update regardless. Handling this 334 | // comprehensively means tracking all environment probe anchors, since ARKit 335 | // may create multiple probes as the device moves around the environment. 336 | break 337 | } 338 | } 339 | 340 | private func handleSpatialInput(_ event: SpatialInputEvent) { 341 | switch event.phase { 342 | case .active: 343 | break 344 | case .ended: 345 | if let ray = event.selectionRay { 346 | let origin = simd_float3(ray.origin) 347 | let toward = simd_float3(ray.direction) 348 | let hitResults = physicsBridge.hitTestWithSegment(from: origin, to: origin + toward * 3.0) 349 | for hit in hitResults.sorted(by: { simd_length($0.worldPosition - origin) < simd_length($1.worldPosition - origin)}) { 350 | if scenePhase == .selectingPlacement { 351 | if let hitEntity = hit.entity, candidatePlanes.contains(hitEntity) { 352 | selectPlaneForPlacement(hitEntity, pose: Transform(position: hit.worldPosition)) 353 | } 354 | } 355 | } 356 | } 357 | case .cancelled: 358 | break 359 | @unknown default: 360 | break 361 | } 362 | } 363 | 364 | private func selectPlaneForPlacement(_ planeEntity: Entity, pose: Transform) { 365 | selectedPlane = planeEntity 366 | scenePhase = .playing 367 | print("Transitioned to playing state with selected plane \(selectedPlane?.name ?? "")") 368 | Task { 369 | await buildTower(pose) 370 | } 371 | } 372 | 373 | private func buildTower(_ pose: Transform) async { 374 | guard blockPrototypes.count > 0 else { 375 | print("Failed to load block models; cannot construct tower") 376 | return 377 | } 378 | 379 | do { 380 | try await sessionManager.removeAllWorldAnchors() 381 | let worldAnchor = try await sessionManager.addWorldAnchor(originFromAnchorTransform: pose.matrix) 382 | let towerAnchor = AnchorEntity() 383 | towerAnchor.name = "Tower Anchor" 384 | towerAnchor.updatePose(from: worldAnchor) 385 | anchorEntities[worldAnchor.id] = towerAnchor 386 | rootEntity.addChild(towerAnchor) 387 | self.anchoredContentRoot = towerAnchor 388 | } catch { 389 | print("Could not create world anchor to place tower; creating default placement") 390 | let towerAnchor = AnchorEntity(transform: Transform(position: simd_float3(0, 0, -0.5))) 391 | rootEntity.addChild(towerAnchor) 392 | self.anchoredContentRoot = towerAnchor 393 | } 394 | 395 | let layerCount = 7 396 | let lateralMargin: Float = 0.0025 397 | let verticalMargin: Float = 0.005 398 | 399 | // Calculate the block bounds, assuming all prototypes have the same size 400 | let defaultBlockBounds = blockPrototypes[0].mesh!.boundingBox 401 | let blockWidth = defaultBlockBounds.max.x - defaultBlockBounds.min.x 402 | let blockHeight = defaultBlockBounds.max.y - defaultBlockBounds.min.y 403 | let blockDepth = defaultBlockBounds.max.z - defaultBlockBounds.min.z 404 | 405 | let rotation = Transform(orientation: simd_quatf(angle: .pi / 2, axis: simd_float3(0, 1, 0))) 406 | var layerIsRotated = false 407 | var yPosition = blockHeight * 0.5 408 | var blockNumber = 0 409 | for _ in 0...stride 49 | 50 | let positionView = BufferView(buffer: geometry.vertices.buffer, offset: geometry.vertices.offset) 51 | let normalView = BufferView(buffer: geometry.normals.buffer, offset: geometry.normals.offset) 52 | let texCoords = [simd_float2](repeating: simd_float2(), count: vertexCount) 53 | let texCoordBuffer = device.makeBuffer(bytes: texCoords, 54 | length: MemoryLayout.stride * vertexCount, 55 | options: .storageModeShared)! 56 | texCoordBuffer.label = "Mesh Anchor Texture Coordinates" 57 | let texCoordView = BufferView(buffer: texCoordBuffer) 58 | 59 | let indexBuffer = BufferView(buffer: geometry.faces.buffer) 60 | 61 | let submesh = Submesh(primitiveType: .triangle, 62 | indexBuffer: indexBuffer, 63 | indexType: geometry.faces.bytesPerIndex == 4 ? .uint32 : .uint16, 64 | indexCount: geometry.faces.count * geometry.faces.primitive.indexCount, 65 | materialIndex: 0) 66 | 67 | self.init(vertexCount: vertexCount, 68 | vertexBuffers: [positionView, normalView, texCoordView], 69 | vertexDescriptor: vertexDescriptor, 70 | submeshes: [submesh], 71 | materials: [PhysicallyBasedMaterial.default]) 72 | } 73 | 74 | convenience init(_ planeAnchor: ARKit.PlaneAnchor) { 75 | let geometry = planeAnchor.geometry 76 | precondition(geometry.meshVertices.format == .float3) 77 | precondition(geometry.meshFaces.primitive == .triangle) 78 | precondition(geometry.meshFaces.bytesPerIndex == 2 || geometry.meshFaces.bytesPerIndex == 4) 79 | 80 | let device = geometry.meshVertices.buffer.device 81 | let vertexCount = geometry.meshVertices.count 82 | 83 | let vertexDescriptor = MDLVertexDescriptor() 84 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition 85 | vertexDescriptor.vertexAttributes[0].format = .float3 86 | vertexDescriptor.vertexAttributes[0].offset = 0 87 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0 88 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal 89 | vertexDescriptor.vertexAttributes[1].format = .float3 90 | vertexDescriptor.vertexAttributes[1].offset = 0 91 | vertexDescriptor.vertexAttributes[1].bufferIndex = 1 92 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate 93 | vertexDescriptor.vertexAttributes[2].format = .float2 94 | vertexDescriptor.vertexAttributes[2].offset = 0 95 | vertexDescriptor.vertexAttributes[2].bufferIndex = 2 96 | vertexDescriptor.bufferLayouts[0].stride = geometry.meshVertices.stride 97 | vertexDescriptor.bufferLayouts[1].stride = MemoryLayout.stride 98 | vertexDescriptor.bufferLayouts[2].stride = MemoryLayout.stride 99 | 100 | let positionView = BufferView(buffer: geometry.meshVertices.buffer, offset: geometry.meshVertices.offset) 101 | let normalData = [simd_float3](repeating: simd_float3(0, 1, 0), count: vertexCount) 102 | let normalBuffer = device.makeBuffer(bytes: normalData, 103 | length: MemoryLayout.stride * vertexCount, 104 | options: .storageModeShared)! 105 | normalBuffer.label = "Plane Anchor Normals" 106 | let normalView = BufferView(buffer: normalBuffer) 107 | let texCoordData = [simd_float2](repeating: simd_float2(), count: vertexCount) 108 | let texCoordBuffer = device.makeBuffer(bytes: texCoordData, 109 | length: MemoryLayout.stride * vertexCount, 110 | options: .storageModeShared)! 111 | texCoordBuffer.label = "Plane Anchor Texture Coordinates" 112 | let texCoordView = BufferView(buffer: texCoordBuffer) 113 | 114 | let indexBuffer = BufferView(buffer: geometry.meshFaces.buffer) 115 | 116 | let submesh = Submesh(primitiveType: .triangle, 117 | indexBuffer: indexBuffer, 118 | indexType: geometry.meshFaces.bytesPerIndex == 4 ? .uint32 : .uint16, 119 | indexCount: geometry.meshFaces.count * geometry.meshFaces.primitive.indexCount, 120 | materialIndex: 0) 121 | 122 | self.init(vertexCount: vertexCount, 123 | vertexBuffers: [positionView, normalView, texCoordView], 124 | vertexDescriptor: vertexDescriptor, 125 | submeshes: [submesh], 126 | materials: [PhysicallyBasedMaterial.default]) 127 | } 128 | } 129 | 130 | #endif 131 | -------------------------------------------------------------------------------- /Engine/ARSessionManager.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #if os(visionOS) 18 | import ARKit 19 | 20 | typealias WorldAnchorUpdateHandler = (WorldAnchor, AnchorEvent) -> Void 21 | typealias MeshAnchorUpdateHandler = (MeshAnchor, AnchorEvent) -> Void 22 | typealias PlaneAnchorUpdateHandler = (PlaneAnchor, AnchorEvent) -> Void 23 | typealias HandAnchorUpdateHandler = (HandAnchor, AnchorEvent) -> Void 24 | typealias EnvironmentLightAnchorUpdateHandler = (EnvironmentLightAnchor, AnchorEvent) -> Void 25 | 26 | /// This extension maps from ARKit anchor update events to our abstract anchor event type 27 | extension AnchorEvent { 28 | init(_ event: AnchorUpdate.Event) where AnchorType : ARKit.Anchor { 29 | switch event { 30 | case .added: 31 | self = .added 32 | case .updated: 33 | self = .updated 34 | case .removed: 35 | self = .removed 36 | } 37 | } 38 | } 39 | 40 | class ARSessionManager { 41 | struct Options: OptionSet { 42 | let rawValue: Int 43 | 44 | static let planeDetection = Options(rawValue: 1 << 0) 45 | static let handTracking = Options(rawValue: 1 << 1) 46 | static let sceneReconstruction = Options(rawValue: 1 << 2) 47 | static let lightEstimation = Options(rawValue: 1 << 3) 48 | 49 | static let all: Options = [ 50 | .planeDetection, .handTracking, .sceneReconstruction, .lightEstimation 51 | ] 52 | } 53 | 54 | let arSession: ARKitSession 55 | 56 | var onWorldAnchorUpdate: WorldAnchorUpdateHandler? 57 | var onPlaneAnchorUpdate: PlaneAnchorUpdateHandler? 58 | var onMeshAnchorUpdate: MeshAnchorUpdateHandler? 59 | var onHandAnchorUpdate: HandAnchorUpdateHandler? 60 | var onEnvironmentLightAnchorUpdate: EnvironmentLightAnchorUpdateHandler? 61 | 62 | private var worldTrackingProvider: WorldTrackingProvider 63 | private var handTrackingProvider: HandTrackingProvider? 64 | 65 | init() { 66 | arSession = ARKitSession() 67 | worldTrackingProvider = WorldTrackingProvider() 68 | } 69 | 70 | func start(options: Options) async throws { 71 | var authorizationTypes = Set(WorldTrackingProvider.requiredAuthorizations) 72 | 73 | var providers: [DataProvider] = [worldTrackingProvider] 74 | 75 | if WorldTrackingProvider.isSupported { 76 | startMonitoringWorldAnchorUpdates(worldTrackingProvider) 77 | } 78 | if PlaneDetectionProvider.isSupported && options.contains(.planeDetection) { 79 | let provider = PlaneDetectionProvider(alignments: [.horizontal]) 80 | authorizationTypes.formUnion(PlaneDetectionProvider.requiredAuthorizations) 81 | providers.append(provider) 82 | startMonitoringPlaneAnchorUpdates(provider) 83 | } 84 | if HandTrackingProvider.isSupported && options.contains(.handTracking) { 85 | handTrackingProvider = HandTrackingProvider() 86 | authorizationTypes.formUnion(HandTrackingProvider.requiredAuthorizations) 87 | providers.append(handTrackingProvider!) 88 | startMonitoringHandTrackingUpdates(handTrackingProvider!) 89 | } 90 | if SceneReconstructionProvider.isSupported && options.contains(.sceneReconstruction) { 91 | let provider = SceneReconstructionProvider(modes: []) 92 | authorizationTypes.formUnion(SceneReconstructionProvider.requiredAuthorizations) 93 | providers.append(provider) 94 | startMonitoringSceneReconstructionUpdates(provider) 95 | } 96 | if EnvironmentLightEstimationProvider.isSupported && options.contains(.lightEstimation) { 97 | let provider = EnvironmentLightEstimationProvider() 98 | authorizationTypes.formUnion(EnvironmentLightEstimationProvider.requiredAuthorizations) 99 | providers.append(provider) 100 | startMonitoringLightingUpdates(provider) 101 | } 102 | 103 | #if !targetEnvironment(simulator) 104 | print("Will query authorization for: \(authorizationTypes)") 105 | let authorization = await arSession.queryAuthorization(for: Array(authorizationTypes)) 106 | // When running an AR session, the system automatically requests the authorization 107 | // types contained by the Info.plist, so we don't need to explicitly request anything. 108 | // However, we regard all required authorization types to be mandatory to proceed, 109 | // so it is an error for the user to have explicitly denied access. In an app 110 | // where world sensing or other authorization types are optional, we'd handle this 111 | // more gracefully. 112 | if authorization.values.contains(.denied) { 113 | fatalError("Required authorization has been denied. Aborting.") 114 | } 115 | #endif 116 | 117 | try await arSession.run(providers) 118 | } 119 | 120 | func stop() { 121 | arSession.stop() 122 | } 123 | 124 | func addWorldAnchor(originFromAnchorTransform: simd_float4x4) async throws -> WorldAnchor { 125 | let anchor = ARKit.WorldAnchor(originFromAnchorTransform: originFromAnchorTransform) 126 | try await worldTrackingProvider.addAnchor(anchor) 127 | return WorldAnchor(anchor) 128 | } 129 | 130 | func removeAllWorldAnchors() async throws { 131 | let anchors = await worldTrackingProvider.allAnchors ?? [] 132 | for i in 0.. ARKit.DeviceAnchor? { 138 | if worldTrackingProvider.state == .running { 139 | let anchor = worldTrackingProvider.queryDeviceAnchor(atTimestamp: timestamp) 140 | if anchor == nil { 141 | print("World tracking provider is running but failed to provide a device anchor at \(timestamp)") 142 | } 143 | return anchor 144 | } else { 145 | print("World tracking provider is not running; cannot retrieve device pose.") 146 | return nil 147 | } 148 | } 149 | 150 | func queryHandAnchors(at timestamp: TimeInterval) -> (ARKit.HandAnchor?, ARKit.HandAnchor?) { 151 | if let provider = handTrackingProvider, provider.state == .running { 152 | return provider.handAnchors(at: timestamp) 153 | } 154 | return (nil, nil) 155 | } 156 | 157 | private func startMonitoringWorldAnchorUpdates(_ provider: WorldTrackingProvider) { 158 | Task(priority: .high) { [weak self] in 159 | for await update in provider.anchorUpdates { 160 | self?.onWorldAnchorUpdate?(WorldAnchor(update.anchor), AnchorEvent(update.event)) 161 | } 162 | } 163 | } 164 | 165 | private func startMonitoringPlaneAnchorUpdates(_ provider: PlaneDetectionProvider) { 166 | Task(priority: .low) { [weak self] in 167 | for await update in provider.anchorUpdates { 168 | self?.onPlaneAnchorUpdate?(PlaneAnchor(update.anchor), AnchorEvent(update.event)) 169 | } 170 | } 171 | } 172 | 173 | private func startMonitoringSceneReconstructionUpdates(_ provider: SceneReconstructionProvider) { 174 | Task(priority: .low) { [weak self] in 175 | for await update in provider.anchorUpdates { 176 | self?.onMeshAnchorUpdate?(MeshAnchor(update.anchor), AnchorEvent(update.event)) 177 | } 178 | } 179 | } 180 | 181 | private func startMonitoringHandTrackingUpdates(_ provider: HandTrackingProvider) { 182 | Task(priority: .high) { [weak self] in 183 | for await update in provider.anchorUpdates { 184 | self?.onHandAnchorUpdate?(HandAnchor(update.anchor), AnchorEvent(update.event)) 185 | } 186 | } 187 | } 188 | 189 | private func startMonitoringLightingUpdates(_ provider: EnvironmentLightEstimationProvider) { 190 | Task(priority: .low) { [weak self] in 191 | for await update in provider.anchorUpdates { 192 | self?.onEnvironmentLightAnchorUpdate?(EnvironmentLightAnchor(update.anchor), AnchorEvent(update.event)) 193 | } 194 | } 195 | } 196 | } 197 | 198 | #endif 199 | -------------------------------------------------------------------------------- /Engine/Anchor.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import simd 19 | 20 | public enum AnchorEvent : Sendable { 21 | case added 22 | case updated 23 | case removed 24 | } 25 | 26 | public protocol Anchor : Identifiable, Sendable { 27 | var originFromAnchorTransform: simd_float4x4 { get } 28 | } 29 | 30 | public struct WorldAnchor : Anchor { 31 | public typealias ID = UUID 32 | 33 | public let id: UUID 34 | public let originFromAnchorTransform: simd_float4x4 35 | 36 | init (id: UUID, originFromAnchorTransform: simd_float4x4) { 37 | self.id = id 38 | self.originFromAnchorTransform = originFromAnchorTransform 39 | } 40 | } 41 | 42 | public struct PlaneAnchor : Anchor, @unchecked Sendable { 43 | public typealias ID = UUID 44 | 45 | public struct Extent { 46 | let width: Float 47 | let height: Float 48 | let anchorFromExtentTransform: simd_float4x4 49 | 50 | init(width: Float, height: Float, anchorFromExtentTransform: simd_float4x4) { 51 | self.width = width 52 | self.height = height 53 | self.anchorFromExtentTransform = anchorFromExtentTransform 54 | } 55 | } 56 | 57 | public enum Alignment { 58 | case horizontal 59 | case vertical 60 | case slanted 61 | } 62 | 63 | public enum Classification : Int { 64 | case unknown 65 | case wall 66 | case floor 67 | case ceiling 68 | case table 69 | case seat 70 | case window 71 | case door 72 | } 73 | 74 | public let id: UUID 75 | public let originFromAnchorTransform: simd_float4x4 76 | public let alignment: PlaneAnchor.Alignment 77 | public let classification: PlaneAnchor.Classification 78 | public let extent: PlaneAnchor.Extent 79 | public let mesh: Mesh 80 | 81 | init(id: UUID, 82 | originFromAnchorTransform: simd_float4x4, 83 | alignment: PlaneAnchor.Alignment, 84 | classification: PlaneAnchor.Classification, 85 | extent: PlaneAnchor.Extent, 86 | mesh: Mesh) 87 | { 88 | self.id = id 89 | self.originFromAnchorTransform = originFromAnchorTransform 90 | self.alignment = alignment 91 | self.classification = classification 92 | self.extent = extent 93 | self.mesh = mesh 94 | } 95 | } 96 | 97 | public struct MeshAnchor : Anchor, @unchecked Sendable { 98 | public typealias ID = UUID 99 | 100 | public let id: UUID 101 | public let originFromAnchorTransform: simd_float4x4 102 | public let mesh: Mesh 103 | 104 | init(id: UUID, originFromAnchorTransform: simd_float4x4, mesh: Mesh) { 105 | self.id = id 106 | self.originFromAnchorTransform = originFromAnchorTransform 107 | self.mesh = mesh 108 | } 109 | } 110 | 111 | public struct HandSkeleton : Sendable { 112 | public enum JointName : String, CaseIterable, Sendable { 113 | case wrist 114 | case thumbKnuckle 115 | case thumbIntermediateBase 116 | case thumbIntermediateTip 117 | case thumbTip 118 | case indexFingerMetacarpal 119 | case indexFingerKnuckle 120 | case indexFingerIntermediateBase 121 | case indexFingerIntermediateTip 122 | case indexFingerTip 123 | case middleFingerMetacarpal 124 | case middleFingerKnuckle 125 | case middleFingerIntermediateBase 126 | case middleFingerIntermediateTip 127 | case middleFingerTip 128 | case ringFingerMetacarpal 129 | case ringFingerKnuckle 130 | case ringFingerIntermediateBase 131 | case ringFingerIntermediateTip 132 | case ringFingerTip 133 | case littleFingerMetacarpal 134 | case littleFingerKnuckle 135 | case littleFingerIntermediateBase 136 | case littleFingerIntermediateTip 137 | case littleFingerTip 138 | case forearmWrist 139 | case forearmArm 140 | } 141 | 142 | public struct Joint : Sendable { 143 | let name: HandSkeleton.JointName 144 | let anchorFromJointTransform: simd_float4x4 145 | let estimatedLinearVelocity: simd_float3 146 | let isTracked: Bool 147 | } 148 | 149 | let joints: [HandSkeleton.Joint] 150 | 151 | init(joints: [HandSkeleton.Joint]) { 152 | self.joints = joints 153 | } 154 | 155 | func joint(named name: JointName) -> Joint? { 156 | joints.first(where: { $0.name == name }) 157 | } 158 | } 159 | 160 | extension HandSkeleton.JointName { 161 | init?(webXRJointName: String) { 162 | switch webXRJointName { 163 | case "wrist" : self = .wrist 164 | case "thumb_metacarpal" : self = .thumbKnuckle 165 | case "thumb_phalanx_proximal" : self = .thumbIntermediateBase 166 | case "thumb_phalanx_distal" : self = .thumbIntermediateTip 167 | case "thumb_tip" : self = .thumbTip 168 | case "index_finger_metacarpal" : self = .indexFingerMetacarpal 169 | case "index_finger_phalanx_proximal" : self = .indexFingerKnuckle 170 | case "index_finger_phalanx_intermediate" : self = .indexFingerIntermediateBase 171 | case "index_finger_phalanx_distal" : self = .indexFingerIntermediateTip 172 | case "index_finger_tip" : self = .indexFingerTip 173 | case "middle_finger_metacarpal" : self = .middleFingerMetacarpal 174 | case "middle_finger_phalanx_proximal" : self = .middleFingerKnuckle 175 | case "middle_finger_phalanx_intermediate" : self = .middleFingerIntermediateBase 176 | case "middle_finger_phalanx_distal" : self = .middleFingerIntermediateTip 177 | case "middle_finger_tip" : self = .middleFingerTip 178 | case "ring_finger_metacarpal" : self = .ringFingerMetacarpal 179 | case "ring_finger_phalanx_proximal" : self = .ringFingerKnuckle 180 | case "ring_finger_phalanx_intermediate" : self = .ringFingerIntermediateBase 181 | case "ring_finger_phalanx_distal" : self = .ringFingerIntermediateTip 182 | case "ring_finger_tip" : self = .ringFingerTip 183 | case "pinky_finger_metacarpal" : self = .littleFingerMetacarpal 184 | case "pinky_finger_phalanx_proximal" : self = .littleFingerKnuckle 185 | case "pinky_finger_phalanx_intermediate" : self = .littleFingerIntermediateBase 186 | case "pinky_finger_phalanx_distal" : self = .littleFingerIntermediateTip 187 | case "pinky_finger_tip" : self = .littleFingerTip 188 | default: 189 | return nil 190 | } 191 | } 192 | 193 | var webXRJointName: String? { 194 | switch self { 195 | case .wrist: return "wrist" 196 | case .thumbKnuckle: return "thumb_metacarpal" 197 | case .thumbIntermediateBase: return "thumb_phalanx_proximal" 198 | case .thumbIntermediateTip: return "thumb_phalanx_distal" 199 | case .thumbTip: return "thumb_tip" 200 | case .indexFingerMetacarpal: return "index_finger_metacarpal" 201 | case .indexFingerKnuckle: return "index_finger_phalanx_proximal" 202 | case .indexFingerIntermediateBase: return "index_finger_phalanx_intermediate" 203 | case .indexFingerIntermediateTip: return "index_finger_phalanx_distal" 204 | case .indexFingerTip: return "index_finger_tip" 205 | case .middleFingerMetacarpal: return "middle_finger_metacarpal" 206 | case .middleFingerKnuckle: return "middle_finger_phalanx_proximal" 207 | case .middleFingerIntermediateBase: return "middle_finger_phalanx_intermediate" 208 | case .middleFingerIntermediateTip: return "middle_finger_phalanx_distal" 209 | case .middleFingerTip: return "middle_finger_tip" 210 | case .ringFingerMetacarpal: return "ring_finger_metacarpal" 211 | case .ringFingerKnuckle: return "ring_finger_phalanx_proximal" 212 | case .ringFingerIntermediateBase: return "ring_finger_phalanx_intermediate" 213 | case .ringFingerIntermediateTip: return "ring_finger_phalanx_distal" 214 | case .ringFingerTip: return "ring_finger_tip" 215 | case .littleFingerMetacarpal: return "pinky_finger_metacarpal" 216 | case .littleFingerKnuckle: return "pinky_finger_phalanx_proximal" 217 | case .littleFingerIntermediateBase: return "pinky_finger_phalanx_intermediate" 218 | case .littleFingerIntermediateTip: return "pinky_finger_phalanx_distal" 219 | case .littleFingerTip: return "pinky_finger_tip" 220 | default: return nil 221 | } 222 | } 223 | } 224 | 225 | public enum Handedness : Sendable { 226 | case left 227 | case right 228 | case none 229 | } 230 | 231 | public struct HandAnchor : Anchor { 232 | public typealias ID = UUID 233 | 234 | public let id: UUID 235 | public let originFromAnchorTransform: simd_float4x4 236 | public let estimatedLinearVelocity: simd_float3 237 | public let handSkeleton: HandSkeleton? 238 | public let handedness: Handedness 239 | public let isTracked: Bool 240 | 241 | init(id: UUID, 242 | originFromAnchorTransform: simd_float4x4, 243 | estimatedLinearVelocity: simd_float3, 244 | handSkeleton: HandSkeleton?, 245 | handedness: Handedness, 246 | isTracked: Bool) 247 | { 248 | self.id = id 249 | self.originFromAnchorTransform = originFromAnchorTransform 250 | self.estimatedLinearVelocity = estimatedLinearVelocity 251 | self.handSkeleton = handSkeleton 252 | self.handedness = handedness 253 | self.isTracked = isTracked 254 | } 255 | } 256 | 257 | public struct EnvironmentLightAnchor : Anchor { 258 | public typealias ID = UUID 259 | 260 | public let id: UUID 261 | 262 | public let originFromAnchorTransform: simd_float4x4 263 | public let light: EnvironmentLight 264 | 265 | init(id: UUID, originFromAnchorTransform: simd_float4x4, light: EnvironmentLight) { 266 | self.id = id 267 | self.originFromAnchorTransform = originFromAnchorTransform 268 | self.light = light 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /Engine/Bridging.h: -------------------------------------------------------------------------------- 1 | #include "JoltPhysics.h" 2 | #include "ShaderTypes.h" 3 | -------------------------------------------------------------------------------- /Engine/BufferUtilities.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Metal 18 | import ModelIO 19 | 20 | // A buffer view is a simple abstraction over a Metal buffer that 21 | // holds a strong reference to its buffer and an optional offset. 22 | class BufferView { 23 | let buffer: MTLBuffer 24 | let offset: Int 25 | 26 | init(buffer: MTLBuffer, offset: Int = 0) { 27 | self.buffer = buffer 28 | self.offset = offset 29 | } 30 | } 31 | 32 | // A ring buffer is a fixed-size buffer that can be used for small 33 | // temporary allocations. When the allocation offset of the buffer 34 | // exceeds its size, it "wraps" around to zero. Allocating an object 35 | // larger than the buffer itself is an error. Since this type does 36 | // not do any lifetime management, any objects copied into it should 37 | // be POD. 38 | class RingBuffer { 39 | let buffer: MTLBuffer 40 | let bufferLength: Int 41 | let minimumAlignment: Int 42 | 43 | private var nextOffset = 0 44 | 45 | init(device: MTLDevice, 46 | length: Int, 47 | options: MTLResourceOptions = .storageModeShared, 48 | label: String? = nil) throws 49 | { 50 | guard let buffer = device.makeBuffer(length: length, options: options) else { 51 | throw ResourceError.allocationFailure 52 | } 53 | buffer.label = label 54 | self.buffer = buffer 55 | self.bufferLength = length 56 | self.minimumAlignment = device.minimumConstantBufferOffsetAlignment 57 | } 58 | 59 | func alloc(length: Int, alignment: Int) -> (UnsafeMutableRawPointer, Int) { 60 | precondition(length <= bufferLength) 61 | let effectiveAlignment = max(minimumAlignment, alignment) 62 | var offset = alignUp(nextOffset, alignment: effectiveAlignment) 63 | if offset + length >= bufferLength { 64 | offset = 0 65 | } 66 | nextOffset = offset + length 67 | return (buffer.contents().advanced(by: offset), offset) 68 | } 69 | 70 | func copy(_ value: UnsafeMutablePointer) -> Int { 71 | precondition(_isPOD(T.self)) 72 | let layout = MemoryLayout.self 73 | let (ptr, offset) = alloc(length: layout.size, alignment: layout.alignment) 74 | ptr.copyMemory(from: value, byteCount: layout.size) 75 | return offset 76 | } 77 | 78 | func copy(_ array: [T]) -> Int { 79 | precondition(_isPOD(T.self)) 80 | if array.count == 0 { return 0 } 81 | let layout = MemoryLayout.self 82 | let regionLength = ((array.count - 1) * layout.stride) + layout.size 83 | let (ptr, offset) = alloc(length: regionLength, alignment: layout.alignment) 84 | array.withUnsafeBytes { elementsPtr in 85 | guard let elementPtr = elementsPtr.baseAddress else { return } 86 | ptr.copyMemory(from: elementPtr, byteCount: regionLength) 87 | } 88 | return offset 89 | } 90 | } 91 | 92 | // A strided view is a sequence type that affords random access to an underlying 93 | // data buffer containing homogeneous elements that may be strided (i.e. not contiguous 94 | // in memory). It is the responsibility of a view's creator to ensure that the 95 | // underlying buffer lives at least as long as any views created on it. 96 | struct StridedView : Sequence { 97 | struct Iterator : Swift.IteratorProtocol { 98 | private let storage: StridedView 99 | private var currentIndex = 0 100 | 101 | init(_ storage: StridedView) { 102 | self.storage = storage 103 | } 104 | 105 | mutating func next() -> Element? { 106 | if currentIndex < storage.count { 107 | let index = currentIndex 108 | currentIndex += 1 109 | return storage[index] 110 | } else { 111 | return nil 112 | } 113 | } 114 | } 115 | 116 | let basePointer: UnsafeRawPointer 117 | let offset: Int 118 | let stride: Int 119 | let count: Int 120 | 121 | init(_ basePointer: UnsafeRawPointer, offset: Int, stride: Int, count: Int) { 122 | precondition(_isPOD(Element.self)) 123 | self.basePointer = basePointer 124 | self.offset = offset 125 | self.stride = stride 126 | self.count = count 127 | } 128 | 129 | var underestimatedCount: Int { 130 | return count 131 | } 132 | 133 | func makeIterator() -> Iterator { 134 | return Iterator(self) 135 | } 136 | 137 | func withContiguousStorageIfAvailable(_ body: (UnsafeBufferPointer) throws -> R) rethrows -> R? { 138 | if stride == MemoryLayout.stride { 139 | let elementPtr = basePointer.advanced(by: offset).assumingMemoryBound(to: Element.self) 140 | let elementBufferPtr = UnsafeBufferPointer(start: elementPtr, count: count) 141 | return try body(elementBufferPtr) 142 | } 143 | return nil 144 | } 145 | 146 | subscript(index: Int) -> Element { 147 | get { 148 | precondition(index >= 0 && index < count) 149 | let elementPtr = basePointer.advanced(by: offset + stride * index).assumingMemoryBound(to: Element.self) 150 | return elementPtr.pointee 151 | } 152 | } 153 | } 154 | 155 | // A small utility for determining the minimum alignment of offsets 156 | // of buffers bound with a command encoder. 157 | extension MTLDevice { 158 | var minimumConstantBufferOffsetAlignment: Int { 159 | #if targetEnvironment(simulator) 160 | let isSimulator = true 161 | #else 162 | let isSimulator = false 163 | #endif 164 | 165 | if supportsFamily(.apple2) && !isSimulator { 166 | // A8 and later (and Apple Silicon) support 4 byte alignment 167 | return 4 168 | } else if supportsFamily(.mac2) && !isSimulator { 169 | // Recent Macs support 32 byte alignment 170 | return 32 171 | } else { 172 | // Worst-case scenario for simulator, old Nvidia Macs, etc. 173 | return 256 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /Engine/Camera.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import CoreGraphics 19 | import simd 20 | 21 | protocol Camera { 22 | var transform: Transform { get } 23 | var viewMatrix: simd_float4x4 { get } 24 | func projectionMatrix(for viewportSize: CGSize) -> simd_float4x4 25 | } 26 | 27 | class OrthographicCamera : Camera { 28 | var near: Float = 0.0 29 | var far: Float = 1.0 30 | 31 | var transform = Transform() 32 | 33 | var viewMatrix: simd_float4x4 { 34 | return transform.matrix.inverse 35 | } 36 | 37 | func projectionMatrix(for viewportSize: CGSize) -> simd_float4x4 { 38 | let width = Float(viewportSize.width), height = Float(viewportSize.height) 39 | return simd_float4x4(orthographicProjectionLeft: 0.0, top: 0.0, 40 | right: width, bottom: height, 41 | near: near, far: far) 42 | } 43 | } 44 | 45 | // A camera that produces a perspective projection based on a field-of-view and near clipping plane distance 46 | // The projection matrix produced by this camera assumes an infinitely distant far viewing distance and reverse-Z 47 | // (i.e. clip space depth runs from 1 to 0, near to far) 48 | class PerspectiveCamera : Camera { 49 | // The total vertical field of view angle of the camera 50 | var fieldOfView: Angle = .degrees(60) 51 | 52 | // The distance to the camera's near viewing plane 53 | var near: Float = 0.005 54 | 55 | var transform = Transform() 56 | 57 | var viewMatrix: simd_float4x4 { 58 | return transform.matrix.inverse 59 | } 60 | 61 | func projectionMatrix(for viewportSize: CGSize) -> simd_float4x4 { 62 | let width = Float(viewportSize.width), height = Float(viewportSize.height) 63 | let aspectRatio = width / height 64 | return simd_float4x4(perspectiveProjectionFOV: Float(fieldOfView.radians), 65 | aspectRatio: aspectRatio, 66 | near: near) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /Engine/Entity.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | 19 | // An entity is a named object that participates in a transform hierarchy 20 | // and can have an associated mesh and/or physics body. 21 | public class Entity { 22 | var name: String = "" 23 | var modelTransform: Transform 24 | public private(set) var parent: Entity? 25 | public private(set) var children: [Entity] = [] 26 | var mesh: Mesh? 27 | var isHidden = false 28 | 29 | var worldTransform: Transform { 30 | get { 31 | if let parent { 32 | return parent.worldTransform * modelTransform 33 | } else { 34 | return modelTransform 35 | } 36 | } 37 | set { 38 | if let parent { 39 | let ancestorTransform = parent.worldTransform 40 | let newTransform = ancestorTransform.inverse * newValue 41 | modelTransform = newTransform 42 | } else { 43 | modelTransform = newValue 44 | } 45 | } 46 | } 47 | 48 | var skinner: Skinner? 49 | 50 | var physicsBody: PhysicsBody? 51 | 52 | init(mesh: Mesh? = nil, transform: Transform = Transform()) { 53 | self.mesh = mesh 54 | self.modelTransform = transform 55 | } 56 | 57 | func addChild(_ child: Entity) { 58 | child.removeFromParent() 59 | child.parent = self 60 | children.append(child) 61 | parent?.didAddChild(child) 62 | } 63 | 64 | func removeFromParent() { 65 | parent?.removeChild(self) 66 | parent = nil 67 | } 68 | 69 | private func removeChild(_ child: Entity) { 70 | children.removeAll { 71 | $0 == child 72 | } 73 | parent?.didRemoveChild(child) 74 | } 75 | 76 | // Creates a recursive "clone" of this entity and its children. 77 | // n.b. that this method "slices" subclasses of Entity, so only 78 | // use it to clone hierarchies of instances of this base class. 79 | // Also note that physics bodies and shapes are not cloned. 80 | func clone(recursively: Bool) -> Entity { 81 | let entity = Entity(mesh: mesh, transform: modelTransform) 82 | entity.isHidden = isHidden 83 | entity.skinner = skinner 84 | for child in children { 85 | entity.addChild(child.clone(recursively: recursively)) 86 | } 87 | return entity 88 | } 89 | 90 | func update(_ timestep: TimeInterval) {} 91 | 92 | func didAddChild(_ entity: Entity) { 93 | parent?.didAddChild(entity) 94 | } 95 | 96 | func didRemoveChild(_ entity: Entity) { 97 | parent?.didRemoveChild(entity) 98 | } 99 | 100 | func generateCollisionShapes(recursive: Bool, bodyMode: PhysicsBodyMode) { 101 | if let mesh { 102 | let shape = PhysicsShape(convexHullFromMesh: mesh) 103 | let body = PhysicsBody(mode: bodyMode, shape: shape) 104 | physicsBody = body 105 | } 106 | if recursive { 107 | for child in children { 108 | child.generateCollisionShapes(recursive: recursive, bodyMode: bodyMode) 109 | } 110 | } 111 | } 112 | } 113 | 114 | extension Entity : Equatable { 115 | public static func == (lhs: Entity, rhs: Entity) -> Bool { 116 | return lhs === rhs 117 | } 118 | } 119 | 120 | extension Entity : Hashable { 121 | public func hash(into hasher: inout Hasher) { 122 | hasher.combine(ObjectIdentifier(self)) 123 | } 124 | } 125 | 126 | extension Entity { 127 | func visitBreadthFirst(_ root: Entity, _ body: (Entity) -> Void) { 128 | var queue = [root] 129 | while !queue.isEmpty { 130 | let current = queue.removeFirst() 131 | body(current) 132 | for child in current.children { 133 | queue.append(child) 134 | } 135 | } 136 | } 137 | 138 | func flattenedHierarchy(_ root: Entity) -> [Entity] { 139 | var entities: [Entity] = [] 140 | visitBreadthFirst(root) { entities.append($0) } 141 | return entities 142 | } 143 | 144 | func child(named name: String, recursive: Bool = true) -> Entity? { 145 | if let immediateMatch = children.first(where: { $0.name == name }) { 146 | return immediateMatch 147 | } else if recursive { 148 | for child in children { 149 | if let match = child.child(named: name, recursive: recursive) { 150 | return match 151 | } 152 | } 153 | } 154 | return nil 155 | } 156 | 157 | func childEntities(matching predicate: (Entity) -> Bool) -> [Entity] { 158 | var matches: [Entity] = [] 159 | visitBreadthFirst(self) { if predicate($0) && $0 !== self { matches.append($0) } } 160 | return matches 161 | } 162 | } 163 | 164 | class AnchorEntity : Entity { 165 | override init(mesh: Mesh? = nil, transform: Transform = Transform()) { 166 | super.init(mesh: mesh, transform: transform) 167 | } 168 | 169 | func updatePose(from anchor: WorldAnchor) { 170 | worldTransform = Transform(anchor.originFromAnchorTransform) 171 | } 172 | } 173 | 174 | class HandEntity: Entity { 175 | let handedness: Handedness 176 | private let meshEntityName = "Mesh" 177 | private let jointConformationTransform: simd_float4x4 178 | private let colliderEntity = Entity() 179 | 180 | init(handedness: Handedness, mesh: Mesh? = nil, transform: Transform = Transform(), context: MetalContext) { 181 | self.handedness = handedness 182 | if handedness == .right { 183 | self.jointConformationTransform = simd_float4x4(rotationAbout: simd_float3(0, 1, 0), by: .pi / 2) 184 | } else { 185 | self.jointConformationTransform = simd_float4x4(rotationAbout: simd_normalize(simd_float3(1, 0, -1)), by: .pi) 186 | } 187 | super.init(mesh: mesh, transform: transform) 188 | 189 | let colliderRadius: Float = 0.0075 190 | let colliderColor = simd_float4(handedness == .right ? 0.8 : 0, handedness == .left ? 0.8 : 0, 0, 1) 191 | let colliderMesh = Mesh.generateSphere(radius: colliderRadius, context: context) 192 | colliderMesh.materials = [PhysicallyBasedMaterial(baseColor: colliderColor, roughness: 0.5, isMetal: false)] 193 | colliderEntity.mesh = colliderMesh 194 | let colliderBody = PhysicsBody(mode: .kinematic, 195 | shape: PhysicsShape(sphereWithRadius: colliderRadius), mass: 0.5) 196 | colliderEntity.physicsBody = colliderBody 197 | addChild(colliderEntity) 198 | } 199 | 200 | func updatePose(from anchor: HandAnchor) { 201 | guard let meshEntity = child(named: meshEntityName, recursive: true) else { return } 202 | guard let handSkeleton = anchor.handSkeleton else { return } 203 | guard let handSkinner = meshEntity.skinner else { return } 204 | var jointPoses = handSkinner.skeleton.restTransforms 205 | for (destinationIndex, jointPath) in handSkinner.skeleton.jointPaths.enumerated() { 206 | if let jointName = HandSkeleton.JointName(webXRJointName: jointPath), 207 | let sourceJoint = handSkeleton.joint(named: jointName) 208 | { 209 | jointPoses[destinationIndex] = sourceJoint.anchorFromJointTransform * jointConformationTransform 210 | } 211 | } 212 | handSkinner.jointTransforms = jointPoses 213 | if anchor.isTracked { 214 | worldTransform = Transform(anchor.originFromAnchorTransform) 215 | 216 | if let indexTipJoint = handSkeleton.joint(named: .indexFingerTip) { 217 | colliderEntity.isHidden = !indexTipJoint.isTracked 218 | if indexTipJoint.isTracked { 219 | colliderEntity.modelTransform = Transform(indexTipJoint.anchorFromJointTransform) 220 | } 221 | } 222 | } else { 223 | isHidden = true 224 | } 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /Engine/Light.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import simd 18 | import Metal 19 | 20 | public class Light { 21 | enum LightType : Int { 22 | case directional = 0 23 | case point = 1 24 | case spot = 2 25 | } 26 | 27 | var type = LightType.directional 28 | var transform = Transform() 29 | var color = simd_float3(1, 1, 1) 30 | var intensity: Float = 1 31 | var range: Float = 0 32 | var innerConeAngle = Angle.degrees(90) 33 | var outerConeAngle = Angle.degrees(90) 34 | 35 | init(type: LightType = LightType.directional, 36 | color: simd_float3 = simd_float3(1, 1, 1), 37 | intensity: Float = 1) 38 | { 39 | self.type = type 40 | self.color = color 41 | self.intensity = intensity 42 | } 43 | } 44 | 45 | public struct EnvironmentLight : @unchecked Sendable { 46 | let cubeFromWorldTransform: Transform 47 | let environmentTexture: MTLTexture? 48 | let scaleFactor: Float 49 | 50 | init(cubeFromWorldTransform: Transform, environmentTexture: MTLTexture?, scaleFactor: Float) { 51 | self.cubeFromWorldTransform = cubeFromWorldTransform 52 | self.environmentTexture = environmentTexture 53 | self.scaleFactor = scaleFactor 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Engine/LockingQueue.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Dispatch 3 | 4 | // A relatively inefficient queue type for low-contention multithreaded operation. 5 | // All peek/pop operations block the calling thread until all previous writes have 6 | // been resolved. Concurrent enqueues are resolved in the order of insertion. 7 | class LockingQueue : @unchecked Sendable { 8 | private let queue = DispatchQueue(label: "concurrent-read-write-queue", attributes: .concurrent) 9 | 10 | private var _elements: [Element] = [] 11 | 12 | func enqueue(_ element: Element) { 13 | queue.async(flags: .barrier) { 14 | self._elements.append(element) 15 | } 16 | } 17 | 18 | func enqueue(_ elements: [Element]) { 19 | queue.async(flags: .barrier) { 20 | self._elements.append(contentsOf: elements) 21 | } 22 | } 23 | 24 | func peek() -> Element? { 25 | return queue.sync { 26 | return _elements.first 27 | } 28 | } 29 | 30 | func pop() -> Element? { 31 | return queue.sync { 32 | return _elements.removeFirst() 33 | } 34 | } 35 | 36 | func popAll() -> [Element] { 37 | return queue.sync { 38 | let result = _elements 39 | _elements.removeAll() 40 | return result 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Engine/Material.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Metal 18 | import ModelIO 19 | 20 | enum BlendMode { 21 | case opaque 22 | case sourceOverPremultiplied 23 | } 24 | 25 | class Material : Identifiable { 26 | var name: String = "" 27 | var blendMode: BlendMode = .opaque 28 | var fillMode: MTLTriangleFillMode = .fill 29 | var isDoubleSided: Bool = false 30 | var writesDepthBuffer: Bool = true 31 | var readsDepthBuffer: Bool = true 32 | var colorMask: MTLColorWriteMask = [.red, .green, .blue, .alpha] 33 | 34 | var vertexFunctionName: String { 35 | fatalError("Material subclasses must implement vertexFunctionName") 36 | } 37 | 38 | var fragmentFunctionName: String { 39 | fatalError("Material subclasses must implement fragmentFunctionName") 40 | } 41 | 42 | var relativeSortOrder: Int { 43 | 0 44 | } 45 | 46 | func bindResources(constantBuffer: RingBuffer, renderCommandEncoder: MTLRenderCommandEncoder) {} 47 | } 48 | 49 | class OcclusionMaterial : Material { 50 | override init() { 51 | super.init() 52 | colorMask = [] 53 | } 54 | 55 | override var vertexFunctionName: String { 56 | "vertex_main" 57 | } 58 | 59 | override var fragmentFunctionName: String { 60 | "fragment_occlusion" 61 | } 62 | override var relativeSortOrder: Int { 63 | -1 // Occluders render first because they lay down the depth of real-world objects 64 | } 65 | } 66 | 67 | class PhysicallyBasedMaterial : Material { 68 | static var `default` : PhysicallyBasedMaterial { 69 | var material = PhysicallyBasedMaterial() 70 | material.name = "Default" 71 | material.baseColor.baseColorFactor = simd_float4(0.5, 0.5, 0.5, 1.0) 72 | material.roughness.roughnessFactor = 0.5 73 | material.metalness.metalnessFactor = 0.0 74 | return material 75 | } 76 | 77 | struct BaseColor { 78 | var baseColorTexture: TextureResource? 79 | var baseColorFactor: simd_float4 = simd_float4(1, 1, 1, 1) 80 | } 81 | 82 | struct Normal { 83 | var normalTexture: TextureResource? 84 | var normalStrength: Float = 1.0 85 | } 86 | 87 | struct Metalness { 88 | var metalnessTexture: TextureResource? 89 | var metalnessFactor: Float = 1.0 90 | } 91 | 92 | struct Roughness { 93 | var roughnessTexture: TextureResource? 94 | var roughnessFactor: Float = 1.0 95 | } 96 | 97 | struct Emissive { 98 | var emissiveTexture: TextureResource? 99 | var emissiveColor: simd_float3 = simd_float3(0, 0, 0) 100 | var emissiveStrength: Float = 1.0 101 | } 102 | 103 | var baseColor = BaseColor() 104 | var normal = Normal() 105 | var metalness = Metalness() 106 | var roughness = Roughness() 107 | var emissive = Emissive() 108 | 109 | override var vertexFunctionName: String { 110 | "vertex_main" 111 | } 112 | 113 | override var fragmentFunctionName: String { 114 | "fragment_pbr" 115 | } 116 | 117 | override var relativeSortOrder: Int { 118 | if colorMask.isEmpty { 119 | -1 // Occluders render first because they lay down the depth of real-world objects 120 | } else if blendMode != .opaque { 121 | 1 // Transparents render last so they can read the depth buffer without affecting it 122 | } else { 123 | 0 124 | } 125 | } 126 | 127 | private var defaultSampler: MTLSamplerState? 128 | 129 | override init() { 130 | } 131 | 132 | init(baseColor: simd_float4, roughness: Float, isMetal: Bool) { 133 | self.baseColor.baseColorFactor = baseColor 134 | self.roughness.roughnessFactor = roughness 135 | self.metalness.metalnessFactor = isMetal ? 1.0 : 0.0 136 | } 137 | 138 | override func bindResources(constantBuffer: RingBuffer, renderCommandEncoder: MTLRenderCommandEncoder) { 139 | if defaultSampler == nil { 140 | let samplerDescriptor = MTLSamplerDescriptor() 141 | samplerDescriptor.minFilter = .linear 142 | samplerDescriptor.magFilter = .linear 143 | samplerDescriptor.mipFilter = .linear 144 | samplerDescriptor.sAddressMode = .repeat 145 | samplerDescriptor.tAddressMode = .repeat 146 | defaultSampler = renderCommandEncoder.device.makeSamplerState(descriptor: samplerDescriptor) 147 | } 148 | 149 | var material = PBRMaterialConstants(baseColorFactor: baseColor.baseColorFactor, 150 | emissiveColor: emissive.emissiveColor, 151 | normalScale: normal.normalStrength, 152 | metallicFactor: metalness.metalnessFactor, 153 | roughnessFactor: roughness.roughnessFactor, 154 | emissiveStrength: emissive.emissiveStrength) 155 | let materialOffset = constantBuffer.copy(&material) 156 | renderCommandEncoder.setFragmentBuffer(constantBuffer.buffer, 157 | offset: materialOffset, 158 | index: Int(FragmentBufferMaterialConstants)) 159 | 160 | if let baseColorTexture = baseColor.baseColorTexture { 161 | renderCommandEncoder.setFragmentTexture(baseColorTexture.texture, index: Int(FragmentTextureBaseColor)) 162 | renderCommandEncoder.setFragmentSamplerState(baseColorTexture.sampler ?? defaultSampler, index: Int(FragmentTextureBaseColor)) 163 | } else { 164 | renderCommandEncoder.setFragmentTexture(nil, index: Int(FragmentTextureBaseColor)) 165 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureBaseColor)) 166 | } 167 | 168 | if let normalTexture = normal.normalTexture { 169 | renderCommandEncoder.setFragmentTexture(normalTexture.texture, index: Int(FragmentTextureNormal)) 170 | renderCommandEncoder.setFragmentSamplerState(normalTexture.sampler ?? defaultSampler, index: Int(FragmentTextureNormal)) 171 | } else { 172 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureNormal)) 173 | } 174 | 175 | if let metalnessTexture = metalness.metalnessTexture { 176 | renderCommandEncoder.setFragmentTexture(metalnessTexture.texture, index: Int(FragmentTextureMetalness)) 177 | renderCommandEncoder.setFragmentSamplerState(metalnessTexture.sampler ?? defaultSampler, index: Int(FragmentTextureMetalness)) 178 | } else { 179 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureMetalness)) 180 | } 181 | 182 | if let roughnessTexture = roughness.roughnessTexture { 183 | renderCommandEncoder.setFragmentTexture(roughnessTexture.texture, index: Int(FragmentTextureRoughness)) 184 | renderCommandEncoder.setFragmentSamplerState(roughnessTexture.sampler ?? defaultSampler, index: Int(FragmentTextureRoughness)) 185 | } else { 186 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureRoughness)) 187 | } 188 | 189 | if let emissiveTexture = emissive.emissiveTexture { 190 | renderCommandEncoder.setFragmentTexture(emissiveTexture.texture, index: Int(FragmentTextureEmissive)) 191 | renderCommandEncoder.setFragmentSamplerState(emissiveTexture.sampler ?? defaultSampler, index: Int(FragmentTextureEmissive)) 192 | } else { 193 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureEmissive)) 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /Engine/Materials.metal: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #include 18 | using namespace metal; 19 | 20 | #include "ShaderTypes.h" 21 | 22 | constant bool writesRenderTargetSlice [[function_constant(0)]]; 23 | 24 | struct MeshVertexIn { 25 | float3 position [[attribute(0)]]; 26 | float3 normal [[attribute(1)]]; 27 | float2 texCoords [[attribute(2)]]; 28 | }; 29 | 30 | struct MeshVertexOut { 31 | float4 clipPosition [[position]]; 32 | float3 position; // world-space position 33 | float3 normal; 34 | float2 texCoords; 35 | uint renderTargetSlice [[render_target_array_index, function_constant(writesRenderTargetSlice)]]; 36 | uint viewIndex [[viewport_array_index]]; 37 | }; 38 | 39 | struct PBRFragmentIn { 40 | float4 clipPosition [[position]]; 41 | float3 position; 42 | float3 normal; 43 | float2 texCoords; 44 | uint viewIndex [[viewport_array_index]]; 45 | bool frontFacing [[front_facing]]; 46 | }; 47 | 48 | struct Light { 49 | float3 direction; 50 | float3 position; 51 | float3 color; 52 | float range; 53 | float intensity; 54 | float innerConeCos; 55 | float outerConeCos; 56 | unsigned int type; 57 | 58 | float getDistanceAttenuation(float range, float distance) const { 59 | float recipDistanceSq = 1.0f / (distance * distance); 60 | if (range <= 0.0f) { 61 | // negative range means unlimited, so use unmodified inverse-square law 62 | return recipDistanceSq; 63 | } 64 | return max(min(1.0f - powr(distance / range, 4.0f), 1.0f), 0.0f) * recipDistanceSq; 65 | } 66 | 67 | float getSpotAttenuation(float3 pointToLight, float3 spotDirection) const 68 | { 69 | float actualCos = dot(normalize(spotDirection), normalize(-pointToLight)); 70 | if (actualCos > outerConeCos) { 71 | if (actualCos < innerConeCos) { 72 | return smoothstep(outerConeCos, innerConeCos, actualCos); 73 | } 74 | return 1.0f; 75 | } 76 | return 0.0f; 77 | } 78 | 79 | half3 getIntensity(float3 pointToLight) const { 80 | float rangeAttenuation = 1.0; 81 | float spotAttenuation = 1.0; 82 | 83 | if (type != LightTypeDirectional) { 84 | rangeAttenuation = getDistanceAttenuation(range, length(pointToLight)); 85 | } 86 | if (type == LightTypeSpot) { 87 | spotAttenuation = getSpotAttenuation(pointToLight, direction); 88 | } 89 | 90 | return rangeAttenuation * spotAttenuation * half3(intensity * color); 91 | } 92 | }; 93 | 94 | struct TangentSpace { 95 | float3 T; // Geometric tangent 96 | float3 B; // Geometric bitangent 97 | float3 Ng; // Geometric normal 98 | float3 N; // Shading normal (TBN() * Nt) 99 | 100 | float3x3 TBN() const { 101 | return { T, B, Ng }; 102 | } 103 | }; 104 | 105 | static TangentSpace getTangentSpace(PBRFragmentIn v, constant PBRMaterialConstants &material, 106 | texture2d normalTexture, sampler normalSampler) 107 | { 108 | float2 uv = v.texCoords; 109 | float3 uv_dx = dfdx(float3(uv, 0.0f)); 110 | float3 uv_dy = dfdy(float3(uv, 0.0f)); 111 | 112 | if (length(uv_dx) + length(uv_dy) <= 1e-6f) { 113 | uv_dx = float3(1.0f, 0.0f, 0.0f); 114 | uv_dy = float3(0.0f, 1.0f, 0.0f); 115 | } 116 | 117 | float3 Tapprox = (uv_dy.y * dfdx(v.position.xyz) - uv_dx.y * dfdy(v.position.xyz)) / 118 | (uv_dx.x * uv_dy.y - uv_dy.x * uv_dx.y); 119 | 120 | float3 t, b, ng; 121 | 122 | ng = normalize(v.normal.xyz); 123 | t = normalize(Tapprox - ng * dot(ng, Tapprox)); 124 | b = cross(ng, t); 125 | 126 | if (!v.frontFacing) { 127 | t *= -1.0f; 128 | b *= -1.0f; 129 | ng *= -1.0f; 130 | } 131 | 132 | // Apply normal map if available 133 | TangentSpace basis; 134 | basis.Ng = ng; 135 | if (!is_null_texture(normalTexture)) { 136 | float3 Nt = normalTexture.sample(normalSampler, uv).rgb * 2.0f - 1.0f; 137 | Nt *= float3(material.normalScale, material.normalScale, 1.0); 138 | Nt = normalize(Nt); 139 | basis.N = normalize(float3x3(t, b, ng) * Nt); 140 | } else { 141 | basis.N = ng; 142 | } 143 | 144 | basis.T = t; 145 | basis.B = b; 146 | return basis; 147 | } 148 | 149 | static half3 F_Schlick(half3 f0, half3 f90, half VdotH) { 150 | return f0 + (f90 - f0) * powr(saturate(1.h - VdotH), 5.0h); 151 | } 152 | 153 | static half V_GGX(half NdotL, half NdotV, half alphaRoughness) 154 | { 155 | half alphaRoughnessSq = alphaRoughness * alphaRoughness; 156 | half GGXV = NdotL * sqrt(NdotV * NdotV * (1.0h - alphaRoughnessSq) + alphaRoughnessSq); 157 | half GGXL = NdotV * sqrt(NdotL * NdotL * (1.0h - alphaRoughnessSq) + alphaRoughnessSq); 158 | half GGX = GGXV + GGXL; 159 | if (GGX > 0.0) { 160 | return 0.5 / GGX; 161 | } 162 | return 0.0; 163 | } 164 | 165 | static float D_GGX(float NdotH, float alphaRoughness) { 166 | float alphaRoughnessSq = alphaRoughness * alphaRoughness; 167 | float f = (NdotH * NdotH) * (alphaRoughnessSq - 1.0f) + 1.0f; 168 | return alphaRoughnessSq / (M_PI_F * f * f); 169 | } 170 | 171 | static half3 BRDF_lambertian(half3 f0, half3 f90, half3 diffuseColor, float VdotH = 1.0f) { 172 | // see https://seblagarde.wordpress.com/2012/01/08/pi-or-not-to-pi-in-game-lighting-equation/ 173 | return diffuseColor / M_PI_F; 174 | } 175 | 176 | static half3 BRDF_specularGGX(half3 f0, half3 f90, half alphaRoughness, 177 | half VdotH, half NdotL, half NdotV, half NdotH) 178 | { 179 | half3 F = F_Schlick(f0, f90, VdotH); 180 | half V = V_GGX(NdotL, NdotV, alphaRoughness); 181 | half D = D_GGX(NdotH, alphaRoughness); 182 | return F * V * D; 183 | } 184 | 185 | vertex MeshVertexOut vertex_main(MeshVertexIn in [[stage_in]], 186 | constant PassConstants &frame [[buffer(VertexBufferPassConstants)]], 187 | constant InstanceConstants *instances [[buffer(VertexBufferInstanceConstants)]], 188 | uint instanceID [[instance_id]], 189 | uint amplificationID [[amplification_id]], 190 | uint amplificationCount [[amplification_count]]) 191 | { 192 | uint viewIndex = amplificationID; 193 | constant auto& instance = instances[instanceID]; 194 | 195 | float4 worldPosition = instance.modelMatrix * float4(in.position, 1.0f); 196 | float4x4 viewProjectionMatrix = frame.projectionMatrices[viewIndex] * frame.viewMatrices[viewIndex]; 197 | 198 | MeshVertexOut out; 199 | out.position = worldPosition.xyz; 200 | out.normal = normalize(instance.normalMatrix * in.normal); 201 | out.texCoords = in.texCoords; 202 | out.clipPosition = viewProjectionMatrix * worldPosition; 203 | out.viewIndex = viewIndex; 204 | 205 | if (amplificationCount > 1) { 206 | if (writesRenderTargetSlice) { 207 | out.renderTargetSlice = viewIndex; 208 | } 209 | } 210 | 211 | return out; 212 | } 213 | 214 | half3 EnvDFGPolynomial_Knarkowicz(half3 specularColor, float gloss, float NdotV) { 215 | half x = gloss; 216 | half y = NdotV; 217 | 218 | half b1 = -0.1688h; 219 | half b2 = 1.895h; 220 | half b3 = 0.9903h; 221 | half b4 = -4.853h; 222 | half b5 = 8.404h; 223 | half b6 = -5.069h; 224 | half bias = saturate( min( b1 * x + b2 * x * x, b3 + b4 * y + b5 * y * y + b6 * y * y * y ) ); 225 | 226 | half d0 = 0.6045h; 227 | half d1 = 1.699h; 228 | half d2 = -0.5228h; 229 | half d3 = -3.603h; 230 | half d4 = 1.404h; 231 | half d5 = 0.1939h; 232 | half d6 = 2.661h; 233 | half delta = saturate( d0 + d1 * x + d2 * y + d3 * x * x + d4 * x * y + d5 * y * y + d6 * x * x * x ); 234 | half scale = delta - bias; 235 | 236 | bias *= saturate(50.0f * specularColor.y); 237 | return specularColor * scale + bias; 238 | } 239 | 240 | typedef half4 FragmentOut; 241 | 242 | fragment FragmentOut fragment_pbr(PBRFragmentIn in [[stage_in]], 243 | constant PassConstants &frame [[buffer(FragmentBufferPassConstants)]], 244 | constant PBRMaterialConstants &material [[buffer(FragmentBufferMaterialConstants)]], 245 | constant Light *lights [[buffer(FragmentBufferLights)]], 246 | texture2d baseColorTexture [[texture(FragmentTextureBaseColor)]], 247 | texture2d normalTexture [[texture(FragmentTextureNormal)]], 248 | texture2d metalnessTexture [[texture(FragmentTextureMetalness)]], 249 | texture2d roughnessTexture [[texture(FragmentTextureRoughness)]], 250 | texture2d emissiveTexture [[texture(FragmentTextureEmissive)]], 251 | sampler baseColorSampler [[sampler(FragmentTextureBaseColor)]], 252 | sampler normalSampler [[sampler(FragmentTextureNormal)]], 253 | sampler metalnessSampler [[sampler(FragmentTextureMetalness)]], 254 | sampler roughnessSampler [[sampler(FragmentTextureRoughness)]], 255 | sampler emissiveSampler [[sampler(FragmentTextureEmissive)]], 256 | texturecube environmentLightTexture [[texture(FragmentTextureEnvironmentLight)]]) 257 | { 258 | half4 baseColor = half4(material.baseColorFactor); 259 | if (!is_null_texture(baseColorTexture)) { 260 | baseColor *= baseColorTexture.sample(baseColorSampler, in.texCoords); 261 | } 262 | 263 | half metalness = material.metallicFactor; 264 | if (!is_null_texture(metalnessTexture)) { 265 | float sampledMetalness = metalnessTexture.sample(metalnessSampler, in.texCoords).b; 266 | metalness *= sampledMetalness; 267 | } 268 | metalness = saturate(metalness); 269 | 270 | half perceptualRoughness = material.roughnessFactor; 271 | if (!is_null_texture(roughnessTexture)) { 272 | half sampledRoughness = roughnessTexture.sample(roughnessSampler, in.texCoords).g; 273 | perceptualRoughness *= sampledRoughness; 274 | } 275 | perceptualRoughness = clamp(perceptualRoughness, 0.004h, 1.0h); 276 | half alphaRoughness = perceptualRoughness * perceptualRoughness; 277 | 278 | half3 diffuseReflectance = mix(baseColor.rgb, half3(0.0h), metalness); 279 | half3 F0 = mix(half3(0.04h), baseColor.rgb, metalness); 280 | half3 F90 = half3(1.0h); 281 | 282 | float3 V = normalize(frame.cameraPositions[in.viewIndex] - in.position); 283 | TangentSpace tangentSpace = getTangentSpace(in, material, normalTexture, normalSampler); 284 | float3 N = tangentSpace.N; 285 | 286 | half NdotV = saturate(dot(N, V)); 287 | 288 | half3 f_diffuse {}; 289 | half3 f_specular {}; 290 | half3 f_emissive {}; 291 | 292 | if (!is_null_texture(emissiveTexture)) { 293 | half3 sampledEmission = emissiveTexture.sample(emissiveSampler, in.texCoords).rgb; 294 | f_emissive = sampledEmission * material.emissiveStrength; 295 | } else { 296 | f_emissive = half3(material.emissiveColor * material.emissiveStrength); 297 | } 298 | 299 | if (!is_null_texture(environmentLightTexture)) { 300 | float3 N_cube = (frame.environmentLightMatrix * float4(N, 0.0f)).xyz; 301 | constexpr sampler environmentSampler(filter::linear, mip_filter::linear); 302 | float mipCount = environmentLightTexture.get_num_mip_levels(); 303 | // Sample the highest mip level available, since this is essentially a per cube face average lighting estimate 304 | half3 I_diff = environmentLightTexture.sample(environmentSampler, N_cube, level(mipCount - 1)).rgb * M_PI_H; 305 | f_diffuse += I_diff * diffuseReflectance; 306 | // For specular environment lighting we use Knarkowicz's analytic DFG. Realitistically, 307 | // we should sample a much lower LOD than we do, since the environment maps from ARKit 308 | // are much coarser than we'd use with classical IBL. Conversely, they're not properly 309 | // prefiltered with an appropriate BRDF, so this is all just hacks anyway. 310 | float lod = perceptualRoughness * (mipCount - 1); 311 | half gloss = powr(1.0h - perceptualRoughness, 4.0h); 312 | half3 I_spec = environmentLightTexture.sample(environmentSampler, N_cube, level(lod)).rgb; 313 | f_specular += I_spec * EnvDFGPolynomial_Knarkowicz(F0, gloss, NdotV); 314 | } 315 | 316 | for (uint i = 0; i < frame.activeLightCount; ++i) { 317 | Light light = lights[i]; 318 | 319 | float3 pointToLight; 320 | if (light.type != LightTypeDirectional) { 321 | pointToLight = light.position - in.position; 322 | } else { 323 | pointToLight = -light.direction; 324 | } 325 | 326 | float3 L = normalize(pointToLight); 327 | float3 H = normalize(L + V); 328 | half NdotL = saturate(dot(N, L)); 329 | half NdotH = saturate(dot(N, H)); 330 | half VdotH = saturate(dot(V, H)); 331 | if (NdotL > 0.0h) { 332 | half3 intensity = light.getIntensity(pointToLight); 333 | f_diffuse += intensity * NdotL * BRDF_lambertian(F0, F90, diffuseReflectance, VdotH); 334 | if (NdotV > 0.0h) { 335 | f_specular += intensity * NdotL * BRDF_specularGGX(F0, F90, alphaRoughness, VdotH, NdotL, NdotV, NdotH); 336 | } 337 | } 338 | } 339 | 340 | half3 color = f_emissive + f_diffuse + f_specular; 341 | FragmentOut out = half4(color * baseColor.a, baseColor.a); 342 | return out; 343 | } 344 | 345 | using OcclusionFragmentIn = MeshVertexOut; 346 | 347 | fragment FragmentOut fragment_occlusion(OcclusionFragmentIn in [[stage_in]]) { 348 | // What we return here doesn't matter because our color mask is zero. 349 | return {}; 350 | } 351 | -------------------------------------------------------------------------------- /Engine/Math.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import simd 18 | import Spatial 19 | 20 | func alignUp(_ base: Int, alignment: Int) -> Int { 21 | precondition(alignment > 0) 22 | return ((base + alignment - 1) / alignment) * alignment 23 | } 24 | 25 | // Given a range, determine the parameter for which lerp would produce the provided value. 26 | // Will extrapolate if given a value outside the range. 27 | func unlerp(_ from: T, _ to: T, _ value: T) -> T { 28 | if to == from { return T(0) } 29 | return (value - from) / (to - from) 30 | } 31 | 32 | // Swift simd doesn't have a packed three-element float vector type, but this 33 | // type has the same layout, size, and alignment such a type would have. 34 | public typealias packed_float3 = (Float, Float, Float) 35 | 36 | public extension simd_float4 { 37 | var xyz: simd_float3 { 38 | return simd_float3(x, y, z) 39 | } 40 | } 41 | 42 | extension simd_float4x4 { 43 | var upperLeft3x3: simd_float3x3 { 44 | return simd_float3x3(columns.0.xyz, columns.1.xyz, columns.2.xyz) 45 | } 46 | 47 | init(translation: simd_float3) { 48 | self.init(simd_float4(1, 0, 0, 0), 49 | simd_float4(0, 1, 0, 0), 50 | simd_float4(0, 0, 1, 0), 51 | simd_float4(translation, 1)) 52 | } 53 | 54 | init(rotationAbout axis: simd_float3, by angleRadians: Float) { 55 | let x = axis.x, y = axis.y, z = axis.z 56 | let c = cosf(angleRadians) 57 | let s = sinf(angleRadians) 58 | let t = 1 - c 59 | self.init(simd_float4( t * x * x + c, t * x * y + z * s, t * x * z - y * s, 0), 60 | simd_float4( t * x * y - z * s, t * y * y + c, t * y * z + x * s, 0), 61 | simd_float4( t * x * z + y * s, t * y * z - x * s, t * z * z + c, 0), 62 | simd_float4( 0, 0, 0, 1)) 63 | } 64 | 65 | init(orthographicProjectionLeft left: Float, top: Float, right: Float, bottom: Float, near: Float, far: Float) 66 | { 67 | let sx = 2.0 / (right - left) 68 | let sy = 2.0 / (top - bottom) 69 | let sz = 1.0 / (near - far) 70 | let tx = (left + right) / (left - right) 71 | let ty = (top + bottom) / (bottom - top) 72 | let tz = near / (near - far) 73 | self.init(simd_float4( sx, 0.0, 0.0, 0.0), 74 | simd_float4(0.0, sy, 0.0, 0.0), 75 | simd_float4(0.0, 0.0, sz, 0.0), 76 | simd_float4( tx, ty, tz, 1.0)) 77 | } 78 | 79 | // Produces a perspective projection with an infinitely distant "far plane" that 80 | // maps clip-space depth from 1 (near) to 0 (infinitely far)—so-called reverse-Z. 81 | // It also assumes that view space is right-handed, because...c'mon. 82 | init(perspectiveProjectionFOV fovYRadians: Float, aspectRatio: Float, near: Float) 83 | { 84 | let sy = 1 / tan(fovYRadians * 0.5) 85 | let sx = sy / aspectRatio 86 | self.init(simd_float4(sx, 0, 0, 0.0), 87 | simd_float4(0, sy, 0, 0.0), 88 | simd_float4(0, 0, 0, -1.0), 89 | simd_float4(0, 0, near, 0.0)) 90 | } 91 | } 92 | 93 | extension simd_float4x4 { 94 | init(_ mat: simd_double4x4) { 95 | self.init(SIMD4(mat.columns.0), 96 | SIMD4(mat.columns.1), 97 | SIMD4(mat.columns.2), 98 | SIMD4(mat.columns.3)) 99 | } 100 | } 101 | 102 | /// Extracts the six frustum planes determined by the provided matrix. 103 | // Ref. https://www8.cs.umu.se/kurser/5DV051/HT12/lab/plane_extraction.pdf 104 | // Ref. https://fgiesen.wordpress.com/2012/08/31/frustum-planes-from-the-projection-matrix/ 105 | func frustumPlanes(from matrix: simd_float4x4) -> [simd_float4] { 106 | let mt = matrix.transpose 107 | let planes = [simd_float4](unsafeUninitializedCapacity: 6, 108 | initializingWith: { buffer, initializedCount in 109 | buffer[0] = mt[3] + mt[0] // left 110 | buffer[1] = mt[3] - mt[0] // right 111 | buffer[2] = mt[3] - mt[1] // top 112 | buffer[3] = mt[3] + mt[1] // bottom 113 | buffer[4] = mt[2] // near 114 | buffer[5] = mt[3] - mt[2] // far 115 | for i in 0..<6 { 116 | buffer[i] /= simd_length(buffer[i].xyz) 117 | } 118 | initializedCount = 6 119 | }) 120 | return planes 121 | } 122 | 123 | public struct Angle { 124 | static func radians(_ radians: Double) -> Angle { 125 | return Angle(radians: radians) 126 | } 127 | 128 | static func degrees(_ degrees: Double) -> Angle { 129 | return Angle(degrees: degrees) 130 | } 131 | 132 | var radians: Double 133 | 134 | var degrees: Double { 135 | return radians * (180 / .pi) 136 | } 137 | 138 | init() { 139 | radians = 0.0 140 | } 141 | 142 | init(radians: Double) { 143 | self.radians = radians 144 | } 145 | 146 | init(degrees: Double) { 147 | self.radians = degrees * (.pi / 180) 148 | } 149 | 150 | static func + (lhs: Angle, rhs: Angle) -> Angle { 151 | return Angle(radians: lhs.radians + rhs.radians) 152 | } 153 | 154 | static func += (lhs: inout Angle, rhs: Angle) { 155 | lhs.radians += rhs.radians 156 | } 157 | } 158 | 159 | public struct Ray { 160 | let origin: simd_float3 161 | let direction: simd_float3 162 | } 163 | 164 | /* 165 | struct BoundingSphere { 166 | var center = simd_float3() 167 | var radius: Float = 0.0 168 | 169 | init(points: StridedView) { 170 | if points.count > 0 { 171 | // Ref. "A Simple Streaming Algorithm for Minimum Enclosing Balls" Zarrabi-Zadeh, Chan (2006) 172 | var center = simd_float3(points[0].0, points[0].1, points[0].2) 173 | var radius: Float = 0.0 174 | for pointTuple in points { 175 | let point = simd_float3(pointTuple.0, pointTuple.1, pointTuple.2) 176 | let distance = simd_distance(center, point) 177 | if distance > radius { 178 | let deltaRadius = 0.5 * (distance - radius) 179 | radius += deltaRadius 180 | let deltaCenter = (deltaRadius / distance) * (point - center) 181 | center += deltaCenter 182 | } 183 | } 184 | self.center = center 185 | self.radius = radius 186 | } 187 | } 188 | 189 | init(center: simd_float3 = simd_float3(), radius: Float = 0.0) { 190 | self.center = center 191 | self.radius = abs(radius) 192 | } 193 | 194 | func transformed(by transform: simd_float4x4) -> BoundingSphere { 195 | let newCenter = transform * simd_float4(center, 1.0) 196 | let stretchedRadius = transform * simd_float4(radius, radius, radius, 0.0) 197 | let newRadius = max(stretchedRadius.x, max(stretchedRadius.y, stretchedRadius.z)) 198 | return BoundingSphere(center: newCenter.xyz, radius: newRadius) 199 | } 200 | } 201 | */ 202 | 203 | struct BoundingBox { 204 | var min = simd_float3() 205 | var max = simd_float3() 206 | 207 | init(points: StridedView) { 208 | if points.count > 0 { 209 | var min = simd_float3(points[0].0, points[0].1, points[0].2) 210 | var max = simd_float3(points[0].0, points[0].1, points[0].2) 211 | for pointTuple in points { 212 | let point = simd_float3(pointTuple.0, pointTuple.1, pointTuple.2) 213 | if point.x < min.x { min.x = point.x } 214 | if point.y < min.y { min.y = point.y } 215 | if point.z < min.z { min.z = point.z } 216 | if point.x > max.x { max.x = point.x } 217 | if point.y > max.y { max.y = point.y } 218 | if point.z > max.z { max.z = point.z } 219 | } 220 | self.min = min 221 | self.max = max 222 | } 223 | } 224 | 225 | init(min: simd_float3 = simd_float3(), max: simd_float3 = simd_float3()) { 226 | self.min = min 227 | self.max = max 228 | } 229 | 230 | func transformed(by transform: simd_float4x4) -> BoundingBox { 231 | let transformedCorners: [simd_float3] = [ 232 | (transform * simd_float4(min.x, min.y, min.z, 1.0)).xyz, 233 | (transform * simd_float4(min.x, min.y, max.z, 1.0)).xyz, 234 | (transform * simd_float4(min.x, max.y, min.z, 1.0)).xyz, 235 | (transform * simd_float4(min.x, max.y, max.z, 1.0)).xyz, 236 | (transform * simd_float4(max.x, min.y, min.z, 1.0)).xyz, 237 | (transform * simd_float4(max.x, min.y, max.z, 1.0)).xyz, 238 | (transform * simd_float4(max.x, max.y, min.z, 1.0)).xyz, 239 | (transform * simd_float4(max.x, max.y, max.z, 1.0)).xyz, 240 | ] 241 | return transformedCorners.withUnsafeBufferPointer { 242 | BoundingBox(points: StridedView($0.baseAddress!, 243 | offset: 0, 244 | stride: MemoryLayout.stride, 245 | count: transformedCorners.count)) 246 | } 247 | } 248 | } 249 | 250 | public struct Transform { 251 | var position: simd_float3 252 | var scale: simd_float3 253 | var orientation: simd_quatf 254 | 255 | var matrix: simd_float4x4 { 256 | let R = matrix_float3x3(orientation) 257 | let TRS = simd_float4x4(simd_float4(scale.x * R.columns.0, 0), 258 | simd_float4(scale.y * R.columns.1, 0), 259 | simd_float4(scale.z * R.columns.2, 0), 260 | simd_float4(position, 1)) 261 | return TRS 262 | } 263 | 264 | var inverse: Transform { 265 | let RSinv = self.matrix.upperLeft3x3.inverse 266 | let TRSInv = simd_float4x4(simd_float4(RSinv.columns.0, 0.0), 267 | simd_float4(RSinv.columns.1, 0.0), 268 | simd_float4(RSinv.columns.2, 0.0), 269 | simd_float4(-RSinv * position, 1.0)) 270 | return Transform(TRSInv) 271 | } 272 | 273 | init(position: simd_float3 = simd_float3(0, 0, 0), 274 | scale: simd_float3 = simd_float3(1, 1, 1), 275 | orientation: simd_quatf = simd_quatf(ix: 0, iy: 0, iz: 0, r: 1)) 276 | { 277 | self.position = position 278 | self.scale = scale 279 | self.orientation = orientation 280 | } 281 | 282 | @available(macOS 13.0, *) 283 | init(_ pose: Pose3D) { 284 | self.position = simd_float3(pose.position) 285 | self.scale = simd_float3(repeating: 1.0) 286 | self.orientation = simd_quatf(pose.rotation) 287 | } 288 | 289 | /// Initializes a Transform by decomposing the provided affine transformation matrix. 290 | /// Assumes the provided matrix is a valid TRS matrix. 291 | init(_ matrix: simd_float4x4) { 292 | self.position = matrix.columns.3.xyz 293 | let RS = matrix.upperLeft3x3 294 | let sx = simd_length(RS.columns.0) 295 | let sy = simd_length(RS.columns.1) 296 | let sz = simd_length(RS.columns.2) 297 | let R = simd_float3x3(RS.columns.0 / sx, RS.columns.1 / sy, RS.columns.2 / sz) 298 | self.scale = simd_float3(sx, sy, sz) 299 | self.orientation = simd_quatf(R) 300 | } 301 | 302 | mutating func setRotation(axis: simd_float3, angle angleRadians: Float) { 303 | orientation = simd_quatf(angle: angleRadians, axis: axis) 304 | } 305 | 306 | mutating func look(at toPoint: simd_float3, from fromPoint: simd_float3, up upVector: simd_float3) { 307 | let zNeg = simd_normalize(toPoint - fromPoint) 308 | let x = simd_normalize(simd_cross(zNeg, upVector)) 309 | let y = simd_normalize(simd_cross(x, zNeg)) 310 | let R = matrix_float3x3(x, y, -zNeg) 311 | orientation = simd_quaternion(R) 312 | position = fromPoint 313 | } 314 | 315 | static func * (lhs: Transform, rhs: Transform) -> Transform { 316 | // TODO: Write a more efficient implementation exploiting affine matrix properties 317 | return Transform(lhs.matrix * rhs.matrix) 318 | } 319 | } 320 | -------------------------------------------------------------------------------- /Engine/Mesh.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import Metal 19 | import ModelIO 20 | 21 | public class Submesh : Identifiable { 22 | let primitiveType: MTLPrimitiveType 23 | let indexBuffer: BufferView? 24 | let indexType: MTLIndexType 25 | let indexCount: Int 26 | let materialIndex: Int 27 | 28 | init(primitiveType: MTLPrimitiveType, 29 | indexBuffer: BufferView?, 30 | indexType: MTLIndexType = .uint32, 31 | indexCount: Int = 0, 32 | materialIndex: Int) 33 | { 34 | self.primitiveType = primitiveType 35 | self.indexBuffer = indexBuffer 36 | self.indexType = indexType 37 | self.indexCount = indexCount 38 | self.materialIndex = materialIndex 39 | } 40 | } 41 | 42 | public class Mesh : Identifiable { 43 | let vertexCount: Int 44 | let vertexBuffers: [BufferView] 45 | let vertexDescriptor: MDLVertexDescriptor 46 | let boundingBox: BoundingBox 47 | let submeshes: [Submesh] 48 | var materials: [Material] 49 | 50 | init(vertexCount: Int, 51 | vertexBuffers: [BufferView], 52 | vertexDescriptor: MDLVertexDescriptor, 53 | submeshes: [Submesh], 54 | materials: [Material], 55 | boundingBox: BoundingBox? = nil) 56 | { 57 | self.vertexCount = vertexCount 58 | self.vertexBuffers = vertexBuffers 59 | self.vertexDescriptor = vertexDescriptor 60 | if let boundingBox { 61 | self.boundingBox = boundingBox 62 | } else { 63 | if let positionAttribute = vertexDescriptor.attributeNamed(MDLVertexAttributePosition) { 64 | let positionView = vertexBuffers[positionAttribute.bufferIndex] 65 | let positionBufferLayout = vertexDescriptor.bufferLayouts[positionAttribute.bufferIndex] 66 | let positionAccessor = StridedView(positionView.buffer.contents() + positionView.offset, 67 | offset: positionAttribute.offset, 68 | stride: positionBufferLayout.stride, 69 | count: vertexCount) 70 | self.boundingBox = BoundingBox(points: positionAccessor) 71 | } else { 72 | self.boundingBox = BoundingBox() 73 | } 74 | } 75 | self.submeshes = submeshes 76 | self.materials = materials 77 | } 78 | } 79 | 80 | extension Mesh { 81 | static var defaultVertexDescriptor: MDLVertexDescriptor { 82 | let vertexDescriptor = MDLVertexDescriptor() 83 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition 84 | vertexDescriptor.vertexAttributes[0].format = .float3 85 | vertexDescriptor.vertexAttributes[0].offset = 0 86 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0 87 | vertexDescriptor.bufferLayouts[0].stride = MemoryLayout.stride 88 | 89 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal 90 | vertexDescriptor.vertexAttributes[1].format = .float3 91 | vertexDescriptor.vertexAttributes[1].offset = 0 92 | vertexDescriptor.vertexAttributes[1].bufferIndex = 1 93 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate 94 | vertexDescriptor.vertexAttributes[2].format = .float2 95 | vertexDescriptor.vertexAttributes[2].offset = MemoryLayout.stride 96 | vertexDescriptor.vertexAttributes[2].bufferIndex = 1 97 | vertexDescriptor.bufferLayouts[1].stride = MemoryLayout.stride + MemoryLayout.stride 98 | 99 | return vertexDescriptor 100 | } 101 | 102 | static var skinnedVertexDescriptor: MDLVertexDescriptor { 103 | let vertexDescriptor = MDLVertexDescriptor() 104 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition 105 | vertexDescriptor.vertexAttributes[0].format = .float3 106 | vertexDescriptor.vertexAttributes[0].offset = 0 107 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0 108 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal 109 | vertexDescriptor.vertexAttributes[1].format = .float3 110 | vertexDescriptor.vertexAttributes[1].offset = MemoryLayout.stride * 3 111 | vertexDescriptor.vertexAttributes[1].bufferIndex = 0 112 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate 113 | vertexDescriptor.vertexAttributes[2].format = .float2 114 | vertexDescriptor.vertexAttributes[2].offset = MemoryLayout.stride * 6 115 | vertexDescriptor.vertexAttributes[2].bufferIndex = 0 116 | vertexDescriptor.vertexAttributes[3].name = MDLVertexAttributeJointWeights 117 | vertexDescriptor.vertexAttributes[3].format = .float4 118 | vertexDescriptor.vertexAttributes[3].offset = MemoryLayout.stride * 8 119 | vertexDescriptor.vertexAttributes[3].bufferIndex = 0 120 | vertexDescriptor.vertexAttributes[4].name = MDLVertexAttributeJointIndices 121 | vertexDescriptor.vertexAttributes[4].format = .uShort4 122 | vertexDescriptor.vertexAttributes[4].offset = MemoryLayout.stride * 12 123 | vertexDescriptor.vertexAttributes[4].bufferIndex = 0 124 | vertexDescriptor.bufferLayouts[0].stride = MemoryLayout.stride * 12 + MemoryLayout.stride 125 | 126 | return vertexDescriptor 127 | } 128 | } 129 | 130 | public class Skeleton { 131 | var name: String 132 | var jointPaths: [String] 133 | var inverseBindTransforms: [simd_float4x4] 134 | var restTransforms: [simd_float4x4] 135 | 136 | init(name: String, jointPaths: [String], inverseBindTransforms: [simd_float4x4], restTransforms: [simd_float4x4]) { 137 | self.name = name 138 | self.jointPaths = jointPaths 139 | self.inverseBindTransforms = inverseBindTransforms 140 | self.restTransforms = restTransforms 141 | } 142 | } 143 | 144 | public class Skinner { 145 | let skeleton: Skeleton 146 | let baseMesh: Mesh 147 | 148 | var jointTransforms: [simd_float4x4] { 149 | didSet { 150 | precondition(jointTransforms.count == skeleton.jointPaths.count) 151 | isDirty = true 152 | } 153 | } 154 | 155 | var isDirty = true 156 | 157 | init(skeleton: Skeleton, baseMesh: Mesh) { 158 | self.skeleton = skeleton 159 | self.baseMesh = baseMesh 160 | self.jointTransforms = skeleton.restTransforms 161 | } 162 | } 163 | 164 | extension Mesh { 165 | /// A single-buffer, packed, interleaved vertex data layout suitable for simple vertex skinning use cases 166 | static let postSkinningVertexDescriptor: MDLVertexDescriptor = { 167 | let vertexDescriptor = MDLVertexDescriptor() 168 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition 169 | vertexDescriptor.vertexAttributes[0].format = .float3 170 | vertexDescriptor.vertexAttributes[0].offset = 0 171 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0 172 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal 173 | vertexDescriptor.vertexAttributes[1].format = .float3 174 | vertexDescriptor.vertexAttributes[1].offset = MemoryLayout.stride * 3 175 | vertexDescriptor.vertexAttributes[1].bufferIndex = 0 176 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate 177 | vertexDescriptor.vertexAttributes[2].format = .float2 178 | vertexDescriptor.vertexAttributes[2].offset = MemoryLayout.stride * 6 179 | vertexDescriptor.vertexAttributes[2].bufferIndex = 0 180 | vertexDescriptor.bufferLayouts[0].stride = MemoryLayout.stride * 8 181 | return vertexDescriptor 182 | }() 183 | 184 | /// Creates a copy of this mesh that is suitable as a post-skinning destination for skinned vertices. 185 | /// Submeshes and materials are copied by reference. 186 | func copyForSkinning(context: MetalContext) -> Mesh { 187 | let vertexDescriptor = Mesh.postSkinningVertexDescriptor 188 | let vertexBufferLength = vertexDescriptor.bufferLayouts[0].stride * vertexCount 189 | let vertexBuffer = context.device.makeBuffer(length: vertexBufferLength, 190 | options: [.storageModePrivate])! 191 | vertexBuffer.label = "Post-Skinning Vertex Attributes" 192 | return Mesh(vertexCount: self.vertexCount, 193 | vertexBuffers: [BufferView(buffer: vertexBuffer)], 194 | vertexDescriptor: vertexDescriptor, 195 | submeshes: submeshes, 196 | materials: materials, 197 | boundingBox: boundingBox) 198 | } 199 | } 200 | 201 | extension Mesh { 202 | class func generateSphere(radius: Float, context: MetalContext) -> Mesh { 203 | let mdlMesh = MDLMesh.init(sphereWithExtent: simd_float3(repeating: radius), 204 | segments: simd_uint2(24, 24), 205 | inwardNormals: false, 206 | geometryType: .triangles, 207 | allocator: nil) 208 | mdlMesh.vertexDescriptor = defaultVertexDescriptor 209 | let tempResourceCache = MetalResourceCache(context: context) 210 | let mesh = Mesh(mdlMesh, context: context, resourceCache: tempResourceCache) 211 | mesh.materials = [PhysicallyBasedMaterial.default] 212 | return mesh 213 | } 214 | 215 | class func generateBox(extents: simd_float3, context: MetalContext) -> Mesh { 216 | let mdlMesh = MDLMesh.init(boxWithExtent: extents, 217 | segments: simd_uint3(1, 1, 1), 218 | inwardNormals: false, 219 | geometryType: .triangles, 220 | allocator: nil) 221 | mdlMesh.vertexDescriptor = defaultVertexDescriptor 222 | let tempResourceCache = MetalResourceCache(context: context) 223 | let mesh = Mesh(mdlMesh, context: context, resourceCache: tempResourceCache) 224 | mesh.materials = [PhysicallyBasedMaterial.default] 225 | return mesh 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /Engine/MetalContext.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import Metal 19 | 20 | enum ResourceError : Error { 21 | case allocationFailure 22 | case imageLoadFailure 23 | case invalidImageFormat 24 | case invalidState 25 | } 26 | 27 | class MetalContext : @unchecked Sendable { 28 | nonisolated(unsafe) static var shared = MetalContext() 29 | 30 | #if os(visionOS) 31 | let preferredColorPixelFormat = MTLPixelFormat.rgba16Float 32 | #else 33 | let preferredColorPixelFormat = MTLPixelFormat.bgra8Unorm_srgb 34 | #endif 35 | 36 | let preferredDepthPixelFormat = MTLPixelFormat.depth32Float 37 | let preferredRasterSampleCount: Int 38 | 39 | let device: MTLDevice 40 | let commandQueue: MTLCommandQueue 41 | let defaultLibrary: MTLLibrary? 42 | 43 | init(device: MTLDevice? = nil, commandQueue: MTLCommandQueue? = nil) { 44 | guard let metalDevice = device ?? MTLCreateSystemDefaultDevice() else { 45 | fatalError("Metal is not supported") 46 | } 47 | guard let metalCommandQueue = commandQueue ?? metalDevice.makeCommandQueue() else { 48 | fatalError() 49 | } 50 | assert(metalCommandQueue.device.isEqual(metalDevice)) 51 | 52 | self.device = metalDevice 53 | self.commandQueue = metalCommandQueue 54 | self.defaultLibrary = metalDevice.makeDefaultLibrary() 55 | 56 | #if targetEnvironment(simulator) 57 | // The Metal API validation layer on Vision Pro simulator claims not to be able 58 | // to resolve Depth32 MSAA targets (it lies), so force MSAA off there. 59 | let candidateSampleCounts = [1] 60 | #else 61 | let candidateSampleCounts = [16, 8, 5, 4, 2, 1] 62 | #endif 63 | let supportedSampleCounts = candidateSampleCounts.filter { metalDevice.supportsTextureSampleCount($0) } 64 | preferredRasterSampleCount = supportedSampleCounts.first ?? 1 65 | print("Supported raster sample counts: \(supportedSampleCounts). Selected \(preferredRasterSampleCount).") 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Engine/Model.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import Metal 19 | import ModelIO 20 | 21 | class Model { 22 | let rootEntities: [Entity] 23 | 24 | // Model I/O doesn't expose this property, which in USD is configurable via asset metadata, 25 | // but most USDZ files in the wild seem to favor USD's default unit of centimeters, so we 26 | // use this as a heuristic to match content to the real world. 27 | var metersPerUnit: Float = 0.01 28 | 29 | init(fileURL url: URL, context: MetalContext) throws { 30 | var rootEntities = [Entity]() 31 | let resourceCache = MetalResourceCache(context: context) 32 | var entityMap = [ObjectIdentifier : Entity]() 33 | autoreleasepool { 34 | let asset = MDLAsset(url: url, vertexDescriptor: nil, bufferAllocator: nil) 35 | 36 | asset.loadTextures() 37 | 38 | var skeletonMap = [ObjectIdentifier : Skeleton]() 39 | 40 | for mdlObject in asset.childObjects(of: MDLObject.self) { 41 | let entity = Entity() 42 | entity.name = mdlObject.name 43 | entityMap[ObjectIdentifier(mdlObject)] = entity 44 | 45 | if let transformComponent = mdlObject.transform { 46 | entity.modelTransform = Transform(transformComponent.matrix) 47 | } 48 | 49 | if let mdlMesh = mdlObject as? MDLMesh { 50 | // If a mesh has an associated animation bind component, it may have a skeleton and be 51 | // skeletally animated, so we need to apply a vertex descriptor that preserves any joint 52 | // weight/index data. Otherwise, we conform the mesh to our default vertex descriptor. 53 | let isSkinned = (mdlObject.animationBind != nil) 54 | let vertexDescriptor = isSkinned ? Mesh.skinnedVertexDescriptor : Mesh.defaultVertexDescriptor 55 | mdlMesh.vertexDescriptor = vertexDescriptor 56 | 57 | let mesh = Mesh(mdlMesh, context: context, resourceCache: resourceCache) 58 | entity.mesh = mesh 59 | } 60 | 61 | if let mdlSkeleton = mdlObject as? MDLSkeleton { 62 | let skeleton = Skeleton(mdlSkeleton, context: context) 63 | skeletonMap[ObjectIdentifier(mdlSkeleton)] = skeleton 64 | } 65 | 66 | if let animationBind = mdlObject.animationBind, let mdlSkeleton = animationBind.skeleton { 67 | if let skeleton = skeletonMap[ObjectIdentifier(mdlSkeleton)], let baseMesh = entity.mesh { 68 | entity.skinner = Skinner(skeleton: skeleton, baseMesh: baseMesh) 69 | entity.mesh = baseMesh.copyForSkinning(context: context) 70 | } 71 | } 72 | 73 | if let parent = mdlObject.parent { 74 | let parentEntity = entityMap[ObjectIdentifier(parent)] 75 | parentEntity?.addChild(entity) 76 | if parentEntity == nil { 77 | print("Could not retrieve object parent (\"\(parent.name)\") from entity map; hierarchy will be broken") 78 | } 79 | } 80 | 81 | if entity.parent == nil { 82 | rootEntities.append(entity) 83 | } 84 | } 85 | } 86 | self.rootEntities = rootEntities 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Engine/ModelIOSupport.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Metal 18 | import MetalKit 19 | import ModelIO 20 | 21 | // The Swift interface for Model I/O has been broken since it was released in 2015. 22 | // These extensions correct the types of a number of properties on Model I/O types. 23 | 24 | extension MDLVertexDescriptor { 25 | var vertexAttributes: [MDLVertexAttribute] { 26 | return attributes as! [MDLVertexAttribute] 27 | } 28 | 29 | var bufferLayouts: [MDLVertexBufferLayout] { 30 | return layouts as! [MDLVertexBufferLayout] 31 | } 32 | } 33 | 34 | extension MDLMesh { 35 | var mdlSubmeshes: [MDLSubmesh]? { 36 | return submeshes as! [MDLSubmesh]? 37 | } 38 | } 39 | 40 | extension MTLSamplerDescriptor { 41 | convenience init(mdlTextureFilter: MDLTextureFilter?) { 42 | self.init() 43 | let metalAddressMode: (MDLMaterialTextureWrapMode) -> MTLSamplerAddressMode = { mode in 44 | switch mode { 45 | case .clamp: 46 | return .clampToEdge 47 | case .repeat: 48 | return .repeat 49 | case .mirror: 50 | return .mirrorRepeat 51 | @unknown default: 52 | return .repeat 53 | } 54 | } 55 | if let mdlTextureFilter { 56 | self.minFilter = mdlTextureFilter.minFilter == .nearest ? .nearest : .linear 57 | self.magFilter = mdlTextureFilter.magFilter == .nearest ? .nearest : .linear 58 | self.mipFilter = mdlTextureFilter.mipFilter == .nearest ? .nearest : .linear 59 | self.sAddressMode = metalAddressMode(mdlTextureFilter.sWrapMode) 60 | self.tAddressMode = metalAddressMode(mdlTextureFilter.tWrapMode) 61 | } else { 62 | self.minFilter = .linear 63 | self.magFilter = .linear 64 | self.mipFilter = .linear 65 | self.sAddressMode = .repeat 66 | self.tAddressMode = .repeat 67 | } 68 | } 69 | } 70 | 71 | // This extension enables strided random access into a ModelIO vertex attribute data object. 72 | // The type contained by the underlying data buffer must exactly match the Element type of 73 | // the created view; otherwise behavior is undefined. 74 | extension StridedView { 75 | init(attributeData: MDLVertexAttributeData, count: Int) { 76 | self.init(attributeData.dataStart, offset: 0, stride: attributeData.stride, count: count) 77 | } 78 | } 79 | 80 | class MetalResourceCache { 81 | let context: MetalContext 82 | 83 | private var textureCache: [ObjectIdentifier: MTLTexture] = [:] 84 | private var samplerCache: [MTLSamplerDescriptor : MTLSamplerState] = [:] 85 | 86 | init(context: MetalContext) { 87 | self.context = context 88 | } 89 | 90 | func makeTextureResource(for mdlMaterialProperty: MDLMaterialProperty, 91 | contentType: TextureResource.ContentType) throws -> TextureResource 92 | { 93 | precondition(mdlMaterialProperty.type == .texture) 94 | 95 | let device = context.device 96 | 97 | guard let mdlTextureSampler = mdlMaterialProperty.textureSamplerValue else { 98 | throw ResourceError.invalidState 99 | } 100 | 101 | let samplerDescriptor = MTLSamplerDescriptor(mdlTextureFilter: mdlTextureSampler.hardwareFilter) 102 | 103 | let sampler = samplerCache[samplerDescriptor] ?? { 104 | let sampler = device.makeSamplerState(descriptor: samplerDescriptor)! 105 | samplerCache[samplerDescriptor] = sampler 106 | return sampler 107 | }() 108 | 109 | guard let mdlTexture = mdlTextureSampler.texture else { 110 | throw ResourceError.imageLoadFailure 111 | } 112 | 113 | if let existingTexture = textureCache[ObjectIdentifier(mdlTexture)] { 114 | return .init(texture: existingTexture, sampler: sampler) 115 | } 116 | 117 | let textureLoader = TextureLoader(context: .shared) 118 | 119 | var options: [TextureLoader.Option : Any] = [ 120 | .generateMipmaps : true, 121 | // Model I/O retains the lower-left origin texture coordinate convention of formats like USD[Z] 122 | // and OBJ, so we request that texel data be flipped on load. We could instead flip texture 123 | // coordinates when loading models or do this at runtime in the vertex shader, but this 124 | // seems the simplest approach, assuming that textures are always used with their corresponding 125 | // models and we don't mind that models' runtime texture coordinates follow the oppposite of 126 | // our preferred convention. 127 | .flipVertically : true 128 | ] 129 | if contentType == .raw { 130 | options[.sRGB] = false 131 | } else { 132 | options[.sRGB] = true 133 | } 134 | if device.supportsFamily(.apple2) { 135 | options[.storageMode] = MTLStorageMode.shared.rawValue 136 | } 137 | 138 | let texture = try textureLoader.makeTexture(mdlTexture: mdlTexture, options: options) 139 | texture.label = mdlMaterialProperty.name 140 | 141 | textureCache[ObjectIdentifier(mdlTexture)] = texture 142 | 143 | return .init(texture: texture, sampler: sampler) 144 | } 145 | } 146 | 147 | extension PhysicallyBasedMaterial { 148 | convenience init(mdlMaterial: MDLMaterial, resourceCache: MetalResourceCache) { 149 | self.init() 150 | 151 | if let baseColorProperty = mdlMaterial.property(with: MDLMaterialSemantic.baseColor) { 152 | if baseColorProperty.type == .texture { 153 | baseColor.baseColorTexture = try? resourceCache.makeTextureResource(for: baseColorProperty, 154 | contentType: .color) 155 | } else if baseColorProperty.type == .float3 { 156 | let color = baseColorProperty.float3Value 157 | baseColor.baseColorFactor = simd_float4(color, 1.0) 158 | } 159 | } 160 | if let normalProperty = mdlMaterial.property(with: MDLMaterialSemantic.tangentSpaceNormal) { 161 | if normalProperty.type == .texture { 162 | normal.normalTexture = try? resourceCache.makeTextureResource(for: normalProperty, 163 | contentType: .raw) 164 | } 165 | } 166 | if let metalnessProperty = mdlMaterial.property(with: MDLMaterialSemantic.metallic) { 167 | if metalnessProperty.type == .texture { 168 | metalness.metalnessTexture = try? resourceCache.makeTextureResource(for: metalnessProperty, 169 | contentType: .raw) 170 | } else if metalnessProperty.type == .float { 171 | metalness.metalnessFactor = metalnessProperty.floatValue 172 | } 173 | } 174 | if let roughnessProperty = mdlMaterial.property(with: MDLMaterialSemantic.roughness) { 175 | if roughnessProperty.type == .texture { 176 | roughness.roughnessTexture = try? resourceCache.makeTextureResource(for: roughnessProperty, 177 | contentType: .raw) 178 | } else if roughnessProperty.type == .float { 179 | roughness.roughnessFactor = roughnessProperty.floatValue 180 | } 181 | } 182 | if let emissionProperty = mdlMaterial.property(with: MDLMaterialSemantic.emission) { 183 | if emissionProperty.type == .texture { 184 | emissive.emissiveTexture = try? resourceCache.makeTextureResource(for: emissionProperty, 185 | contentType: .color) 186 | } else if emissionProperty.type == .float3 { 187 | emissive.emissiveColor = emissionProperty.float3Value 188 | } 189 | } 190 | if let _ = mdlMaterial.property(with: MDLMaterialSemantic.opacity) { 191 | // There are probably more precise checks we could do to determine if we should enable 192 | // alpha blending, but Model I/O doesn't have a convenient "blend mode" property, so 193 | // we use the presence of the opacity property as a simple heuristic. 194 | blendMode = .sourceOverPremultiplied 195 | } 196 | } 197 | } 198 | 199 | extension Mesh { 200 | convenience init(_ mdlMesh: MDLMesh, context: MetalContext, resourceCache: MetalResourceCache) { 201 | let device = context.device 202 | 203 | let vertexBuffers = mdlMesh.vertexBuffers.map { mdlBuffer in 204 | let bufferMap = mdlBuffer.map() 205 | let buffer = device.makeBuffer(bytes: bufferMap.bytes, 206 | length: mdlBuffer.length, 207 | options: .storageModeShared)! 208 | buffer.label = "Vertex Attributes" 209 | return BufferView(buffer: buffer) 210 | } 211 | 212 | var submeshes = [Submesh]() 213 | var materials = [Material]() 214 | for mdlSubmesh in (mdlMesh.mdlSubmeshes ?? []) { 215 | guard mdlSubmesh.geometryType == .triangles else { continue } 216 | 217 | let mdlIndexBuffer = mdlSubmesh.indexBuffer 218 | let bufferMap = mdlIndexBuffer.map() 219 | let buffer = device.makeBuffer(bytes: bufferMap.bytes, 220 | length: mdlIndexBuffer.length, 221 | options: .storageModeShared)! 222 | buffer.label = "Submesh Indices" 223 | let indexBuffer = BufferView(buffer: buffer) 224 | 225 | let indexType: MTLIndexType = mdlSubmesh.indexType == .uint16 ? .uint16 : .uint32 226 | 227 | var material = PhysicallyBasedMaterial.default 228 | if let mdlMaterial = mdlSubmesh.material { 229 | material = PhysicallyBasedMaterial(mdlMaterial: mdlMaterial, resourceCache: resourceCache) 230 | material.name = mdlMaterial.name 231 | } 232 | // ModelIO doesn't have any means of expressing whether a material is 233 | // double-sided, so we just assume all imported materials are. 234 | material.isDoubleSided = true 235 | 236 | let submesh = Submesh(primitiveType: .triangle, 237 | indexBuffer: indexBuffer, 238 | indexType: indexType, 239 | indexCount: mdlSubmesh.indexCount, 240 | materialIndex: materials.count) 241 | 242 | materials.append(material) 243 | submeshes.append(submesh) 244 | } 245 | 246 | let boundingBox = BoundingBox(min: mdlMesh.boundingBox.minBounds, max: mdlMesh.boundingBox.maxBounds) 247 | 248 | self.init(vertexCount: mdlMesh.vertexCount, 249 | vertexBuffers: vertexBuffers, 250 | vertexDescriptor: mdlMesh.vertexDescriptor, 251 | submeshes: submeshes, 252 | materials: materials, 253 | boundingBox: boundingBox) 254 | } 255 | } 256 | 257 | extension Skeleton { 258 | convenience init(_ mdlSkeleton: MDLSkeleton, context: MetalContext) { 259 | let jointBindMatrices = mdlSkeleton.jointBindTransforms.float4x4Array.map { $0.inverse } 260 | let restMatrices = mdlSkeleton.jointRestTransforms.float4x4Array 261 | self.init(name: mdlSkeleton.name, 262 | jointPaths: mdlSkeleton.jointPaths, 263 | inverseBindTransforms: jointBindMatrices, 264 | restTransforms: restMatrices) 265 | } 266 | } 267 | 268 | extension MDLObject { 269 | var animationBind: MDLAnimationBindComponent? { 270 | return components.filter({ 271 | $0 is MDLAnimationBindComponent 272 | }).first as? MDLAnimationBindComponent 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /Engine/Physics/JoltPhysics.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #import 18 | #import 19 | 20 | NS_ASSUME_NONNULL_BEGIN 21 | 22 | typedef NS_ENUM(NSInteger, SRJBodyType) { 23 | SRJBodyTypeStatic, 24 | SRJBodyTypeDynamic, 25 | SRJBodyTypeKinematic, 26 | }; 27 | 28 | typedef struct SRJRigidBodyTransform_t { 29 | simd_float3 position; 30 | simd_quatf orientation; 31 | } SRJRigidBodyTransform; 32 | 33 | typedef struct SRJBodyProperties_t { 34 | float mass; 35 | float friction; 36 | float restitution; 37 | bool isAffectedByGravity; 38 | } SRJBodyProperties; 39 | 40 | @interface SRJPhysicsShape : NSObject 41 | + (instancetype)newSphereShapeWithRadius:(float)radius scale:(simd_float3)scale 42 | NS_SWIFT_NAME(makeSphereShape(radius:scale:)); 43 | 44 | + (instancetype)newBoxShapeWithExtents:(simd_float3)extents scale:(simd_float3)scale 45 | NS_SWIFT_NAME(makeBoxShape(extents:scale:)); 46 | 47 | + (nullable instancetype)newConvexHullShapeWithVertices:(const simd_float3 *)vertices 48 | vertexCount:(NSInteger)vertexCount 49 | scale:(simd_float3)scale 50 | error:(NSError **)error 51 | NS_SWIFT_NAME(makeConvexHullShape(vertices:vertexCount:scale:)) 52 | NS_REFINED_FOR_SWIFT; 53 | 54 | + (instancetype)newConcavePolyhedronShapeWithVertices:(const simd_float3 *)vertices 55 | vertexCount:(NSInteger)vertexCount 56 | indices:(const uint32_t *)indices 57 | indexCount:(NSInteger)indexCount 58 | scale:(simd_float3)scale 59 | NS_SWIFT_NAME(makeConcavePolyhedronShape(vertices:vertexCount:indices:indexCount:scale:)) NS_REFINED_FOR_SWIFT; 60 | 61 | - (instancetype)init NS_UNAVAILABLE; 62 | @end 63 | 64 | @interface SRJPhysicsBody : NSObject 65 | - (instancetype)init NS_UNAVAILABLE; 66 | 67 | @property (nonatomic, assign) SRJRigidBodyTransform transform; 68 | 69 | @end 70 | 71 | @interface SRJHitTestResult : NSObject 72 | @property (nonatomic, weak) SRJPhysicsBody *body; 73 | @property (nonatomic, assign) simd_float3 position; 74 | @property (nonatomic, assign) CGFloat distance; 75 | @end 76 | 77 | @interface SRJPhysicsWorld : NSObject 78 | 79 | + (void)initializeJoltPhysics; 80 | + (void)deinitializeJoltPhysics; 81 | 82 | - (SRJPhysicsBody *)createAndAddPhysicsBodyWithType:(SRJBodyType)type 83 | properties:(SRJBodyProperties)bodyProperties 84 | physicsShape:(SRJPhysicsShape *)shape 85 | initialTransform:(SRJRigidBodyTransform)transform 86 | NS_SWIFT_NAME(addPhysicsBody(type:bodyProperties:physicsShape:initialTransform:)); 87 | 88 | - (void)removePhysicsBody:(SRJPhysicsBody *)physicsBody; 89 | 90 | - (void)updateWithTimestep:(NSTimeInterval)timestep NS_SWIFT_NAME(update(timestep:)); 91 | 92 | - (NSArray *)hitTestWithSegmentFromPoint:(simd_float3)origin toPoint:(simd_float3)dest 93 | NS_SWIFT_NAME(hitTestWithSegment(from:to:)); 94 | 95 | @end 96 | 97 | NS_ASSUME_NONNULL_END 98 | -------------------------------------------------------------------------------- /Engine/Physics/JoltPhysics.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import ModelIO 19 | 20 | extension Mesh { 21 | func getPackedVertexPositions() -> [simd_float3] { 22 | let attribute = vertexDescriptor.attributeNamed(MDLVertexAttributePosition)! 23 | let layout = vertexDescriptor.bufferLayouts[attribute.bufferIndex] 24 | precondition(attribute.format == .float3) 25 | 26 | let vertexBuffer = vertexBuffers[attribute.bufferIndex] 27 | let stride = layout.stride 28 | let bufferBaseAddr = vertexBuffer.buffer.contents().advanced(by: vertexBuffer.offset) 29 | let attributeBaseAddr = bufferBaseAddr.advanced(by: attribute.offset) 30 | let positions = [simd_float3].init(unsafeUninitializedCapacity: vertexCount) { vertexPtr, initializedCount in 31 | for i in 0.. [UInt32] { 41 | let indexCount = submeshes.reduce(0) { return $0 + $1.indexCount } 42 | var indices = [UInt32]() 43 | indices.reserveCapacity(indexCount) 44 | for submesh in self.submeshes { 45 | guard let indexBuffer = submesh.indexBuffer else { continue } 46 | if submesh.indexType == .uint16 { 47 | let indexBaseAddr = indexBuffer.buffer.contents().advanced(by: indexBuffer.offset).assumingMemoryBound(to: UInt16.self) 48 | for i in 0.. SRJPhysicsShape { 64 | return try points.withUnsafeBufferPointer { ptr in 65 | return try makeConvexHullShape(vertices: ptr.baseAddress!, vertexCount: points.count, scale: scale) 66 | } 67 | } 68 | 69 | class func makeConcavePolyhedronShape(points: [simd_float3], indices: [UInt32], scale:simd_float3) -> SRJPhysicsShape { 70 | return points.withUnsafeBufferPointer { pointPtr in 71 | indices.withUnsafeBufferPointer { indexPtr in 72 | return makeConcavePolyhedronShape(vertices: pointPtr.baseAddress!, 73 | vertexCount: points.count, 74 | indices: indexPtr.baseAddress!, 75 | indexCount: indices.count, 76 | scale: scale) 77 | 78 | } 79 | } 80 | } 81 | } 82 | 83 | extension PhysicsShape { 84 | func makeJoltPhysicsShape(scale: simd_float3) throws -> SRJPhysicsShape { 85 | switch type { 86 | case .sphere(let radius): 87 | return SRJPhysicsShape.makeSphereShape(radius: radius, scale: scale) 88 | case .box(let extents): 89 | return SRJPhysicsShape.makeBoxShape(extents: extents, scale: scale) 90 | case .convexHull(let mesh): 91 | let points = mesh.getPackedVertexPositions() 92 | return try SRJPhysicsShape.makeConvexHullShape(points: points, scale: scale) 93 | case .concaveMesh(let mesh): 94 | let points = mesh.getPackedVertexPositions() 95 | let indices = mesh.getPackedSubmeshIndices() 96 | return SRJPhysicsShape.makeConcavePolyhedronShape(points: points, indices: indices, scale: scale) 97 | } 98 | } 99 | } 100 | 101 | class JoltPhysicsBridge : PhysicsBridge { 102 | let physicsWorld: SRJPhysicsWorld 103 | var registeredEntities: Set = [] 104 | var bodiesForEntities: [ObjectIdentifier: SRJPhysicsBody] = [:] 105 | var entitiesForBodies: [ObjectIdentifier: Entity] = [:] 106 | 107 | init() { 108 | physicsWorld = SRJPhysicsWorld() 109 | } 110 | 111 | func addEntity(_ entity: Entity) { 112 | if registeredEntities.contains(entity) { 113 | print("Tried to add entity \(entity.name) to physics bridge when it was already registered!") 114 | return 115 | } 116 | registeredEntities.insert(entity) 117 | if let physicsBody = entity.physicsBody { 118 | do { 119 | let bodyType: SRJBodyType = switch physicsBody.mode { 120 | case .static: .`static` 121 | case .dynamic: .dynamic 122 | case .kinematic: .kinematic 123 | } 124 | let worldTransform = entity.worldTransform 125 | let bodyTransform = SRJRigidBodyTransform(position: worldTransform.position, 126 | orientation: worldTransform.orientation) 127 | let properties = SRJBodyProperties(mass: physicsBody.mass, 128 | friction: physicsBody.friction, 129 | restitution: physicsBody.restitution, 130 | isAffectedByGravity: physicsBody.isAffectedByGravity) 131 | let backendShape = try physicsBody.shape.makeJoltPhysicsShape(scale: worldTransform.scale) 132 | let backendBody = physicsWorld.addPhysicsBody(type: bodyType, 133 | bodyProperties: properties, 134 | physicsShape: backendShape, 135 | initialTransform: bodyTransform) 136 | bodiesForEntities[ObjectIdentifier(entity)] = backendBody 137 | entitiesForBodies[ObjectIdentifier(backendBody)] = entity 138 | } catch { 139 | print("Failed to register entity \(entity.name) to physics bridge with error: \(error.localizedDescription)") 140 | } 141 | } else { 142 | // TODO: Handle entities with no physics body but with a mesh that should be included in hit-testing? 143 | } 144 | } 145 | 146 | func removeEntity(_ entity: Entity) { 147 | if let body = bodiesForEntities[ObjectIdentifier(entity)] { 148 | physicsWorld.remove(body) 149 | entitiesForBodies.removeValue(forKey: ObjectIdentifier(body)) 150 | bodiesForEntities.removeValue(forKey: ObjectIdentifier(entity)) 151 | } 152 | registeredEntities.remove(entity) 153 | } 154 | 155 | func update(entities: [Entity], timestep: TimeInterval) { 156 | for physicsEntity in entities.filter({ $0.physicsBody != nil }) { 157 | if let body = bodiesForEntities[ObjectIdentifier(physicsEntity)] { 158 | let entityTransform = physicsEntity.worldTransform 159 | let bodyTransform = SRJRigidBodyTransform(position: entityTransform.position, 160 | orientation: entityTransform.orientation) 161 | body.transform = bodyTransform 162 | } else { 163 | print("Tried to update entity \(physicsEntity.name) before it was added to the physics bridge.") 164 | } 165 | } 166 | 167 | physicsWorld.update(timestep: timestep) 168 | 169 | for dynamicEntity in entities.filter({ $0.physicsBody?.mode == .dynamic }) { 170 | if let dynamicBody = bodiesForEntities[ObjectIdentifier(dynamicEntity)] { 171 | let bodyTransform = dynamicBody.transform 172 | dynamicEntity.worldTransform = Transform(position: bodyTransform.position, 173 | scale: dynamicEntity.modelTransform.scale, 174 | orientation: bodyTransform.orientation) 175 | } else { 176 | print("Tried to update dynamic entity \(dynamicEntity.name) before it was added to the physics bridge.") 177 | } 178 | } 179 | } 180 | 181 | func hitTestWithSegment(from: simd_float3, to: simd_float3) -> [HitTestResult] { 182 | let results = physicsWorld.hitTestWithSegment(from: from, to: to) 183 | return results.map { result in 184 | if let backendBody = result.body { 185 | let entity = entitiesForBodies[ObjectIdentifier(backendBody)] 186 | return HitTestResult(entity: entity, worldPosition: result.position) 187 | } 188 | return HitTestResult(entity: nil, worldPosition: result.position) 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /Engine/Physics/PhysicsBridge.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | import simd 19 | 20 | struct HitTestResult { 21 | let entity: Entity? 22 | let worldPosition: simd_float3 23 | } 24 | 25 | protocol PhysicsBridge { 26 | func addEntity(_ entity: Entity) 27 | func removeEntity(_ entity: Entity) 28 | func update(entities: [Entity], timestep: TimeInterval) 29 | func hitTestWithSegment(from: simd_float3, to: simd_float3) -> [HitTestResult] 30 | } 31 | -------------------------------------------------------------------------------- /Engine/PhysicsBody.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Foundation 18 | 19 | enum PhysicsShapeType { 20 | case box(_ extents: simd_float3) 21 | case sphere(_ radius: Float) 22 | case convexHull(_ mesh: Mesh) 23 | case concaveMesh(_ mesh: Mesh) 24 | } 25 | 26 | enum PhysicsBodyMode { 27 | case `static` 28 | case dynamic 29 | case kinematic 30 | } 31 | 32 | struct CollisionGroup : OptionSet { 33 | let rawValue: UInt32 34 | 35 | static let `default`: CollisionGroup = CollisionGroup(rawValue: 1 << 0) 36 | static let sceneUnderstanding: CollisionGroup = CollisionGroup(rawValue: 1 << 1) 37 | static let all: CollisionGroup = CollisionGroup(rawValue: UInt32.max) 38 | } 39 | 40 | struct CollisionFilter : Equatable { 41 | static let `default`: CollisionFilter = .init(group: .default, mask: .all) 42 | 43 | var group: CollisionGroup 44 | var mask: CollisionGroup 45 | 46 | init(group: CollisionGroup, mask: CollisionGroup) { 47 | self.group = group 48 | self.mask = mask 49 | } 50 | } 51 | 52 | struct Contact : Sendable { 53 | let point: simd_float3 54 | //let normal: simd_float3 55 | let impulse: Float 56 | let impulseDirection: simd_float3 57 | let penetrationDistance: Float 58 | } 59 | 60 | class PhysicsShape { 61 | let type: PhysicsShapeType 62 | 63 | init(boxWithExtents extents: simd_float3) { 64 | self.type = .box(extents) 65 | } 66 | 67 | init(sphereWithRadius radius: Float) { 68 | self.type = .sphere(radius) 69 | } 70 | 71 | init(convexHullFromMesh mesh: Mesh) { 72 | self.type = .convexHull(mesh) 73 | } 74 | 75 | init(concaveMeshFromMesh mesh: Mesh) { 76 | self.type = .concaveMesh(mesh) 77 | } 78 | } 79 | 80 | struct PhysicsBody { 81 | let mode: PhysicsBodyMode 82 | let shape: PhysicsShape 83 | let mass: Float // For dynamic/kinematic bodies, mass=0 tells the system to calculate the mass. Ignored otherwise. 84 | let friction: Float 85 | let restitution: Float 86 | let filter: CollisionFilter = .default 87 | let isAffectedByGravity: Bool = true 88 | 89 | init(mode: PhysicsBodyMode, 90 | shape: PhysicsShape, 91 | mass: Float = 0.0, 92 | friction: Float = 0.5, 93 | restitution: Float = 0.0) 94 | { 95 | self.mode = mode 96 | self.shape = shape 97 | self.mass = mass 98 | self.friction = friction 99 | self.restitution = restitution 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /Engine/ShaderTypes.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #if __METAL_VERSION__ 22 | #define CONST constant 23 | #else 24 | #define CONST static const 25 | #endif 26 | 27 | CONST unsigned int LightTypeDirectional = 0; 28 | CONST unsigned int LightTypePoint = 1; 29 | CONST unsigned int LightTypeSpot = 2; 30 | 31 | // We reserve vertex buffer indices 0-3 for vertex attributes, 32 | // since different materials and meshes may prefer different layouts. 33 | CONST unsigned int VertexBufferPassConstants = 4; 34 | CONST unsigned int VertexBufferInstanceConstants = 5; 35 | CONST unsigned int VertexBufferSkinningJointTransforms = 6; 36 | CONST unsigned int VertexBufferSkinningVerticesOut = 16; 37 | 38 | CONST unsigned int FragmentBufferPassConstants = 0; 39 | CONST unsigned int FragmentBufferMaterialConstants = 1; 40 | CONST unsigned int FragmentBufferLights = 2; 41 | 42 | CONST unsigned int FragmentTextureBaseColor = 0; 43 | CONST unsigned int FragmentTextureNormal = 1; 44 | CONST unsigned int FragmentTextureMetalness = 2; 45 | CONST unsigned int FragmentTextureRoughness = 3; 46 | CONST unsigned int FragmentTextureEmissive = 4; 47 | CONST unsigned int FragmentTextureEnvironmentLight = 30; 48 | 49 | struct Frustum { 50 | simd_float4 planes[6]; 51 | }; 52 | 53 | #define MAX_VIEW_COUNT 2 54 | 55 | struct PassConstants { 56 | simd_float4x4 viewMatrices[MAX_VIEW_COUNT]; 57 | simd_float4x4 projectionMatrices[MAX_VIEW_COUNT]; 58 | simd_float3 cameraPositions[MAX_VIEW_COUNT]; // world space 59 | simd_float4x4 environmentLightMatrix; 60 | unsigned int activeLightCount; 61 | }; 62 | 63 | struct InstanceConstants { 64 | simd_float4x4 modelMatrix; 65 | simd_float3x3 normalMatrix; 66 | }; 67 | 68 | struct PBRMaterialConstants { 69 | simd_float4 baseColorFactor; 70 | simd_float3 emissiveColor; 71 | float normalScale; 72 | float metallicFactor; 73 | float roughnessFactor; 74 | float emissiveStrength; 75 | } ; 76 | 77 | struct PBRLight { 78 | simd_float3 direction; 79 | simd_float3 position; 80 | simd_float3 color; 81 | float range; 82 | float intensity; 83 | float innerConeCos; 84 | float outerConeCos; 85 | unsigned int type; 86 | }; 87 | -------------------------------------------------------------------------------- /Engine/Skinning.metal: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #include 18 | using namespace metal; 19 | 20 | #include "ShaderTypes.h" 21 | 22 | struct SkinnedVertexIn { 23 | float3 position [[attribute(0)]]; 24 | float3 normal [[attribute(1)]]; 25 | float2 texCoords [[attribute(2)]]; 26 | float4 jointWeights [[attribute(3)]]; 27 | ushort4 jointIndices [[attribute(4)]]; 28 | }; 29 | 30 | // Since we use a fixed vertex layout for post-skinned vertices, this 31 | // structure must exactly match Mesh.postSkinningVertexDescriptor. 32 | struct SkinnedVertexOut { 33 | packed_float3 position; 34 | packed_float3 normal; 35 | packed_float2 texCoords; 36 | }; 37 | 38 | [[vertex]] 39 | void vertex_skin(SkinnedVertexIn in [[stage_in]], 40 | constant float4x4 *jointTransforms [[buffer(VertexBufferSkinningJointTransforms)]], 41 | device SkinnedVertexOut *outVertices [[buffer(VertexBufferSkinningVerticesOut)]], 42 | uint vertexID [[vertex_id]]) 43 | { 44 | float4 weights = in.jointWeights; 45 | //float weightSum = weights[0] + weights[1] + weights[2] + weights[3]; 46 | //weights /= weightSum; 47 | 48 | float4x4 skinningMatrix = weights[0] * jointTransforms[in.jointIndices[0]] + 49 | weights[1] * jointTransforms[in.jointIndices[1]] + 50 | weights[2] * jointTransforms[in.jointIndices[2]] + 51 | weights[3] * jointTransforms[in.jointIndices[3]]; 52 | 53 | float3 skinnedPosition = (skinningMatrix * float4(in.position, 1.0f)).xyz; 54 | // n.b. We'd ordinarily use the inverse transpose of the skinning matrix here to 55 | // get correct normal transformation, but if we assume uniform scale (which skinning 56 | // matrices almost always have), the inverse transpose produces the same vector 57 | // as the matrix itself, up to scale, so we avoid the expense of inverting. 58 | float3 skinnedNormal = normalize((skinningMatrix * float4(in.normal, 0.0f)).xyz); 59 | 60 | outVertices[vertexID].position = skinnedPosition; 61 | outVertices[vertexID].normal = skinnedNormal; 62 | outVertices[vertexID].texCoords = in.texCoords; 63 | } 64 | -------------------------------------------------------------------------------- /Engine/SpatialEvent.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import SwiftUI 18 | import Spatial 19 | 20 | public enum Event : Sendable { 21 | case worldAnchor(WorldAnchor, AnchorEvent) 22 | case meshAnchor(MeshAnchor, AnchorEvent) 23 | case planeAnchor(PlaneAnchor, AnchorEvent) 24 | case handAnchor(HandAnchor, AnchorEvent) 25 | case environmentLightAnchor(EnvironmentLightAnchor, AnchorEvent) 26 | case spatialInput(SpatialInputEvent) 27 | } 28 | 29 | public struct SpatialInputEvent : Identifiable, Hashable, Sendable { 30 | public typealias ID = UUID 31 | 32 | public var id: ID 33 | 34 | public enum Kind : Hashable, Sendable { 35 | case touch 36 | case directPinch 37 | case indirectPinch 38 | case pointer 39 | } 40 | 41 | public enum Phase : Hashable, Sendable { 42 | case active 43 | case ended 44 | case cancelled 45 | } 46 | 47 | public struct InputDevicePose : Hashable, Sendable { 48 | public var altitude: Angle2D 49 | public var azimuth: Angle2D 50 | public var pose3D: Pose3D 51 | } 52 | 53 | public var timestamp: TimeInterval 54 | public var kind: SpatialInputEvent.Kind 55 | public var location: CGPoint 56 | public var phase: SpatialInputEvent.Phase 57 | public var inputDevicePose: SpatialInputEvent.InputDevicePose? 58 | public var location3D: Point3D 59 | public var selectionRay: Ray3D? 60 | public var handedness: Handedness 61 | } 62 | 63 | extension SpatialInputEvent { 64 | init(_ event: SpatialEventCollection.Event) { 65 | self.id = UUID() 66 | self.timestamp = event.timestamp 67 | self.kind = switch event.kind { 68 | case .touch: .touch 69 | case .directPinch: .directPinch 70 | case .indirectPinch: .indirectPinch 71 | case .pointer: .pointer 72 | @unknown default: fatalError() 73 | } 74 | self.location = event.location 75 | self.phase = switch event.phase { 76 | case .active: .active 77 | case .ended: .ended 78 | case .cancelled: .cancelled 79 | @unknown default: fatalError() 80 | } 81 | #if os(visionOS) 82 | if let pose = event.inputDevicePose { 83 | self.inputDevicePose = InputDevicePose(altitude: Angle2D(radians: pose.altitude.radians), 84 | azimuth: Angle2D(radians: pose.azimuth.radians), 85 | pose3D: pose.pose3D) 86 | } 87 | self.location3D = event.location3D 88 | self.selectionRay = event.selectionRay 89 | self.handedness = switch event.chirality { 90 | case .left: .left 91 | case .right: .right 92 | case .none: .none 93 | } 94 | #else 95 | self.location3D = .zero 96 | self.handedness = .none 97 | #endif 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /Engine/SpatialRenderLoop.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #if os(visionOS) 18 | import ARKit 19 | import CompositorServices 20 | import Metal 21 | import SwiftUI 22 | 23 | extension LayerRenderer.Drawable : FrameResourceProvider { 24 | var storeDepth: Bool { 25 | // It is important that we provide accurate depth information to 26 | // Compositor Services so it can accurately reproject content, 27 | // especially in mixed immersion ("passthrough") mode. 28 | true 29 | } 30 | } 31 | 32 | extension LayerRenderer.Clock.Instant { 33 | var timeInterval: TimeInterval { 34 | let components = LayerRenderer.Clock.Instant.epoch.duration(to: self).components 35 | return TimeInterval(components.seconds) + TimeInterval(components.attoseconds) * 1e-18 36 | } 37 | } 38 | 39 | class SpatialRenderLoop { 40 | let context: MetalContext 41 | let renderer: SceneRenderer 42 | let scene: SceneContent 43 | let sessionManager: ARSessionManager 44 | 45 | private let clock = LayerRenderer.Clock() 46 | private var layerRenderer: LayerRenderer! 47 | private var lastFramePresentationTime: TimeInterval 48 | 49 | init(scene: SceneContent, renderer: SceneRenderer, sessionManager: ARSessionManager) { 50 | self.renderer = renderer 51 | self.scene = scene 52 | self.context = renderer.context 53 | self.sessionManager = sessionManager 54 | 55 | lastFramePresentationTime = clock.now.timeInterval 56 | } 57 | 58 | func run(_ layerRenderer: LayerRenderer) async { 59 | self.layerRenderer = layerRenderer 60 | 61 | var isRendering = true 62 | while isRendering { 63 | switch layerRenderer.state { 64 | case .paused: 65 | print("Layer renderer is in paused state.") 66 | layerRenderer.waitUntilRunning() 67 | print("Layer renderer is running.") 68 | case .running: 69 | autoreleasepool { 70 | frame() 71 | } 72 | case .invalidated: 73 | print("Layer renderer was invalidated.") 74 | fallthrough 75 | default: 76 | isRendering = false 77 | } 78 | } 79 | } 80 | 81 | func frame() { 82 | guard let frame = layerRenderer.queryNextFrame() else { return } 83 | guard let predictedTiming = frame.predictTiming() else { return } 84 | 85 | // Calculate timestep from previous frame to this frame, capping it at 33 ms so 86 | // animations don't jump way ahead in case we were paused or had a long hitch. 87 | let deltaTime = min(predictedTiming.presentationTime.timeInterval - lastFramePresentationTime, 1.0 / 30.0) 88 | 89 | frame.startUpdate() 90 | updateHandAnchors(at: predictedTiming.trackableAnchorTime.timeInterval) 91 | scene.update(deltaTime) 92 | frame.endUpdate() 93 | 94 | clock.wait(until: predictedTiming.optimalInputTime) 95 | 96 | frame.startSubmission() 97 | 98 | guard let drawable = frame.queryDrawable() else { return } 99 | let finalTiming = drawable.frameTiming 100 | let presentationTimestamp = finalTiming.presentationTime.timeInterval 101 | let deviceAnchor = sessionManager.queryDeviceAnchor(at: presentationTimestamp) 102 | drawable.deviceAnchor = deviceAnchor 103 | if let deviceAnchor { 104 | renderFrame(frame: frame, drawable: drawable, deviceAnchor: deviceAnchor) 105 | } 106 | 107 | let commandBuffer = context.commandQueue.makeCommandBuffer()! 108 | commandBuffer.label = "Present Drawable" 109 | drawable.encodePresent(commandBuffer: commandBuffer) 110 | commandBuffer.commit() 111 | 112 | lastFramePresentationTime = presentationTimestamp 113 | 114 | frame.endSubmission() 115 | } 116 | 117 | private func updateHandAnchors(at time: TimeInterval) { 118 | let (maybeLeftHand, maybeRightHand) = sessionManager.queryHandAnchors(at: time) 119 | if let hand = maybeLeftHand { 120 | scene.enqueueEvent(.handAnchor(HandAnchor(hand), .updated)) 121 | } 122 | if let hand = maybeRightHand { 123 | scene.enqueueEvent(.handAnchor(HandAnchor(hand), .updated)) 124 | } 125 | } 126 | 127 | func renderFrame(frame: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, deviceAnchor: DeviceAnchor) { 128 | let viewCount = drawable.views.count 129 | let deviceTransform = deviceAnchor.originFromAnchorTransform 130 | let cameraTransforms = drawable.views.map { view in deviceTransform * view.transform } 131 | let viewMatrices = cameraTransforms.map(\.inverse) 132 | let projectionMatrices = (0.. MTLTexture { 38 | let device = context.device 39 | 40 | let bytesPerPixel = 4 41 | var pixelFormat = MTLPixelFormat.rgba8Unorm 42 | 43 | if let srgbOption = options?[.sRGB] as? NSNumber { 44 | if srgbOption.boolValue { 45 | pixelFormat = MTLPixelFormat.rgba8Unorm_srgb 46 | } 47 | } 48 | 49 | var storageMode = MTLStorageMode.shared 50 | #if os(macOS) 51 | if !device.hasUnifiedMemory { 52 | storageMode = MTLStorageMode.managed 53 | } 54 | #endif 55 | if let storageModeOption = options?[.storageMode] as? NSNumber { 56 | if let preferredStorageMode = MTLStorageMode(rawValue: storageModeOption.uintValue) { 57 | storageMode = preferredStorageMode 58 | } 59 | } 60 | 61 | var usageMode = MTLTextureUsage.shaderRead 62 | if let usageModeOption = options?[.usageMode] as? NSNumber { 63 | let preferredUsageMode = MTLTextureUsage(rawValue: usageModeOption.uintValue) 64 | usageMode = preferredUsageMode 65 | } 66 | 67 | var wantsMipmaps = false 68 | if let mipmapOption = options?[.generateMipmaps] as? NSNumber { 69 | wantsMipmaps = mipmapOption.boolValue 70 | } 71 | 72 | var flipVertically = false 73 | if let flipOption = options?[.flipVertically] as? NSNumber { 74 | flipVertically = flipOption.boolValue 75 | } 76 | 77 | let getImageData: () -> Data? = { 78 | if flipVertically { 79 | return mdlTexture.texelDataWithBottomLeftOrigin(atMipLevel: 0, create: true) 80 | } else { 81 | return mdlTexture.texelDataWithTopLeftOrigin(atMipLevel: 0, create: true) 82 | } 83 | } 84 | 85 | guard let data = getImageData() else { 86 | throw ResourceError.imageLoadFailure 87 | } 88 | 89 | let width = Int(mdlTexture.dimensions.x) 90 | let height = Int(mdlTexture.dimensions.y) 91 | 92 | let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: pixelFormat, 93 | width: width, 94 | height: height, 95 | mipmapped: wantsMipmaps) 96 | textureDescriptor.usage = usageMode 97 | textureDescriptor.storageMode = storageMode 98 | 99 | guard let texture = device.makeTexture(descriptor: textureDescriptor) else { 100 | throw ResourceError.allocationFailure 101 | } 102 | 103 | let bytesPerRow = width * bytesPerPixel 104 | 105 | let region = MTLRegionMake2D(0, 0, width, height) 106 | data.withUnsafeBytes { imageBytes in 107 | texture.replace(region: region, mipmapLevel: 0, withBytes: imageBytes.baseAddress!, bytesPerRow: bytesPerRow) 108 | } 109 | 110 | if wantsMipmaps { 111 | let commandQueue = context.commandQueue 112 | if let commandBuffer = commandQueue.makeCommandBuffer() { 113 | if let mipmapEncoder = commandBuffer.makeBlitCommandEncoder() { 114 | mipmapEncoder.generateMipmaps(for: texture) 115 | mipmapEncoder.endEncoding() 116 | } 117 | commandBuffer.commit() 118 | } 119 | } 120 | 121 | return texture 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /Engine/TextureResource.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2024 Warren Moore 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | import Metal 18 | 19 | class TextureResource { 20 | enum ContentType { 21 | case raw 22 | case color 23 | } 24 | 25 | let texture: MTLTexture 26 | let sampler: MTLSamplerState? 27 | 28 | init(texture: MTLTexture, sampler: MTLSamplerState? = nil) { 29 | self.texture = texture 30 | self.sampler = sampler 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 13 | 14 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 15 | 16 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 17 | 18 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 19 | 20 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 21 | 22 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 23 | 24 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 25 | 26 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 27 | 28 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 29 | 30 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 31 | 32 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 33 | 34 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 35 | 36 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 37 | You must cause any modified files to carry prominent notices stating that You changed the files; and 38 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 39 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 40 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 41 | 42 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 43 | 44 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 45 | 46 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 47 | 48 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 49 | 50 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 51 | 52 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spatial Rendering for Apple Vision Pro 2 | 3 | **Important**: This repository uses git submodules. Use `--recursive` when cloning. e.g.: 4 | 5 | ``` 6 | $ git clone --recursive --depth 1 git@github.com:metal-by-example/spatial-rendering.git 7 | ``` 8 | 9 | ## Talk 10 | 11 | A recording of the talk for which this sample was created is available on [YouTube](https://youtu.be/vO0M4c9mb2E). 12 | 13 | See also the slides that accompanied this talk: 14 | 15 | [![Title Slide](images/title-slide.png)](https://speakerdeck.com/warrenm/spatial-rendering-for-apple-vision-pro) 16 | 17 | ## Overview 18 | 19 | Modern RealityKit is a powerful framework for building spatial experiences, but sometimes you need to use lower level facilities like ARKit, Metal, and Compositor Services. 20 | 21 | This sample is a medium-scale example of how to use various features from ARKit and Metal to perform spatial rendering in a passthrough style. It allows the user to scan their environment for flat surfaces and place a wooden block tower, then interact with it with their hands. 22 | 23 | ## Usage 24 | 25 | It is recommended to run this app on Vision Pro hardware rather than the simulator. Although it will run on the simulator, it has no interactive features in this context because of its reliance on hand tracking. 26 | 27 | To build and run the app, you will need to ensure that the dependency on the Jolt physics engine is fulfilled (ideally by cloning recursively, or by a subsequent `git submodule init & git submodule update`)—just downloading the code from GitHub is **not** sufficient. 28 | 29 | You will also need to set your team identifier in the project in order for code-signing to work. 30 | 31 | 32 | -------------------------------------------------------------------------------- /Resources/Blocks.usdz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Blocks.usdz -------------------------------------------------------------------------------- /Resources/Hand_Left.usdz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Hand_Left.usdz -------------------------------------------------------------------------------- /Resources/Hand_Right.usdz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Hand_Right.usdz -------------------------------------------------------------------------------- /Resources/Hand_Right_backup.usdz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Hand_Right_backup.usdz -------------------------------------------------------------------------------- /Resources/LICENSE: -------------------------------------------------------------------------------- 1 | Hand models, obtained from https://github.com/immersive-web/webxr-input-profiles, 2 | have been converted from glTF format to USDZ format and are subject to the 3 | 4 | MIT License 5 | 6 | Copyright (c) 2019 Amazon 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is furnished 13 | to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice (including the next 16 | paragraph) shall be included in all copies or substantial portions of the 17 | Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 21 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS 22 | OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF 24 | OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | 26 | Wooden block models, obtained from 27 | https://sketchfab.com/3d-models/efd543fd64e041a8937baa3c8a4c851e 28 | are originally by Sketchfab user Anushka (https://sketchfab.com/nerdysickness) 29 | and subject to the Creative Commons Attribution License 4.0: 30 | https://creativecommons.org/licenses/by/4.0/ 31 | -------------------------------------------------------------------------------- /Resources/Placement_Reticle.usdz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Placement_Reticle.usdz -------------------------------------------------------------------------------- /SpatialRendering.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /SpatialRendering.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /SpatialRendering.xcodeproj/xcshareddata/xcschemes/SpatialDemo.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 9 | 10 | 16 | 22 | 23 | 24 | 25 | 26 | 32 | 33 | 44 | 46 | 52 | 53 | 54 | 55 | 61 | 63 | 69 | 70 | 71 | 72 | 74 | 75 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /SpatialRendering.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /ThirdParty/JoltPhysics.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | BuildSystemType 6 | Latest 7 | 8 | 9 | -------------------------------------------------------------------------------- /ThirdParty/JoltPhysics.xcodeproj/xcshareddata/xcschemes/Jolt.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 9 | 10 | 16 | 22 | 23 | 24 | 25 | 26 | 32 | 33 | 43 | 44 | 50 | 51 | 57 | 58 | 59 | 60 | 62 | 63 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /images/title-slide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/images/title-slide.png --------------------------------------------------------------------------------