├── .gitignore
├── .gitmodules
├── App
├── App.swift
├── Assets.xcassets
│ ├── AccentColor.colorset
│ │ └── Contents.json
│ ├── AppIcon.appiconset
│ │ └── Contents.json
│ └── Contents.json
├── Entitlements.entitlements
├── SpatialDemo-Info.plist
└── SpatialScene.swift
├── Engine
├── ARKitAnchorSupport.swift
├── ARKitMeshSupport.swift
├── ARSessionManager.swift
├── Anchor.swift
├── Bridging.h
├── BufferUtilities.swift
├── Camera.swift
├── Entity.swift
├── Light.swift
├── LockingQueue.swift
├── Material.swift
├── Materials.metal
├── Math.swift
├── Mesh.swift
├── MetalContext.swift
├── Model.swift
├── ModelIOSupport.swift
├── Physics
│ ├── JoltPhysics.h
│ ├── JoltPhysics.mm
│ ├── JoltPhysics.swift
│ └── PhysicsBridge.swift
├── PhysicsBody.swift
├── SceneRenderer.swift
├── ShaderTypes.h
├── Skinning.metal
├── SpatialEvent.swift
├── SpatialRenderLoop.swift
├── TextureLoader.swift
└── TextureResource.swift
├── LICENSE
├── README.md
├── Resources
├── Blocks.usdz
├── Hand_Left.usdz
├── Hand_Right.usdz
├── Hand_Right_backup.usdz
├── LICENSE
└── Placement_Reticle.usdz
├── SpatialRendering.xcodeproj
├── project.pbxproj
├── project.xcworkspace
│ ├── contents.xcworkspacedata
│ └── xcshareddata
│ │ └── IDEWorkspaceChecks.plist
└── xcshareddata
│ └── xcschemes
│ └── SpatialDemo.xcscheme
├── SpatialRendering.xcworkspace
└── contents.xcworkspacedata
├── ThirdParty
└── JoltPhysics.xcodeproj
│ ├── project.pbxproj
│ ├── project.xcworkspace
│ └── xcshareddata
│ │ └── WorkspaceSettings.xcsettings
│ └── xcshareddata
│ └── xcschemes
│ └── Jolt.xcscheme
└── images
└── title-slide.png
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | xcuserdata
3 |
--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "ThirdParty/JoltPhysics"]
2 | path = ThirdParty/JoltPhysics
3 | url = https://github.com/jrouwe/JoltPhysics
4 |
--------------------------------------------------------------------------------
/App/App.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import SwiftUI
18 |
19 | #if os(visionOS)
20 | import CompositorServices
21 |
22 | extension RenderLayout {
23 | init (_ layout: LayerRenderer.Layout) {
24 | switch layout {
25 | case .shared:
26 | self = .shared
27 | case .layered:
28 | self = .layered
29 | case .dedicated:
30 | fallthrough
31 | default:
32 | self = .dedicated
33 | }
34 | }
35 | }
36 |
37 | // A type that can be used to customize the creation of a LayerRenderer
38 | struct LayerConfiguration : CompositorLayerConfiguration {
39 | let context: MetalContext
40 |
41 | func makeConfiguration(capabilities: LayerRenderer.Capabilities,
42 | configuration: inout LayerRenderer.Configuration)
43 | {
44 | // Enable foveated rendering if available
45 | let supportsFoveation = capabilities.supportsFoveation
46 | configuration.isFoveationEnabled = supportsFoveation
47 |
48 | // Select a layout based on device capabilities
49 | let canUseAmplification = context.device.supportsVertexAmplificationCount(2)
50 | let supportedLayouts = capabilities.supportedLayouts(options: supportsFoveation ? .foveationEnabled : [])
51 | if supportedLayouts.contains(.layered) {
52 | // Layered rendering isn't supported everywhere, but when available, it is the most efficient option.
53 | configuration.layout = .layered
54 | print("Selected layered renderer layout")
55 | } else if canUseAmplification && !configuration.isFoveationEnabled {
56 | // On platforms where layered rendering isn't supported, we can use a
57 | // shared layout, provided we don't use foeveated rendering.
58 | configuration.layout = .shared
59 | print("Selected shared renderer layout")
60 | } else {
61 | // If all else fails, we can use a dedicated layout, but this requires one pass per view.
62 | configuration.layout = .dedicated
63 | print("Selected dedicated renderer layout")
64 | }
65 |
66 | // Use the global preferred pixel formats as our drawable formats
67 | configuration.colorFormat = context.preferredColorPixelFormat
68 | configuration.depthFormat = context.preferredDepthPixelFormat
69 | }
70 | }
71 |
72 | @main
73 | struct SpatialApp: App {
74 | @State var context = MetalContext.shared
75 | @State var selectedImmersionStyle: (any ImmersionStyle) = .mixed
76 |
77 | init() {
78 | SRJPhysicsWorld.initializeJoltPhysics()
79 | }
80 |
81 | var body: some Scene {
82 | ImmersiveSpace(id: "Immersive Space") {
83 | // The content of this immersive space is a compositor layer, which
84 | // allows us to render mixed and fully immersive 3D content via a
85 | // layer renderer
86 | CompositorLayer(configuration: LayerConfiguration(context: context)) { layerRenderer in
87 | // Spin up a high-priority task so updates and rendering happen off the main thread
88 | Task.detached(priority: .high) {
89 | await runRenderLoop(layerRenderer)
90 | }
91 | }
92 | }
93 | .immersionStyle(selection: $selectedImmersionStyle, in: .mixed, .full)
94 | // Prevent hands from occluding rendered content
95 | .upperLimbVisibility(.hidden)
96 | // Prevent home button from interfering with interaction/gestures
97 | .persistentSystemOverlays(.hidden)
98 | }
99 |
100 | func runRenderLoop(_ layerRenderer: LayerRenderer) async {
101 | do {
102 | // We need an ARKit session to be running in order to determine where
103 | // spatial content should be drawn, so get that going first.
104 | let sessionManager = ARSessionManager()
105 | try await sessionManager.start(options: [.planeDetection, .sceneReconstruction, .handTracking, .lightEstimation])
106 |
107 | // Load scene content
108 | let scene = try SpatialScene(sessionManager: sessionManager, context: context)
109 |
110 | // Create a renderer
111 | let renderer = try SceneRenderer(context: context, layout: RenderLayout(layerRenderer.configuration.layout))
112 |
113 | // Subscribe to world-sensing updates
114 | sessionManager.onWorldAnchorUpdate = { worldAnchor, event in
115 | scene.enqueueEvent(.worldAnchor(worldAnchor, event))
116 | }
117 | sessionManager.onPlaneAnchorUpdate = { planeAnchor, event in
118 | scene.enqueueEvent(.planeAnchor(planeAnchor, event))
119 | }
120 | sessionManager.onMeshAnchorUpdate = { meshAnchor, event in
121 | scene.enqueueEvent(.meshAnchor(meshAnchor, event))
122 | }
123 | sessionManager.onEnvironmentLightAnchorUpdate = { lightAnchor, event in
124 | scene.enqueueEvent(.environmentLightAnchor(lightAnchor, event))
125 | }
126 |
127 | // Subscribe to spatial events from the layer renderer
128 | layerRenderer.onSpatialEvent = { events in
129 | for event in events {
130 | scene.enqueueEvent(.spatialInput(SpatialInputEvent(event)))
131 | }
132 | }
133 |
134 | // Create a render loop to draw our immersive content on a cadence determined by the system
135 | let renderLoop = SpatialRenderLoop(scene: scene, renderer: renderer, sessionManager: sessionManager)
136 | await renderLoop.run(layerRenderer)
137 | } catch {
138 | print("Failed to start render loop: \(error.localizedDescription)")
139 | }
140 | print("Exiting render loop...")
141 | }
142 | }
143 |
144 | #endif
145 |
146 |
--------------------------------------------------------------------------------
/App/Assets.xcassets/AccentColor.colorset/Contents.json:
--------------------------------------------------------------------------------
1 | {
2 | "colors" : [
3 | {
4 | "idiom" : "universal"
5 | }
6 | ],
7 | "info" : {
8 | "author" : "xcode",
9 | "version" : 1
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/App/Assets.xcassets/AppIcon.appiconset/Contents.json:
--------------------------------------------------------------------------------
1 | {
2 | "images" : [
3 | {
4 | "idiom" : "universal",
5 | "platform" : "ios",
6 | "size" : "1024x1024"
7 | },
8 | {
9 | "idiom" : "mac",
10 | "scale" : "1x",
11 | "size" : "16x16"
12 | },
13 | {
14 | "idiom" : "mac",
15 | "scale" : "2x",
16 | "size" : "16x16"
17 | },
18 | {
19 | "idiom" : "mac",
20 | "scale" : "1x",
21 | "size" : "32x32"
22 | },
23 | {
24 | "idiom" : "mac",
25 | "scale" : "2x",
26 | "size" : "32x32"
27 | },
28 | {
29 | "idiom" : "mac",
30 | "scale" : "1x",
31 | "size" : "128x128"
32 | },
33 | {
34 | "idiom" : "mac",
35 | "scale" : "2x",
36 | "size" : "128x128"
37 | },
38 | {
39 | "idiom" : "mac",
40 | "scale" : "1x",
41 | "size" : "256x256"
42 | },
43 | {
44 | "idiom" : "mac",
45 | "scale" : "2x",
46 | "size" : "256x256"
47 | },
48 | {
49 | "idiom" : "mac",
50 | "scale" : "1x",
51 | "size" : "512x512"
52 | },
53 | {
54 | "idiom" : "mac",
55 | "scale" : "2x",
56 | "size" : "512x512"
57 | }
58 | ],
59 | "info" : {
60 | "author" : "xcode",
61 | "version" : 1
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/App/Assets.xcassets/Contents.json:
--------------------------------------------------------------------------------
1 | {
2 | "info" : {
3 | "author" : "xcode",
4 | "version" : 1
5 | }
6 | }
7 |
--------------------------------------------------------------------------------
/App/Entitlements.entitlements:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | com.apple.security.app-sandbox
6 |
7 | com.apple.security.files.user-selected.read-only
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/App/SpatialDemo-Info.plist:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | NSWorldSensingUsageDescription
6 | This app uses world sensing to enable interaction between virtual content and the real world. It does not collect or store any personally identifiable information.
7 | NSHandsTrackingUsageDescription
8 | This app uses hand tracking to enable interaction between virtual content and the user's hands.
9 | UIApplicationSceneManifest
10 |
11 | UIApplicationSupportsMultipleScenes
12 |
13 | UIApplicationPreferredDefaultSceneSessionRole
14 | CPSceneSessionRoleImmersiveSpaceApplication
15 | UISceneConfigurations
16 |
17 | CPSceneSessionRoleImmersiveSpaceApplication
18 |
19 |
20 | UISceneInitialImmersionStyle
21 | UIImmersionStyleMixed
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/App/SpatialScene.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import SwiftUI
19 | import Metal
20 | import MetalKit
21 | import ModelIO
22 |
23 | class SpatialScene : Entity, SceneContent {
24 | enum ScenePhase {
25 | case selectingPlacement
26 | case playing
27 | }
28 | var scenePhase = ScenePhase.selectingPlacement
29 |
30 | let sessionManager: ARSessionManager
31 | let context: MetalContext
32 | let rootEntity: Entity
33 | var anchoredContentRoot: AnchorEntity?
34 |
35 | var entities: [Entity] {
36 | return [Entity](flatEntities)
37 | }
38 | var lights: [Light] = []
39 |
40 | var environmentLight: EnvironmentLight?
41 |
42 | private let physicsBridge: PhysicsBridge
43 | private let eventQueue = LockingQueue()
44 | private var flatEntities = Set()
45 | private var anchorEntities: [UUID : AnchorEntity] = [:]
46 | private var meshEntities: [UUID : Entity] = [:]
47 | private var planeEntities: [UUID : Entity] = [:]
48 | private var handEntities: [Handedness : HandEntity] = [:]
49 | private var environmentMeshMaterial: Material!
50 | private var reticleEntity: Entity?
51 | private var candidatePlanes = Set()
52 | private var selectedPlane: Entity?
53 | private var blockPrototypes = [Entity]()
54 |
55 | #if targetEnvironment(simulator)
56 | let floorLevel: Float = -0.5
57 | #else
58 | let floorLevel: Float = 0.0
59 | #endif
60 |
61 | init(sessionManager: ARSessionManager, context: MetalContext) throws {
62 | self.sessionManager = sessionManager
63 | self.context = context
64 | self.rootEntity = Entity()
65 | self.physicsBridge = JoltPhysicsBridge()
66 |
67 | super.init()
68 |
69 | self.addChild(rootEntity)
70 | try makeScene()
71 | }
72 |
73 | func makeScene() throws {
74 | environmentMeshMaterial = OcclusionMaterial()
75 | environmentMeshMaterial.name = "Occlusion"
76 |
77 | let sunLight = Light(type: .directional, color: simd_float3(1, 1, 1), intensity: 1)
78 | sunLight.transform.look(at: simd_float3(0, 0, 0), from: simd_float3(0.1, 1, 0.1), up: simd_float3(0, 1, 0))
79 | lights.append(sunLight)
80 |
81 | #if targetEnvironment(simulator)
82 | let accentTop = Light(type: .point, color: simd_float3(1, 1, 1), intensity: 25)
83 | accentTop.transform.look(at: simd_float3(0, 0, 0), from: simd_float3(-3, 3, 0), up: simd_float3(0, 1, 0))
84 | lights.append(accentTop)
85 |
86 | let accentBottom = Light(type: .point, color: simd_float3(1, 1, 1), intensity: 25)
87 | accentBottom.transform.look(at: simd_float3(0, 0, 0), from: simd_float3(-3, -3, 0), up: simd_float3(0, 1, 0))
88 | lights.append(accentBottom)
89 | #endif
90 |
91 | if let handURL = Bundle.main.url(forResource: "Hand_Left", withExtension: "usdz") {
92 | let model = try Model(fileURL: handURL, context: context)
93 | if let handArmature = model.rootEntities.first {
94 | let handEntity = HandEntity(handedness: .left, context: context)
95 | handEntity.addChild(handArmature)
96 | rootEntity.addChild(handEntity)
97 | handEntities[.left] = handEntity
98 | handEntity.worldTransform = Transform(position: simd_float3(-0.15, floorLevel + 0.8, -0.15))
99 | if let handMaterial = handEntity.child(named: "Mesh")?.mesh?.materials.first as? PhysicallyBasedMaterial {
100 | handMaterial.blendMode = .sourceOverPremultiplied
101 | handMaterial.isDoubleSided = false
102 | handMaterial.baseColor.baseColorFactor.w = 0.1
103 | }
104 | }
105 | }
106 | if let handURL = Bundle.main.url(forResource: "Hand_Right", withExtension: "usdz") {
107 | let model = try Model(fileURL: handURL, context: context)
108 | if let handArmature = model.rootEntities.first {
109 | let handEntity = HandEntity(handedness: .right, context: context)
110 | handEntity.addChild(handArmature)
111 | rootEntity.addChild(handEntity)
112 | handEntities[.right] = handEntity
113 | handEntity.worldTransform = Transform(position: simd_float3(0.15, floorLevel + 0.8, -0.15))
114 | if let handMaterial = handEntity.child(named: "Mesh")?.mesh?.materials.first as? PhysicallyBasedMaterial {
115 | handMaterial.blendMode = .sourceOverPremultiplied
116 | handMaterial.isDoubleSided = false
117 | handMaterial.baseColor.baseColorFactor.w = 0.1
118 | }
119 | }
120 | }
121 |
122 | if let reticleURL = Bundle.main.url(forResource: "Placement_Reticle", withExtension: "usdz") {
123 | let model = try Model(fileURL: reticleURL, context: context)
124 | // USD tends to create unnecessarily deep prim hierarchies, so just reach in and grab the mesh node.
125 | if let meshEntity = model.rootEntities.first?.child(named: "mesh") {
126 | let reticleTransform = meshEntity.worldTransform
127 | meshEntity.removeFromParent()
128 | meshEntity.modelTransform = reticleTransform
129 | reticleEntity = meshEntity
130 | }
131 | }
132 |
133 | if let modelURL = Bundle.main.url(forResource: "Blocks", withExtension: "usdz") {
134 | let model = try Model(fileURL: modelURL, context: context)
135 | let blockEntities = model.rootEntities.first?.childEntities(matching: { $0.mesh != nil }) ?? []
136 | blockEntities.forEach { entity in
137 | // Since we want to apply physics to each object in the model, we move it to the scene root
138 | // by de-parenting it and assigning its initial world transform as its model transform.
139 | // This allows us to handle object scaling correctly in the rendering and physics systems.
140 | let placementTransform = entity.worldTransform
141 | entity.removeFromParent()
142 | entity.modelTransform = placementTransform
143 | entity.generateCollisionShapes(recursive: false, bodyMode: .dynamic)
144 | }
145 | self.blockPrototypes = blockEntities
146 | }
147 |
148 | // Create a large ground plane so that even if we don't have plane detection
149 | // or scene reconstruction enabled, virtual objects don't fall infinitely far
150 | let floorExtent = simd_float3(10, 0.02, 10)
151 | let groundPlane = Entity()
152 | groundPlane.name = "Ground Plane"
153 | groundPlane.isHidden = true
154 | groundPlane.modelTransform.position = simd_float3(0, floorLevel - floorExtent.y * 0.5, 0)
155 | let groundShape = PhysicsShape(boxWithExtents: floorExtent)
156 | groundPlane.physicsBody = PhysicsBody(mode: .static, shape: groundShape)
157 | rootEntity.addChild(groundPlane)
158 |
159 | #if targetEnvironment(simulator) || !os(visionOS)
160 | selectPlaneForPlacement(groundPlane, pose: Transform())
161 | #endif
162 | }
163 |
164 | func enqueueEvent(_ event: Event) {
165 | eventQueue.enqueue(event)
166 | }
167 |
168 | override func update(_ timestep: TimeInterval) {
169 | for event in eventQueue.popAll() {
170 | switch event {
171 | case .worldAnchor(let anchor, let anchorEvent):
172 | handleWorldAnchorEvent(anchor, event: anchorEvent)
173 | case .meshAnchor(let anchor, let anchorEvent):
174 | handleMeshAnchorEvent(anchor, event: anchorEvent)
175 | case .planeAnchor(let anchor, let anchorEvent):
176 | handlePlaneAnchorEvent(anchor, event: anchorEvent)
177 | case .handAnchor(let anchor, let anchorEvent):
178 | handleHandAnchorEvent(anchor, event: anchorEvent)
179 | case .spatialInput(let spatialEvents):
180 | handleSpatialInput(spatialEvents)
181 | case .environmentLightAnchor(let anchor, let anchorEvent):
182 | handleEnvironmentLightEvent(anchor, event: anchorEvent)
183 | }
184 | }
185 |
186 | for entity in entities {
187 | entity.update(timestep)
188 | }
189 |
190 | physicsBridge.update(entities: entities, timestep: timestep)
191 | }
192 |
193 | override func didAddChild(_ entity: Entity) {
194 | let flattenedSubtree = flattenedHierarchy(entity)
195 | for element in flattenedSubtree {
196 | flatEntities.insert(element)
197 | physicsBridge.addEntity(element)
198 | }
199 | super.didAddChild(entity)
200 | }
201 |
202 | override func didRemoveChild(_ entity: Entity) {
203 | let flattenedSubtree = flattenedHierarchy(entity)
204 | for element in flattenedSubtree {
205 | flatEntities.remove(element)
206 | physicsBridge.removeEntity(element)
207 | }
208 | super.didRemoveChild(entity)
209 | }
210 |
211 | private func updatePhysicsShapeForStaticEntity(_ entity: Entity) {
212 | physicsBridge.removeEntity(entity)
213 | if let mesh = entity.mesh {
214 | let shape = PhysicsShape(concaveMeshFromMesh: mesh)
215 | entity.physicsBody = PhysicsBody(mode: .static, shape: shape)
216 | } else {
217 | entity.physicsBody = nil
218 | }
219 | physicsBridge.addEntity(entity)
220 | }
221 |
222 | func handlePlaneAnchorEvent(_ anchor: PlaneAnchor, event: AnchorEvent) {
223 | let desiredPlacements: [PlaneAnchor.Classification] = [.table]
224 | switch event {
225 | case .added:
226 | let planeEntity = Entity()
227 | planeEntity.name = "Plane \(anchor.id)"
228 | planeEntity.worldTransform = Transform(anchor.originFromAnchorTransform)
229 | planeEntity.mesh = anchor.mesh
230 | planeEntity.mesh?.materials = [environmentMeshMaterial]
231 | planeEntities[anchor.id] = planeEntity
232 | if scenePhase == .selectingPlacement {
233 | if desiredPlacements.contains(anchor.classification) {
234 | candidatePlanes.insert(planeEntity)
235 | if let planeReticle = reticleEntity?.clone(recursively: false) {
236 | let reticlePivot = Entity(transform: Transform(position: simd_float3(0, 0.005, 0)))
237 | reticlePivot.addChild(planeReticle)
238 | planeEntity.addChild(reticlePivot)
239 | }
240 | }
241 | }
242 | rootEntity.addChild(planeEntity)
243 | case .updated:
244 | if let planeEntity = planeEntities[anchor.id] {
245 | planeEntity.worldTransform = Transform(anchor.originFromAnchorTransform)
246 | planeEntity.mesh = anchor.mesh
247 | planeEntity.mesh?.materials = [environmentMeshMaterial]
248 | updatePhysicsShapeForStaticEntity(planeEntity)
249 | }
250 | case .removed:
251 | if let planeEntity = planeEntities[anchor.id] {
252 | planeEntities.removeValue(forKey: anchor.id)
253 | candidatePlanes.remove(planeEntity)
254 | planeEntity.removeFromParent()
255 | }
256 | }
257 | }
258 |
259 | func handleMeshAnchorEvent(_ anchor: MeshAnchor, event: AnchorEvent) {
260 | switch event {
261 | case .added:
262 | let meshEntity = Entity()
263 | meshEntity.name = "World Mesh \(anchor.id)"
264 | meshEntity.worldTransform = Transform(anchor.originFromAnchorTransform)
265 | meshEntity.mesh = anchor.mesh
266 | meshEntity.mesh?.materials = [environmentMeshMaterial]
267 | meshEntities[anchor.id] = meshEntity
268 | rootEntity.addChild(meshEntity)
269 | case .updated:
270 | if let meshEntity = meshEntities[anchor.id] {
271 | meshEntity.worldTransform = Transform(anchor.originFromAnchorTransform)
272 | meshEntity.mesh = anchor.mesh
273 | meshEntity.mesh?.materials = [environmentMeshMaterial]
274 | updatePhysicsShapeForStaticEntity(meshEntity)
275 | }
276 | case .removed:
277 | if let meshEntity = meshEntities[anchor.id] {
278 | meshEntities.removeValue(forKey: anchor.id)
279 | meshEntity.removeFromParent()
280 | }
281 | }
282 | }
283 |
284 | func handleHandAnchorEvent(_ anchor: HandAnchor, event: AnchorEvent) {
285 | switch event {
286 | case .added:
287 | if let handEntity = handEntities[anchor.handedness] {
288 | handEntity.isHidden = false
289 | handEntity.updatePose(from: anchor)
290 | }
291 | case .updated:
292 | if let handEntity = handEntities[anchor.handedness] {
293 | handEntity.updatePose(from: anchor)
294 | }
295 | case .removed:
296 | if let handEntity = handEntities[anchor.handedness] {
297 | handEntity.isHidden = true
298 | }
299 | }
300 | }
301 |
302 | func handleWorldAnchorEvent(_ anchor: WorldAnchor, event: AnchorEvent) {
303 | switch event {
304 | case .added:
305 | if let anchorEntity = anchorEntities[anchor.id] {
306 | anchorEntity.updatePose(from: anchor)
307 | } else {
308 | let anchorEntity = AnchorEntity(transform: Transform(anchor.originFromAnchorTransform))
309 | anchorEntity.name = "Anchor \(anchor.id)"
310 | anchorEntities[anchor.id] = anchorEntity
311 | }
312 | case .updated:
313 | if let anchorEntity = anchorEntities[anchor.id] {
314 | anchorEntity.updatePose(from: anchor)
315 | }
316 | case .removed:
317 | anchorEntities.removeValue(forKey: anchor.id)
318 | }
319 | }
320 |
321 | private func handleEnvironmentLightEvent(_ anchor: EnvironmentLightAnchor, event: AnchorEvent) {
322 | switch event {
323 | case .added:
324 | fallthrough
325 | case .updated:
326 | // Only bother updating environmental lighting once we have an environment cube map
327 | if anchor.light.environmentTexture != nil {
328 | self.environmentLight = anchor.light
329 | }
330 | case .removed:
331 | // Although it might be more correct to remove environment lights when their
332 | // corresponding anchors go away, we prefer to use environment lighting even
333 | // if it's stale, so we retain the most recent update regardless. Handling this
334 | // comprehensively means tracking all environment probe anchors, since ARKit
335 | // may create multiple probes as the device moves around the environment.
336 | break
337 | }
338 | }
339 |
340 | private func handleSpatialInput(_ event: SpatialInputEvent) {
341 | switch event.phase {
342 | case .active:
343 | break
344 | case .ended:
345 | if let ray = event.selectionRay {
346 | let origin = simd_float3(ray.origin)
347 | let toward = simd_float3(ray.direction)
348 | let hitResults = physicsBridge.hitTestWithSegment(from: origin, to: origin + toward * 3.0)
349 | for hit in hitResults.sorted(by: { simd_length($0.worldPosition - origin) < simd_length($1.worldPosition - origin)}) {
350 | if scenePhase == .selectingPlacement {
351 | if let hitEntity = hit.entity, candidatePlanes.contains(hitEntity) {
352 | selectPlaneForPlacement(hitEntity, pose: Transform(position: hit.worldPosition))
353 | }
354 | }
355 | }
356 | }
357 | case .cancelled:
358 | break
359 | @unknown default:
360 | break
361 | }
362 | }
363 |
364 | private func selectPlaneForPlacement(_ planeEntity: Entity, pose: Transform) {
365 | selectedPlane = planeEntity
366 | scenePhase = .playing
367 | print("Transitioned to playing state with selected plane \(selectedPlane?.name ?? "")")
368 | Task {
369 | await buildTower(pose)
370 | }
371 | }
372 |
373 | private func buildTower(_ pose: Transform) async {
374 | guard blockPrototypes.count > 0 else {
375 | print("Failed to load block models; cannot construct tower")
376 | return
377 | }
378 |
379 | do {
380 | try await sessionManager.removeAllWorldAnchors()
381 | let worldAnchor = try await sessionManager.addWorldAnchor(originFromAnchorTransform: pose.matrix)
382 | let towerAnchor = AnchorEntity()
383 | towerAnchor.name = "Tower Anchor"
384 | towerAnchor.updatePose(from: worldAnchor)
385 | anchorEntities[worldAnchor.id] = towerAnchor
386 | rootEntity.addChild(towerAnchor)
387 | self.anchoredContentRoot = towerAnchor
388 | } catch {
389 | print("Could not create world anchor to place tower; creating default placement")
390 | let towerAnchor = AnchorEntity(transform: Transform(position: simd_float3(0, 0, -0.5)))
391 | rootEntity.addChild(towerAnchor)
392 | self.anchoredContentRoot = towerAnchor
393 | }
394 |
395 | let layerCount = 7
396 | let lateralMargin: Float = 0.0025
397 | let verticalMargin: Float = 0.005
398 |
399 | // Calculate the block bounds, assuming all prototypes have the same size
400 | let defaultBlockBounds = blockPrototypes[0].mesh!.boundingBox
401 | let blockWidth = defaultBlockBounds.max.x - defaultBlockBounds.min.x
402 | let blockHeight = defaultBlockBounds.max.y - defaultBlockBounds.min.y
403 | let blockDepth = defaultBlockBounds.max.z - defaultBlockBounds.min.z
404 |
405 | let rotation = Transform(orientation: simd_quatf(angle: .pi / 2, axis: simd_float3(0, 1, 0)))
406 | var layerIsRotated = false
407 | var yPosition = blockHeight * 0.5
408 | var blockNumber = 0
409 | for _ in 0...stride
49 |
50 | let positionView = BufferView(buffer: geometry.vertices.buffer, offset: geometry.vertices.offset)
51 | let normalView = BufferView(buffer: geometry.normals.buffer, offset: geometry.normals.offset)
52 | let texCoords = [simd_float2](repeating: simd_float2(), count: vertexCount)
53 | let texCoordBuffer = device.makeBuffer(bytes: texCoords,
54 | length: MemoryLayout.stride * vertexCount,
55 | options: .storageModeShared)!
56 | texCoordBuffer.label = "Mesh Anchor Texture Coordinates"
57 | let texCoordView = BufferView(buffer: texCoordBuffer)
58 |
59 | let indexBuffer = BufferView(buffer: geometry.faces.buffer)
60 |
61 | let submesh = Submesh(primitiveType: .triangle,
62 | indexBuffer: indexBuffer,
63 | indexType: geometry.faces.bytesPerIndex == 4 ? .uint32 : .uint16,
64 | indexCount: geometry.faces.count * geometry.faces.primitive.indexCount,
65 | materialIndex: 0)
66 |
67 | self.init(vertexCount: vertexCount,
68 | vertexBuffers: [positionView, normalView, texCoordView],
69 | vertexDescriptor: vertexDescriptor,
70 | submeshes: [submesh],
71 | materials: [PhysicallyBasedMaterial.default])
72 | }
73 |
74 | convenience init(_ planeAnchor: ARKit.PlaneAnchor) {
75 | let geometry = planeAnchor.geometry
76 | precondition(geometry.meshVertices.format == .float3)
77 | precondition(geometry.meshFaces.primitive == .triangle)
78 | precondition(geometry.meshFaces.bytesPerIndex == 2 || geometry.meshFaces.bytesPerIndex == 4)
79 |
80 | let device = geometry.meshVertices.buffer.device
81 | let vertexCount = geometry.meshVertices.count
82 |
83 | let vertexDescriptor = MDLVertexDescriptor()
84 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition
85 | vertexDescriptor.vertexAttributes[0].format = .float3
86 | vertexDescriptor.vertexAttributes[0].offset = 0
87 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0
88 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal
89 | vertexDescriptor.vertexAttributes[1].format = .float3
90 | vertexDescriptor.vertexAttributes[1].offset = 0
91 | vertexDescriptor.vertexAttributes[1].bufferIndex = 1
92 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate
93 | vertexDescriptor.vertexAttributes[2].format = .float2
94 | vertexDescriptor.vertexAttributes[2].offset = 0
95 | vertexDescriptor.vertexAttributes[2].bufferIndex = 2
96 | vertexDescriptor.bufferLayouts[0].stride = geometry.meshVertices.stride
97 | vertexDescriptor.bufferLayouts[1].stride = MemoryLayout.stride
98 | vertexDescriptor.bufferLayouts[2].stride = MemoryLayout.stride
99 |
100 | let positionView = BufferView(buffer: geometry.meshVertices.buffer, offset: geometry.meshVertices.offset)
101 | let normalData = [simd_float3](repeating: simd_float3(0, 1, 0), count: vertexCount)
102 | let normalBuffer = device.makeBuffer(bytes: normalData,
103 | length: MemoryLayout.stride * vertexCount,
104 | options: .storageModeShared)!
105 | normalBuffer.label = "Plane Anchor Normals"
106 | let normalView = BufferView(buffer: normalBuffer)
107 | let texCoordData = [simd_float2](repeating: simd_float2(), count: vertexCount)
108 | let texCoordBuffer = device.makeBuffer(bytes: texCoordData,
109 | length: MemoryLayout.stride * vertexCount,
110 | options: .storageModeShared)!
111 | texCoordBuffer.label = "Plane Anchor Texture Coordinates"
112 | let texCoordView = BufferView(buffer: texCoordBuffer)
113 |
114 | let indexBuffer = BufferView(buffer: geometry.meshFaces.buffer)
115 |
116 | let submesh = Submesh(primitiveType: .triangle,
117 | indexBuffer: indexBuffer,
118 | indexType: geometry.meshFaces.bytesPerIndex == 4 ? .uint32 : .uint16,
119 | indexCount: geometry.meshFaces.count * geometry.meshFaces.primitive.indexCount,
120 | materialIndex: 0)
121 |
122 | self.init(vertexCount: vertexCount,
123 | vertexBuffers: [positionView, normalView, texCoordView],
124 | vertexDescriptor: vertexDescriptor,
125 | submeshes: [submesh],
126 | materials: [PhysicallyBasedMaterial.default])
127 | }
128 | }
129 |
130 | #endif
131 |
--------------------------------------------------------------------------------
/Engine/ARSessionManager.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | #if os(visionOS)
18 | import ARKit
19 |
20 | typealias WorldAnchorUpdateHandler = (WorldAnchor, AnchorEvent) -> Void
21 | typealias MeshAnchorUpdateHandler = (MeshAnchor, AnchorEvent) -> Void
22 | typealias PlaneAnchorUpdateHandler = (PlaneAnchor, AnchorEvent) -> Void
23 | typealias HandAnchorUpdateHandler = (HandAnchor, AnchorEvent) -> Void
24 | typealias EnvironmentLightAnchorUpdateHandler = (EnvironmentLightAnchor, AnchorEvent) -> Void
25 |
26 | /// This extension maps from ARKit anchor update events to our abstract anchor event type
27 | extension AnchorEvent {
28 | init(_ event: AnchorUpdate.Event) where AnchorType : ARKit.Anchor {
29 | switch event {
30 | case .added:
31 | self = .added
32 | case .updated:
33 | self = .updated
34 | case .removed:
35 | self = .removed
36 | }
37 | }
38 | }
39 |
40 | class ARSessionManager {
41 | struct Options: OptionSet {
42 | let rawValue: Int
43 |
44 | static let planeDetection = Options(rawValue: 1 << 0)
45 | static let handTracking = Options(rawValue: 1 << 1)
46 | static let sceneReconstruction = Options(rawValue: 1 << 2)
47 | static let lightEstimation = Options(rawValue: 1 << 3)
48 |
49 | static let all: Options = [
50 | .planeDetection, .handTracking, .sceneReconstruction, .lightEstimation
51 | ]
52 | }
53 |
54 | let arSession: ARKitSession
55 |
56 | var onWorldAnchorUpdate: WorldAnchorUpdateHandler?
57 | var onPlaneAnchorUpdate: PlaneAnchorUpdateHandler?
58 | var onMeshAnchorUpdate: MeshAnchorUpdateHandler?
59 | var onHandAnchorUpdate: HandAnchorUpdateHandler?
60 | var onEnvironmentLightAnchorUpdate: EnvironmentLightAnchorUpdateHandler?
61 |
62 | private var worldTrackingProvider: WorldTrackingProvider
63 | private var handTrackingProvider: HandTrackingProvider?
64 |
65 | init() {
66 | arSession = ARKitSession()
67 | worldTrackingProvider = WorldTrackingProvider()
68 | }
69 |
70 | func start(options: Options) async throws {
71 | var authorizationTypes = Set(WorldTrackingProvider.requiredAuthorizations)
72 |
73 | var providers: [DataProvider] = [worldTrackingProvider]
74 |
75 | if WorldTrackingProvider.isSupported {
76 | startMonitoringWorldAnchorUpdates(worldTrackingProvider)
77 | }
78 | if PlaneDetectionProvider.isSupported && options.contains(.planeDetection) {
79 | let provider = PlaneDetectionProvider(alignments: [.horizontal])
80 | authorizationTypes.formUnion(PlaneDetectionProvider.requiredAuthorizations)
81 | providers.append(provider)
82 | startMonitoringPlaneAnchorUpdates(provider)
83 | }
84 | if HandTrackingProvider.isSupported && options.contains(.handTracking) {
85 | handTrackingProvider = HandTrackingProvider()
86 | authorizationTypes.formUnion(HandTrackingProvider.requiredAuthorizations)
87 | providers.append(handTrackingProvider!)
88 | startMonitoringHandTrackingUpdates(handTrackingProvider!)
89 | }
90 | if SceneReconstructionProvider.isSupported && options.contains(.sceneReconstruction) {
91 | let provider = SceneReconstructionProvider(modes: [])
92 | authorizationTypes.formUnion(SceneReconstructionProvider.requiredAuthorizations)
93 | providers.append(provider)
94 | startMonitoringSceneReconstructionUpdates(provider)
95 | }
96 | if EnvironmentLightEstimationProvider.isSupported && options.contains(.lightEstimation) {
97 | let provider = EnvironmentLightEstimationProvider()
98 | authorizationTypes.formUnion(EnvironmentLightEstimationProvider.requiredAuthorizations)
99 | providers.append(provider)
100 | startMonitoringLightingUpdates(provider)
101 | }
102 |
103 | #if !targetEnvironment(simulator)
104 | print("Will query authorization for: \(authorizationTypes)")
105 | let authorization = await arSession.queryAuthorization(for: Array(authorizationTypes))
106 | // When running an AR session, the system automatically requests the authorization
107 | // types contained by the Info.plist, so we don't need to explicitly request anything.
108 | // However, we regard all required authorization types to be mandatory to proceed,
109 | // so it is an error for the user to have explicitly denied access. In an app
110 | // where world sensing or other authorization types are optional, we'd handle this
111 | // more gracefully.
112 | if authorization.values.contains(.denied) {
113 | fatalError("Required authorization has been denied. Aborting.")
114 | }
115 | #endif
116 |
117 | try await arSession.run(providers)
118 | }
119 |
120 | func stop() {
121 | arSession.stop()
122 | }
123 |
124 | func addWorldAnchor(originFromAnchorTransform: simd_float4x4) async throws -> WorldAnchor {
125 | let anchor = ARKit.WorldAnchor(originFromAnchorTransform: originFromAnchorTransform)
126 | try await worldTrackingProvider.addAnchor(anchor)
127 | return WorldAnchor(anchor)
128 | }
129 |
130 | func removeAllWorldAnchors() async throws {
131 | let anchors = await worldTrackingProvider.allAnchors ?? []
132 | for i in 0.. ARKit.DeviceAnchor? {
138 | if worldTrackingProvider.state == .running {
139 | let anchor = worldTrackingProvider.queryDeviceAnchor(atTimestamp: timestamp)
140 | if anchor == nil {
141 | print("World tracking provider is running but failed to provide a device anchor at \(timestamp)")
142 | }
143 | return anchor
144 | } else {
145 | print("World tracking provider is not running; cannot retrieve device pose.")
146 | return nil
147 | }
148 | }
149 |
150 | func queryHandAnchors(at timestamp: TimeInterval) -> (ARKit.HandAnchor?, ARKit.HandAnchor?) {
151 | if let provider = handTrackingProvider, provider.state == .running {
152 | return provider.handAnchors(at: timestamp)
153 | }
154 | return (nil, nil)
155 | }
156 |
157 | private func startMonitoringWorldAnchorUpdates(_ provider: WorldTrackingProvider) {
158 | Task(priority: .high) { [weak self] in
159 | for await update in provider.anchorUpdates {
160 | self?.onWorldAnchorUpdate?(WorldAnchor(update.anchor), AnchorEvent(update.event))
161 | }
162 | }
163 | }
164 |
165 | private func startMonitoringPlaneAnchorUpdates(_ provider: PlaneDetectionProvider) {
166 | Task(priority: .low) { [weak self] in
167 | for await update in provider.anchorUpdates {
168 | self?.onPlaneAnchorUpdate?(PlaneAnchor(update.anchor), AnchorEvent(update.event))
169 | }
170 | }
171 | }
172 |
173 | private func startMonitoringSceneReconstructionUpdates(_ provider: SceneReconstructionProvider) {
174 | Task(priority: .low) { [weak self] in
175 | for await update in provider.anchorUpdates {
176 | self?.onMeshAnchorUpdate?(MeshAnchor(update.anchor), AnchorEvent(update.event))
177 | }
178 | }
179 | }
180 |
181 | private func startMonitoringHandTrackingUpdates(_ provider: HandTrackingProvider) {
182 | Task(priority: .high) { [weak self] in
183 | for await update in provider.anchorUpdates {
184 | self?.onHandAnchorUpdate?(HandAnchor(update.anchor), AnchorEvent(update.event))
185 | }
186 | }
187 | }
188 |
189 | private func startMonitoringLightingUpdates(_ provider: EnvironmentLightEstimationProvider) {
190 | Task(priority: .low) { [weak self] in
191 | for await update in provider.anchorUpdates {
192 | self?.onEnvironmentLightAnchorUpdate?(EnvironmentLightAnchor(update.anchor), AnchorEvent(update.event))
193 | }
194 | }
195 | }
196 | }
197 |
198 | #endif
199 |
--------------------------------------------------------------------------------
/Engine/Anchor.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import simd
19 |
20 | public enum AnchorEvent : Sendable {
21 | case added
22 | case updated
23 | case removed
24 | }
25 |
26 | public protocol Anchor : Identifiable, Sendable {
27 | var originFromAnchorTransform: simd_float4x4 { get }
28 | }
29 |
30 | public struct WorldAnchor : Anchor {
31 | public typealias ID = UUID
32 |
33 | public let id: UUID
34 | public let originFromAnchorTransform: simd_float4x4
35 |
36 | init (id: UUID, originFromAnchorTransform: simd_float4x4) {
37 | self.id = id
38 | self.originFromAnchorTransform = originFromAnchorTransform
39 | }
40 | }
41 |
42 | public struct PlaneAnchor : Anchor, @unchecked Sendable {
43 | public typealias ID = UUID
44 |
45 | public struct Extent {
46 | let width: Float
47 | let height: Float
48 | let anchorFromExtentTransform: simd_float4x4
49 |
50 | init(width: Float, height: Float, anchorFromExtentTransform: simd_float4x4) {
51 | self.width = width
52 | self.height = height
53 | self.anchorFromExtentTransform = anchorFromExtentTransform
54 | }
55 | }
56 |
57 | public enum Alignment {
58 | case horizontal
59 | case vertical
60 | case slanted
61 | }
62 |
63 | public enum Classification : Int {
64 | case unknown
65 | case wall
66 | case floor
67 | case ceiling
68 | case table
69 | case seat
70 | case window
71 | case door
72 | }
73 |
74 | public let id: UUID
75 | public let originFromAnchorTransform: simd_float4x4
76 | public let alignment: PlaneAnchor.Alignment
77 | public let classification: PlaneAnchor.Classification
78 | public let extent: PlaneAnchor.Extent
79 | public let mesh: Mesh
80 |
81 | init(id: UUID,
82 | originFromAnchorTransform: simd_float4x4,
83 | alignment: PlaneAnchor.Alignment,
84 | classification: PlaneAnchor.Classification,
85 | extent: PlaneAnchor.Extent,
86 | mesh: Mesh)
87 | {
88 | self.id = id
89 | self.originFromAnchorTransform = originFromAnchorTransform
90 | self.alignment = alignment
91 | self.classification = classification
92 | self.extent = extent
93 | self.mesh = mesh
94 | }
95 | }
96 |
97 | public struct MeshAnchor : Anchor, @unchecked Sendable {
98 | public typealias ID = UUID
99 |
100 | public let id: UUID
101 | public let originFromAnchorTransform: simd_float4x4
102 | public let mesh: Mesh
103 |
104 | init(id: UUID, originFromAnchorTransform: simd_float4x4, mesh: Mesh) {
105 | self.id = id
106 | self.originFromAnchorTransform = originFromAnchorTransform
107 | self.mesh = mesh
108 | }
109 | }
110 |
111 | public struct HandSkeleton : Sendable {
112 | public enum JointName : String, CaseIterable, Sendable {
113 | case wrist
114 | case thumbKnuckle
115 | case thumbIntermediateBase
116 | case thumbIntermediateTip
117 | case thumbTip
118 | case indexFingerMetacarpal
119 | case indexFingerKnuckle
120 | case indexFingerIntermediateBase
121 | case indexFingerIntermediateTip
122 | case indexFingerTip
123 | case middleFingerMetacarpal
124 | case middleFingerKnuckle
125 | case middleFingerIntermediateBase
126 | case middleFingerIntermediateTip
127 | case middleFingerTip
128 | case ringFingerMetacarpal
129 | case ringFingerKnuckle
130 | case ringFingerIntermediateBase
131 | case ringFingerIntermediateTip
132 | case ringFingerTip
133 | case littleFingerMetacarpal
134 | case littleFingerKnuckle
135 | case littleFingerIntermediateBase
136 | case littleFingerIntermediateTip
137 | case littleFingerTip
138 | case forearmWrist
139 | case forearmArm
140 | }
141 |
142 | public struct Joint : Sendable {
143 | let name: HandSkeleton.JointName
144 | let anchorFromJointTransform: simd_float4x4
145 | let estimatedLinearVelocity: simd_float3
146 | let isTracked: Bool
147 | }
148 |
149 | let joints: [HandSkeleton.Joint]
150 |
151 | init(joints: [HandSkeleton.Joint]) {
152 | self.joints = joints
153 | }
154 |
155 | func joint(named name: JointName) -> Joint? {
156 | joints.first(where: { $0.name == name })
157 | }
158 | }
159 |
160 | extension HandSkeleton.JointName {
161 | init?(webXRJointName: String) {
162 | switch webXRJointName {
163 | case "wrist" : self = .wrist
164 | case "thumb_metacarpal" : self = .thumbKnuckle
165 | case "thumb_phalanx_proximal" : self = .thumbIntermediateBase
166 | case "thumb_phalanx_distal" : self = .thumbIntermediateTip
167 | case "thumb_tip" : self = .thumbTip
168 | case "index_finger_metacarpal" : self = .indexFingerMetacarpal
169 | case "index_finger_phalanx_proximal" : self = .indexFingerKnuckle
170 | case "index_finger_phalanx_intermediate" : self = .indexFingerIntermediateBase
171 | case "index_finger_phalanx_distal" : self = .indexFingerIntermediateTip
172 | case "index_finger_tip" : self = .indexFingerTip
173 | case "middle_finger_metacarpal" : self = .middleFingerMetacarpal
174 | case "middle_finger_phalanx_proximal" : self = .middleFingerKnuckle
175 | case "middle_finger_phalanx_intermediate" : self = .middleFingerIntermediateBase
176 | case "middle_finger_phalanx_distal" : self = .middleFingerIntermediateTip
177 | case "middle_finger_tip" : self = .middleFingerTip
178 | case "ring_finger_metacarpal" : self = .ringFingerMetacarpal
179 | case "ring_finger_phalanx_proximal" : self = .ringFingerKnuckle
180 | case "ring_finger_phalanx_intermediate" : self = .ringFingerIntermediateBase
181 | case "ring_finger_phalanx_distal" : self = .ringFingerIntermediateTip
182 | case "ring_finger_tip" : self = .ringFingerTip
183 | case "pinky_finger_metacarpal" : self = .littleFingerMetacarpal
184 | case "pinky_finger_phalanx_proximal" : self = .littleFingerKnuckle
185 | case "pinky_finger_phalanx_intermediate" : self = .littleFingerIntermediateBase
186 | case "pinky_finger_phalanx_distal" : self = .littleFingerIntermediateTip
187 | case "pinky_finger_tip" : self = .littleFingerTip
188 | default:
189 | return nil
190 | }
191 | }
192 |
193 | var webXRJointName: String? {
194 | switch self {
195 | case .wrist: return "wrist"
196 | case .thumbKnuckle: return "thumb_metacarpal"
197 | case .thumbIntermediateBase: return "thumb_phalanx_proximal"
198 | case .thumbIntermediateTip: return "thumb_phalanx_distal"
199 | case .thumbTip: return "thumb_tip"
200 | case .indexFingerMetacarpal: return "index_finger_metacarpal"
201 | case .indexFingerKnuckle: return "index_finger_phalanx_proximal"
202 | case .indexFingerIntermediateBase: return "index_finger_phalanx_intermediate"
203 | case .indexFingerIntermediateTip: return "index_finger_phalanx_distal"
204 | case .indexFingerTip: return "index_finger_tip"
205 | case .middleFingerMetacarpal: return "middle_finger_metacarpal"
206 | case .middleFingerKnuckle: return "middle_finger_phalanx_proximal"
207 | case .middleFingerIntermediateBase: return "middle_finger_phalanx_intermediate"
208 | case .middleFingerIntermediateTip: return "middle_finger_phalanx_distal"
209 | case .middleFingerTip: return "middle_finger_tip"
210 | case .ringFingerMetacarpal: return "ring_finger_metacarpal"
211 | case .ringFingerKnuckle: return "ring_finger_phalanx_proximal"
212 | case .ringFingerIntermediateBase: return "ring_finger_phalanx_intermediate"
213 | case .ringFingerIntermediateTip: return "ring_finger_phalanx_distal"
214 | case .ringFingerTip: return "ring_finger_tip"
215 | case .littleFingerMetacarpal: return "pinky_finger_metacarpal"
216 | case .littleFingerKnuckle: return "pinky_finger_phalanx_proximal"
217 | case .littleFingerIntermediateBase: return "pinky_finger_phalanx_intermediate"
218 | case .littleFingerIntermediateTip: return "pinky_finger_phalanx_distal"
219 | case .littleFingerTip: return "pinky_finger_tip"
220 | default: return nil
221 | }
222 | }
223 | }
224 |
225 | public enum Handedness : Sendable {
226 | case left
227 | case right
228 | case none
229 | }
230 |
231 | public struct HandAnchor : Anchor {
232 | public typealias ID = UUID
233 |
234 | public let id: UUID
235 | public let originFromAnchorTransform: simd_float4x4
236 | public let estimatedLinearVelocity: simd_float3
237 | public let handSkeleton: HandSkeleton?
238 | public let handedness: Handedness
239 | public let isTracked: Bool
240 |
241 | init(id: UUID,
242 | originFromAnchorTransform: simd_float4x4,
243 | estimatedLinearVelocity: simd_float3,
244 | handSkeleton: HandSkeleton?,
245 | handedness: Handedness,
246 | isTracked: Bool)
247 | {
248 | self.id = id
249 | self.originFromAnchorTransform = originFromAnchorTransform
250 | self.estimatedLinearVelocity = estimatedLinearVelocity
251 | self.handSkeleton = handSkeleton
252 | self.handedness = handedness
253 | self.isTracked = isTracked
254 | }
255 | }
256 |
257 | public struct EnvironmentLightAnchor : Anchor {
258 | public typealias ID = UUID
259 |
260 | public let id: UUID
261 |
262 | public let originFromAnchorTransform: simd_float4x4
263 | public let light: EnvironmentLight
264 |
265 | init(id: UUID, originFromAnchorTransform: simd_float4x4, light: EnvironmentLight) {
266 | self.id = id
267 | self.originFromAnchorTransform = originFromAnchorTransform
268 | self.light = light
269 | }
270 | }
271 |
--------------------------------------------------------------------------------
/Engine/Bridging.h:
--------------------------------------------------------------------------------
1 | #include "JoltPhysics.h"
2 | #include "ShaderTypes.h"
3 |
--------------------------------------------------------------------------------
/Engine/BufferUtilities.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Metal
18 | import ModelIO
19 |
20 | // A buffer view is a simple abstraction over a Metal buffer that
21 | // holds a strong reference to its buffer and an optional offset.
22 | class BufferView {
23 | let buffer: MTLBuffer
24 | let offset: Int
25 |
26 | init(buffer: MTLBuffer, offset: Int = 0) {
27 | self.buffer = buffer
28 | self.offset = offset
29 | }
30 | }
31 |
32 | // A ring buffer is a fixed-size buffer that can be used for small
33 | // temporary allocations. When the allocation offset of the buffer
34 | // exceeds its size, it "wraps" around to zero. Allocating an object
35 | // larger than the buffer itself is an error. Since this type does
36 | // not do any lifetime management, any objects copied into it should
37 | // be POD.
38 | class RingBuffer {
39 | let buffer: MTLBuffer
40 | let bufferLength: Int
41 | let minimumAlignment: Int
42 |
43 | private var nextOffset = 0
44 |
45 | init(device: MTLDevice,
46 | length: Int,
47 | options: MTLResourceOptions = .storageModeShared,
48 | label: String? = nil) throws
49 | {
50 | guard let buffer = device.makeBuffer(length: length, options: options) else {
51 | throw ResourceError.allocationFailure
52 | }
53 | buffer.label = label
54 | self.buffer = buffer
55 | self.bufferLength = length
56 | self.minimumAlignment = device.minimumConstantBufferOffsetAlignment
57 | }
58 |
59 | func alloc(length: Int, alignment: Int) -> (UnsafeMutableRawPointer, Int) {
60 | precondition(length <= bufferLength)
61 | let effectiveAlignment = max(minimumAlignment, alignment)
62 | var offset = alignUp(nextOffset, alignment: effectiveAlignment)
63 | if offset + length >= bufferLength {
64 | offset = 0
65 | }
66 | nextOffset = offset + length
67 | return (buffer.contents().advanced(by: offset), offset)
68 | }
69 |
70 | func copy(_ value: UnsafeMutablePointer) -> Int {
71 | precondition(_isPOD(T.self))
72 | let layout = MemoryLayout.self
73 | let (ptr, offset) = alloc(length: layout.size, alignment: layout.alignment)
74 | ptr.copyMemory(from: value, byteCount: layout.size)
75 | return offset
76 | }
77 |
78 | func copy(_ array: [T]) -> Int {
79 | precondition(_isPOD(T.self))
80 | if array.count == 0 { return 0 }
81 | let layout = MemoryLayout.self
82 | let regionLength = ((array.count - 1) * layout.stride) + layout.size
83 | let (ptr, offset) = alloc(length: regionLength, alignment: layout.alignment)
84 | array.withUnsafeBytes { elementsPtr in
85 | guard let elementPtr = elementsPtr.baseAddress else { return }
86 | ptr.copyMemory(from: elementPtr, byteCount: regionLength)
87 | }
88 | return offset
89 | }
90 | }
91 |
92 | // A strided view is a sequence type that affords random access to an underlying
93 | // data buffer containing homogeneous elements that may be strided (i.e. not contiguous
94 | // in memory). It is the responsibility of a view's creator to ensure that the
95 | // underlying buffer lives at least as long as any views created on it.
96 | struct StridedView : Sequence {
97 | struct Iterator : Swift.IteratorProtocol {
98 | private let storage: StridedView
99 | private var currentIndex = 0
100 |
101 | init(_ storage: StridedView) {
102 | self.storage = storage
103 | }
104 |
105 | mutating func next() -> Element? {
106 | if currentIndex < storage.count {
107 | let index = currentIndex
108 | currentIndex += 1
109 | return storage[index]
110 | } else {
111 | return nil
112 | }
113 | }
114 | }
115 |
116 | let basePointer: UnsafeRawPointer
117 | let offset: Int
118 | let stride: Int
119 | let count: Int
120 |
121 | init(_ basePointer: UnsafeRawPointer, offset: Int, stride: Int, count: Int) {
122 | precondition(_isPOD(Element.self))
123 | self.basePointer = basePointer
124 | self.offset = offset
125 | self.stride = stride
126 | self.count = count
127 | }
128 |
129 | var underestimatedCount: Int {
130 | return count
131 | }
132 |
133 | func makeIterator() -> Iterator {
134 | return Iterator(self)
135 | }
136 |
137 | func withContiguousStorageIfAvailable(_ body: (UnsafeBufferPointer) throws -> R) rethrows -> R? {
138 | if stride == MemoryLayout.stride {
139 | let elementPtr = basePointer.advanced(by: offset).assumingMemoryBound(to: Element.self)
140 | let elementBufferPtr = UnsafeBufferPointer(start: elementPtr, count: count)
141 | return try body(elementBufferPtr)
142 | }
143 | return nil
144 | }
145 |
146 | subscript(index: Int) -> Element {
147 | get {
148 | precondition(index >= 0 && index < count)
149 | let elementPtr = basePointer.advanced(by: offset + stride * index).assumingMemoryBound(to: Element.self)
150 | return elementPtr.pointee
151 | }
152 | }
153 | }
154 |
155 | // A small utility for determining the minimum alignment of offsets
156 | // of buffers bound with a command encoder.
157 | extension MTLDevice {
158 | var minimumConstantBufferOffsetAlignment: Int {
159 | #if targetEnvironment(simulator)
160 | let isSimulator = true
161 | #else
162 | let isSimulator = false
163 | #endif
164 |
165 | if supportsFamily(.apple2) && !isSimulator {
166 | // A8 and later (and Apple Silicon) support 4 byte alignment
167 | return 4
168 | } else if supportsFamily(.mac2) && !isSimulator {
169 | // Recent Macs support 32 byte alignment
170 | return 32
171 | } else {
172 | // Worst-case scenario for simulator, old Nvidia Macs, etc.
173 | return 256
174 | }
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/Engine/Camera.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import CoreGraphics
19 | import simd
20 |
21 | protocol Camera {
22 | var transform: Transform { get }
23 | var viewMatrix: simd_float4x4 { get }
24 | func projectionMatrix(for viewportSize: CGSize) -> simd_float4x4
25 | }
26 |
27 | class OrthographicCamera : Camera {
28 | var near: Float = 0.0
29 | var far: Float = 1.0
30 |
31 | var transform = Transform()
32 |
33 | var viewMatrix: simd_float4x4 {
34 | return transform.matrix.inverse
35 | }
36 |
37 | func projectionMatrix(for viewportSize: CGSize) -> simd_float4x4 {
38 | let width = Float(viewportSize.width), height = Float(viewportSize.height)
39 | return simd_float4x4(orthographicProjectionLeft: 0.0, top: 0.0,
40 | right: width, bottom: height,
41 | near: near, far: far)
42 | }
43 | }
44 |
45 | // A camera that produces a perspective projection based on a field-of-view and near clipping plane distance
46 | // The projection matrix produced by this camera assumes an infinitely distant far viewing distance and reverse-Z
47 | // (i.e. clip space depth runs from 1 to 0, near to far)
48 | class PerspectiveCamera : Camera {
49 | // The total vertical field of view angle of the camera
50 | var fieldOfView: Angle = .degrees(60)
51 |
52 | // The distance to the camera's near viewing plane
53 | var near: Float = 0.005
54 |
55 | var transform = Transform()
56 |
57 | var viewMatrix: simd_float4x4 {
58 | return transform.matrix.inverse
59 | }
60 |
61 | func projectionMatrix(for viewportSize: CGSize) -> simd_float4x4 {
62 | let width = Float(viewportSize.width), height = Float(viewportSize.height)
63 | let aspectRatio = width / height
64 | return simd_float4x4(perspectiveProjectionFOV: Float(fieldOfView.radians),
65 | aspectRatio: aspectRatio,
66 | near: near)
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/Engine/Entity.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 |
19 | // An entity is a named object that participates in a transform hierarchy
20 | // and can have an associated mesh and/or physics body.
21 | public class Entity {
22 | var name: String = ""
23 | var modelTransform: Transform
24 | public private(set) var parent: Entity?
25 | public private(set) var children: [Entity] = []
26 | var mesh: Mesh?
27 | var isHidden = false
28 |
29 | var worldTransform: Transform {
30 | get {
31 | if let parent {
32 | return parent.worldTransform * modelTransform
33 | } else {
34 | return modelTransform
35 | }
36 | }
37 | set {
38 | if let parent {
39 | let ancestorTransform = parent.worldTransform
40 | let newTransform = ancestorTransform.inverse * newValue
41 | modelTransform = newTransform
42 | } else {
43 | modelTransform = newValue
44 | }
45 | }
46 | }
47 |
48 | var skinner: Skinner?
49 |
50 | var physicsBody: PhysicsBody?
51 |
52 | init(mesh: Mesh? = nil, transform: Transform = Transform()) {
53 | self.mesh = mesh
54 | self.modelTransform = transform
55 | }
56 |
57 | func addChild(_ child: Entity) {
58 | child.removeFromParent()
59 | child.parent = self
60 | children.append(child)
61 | parent?.didAddChild(child)
62 | }
63 |
64 | func removeFromParent() {
65 | parent?.removeChild(self)
66 | parent = nil
67 | }
68 |
69 | private func removeChild(_ child: Entity) {
70 | children.removeAll {
71 | $0 == child
72 | }
73 | parent?.didRemoveChild(child)
74 | }
75 |
76 | // Creates a recursive "clone" of this entity and its children.
77 | // n.b. that this method "slices" subclasses of Entity, so only
78 | // use it to clone hierarchies of instances of this base class.
79 | // Also note that physics bodies and shapes are not cloned.
80 | func clone(recursively: Bool) -> Entity {
81 | let entity = Entity(mesh: mesh, transform: modelTransform)
82 | entity.isHidden = isHidden
83 | entity.skinner = skinner
84 | for child in children {
85 | entity.addChild(child.clone(recursively: recursively))
86 | }
87 | return entity
88 | }
89 |
90 | func update(_ timestep: TimeInterval) {}
91 |
92 | func didAddChild(_ entity: Entity) {
93 | parent?.didAddChild(entity)
94 | }
95 |
96 | func didRemoveChild(_ entity: Entity) {
97 | parent?.didRemoveChild(entity)
98 | }
99 |
100 | func generateCollisionShapes(recursive: Bool, bodyMode: PhysicsBodyMode) {
101 | if let mesh {
102 | let shape = PhysicsShape(convexHullFromMesh: mesh)
103 | let body = PhysicsBody(mode: bodyMode, shape: shape)
104 | physicsBody = body
105 | }
106 | if recursive {
107 | for child in children {
108 | child.generateCollisionShapes(recursive: recursive, bodyMode: bodyMode)
109 | }
110 | }
111 | }
112 | }
113 |
114 | extension Entity : Equatable {
115 | public static func == (lhs: Entity, rhs: Entity) -> Bool {
116 | return lhs === rhs
117 | }
118 | }
119 |
120 | extension Entity : Hashable {
121 | public func hash(into hasher: inout Hasher) {
122 | hasher.combine(ObjectIdentifier(self))
123 | }
124 | }
125 |
126 | extension Entity {
127 | func visitBreadthFirst(_ root: Entity, _ body: (Entity) -> Void) {
128 | var queue = [root]
129 | while !queue.isEmpty {
130 | let current = queue.removeFirst()
131 | body(current)
132 | for child in current.children {
133 | queue.append(child)
134 | }
135 | }
136 | }
137 |
138 | func flattenedHierarchy(_ root: Entity) -> [Entity] {
139 | var entities: [Entity] = []
140 | visitBreadthFirst(root) { entities.append($0) }
141 | return entities
142 | }
143 |
144 | func child(named name: String, recursive: Bool = true) -> Entity? {
145 | if let immediateMatch = children.first(where: { $0.name == name }) {
146 | return immediateMatch
147 | } else if recursive {
148 | for child in children {
149 | if let match = child.child(named: name, recursive: recursive) {
150 | return match
151 | }
152 | }
153 | }
154 | return nil
155 | }
156 |
157 | func childEntities(matching predicate: (Entity) -> Bool) -> [Entity] {
158 | var matches: [Entity] = []
159 | visitBreadthFirst(self) { if predicate($0) && $0 !== self { matches.append($0) } }
160 | return matches
161 | }
162 | }
163 |
164 | class AnchorEntity : Entity {
165 | override init(mesh: Mesh? = nil, transform: Transform = Transform()) {
166 | super.init(mesh: mesh, transform: transform)
167 | }
168 |
169 | func updatePose(from anchor: WorldAnchor) {
170 | worldTransform = Transform(anchor.originFromAnchorTransform)
171 | }
172 | }
173 |
174 | class HandEntity: Entity {
175 | let handedness: Handedness
176 | private let meshEntityName = "Mesh"
177 | private let jointConformationTransform: simd_float4x4
178 | private let colliderEntity = Entity()
179 |
180 | init(handedness: Handedness, mesh: Mesh? = nil, transform: Transform = Transform(), context: MetalContext) {
181 | self.handedness = handedness
182 | if handedness == .right {
183 | self.jointConformationTransform = simd_float4x4(rotationAbout: simd_float3(0, 1, 0), by: .pi / 2)
184 | } else {
185 | self.jointConformationTransform = simd_float4x4(rotationAbout: simd_normalize(simd_float3(1, 0, -1)), by: .pi)
186 | }
187 | super.init(mesh: mesh, transform: transform)
188 |
189 | let colliderRadius: Float = 0.0075
190 | let colliderColor = simd_float4(handedness == .right ? 0.8 : 0, handedness == .left ? 0.8 : 0, 0, 1)
191 | let colliderMesh = Mesh.generateSphere(radius: colliderRadius, context: context)
192 | colliderMesh.materials = [PhysicallyBasedMaterial(baseColor: colliderColor, roughness: 0.5, isMetal: false)]
193 | colliderEntity.mesh = colliderMesh
194 | let colliderBody = PhysicsBody(mode: .kinematic,
195 | shape: PhysicsShape(sphereWithRadius: colliderRadius), mass: 0.5)
196 | colliderEntity.physicsBody = colliderBody
197 | addChild(colliderEntity)
198 | }
199 |
200 | func updatePose(from anchor: HandAnchor) {
201 | guard let meshEntity = child(named: meshEntityName, recursive: true) else { return }
202 | guard let handSkeleton = anchor.handSkeleton else { return }
203 | guard let handSkinner = meshEntity.skinner else { return }
204 | var jointPoses = handSkinner.skeleton.restTransforms
205 | for (destinationIndex, jointPath) in handSkinner.skeleton.jointPaths.enumerated() {
206 | if let jointName = HandSkeleton.JointName(webXRJointName: jointPath),
207 | let sourceJoint = handSkeleton.joint(named: jointName)
208 | {
209 | jointPoses[destinationIndex] = sourceJoint.anchorFromJointTransform * jointConformationTransform
210 | }
211 | }
212 | handSkinner.jointTransforms = jointPoses
213 | if anchor.isTracked {
214 | worldTransform = Transform(anchor.originFromAnchorTransform)
215 |
216 | if let indexTipJoint = handSkeleton.joint(named: .indexFingerTip) {
217 | colliderEntity.isHidden = !indexTipJoint.isTracked
218 | if indexTipJoint.isTracked {
219 | colliderEntity.modelTransform = Transform(indexTipJoint.anchorFromJointTransform)
220 | }
221 | }
222 | } else {
223 | isHidden = true
224 | }
225 | }
226 | }
227 |
--------------------------------------------------------------------------------
/Engine/Light.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import simd
18 | import Metal
19 |
20 | public class Light {
21 | enum LightType : Int {
22 | case directional = 0
23 | case point = 1
24 | case spot = 2
25 | }
26 |
27 | var type = LightType.directional
28 | var transform = Transform()
29 | var color = simd_float3(1, 1, 1)
30 | var intensity: Float = 1
31 | var range: Float = 0
32 | var innerConeAngle = Angle.degrees(90)
33 | var outerConeAngle = Angle.degrees(90)
34 |
35 | init(type: LightType = LightType.directional,
36 | color: simd_float3 = simd_float3(1, 1, 1),
37 | intensity: Float = 1)
38 | {
39 | self.type = type
40 | self.color = color
41 | self.intensity = intensity
42 | }
43 | }
44 |
45 | public struct EnvironmentLight : @unchecked Sendable {
46 | let cubeFromWorldTransform: Transform
47 | let environmentTexture: MTLTexture?
48 | let scaleFactor: Float
49 |
50 | init(cubeFromWorldTransform: Transform, environmentTexture: MTLTexture?, scaleFactor: Float) {
51 | self.cubeFromWorldTransform = cubeFromWorldTransform
52 | self.environmentTexture = environmentTexture
53 | self.scaleFactor = scaleFactor
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/Engine/LockingQueue.swift:
--------------------------------------------------------------------------------
1 | import Foundation
2 | import Dispatch
3 |
4 | // A relatively inefficient queue type for low-contention multithreaded operation.
5 | // All peek/pop operations block the calling thread until all previous writes have
6 | // been resolved. Concurrent enqueues are resolved in the order of insertion.
7 | class LockingQueue : @unchecked Sendable {
8 | private let queue = DispatchQueue(label: "concurrent-read-write-queue", attributes: .concurrent)
9 |
10 | private var _elements: [Element] = []
11 |
12 | func enqueue(_ element: Element) {
13 | queue.async(flags: .barrier) {
14 | self._elements.append(element)
15 | }
16 | }
17 |
18 | func enqueue(_ elements: [Element]) {
19 | queue.async(flags: .barrier) {
20 | self._elements.append(contentsOf: elements)
21 | }
22 | }
23 |
24 | func peek() -> Element? {
25 | return queue.sync {
26 | return _elements.first
27 | }
28 | }
29 |
30 | func pop() -> Element? {
31 | return queue.sync {
32 | return _elements.removeFirst()
33 | }
34 | }
35 |
36 | func popAll() -> [Element] {
37 | return queue.sync {
38 | let result = _elements
39 | _elements.removeAll()
40 | return result
41 | }
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/Engine/Material.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Metal
18 | import ModelIO
19 |
20 | enum BlendMode {
21 | case opaque
22 | case sourceOverPremultiplied
23 | }
24 |
25 | class Material : Identifiable {
26 | var name: String = ""
27 | var blendMode: BlendMode = .opaque
28 | var fillMode: MTLTriangleFillMode = .fill
29 | var isDoubleSided: Bool = false
30 | var writesDepthBuffer: Bool = true
31 | var readsDepthBuffer: Bool = true
32 | var colorMask: MTLColorWriteMask = [.red, .green, .blue, .alpha]
33 |
34 | var vertexFunctionName: String {
35 | fatalError("Material subclasses must implement vertexFunctionName")
36 | }
37 |
38 | var fragmentFunctionName: String {
39 | fatalError("Material subclasses must implement fragmentFunctionName")
40 | }
41 |
42 | var relativeSortOrder: Int {
43 | 0
44 | }
45 |
46 | func bindResources(constantBuffer: RingBuffer, renderCommandEncoder: MTLRenderCommandEncoder) {}
47 | }
48 |
49 | class OcclusionMaterial : Material {
50 | override init() {
51 | super.init()
52 | colorMask = []
53 | }
54 |
55 | override var vertexFunctionName: String {
56 | "vertex_main"
57 | }
58 |
59 | override var fragmentFunctionName: String {
60 | "fragment_occlusion"
61 | }
62 | override var relativeSortOrder: Int {
63 | -1 // Occluders render first because they lay down the depth of real-world objects
64 | }
65 | }
66 |
67 | class PhysicallyBasedMaterial : Material {
68 | static var `default` : PhysicallyBasedMaterial {
69 | var material = PhysicallyBasedMaterial()
70 | material.name = "Default"
71 | material.baseColor.baseColorFactor = simd_float4(0.5, 0.5, 0.5, 1.0)
72 | material.roughness.roughnessFactor = 0.5
73 | material.metalness.metalnessFactor = 0.0
74 | return material
75 | }
76 |
77 | struct BaseColor {
78 | var baseColorTexture: TextureResource?
79 | var baseColorFactor: simd_float4 = simd_float4(1, 1, 1, 1)
80 | }
81 |
82 | struct Normal {
83 | var normalTexture: TextureResource?
84 | var normalStrength: Float = 1.0
85 | }
86 |
87 | struct Metalness {
88 | var metalnessTexture: TextureResource?
89 | var metalnessFactor: Float = 1.0
90 | }
91 |
92 | struct Roughness {
93 | var roughnessTexture: TextureResource?
94 | var roughnessFactor: Float = 1.0
95 | }
96 |
97 | struct Emissive {
98 | var emissiveTexture: TextureResource?
99 | var emissiveColor: simd_float3 = simd_float3(0, 0, 0)
100 | var emissiveStrength: Float = 1.0
101 | }
102 |
103 | var baseColor = BaseColor()
104 | var normal = Normal()
105 | var metalness = Metalness()
106 | var roughness = Roughness()
107 | var emissive = Emissive()
108 |
109 | override var vertexFunctionName: String {
110 | "vertex_main"
111 | }
112 |
113 | override var fragmentFunctionName: String {
114 | "fragment_pbr"
115 | }
116 |
117 | override var relativeSortOrder: Int {
118 | if colorMask.isEmpty {
119 | -1 // Occluders render first because they lay down the depth of real-world objects
120 | } else if blendMode != .opaque {
121 | 1 // Transparents render last so they can read the depth buffer without affecting it
122 | } else {
123 | 0
124 | }
125 | }
126 |
127 | private var defaultSampler: MTLSamplerState?
128 |
129 | override init() {
130 | }
131 |
132 | init(baseColor: simd_float4, roughness: Float, isMetal: Bool) {
133 | self.baseColor.baseColorFactor = baseColor
134 | self.roughness.roughnessFactor = roughness
135 | self.metalness.metalnessFactor = isMetal ? 1.0 : 0.0
136 | }
137 |
138 | override func bindResources(constantBuffer: RingBuffer, renderCommandEncoder: MTLRenderCommandEncoder) {
139 | if defaultSampler == nil {
140 | let samplerDescriptor = MTLSamplerDescriptor()
141 | samplerDescriptor.minFilter = .linear
142 | samplerDescriptor.magFilter = .linear
143 | samplerDescriptor.mipFilter = .linear
144 | samplerDescriptor.sAddressMode = .repeat
145 | samplerDescriptor.tAddressMode = .repeat
146 | defaultSampler = renderCommandEncoder.device.makeSamplerState(descriptor: samplerDescriptor)
147 | }
148 |
149 | var material = PBRMaterialConstants(baseColorFactor: baseColor.baseColorFactor,
150 | emissiveColor: emissive.emissiveColor,
151 | normalScale: normal.normalStrength,
152 | metallicFactor: metalness.metalnessFactor,
153 | roughnessFactor: roughness.roughnessFactor,
154 | emissiveStrength: emissive.emissiveStrength)
155 | let materialOffset = constantBuffer.copy(&material)
156 | renderCommandEncoder.setFragmentBuffer(constantBuffer.buffer,
157 | offset: materialOffset,
158 | index: Int(FragmentBufferMaterialConstants))
159 |
160 | if let baseColorTexture = baseColor.baseColorTexture {
161 | renderCommandEncoder.setFragmentTexture(baseColorTexture.texture, index: Int(FragmentTextureBaseColor))
162 | renderCommandEncoder.setFragmentSamplerState(baseColorTexture.sampler ?? defaultSampler, index: Int(FragmentTextureBaseColor))
163 | } else {
164 | renderCommandEncoder.setFragmentTexture(nil, index: Int(FragmentTextureBaseColor))
165 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureBaseColor))
166 | }
167 |
168 | if let normalTexture = normal.normalTexture {
169 | renderCommandEncoder.setFragmentTexture(normalTexture.texture, index: Int(FragmentTextureNormal))
170 | renderCommandEncoder.setFragmentSamplerState(normalTexture.sampler ?? defaultSampler, index: Int(FragmentTextureNormal))
171 | } else {
172 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureNormal))
173 | }
174 |
175 | if let metalnessTexture = metalness.metalnessTexture {
176 | renderCommandEncoder.setFragmentTexture(metalnessTexture.texture, index: Int(FragmentTextureMetalness))
177 | renderCommandEncoder.setFragmentSamplerState(metalnessTexture.sampler ?? defaultSampler, index: Int(FragmentTextureMetalness))
178 | } else {
179 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureMetalness))
180 | }
181 |
182 | if let roughnessTexture = roughness.roughnessTexture {
183 | renderCommandEncoder.setFragmentTexture(roughnessTexture.texture, index: Int(FragmentTextureRoughness))
184 | renderCommandEncoder.setFragmentSamplerState(roughnessTexture.sampler ?? defaultSampler, index: Int(FragmentTextureRoughness))
185 | } else {
186 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureRoughness))
187 | }
188 |
189 | if let emissiveTexture = emissive.emissiveTexture {
190 | renderCommandEncoder.setFragmentTexture(emissiveTexture.texture, index: Int(FragmentTextureEmissive))
191 | renderCommandEncoder.setFragmentSamplerState(emissiveTexture.sampler ?? defaultSampler, index: Int(FragmentTextureEmissive))
192 | } else {
193 | renderCommandEncoder.setFragmentSamplerState(defaultSampler, index: Int(FragmentTextureEmissive))
194 | }
195 | }
196 | }
197 |
--------------------------------------------------------------------------------
/Engine/Materials.metal:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | #include
18 | using namespace metal;
19 |
20 | #include "ShaderTypes.h"
21 |
22 | constant bool writesRenderTargetSlice [[function_constant(0)]];
23 |
24 | struct MeshVertexIn {
25 | float3 position [[attribute(0)]];
26 | float3 normal [[attribute(1)]];
27 | float2 texCoords [[attribute(2)]];
28 | };
29 |
30 | struct MeshVertexOut {
31 | float4 clipPosition [[position]];
32 | float3 position; // world-space position
33 | float3 normal;
34 | float2 texCoords;
35 | uint renderTargetSlice [[render_target_array_index, function_constant(writesRenderTargetSlice)]];
36 | uint viewIndex [[viewport_array_index]];
37 | };
38 |
39 | struct PBRFragmentIn {
40 | float4 clipPosition [[position]];
41 | float3 position;
42 | float3 normal;
43 | float2 texCoords;
44 | uint viewIndex [[viewport_array_index]];
45 | bool frontFacing [[front_facing]];
46 | };
47 |
48 | struct Light {
49 | float3 direction;
50 | float3 position;
51 | float3 color;
52 | float range;
53 | float intensity;
54 | float innerConeCos;
55 | float outerConeCos;
56 | unsigned int type;
57 |
58 | float getDistanceAttenuation(float range, float distance) const {
59 | float recipDistanceSq = 1.0f / (distance * distance);
60 | if (range <= 0.0f) {
61 | // negative range means unlimited, so use unmodified inverse-square law
62 | return recipDistanceSq;
63 | }
64 | return max(min(1.0f - powr(distance / range, 4.0f), 1.0f), 0.0f) * recipDistanceSq;
65 | }
66 |
67 | float getSpotAttenuation(float3 pointToLight, float3 spotDirection) const
68 | {
69 | float actualCos = dot(normalize(spotDirection), normalize(-pointToLight));
70 | if (actualCos > outerConeCos) {
71 | if (actualCos < innerConeCos) {
72 | return smoothstep(outerConeCos, innerConeCos, actualCos);
73 | }
74 | return 1.0f;
75 | }
76 | return 0.0f;
77 | }
78 |
79 | half3 getIntensity(float3 pointToLight) const {
80 | float rangeAttenuation = 1.0;
81 | float spotAttenuation = 1.0;
82 |
83 | if (type != LightTypeDirectional) {
84 | rangeAttenuation = getDistanceAttenuation(range, length(pointToLight));
85 | }
86 | if (type == LightTypeSpot) {
87 | spotAttenuation = getSpotAttenuation(pointToLight, direction);
88 | }
89 |
90 | return rangeAttenuation * spotAttenuation * half3(intensity * color);
91 | }
92 | };
93 |
94 | struct TangentSpace {
95 | float3 T; // Geometric tangent
96 | float3 B; // Geometric bitangent
97 | float3 Ng; // Geometric normal
98 | float3 N; // Shading normal (TBN() * Nt)
99 |
100 | float3x3 TBN() const {
101 | return { T, B, Ng };
102 | }
103 | };
104 |
105 | static TangentSpace getTangentSpace(PBRFragmentIn v, constant PBRMaterialConstants &material,
106 | texture2d normalTexture, sampler normalSampler)
107 | {
108 | float2 uv = v.texCoords;
109 | float3 uv_dx = dfdx(float3(uv, 0.0f));
110 | float3 uv_dy = dfdy(float3(uv, 0.0f));
111 |
112 | if (length(uv_dx) + length(uv_dy) <= 1e-6f) {
113 | uv_dx = float3(1.0f, 0.0f, 0.0f);
114 | uv_dy = float3(0.0f, 1.0f, 0.0f);
115 | }
116 |
117 | float3 Tapprox = (uv_dy.y * dfdx(v.position.xyz) - uv_dx.y * dfdy(v.position.xyz)) /
118 | (uv_dx.x * uv_dy.y - uv_dy.x * uv_dx.y);
119 |
120 | float3 t, b, ng;
121 |
122 | ng = normalize(v.normal.xyz);
123 | t = normalize(Tapprox - ng * dot(ng, Tapprox));
124 | b = cross(ng, t);
125 |
126 | if (!v.frontFacing) {
127 | t *= -1.0f;
128 | b *= -1.0f;
129 | ng *= -1.0f;
130 | }
131 |
132 | // Apply normal map if available
133 | TangentSpace basis;
134 | basis.Ng = ng;
135 | if (!is_null_texture(normalTexture)) {
136 | float3 Nt = normalTexture.sample(normalSampler, uv).rgb * 2.0f - 1.0f;
137 | Nt *= float3(material.normalScale, material.normalScale, 1.0);
138 | Nt = normalize(Nt);
139 | basis.N = normalize(float3x3(t, b, ng) * Nt);
140 | } else {
141 | basis.N = ng;
142 | }
143 |
144 | basis.T = t;
145 | basis.B = b;
146 | return basis;
147 | }
148 |
149 | static half3 F_Schlick(half3 f0, half3 f90, half VdotH) {
150 | return f0 + (f90 - f0) * powr(saturate(1.h - VdotH), 5.0h);
151 | }
152 |
153 | static half V_GGX(half NdotL, half NdotV, half alphaRoughness)
154 | {
155 | half alphaRoughnessSq = alphaRoughness * alphaRoughness;
156 | half GGXV = NdotL * sqrt(NdotV * NdotV * (1.0h - alphaRoughnessSq) + alphaRoughnessSq);
157 | half GGXL = NdotV * sqrt(NdotL * NdotL * (1.0h - alphaRoughnessSq) + alphaRoughnessSq);
158 | half GGX = GGXV + GGXL;
159 | if (GGX > 0.0) {
160 | return 0.5 / GGX;
161 | }
162 | return 0.0;
163 | }
164 |
165 | static float D_GGX(float NdotH, float alphaRoughness) {
166 | float alphaRoughnessSq = alphaRoughness * alphaRoughness;
167 | float f = (NdotH * NdotH) * (alphaRoughnessSq - 1.0f) + 1.0f;
168 | return alphaRoughnessSq / (M_PI_F * f * f);
169 | }
170 |
171 | static half3 BRDF_lambertian(half3 f0, half3 f90, half3 diffuseColor, float VdotH = 1.0f) {
172 | // see https://seblagarde.wordpress.com/2012/01/08/pi-or-not-to-pi-in-game-lighting-equation/
173 | return diffuseColor / M_PI_F;
174 | }
175 |
176 | static half3 BRDF_specularGGX(half3 f0, half3 f90, half alphaRoughness,
177 | half VdotH, half NdotL, half NdotV, half NdotH)
178 | {
179 | half3 F = F_Schlick(f0, f90, VdotH);
180 | half V = V_GGX(NdotL, NdotV, alphaRoughness);
181 | half D = D_GGX(NdotH, alphaRoughness);
182 | return F * V * D;
183 | }
184 |
185 | vertex MeshVertexOut vertex_main(MeshVertexIn in [[stage_in]],
186 | constant PassConstants &frame [[buffer(VertexBufferPassConstants)]],
187 | constant InstanceConstants *instances [[buffer(VertexBufferInstanceConstants)]],
188 | uint instanceID [[instance_id]],
189 | uint amplificationID [[amplification_id]],
190 | uint amplificationCount [[amplification_count]])
191 | {
192 | uint viewIndex = amplificationID;
193 | constant auto& instance = instances[instanceID];
194 |
195 | float4 worldPosition = instance.modelMatrix * float4(in.position, 1.0f);
196 | float4x4 viewProjectionMatrix = frame.projectionMatrices[viewIndex] * frame.viewMatrices[viewIndex];
197 |
198 | MeshVertexOut out;
199 | out.position = worldPosition.xyz;
200 | out.normal = normalize(instance.normalMatrix * in.normal);
201 | out.texCoords = in.texCoords;
202 | out.clipPosition = viewProjectionMatrix * worldPosition;
203 | out.viewIndex = viewIndex;
204 |
205 | if (amplificationCount > 1) {
206 | if (writesRenderTargetSlice) {
207 | out.renderTargetSlice = viewIndex;
208 | }
209 | }
210 |
211 | return out;
212 | }
213 |
214 | half3 EnvDFGPolynomial_Knarkowicz(half3 specularColor, float gloss, float NdotV) {
215 | half x = gloss;
216 | half y = NdotV;
217 |
218 | half b1 = -0.1688h;
219 | half b2 = 1.895h;
220 | half b3 = 0.9903h;
221 | half b4 = -4.853h;
222 | half b5 = 8.404h;
223 | half b6 = -5.069h;
224 | half bias = saturate( min( b1 * x + b2 * x * x, b3 + b4 * y + b5 * y * y + b6 * y * y * y ) );
225 |
226 | half d0 = 0.6045h;
227 | half d1 = 1.699h;
228 | half d2 = -0.5228h;
229 | half d3 = -3.603h;
230 | half d4 = 1.404h;
231 | half d5 = 0.1939h;
232 | half d6 = 2.661h;
233 | half delta = saturate( d0 + d1 * x + d2 * y + d3 * x * x + d4 * x * y + d5 * y * y + d6 * x * x * x );
234 | half scale = delta - bias;
235 |
236 | bias *= saturate(50.0f * specularColor.y);
237 | return specularColor * scale + bias;
238 | }
239 |
240 | typedef half4 FragmentOut;
241 |
242 | fragment FragmentOut fragment_pbr(PBRFragmentIn in [[stage_in]],
243 | constant PassConstants &frame [[buffer(FragmentBufferPassConstants)]],
244 | constant PBRMaterialConstants &material [[buffer(FragmentBufferMaterialConstants)]],
245 | constant Light *lights [[buffer(FragmentBufferLights)]],
246 | texture2d baseColorTexture [[texture(FragmentTextureBaseColor)]],
247 | texture2d normalTexture [[texture(FragmentTextureNormal)]],
248 | texture2d metalnessTexture [[texture(FragmentTextureMetalness)]],
249 | texture2d roughnessTexture [[texture(FragmentTextureRoughness)]],
250 | texture2d emissiveTexture [[texture(FragmentTextureEmissive)]],
251 | sampler baseColorSampler [[sampler(FragmentTextureBaseColor)]],
252 | sampler normalSampler [[sampler(FragmentTextureNormal)]],
253 | sampler metalnessSampler [[sampler(FragmentTextureMetalness)]],
254 | sampler roughnessSampler [[sampler(FragmentTextureRoughness)]],
255 | sampler emissiveSampler [[sampler(FragmentTextureEmissive)]],
256 | texturecube environmentLightTexture [[texture(FragmentTextureEnvironmentLight)]])
257 | {
258 | half4 baseColor = half4(material.baseColorFactor);
259 | if (!is_null_texture(baseColorTexture)) {
260 | baseColor *= baseColorTexture.sample(baseColorSampler, in.texCoords);
261 | }
262 |
263 | half metalness = material.metallicFactor;
264 | if (!is_null_texture(metalnessTexture)) {
265 | float sampledMetalness = metalnessTexture.sample(metalnessSampler, in.texCoords).b;
266 | metalness *= sampledMetalness;
267 | }
268 | metalness = saturate(metalness);
269 |
270 | half perceptualRoughness = material.roughnessFactor;
271 | if (!is_null_texture(roughnessTexture)) {
272 | half sampledRoughness = roughnessTexture.sample(roughnessSampler, in.texCoords).g;
273 | perceptualRoughness *= sampledRoughness;
274 | }
275 | perceptualRoughness = clamp(perceptualRoughness, 0.004h, 1.0h);
276 | half alphaRoughness = perceptualRoughness * perceptualRoughness;
277 |
278 | half3 diffuseReflectance = mix(baseColor.rgb, half3(0.0h), metalness);
279 | half3 F0 = mix(half3(0.04h), baseColor.rgb, metalness);
280 | half3 F90 = half3(1.0h);
281 |
282 | float3 V = normalize(frame.cameraPositions[in.viewIndex] - in.position);
283 | TangentSpace tangentSpace = getTangentSpace(in, material, normalTexture, normalSampler);
284 | float3 N = tangentSpace.N;
285 |
286 | half NdotV = saturate(dot(N, V));
287 |
288 | half3 f_diffuse {};
289 | half3 f_specular {};
290 | half3 f_emissive {};
291 |
292 | if (!is_null_texture(emissiveTexture)) {
293 | half3 sampledEmission = emissiveTexture.sample(emissiveSampler, in.texCoords).rgb;
294 | f_emissive = sampledEmission * material.emissiveStrength;
295 | } else {
296 | f_emissive = half3(material.emissiveColor * material.emissiveStrength);
297 | }
298 |
299 | if (!is_null_texture(environmentLightTexture)) {
300 | float3 N_cube = (frame.environmentLightMatrix * float4(N, 0.0f)).xyz;
301 | constexpr sampler environmentSampler(filter::linear, mip_filter::linear);
302 | float mipCount = environmentLightTexture.get_num_mip_levels();
303 | // Sample the highest mip level available, since this is essentially a per cube face average lighting estimate
304 | half3 I_diff = environmentLightTexture.sample(environmentSampler, N_cube, level(mipCount - 1)).rgb * M_PI_H;
305 | f_diffuse += I_diff * diffuseReflectance;
306 | // For specular environment lighting we use Knarkowicz's analytic DFG. Realitistically,
307 | // we should sample a much lower LOD than we do, since the environment maps from ARKit
308 | // are much coarser than we'd use with classical IBL. Conversely, they're not properly
309 | // prefiltered with an appropriate BRDF, so this is all just hacks anyway.
310 | float lod = perceptualRoughness * (mipCount - 1);
311 | half gloss = powr(1.0h - perceptualRoughness, 4.0h);
312 | half3 I_spec = environmentLightTexture.sample(environmentSampler, N_cube, level(lod)).rgb;
313 | f_specular += I_spec * EnvDFGPolynomial_Knarkowicz(F0, gloss, NdotV);
314 | }
315 |
316 | for (uint i = 0; i < frame.activeLightCount; ++i) {
317 | Light light = lights[i];
318 |
319 | float3 pointToLight;
320 | if (light.type != LightTypeDirectional) {
321 | pointToLight = light.position - in.position;
322 | } else {
323 | pointToLight = -light.direction;
324 | }
325 |
326 | float3 L = normalize(pointToLight);
327 | float3 H = normalize(L + V);
328 | half NdotL = saturate(dot(N, L));
329 | half NdotH = saturate(dot(N, H));
330 | half VdotH = saturate(dot(V, H));
331 | if (NdotL > 0.0h) {
332 | half3 intensity = light.getIntensity(pointToLight);
333 | f_diffuse += intensity * NdotL * BRDF_lambertian(F0, F90, diffuseReflectance, VdotH);
334 | if (NdotV > 0.0h) {
335 | f_specular += intensity * NdotL * BRDF_specularGGX(F0, F90, alphaRoughness, VdotH, NdotL, NdotV, NdotH);
336 | }
337 | }
338 | }
339 |
340 | half3 color = f_emissive + f_diffuse + f_specular;
341 | FragmentOut out = half4(color * baseColor.a, baseColor.a);
342 | return out;
343 | }
344 |
345 | using OcclusionFragmentIn = MeshVertexOut;
346 |
347 | fragment FragmentOut fragment_occlusion(OcclusionFragmentIn in [[stage_in]]) {
348 | // What we return here doesn't matter because our color mask is zero.
349 | return {};
350 | }
351 |
--------------------------------------------------------------------------------
/Engine/Math.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import simd
18 | import Spatial
19 |
20 | func alignUp(_ base: Int, alignment: Int) -> Int {
21 | precondition(alignment > 0)
22 | return ((base + alignment - 1) / alignment) * alignment
23 | }
24 |
25 | // Given a range, determine the parameter for which lerp would produce the provided value.
26 | // Will extrapolate if given a value outside the range.
27 | func unlerp(_ from: T, _ to: T, _ value: T) -> T {
28 | if to == from { return T(0) }
29 | return (value - from) / (to - from)
30 | }
31 |
32 | // Swift simd doesn't have a packed three-element float vector type, but this
33 | // type has the same layout, size, and alignment such a type would have.
34 | public typealias packed_float3 = (Float, Float, Float)
35 |
36 | public extension simd_float4 {
37 | var xyz: simd_float3 {
38 | return simd_float3(x, y, z)
39 | }
40 | }
41 |
42 | extension simd_float4x4 {
43 | var upperLeft3x3: simd_float3x3 {
44 | return simd_float3x3(columns.0.xyz, columns.1.xyz, columns.2.xyz)
45 | }
46 |
47 | init(translation: simd_float3) {
48 | self.init(simd_float4(1, 0, 0, 0),
49 | simd_float4(0, 1, 0, 0),
50 | simd_float4(0, 0, 1, 0),
51 | simd_float4(translation, 1))
52 | }
53 |
54 | init(rotationAbout axis: simd_float3, by angleRadians: Float) {
55 | let x = axis.x, y = axis.y, z = axis.z
56 | let c = cosf(angleRadians)
57 | let s = sinf(angleRadians)
58 | let t = 1 - c
59 | self.init(simd_float4( t * x * x + c, t * x * y + z * s, t * x * z - y * s, 0),
60 | simd_float4( t * x * y - z * s, t * y * y + c, t * y * z + x * s, 0),
61 | simd_float4( t * x * z + y * s, t * y * z - x * s, t * z * z + c, 0),
62 | simd_float4( 0, 0, 0, 1))
63 | }
64 |
65 | init(orthographicProjectionLeft left: Float, top: Float, right: Float, bottom: Float, near: Float, far: Float)
66 | {
67 | let sx = 2.0 / (right - left)
68 | let sy = 2.0 / (top - bottom)
69 | let sz = 1.0 / (near - far)
70 | let tx = (left + right) / (left - right)
71 | let ty = (top + bottom) / (bottom - top)
72 | let tz = near / (near - far)
73 | self.init(simd_float4( sx, 0.0, 0.0, 0.0),
74 | simd_float4(0.0, sy, 0.0, 0.0),
75 | simd_float4(0.0, 0.0, sz, 0.0),
76 | simd_float4( tx, ty, tz, 1.0))
77 | }
78 |
79 | // Produces a perspective projection with an infinitely distant "far plane" that
80 | // maps clip-space depth from 1 (near) to 0 (infinitely far)—so-called reverse-Z.
81 | // It also assumes that view space is right-handed, because...c'mon.
82 | init(perspectiveProjectionFOV fovYRadians: Float, aspectRatio: Float, near: Float)
83 | {
84 | let sy = 1 / tan(fovYRadians * 0.5)
85 | let sx = sy / aspectRatio
86 | self.init(simd_float4(sx, 0, 0, 0.0),
87 | simd_float4(0, sy, 0, 0.0),
88 | simd_float4(0, 0, 0, -1.0),
89 | simd_float4(0, 0, near, 0.0))
90 | }
91 | }
92 |
93 | extension simd_float4x4 {
94 | init(_ mat: simd_double4x4) {
95 | self.init(SIMD4(mat.columns.0),
96 | SIMD4(mat.columns.1),
97 | SIMD4(mat.columns.2),
98 | SIMD4(mat.columns.3))
99 | }
100 | }
101 |
102 | /// Extracts the six frustum planes determined by the provided matrix.
103 | // Ref. https://www8.cs.umu.se/kurser/5DV051/HT12/lab/plane_extraction.pdf
104 | // Ref. https://fgiesen.wordpress.com/2012/08/31/frustum-planes-from-the-projection-matrix/
105 | func frustumPlanes(from matrix: simd_float4x4) -> [simd_float4] {
106 | let mt = matrix.transpose
107 | let planes = [simd_float4](unsafeUninitializedCapacity: 6,
108 | initializingWith: { buffer, initializedCount in
109 | buffer[0] = mt[3] + mt[0] // left
110 | buffer[1] = mt[3] - mt[0] // right
111 | buffer[2] = mt[3] - mt[1] // top
112 | buffer[3] = mt[3] + mt[1] // bottom
113 | buffer[4] = mt[2] // near
114 | buffer[5] = mt[3] - mt[2] // far
115 | for i in 0..<6 {
116 | buffer[i] /= simd_length(buffer[i].xyz)
117 | }
118 | initializedCount = 6
119 | })
120 | return planes
121 | }
122 |
123 | public struct Angle {
124 | static func radians(_ radians: Double) -> Angle {
125 | return Angle(radians: radians)
126 | }
127 |
128 | static func degrees(_ degrees: Double) -> Angle {
129 | return Angle(degrees: degrees)
130 | }
131 |
132 | var radians: Double
133 |
134 | var degrees: Double {
135 | return radians * (180 / .pi)
136 | }
137 |
138 | init() {
139 | radians = 0.0
140 | }
141 |
142 | init(radians: Double) {
143 | self.radians = radians
144 | }
145 |
146 | init(degrees: Double) {
147 | self.radians = degrees * (.pi / 180)
148 | }
149 |
150 | static func + (lhs: Angle, rhs: Angle) -> Angle {
151 | return Angle(radians: lhs.radians + rhs.radians)
152 | }
153 |
154 | static func += (lhs: inout Angle, rhs: Angle) {
155 | lhs.radians += rhs.radians
156 | }
157 | }
158 |
159 | public struct Ray {
160 | let origin: simd_float3
161 | let direction: simd_float3
162 | }
163 |
164 | /*
165 | struct BoundingSphere {
166 | var center = simd_float3()
167 | var radius: Float = 0.0
168 |
169 | init(points: StridedView) {
170 | if points.count > 0 {
171 | // Ref. "A Simple Streaming Algorithm for Minimum Enclosing Balls" Zarrabi-Zadeh, Chan (2006)
172 | var center = simd_float3(points[0].0, points[0].1, points[0].2)
173 | var radius: Float = 0.0
174 | for pointTuple in points {
175 | let point = simd_float3(pointTuple.0, pointTuple.1, pointTuple.2)
176 | let distance = simd_distance(center, point)
177 | if distance > radius {
178 | let deltaRadius = 0.5 * (distance - radius)
179 | radius += deltaRadius
180 | let deltaCenter = (deltaRadius / distance) * (point - center)
181 | center += deltaCenter
182 | }
183 | }
184 | self.center = center
185 | self.radius = radius
186 | }
187 | }
188 |
189 | init(center: simd_float3 = simd_float3(), radius: Float = 0.0) {
190 | self.center = center
191 | self.radius = abs(radius)
192 | }
193 |
194 | func transformed(by transform: simd_float4x4) -> BoundingSphere {
195 | let newCenter = transform * simd_float4(center, 1.0)
196 | let stretchedRadius = transform * simd_float4(radius, radius, radius, 0.0)
197 | let newRadius = max(stretchedRadius.x, max(stretchedRadius.y, stretchedRadius.z))
198 | return BoundingSphere(center: newCenter.xyz, radius: newRadius)
199 | }
200 | }
201 | */
202 |
203 | struct BoundingBox {
204 | var min = simd_float3()
205 | var max = simd_float3()
206 |
207 | init(points: StridedView) {
208 | if points.count > 0 {
209 | var min = simd_float3(points[0].0, points[0].1, points[0].2)
210 | var max = simd_float3(points[0].0, points[0].1, points[0].2)
211 | for pointTuple in points {
212 | let point = simd_float3(pointTuple.0, pointTuple.1, pointTuple.2)
213 | if point.x < min.x { min.x = point.x }
214 | if point.y < min.y { min.y = point.y }
215 | if point.z < min.z { min.z = point.z }
216 | if point.x > max.x { max.x = point.x }
217 | if point.y > max.y { max.y = point.y }
218 | if point.z > max.z { max.z = point.z }
219 | }
220 | self.min = min
221 | self.max = max
222 | }
223 | }
224 |
225 | init(min: simd_float3 = simd_float3(), max: simd_float3 = simd_float3()) {
226 | self.min = min
227 | self.max = max
228 | }
229 |
230 | func transformed(by transform: simd_float4x4) -> BoundingBox {
231 | let transformedCorners: [simd_float3] = [
232 | (transform * simd_float4(min.x, min.y, min.z, 1.0)).xyz,
233 | (transform * simd_float4(min.x, min.y, max.z, 1.0)).xyz,
234 | (transform * simd_float4(min.x, max.y, min.z, 1.0)).xyz,
235 | (transform * simd_float4(min.x, max.y, max.z, 1.0)).xyz,
236 | (transform * simd_float4(max.x, min.y, min.z, 1.0)).xyz,
237 | (transform * simd_float4(max.x, min.y, max.z, 1.0)).xyz,
238 | (transform * simd_float4(max.x, max.y, min.z, 1.0)).xyz,
239 | (transform * simd_float4(max.x, max.y, max.z, 1.0)).xyz,
240 | ]
241 | return transformedCorners.withUnsafeBufferPointer {
242 | BoundingBox(points: StridedView($0.baseAddress!,
243 | offset: 0,
244 | stride: MemoryLayout.stride,
245 | count: transformedCorners.count))
246 | }
247 | }
248 | }
249 |
250 | public struct Transform {
251 | var position: simd_float3
252 | var scale: simd_float3
253 | var orientation: simd_quatf
254 |
255 | var matrix: simd_float4x4 {
256 | let R = matrix_float3x3(orientation)
257 | let TRS = simd_float4x4(simd_float4(scale.x * R.columns.0, 0),
258 | simd_float4(scale.y * R.columns.1, 0),
259 | simd_float4(scale.z * R.columns.2, 0),
260 | simd_float4(position, 1))
261 | return TRS
262 | }
263 |
264 | var inverse: Transform {
265 | let RSinv = self.matrix.upperLeft3x3.inverse
266 | let TRSInv = simd_float4x4(simd_float4(RSinv.columns.0, 0.0),
267 | simd_float4(RSinv.columns.1, 0.0),
268 | simd_float4(RSinv.columns.2, 0.0),
269 | simd_float4(-RSinv * position, 1.0))
270 | return Transform(TRSInv)
271 | }
272 |
273 | init(position: simd_float3 = simd_float3(0, 0, 0),
274 | scale: simd_float3 = simd_float3(1, 1, 1),
275 | orientation: simd_quatf = simd_quatf(ix: 0, iy: 0, iz: 0, r: 1))
276 | {
277 | self.position = position
278 | self.scale = scale
279 | self.orientation = orientation
280 | }
281 |
282 | @available(macOS 13.0, *)
283 | init(_ pose: Pose3D) {
284 | self.position = simd_float3(pose.position)
285 | self.scale = simd_float3(repeating: 1.0)
286 | self.orientation = simd_quatf(pose.rotation)
287 | }
288 |
289 | /// Initializes a Transform by decomposing the provided affine transformation matrix.
290 | /// Assumes the provided matrix is a valid TRS matrix.
291 | init(_ matrix: simd_float4x4) {
292 | self.position = matrix.columns.3.xyz
293 | let RS = matrix.upperLeft3x3
294 | let sx = simd_length(RS.columns.0)
295 | let sy = simd_length(RS.columns.1)
296 | let sz = simd_length(RS.columns.2)
297 | let R = simd_float3x3(RS.columns.0 / sx, RS.columns.1 / sy, RS.columns.2 / sz)
298 | self.scale = simd_float3(sx, sy, sz)
299 | self.orientation = simd_quatf(R)
300 | }
301 |
302 | mutating func setRotation(axis: simd_float3, angle angleRadians: Float) {
303 | orientation = simd_quatf(angle: angleRadians, axis: axis)
304 | }
305 |
306 | mutating func look(at toPoint: simd_float3, from fromPoint: simd_float3, up upVector: simd_float3) {
307 | let zNeg = simd_normalize(toPoint - fromPoint)
308 | let x = simd_normalize(simd_cross(zNeg, upVector))
309 | let y = simd_normalize(simd_cross(x, zNeg))
310 | let R = matrix_float3x3(x, y, -zNeg)
311 | orientation = simd_quaternion(R)
312 | position = fromPoint
313 | }
314 |
315 | static func * (lhs: Transform, rhs: Transform) -> Transform {
316 | // TODO: Write a more efficient implementation exploiting affine matrix properties
317 | return Transform(lhs.matrix * rhs.matrix)
318 | }
319 | }
320 |
--------------------------------------------------------------------------------
/Engine/Mesh.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import Metal
19 | import ModelIO
20 |
21 | public class Submesh : Identifiable {
22 | let primitiveType: MTLPrimitiveType
23 | let indexBuffer: BufferView?
24 | let indexType: MTLIndexType
25 | let indexCount: Int
26 | let materialIndex: Int
27 |
28 | init(primitiveType: MTLPrimitiveType,
29 | indexBuffer: BufferView?,
30 | indexType: MTLIndexType = .uint32,
31 | indexCount: Int = 0,
32 | materialIndex: Int)
33 | {
34 | self.primitiveType = primitiveType
35 | self.indexBuffer = indexBuffer
36 | self.indexType = indexType
37 | self.indexCount = indexCount
38 | self.materialIndex = materialIndex
39 | }
40 | }
41 |
42 | public class Mesh : Identifiable {
43 | let vertexCount: Int
44 | let vertexBuffers: [BufferView]
45 | let vertexDescriptor: MDLVertexDescriptor
46 | let boundingBox: BoundingBox
47 | let submeshes: [Submesh]
48 | var materials: [Material]
49 |
50 | init(vertexCount: Int,
51 | vertexBuffers: [BufferView],
52 | vertexDescriptor: MDLVertexDescriptor,
53 | submeshes: [Submesh],
54 | materials: [Material],
55 | boundingBox: BoundingBox? = nil)
56 | {
57 | self.vertexCount = vertexCount
58 | self.vertexBuffers = vertexBuffers
59 | self.vertexDescriptor = vertexDescriptor
60 | if let boundingBox {
61 | self.boundingBox = boundingBox
62 | } else {
63 | if let positionAttribute = vertexDescriptor.attributeNamed(MDLVertexAttributePosition) {
64 | let positionView = vertexBuffers[positionAttribute.bufferIndex]
65 | let positionBufferLayout = vertexDescriptor.bufferLayouts[positionAttribute.bufferIndex]
66 | let positionAccessor = StridedView(positionView.buffer.contents() + positionView.offset,
67 | offset: positionAttribute.offset,
68 | stride: positionBufferLayout.stride,
69 | count: vertexCount)
70 | self.boundingBox = BoundingBox(points: positionAccessor)
71 | } else {
72 | self.boundingBox = BoundingBox()
73 | }
74 | }
75 | self.submeshes = submeshes
76 | self.materials = materials
77 | }
78 | }
79 |
80 | extension Mesh {
81 | static var defaultVertexDescriptor: MDLVertexDescriptor {
82 | let vertexDescriptor = MDLVertexDescriptor()
83 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition
84 | vertexDescriptor.vertexAttributes[0].format = .float3
85 | vertexDescriptor.vertexAttributes[0].offset = 0
86 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0
87 | vertexDescriptor.bufferLayouts[0].stride = MemoryLayout.stride
88 |
89 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal
90 | vertexDescriptor.vertexAttributes[1].format = .float3
91 | vertexDescriptor.vertexAttributes[1].offset = 0
92 | vertexDescriptor.vertexAttributes[1].bufferIndex = 1
93 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate
94 | vertexDescriptor.vertexAttributes[2].format = .float2
95 | vertexDescriptor.vertexAttributes[2].offset = MemoryLayout.stride
96 | vertexDescriptor.vertexAttributes[2].bufferIndex = 1
97 | vertexDescriptor.bufferLayouts[1].stride = MemoryLayout.stride + MemoryLayout.stride
98 |
99 | return vertexDescriptor
100 | }
101 |
102 | static var skinnedVertexDescriptor: MDLVertexDescriptor {
103 | let vertexDescriptor = MDLVertexDescriptor()
104 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition
105 | vertexDescriptor.vertexAttributes[0].format = .float3
106 | vertexDescriptor.vertexAttributes[0].offset = 0
107 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0
108 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal
109 | vertexDescriptor.vertexAttributes[1].format = .float3
110 | vertexDescriptor.vertexAttributes[1].offset = MemoryLayout.stride * 3
111 | vertexDescriptor.vertexAttributes[1].bufferIndex = 0
112 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate
113 | vertexDescriptor.vertexAttributes[2].format = .float2
114 | vertexDescriptor.vertexAttributes[2].offset = MemoryLayout.stride * 6
115 | vertexDescriptor.vertexAttributes[2].bufferIndex = 0
116 | vertexDescriptor.vertexAttributes[3].name = MDLVertexAttributeJointWeights
117 | vertexDescriptor.vertexAttributes[3].format = .float4
118 | vertexDescriptor.vertexAttributes[3].offset = MemoryLayout.stride * 8
119 | vertexDescriptor.vertexAttributes[3].bufferIndex = 0
120 | vertexDescriptor.vertexAttributes[4].name = MDLVertexAttributeJointIndices
121 | vertexDescriptor.vertexAttributes[4].format = .uShort4
122 | vertexDescriptor.vertexAttributes[4].offset = MemoryLayout.stride * 12
123 | vertexDescriptor.vertexAttributes[4].bufferIndex = 0
124 | vertexDescriptor.bufferLayouts[0].stride = MemoryLayout.stride * 12 + MemoryLayout.stride
125 |
126 | return vertexDescriptor
127 | }
128 | }
129 |
130 | public class Skeleton {
131 | var name: String
132 | var jointPaths: [String]
133 | var inverseBindTransforms: [simd_float4x4]
134 | var restTransforms: [simd_float4x4]
135 |
136 | init(name: String, jointPaths: [String], inverseBindTransforms: [simd_float4x4], restTransforms: [simd_float4x4]) {
137 | self.name = name
138 | self.jointPaths = jointPaths
139 | self.inverseBindTransforms = inverseBindTransforms
140 | self.restTransforms = restTransforms
141 | }
142 | }
143 |
144 | public class Skinner {
145 | let skeleton: Skeleton
146 | let baseMesh: Mesh
147 |
148 | var jointTransforms: [simd_float4x4] {
149 | didSet {
150 | precondition(jointTransforms.count == skeleton.jointPaths.count)
151 | isDirty = true
152 | }
153 | }
154 |
155 | var isDirty = true
156 |
157 | init(skeleton: Skeleton, baseMesh: Mesh) {
158 | self.skeleton = skeleton
159 | self.baseMesh = baseMesh
160 | self.jointTransforms = skeleton.restTransforms
161 | }
162 | }
163 |
164 | extension Mesh {
165 | /// A single-buffer, packed, interleaved vertex data layout suitable for simple vertex skinning use cases
166 | static let postSkinningVertexDescriptor: MDLVertexDescriptor = {
167 | let vertexDescriptor = MDLVertexDescriptor()
168 | vertexDescriptor.vertexAttributes[0].name = MDLVertexAttributePosition
169 | vertexDescriptor.vertexAttributes[0].format = .float3
170 | vertexDescriptor.vertexAttributes[0].offset = 0
171 | vertexDescriptor.vertexAttributes[0].bufferIndex = 0
172 | vertexDescriptor.vertexAttributes[1].name = MDLVertexAttributeNormal
173 | vertexDescriptor.vertexAttributes[1].format = .float3
174 | vertexDescriptor.vertexAttributes[1].offset = MemoryLayout.stride * 3
175 | vertexDescriptor.vertexAttributes[1].bufferIndex = 0
176 | vertexDescriptor.vertexAttributes[2].name = MDLVertexAttributeTextureCoordinate
177 | vertexDescriptor.vertexAttributes[2].format = .float2
178 | vertexDescriptor.vertexAttributes[2].offset = MemoryLayout.stride * 6
179 | vertexDescriptor.vertexAttributes[2].bufferIndex = 0
180 | vertexDescriptor.bufferLayouts[0].stride = MemoryLayout.stride * 8
181 | return vertexDescriptor
182 | }()
183 |
184 | /// Creates a copy of this mesh that is suitable as a post-skinning destination for skinned vertices.
185 | /// Submeshes and materials are copied by reference.
186 | func copyForSkinning(context: MetalContext) -> Mesh {
187 | let vertexDescriptor = Mesh.postSkinningVertexDescriptor
188 | let vertexBufferLength = vertexDescriptor.bufferLayouts[0].stride * vertexCount
189 | let vertexBuffer = context.device.makeBuffer(length: vertexBufferLength,
190 | options: [.storageModePrivate])!
191 | vertexBuffer.label = "Post-Skinning Vertex Attributes"
192 | return Mesh(vertexCount: self.vertexCount,
193 | vertexBuffers: [BufferView(buffer: vertexBuffer)],
194 | vertexDescriptor: vertexDescriptor,
195 | submeshes: submeshes,
196 | materials: materials,
197 | boundingBox: boundingBox)
198 | }
199 | }
200 |
201 | extension Mesh {
202 | class func generateSphere(radius: Float, context: MetalContext) -> Mesh {
203 | let mdlMesh = MDLMesh.init(sphereWithExtent: simd_float3(repeating: radius),
204 | segments: simd_uint2(24, 24),
205 | inwardNormals: false,
206 | geometryType: .triangles,
207 | allocator: nil)
208 | mdlMesh.vertexDescriptor = defaultVertexDescriptor
209 | let tempResourceCache = MetalResourceCache(context: context)
210 | let mesh = Mesh(mdlMesh, context: context, resourceCache: tempResourceCache)
211 | mesh.materials = [PhysicallyBasedMaterial.default]
212 | return mesh
213 | }
214 |
215 | class func generateBox(extents: simd_float3, context: MetalContext) -> Mesh {
216 | let mdlMesh = MDLMesh.init(boxWithExtent: extents,
217 | segments: simd_uint3(1, 1, 1),
218 | inwardNormals: false,
219 | geometryType: .triangles,
220 | allocator: nil)
221 | mdlMesh.vertexDescriptor = defaultVertexDescriptor
222 | let tempResourceCache = MetalResourceCache(context: context)
223 | let mesh = Mesh(mdlMesh, context: context, resourceCache: tempResourceCache)
224 | mesh.materials = [PhysicallyBasedMaterial.default]
225 | return mesh
226 | }
227 | }
228 |
--------------------------------------------------------------------------------
/Engine/MetalContext.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import Metal
19 |
20 | enum ResourceError : Error {
21 | case allocationFailure
22 | case imageLoadFailure
23 | case invalidImageFormat
24 | case invalidState
25 | }
26 |
27 | class MetalContext : @unchecked Sendable {
28 | nonisolated(unsafe) static var shared = MetalContext()
29 |
30 | #if os(visionOS)
31 | let preferredColorPixelFormat = MTLPixelFormat.rgba16Float
32 | #else
33 | let preferredColorPixelFormat = MTLPixelFormat.bgra8Unorm_srgb
34 | #endif
35 |
36 | let preferredDepthPixelFormat = MTLPixelFormat.depth32Float
37 | let preferredRasterSampleCount: Int
38 |
39 | let device: MTLDevice
40 | let commandQueue: MTLCommandQueue
41 | let defaultLibrary: MTLLibrary?
42 |
43 | init(device: MTLDevice? = nil, commandQueue: MTLCommandQueue? = nil) {
44 | guard let metalDevice = device ?? MTLCreateSystemDefaultDevice() else {
45 | fatalError("Metal is not supported")
46 | }
47 | guard let metalCommandQueue = commandQueue ?? metalDevice.makeCommandQueue() else {
48 | fatalError()
49 | }
50 | assert(metalCommandQueue.device.isEqual(metalDevice))
51 |
52 | self.device = metalDevice
53 | self.commandQueue = metalCommandQueue
54 | self.defaultLibrary = metalDevice.makeDefaultLibrary()
55 |
56 | #if targetEnvironment(simulator)
57 | // The Metal API validation layer on Vision Pro simulator claims not to be able
58 | // to resolve Depth32 MSAA targets (it lies), so force MSAA off there.
59 | let candidateSampleCounts = [1]
60 | #else
61 | let candidateSampleCounts = [16, 8, 5, 4, 2, 1]
62 | #endif
63 | let supportedSampleCounts = candidateSampleCounts.filter { metalDevice.supportsTextureSampleCount($0) }
64 | preferredRasterSampleCount = supportedSampleCounts.first ?? 1
65 | print("Supported raster sample counts: \(supportedSampleCounts). Selected \(preferredRasterSampleCount).")
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/Engine/Model.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import Metal
19 | import ModelIO
20 |
21 | class Model {
22 | let rootEntities: [Entity]
23 |
24 | // Model I/O doesn't expose this property, which in USD is configurable via asset metadata,
25 | // but most USDZ files in the wild seem to favor USD's default unit of centimeters, so we
26 | // use this as a heuristic to match content to the real world.
27 | var metersPerUnit: Float = 0.01
28 |
29 | init(fileURL url: URL, context: MetalContext) throws {
30 | var rootEntities = [Entity]()
31 | let resourceCache = MetalResourceCache(context: context)
32 | var entityMap = [ObjectIdentifier : Entity]()
33 | autoreleasepool {
34 | let asset = MDLAsset(url: url, vertexDescriptor: nil, bufferAllocator: nil)
35 |
36 | asset.loadTextures()
37 |
38 | var skeletonMap = [ObjectIdentifier : Skeleton]()
39 |
40 | for mdlObject in asset.childObjects(of: MDLObject.self) {
41 | let entity = Entity()
42 | entity.name = mdlObject.name
43 | entityMap[ObjectIdentifier(mdlObject)] = entity
44 |
45 | if let transformComponent = mdlObject.transform {
46 | entity.modelTransform = Transform(transformComponent.matrix)
47 | }
48 |
49 | if let mdlMesh = mdlObject as? MDLMesh {
50 | // If a mesh has an associated animation bind component, it may have a skeleton and be
51 | // skeletally animated, so we need to apply a vertex descriptor that preserves any joint
52 | // weight/index data. Otherwise, we conform the mesh to our default vertex descriptor.
53 | let isSkinned = (mdlObject.animationBind != nil)
54 | let vertexDescriptor = isSkinned ? Mesh.skinnedVertexDescriptor : Mesh.defaultVertexDescriptor
55 | mdlMesh.vertexDescriptor = vertexDescriptor
56 |
57 | let mesh = Mesh(mdlMesh, context: context, resourceCache: resourceCache)
58 | entity.mesh = mesh
59 | }
60 |
61 | if let mdlSkeleton = mdlObject as? MDLSkeleton {
62 | let skeleton = Skeleton(mdlSkeleton, context: context)
63 | skeletonMap[ObjectIdentifier(mdlSkeleton)] = skeleton
64 | }
65 |
66 | if let animationBind = mdlObject.animationBind, let mdlSkeleton = animationBind.skeleton {
67 | if let skeleton = skeletonMap[ObjectIdentifier(mdlSkeleton)], let baseMesh = entity.mesh {
68 | entity.skinner = Skinner(skeleton: skeleton, baseMesh: baseMesh)
69 | entity.mesh = baseMesh.copyForSkinning(context: context)
70 | }
71 | }
72 |
73 | if let parent = mdlObject.parent {
74 | let parentEntity = entityMap[ObjectIdentifier(parent)]
75 | parentEntity?.addChild(entity)
76 | if parentEntity == nil {
77 | print("Could not retrieve object parent (\"\(parent.name)\") from entity map; hierarchy will be broken")
78 | }
79 | }
80 |
81 | if entity.parent == nil {
82 | rootEntities.append(entity)
83 | }
84 | }
85 | }
86 | self.rootEntities = rootEntities
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/Engine/ModelIOSupport.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Metal
18 | import MetalKit
19 | import ModelIO
20 |
21 | // The Swift interface for Model I/O has been broken since it was released in 2015.
22 | // These extensions correct the types of a number of properties on Model I/O types.
23 |
24 | extension MDLVertexDescriptor {
25 | var vertexAttributes: [MDLVertexAttribute] {
26 | return attributes as! [MDLVertexAttribute]
27 | }
28 |
29 | var bufferLayouts: [MDLVertexBufferLayout] {
30 | return layouts as! [MDLVertexBufferLayout]
31 | }
32 | }
33 |
34 | extension MDLMesh {
35 | var mdlSubmeshes: [MDLSubmesh]? {
36 | return submeshes as! [MDLSubmesh]?
37 | }
38 | }
39 |
40 | extension MTLSamplerDescriptor {
41 | convenience init(mdlTextureFilter: MDLTextureFilter?) {
42 | self.init()
43 | let metalAddressMode: (MDLMaterialTextureWrapMode) -> MTLSamplerAddressMode = { mode in
44 | switch mode {
45 | case .clamp:
46 | return .clampToEdge
47 | case .repeat:
48 | return .repeat
49 | case .mirror:
50 | return .mirrorRepeat
51 | @unknown default:
52 | return .repeat
53 | }
54 | }
55 | if let mdlTextureFilter {
56 | self.minFilter = mdlTextureFilter.minFilter == .nearest ? .nearest : .linear
57 | self.magFilter = mdlTextureFilter.magFilter == .nearest ? .nearest : .linear
58 | self.mipFilter = mdlTextureFilter.mipFilter == .nearest ? .nearest : .linear
59 | self.sAddressMode = metalAddressMode(mdlTextureFilter.sWrapMode)
60 | self.tAddressMode = metalAddressMode(mdlTextureFilter.tWrapMode)
61 | } else {
62 | self.minFilter = .linear
63 | self.magFilter = .linear
64 | self.mipFilter = .linear
65 | self.sAddressMode = .repeat
66 | self.tAddressMode = .repeat
67 | }
68 | }
69 | }
70 |
71 | // This extension enables strided random access into a ModelIO vertex attribute data object.
72 | // The type contained by the underlying data buffer must exactly match the Element type of
73 | // the created view; otherwise behavior is undefined.
74 | extension StridedView {
75 | init(attributeData: MDLVertexAttributeData, count: Int) {
76 | self.init(attributeData.dataStart, offset: 0, stride: attributeData.stride, count: count)
77 | }
78 | }
79 |
80 | class MetalResourceCache {
81 | let context: MetalContext
82 |
83 | private var textureCache: [ObjectIdentifier: MTLTexture] = [:]
84 | private var samplerCache: [MTLSamplerDescriptor : MTLSamplerState] = [:]
85 |
86 | init(context: MetalContext) {
87 | self.context = context
88 | }
89 |
90 | func makeTextureResource(for mdlMaterialProperty: MDLMaterialProperty,
91 | contentType: TextureResource.ContentType) throws -> TextureResource
92 | {
93 | precondition(mdlMaterialProperty.type == .texture)
94 |
95 | let device = context.device
96 |
97 | guard let mdlTextureSampler = mdlMaterialProperty.textureSamplerValue else {
98 | throw ResourceError.invalidState
99 | }
100 |
101 | let samplerDescriptor = MTLSamplerDescriptor(mdlTextureFilter: mdlTextureSampler.hardwareFilter)
102 |
103 | let sampler = samplerCache[samplerDescriptor] ?? {
104 | let sampler = device.makeSamplerState(descriptor: samplerDescriptor)!
105 | samplerCache[samplerDescriptor] = sampler
106 | return sampler
107 | }()
108 |
109 | guard let mdlTexture = mdlTextureSampler.texture else {
110 | throw ResourceError.imageLoadFailure
111 | }
112 |
113 | if let existingTexture = textureCache[ObjectIdentifier(mdlTexture)] {
114 | return .init(texture: existingTexture, sampler: sampler)
115 | }
116 |
117 | let textureLoader = TextureLoader(context: .shared)
118 |
119 | var options: [TextureLoader.Option : Any] = [
120 | .generateMipmaps : true,
121 | // Model I/O retains the lower-left origin texture coordinate convention of formats like USD[Z]
122 | // and OBJ, so we request that texel data be flipped on load. We could instead flip texture
123 | // coordinates when loading models or do this at runtime in the vertex shader, but this
124 | // seems the simplest approach, assuming that textures are always used with their corresponding
125 | // models and we don't mind that models' runtime texture coordinates follow the oppposite of
126 | // our preferred convention.
127 | .flipVertically : true
128 | ]
129 | if contentType == .raw {
130 | options[.sRGB] = false
131 | } else {
132 | options[.sRGB] = true
133 | }
134 | if device.supportsFamily(.apple2) {
135 | options[.storageMode] = MTLStorageMode.shared.rawValue
136 | }
137 |
138 | let texture = try textureLoader.makeTexture(mdlTexture: mdlTexture, options: options)
139 | texture.label = mdlMaterialProperty.name
140 |
141 | textureCache[ObjectIdentifier(mdlTexture)] = texture
142 |
143 | return .init(texture: texture, sampler: sampler)
144 | }
145 | }
146 |
147 | extension PhysicallyBasedMaterial {
148 | convenience init(mdlMaterial: MDLMaterial, resourceCache: MetalResourceCache) {
149 | self.init()
150 |
151 | if let baseColorProperty = mdlMaterial.property(with: MDLMaterialSemantic.baseColor) {
152 | if baseColorProperty.type == .texture {
153 | baseColor.baseColorTexture = try? resourceCache.makeTextureResource(for: baseColorProperty,
154 | contentType: .color)
155 | } else if baseColorProperty.type == .float3 {
156 | let color = baseColorProperty.float3Value
157 | baseColor.baseColorFactor = simd_float4(color, 1.0)
158 | }
159 | }
160 | if let normalProperty = mdlMaterial.property(with: MDLMaterialSemantic.tangentSpaceNormal) {
161 | if normalProperty.type == .texture {
162 | normal.normalTexture = try? resourceCache.makeTextureResource(for: normalProperty,
163 | contentType: .raw)
164 | }
165 | }
166 | if let metalnessProperty = mdlMaterial.property(with: MDLMaterialSemantic.metallic) {
167 | if metalnessProperty.type == .texture {
168 | metalness.metalnessTexture = try? resourceCache.makeTextureResource(for: metalnessProperty,
169 | contentType: .raw)
170 | } else if metalnessProperty.type == .float {
171 | metalness.metalnessFactor = metalnessProperty.floatValue
172 | }
173 | }
174 | if let roughnessProperty = mdlMaterial.property(with: MDLMaterialSemantic.roughness) {
175 | if roughnessProperty.type == .texture {
176 | roughness.roughnessTexture = try? resourceCache.makeTextureResource(for: roughnessProperty,
177 | contentType: .raw)
178 | } else if roughnessProperty.type == .float {
179 | roughness.roughnessFactor = roughnessProperty.floatValue
180 | }
181 | }
182 | if let emissionProperty = mdlMaterial.property(with: MDLMaterialSemantic.emission) {
183 | if emissionProperty.type == .texture {
184 | emissive.emissiveTexture = try? resourceCache.makeTextureResource(for: emissionProperty,
185 | contentType: .color)
186 | } else if emissionProperty.type == .float3 {
187 | emissive.emissiveColor = emissionProperty.float3Value
188 | }
189 | }
190 | if let _ = mdlMaterial.property(with: MDLMaterialSemantic.opacity) {
191 | // There are probably more precise checks we could do to determine if we should enable
192 | // alpha blending, but Model I/O doesn't have a convenient "blend mode" property, so
193 | // we use the presence of the opacity property as a simple heuristic.
194 | blendMode = .sourceOverPremultiplied
195 | }
196 | }
197 | }
198 |
199 | extension Mesh {
200 | convenience init(_ mdlMesh: MDLMesh, context: MetalContext, resourceCache: MetalResourceCache) {
201 | let device = context.device
202 |
203 | let vertexBuffers = mdlMesh.vertexBuffers.map { mdlBuffer in
204 | let bufferMap = mdlBuffer.map()
205 | let buffer = device.makeBuffer(bytes: bufferMap.bytes,
206 | length: mdlBuffer.length,
207 | options: .storageModeShared)!
208 | buffer.label = "Vertex Attributes"
209 | return BufferView(buffer: buffer)
210 | }
211 |
212 | var submeshes = [Submesh]()
213 | var materials = [Material]()
214 | for mdlSubmesh in (mdlMesh.mdlSubmeshes ?? []) {
215 | guard mdlSubmesh.geometryType == .triangles else { continue }
216 |
217 | let mdlIndexBuffer = mdlSubmesh.indexBuffer
218 | let bufferMap = mdlIndexBuffer.map()
219 | let buffer = device.makeBuffer(bytes: bufferMap.bytes,
220 | length: mdlIndexBuffer.length,
221 | options: .storageModeShared)!
222 | buffer.label = "Submesh Indices"
223 | let indexBuffer = BufferView(buffer: buffer)
224 |
225 | let indexType: MTLIndexType = mdlSubmesh.indexType == .uint16 ? .uint16 : .uint32
226 |
227 | var material = PhysicallyBasedMaterial.default
228 | if let mdlMaterial = mdlSubmesh.material {
229 | material = PhysicallyBasedMaterial(mdlMaterial: mdlMaterial, resourceCache: resourceCache)
230 | material.name = mdlMaterial.name
231 | }
232 | // ModelIO doesn't have any means of expressing whether a material is
233 | // double-sided, so we just assume all imported materials are.
234 | material.isDoubleSided = true
235 |
236 | let submesh = Submesh(primitiveType: .triangle,
237 | indexBuffer: indexBuffer,
238 | indexType: indexType,
239 | indexCount: mdlSubmesh.indexCount,
240 | materialIndex: materials.count)
241 |
242 | materials.append(material)
243 | submeshes.append(submesh)
244 | }
245 |
246 | let boundingBox = BoundingBox(min: mdlMesh.boundingBox.minBounds, max: mdlMesh.boundingBox.maxBounds)
247 |
248 | self.init(vertexCount: mdlMesh.vertexCount,
249 | vertexBuffers: vertexBuffers,
250 | vertexDescriptor: mdlMesh.vertexDescriptor,
251 | submeshes: submeshes,
252 | materials: materials,
253 | boundingBox: boundingBox)
254 | }
255 | }
256 |
257 | extension Skeleton {
258 | convenience init(_ mdlSkeleton: MDLSkeleton, context: MetalContext) {
259 | let jointBindMatrices = mdlSkeleton.jointBindTransforms.float4x4Array.map { $0.inverse }
260 | let restMatrices = mdlSkeleton.jointRestTransforms.float4x4Array
261 | self.init(name: mdlSkeleton.name,
262 | jointPaths: mdlSkeleton.jointPaths,
263 | inverseBindTransforms: jointBindMatrices,
264 | restTransforms: restMatrices)
265 | }
266 | }
267 |
268 | extension MDLObject {
269 | var animationBind: MDLAnimationBindComponent? {
270 | return components.filter({
271 | $0 is MDLAnimationBindComponent
272 | }).first as? MDLAnimationBindComponent
273 | }
274 | }
275 |
--------------------------------------------------------------------------------
/Engine/Physics/JoltPhysics.h:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | #import
18 | #import
19 |
20 | NS_ASSUME_NONNULL_BEGIN
21 |
22 | typedef NS_ENUM(NSInteger, SRJBodyType) {
23 | SRJBodyTypeStatic,
24 | SRJBodyTypeDynamic,
25 | SRJBodyTypeKinematic,
26 | };
27 |
28 | typedef struct SRJRigidBodyTransform_t {
29 | simd_float3 position;
30 | simd_quatf orientation;
31 | } SRJRigidBodyTransform;
32 |
33 | typedef struct SRJBodyProperties_t {
34 | float mass;
35 | float friction;
36 | float restitution;
37 | bool isAffectedByGravity;
38 | } SRJBodyProperties;
39 |
40 | @interface SRJPhysicsShape : NSObject
41 | + (instancetype)newSphereShapeWithRadius:(float)radius scale:(simd_float3)scale
42 | NS_SWIFT_NAME(makeSphereShape(radius:scale:));
43 |
44 | + (instancetype)newBoxShapeWithExtents:(simd_float3)extents scale:(simd_float3)scale
45 | NS_SWIFT_NAME(makeBoxShape(extents:scale:));
46 |
47 | + (nullable instancetype)newConvexHullShapeWithVertices:(const simd_float3 *)vertices
48 | vertexCount:(NSInteger)vertexCount
49 | scale:(simd_float3)scale
50 | error:(NSError **)error
51 | NS_SWIFT_NAME(makeConvexHullShape(vertices:vertexCount:scale:))
52 | NS_REFINED_FOR_SWIFT;
53 |
54 | + (instancetype)newConcavePolyhedronShapeWithVertices:(const simd_float3 *)vertices
55 | vertexCount:(NSInteger)vertexCount
56 | indices:(const uint32_t *)indices
57 | indexCount:(NSInteger)indexCount
58 | scale:(simd_float3)scale
59 | NS_SWIFT_NAME(makeConcavePolyhedronShape(vertices:vertexCount:indices:indexCount:scale:)) NS_REFINED_FOR_SWIFT;
60 |
61 | - (instancetype)init NS_UNAVAILABLE;
62 | @end
63 |
64 | @interface SRJPhysicsBody : NSObject
65 | - (instancetype)init NS_UNAVAILABLE;
66 |
67 | @property (nonatomic, assign) SRJRigidBodyTransform transform;
68 |
69 | @end
70 |
71 | @interface SRJHitTestResult : NSObject
72 | @property (nonatomic, weak) SRJPhysicsBody *body;
73 | @property (nonatomic, assign) simd_float3 position;
74 | @property (nonatomic, assign) CGFloat distance;
75 | @end
76 |
77 | @interface SRJPhysicsWorld : NSObject
78 |
79 | + (void)initializeJoltPhysics;
80 | + (void)deinitializeJoltPhysics;
81 |
82 | - (SRJPhysicsBody *)createAndAddPhysicsBodyWithType:(SRJBodyType)type
83 | properties:(SRJBodyProperties)bodyProperties
84 | physicsShape:(SRJPhysicsShape *)shape
85 | initialTransform:(SRJRigidBodyTransform)transform
86 | NS_SWIFT_NAME(addPhysicsBody(type:bodyProperties:physicsShape:initialTransform:));
87 |
88 | - (void)removePhysicsBody:(SRJPhysicsBody *)physicsBody;
89 |
90 | - (void)updateWithTimestep:(NSTimeInterval)timestep NS_SWIFT_NAME(update(timestep:));
91 |
92 | - (NSArray *)hitTestWithSegmentFromPoint:(simd_float3)origin toPoint:(simd_float3)dest
93 | NS_SWIFT_NAME(hitTestWithSegment(from:to:));
94 |
95 | @end
96 |
97 | NS_ASSUME_NONNULL_END
98 |
--------------------------------------------------------------------------------
/Engine/Physics/JoltPhysics.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import ModelIO
19 |
20 | extension Mesh {
21 | func getPackedVertexPositions() -> [simd_float3] {
22 | let attribute = vertexDescriptor.attributeNamed(MDLVertexAttributePosition)!
23 | let layout = vertexDescriptor.bufferLayouts[attribute.bufferIndex]
24 | precondition(attribute.format == .float3)
25 |
26 | let vertexBuffer = vertexBuffers[attribute.bufferIndex]
27 | let stride = layout.stride
28 | let bufferBaseAddr = vertexBuffer.buffer.contents().advanced(by: vertexBuffer.offset)
29 | let attributeBaseAddr = bufferBaseAddr.advanced(by: attribute.offset)
30 | let positions = [simd_float3].init(unsafeUninitializedCapacity: vertexCount) { vertexPtr, initializedCount in
31 | for i in 0.. [UInt32] {
41 | let indexCount = submeshes.reduce(0) { return $0 + $1.indexCount }
42 | var indices = [UInt32]()
43 | indices.reserveCapacity(indexCount)
44 | for submesh in self.submeshes {
45 | guard let indexBuffer = submesh.indexBuffer else { continue }
46 | if submesh.indexType == .uint16 {
47 | let indexBaseAddr = indexBuffer.buffer.contents().advanced(by: indexBuffer.offset).assumingMemoryBound(to: UInt16.self)
48 | for i in 0.. SRJPhysicsShape {
64 | return try points.withUnsafeBufferPointer { ptr in
65 | return try makeConvexHullShape(vertices: ptr.baseAddress!, vertexCount: points.count, scale: scale)
66 | }
67 | }
68 |
69 | class func makeConcavePolyhedronShape(points: [simd_float3], indices: [UInt32], scale:simd_float3) -> SRJPhysicsShape {
70 | return points.withUnsafeBufferPointer { pointPtr in
71 | indices.withUnsafeBufferPointer { indexPtr in
72 | return makeConcavePolyhedronShape(vertices: pointPtr.baseAddress!,
73 | vertexCount: points.count,
74 | indices: indexPtr.baseAddress!,
75 | indexCount: indices.count,
76 | scale: scale)
77 |
78 | }
79 | }
80 | }
81 | }
82 |
83 | extension PhysicsShape {
84 | func makeJoltPhysicsShape(scale: simd_float3) throws -> SRJPhysicsShape {
85 | switch type {
86 | case .sphere(let radius):
87 | return SRJPhysicsShape.makeSphereShape(radius: radius, scale: scale)
88 | case .box(let extents):
89 | return SRJPhysicsShape.makeBoxShape(extents: extents, scale: scale)
90 | case .convexHull(let mesh):
91 | let points = mesh.getPackedVertexPositions()
92 | return try SRJPhysicsShape.makeConvexHullShape(points: points, scale: scale)
93 | case .concaveMesh(let mesh):
94 | let points = mesh.getPackedVertexPositions()
95 | let indices = mesh.getPackedSubmeshIndices()
96 | return SRJPhysicsShape.makeConcavePolyhedronShape(points: points, indices: indices, scale: scale)
97 | }
98 | }
99 | }
100 |
101 | class JoltPhysicsBridge : PhysicsBridge {
102 | let physicsWorld: SRJPhysicsWorld
103 | var registeredEntities: Set = []
104 | var bodiesForEntities: [ObjectIdentifier: SRJPhysicsBody] = [:]
105 | var entitiesForBodies: [ObjectIdentifier: Entity] = [:]
106 |
107 | init() {
108 | physicsWorld = SRJPhysicsWorld()
109 | }
110 |
111 | func addEntity(_ entity: Entity) {
112 | if registeredEntities.contains(entity) {
113 | print("Tried to add entity \(entity.name) to physics bridge when it was already registered!")
114 | return
115 | }
116 | registeredEntities.insert(entity)
117 | if let physicsBody = entity.physicsBody {
118 | do {
119 | let bodyType: SRJBodyType = switch physicsBody.mode {
120 | case .static: .`static`
121 | case .dynamic: .dynamic
122 | case .kinematic: .kinematic
123 | }
124 | let worldTransform = entity.worldTransform
125 | let bodyTransform = SRJRigidBodyTransform(position: worldTransform.position,
126 | orientation: worldTransform.orientation)
127 | let properties = SRJBodyProperties(mass: physicsBody.mass,
128 | friction: physicsBody.friction,
129 | restitution: physicsBody.restitution,
130 | isAffectedByGravity: physicsBody.isAffectedByGravity)
131 | let backendShape = try physicsBody.shape.makeJoltPhysicsShape(scale: worldTransform.scale)
132 | let backendBody = physicsWorld.addPhysicsBody(type: bodyType,
133 | bodyProperties: properties,
134 | physicsShape: backendShape,
135 | initialTransform: bodyTransform)
136 | bodiesForEntities[ObjectIdentifier(entity)] = backendBody
137 | entitiesForBodies[ObjectIdentifier(backendBody)] = entity
138 | } catch {
139 | print("Failed to register entity \(entity.name) to physics bridge with error: \(error.localizedDescription)")
140 | }
141 | } else {
142 | // TODO: Handle entities with no physics body but with a mesh that should be included in hit-testing?
143 | }
144 | }
145 |
146 | func removeEntity(_ entity: Entity) {
147 | if let body = bodiesForEntities[ObjectIdentifier(entity)] {
148 | physicsWorld.remove(body)
149 | entitiesForBodies.removeValue(forKey: ObjectIdentifier(body))
150 | bodiesForEntities.removeValue(forKey: ObjectIdentifier(entity))
151 | }
152 | registeredEntities.remove(entity)
153 | }
154 |
155 | func update(entities: [Entity], timestep: TimeInterval) {
156 | for physicsEntity in entities.filter({ $0.physicsBody != nil }) {
157 | if let body = bodiesForEntities[ObjectIdentifier(physicsEntity)] {
158 | let entityTransform = physicsEntity.worldTransform
159 | let bodyTransform = SRJRigidBodyTransform(position: entityTransform.position,
160 | orientation: entityTransform.orientation)
161 | body.transform = bodyTransform
162 | } else {
163 | print("Tried to update entity \(physicsEntity.name) before it was added to the physics bridge.")
164 | }
165 | }
166 |
167 | physicsWorld.update(timestep: timestep)
168 |
169 | for dynamicEntity in entities.filter({ $0.physicsBody?.mode == .dynamic }) {
170 | if let dynamicBody = bodiesForEntities[ObjectIdentifier(dynamicEntity)] {
171 | let bodyTransform = dynamicBody.transform
172 | dynamicEntity.worldTransform = Transform(position: bodyTransform.position,
173 | scale: dynamicEntity.modelTransform.scale,
174 | orientation: bodyTransform.orientation)
175 | } else {
176 | print("Tried to update dynamic entity \(dynamicEntity.name) before it was added to the physics bridge.")
177 | }
178 | }
179 | }
180 |
181 | func hitTestWithSegment(from: simd_float3, to: simd_float3) -> [HitTestResult] {
182 | let results = physicsWorld.hitTestWithSegment(from: from, to: to)
183 | return results.map { result in
184 | if let backendBody = result.body {
185 | let entity = entitiesForBodies[ObjectIdentifier(backendBody)]
186 | return HitTestResult(entity: entity, worldPosition: result.position)
187 | }
188 | return HitTestResult(entity: nil, worldPosition: result.position)
189 | }
190 | }
191 | }
192 |
--------------------------------------------------------------------------------
/Engine/Physics/PhysicsBridge.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 | import simd
19 |
20 | struct HitTestResult {
21 | let entity: Entity?
22 | let worldPosition: simd_float3
23 | }
24 |
25 | protocol PhysicsBridge {
26 | func addEntity(_ entity: Entity)
27 | func removeEntity(_ entity: Entity)
28 | func update(entities: [Entity], timestep: TimeInterval)
29 | func hitTestWithSegment(from: simd_float3, to: simd_float3) -> [HitTestResult]
30 | }
31 |
--------------------------------------------------------------------------------
/Engine/PhysicsBody.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Foundation
18 |
19 | enum PhysicsShapeType {
20 | case box(_ extents: simd_float3)
21 | case sphere(_ radius: Float)
22 | case convexHull(_ mesh: Mesh)
23 | case concaveMesh(_ mesh: Mesh)
24 | }
25 |
26 | enum PhysicsBodyMode {
27 | case `static`
28 | case dynamic
29 | case kinematic
30 | }
31 |
32 | struct CollisionGroup : OptionSet {
33 | let rawValue: UInt32
34 |
35 | static let `default`: CollisionGroup = CollisionGroup(rawValue: 1 << 0)
36 | static let sceneUnderstanding: CollisionGroup = CollisionGroup(rawValue: 1 << 1)
37 | static let all: CollisionGroup = CollisionGroup(rawValue: UInt32.max)
38 | }
39 |
40 | struct CollisionFilter : Equatable {
41 | static let `default`: CollisionFilter = .init(group: .default, mask: .all)
42 |
43 | var group: CollisionGroup
44 | var mask: CollisionGroup
45 |
46 | init(group: CollisionGroup, mask: CollisionGroup) {
47 | self.group = group
48 | self.mask = mask
49 | }
50 | }
51 |
52 | struct Contact : Sendable {
53 | let point: simd_float3
54 | //let normal: simd_float3
55 | let impulse: Float
56 | let impulseDirection: simd_float3
57 | let penetrationDistance: Float
58 | }
59 |
60 | class PhysicsShape {
61 | let type: PhysicsShapeType
62 |
63 | init(boxWithExtents extents: simd_float3) {
64 | self.type = .box(extents)
65 | }
66 |
67 | init(sphereWithRadius radius: Float) {
68 | self.type = .sphere(radius)
69 | }
70 |
71 | init(convexHullFromMesh mesh: Mesh) {
72 | self.type = .convexHull(mesh)
73 | }
74 |
75 | init(concaveMeshFromMesh mesh: Mesh) {
76 | self.type = .concaveMesh(mesh)
77 | }
78 | }
79 |
80 | struct PhysicsBody {
81 | let mode: PhysicsBodyMode
82 | let shape: PhysicsShape
83 | let mass: Float // For dynamic/kinematic bodies, mass=0 tells the system to calculate the mass. Ignored otherwise.
84 | let friction: Float
85 | let restitution: Float
86 | let filter: CollisionFilter = .default
87 | let isAffectedByGravity: Bool = true
88 |
89 | init(mode: PhysicsBodyMode,
90 | shape: PhysicsShape,
91 | mass: Float = 0.0,
92 | friction: Float = 0.5,
93 | restitution: Float = 0.0)
94 | {
95 | self.mode = mode
96 | self.shape = shape
97 | self.mass = mass
98 | self.friction = friction
99 | self.restitution = restitution
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/Engine/ShaderTypes.h:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | #pragma once
18 |
19 | #include
20 |
21 | #if __METAL_VERSION__
22 | #define CONST constant
23 | #else
24 | #define CONST static const
25 | #endif
26 |
27 | CONST unsigned int LightTypeDirectional = 0;
28 | CONST unsigned int LightTypePoint = 1;
29 | CONST unsigned int LightTypeSpot = 2;
30 |
31 | // We reserve vertex buffer indices 0-3 for vertex attributes,
32 | // since different materials and meshes may prefer different layouts.
33 | CONST unsigned int VertexBufferPassConstants = 4;
34 | CONST unsigned int VertexBufferInstanceConstants = 5;
35 | CONST unsigned int VertexBufferSkinningJointTransforms = 6;
36 | CONST unsigned int VertexBufferSkinningVerticesOut = 16;
37 |
38 | CONST unsigned int FragmentBufferPassConstants = 0;
39 | CONST unsigned int FragmentBufferMaterialConstants = 1;
40 | CONST unsigned int FragmentBufferLights = 2;
41 |
42 | CONST unsigned int FragmentTextureBaseColor = 0;
43 | CONST unsigned int FragmentTextureNormal = 1;
44 | CONST unsigned int FragmentTextureMetalness = 2;
45 | CONST unsigned int FragmentTextureRoughness = 3;
46 | CONST unsigned int FragmentTextureEmissive = 4;
47 | CONST unsigned int FragmentTextureEnvironmentLight = 30;
48 |
49 | struct Frustum {
50 | simd_float4 planes[6];
51 | };
52 |
53 | #define MAX_VIEW_COUNT 2
54 |
55 | struct PassConstants {
56 | simd_float4x4 viewMatrices[MAX_VIEW_COUNT];
57 | simd_float4x4 projectionMatrices[MAX_VIEW_COUNT];
58 | simd_float3 cameraPositions[MAX_VIEW_COUNT]; // world space
59 | simd_float4x4 environmentLightMatrix;
60 | unsigned int activeLightCount;
61 | };
62 |
63 | struct InstanceConstants {
64 | simd_float4x4 modelMatrix;
65 | simd_float3x3 normalMatrix;
66 | };
67 |
68 | struct PBRMaterialConstants {
69 | simd_float4 baseColorFactor;
70 | simd_float3 emissiveColor;
71 | float normalScale;
72 | float metallicFactor;
73 | float roughnessFactor;
74 | float emissiveStrength;
75 | } ;
76 |
77 | struct PBRLight {
78 | simd_float3 direction;
79 | simd_float3 position;
80 | simd_float3 color;
81 | float range;
82 | float intensity;
83 | float innerConeCos;
84 | float outerConeCos;
85 | unsigned int type;
86 | };
87 |
--------------------------------------------------------------------------------
/Engine/Skinning.metal:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | #include
18 | using namespace metal;
19 |
20 | #include "ShaderTypes.h"
21 |
22 | struct SkinnedVertexIn {
23 | float3 position [[attribute(0)]];
24 | float3 normal [[attribute(1)]];
25 | float2 texCoords [[attribute(2)]];
26 | float4 jointWeights [[attribute(3)]];
27 | ushort4 jointIndices [[attribute(4)]];
28 | };
29 |
30 | // Since we use a fixed vertex layout for post-skinned vertices, this
31 | // structure must exactly match Mesh.postSkinningVertexDescriptor.
32 | struct SkinnedVertexOut {
33 | packed_float3 position;
34 | packed_float3 normal;
35 | packed_float2 texCoords;
36 | };
37 |
38 | [[vertex]]
39 | void vertex_skin(SkinnedVertexIn in [[stage_in]],
40 | constant float4x4 *jointTransforms [[buffer(VertexBufferSkinningJointTransforms)]],
41 | device SkinnedVertexOut *outVertices [[buffer(VertexBufferSkinningVerticesOut)]],
42 | uint vertexID [[vertex_id]])
43 | {
44 | float4 weights = in.jointWeights;
45 | //float weightSum = weights[0] + weights[1] + weights[2] + weights[3];
46 | //weights /= weightSum;
47 |
48 | float4x4 skinningMatrix = weights[0] * jointTransforms[in.jointIndices[0]] +
49 | weights[1] * jointTransforms[in.jointIndices[1]] +
50 | weights[2] * jointTransforms[in.jointIndices[2]] +
51 | weights[3] * jointTransforms[in.jointIndices[3]];
52 |
53 | float3 skinnedPosition = (skinningMatrix * float4(in.position, 1.0f)).xyz;
54 | // n.b. We'd ordinarily use the inverse transpose of the skinning matrix here to
55 | // get correct normal transformation, but if we assume uniform scale (which skinning
56 | // matrices almost always have), the inverse transpose produces the same vector
57 | // as the matrix itself, up to scale, so we avoid the expense of inverting.
58 | float3 skinnedNormal = normalize((skinningMatrix * float4(in.normal, 0.0f)).xyz);
59 |
60 | outVertices[vertexID].position = skinnedPosition;
61 | outVertices[vertexID].normal = skinnedNormal;
62 | outVertices[vertexID].texCoords = in.texCoords;
63 | }
64 |
--------------------------------------------------------------------------------
/Engine/SpatialEvent.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import SwiftUI
18 | import Spatial
19 |
20 | public enum Event : Sendable {
21 | case worldAnchor(WorldAnchor, AnchorEvent)
22 | case meshAnchor(MeshAnchor, AnchorEvent)
23 | case planeAnchor(PlaneAnchor, AnchorEvent)
24 | case handAnchor(HandAnchor, AnchorEvent)
25 | case environmentLightAnchor(EnvironmentLightAnchor, AnchorEvent)
26 | case spatialInput(SpatialInputEvent)
27 | }
28 |
29 | public struct SpatialInputEvent : Identifiable, Hashable, Sendable {
30 | public typealias ID = UUID
31 |
32 | public var id: ID
33 |
34 | public enum Kind : Hashable, Sendable {
35 | case touch
36 | case directPinch
37 | case indirectPinch
38 | case pointer
39 | }
40 |
41 | public enum Phase : Hashable, Sendable {
42 | case active
43 | case ended
44 | case cancelled
45 | }
46 |
47 | public struct InputDevicePose : Hashable, Sendable {
48 | public var altitude: Angle2D
49 | public var azimuth: Angle2D
50 | public var pose3D: Pose3D
51 | }
52 |
53 | public var timestamp: TimeInterval
54 | public var kind: SpatialInputEvent.Kind
55 | public var location: CGPoint
56 | public var phase: SpatialInputEvent.Phase
57 | public var inputDevicePose: SpatialInputEvent.InputDevicePose?
58 | public var location3D: Point3D
59 | public var selectionRay: Ray3D?
60 | public var handedness: Handedness
61 | }
62 |
63 | extension SpatialInputEvent {
64 | init(_ event: SpatialEventCollection.Event) {
65 | self.id = UUID()
66 | self.timestamp = event.timestamp
67 | self.kind = switch event.kind {
68 | case .touch: .touch
69 | case .directPinch: .directPinch
70 | case .indirectPinch: .indirectPinch
71 | case .pointer: .pointer
72 | @unknown default: fatalError()
73 | }
74 | self.location = event.location
75 | self.phase = switch event.phase {
76 | case .active: .active
77 | case .ended: .ended
78 | case .cancelled: .cancelled
79 | @unknown default: fatalError()
80 | }
81 | #if os(visionOS)
82 | if let pose = event.inputDevicePose {
83 | self.inputDevicePose = InputDevicePose(altitude: Angle2D(radians: pose.altitude.radians),
84 | azimuth: Angle2D(radians: pose.azimuth.radians),
85 | pose3D: pose.pose3D)
86 | }
87 | self.location3D = event.location3D
88 | self.selectionRay = event.selectionRay
89 | self.handedness = switch event.chirality {
90 | case .left: .left
91 | case .right: .right
92 | case .none: .none
93 | }
94 | #else
95 | self.location3D = .zero
96 | self.handedness = .none
97 | #endif
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/Engine/SpatialRenderLoop.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | #if os(visionOS)
18 | import ARKit
19 | import CompositorServices
20 | import Metal
21 | import SwiftUI
22 |
23 | extension LayerRenderer.Drawable : FrameResourceProvider {
24 | var storeDepth: Bool {
25 | // It is important that we provide accurate depth information to
26 | // Compositor Services so it can accurately reproject content,
27 | // especially in mixed immersion ("passthrough") mode.
28 | true
29 | }
30 | }
31 |
32 | extension LayerRenderer.Clock.Instant {
33 | var timeInterval: TimeInterval {
34 | let components = LayerRenderer.Clock.Instant.epoch.duration(to: self).components
35 | return TimeInterval(components.seconds) + TimeInterval(components.attoseconds) * 1e-18
36 | }
37 | }
38 |
39 | class SpatialRenderLoop {
40 | let context: MetalContext
41 | let renderer: SceneRenderer
42 | let scene: SceneContent
43 | let sessionManager: ARSessionManager
44 |
45 | private let clock = LayerRenderer.Clock()
46 | private var layerRenderer: LayerRenderer!
47 | private var lastFramePresentationTime: TimeInterval
48 |
49 | init(scene: SceneContent, renderer: SceneRenderer, sessionManager: ARSessionManager) {
50 | self.renderer = renderer
51 | self.scene = scene
52 | self.context = renderer.context
53 | self.sessionManager = sessionManager
54 |
55 | lastFramePresentationTime = clock.now.timeInterval
56 | }
57 |
58 | func run(_ layerRenderer: LayerRenderer) async {
59 | self.layerRenderer = layerRenderer
60 |
61 | var isRendering = true
62 | while isRendering {
63 | switch layerRenderer.state {
64 | case .paused:
65 | print("Layer renderer is in paused state.")
66 | layerRenderer.waitUntilRunning()
67 | print("Layer renderer is running.")
68 | case .running:
69 | autoreleasepool {
70 | frame()
71 | }
72 | case .invalidated:
73 | print("Layer renderer was invalidated.")
74 | fallthrough
75 | default:
76 | isRendering = false
77 | }
78 | }
79 | }
80 |
81 | func frame() {
82 | guard let frame = layerRenderer.queryNextFrame() else { return }
83 | guard let predictedTiming = frame.predictTiming() else { return }
84 |
85 | // Calculate timestep from previous frame to this frame, capping it at 33 ms so
86 | // animations don't jump way ahead in case we were paused or had a long hitch.
87 | let deltaTime = min(predictedTiming.presentationTime.timeInterval - lastFramePresentationTime, 1.0 / 30.0)
88 |
89 | frame.startUpdate()
90 | updateHandAnchors(at: predictedTiming.trackableAnchorTime.timeInterval)
91 | scene.update(deltaTime)
92 | frame.endUpdate()
93 |
94 | clock.wait(until: predictedTiming.optimalInputTime)
95 |
96 | frame.startSubmission()
97 |
98 | guard let drawable = frame.queryDrawable() else { return }
99 | let finalTiming = drawable.frameTiming
100 | let presentationTimestamp = finalTiming.presentationTime.timeInterval
101 | let deviceAnchor = sessionManager.queryDeviceAnchor(at: presentationTimestamp)
102 | drawable.deviceAnchor = deviceAnchor
103 | if let deviceAnchor {
104 | renderFrame(frame: frame, drawable: drawable, deviceAnchor: deviceAnchor)
105 | }
106 |
107 | let commandBuffer = context.commandQueue.makeCommandBuffer()!
108 | commandBuffer.label = "Present Drawable"
109 | drawable.encodePresent(commandBuffer: commandBuffer)
110 | commandBuffer.commit()
111 |
112 | lastFramePresentationTime = presentationTimestamp
113 |
114 | frame.endSubmission()
115 | }
116 |
117 | private func updateHandAnchors(at time: TimeInterval) {
118 | let (maybeLeftHand, maybeRightHand) = sessionManager.queryHandAnchors(at: time)
119 | if let hand = maybeLeftHand {
120 | scene.enqueueEvent(.handAnchor(HandAnchor(hand), .updated))
121 | }
122 | if let hand = maybeRightHand {
123 | scene.enqueueEvent(.handAnchor(HandAnchor(hand), .updated))
124 | }
125 | }
126 |
127 | func renderFrame(frame: LayerRenderer.Frame, drawable: LayerRenderer.Drawable, deviceAnchor: DeviceAnchor) {
128 | let viewCount = drawable.views.count
129 | let deviceTransform = deviceAnchor.originFromAnchorTransform
130 | let cameraTransforms = drawable.views.map { view in deviceTransform * view.transform }
131 | let viewMatrices = cameraTransforms.map(\.inverse)
132 | let projectionMatrices = (0.. MTLTexture {
38 | let device = context.device
39 |
40 | let bytesPerPixel = 4
41 | var pixelFormat = MTLPixelFormat.rgba8Unorm
42 |
43 | if let srgbOption = options?[.sRGB] as? NSNumber {
44 | if srgbOption.boolValue {
45 | pixelFormat = MTLPixelFormat.rgba8Unorm_srgb
46 | }
47 | }
48 |
49 | var storageMode = MTLStorageMode.shared
50 | #if os(macOS)
51 | if !device.hasUnifiedMemory {
52 | storageMode = MTLStorageMode.managed
53 | }
54 | #endif
55 | if let storageModeOption = options?[.storageMode] as? NSNumber {
56 | if let preferredStorageMode = MTLStorageMode(rawValue: storageModeOption.uintValue) {
57 | storageMode = preferredStorageMode
58 | }
59 | }
60 |
61 | var usageMode = MTLTextureUsage.shaderRead
62 | if let usageModeOption = options?[.usageMode] as? NSNumber {
63 | let preferredUsageMode = MTLTextureUsage(rawValue: usageModeOption.uintValue)
64 | usageMode = preferredUsageMode
65 | }
66 |
67 | var wantsMipmaps = false
68 | if let mipmapOption = options?[.generateMipmaps] as? NSNumber {
69 | wantsMipmaps = mipmapOption.boolValue
70 | }
71 |
72 | var flipVertically = false
73 | if let flipOption = options?[.flipVertically] as? NSNumber {
74 | flipVertically = flipOption.boolValue
75 | }
76 |
77 | let getImageData: () -> Data? = {
78 | if flipVertically {
79 | return mdlTexture.texelDataWithBottomLeftOrigin(atMipLevel: 0, create: true)
80 | } else {
81 | return mdlTexture.texelDataWithTopLeftOrigin(atMipLevel: 0, create: true)
82 | }
83 | }
84 |
85 | guard let data = getImageData() else {
86 | throw ResourceError.imageLoadFailure
87 | }
88 |
89 | let width = Int(mdlTexture.dimensions.x)
90 | let height = Int(mdlTexture.dimensions.y)
91 |
92 | let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: pixelFormat,
93 | width: width,
94 | height: height,
95 | mipmapped: wantsMipmaps)
96 | textureDescriptor.usage = usageMode
97 | textureDescriptor.storageMode = storageMode
98 |
99 | guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
100 | throw ResourceError.allocationFailure
101 | }
102 |
103 | let bytesPerRow = width * bytesPerPixel
104 |
105 | let region = MTLRegionMake2D(0, 0, width, height)
106 | data.withUnsafeBytes { imageBytes in
107 | texture.replace(region: region, mipmapLevel: 0, withBytes: imageBytes.baseAddress!, bytesPerRow: bytesPerRow)
108 | }
109 |
110 | if wantsMipmaps {
111 | let commandQueue = context.commandQueue
112 | if let commandBuffer = commandQueue.makeCommandBuffer() {
113 | if let mipmapEncoder = commandBuffer.makeBlitCommandEncoder() {
114 | mipmapEncoder.generateMipmaps(for: texture)
115 | mipmapEncoder.endEncoding()
116 | }
117 | commandBuffer.commit()
118 | }
119 | }
120 |
121 | return texture
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/Engine/TextureResource.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2024 Warren Moore
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | import Metal
18 |
19 | class TextureResource {
20 | enum ContentType {
21 | case raw
22 | case color
23 | }
24 |
25 | let texture: MTLTexture
26 | let sampler: MTLSamplerState?
27 |
28 | init(texture: MTLTexture, sampler: MTLSamplerState? = nil) {
29 | self.texture = texture
30 | self.sampler = sampler
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
13 |
14 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
15 |
16 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
17 |
18 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
19 |
20 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
21 |
22 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
23 |
24 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
25 |
26 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
27 |
28 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
29 |
30 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
31 |
32 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
33 |
34 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
35 |
36 | You must give any other recipients of the Work or Derivative Works a copy of this License; and
37 | You must cause any modified files to carry prominent notices stating that You changed the files; and
38 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
39 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
40 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
41 |
42 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
43 |
44 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
45 |
46 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
47 |
48 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
49 |
50 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
51 |
52 | END OF TERMS AND CONDITIONS
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Spatial Rendering for Apple Vision Pro
2 |
3 | **Important**: This repository uses git submodules. Use `--recursive` when cloning. e.g.:
4 |
5 | ```
6 | $ git clone --recursive --depth 1 git@github.com:metal-by-example/spatial-rendering.git
7 | ```
8 |
9 | ## Talk
10 |
11 | A recording of the talk for which this sample was created is available on [YouTube](https://youtu.be/vO0M4c9mb2E).
12 |
13 | See also the slides that accompanied this talk:
14 |
15 | [](https://speakerdeck.com/warrenm/spatial-rendering-for-apple-vision-pro)
16 |
17 | ## Overview
18 |
19 | Modern RealityKit is a powerful framework for building spatial experiences, but sometimes you need to use lower level facilities like ARKit, Metal, and Compositor Services.
20 |
21 | This sample is a medium-scale example of how to use various features from ARKit and Metal to perform spatial rendering in a passthrough style. It allows the user to scan their environment for flat surfaces and place a wooden block tower, then interact with it with their hands.
22 |
23 | ## Usage
24 |
25 | It is recommended to run this app on Vision Pro hardware rather than the simulator. Although it will run on the simulator, it has no interactive features in this context because of its reliance on hand tracking.
26 |
27 | To build and run the app, you will need to ensure that the dependency on the Jolt physics engine is fulfilled (ideally by cloning recursively, or by a subsequent `git submodule init & git submodule update`)—just downloading the code from GitHub is **not** sufficient.
28 |
29 | You will also need to set your team identifier in the project in order for code-signing to work.
30 |
31 |
32 |
--------------------------------------------------------------------------------
/Resources/Blocks.usdz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Blocks.usdz
--------------------------------------------------------------------------------
/Resources/Hand_Left.usdz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Hand_Left.usdz
--------------------------------------------------------------------------------
/Resources/Hand_Right.usdz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Hand_Right.usdz
--------------------------------------------------------------------------------
/Resources/Hand_Right_backup.usdz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Hand_Right_backup.usdz
--------------------------------------------------------------------------------
/Resources/LICENSE:
--------------------------------------------------------------------------------
1 | Hand models, obtained from https://github.com/immersive-web/webxr-input-profiles,
2 | have been converted from glTF format to USDZ format and are subject to the
3 |
4 | MIT License
5 |
6 | Copyright (c) 2019 Amazon
7 |
8 | Permission is hereby granted, free of charge, to any person obtaining a copy
9 | of this software and associated documentation files (the "Software"), to deal
10 | in the Software without restriction, including without limitation the rights
11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 | copies of the Software, and to permit persons to whom the Software is furnished
13 | to do so, subject to the following conditions:
14 |
15 | The above copyright notice and this permission notice (including the next
16 | paragraph) shall be included in all copies or substantial portions of the
17 | Software.
18 |
19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
21 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
22 | OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
23 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
24 | OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 |
26 | Wooden block models, obtained from
27 | https://sketchfab.com/3d-models/efd543fd64e041a8937baa3c8a4c851e
28 | are originally by Sketchfab user Anushka (https://sketchfab.com/nerdysickness)
29 | and subject to the Creative Commons Attribution License 4.0:
30 | https://creativecommons.org/licenses/by/4.0/
31 |
--------------------------------------------------------------------------------
/Resources/Placement_Reticle.usdz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/Resources/Placement_Reticle.usdz
--------------------------------------------------------------------------------
/SpatialRendering.xcodeproj/project.xcworkspace/contents.xcworkspacedata:
--------------------------------------------------------------------------------
1 |
2 |
4 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/SpatialRendering.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | IDEDidComputeMac32BitWarning
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/SpatialRendering.xcodeproj/xcshareddata/xcschemes/SpatialDemo.xcscheme:
--------------------------------------------------------------------------------
1 |
2 |
5 |
9 |
10 |
16 |
22 |
23 |
24 |
25 |
26 |
32 |
33 |
44 |
46 |
52 |
53 |
54 |
55 |
61 |
63 |
69 |
70 |
71 |
72 |
74 |
75 |
78 |
79 |
80 |
--------------------------------------------------------------------------------
/SpatialRendering.xcworkspace/contents.xcworkspacedata:
--------------------------------------------------------------------------------
1 |
2 |
4 |
6 |
7 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/ThirdParty/JoltPhysics.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | BuildSystemType
6 | Latest
7 |
8 |
9 |
--------------------------------------------------------------------------------
/ThirdParty/JoltPhysics.xcodeproj/xcshareddata/xcschemes/Jolt.xcscheme:
--------------------------------------------------------------------------------
1 |
2 |
5 |
9 |
10 |
16 |
22 |
23 |
24 |
25 |
26 |
32 |
33 |
43 |
44 |
50 |
51 |
57 |
58 |
59 |
60 |
62 |
63 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/images/title-slide.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/metal-by-example/spatial-rendering/abc14ae118c2140f44cacbeba0d4e267eb8de0ba/images/title-slide.png
--------------------------------------------------------------------------------