├── .gitignore ├── Sources ├── Util.swift ├── ARSupport.swift ├── GodotToolbox.swift ├── GodotVision+SwiftUI.swift ├── GodotRealityKit+Extensions.swift ├── Multiplayer+GroupActivities.swift ├── GodotResourceConversion.swift └── GodotVisionCoordinator.swift ├── LICENSE ├── Package.swift └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .build/ 2 | .swiftpm/ 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /Sources/Util.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Util.swift 3 | // 4 | // 5 | // Created by Adam Watters on 4/16/24. 6 | // 7 | 8 | import Foundation 9 | 10 | class Debouncer { 11 | private let delay: TimeInterval 12 | private var workItem: DispatchWorkItem? 13 | 14 | init(delay: TimeInterval) { 15 | self.delay = delay 16 | } 17 | 18 | func debounce(action: @escaping (() -> Void)) { 19 | workItem?.cancel() 20 | let workItem = DispatchWorkItem(block: action) 21 | self.workItem = workItem 22 | DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: workItem) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Kevin Watters and Adam Watters 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | 3 | import PackageDescription 4 | 5 | let package = Package( 6 | name: "GodotVision", 7 | platforms: [ 8 | .macOS(.v13), 9 | .visionOS(.v1) 10 | ], 11 | products: [ 12 | .library(name: "GodotVision", targets: ["GodotVision"]) 13 | ], 14 | dependencies: [ 15 | //.package(path: "../SwiftGodotKit") 16 | .package(url: "https://github.com/multijam/SwiftGodotKit", revision: "526ae902f84c1604d30680d8198c09fdfa566848"), 17 | ], 18 | targets: [ 19 | .target( 20 | name: "GodotVision", 21 | /* static build experiment... 22 | dependencies: ["binary_SwiftGodotKit", "binary_libgodot"], 23 | */ 24 | dependencies: ["SwiftGodotKit"], 25 | path: "Sources" 26 | // sources are implicitly in 'Sources' directory. 27 | ), 28 | 29 | /* 30 | static build experiment... 31 | .binaryTarget ( 32 | name: "binary_SwiftGodotKit", 33 | path: "../SwiftGodotKit/SwiftGodotKit.xcframework" 34 | ), 35 | .binaryTarget(name: "binary_libgodot", path: "../SwiftGodot/libgodot.xcframework") 36 | */ 37 | ] 38 | ) 39 | -------------------------------------------------------------------------------- /Sources/ARSupport.swift: -------------------------------------------------------------------------------- 1 | // Created by Kevin Watters on 4/29/24. 2 | 3 | import Foundation 4 | import ARKit 5 | import UIKit 6 | import SwiftGodot 7 | import RealityKit 8 | 9 | struct ARState { 10 | var nodes: Set = .init() 11 | fileprivate var session: ARKitSession = .init() 12 | fileprivate var worldTracking: WorldTrackingProvider = .init() 13 | 14 | fileprivate var inited = false 15 | } 16 | 17 | extension GodotVisionCoordinator { 18 | func ensureARInited() { 19 | if ar.inited { return } 20 | ar.inited = true 21 | Task { await initAR() } 22 | } 23 | 24 | func initAR() async { 25 | do { 26 | try await ar.session.run([ar.worldTracking]) 27 | } catch { 28 | logError("could not run ARSession wtp: \(error)") 29 | } 30 | } 31 | 32 | func arUpdate() { 33 | guard ar.nodes.count > 0, ar.worldTracking.state == .running, let deviceTransform = getDeviceTransform() else { 34 | return 35 | } 36 | 37 | let rkGlobalTransform = RealityKit.Transform(matrix: deviceTransform) 38 | let godotSpaceGlobalTransform = godotEntitiesParent.convert(transform: rkGlobalTransform, from: nil) 39 | let xform = Transform3D(godotSpaceGlobalTransform) 40 | 41 | Array(ar.nodes).forEach { node in 42 | if node.isInsideTree() { 43 | node.globalTransform = xform 44 | } else { 45 | ar.nodes.remove(node) 46 | } 47 | } 48 | } 49 | 50 | func getDeviceTransform() -> simd_float4x4? { 51 | guard let deviceAnchor = ar.worldTracking.queryDeviceAnchor(atTimestamp: CACurrentMediaTime()) else { return nil } 52 | return deviceAnchor.originFromAnchorTransform 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Sources/GodotToolbox.swift: -------------------------------------------------------------------------------- 1 | // Created by Kevin Watters on 1/10/24. 2 | 3 | import Foundation 4 | import libgodot 5 | import SwiftGodot 6 | import SwiftGodotKit 7 | 8 | extension Object { 9 | func getMetaBool(_ name: StringName, defaultValue: Bool) -> Bool { 10 | Bool(getMeta(name: name, default: Variant(defaultValue))) ?? defaultValue 11 | } 12 | func getMetaBool(_ name: String, defaultValue: Bool) -> Bool { 13 | getMetaBool(StringName(name), defaultValue: defaultValue) 14 | } 15 | } 16 | 17 | extension PackedByteArray { 18 | /// Returns a new Data object with a copy of the data contained by this PackedByteArray 19 | public func asDataNoCopy() -> Data? { 20 | withUnsafeConstAccessToData { ptr, count in 21 | Data(bytesNoCopy: .init(mutating: ptr), count: count, deallocator: .none) 22 | } 23 | } 24 | } 25 | 26 | extension String { 27 | func removingStringPrefix(_ prefix: String) -> String { 28 | if starts(with: prefix) { 29 | let r = index(startIndex, offsetBy: prefix.count).. String { 52 | if let idx = functionName.firstIndex(of: Character("(")) { 53 | return String(functionName.prefix(upTo: idx)) 54 | } else { 55 | return functionName 56 | } 57 | } 58 | 59 | func logError(_ message: String, functionName: String = #function) { 60 | print("⚠️ \(stripFunctionName(functionName)) ERROR: \(message)") 61 | } 62 | 63 | func logError(_ error: any Error, functionName: String = #function) { 64 | print("⚠️ \(stripFunctionName(functionName)) ERROR: \(error)") 65 | } 66 | 67 | func logError(_ message: String, _ error: any Error, functionName: String = #function) { 68 | print("⚠️ \(stripFunctionName(functionName)) ERROR: \(message) - \(error)") 69 | } 70 | 71 | func doLoggingErrors(_ block: () throws -> R, functionName: String = #function) -> R? { 72 | do { 73 | return try block() 74 | } catch { 75 | logError(error, functionName: functionName) 76 | } 77 | 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This Project is Archived 2 | 3 | > [!IMPORTANT] 4 | > Native visionOS support is landing in Godot Engine. Check out the [latest Godot 4.5 dev 5 snapshot](https://godotengine.org/article/dev-snapshot-godot-4-5-dev-5/#native-visionos-support) for details on Apple's contributions back to Godot to bring native XR support for visionOS. This project is going into "archive" mode. 5 | 6 | # GodotVision 7 | 8 | Godot headless on visionOS, rendered with RealityKit, so you can create shared-space visionOS experiences from Godot. 9 | 10 | ### Questions? Join the [GodotVision Discord](https://discord.gg/XvB4dwGUtF) 11 | 12 | ### Documentation 13 | 14 | Documentation lives at the website — [https://godot.vision](https://godot.vision) 15 | 16 | ### Example Repo 17 | 18 | See [GodotVisionExample](https://github.com/kevinw/GodotVisionExample) for a simple repository you can clone and run in Xcode. 19 | 20 | ![example image](https://raw.githubusercontent.com/kevinw/GodotVisionExample/main/docs/screenshot1.jpg) 21 | 22 | ## What is this? 23 | 24 | [SwiftGodotKit](https://github.com/migueldeicaza/SwiftGodotKit)'s Godot-as-a-library, compiled for visionOS. Godot ([slightly modified](https://github.com/multijam/godot/commits/visionos/?author=kevinw)) thinks it is a headless iOS template release build, and we add an extra ability--to tick the Godot loop from the "outside" host process. Then we intersperse the RealityKit main loop and the Godot main loop, and watch the Godot SceneTree for Node3Ds, and mirror meshes, textures, and sounds to the RealityKit world. 25 | 26 | Check out the amazing [SwiftGodot](https://migueldeicaza.github.io/SwiftGodotDocs/documentation/swiftgodot/) documentation for how to hack on SwiftGodot/this project. 27 | 28 | # Setup 29 | Steps to add GodotVision to an existing VisionOS XCode project: 30 | 31 | ## Import Godot_Project 32 | 1. Copy Godot_Project folder from this repo to your target repo 33 | 34 | ## Add GodotVision Package dependency 35 | 1. Open App Settings by clicking on the App in the Navigator 36 | 1. Choose your Target from the target list 37 | 1. General -> Frameworks -> `+` -> "Add Other..." -> "Add Package Dependency..." 38 | 1. In "Search or Enter Package URL": Enter `https://github.com/kevinw/GodotVision.git` (Make sure to add `.git`) 39 | 1. "Add Package" 40 | 41 | ## Change Swift Compiler Language 42 | 1. Stay on the Target subpanel 43 | 1. Build Settings 44 | 1. "All" to open edit all options 45 | 1. Under "Swift Compiler - Language", change "Swift Compiler - Language" to "C++ / Objective-C++" 46 | 47 | ## Build 48 | 1. Product -> Build 49 | 50 | ## Limitations 51 | 52 | * Missing: 53 | * Shaders 54 | * Particle systems 55 | * Many PBR material options 56 | * Any native GDExtensions you want to use need to be compiled for visionOS. 57 | -------------------------------------------------------------------------------- /Sources/GodotVision+SwiftUI.swift: -------------------------------------------------------------------------------- 1 | // Created by Kevin Watters on 2/5/24. 2 | 3 | import Foundation 4 | import SwiftUI 5 | import RealityKit 6 | 7 | public struct GodotVisionRealityViewModifier: ViewModifier { 8 | @Environment(\.scenePhase) private var scenePhase 9 | var coordinator: GodotVisionCoordinator 10 | 11 | public init(coordinator: GodotVisionCoordinator) { 12 | self.coordinator = coordinator 13 | } 14 | 15 | public func body(content: Content) -> some View { 16 | 17 | // MARK: - Tap Gesture Definition 18 | 19 | let tapGesture = SpatialTapGesture().targetedToAnyEntity().onEnded { event in 20 | if event.entity.components.has(InputTargetComponent.self) { 21 | coordinator.receivedTap(event: event) 22 | } 23 | } 24 | 25 | // MARK: - Drag Gesture Definition 26 | 27 | let dragGesture = DragGesture(minimumDistance: 0.1, coordinateSpace: .local).targetedToAnyEntity().onChanged({ value in 28 | if value.entity.components.has(InputTargetComponent.self) { 29 | coordinator.receivedDrag(value) 30 | } 31 | }).onEnded({value in 32 | if value.entity.components.has(InputTargetComponent.self) { 33 | coordinator.receivedDragEnded(value) 34 | } 35 | }) 36 | 37 | // MARK: - Magnify Gesture Definition 38 | 39 | let magnifyGesture = MagnifyGesture(minimumScaleDelta: 0.01).targetedToAnyEntity() 40 | .onChanged({ value in 41 | if value.entity.components.has(InputTargetComponent.self) { 42 | coordinator.receivedMagnify(value) 43 | } 44 | }) 45 | .onEnded({value in 46 | if value.entity.components.has(InputTargetComponent.self) { 47 | coordinator.receivedMagnifyEnded(value) 48 | } 49 | }) 50 | 51 | // MARK: - Rotate3D Gesture Definition 52 | 53 | // constrainedToAxis can be set to nil, or an axis (like RotationAxis3D.y) 54 | // a combination axis (like RotationAxis3D.yz which is a single axis defined by the combo of the two), 55 | // nil allows for full rotation by default. 56 | let rotateGesture3D = RotateGesture3D(constrainedToAxis: nil, minimumAngleDelta: .degrees(1)).targetedToAnyEntity() 57 | .onChanged({ value in 58 | if value.entity.components.has(InputTargetComponent.self) { 59 | coordinator.receivedRotate3D(value) 60 | } 61 | }) 62 | .onEnded({ value in 63 | if value.entity.components.has(InputTargetComponent.self) { 64 | coordinator.receivedRotate3DEnded(value) 65 | } 66 | }) 67 | 68 | // MARK: - Content View Instantiation 69 | 70 | content 71 | //.gesture(tapGesture) 72 | //.gesture(dragGesture) 73 | //.gesture(rotateGesture3D) 74 | //.gesture(magnifyGesture) 75 | //.gesture(magnifyGesture.exclusively(before: tapGesture)) 76 | .gesture(rotateGesture3D.simultaneously(with: dragGesture).simultaneously(with: magnifyGesture).exclusively(before: tapGesture)) 77 | 78 | 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /Sources/GodotRealityKit+Extensions.swift: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | Extensions for interop between RealityKit and SwiftGodot types. 4 | 5 | */ 6 | 7 | // Created by Kevin Watters on 1/3/24. 8 | 9 | import Foundation 10 | import SwiftGodot 11 | import RealityKit 12 | import Spatial 13 | import SwiftUI 14 | 15 | // 16 | // RealityKit 17 | // 18 | 19 | func uiColor(forGodotColor c: SwiftGodot.Color) -> UIColor { 20 | .init(red: .init(c.red), green: .init(c.green), blue: .init(c.blue), alpha: .init(c.alpha)) 21 | } 22 | 23 | extension SwiftGodot.Transform3D { 24 | init(_ rkTransform: RealityKit.Transform) { 25 | self.init(basis: .init(from: Quaternion(rkTransform.rotation)).scaled(scale: .init(rkTransform.scale)), 26 | origin: .init(rkTransform.translation)) 27 | // TODO great place for some unit tests, trying converting matrices back and forth 28 | } 29 | } 30 | 31 | extension RealityKit.Transform { 32 | init(_ godotTransform: SwiftGodot.Transform3D) { 33 | self.init(scale: .init(godotTransform.basis.getScale()), 34 | rotation: .init(godotTransform.basis.getRotationQuaternion()), 35 | translation: .init(godotTransform.origin)) 36 | } 37 | } 38 | 39 | extension simd_float4x4 { 40 | init(_ godotTransform: SwiftGodot.Transform3D) { 41 | self = RealityKit.Transform(godotTransform).matrix 42 | } 43 | } 44 | 45 | extension simd_float3 { 46 | init(_ godotVector: SwiftGodot.Vector3) { 47 | self.init(godotVector.x, godotVector.y, godotVector.z) 48 | } 49 | 50 | func isApproximatelyEqualTo(_ v: Self) -> Bool { 51 | x.isApproximatelyEqualTo(v.x) && 52 | y.isApproximatelyEqualTo(v.y) && 53 | z.isApproximatelyEqualTo(v.z) 54 | } 55 | } 56 | 57 | extension simd_float4 { 58 | init(_ godotVector: SwiftGodot.Vector4) { 59 | self.init(godotVector.x, godotVector.y, godotVector.z, godotVector.w) 60 | } 61 | } 62 | 63 | extension simd_float2 { 64 | init(_ godotVector: SwiftGodot.Vector2) { 65 | self.init(godotVector.x, godotVector.y) 66 | } 67 | } 68 | 69 | extension SwiftGodot.Quaternion { 70 | init(_ quat: simd_quatf) { 71 | self.init() 72 | let imaginary = quat.imag 73 | self.w = quat.real 74 | self.z = imaginary.z 75 | self.y = imaginary.y 76 | self.x = imaginary.x 77 | } 78 | 79 | init(_ quat: simd_quatd) { 80 | self.init() 81 | let imaginary = quat.imag 82 | self.w = Float(quat.real) 83 | self.z = Float(imaginary.z) 84 | self.y = Float(imaginary.y) 85 | self.x = Float(imaginary.x) 86 | } 87 | } 88 | 89 | extension simd_quatf { 90 | init(_ godotQuat: SwiftGodot.Quaternion) { 91 | self.init(ix: godotQuat.x, iy: godotQuat.y, iz: godotQuat.z, r: godotQuat.w) 92 | } 93 | 94 | init(_ q: simd_quatd) { 95 | self.init(ix: Float(q.imag.x), iy: Float(q.imag.y), iz: Float(q.imag.z), r: Float(q.real)) 96 | } 97 | } 98 | 99 | // 100 | // Godot 101 | // 102 | 103 | extension SwiftGodot.Variant { 104 | /// A utility method for initializing a specific type from a Variant. Prints an error message if the cast fails. 105 | func cast(as t: T.Type, debugName: String = "value", printError: Bool = true, functionName: String = #function) -> T? where T: InitsFromVariant { 106 | guard let result = T(self) else { 107 | if printError { 108 | logError("expected \(debugName) to be castable to a \(T.self)", functionName: functionName) 109 | } 110 | return nil 111 | } 112 | 113 | return result 114 | } 115 | } 116 | 117 | extension SwiftGodot.Vector3 { 118 | init(_ point3D: Spatial.Point3D) { 119 | self.init(x: Float(point3D.x), y: Float(point3D.y), z: Float(point3D.z)) 120 | } 121 | 122 | init(_ v: simd_float3) { 123 | self.init(x: v.x, y: v.y, z: v.z) 124 | } 125 | } 126 | 127 | 128 | extension Float { 129 | func isApproximatelyEqualTo(_ f: Self, epsilon: Float = 0.000001) -> Bool { 130 | abs(self - f) < epsilon 131 | } 132 | } 133 | 134 | protocol InitsFromVariant { init?(_ variant: Variant) } 135 | 136 | extension PackedInt32Array: InitsFromVariant {} 137 | extension PackedFloat32Array: InitsFromVariant {} 138 | extension PackedFloat64Array: InitsFromVariant {} 139 | extension PackedVector3Array: InitsFromVariant {} 140 | extension PackedVector2Array: InitsFromVariant {} 141 | 142 | func printGodotTree(_ node: Node, indent: String = "") { 143 | print(indent, node, node.getInstanceId()) 144 | let newIndent = " \(indent)" 145 | for child in node.getChildren() { 146 | printGodotTree(child, indent: newIndent) 147 | } 148 | 149 | } 150 | func printEntityTree(_ rkEntity: RealityKit.Entity, indent: String = "") { 151 | func ori(_ o: simd_quatf) -> String { 152 | "(r=\(o.real), image: \(o.imag.x), \(o.imag.y), \(o.imag.z))" 153 | } 154 | 155 | func v3(_ p: simd_float3) -> String { "(\(p.x), \(p.y), \(p.z))" } 156 | 157 | print(indent, rkEntity.name, "pos=\(v3(rkEntity.position)), ori=\(ori(rkEntity.orientation))") 158 | 159 | let newIndent = " \(indent)" 160 | for child in rkEntity.children { 161 | printEntityTree(child, indent: newIndent) 162 | } 163 | } 164 | 165 | -------------------------------------------------------------------------------- /Sources/Multiplayer+GroupActivities.swift: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | Multiplayer functionality backed by Apple's GroupActivities Framework. 4 | 5 | */ 6 | // Created by Kevin Watters on 4/23/24. 7 | 8 | import Foundation 9 | import GroupActivities 10 | import CoreTransferable 11 | import Combine 12 | import SwiftUI 13 | 14 | public class LogCoordinator: NSObject, ObservableObject { 15 | static var _instance: LogCoordinator = LogCoordinator() 16 | public static var instance: LogCoordinator { _instance } 17 | 18 | @Published public var lines: [(Int, String)] = [] 19 | } 20 | 21 | public struct LogView: View { 22 | @EnvironmentObject private var logCoordinator: LogCoordinator 23 | public init() { 24 | 25 | } 26 | public var body: some View { 27 | ForEach(logCoordinator.lines, id: \.0) { line in 28 | Text(line.1).monospaced() 29 | } 30 | } 31 | } 32 | 33 | fileprivate var messageCount = 0 34 | 35 | public func log(_ message: String) { 36 | print(message) 37 | 38 | DispatchQueue.main.async { 39 | messageCount += 1 40 | let id = messageCount 41 | var oldLines = LogCoordinator.instance.lines 42 | oldLines.insert((id, message), at: 0) 43 | if oldLines.count > 50 { 44 | oldLines = Array(oldLines.prefix(50)) 45 | } 46 | LogCoordinator.instance.lines = oldLines 47 | } 48 | } 49 | 50 | public class ShareModel: ObservableObject { 51 | public typealias ActivityType = GodotVisionGroupActivity 52 | 53 | public var groupSession: GroupSession? = nil 54 | public var activityIdentifier: String? = nil // set from outside 55 | public var automaticallyShareInput = false 56 | 57 | static var instance: ShareModel? = nil 58 | 59 | @Published public var isEligibleForGroupSession = false 60 | var session: GroupSession? = nil 61 | 62 | let activity = ActivityType() 63 | private let groupStateObserver = GroupStateObserver() 64 | var unreliableMessenger: GroupSessionMessenger? 65 | var reliableMessenger: GroupSessionMessenger? 66 | private var subs: Set = [] 67 | private var sessionSubs: Set = [] 68 | private var tasks = Set>() 69 | private var awaitSessionsTask: Task? = nil 70 | private var preparingSession = false 71 | public var onMessageCallback: ((Data) -> Void)? = nil 72 | 73 | #if os(visionOS) 74 | var systemCoordinatorConfig: SystemCoordinator.Configuration? 75 | var spatialTemplatePreference: SpatialTemplatePreference = .none 76 | #endif 77 | 78 | var didSetup = false 79 | 80 | var godotData: GodotData = .init() 81 | 82 | public init() { 83 | if Self.instance == nil { 84 | Self.instance = self 85 | } else { 86 | logError("more than one ShareModel") 87 | log("more than one ShareModel") 88 | } 89 | 90 | groupStateObserver.$isEligibleForGroupSession.sink { [weak self] value in 91 | self?.isEligibleForGroupSession = value 92 | }.store(in: &subs) 93 | } 94 | 95 | public func prepareSession() async { 96 | log("prepareSession") 97 | // Await the result of the preparation call. 98 | let result = await activity.prepareForActivation() 99 | switch result { 100 | case .activationDisabled: 101 | log("Activation is disabled") 102 | case .activationPreferred: 103 | do { 104 | log("Activation is preferred") 105 | _ = try await activity.activate() 106 | } catch { 107 | log("Unable to activate the activity: \(error)") 108 | } 109 | case .cancelled: 110 | log("Activation Cancelled") 111 | default: 112 | logError("unknown activation \(result)") 113 | log("unknown activation \(result)") 114 | } 115 | } 116 | 117 | public func startSessionHandlerTask() { 118 | awaitSessionsTask?.cancel() 119 | 120 | log("starting session handler task") 121 | awaitSessionsTask = Task { 122 | for await session in ActivityType.sessions() { 123 | log("SESSION \(session)") 124 | log(" FOR ACTIVITY \(session.activity)") 125 | #if os(visionOS) 126 | guard let systemCoordinator = await session.systemCoordinator else { 127 | logError("no system coordinator") 128 | continue 129 | } 130 | 131 | let isSpatial = systemCoordinator.localParticipantState.isSpatial 132 | print(" isSpatial", isSpatial) 133 | 134 | if isSpatial { 135 | var configuration = SystemCoordinator.Configuration() 136 | configuration.spatialTemplatePreference = spatialTemplatePreference 137 | configuration.supportsGroupImmersiveSpace = true 138 | systemCoordinator.configuration = configuration 139 | systemCoordinatorConfig = configuration 140 | } 141 | #endif 142 | let reliableMessenger = GroupSessionMessenger(session: session, deliveryMode: .reliable) 143 | self.reliableMessenger = reliableMessenger 144 | 145 | let unreliableMessenger = GroupSessionMessenger(session: session, deliveryMode: .unreliable) 146 | self.unreliableMessenger = unreliableMessenger 147 | 148 | session.$state.sink { [weak self] sessionState in 149 | guard let self else { return } 150 | log("sesssion state \(sessionState)") 151 | switch sessionState { 152 | case .invalidated(let reason): 153 | if self.session === session { 154 | log("calling endSession because \(reason)") 155 | self.endSession() 156 | } 157 | case .joined: 158 | () 159 | case .waiting: 160 | () 161 | @unknown default: 162 | fatalError("unhandled sessionState \(sessionState)") 163 | } 164 | }.store(in: &sessionSubs) 165 | 166 | for messenger in [reliableMessenger, unreliableMessenger] { 167 | tasks.insert(Task.detached { 168 | for await (data, messageContext) in messenger.messages(of: Data.self) { 169 | log("msg data from \(messageContext.source.id) -> \(data)") 170 | self.onMessageCallback?(data) 171 | } 172 | }) 173 | } 174 | 175 | self.session = session 176 | 177 | session.join() 178 | log("calling join") 179 | } 180 | } 181 | } 182 | 183 | func endSession() { 184 | sessionSubs.forEach { $0.cancel() } 185 | sessionSubs.removeAll() 186 | 187 | unreliableMessenger = nil 188 | reliableMessenger = nil 189 | 190 | tasks.forEach { $0.cancel() } 191 | tasks.removeAll() 192 | 193 | session = nil 194 | } 195 | 196 | deinit { 197 | awaitSessionsTask?.cancel() 198 | awaitSessionsTask = nil 199 | 200 | 201 | subs.forEach { $0.cancel() } 202 | subs.removeAll() 203 | godotData.cleanup() 204 | } 205 | 206 | public func maybeJoin() { 207 | if preparingSession { 208 | log("already preparing session...") 209 | return 210 | } 211 | 212 | if !isReadyToJoinGroupActivity { 213 | log("maybeJoin failed, not ready to join group activity") 214 | return 215 | } 216 | 217 | if session != nil { 218 | log("maybeJoin failed already have a session") 219 | return 220 | } 221 | 222 | print("maybeJoin, current session is", String(describing: session)) 223 | print("activity", activityIdentifier ?? "") 224 | if !preparingSession { 225 | preparingSession = true 226 | Task { 227 | await prepareSession() 228 | preparingSession = false 229 | } 230 | } 231 | } 232 | 233 | // nocommit: change to closure? 234 | public func sendInput(_ inputMessage: T, reliable: Bool) where T: Codable { 235 | let messengerOptional = reliable ? reliableMessenger : unreliableMessenger 236 | guard let messenger = messengerOptional else { 237 | return 238 | } 239 | 240 | tasks.insert(Task { 241 | do { 242 | let data = try JSONEncoder().encode(inputMessage) 243 | try await messenger.send(data) 244 | } catch { 245 | logError(error) 246 | } 247 | }) 248 | } 249 | 250 | private var isReadyToJoinGroupActivity: Bool { 251 | if activityIdentifier == nil { 252 | return false 253 | } 254 | if activityIdentifier!.isEmpty { 255 | return false 256 | } 257 | if !automaticallyShareInput { 258 | return false 259 | } 260 | if !groupStateObserver.isEligibleForGroupSession { 261 | return false 262 | } 263 | return true 264 | } 265 | } 266 | 267 | 268 | public struct GodotVisionGroupActivity: GroupActivity { 269 | public static var activityIdentifier: String { 270 | ShareModel.instance?.activityIdentifier ?? Bundle.main.bundleIdentifier!.appending(".GodotVistionActivity") 271 | } 272 | 273 | public var metadata: GroupActivityMetadata { 274 | var metaData = GroupActivityMetadata() 275 | metaData.type = .generic 276 | metaData.title = "Play Together" 277 | //metaData.sceneAssociationBehavior = .content(Self.activityIdentifier) 278 | metaData.sceneAssociationBehavior = .default 279 | return metaData 280 | } 281 | } 282 | 283 | struct GodotVisionTransferable: Transferable { 284 | static var transferRepresentation: some TransferRepresentation { 285 | GroupActivityTransferRepresentation { _ in 286 | GodotVisionGroupActivity() 287 | } 288 | } 289 | } 290 | 291 | // 292 | // GODOT specific stuff below 293 | // 294 | 295 | import SwiftGodot 296 | 297 | struct GodotData { 298 | var node: Node? = nil 299 | var signal_cb: Callable? = nil 300 | 301 | mutating func cleanup() { 302 | if let node { 303 | if let signal_cb { 304 | node.disconnect(signal: join_activity, callable: signal_cb) 305 | self.signal_cb = nil 306 | } 307 | self.node = nil 308 | } 309 | } 310 | } 311 | 312 | fileprivate let config_changed = StringName("config_changed") /// Signal name for SharePlay config change 313 | fileprivate let peer_connected = StringName("peer_connected") 314 | fileprivate let join_activity = StringName("join_activity") 315 | 316 | extension ShareModel { 317 | 318 | func setup(sceneTree: SceneTree, onMessageData: @escaping (Data) -> Void) { 319 | self.onMessageCallback = onMessageData 320 | guard let _sharePlayNode = findSharePlayNode(sceneTree: sceneTree) else { 321 | logError("no share play node") 322 | return 323 | } 324 | 325 | let id = Int64(_sharePlayNode.getInstanceId()) 326 | 327 | guard let node = GD.instanceFromId(instanceId: id) as? Node else { 328 | logError("could not get Node instance from ID \(id)") 329 | return 330 | } 331 | 332 | godotData.node = node 333 | 334 | let callable = Callable(onGodotRequestJoin) 335 | let godotError = node.connect(signal: join_activity, callable: callable) 336 | if godotError != .ok { 337 | logError("Could not connect to \(join_activity) signal on SharePlay node: \(godotError)") 338 | } else { 339 | godotData.signal_cb = callable 340 | } 341 | } 342 | 343 | private func onGodotRequestJoin(arguments: borrowing Arguments) -> Variant { 344 | guard let node = godotData.node else { return .init() } 345 | automaticallyShareInput = Bool(node.get(property: "automatically_share_input")) ?? false 346 | maybeJoin() 347 | return .init() 348 | } 349 | 350 | } 351 | 352 | fileprivate func findSharePlayNode(sceneTree: SceneTree) -> Node? { 353 | guard let root = sceneTree.root else { 354 | logError("sceneTree had no root") 355 | return nil 356 | } 357 | 358 | // TODO: find a better way to find the GodotVision plugin singleton? 359 | for GV in root.getChildren() where GV.name == "GodotVision" { 360 | for sharePlay in GV.getChildren() where sharePlay.name == "SharePlay" && sharePlay.hasSignal(peer_connected) { 361 | return sharePlay 362 | } 363 | } 364 | 365 | return nil 366 | } 367 | 368 | 369 | -------------------------------------------------------------------------------- /Sources/GodotResourceConversion.swift: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | Convert Godot Resources (meshes, textures, materials, etc.) to their RealityKit counterparts. 4 | 5 | */ 6 | 7 | // Created by Kevin Watters on 1/19/24. 8 | 9 | import SwiftGodot 10 | import RealityKit 11 | import Foundation 12 | 13 | class ResourceCache { 14 | var projectContext: GodotProjectContext? = nil 15 | var materials: [SwiftGodot.Material: MaterialEntry] = .init() 16 | var textures: [SwiftGodot.Texture: TextureEntry] = .init() 17 | 18 | func reset() { 19 | materials.removeAll() 20 | textures.removeAll() 21 | } 22 | 23 | func materialEntry(forGodotMaterial godotMaterial: SwiftGodot.Material) -> MaterialEntry { 24 | if materials[godotMaterial] == nil { 25 | materials[godotMaterial] = MaterialEntry(godotResource: godotMaterial) 26 | } 27 | return materials[godotMaterial]! 28 | } 29 | 30 | func textureEntry(forGodotTexture godotTexture: SwiftGodot.Texture) -> TextureEntry { 31 | if textures[godotTexture] == nil { 32 | textures[godotTexture] = TextureEntry(godotResource: godotTexture) 33 | } 34 | return textures[godotTexture]! 35 | } 36 | 37 | func rkTexture(forGodotTexture godotTexture: SwiftGodot.Texture) -> RealityKit.TextureResource? { 38 | textureEntry(forGodotTexture: godotTexture).getTexture(resourceCache: self) 39 | } 40 | 41 | func rkMaterial(forGodotMaterial godotMaterial: SwiftGodot.Material) -> RealityKit.Material { 42 | materialEntry(forGodotMaterial: godotMaterial).getMaterial(resourceCache: self) 43 | } 44 | } 45 | 46 | class ResourceEntry where G: SwiftGodot.Resource { 47 | fileprivate var godotResource: G? = nil 48 | fileprivate var rkResource: R? = nil 49 | fileprivate var changedToken: SwiftGodot.Object? = nil 50 | 51 | private func onChanged() { 52 | // TODO: haven't actually verified this works... 53 | 54 | print("RESOURCE CHANGED \(Self.self)", String(describing: godotResource)) 55 | DispatchQueue.main.async { 56 | self.rkResource = nil 57 | } 58 | } 59 | 60 | init(godotResource: G) { 61 | self.godotResource = godotResource 62 | changedToken = godotResource.changed.connect(onChanged) 63 | } 64 | 65 | deinit { 66 | if let changedToken, let godotResource { 67 | godotResource.changed.disconnect(changedToken) 68 | } 69 | 70 | changedToken = nil 71 | godotResource = nil 72 | } 73 | } 74 | 75 | class TextureEntry: ResourceEntry { 76 | func getTexture(resourceCache: ResourceCache) -> RealityKit.TextureResource? { 77 | if let godotViewportTex = godotResource as? SwiftGodot.ViewportTexture { 78 | logError("ViewportTextures are not currently supported in headless GodotVision builds: \(godotViewportTex)") 79 | /* 80 | if let image = godotViewportTex.getImage() { 81 | let imageData = image.getData() 82 | if let data = imageData.asDataNoCopy() { 83 | do { 84 | let format = image.getFormat() 85 | let mtlFormat: MTLPixelFormat 86 | switch format { 87 | case .rgba8: 88 | mtlFormat = .rgba8Uint 89 | default: 90 | fatalError("unknown image format \(format)") 91 | } 92 | return try .init(dimensions: .dimensions(width: Int(image.getWidth()), height: Int(image.getHeight())), 93 | format: .raw(pixelFormat: mtlFormat), 94 | contents: .init(mipmapLevels: [.mip(data: data, bytesPerRow: data.count / Int(image.getWidth()))])) 95 | } catch { 96 | logError("creating viewport texture \(error)") 97 | } 98 | } 99 | } 100 | */ 101 | } else if let godotTex = godotResource as? SwiftGodot.Texture2D { 102 | guard let projectContext = resourceCache.projectContext else { 103 | logError("projectContext not set in ResourceCache") 104 | return nil 105 | } 106 | let resourcePath = godotTex.resourcePath 107 | if resourcePath.isEmpty { 108 | logError("Texture2D had empty resourcePath") 109 | return nil 110 | } 111 | 112 | #if false 113 | if #available(visionOS 2.0, *) { 114 | if let resource = ResourceLoader.load(path: resourcePath) { 115 | if let cTex = resource as? CompressedTexture2D { 116 | if let image = cTex.getImage() { 117 | let packedData = image.getData() 118 | if let data = packedData.asDataNoCopy() { 119 | let s = String(decoding: data.prefix(100), as: Unicode.ASCII.self) 120 | print(s) 121 | } 122 | } 123 | } 124 | } 125 | } 126 | #endif 127 | 128 | do { 129 | return try .load(contentsOf: projectContext.fileUrl(forGodotResourcePath: resourcePath)) 130 | } catch { 131 | logError("loading texture with resourcePath: \(resourcePath)", error) 132 | } 133 | 134 | } 135 | 136 | return nil 137 | } 138 | } 139 | 140 | class MaterialEntry: ResourceEntry { 141 | func getMaterial(resourceCache: ResourceCache) -> RealityKit.Material { 142 | if rkResource == nil, let godotResource { 143 | if let stdMat = godotResource as? StandardMaterial3D { 144 | var rkMat = PhysicallyBasedMaterial() 145 | 146 | // TODO: flesh this out so that we respect as many Godot PBR fields as possible. 147 | // also should work for godot's ORMMaterial as well 148 | 149 | // 150 | // ALBEDO (base color) 151 | // 152 | if let albedoTexture = stdMat.albedoTexture { 153 | let tex = resourceCache.rkTexture(forGodotTexture: albedoTexture) 154 | if let tex { 155 | rkMat.baseColor = .init(tint: uiColor(forGodotColor: stdMat.albedoColor), texture: .init(tex)) 156 | } else { 157 | rkMat.baseColor = .init(tint: uiColor(forGodotColor: stdMat.albedoColor)) 158 | } 159 | } else { 160 | if stdMat.transparency == .alpha { 161 | rkMat.baseColor = .init(tint: uiColor(forGodotColor: stdMat.albedoColor).withAlphaComponent(1.0)) 162 | rkMat.blending = .transparent(opacity: .init(floatLiteral: stdMat.albedoColor.alpha)) 163 | } else { 164 | rkMat.baseColor = .init(tint: uiColor(forGodotColor: stdMat.albedoColor)) 165 | } 166 | } 167 | 168 | rkMat.metallic = PhysicallyBasedMaterial.Metallic(floatLiteral: Float(stdMat.metallic)) 169 | rkMat.roughness = PhysicallyBasedMaterial.Roughness(floatLiteral: Float(stdMat.roughness)) 170 | rkMat.textureCoordinateTransform = .init(offset: SIMD2(x: stdMat.uv1Offset.x, y: stdMat.uv1Offset.y), scale: SIMD2(x: stdMat.uv1Scale.x, y: stdMat.uv1Scale.y)) 171 | 172 | // 173 | // EMISSION 174 | // 175 | if stdMat.emissionEnabled { 176 | let emissiveColor: PhysicallyBasedMaterial.EmissiveColor 177 | if let emissionTexture = stdMat.emissionTexture, let tex = resourceCache.rkTexture(forGodotTexture: emissionTexture) { 178 | emissiveColor = .init(color: uiColor(forGodotColor: stdMat.emission), texture: .init(tex)) 179 | } else { 180 | emissiveColor = .init(color: uiColor(forGodotColor: stdMat.emission)) 181 | } 182 | 183 | rkMat.emissiveColor = emissiveColor 184 | rkMat.emissiveIntensity = Float(stdMat.emissionIntensity / 1000.0) // TODO: Godot docs say Material.emission_intensity is specified in nits and defaults to 1000. not sure how to convert this, since the RealityKit docs don't specify a unit. 185 | } 186 | 187 | rkResource = rkMat 188 | } 189 | } 190 | 191 | if rkResource == nil { 192 | logError("generating material from \(String(describing: godotResource))") 193 | rkResource = SimpleMaterial(color: .systemPink, isMetallic: false) 194 | } 195 | 196 | return rkResource! 197 | } 198 | } 199 | 200 | /// Describes a Godot mesh and the information needed to convert it to a RealityKit mesh. 201 | struct MeshCreationInfo: Hashable { 202 | var godotMesh: SwiftGodot.Mesh 203 | var godotSkeleton: SwiftGodot.Skeleton3D? = nil 204 | var flipFacesIfNoIndexBuffer: Bool = false 205 | var instanceTransforms: [simd_float4x4]? = nil /// If not nil, the created mesh will be instanced, one instance for each transform. 206 | 207 | func hash(into hasher: inout Hasher) { 208 | hasher.combine(godotMesh) 209 | hasher.combine(godotSkeleton) 210 | hasher.combine(flipFacesIfNoIndexBuffer) 211 | hasher.combine(instanceTransforms?.count ?? -1) // TODO: the notion of memoizing these MeshCreationInfos when there are instance arrays might not really make sense?? 212 | } 213 | } 214 | 215 | class MeshEntry { 216 | init(meshResource: MeshResource) { 217 | self.meshResource = meshResource 218 | } 219 | 220 | let meshResource: MeshResource 221 | var entities: Set = .init() 222 | 223 | var changedListeners: [() -> Void] = [] 224 | } 225 | 226 | private var meshCache: [MeshCreationInfo: MeshEntry] = [:] // TODO @Leak 227 | 228 | func createRealityKitMesh(entity: Entity, 229 | debugName: String, 230 | meshCreationInfo: MeshCreationInfo, 231 | onResourceChange: @escaping (MeshEntry) -> Void, 232 | onMeshEntry: @escaping (MeshEntry?) -> Void, 233 | verbose: Bool = false 234 | ) { 235 | if let meshEntry = meshCache[meshCreationInfo] { 236 | return cb(meshEntry) 237 | } 238 | 239 | doLoggingErrors { 240 | let meshContents = try meshContents(debugName: debugName, meshCreationInfo: meshCreationInfo, verbose: verbose) 241 | #if true // Call MeshResource async version so GPU buffers get filled on bg thread. 242 | Task(priority: .high) { 243 | let meshResource = try await MeshResource(from: meshContents) 244 | await MainActor.run { 245 | onMeshResource(meshResource: meshResource) 246 | } 247 | } 248 | #else 249 | let meshResource = try MeshResource.generate(from: meshContents) 250 | onMeshResource(meshResource: meshResource) 251 | #endif 252 | } 253 | 254 | @Sendable func cb(_ meshEntry: MeshEntry?) { 255 | if let meshEntry, let modelEntity = entity as? ModelEntity { 256 | meshEntry.entities.insert(modelEntity) 257 | } 258 | if verbose { 259 | print("calling onMeshEntry \(debugName)") 260 | } 261 | onMeshEntry(meshEntry) 262 | } 263 | 264 | @Sendable func onMeshResource(meshResource: MeshResource) { 265 | let meshEntry = MeshEntry(meshResource: meshResource) 266 | if !meshCreationInfo.godotMesh.getMetaBool("_didRegisterChanged", defaultValue: false) { 267 | meshCreationInfo.godotMesh.setMeta(name: "_didRegisterChanged", value: Variant(true)) 268 | meshCreationInfo.godotMesh.changed.connect { 269 | meshCache.removeValue(forKey: meshCreationInfo) 270 | onResourceChange(meshEntry) 271 | } 272 | } 273 | 274 | meshCache[meshCreationInfo] = meshEntry 275 | cb(meshEntry) 276 | } 277 | 278 | } 279 | 280 | private func getInverseBindPoseMatrix(skeleton: Skeleton3D, boneIdx: Int32) -> simd_float4x4 { 281 | var mat: simd_float4x4 = .init(diagonal: .one) 282 | 283 | if boneIdx == -1 { 284 | return mat 285 | } 286 | 287 | var boneIdx = boneIdx 288 | while true { 289 | mat *= simd_inverse(simd_float4x4(skeleton.getBonePose(boneIdx: boneIdx))) 290 | 291 | let parentIdx = skeleton.getBoneParent(boneIdx: boneIdx) 292 | if parentIdx != -1 { 293 | boneIdx = parentIdx 294 | } else { 295 | break 296 | } 297 | } 298 | 299 | return mat 300 | } 301 | 302 | 303 | func createRealityKitSkeleton(skeleton: SwiftGodot.Skeleton3D) -> MeshResource.Skeleton? { 304 | var jointNames: [String] = [] 305 | var inverseBindPoseMatrices: [simd_float4x4] = [] 306 | var restPoseTransforms: [RealityKit.Transform] = [] 307 | var parentIndices: [Int] = [] 308 | 309 | for boneIdx in 0.. 0 ? skeletonName : "skeleton", 318 | jointNames: jointNames, 319 | inverseBindPoseMatrices: inverseBindPoseMatrices, 320 | restPoseTransforms: restPoseTransforms, 321 | parentIndices: parentIndices 322 | ) 323 | } 324 | 325 | /// Converts Godot mesh data to RealityKit MeshResource.Contents. 326 | private func meshContents(debugName: String, 327 | meshCreationInfo: MeshCreationInfo, 328 | verbose: Bool = false) throws -> MeshResource.Contents 329 | { 330 | 331 | let mesh = meshCreationInfo.godotMesh 332 | let skeleton = meshCreationInfo.godotSkeleton 333 | 334 | if mesh.getSurfaceCount() == 0 { 335 | return MeshResource.Contents() 336 | } 337 | 338 | enum ArrayType: Int { // TODO: are these already exposed from SwiftGodot somewhere? 339 | case ARRAY_VERTEX = 0 340 | case ARRAY_NORMAL = 1 341 | case ARRAY_TANGENT = 2 342 | case ARRAY_TEX_UV = 4 343 | case ARRAY_BONES = 10 /// PackedFloat32Array or PackedInt32Array of bone indices. Contains either 4 or 8 numbers per vertex depending on the presence of the ARRAY_FLAG_USE_8_BONE_WEIGHTS flag. 344 | case ARRAY_WEIGHTS = 11 /// PackedFloat32Array or PackedFloat64Array of bone weights in the range 0.0 to 1.0 (inclusive). Contains either 4 or 8 numbers per vertex depending on the presence of the ARRAY_FLAG_USE_8_BONE_WEIGHTS flag. 345 | case ARRAY_INDEX = 12 346 | } 347 | 348 | if verbose { 349 | print("--------\nmeshContents for node: \(debugName)") 350 | } 351 | 352 | var newContents = MeshResource.Contents() 353 | 354 | var rkSkeleton: MeshResource.Skeleton? = nil 355 | if let skeleton, let newSkeleton = createRealityKitSkeleton(skeleton: skeleton) { 356 | rkSkeleton = newSkeleton 357 | newContents.skeletons = MeshSkeletonCollection([newSkeleton]) 358 | } 359 | 360 | var meshParts: [MeshResource.Part] = [] 361 | for surfIdx in 0.. 0 { 476 | print(" jointInfluences.count: \(jointInfluences.count)") 477 | } 478 | 479 | // ARRAY_WEIGHTS 480 | do { 481 | let weightsVariant = surfaceArrays[ArrayType.ARRAY_WEIGHTS.rawValue] 482 | switch weightsVariant.gtype { 483 | case .nil: 484 | if jointInfluences.count > 0 { 485 | logError("nil ARRAY_WEIGHTS but ARRAY_BONES present") 486 | } 487 | case .packedFloat32Array: 488 | if let weights = weightsVariant.cast(as: PackedFloat32Array.self, debugName: "ARRAY_WEIGHTS") { 489 | if jointInfluences.count <= weights.count { 490 | weights.bufPtr().enumerated().forEach { (idx, weight) in jointInfluences[idx].weight = weight } 491 | } else { 492 | logError("more weights than bone indices") 493 | } 494 | } 495 | case .packedFloat64Array: 496 | if let weights = weightsVariant.cast(as: PackedFloat64Array.self, debugName: "ARRAY_WEIGHTS") { 497 | if jointInfluences.count <= weights.count { 498 | weights.bufPtr().enumerated().forEach { (idx, weight) in jointInfluences[idx].weight = Float(weight) } // note: precision loss from 64 to 32 bit 499 | } else { 500 | logError("more weights than bone indices") 501 | } 502 | } 503 | default: 504 | logError("ARRAY_WEIGHTS array had unexpected gtype: \(weightsVariant.gtype)") 505 | } 506 | } 507 | 508 | let influencesPerVertex = jointInfluences.count / verticesArray.count 509 | if influencesPerVertex > 0 { 510 | if !(influencesPerVertex == 4 || influencesPerVertex == 8) { 511 | logError("expected influencesPerVertex to be 4 or 8, but it was \(influencesPerVertex) - omitting jointInfluences") 512 | } else { 513 | meshPart.jointInfluences = .init(influences: MeshBuffers.JointInfluences(jointInfluences), influencesPerVertex: influencesPerVertex) 514 | meshPart.skeletonID = rkSkeleton?.id 515 | if let skeletonID = meshPart.skeletonID, newContents.skeletons[skeletonID] == nil { 516 | logError("no skeleton passed for id '\(skeletonID)'") 517 | } 518 | } 519 | } 520 | } 521 | 522 | var modelCollection = MeshModelCollection() 523 | let modelName: String = "model_\(debugName)" 524 | modelCollection.insert(MeshResource.Model(id: modelName, parts: meshParts)) 525 | newContents.models = modelCollection 526 | 527 | // 528 | // Instances 529 | // 530 | if let instanceTransforms = meshCreationInfo.instanceTransforms { 531 | newContents.instances = .init(instanceTransforms.enumerated().map { (idx, xform) in 532 | MeshResource.Instance(id: idx.description, model: modelName, at: xform) 533 | }) 534 | } 535 | 536 | return newContents 537 | } 538 | 539 | private func reverseWindingOrderOfIndexBuffer(_ buffer: [T]) -> [T] where T: BinaryInteger { 540 | var result: [T] = Array.init(repeating: T(), count: buffer.count) 541 | 542 | assert(buffer.count % 3 == 0) 543 | 544 | var i = 0 545 | while i < buffer.count { 546 | result[i + 0] = buffer[i + 0] 547 | result[i + 1] = buffer[i + 2] 548 | result[i + 2] = buffer[i + 1] 549 | 550 | i += 3 551 | } 552 | 553 | return result 554 | } 555 | -------------------------------------------------------------------------------- /Sources/GodotVisionCoordinator.swift: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | Mirror Node3Ds from Godot to RealityKit. 4 | 5 | */ 6 | 7 | // Created by Kevin Watters on 1/3/24. 8 | 9 | let HANDLE_RUNLOOP_MANUALLY = true 10 | let DO_EXTRA_DEBUGGING = false 11 | 12 | import ARKit 13 | import Foundation 14 | import SwiftUI 15 | import RealityKit 16 | import SwiftGodot 17 | import SwiftGodotKit 18 | 19 | private let whiteNonMetallic = SimpleMaterial(color: .white, isMetallic: false) 20 | 21 | let VISION_VOLUME_CAMERA_GODOT_NODE_NAME = "VisionVolumeCamera" 22 | let SHOW_CORNERS = false 23 | let DEFAULT_PROJECT_FOLDER_NAME = "Godot_Project" 24 | 25 | // StringNames for interacting with Godot API efficiently. 26 | let spatial_drag: StringName = "spatial_drag" 27 | let spatial_magnify: StringName = "spatial_magnify" 28 | let spatial_rotate3D: StringName = "spatial_rotate3D" 29 | let gv_auto_prepare: StringName = "gv.auto_prepare" 30 | let hover_effect: StringName = "hover_effect" 31 | let grounding_shadow: StringName = "grounding_shadow" 32 | 33 | struct AudioStreamPlay { 34 | var godotInstanceID: Int64 = 0 35 | var resourcePath: String 36 | var volumeDb: Double = 0 37 | var prepareOnly: Bool = false 38 | var retryCount = 0 39 | } 40 | 41 | 42 | public class GodotVisionCoordinator: NSObject, ObservableObject { 43 | @Published public var paused: Bool = false 44 | public var sceneTree: SceneTree? = nil /// The main Godot SceneTree object. 45 | public var extraScale: Float = 1.0 46 | public var extraOffset: simd_float3 = .zero 47 | public var scenePhase: ScenePhase = .active 48 | 49 | 50 | 51 | /// Allow the user to set how many physics ticks per second (90 is the normal FPS of the vision headset) 52 | public var physicsTicksPerSecond: Int { 53 | get { 54 | Int(Engine.physicsTicksPerSecond) 55 | } 56 | set { 57 | if physicsTicksPerSecond != newValue { 58 | Engine.physicsTicksPerSecond = Int32(newValue) 59 | objectWillChange.send() 60 | } 61 | } 62 | } 63 | 64 | var godotEntitiesParent = Entity() /// The tree of RealityKit Entities mirroring Godot Node3Ds gets parented here. 65 | var shareModel = ShareModel() 66 | 67 | private var godotInstanceIDToEntity: [Int64: Entity] = [:] 68 | private var godotEntitiesScale = Entity() /// Applies a global scale to the godot scene based on the volume size and the godot camera volume. 69 | private var eventSubscription: EventSubscription? = nil 70 | private var nodeTransformsChangedToken: SwiftGodot.Object? = nil 71 | private var nodeAddedToken: SwiftGodot.Object? = nil 72 | private var nodeRemovedToken: SwiftGodot.Object? = nil 73 | private var audioStreamPlayerPlaybackToken: SwiftGodot.Object? = nil 74 | private var receivedRootNode = false 75 | private var resourceCache: ResourceCache = .init() 76 | private var volumeCamera: SwiftGodot.Node3D? = nil 77 | private var audioStreamPlays: [AudioStreamPlay] = [] 78 | private var audioPlaybackControllers: [AudioStreamPlayer3D: AudioPlaybackController] = [:] 79 | private var _audioResources: [String: AudioFileResource] = [:] 80 | private var godotInstanceIDsRemovedFromTree: Set = .init() 81 | private var nodeIdsForNewlyEnteredNodes: [(godotInstanceId: Int64, isRoot: Bool)] = [] 82 | private var volumeCameraBoxSize: simd_float3 = .one 83 | private var realityKitVolumeSize: simd_double3 = .one /// The size we think the RealityKit volume is, in meters, as an application in the user's AR space. 84 | private var godotToRealityKitRatio: Float = 0.05 // default ratio - this is adjusted when realityKitVolumeSize size changes 85 | private var projectContext: GodotProjectContext = .init() 86 | private var skeletonsToUpdate: Dictionary = .init() 87 | private static var didInitGodot = false 88 | 89 | var ar: ARState = .init() /// State for AR functionality. 90 | 91 | private let volumeRatioDebouncer = Debouncer(delay: 2) // used in changeScaleIfVolumeSizeChanged 92 | 93 | // we expect changeScaleIfVolumeSizeChanged to be called... 94 | // - from didReceiveGodotVolumeCamera, when Volume Camera node is first recognized 95 | // - from SwiftUI RealityView update method in ContentView 96 | // - VisionOS seems to size the volume in a few steps 97 | // - when a user changes the zoom level in system settings while the app is open 98 | // we debounce ratio mismatch error log so we only check the ratio when the visionOS volume size settles 99 | // one other note, visionOS clamps volume size in all dimensions to 0.2352 to 1.9852 100 | public func changeScaleIfVolumeSizeChanged(_ volumeSize: simd_double3) { 101 | if volumeSize == realityKitVolumeSize { 102 | return 103 | } 104 | realityKitVolumeSize = volumeSize 105 | let ratio = simd_float3(realityKitVolumeSize) / volumeCameraBoxSize 106 | godotToRealityKitRatio = max(max(ratio.x, ratio.y), ratio.z) 107 | volumeRatioDebouncer.debounce { 108 | let epsilon: Float = 0.01 109 | if !(ratio.x.isApproximatelyEqualTo(ratio.y, epsilon: epsilon) && ratio.y.isApproximatelyEqualTo(ratio.z, epsilon: epsilon)) { 110 | logError("expected the proportions of the RealityKit volume to match the godot volume! the camera volume may be off:\n" + 111 | " realityKitVolumeSize: \(self.realityKitVolumeSize)\n" + 112 | " volumeCameraBoxSize: \(self.volumeCameraBoxSize)\n" + 113 | " ratio: \(ratio))") 114 | } 115 | } 116 | } 117 | 118 | @MainActor 119 | public func reloadScene() { 120 | // print("reloadScene currently doesn't work for loading a new version of the scene saved from the editor, since Xcode copies the Godot_Project into the application's bundle only once at build time.") 121 | 122 | guard let sceneFilePath = self.sceneTree?.currentScene?.sceneFilePath else { 123 | logError("cannot reload, no .sceneFilePath") 124 | return 125 | } 126 | 127 | resetRealityKit() 128 | changeSceneToFile(atResourcePath: sceneFilePath) 129 | } 130 | 131 | @MainActor 132 | public func changeSceneToFile(atResourcePath sceneResourcePath: String) { 133 | guard let sceneTree else { return } 134 | print("CHANGING SCENE TO", sceneResourcePath) 135 | 136 | self.receivedRootNode = false // we will "regrab" the root node and do init stuff with it again. 137 | 138 | for (godotInstanceId, _) in godotInstanceIDToEntity { 139 | godotInstanceIDsRemovedFromTree.insert(UInt(godotInstanceId)) 140 | } 141 | flushRemovedInstances() 142 | 143 | let result = sceneTree.changeSceneToFile(path: sceneResourcePath) 144 | if SwiftGodot.GodotError.ok != result { 145 | logError("changeSceneToFile result was not ok: \(result)") 146 | } 147 | } 148 | 149 | func initGodot() { 150 | var args = [ 151 | ".", // TODO: not sure about this, I think it's the "project directory" -- ios build command line eats the first argument. 152 | 153 | // We want Godot to run in headless mode, meaning: don't open windows, don't render anything, don't play sounds. 154 | "--headless", 155 | 156 | // The GodotVision Godot fork sees this argument and assumes that we will tick the main loop manually. 157 | "--runLoopHandledByHost" 158 | ] 159 | 160 | // Here we tell Godot where to find our Godot project. 161 | 162 | args.append(contentsOf: ["--path", projectContext.getProjectDir()]) 163 | 164 | // This is the SwiftGodotKit "entry" point 165 | runGodot(args: args, 166 | initHook: initHook, 167 | loadScene: receivedSceneTree, 168 | loadProjectSettings: { _ in 169 | // WARNING: this never gets called by SwiftGodotKit (see https://github.com/migueldeicaza/SwiftGodotKit/issues/10) 170 | }, 171 | verbose: true) 172 | } 173 | 174 | private func receivedSceneTree(sceneTree: SwiftGodot.SceneTree) { 175 | // print("receivedSceneTree", sceneTree) 176 | // print("loadSceneCallback", sceneTree.getInstanceId(), sceneTree) 177 | // print(sceneTree.currentScene?.sceneFilePath) 178 | 179 | nodeTransformsChangedToken = sceneTree.nodeTransformsChanged.connect(onNodeTransformsChanged) 180 | nodeAddedToken = sceneTree.nodeAdded.connect(onNodeAdded) 181 | nodeRemovedToken = sceneTree.nodeRemoved.connect(onNodeRemoved) 182 | audioStreamPlayerPlaybackToken = sceneTree.audioStreamPlayerPlayback.connect(onAudioStreamPlayerPlayback) 183 | 184 | self.sceneTree = sceneTree 185 | 186 | #if false // show the nodes in the RealityKit hierarchy 187 | DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { 188 | print("RK HIERARCHY-------") 189 | printEntityTree(self.godotEntitiesParent) 190 | } 191 | #endif 192 | 193 | #if false // show the nodes in the Godot hierarchy 194 | DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { 195 | print("GODOT HIERARCHY-----") 196 | printGodotTree(sceneTree.root!) 197 | } 198 | #endif 199 | } 200 | 201 | private var audioToProcess: [AudioToProcess] = [] 202 | 203 | // callback for custom signal on SceneTree from GodotVision's libgodot fork which gets called for all AudioStreamPlayer[3D] play/stop 204 | private func onAudioStreamPlayerPlayback(obj: SwiftGodot.Object, state: Int64, flags: Int64, from_pos: Double) { 205 | // TODO: once we fix the casting of Nodes from signal calls, we don't need audioToProcess at all. but until then, obj as? AudioStreamPlayer3D will always fail from within this method. :SignalsWithNodeArguments 206 | audioToProcess.append(.init(instanceId: Int64(obj.getInstanceId()), 207 | state: state, 208 | flags: flags, 209 | from_pos: from_pos)) 210 | } 211 | 212 | private func onNodeRemoved(_ node: SwiftGodot.Node) { 213 | let _ = godotInstanceIDsRemovedFromTree.insert(node.getInstanceId()) 214 | if let node3D = node as? Node3D { 215 | ar.nodes.remove(node3D) 216 | if let asp = node as? AudioStreamPlayer3D { 217 | audioPlaybackControllers.removeValue(forKey: asp) 218 | } 219 | } 220 | } 221 | 222 | private func onNodeTransformsChanged(_ packedByteArray: PackedByteArray) { 223 | guard let data = packedByteArray.asDataNoCopy() else { 224 | return 225 | } 226 | 227 | struct NodeData { 228 | enum Flags: UInt32 { 229 | case VISIBLE = 1 230 | } 231 | 232 | // We receive a PackedByteArray of these structs from a special Godot SceneTree signal created for GodotVision. See "node_transforms_changed" in the Godot source. 233 | // The idea is, for performance, to just receive an array of plain data structs, and not incur the costs of gdextension method calls for each Node. 234 | 235 | var objectID: Int64 236 | var parentID: Int64 237 | var pos: simd_float4 238 | var rot: simd_quatf 239 | var scl: simd_float4 240 | 241 | var nativeHandle: UnsafeRawPointer 242 | var flags: UInt32 243 | var pad0: Float 244 | } 245 | 246 | data.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) in 247 | ptr.withMemoryRebound(to: NodeData.self) { nodeDatas in 248 | _receivedNodeDatas(nodeDatas) 249 | } 250 | } 251 | 252 | func _receivedNodeDatas(_ nodeDatas: UnsafeBufferPointer) { 253 | #if false 254 | var transformSetCount = 0 255 | let swiftStride = MemoryLayout.stride 256 | let swiftSize = MemoryLayout.size 257 | //print("SWIFT offsetof transform", MemoryLayout.offset(of: \.transform)) 258 | print("SWIFT offsetof rotation", MemoryLayout.offset(of: \.rot)) 259 | print("SWIFT stride", swiftStride, "size", swiftSize) 260 | print("Swift bound array has", nodeDatas.count, "items.") 261 | print("DATA", ptr.baseAddress, "has", data.count, "bytes of raw data") 262 | #endif 263 | for nodeData in nodeDatas { 264 | if nodeData.objectID == 0 { continue } 265 | 266 | guard let entity = godotInstanceIDToEntity[nodeData.objectID] else { 267 | continue 268 | } 269 | 270 | // Update Transform 271 | entity.transform = .init(scale: .init(nodeData.scl.x, nodeData.scl.y, nodeData.scl.z), 272 | rotation: nodeData.rot, 273 | translation: .init(nodeData.pos.x, nodeData.pos.y, nodeData.pos.z)) 274 | 275 | let isEnabled = (nodeData.flags & NodeData.Flags.VISIBLE.rawValue) != 0 276 | if entity.isEnabled != isEnabled { 277 | entity.isEnabled = isEnabled 278 | } 279 | } 280 | } 281 | 282 | } 283 | 284 | private func onNodeAdded(_ node: SwiftGodot.Node) { 285 | let isRoot = node.getParent() == node.getTree()?.root 286 | 287 | // TODO: Hack to get around the fact that the nodes coming in don't cast properly as their subclass yet. outside of this signal handler they do, so we store the id for later. :SignalsWithNodeArguments 288 | nodeIdsForNewlyEnteredNodes.append((Int64(node.getInstanceId()), isRoot)) 289 | 290 | if isRoot && !receivedRootNode { 291 | receivedRootNode = true 292 | resetRealityKit() 293 | if let sceneTree, !shareModel.didSetup, sharePlayEnabled { 294 | shareModel.didSetup = true 295 | shareModel.setup(sceneTree: sceneTree, onMessageData: onMultiplayerData) 296 | } 297 | } 298 | } 299 | 300 | private func didReceiveGodotVolumeCamera(_ node: Node3D) { 301 | volumeCamera = node 302 | 303 | guard let area3D = volumeCamera as? Area3D else { 304 | logError("expected node '\(VISION_VOLUME_CAMERA_GODOT_NODE_NAME)' to be an Area3D, but it was a \(node.godotClassName)") 305 | return 306 | } 307 | 308 | let collisionShape3Ds = area3D.findChildren(pattern: "*", type: "CollisionShape3D") 309 | if collisionShape3Ds.count != 1 { 310 | logError("expected \(VISION_VOLUME_CAMERA_GODOT_NODE_NAME) to have exactly one child with CollisionShape3D, but got \(collisionShape3Ds.count)") 311 | return 312 | } 313 | 314 | guard let collisionShape3D = collisionShape3Ds[0] as? CollisionShape3D else { 315 | logError("Could not cast child as CollisionShape3D") 316 | return 317 | } 318 | 319 | guard let boxShape3D = collisionShape3D.shape as? BoxShape3D else { 320 | logError("Could not cast shape as BoxShape3D in CollisionShape3D child of \(VISION_VOLUME_CAMERA_GODOT_NODE_NAME)") 321 | return 322 | } 323 | 324 | let godotVolumeBoxSize = simd_float3(boxShape3D.size) 325 | volumeCameraBoxSize = godotVolumeBoxSize 326 | 327 | changeScaleIfVolumeSizeChanged(realityKitVolumeSize) 328 | } 329 | 330 | func resetRealityKit() { 331 | for (godotInstanceId, _) in godotInstanceIDToEntity { 332 | godotInstanceIDsRemovedFromTree.insert(UInt(godotInstanceId)) 333 | } 334 | flushRemovedInstances() 335 | audioStreamPlays.removeAll() 336 | audioPlaybackControllers.removeAll() 337 | resourceCache.reset() 338 | skeletonsToUpdate.removeAll() 339 | 340 | for child in godotEntitiesParent.children.map({ $0 }) { 341 | child.removeFromParent() 342 | } 343 | } 344 | 345 | var sharePlayEnabled: Bool { shareModel.activityIdentifier != nil } 346 | 347 | @MainActor 348 | @discardableResult 349 | public func setupRealityKitScene(_ rootEntity: Entity, 350 | volumeSize: simd_double3, 351 | projectFileDir: String? = nil, 352 | sharePlayActivityId: String? = nil) -> Entity 353 | { 354 | projectContext.projectFolderName = projectFileDir ?? DEFAULT_PROJECT_FOLDER_NAME 355 | resourceCache.projectContext = projectContext 356 | 357 | if sharePlayActivityId != nil { 358 | log("Starting shareplay session handler...") 359 | shareModel.activityIdentifier = sharePlayActivityId // ENABLE MULTIPLAYER if not nil 360 | shareModel.startSessionHandlerTask() 361 | } 362 | 363 | if !Self.didInitGodot { 364 | initGodot() // TODO: once libgodot is properly embeddable/teardownable this will change. 365 | Self.didInitGodot = true 366 | } 367 | 368 | realityKitVolumeSize = volumeSize 369 | 370 | // Create a root Entity to store all our mirrored Godot nodes-turned-RealityKit entities. 371 | godotEntitiesParent.name = "GODOTRK_ROOT" 372 | godotEntitiesScale.addChild(godotEntitiesParent) 373 | 374 | // A root of that entity will also apply scale. 375 | godotEntitiesScale.name = "GODOTRK_SCALE" 376 | rootEntity.addChild(godotEntitiesScale) 377 | 378 | return godotEntitiesParent 379 | } 380 | 381 | public func cleanup() { 382 | print("Cleaning up GodotVisionCoordinator.") 383 | 384 | if let eventSubscription { 385 | eventSubscription.cancel() 386 | self.eventSubscription = nil 387 | } 388 | 389 | // Disconnect SceneTree signals 390 | if let sceneTree { 391 | if let nodeTransformsChangedToken { 392 | sceneTree.nodeTransformsChanged.disconnect(nodeTransformsChangedToken) 393 | self.nodeTransformsChangedToken = nil 394 | } 395 | if let nodeAddedToken { 396 | sceneTree.nodeAdded.disconnect(nodeAddedToken) 397 | self.nodeAddedToken = nil 398 | } 399 | if let nodeRemovedToken { 400 | sceneTree.nodeRemoved.disconnect(nodeRemovedToken) 401 | self.nodeRemovedToken = nil 402 | } 403 | self.sceneTree = nil 404 | } 405 | 406 | godotEntitiesScale.removeFromParent() 407 | } 408 | 409 | private func cacheAudioResource(resourcePath: String, godotInstanceID: Int64) -> AudioFileResource? { 410 | var audioResource: AudioFileResource? = _audioResources[resourcePath] 411 | if audioResource == nil { 412 | var configuration = AudioFileResource.Configuration() 413 | 414 | // TODO: see if there's a way to use metadata to do shouldRandomizeStartTime 415 | 416 | // See if we need to loop. 417 | // TODO: different AudioStreams may point at the same resource path? 418 | if let audioStreamPlayer3D = GD.instanceFromId(instanceId: godotInstanceID) as? AudioStreamPlayer3D, 419 | let audioStreamWAV = audioStreamPlayer3D.stream as? AudioStreamWAV, 420 | audioStreamWAV.loopMode == .forward 421 | { 422 | switch audioStreamWAV.loopMode { 423 | case .disabled: 424 | () 425 | case .forward: 426 | configuration.shouldLoop = true 427 | default: 428 | logError("AudioStreamWAV.loopMode '\(audioStreamWAV.loopMode)' not supported yet.") 429 | } 430 | } 431 | 432 | do { 433 | audioResource = try AudioFileResource.load(contentsOf: projectContext.fileUrl(forGodotResourcePath: resourcePath), configuration: configuration) 434 | } catch { 435 | logError(error) 436 | return nil 437 | } 438 | 439 | _audioResources[resourcePath] = audioResource 440 | } 441 | return audioResource 442 | } 443 | 444 | private func _onMeshResourceChanged(meshEntry: MeshEntry) { 445 | for entity in meshEntry.entities { 446 | if let node = entity.components[GodotNode.self]?.node3D { 447 | let _ = _updateModelEntityMesh(node: node, existingEntity: entity) 448 | } 449 | } 450 | } 451 | 452 | private func _updateModelEntityMesh(node: Node, existingEntity: Entity?, cb: ((Entity?) -> Void)? = nil) -> Entity { 453 | var mesh: SwiftGodot.Mesh? = nil 454 | var materials: [SwiftGodot.Material]? = nil 455 | var skeleton3D: SwiftGodot.Skeleton3D? = nil 456 | var flipFacesIfNoIndexBuffer: Bool = false 457 | var instanceTransforms: [simd_float4x4]? = nil 458 | 459 | // Inspect the Godot node to see if we have a mesh, materials, and a skeleton. 460 | 461 | if let meshInstance3D = node as? MeshInstance3D { 462 | // MeshInstance3D 463 | skeleton3D = meshInstance3D.skeleton.isEmpty() ? nil : (meshInstance3D.getNode(path: meshInstance3D.skeleton) as? Skeleton3D) 464 | mesh = meshInstance3D.mesh 465 | if let mesh { 466 | materials = (0..= 0 ? multimesh.visibleInstanceCount : multimesh.instanceCount 473 | instanceTransforms = (0..= 2 { 479 | flipFacesIfNoIndexBuffer = true 480 | mesh = transformAndMesh[1].asObject(SwiftGodot.Mesh.self) 481 | } 482 | } 483 | 484 | // If getActiveMaterial did not provide materials, use the materials from the mesh directly. 485 | if materials == nil, let mesh { 486 | materials = (0.. Entity { 540 | let inputRayPickable: Bool = (node as? CollisionObject3D)?.inputRayPickable ?? false 541 | // If there's a metadata entry for 'hover_effect' with the boolean value of true, we add a RealityKit HoverEffectComponent. 542 | let hoverEffect: Bool = inputRayPickable && node.getMetaBool(hover_effect, defaultValue: false) 543 | 544 | // Construct a ModelEntity with our mesh data if we have one. 545 | let entity = _updateModelEntityMesh(node: node, existingEntity: nil) { entity in 546 | if inputRayPickable, let entity { 547 | DispatchQueue.main.async { 548 | updateCollision(entity: entity) 549 | } 550 | } 551 | } 552 | entity.name = "\(node.name)" 553 | if inputRayPickable { 554 | entity.components.set(InputTargetComponent()) 555 | if hoverEffect { 556 | entity.components.set(HoverEffectComponent()) 557 | } 558 | } 559 | 560 | let instanceID = Int64(node.getInstanceId()) 561 | godotInstanceIDToEntity[instanceID] = entity 562 | godotEntitiesParent.addChild(entity) 563 | 564 | // Setup a node to be tappable with the `InputTargetComponent()` if necessary.. 565 | 566 | // Finally, initialize the position/rotation/scale. 567 | if let node3D = node as? Node3D { 568 | entity.transform = RealityKit.Transform(node3D.transform) 569 | entity.isEnabled = node3D.visible 570 | } 571 | 572 | // Remove RealityKit physics since we rely on Godot's. 573 | entity.components[PhysicsBodyComponent.self] = .none 574 | 575 | return entity 576 | } 577 | 578 | public var perFrameTick: ((_ deltaTime: TimeInterval) -> Void)? = nil 579 | 580 | public func realityKitPerFrameTick(_ event: SceneEvents.Update) { 581 | if paused { 582 | return 583 | } 584 | 585 | Input.flushBufferedEvents() // our iOS loop doesn't currently do this, so flush events manually 586 | 587 | let godotDidQuit = stepGodotFrame() 588 | 589 | defer { 590 | if godotDidQuit { 591 | print("GODOT HAS QUIT") 592 | paused = true 593 | } 594 | } 595 | 596 | perFrameTick?(event.deltaTime) 597 | 598 | // update playing audio pitches 599 | for (godotAudioStreamPlayer, pbCon) in audioPlaybackControllers { 600 | if pbCon.isPlaying, godotAudioStreamPlayer.isPlaying() { 601 | pbCon.speed = godotAudioStreamPlayer.pitchScale 602 | } 603 | } 604 | 605 | // handle new nodes 606 | for (id, isRoot) in nodeIdsForNewlyEnteredNodes { 607 | guard let node = GD.instanceFromId(instanceId: id) as? Node else { 608 | logError("No new Node instance for id \(id)") 609 | continue 610 | } 611 | 612 | let entity = _createRealityKitEntityForNewNode(node) 613 | let node3D = node as? Node3D 614 | 615 | let skeleton = node3D != nil ? getSkeletonNode(node: node3D!, entity: entity) : nil 616 | if let node3D, let skeleton { 617 | entity.components.set(GodotNode(node3D: node3D)) 618 | if let modelEntity = entity as? ModelEntity { 619 | if skeletonsToUpdate[skeleton] == nil { 620 | skeletonsToUpdate[skeleton] = .init() 621 | } 622 | skeletonsToUpdate[skeleton]?.append(modelEntity) 623 | } else { 624 | logError("expected to cast as ModelEntity but failed: \(entity)") 625 | } 626 | } 627 | 628 | // we notice a specially named node for the "bounds" 629 | if node.name == VISION_VOLUME_CAMERA_GODOT_NODE_NAME, let node3D = node as? Node3D { 630 | didReceiveGodotVolumeCamera(node3D) 631 | } 632 | 633 | // MARK: - Create gesture context components on applicable nodes 634 | if node.hasSignal(spatial_drag) { 635 | // TODO: check if it's a collision object? warn if not? 636 | entity.components.set(GodotVisionDraggable()) 637 | } 638 | if node.hasSignal(spatial_magnify) { 639 | // TODO: check if it's a collision object? warn if not? 640 | entity.components.set(GodotVisionMagnifiable()) 641 | } 642 | if node.hasSignal(spatial_rotate3D) { 643 | // TODO: check if it's a collision object? warn if not? 644 | entity.components.set(GodotVisionRotateable()) 645 | } 646 | 647 | if let audioStreamPlayer3D = node as? AudioStreamPlayer3D { 648 | // See if we need to prepare the audio resource (prevents a hitch on first play). 649 | if audioStreamPlayer3D.getMetaBool(gv_auto_prepare, defaultValue: true) { 650 | if let resourcePath = audioStreamPlayer3D.stream?.resourcePath { 651 | audioStreamPlays.append(.init(godotInstanceID: Int64(audioStreamPlayer3D.getInstanceId()), 652 | resourcePath: resourcePath, 653 | volumeDb: audioStreamPlayer3D.volumeDb, 654 | prepareOnly: true)) 655 | } 656 | } 657 | } 658 | 659 | // XXX TEMP HACK while we build out the API for requesting head tracking. 660 | if node.name == "ARTracking", let node3D = node as? Node3D { 661 | ensureARInited() 662 | ar.nodes.insert(node3D) 663 | } 664 | 665 | if !isRoot, let nodeParent = node.getParent() { 666 | let parentId = Int64(nodeParent.getInstanceId()) 667 | if let parent = godotInstanceIDToEntity[parentId] { 668 | entity.setParent(parent) 669 | } else { 670 | logError("could not find parent for id \(parentId)") 671 | } 672 | } 673 | } 674 | nodeIdsForNewlyEnteredNodes.removeAll() 675 | 676 | var retries: [AudioStreamPlay] = [] 677 | defer { 678 | audioStreamPlays = retries 679 | godotInstanceIDsRemovedFromTree.removeAll() 680 | } 681 | 682 | // new audio plays 683 | do { 684 | let newAudios = audioToProcess 685 | audioToProcess.removeAll() 686 | enum State: Int64 { 687 | case UNKNOWN = 0 688 | case PLAY = 1 689 | case STOP = 2 690 | } 691 | 692 | for newAudio in newAudios { 693 | let node = GD.instanceFromId(instanceId: newAudio.instanceId) 694 | if let audioStreamPlayer3D = node as? AudioStreamPlayer3D { 695 | if let resourcePath = audioStreamPlayer3D.stream?.resourcePath { 696 | if newAudio.state == State.PLAY.rawValue { 697 | audioStreamPlays.append(.init(godotInstanceID: Int64(audioStreamPlayer3D.getInstanceId()), 698 | resourcePath: resourcePath, 699 | volumeDb: audioStreamPlayer3D.volumeDb, 700 | prepareOnly: false)) 701 | } 702 | } 703 | } 704 | } 705 | } 706 | 707 | // Play any AudioStreamPlayer3D sounds 708 | for var audioStreamPlay in audioStreamPlays { 709 | guard let entity = godotInstanceIDToEntity[audioStreamPlay.godotInstanceID] else { 710 | audioStreamPlay.retryCount += 1 711 | if audioStreamPlay.retryCount < 100 { 712 | retries.append(audioStreamPlay) // we may not have seen the instance yet. 713 | } 714 | continue 715 | } 716 | 717 | if let audioResource = cacheAudioResource(resourcePath: audioStreamPlay.resourcePath, godotInstanceID: audioStreamPlay.godotInstanceID) { 718 | if audioStreamPlay.prepareOnly { 719 | let _ = entity.prepareAudio(audioResource) 720 | } else { 721 | let audioPlaybackController = entity.playAudio(audioResource) 722 | audioPlaybackController.gain = audioStreamPlay.volumeDb 723 | if let audioStreamPlayer = GD.instanceFromId(instanceId: audioStreamPlay.godotInstanceID) as? AudioStreamPlayer3D { 724 | audioPlaybackControllers[audioStreamPlayer] = audioPlaybackController 725 | } 726 | } 727 | } 728 | } 729 | 730 | flushRemovedInstances() 731 | arUpdate() 732 | 733 | // Update skeletons 734 | if !skeletonsToUpdate.isEmpty { 735 | let skeletons = Array(skeletonsToUpdate.keys) 736 | var results = Array(Skeleton3D.getMultipleSkeletonBonePoses(skeletons: ObjectCollection(skeletons))) 737 | // returns [stride, num_bones_1, transform_ptr_1, num_bones_2, transform_ptr_2] where each ptr points to a block of transforms num_bones long, `stride` bytes apart 738 | assert(results.count > 0) 739 | let stride = results.removeFirst() 740 | assert(results.count == skeletons.count * 2) 741 | for (skeletonIdx, skeleton) in skeletons.enumerated() { 742 | guard let modelEntities = skeletonsToUpdate[skeleton] else { 743 | logError("no modelEntities for skeleton \(skeleton)") 744 | continue 745 | } 746 | 747 | let numBonesIdx = skeletonIdx * 2 748 | let (numBones, transformPtrInt) = (Int(results[numBonesIdx]), results[numBonesIdx + 1]) 749 | let transforms = (0.., ended: Bool = false) { 788 | let entity = value.entity 789 | 790 | // See if this Node has a 'drag' signal 791 | guard let obj = entity.godotNode else { return } 792 | if !obj.hasSignal(spatial_drag) { 793 | return 794 | } 795 | 796 | guard var dragCtx = entity.components[GodotVisionDraggable.self] else { return } 797 | defer { entity.components.set(dragCtx) } 798 | 799 | var origPos = dragCtx.originalPosition 800 | var began = false 801 | if origPos == nil { 802 | origPos = entity.position(relativeTo: nil) 803 | dragCtx.originalPosition = origPos 804 | began = true 805 | } 806 | 807 | var origRotation = dragCtx.originalRotation 808 | if origRotation == nil { 809 | origRotation = Rotation3D(entity.orientation(relativeTo: nil)) 810 | dragCtx.originalRotation = origRotation 811 | } 812 | 813 | // Rotation 814 | var newOrientation = entity.orientation(relativeTo: nil) 815 | let localToScene = value.transform(from: .local, to: .scene) // an AffineTransform3D which maps .local to .scene 816 | if let startInputDevicePose3D = value.startInputDevicePose3D, 817 | let inputDevicePose3D = value.inputDevicePose3D, 818 | let startRotation = (localToScene * AffineTransform3D(pose: startInputDevicePose3D)).rotation, 819 | let currentRotation = (localToScene * AffineTransform3D(pose: inputDevicePose3D)).rotation 820 | { 821 | // "add" the original pose and the delta of the (current pose - start pose) 822 | newOrientation = .init((currentRotation * startRotation.inverse) * origRotation!) 823 | } 824 | 825 | // Position 826 | // Note that we ignore the position of the pose3D objects above--they are actually more noisy than the smoothed out version we get in 827 | // the gesture value. 828 | let startPos = value.convert(value.startLocation3D, from: .local, to: .scene) 829 | let currentPos = value.convert(value.location3D, from: .local, to: .scene) 830 | let newPosition = (currentPos - startPos) + origPos! 831 | 832 | let scale = entity.scale(relativeTo: nil) // TODO: test dragging nodes with different scales to make sure this is right 833 | 834 | let gdStartGlobalTransform = Transform3D(godotEntitiesParent.convert(transform: .init(scale: scale, rotation: .init(origRotation!), translation: .init(origPos!)), from: nil)) 835 | let gdGlobalTransform = Transform3D(godotEntitiesParent.convert(transform: .init(scale: scale, rotation: newOrientation, translation: newPosition), from: nil)) 836 | let phase = ended ? "ended" : (began ? "began" : "changed") 837 | 838 | 839 | // pass a dictionary of values to the drag signal 840 | let dict: GDictionary = .init() 841 | dict["global_transform"] = Variant(gdGlobalTransform) 842 | dict["start_global_transform"] = Variant(gdStartGlobalTransform) 843 | dict["phase"] = Variant(phase) 844 | 845 | obj.emitSignal(spatial_drag, Variant(dict)) 846 | 847 | if shareModel.automaticallyShareInput { 848 | let nodePathString = obj.getPath().description 849 | let params: SpatialDragParams = .init(global_transform: .init(gdGlobalTransform), start_global_transform: .init(gdStartGlobalTransform), phase: phase) 850 | let inputMessage: InputMessage = .init(nodePath: nodePathString, signalName: "spatial_drag", params: params) 851 | shareModel.sendInput(inputMessage, reliable: phase != "changed") 852 | } 853 | 854 | if ended { 855 | dragCtx.reset() 856 | } 857 | } 858 | 859 | /// A visionOS drag is starting or being updated. We emit a signal with information about the gesture so that Godot code can respond. 860 | func receivedMagnify(_ value: EntityTargetValue, ended: Bool = false) { 861 | let entity = value.entity 862 | 863 | // See if this Node has a 'magnify' signal 864 | guard let obj = entity.godotNode else { return } 865 | if !obj.hasSignal(spatial_magnify) { 866 | return 867 | } 868 | 869 | guard var magCtx = entity.components[GodotVisionMagnifiable.self] else { return } 870 | defer { entity.components.set(magCtx) } 871 | 872 | var origScale = magCtx.originalScale 873 | var began = false 874 | if origScale == nil { 875 | origScale = entity.scale(relativeTo: nil) 876 | magCtx.originalScale = origScale 877 | began = true 878 | } 879 | 880 | var origPos = magCtx.originalPosition 881 | if origPos == nil { 882 | origPos = entity.position(relativeTo: nil) 883 | magCtx.originalPosition = origPos 884 | } 885 | 886 | var origRotation = magCtx.originalRotation 887 | if origRotation == nil { 888 | origRotation = Rotation3D(entity.orientation(relativeTo: nil)) 889 | magCtx.originalRotation = origRotation 890 | } 891 | 892 | let magnification = Float(value.magnification) 893 | let newScale = origScale! * magnification 894 | 895 | let gdStartGlobalTransform = Transform3D(godotEntitiesParent.convert(transform: .init(scale: origScale!, rotation: .init(origRotation!), translation: .init(origPos!)), from: nil)) 896 | let gdGlobalTransform = Transform3D(godotEntitiesParent.convert(transform: .init(scale: newScale, rotation: .init(origRotation!), translation: .init(origPos!)), from: nil)) 897 | 898 | let phase = ended ? "ended" : (began ? "began" : "changed") 899 | 900 | // pass a dictionary of values to the signal 901 | let dict: GDictionary = .init() 902 | dict["global_transform"] = Variant(gdGlobalTransform) 903 | dict["start_global_transform"] = Variant(gdStartGlobalTransform) 904 | dict["phase"] = Variant(phase) 905 | 906 | obj.emitSignal(spatial_magnify, Variant(dict)) 907 | 908 | if shareModel.automaticallyShareInput { 909 | let nodePathString = obj.getPath().description 910 | 911 | let params: SpatialDragParams = .init(global_transform: .init(gdGlobalTransform), start_global_transform: .init(gdStartGlobalTransform), phase: phase) 912 | let inputMessage: InputMessage = .init(nodePath: nodePathString, signalName: "spatial_magnify", params: params) 913 | self.shareModel.sendInput(inputMessage, reliable: phase != "changed") 914 | } 915 | 916 | if ended { 917 | magCtx.reset() 918 | } 919 | } 920 | 921 | /// A visionOS drag is starting or being updated. We emit a signal with information about the gesture so that Godot code can respond. 922 | func receivedRotate3D(_ value: EntityTargetValue, ended: Bool = false) { 923 | let entity = value.entity 924 | 925 | // See if this Node has a 'rotate3D' signal 926 | guard let obj = entity.godotNode else { return } 927 | if !obj.hasSignal(spatial_rotate3D) { 928 | return 929 | } 930 | 931 | guard var rotCtx = entity.components[GodotVisionRotateable.self] else { return } 932 | defer { entity.components.set(rotCtx) } 933 | 934 | var origPos = rotCtx.originalPosition 935 | var began = false 936 | if origPos == nil { 937 | origPos = entity.position(relativeTo: nil) 938 | rotCtx.originalPosition = origPos 939 | began = true 940 | } 941 | 942 | var origRotation = rotCtx.originalRotation 943 | if origRotation == nil { 944 | origRotation = Rotation3D(entity.orientation(relativeTo: nil)) 945 | rotCtx.originalRotation = origRotation 946 | } 947 | 948 | // Rotation 949 | let rotation = value.rotation 950 | let flippedRotation = Rotation3D(angle: rotation.angle, 951 | axis: RotationAxis3D(x: -rotation.axis.x, 952 | y: rotation.axis.y, 953 | z: -rotation.axis.z)) 954 | 955 | var newOrientation = entity.orientation(relativeTo: nil) 956 | let startRotation = origRotation 957 | 958 | newOrientation = .init(flippedRotation * startRotation!) 959 | 960 | let scale = entity.scale(relativeTo: nil) 961 | 962 | let gdStartGlobalTransform = Transform3D(godotEntitiesParent.convert(transform: .init(scale: scale, rotation: .init(origRotation!), translation: .init(origPos!)), from: nil)) 963 | let gdGlobalTransform = Transform3D(godotEntitiesParent.convert(transform: .init(scale: scale, rotation: newOrientation, translation: .init(origPos!)), from: nil)) 964 | let phase = ended ? "ended" : (began ? "began" : "changed") 965 | 966 | 967 | // pass a dictionary of values to the drag signal 968 | let dict: GDictionary = .init() 969 | let source = "rotate3D" 970 | dict["global_transform"] = Variant(gdGlobalTransform) 971 | dict["start_global_transform"] = Variant(gdStartGlobalTransform) 972 | dict["phase"] = Variant(phase) 973 | dict["source"] = Variant(source) 974 | 975 | obj.emitSignal(spatial_rotate3D, Variant(dict)) 976 | 977 | if shareModel.automaticallyShareInput { 978 | let nodePathString = obj.getPath().description 979 | let params: SpatialDragParams = .init(global_transform: .init(gdGlobalTransform), start_global_transform: .init(gdStartGlobalTransform), phase: phase) 980 | let inputMessage: InputMessage = .init(nodePath: nodePathString, signalName: "spatial_rotate3D", params: params) 981 | shareModel.sendInput(inputMessage, reliable: phase != "changed") 982 | } 983 | 984 | if ended { 985 | rotCtx.reset() 986 | } 987 | } 988 | 989 | func onMultiplayerData(_ data: Data) { 990 | let inputMessage: InputMessage 991 | do { 992 | inputMessage = try JSONDecoder().decode(InputMessage.self, from: data) 993 | } catch { 994 | log("error decoding json \(error)") 995 | return 996 | } 997 | 998 | guard let sceneTree else { 999 | return 1000 | } 1001 | 1002 | DispatchQueue.main.async { 1003 | 1004 | let nodePath = NodePath(inputMessage.nodePath) 1005 | guard let node = sceneTree.root?.getNode(path: nodePath) as? Node else { 1006 | log("Could not find Node for node path \(nodePath)") 1007 | return 1008 | } 1009 | 1010 | let dict = GDictionary() 1011 | dict["global_transform"] = Variant(Transform3D(inputMessage.params.global_transform)) 1012 | dict["start_global_transform"] = Variant(Transform3D(inputMessage.params.start_global_transform)) 1013 | dict["phase"] = Variant(inputMessage.params.phase) 1014 | node.emitSignal(StringName(inputMessage.signalName), Variant(dict)) 1015 | log("did emit \(inputMessage.signalName) from remote!") 1016 | } 1017 | } 1018 | 1019 | /// A visionOS drag has ended. We emit a signal to inform Godot land. 1020 | func receivedDragEnded(_ value: EntityTargetValue) { 1021 | receivedDrag(value, ended: true) 1022 | } 1023 | 1024 | /// A visionOS magnify has ended. We emit a signal to inform Godot land. 1025 | func receivedMagnifyEnded(_ value: EntityTargetValue) { 1026 | receivedMagnify(value, ended: true) 1027 | } 1028 | 1029 | /// A visionOS rotate3D has ended. We emit a signal to inform Godot land. 1030 | func receivedRotate3DEnded(_ value: EntityTargetValue) { 1031 | receivedRotate3D(value, ended: true) 1032 | } 1033 | 1034 | /// RealityKit has received a SpatialTapGesture in the RealityView 1035 | func receivedTap(event: EntityTargetValue) { 1036 | let sceneLoc = event.convert(event.location3D, from: .local, to: .scene) 1037 | let godotLocation = godotEntitiesParent.convert(position: simd_float3(sceneLoc), from: nil) 1038 | 1039 | // TODO: hack. this normal is not always right. currently this just pretends everything you tap is a sphere??? 1040 | 1041 | let realityKitNormal = normalize(event.entity.position(relativeTo: nil) - sceneLoc) 1042 | 1043 | // TODO: need to convert the normal to global or local space? 1044 | 1045 | let godotNormal = realityKitNormal 1046 | 1047 | guard let obj = event.entity.godotNode else { return } 1048 | 1049 | // Construct an InputEventMouseButton to send to Godot. 1050 | 1051 | let godotEvent = InputEventMouseButton() 1052 | godotEvent.buttonIndex = .left 1053 | godotEvent.doubleClick = false // TODO 1054 | godotEvent.pressed = true 1055 | 1056 | let position = SwiftGodot.Vector3(godotLocation) // TODO: check if this should be global or local. 1057 | let normal = SwiftGodot.Vector3(godotNormal) 1058 | let shape_idx = 0 1059 | 1060 | // TODO: I think SwiftGodot provides a better way to emit signals than all these Variant() constructors 1061 | obj.emitSignal("input_event", Variant.init(), Variant(godotEvent), Variant(position), Variant(normal), Variant(shape_idx)) 1062 | } 1063 | } 1064 | 1065 | 1066 | /// Stores a reference to a Godot Node3D in the RealityKit ECS for an Entity. 1067 | struct GodotNode: Component { 1068 | var node: Node? = nil 1069 | var node3D: Node3D? = nil 1070 | var meshEntry: MeshEntry? = nil 1071 | } 1072 | 1073 | /// Contains the notion of where in the application bundle to find the Godot .project file. 1074 | class GodotProjectContext { 1075 | var projectFolderName: String? = nil 1076 | 1077 | func fileUrl(forGodotResourcePath resourcePath: String) -> URL { 1078 | getGodotProjectURL().appendingPathComponent(resourcePath.removingStringPrefix("res://")) 1079 | } 1080 | 1081 | func getGodotProjectURL() -> URL { 1082 | if projectFolderName == nil { 1083 | print("*** WARNING, defaulting to DEFAULT_PROJECT_FOLDER_NAME") 1084 | } 1085 | let dirName = projectFolderName ?? DEFAULT_PROJECT_FOLDER_NAME 1086 | guard let url = Bundle.main.url(forResource: dirName, withExtension: nil) else { 1087 | fatalError("ERROR: could not find '\(dirName)' Godot project folder in Bundle.main") 1088 | } 1089 | return url 1090 | } 1091 | 1092 | func getProjectDir() -> String { 1093 | // Godot is expecting a path without the file:// part for the packfile 1094 | getGodotProjectURL().absoluteString.removingStringPrefix("file://") 1095 | } 1096 | } 1097 | 1098 | 1099 | struct GodotVisionDraggable: Component { 1100 | var originalPosition: simd_float3? = nil 1101 | var originalRotation: Rotation3D? = nil 1102 | var originalScale: SIMD3? = nil 1103 | 1104 | mutating func reset() { 1105 | originalPosition = nil 1106 | originalRotation = nil 1107 | originalScale = nil 1108 | } 1109 | } 1110 | 1111 | struct GodotVisionMagnifiable: Component { 1112 | var originalPosition: simd_float3? = nil 1113 | var originalRotation: Rotation3D? = nil 1114 | var originalScale: SIMD3? = nil 1115 | 1116 | mutating func reset() { 1117 | originalPosition = nil 1118 | originalRotation = nil 1119 | originalScale = nil 1120 | } 1121 | } 1122 | 1123 | struct GodotVisionRotateable: Component { 1124 | var originalPosition: simd_float3? = nil 1125 | var originalRotation: Rotation3D? = nil 1126 | var originalScale: SIMD3? = nil 1127 | 1128 | mutating func reset() { 1129 | originalPosition = nil 1130 | originalRotation = nil 1131 | originalScale = nil 1132 | } 1133 | } 1134 | 1135 | private func getSkeletonNode(node: SwiftGodot.Node3D, entity: RealityKit.Entity, verbose: Bool = false) -> Skeleton3D? { 1136 | guard let _ = entity as? ModelEntity else { 1137 | if verbose { print("RK entity is not a ModelEntity") } 1138 | return nil 1139 | } 1140 | 1141 | guard let meshInstance3D = node as? MeshInstance3D else { 1142 | if verbose { print("the godot Node is not a MeshInstance3D") } 1143 | return nil 1144 | } 1145 | 1146 | let skeletonNodePath = meshInstance3D.skeleton 1147 | if !skeletonNodePath.isEmpty(), let skeleton = meshInstance3D.getNode(path: skeletonNodePath) as? Skeleton3D { 1148 | return skeleton 1149 | } 1150 | 1151 | return nil 1152 | } 1153 | 1154 | protocol TriviallyInitializableComponent: Component { init() } 1155 | extension GodotNode: TriviallyInitializableComponent {} 1156 | 1157 | extension Entity.ComponentSet { 1158 | mutating func getOrMake(_ cb: (inout T) -> Void) where T: TriviallyInitializableComponent { 1159 | var componentData = self[T.self] ?? .init() 1160 | cb(&componentData) 1161 | self[T.self] = componentData 1162 | } 1163 | 1164 | mutating func modifyIfExists(_ cb: (inout T) -> Void) where T: Component { 1165 | if var componentData = self[T.self] { 1166 | cb(&componentData) 1167 | self[T.self] = componentData 1168 | } 1169 | } 1170 | } 1171 | 1172 | import simd 1173 | public struct SpatialDragParams: Codable { 1174 | var global_transform: CodableTransform3D 1175 | var start_global_transform: CodableTransform3D 1176 | var phase: String 1177 | } 1178 | 1179 | public struct InputMessage: Codable where T: Codable { 1180 | var nodePath: String 1181 | var signalName: String 1182 | var params: T 1183 | } 1184 | 1185 | public struct CodableTransform3D: Codable { 1186 | var basis0: simd_float3 1187 | var basis1: simd_float3 1188 | var basis2: simd_float3 1189 | var origin: simd_float3 1190 | 1191 | init(_ t: Transform3D) { 1192 | basis0 = .init(t.basis.x) 1193 | basis1 = .init(t.basis.y) 1194 | basis2 = .init(t.basis.z) 1195 | origin = .init(t.origin) 1196 | } 1197 | } 1198 | 1199 | extension Transform3D { 1200 | init(_ c: CodableTransform3D) { 1201 | self.init(xAxis: .init(c.basis0), yAxis: .init(c.basis1), zAxis: .init(c.basis2), origin: .init(c.origin)) 1202 | } 1203 | } 1204 | 1205 | func updateCollision(entity: Entity, count: Int = 0) { 1206 | if !entity.components.has(InputTargetComponent.self) { 1207 | return 1208 | } 1209 | 1210 | let bounds = entity.visualBounds(relativeTo: entity.parent, excludeInactive: true) 1211 | if bounds.isEmpty { 1212 | if entity.parent != nil, count < 500 { 1213 | // HACK: Now that we're generating meshes asynchronously, RealityKit doesn't give a "Correct" visual bounds until sometime after the mesh actually becomes visible. 1214 | // If in `updateCollision` we generate the collision shapes directly from the Godot collision shapes, we don't need a random retry here. 1215 | DispatchQueue.main.asyncAfter(deadline: .now() + Double.random(in: 0.01...0.5)) { 1216 | updateCollision(entity: entity, count: count + 1) 1217 | } 1218 | } 1219 | 1220 | return 1221 | } 1222 | 1223 | // TODO: we could have more fine-graned input ray pickable shapes based on the Godot collision shapes. Currently we just put a box around the visual bounds. 1224 | var collision = CollisionComponent(shapes: [.generateBox(size: bounds.extents)]) 1225 | collision.filter = .init(group: [], mask: []) // disable for collision detection 1226 | entity.components.set(collision) 1227 | } 1228 | 1229 | public extension Entity { 1230 | /// Returns the SwiftGodot Object associated with this RealityKit Entity. 1231 | var godotNode: SwiftGodot.Node? { 1232 | components[GodotNode.self]?.node 1233 | } 1234 | } 1235 | 1236 | fileprivate struct AudioToProcess { 1237 | var instanceId: Int64 1238 | var state: Int64 1239 | var flags: Int64 1240 | var from_pos: Double 1241 | } 1242 | --------------------------------------------------------------------------------