├── .gitignore ├── Demo └── HandDemo │ ├── HandVectorDemo.xcodeproj │ ├── project.pbxproj │ ├── project.xcworkspace │ │ ├── contents.xcworkspacedata │ │ └── xcshareddata │ │ │ ├── IDEWorkspaceChecks.plist │ │ │ └── WorkspaceSettings.xcsettings │ └── xcshareddata │ │ └── xcschemes │ │ └── HandVectorDemo.xcscheme │ └── HandVectorDemo │ ├── Assets.xcassets │ ├── AppIcon.solidimagestack │ │ ├── Back.solidimagestacklayer │ │ │ ├── Content.imageset │ │ │ │ └── Contents.json │ │ │ └── Contents.json │ │ ├── Contents.json │ │ ├── Front.solidimagestacklayer │ │ │ ├── Content.imageset │ │ │ │ └── Contents.json │ │ │ └── Contents.json │ │ └── Middle.solidimagestacklayer │ │ │ ├── Content.imageset │ │ │ └── Contents.json │ │ │ └── Contents.json │ └── Contents.json │ ├── Extensions │ ├── Codable+Extensions.swift │ └── Entity+Extensions.swift │ ├── HandVectorApp.swift │ ├── HandVectorView.swift │ ├── HandViewModel.swift │ ├── Info.plist │ ├── Module.swift │ ├── Preview Content │ └── Preview Assets.xcassets │ │ └── Contents.json │ ├── RealityViews │ ├── FingerShapeImmersiveView.swift │ ├── MatchBuildinImmersiveView.swift │ └── RecordAndMatchImmersiveView.swift │ ├── Views │ ├── FingerShape.swift │ ├── MatchAllBuiltin.swift │ └── RecordAndMatch.swift │ ├── en.lproj │ └── en.lproj │ │ └── Localizable.strings │ └── zh-Hans.lproj │ └── zh-Hans.lproj │ └── Localizable.strings ├── LICENSE ├── Package.swift ├── README.md ├── README_CN.md ├── READMEv1 ├── README.md ├── README_CN.md └── Resources │ ├── 960x540mv.webp │ ├── HandVectorLogo.png │ ├── handVectorDemoMatchOK.gif │ ├── handVectorDemoRecordMatch.gif │ ├── handVectorTest.gif │ └── skygestures_demo1.gif ├── Resources ├── FingerShapFullCurl.png ├── FingerShapeBaseCurl.png ├── FingerShapeDifferenceCurl.png ├── FingerShapePinch.png ├── FingerShapeSpread.png ├── FingerShapeTipCurl.png ├── FingerShaper.gif ├── HandVectorFileStructure.png ├── HandVectorLogo.png ├── MatchAllBuiltin.gif ├── RecordAndMatch.gif ├── UntityXRHandsCoverImage.png └── handVectorTest.gif ├── Sources └── HandVector │ ├── BuiltinJson │ └── BuiltinHand.json │ ├── Extensions │ ├── Codable+Extensions.swift │ ├── HandAnchor+Extensions.swift │ └── HandSkeleton+Extensions.swift │ ├── HVFingerShape.swift │ ├── HVHandInfo+CosineSimilary.swift │ ├── HVHandInfo+Direction.swift │ ├── HVHandInfo.swift │ ├── HVHandJsonModel.swift │ ├── HVJointInfo.swift │ ├── HVJointOfFinger.swift │ ├── HandVector.swift │ ├── HandVectorManager.swift │ └── SimHands │ ├── Bonjour │ ├── BonjourSession.swift │ ├── Extensions │ │ └── MCPeerID+Extensions.swift │ ├── Peer.swift │ ├── ProgressWatcher.swift │ └── readme.md │ └── VisionSimHands.swift ├── Tests └── HandVectorTests │ └── HandVectorTests.swift └── macOS Helper ├── VisionSimHands.xcodeproj ├── project.pbxproj └── project.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ ├── IDEWorkspaceChecks.plist │ └── swiftpm │ └── Package.resolved └── VisionSimHands ├── AppDelegate.swift ├── Assets.xcassets ├── AccentColor.colorset │ └── Contents.json ├── AppIcon.appiconset │ └── Contents.json └── Contents.json ├── Base.lproj └── Main.storyboard ├── Info.plist ├── ViewController.swift ├── VisionSimHands.entitlements └── index.html /.gitignore: -------------------------------------------------------------------------------- 1 | # SPM defaults 2 | .swiftpm/ 3 | *.xcuserstate 4 | 5 | .DS_Store 6 | /.build 7 | /Packages 8 | xcuserdata/ 9 | DerivedData/ 10 | .swiftpm/configuration/registries.json 11 | .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata 12 | .netrc 13 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo.xcodeproj/xcshareddata/xcschemes/HandVectorDemo.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 9 | 10 | 16 | 22 | 23 | 24 | 25 | 26 | 32 | 33 | 44 | 46 | 52 | 53 | 54 | 55 | 61 | 63 | 69 | 70 | 71 | 72 | 74 | 75 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Back.solidimagestacklayer/Content.imageset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "reality", 5 | "scale" : "2x" 6 | } 7 | ], 8 | "info" : { 9 | "author" : "xcode", 10 | "version" : 1 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Back.solidimagestacklayer/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | }, 6 | "layers" : [ 7 | { 8 | "filename" : "Front.solidimagestacklayer" 9 | }, 10 | { 11 | "filename" : "Middle.solidimagestacklayer" 12 | }, 13 | { 14 | "filename" : "Back.solidimagestacklayer" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Front.solidimagestacklayer/Content.imageset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "reality", 5 | "scale" : "2x" 6 | } 7 | ], 8 | "info" : { 9 | "author" : "xcode", 10 | "version" : 1 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Front.solidimagestacklayer/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Middle.solidimagestacklayer/Content.imageset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "reality", 5 | "scale" : "2x" 6 | } 7 | ], 8 | "info" : { 9 | "author" : "xcode", 10 | "version" : 1 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/AppIcon.solidimagestack/Middle.solidimagestacklayer/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Extensions/Codable+Extensions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Codable+Extensions.swift 3 | // HandVector 4 | // 5 | // Created by 许同学 on 2023/11/30. 6 | // 7 | 8 | import Foundation 9 | 10 | 11 | extension Encodable { 12 | func toJson(encoding: String.Encoding = .utf8) -> String? { 13 | guard let data = try? JSONEncoder().encode(self) else { return nil } 14 | return String(data: data, encoding: encoding) 15 | } 16 | } 17 | 18 | 19 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Extensions/Entity+Extensions.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See the LICENSE.txt file for this sample’s licensing information. 3 | 4 | Abstract: 5 | Extensions and utilities. 6 | */ 7 | 8 | import SwiftUI 9 | import RealityKit 10 | 11 | extension Entity { 12 | /// Property for getting or setting an entity's `modelComponent`. 13 | public var modelComponent: ModelComponent? { 14 | get { components[ModelComponent.self] } 15 | set { components[ModelComponent.self] = newValue } 16 | } 17 | public var descendentsWithModelComponent: [Entity] { 18 | var descendents = [Entity]() 19 | 20 | for child in children { 21 | if child.components[ModelComponent.self] != nil { 22 | descendents.append(child) 23 | } 24 | descendents.append(contentsOf: child.descendentsWithModelComponent) 25 | } 26 | return descendents 27 | } 28 | } 29 | 30 | extension Entity { 31 | public subscript(parentMatching targetName: String) -> Entity? { 32 | if name.contains(targetName) { 33 | return self 34 | } 35 | 36 | guard let nextParent = parent else { 37 | return nil 38 | } 39 | 40 | return nextParent[parentMatching: targetName] 41 | } 42 | public func getParent(nameBeginsWith name: String) -> Entity? { 43 | if self.name.hasPrefix(name) { 44 | return self 45 | } 46 | guard let nextParent = parent else { 47 | return nil 48 | } 49 | 50 | return nextParent.getParent(nameBeginsWith: name) 51 | } 52 | public func getParent(withName name: String) -> Entity? { 53 | if self.name == name { 54 | return self 55 | } 56 | guard let nextParent = parent else { 57 | return nil 58 | } 59 | 60 | return nextParent.getParent(withName: name) 61 | } 62 | public subscript(descendentMatching targetName: String) -> Entity? { 63 | if name.contains(targetName) { 64 | return self 65 | } 66 | 67 | var match: Entity? = nil 68 | for child in children { 69 | match = child[descendentMatching: targetName] 70 | if let match = match { 71 | return match 72 | } 73 | } 74 | 75 | return match 76 | } 77 | public func getSelfOrDescendent(withName name: String) -> Entity? { 78 | if self.name == name { 79 | return self 80 | } 81 | var match: Entity? = nil 82 | for child in children { 83 | match = child.getSelfOrDescendent(withName: name) 84 | if match != nil { 85 | return match 86 | } 87 | } 88 | 89 | return match 90 | } 91 | public func forward(relativeTo referenceEntity: Entity?) -> SIMD3 { 92 | normalize(convert(direction: SIMD3(0, 0, +1), to: referenceEntity)) 93 | } 94 | 95 | public var forward: SIMD3 { 96 | forward(relativeTo: nil) 97 | } 98 | } 99 | 100 | extension SIMD4 { 101 | public var xyz: SIMD3 { 102 | self[SIMD3(0, 1, 2)] 103 | } 104 | } 105 | 106 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/HandVectorApp.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandVectorApp.swift 3 | // HandVector 4 | // 5 | // Created by 许同学 on 2023/9/17. 6 | // 7 | 8 | import SwiftUI 9 | 10 | @main 11 | struct HandVectorApp: App { 12 | @State private var model = HandViewModel() 13 | var body: some Scene { 14 | WindowGroup { 15 | HandVectorView() 16 | .environment(model) 17 | // .environment(\.locale, .init(identifier: "en")) 18 | // .environment(\.locale, .init(identifier: "zh-Hans")) 19 | } 20 | .windowResizability(.contentSize) 21 | .defaultSize(width: 1, height: 0.6, depth: 0.1, in: .meters) 22 | 23 | ImmersiveSpace(id: Module.matchAllBuiltinHands.immersiveId) { 24 | MatchBuildinImmersiveView() 25 | .environment(model) 26 | } 27 | .immersionStyle(selection: .constant(.mixed), in: .mixed) 28 | 29 | ImmersiveSpace(id: Module.recordAndMatchHand.immersiveId) { 30 | RecordAndMatchImmersiveView() 31 | .environment(model) 32 | } 33 | .immersionStyle(selection: .constant(.mixed), in: .mixed) 34 | 35 | ImmersiveSpace(id: Module.calculateFingerShape.immersiveId) { 36 | FingerShapeImmersiveView() 37 | .environment(model) 38 | } 39 | .immersionStyle(selection: .constant(.mixed), in: .mixed) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/HandVectorView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandVectorView.swift 3 | // HandVectorDemo 4 | // 5 | // Created by 许同学 on 2024/4/14. 6 | // 7 | 8 | import SwiftUI 9 | 10 | struct HandVectorView: View { 11 | @State private var selectedModule: Module = .matchAllBuiltinHands 12 | 13 | @Environment(HandViewModel.self) private var model 14 | @Environment(\.openImmersiveSpace) var openImmersiveSpace 15 | @Environment(\.dismissImmersiveSpace) var dismissImmersiveSpace 16 | 17 | var body: some View { 18 | NavigationSplitView { 19 | List(Module.allCases) { module in 20 | Button(action: { 21 | selectedModule = module 22 | }, label: { 23 | Text(module.name) 24 | }) 25 | .listRowBackground( 26 | RoundedRectangle(cornerRadius: 8) 27 | .background(Color.clear) 28 | .foregroundColor((module == selectedModule) ? Color.teal.opacity(0.3) : .clear) 29 | ) 30 | 31 | } 32 | .navigationTitle("Hand Vector Demo") 33 | } detail: { 34 | switch selectedModule { 35 | case .matchAllBuiltinHands: 36 | MatchAllBuiltin() 37 | .navigationTitle(selectedModule.name) 38 | case .recordAndMatchHand: 39 | RecordAndMatch() 40 | .navigationTitle(selectedModule.name) 41 | case .calculateFingerShape: 42 | FingerShape() 43 | .navigationTitle(selectedModule.name) 44 | } 45 | 46 | } 47 | .frame(minWidth: 800, minHeight: 500) 48 | .onChange(of: selectedModule) { _, newValue in 49 | Task { 50 | if model.turnOnImmersiveSpace { 51 | model.turnOnImmersiveSpace = false 52 | } 53 | } 54 | } 55 | .onChange(of: model.turnOnImmersiveSpace) { _, newValue in 56 | Task { 57 | if newValue { 58 | await openImmersiveSpace(id: selectedModule.immersiveId) 59 | } else { 60 | await dismissImmersiveSpace() 61 | model.reset() 62 | } 63 | } 64 | } 65 | } 66 | } 67 | 68 | #Preview { 69 | HandVectorView() 70 | } 71 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/HandViewModel.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandViewModel.swift 3 | // Demo 4 | // 5 | // Created by 许同学 on 2023/12/5. 6 | // 7 | 8 | 9 | import ARKit 10 | import SwiftUI 11 | import RealityKit 12 | import HandVector 13 | 14 | /// A model that contains up-to-date hand coordinate information. 15 | @Observable 16 | class HandViewModel: @unchecked Sendable { 17 | var turnOnImmersiveSpace = false 18 | 19 | var rootEntity: Entity? 20 | 21 | var latestHandTracking: HandVectorManager = .init(left: nil, right: nil) 22 | var recordHand: HVHandInfo? 23 | var averageAndEachLeftScores: (average: Float, eachFinger: [HVJointOfFinger: Float])? 24 | var averageAndEachRightScores: (average: Float, eachFinger: [HVJointOfFinger: Float])? 25 | 26 | var leftScores: [String: Float] = [:] 27 | var rightScores: [String: Float] = [:] 28 | 29 | var leftFingerShapes: [HVJointOfFinger: HVFingerShape] = [:] 30 | var rightFingerShapes: [HVJointOfFinger: HVFingerShape] = [:] 31 | 32 | private let session = ARKitSession() 33 | private let worldTracking = WorldTrackingProvider() 34 | private let handTracking = HandTrackingProvider() 35 | private let simHandProvider = SimulatorHandTrackingProvider() 36 | 37 | init() { 38 | latestHandTracking.isCollisionEnable = true 39 | } 40 | func clear() { 41 | rootEntity?.children.removeAll() 42 | latestHandTracking.left?.removeFromParent() 43 | latestHandTracking.right?.removeFromParent() 44 | } 45 | 46 | /// Resets game state information. 47 | func reset() { 48 | debugPrint(#function) 49 | 50 | leftScores = [:] 51 | rightScores = [:] 52 | 53 | averageAndEachLeftScores = nil 54 | averageAndEachRightScores = nil 55 | 56 | leftFingerShapes = [:] 57 | rightFingerShapes = [:] 58 | 59 | clear() 60 | } 61 | func matchAllBuiltinHands() { 62 | let builtinHands = HVHandInfo.builtinHandInfo 63 | builtinHands.forEach { (key, value) in 64 | leftScores[key] = latestHandTracking.leftHandVector?.similarity(of: .fiveFingers, to: value) 65 | rightScores[key] = latestHandTracking.rightHandVector?.similarity(of: .fiveFingers, to: value) 66 | } 67 | } 68 | func matchRecordHandAndFingers() { 69 | if let recordHand { 70 | averageAndEachLeftScores = latestHandTracking.leftHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: recordHand) 71 | averageAndEachRightScores = latestHandTracking.rightHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: recordHand) 72 | } 73 | } 74 | func calculateFingerShapes() { 75 | leftFingerShapes[.thumb] = latestHandTracking.leftHandVector?.calculateFingerShape(finger: .thumb) 76 | leftFingerShapes[.indexFinger] = latestHandTracking.leftHandVector?.calculateFingerShape(finger: .indexFinger) 77 | leftFingerShapes[.middleFinger] = latestHandTracking.leftHandVector?.calculateFingerShape(finger: .middleFinger) 78 | leftFingerShapes[.ringFinger] = latestHandTracking.leftHandVector?.calculateFingerShape(finger: .ringFinger) 79 | leftFingerShapes[.littleFinger] = latestHandTracking.leftHandVector?.calculateFingerShape(finger: .littleFinger) 80 | 81 | rightFingerShapes[.thumb] = latestHandTracking.rightHandVector?.calculateFingerShape(finger: .thumb) 82 | rightFingerShapes[.indexFinger] = latestHandTracking.rightHandVector?.calculateFingerShape(finger: .indexFinger) 83 | rightFingerShapes[.middleFinger] = latestHandTracking.rightHandVector?.calculateFingerShape(finger: .middleFinger) 84 | rightFingerShapes[.ringFinger] = latestHandTracking.rightHandVector?.calculateFingerShape(finger: .ringFinger) 85 | rightFingerShapes[.littleFinger] = latestHandTracking.rightHandVector?.calculateFingerShape(finger: .littleFinger) 86 | } 87 | func startHandTracking() async { 88 | do { 89 | if HandTrackingProvider.isSupported { 90 | print("ARKitSession starting.") 91 | try await session.run([handTracking]) 92 | } 93 | } catch { 94 | print("ARKitSession error:", error) 95 | } 96 | 97 | } 98 | 99 | func publishHandTrackingUpdates() async { 100 | for await update in handTracking.anchorUpdates { 101 | switch update.event { 102 | case .added, .updated: 103 | let anchor = update.anchor 104 | guard anchor.isTracked else { 105 | continue 106 | } 107 | let handInfo = latestHandTracking.generateHandInfo(from: anchor) 108 | if let handInfo { 109 | await latestHandTracking.updateHandSkeletonEntity(from: handInfo) 110 | if let left = latestHandTracking.left { 111 | await rootEntity?.addChild(left) 112 | } 113 | if let right = latestHandTracking.right { 114 | await rootEntity?.addChild(right) 115 | } 116 | } 117 | case .removed: 118 | let anchor = update.anchor 119 | latestHandTracking.removeHand(from: anchor) 120 | } 121 | } 122 | } 123 | 124 | func monitorSessionEvents() async { 125 | for await event in session.events { 126 | switch event { 127 | case .authorizationChanged(let type, let status): 128 | if type == .handTracking && status != .allowed { 129 | // Stop the game, ask the user to grant hand tracking authorization again in Settings. 130 | print("handTracking authorizationChanged \(status)") 131 | } 132 | default: 133 | print("Session event \(event)") 134 | } 135 | } 136 | } 137 | 138 | func publishSimHandTrackingUpdates() async { 139 | for await simHand in simHandProvider.simHands { 140 | if simHand.landmarks.isEmpty { continue } 141 | await latestHandTracking.updateHand(from: simHand) 142 | if let left = latestHandTracking.left { 143 | await rootEntity?.addChild(left) 144 | } 145 | if let right = latestHandTracking.right { 146 | await rootEntity?.addChild(right) 147 | } 148 | } 149 | } 150 | 151 | } 152 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | NSHandsTrackingUsageDescription 6 | Track your hands to detect a heart gesture. 7 | UIApplicationSceneManifest 8 | 9 | UIApplicationPreferredDefaultSceneSessionRole 10 | UIWindowSceneSessionRoleApplication 11 | UIApplicationSupportsMultipleScenes 12 | 13 | UISceneConfigurations 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Module.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See the LICENSE.txt file for this sample’s licensing information. 3 | 4 | Abstract: 5 | The modules that the app can present. 6 | */ 7 | 8 | 9 | import Foundation 10 | import SwiftUI 11 | 12 | /// A description of the modules that the app can present. 13 | enum Module: String, Identifiable, CaseIterable, Equatable { 14 | case matchAllBuiltinHands 15 | case recordAndMatchHand 16 | case calculateFingerShape 17 | 18 | var id: Self { self } 19 | var name: LocalizedStringKey { 20 | LocalizedStringKey(rawValue) 21 | } 22 | 23 | var immersiveId: String { 24 | self.rawValue + "ID" 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Preview Content/Preview Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/RealityViews/FingerShapeImmersiveView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // FingerShapeImmersiveView.swift 3 | // HandVectorDemo 4 | // 5 | // Created by 许同学 on 2024/8/26. 6 | // 7 | 8 | import SwiftUI 9 | import RealityKit 10 | import HandVector 11 | 12 | struct FingerShapeImmersiveView: View { 13 | @Environment(HandViewModel.self) private var model 14 | @State private var subscriptions = [EventSubscription]() 15 | @State private var updateCount: Int = 0 16 | var body: some View { 17 | RealityView { content in 18 | 19 | let entity = Entity() 20 | entity.name = "GameRoot" 21 | model.rootEntity = entity 22 | content.add(entity) 23 | 24 | subscriptions.append(content.subscribe(to: SceneEvents.Update.self, on: nil, { event in 25 | updateCount += 1 26 | // low down the update rate 27 | if updateCount % 15 == 0 { 28 | updateCount = 0 29 | model.calculateFingerShapes() 30 | } 31 | })) 32 | 33 | } update: { content in 34 | 35 | 36 | } 37 | .upperLimbVisibility(model.latestHandTracking.isSkeletonVisible ? .hidden : .automatic) 38 | .task { 39 | await model.startHandTracking() 40 | } 41 | .task { 42 | await model.publishHandTrackingUpdates() 43 | } 44 | .task { 45 | await model.monitorSessionEvents() 46 | } 47 | #if targetEnvironment(simulator) 48 | .task { 49 | await model.publishSimHandTrackingUpdates() 50 | } 51 | #endif 52 | } 53 | } 54 | 55 | #Preview { 56 | FingerShapeImmersiveView() 57 | } 58 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/RealityViews/MatchBuildinImmersiveView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // MatchBuildinImmersiveView.swift 3 | // FingerDance 4 | // 5 | // Created by 许同学 on 2024/1/8. 6 | // 7 | 8 | import SwiftUI 9 | import RealityKit 10 | import HandVector 11 | 12 | struct MatchBuildinImmersiveView: View { 13 | @Environment(HandViewModel.self) private var model 14 | @State private var subscriptions = [EventSubscription]() 15 | @State private var updateCount: Int = 0 16 | var body: some View { 17 | RealityView { content in 18 | 19 | let entity = Entity() 20 | entity.name = "GameRoot" 21 | model.rootEntity = entity 22 | content.add(entity) 23 | 24 | subscriptions.append(content.subscribe(to: SceneEvents.Update.self, on: nil, { event in 25 | updateCount += 1 26 | // low down the update rate 27 | if updateCount % 15 == 0 { 28 | updateCount = 0 29 | model.matchAllBuiltinHands() 30 | } 31 | })) 32 | 33 | 34 | } update: { content in 35 | 36 | 37 | } 38 | .upperLimbVisibility(model.latestHandTracking.isSkeletonVisible ? .hidden : .automatic) 39 | 40 | .task { 41 | await model.startHandTracking() 42 | } 43 | .task { 44 | await model.publishHandTrackingUpdates() 45 | } 46 | .task { 47 | await model.monitorSessionEvents() 48 | } 49 | #if targetEnvironment(simulator) 50 | .task { 51 | await model.publishSimHandTrackingUpdates() 52 | } 53 | #endif 54 | } 55 | } 56 | 57 | #Preview { 58 | MatchBuildinImmersiveView() 59 | } 60 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/RealityViews/RecordAndMatchImmersiveView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // RecordAndMatchImmersiveView.swift 3 | // HandVectorDemo 4 | // 5 | // Created by 许同学 on 2024/4/14. 6 | // 7 | 8 | import SwiftUI 9 | import RealityKit 10 | import HandVector 11 | 12 | struct RecordAndMatchImmersiveView: View { 13 | @Environment(HandViewModel.self) private var model 14 | @State private var subscriptions = [EventSubscription]() 15 | @State private var updateCount: Int = 0 16 | var body: some View { 17 | RealityView { content in 18 | 19 | let entity = Entity() 20 | entity.name = "GameRoot" 21 | model.rootEntity = entity 22 | content.add(entity) 23 | 24 | subscriptions.append(content.subscribe(to: SceneEvents.Update.self, on: nil, { event in 25 | updateCount += 1 26 | // low down the update rate 27 | if updateCount % 15 == 0 { 28 | updateCount = 0 29 | model.matchRecordHandAndFingers() 30 | } 31 | })) 32 | 33 | } update: { content in 34 | 35 | 36 | } 37 | .upperLimbVisibility(model.latestHandTracking.isSkeletonVisible ? .hidden : .automatic) 38 | .task { 39 | await model.startHandTracking() 40 | } 41 | .task { 42 | await model.publishHandTrackingUpdates() 43 | } 44 | .task { 45 | await model.monitorSessionEvents() 46 | } 47 | #if targetEnvironment(simulator) 48 | .task { 49 | await model.publishSimHandTrackingUpdates() 50 | } 51 | #endif 52 | } 53 | } 54 | 55 | #Preview { 56 | RecordAndMatchImmersiveView() 57 | } 58 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Views/FingerShape.swift: -------------------------------------------------------------------------------- 1 | // 2 | // FingerShape.swift 3 | // HandVectorDemo 4 | // 5 | // Created by 许同学 on 2024/8/26. 6 | // 7 | 8 | import SwiftUI 9 | 10 | struct FingerShape: View { 11 | struct TextProgressView: View { 12 | var text: String 13 | var value: Float 14 | var body: some View { 15 | HStack { 16 | Text(text) 17 | .font(.system(size: 16)) 18 | .frame(width: 80) 19 | // .border(.yellow, width: 1) 20 | 21 | ProgressView(value: abs(value), total: 1, label: { 22 | Text(value.formatted()) 23 | .font(.system(size: 10)) 24 | }) 25 | } 26 | // .border(.black, width: 1) 27 | } 28 | } 29 | @Environment(HandViewModel.self) private var model 30 | 31 | var body: some View { 32 | @Bindable var model = model 33 | HStack(spacing: 10) { 34 | VStack { 35 | let fingerShapes = model.leftFingerShapes 36 | Spacer() 37 | HStack { 38 | Text("thumb").frame(width: 65) 39 | VStack { 40 | TextProgressView(text: "fullCurl", value: fingerShapes[.thumb]?.fullCurl ?? 0) 41 | TextProgressView(text: "baseCurl", value: fingerShapes[.thumb]?.baseCurl ?? 0) 42 | TextProgressView(text: "tipCurl", value: fingerShapes[.thumb]?.tipCurl ?? 0) 43 | TextProgressView(text: "pinch", value: fingerShapes[.thumb]?.pinch ?? 0) 44 | TextProgressView(text: "spread", value: fingerShapes[.thumb]?.spread ?? 0) 45 | } 46 | } 47 | .background() 48 | Spacer() 49 | 50 | HStack { 51 | Text("index").frame(width: 65) 52 | VStack { 53 | TextProgressView(text: "fullCurl", value: fingerShapes[.indexFinger]?.fullCurl ?? 0) 54 | TextProgressView(text: "baseCurl", value: fingerShapes[.indexFinger]?.baseCurl ?? 0) 55 | TextProgressView(text: "tipCurl", value: fingerShapes[.indexFinger]?.tipCurl ?? 0) 56 | TextProgressView(text: "pinch", value: fingerShapes[.indexFinger]?.pinch ?? 0) 57 | TextProgressView(text: "spread", value: fingerShapes[.indexFinger]?.spread ?? 0) 58 | } 59 | } 60 | .background() 61 | Spacer() 62 | 63 | HStack { 64 | Text("middle").frame(width: 65) 65 | VStack { 66 | TextProgressView(text: "fullCurl", value: fingerShapes[.middleFinger]?.fullCurl ?? 0) 67 | TextProgressView(text: "baseCurl", value: fingerShapes[.middleFinger]?.baseCurl ?? 0) 68 | TextProgressView(text: "tipCurl", value: fingerShapes[.middleFinger]?.tipCurl ?? 0) 69 | TextProgressView(text: "pinch", value: fingerShapes[.middleFinger]?.pinch ?? 0) 70 | TextProgressView(text: "spread", value: fingerShapes[.middleFinger]?.spread ?? 0) 71 | } 72 | } 73 | .background() 74 | Spacer() 75 | 76 | HStack { 77 | Text("ring").frame(width: 65) 78 | VStack { 79 | TextProgressView(text: "fullCurl", value: fingerShapes[.ringFinger]?.fullCurl ?? 0) 80 | TextProgressView(text: "baseCurl", value: fingerShapes[.ringFinger]?.baseCurl ?? 0) 81 | TextProgressView(text: "tipCurl", value: fingerShapes[.ringFinger]?.tipCurl ?? 0) 82 | TextProgressView(text: "pinch", value: fingerShapes[.ringFinger]?.pinch ?? 0) 83 | TextProgressView(text: "spread", value: fingerShapes[.ringFinger]?.spread ?? 0) 84 | } 85 | } 86 | .background() 87 | Spacer() 88 | 89 | HStack { 90 | Text("little").frame(width: 65) 91 | VStack { 92 | TextProgressView(text: "fullCurl", value: fingerShapes[.littleFinger]?.fullCurl ?? 0) 93 | TextProgressView(text: "baseCurl", value: fingerShapes[.littleFinger]?.baseCurl ?? 0) 94 | TextProgressView(text: "tipCurl", value: fingerShapes[.littleFinger]?.tipCurl ?? 0) 95 | TextProgressView(text: "pinch", value: fingerShapes[.littleFinger]?.pinch ?? 0) 96 | TextProgressView(text: "spread", value: fingerShapes[.littleFinger]?.spread ?? 0) 97 | } 98 | } 99 | .background() 100 | Spacer() 101 | } 102 | .frame(width: 350) 103 | 104 | 105 | VStack(alignment: .center, spacing: 10) { 106 | Toggle("Start hand tracking and matching", isOn: $model.turnOnImmersiveSpace) 107 | .toggleStyle(ButtonToggleStyle()) 108 | .font(.system(size: 16, weight: .bold)) 109 | .padding(.bottom, 40) 110 | 111 | Toggle("Show hand skeleton", isOn: $model.latestHandTracking.isSkeletonVisible) 112 | .toggleStyle(ButtonToggleStyle()) 113 | .font(.system(size: 16, weight: .bold)) 114 | .disabled(!model.turnOnImmersiveSpace) 115 | .padding(.bottom, 40) 116 | 117 | 118 | } 119 | // .border(.red, width: 1.0) 120 | 121 | 122 | VStack { 123 | let fingerShapes = model.rightFingerShapes 124 | Spacer() 125 | HStack { 126 | Text("thumb").frame(width: 65) 127 | VStack { 128 | TextProgressView(text: "fullCurl", value: fingerShapes[.thumb]?.fullCurl ?? 0) 129 | TextProgressView(text: "baseCurl", value: fingerShapes[.thumb]?.baseCurl ?? 0) 130 | TextProgressView(text: "tipCurl", value: fingerShapes[.thumb]?.tipCurl ?? 0) 131 | TextProgressView(text: "pinch", value: fingerShapes[.thumb]?.pinch ?? 0) 132 | TextProgressView(text: "spread", value: fingerShapes[.thumb]?.spread ?? 0) 133 | } 134 | } 135 | .background() 136 | Spacer() 137 | 138 | HStack { 139 | Text("index").frame(width: 65) 140 | VStack { 141 | TextProgressView(text: "fullCurl", value: fingerShapes[.indexFinger]?.fullCurl ?? 0) 142 | TextProgressView(text: "baseCurl", value: fingerShapes[.indexFinger]?.baseCurl ?? 0) 143 | TextProgressView(text: "tipCurl", value: fingerShapes[.indexFinger]?.tipCurl ?? 0) 144 | TextProgressView(text: "pinch", value: fingerShapes[.indexFinger]?.pinch ?? 0) 145 | TextProgressView(text: "spread", value: fingerShapes[.indexFinger]?.spread ?? 0) 146 | } 147 | } 148 | .background() 149 | Spacer() 150 | 151 | HStack { 152 | Text("middle").frame(width: 65) 153 | VStack { 154 | TextProgressView(text: "fullCurl", value: fingerShapes[.middleFinger]?.fullCurl ?? 0) 155 | TextProgressView(text: "baseCurl", value: fingerShapes[.middleFinger]?.baseCurl ?? 0) 156 | TextProgressView(text: "tipCurl", value: fingerShapes[.middleFinger]?.tipCurl ?? 0) 157 | TextProgressView(text: "pinch", value: fingerShapes[.middleFinger]?.pinch ?? 0) 158 | TextProgressView(text: "spread", value: fingerShapes[.middleFinger]?.spread ?? 0) 159 | } 160 | } 161 | .background() 162 | Spacer() 163 | 164 | HStack { 165 | Text("ring").frame(width: 65) 166 | VStack { 167 | TextProgressView(text: "fullCurl", value: fingerShapes[.ringFinger]?.fullCurl ?? 0) 168 | TextProgressView(text: "baseCurl", value: fingerShapes[.ringFinger]?.baseCurl ?? 0) 169 | TextProgressView(text: "tipCurl", value: fingerShapes[.ringFinger]?.tipCurl ?? 0) 170 | TextProgressView(text: "pinch", value: fingerShapes[.ringFinger]?.pinch ?? 0) 171 | TextProgressView(text: "spread", value: fingerShapes[.ringFinger]?.spread ?? 0) 172 | } 173 | } 174 | .background() 175 | Spacer() 176 | 177 | HStack { 178 | Text("little").frame(width: 65) 179 | VStack { 180 | TextProgressView(text: "fullCurl", value: fingerShapes[.littleFinger]?.fullCurl ?? 0) 181 | TextProgressView(text: "baseCurl", value: fingerShapes[.littleFinger]?.baseCurl ?? 0) 182 | TextProgressView(text: "tipCurl", value: fingerShapes[.littleFinger]?.tipCurl ?? 0) 183 | TextProgressView(text: "pinch", value: fingerShapes[.littleFinger]?.pinch ?? 0) 184 | TextProgressView(text: "spread", value: fingerShapes[.littleFinger]?.spread ?? 0) 185 | } 186 | } 187 | .background() 188 | Spacer() 189 | } 190 | .frame(width: 350) 191 | } 192 | } 193 | } 194 | 195 | #Preview { 196 | FingerShape() 197 | } 198 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Views/MatchAllBuiltin.swift: -------------------------------------------------------------------------------- 1 | // 2 | // SwiftUIView.swift 3 | // FingerDance 4 | // 5 | // Created by 许同学 on 2024/1/11. 6 | // 7 | 8 | import SwiftUI 9 | 10 | 11 | struct MatchAllBuiltin: View { 12 | struct TextProgressView: View { 13 | var text: String 14 | var value: Float 15 | var body: some View { 16 | HStack { 17 | Text(text).font(.system(size: 50)) 18 | 19 | ProgressView(value: abs(value), total: 1, label: { 20 | Text(value.formatted()) 21 | }) 22 | } 23 | .padding(.horizontal) 24 | } 25 | } 26 | @Environment(HandViewModel.self) private var model 27 | 28 | var body: some View { 29 | @Bindable var model = model 30 | HStack { 31 | VStack { 32 | TextProgressView(text: "👆", value: model.leftScores["👆"] ?? 0) 33 | TextProgressView(text: "✌️", value: model.leftScores["✌️"] ?? 0) 34 | TextProgressView(text: "✋", value: model.leftScores["✋"] ?? 0) 35 | TextProgressView(text: "👌", value: model.leftScores["👌"] ?? 0) 36 | 37 | TextProgressView(text: "✊", value: model.leftScores["✊"] ?? 0) 38 | TextProgressView(text: "🤘", value: model.leftScores["🤘"] ?? 0) 39 | TextProgressView(text: "🤙", value: model.leftScores["🤙"] ?? 0) 40 | TextProgressView(text: "🫱🏿‍🫲🏻", value: model.leftScores["🫱🏿‍🫲🏻"] ?? 0) 41 | } 42 | .frame(width: 300) 43 | 44 | 45 | VStack(alignment: .center, spacing: 10) { 46 | Toggle("Start hand tracking and matching", isOn: $model.turnOnImmersiveSpace) 47 | .toggleStyle(ButtonToggleStyle()) 48 | .font(.system(size: 16, weight: .bold)) 49 | .padding(.bottom, 40) 50 | 51 | Toggle("Show hand skeleton", isOn: $model.latestHandTracking.isSkeletonVisible) 52 | .toggleStyle(ButtonToggleStyle()) 53 | .font(.system(size: 16, weight: .bold)) 54 | .disabled(!model.turnOnImmersiveSpace) 55 | .padding(.bottom, 40) 56 | 57 | VStack { 58 | Text("👆✌️✋👌\n✊🤘🤙🫱🏿‍🫲🏻") 59 | .font(.system(size: 60)) 60 | .padding(.bottom, 20) 61 | 62 | Text("Try to make the same gesture,\nyou can use left or right hand") 63 | .multilineTextAlignment(.center) 64 | .font(.system(size: 20, weight: .bold)) 65 | } 66 | 67 | } 68 | // .border(.red, width: 1.0) 69 | 70 | VStack { 71 | TextProgressView(text: "👆", value: model.rightScores["👆"] ?? 0) 72 | TextProgressView(text: "✌️", value: model.rightScores["✌️"] ?? 0) 73 | TextProgressView(text: "✋", value: model.rightScores["✋"] ?? 0) 74 | TextProgressView(text: "👌", value: model.rightScores["👌"] ?? 0) 75 | 76 | TextProgressView(text: "✊", value: model.rightScores["✊"] ?? 0) 77 | TextProgressView(text: "🤘", value: model.rightScores["🤘"] ?? 0) 78 | TextProgressView(text: "🤙", value: model.rightScores["🤙"] ?? 0) 79 | TextProgressView(text: "🫱🏿‍🫲🏻", value: model.rightScores["🫱🏿‍🫲🏻"] ?? 0) 80 | } 81 | .frame(width: 300) 82 | } 83 | } 84 | 85 | } 86 | 87 | #Preview { 88 | MatchAllBuiltin() 89 | .environment(HandViewModel()) 90 | .glassBackgroundEffect( 91 | in: RoundedRectangle( 92 | cornerRadius: 32, 93 | style: .continuous 94 | ) 95 | ) 96 | } 97 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/Views/RecordAndMatch.swift: -------------------------------------------------------------------------------- 1 | // 2 | // RecordAndMatch.swift 3 | // HandVectorDemo 4 | // 5 | // Created by 许同学 on 2024/4/14. 6 | // 7 | 8 | import SwiftUI 9 | import HandVector 10 | 11 | struct RecordAndMatch: View { 12 | struct TextProgressView: View { 13 | var text: String 14 | var value: Float 15 | var body: some View { 16 | HStack { 17 | Text(text) 18 | .font(.system(size: 20)) 19 | .frame(width: 120) 20 | 21 | ProgressView(value: abs(value), total: 1, label: { 22 | Text(value.formatted()) 23 | }) 24 | } 25 | .padding() 26 | } 27 | } 28 | @Environment(HandViewModel.self) private var model 29 | @State private var recordIndex = 0 30 | 31 | @State private var timer = Timer.publish(every: 1, on: .main, in: .common).autoconnect() 32 | @State private var countDown = -1 33 | 34 | @State private var jsonString: String? 35 | @State private var showJsonSheet: Bool = false 36 | @State private var showPasteAlert: Bool = false 37 | 38 | var progressValue: Float { 39 | min(1, max(0, Float(countDown) / 3.0 + 0.001)) 40 | } 41 | 42 | var body: some View { 43 | @Bindable var model = model 44 | HStack { 45 | VStack { 46 | let leftScores = model.averageAndEachLeftScores 47 | TextProgressView(text: "thumb", value: leftScores?.eachFinger[.thumb] ?? 0) 48 | TextProgressView(text: "indexFinger", value: leftScores?.eachFinger[.indexFinger] ?? 0) 49 | TextProgressView(text: "middleFinger", value: leftScores?.eachFinger[.middleFinger] ?? 0) 50 | TextProgressView(text: "ringFinger", value: leftScores?.eachFinger[.ringFinger] ?? 0) 51 | TextProgressView(text: "littleFinger", value: leftScores?.eachFinger[.littleFinger] ?? 0) 52 | } 53 | .frame(width: 350) 54 | 55 | 56 | VStack(alignment: .center, spacing: 10) { 57 | Toggle("Start hand tracking and matching", isOn: $model.turnOnImmersiveSpace) 58 | .toggleStyle(ButtonToggleStyle()) 59 | .font(.system(size: 16, weight: .bold)) 60 | .padding(.bottom, 40) 61 | 62 | Toggle("Show hand skeleton", isOn: $model.latestHandTracking.isSkeletonVisible) 63 | .toggleStyle(ButtonToggleStyle()) 64 | .font(.system(size: 16, weight: .bold)) 65 | .disabled(!model.turnOnImmersiveSpace) 66 | .padding(.bottom, 40) 67 | 68 | 69 | 70 | Group { 71 | Picker("Choose Left or Right hand to recrod", selection: $recordIndex) { 72 | Text("Left Hand").tag(0) 73 | Text("Right Hand").tag(1) 74 | } 75 | .pickerStyle(.segmented) 76 | .padding(.bottom, 20) 77 | .frame(width: 300) 78 | 79 | 80 | Button { 81 | countDown = 3 82 | } label: { 83 | Text("Ready to record") 84 | .font(.system(size: 16, weight: .bold)) 85 | } 86 | .disabled(countDown > -1 || !model.turnOnImmersiveSpace) 87 | 88 | 89 | Text(verbatim: countDown < 0 ? "..." : "\(countDown)") 90 | .animation(.none, value: progressValue) 91 | .font(.system(size: 64)) 92 | .bold() 93 | .padding(.bottom, 30) 94 | } 95 | // .border(.green, width: 1.0) 96 | 97 | 98 | Group { 99 | HStack { 100 | 101 | Text("left score:\(model.averageAndEachLeftScores?.average.formatted(.number.precision(.fractionLength(4))) ?? "0")") 102 | .frame(width: 150) 103 | Text("right score:\(model.averageAndEachRightScores?.average.formatted(.number.precision(.fractionLength(4))) ?? "0")") 104 | .frame(width: 150) 105 | } 106 | .padding(.bottom, 30) 107 | 108 | Button { 109 | showJsonSheet = true 110 | } label: { 111 | Text("Check recorded json string") 112 | .font(.system(size: 16, weight: .bold)) 113 | } 114 | .disabled(jsonString?.isEmpty ?? true) 115 | .sheet(isPresented: $showJsonSheet) { 116 | VStack { 117 | ScrollView { 118 | Text(verbatim: jsonString ?? "") 119 | .padding() 120 | } 121 | 122 | HStack(spacing: 10) { 123 | Button(action: { 124 | showJsonSheet = false 125 | }) { 126 | Text("OK") 127 | Image(systemName: "xmark.circle") 128 | } 129 | 130 | Button(action: { 131 | UIPasteboard.general.string = jsonString 132 | showPasteAlert = true 133 | }) { 134 | Text("Copy") 135 | Image(systemName: "doc.on.doc") 136 | } 137 | .alert( 138 | "Copied to clipboard", 139 | isPresented: $showPasteAlert 140 | ) { 141 | Button("OK") { 142 | // Handle the acknowledgement. 143 | } 144 | } message: { 145 | // Text("Please paste to anywhere you want.") 146 | } 147 | 148 | } 149 | .padding(.vertical) 150 | } 151 | .frame(minHeight: 600) 152 | } 153 | 154 | } 155 | .font(.system(size: 16, weight: .bold)) 156 | // .border(.green, width: 1.0) 157 | } 158 | // .border(.red, width: 1.0) 159 | 160 | 161 | VStack { 162 | let rightScores = model.averageAndEachRightScores 163 | TextProgressView(text: "thumb", value: rightScores?.eachFinger[.thumb] ?? 0) 164 | TextProgressView(text: "indexFinger", value: rightScores?.eachFinger[.indexFinger] ?? 0) 165 | TextProgressView(text: "middleFinger", value: rightScores?.eachFinger[.middleFinger] ?? 0) 166 | TextProgressView(text: "ringFinger", value: rightScores?.eachFinger[.ringFinger] ?? 0) 167 | TextProgressView(text: "littleFinger", value: rightScores?.eachFinger[.littleFinger] ?? 0) 168 | } 169 | .frame(width: 350) 170 | } 171 | .onReceive(timer) { _ in 172 | if !model.turnOnImmersiveSpace { 173 | return 174 | } 175 | if countDown > 0 { 176 | countDown -= 1 177 | } else if countDown == 0 { 178 | countDown = -1 179 | switch recordIndex { 180 | case 0: 181 | if let left = model.latestHandTracking.leftHandVector { 182 | let para = HVHandJsonModel.generateJsonModel(name: "left", handVector: left) 183 | model.recordHand = left 184 | jsonString = para.toJson() 185 | } 186 | case 1: 187 | if let right = model.latestHandTracking.rightHandVector { 188 | let para = HVHandJsonModel.generateJsonModel(name: "right", handVector: right) 189 | model.recordHand = right 190 | jsonString = para.toJson() 191 | } 192 | 193 | default: 194 | break 195 | } 196 | 197 | } 198 | } 199 | .onAppear { 200 | if let hand = model.recordHand { 201 | let para = HVHandJsonModel.generateJsonModel(name: hand.chirality.codableName.rawValue, handVector: hand) 202 | jsonString = para.toJson() 203 | } 204 | } 205 | } 206 | } 207 | 208 | #Preview { 209 | RecordAndMatch() 210 | } 211 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/en.lproj/en.lproj/Localizable.strings: -------------------------------------------------------------------------------- 1 | /* 2 | Localizable.strings 3 | HandVectorDemo 4 | 5 | Created by 许同学 on 2024/3/17. 6 | 7 | */ 8 | "matchAllBuiltinHands" = "Match all builtin Hand gestures"; 9 | "recordAndMatchHand" = "Record new gesture and match it"; 10 | "calculateFingerShape" = "Calculate Curl and Spread of finger"; 11 | -------------------------------------------------------------------------------- /Demo/HandDemo/HandVectorDemo/zh-Hans.lproj/zh-Hans.lproj/Localizable.strings: -------------------------------------------------------------------------------- 1 | /* 2 | Localizable.strings 3 | HandVectorDemo 4 | 5 | Created by 许同学 on 2024/3/17. 6 | 7 | */ 8 | "matchAllBuiltinHands" = "匹配全部预置的手势"; 9 | "recordAndMatchHand" = "录制并匹配新的手势"; 10 | "calculateFingerShape" = "计算每根手指的弯曲程度和分散程序"; 11 | 12 | "Start hand tracking and matching" = "开启手势追踪与匹配"; 13 | "Show hand skeleton" = "显示手部关节"; 14 | "Try to make the same gesture,\nyou can use left or right hand" = "试着做出相同的手势\n左手或右手都可以"; 15 | "left score:%@" = "左手得分:%@"; 16 | "right score:%@" = "右手得分:%@"; 17 | 18 | "Start hand tracking" = "开启手势追踪"; 19 | "Choose Left or Right hand to recrod" = "选择录制左手或右手"; 20 | "Left Hand" = "左手"; 21 | "Right Hand" = "右手"; 22 | "Ready to record" = "准备开始录制"; 23 | "Check recorded json string" = "查看录制好的 json 字符串"; 24 | "Copy" = "复制"; 25 | "Copied to clipboard" = "已复制到剪贴板"; 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-present Yannick Loriot 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "HandVector", 8 | platforms: [.visionOS(.v1)], 9 | products: [ 10 | // Products define the executables and libraries a package produces, making them visible to other packages. 11 | .library( 12 | name: "HandVector", 13 | targets: ["HandVector"]), 14 | ], 15 | dependencies: [ 16 | // Dependencies declare other packages that this package depends on. 17 | // .package(url: /* package url */, from: "1.0.0"), 18 | ], 19 | targets: [ 20 | // Targets are the basic building blocks of a package, defining a module or a test suite. 21 | // Targets can depend on other targets in this package and products from dependencies. 22 | .target( 23 | name: "HandVector", 24 | dependencies: [], 25 | resources: [.process("BuiltinJson")] 26 | ) 27 | ] 28 | ) 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | HandVector Logo 3 |

4 |

5 | Swift Package Manager compatible 6 | Swift 5.9 7 | Swift 5.9 8 |

9 | 10 | [中文版本](./README_CN.md) 11 | 12 | [old version](./READMEv1/README.md) 13 | 14 | 15 | 16 | 17 | **HandVector** calculates the similarity between different static gestures on visionOS and comes with a macOS utility class that allows you to use gesture tracking in the visionOS simulator as well. 18 | 19 | HandVector version 2.0 is a major update, bringing the improved **Cosine Similarity** and the **FingerShape** feature for easier customization. 20 | 21 | > Note: HandVector 2.0 has significant API changes and is not compatible with older versions. 22 | 23 |

24 | Requirements • Usage • Installation • Contribution • Contact • License 25 |

26 | 27 | ## Requirements 28 | 29 | - visionOS 1.0+ 30 | - Xcode 15.2+ 31 | - Swift 5.9+ 32 | 33 | ## Usage 34 | 35 | `HandVector 2.0` supports two kinds of gesture matching methods, which differ in their calculation principles and are suitable for different scenarios. They can also be mixed used together in a project: 36 | 37 | * **Cosine Similarity**: This method matches each joint of the specified fingers precisely, using the matrix information of each joint relative to its parent joint, resulting in high accuracy. Advantages: High precision, applicable to fingers and wrists; Disadvantages: Poor interpretability, difficult to adjust the range. 38 | * **FingerShape**: Referencing Unity's [XRHands](https://docs.unity3d.com/Packages/com.unity.xr.hands@1.5/manual/index.html) framework, this method simplifies the finger shape into five parameters: `baseCurl` (curl at the base of the finger), `tipCurl` (curl at the tip of the finger), `fullCurl` (overall curl of the finger), `pinch` (distance of pinching with the thumb), and `spread` (degree of separation from the adjacent outer finger). Advantages: The values are easy to understand and convenient to control and adjust; Disadvantages: Does not fully utilize joint pose information, thus not as precise, and is only applicable to five fingers. 39 | 40 | 41 | This upgrade also includes a reorganization of the file structure, with some classes and structures being renamed for clearer functionality. Therefore, it is not compatible with the previous major version API. 42 | 43 | ![HandVectorFileStructure](./Resources/HandVectorFileStructure.png) 44 | 45 | ### 1. Cosine Similarity Gesture Matching 46 | `HandVector` supports matching built-in gestures as well as recording and saving custom gestures for later use. Currently, there are 8 built-in gestures: 👆✌️✋👌✊🤘🤙🫱🏿‍🫲🏻 47 | > 🫱🏿‍🫲🏻: Grab, grasp 48 | #### a. Matching Built-in Gestures 49 | ![MatchAllBuiltin](./Resources/MatchAllBuiltin.gif) 50 | 51 | ```swift 52 | 53 | import HandVector 54 | 55 | 56 | 57 | //Get current Hand info from `HandTrackingProvider` , and convert to `HVHandInfo` 58 | 59 | for await update in handTracking.anchorUpdates { 60 | 61 | switch update.event { 62 | 63 | case .added, .updated: 64 | 65 | let anchor = update.anchor 66 | 67 | guard anchor.isTracked else { continue } 68 | 69 | let handInfo = latestHandTracking.generateHandInfo(from: anchor) 70 | 71 | case .removed: 72 | 73 | ... 74 | 75 | } 76 | 77 | } 78 | 79 | 80 | 81 | //Load built-in gesture from json file 82 | 83 | let builtinHands = HVHandInfo.builtinHandInfo 84 | 85 | //Calculate the similarity with the built-in gestures, `.fiveFingers` indicates matching only the 5 fingers, ignoring the wrist and palm. 86 | 87 | builtinHands.forEach { (key, value) in 88 | 89 | leftScores[key] = latestHandTracking.leftHandVector?.similarity(of: .fiveFingers, to: value) 90 | 91 | rightScores[key] = latestHandTracking.rightHandVector?.similarity(of: .fiveFingers, to: value) 92 | 93 | } 94 | 95 | ``` 96 | 97 | the score should be in `[-1.0,1.0]`, `1.0` means fully matched and both are left or right hands, `-1.0 `means fully matched but one is left hand, another is right hand, and `0` means not matched. 98 | 99 | #### b. Record custom gesture and match it 100 | 101 | 102 | 103 | ![RecordAndMatch](./Resources/RecordAndMatch.gif) 104 | 105 | 106 | 107 | Record a custom gesture and save it as a JSON string using `HVHandJsonModel`: 108 | 109 | 110 | 111 | ```swift 112 | 113 | if let left = model.latestHandTracking.leftHandVector { 114 | 115 | let para = HVHandJsonModel.generateJsonModel(name: "YourHand", handVector: left) 116 | 117 | jsonString = para.toJson() 118 | 119 | //Save jsonString to disk or network 120 | 121 | ... 122 | 123 | } 124 | 125 | ``` 126 | 127 | 128 | 129 | Next, convert the saved JSON string into the `HVHandInfo` type for gesture matching: 130 | 131 | 132 | 133 | ```swift 134 | 135 | //Convert from JSON string 136 | 137 | let handInfo = jsonStr.toModel(HVHandJsonModel.self)!.convertToHVHandInfo() 138 | 139 | //Load JSON file from disk, and convert 140 | 141 | let handInfo = HVHandJsonModel.loadHandJsonModel(fileName: "YourJsonFileName")!.convertToHVHandInfo() 142 | 143 | 144 | 145 | //Using the `HVHandInfo` type for gesture matching allows you to calculate the similarity for each finger individually. 146 | 147 | if let handInfo { 148 | 149 | averageAndEachLeftScores = latestHandTracking.leftHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: recordHand) 150 | 151 | averageAndEachRightScores = latestHandTracking.rightHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: recordHand) 152 | 153 | } 154 | 155 | 156 | 157 | ``` 158 | 159 | 160 | 161 | ### 2.Finger Shape Parameter 162 | 163 | 164 | 165 | ![XRHandsCoverImage](./Resources//UntityXRHandsCoverImage.png) 166 | 167 | 168 | 169 | This method draws significant reference from the well-known XR gesture framework in Unity: [XRHands](https://docs.unity3d.com/Packages/com.unity.xr.hands@1.5/manual/index.html). 170 | 171 | 172 | 173 | ![FingerShaper](./Resources/FingerShaper.gif) 174 | 175 | 176 | 177 | The definitions of the related parameters are similar: 178 | * **baseCurl**: The degree of curl at the root joint of the finger. For the thumb, it is the `IntermediateBase` joint, and for the other fingers, it is the `Knuckle` joint, with a range of 0 to 1. 179 | 180 | ![FingerShapeBaseCurl](./Resources/FingerShapeBaseCurl.png) 181 | 182 | 183 | 184 | * **tipCurl**：The degree of curl at the upper joint of the finger. For the thumb, it is the `IntermediateTip` joint, and for the other fingers, it is the average value of the `IntermediateBase` and `IntermediateTip` joints, with a range of 0 to 1. 185 | 186 | ![FingerShapeTipCurl](./Resources/FingerShapeTipCurl.png) 187 | 188 | * **fullCurl**：The average value of baseCurl and tipCurl, with a range of 0 to 1. 189 | 190 | ![FingerShapFullCurl](./Resources/FingerShapFullCurl.png) 191 | 192 | * **pinch**：The distance from the tip of the thumb, with a range of 0 to 1. For the thumb, this parameter is `nil`. 193 | 194 | ![FingerShapePinch](./Resources/FingerShapePinch.png) 195 | 196 | * **spread**：Only the horizontal spread angle is calculated, with a range of 0 to 1. For the little finger, this parameter is `nil`. 197 | 198 | ![FingerShapeSpread](./Resources/FingerShapeSpread.png) 199 | 200 | Regarding the differences between the three types of curl degrees, you can refer to the following image: 201 | 202 | ![FingerShapeDifferenceCurl](./Resources/FingerShapeDifferenceCurl.png) 203 | 204 | ### 3. Test hand gesture on Mac simulator 205 | 206 | The test method of`HandVector` is inspired by [VisionOS Simulator hands](https://github.com/BenLumenDigital/VisionOS-SimHands), it allow you to test hand tracking on visionOS simulator: 207 | 208 | It uses 2 things: 209 | 210 | 1. A macOS helper app, with a bonjour service 211 | 2. A Swift class for your VisionOS project which connects to the bonjour service (It comes with this package, and automatically receives and converts to the corresponding gesture; HandVector 2.0 version has updated mathematical "black magic" to achieve the new matching algorithm.) 212 | 213 | #### macOS Helper App 214 | 215 | The helper app uses Google MediaPipes for 3D hand tracking. This is a very basic setup - it uses a WKWebView to run the Google sample code, and that passed the hand data as JSON into native Swift. 216 | 217 | The Swift code then spits out the JSON over a Bonjour service. 218 | 219 | > If hand tracking can't start for a long time(Start button still can't be pressed), please check your network to google MediaPipes. 220 | 221 | ![](./Resources/handVectorTest.gif) 222 | 223 | ### And many more... 224 | 225 | To go further, take a look at the documentation and the demo project. 226 | 227 | *Note: All contributions are welcome* 228 | 229 | ## Installation 230 | 231 | #### Swift Package Manager 232 | 233 | To integrate using Apple's Swift package manager, without Xcode integration, add the following as a dependency to your `Package.swift`: 234 | 235 | ``` 236 | .package(url: "https://github.com/XanderXu/HandVector.git", .upToNextMajor(from: "2.0.0")) 237 | ``` 238 | 239 | #### Manually 240 | 241 | [Download](https://github.com/XanderXu/HandVector/archive/master.zip) the project and copy the `HandVector` folder into your project to use it. 242 | 243 | ## Contribution 244 | 245 | Contributions are welcomed and encouraged *♡*. 246 | 247 | ## Contact 248 | 249 | Xander: API 搬运工 250 | 251 | * [https://twitter.com/XanderARKit](https://twitter.com/XanderARKit) 252 | * [https://github.com/XanderXu](https://github.com/XanderXu) 253 | 254 | - [https://juejin.cn/user/2629687543092056](https://juejin.cn/user/2629687543092056) 255 | 256 | 257 | 258 | ## License 259 | 260 | HandVector is released under an MIT license. See [LICENSE](./LICENSE) for more information. 261 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 |

2 | HandVector Logo 3 |

4 |

5 | Swift Package Manager compatible 6 | Swift 5.9 7 | Swift 5.9 8 |

9 | 10 | [English](./README.md) 11 | 12 | [旧版本](./READMEv1/README_CN.md) 13 | 14 | **HandVector** 在 visionOS 上计算不同静态手势之间的相似度，并且带有一个 macOS 的工具类能让你在 visionOS 模拟器上也能使用手势追踪功能。 15 | 16 | HandVector 2.0 版本是一个大更新，带来更好的 **余弦相似度Cosine Similarity** 匹配效果和 **手指形状参数FingerShape** 功能，更方便自定义使用。 17 | 18 | > 注意：HandVector 2.0 有重大 API 改动，与旧版本不兼容。 19 | 20 |

21 | 环境要求 • 用法 • 安装 • 贡献 • 联系方式 • 许可证 22 |

23 | 24 | 25 | ## 环境要求 26 | 27 | - visionOS 1.0+ 28 | - Xcode 15.2+ 29 | - Swift 5.9+ 30 | 31 | ## 用法 32 | 33 | `HandVector 2.0` 支持两种手势匹配方法，它们计算理念不同，适用不同情况，也可一起混合使用： 34 | 35 | * **余弦相似度Cosine Similarity**：完全匹配指定手指的每个关节，使用每个关节点相对于父关节的矩阵信息，精准度高。优点：精度高，可适用手指与手腕；缺点：可解释性差，难以调整范围。 36 | * **手指形状参数FingerShape**：参考 Unity 的 [XRHands](https://docs.unity3d.com/Packages/com.unity.xr.hands@1.5/manual/index.html) 框架，将手指形状化简为： `指根卷曲度 baseCurl`、 `指尖卷曲度 tipCurl`、 `整体卷曲度 fullCurl`、 `（与拇指）捏合度 pinch`、 `（与外侧相邻手指）分离度 spread` 5 个参数。优点：数值方便理解，方便控制与调整；缺点：未充分利用关节位姿信息，精度不够高，且只适用 5 根手指。 37 | 38 | 本次升级也同时整理了文件结构，重命名了部分类和结构体，作用更清晰，所以与之间大版本 API 不兼容。 39 | 40 | ![HandVectorFileStructure](./Resources/HandVectorFileStructure.png) 41 | 42 | ### 1.余弦相似度手势匹配 43 | 44 | `HandVector` 支持匹配内置的手势，也支持自行录制保存自定义手势。目前内置的手势有 8 种：👆✌️✋👌✊🤘🤙🫱🏿‍🫲🏻 45 | 46 | > 🫱🏿‍🫲🏻：握、抓住 47 | 48 | #### a.匹配内置的手势 49 | 50 | ![MatchAllBuiltin](./Resources/MatchAllBuiltin.gif) 51 | 52 | 追踪双手关节的姿态，并与内置的手势相比较，得出它们的相似度。 53 | 54 | ```swift 55 | import HandVector 56 | 57 | //从 HandTrackingProvider 中获取当前手部信息，并转换为 `HVHandInfo` 58 | for await update in handTracking.anchorUpdates { 59 | switch update.event { 60 | case .added, .updated: 61 | let anchor = update.anchor 62 | guard anchor.isTracked else { continue } 63 | let handInfo = latestHandTracking.generateHandInfo(from: anchor) 64 | case .removed: 65 | ... 66 | } 67 | } 68 | 69 | //从 json 文件中加载内置的手势 70 | let builtinHands = HVHandInfo.builtinHandInfo 71 | //计算与内置手势的相似度,`.fiveFingers`表示只匹配 5 根手指，忽略手腕和手掌 72 | builtinHands.forEach { (key, value) in 73 | leftScores[key] = latestHandTracking.leftHandVector?.similarity(of: .fiveFingers, to: value) 74 | rightScores[key] = latestHandTracking.rightHandVector?.similarity(of: .fiveFingers, to: value) 75 | } 76 | ``` 77 | 78 | 相似度得分在 `[-1.0,1.0]` 之间， `1.0` 含义为手势完全匹配并且左右手也匹配， `-1.0 ` 含义为手势完全匹配但一个是左手一个是右手， `0` 含义为完全不匹配。 79 | 80 | #### b. 录制自定义的新手势并匹配它 81 | 82 | ![RecordAndMatch](./Resources/RecordAndMatch.gif) 83 | 84 | 录制自定义手势，并利用 `HVHandJsonModel` 保存为 JSON 字符串: 85 | 86 | ```swift 87 | if let left = model.latestHandTracking.leftHandVector { 88 | let para = HVHandJsonModel.generateJsonModel(name: "YourHand", handVector: left) 89 | jsonString = para.toJson() 90 | //保存 jsonString 到磁盘或网络 91 | ... 92 | } 93 | ``` 94 | 95 | 然后，将保存的 JSON 字符串转换为 `HVHandInfo` 类型进行手势匹配： 96 | 97 | ```swift 98 | //从 JSON 字符串转换 99 | let handInfo = jsonStr.toModel(HVHandJsonModel.self)!.convertToHVHandInfo() 100 | //从磁盘加载 JSON 并转换 101 | let handInfo = HVHandJsonModel.loadHandJsonModel(fileName: "YourJsonFileName")!.convertToHVHandInfo() 102 | 103 | //用 `HVHandInfo` 类型进行手势匹配，可以将每根手指单独计算相似度 104 | if let handInfo { 105 | averageAndEachLeftScores = latestHandTracking.leftHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: recordHand) 106 | averageAndEachRightScores = latestHandTracking.rightHandVector?.averageAndEachSimilarities(of: .fiveFingers, to: recordHand) 107 | } 108 | 109 | ``` 110 | 111 | ### 2.手指形状参数FingerShape 112 | 113 | ![XRHandsCoverImage](./Resources//UntityXRHandsCoverImage.png) 114 | 115 | 该方法重点参考了 Unity 中知名 XR 手势框架：[XRHands](https://docs.unity3d.com/Packages/com.unity.xr.hands@1.5/manual/index.html) 。 116 | 117 | ![FingerShaper](./Resources/FingerShaper.gif) 118 | 119 | 相关参数的定义也类似： 120 | 121 | * **指根卷曲度 baseCurl**：手指根部关节的卷曲度，大拇指为 `IntermediateBase` 关节，其余手指为 `Knuckle` 关节，范围 0～1 122 | 123 | ![FingerShapeBaseCurl](./Resources/FingerShapeBaseCurl.png) 124 | 125 | * **指尖卷曲度 tipCurl**：手指上部关节的卷曲度，大拇指为 `IntermediateTip` 关节，其余手指为 `IntermediateBase` 和 `IntermediateTip` 两个关节的平均值，范围 0～1 126 | 127 | ![FingerShapeTipCurl](./Resources/FingerShapeTipCurl.png) 128 | 129 | * **整体卷曲度 fullCurl**：baseCurl 与 tipCurl 的平均值，范围 0～1 130 | 131 | ![FingerShapFullCurl](./Resources/FingerShapFullCurl.png) 132 | 133 | * **（与拇指）捏合度 pinch**：计算与拇指指尖的距离，范围 0～1，拇指该参数为 `nil` 134 | 135 | ![FingerShapePinch](./Resources/FingerShapePinch.png) 136 | 137 | * **（与外侧相邻手指）分离度 spread**：只计算水平方向上的夹角，范围 0～1，小拇指该参数为 `nil` 138 | 139 | ![FingerShapeSpread](./Resources/FingerShapeSpread.png) 140 | 141 | 关于三个不同的卷曲度有什么区别，可以参考下图： 142 | 143 | ![FingerShapeDifferenceCurl](./Resources/FingerShapeDifferenceCurl.png) 144 | 145 | ### 3. 在模拟器上测试 146 | 147 | `HandVector` 中的模拟器测试方法来自于 [VisionOS Simulator hands](https://github.com/BenLumenDigital/VisionOS-SimHands) 项目, 它提供了一种可以在模拟器上测试手部追踪的方法: 148 | 149 | 它分为 2 部分: 150 | 151 | 1. 一个 macOS 工具 app, 带有 bonjour 网络服务 152 | 2. 一个 Swift 类，用来在你的项目中连接到 bonjour 服务（本 package 中已自带，并自动接收转换为对应手势，HandVector 2.0 版本中更新了大量数学“黑魔法”来实现新的匹配算法) 153 | 154 | #### macOS Helper App 155 | 156 | 这个工具 app 使用了 Google 的 MediaPipes 来实现 3D 手势追踪。工具中只是一段非常简单的代码——它使用一个WKWebView 来运行 Google 的示例代码，并将捕获到的手部数据作用 JSON 传递到原生 Swift 代码中。 157 | 158 | 然后通过 Swift 代码将 JSON 信息通过 Bonjour 服务广播出去。 159 | 160 | > 如果手势识别长时间无法启动（按钮一直无法点击），请检查网络是否能连接到 google MediaPipes。（中国用户请特别注意网络） 161 | 162 | ![](./Resources/handVectorTest.gif) 163 | 164 | ### 其他... 165 | 166 | 更多详情，请查看 demo 工程。 167 | 168 | 169 | 170 | ## 安装 171 | 172 | #### Swift Package Manager 173 | 174 | 要使用苹果的 Swift Package Manager 集成，将以下内容作为依赖添加到你的 `Package.swift`： 175 | 176 | ``` 177 | .package(url: "https://github.com/XanderXu/HandVector.git", .upToNextMajor(from: "2.0.0")) 178 | ``` 179 | 180 | #### 手动 181 | 182 | [下载](https://github.com/XanderXu/HandVector/archive/master.zip) 项目，然后复制 `HandVector` 文件夹到你的工程中就可以使用了。 183 | 184 | ## 贡献 185 | 186 | 欢迎贡献代码 *♡*. 187 | 188 | ## 联系我 189 | 190 | Xander: API 搬运工 191 | 192 | * [https://twitter.com/XanderARKit](https://twitter.com/XanderARKit) 193 | * [https://github.com/XanderXu](https://github.com/XanderXu) 194 | 195 | - [https://juejin.cn/user/2629687543092056](https://juejin.cn/user/2629687543092056) 196 | 197 | 198 | 199 | ## 许可证 200 | 201 | HandVector 是在 MIT license 下发布的。更多信息可以查看 [LICENSE](./LICENSE)。 202 | -------------------------------------------------------------------------------- /READMEv1/README.md: -------------------------------------------------------------------------------- 1 |

2 | HandVector Logo 3 |

4 |

5 | Swift Package Manager compatible 6 | Swift 5.9 7 | Swift 5.9 8 |

9 | 10 | [中文版](./README_CN.md) 11 | 12 | **HandVector** uses **Cosine Similarity** Algorithm to calculate the similarity of hand gestures in visionOS, and with a macOS tool to test hand tracking in visionOS simulator. 13 | 14 |

15 | Requirements • Usage • Installation • Contribution • Contact • License 16 |

17 | 18 | ## Requirements 19 | 20 | - visionOS 1.0+ 21 | - Xcode 15.2+ 22 | - Swift 5.9+ 23 | 24 | ## Usage 25 | 26 | Your can run demo in package to see how to use it, and also can try an Vision Pro App. And also can see the App in App Store whitch uses `HandVector` to match gesture: 27 | 28 | 1. [FingerEmoji](https://apps.apple.com/us/app/fingeremoji/id6476075901) : FingerEmoji Let your finger dance with Emoji, you can Hit the emoji card by hand with the same gesture. 29 | 30 | ![960x540mv](./Resources/960x540mv.webp) 31 | 32 | 2. [SkyGestures](https://apps.apple.com/us/app/skygestures/id6499123392): **[SkyGestures](https://github.com/zlinoliver/SkyGestures)** is an innovative app that uses hand gestures to control DJI Tello drones via the Vision Pro platform. It's [Open Source](https://github.com/zlinoliver/SkyGestures) now. 33 | 34 | ![](./Resources/skygestures_demo1.gif) 35 | 36 | 37 | 38 | ### 1. Match builtin hand gesture: OK 39 | 40 | ![](./Resources/handVectorDemoMatchOK.gif) 41 | 42 | `HandVector` allows you to track your hands, and calculate the similarity between your current hand to another recorded hand gesture: 43 | 44 | ```swift 45 | import HandVector 46 | 47 | //load recorded hand gesture from json file 48 | model.handEmojiDict = HandEmojiParameter.generateParametersDict(fileName: "HandEmojiTotalJson")! 49 | guard let okVector = model.handEmojiDict["👌"]?.convertToHandVectorMatcher(), let leftOKVector = okVector.left else { return } 50 | 51 | //update current handTracking from HandTrackingProvider 52 | for await update in handTracking.anchorUpdates { 53 | switch update.event { 54 | case .added, .updated: 55 | let anchor = update.anchor 56 | guard anchor.isTracked else { continue } 57 | await latestHandTracking.updateHand(from: anchor) 58 | case .removed: 59 | ... 60 | } 61 | } 62 | 63 | 64 | //calculate the similarity 65 | let leftScore = model.latestHandTracking.leftHandVector?.similarity(to: leftOKVector) ?? 0 66 | model.leftScore = Int(abs(leftScore) * 100) 67 | let rightScore = model.latestHandTracking.rightHandVector?.similarity(to: leftOKVector) ?? 0 68 | model.rightScore = Int(abs(rightScore) * 100) 69 | ``` 70 | 71 | the score should be in `[-1.0,1.0]`, `1.0` means fully matched and both are left or right hands, `-1.0 `means fully matched but one is left hand, another is right hand, and `0` means not matched. 72 | 73 | ### 2. Record a new gesture and match 74 | 75 | ![](./Resources/handVectorDemoRecordMatch.gif) 76 | 77 | `HandVector` allows you to record your custom hands gesture, and save as JSON string: 78 | 79 | ```swift 80 | let para = HandEmojiParameter.generateParameters(name: "both", leftHandVector: model.latestHandTracking.leftHandVector, rightHandVector: model.latestHandTracking.rightHandVector) 81 | model.recordHand = para 82 | 83 | jsonString = para?.toJson() 84 | ``` 85 | 86 | And then, you can turn this JSON string to `HandVectorMatcher`，so you can use it to match your gesture now: 87 | 88 | ```swift 89 | guard let targetVector = model.recordHand?.convertToHandVectorMatcher(), targetVector.left != nil || targetVector.right != nil else { return } 90 | 91 | let targetLeft = targetVector.left ?? targetVector.right 92 | let targetRight = targetVector.right ?? targetVector.left 93 | 94 | let leftScore = model.latestHandTracking.leftHandVector?.similarity(of: HandVectorMatcher.allFingers, to: targetLeft!) ?? 0 95 | model.leftScore = Int(abs(leftScore) * 100) 96 | let rightScore = model.latestHandTracking.rightHandVector?.similarity(of: HandVectorMatcher.allFingers, to: targetRight!) ?? 0 97 | model.rightScore = Int(abs(rightScore) * 100) 98 | ``` 99 | 100 | 101 | 102 | ### 3. Test hand gesture on Mac simulator 103 | 104 | The test method of`HandVector` is inspired by [VisionOS Simulator hands](https://github.com/BenLumenDigital/VisionOS-SimHands), it allow you to test hand tracking on visionOS simulator: 105 | 106 | It uses 2 things: 107 | 108 | 1. A macOS helper app, with a bonjour service 109 | 2. A Swift class for your VisionOS project which connects to the bonjour service (already in this package, and already turn JSON data to hand gestures) 110 | 111 | #### macOS Helper App 112 | 113 | The helper app uses Google MediaPipes for 3D hand tracking. This is a very basic setup - it uses a WKWebView to run the Google sample code, and that passed the hand data as JSON into native Swift. 114 | 115 | The Swift code then spits out the JSON over a Bonjour service. 116 | 117 | > If hand tracking can't start for a long time(Start button still can't be pressed), please check your network to google MediaPipes. 118 | 119 | ![](./Resources/handVectorTest.gif) 120 | 121 | ### And many more... 122 | 123 | To go further, take a look at the documentation and the demo project. 124 | 125 | *Note: All contributions are welcome* 126 | 127 | ## Installation 128 | 129 | #### Swift Package Manager 130 | 131 | To integrate using Apple's Swift package manager, without Xcode integration, add the following as a dependency to your `Package.swift`: 132 | 133 | ``` 134 | .package(url: "https://github.com/XanderXu/HandVector.git", .upToNextMajor(from: "0.3.0")) 135 | ``` 136 | 137 | #### Manually 138 | 139 | [Download](https://github.com/XanderXu/HandVector/archive/master.zip) the project and copy the `HandVector` folder into your project to use it. 140 | 141 | ## Contribution 142 | 143 | Contributions are welcomed and encouraged *♡*. 144 | 145 | ## Contact 146 | 147 | Xander: API 搬运工 148 | 149 | * [https://twitter.com/XanderARKit](https://twitter.com/XanderARKit) 150 | * [https://github.com/XanderXu](https://github.com/XanderXu) 151 | 152 | - [https://juejin.cn/user/2629687543092056](https://juejin.cn/user/2629687543092056) 153 | 154 | 155 | 156 | ## License 157 | 158 | HandVector is released under an MIT license. See [LICENSE](./LICENSE) for more information. 159 | -------------------------------------------------------------------------------- /READMEv1/README_CN.md: -------------------------------------------------------------------------------- 1 |

2 | HandVector Logo 3 |

4 |

5 | Swift Package Manager compatible 6 | Swift 5.9 7 | Swift 5.9 8 |

9 | [English Version](./README.md) 10 | 11 | [旧版本](./READMEv1/README_CN.md) 12 | 13 | **HandVector** 使用**余弦相似度**算法，在 vsionOS 上计算不同手势之间的相似度，还带有一个 macOS 的工具类能让你在 visionOS 模拟器上也能使用手势追踪功能。 14 | 15 |

16 | 环境要求 • 用法 • 安装 • 贡献 • 联系方式 • 许可证 17 |

18 | 19 | 20 | ## 环境要求 21 | 22 | - visionOS 1.0+ 23 | - Xcode 15.2+ 24 | - Swift 5.9+ 25 | 26 | ## 用法 27 | 28 | 你可以下载运行软件包中的 demo 工程来查看如何使用，也可以从 App Store 中下载使用了 **HandVector** 的 App 来查看功能演示： 29 | 30 | 1. [FingerEmoji](https://apps.apple.com/us/app/fingeremoji/id6476075901) : FingerEmoji 让你的手指与 Emoji 共舞，你可以做出相同的手势并撞击 Emoji 卡片得分。 31 | 32 | ![960x540mv](./Resources/960x540mv.webp) 33 | 34 | 2. [SkyGestures](https://apps.apple.com/us/app/skygestures/id6499123392): **[SkyGestures](https://github.com/zlinoliver/SkyGestures)** 使用 Vision Pro 上的手势来控制大疆 DJI Tello 无人机，并且目前它已经 [开源](https://github.com/zlinoliver/SkyGestures) 了！ 35 | 36 | ![](./Resources/skygestures_demo1.gif) 37 | 38 | 39 | 40 | ### 1.匹配内置的 OK 手势 41 | 42 | ![](./Resources/handVectorDemoMatchOK.gif) 43 | 44 | `HandVector` 可以让你追踪双手关节的姿态，并与先前记录下的手势相比较，得出它们的相似度: 45 | 46 | ```swift 47 | import HandVector 48 | 49 | //从 json 文件中加载先前记录下的手势 50 | model.handEmojiDict = HandEmojiParameter.generateParametersDict(fileName: "HandEmojiTotalJson")! 51 | guard let okVector = model.handEmojiDict["👌"]?.convertToHandVectorMatcher(), let leftOKVector = okVector.left else { return } 52 | 53 | //从 HandTrackingProvider 中获取当前手势，并更新 54 | for await update in handTracking.anchorUpdates { 55 | switch update.event { 56 | case .added, .updated: 57 | let anchor = update.anchor 58 | guard anchor.isTracked else { continue } 59 | await latestHandTracking.updateHand(from: anchor) 60 | case .removed: 61 | ... 62 | } 63 | } 64 | 65 | 66 | //计算相似度 67 | let leftScore = model.latestHandTracking.leftHandVector?.similarity(to: leftOKVector) ?? 0 68 | model.leftScore = Int(abs(leftScore) * 100) 69 | let rightScore = model.latestHandTracking.rightHandVector?.similarity(to: leftOKVector) ?? 0 70 | model.rightScore = Int(abs(rightScore) * 100) 71 | ``` 72 | 73 | 相似度得分在 `[-1.0,1.0]` 之间， `1.0` 含义为手势完全匹配并且左右手也匹配， `-1.0 ` 含义为手势完全匹配但一个是左手一个是右手， `0` 含义为完全不匹配。 74 | 75 | ### 2. 录制自定义的新手势并匹配它 76 | 77 | ![](./Resources/handVectorDemoRecordMatch.gif) 78 | 79 | `HandVector` 允许你录制你的自定义手势，并保存为 JSON 字符串: 80 | 81 | ```swift 82 | let para = HandEmojiParameter.generateParameters(name: "both", leftHandVector: model.latestHandTracking.leftHandVector, rightHandVector: model.latestHandTracking.rightHandVector) 83 | model.recordHand = para 84 | 85 | jsonString = para?.toJson() 86 | ``` 87 | 88 | 然后，你还可以将 JSON 字符串转换为 `HandVectorMatcher` 类型，这样就可以进行手势匹配了： 89 | 90 | ```swift 91 | guard let targetVector = model.recordHand?.convertToHandVectorMatcher(), targetVector.left != nil || targetVector.right != nil else { return } 92 | 93 | let targetLeft = targetVector.left ?? targetVector.right 94 | let targetRight = targetVector.right ?? targetVector.left 95 | 96 | let leftScore = model.latestHandTracking.leftHandVector?.similarity(of: HandVectorMatcher.allFingers, to: targetLeft!) ?? 0 97 | model.leftScore = Int(abs(leftScore) * 100) 98 | let rightScore = model.latestHandTracking.rightHandVector?.similarity(of: HandVectorMatcher.allFingers, to: targetRight!) ?? 0 99 | model.rightScore = Int(abs(rightScore) * 100) 100 | ``` 101 | 102 | 103 | 104 | ### 3. 在模拟器上测试 105 | 106 | `HandVector` 中的模拟器测试方法来自于 [VisionOS Simulator hands](https://github.com/BenLumenDigital/VisionOS-SimHands) 项目, 它提供了一种可以在模拟器上测试手部追踪的方法: 107 | 108 | 它分为 2 部分: 109 | 110 | 1. 一个 macOS 工具 app, 带有 bonjour 网络服务 111 | 2. 一个 Swift 类，用来在你的项目中连接到 bonjour 服务（本 package 中已自带，并自动接收转换为对应手势) 112 | 113 | #### macOS Helper App 114 | 115 | 这个工具 app 使用了 Google 的 MediaPipes 来实现 3D 手势追踪。工具中只是一段非常简单的代码——它使用一个WKWebView 来运行 Google 的示例代码，并将捕获到的手部数据作用 JSON 传递到原生 Swift 代码中。 116 | 117 | 然后通过 Swift 代码将 JSON 信息通过 Bonjour 服务广播出去。 118 | 119 | > 如果手势识别长时间无法启动（按钮一直无法点击），请检查网络是否能连接到 google MediaPipes。（中国用户请特别注意网络） 120 | 121 | ![](./Resources/handVectorTest.gif) 122 | 123 | ### 其他... 124 | 125 | 更多详情，请查看 demo 工程。 126 | 127 | 128 | 129 | ## 安装 130 | 131 | #### Swift Package Manager 132 | 133 | 要使用苹果的 Swift Package Manager 集成，将以下内容作为依赖添加到你的 `Package.swift`： 134 | 135 | ``` 136 | .package(url: "https://github.com/XanderXu/HandVector.git", .upToNextMajor(from: "0.3.0")) 137 | ``` 138 | 139 | #### 手动 140 | 141 | [下载](https://github.com/XanderXu/HandVector/archive/master.zip) 项目，然后复制 `HandVector` 文件夹到你的工程中就可以使用了。 142 | 143 | ## 贡献 144 | 145 | 欢迎贡献代码 *♡*. 146 | 147 | ## 联系我 148 | 149 | Xander: API 搬运工 150 | 151 | * [https://twitter.com/XanderARKit](https://twitter.com/XanderARKit) 152 | * [https://github.com/XanderXu](https://github.com/XanderXu) 153 | 154 | - [https://juejin.cn/user/2629687543092056](https://juejin.cn/user/2629687543092056) 155 | 156 | 157 | 158 | ## 许可证 159 | 160 | HandVector 是在 MIT license 下发布的。更多信息可以查看 [LICENSE](./LICENSE)。 161 | -------------------------------------------------------------------------------- /READMEv1/Resources/960x540mv.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/READMEv1/Resources/960x540mv.webp -------------------------------------------------------------------------------- /READMEv1/Resources/HandVectorLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/READMEv1/Resources/HandVectorLogo.png -------------------------------------------------------------------------------- /READMEv1/Resources/handVectorDemoMatchOK.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/READMEv1/Resources/handVectorDemoMatchOK.gif -------------------------------------------------------------------------------- /READMEv1/Resources/handVectorDemoRecordMatch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/READMEv1/Resources/handVectorDemoRecordMatch.gif -------------------------------------------------------------------------------- /READMEv1/Resources/handVectorTest.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/READMEv1/Resources/handVectorTest.gif -------------------------------------------------------------------------------- /READMEv1/Resources/skygestures_demo1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/READMEv1/Resources/skygestures_demo1.gif -------------------------------------------------------------------------------- /Resources/FingerShapFullCurl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShapFullCurl.png -------------------------------------------------------------------------------- /Resources/FingerShapeBaseCurl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShapeBaseCurl.png -------------------------------------------------------------------------------- /Resources/FingerShapeDifferenceCurl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShapeDifferenceCurl.png -------------------------------------------------------------------------------- /Resources/FingerShapePinch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShapePinch.png -------------------------------------------------------------------------------- /Resources/FingerShapeSpread.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShapeSpread.png -------------------------------------------------------------------------------- /Resources/FingerShapeTipCurl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShapeTipCurl.png -------------------------------------------------------------------------------- /Resources/FingerShaper.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/FingerShaper.gif -------------------------------------------------------------------------------- /Resources/HandVectorFileStructure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/HandVectorFileStructure.png -------------------------------------------------------------------------------- /Resources/HandVectorLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/HandVectorLogo.png -------------------------------------------------------------------------------- /Resources/MatchAllBuiltin.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/MatchAllBuiltin.gif -------------------------------------------------------------------------------- /Resources/RecordAndMatch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/RecordAndMatch.gif -------------------------------------------------------------------------------- /Resources/UntityXRHandsCoverImage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/UntityXRHandsCoverImage.png -------------------------------------------------------------------------------- /Resources/handVectorTest.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XanderXu/HandVector/efd0d2f0f2dfe4d1f98e4ad0817df24e28a0a59d/Resources/handVectorTest.gif -------------------------------------------------------------------------------- /Sources/HandVector/Extensions/Codable+Extensions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Codable+Extensions.swift 3 | // HandVector 4 | // 5 | // Created by 许同学 on 2023/11/30. 6 | // 7 | 8 | import Foundation 9 | 10 | 11 | extension Encodable { 12 | func toJson(encoding: String.Encoding = .utf8) -> String? { 13 | guard let data = try? JSONEncoder().encode(self) else { return nil } 14 | return String(data: data, encoding: encoding) 15 | } 16 | } 17 | extension String { 18 | func toModel(_ type: T.Type, using encoding: String.Encoding = .utf8) -> T? where T : Decodable { 19 | guard let data = self.data(using: encoding) else { return nil } 20 | do { 21 | return try JSONDecoder().decode(T.self, from: data) 22 | } catch { 23 | print(error) 24 | } 25 | return nil 26 | } 27 | } 28 | 29 | //extension simd_float4x4: Codable { 30 | // public init(from decoder: any Decoder) throws { 31 | // let container = try decoder.singleValueContainer() 32 | // 33 | // let cols = try container.decode([SIMD4].self) 34 | // self.init(columns: (cols[0], cols[1], cols[2], cols[3])) 35 | // } 36 | // public func encode(to encoder: any Encoder) throws { 37 | // var container = encoder.singleValueContainer() 38 | // try container.encode([columns.0, columns.1, columns.2, columns.3]) 39 | // } 40 | //} 41 | -------------------------------------------------------------------------------- /Sources/HandVector/Extensions/HandAnchor+Extensions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandAnchor+Extensions.swift 3 | // HandVector 4 | // 5 | // Created by xu on 2023/9/28. 6 | // 7 | 8 | import Foundation 9 | import simd 10 | import ARKit 11 | 12 | 13 | public extension HandAnchor.Chirality { 14 | enum NameCodingKey: String, Codable, CodingKey { 15 | case right 16 | case left 17 | 18 | public var description: String { 19 | return self.rawValue 20 | } 21 | 22 | public var chirality: HandAnchor.Chirality { 23 | switch self { 24 | case .right: 25 | return .right 26 | case .left: 27 | return .left 28 | } 29 | } 30 | } 31 | var codableName: NameCodingKey { 32 | switch self { 33 | case .right: 34 | return .right 35 | case .left: 36 | return .left 37 | } 38 | } 39 | } 40 | public extension HandSkeleton.JointName { 41 | enum NameCodingKey: String, Codable, CodingKey, CaseIterable { 42 | case wrist 43 | 44 | /// The thumb knuckle joint of a hand skeleton. 45 | case thumbKnuckle 46 | 47 | /// The thumb intermediate base joint of a hand skeleton. 48 | case thumbIntermediateBase 49 | 50 | /// The thumb intermediate tip joint of a hand skeleton. 51 | case thumbIntermediateTip 52 | 53 | /// The thumb tip joint of a hand skeleton. 54 | case thumbTip 55 | 56 | /// The index finger metacarpal joint of a hand skeleton. 57 | case indexFingerMetacarpal 58 | 59 | /// The index finger knuckle joint of a hand skeleton. 60 | case indexFingerKnuckle 61 | 62 | /// The index finger intermediate base joint of a hand skeleton. 63 | case indexFingerIntermediateBase 64 | 65 | /// The index finger intermediate tip joint of a hand skeleton. 66 | case indexFingerIntermediateTip 67 | 68 | /// The index finger tip joint of a hand skeleton. 69 | case indexFingerTip 70 | 71 | /// The middle finger metacarpal joint of a hand skeleton. 72 | case middleFingerMetacarpal 73 | 74 | /// The middle finger knuckle joint of a hand skeleton. 75 | case middleFingerKnuckle 76 | 77 | /// The middle finger intermediate base joint of a hand skeleton. 78 | case middleFingerIntermediateBase 79 | 80 | /// The middle finger intermediate tip joint of a hand skeleton. 81 | case middleFingerIntermediateTip 82 | 83 | /// The middle finger tip joint of a hand skeleton. 84 | case middleFingerTip 85 | 86 | /// The ring finger metacarpal joint of a hand skeleton. 87 | case ringFingerMetacarpal 88 | 89 | /// The ring finger knuckle joint of a hand skeleton. 90 | case ringFingerKnuckle 91 | 92 | /// The ring finger intermediate base joint of a hand skeleton. 93 | case ringFingerIntermediateBase 94 | 95 | /// The ring finger intermediate tip joint of a hand skeleton. 96 | case ringFingerIntermediateTip 97 | 98 | /// The ring finger tip joint of a hand skeleton. 99 | case ringFingerTip 100 | 101 | /// The little finger metacarpal joint of a hand skeleton. 102 | case littleFingerMetacarpal 103 | 104 | /// The little finger knuckle joint of a hand skeleton. 105 | case littleFingerKnuckle 106 | 107 | /// The little finger intermediate base joint of a hand skeleton. 108 | case littleFingerIntermediateBase 109 | 110 | /// The little finger intermediate tip joint of a hand skeleton. 111 | case littleFingerIntermediateTip 112 | 113 | /// The little finger tip joint of a hand skeleton. 114 | case littleFingerTip 115 | 116 | /// The wrist joint at the forearm of a hand skeleton. 117 | case forearmWrist 118 | 119 | /// The forearm joint of a hand skeleton. 120 | case forearmArm 121 | 122 | case unknown 123 | 124 | public var description: String { 125 | return self.rawValue 126 | } 127 | 128 | public var jointName: HandSkeleton.JointName? { 129 | switch self { 130 | case .wrist: 131 | return .wrist 132 | case .thumbKnuckle: 133 | return .thumbKnuckle 134 | case .thumbIntermediateBase: 135 | return .thumbIntermediateBase 136 | case .thumbIntermediateTip: 137 | return .thumbIntermediateTip 138 | case .thumbTip: 139 | return .thumbTip 140 | case .indexFingerMetacarpal: 141 | return .indexFingerMetacarpal 142 | case .indexFingerKnuckle: 143 | return .indexFingerKnuckle 144 | case .indexFingerIntermediateBase: 145 | return .indexFingerIntermediateBase 146 | case .indexFingerIntermediateTip: 147 | return .indexFingerIntermediateTip 148 | case .indexFingerTip: 149 | return .indexFingerTip 150 | case .middleFingerMetacarpal: 151 | return .middleFingerMetacarpal 152 | case .middleFingerKnuckle: 153 | return .middleFingerKnuckle 154 | case .middleFingerIntermediateBase: 155 | return .middleFingerIntermediateBase 156 | case .middleFingerIntermediateTip: 157 | return .middleFingerIntermediateTip 158 | case .middleFingerTip: 159 | return .middleFingerTip 160 | case .ringFingerMetacarpal: 161 | return .ringFingerMetacarpal 162 | case .ringFingerKnuckle: 163 | return .ringFingerKnuckle 164 | case .ringFingerIntermediateBase: 165 | return .ringFingerIntermediateBase 166 | case .ringFingerIntermediateTip: 167 | return .ringFingerIntermediateTip 168 | case .ringFingerTip: 169 | return .ringFingerTip 170 | case .littleFingerMetacarpal: 171 | return .littleFingerMetacarpal 172 | case .littleFingerKnuckle: 173 | return .littleFingerKnuckle 174 | case .littleFingerIntermediateBase: 175 | return .littleFingerIntermediateBase 176 | case .littleFingerIntermediateTip: 177 | return .littleFingerIntermediateTip 178 | case .littleFingerTip: 179 | return .littleFingerTip 180 | case .forearmWrist: 181 | return .forearmWrist 182 | case .forearmArm: 183 | return .forearmArm 184 | case .unknown: 185 | return nil 186 | } 187 | } 188 | } 189 | var codableName: NameCodingKey { 190 | switch self { 191 | case .wrist: 192 | return .wrist 193 | case .thumbKnuckle: 194 | return .thumbKnuckle 195 | case .thumbIntermediateBase: 196 | return .thumbIntermediateBase 197 | case .thumbIntermediateTip: 198 | return .thumbIntermediateTip 199 | case .thumbTip: 200 | return .thumbTip 201 | case .indexFingerMetacarpal: 202 | return .indexFingerMetacarpal 203 | case .indexFingerKnuckle: 204 | return .indexFingerKnuckle 205 | case .indexFingerIntermediateBase: 206 | return .indexFingerIntermediateBase 207 | case .indexFingerIntermediateTip: 208 | return .indexFingerIntermediateTip 209 | case .indexFingerTip: 210 | return .indexFingerTip 211 | case .middleFingerMetacarpal: 212 | return .middleFingerMetacarpal 213 | case .middleFingerKnuckle: 214 | return .middleFingerKnuckle 215 | case .middleFingerIntermediateBase: 216 | return .middleFingerIntermediateBase 217 | case .middleFingerIntermediateTip: 218 | return .middleFingerIntermediateTip 219 | case .middleFingerTip: 220 | return .middleFingerTip 221 | case .ringFingerMetacarpal: 222 | return .ringFingerMetacarpal 223 | case .ringFingerKnuckle: 224 | return .ringFingerKnuckle 225 | case .ringFingerIntermediateBase: 226 | return .ringFingerIntermediateBase 227 | case .ringFingerIntermediateTip: 228 | return .ringFingerIntermediateTip 229 | case .ringFingerTip: 230 | return .ringFingerTip 231 | case .littleFingerMetacarpal: 232 | return .littleFingerMetacarpal 233 | case .littleFingerKnuckle: 234 | return .littleFingerKnuckle 235 | case .littleFingerIntermediateBase: 236 | return .littleFingerIntermediateBase 237 | case .littleFingerIntermediateTip: 238 | return .littleFingerIntermediateTip 239 | case .littleFingerTip: 240 | return .littleFingerTip 241 | case .forearmWrist: 242 | return .forearmWrist 243 | case .forearmArm: 244 | return .forearmArm 245 | @unknown default: 246 | return .unknown 247 | } 248 | } 249 | 250 | public var parentName: HandSkeleton.JointName? { 251 | switch self { 252 | case .wrist: 253 | return nil 254 | case .thumbKnuckle: 255 | return .wrist 256 | case .thumbIntermediateBase: 257 | return .thumbKnuckle 258 | case .thumbIntermediateTip: 259 | return .thumbIntermediateBase 260 | case .thumbTip: 261 | return .thumbIntermediateTip 262 | case .indexFingerMetacarpal: 263 | return .wrist 264 | case .indexFingerKnuckle: 265 | return .indexFingerMetacarpal 266 | case .indexFingerIntermediateBase: 267 | return .indexFingerKnuckle 268 | case .indexFingerIntermediateTip: 269 | return .indexFingerIntermediateBase 270 | case .indexFingerTip: 271 | return .indexFingerIntermediateTip 272 | case .middleFingerMetacarpal: 273 | return .wrist 274 | case .middleFingerKnuckle: 275 | return .middleFingerMetacarpal 276 | case .middleFingerIntermediateBase: 277 | return .middleFingerKnuckle 278 | case .middleFingerIntermediateTip: 279 | return .middleFingerIntermediateBase 280 | case .middleFingerTip: 281 | return .middleFingerIntermediateTip 282 | case .ringFingerMetacarpal: 283 | return .wrist 284 | case .ringFingerKnuckle: 285 | return .ringFingerMetacarpal 286 | case .ringFingerIntermediateBase: 287 | return .ringFingerKnuckle 288 | case .ringFingerIntermediateTip: 289 | return .ringFingerIntermediateBase 290 | case .ringFingerTip: 291 | return .ringFingerIntermediateTip 292 | case .littleFingerMetacarpal: 293 | return .wrist 294 | case .littleFingerKnuckle: 295 | return .littleFingerMetacarpal 296 | case .littleFingerIntermediateBase: 297 | return .littleFingerKnuckle 298 | case .littleFingerIntermediateTip: 299 | return .littleFingerIntermediateBase 300 | case .littleFingerTip: 301 | return .littleFingerIntermediateTip 302 | case .forearmWrist: 303 | return .wrist 304 | case .forearmArm: 305 | return .forearmWrist 306 | @unknown default: 307 | return nil 308 | } 309 | } 310 | } 311 | 312 | -------------------------------------------------------------------------------- /Sources/HandVector/Extensions/HandSkeleton+Extensions.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See the LICENSE.txt file for this sample’s licensing information. 3 | 4 | Abstract: 5 | Extensions and utilities. 6 | */ 7 | 8 | import ARKit 9 | 10 | extension HandSkeleton.Joint { 11 | var localPosition: simd_float3 { 12 | return anchorFromJointTransform.columns.3.xyz 13 | } 14 | } 15 | extension simd_float4x4 { 16 | var float4Array: [SIMD4] { 17 | [columns.0, columns.1, columns.2, columns.3] 18 | } 19 | var positionReversed: simd_float4x4 { 20 | simd_float4x4( 21 | [columns.0, 22 | columns.1, 23 | columns.2, 24 | SIMD4(-columns.3.xyz, 1)] 25 | ) 26 | } 27 | } 28 | extension SIMD4 { 29 | var xyz: SIMD3 { 30 | self[SIMD3(0, 1, 2)] 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /Sources/HandVector/HVFingerShape.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HVFingerShape.swift 3 | // HandVector 4 | // 5 | // Created by 许同学 on 2024/8/20. 6 | // 7 | 8 | import ARKit 9 | 10 | public struct HVFingerShape: Sendable, Equatable { 11 | public enum FingerShapeType: Int, Sendable, Equatable, CaseIterable { 12 | case baseCurl = 1 13 | case tipCurl = 2 14 | case fullCurl = 4 15 | case pinch = 8 16 | case spread = 16 17 | } 18 | 19 | 20 | public let finger: HVJointOfFinger 21 | public let fingerShapeTypes: Set 22 | 23 | public let fullCurl: Float 24 | public let baseCurl: Float 25 | public let tipCurl: Float 26 | /// not avalible on thumb 27 | public let pinch: Float? 28 | /// not avalible on littleFinger 29 | public let spread: Float? 30 | 31 | 32 | public init(finger: HVJointOfFinger, fingerShapeType: HVFingerShape.FingerShapeType, joints: [HandSkeleton.JointName: HVJointInfo]) { 33 | self.init(finger: finger, fingerShapeTypes: [fingerShapeType], joints: joints) 34 | } 35 | public init(finger: HVJointOfFinger, fingerShapeTypes: Set = .all, joints: [HandSkeleton.JointName: HVJointInfo]) { 36 | func linearInterpolate(lowerBound: Float, upperBound: Float, value: Float, clamp: Bool = true) -> Float { 37 | let p = (value-lowerBound)/(upperBound-lowerBound) 38 | if clamp { 39 | return simd_clamp(p, 0, 1) 40 | } else { 41 | return p 42 | } 43 | } 44 | 45 | self.finger = finger 46 | self.fingerShapeTypes = fingerShapeTypes 47 | 48 | var baseCurl: Float = 0 49 | var tipCurl: Float = 0 50 | var fullCurl: Float = 0 51 | var pinch: Float? = nil 52 | var spread: Float? = nil 53 | let config = finger.fingerShapeConfiguration 54 | 55 | if finger == .thumb { 56 | if fingerShapeTypes.contains(.baseCurl) { 57 | let joint = joints[finger.jointGroupNames[1]]! 58 | let xAxis = joint.transformToParent.columns.0 59 | let angle = atan2(xAxis.y, xAxis.x) / .pi * 180 60 | baseCurl = linearInterpolate(lowerBound: config.minimumBaseCurlDegrees, upperBound: config.maximumBaseCurlDegrees, value: angle) 61 | 62 | // print("baseCurl", angle, baseCurl) 63 | } else { 64 | baseCurl = 0 65 | } 66 | 67 | if fingerShapeTypes.contains(.tipCurl) { 68 | let joint = joints[finger.jointGroupNames[2]]! 69 | let xAxis = joint.transformToParent.columns.0 70 | let angle = atan2(xAxis.y, xAxis.x) / .pi * 180 71 | tipCurl = linearInterpolate(lowerBound: config.minimumTipCurlDegrees1, upperBound: config.maximumTipCurlDegrees1, value: angle) 72 | 73 | // print("tipCurl", angle, tipCurl) 74 | } else { 75 | tipCurl = 0 76 | } 77 | 78 | if fingerShapeTypes.contains(.fullCurl) { 79 | fullCurl = (baseCurl + tipCurl)/2 80 | } 81 | 82 | if fingerShapeTypes.contains(.spread) { 83 | let joint = joints[finger.jointGroupNames[1]]! 84 | let xAxis = joint.transform.columns.0 85 | 86 | let angle = -atan2(xAxis.z, xAxis.x) / .pi * 180 87 | spread = linearInterpolate(lowerBound: config.minimumSpreadDegrees, upperBound: config.maximumSpreadDegrees, value: angle) 88 | 89 | // print("spread", angle, spread) 90 | } else { 91 | spread = nil 92 | } 93 | } else { 94 | if fingerShapeTypes.contains(.baseCurl) { 95 | let joint = joints[finger.jointGroupNames.first!]! 96 | let xAxis = joint.transformToParent.columns.0 97 | let angle = atan2(xAxis.y, xAxis.x) / .pi * 180 98 | baseCurl = linearInterpolate(lowerBound: config.minimumBaseCurlDegrees, upperBound: config.maximumBaseCurlDegrees, value: angle) 99 | 100 | // print("baseCurl", angle, baseCurl) 101 | } else { 102 | baseCurl = 0 103 | } 104 | 105 | if fingerShapeTypes.contains(.tipCurl) { 106 | let joint1 = joints[finger.jointGroupNames[1]]! 107 | let xAxis1 = joint1.transformToParent.columns.0 108 | let angle1 = atan2(xAxis1.y, xAxis1.x) / .pi * 180 109 | let tipCurl1 = linearInterpolate(lowerBound: config.minimumTipCurlDegrees1, upperBound: config.maximumTipCurlDegrees1, value: angle1) 110 | 111 | let joint2 = joints[finger.jointGroupNames[2]]! 112 | let xAxis2 = joint2.transformToParent.columns.0 113 | let angle2 = atan2(xAxis2.y, xAxis2.x) / .pi * 180 114 | let tipCurl2 = linearInterpolate(lowerBound: config.minimumTipCurlDegrees2, upperBound: config.maximumTipCurlDegrees2, value: angle2) 115 | 116 | tipCurl = (tipCurl1 + tipCurl2)/2 117 | // print("tipCurl", angle1, angle2, tipCurl) 118 | } else { 119 | tipCurl = 0 120 | } 121 | 122 | if fingerShapeTypes.contains(.fullCurl) { 123 | fullCurl = (baseCurl + tipCurl)/2 124 | } 125 | 126 | if fingerShapeTypes.contains(.pinch) { 127 | let joint1 = joints[.thumbTip]! 128 | let joint2 = joints[finger.jointGroupNames.last!]! 129 | let distance = simd_distance(joint1.position, joint2.position) 130 | pinch = linearInterpolate(lowerBound: config.maximumPinchDistance, upperBound: config.minimumPinchDistance, value: distance) 131 | 132 | // print("pinch", distance,pinch) 133 | } else { 134 | pinch = nil 135 | } 136 | 137 | if fingerShapeTypes.contains(.spread), finger != .littleFinger { 138 | let joint1 = joints[finger.jointGroupNames[0]]! 139 | let joint2 = joints[finger.nextNeighbourFinger!.jointGroupNames[0]]! 140 | let xAxis1 = joint1.transform.columns.0 141 | let xAxis2 = joint2.transform.columns.0 142 | 143 | let xAxis = normalize(xAxis1 + xAxis2) 144 | let zAxis = normalize(joint1.transform.columns.2 + joint2.transform.columns.2) 145 | let yAxis = SIMD4(cross(zAxis.xyz, xAxis.xyz), 0) 146 | let p = (joint1.transform.columns.3 + joint2.transform.columns.3)/2 147 | 148 | let matrix = simd_float4x4(xAxis, yAxis, zAxis, p) 149 | let xAxis1R = matrix.inverse * xAxis1 150 | let xAxis2R = matrix.inverse * xAxis2 151 | 152 | let xAxis1H = SIMD3(xAxis1R.x, 0, xAxis1R.z) 153 | let xAxis2H = SIMD3(xAxis2R.x, 0, xAxis2R.z) 154 | 155 | let angle1 = atan2(xAxis1H.z, xAxis1H.x) / .pi * 180 156 | let angle2 = atan2(xAxis2H.z, xAxis2H.x) / .pi * 180 157 | 158 | let angle = angle2 - angle1 159 | spread = linearInterpolate(lowerBound: config.minimumSpreadDegrees, upperBound: config.maximumSpreadDegrees, value: angle) 160 | 161 | // print("spread",angle1, angle2, angle, spread) 162 | } else { 163 | spread = nil 164 | } 165 | 166 | } 167 | self.baseCurl = baseCurl 168 | self.tipCurl = tipCurl 169 | self.fullCurl = fullCurl 170 | self.pinch = pinch 171 | self.spread = spread 172 | } 173 | 174 | } 175 | fileprivate extension HVFingerShape { 176 | struct FingerShapeConfiguration { 177 | var maximumBaseCurlDegrees: Float 178 | var maximumTipCurlDegrees1: Float 179 | var maximumTipCurlDegrees2: Float 180 | var maximumPinchDistance: Float 181 | var maximumSpreadDegrees: Float 182 | 183 | var minimumBaseCurlDegrees: Float 184 | var minimumTipCurlDegrees1: Float 185 | var minimumTipCurlDegrees2: Float 186 | var minimumPinchDistance: Float 187 | var minimumSpreadDegrees: Float 188 | 189 | 190 | static var thumb: FingerShapeConfiguration { 191 | return FingerShapeConfiguration(maximumBaseCurlDegrees: 55, maximumTipCurlDegrees1: 55, maximumTipCurlDegrees2: 0, maximumPinchDistance: 0, maximumSpreadDegrees: 40, minimumBaseCurlDegrees: 5, minimumTipCurlDegrees1: -15, minimumTipCurlDegrees2: 0, minimumPinchDistance: 0, minimumSpreadDegrees: 0) 192 | } 193 | static var indexFinger: FingerShapeConfiguration { 194 | return FingerShapeConfiguration(maximumBaseCurlDegrees: 70, maximumTipCurlDegrees1: 100, maximumTipCurlDegrees2: 60, maximumPinchDistance: 0.05, maximumSpreadDegrees: 35, minimumBaseCurlDegrees: -5, minimumTipCurlDegrees1: 0, minimumTipCurlDegrees2: 0, minimumPinchDistance: 0.008, minimumSpreadDegrees: 0) 195 | } 196 | static var middleFinger: FingerShapeConfiguration { 197 | return FingerShapeConfiguration(maximumBaseCurlDegrees: 75, maximumTipCurlDegrees1: 100, maximumTipCurlDegrees2: 70, maximumPinchDistance: 0.05, maximumSpreadDegrees: 25, minimumBaseCurlDegrees: -5, minimumTipCurlDegrees1: 0, minimumTipCurlDegrees2: 0, minimumPinchDistance: 0.008, minimumSpreadDegrees: 0) 198 | } 199 | static var ringFinger: FingerShapeConfiguration { 200 | return FingerShapeConfiguration(maximumBaseCurlDegrees: 80, maximumTipCurlDegrees1: 100, maximumTipCurlDegrees2: 70, maximumPinchDistance: 0.05, maximumSpreadDegrees: 30, minimumBaseCurlDegrees: -5, minimumTipCurlDegrees1: 0, minimumTipCurlDegrees2: 0, minimumPinchDistance: 0.015, minimumSpreadDegrees: 0) 201 | } 202 | static var littleFinger: FingerShapeConfiguration { 203 | return FingerShapeConfiguration(maximumBaseCurlDegrees: 80, maximumTipCurlDegrees1:85, maximumTipCurlDegrees2: 80, maximumPinchDistance: 0.05, maximumSpreadDegrees: 0, minimumBaseCurlDegrees: -5, minimumTipCurlDegrees1: 0, minimumTipCurlDegrees2: 0, minimumPinchDistance: 0.015, minimumSpreadDegrees: 0) 204 | } 205 | 206 | } 207 | } 208 | fileprivate extension HVJointOfFinger { 209 | fileprivate var fingerShapeConfiguration: HVFingerShape.FingerShapeConfiguration { 210 | switch self { 211 | case .thumb: 212 | return .thumb 213 | case .indexFinger: 214 | return .indexFinger 215 | case .middleFinger: 216 | return .middleFinger 217 | case .ringFinger: 218 | return .ringFinger 219 | case .littleFinger: 220 | return .littleFinger 221 | default: 222 | return .indexFinger 223 | } 224 | } 225 | 226 | fileprivate var nextNeighbourFinger: HVJointOfFinger? { 227 | switch self { 228 | case .thumb: 229 | return .indexFinger 230 | case .indexFinger: 231 | return .middleFinger 232 | case .middleFinger: 233 | return .ringFinger 234 | case .ringFinger: 235 | return .littleFinger 236 | case .littleFinger: 237 | return nil 238 | default: 239 | return nil 240 | } 241 | } 242 | } 243 | public extension Set { 244 | public static let all: Set = [.baseCurl, .tipCurl, .fullCurl, .pinch, .spread] 245 | } 246 | -------------------------------------------------------------------------------- /Sources/HandVector/HVHandInfo+CosineSimilary.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandVectorMatcher+CosineSimilary.swift 3 | // HandVector 4 | // 5 | // Created by 许同学 on 2024/8/20. 6 | // 7 | 8 | import ARKit 9 | 10 | public extension HVHandInfo { 11 | /// Fingers joint your selected 12 | func similarity(of joints: Set, to vector: HVHandInfo) -> Float { 13 | var similarity: Float = 0 14 | similarity = joints.map { name in 15 | let dv = dot(vector.vectorEndTo(name).normalizedVector, self.vectorEndTo(name).normalizedVector) 16 | return dv 17 | }.reduce(0) { $0 + $1 } 18 | 19 | similarity /= Float(joints.count) 20 | return similarity 21 | } 22 | 23 | /// Finger your selected 24 | func similarity(of finger: HVJointOfFinger, to vector: HVHandInfo) -> Float { 25 | return similarity(of: [finger], to: vector) 26 | } 27 | /// Fingers your selected 28 | func similarity(of fingers: Set, to vector: HVHandInfo) -> Float { 29 | var similarity: Float = 0 30 | let jointNames = fingers.jointGroupNames 31 | similarity = jointNames.map { name in 32 | let dv = dot(vector.vectorEndTo(name).normalizedVector, self.vectorEndTo(name).normalizedVector) 33 | return dv 34 | }.reduce(0) { $0 + $1 } 35 | 36 | similarity /= Float(jointNames.count) 37 | return similarity 38 | } 39 | /// Fingers and wrist and forearm 40 | func similarity(to vector: HVHandInfo) -> Float { 41 | return similarity(of: .all, to: vector) 42 | } 43 | /// all 44 | func similarities(to vector: HVHandInfo) -> (average: Float, eachFinger: [HVJointOfFinger: Float]) { 45 | return averageAndEachSimilarities(of: .all, to: vector) 46 | } 47 | func averageAndEachSimilarities(of fingers: Set, to vector: HVHandInfo) -> (average: Float, eachFinger: [HVJointOfFinger: Float]) { 48 | 49 | let fingerTotal = fingers.reduce(into: [HVJointOfFinger: Float]()) { partialResult, finger in 50 | let fingerResult = finger.jointGroupNames.reduce(into: Float.zero) { partialResult, name in 51 | let dv = dot(vector.vectorEndTo(name).normalizedVector, self.vectorEndTo(name).normalizedVector) 52 | partialResult += dv 53 | } 54 | partialResult[finger] = fingerResult 55 | } 56 | let fingerScore = fingerTotal.reduce(into: [HVJointOfFinger: Float]()) { partialResult, ele in 57 | partialResult[ele.key] = ele.value / Float(ele.key.jointGroupNames.count) 58 | } 59 | 60 | let jointTotal = fingerTotal.reduce(into: Float.zero) { partialResult, element in 61 | partialResult += element.value 62 | } 63 | let jointCount = fingers.jointGroupNames.count 64 | return (average: jointTotal / Float(jointCount), eachFinger: fingerScore) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Sources/HandVector/HVHandInfo+Direction.swift: -------------------------------------------------------------------------------- 1 | // 2 | // File.swift 3 | // 4 | // 5 | // Created by 许同学 on 2024/6/5. 6 | // 7 | 8 | import ARKit 9 | 10 | public extension HVHandInfo { 11 | //world space direction 12 | public var fingersExtendedDirection: simd_float3 { 13 | return chirality == .left ? transform.columns.0.xyz : -transform.columns.0.xyz 14 | } 15 | public var thumbExtendedDirection: simd_float3 { 16 | return chirality == .left ? -transform.columns.2.xyz : transform.columns.2.xyz 17 | } 18 | public var palmDirection: simd_float3 { 19 | return chirality == .left ? transform.columns.1.xyz : -transform.columns.1.xyz 20 | } 21 | 22 | //in world space 23 | //direction: from knukle to tip of a finger 24 | public func fingerPositionDirection(of finger: HVJointOfFinger) -> (position: SIMD3, direction: SIMD3) { 25 | let tip = finger.jointGroupNames.last! 26 | let tipLocal = allJoints[tip]?.position ?? .zero 27 | let tipWorld = transform * SIMD4(tipLocal, 1) 28 | 29 | let back = chirality == .left ? SIMD3(1, 0, 0) : SIMD3(-1, 0, 0) 30 | let knukle = finger.jointGroupNames.first! 31 | let knukleLocal = allJoints[knukle]?.position ?? back 32 | let knukleWorld = transform * SIMD4(knukleLocal, 1) 33 | 34 | return (position: tipWorld.xyz, direction: simd_normalize(tipWorld.xyz - knukleWorld.xyz)) 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /Sources/HandVector/HVHandInfo.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandVectorMatcher.swift 3 | // FingerDance 4 | // 5 | // Created by 许同学 on 2024/1/2. 6 | // 7 | 8 | import Foundation 9 | import simd 10 | import ARKit 11 | 12 | public struct HVHandInfo: Sendable, Equatable { 13 | public let chirality: HandAnchor.Chirality 14 | public let allJoints: [HandSkeleton.JointName: HVJointInfo] 15 | public let transform: simd_float4x4 16 | 17 | internal let internalVectors: [HandSkeleton.JointName: InternalVectorInfo] 18 | internal func vectorEndTo(_ named: HandSkeleton.JointName) -> InternalVectorInfo { 19 | return internalVectors[named]! 20 | } 21 | public static var builtinHandInfo: [String : HVHandInfo] = { 22 | let dict = HVHandJsonModel.loadHandJsonModelDict(fileName: "BuiltinHand", bundle: handAssetsBundle)!.reduce(into: [String: HVHandInfo](), { 23 | $0[$1.key] = $1.value.convertToHVHandInfo() 24 | }) 25 | return dict 26 | }() 27 | 28 | public func calculateFingerShape(finger: HVJointOfFinger, fingerShapeTypes: Set = .all) -> HVFingerShape { 29 | let shape = HVFingerShape(finger: finger, fingerShapeTypes: fingerShapeTypes, joints: allJoints) 30 | return shape 31 | } 32 | 33 | public init?(chirality: HandAnchor.Chirality, allJoints: [HandSkeleton.JointName: HVJointInfo], transform: simd_float4x4) { 34 | if allJoints.count >= HandSkeleton.JointName.allCases.count { 35 | self.chirality = chirality 36 | self.allJoints = allJoints 37 | self.transform = transform 38 | self.internalVectors = Self.genetateVectors(from: allJoints) 39 | } else { 40 | return nil 41 | } 42 | } 43 | public init?(handAnchor: HandAnchor) { 44 | guard let handSkeleton = handAnchor.handSkeleton else { 45 | return nil 46 | } 47 | self.init(chirality: handAnchor.chirality, handSkeleton: handSkeleton, transform: handAnchor.originFromAnchorTransform) 48 | } 49 | public init(chirality: HandAnchor.Chirality, handSkeleton: HandSkeleton, transform: simd_float4x4) { 50 | self.chirality = chirality 51 | self.allJoints = Self.genetateJoints(from: handSkeleton) 52 | self.transform = transform 53 | self.internalVectors = Self.genetateVectors(from: allJoints) 54 | } 55 | 56 | 57 | public func reversedChirality() -> HVHandInfo { 58 | var infoNew: [HandSkeleton.JointName: HVJointInfo] = [:] 59 | for (name, info) in allJoints { 60 | infoNew[name] = info.reversedChirality() 61 | } 62 | let m = HVHandInfo(chirality: chirality == .left ? .right : .left, allJoints: infoNew, transform: simd_float4x4([-transform.columns.0, transform.columns.1, -transform.columns.2, transform.columns.3]))! 63 | return m 64 | } 65 | } 66 | 67 | private extension HVHandInfo { 68 | private static func genetateJoints(from handSkeleton: HandSkeleton) -> [HandSkeleton.JointName: HVJointInfo] { 69 | var joints: [HandSkeleton.JointName: HVJointInfo] = [:] 70 | HandSkeleton.JointName.allCases.forEach { jointName in 71 | joints[jointName] = HVJointInfo(joint: handSkeleton.joint(jointName)) 72 | } 73 | return joints 74 | } 75 | private static func genetateVectors(from positions: [HandSkeleton.JointName: HVJointInfo]) -> [HandSkeleton.JointName: InternalVectorInfo] { 76 | var vectors: [HandSkeleton.JointName: InternalVectorInfo] = [:] 77 | 78 | let wrist = positions[.wrist]! 79 | let forearmWrist = positions[.forearmWrist]! 80 | let forearmArm = positions[.forearmArm]! 81 | vectors[.forearmWrist] = InternalVectorInfo(from: forearmArm, to: forearmWrist) 82 | vectors[.forearmArm] = InternalVectorInfo(from: forearmWrist, to: forearmArm) 83 | vectors[.wrist] = InternalVectorInfo(from: forearmArm, to: wrist) 84 | 85 | let thumbKnuckle = positions[.thumbKnuckle]! 86 | let thumbIntermediateBase = positions[.thumbIntermediateBase]! 87 | let thumbIntermediateTip = positions[.thumbIntermediateTip]! 88 | let thumbTip = positions[.thumbTip]! 89 | vectors[.thumbKnuckle] = InternalVectorInfo(from: wrist, to: thumbKnuckle) 90 | vectors[.thumbIntermediateBase] = InternalVectorInfo(from: thumbKnuckle, to: thumbIntermediateBase) 91 | vectors[.thumbIntermediateTip] = InternalVectorInfo(from: thumbIntermediateBase, to: thumbIntermediateTip) 92 | vectors[.thumbTip] = InternalVectorInfo(from: thumbIntermediateTip, to: thumbTip) 93 | 94 | let indexFingerMetacarpal = positions[.indexFingerMetacarpal]! 95 | let indexFingerKnuckle = positions[.indexFingerKnuckle]! 96 | let indexFingerIntermediateBase = positions[.indexFingerIntermediateBase]! 97 | let indexFingerIntermediateTip = positions[.indexFingerIntermediateTip]! 98 | let indexFingerTip = positions[.indexFingerTip]! 99 | 100 | vectors[.indexFingerMetacarpal] = InternalVectorInfo(from: wrist, to: indexFingerMetacarpal) 101 | vectors[.indexFingerKnuckle] = InternalVectorInfo(from: indexFingerMetacarpal, to: indexFingerKnuckle) 102 | vectors[.indexFingerIntermediateBase] = InternalVectorInfo(from: indexFingerKnuckle, to: indexFingerIntermediateBase) 103 | vectors[.indexFingerIntermediateTip] = InternalVectorInfo(from: indexFingerIntermediateBase, to: indexFingerIntermediateTip) 104 | vectors[.indexFingerTip] = InternalVectorInfo(from: indexFingerIntermediateTip, to: indexFingerTip) 105 | 106 | let middleFingerMetacarpal = positions[.middleFingerMetacarpal]! 107 | let middleFingerKnuckle = positions[.middleFingerKnuckle]! 108 | let middleFingerIntermediateBase = positions[.middleFingerIntermediateBase]! 109 | let middleFingerIntermediateTip = positions[.middleFingerIntermediateTip]! 110 | let middleFingerTip = positions[.middleFingerTip]! 111 | 112 | vectors[.middleFingerMetacarpal] = InternalVectorInfo(from: wrist, to: middleFingerMetacarpal) 113 | vectors[.middleFingerKnuckle] = InternalVectorInfo(from: middleFingerMetacarpal, to: middleFingerKnuckle) 114 | vectors[.middleFingerIntermediateBase] = InternalVectorInfo(from: middleFingerKnuckle, to: middleFingerIntermediateBase) 115 | vectors[.middleFingerIntermediateTip] = InternalVectorInfo(from: middleFingerIntermediateBase, to: middleFingerIntermediateTip) 116 | vectors[.middleFingerTip] = InternalVectorInfo(from: middleFingerIntermediateTip, to: middleFingerTip) 117 | 118 | 119 | let ringFingerMetacarpal = positions[.ringFingerMetacarpal]! 120 | let ringFingerKnuckle = positions[.ringFingerKnuckle]! 121 | let ringFingerIntermediateBase = positions[.ringFingerIntermediateBase]! 122 | let ringFingerIntermediateTip = positions[.ringFingerIntermediateTip]! 123 | let ringFingerTip = positions[.ringFingerTip]! 124 | 125 | vectors[.ringFingerMetacarpal] = InternalVectorInfo(from: wrist, to: ringFingerMetacarpal) 126 | vectors[.ringFingerKnuckle] = InternalVectorInfo(from: ringFingerMetacarpal, to: ringFingerKnuckle) 127 | vectors[.ringFingerIntermediateBase] = InternalVectorInfo(from: ringFingerKnuckle, to: ringFingerIntermediateBase) 128 | vectors[.ringFingerIntermediateTip] = InternalVectorInfo(from: ringFingerIntermediateBase, to: ringFingerIntermediateTip) 129 | vectors[.ringFingerTip] = InternalVectorInfo(from: ringFingerIntermediateTip, to: ringFingerTip) 130 | 131 | 132 | let littleFingerMetacarpal = positions[.littleFingerMetacarpal]! 133 | let littleFingerKnuckle = positions[.littleFingerKnuckle]! 134 | let littleFingerIntermediateBase = positions[.littleFingerIntermediateBase]! 135 | let littleFingerIntermediateTip = positions[.littleFingerIntermediateTip]! 136 | let littleFingerTip = positions[.littleFingerTip]! 137 | 138 | vectors[.littleFingerMetacarpal] = InternalVectorInfo(from: wrist, to: littleFingerMetacarpal) 139 | vectors[.littleFingerKnuckle] = InternalVectorInfo(from: littleFingerMetacarpal, to: littleFingerKnuckle) 140 | vectors[.littleFingerIntermediateBase] = InternalVectorInfo(from: littleFingerKnuckle, to: littleFingerIntermediateBase) 141 | vectors[.littleFingerIntermediateTip] = InternalVectorInfo(from: littleFingerIntermediateBase, to: littleFingerIntermediateTip) 142 | vectors[.littleFingerTip] = InternalVectorInfo(from: littleFingerIntermediateTip, to: littleFingerTip) 143 | return vectors 144 | } 145 | } 146 | 147 | extension HVHandInfo { 148 | struct InternalVectorInfo: Hashable, Sendable, CustomStringConvertible { 149 | public let from: HandSkeleton.JointName 150 | public let to: HandSkeleton.JointName 151 | // relative to 'from' joint 152 | public let vector: simd_float3 153 | public let normalizedVector: simd_float3 154 | 155 | public func reversedChirality() -> InternalVectorInfo { 156 | return InternalVectorInfo(from: from, to: to, vector: -vector) 157 | } 158 | 159 | public init(from: HVJointInfo, to: HVJointInfo) { 160 | self.from = from.name 161 | self.to = to.name 162 | let position4 = SIMD4(to.position, 0) 163 | self.vector = (from.transformToParent * position4).xyz 164 | if vector == .zero { 165 | self.normalizedVector = .zero 166 | } else { 167 | self.normalizedVector = normalize(self.vector) 168 | } 169 | } 170 | private init(from: HandSkeleton.JointName, to: HandSkeleton.JointName, vector: simd_float3) { 171 | self.from = from 172 | self.to = to 173 | self.vector = vector 174 | if vector == .zero { 175 | self.normalizedVector = .zero 176 | } else { 177 | self.normalizedVector = normalize(vector) 178 | } 179 | } 180 | 181 | public var description: String { 182 | return "from: \(from),\nto: \(to),\nvector: \(vector), normalizedVector:\(normalizedVector)" 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /Sources/HandVector/HVHandJsonModel.swift: -------------------------------------------------------------------------------- 1 | // 2 | // File.swift 3 | // HandVector 4 | // 5 | // Created by 许同学 on 2024/8/5. 6 | // 7 | 8 | import RealityFoundation 9 | import ARKit 10 | 11 | public struct HVJointJsonModel: Sendable, Equatable { 12 | public let name: HandSkeleton.JointName.NameCodingKey 13 | public let isTracked: Bool 14 | public let transform: simd_float4x4 15 | } 16 | 17 | extension HVJointJsonModel: Codable { 18 | enum CodingKeys: CodingKey { 19 | case name 20 | case isTracked 21 | case transform 22 | } 23 | 24 | public init(from decoder: Decoder) throws { 25 | let container: KeyedDecodingContainer = try decoder.container(keyedBy: HVJointJsonModel.CodingKeys.self) 26 | 27 | self.name = try container.decode(HandSkeleton.JointName.NameCodingKey.self, forKey: HVJointJsonModel.CodingKeys.name) 28 | self.isTracked = try container.decode(Bool.self, forKey: HVJointJsonModel.CodingKeys.isTracked) 29 | self.transform = try simd_float4x4(container.decode([SIMD4].self, forKey: HVJointJsonModel.CodingKeys.transform)) 30 | } 31 | 32 | public func encode(to encoder: Encoder) throws { 33 | var container: KeyedEncodingContainer = encoder.container(keyedBy: HVJointJsonModel.CodingKeys.self) 34 | 35 | try container.encode(self.name, forKey: HVJointJsonModel.CodingKeys.name) 36 | try container.encode(self.isTracked, forKey: HVJointJsonModel.CodingKeys.isTracked) 37 | try container.encode(self.transform.float4Array, forKey: HVJointJsonModel.CodingKeys.transform) 38 | } 39 | } 40 | 41 | public struct HVHandJsonModel:Sendable, Equatable { 42 | public let name: String 43 | public let chirality: HandAnchor.Chirality.NameCodingKey 44 | public let transform: simd_float4x4 45 | public let joints: [HVJointJsonModel] 46 | public var description: String? 47 | 48 | public static func loadHandJsonModel(fileName: String, bundle: Bundle = Bundle.main) -> HVHandJsonModel? { 49 | guard let path = bundle.path(forResource: fileName, ofType: "json") else {return nil} 50 | do { 51 | let jsonStr = try String(contentsOfFile: path, encoding: .utf8) 52 | return jsonStr.toModel(HVHandJsonModel.self) 53 | } catch { 54 | print(error) 55 | } 56 | return nil 57 | } 58 | public static func loadHandJsonModelDict(fileName: String, bundle: Bundle = Bundle.main) -> [String: HVHandJsonModel]? { 59 | guard let path = bundle.path(forResource: fileName, ofType: "json") else {return nil} 60 | do { 61 | let jsonStr = try String(contentsOfFile: path, encoding: .utf8) 62 | return jsonStr.toModel([String: HVHandJsonModel].self) 63 | } catch { 64 | print(error) 65 | } 66 | return nil 67 | } 68 | 69 | public func convertToHVHandInfo() -> HVHandInfo? { 70 | let jsonDict = joints.reduce(into: [HandSkeleton.JointName: HVJointJsonModel]()) { 71 | $0[$1.name.jointName!] = $1 72 | } 73 | let identity = simd_float4x4.init(diagonal: .one) 74 | let allJoints = HandSkeleton.JointName.allCases.reduce(into: [HandSkeleton.JointName: HVJointInfo]()) { 75 | if let jsonJoint = jsonDict[$1] { 76 | if let parentName = $1.parentName, let parentTransform = jsonDict[parentName]?.transform { 77 | let parentIT = parentTransform.inverse * jsonJoint.transform 78 | let joint = HVJointInfo(name: jsonJoint.name.jointName!, isTracked: jsonJoint.isTracked, anchorFromJointTransform: jsonJoint.transform, parentFromJointTransform: parentIT) 79 | $0[$1] = joint 80 | } else { 81 | let joint = HVJointInfo(name: jsonJoint.name.jointName!, isTracked: jsonJoint.isTracked, anchorFromJointTransform: jsonJoint.transform, parentFromJointTransform: identity) 82 | $0[$1] = joint 83 | } 84 | } 85 | } 86 | 87 | let vector = HVHandInfo(chirality: chirality.chirality, allJoints: allJoints, transform: transform) 88 | return vector 89 | } 90 | 91 | public static func generateJsonModel(name: String, handVector: HVHandInfo, description: String? = nil) -> HVHandJsonModel { 92 | let joints = HandSkeleton.JointName.allCases.map { jointName in 93 | let joint = handVector.allJoints[jointName]! 94 | return HVJointJsonModel(name: joint.name.codableName, isTracked: joint.isTracked, transform: joint.transform) 95 | } 96 | return HVHandJsonModel(name: name, chirality: handVector.chirality.codableName, transform: handVector.transform, joints: joints, description: description) 97 | } 98 | 99 | } 100 | 101 | extension HVHandJsonModel: Codable { 102 | enum CodingKeys: CodingKey { 103 | case name 104 | case chirality 105 | case joints 106 | case transform 107 | case description 108 | } 109 | 110 | public init(from decoder: Decoder) throws { 111 | let container: KeyedDecodingContainer = try decoder.container(keyedBy: HVHandJsonModel.CodingKeys.self) 112 | 113 | self.name = try container.decode(String.self, forKey: .name) 114 | self.chirality = try container.decode(HandAnchor.Chirality.NameCodingKey.self, forKey: .chirality) 115 | self.joints = try container.decode([HVJointJsonModel].self, forKey: .joints) 116 | self.transform = try simd_float4x4(container.decode([SIMD4].self, forKey: .transform)) 117 | self.description = try container.decodeIfPresent(String.self, forKey: .description) 118 | } 119 | 120 | public func encode(to encoder: Encoder) throws { 121 | var container: KeyedEncodingContainer = encoder.container(keyedBy: HVHandJsonModel.CodingKeys.self) 122 | 123 | try container.encode(self.name, forKey: .name) 124 | try container.encode(self.chirality, forKey: .chirality) 125 | try container.encode(self.joints, forKey: .joints) 126 | try container.encode(self.transform.float4Array, forKey: .transform) 127 | try container.encodeIfPresent(self.description, forKey: .description) 128 | } 129 | } 130 | 131 | -------------------------------------------------------------------------------- /Sources/HandVector/HVJointInfo.swift: -------------------------------------------------------------------------------- 1 | // 2 | // File.swift 3 | // 4 | // 5 | // Created by 许同学 on 2024/6/5. 6 | // 7 | 8 | import ARKit 9 | 10 | 11 | public struct HVJointInfo: Sendable, Equatable { 12 | public let name: HandSkeleton.JointName 13 | public let isTracked: Bool 14 | // relative to anchor 15 | public let transform: simd_float4x4 16 | // relative to parent joint 17 | public let transformToParent: simd_float4x4 18 | 19 | public init(joint: HandSkeleton.Joint) { 20 | self.name = joint.name 21 | self.isTracked = joint.isTracked 22 | self.transform = joint.anchorFromJointTransform 23 | self.transformToParent = joint.parentFromJointTransform 24 | } 25 | 26 | public init(name: HandSkeleton.JointName, isTracked: Bool, anchorFromJointTransform: simd_float4x4, parentFromJointTransform: simd_float4x4) { 27 | self.name = name 28 | self.isTracked = isTracked 29 | self.transform = anchorFromJointTransform 30 | self.transformToParent = parentFromJointTransform 31 | } 32 | 33 | public func reversedChirality() -> HVJointInfo { 34 | return HVJointInfo(name: name, isTracked: isTracked, anchorFromJointTransform: transform.positionReversed, parentFromJointTransform: transformToParent.positionReversed) 35 | } 36 | 37 | public var position: SIMD3 { 38 | return transform.columns.3.xyz 39 | } 40 | 41 | public var description: String { 42 | return "name: \(name), isTracked: \(isTracked), position: \(transform.columns.0.xyz)" 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /Sources/HandVector/HVJointOfFinger.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HVJointOfFinger.swift 3 | // 4 | // 5 | // Created by xu on 2024/8/16. 6 | // 7 | 8 | import ARKit 9 | 10 | 11 | public enum HVJointOfFinger:Sendable, Equatable, CaseIterable { 12 | case thumb 13 | case indexFinger 14 | case middleFinger 15 | case ringFinger 16 | case littleFinger 17 | case metacarpal 18 | case forearm 19 | 20 | public var jointGroupNames: [HandSkeleton.JointName] { 21 | switch self { 22 | case .thumb: 23 | [.thumbKnuckle, .thumbIntermediateBase, .thumbIntermediateTip, .thumbTip] 24 | case .indexFinger: 25 | [.indexFingerKnuckle, .indexFingerIntermediateBase, .indexFingerIntermediateTip, .indexFingerTip] 26 | case .middleFinger: 27 | [.middleFingerKnuckle, .middleFingerIntermediateBase, .middleFingerIntermediateTip, .middleFingerTip] 28 | case .ringFinger: 29 | [.ringFingerKnuckle, .ringFingerIntermediateBase, .ringFingerIntermediateTip, .ringFingerTip] 30 | case .littleFinger: 31 | [.littleFingerKnuckle, .littleFingerIntermediateBase, .littleFingerIntermediateTip, .littleFingerTip] 32 | case .metacarpal: 33 | [.indexFingerMetacarpal, .middleFingerMetacarpal, .ringFingerMetacarpal, .littleFingerMetacarpal] 34 | case .forearm: 35 | [.forearmWrist, .forearmArm] 36 | } 37 | } 38 | } 39 | public extension Set { 40 | 41 | public static let fiveFingers: Set = [.thumb, .indexFinger, .middleFinger, .ringFinger, .littleFinger] 42 | public static let fiveFingersAndForeArm: Set = [.thumb, .indexFinger, .middleFinger, .ringFinger, .littleFinger, .forearm] 43 | public static let fiveFingersAndWrist: Set = [.thumb, .indexFinger, .middleFinger, .ringFinger, .littleFinger, .metacarpal] 44 | public static let all: Set = [.thumb, .indexFinger, .middleFinger, .ringFinger, .littleFinger, .metacarpal, .forearm] 45 | 46 | public var jointGroupNames: [HandSkeleton.JointName] { 47 | var jointNames: [HandSkeleton.JointName] = [] 48 | for finger in HVJointOfFinger.allCases { 49 | if contains(finger) { 50 | jointNames.append(contentsOf: finger.jointGroupNames) 51 | } 52 | } 53 | return jointNames 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Sources/HandVector/HandVector.swift: -------------------------------------------------------------------------------- 1 | // 2 | // File.swift 3 | // 4 | // 5 | // Created by 许同学 on 2024/2/21. 6 | // 7 | 8 | import Foundation 9 | 10 | public let handAssetsBundle = Bundle.module 11 | -------------------------------------------------------------------------------- /Sources/HandVector/HandVectorManager.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HandVectorTool.swift 3 | // FingerDance 4 | // 5 | // Created by 许同学 on 2023/12/24. 6 | // 7 | 8 | import RealityKit 9 | import ARKit 10 | 11 | public class HandVectorManager { 12 | public var left: Entity? 13 | public var right: Entity? 14 | 15 | public var leftHandVector: HVHandInfo? 16 | public var rightHandVector: HVHandInfo? 17 | 18 | public init(left: Entity? = nil, right: Entity? = nil, leftHandVector: HVHandInfo? = nil, rightHandVector: HVHandInfo? = nil) { 19 | self.left = left 20 | self.right = right 21 | self.leftHandVector = leftHandVector 22 | self.rightHandVector = rightHandVector 23 | } 24 | 25 | public var isSkeletonVisible: Bool = false { 26 | didSet { 27 | for name in HandSkeleton.JointName.allCases { 28 | let modelEntityLeft = left?.findEntity(named: name.codableName.rawValue + "-model") 29 | modelEntityLeft?.isEnabled = isSkeletonVisible 30 | 31 | let modelEntityRight = right?.findEntity(named: name.codableName.rawValue + "-model") 32 | modelEntityRight?.isEnabled = isSkeletonVisible 33 | } 34 | } 35 | } 36 | public var isCollisionEnable: Bool = false { 37 | didSet { 38 | for name in HandSkeleton.JointName.allCases { 39 | let modelEntityLeft = left?.findEntity(named: name.codableName.rawValue + "-collision") 40 | modelEntityLeft?.isEnabled = isCollisionEnable 41 | 42 | let modelEntityRight = right?.findEntity(named: name.codableName.rawValue + "-collision") 43 | modelEntityRight?.isEnabled = isCollisionEnable 44 | } 45 | } 46 | } 47 | 48 | @discardableResult 49 | public func generateHandInfo(from handAnchor: HandAnchor) -> HVHandInfo? { 50 | let handInfo = HVHandInfo(handAnchor: handAnchor) 51 | if handAnchor.chirality == .left { 52 | leftHandVector = handInfo 53 | } else { 54 | rightHandVector = handInfo 55 | } 56 | return handInfo 57 | } 58 | 59 | 60 | 61 | @MainActor 62 | public func updateHandSkeletonEntity(from handInfo: HVHandInfo, filter: CollisionFilter = .default) async { 63 | if handInfo.chirality == .left { 64 | if let left { 65 | updateHandEntity(from: handInfo, inEntiy: left) 66 | } else { 67 | left = generateHandEntity(from: handInfo, filter: filter) 68 | } 69 | } else if handInfo.chirality == .right { 70 | if let right { 71 | updateHandEntity(from: handInfo, inEntiy: right) 72 | } else { 73 | right = generateHandEntity(from: handInfo, filter: filter) 74 | } 75 | } 76 | } 77 | public func removeHand(from handAnchor: HandAnchor) { 78 | if handAnchor.chirality == .left { 79 | left?.removeFromParent() 80 | left = nil 81 | leftHandVector = nil 82 | } else if handAnchor.chirality == .right { // Update right hand info. 83 | right?.removeFromParent() 84 | right = nil 85 | rightHandVector = nil 86 | } 87 | } 88 | 89 | 90 | public static let simHandPositionY: Float = 1.4 91 | @MainActor 92 | public func updateHand(from simHand: SimHand, filter: CollisionFilter = .default) async { 93 | let handVectors = simHand.convertToHandVector(offset: .init(0, Self.simHandPositionY, -0.2)) 94 | leftHandVector = handVectors.left 95 | if let leftHandVector { 96 | if left == nil { 97 | left = generateHandEntity(from: leftHandVector, filter: filter) 98 | } else { 99 | left?.isEnabled = true 100 | updateHandEntity(from: leftHandVector, inEntiy: left!) 101 | } 102 | } else { 103 | left?.isEnabled = false 104 | } 105 | 106 | rightHandVector = handVectors.right 107 | if let rightHandVector { 108 | if right == nil { 109 | right = generateHandEntity(from: rightHandVector, filter: filter) 110 | } else { 111 | right?.isEnabled = true 112 | updateHandEntity(from: rightHandVector, inEntiy: right!) 113 | } 114 | } else { 115 | right?.isEnabled = false 116 | } 117 | } 118 | 119 | @MainActor 120 | private func generateHandEntity(from handVector: HVHandInfo, filter: CollisionFilter = .default) -> Entity { 121 | let hand = Entity() 122 | hand.name = handVector.chirality == .left ? "leftHand" : "rightHand" 123 | hand.transform.matrix = handVector.transform 124 | 125 | let wm = SimpleMaterial(color: .white, isMetallic: false) 126 | let rm = SimpleMaterial(color: .red, isMetallic: false) 127 | let gm = SimpleMaterial(color: .green, isMetallic: false) 128 | let bm = SimpleMaterial(color: .blue, isMetallic: false) 129 | let rnm = SimpleMaterial(color: .init(red: 0.5, green: 0, blue: 0, alpha: 1), isMetallic: false) 130 | let gnm = SimpleMaterial(color: .init(red: 0, green: 0.5, blue: 0, alpha: 1), isMetallic: false) 131 | let bnm = SimpleMaterial(color: .init(red: 0, green: 0, blue: 0.5, alpha: 1), isMetallic: false) 132 | for positionInfo in handVector.allJoints.values { 133 | let modelEntity = ModelEntity(mesh: .generateBox(width: 0.015, height: 0.015, depth: 0.015, splitFaces: true), materials: [bm, gm, bnm, gnm, rm, rnm])//[+z, +y, -z, -y, +x, -x] 134 | modelEntity.transform.matrix = positionInfo.transform 135 | modelEntity.name = positionInfo.name.codableName.rawValue + "-model" 136 | modelEntity.isEnabled = isSkeletonVisible 137 | hand.addChild(modelEntity) 138 | 139 | let collisionEntity = Entity() 140 | collisionEntity.components.set(CollisionComponent(shapes: [.generateBox(width: 0.015, height: 0.015, depth: 0.015)], filter: filter)) 141 | collisionEntity.transform.matrix = positionInfo.transform 142 | collisionEntity.name = positionInfo.name.codableName.rawValue + "-collision" 143 | collisionEntity.isEnabled = isCollisionEnable 144 | hand.addChild(collisionEntity) 145 | } 146 | return hand 147 | } 148 | private func updateHandEntity(from handVector: HVHandInfo, inEntiy: Entity) { 149 | inEntiy.transform.matrix = handVector.transform 150 | for positionInfo in handVector.allJoints.values { 151 | let modelEntity = inEntiy.findEntity(named: positionInfo.name.codableName.rawValue + "-model") 152 | modelEntity?.transform.matrix = positionInfo.transform 153 | modelEntity?.isEnabled = isSkeletonVisible 154 | 155 | let collisionEntity = inEntiy.findEntity(named: positionInfo.name.codableName.rawValue + "-collision") 156 | collisionEntity?.transform.matrix = positionInfo.transform 157 | collisionEntity?.isEnabled = isCollisionEnable 158 | } 159 | } 160 | } 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /Sources/HandVector/SimHands/Bonjour/BonjourSession.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import MultipeerConnectivity 3 | import os.log 4 | 5 | public typealias InvitationCompletionHandler = (_ result: Result) -> Void 6 | 7 | final public class BonjourSession: NSObject { 8 | 9 | // MARK: - Type Definitions 10 | 11 | public struct Configuration { 12 | 13 | public enum Invitation { 14 | case automatic 15 | case custom((Peer) throws -> (context: Data, timeout: TimeInterval)?) 16 | case none 17 | } 18 | 19 | public struct Security { 20 | 21 | public typealias InvitationHandler = (Peer, Data?, @escaping (Bool) -> Void) -> Void 22 | public typealias CertificateHandler = ([Any]?, MCPeerID, @escaping (Bool) -> Void) -> Void 23 | 24 | public var identity: [Any]? 25 | public var encryptionPreference: MCEncryptionPreference 26 | public var invitationHandler: InvitationHandler 27 | public var certificateHandler: CertificateHandler 28 | 29 | public init(identity: [Any]?, 30 | encryptionPreference: MCEncryptionPreference, 31 | invitationHandler: @escaping InvitationHandler, 32 | certificateHandler: @escaping CertificateHandler) { 33 | self.identity = identity 34 | self.encryptionPreference = encryptionPreference 35 | self.invitationHandler = invitationHandler 36 | self.certificateHandler = certificateHandler 37 | } 38 | 39 | public static let `default` = Security(identity: nil, 40 | encryptionPreference: .none, 41 | invitationHandler: { _, _, handler in handler(true) }, 42 | certificateHandler: { _, _, handler in handler(true) }) 43 | 44 | } 45 | 46 | public var serviceType: String 47 | public var peerName: String 48 | public var defaults: UserDefaults 49 | public var security: Security 50 | public var invitation: Invitation 51 | 52 | public init(serviceType: String, 53 | peerName: String, 54 | defaults: UserDefaults, 55 | security: Security, 56 | invitation: Invitation) { 57 | precondition(peerName.utf8.count <= 63, "peerName can't be longer than 63 bytes") 58 | 59 | self.serviceType = serviceType 60 | self.peerName = peerName 61 | self.defaults = defaults 62 | self.security = security 63 | self.invitation = invitation 64 | } 65 | 66 | public static let `default` = Configuration(serviceType: "Bonjour", 67 | peerName: MCPeerID.defaultDisplayName, 68 | defaults: .standard, 69 | security: .default, 70 | invitation: .automatic) 71 | } 72 | 73 | 74 | public enum BonjourSessionError: LocalizedError { 75 | case connectionToPeerfailed 76 | 77 | var localizedDescription: String { 78 | switch self { 79 | case .connectionToPeerfailed: return "Failed to connect to peer." 80 | } 81 | } 82 | } 83 | 84 | public struct Usage: OptionSet { 85 | public let rawValue: UInt 86 | 87 | public init(rawValue: UInt) { 88 | self.rawValue = rawValue 89 | } 90 | 91 | public static let receive = Usage(rawValue: 0x1) 92 | public static let transmit = Usage(rawValue: 0x2) 93 | public static let combined: Usage = [.receive, .transmit] 94 | } 95 | 96 | // MARK: - Public Properties 97 | 98 | public let usage: Usage 99 | public let configuration: Configuration 100 | public let localPeerID: MCPeerID 101 | 102 | public private(set) var availablePeers: Set = [] { 103 | didSet { 104 | guard self.availablePeers != oldValue 105 | else { return } 106 | self.sessionQueue.async { 107 | self.onAvailablePeersDidChange?(Array(self.availablePeers)) 108 | } 109 | } 110 | } 111 | public var connectedPeers: Set { self.availablePeers.filter { $0.isConnected } } 112 | 113 | // MARK: - Handlers 114 | 115 | public var onStartReceiving: ((_ resourceName: String, _ peer: Peer) -> Void)? 116 | public var onReceiving: ((_ resourceName: String, _ peer: Peer, _ progress: Double) -> Void)? 117 | public var onFinishReceiving: ((_ resourceName: String, _ peer: Peer, _ localURL: URL?, _ error: Error?) -> Void)? 118 | public var onReceive: ((_ data: Data, _ peer: Peer) -> Void)? 119 | public var onPeerDiscovery: ((_ peer: Peer) -> Void)? 120 | public var onPeerLoss: ((_ peer: Peer) -> Void)? 121 | public var onPeerConnection: ((_ peer: Peer) -> Void)? 122 | public var onPeerDisconnection: ((_ peer: Peer) -> Void)? 123 | public var onAvailablePeersDidChange: ((_ peers: [Peer]) -> Void)? 124 | 125 | // MARK: - Private Properties 126 | 127 | private lazy var session: MCSession = { 128 | let session = MCSession(peer: self.localPeerID, 129 | securityIdentity: self.configuration.security.identity, 130 | encryptionPreference: self.configuration.security.encryptionPreference) 131 | session.delegate = self 132 | return session 133 | }() 134 | 135 | private lazy var browser: MCNearbyServiceBrowser = { 136 | let browser = MCNearbyServiceBrowser(peer: self.localPeerID, 137 | serviceType: self.configuration.serviceType) 138 | browser.delegate = self 139 | return browser 140 | }() 141 | 142 | private lazy var advertiser: MCNearbyServiceAdvertiser = { 143 | let advertiser = MCNearbyServiceAdvertiser(peer: self.localPeerID, 144 | discoveryInfo: nil, 145 | serviceType: self.configuration.serviceType) 146 | advertiser.delegate = self 147 | return advertiser 148 | }() 149 | 150 | private var invitationCompletionHandlers: [MCPeerID: InvitationCompletionHandler] = [:] 151 | private var progressWatchers: [String: ProgressWatcher] = [:] 152 | private let sessionQueue = DispatchQueue(label: "Bonjour.Session", qos: .userInteractive) 153 | 154 | 155 | // MARK: - Init 156 | 157 | public init(usage: Usage = .combined, 158 | configuration: Configuration = .default) { 159 | self.usage = usage 160 | self.configuration = configuration 161 | self.localPeerID = MCPeerID.fetchOrCreate(with: configuration) 162 | } 163 | 164 | public func start() { 165 | #if DEBUG 166 | os_log("%{public}@", 167 | log: .default, 168 | type: .debug, 169 | #function) 170 | #endif 171 | 172 | if self.usage.contains(.receive) { 173 | self.advertiser.startAdvertisingPeer() 174 | } 175 | if self.usage.contains(.transmit) { 176 | self.browser.startBrowsingForPeers() 177 | } 178 | } 179 | 180 | public func stop() { 181 | #if DEBUG 182 | os_log("%{public}@", 183 | log: .default, 184 | type: .debug, 185 | #function) 186 | #endif 187 | 188 | if self.usage.contains(.receive) { 189 | self.advertiser.stopAdvertisingPeer() 190 | } 191 | if self.usage.contains(.transmit) { 192 | self.browser.stopBrowsingForPeers() 193 | } 194 | } 195 | 196 | public func invite(_ peer: Peer, 197 | with context: Data?, 198 | timeout: TimeInterval, 199 | completion: InvitationCompletionHandler?) { 200 | self.invitationCompletionHandlers[peer.peerID] = completion 201 | 202 | self.browser.invitePeer(peer.peerID, 203 | to: self.session, 204 | withContext: context, 205 | timeout: timeout) 206 | } 207 | 208 | public func broadcast(_ data: Data) { 209 | guard !self.session.connectedPeers.isEmpty 210 | else { 211 | #if DEBUG 212 | os_log("Not broadcasting message: no connected peers", 213 | log: .default, 214 | type: .error) 215 | #endif 216 | return 217 | } 218 | 219 | do { 220 | try self.session.send(data, 221 | toPeers: self.session.connectedPeers, 222 | with: .reliable) 223 | } catch { 224 | #if DEBUG 225 | os_log("Could not send data", 226 | log: .default, 227 | type: .error) 228 | #endif 229 | return 230 | } 231 | } 232 | 233 | public func send(_ data: Data, 234 | to peers: [Peer]) { 235 | do { 236 | try self.session.send(data, 237 | toPeers: peers.map { $0.peerID }, 238 | with: .reliable) 239 | } catch { 240 | #if DEBUG 241 | os_log("Could not send data", 242 | log: .default, 243 | type: .error) 244 | #endif 245 | return 246 | } 247 | } 248 | 249 | public func sendResource(at url: URL, 250 | resourceName: String, 251 | to peer: Peer, 252 | progressHandler: ((Double) -> Void)?, 253 | completionHandler: ((Error?) -> Void)?) { 254 | let completion: ((Error?) -> Void)? = { error in 255 | self.progressWatchers[resourceName] = nil 256 | completionHandler?(error) 257 | } 258 | 259 | let progress = self.session.sendResource(at: url, 260 | withName: resourceName, 261 | toPeer: peer.peerID, 262 | withCompletionHandler: completion) 263 | if let progress = progress, 264 | let progressHandler = progressHandler { 265 | let progressWatcher = ProgressWatcher(progress: progress) 266 | self.progressWatchers[resourceName] = progressWatcher 267 | progressWatcher.progressHandler = progressHandler 268 | } 269 | } 270 | 271 | // MARK: - Private 272 | 273 | private func didDiscover(_ peer: Peer) { 274 | self.availablePeers.insert(peer) 275 | self.onPeerDiscovery?(peer) 276 | } 277 | 278 | private func handleDidStartReceiving(resourceName: String, 279 | from peerID: MCPeerID, 280 | progress: Progress) { 281 | guard let peer = self.availablePeers.first(where: { $0.peerID == peerID }) 282 | else { return } 283 | let progressWatcher = ProgressWatcher(progress: progress) 284 | self.progressWatchers[resourceName] = progressWatcher 285 | progressWatcher.progressHandler = { progress in 286 | self.onReceiving?(resourceName, peer, progress) 287 | } 288 | self.onStartReceiving?(resourceName, peer) 289 | } 290 | 291 | private func handleDidFinishReceiving(resourceName: String, 292 | from peerID: MCPeerID, 293 | at localURL: URL?, 294 | withError error: Error?) { 295 | guard let peer = self.availablePeers.first(where: { $0.peerID == peerID }) 296 | else { return } 297 | self.progressWatchers[resourceName] = nil 298 | self.onFinishReceiving?(resourceName, peer, localURL, error) 299 | } 300 | 301 | private func handleDidReceived(_ data: Data, 302 | peerID: MCPeerID) { 303 | guard let peer = self.availablePeers.first(where: { $0.peerID == peerID }) 304 | else { return } 305 | self.onReceive?(data, peer) 306 | } 307 | 308 | private func handlePeerRemoved(_ peerID: MCPeerID) { 309 | guard let peer = self.availablePeers.first(where: { $0.peerID == peerID }) 310 | else { return } 311 | self.availablePeers.remove(peer) 312 | self.onPeerLoss?(peer) 313 | } 314 | 315 | private func handlePeerConnected(_ peer: Peer) { 316 | self.setConnected(true, on: peer) 317 | self.onPeerConnection?(peer) 318 | } 319 | 320 | private func handlePeerDisconnected(_ peer: Peer) { 321 | self.setConnected(false, on: peer) 322 | self.onPeerDisconnection?(peer) 323 | } 324 | 325 | private func setConnected(_ connected: Bool, on peer: Peer) { 326 | guard let idx = self.availablePeers.firstIndex(where: { $0.peerID == peer.peerID }) 327 | else { return } 328 | 329 | var mutablePeer = self.availablePeers[idx] 330 | mutablePeer.isConnected = connected 331 | self.availablePeers.remove(peer) 332 | self.availablePeers.insert(mutablePeer) 333 | } 334 | 335 | } 336 | 337 | // MARK: - Session delegate 338 | 339 | extension BonjourSession: MCSessionDelegate { 340 | 341 | public func session(_ session: MCSession, 342 | peer peerID: MCPeerID, 343 | didChange state: MCSessionState) { 344 | #if DEBUG 345 | os_log("%{public}@", 346 | log: .default, 347 | type: .debug, 348 | #function) 349 | #endif 350 | 351 | guard let peer = self.availablePeers.first(where: { $0.peerID == peerID }) 352 | else { return } 353 | 354 | let handler = self.invitationCompletionHandlers[peerID] 355 | 356 | self.sessionQueue.async { 357 | switch state { 358 | case .connected: 359 | handler?(.success(peer)) 360 | self.invitationCompletionHandlers[peerID] = nil 361 | self.handlePeerConnected(peer) 362 | case .notConnected: 363 | handler?(.failure(BonjourSessionError.connectionToPeerfailed)) 364 | self.invitationCompletionHandlers[peerID] = nil 365 | self.handlePeerDisconnected(peer) 366 | case .connecting: 367 | break 368 | @unknown default: 369 | break 370 | } 371 | } 372 | } 373 | 374 | public func session(_ session: MCSession, 375 | didReceive data: Data, 376 | fromPeer peerID: MCPeerID) { 377 | #if DEBUG 378 | os_log("%{public}@", 379 | log: .default, 380 | type: .debug, 381 | #function) 382 | #endif 383 | self.handleDidReceived(data, peerID: peerID) 384 | } 385 | 386 | public func session(_ session: MCSession, 387 | didReceive stream: InputStream, 388 | withName streamName: String, 389 | fromPeer peerID: MCPeerID) { 390 | #if DEBUG 391 | os_log("%{public}@", 392 | log: .default, 393 | type: .debug, 394 | #function) 395 | #endif 396 | } 397 | 398 | public func session(_ session: MCSession, 399 | didStartReceivingResourceWithName resourceName: String, 400 | fromPeer peerID: MCPeerID, 401 | with progress: Progress) { 402 | self.handleDidStartReceiving(resourceName: resourceName, 403 | from: peerID, 404 | progress: progress) 405 | #if DEBUG 406 | os_log("%{public}@", 407 | log: .default, 408 | type: .debug, 409 | #function) 410 | #endif 411 | } 412 | 413 | public func session(_ session: MCSession, 414 | didFinishReceivingResourceWithName resourceName: String, 415 | fromPeer peerID: MCPeerID, 416 | at localURL: URL?, 417 | withError error: Error?) { 418 | self.handleDidFinishReceiving(resourceName: resourceName, 419 | from: peerID, 420 | at: localURL, 421 | withError: error) 422 | #if DEBUG 423 | os_log("%{public}@", 424 | log: .default, 425 | type: .debug, 426 | #function) 427 | #endif 428 | } 429 | 430 | public func session(_ session: MCSession, 431 | didReceiveCertificate certificate: [Any]?, 432 | fromPeer peerID: MCPeerID, 433 | certificateHandler: @escaping (Bool) -> Void) { 434 | self.configuration.security.certificateHandler(certificate, 435 | peerID, 436 | certificateHandler) 437 | #if DEBUG 438 | os_log("%{public}@", 439 | log: .default, 440 | type: .debug, 441 | #function) 442 | #endif 443 | } 444 | 445 | } 446 | 447 | // MARK: - Browser delegate 448 | 449 | extension BonjourSession: MCNearbyServiceBrowserDelegate { 450 | 451 | public func browser(_ browser: MCNearbyServiceBrowser, 452 | foundPeer peerID: MCPeerID, 453 | withDiscoveryInfo info: [String : String]?) { 454 | #if DEBUG 455 | os_log("%{public}@", 456 | log: .default, 457 | type: .debug, 458 | #function) 459 | #endif 460 | 461 | do { 462 | let peer = try Peer(peer: peerID, discoveryInfo: info) 463 | 464 | self.didDiscover(peer) 465 | 466 | switch configuration.invitation { 467 | case .automatic: 468 | browser.invitePeer(peerID, 469 | to: self.session, 470 | withContext: nil, 471 | timeout: 10.0) 472 | case .custom(let inviter): 473 | guard let invite = try inviter(peer) 474 | else { 475 | #if DEBUG 476 | os_log("Custom invite not sent for peer %@", 477 | log: .default, 478 | type: .error, 479 | String(describing: peer)) 480 | #endif 481 | return 482 | } 483 | 484 | browser.invitePeer(peerID, 485 | to: self.session, 486 | withContext: invite.context, 487 | timeout: invite.timeout) 488 | case .none: 489 | #if DEBUG 490 | os_log("Auto-invite disabled", 491 | log: .default, 492 | type: .debug) 493 | #endif 494 | return 495 | } 496 | } catch { 497 | #if DEBUG 498 | os_log("Failed to initialize peer based on peer ID %@: %{public}@", 499 | log: .default, 500 | type: .error, 501 | String(describing: peerID), 502 | String(describing: error)) 503 | #endif 504 | } 505 | } 506 | 507 | public func browser(_ browser: MCNearbyServiceBrowser, 508 | lostPeer peerID: MCPeerID) { 509 | #if DEBUG 510 | os_log("%{public}@", 511 | log: .default, 512 | type: .debug, 513 | #function) 514 | #endif 515 | self.handlePeerRemoved(peerID) 516 | } 517 | 518 | public func browser(_ browser: MCNearbyServiceBrowser, 519 | didNotStartBrowsingForPeers error: Error) { 520 | #if DEBUG 521 | os_log("%{public}@", 522 | log: .default, 523 | type: .error, 524 | #function) 525 | #endif 526 | } 527 | 528 | 529 | } 530 | 531 | // MARK: - Advertiser delegate 532 | 533 | extension BonjourSession: MCNearbyServiceAdvertiserDelegate { 534 | 535 | public func advertiser(_ advertiser: MCNearbyServiceAdvertiser, 536 | didReceiveInvitationFromPeer peerID: MCPeerID, 537 | withContext context: Data?, 538 | invitationHandler: @escaping (Bool, MCSession?) -> Void) { 539 | #if DEBUG 540 | os_log("%{public}@", 541 | log: .default, 542 | type: .debug, 543 | #function) 544 | #endif 545 | 546 | guard let peer = self.availablePeers.first(where: { $0.peerID == peerID }) 547 | else { return } 548 | 549 | self.configuration.security.invitationHandler(peer, context, { [weak self] decision in 550 | guard let self = self 551 | else { return } 552 | invitationHandler(decision, decision ? self.session : nil) 553 | }) 554 | } 555 | 556 | public func advertiser(_ advertiser: MCNearbyServiceAdvertiser, 557 | didNotStartAdvertisingPeer error: Error) { 558 | #if DEBUG 559 | os_log("%{public}@", 560 | log: .default, 561 | type: .error, 562 | #function) 563 | #endif 564 | } 565 | 566 | } 567 | -------------------------------------------------------------------------------- /Sources/HandVector/SimHands/Bonjour/Extensions/MCPeerID+Extensions.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import MultipeerConnectivity.MCPeerID 3 | 4 | extension MCPeerID { 5 | 6 | static func fetchOrCreate(with config: BonjourSession.Configuration) -> MCPeerID { 7 | self.fetchExisting(with: config) ?? .init(displayName: config.peerName) 8 | } 9 | private static func fetchExisting(with config: BonjourSession.Configuration) -> MCPeerID? { 10 | guard let data = config.defaults.data(forKey: Self.defaultsKey) 11 | else { return nil } 12 | do { 13 | let peer = try NSKeyedUnarchiver.unarchivedObject(ofClass: MCPeerID.self, from: data) 14 | guard peer?.displayName == config.peerName 15 | else { return nil } 16 | return peer 17 | } catch { 18 | return nil 19 | } 20 | } 21 | private static let defaultsKey = "_bonjour.peerID" 22 | 23 | 24 | } 25 | 26 | #if os(iOS) || os(tvOS) 27 | import UIKit 28 | 29 | public extension MCPeerID { 30 | static var defaultDisplayName: String { UIDevice.current.name } 31 | } 32 | 33 | #else 34 | 35 | //import Cocoa 36 | 37 | public extension MCPeerID { 38 | static var defaultDisplayName: String { "Unknown Mac" } 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /Sources/HandVector/SimHands/Bonjour/Peer.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import MultipeerConnectivity.MCPeerID 3 | import CommonCrypto 4 | 5 | public struct Peer: Hashable, Identifiable { 6 | public let id: String 7 | public let name: String 8 | public let discoveryInfo: [String: String]? 9 | let peerID: MCPeerID 10 | public internal(set) var isConnected: Bool 11 | 12 | public init(peer: MCPeerID, discoveryInfo: [String: String]?) throws { 13 | /** 14 | According to Apple's docs, every MCPeerID is unique, therefore encoding it 15 | and hashing the resulting data is a good way to generate an unique identifier 16 | that will be always the same for the same peer ID. 17 | */ 18 | let peerData = try NSKeyedArchiver.archivedData(withRootObject: peer, requiringSecureCoding: true) 19 | self.id = peerData.idHash 20 | 21 | self.peerID = peer 22 | self.name = peer.displayName 23 | self.discoveryInfo = discoveryInfo 24 | self.isConnected = false 25 | } 26 | } 27 | 28 | fileprivate extension Data { 29 | var idHash: String { 30 | var sha1 = [UInt8](repeating: 0, count: Int(CC_SHA1_DIGEST_LENGTH)) 31 | withUnsafeBytes { _ = CC_SHA1($0.baseAddress, CC_LONG(count), &sha1) } 32 | return sha1.map({ String(format: "%02hhx", $0) }).joined() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Sources/HandVector/SimHands/Bonjour/ProgressWatcher.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | class ProgressWatcher: NSObject { 4 | 5 | enum ObservationKeyPath: String { 6 | case fractionCompleted 7 | } 8 | 9 | var progressHandler: ((Double) -> Void)? 10 | 11 | let progress: Progress 12 | private var kvoContext = 0 13 | private let observationQueue = DispatchQueue(label: "Bonjour.ProgressWatcher", qos: .userInteractive) 14 | 15 | init(progress: Progress) { 16 | self.progress = progress 17 | super.init() 18 | progress.addObserver(self, 19 | forKeyPath: ObservationKeyPath.fractionCompleted.rawValue, 20 | options: [], 21 | context: &self.kvoContext) 22 | } 23 | deinit { 24 | self.progress.removeObserver(self, 25 | forKeyPath: "fractionCompleted", 26 | context: &self.kvoContext) 27 | } 28 | 29 | override func observeValue(forKeyPath keyPath: String?, 30 | of object: Any?, 31 | change: [NSKeyValueChangeKey : Any]?, 32 | context: UnsafeMutableRawPointer?) { 33 | self.observationQueue.async { 34 | if context == &self.kvoContext { 35 | switch keyPath { 36 | case ObservationKeyPath.fractionCompleted.rawValue: 37 | if let progress = object as? Progress { 38 | self.progressHandler?(progress.fractionCompleted) 39 | } 40 | default: break 41 | } 42 | } else { 43 | super.observeValue(forKeyPath: keyPath, 44 | of: object, 45 | change: change, 46 | context: context) 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Sources/HandVector/SimHands/Bonjour/readme.md: -------------------------------------------------------------------------------- 1 | # Bonjour - by eugenebokhan 2 | 3 | This is a small for from: 4 | 5 | https://github.com/eugenebokhan/bonjour 6 | 7 | It's been modified slightly to work on VisionOS, by changing MCPeerID+Extensions.swift 8 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 56; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | 26203DBD2B2C553700840F92 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26203DBC2B2C553700840F92 /* AppDelegate.swift */; }; 11 | 26203DBF2B2C553700840F92 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26203DBE2B2C553700840F92 /* ViewController.swift */; }; 12 | 26203DC12B2C553800840F92 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 26203DC02B2C553800840F92 /* Assets.xcassets */; }; 13 | 26203DC42B2C553800840F92 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 26203DC22B2C553800840F92 /* Main.storyboard */; }; 14 | 26AF84342B2C5BF900C2458A /* index.html in Resources */ = {isa = PBXBuildFile; fileRef = 26AF84332B2C5BF900C2458A /* index.html */; }; 15 | 26AF84372B2C5FCE00C2458A /* Bonjour in Frameworks */ = {isa = PBXBuildFile; productRef = 26AF84362B2C5FCE00C2458A /* Bonjour */; }; 16 | /* End PBXBuildFile section */ 17 | 18 | /* Begin PBXFileReference section */ 19 | 26203DB92B2C553700840F92 /* VisionSimHands.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VisionSimHands.app; sourceTree = BUILT_PRODUCTS_DIR; }; 20 | 26203DBC2B2C553700840F92 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 21 | 26203DBE2B2C553700840F92 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; 22 | 26203DC02B2C553800840F92 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 23 | 26203DC32B2C553800840F92 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 24 | 26203DC52B2C553800840F92 /* VisionSimHands.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = VisionSimHands.entitlements; sourceTree = ""; }; 25 | 262918F72B2C6747000F0036 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; 26 | 26AF84332B2C5BF900C2458A /* index.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = index.html; sourceTree = ""; }; 27 | /* End PBXFileReference section */ 28 | 29 | /* Begin PBXFrameworksBuildPhase section */ 30 | 26203DB62B2C553700840F92 /* Frameworks */ = { 31 | isa = PBXFrameworksBuildPhase; 32 | buildActionMask = 2147483647; 33 | files = ( 34 | 26AF84372B2C5FCE00C2458A /* Bonjour in Frameworks */, 35 | ); 36 | runOnlyForDeploymentPostprocessing = 0; 37 | }; 38 | /* End PBXFrameworksBuildPhase section */ 39 | 40 | /* Begin PBXGroup section */ 41 | 26203DB02B2C553700840F92 = { 42 | isa = PBXGroup; 43 | children = ( 44 | 26203DBB2B2C553700840F92 /* VisionSimHands */, 45 | 26203DBA2B2C553700840F92 /* Products */, 46 | ); 47 | sourceTree = ""; 48 | }; 49 | 26203DBA2B2C553700840F92 /* Products */ = { 50 | isa = PBXGroup; 51 | children = ( 52 | 26203DB92B2C553700840F92 /* VisionSimHands.app */, 53 | ); 54 | name = Products; 55 | sourceTree = ""; 56 | }; 57 | 26203DBB2B2C553700840F92 /* VisionSimHands */ = { 58 | isa = PBXGroup; 59 | children = ( 60 | 262918F72B2C6747000F0036 /* Info.plist */, 61 | 26AF84332B2C5BF900C2458A /* index.html */, 62 | 26203DBC2B2C553700840F92 /* AppDelegate.swift */, 63 | 26203DBE2B2C553700840F92 /* ViewController.swift */, 64 | 26203DC02B2C553800840F92 /* Assets.xcassets */, 65 | 26203DC22B2C553800840F92 /* Main.storyboard */, 66 | 26203DC52B2C553800840F92 /* VisionSimHands.entitlements */, 67 | ); 68 | path = VisionSimHands; 69 | sourceTree = ""; 70 | }; 71 | /* End PBXGroup section */ 72 | 73 | /* Begin PBXNativeTarget section */ 74 | 26203DB82B2C553700840F92 /* VisionSimHands */ = { 75 | isa = PBXNativeTarget; 76 | buildConfigurationList = 26203DC82B2C553800840F92 /* Build configuration list for PBXNativeTarget "VisionSimHands" */; 77 | buildPhases = ( 78 | 26203DB52B2C553700840F92 /* Sources */, 79 | 26203DB62B2C553700840F92 /* Frameworks */, 80 | 26203DB72B2C553700840F92 /* Resources */, 81 | ); 82 | buildRules = ( 83 | ); 84 | dependencies = ( 85 | ); 86 | name = VisionSimHands; 87 | packageProductDependencies = ( 88 | 26AF84362B2C5FCE00C2458A /* Bonjour */, 89 | ); 90 | productName = VisionSimHands; 91 | productReference = 26203DB92B2C553700840F92 /* VisionSimHands.app */; 92 | productType = "com.apple.product-type.application"; 93 | }; 94 | /* End PBXNativeTarget section */ 95 | 96 | /* Begin PBXProject section */ 97 | 26203DB12B2C553700840F92 /* Project object */ = { 98 | isa = PBXProject; 99 | attributes = { 100 | BuildIndependentTargetsInParallel = 1; 101 | LastSwiftUpdateCheck = 1500; 102 | LastUpgradeCheck = 1500; 103 | TargetAttributes = { 104 | 26203DB82B2C553700840F92 = { 105 | CreatedOnToolsVersion = 15.0; 106 | }; 107 | }; 108 | }; 109 | buildConfigurationList = 26203DB42B2C553700840F92 /* Build configuration list for PBXProject "VisionSimHands" */; 110 | compatibilityVersion = "Xcode 14.0"; 111 | developmentRegion = en; 112 | hasScannedForEncodings = 0; 113 | knownRegions = ( 114 | en, 115 | Base, 116 | ); 117 | mainGroup = 26203DB02B2C553700840F92; 118 | packageReferences = ( 119 | 26AF84352B2C5FCE00C2458A /* XCRemoteSwiftPackageReference "Bonjour" */, 120 | ); 121 | productRefGroup = 26203DBA2B2C553700840F92 /* Products */; 122 | projectDirPath = ""; 123 | projectRoot = ""; 124 | targets = ( 125 | 26203DB82B2C553700840F92 /* VisionSimHands */, 126 | ); 127 | }; 128 | /* End PBXProject section */ 129 | 130 | /* Begin PBXResourcesBuildPhase section */ 131 | 26203DB72B2C553700840F92 /* Resources */ = { 132 | isa = PBXResourcesBuildPhase; 133 | buildActionMask = 2147483647; 134 | files = ( 135 | 26AF84342B2C5BF900C2458A /* index.html in Resources */, 136 | 26203DC12B2C553800840F92 /* Assets.xcassets in Resources */, 137 | 26203DC42B2C553800840F92 /* Main.storyboard in Resources */, 138 | ); 139 | runOnlyForDeploymentPostprocessing = 0; 140 | }; 141 | /* End PBXResourcesBuildPhase section */ 142 | 143 | /* Begin PBXSourcesBuildPhase section */ 144 | 26203DB52B2C553700840F92 /* Sources */ = { 145 | isa = PBXSourcesBuildPhase; 146 | buildActionMask = 2147483647; 147 | files = ( 148 | 26203DBF2B2C553700840F92 /* ViewController.swift in Sources */, 149 | 26203DBD2B2C553700840F92 /* AppDelegate.swift in Sources */, 150 | ); 151 | runOnlyForDeploymentPostprocessing = 0; 152 | }; 153 | /* End PBXSourcesBuildPhase section */ 154 | 155 | /* Begin PBXVariantGroup section */ 156 | 26203DC22B2C553800840F92 /* Main.storyboard */ = { 157 | isa = PBXVariantGroup; 158 | children = ( 159 | 26203DC32B2C553800840F92 /* Base */, 160 | ); 161 | name = Main.storyboard; 162 | sourceTree = ""; 163 | }; 164 | /* End PBXVariantGroup section */ 165 | 166 | /* Begin XCBuildConfiguration section */ 167 | 26203DC62B2C553800840F92 /* Debug */ = { 168 | isa = XCBuildConfiguration; 169 | buildSettings = { 170 | ALWAYS_SEARCH_USER_PATHS = NO; 171 | ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; 172 | CLANG_ANALYZER_NONNULL = YES; 173 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 174 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; 175 | CLANG_ENABLE_MODULES = YES; 176 | CLANG_ENABLE_OBJC_ARC = YES; 177 | CLANG_ENABLE_OBJC_WEAK = YES; 178 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 179 | CLANG_WARN_BOOL_CONVERSION = YES; 180 | CLANG_WARN_COMMA = YES; 181 | CLANG_WARN_CONSTANT_CONVERSION = YES; 182 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 183 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 184 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 185 | CLANG_WARN_EMPTY_BODY = YES; 186 | CLANG_WARN_ENUM_CONVERSION = YES; 187 | CLANG_WARN_INFINITE_RECURSION = YES; 188 | CLANG_WARN_INT_CONVERSION = YES; 189 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 190 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 191 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 192 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 193 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 194 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 195 | CLANG_WARN_STRICT_PROTOTYPES = YES; 196 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 197 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 198 | CLANG_WARN_UNREACHABLE_CODE = YES; 199 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 200 | COPY_PHASE_STRIP = NO; 201 | DEBUG_INFORMATION_FORMAT = dwarf; 202 | ENABLE_STRICT_OBJC_MSGSEND = YES; 203 | ENABLE_TESTABILITY = YES; 204 | ENABLE_USER_SCRIPT_SANDBOXING = YES; 205 | GCC_C_LANGUAGE_STANDARD = gnu17; 206 | GCC_DYNAMIC_NO_PIC = NO; 207 | GCC_NO_COMMON_BLOCKS = YES; 208 | GCC_OPTIMIZATION_LEVEL = 0; 209 | GCC_PREPROCESSOR_DEFINITIONS = ( 210 | "DEBUG=1", 211 | "$(inherited)", 212 | ); 213 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 214 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 215 | GCC_WARN_UNDECLARED_SELECTOR = YES; 216 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 217 | GCC_WARN_UNUSED_FUNCTION = YES; 218 | GCC_WARN_UNUSED_VARIABLE = YES; 219 | LOCALIZATION_PREFERS_STRING_CATALOGS = YES; 220 | MACOSX_DEPLOYMENT_TARGET = 14.0; 221 | MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; 222 | MTL_FAST_MATH = YES; 223 | ONLY_ACTIVE_ARCH = YES; 224 | SDKROOT = macosx; 225 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; 226 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 227 | }; 228 | name = Debug; 229 | }; 230 | 26203DC72B2C553800840F92 /* Release */ = { 231 | isa = XCBuildConfiguration; 232 | buildSettings = { 233 | ALWAYS_SEARCH_USER_PATHS = NO; 234 | ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; 235 | CLANG_ANALYZER_NONNULL = YES; 236 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 237 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; 238 | CLANG_ENABLE_MODULES = YES; 239 | CLANG_ENABLE_OBJC_ARC = YES; 240 | CLANG_ENABLE_OBJC_WEAK = YES; 241 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 242 | CLANG_WARN_BOOL_CONVERSION = YES; 243 | CLANG_WARN_COMMA = YES; 244 | CLANG_WARN_CONSTANT_CONVERSION = YES; 245 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 246 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 247 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 248 | CLANG_WARN_EMPTY_BODY = YES; 249 | CLANG_WARN_ENUM_CONVERSION = YES; 250 | CLANG_WARN_INFINITE_RECURSION = YES; 251 | CLANG_WARN_INT_CONVERSION = YES; 252 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 253 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 254 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 255 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 256 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 257 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 258 | CLANG_WARN_STRICT_PROTOTYPES = YES; 259 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 260 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 261 | CLANG_WARN_UNREACHABLE_CODE = YES; 262 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 263 | COPY_PHASE_STRIP = NO; 264 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 265 | ENABLE_NS_ASSERTIONS = NO; 266 | ENABLE_STRICT_OBJC_MSGSEND = YES; 267 | ENABLE_USER_SCRIPT_SANDBOXING = YES; 268 | GCC_C_LANGUAGE_STANDARD = gnu17; 269 | GCC_NO_COMMON_BLOCKS = YES; 270 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 271 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 272 | GCC_WARN_UNDECLARED_SELECTOR = YES; 273 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 274 | GCC_WARN_UNUSED_FUNCTION = YES; 275 | GCC_WARN_UNUSED_VARIABLE = YES; 276 | LOCALIZATION_PREFERS_STRING_CATALOGS = YES; 277 | MACOSX_DEPLOYMENT_TARGET = 14.0; 278 | MTL_ENABLE_DEBUG_INFO = NO; 279 | MTL_FAST_MATH = YES; 280 | SDKROOT = macosx; 281 | SWIFT_COMPILATION_MODE = wholemodule; 282 | }; 283 | name = Release; 284 | }; 285 | 26203DC92B2C553800840F92 /* Debug */ = { 286 | isa = XCBuildConfiguration; 287 | buildSettings = { 288 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 289 | ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; 290 | CODE_SIGN_ENTITLEMENTS = VisionSimHands/VisionSimHands.entitlements; 291 | CODE_SIGN_STYLE = Automatic; 292 | COMBINE_HIDPI_IMAGES = YES; 293 | CURRENT_PROJECT_VERSION = 1; 294 | DEVELOPMENT_TEAM = 67UBXXU7S9; 295 | ENABLE_HARDENED_RUNTIME = YES; 296 | GENERATE_INFOPLIST_FILE = YES; 297 | INFOPLIST_FILE = VisionSimHands/Info.plist; 298 | INFOPLIST_KEY_NSCameraUsageDescription = "We need camera access to track your hands"; 299 | INFOPLIST_KEY_NSHumanReadableCopyright = ""; 300 | INFOPLIST_KEY_NSLocalNetworkUsageDescription = "We need to access your local network to stream hand data"; 301 | INFOPLIST_KEY_NSMainStoryboardFile = Main; 302 | INFOPLIST_KEY_NSMicrophoneUsageDescription = "We need microphone access to allow the embedded webview to start the gerUserMedia session"; 303 | INFOPLIST_KEY_NSPrincipalClass = NSApplication; 304 | LD_RUNPATH_SEARCH_PATHS = ( 305 | "$(inherited)", 306 | "@executable_path/../Frameworks", 307 | ); 308 | MARKETING_VERSION = 1.0; 309 | PRODUCT_BUNDLE_IDENTIFIER = "com.lumen-digital.VisionSimHands.xu"; 310 | PRODUCT_NAME = "$(TARGET_NAME)"; 311 | SWIFT_EMIT_LOC_STRINGS = YES; 312 | SWIFT_VERSION = 5.0; 313 | }; 314 | name = Debug; 315 | }; 316 | 26203DCA2B2C553800840F92 /* Release */ = { 317 | isa = XCBuildConfiguration; 318 | buildSettings = { 319 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 320 | ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; 321 | CODE_SIGN_ENTITLEMENTS = VisionSimHands/VisionSimHands.entitlements; 322 | CODE_SIGN_STYLE = Automatic; 323 | COMBINE_HIDPI_IMAGES = YES; 324 | CURRENT_PROJECT_VERSION = 1; 325 | DEVELOPMENT_TEAM = 67UBXXU7S9; 326 | ENABLE_HARDENED_RUNTIME = YES; 327 | GENERATE_INFOPLIST_FILE = YES; 328 | INFOPLIST_FILE = VisionSimHands/Info.plist; 329 | INFOPLIST_KEY_NSCameraUsageDescription = "We need camera access to track your hands"; 330 | INFOPLIST_KEY_NSHumanReadableCopyright = ""; 331 | INFOPLIST_KEY_NSLocalNetworkUsageDescription = "We need to access your local network to stream hand data"; 332 | INFOPLIST_KEY_NSMainStoryboardFile = Main; 333 | INFOPLIST_KEY_NSMicrophoneUsageDescription = "We need microphone access to allow the embedded webview to start the gerUserMedia session"; 334 | INFOPLIST_KEY_NSPrincipalClass = NSApplication; 335 | LD_RUNPATH_SEARCH_PATHS = ( 336 | "$(inherited)", 337 | "@executable_path/../Frameworks", 338 | ); 339 | MARKETING_VERSION = 1.0; 340 | PRODUCT_BUNDLE_IDENTIFIER = "com.lumen-digital.VisionSimHands.xu"; 341 | PRODUCT_NAME = "$(TARGET_NAME)"; 342 | SWIFT_EMIT_LOC_STRINGS = YES; 343 | SWIFT_VERSION = 5.0; 344 | }; 345 | name = Release; 346 | }; 347 | /* End XCBuildConfiguration section */ 348 | 349 | /* Begin XCConfigurationList section */ 350 | 26203DB42B2C553700840F92 /* Build configuration list for PBXProject "VisionSimHands" */ = { 351 | isa = XCConfigurationList; 352 | buildConfigurations = ( 353 | 26203DC62B2C553800840F92 /* Debug */, 354 | 26203DC72B2C553800840F92 /* Release */, 355 | ); 356 | defaultConfigurationIsVisible = 0; 357 | defaultConfigurationName = Release; 358 | }; 359 | 26203DC82B2C553800840F92 /* Build configuration list for PBXNativeTarget "VisionSimHands" */ = { 360 | isa = XCConfigurationList; 361 | buildConfigurations = ( 362 | 26203DC92B2C553800840F92 /* Debug */, 363 | 26203DCA2B2C553800840F92 /* Release */, 364 | ); 365 | defaultConfigurationIsVisible = 0; 366 | defaultConfigurationName = Release; 367 | }; 368 | /* End XCConfigurationList section */ 369 | 370 | /* Begin XCRemoteSwiftPackageReference section */ 371 | 26AF84352B2C5FCE00C2458A /* XCRemoteSwiftPackageReference "Bonjour" */ = { 372 | isa = XCRemoteSwiftPackageReference; 373 | repositoryURL = "https://github.com/eugenebokhan/Bonjour.git"; 374 | requirement = { 375 | kind = upToNextMajorVersion; 376 | minimumVersion = 2.1.0; 377 | }; 378 | }; 379 | /* End XCRemoteSwiftPackageReference section */ 380 | 381 | /* Begin XCSwiftPackageProductDependency section */ 382 | 26AF84362B2C5FCE00C2458A /* Bonjour */ = { 383 | isa = XCSwiftPackageProductDependency; 384 | package = 26AF84352B2C5FCE00C2458A /* XCRemoteSwiftPackageReference "Bonjour" */; 385 | productName = Bonjour; 386 | }; 387 | /* End XCSwiftPackageProductDependency section */ 388 | }; 389 | rootObject = 26203DB12B2C553700840F92 /* Project object */; 390 | } 391 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "pins" : [ 3 | { 4 | "identity" : "bonjour", 5 | "kind" : "remoteSourceControl", 6 | "location" : "https://github.com/eugenebokhan/Bonjour.git", 7 | "state" : { 8 | "revision" : "4e2912cddb3dcc8ec48370851ba0eb19f1a3e6af", 9 | "version" : "2.1.0" 10 | } 11 | } 12 | ], 13 | "version" : 2 14 | } 15 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AppDelegate.swift 3 | // VisionSimHands 4 | // 5 | // Created by Ben Harraway on 15/12/2023. 6 | // 7 | 8 | import Cocoa 9 | 10 | @main 11 | class AppDelegate: NSObject, NSApplicationDelegate { 12 | 13 | 14 | 15 | 16 | func applicationDidFinishLaunching(_ aNotification: Notification) { 17 | // Insert code here to initialize your application 18 | } 19 | 20 | func applicationWillTerminate(_ aNotification: Notification) { 21 | // Insert code here to tear down your application 22 | } 23 | 24 | func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { 25 | return true 26 | } 27 | 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/Assets.xcassets/AccentColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "idiom" : "universal" 5 | } 6 | ], 7 | "info" : { 8 | "author" : "xcode", 9 | "version" : 1 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "mac", 5 | "scale" : "1x", 6 | "size" : "16x16" 7 | }, 8 | { 9 | "idiom" : "mac", 10 | "scale" : "2x", 11 | "size" : "16x16" 12 | }, 13 | { 14 | "idiom" : "mac", 15 | "scale" : "1x", 16 | "size" : "32x32" 17 | }, 18 | { 19 | "idiom" : "mac", 20 | "scale" : "2x", 21 | "size" : "32x32" 22 | }, 23 | { 24 | "idiom" : "mac", 25 | "scale" : "1x", 26 | "size" : "128x128" 27 | }, 28 | { 29 | "idiom" : "mac", 30 | "scale" : "2x", 31 | "size" : "128x128" 32 | }, 33 | { 34 | "idiom" : "mac", 35 | "scale" : "1x", 36 | "size" : "256x256" 37 | }, 38 | { 39 | "idiom" : "mac", 40 | "scale" : "2x", 41 | "size" : "256x256" 42 | }, 43 | { 44 | "idiom" : "mac", 45 | "scale" : "1x", 46 | "size" : "512x512" 47 | }, 48 | { 49 | "idiom" : "mac", 50 | "scale" : "2x", 51 | "size" : "512x512" 52 | } 53 | ], 54 | "info" : { 55 | "author" : "xcode", 56 | "version" : 1 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | NSBonjourServices 6 | 7 | _Bonjour._tcp 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/ViewController.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ViewController.swift 3 | // VisionSimHands 4 | // 5 | // Created by Ben Harraway on 15/12/2023. 6 | // 7 | 8 | import Cocoa 9 | import WebKit 10 | import Bonjour 11 | 12 | class ViewController: NSViewController, WKUIDelegate, WKScriptMessageHandler { 13 | 14 | let bonjour = BonjourSession(configuration: .default) 15 | var webView: WKWebView! 16 | 17 | override func loadView() { 18 | let contentController = WKUserContentController() 19 | contentController.add(self, name: "callback") 20 | 21 | let webConfiguration = WKWebViewConfiguration () 22 | webConfiguration.preferences.setValue(true, forKey: "allowFileAccessFromFileURLs") 23 | webConfiguration.userContentController = contentController 24 | 25 | webView = WKWebView (frame: CGRect(x:0, y:0, width:800, height:600), configuration:webConfiguration) 26 | webView.uiDelegate = self 27 | 28 | view = webView 29 | } 30 | 31 | override func viewDidLoad() { 32 | super.viewDidLoad() 33 | 34 | self.view.wantsLayer = true 35 | self.view.layer?.backgroundColor = .init(red: 255.0, green: 0.0, blue: 0.0, alpha: 1.0) 36 | 37 | webView.translatesAutoresizingMaskIntoConstraints = false 38 | webView.leadingAnchor.constraint(equalTo: self.view.leadingAnchor).isActive = true 39 | webView.trailingAnchor.constraint(equalTo: self.view.trailingAnchor).isActive = true 40 | webView.topAnchor.constraint(equalTo: self.view.topAnchor).isActive = true 41 | webView.bottomAnchor.constraint(equalTo: self.view.bottomAnchor).isActive = true 42 | 43 | webView.loadFileURL(Bundle.main.url(forResource: "index", withExtension: "html")!, allowingReadAccessTo: Bundle.main.bundleURL) 44 | } 45 | 46 | override var representedObject: Any? { 47 | didSet { 48 | // Update the view, if already loaded. 49 | } 50 | } 51 | 52 | func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) { 53 | // Data incoming from WKWebView... 54 | guard let response = message.body as? String else { return } 55 | 56 | if (response == "start_server") { 57 | bonjour.start() 58 | bonjour.onPeerConnection = { peer in 59 | print("Hello peer", peer) 60 | } 61 | bonjour.onPeerLoss = { peer in 62 | print("Lost peer", peer) 63 | } 64 | bonjour.onPeerDisconnection = { peer in 65 | print("Disconnected peer", peer) 66 | } 67 | 68 | } else if (response == "stop_server") { 69 | bonjour.stop() 70 | 71 | } else if let data = response.data(using: .utf8) { 72 | bonjour.broadcast(data) 73 | } 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/VisionSimHands.entitlements: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.apple.security.app-sandbox 6 | 7 | com.apple.security.device.camera 8 | 9 | com.apple.security.files.user-selected.read-only 10 | 11 | com.apple.security.network.client 12 | 13 | com.apple.security.network.server 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /macOS Helper/VisionSimHands/index.html: -------------------------------------------------------------------------------- 1 | 14 | 15 | 16 | 17 | 18 | 19 | 139 | 140 | 141 | 142 | 155 | 156 | 319 | --------------------------------------------------------------------------------