├── .gitattributes ├── .gitignore ├── Content └── Demo │ ├── BP_OXRPawn.uasset │ ├── IA_OXR_Submit.uasset │ ├── IMC_OXRDemo.uasset │ ├── M_OXRHand.uasset │ ├── OXR_TestMap.umap │ └── WBP_DebugHandTracking.uasset ├── FSOpenXRHandTracking.uplugin ├── LICENSE.md ├── README.md ├── Resources └── Icon128.png └── Source └── FSOpenXRHandTracking ├── FSOpenXRHandTracking.Build.cs ├── Private ├── FSInstancedHand.cpp └── FSOpenXRHandTracking.cpp └── Public ├── FSInstancedHand.h └── FSOpenXRHandTracking.h /.gitattributes: -------------------------------------------------------------------------------- 1 | # UE file types 2 | *.uasset filter=lfs diff=lfs merge=lfs -text 3 | *.umap filter=lfs diff=lfs merge=lfs -text 4 | *.so filter=lfs diff=lfs merge=lfs -text 5 | *.dylib filter=lfs diff=lfs merge=lfs -text 6 | *.a filter=lfs diff=lfs merge=lfs -text 7 | *.dll filter=lfs diff=lfs merge=lfs -text 8 | *.lib filter=lfs diff=lfs merge=lfs -text 9 | 10 | # Raw Content types 11 | *.fbx filter=lfs diff=lfs merge=lfs -text 12 | *.3ds filter=lfs diff=lfs merge=lfs -text 13 | *.psd filter=lfs diff=lfs merge=lfs -text 14 | *.png filter=lfs diff=lfs merge=lfs -text 15 | *.mp3 filter=lfs diff=lfs merge=lfs -text 16 | *.wav filter=lfs diff=lfs merge=lfs -text 17 | *.xcf filter=lfs diff=lfs merge=lfs -text 18 | *.jpg filter=lfs diff=lfs merge=lfs -text -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Visual Studio 2015 user specific files 2 | .vs/ 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | 22 | # Compiled Static libraries 23 | *.lai 24 | *.la 25 | *.a 26 | *.lib 27 | 28 | # Executables 29 | *.exe 30 | *.out 31 | *.app 32 | *.ipa 33 | 34 | # These project files can be generated by the engine 35 | *.xcodeproj 36 | *.xcworkspace 37 | *.sln 38 | *.suo 39 | *.opensdf 40 | *.sdf 41 | *.VC.db 42 | *.VC.opendb 43 | 44 | # Precompiled Assets 45 | SourceArt/**/*.png 46 | SourceArt/**/*.tga 47 | 48 | # Binary Files 49 | Binaries/* 50 | Plugins/*/Binaries/* 51 | 52 | # Builds 53 | Build/* 54 | 55 | # Whitelist PakBlacklist-.txt files 56 | !Build/*/ 57 | Build/*/** 58 | !Build/*/PakBlacklist*.txt 59 | 60 | # Don't ignore icon files in Build 61 | !Build/**/*.ico 62 | 63 | # Built data for maps 64 | #*_BuiltData.uasset 65 | 66 | # Configuration files generated by the Editor 67 | Saved/* 68 | 69 | # Compiled source files for the engine to use 70 | Intermediate/* 71 | Plugins/*/Intermediate/* 72 | 73 | # Cache files for the editor to use 74 | DerivedDataCache/* 75 | 76 | .idea 77 | .vsconfig 78 | .github 79 | Config/.DS_Store 80 | Plugins/.DS_Store 81 | *.DS_Store 82 | 83 | Makefile 84 | *.code-workspace -------------------------------------------------------------------------------- /Content/Demo/BP_OXRPawn.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b5e4223c3efbac294be557b01d0534a8c2da7e6a1e329f9551645aff365d3785 3 | size 209021 4 | -------------------------------------------------------------------------------- /Content/Demo/IA_OXR_Submit.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f6165c438f7618f1360f50a0dd71b20176e6a4c5a2d349162096e3f1cba1dca0 3 | size 1408 4 | -------------------------------------------------------------------------------- /Content/Demo/IMC_OXRDemo.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:062273dc07f291546a8587386c39242fc29983c795c0c35df92f11864ad7402b 3 | size 2820 4 | -------------------------------------------------------------------------------- /Content/Demo/M_OXRHand.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:87fe47a1ac582f9d52240bf2f410bec3d7bcd49cb9b60577445a965dc2148042 3 | size 10230 4 | -------------------------------------------------------------------------------- /Content/Demo/OXR_TestMap.umap: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:6b765d3370ca24327b951abb80e022a8857ddfa2052fc1378b355bfa1fd4b3db 3 | size 52518 4 | -------------------------------------------------------------------------------- /Content/Demo/WBP_DebugHandTracking.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7a68e7c3becb6318eab93a7cad029fb9b2aa3fbbdf9d1752fb193ec35895b97b 3 | size 205953 4 | -------------------------------------------------------------------------------- /FSOpenXRHandTracking.uplugin: -------------------------------------------------------------------------------- 1 | { 2 | "FileVersion": 3, 3 | "Version": 1, 4 | "VersionName": "1.0", 5 | "FriendlyName": "FSOpenXRHandTracking", 6 | "Description": "Hand Tracking plugin using OpenXR Hand Tracking extensions", 7 | "Category": "Virtual Reality", 8 | "CreatedBy": "Yannick Comte", 9 | "CreatedByURL": "https://www.demonixis.net", 10 | "DocsURL": "https://www.demonixis.net", 11 | "MarketplaceURL": "", 12 | "SupportURL": "contact@demonixis.net", 13 | "CanContainContent": true, 14 | "IsBetaVersion": false, 15 | "IsExperimentalVersion": false, 16 | "Installed": false, 17 | "Modules": [ 18 | { 19 | "Name": "FSOpenXRHandTracking", 20 | "Type": "Runtime", 21 | "LoadingPhase": "Default" 22 | } 23 | ], 24 | "Plugins": [ 25 | { 26 | "Name": "OpenXR", 27 | "Enabled": true, 28 | "SupportedTargetPlatforms": [ 29 | "Win64", 30 | "Linux", 31 | "Android", 32 | "VisionOS" 33 | ] 34 | }, 35 | { 36 | "Name": "OpenXRHandTracking", 37 | "Enabled": true, 38 | "SupportedTargetPlatforms": [ 39 | "Win64", 40 | "Linux", 41 | "Android", 42 | "VisionOS" 43 | ] 44 | }, 45 | { 46 | "Name": "XRVisualization", 47 | "Enabled": true 48 | }, 49 | { 50 | "Name": "EnhancedInput", 51 | "Enabled": true 52 | }, 53 | { 54 | "Name": "OculusXR", 55 | "Optional": true, 56 | "Enabled": true, 57 | "SupportedTargetPlatforms": [ 58 | "Win64", 59 | "Android" 60 | ] 61 | } 62 | ] 63 | } -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Yannick Comte 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenXR Hand Tracking for Unreal Engine 5 2 | 3 | This is a plugin that allows you to use hand tracking using the OpenXR plugin and the Hand Tracking extension. 4 | This plugin was tested on Meta Quest 3, but must work on any other device that supports the Hand Tracking extension. 5 | 6 | ## Requirements 7 | - Current Unreal Engine version: 5.4.x 8 | 9 | ## Features 10 | - Basic hand rendering with many options 11 | - Pinch detection (optional) 12 | - Enhanced Input Support (optional) 13 | - Hand Ray support (optional) 14 | - MetaXR bridge (Experimental) 15 | 16 | ## Getting started 17 | You've to add an `UFSInstancedHand` component parented to your `VROrigin` component and select weither it's a `left` or `right` hand. The `UFSInstancedHand` component inherits from `UInstancedStaticMeshComponent`, that means the hand is rendered into one draw call. You can assign a material for each hand, or keep the default material. You can use this plugin in a C++ and Blueprint projects, all function are exposed to Blueprint. 18 | The final step is to call the `UpdateHand(const FXRMotionControllerData& InData, const float DeltaTime)` function in the `Tick` function. 19 | 20 | ### MetaXR Support 21 | When MetaXR plugin is enabled, Hand Tracking data are not valid and the plugin can't work. That's why there is a function called `GetDataFromSkeleton(UPoseableMeshComponent* Target, const bool bLeft, FXRMotionControllerData& OutData)`, that allows you to retrieve a valid `FXRMotionControllerData`. 22 | 23 | You've to add the OculusXRHands to your Pawn, then pass each hand to the function. You can use this function in both Blueprint and C++, here is a C++ example: 24 | 25 | ```cpp 26 | FXRMotionControllerData HandData; 27 | 28 | UHeadMountedDisplayFunctionLibrary::GetMotionControllerData(this, EControllerHand::Left, HandData); 29 | 30 | #if WITH_METAXR 31 | UFSInstancedHand::GetDataFromSkeleton(LeftHandSkeleton, true, HandData); 32 | #endif 33 | 34 | LeftHandTracking->UpdateHand(HandData, DeltaSeconds); 35 | ``` 36 | 37 | Finally, you've to enable MetaXR support by passing the `bMetaXREnabled` private var to `true` in `FSOpenXRHandTracking.Build.cs`. 38 | 39 | ### Hand rendering 40 | Here are the supported rendering modes: 41 | - `EFSOpenXRHandRendering::InstancedMesh` displays a mesh on each bone 42 | - `EFSOpenXRHandRendering::Wireframe` displays wireframe 43 | - `EFSOpenXRHandRendering::Both` (default), displays both meshes and wireframe 44 | 45 | #### Basic settings 46 | | Parameter | Description | Default | 47 | |-----------|-------------|---------| 48 | | `BoneScale` | Size of a rendered bone | `0.015f` | 49 | | `bLeftHand` | Set to `true` for the left hand | `false` | 50 | | `PinchThreshold` | Pinch detection threshold | `1.5f` | 51 | | `bHideHand` | Hide 3D hands but keeps the logic working (pinch/gestures) | `False` | 52 | | `bOnlyDisplayTups` | Only display tip bones | `False` | 53 | | `bComputeRelativeRotations` | Compute relative rotations, for use with gesture recognizer for instance | `False` | 54 | 55 | #### Wireframe settings 56 | | Parameter | Description | Default | 57 | |-----------|-------------|---------| 58 | | `HandRendering` | Hand rendering mode | `Both` | 59 | | `WireframeColor` | Wireframe color | `FColor::Blue` | 60 | | `WireframeThickness` | Wireframe Thickness | `0.5f ` | 61 | | `bRenderWireframePalm` | Render palm wireframe | `false` | 62 | | `bRenderWireframeBones` | Render wireframe bones | `false` | 63 | | `HandPointerDepth` | Pointer Depth when rendering wireframe | `1.0f` | 64 | 65 | ### Pinch detection & Enhanced Input System 66 | You can check using the `IsPinching(const EFSOpenXRPinchFingers Finger)` function if a finger is pinching or not. 67 | It's also possible to use the Enhanced Input System to trigger an `UInputAction` during a finger pinch. Check the `RegisterInputAction(UInputAction* InInputAction, const EFSOpenXRPinchFingers Finger)` function. You can control the pinch detection threshold using the `PinchThreshold` parameter. 68 | 69 | ### Hand Ray follower 70 | You can setup a hand ray (for instance a scaled cylinder and a `UWidgetInteractionComponent`). The system will move the hand ray at the correct location and rotation, adding an angle to the ray and some lag. The lag is required because the hand is constantly moving and you can't interact easily with UI elements without that. To enable this feature, you've to first register a `USceneComponent` node, that will be moved using the `RegisterHandRay(USceneComponent* InRayContainer)` 71 | 72 | #### Hand Ray settings 73 | | Parameter | Description | Default | 74 | |-----------|-------------|---------| 75 | | `HandPointerAngleFromPalm` | The angle applied from the palm to the Ray | `-45.0f` | 76 | | `bHideHandPointerWhenNotTracked` | Hide hand ray if hands are not tracked | `false` | 77 | | `HandPointerLocationSpeed` | Ray movement speed | `8.0f` | 78 | | `HandPointerRotationSpeed` | Ray rotation speed | `2.0f` | 79 | 80 | ## What's planned 81 | - Basic Gesture detector 82 | - Enhanced Input System support for the gesture detector 83 | - Skeletal mesh support 84 | 85 | ## Contribution 86 | Feel free to fork and contribute :) 87 | 88 | ## License 89 | This project is released under the MIT license. Please read the `LICENSE.md` file for more informations. 90 | -------------------------------------------------------------------------------- /Resources/Icon128.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f7239efaeefbd82de33ebe18518e50de075ea4188a468a9e4991396433d2275f 3 | size 12699 4 | -------------------------------------------------------------------------------- /Source/FSOpenXRHandTracking/FSOpenXRHandTracking.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | 5 | public class FSOpenXRHandTracking : ModuleRules 6 | { 7 | private const bool bMetaXREnabled = false; 8 | 9 | public FSOpenXRHandTracking(ReadOnlyTargetRules Target) : base(Target) 10 | { 11 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 12 | 13 | PublicDependencyModuleNames.AddRange(new [] 14 | { 15 | "Core", 16 | "EnhancedInput", 17 | "HeadMountedDisplay", 18 | "XRBase", 19 | "XRVisualization" 20 | }); 21 | 22 | PrivateDependencyModuleNames.AddRange(new [] { 23 | "CoreUObject", 24 | "Engine", 25 | "Slate", 26 | "SlateCore" 27 | }); 28 | 29 | bool bMetaXRSupported = (Target.Platform == UnrealTargetPlatform.Android || 30 | Target.Platform == UnrealTargetPlatform.Win64) && 31 | bMetaXREnabled; 32 | 33 | if (bMetaXRSupported) 34 | { 35 | PrivateDependencyModuleNames.Add("OculusXRInput"); 36 | } 37 | 38 | PublicDefinitions.Add($"WITH_METAXR={(bMetaXRSupported ? 1 : 0)}"); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Source/FSOpenXRHandTracking/Private/FSInstancedHand.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | 4 | #include "FSInstancedHand.h" 5 | #include "EnhancedInputSubsystems.h" 6 | #include "XRVisualizationFunctionLibrary.h" 7 | #include "InputActionValue.h" 8 | #include "InputModifiers.h" 9 | #include "InputTriggers.h" 10 | #if WITH_METAXR 11 | #include "OculusXRInputFunctionLibrary.h" 12 | #endif 13 | #include "Components/PoseableMeshComponent.h" 14 | #include "Kismet/GameplayStatics.h" 15 | 16 | UFSInstancedHand::UFSInstancedHand() 17 | { 18 | bHandTracked = false; 19 | bPreviousHandTracked = false; 20 | bLeftHand = false; 21 | HandRendering = EFSOpenXRHandRendering::Both; 22 | PinchThreshold = 1.5f; 23 | BoneScale = 0.015f; 24 | WireframeColor = FColor::Blue; 25 | WireframeThickness = 0.35f; 26 | bRenderWireframePalm = false; 27 | bRenderWireframeBones = false; 28 | bUpdateHandPointer = false; 29 | PointerContainer = nullptr; 30 | HandPointerAngleFromPalm = -45.0f; 31 | HandPointerLocationSpeed = 8.0f; 32 | HandPointerRotationSpeed = 2.0f; 33 | HandPointerDepth = 0; 34 | bHideHand = false; 35 | bHideHandPointerWhenNotTracked = false; 36 | BoneLocations.Init(FVector::ZeroVector, EHandKeypointCount); 37 | BoneRotations.Init(FRotator::ZeroRotator, EHandKeypointCount); 38 | BoneRelativeRotations.Init(FRotator::ZeroRotator, EHandKeypointCount); 39 | 40 | // Use the default Cube by default 41 | const auto MeshAsset = 42 | ConstructorHelpers::FObjectFinder(TEXT("StaticMesh'/Engine/BasicShapes/Cube.Cube'")); 43 | 44 | if (MeshAsset.Object != nullptr) 45 | UStaticMeshComponent::SetStaticMesh(MeshAsset.Object); 46 | 47 | constexpr int InputActionCount = static_cast(EFSOpenXRPinchFingers::Little) + 1; 48 | InputActions.Init(nullptr, InputActionCount); // 4 Pinchable fingers. 49 | } 50 | 51 | FTransform UFSInstancedHand::GetHandTransform() const 52 | { 53 | return bHandTracked ? CurrentHandTransform : FallbackTransform; 54 | } 55 | 56 | bool UFSInstancedHand::UpdateHand(const FXRMotionControllerData& InData, const float DeltaTime) 57 | { 58 | ClearInstances(); 59 | 60 | bHandTracked = InData.bValid; 61 | 62 | if (bHandTracked != bPreviousHandTracked) 63 | { 64 | bPreviousHandTracked = bHandTracked; 65 | 66 | // Enable/Disable the hand ray if needed. 67 | if (bUpdateHandPointer && PointerContainer != nullptr) 68 | PointerContainer->SetVisibility(bHandTracked, true); 69 | 70 | HandTrackingEnableChanged.Broadcast(bLeftHand, bHandTracked); 71 | } 72 | 73 | if (!bHandTracked) return false; 74 | 75 | // Render wireframe if needed 76 | if (!bHideHand && HandRendering == EFSOpenXRHandRendering::Both || HandRendering == 77 | EFSOpenXRHandRendering::Wireframe) 78 | { 79 | RenderFinger(InData, EHandKeypoint::ThumbMetacarpal, EHandKeypoint::ThumbTip); 80 | RenderFinger(InData, EHandKeypoint::IndexMetacarpal, EHandKeypoint::IndexTip); 81 | RenderFinger(InData, EHandKeypoint::MiddleMetacarpal, EHandKeypoint::MiddleTip); 82 | RenderFinger(InData, EHandKeypoint::RingMetacarpal, EHandKeypoint::RingTip); 83 | RenderFinger(InData, EHandKeypoint::LittleMetacarpal, EHandKeypoint::LittleTip); 84 | } 85 | 86 | FTransform BoneTransform; 87 | 88 | constexpr uint8 PalmIndex = static_cast(EHandKeypoint::Palm); 89 | 90 | // Populate array data and add instances. 91 | for (int i = 0; i < InData.HandKeyPositions.Num(); i++) 92 | { 93 | BoneLocations[i] = InData.HandKeyPositions[i]; 94 | BoneRotations[i] = InData.HandKeyRotations[i].Rotator(); 95 | 96 | if (i == PalmIndex) 97 | { 98 | CurrentHandTransform.SetLocation(InData.HandKeyPositions[i]); 99 | CurrentHandTransform.SetRotation(InData.HandKeyRotations[i]); 100 | } 101 | 102 | BoneTransform.SetLocation(InData.HandKeyPositions[i]); 103 | BoneTransform.SetRotation(InData.HandKeyRotations[i]); 104 | 105 | const float BoneScaleFactor = InData.HandKeyRadii[i] * BoneScale; 106 | FVector BoneScale3D = FVector(BoneScaleFactor, BoneScaleFactor, BoneScaleFactor); 107 | BoneTransform.SetScale3D(BoneScale3D); 108 | 109 | bool bDisplayBone = !bHideHand; 110 | 111 | if (bDisplayBone && bOnlyDisplayTips) 112 | { 113 | bDisplayBone = 114 | i == static_cast(EHandKeypoint::ThumbTip) || 115 | i == static_cast(EHandKeypoint::IndexTip) || 116 | i == static_cast(EHandKeypoint::MiddleTip) || 117 | i == static_cast(EHandKeypoint::RingTip) || 118 | i == static_cast(EHandKeypoint::LittleTip); 119 | } 120 | 121 | if (bDisplayBone) 122 | AddInstance(BoneTransform, true); 123 | } 124 | 125 | // Broadcast Pinch Events 126 | for (int i = 0; i < InputActions.Num(); i++) 127 | { 128 | if (InputActions[i] == nullptr) continue; 129 | const EFSOpenXRPinchFingers Finger = static_cast(i); 130 | 131 | if (IsPinching(Finger)) 132 | OverrideInputWithAction(InputActions[i], 1.0f); 133 | } 134 | 135 | // Update the Hand Pointer if needed 136 | if (bUpdateHandPointer && PointerContainer != nullptr) 137 | { 138 | // Get the current pointer transform 139 | const FVector PointerLocation = PointerContainer->GetComponentLocation(); 140 | const FRotator PointerRotation = PointerContainer->GetComponentRotation(); 141 | 142 | // Get the target pointer transform and add an angle to the ray 143 | const FVector PalmLocation = BoneLocations[PalmIndex]; 144 | FRotator PalmRotation = BoneRotations[PalmIndex]; 145 | PalmRotation.Pitch += HandPointerAngleFromPalm; 146 | 147 | // Move the container 148 | const FVector TargetLocation = FMath::Lerp(PointerLocation, PalmLocation, DeltaTime * HandPointerLocationSpeed); 149 | const FRotator TargetRotation = 150 | FMath::Lerp(PointerRotation, PalmRotation, DeltaTime * HandPointerRotationSpeed); 151 | PointerContainer->SetWorldLocationAndRotation(TargetLocation, TargetRotation); 152 | } 153 | 154 | #if !WITH_METAXR 155 | if (bComputeRelativeRotations) 156 | { 157 | for (int i = 0; i < InData.HandKeyRotations.Num(); ++i) 158 | { 159 | const int ParentIndex = GetParentIndex(static_cast(i)); 160 | if (ParentIndex != INDEX_NONE) 161 | { 162 | FQuat ParentQuat = InData.HandKeyRotations[ParentIndex]; 163 | FQuat BoneQuat = InData.HandKeyRotations[i]; 164 | FQuat RelativeQuat = ParentQuat.Inverse() * BoneQuat; 165 | BoneRelativeRotations[i] = RelativeQuat.Rotator(); 166 | } 167 | else 168 | { 169 | BoneRelativeRotations[i] = InData.HandKeyRotations[i].Rotator(); 170 | } 171 | } 172 | } 173 | #endif 174 | 175 | return true; 176 | } 177 | 178 | void UFSInstancedHand::GetDataFromSkeleton(UPoseableMeshComponent* Target, const bool bLeft, 179 | FXRMotionControllerData& OutData) 180 | { 181 | #if WITH_METAXR 182 | const FTransform Pose = UOculusXRInputFunctionLibrary::GetPointerPose( 183 | bLeft ? EOculusXRHandType::HandLeft : EOculusXRHandType::HandRight); 184 | 185 | OutData.bValid = Target->IsVisible(); 186 | OutData.HandIndex = bLeft ? EControllerHand::Left : EControllerHand::Right; 187 | OutData.AimPosition = Pose.GetLocation(); 188 | OutData.AimRotation = Pose.GetRotation(); 189 | OutData.DeviceName = bLeft ? "OculusXRLeftHand" : "OculusXRRightHand"; 190 | OutData.TrackingStatus = OutData.bValid ? ETrackingStatus::Tracked : ETrackingStatus::NotTracked; 191 | OutData.DeviceVisualType = EXRVisualType::Hand; 192 | OutData.HandKeyPositions.Empty(); 193 | OutData.HandKeyRotations.Empty(); 194 | OutData.HandKeyRadii.Empty(); 195 | 196 | EHandKeypoint HandKeyPoint; 197 | uint8 BoneIndex; 198 | EOculusXRBone Bone; 199 | FString BoneName; 200 | FName BoneFName; 201 | FVector WorldLocation; 202 | FRotator WorldRotation; 203 | 204 | for (int i = 0; i < EHandKeypointCount; i++) 205 | { 206 | HandKeyPoint = static_cast(i); 207 | BoneIndex = GetOculusBone(HandKeyPoint); 208 | Bone = static_cast(BoneIndex); 209 | BoneName = UOculusXRInputFunctionLibrary::GetBoneName(Bone); 210 | BoneFName = FName(*BoneName); 211 | 212 | WorldLocation = Target->GetBoneLocationByName(BoneFName, EBoneSpaces::WorldSpace); 213 | WorldRotation = Target->GetBoneRotationByName(BoneFName, EBoneSpaces::WorldSpace); 214 | 215 | if (HandKeyPoint == EHandKeypoint::Palm) 216 | { 217 | OutData.PalmPosition = WorldLocation; 218 | OutData.PalmRotation = WorldRotation.Quaternion(); 219 | OutData.GripPosition = WorldLocation; 220 | OutData.GripRotation = WorldRotation.Quaternion(); 221 | } 222 | 223 | OutData.HandKeyPositions.Add(WorldLocation); 224 | OutData.HandKeyRotations.Add(WorldRotation.Quaternion()); 225 | OutData.HandKeyRadii.Add(0.2f); 226 | } 227 | #endif 228 | } 229 | 230 | bool UFSInstancedHand::IsHandTracked() const 231 | { 232 | return bHandTracked; 233 | } 234 | 235 | bool UFSInstancedHand::IsPinching(const EFSOpenXRPinchFingers Finger) const 236 | { 237 | constexpr uint8 ThumbIndex = static_cast(EHandKeypoint::ThumbTip); 238 | uint8 OtherIndex = static_cast(EHandKeypoint::IndexTip); 239 | 240 | if (Finger == EFSOpenXRPinchFingers::Middle) 241 | OtherIndex = static_cast(EHandKeypoint::MiddleTip); 242 | else if (Finger == EFSOpenXRPinchFingers::Ring) 243 | OtherIndex = static_cast(EHandKeypoint::RingTip); 244 | else if (Finger == EFSOpenXRPinchFingers::Little) 245 | OtherIndex = static_cast(EHandKeypoint::LittleTip); 246 | 247 | return FVector::Dist(BoneLocations[ThumbIndex], BoneLocations[OtherIndex]) <= PinchThreshold; 248 | } 249 | 250 | void UFSInstancedHand::RegisterInputAction(const EFSOpenXRPinchFingers Finger, UInputAction* InInputAction) 251 | { 252 | const int Index = static_cast(Finger); 253 | InputActions[Index] = InInputAction; 254 | } 255 | 256 | void UFSInstancedHand::RegisterHandRay(USceneComponent* InRayContainer) 257 | { 258 | PointerContainer = InRayContainer; 259 | bUpdateHandPointer = InRayContainer != nullptr; 260 | } 261 | 262 | FRotator UFSInstancedHand::GetBoneRotation(const EHandKeypoint Keypoint) const 263 | { 264 | const int Index = static_cast(Keypoint); 265 | return BoneRotations[Index]; 266 | } 267 | 268 | FVector UFSInstancedHand::GetBoneLocation(const EHandKeypoint Keypoint) const 269 | { 270 | const int Index = static_cast(Keypoint); 271 | return BoneLocations[Index]; 272 | } 273 | 274 | FRotator UFSInstancedHand::GetBoneRelativeRotation(const EHandKeypoint Keypoint) const 275 | { 276 | #if WITH_METAXR 277 | const uint8 BoneId = GetOculusBone(Keypoint); 278 | const EOculusXRBone BoneIndex = static_cast(BoneId); 279 | const FQuat BoneQuat = UOculusXRInputFunctionLibrary::GetBoneRotation( 280 | bLeftHand ? EOculusXRHandType::HandLeft : EOculusXRHandType::HandRight, BoneIndex); 281 | return BoneQuat.Rotator(); 282 | #else 283 | const int Index = static_cast(Keypoint); 284 | return BoneRelativeRotations[Index]; 285 | #endif 286 | } 287 | 288 | void UFSInstancedHand::OverrideInputWithAction(const UInputAction* InInputAction, const float Value) const 289 | { 290 | const APlayerController* PC = UGameplayStatics::GetPlayerController(GetWorld(), 0); 291 | auto* Subsystem = ULocalPlayer::GetSubsystem(PC->GetLocalPlayer()); 292 | 293 | Subsystem->InjectInputForAction( 294 | InInputAction, 295 | FInputActionValue(Value), 296 | TArray(), 297 | TArray()); 298 | } 299 | 300 | int32 UFSInstancedHand::GetParentIndex(EHandKeypoint Keypoint) 301 | { 302 | static TMap ParentMap; 303 | if (ParentMap.Num() == 0) 304 | { 305 | // Thumb 306 | ParentMap.Add(EHandKeypoint::ThumbTip, EHandKeypoint::ThumbDistal); 307 | ParentMap.Add(EHandKeypoint::ThumbDistal, EHandKeypoint::ThumbProximal); 308 | ParentMap.Add(EHandKeypoint::ThumbProximal, EHandKeypoint::ThumbMetacarpal); 309 | 310 | // Index 311 | ParentMap.Add(EHandKeypoint::IndexTip, EHandKeypoint::IndexDistal); 312 | ParentMap.Add(EHandKeypoint::IndexDistal, EHandKeypoint::IndexIntermediate); 313 | ParentMap.Add(EHandKeypoint::IndexIntermediate, EHandKeypoint::IndexProximal); 314 | ParentMap.Add(EHandKeypoint::IndexProximal, EHandKeypoint::IndexMetacarpal); 315 | 316 | // Middle 317 | ParentMap.Add(EHandKeypoint::MiddleTip, EHandKeypoint::MiddleDistal); 318 | ParentMap.Add(EHandKeypoint::MiddleDistal, EHandKeypoint::MiddleIntermediate); 319 | ParentMap.Add(EHandKeypoint::MiddleIntermediate, EHandKeypoint::MiddleProximal); 320 | ParentMap.Add(EHandKeypoint::MiddleProximal, EHandKeypoint::MiddleMetacarpal); 321 | 322 | // Ring 323 | ParentMap.Add(EHandKeypoint::RingTip, EHandKeypoint::RingDistal); 324 | ParentMap.Add(EHandKeypoint::RingDistal, EHandKeypoint::RingIntermediate); 325 | ParentMap.Add(EHandKeypoint::RingIntermediate, EHandKeypoint::RingProximal); 326 | ParentMap.Add(EHandKeypoint::RingProximal, EHandKeypoint::RingMetacarpal); 327 | 328 | // Little 329 | ParentMap.Add(EHandKeypoint::LittleTip, EHandKeypoint::LittleDistal); 330 | ParentMap.Add(EHandKeypoint::LittleDistal, EHandKeypoint::LittleIntermediate); 331 | ParentMap.Add(EHandKeypoint::LittleIntermediate, EHandKeypoint::LittleProximal); 332 | ParentMap.Add(EHandKeypoint::LittleProximal, EHandKeypoint::LittleMetacarpal); 333 | 334 | // Metacarpals to Palm 335 | ParentMap.Add(EHandKeypoint::ThumbMetacarpal, EHandKeypoint::Palm); 336 | ParentMap.Add(EHandKeypoint::IndexMetacarpal, EHandKeypoint::Palm); 337 | ParentMap.Add(EHandKeypoint::MiddleMetacarpal, EHandKeypoint::Palm); 338 | ParentMap.Add(EHandKeypoint::RingMetacarpal, EHandKeypoint::Palm); 339 | ParentMap.Add(EHandKeypoint::LittleMetacarpal, EHandKeypoint::Palm); 340 | 341 | // Wrist to Palm 342 | ParentMap.Add(EHandKeypoint::Palm, EHandKeypoint::Wrist); 343 | } 344 | 345 | if (ParentMap.Contains(Keypoint)) 346 | { 347 | return static_cast(ParentMap[Keypoint]); 348 | } 349 | return INDEX_NONE; 350 | } 351 | 352 | uint8 UFSInstancedHand::GetOculusBone(EHandKeypoint Keypoint) 353 | { 354 | #if WITH_METAXR 355 | if (Keypoint == EHandKeypoint::Wrist || Keypoint == EHandKeypoint::Palm) 356 | return static_cast(EOculusXRBone::Hand_Start); 357 | 358 | if (Keypoint == EHandKeypoint::ThumbMetacarpal) 359 | return static_cast(EOculusXRBone::Thumb_1); 360 | if (Keypoint == EHandKeypoint::ThumbProximal) 361 | return static_cast(EOculusXRBone::Thumb_2); 362 | if (Keypoint == EHandKeypoint::ThumbDistal) 363 | return static_cast(EOculusXRBone::Thumb_3); 364 | if (Keypoint == EHandKeypoint::ThumbTip) 365 | return static_cast(EOculusXRBone::Thumb_Tip); 366 | 367 | if (Keypoint == EHandKeypoint::IndexProximal || Keypoint == EHandKeypoint::IndexMetacarpal) 368 | return static_cast(EOculusXRBone::Index_1); 369 | if (Keypoint == EHandKeypoint::IndexIntermediate) 370 | return static_cast(EOculusXRBone::Index_2); 371 | if (Keypoint == EHandKeypoint::IndexDistal) 372 | return static_cast(EOculusXRBone::Index_3); 373 | if (Keypoint == EHandKeypoint::IndexTip) 374 | return static_cast(EOculusXRBone::Index_Tip); 375 | 376 | if (Keypoint == EHandKeypoint::MiddleProximal || Keypoint == EHandKeypoint::MiddleMetacarpal) 377 | return static_cast(EOculusXRBone::Middle_1); 378 | if (Keypoint == EHandKeypoint::MiddleIntermediate) 379 | return static_cast(EOculusXRBone::Middle_2); 380 | if (Keypoint == EHandKeypoint::MiddleDistal) 381 | return static_cast(EOculusXRBone::Middle_3); 382 | if (Keypoint == EHandKeypoint::MiddleTip) 383 | return static_cast(EOculusXRBone::Middle_Tip); 384 | 385 | if (Keypoint == EHandKeypoint::RingProximal || Keypoint == EHandKeypoint::RingMetacarpal) 386 | return static_cast(EOculusXRBone::Ring_1); 387 | if (Keypoint == EHandKeypoint::RingIntermediate) 388 | return static_cast(EOculusXRBone::Ring_2); 389 | if (Keypoint == EHandKeypoint::RingDistal) 390 | return static_cast(EOculusXRBone::Ring_3); 391 | if (Keypoint == EHandKeypoint::RingTip) 392 | return static_cast(EOculusXRBone::Ring_Tip); 393 | 394 | if (Keypoint == EHandKeypoint::LittleMetacarpal) 395 | return static_cast(EOculusXRBone::Pinky_0); 396 | if (Keypoint == EHandKeypoint::LittleProximal) 397 | return static_cast(EOculusXRBone::Pinky_1); 398 | if (Keypoint == EHandKeypoint::LittleIntermediate) 399 | return static_cast(EOculusXRBone::Pinky_2); 400 | if (Keypoint == EHandKeypoint::LittleDistal) 401 | return static_cast(EOculusXRBone::Pinky_3); 402 | if (Keypoint == EHandKeypoint::LittleTip) 403 | return static_cast(EOculusXRBone::Pinky_Tip); 404 | #endif 405 | 406 | return 0; 407 | } 408 | 409 | // Code forked from UXRVisualizationFunctionLibrary::RenderFinger 410 | void UFSInstancedHand::RenderFinger(const FXRMotionControllerData& InData, const EHandKeypoint FingerStart, 411 | const EHandKeypoint FingerEnd) const 412 | { 413 | const uint32 IndexStart = static_cast(FingerStart); 414 | const uint32 IndexStop = static_cast(FingerEnd); 415 | 416 | const bool bValid = ((IndexStart > 0) && (IndexStart < EHandKeypointCount)) && 417 | ((IndexStop > 0) && (IndexStop < EHandKeypointCount)) && 418 | (EHandKeypointCount == InData.HandKeyPositions.Num()) && (EHandKeypointCount == InData.HandKeyRadii.Num()); 419 | 420 | if (!bValid) return; 421 | 422 | const UWorld* World = GetWorld(); 423 | 424 | if (bRenderWireframePalm) 425 | { 426 | DrawDebugLine(World, InData.HandKeyPositions[0], InData.HandKeyPositions[1], WireframeColor, false, 427 | -1, HandPointerDepth, WireframeThickness); 428 | DrawDebugLine(World, InData.HandKeyPositions[1], InData.HandKeyPositions[IndexStart], WireframeColor, 429 | false, -1, HandPointerDepth, WireframeThickness); 430 | } 431 | 432 | //Iterate from FingerStart to Finger End 433 | for (uint32 DigitIndex = IndexStart; DigitIndex < IndexStop; ++DigitIndex) 434 | { 435 | DrawDebugLine(World, InData.HandKeyPositions[DigitIndex], InData.HandKeyPositions[DigitIndex + 1], 436 | WireframeColor, false, -1, HandPointerDepth, WireframeThickness); 437 | 438 | if (bRenderWireframeBones) 439 | { 440 | DrawDebugSphere(World, InData.HandKeyPositions[DigitIndex + 1], InData.HandKeyRadii[DigitIndex + 1], 4, 441 | WireframeColor, false, -1, HandPointerDepth, WireframeThickness); 442 | } 443 | } 444 | } 445 | -------------------------------------------------------------------------------- /Source/FSOpenXRHandTracking/Private/FSOpenXRHandTracking.cpp: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #include "FSOpenXRHandTracking.h" 4 | 5 | #define LOCTEXT_NAMESPACE "FFSOpenXRHandTrackingModule" 6 | 7 | void FFSOpenXRHandTrackingModule::StartupModule() 8 | { 9 | // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module 10 | } 11 | 12 | void FFSOpenXRHandTrackingModule::ShutdownModule() 13 | { 14 | // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, 15 | // we call this function before unloading the module. 16 | } 17 | 18 | #undef LOCTEXT_NAMESPACE 19 | 20 | IMPLEMENT_MODULE(FFSOpenXRHandTrackingModule, FSOpenXRHandTracking) -------------------------------------------------------------------------------- /Source/FSOpenXRHandTracking/Public/FSInstancedHand.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "HeadMountedDisplayTypes.h" 7 | #include "InputAction.h" 8 | #include "Components/InstancedStaticMeshComponent.h" 9 | #include "FSInstancedHand.generated.h" 10 | 11 | UENUM(BlueprintType) 12 | enum class EFSOpenXRPinchFingers : uint8 13 | { 14 | Index UMETA(DisplayName="Index"), 15 | Middle UMETA(DisplayName="Middle"), 16 | Ring UMETA(DisplayName="Ring"), 17 | Little UMETA(DisplayName="Little") 18 | }; 19 | 20 | UENUM(BlueprintType) 21 | enum class EFSOpenXRHandRendering : uint8 22 | { 23 | InstancedMesh UMETA(DisplayName="InstancedMesh"), 24 | Wireframe UMETA(DisplayName="Wireframe"), 25 | Both UMETA(DisplayName="Both") 26 | }; 27 | 28 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FHandTrackingEnabledDelegate, bool, bLeft, bool, bEnabled); 29 | 30 | UCLASS(ClassGroup=(Custom), meta=(BlueprintSpawnableComponent)) 31 | class FSOPENXRHANDTRACKING_API UFSInstancedHand : public UInstancedStaticMeshComponent 32 | { 33 | GENERATED_BODY() 34 | 35 | bool bHandTracked; 36 | bool bPreviousHandTracked; 37 | FTransform CurrentHandTransform; 38 | 39 | UPROPERTY() 40 | TArray BoneLocations; 41 | UPROPERTY() 42 | TArray BoneRotations; 43 | UPROPERTY() 44 | TArray BoneRelativeRotations; 45 | UPROPERTY() 46 | TArray InputActions; 47 | 48 | public: 49 | // Settings 50 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 51 | bool bHideHand; 52 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 53 | bool bOnlyDisplayTips; 54 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 55 | float BoneScale; 56 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 57 | bool bLeftHand; 58 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 59 | float PinchThreshold; 60 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 61 | FTransform FallbackTransform; 62 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Settings") 63 | bool bComputeRelativeRotations; 64 | 65 | // Rendering 66 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Rendering") 67 | EFSOpenXRHandRendering HandRendering; 68 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Rendering") 69 | FColor WireframeColor; 70 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Rendering") 71 | float WireframeThickness; 72 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Rendering") 73 | bool bRenderWireframePalm; 74 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Rendering") 75 | bool bRenderWireframeBones; 76 | 77 | // Hand Pointer 78 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 79 | bool bUpdateHandPointer; 80 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 81 | USceneComponent* PointerContainer; 82 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 83 | float HandPointerAngleFromPalm; 84 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 85 | bool bHideHandPointerWhenNotTracked; 86 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 87 | float HandPointerLocationSpeed; 88 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 89 | float HandPointerRotationSpeed; 90 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category="FSOpenXRHandTracking|Hand Pointer") 91 | int HandPointerDepth; 92 | 93 | // Events 94 | UPROPERTY(VisibleAnywhere, BlueprintAssignable) 95 | FHandTrackingEnabledDelegate HandTrackingEnableChanged; 96 | 97 | UFSInstancedHand(); 98 | 99 | // Public & Blueprint functions 100 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 101 | FTransform GetHandTransform() const; 102 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 103 | bool UpdateHand(const FXRMotionControllerData& InData, const float DeltaTime); 104 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 105 | static void GetDataFromSkeleton(UPoseableMeshComponent* Target, const bool bLeft, FXRMotionControllerData& OutData); 106 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 107 | bool IsHandTracked() const; 108 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 109 | bool IsPinching(const EFSOpenXRPinchFingers Finger) const; 110 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 111 | void RegisterInputAction(const EFSOpenXRPinchFingers Finger, UInputAction* InInputAction); 112 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 113 | void RegisterHandRay(USceneComponent* InRayContainer); 114 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 115 | FRotator GetBoneRotation(const EHandKeypoint Keypoint) const; 116 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 117 | FVector GetBoneLocation(const EHandKeypoint Keypoint) const; 118 | UFUNCTION(BlueprintCallable, Category="FSOpenXRHandTracking|Blueprint") 119 | FRotator GetBoneRelativeRotation(const EHandKeypoint Keypoint) const; 120 | 121 | private: 122 | void RenderFinger(const FXRMotionControllerData& InData, const EHandKeypoint FingerStart, 123 | const EHandKeypoint FingerEnd) const; 124 | void OverrideInputWithAction(const UInputAction* InInputAction, const float Value) const; 125 | static int32 GetParentIndex(EHandKeypoint Keypoint); 126 | 127 | static uint8 GetOculusBone(EHandKeypoint Keypoint); 128 | }; 129 | -------------------------------------------------------------------------------- /Source/FSOpenXRHandTracking/Public/FSOpenXRHandTracking.h: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Modules/ModuleManager.h" 7 | 8 | class FFSOpenXRHandTrackingModule : public IModuleInterface 9 | { 10 | public: 11 | 12 | /** IModuleInterface implementation */ 13 | virtual void StartupModule() override; 14 | virtual void ShutdownModule() override; 15 | }; 16 | --------------------------------------------------------------------------------