├── .gitattributes ├── .gitignore ├── AzureKinect.uplugin ├── Content ├── ABP_AureKinectSkeletonToHumanoid.uasset ├── AzureKinectActor.uasset ├── AzureKinectDevice.uasset ├── BP_AzureKinectActor.uasset ├── BP_AzureKinectActor_MultiUser.uasset ├── NS_KinectParticle.uasset └── RenderTargets │ ├── RT_KinectBodyIndex.uasset │ ├── RT_KinectColor.uasset │ ├── RT_KinectDepth.uasset │ └── RT_KinectIR.uasset ├── Docs ├── Sequence 01.gif ├── animation.gif ├── animgraph.jpg ├── bp.png ├── in-editor.gif └── kinect.png ├── README.md └── Source ├── AzureKinect ├── AzureKinect.Build.cs ├── Private │ ├── AnimNode_AzureKinectPose.cpp │ ├── AzureKinectDevice.cpp │ ├── AzureKinectDeviceThread.cpp │ └── AzureKinectModule.cpp └── Public │ ├── AnimNode_AzureKinectPose.h │ ├── AzureKinectDevice.h │ ├── AzureKinectDeviceThread.h │ └── AzureKinectEnum.h └── AzureKinectEditor ├── AzureKinectEditor.Build.cs └── Private ├── AnimNodes ├── AnimGraphNode_AzureKinectPose.cpp └── AnimGraphNode_AzureKinectPose.h ├── AssetTools ├── AzureKinectDeviceActions.cpp └── AzureKinectDeviceActions.h ├── AzureKinectEditor.cpp ├── Customizations ├── AzureKinectDeviceCustomization.cpp └── AzureKinectDeviceCustomization.h └── Factories ├── AzureKinectDeviceFactory.cpp └── AzureKinectDeviceFactory.h /.gitattributes: -------------------------------------------------------------------------------- 1 | # Unreal Engine file types. 2 | *.uasset filter=lfs diff=lfs merge=lfs -text 3 | *.umap filter=lfs diff=lfs merge=lfs -text 4 | 5 | # Raw Content file types. 6 | *.fbx filter=lfs diff=lfs merge=lfs -text 7 | *.3ds filter=lfs diff=lfs merge=lfs -text 8 | *.psd filter=lfs diff=lfs merge=lfs -text 9 | *.png filter=lfs diff=lfs merge=lfs -text 10 | *.mp3 filter=lfs diff=lfs merge=lfs -text 11 | *.wav filter=lfs diff=lfs merge=lfs -text 12 | *.xcf filter=lfs diff=lfs merge=lfs -text 13 | *.jpg filter=lfs diff=lfs merge=lfs -text 14 | *.gif filter=lfs diff=lfs merge=lfs -text 15 | 16 | # Anything in `/RawContent` dir. 17 | /RawContent/**/* filter=lfs diff=lfs merge=lfs -text -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /Binaries 2 | /Intermediate 3 | /Resources 4 | /.vs/ -------------------------------------------------------------------------------- /AzureKinect.uplugin: -------------------------------------------------------------------------------- 1 | { 2 | "FileVersion": 3, 3 | "Version": 1, 4 | "VersionName": "1.0", 5 | "FriendlyName": "Azure Kinect", 6 | "Description": "Exposes Azure Kinect Support for integration into Unreal Engine Applications", 7 | "Category": "Virtual Production", 8 | "CreatedBy": "Ayumu Nagamatsu", 9 | "CreatedByURL": "https://ayumu-nagamatsu.com/", 10 | "DocsURL": "https://github.com/nama-gatsuo/AzureKinectForUE", 11 | "MarketplaceURL": "", 12 | "SupportURL": "", 13 | "CanContainContent": true, 14 | "IsBetaVersion": true, 15 | "IsExperimentalVersion": false, 16 | "Installed": false, 17 | "SupportedTargetPlatforms": [ "Win64" ], 18 | "Modules": [ 19 | { 20 | "Name": "AzureKinect", 21 | "Type": "Runtime", 22 | "LoadingPhase": "Default" 23 | }, 24 | { 25 | "Name": "AzureKinectEditor", 26 | "Type": "UncookedOnly", 27 | "LoadingPhase": "Default" 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /Content/ABP_AureKinectSkeletonToHumanoid.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:d143134e110ced4aed3d3357cd49f926eb3789e8aa3bb6a28e24ede0b509c71e 3 | size 64530 4 | -------------------------------------------------------------------------------- /Content/AzureKinectActor.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:6b12dc821ef481cea7586e9cc31960fc947e84ab967551a6871c00141ec381b0 3 | size 2311 4 | -------------------------------------------------------------------------------- /Content/AzureKinectDevice.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0d17d7f058d3f83cf6e5ae452bff91f2ff6dbf9ccc6d746587e76bfe0e5d1d97 3 | size 2655 4 | -------------------------------------------------------------------------------- /Content/BP_AzureKinectActor.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e03af29176162d5e48c8ec704503c81fd96ed5903aea6c471d79c0963ff76a63 3 | size 106017 4 | -------------------------------------------------------------------------------- /Content/BP_AzureKinectActor_MultiUser.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:adf0087e305ffbedd7398b181da820ef42a83b5fa5bc0ddbec77f054d4ea21d6 3 | size 511589 4 | -------------------------------------------------------------------------------- /Content/NS_KinectParticle.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ebcdee960847b368e1a78c4af5c3ba482b93738a9449ae217de9855c517e2a84 3 | size 754246 4 | -------------------------------------------------------------------------------- /Content/RenderTargets/RT_KinectBodyIndex.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b1008a02b1d51eb73a6097c7f5d8329c7afb401a8247bf82575bb086246eef95 3 | size 5700 4 | -------------------------------------------------------------------------------- /Content/RenderTargets/RT_KinectColor.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ea0baa3d609c8282710ec585738163906a31932c9e0f0463883558241bd519d3 3 | size 8198 4 | -------------------------------------------------------------------------------- /Content/RenderTargets/RT_KinectDepth.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:19e600d2b05ab814c1989f619b38759c9b800fec292e1c36bfe8b1ce1a6d3250 3 | size 3794 4 | -------------------------------------------------------------------------------- /Content/RenderTargets/RT_KinectIR.uasset: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:eb306826af33d560fbd9b84552597c83af8efc973a5e0b6e3759697b174155c0 3 | size 3782 4 | -------------------------------------------------------------------------------- /Docs/Sequence 01.gif: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8224a2a327e9fa04d46dc3ff37482bd393c9c5888b3e6eb05792307d240d23b9 3 | size 116129244 4 | -------------------------------------------------------------------------------- /Docs/animation.gif: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f7234e5624e9a5d214d61ce51bb334b3bce5823bab4abffe59a5201c45f57a6b 3 | size 4458917 4 | -------------------------------------------------------------------------------- /Docs/animgraph.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c0cadc057782c801db4d2aa69e4ccb81748dc3cad261def1e452f30ad90e3d90 3 | size 97799 4 | -------------------------------------------------------------------------------- /Docs/bp.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:6a306ec8db21ece7bd831890e9aacb2201bce4e07f00e064088930486a5565f3 3 | size 119202 4 | -------------------------------------------------------------------------------- /Docs/in-editor.gif: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:efd5130bf40d083e1bfb8752f7ae6f5e15452522a5754725ac544f12686daf0c 3 | size 648981 4 | -------------------------------------------------------------------------------- /Docs/kinect.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7aed07f93d3baef8a82110f48e6839462ed3efa41ed88b42caf08484f5ad0352 3 | size 426986 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Kinect Plugin for Unreal Engine 2 | Exposes Azure Kinect Support for integration into Unreal Engine Applications. 3 | 4 | Updated Version of nama-gatsu's Azure Kinect Unreal Engine Plugin. 5 | - Fixed Importing Binaries Issues 6 | - Added Translation to skeletal tracking 7 | 8 | ![](./Docs/Sequence%2001.gif) 9 | 10 | ## Setup Instructions 11 | * Platform: Win64 12 | * Unreal Engine Version 5.2 13 | * Download and Install `Azure Kinect SDK v1.4.1` from [here](https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/develop/docs/usage.md) 14 | * Download and Install `Azure Kinect Body Tracking SDK v1.1.0` from [here](https://docs.microsoft.com/en-us/azure/Kinect-dk/body-sdk-download) 15 | * Create the following two System Environment variables: 16 | * AZUREKINECT_SDK that points to the Azure Kinect Body Tracking SDK root path `C:\Program Files\Azure Kinect SDK v1.4.1` 17 | * AZUREKINECT_BODY_SDK that points to the Azure Kinect Body Tracking SDK root path `C:\Program Files\Azure Kinect Body Tracking SDK` 18 | * Add the following three paths to the PATH system environment variable 19 | * `C:\Program Files\Azure Kinect Body Tracking SDK\sdk\windows-desktop\amd64\release\bin` 20 | * `C:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\bin` 21 | * `C:\Program Files\Azure Kinect SDK v1.4.1\sdk\netstandard2.0\release` 22 | * Create a Plugins folder inside your unreal project folder (Where the .uprojrct is located) 23 | * Clone this repo into the plugins folder. 24 | ## Features 25 | 26 | ### In-Editor activation 27 | 28 | * Write Depth / Color buffer into `RenderTarget2D`s. 29 | 30 | ![](./Docs/in-editor.gif) 31 | 32 | ### Blueprint activation 33 | 34 | ![](./Docs/bp.png) 35 | 36 | ### Niagara Particle 37 | 38 | * You can modify base a niagara system `NS_KinectParticle`. 39 | 40 | ![](./Docs/animation.gif) 41 | 42 | 43 | ### Skeleton tracking 44 | 45 | ![]() 46 | 47 | * Bone mapping node in Anim Graph 48 | 49 | ![](./Docs/animgraph.jpg) 50 | 51 | ## Notice 52 | 53 | Depthe data are stored `RenderTarget2D` into standard 8bit RGBA texture. 54 | R: first 8bit as `uint8` of original `uint16` sample 55 | G: last 8bit as `uint8` of original `uint16` sample 56 | B: `0x00` or `0xFF` (if depth sample is invalid) 57 | A: `0xFF` (Constant value) 58 | 59 | Thus we need conversion to acquire orignal depth samples. 60 | ``` 61 | // In MaterialEditor or Niagara, sample values in Depth texture are normalized to 0-1. 62 | float DepthSample = (G * 256.0 + R) * 256.0; // millimetor 63 | ``` 64 | 65 | ``` 66 | // In C++ 67 | uint8 R = Sample.R, G = Sample.G; 68 | uint16 DepthSample = G << 8 | R; // millimetor 69 | ``` 70 | 71 | Depth pixel from Azure Kinect SDK is originally a single `uint16` in millimetor. But `RenderTarget2D` can't store `uint16` as texture (`EPixelFormat::PF_R16_UINT` doesn't work for RenderTarget). 72 | 73 | # License 74 | ## MIT License 75 | Copyright 2021 Ayumu Nagamtsu 76 | 77 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 78 | 79 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 80 | 81 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Source/AzureKinect/AzureKinect.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using System.IO; 4 | using UnrealBuildTool; 5 | 6 | public class AzureKinect : ModuleRules 7 | { 8 | public AzureKinect(ReadOnlyTargetRules Target) : base(Target) 9 | { 10 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 11 | 12 | if (Target.Platform == UnrealTargetPlatform.Win64) 13 | { 14 | string sdkPath = System.Environment.GetEnvironmentVariable("AZUREKINECT_SDK"); 15 | string bodySdkPath = System.Environment.GetEnvironmentVariable("AZUREKINECT_BODY_SDK"); 16 | 17 | PublicIncludePaths.AddRange( 18 | new string[] { 19 | Path.Combine(sdkPath, "sdk", "include"), 20 | Path.Combine(bodySdkPath, "sdk", "include") 21 | }); 22 | 23 | PublicAdditionalLibraries.AddRange( 24 | new string[] { 25 | Path.Combine(sdkPath, "sdk", "windows-desktop", "amd64", "release", "lib", "k4a.lib"), 26 | Path.Combine(sdkPath, "sdk", "windows-desktop", "amd64", "release", "lib", "k4arecord.lib"), 27 | Path.Combine(bodySdkPath, "sdk", "windows-desktop", "amd64", "release", "lib", "k4abt.lib") 28 | }); 29 | System.Console.WriteLine("TEST TEST TEST TEST TEST TEST TEST"); 30 | 31 | 32 | 33 | string depthEngineDllPath = Path.Combine(sdkPath, "sdk", "windows-desktop", "amd64", "release", "bin", "depthengine_2_0.dll"); 34 | string k4aDllPath = Path.Combine(sdkPath, "sdk", "windows-desktop", "amd64", "release", "bin", "k4a.dll"); 35 | string k4abtDllPath = Path.Combine(bodySdkPath, "sdk", "windows-desktop", "amd64", "release", "bin", "k4abt.dll"); 36 | 37 | //string bodyTrackingDLL = Path.Combine(bodySdkPath, "sdk", "netstandard2.0", "release", "microsoft.azure.kinect.bodytracking.dll"); 38 | //string bodyTrackingPDB = Path.Combine(bodySdkPath, "sdk", "netstandard2.0", "release", "Microsoft.Azure.Kinect.BodyTracking.pbd"); 39 | 40 | 41 | 42 | System.Console.WriteLine(depthEngineDllPath); 43 | PublicDelayLoadDLLs.AddRange( 44 | new string[] { 45 | depthEngineDllPath, 46 | //bodyTrackingDLL, 47 | //bodyTrackingPDB, 48 | k4aDllPath, 49 | k4abtDllPath, 50 | }); 51 | 52 | RuntimeDependencies.Add(depthEngineDllPath); 53 | RuntimeDependencies.Add(k4aDllPath); 54 | RuntimeDependencies.Add(k4abtDllPath); 55 | } 56 | 57 | 58 | PrivateIncludePaths.AddRange( 59 | new string[] 60 | { 61 | "AzureKinect/Private", 62 | }); 63 | 64 | PrivateDependencyModuleNames.AddRange( 65 | new string[] 66 | { 67 | "Core", 68 | "CoreUObject", 69 | "Engine", 70 | "RenderCore", 71 | "RHI", 72 | "AnimGraphRuntime", 73 | }); 74 | 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /Source/AzureKinect/Private/AnimNode_AzureKinectPose.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #include "AnimNode_AzureKinectPose.h" 4 | #include "Animation/AnimInstanceProxy.h" 5 | #include "AnimationRuntime.h" 6 | #include "k4abttypes.h" 7 | 8 | DEFINE_LOG_CATEGORY(AzureKinectAnimNodeLog); 9 | //General Log 10 | //DECLARE_LOG_CATEGORY_EXTERN(AzureKinectAnimNodeLog, Log, All); 11 | 12 | FAnimNode_AzureKinectPose::FAnimNode_AzureKinectPose() 13 | { 14 | BonesToModify.Reserve(K4ABT_JOINT_COUNT); 15 | for (int i = 0; i < K4ABT_JOINT_COUNT; i++) 16 | { 17 | BonesToModify.Add(static_cast(i), FBoneReference()); 18 | } 19 | } 20 | 21 | void FAnimNode_AzureKinectPose::Update_AnyThread(const FAnimationUpdateContext& Context) 22 | { 23 | DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(Update_AnyThread); 24 | 25 | GetEvaluateGraphExposedInputs().Execute(Context); 26 | 27 | USkeletalMeshComponent* SkelMesh = Context.AnimInstanceProxy->GetSkelMeshComponent(); 28 | USkeleton* mySkeleton = Context.AnimInstanceProxy->GetSkeleton(); 29 | BoneTransforms.Reset(K4ABT_JOINT_COUNT); 30 | 31 | //for each UE Skeleton joint, add the corresponding Kinect Joint to a BoneTransforms Array 32 | for (int i = 0; i < Skeleton.Joints.Num(); i++) 33 | { 34 | EKinectBodyJoint JointIndex = static_cast(i); 35 | if (BonesToModify.Contains(JointIndex)) 36 | { 37 | //UE_LOG(AzureKinectAnimNodeLog, Warning, TEXT("Azure Joint Index: %d"), JointIndex); 38 | int32 BoneIndex = SkelMesh->GetBoneIndex(BonesToModify[JointIndex].BoneName); 39 | //UE_LOG(AzureKinectAnimNodeLog, Warning, TEXT("UE Bone: %s"), *BonesToModify[JointIndex].BoneName.ToString()); 40 | if (BoneIndex != INDEX_NONE) 41 | { 42 | FCompactPoseBoneIndex CompactBoneIndex(BoneIndex); 43 | BoneTransforms.Emplace(CompactBoneIndex, Skeleton.Joints[i]); 44 | } 45 | if (BoneIndex == INDEX_NONE) 46 | { 47 | //UE_LOG(AzureKinectAnimNodeLog, Warning, TEXT("Bone Index is NONE")); 48 | } 49 | } 50 | } 51 | 52 | } 53 | 54 | void FAnimNode_AzureKinectPose::EvaluateComponentSpace_AnyThread(FComponentSpacePoseContext& Output) 55 | { 56 | DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateComponentSpace_AnyThread) 57 | Output.ResetToRefPose(); 58 | 59 | //for each kinect bone set the output pose 60 | for (const FBoneTransform& BoneTransform : BoneTransforms) 61 | { 62 | //transform = transform of UE bone 63 | FTransform Transform = Output.Pose.GetComponentSpaceTransform(BoneTransform.BoneIndex); 64 | 65 | FRotator3d rotator = BoneTransform.Transform.Rotator(); 66 | if (BoneTransform.BoneIndex == 1) { 67 | float xtrans = BoneTransform.Transform.GetTranslation().X; 68 | float ytrans = BoneTransform.Transform.GetTranslation().Y; 69 | float ztrans = BoneTransform.Transform.GetTranslation().Z; 70 | 71 | FVector3d fixedTranslation = FVector3d(ytrans, -ztrans, -xtrans); 72 | Transform.SetTranslation(fixedTranslation); 73 | } 74 | FQuat4d quaternion = rotator.Quaternion(); 75 | Transform.SetRotation(quaternion); 76 | Output.Pose.SetComponentSpaceTransform(BoneTransform.BoneIndex, Transform); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Source/AzureKinect/Private/AzureKinectDevice.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | #include "AzureKinectDevice.h" 3 | #include "Runtime/RHI/Public/RHI.h" 4 | #include 5 | DEFINE_LOG_CATEGORY(AzureKinectDeviceLog); 6 | 7 | UAzureKinectDevice::UAzureKinectDevice() : 8 | NativeDevice(nullptr), 9 | Thread(nullptr), 10 | DeviceIndex(-1), 11 | bOpen(false), 12 | NumTrackedSkeletons(0), 13 | DepthMode(EKinectDepthMode::NFOV_2X2BINNED), 14 | ColorMode(EKinectColorResolution::RESOLUTION_720P), 15 | Fps(EKinectFps::PER_SECOND_30), 16 | SensorOrientation(EKinectSensorOrientation::DEFAULT), 17 | bSkeletonTracking(false), 18 | Capture(nullptr) 19 | { 20 | LoadDevices(); 21 | } 22 | 23 | UAzureKinectDevice::UAzureKinectDevice(const FObjectInitializer& ObjectInitializer) : 24 | Super(ObjectInitializer) 25 | { 26 | LoadDevices(); 27 | } 28 | 29 | void UAzureKinectDevice::LoadDevices() 30 | { 31 | 32 | int32 NumKinect = GetNumConnectedDevices(); 33 | DeviceList.Empty(NumKinect + 1); 34 | DeviceList.Add(MakeShared("No Device")); 35 | 36 | if (NumKinect > 0) 37 | { 38 | for (int32 i = 0; i < NumKinect; i++) 39 | { 40 | try 41 | { 42 | // Open connection to the device. 43 | k4a::device Device = k4a::device::open(i); 44 | // Get and store the device serial number 45 | DeviceList.Add(MakeShared(Device.get_serialnum().c_str())); 46 | Device.close(); 47 | } 48 | catch (const k4a::error& Err) 49 | { 50 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("Can't load: %s"), TCHAR_TO_UTF8(ANSI_TO_TCHAR(Err.what()))); 51 | } 52 | } 53 | } 54 | 55 | } 56 | 57 | bool UAzureKinectDevice::StartDevice() 58 | { 59 | //K4A_ENABLE_LOG_TO_A_FILE = "log\custom.log"; 60 | if (bOpen) 61 | { 62 | UE_LOG(AzureKinectDeviceLog, Warning, TEXT("This Device has been open.")); 63 | return false; 64 | } 65 | 66 | 67 | if (DeviceIndex == -1) 68 | { 69 | UE_LOG(AzureKinectDeviceLog, Warning, TEXT("No Device is selected.")); 70 | return false; 71 | } 72 | 73 | CalcFrameCount(); 74 | 75 | try 76 | { 77 | // Open connection to the device. 78 | //NativeDevice = k4a::device::open(DeviceIndex); 79 | // Start the Camera and make sure the Depth Camera is Enabled 80 | k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; 81 | config.depth_mode = static_cast(DepthMode); 82 | config.color_resolution = static_cast(ColorMode); 83 | config.camera_fps = static_cast(Fps); 84 | config.color_format = k4a_image_format_t::K4A_IMAGE_FORMAT_COLOR_BGRA32; 85 | config.synchronized_images_only = true; 86 | config.wired_sync_mode = K4A_WIRED_SYNC_MODE_STANDALONE; 87 | //NativeDevice.stop_cameras(); 88 | 89 | k4a_device_t testHandle = NativeDevice.handle(); 90 | 91 | //Capture; capture; 92 | //capture = NativeDevice.GetCapture(); 93 | int32_t testInt = 30000; 94 | k4a::device sampleDev = k4a::device::device(testHandle); 95 | k4a_capture_t capHandle = Capture.handle(); 96 | k4a_capture_t* pointHandle = &capHandle; 97 | //k4a::capture cap = 98 | k4a_device_get_capture(testHandle, pointHandle, testInt); 99 | k4a::image im = k4a_capture_get_color_image(capHandle); 100 | k4a::image dep = k4a_capture_get_depth_image(capHandle); 101 | 102 | //bool cap = sampleDev.get_capture(Capture); 103 | 104 | k4a::image dep2 = Capture.get_depth_image(); 105 | //NativeDevice.get_capture(&Capture, FrameTime); 106 | //k4a::image testImage = k4a_capture_get_depth_image(testHandle); 107 | k4a_device_t device = NULL; 108 | if (K4A_FAILED(k4a_device_open(K4A_DEVICE_DEFAULT, &device))) 109 | { 110 | printf("Failed to open k4a device!\n"); 111 | return 1; 112 | } 113 | // Configure a stream of 4096x3072 BRGA color data at 15 frames per second 114 | /* 115 | k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; 116 | 117 | config.camera_fps = K4A_FRAMES_PER_SECOND_30; 118 | config.color_format = K4A_IMAGE_FORMAT_COLOR_BGRA32; 119 | config.depth_mode = K4A_DEPTH_MODE_NFOV_UNBINNED; 120 | config.synchronized_images_only = true; 121 | config.wired_sync_mode = K4A_WIRED_SYNC_MODE_STANDALONE; 122 | //config.color_resolution = static_cast(ColorMode); 123 | config.color_resolution = K4A_COLOR_RESOLUTION_2160P; 124 | */ 125 | //test._handle_name_ 126 | //k4a::handle:: 127 | // Start the camera with the given configuration 128 | //k4a_device_stop_cameras(device); 129 | if (K4A_FAILED(k4a_device_start_cameras(device, &config))) 130 | { 131 | printf("Failed to start cameras!\n"); 132 | k4a_device_close(device); 133 | return 1; 134 | } 135 | NativeDevice = k4a::device(device); 136 | 137 | //NativeDevice.start_cameras(&DeviceConfig); 138 | 139 | KinectCalibration = NativeDevice.get_calibration(config.depth_mode, config.color_resolution); 140 | KinectTransformation = k4a::transformation(KinectCalibration); 141 | 142 | if (bSkeletonTracking) 143 | { 144 | UE_LOG(AzureKinectDeviceLog, Log, TEXT("Skeleton Tracking")); 145 | 146 | k4abt_tracker_configuration_t TrackerConfig = K4ABT_TRACKER_CONFIG_DEFAULT; 147 | TrackerConfig.sensor_orientation = static_cast(SensorOrientation); 148 | 149 | // Retain body tracker 150 | BodyTracker = k4abt::tracker::create(KinectCalibration, TrackerConfig); 151 | } 152 | 153 | } 154 | catch (const k4a::error& Err) 155 | { 156 | if (NativeDevice) 157 | { 158 | NativeDevice.close(); 159 | } 160 | 161 | FString Msg(ANSI_TO_TCHAR(Err.what())); 162 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("Cant't open: %s"), *Msg); 163 | return false; 164 | } 165 | 166 | Thread = new FAzureKinectDeviceThread(this); 167 | 168 | bOpen = true; 169 | 170 | return true; 171 | } 172 | 173 | bool UAzureKinectDevice::StopDevice() 174 | { 175 | 176 | if (!bOpen) 177 | { 178 | UE_LOG(AzureKinectDeviceLog, Warning, TEXT("KinectDevice is not running.")); 179 | return false; 180 | } 181 | 182 | if (Thread) 183 | { 184 | Thread->EnsureCompletion(); 185 | Thread = nullptr; 186 | } 187 | 188 | if (BodyTracker) 189 | { 190 | BodyTracker.shutdown(); 191 | BodyTracker.destroy(); 192 | BodyTracker = nullptr; 193 | } 194 | 195 | if (RemapImage) 196 | { 197 | RemapImage.reset(); 198 | } 199 | 200 | if (NativeDevice) 201 | { 202 | NativeDevice.stop_cameras(); 203 | NativeDevice.close(); 204 | NativeDevice = nullptr; 205 | UE_LOG(AzureKinectDeviceLog, Verbose, TEXT("KinectDevice Camera is Stopped and Closed.")); 206 | } 207 | 208 | bOpen = false; 209 | return true; 210 | } 211 | 212 | int32 UAzureKinectDevice::GetNumConnectedDevices() 213 | { 214 | return k4a_device_get_installed_count(); 215 | } 216 | 217 | int32 UAzureKinectDevice::GetNumTrackedSkeletons() const 218 | { 219 | if (!bOpen) 220 | { 221 | return 0; 222 | } 223 | if (!bSkeletonTracking) 224 | { 225 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("GetNumTrackedBodies: Skeleton Tracking is disabled!")); 226 | return 0; 227 | } 228 | 229 | FScopeLock Lock(Thread->GetCriticalSection()); 230 | return NumTrackedSkeletons; 231 | } 232 | 233 | FAzureKinectSkeleton UAzureKinectDevice::GetSkeleton(int32 Index) const 234 | { 235 | if (bOpen) 236 | { 237 | if (!bSkeletonTracking) 238 | { 239 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("GetSkeleton: Skeleton Tracking is disabled!")); 240 | return FAzureKinectSkeleton(); 241 | } 242 | 243 | FScopeLock Lock(Thread->GetCriticalSection()); 244 | if (Skeletons.IsValidIndex(Index)) 245 | { 246 | return Skeletons[Index]; 247 | } 248 | else 249 | { 250 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("GetSkeleton: Index is out of range!")); 251 | return FAzureKinectSkeleton(); 252 | } 253 | } 254 | else 255 | { 256 | return FAzureKinectSkeleton(); 257 | } 258 | 259 | } 260 | 261 | const TArray& UAzureKinectDevice::GetSkeletons() const { 262 | if (bOpen) 263 | { 264 | FScopeLock Lock(Thread->GetCriticalSection()); 265 | return Skeletons; 266 | } 267 | else 268 | { 269 | return Skeletons; 270 | } 271 | } 272 | 273 | void UAzureKinectDevice::UpdateAsync() 274 | { 275 | // Threaded function 276 | try 277 | { 278 | if (!NativeDevice.get_capture(&Capture, FrameTime)) 279 | { 280 | UE_LOG(AzureKinectDeviceLog, Verbose, TEXT("Timed out waiting for capture.")); 281 | } 282 | } 283 | catch (const k4a::error& Err) 284 | { 285 | FString Msg(ANSI_TO_TCHAR(Err.what())); 286 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("Can't capture frame: %s"), *Msg); 287 | return; 288 | } 289 | 290 | if (ColorMode != EKinectColorResolution::RESOLUTION_OFF && ColorTexture) 291 | { 292 | CaptureColorImage(); 293 | } 294 | 295 | if (DepthMode != EKinectDepthMode::OFF && DepthTexture) 296 | { 297 | CaptureDepthImage(); 298 | } 299 | 300 | if (DepthMode != EKinectDepthMode::OFF && InflaredTexture) 301 | { 302 | CaptureInflaredImage(); 303 | } 304 | 305 | if (bSkeletonTracking && BodyTracker) 306 | { 307 | UpdateSkeletons(); 308 | } 309 | 310 | Capture.reset(); 311 | 312 | } 313 | 314 | void UAzureKinectDevice::CaptureColorImage() 315 | { 316 | int32 Width = 0, Height = 0; 317 | uint8* SourceBuffer; 318 | 319 | if (RemapMode == EKinectRemap::COLOR_TO_DEPTH) 320 | { 321 | k4a::image DepthCapture = Capture.get_depth_image(); 322 | k4a::image ColorCapture = Capture.get_color_image(); 323 | 324 | if (!DepthCapture.is_valid() || !ColorCapture.is_valid()) return; 325 | 326 | Width = DepthCapture.get_width_pixels(); 327 | Height = DepthCapture.get_height_pixels(); 328 | 329 | if (Width == 0 || Height == 0) return; 330 | 331 | // 332 | if (!RemapImage || !RemapImage.is_valid()) 333 | { 334 | RemapImage = k4a::image::create(K4A_IMAGE_FORMAT_COLOR_BGRA32, Width, Height, Width * static_cast(sizeof(uint8) * 4)); 335 | } 336 | 337 | try 338 | { 339 | KinectTransformation.color_image_to_depth_camera(DepthCapture, ColorCapture, &RemapImage); 340 | } 341 | catch (const k4a::error& Err) 342 | { 343 | FString Msg(ANSI_TO_TCHAR(Err.what())); 344 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("Cant't transform Color to Depth: %s"), *Msg); 345 | return; 346 | } 347 | 348 | SourceBuffer = RemapImage.get_buffer(); 349 | 350 | DepthCapture.reset(); 351 | ColorCapture.reset(); 352 | } 353 | else 354 | { 355 | k4a::image ColorCapture = Capture.get_color_image(); 356 | 357 | if (!ColorCapture.is_valid()) return; 358 | 359 | Width = ColorCapture.get_width_pixels(); 360 | Height = ColorCapture.get_height_pixels(); 361 | if (Width == 0 || Height == 0) return; 362 | 363 | SourceBuffer = ColorCapture.get_buffer(); 364 | 365 | ColorCapture.reset(); 366 | } 367 | 368 | if (ColorTexture->GetSurfaceWidth() != Width || ColorTexture->GetSurfaceHeight() != Height) 369 | { 370 | ColorTexture->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, false); 371 | ColorTexture->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; 372 | ColorTexture->UpdateResource(); 373 | } 374 | else 375 | { 376 | 377 | FTextureResource* TextureResource = ColorTexture->Resource; 378 | auto Region = FUpdateTextureRegion2D(0, 0, 0, 0, Width, Height); 379 | 380 | ENQUEUE_RENDER_COMMAND(UpdateTextureData)( 381 | [TextureResource, Region, SourceBuffer](FRHICommandListImmediate& RHICmdList) { 382 | FTexture2DRHIRef Texture2D = TextureResource->TextureRHI ? TextureResource->TextureRHI->GetTexture2D() : nullptr; 383 | if (!Texture2D) 384 | { 385 | return; 386 | } 387 | RHIUpdateTexture2D(Texture2D, 0, Region, 4 * Region.Width, SourceBuffer); 388 | }); 389 | } 390 | 391 | } 392 | 393 | void UAzureKinectDevice::CaptureDepthImage() 394 | { 395 | int32 Width = 0, Height = 0; 396 | uint8* SourceBuffer; 397 | if (RemapMode == EKinectRemap::DEPTH_TO_COLOR) 398 | { 399 | k4a::image DepthCapture = Capture.get_depth_image(); 400 | k4a::image ColorCapture = Capture.get_color_image(); 401 | 402 | if (!DepthCapture.is_valid() || !ColorCapture.is_valid()) return; 403 | 404 | Width = ColorCapture.get_width_pixels(); 405 | Height = ColorCapture.get_height_pixels(); 406 | 407 | if (Width == 0 || Height == 0) return; 408 | 409 | // 410 | if (!RemapImage || !RemapImage.is_valid()) 411 | { 412 | RemapImage = k4a::image::create(K4A_IMAGE_FORMAT_DEPTH16, Width, Height, Width * static_cast(sizeof(uint16))); 413 | } 414 | 415 | try 416 | { 417 | KinectTransformation.depth_image_to_color_camera(DepthCapture, &RemapImage); 418 | } 419 | catch (const k4a::error& Err) 420 | { 421 | FString Msg(ANSI_TO_TCHAR(Err.what())); 422 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("Cant't transform Depth to Color: %s"), *Msg); 423 | return; 424 | } 425 | 426 | SourceBuffer = RemapImage.get_buffer(); 427 | 428 | DepthCapture.reset(); 429 | ColorCapture.reset(); 430 | } 431 | else 432 | { 433 | k4a::image DepthCapture = Capture.get_depth_image(); 434 | if (!DepthCapture.is_valid()) return; 435 | 436 | Width = DepthCapture.get_width_pixels(); 437 | Height = DepthCapture.get_height_pixels(); 438 | 439 | if (Width == 0 || Height == 0) return; 440 | 441 | SourceBuffer = DepthCapture.get_buffer(); 442 | 443 | DepthCapture.reset(); 444 | } 445 | 446 | if (DepthTexture->GetSurfaceWidth() != Width || DepthTexture->GetSurfaceHeight() != Height) 447 | { 448 | DepthTexture->InitCustomFormat(Width, Height, EPixelFormat::PF_R8G8B8A8, true); 449 | DepthTexture->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; 450 | DepthTexture->UpdateResource(); 451 | } 452 | else 453 | { 454 | 455 | TArray SrcData; 456 | SrcData.Reset(Width * Height * 4); 457 | for (int hi = 0; hi < Height; hi++) 458 | { 459 | for (int wi = 0; wi < Width; wi++) 460 | { 461 | int index = hi * Width + wi; 462 | uint16 R = SourceBuffer[index * 2]; 463 | uint16 G = SourceBuffer[index * 2 + 1]; 464 | 465 | uint16 Sample = G << 8 | R; 466 | 467 | SrcData.Push(SourceBuffer[index * 2]); 468 | SrcData.Push(SourceBuffer[index * 2 + 1]); 469 | SrcData.Push(Sample > 0 ? 0x00 : 0xFF); 470 | SrcData.Push(0xFF); 471 | 472 | } 473 | } 474 | 475 | FTextureResource* TextureResource = DepthTexture->Resource; 476 | auto Region = FUpdateTextureRegion2D(0, 0, 0, 0, Width, Height); 477 | 478 | ENQUEUE_RENDER_COMMAND(UpdateTextureData)( 479 | [TextureResource, Region, SrcData](FRHICommandListImmediate& RHICmdList) { 480 | FTexture2DRHIRef Texture2D = TextureResource->TextureRHI ? TextureResource->TextureRHI->GetTexture2D() : nullptr; 481 | if (!Texture2D) 482 | { 483 | return; 484 | } 485 | RHIUpdateTexture2D(Texture2D, 0, Region, 4 * Region.Width, SrcData.GetData()); 486 | 487 | }); 488 | } 489 | 490 | } 491 | 492 | void UAzureKinectDevice::CaptureInflaredImage() 493 | { 494 | const k4a::image& InflaredCapture = Capture.get_ir_image(); 495 | if (!InflaredCapture.is_valid()) return; 496 | 497 | int32 Width = InflaredCapture.get_width_pixels(), Height = InflaredCapture.get_height_pixels(); 498 | if (Width == 0 || Height == 0) return; 499 | 500 | if (InflaredTexture->GetSurfaceWidth() != Width || InflaredTexture->GetSurfaceWidth() != Height) 501 | { 502 | InflaredTexture->InitCustomFormat(Width, Height, EPixelFormat::PF_R8G8B8A8, true); 503 | InflaredTexture->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; 504 | InflaredTexture->UpdateResource(); 505 | } 506 | else 507 | { 508 | const uint8* S = InflaredCapture.get_buffer(); 509 | TArray SrcData; 510 | SrcData.Reset(Width * Height * 4); 511 | for (int hi = 0; hi < Height; hi++) 512 | { 513 | for (int wi = 0; wi < Width; wi++) 514 | { 515 | int index = hi * Width + wi; 516 | 517 | if (S[index * 2] + S[index * 2 + 1] > 0) 518 | { 519 | SrcData.Push(S[index * 2]); 520 | SrcData.Push(S[index * 2 + 1]); 521 | SrcData.Push(0x00); 522 | SrcData.Push(0xff); 523 | } 524 | else 525 | { 526 | SrcData.Push(0x00); 527 | SrcData.Push(0x00); 528 | SrcData.Push(0xff); 529 | SrcData.Push(0xff); 530 | } 531 | } 532 | } 533 | 534 | FTextureResource* TextureResource = InflaredTexture->Resource; 535 | auto Region = FUpdateTextureRegion2D(0, 0, 0, 0, Width, Height); 536 | 537 | ENQUEUE_RENDER_COMMAND(UpdateTextureData)( 538 | [TextureResource, Region, SrcData](FRHICommandListImmediate& RHICmdList) { 539 | FTexture2DRHIRef Texture2D = TextureResource->TextureRHI ? TextureResource->TextureRHI->GetTexture2D() : nullptr; 540 | if (!Texture2D) 541 | { 542 | return; 543 | } 544 | 545 | RHIUpdateTexture2D(Texture2D, 0, Region, 4 * Region.Width, SrcData.GetData()); 546 | }); 547 | } 548 | 549 | } 550 | 551 | void UAzureKinectDevice::CaptureBodyIndexImage(const k4abt::frame& BodyFrame) 552 | { 553 | k4a::image BodyIndexMap = BodyFrame.get_body_index_map(); 554 | 555 | int32 Width = BodyIndexMap.get_width_pixels(), Height = BodyIndexMap.get_height_pixels(); 556 | if (Width == 0 || Height == 0) return; 557 | 558 | if (BodyIndexTexture->GetSurfaceWidth() != Width || BodyIndexTexture->GetSurfaceHeight() != Height) 559 | { 560 | BodyIndexTexture->InitCustomFormat(Width, Height, EPixelFormat::PF_R8G8B8A8, true); 561 | BodyIndexTexture->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; 562 | BodyIndexTexture->UpdateResource(); 563 | } 564 | else 565 | { 566 | uint8* S = BodyIndexMap.get_buffer(); 567 | TArray SrcData; 568 | SrcData.Reset(Width * Height * 4); 569 | for (int i = 0; i < Width * Height; i++) 570 | { 571 | SrcData.Push(S[i]); 572 | SrcData.Push(S[i]); 573 | SrcData.Push(S[i]); 574 | SrcData.Push(0xff); 575 | } 576 | 577 | FTextureResource* TextureResource = BodyIndexTexture->Resource; 578 | auto Region = FUpdateTextureRegion2D(0, 0, 0, 0, Width, Height); 579 | 580 | ENQUEUE_RENDER_COMMAND(UpdateTextureData)( 581 | [TextureResource, Region, SrcData](FRHICommandListImmediate& RHICmdList) { 582 | FTexture2DRHIRef Texture2D = TextureResource->TextureRHI ? TextureResource->TextureRHI->GetTexture2D() : nullptr; 583 | if (!Texture2D) 584 | { 585 | return; 586 | } 587 | RHIUpdateTexture2D(Texture2D, 0, Region, 4 * Region.Width, SrcData.GetData()); 588 | 589 | }); 590 | } 591 | 592 | } 593 | 594 | void UAzureKinectDevice::UpdateSkeletons() 595 | { 596 | 597 | k4abt::frame BodyFrame = nullptr; 598 | TArray BodyIDs; 599 | 600 | try 601 | { 602 | if (!BodyTracker.enqueue_capture(Capture, FrameTime)) 603 | { 604 | UE_LOG(AzureKinectDeviceLog, Warning, TEXT("Failed adding capture to tracker process queue")); 605 | return; 606 | } 607 | 608 | if (!BodyTracker.pop_result(&BodyFrame, FrameTime)) 609 | { 610 | UE_LOG(AzureKinectDeviceLog, Warning, TEXT("Failed Tracker pop body frame")); 611 | return; 612 | } 613 | } 614 | catch (const k4a::error& Err) 615 | { 616 | FString Msg(ANSI_TO_TCHAR(Err.what())); 617 | UE_LOG(AzureKinectDeviceLog, Error, TEXT("Couldn't get Body Frame: %s"), *Msg); 618 | } 619 | 620 | if (BodyIndexTexture) 621 | { 622 | CaptureBodyIndexImage(BodyFrame); 623 | } 624 | 625 | { 626 | FScopeLock Lock(Thread->GetCriticalSection()); 627 | 628 | NumTrackedSkeletons = BodyFrame.get_num_bodies(); 629 | Skeletons.Reset(NumTrackedSkeletons); 630 | 631 | for (int32 i = 0; i < NumTrackedSkeletons; i++) 632 | { 633 | k4abt_body_t Body; 634 | FAzureKinectSkeleton Skeleton; 635 | 636 | BodyFrame.get_body_skeleton(i, Body.skeleton); 637 | Skeleton.ID = BodyFrame.get_body_id(i); 638 | 639 | Skeleton.Joints.Reset(K4ABT_JOINT_COUNT); 640 | 641 | for (int32 j = 0; j < K4ABT_JOINT_COUNT; j++) 642 | { 643 | Skeleton.Joints.Push(JointToTransform(Body.skeleton.joints[j], j)); 644 | } 645 | 646 | Skeletons.Push(Skeleton); 647 | } 648 | } 649 | 650 | 651 | BodyFrame.reset(); 652 | 653 | } 654 | 655 | FTransform UAzureKinectDevice::JointToTransform(const k4abt_joint_t& Joint, int32 Index) 656 | { 657 | 658 | // This transform algorithm is introdeced from 659 | // https://github.com/secretlocation/azure-kinect-unreal/ 660 | // Still there is room to refactor... 661 | 662 | /** 663 | * Convert Azure Kinect Depth and Color camera co-ordinate system 664 | * to Unreal co-ordinate system 665 | * @see https://docs.microsoft.com/en-us/azure/kinect-dk/coordinate-systems 666 | * 667 | * Kinect [mm] Unreal [cm] 668 | * -------------------------------------- 669 | * +ve X-axis Right +ve Y-axis 670 | * +ve Y-axis Down -ve Z-axis 671 | * +ve Z-axis Forward +ve X-axis 672 | */ 673 | FVector Position(Joint.position.xyz.z, Joint.position.xyz.x, - Joint.position.xyz.y); 674 | Position *= 0.1f; 675 | 676 | /** 677 | * Convert the Orientation from Kinect co-ordinate system to Unreal co-ordinate system. 678 | * We negate the x, y components of the JointQuaternion since we are converting from 679 | * Kinect's Right Hand orientation to Unreal's Left Hand orientation. 680 | */ 681 | FQuat Quat( 682 | -Joint.orientation.wxyz.x, 683 | -Joint.orientation.wxyz.y, 684 | Joint.orientation.wxyz.z, 685 | Joint.orientation.wxyz.w 686 | ); 687 | 688 | return FTransform(Quat, Position); 689 | } 690 | 691 | void UAzureKinectDevice::CalcFrameCount() 692 | { 693 | float FrameTimeInMilli = 0.0f; 694 | switch (Fps) 695 | { 696 | case EKinectFps::PER_SECOND_5: 697 | FrameTimeInMilli = 1000.f / 5.f; 698 | break; 699 | case EKinectFps::PER_SECOND_15: 700 | FrameTimeInMilli = 1000.f / 15.f; 701 | break; 702 | case EKinectFps::PER_SECOND_30: 703 | FrameTimeInMilli = 1000.f / 30.f; 704 | break; 705 | default: 706 | break; 707 | } 708 | FrameTime = std::chrono::milliseconds(FMath::CeilToInt(FrameTimeInMilli)); 709 | } 710 | -------------------------------------------------------------------------------- /Source/AzureKinect/Private/AzureKinectDeviceThread.cpp: -------------------------------------------------------------------------------- 1 | #include "AzureKinectDeviceThread.h" 2 | #include "HAL/PlatformProcess.h" 3 | #include "AzureKinectDevice.h" 4 | 5 | DEFINE_LOG_CATEGORY(AzureKinectThreadLog); 6 | 7 | FAzureKinectDeviceThread::FAzureKinectDeviceThread(UAzureKinectDevice* Device) : 8 | KinectDevice(Device), 9 | Thread(nullptr), 10 | StopTaskCounter(0) 11 | { 12 | Thread = FRunnableThread::Create(this, TEXT("FAzureKinectDeviceThread"), 0, TPri_BelowNormal); //windows default = 8mb for thread, could specify more 13 | if (!Thread) 14 | { 15 | UE_LOG(AzureKinectThreadLog, Error, TEXT("Failed to create Azure Kinect thread.")); 16 | } 17 | 18 | } 19 | 20 | FAzureKinectDeviceThread::~FAzureKinectDeviceThread() 21 | { 22 | if (Thread) 23 | { 24 | delete Thread; 25 | Thread = nullptr; 26 | } 27 | } 28 | 29 | bool FAzureKinectDeviceThread::Init() 30 | { 31 | UE_LOG(AzureKinectThreadLog, Verbose, TEXT("Azure Kinect thread started.")); 32 | return true; 33 | } 34 | 35 | uint32 FAzureKinectDeviceThread::Run() 36 | { 37 | if (!KinectDevice) 38 | { 39 | UE_LOG(AzureKinectThreadLog, Error, TEXT("KinectDevice is null, could not run the thread")); 40 | return 1; 41 | } 42 | 43 | while (StopTaskCounter.GetValue() == 0) 44 | { 45 | // Do the Kinect capture, enqueue, pop body frame stuff 46 | KinectDevice->UpdateAsync(); 47 | } 48 | 49 | return 0; 50 | } 51 | 52 | void FAzureKinectDeviceThread::Stop() 53 | { 54 | StopTaskCounter.Increment(); 55 | } 56 | 57 | void FAzureKinectDeviceThread::EnsureCompletion() 58 | { 59 | Stop(); 60 | if (Thread) 61 | { 62 | Thread->WaitForCompletion(); 63 | } 64 | 65 | } 66 | 67 | FCriticalSection* FAzureKinectDeviceThread::GetCriticalSection() 68 | { 69 | return &CriticalSection; 70 | } 71 | -------------------------------------------------------------------------------- /Source/AzureKinect/Private/AzureKinectModule.cpp: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Modules/ModuleManager.h" 7 | 8 | #define LOCTEXT_NAMESPACE "FAzureKinectModule" 9 | 10 | class FAzureKinectModule : public IModuleInterface 11 | { 12 | public: 13 | 14 | /** IModuleInterface implementation */ 15 | virtual void StartupModule() override {}; 16 | virtual void ShutdownModule() override {}; 17 | }; 18 | 19 | #undef LOCTEXT_NAMESPACE 20 | 21 | IMPLEMENT_MODULE(FAzureKinectModule, AzureKinect) -------------------------------------------------------------------------------- /Source/AzureKinect/Public/AnimNode_AzureKinectPose.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "BonePose.h" 7 | #include "BoneControllers/AnimNode_SkeletalControlBase.h" 8 | #include "AzureKinectDevice.h" 9 | 10 | #include "AnimNode_AzureKinectPose.generated.h" 11 | 12 | DECLARE_LOG_CATEGORY_EXTERN(AzureKinectAnimNodeLog, Log, All); 13 | 14 | /** 15 | * 16 | */ 17 | USTRUCT(BlueprintInternalUseOnly) 18 | struct AZUREKINECT_API FAnimNode_AzureKinectPose : public FAnimNode_Base 19 | { 20 | GENERATED_BODY() 21 | 22 | public: 23 | 24 | FAnimNode_AzureKinectPose(); 25 | 26 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Transform", meta = (PinShownByDefault)) 27 | FAzureKinectSkeleton Skeleton; 28 | 29 | UPROPERTY(EditAnywhere, Category="Bone Mapping") 30 | TMap BonesToModify; 31 | 32 | // FAnimNode_Base interface 33 | virtual void Update_AnyThread(const FAnimationUpdateContext& Context) override; 34 | virtual void EvaluateComponentSpace_AnyThread(FComponentSpacePoseContext& Output) override; 35 | 36 | private: 37 | TArray BoneTransforms; 38 | }; 39 | -------------------------------------------------------------------------------- /Source/AzureKinect/Public/AzureKinectDevice.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "CoreMinimal.h" 4 | #include "Engine/TextureRenderTarget2D.h" 5 | #include "Animation/SkeletalMeshActor.h" 6 | 7 | #include "k4a/k4a.hpp" 8 | #include "k4abt.hpp" 9 | #include "AzureKinectEnum.h" 10 | #include "AzureKinectDeviceThread.h" 11 | 12 | #include "AzureKinectDevice.generated.h" 13 | 14 | USTRUCT(BlueprintType) 15 | struct FAzureKinectSkeleton 16 | { 17 | GENERATED_BODY() 18 | 19 | UPROPERTY(BlueprintReadWrite) 20 | int32 ID; 21 | 22 | UPROPERTY(BlueprintReadWrite) 23 | TArray Joints; 24 | }; 25 | 26 | DECLARE_LOG_CATEGORY_EXTERN(AzureKinectDeviceLog, Log, All); 27 | 28 | UCLASS(BlueprintType, hidecategories=(Object)) 29 | class AZUREKINECT_API UAzureKinectDevice : public UObject 30 | { 31 | GENERATED_BODY() 32 | public: 33 | UAzureKinectDevice(); 34 | UAzureKinectDevice(const FObjectInitializer& ObjectInitializer); 35 | 36 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "IO") 37 | UTextureRenderTarget2D* DepthTexture; 38 | 39 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "IO") 40 | UTextureRenderTarget2D* ColorTexture; 41 | 42 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "IO") 43 | UTextureRenderTarget2D* InflaredTexture; 44 | 45 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "IO") 46 | UTextureRenderTarget2D* BodyIndexTexture; 47 | 48 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Config") 49 | EKinectDepthMode DepthMode; 50 | 51 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Config") 52 | EKinectColorResolution ColorMode; 53 | 54 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Config") 55 | EKinectRemap RemapMode; 56 | 57 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Config") 58 | EKinectFps Fps; 59 | 60 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Config") 61 | EKinectSensorOrientation SensorOrientation = EKinectSensorOrientation::DEFAULT; 62 | 63 | UPROPERTY(BlueprintReadWrite, Category = "Config") 64 | int32 DeviceIndex; 65 | 66 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Config") 67 | bool bSkeletonTracking; 68 | 69 | UFUNCTION(BlueprintCallable, Category = "IO") 70 | static int32 GetNumConnectedDevices(); 71 | 72 | UFUNCTION(BlueprintCallable, Category = "IO") 73 | void LoadDevices(); 74 | 75 | /** 76 | * Call "open" and "start_camara" to Native Kinect Device 77 | * and return result. Then start a thread for Kinect's data feed. 78 | * Device Index should be specified in advance; 79 | */ 80 | UFUNCTION(BlueprintCallable, Category = "IO") 81 | bool StartDevice(); 82 | 83 | /** 84 | * Call "stop_camara" and "close" to Native Kinect Device, 85 | * and release all instaces about Native Kinect. 86 | */ 87 | UFUNCTION(BlueprintCallable, Category = "IO") 88 | bool StopDevice(); 89 | 90 | /** 91 | * Check if Kinect Device is open. 92 | */ 93 | UFUNCTION(BlueprintCallable, Category = "IO") 94 | bool IsOpen() const { return bOpen; } 95 | 96 | /** 97 | * Return a number of Skeletons currently aquired and stored. 98 | */ 99 | UFUNCTION(BlueprintCallable, Category = "Skeletons") 100 | int32 GetNumTrackedSkeletons() const; 101 | 102 | /** 103 | * Return an array of Skeletons currently aquired and stored. 104 | */ 105 | UFUNCTION(BlueprintCallable, Category = "Skeletons") 106 | const TArray& GetSkeletons() const; 107 | 108 | /** 109 | * Return a Skeleton struct by Index (not Skeleton ID). 110 | * If given Index is out of range, return a null struct. 111 | */ 112 | UFUNCTION(BlueprintCallable, Category = "Skeletons") 113 | FAzureKinectSkeleton GetSkeleton(int32 Index) const; 114 | 115 | /** 116 | * Update and process raw feed from Kinect Device asynchronously. 117 | * Should be called out of main thread. 118 | */ 119 | void UpdateAsync(); 120 | 121 | TArray> DeviceList; 122 | 123 | private: 124 | bool bOpen; 125 | 126 | void CaptureColorImage(); 127 | void CaptureDepthImage(); 128 | void CaptureInflaredImage(); 129 | void CaptureBodyIndexImage(const k4abt::frame& BodyFrame); 130 | 131 | static FTransform JointToTransform(const k4abt_joint_t& Joint, int32 Index); 132 | void UpdateSkeletons(); 133 | 134 | void CalcFrameCount(); 135 | 136 | k4a::device NativeDevice; 137 | k4a::capture Capture; 138 | std::chrono::milliseconds FrameTime; 139 | k4a::image RemapImage; 140 | k4a::calibration KinectCalibration; 141 | k4a::transformation KinectTransformation; 142 | k4abt::tracker BodyTracker; 143 | 144 | FAzureKinectDeviceThread* Thread; 145 | 146 | int32 NumTrackedSkeletons; 147 | TArray Skeletons; 148 | }; 149 | -------------------------------------------------------------------------------- /Source/AzureKinect/Public/AzureKinectDeviceThread.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "CoreMinimal.h" 3 | #include "HAL/Runnable.h" 4 | #include "HAL/RunnableThread.h" 5 | 6 | DECLARE_LOG_CATEGORY_EXTERN(AzureKinectThreadLog, Log, All); 7 | 8 | class UAzureKinectDevice; 9 | 10 | class FAzureKinectDeviceThread : public FRunnable 11 | { 12 | public: 13 | 14 | FAzureKinectDeviceThread(UAzureKinectDevice* Device); 15 | 16 | virtual ~FAzureKinectDeviceThread(); 17 | 18 | virtual bool Init(); 19 | virtual uint32 Run(); 20 | virtual void Stop(); 21 | 22 | /** Stops the threadand waits for its completion. */ 23 | void EnsureCompletion(); 24 | 25 | FCriticalSection* GetCriticalSection(); 26 | 27 | private: 28 | /** Thread handle.Control the thread using this, with operators like Killand Suspend */ 29 | FRunnableThread* Thread; 30 | 31 | /** Stop this thread? Uses Thread Safe Counter */ 32 | FThreadSafeCounter StopTaskCounter; 33 | 34 | /** The device that starts this thread. */ 35 | UAzureKinectDevice* KinectDevice; 36 | 37 | /** To be used for UScopeLock */ 38 | FCriticalSection CriticalSection; 39 | 40 | }; -------------------------------------------------------------------------------- /Source/AzureKinect/Public/AzureKinectEnum.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "AzureKinectEnum.generated.h" 4 | 5 | /** 6 | * Blueprintable enum defined based on k4a_depth_mode_t from k4atypes.h 7 | * 8 | * @note This should always have the same enum values as k4a_depth_mode_t 9 | */ 10 | UENUM(BlueprintType, Category = "Azure Kinect|Enums") 11 | enum class EKinectDepthMode : uint8 12 | { 13 | OFF = 0 UMETA(DisplayName = "Depth Mode Off"), /**< Depth sensor will be turned off with this setting. */ 14 | NFOV_2X2BINNED UMETA(DisplayName = "NFOV 2x2 Binned (320x288)"), /**< Depth captured at 320x288. Passive IR is also captured at 320x288. */ 15 | NFOV_UNBINNED UMETA(DisplayName = "NFOV Unbinned (640x576)"), /**< Depth captured at 640x576. Passive IR is also captured at 640x576. */ 16 | WFOV_2X2BINNED UMETA(DisplayName = "WFOV 2x2 Binned (512x512)"), /**< Depth captured at 512x512. Passive IR is also captured at 512x512. */ 17 | WFOV_UNBINNED UMETA(DisplayName = "WFOV Unbinned (1024x1024)"), /**< Depth captured at 1024x1024. Passive IR is also captured at 1024x1024. */ 18 | PASSIVE_IR UMETA(DisplayName = "Passive IR (1024x1024)"), /**< Passive IR only, captured at 1024x1024. */ 19 | }; 20 | 21 | UENUM(BlueprintType, Category = "Azure Kinect|Enums") 22 | enum class EKinectColorResolution : uint8 23 | { 24 | RESOLUTION_OFF = 0 UMETA(DisplayName = "Color Camera Turned Off"), /**< Color sensor will be turned off with this setting. */ 25 | RESOLUTION_720P UMETA(DisplayName = "1280 x 720 [16:9]"), /**< Color captured at 1280 x 720. */ 26 | RESOLUTION_1440P UMETA(DisplayName = "2560 x 1440 [16:9]"), /**< Color captured at 2560 x 1440. */ 27 | RESOLUTION_1536P UMETA(DisplayName = "2048 x 1536 [4:3]"), /**< Color captured at 2048 x 1536. */ 28 | RESOLUTION_2160P UMETA(DisplayName = "3840 x 2160 [16:9]"), /**< Color captured at 3840 x 2160. */ 29 | RESOLUTION_3072P UMETA(DisplayName = "4096 x 3072 [4:3]"), /**< Color captured at 4096 x 3072. */ 30 | }; 31 | 32 | UENUM(BlueprintType, Category = "Azure Kinect|Enums") 33 | enum class EKinectFps : uint8 34 | { 35 | PER_SECOND_5 = 0 UMETA(DisplayName = "5 fps"), 36 | PER_SECOND_15 UMETA(DisplayName = "15 fps"), 37 | PER_SECOND_30 UMETA(DisplayName = "30 fps"), 38 | }; 39 | 40 | UENUM(BlueprintType, Category = "Azure Kinect|Enums") 41 | enum class EKinectRemap : uint8 42 | { 43 | COLOR_TO_DEPTH = 0 UMETA(DisplayName = "Color to Depth"), 44 | DEPTH_TO_COLOR UMETA(DisplayName = "Depth to Color"), 45 | }; 46 | 47 | /** 48 | * Blueprintable enum defined based on k4abt_joint_id_t from k4abttypes.h 49 | * This should always have the same enum values as k4abt_joint_id_t 50 | */ 51 | UENUM(BlueprintType, Category = "Azure Kinect|Enums") 52 | enum class EKinectBodyJoint : uint8 53 | { 54 | PELVIS = 0 UMETA(DisplayName = "Pelvis"), 55 | SPINE_NAVEL UMETA(DisplayName = "Spine Navel"), 56 | SPINE_CHEST UMETA(DisplayName = "Spine Chest"), 57 | NECK UMETA(DisplayName = "Neck"), 58 | CLAVICLE_LEFT UMETA(DisplayName = "Clavicle Left"), 59 | SHOULDER_LEFT UMETA(DisplayName = "Shoulder Left"), 60 | ELBOW_LEFT UMETA(DisplayName = "Elbow Left"), 61 | WRIST_LEFT UMETA(DisplayName = "Wrist Left"), 62 | HAND_LEFT UMETA(DisplayName = "Hand Left"), 63 | HANDTIP_LEFT UMETA(DisplayName = "Hand Tip Left"), 64 | THUMB_LEFT UMETA(DisplayName = "Thumb Left"), 65 | CLAVICLE_RIGHT UMETA(DisplayName = "Clavicle Right"), 66 | SHOULDER_RIGHT UMETA(DisplayName = "Shoulder Right"), 67 | ELBOW_RIGHT UMETA(DisplayName = "Elbow Right"), 68 | WRIST_RIGHT UMETA(DisplayName = "Wrist Right"), 69 | HAND_RIGHT UMETA(DisplayName = "Hand Right"), 70 | HANDTIP_RIGHT UMETA(DisplayName = "Hand Tip Right"), 71 | THUMB_RIGHT UMETA(DisplayName = "Thumb Right"), 72 | HIP_LEFT UMETA(DisplayName = "Hip Left"), 73 | KNEE_LEFT UMETA(DisplayName = "Knee Left"), 74 | ANKLE_LEFT UMETA(DisplayName = "Ankle Left"), 75 | FOOT_LEFT UMETA(DisplayName = "Foot Left"), 76 | HIP_RIGHT UMETA(DisplayName = "Hip Right"), 77 | KNEE_RIGHT UMETA(DisplayName = "Knee Right"), 78 | ANKLE_RIGHT UMETA(DisplayName = "Ankle Right"), 79 | FOOT_RIGHT UMETA(DisplayName = "Foot Right"), 80 | HEAD UMETA(DisplayName = "Head"), 81 | NOSE UMETA(DisplayName = "Nose"), 82 | EYE_LEFT UMETA(DisplayName = "Eye Left"), 83 | EAR_LEFT UMETA(DisplayName = "Ear Left"), 84 | EYE_RIGHT UMETA(DisplayName = "Eye Right"), 85 | EAR_RIGHT UMETA(DisplayName = "Ear Right"), 86 | COUNT UMETA(DisplayName = "COUNT", Hidden), 87 | }; 88 | 89 | /** 90 | * This should always have the same enum values as k4abt_sensor_orientation_t 91 | */ 92 | UENUM(BlueprintType, Category = "Azure Kinect|Enums") 93 | enum class EKinectSensorOrientation : uint8 94 | { 95 | DEFAULT = 0 UMETA(DisplayName = "Default"), /**< Mount the sensor at its default orientation */ 96 | CLOCKWISE90 UMETA(DisplayName = "Clockwise 90"), /**< Clockwisely rotate the sensor 90 degree */ 97 | COUNTERCLOCKWISE90 UMETA(DisplayName = "Conter-clockwise 90"), /**< Counter-clockwisely rotate the sensor 90 degrees */ 98 | FLIP180 UMETA(DisplayName = "Flip 180"), /**< Mount the sensor upside-down */ 99 | }; -------------------------------------------------------------------------------- /Source/AzureKinectEditor/AzureKinectEditor.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using System.IO; 4 | using UnrealBuildTool; 5 | 6 | public class AzureKinectEditor : ModuleRules 7 | { 8 | public AzureKinectEditor(ReadOnlyTargetRules Target) : base(Target) 9 | { 10 | //OverridePackageType = PackageOverrideType.GameUncookedOnly; 11 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 12 | 13 | PrivateIncludePaths.AddRange( 14 | new string[] 15 | { 16 | "AzureKinectEditor/Private", 17 | "AzureKinectEditor/Private/AnimNodes", 18 | "AzureKinectEditor/Private/AssetTools", 19 | "AzureKinectEditor/Private/Customizations", 20 | "AzureKinectEditor/Private/Factories", 21 | }); 22 | 23 | PrivateDependencyModuleNames.AddRange( 24 | new string[] 25 | { 26 | "Core", 27 | "CoreUObject", 28 | "Engine", 29 | "Slate", 30 | "SlateCore", 31 | "PropertyEditor", 32 | "InputCore", 33 | "AzureKinect", 34 | "UnrealEd", 35 | "BlueprintGraph", 36 | "AnimGraph", 37 | }); 38 | 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/AnimNodes/AnimGraphNode_AzureKinectPose.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | 4 | #include "AnimNodes/AnimGraphNode_AzureKinectPose.h" 5 | #include "AnimationGraphSchema.h" 6 | 7 | #define LOCTEXT_NAMESPACE "AzureKinectPose" 8 | 9 | FText UAnimGraphNode_AzureKinectPose::GetNodeTitle(ENodeTitleType::Type TitleType) const 10 | { 11 | return LOCTEXT("AzureKinectPose", "Azure Kinect Pose"); 12 | } 13 | 14 | FString UAnimGraphNode_AzureKinectPose::GetNodeCategory() const 15 | { 16 | 17 | return FString("Azure Kinect"); 18 | } 19 | 20 | FText UAnimGraphNode_AzureKinectPose::GetTooltipText() const 21 | { 22 | return LOCTEXT( 23 | "AnimGraphNode_AzureKinectPose_Tooltip", 24 | "Process AzureKinect skeleton input into pose" 25 | ); 26 | } 27 | 28 | FLinearColor UAnimGraphNode_AzureKinectPose::GetNodeTitleColor() const 29 | { 30 | return FLinearColor(0.f, 0.f, 0.f); 31 | } 32 | 33 | void UAnimGraphNode_AzureKinectPose::CreateOutputPins() 34 | { 35 | CreatePin(EGPD_Output, UAnimationGraphSchema::PC_Struct, FComponentSpacePoseLink::StaticStruct(), TEXT("ComponentPose")); 36 | } 37 | 38 | #undef LOCTEXT_NAMESPACE -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/AnimNodes/AnimGraphNode_AzureKinectPose.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "AnimGraphNode_Base.h" 7 | #include "AnimNode_AzureKinectPose.h" 8 | #include "EdGraph/EdGraphNodeUtils.h" 9 | #include "AnimNodeEditModes.h" 10 | 11 | #include "AnimGraphNode_AzureKinectPose.generated.h" 12 | 13 | /** 14 | * 15 | */ 16 | UCLASS(meta = (Kerwords="Azure Kinect")) 17 | class UAnimGraphNode_AzureKinectPose : public UAnimGraphNode_Base 18 | { 19 | GENERATED_BODY() 20 | 21 | public: 22 | UPROPERTY(EditAnywhere, Category = "Settings") 23 | FAnimNode_AzureKinectPose Node; 24 | 25 | virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; 26 | virtual FString GetNodeCategory() const override; 27 | virtual FText GetTooltipText() const override; 28 | virtual FLinearColor GetNodeTitleColor() const override; 29 | virtual void CreateOutputPins() override; 30 | 31 | }; 32 | -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/AssetTools/AzureKinectDeviceActions.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | 4 | #include "AzureKinectDeviceActions.h" 5 | #include "AzureKinectDevice.h" 6 | 7 | FText FAzureKinectDeviceActions::GetName() const 8 | { 9 | return NSLOCTEXT("AzureKinectDeviceActions", "AssetTypeActions_AzureKinectDevice", "Azure Kinect Device"); 10 | } 11 | 12 | FColor FAzureKinectDeviceActions::GetTypeColor() const 13 | { 14 | return FColor::White; 15 | } 16 | 17 | UClass* FAzureKinectDeviceActions::GetSupportedClass() const 18 | { 19 | return UAzureKinectDevice::StaticClass(); 20 | } 21 | 22 | uint32 FAzureKinectDeviceActions::GetCategories() 23 | { 24 | return EAssetTypeCategories::Media; 25 | } 26 | -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/AssetTools/AzureKinectDeviceActions.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "AssetTypeActions_Base.h" 6 | 7 | class FAzureKinectDeviceActions : public FAssetTypeActions_Base 8 | { 9 | public: 10 | 11 | virtual FText GetName() const override; 12 | virtual FColor GetTypeColor() const override; 13 | virtual UClass* GetSupportedClass() const override; 14 | virtual uint32 GetCategories() override; 15 | 16 | //virtual void OpenAssetEditor(const TArray& InObjects, TSharedPtr EditWithinLevelEditor = TSharedPtr()) override; 17 | //virtual void GetActions(const TArray&InObjects, FMenuBuilder & MenuBuilder) override; 18 | //virtual bool HasActions(const TArray& InObjects) const override; 19 | 20 | private: 21 | }; 22 | -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/AzureKinectEditor.cpp: -------------------------------------------------------------------------------- 1 | 2 | 3 | // Copyright Epic Games, Inc. All Rights Reserved. 4 | 5 | #pragma once 6 | 7 | #include "CoreMinimal.h" 8 | #include "Modules/ModuleManager.h" 9 | #include "Modules/ModuleInterface.h" 10 | 11 | #include "AzureKinectDevice.h" 12 | #include "AzureKinectDeviceActions.h" 13 | #include "AzureKinectDeviceCustomization.h" 14 | 15 | 16 | #define LOCTEXT_NAMESPACE "FAzureKinectEditorModule" 17 | 18 | class FAzureKinectEditorModule : public IModuleInterface 19 | { 20 | public: 21 | 22 | /** IModuleInterface implementation */ 23 | 24 | virtual void StartupModule() override 25 | { 26 | AzureKinectDeviceName = UAzureKinectDevice::StaticClass()->GetFName(); 27 | RegisterAssetTools(); 28 | RegisterCustomizations(); 29 | } 30 | 31 | virtual void ShutdownModule() override 32 | { 33 | UnregisterAssetTools(); 34 | UnregisterCustomizations(); 35 | } 36 | 37 | protected: 38 | 39 | /** Registers asset tool actions. */ 40 | void RegisterAssetTools() 41 | { 42 | IAssetTools& AssetTools = FModuleManager::LoadModuleChecked("AssetTools").Get(); 43 | RegisterAssetTypeAction(AssetTools, MakeShareable(new FAzureKinectDeviceActions())); 44 | } 45 | 46 | /** Unregisters asset tool actions. */ 47 | void UnregisterAssetTools() 48 | { 49 | FAssetToolsModule* AssetToolsModule = FModuleManager::GetModulePtr("AssetTools"); 50 | 51 | if (AssetToolsModule) 52 | { 53 | IAssetTools& AssetTools = AssetToolsModule->Get(); 54 | 55 | for (auto Action : RegisteredAssetTypeActions) 56 | { 57 | AssetTools.UnregisterAssetTypeActions(Action); 58 | } 59 | } 60 | } 61 | 62 | /** 63 | * Registers a single asset type action. 64 | * 65 | * @param AssetTools The asset tools object to register with. 66 | * @param Action The asset type action to register. 67 | */ 68 | void RegisterAssetTypeAction(IAssetTools& AssetTools, TSharedRef Action) 69 | { 70 | AssetTools.RegisterAssetTypeActions(Action); 71 | RegisteredAssetTypeActions.Add(Action); 72 | } 73 | 74 | 75 | /** Register details view customizations. */ 76 | void RegisterCustomizations() 77 | { 78 | FPropertyEditorModule& PropertyModule = FModuleManager::LoadModuleChecked("PropertyEditor"); 79 | PropertyModule.RegisterCustomClassLayout(AzureKinectDeviceName, FOnGetDetailCustomizationInstance::CreateStatic(&FAzureKinectDeviceCustomization::MakeInstance)); 80 | } 81 | 82 | void UnregisterCustomizations() 83 | { 84 | FPropertyEditorModule& PropertyModule = FModuleManager::LoadModuleChecked("PropertyEditor"); 85 | PropertyModule.UnregisterCustomClassLayout(AzureKinectDeviceName); 86 | } 87 | 88 | private: 89 | /** The collection of registered asset type actions. */ 90 | TArray> RegisteredAssetTypeActions; 91 | 92 | FName AzureKinectDeviceName; 93 | 94 | }; 95 | 96 | #undef LOCTEXT_NAMESPACE 97 | 98 | IMPLEMENT_MODULE(FAzureKinectEditorModule, AzureKinectEditor) -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/Customizations/AzureKinectDeviceCustomization.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #include "AzureKinectDeviceCustomization.h" 4 | 5 | #include "DetailCategoryBuilder.h" 6 | #include "DetailLayoutBuilder.h" 7 | #include "DetailWidgetRow.h" 8 | #include "Widgets/Input/SComboBox.h" 9 | #include "PropertyCustomizationHelpers.h" 10 | 11 | #define LOCTEXT_NAMESPACE "AzureKinectDeviceCustomization" 12 | 13 | TSharedRef FAzureKinectDeviceCustomization::MakeInstance() 14 | { 15 | return MakeShareable(new FAzureKinectDeviceCustomization()); 16 | } 17 | 18 | void FAzureKinectDeviceCustomization::CustomizeDetails(IDetailLayoutBuilder& DetailBuilder) 19 | { 20 | 21 | // Retrieve target object 22 | TArray> Objects; 23 | DetailBuilder.GetObjectsBeingCustomized(Objects); 24 | if (Objects.Num() == 1) 25 | { 26 | AzureKinectDevice = Cast(Objects[0].Get()); 27 | } 28 | else 29 | { 30 | return; 31 | } 32 | 33 | // Customize 'Config' category 34 | IDetailCategoryBuilder& ConfigCategory = DetailBuilder.EditCategory("Config"); 35 | 36 | TAttribute CheckDeviceOpen = TAttribute::Create(TAttribute::FGetter::CreateLambda( 37 | [this]() { 38 | return !AzureKinectDevice->IsOpen(); 39 | })); 40 | 41 | { 42 | // Add Custom Row of Device selection 43 | CurrentOption = AzureKinectDevice->DeviceList[0]; 44 | 45 | ConfigCategory.AddCustomRow(LOCTEXT("DeviceSelectionFilterString", "Device Selection")) 46 | .NameContent() 47 | [ 48 | SNew(STextBlock) 49 | .Text(LOCTEXT("DeviceSelectionLabel", "Device Selection")) 50 | ] 51 | .ValueContent() 52 | [ 53 | SNew(SComboBox>) 54 | .IsEnabled(CheckDeviceOpen) 55 | .OptionsSource(&(AzureKinectDevice->DeviceList)) 56 | .OnSelectionChanged_Raw(this, &FAzureKinectDeviceCustomization::OnSelectionChanged) 57 | .OnGenerateWidget_Raw(this, &FAzureKinectDeviceCustomization::MakeWidgetForOption) 58 | .InitiallySelectedItem(CurrentOption) 59 | [ 60 | SNew(STextBlock) 61 | .Text(this, &FAzureKinectDeviceCustomization::GetCurrentItemLabel) 62 | ] 63 | ]; 64 | } 65 | 66 | // Alternative of UProperty specifier: meta=(EditCondition="bOpen") 67 | // I don't wanna make "bOpen" editable UProperty. 68 | // Below is a workarround how to make UProperty conditional without condition (Uproperty boolean) 69 | auto DepthMode = DetailBuilder.GetProperty(GET_MEMBER_NAME_CHECKED(UAzureKinectDevice, DepthMode)); 70 | auto ColorMode = DetailBuilder.GetProperty(GET_MEMBER_NAME_CHECKED(UAzureKinectDevice, ColorMode)); 71 | auto Fps = DetailBuilder.GetProperty(GET_MEMBER_NAME_CHECKED(UAzureKinectDevice, Fps)); 72 | auto SensorOrientation = DetailBuilder.GetProperty(GET_MEMBER_NAME_CHECKED(UAzureKinectDevice, SensorOrientation)); 73 | auto RemapMode = DetailBuilder.GetProperty(GET_MEMBER_NAME_CHECKED(UAzureKinectDevice, RemapMode)); 74 | auto SkeletonTracking = DetailBuilder.GetProperty(GET_MEMBER_NAME_CHECKED(UAzureKinectDevice, bSkeletonTracking)); 75 | 76 | ConfigCategory.AddProperty(DepthMode).IsEnabled(CheckDeviceOpen); 77 | ConfigCategory.AddProperty(ColorMode).IsEnabled(CheckDeviceOpen); 78 | ConfigCategory.AddProperty(Fps).IsEnabled(CheckDeviceOpen); 79 | ConfigCategory.AddProperty(SensorOrientation).IsEnabled(CheckDeviceOpen); 80 | ConfigCategory.AddProperty(RemapMode).IsEnabled(CheckDeviceOpen); 81 | ConfigCategory.AddProperty(SkeletonTracking).IsEnabled(CheckDeviceOpen); 82 | 83 | 84 | // Customize 'IO' category 85 | IDetailCategoryBuilder& IOCategory = DetailBuilder.EditCategory("IO"); 86 | 87 | // Add Custom Row of Execution buttons 88 | IOCategory.AddCustomRow(LOCTEXT("ButtonFilterString", "Function Buttons")) 89 | .NameContent() 90 | [ 91 | SNew(STextBlock) 92 | .Text(LOCTEXT("ExecutionLabel", "Execution")) 93 | ] 94 | .ValueContent() 95 | [ 96 | SNew(SHorizontalBox) 97 | + SHorizontalBox::Slot() 98 | .Padding(FMargin(0.f, 2.f, 10.f, 2.f)) 99 | .AutoWidth() 100 | [ 101 | SNew(SButton) 102 | .Text(LOCTEXT("LoadButtonText", "LoadDevice")) 103 | .Visibility_Lambda([this]() { 104 | return AzureKinectDevice->IsOpen() ? EVisibility::Collapsed : EVisibility::Visible; 105 | }) 106 | .OnClicked_Lambda([this]() { 107 | AzureKinectDevice->LoadDevices(); 108 | return FReply::Handled(); 109 | }) 110 | ] 111 | + SHorizontalBox::Slot() 112 | .Padding(FMargin(0.f, 2.f, 10.0f, 2.f)) 113 | .AutoWidth() 114 | [ 115 | SNew(SButton) 116 | .Text(LOCTEXT("StartButtonText", "StartDevice")) 117 | .Visibility_Lambda([this]() { 118 | return AzureKinectDevice->IsOpen() ? EVisibility::Collapsed : EVisibility::Visible; 119 | }) 120 | .OnClicked_Lambda([this]() { 121 | AzureKinectDevice->StartDevice(); 122 | return FReply::Handled(); 123 | }) 124 | ] 125 | + SHorizontalBox::Slot() 126 | .Padding(FMargin(0.f, 0, 10.f, 2.f)) 127 | .AutoWidth() 128 | [ 129 | SNew(SButton) 130 | .Text(LOCTEXT("StopButtonText", "StopDevice")) 131 | .Visibility_Lambda([this]() { 132 | return AzureKinectDevice->IsOpen() ? EVisibility::Visible : EVisibility::Collapsed; 133 | }) 134 | .OnClicked_Lambda([this]() { 135 | AzureKinectDevice->StopDevice(); 136 | return FReply::Handled(); 137 | }) 138 | ] 139 | ]; 140 | 141 | } 142 | 143 | TSharedRef FAzureKinectDeviceCustomization::MakeWidgetForOption(TSharedPtr InOption) 144 | { 145 | return SNew(STextBlock).Text(FText::FromString(*InOption)); 146 | } 147 | 148 | void FAzureKinectDeviceCustomization::OnSelectionChanged(TSharedPtr NewValue, ESelectInfo::Type) 149 | { 150 | CurrentOption = NewValue; 151 | // Also update UAzureKinectDevice's current index 152 | int32 IndexOfFound = AzureKinectDevice->DeviceList.Find(NewValue); 153 | if (IndexOfFound == INDEX_NONE) 154 | { 155 | AzureKinectDevice->DeviceIndex = - 1; 156 | } 157 | else 158 | { 159 | AzureKinectDevice->DeviceIndex = IndexOfFound - 1; 160 | } 161 | } 162 | 163 | FText FAzureKinectDeviceCustomization::GetCurrentItemLabel() const 164 | { 165 | if (CurrentOption.IsValid()) 166 | { 167 | return FText::FromString(*CurrentOption); 168 | } 169 | 170 | return LOCTEXT("InvalidComboEntryText", "No Device"); 171 | } 172 | 173 | #undef LOCTEXT_NAMESPACE -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/Customizations/AzureKinectDeviceCustomization.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "IDetailCustomization.h" 7 | #include "AzureKinectDevice.h" 8 | /** 9 | * 10 | */ 11 | class FAzureKinectDeviceCustomization : public IDetailCustomization 12 | { 13 | public: 14 | static TSharedRef MakeInstance(); 15 | virtual void CustomizeDetails(IDetailLayoutBuilder& DetailBuilder) override; 16 | 17 | TSharedRef MakeWidgetForOption(TSharedPtr InOption); 18 | void OnSelectionChanged(TSharedPtr NewValue, ESelectInfo::Type); 19 | FText GetCurrentItemLabel() const; 20 | 21 | private: 22 | 23 | TWeakObjectPtr AzureKinectDevice; 24 | TSharedPtr CurrentOption; 25 | 26 | }; 27 | -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/Factories/AzureKinectDeviceFactory.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | 4 | #include "AzureKinectDeviceFactory.h" 5 | #include "AzureKinectDevice.h" 6 | #include "AssetTypeCategories.h" 7 | 8 | UAzureKinectDeviceFactory::UAzureKinectDeviceFactory(const FObjectInitializer& ObjectInitializer) : Super(ObjectInitializer) 9 | { 10 | bCreateNew = true; 11 | bEditAfterNew = true; 12 | SupportedClass = UAzureKinectDevice::StaticClass(); 13 | } 14 | 15 | UObject* UAzureKinectDeviceFactory::FactoryCreateNew(UClass* InClass, UObject* InParent, FName InName, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) 16 | { 17 | return NewObject(InParent, InClass, InName, Flags); 18 | } 19 | 20 | bool UAzureKinectDeviceFactory::ShouldShowInNewMenu() const 21 | { 22 | return true; 23 | } 24 | 25 | uint32 UAzureKinectDeviceFactory::GetMenuCategories() const 26 | { 27 | return EAssetTypeCategories::Misc; 28 | } 29 | -------------------------------------------------------------------------------- /Source/AzureKinectEditor/Private/Factories/AzureKinectDeviceFactory.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "Factories/Factory.h" 6 | #include "UObject/ObjectMacros.h" 7 | 8 | #include "AzureKinectDeviceFactory.generated.h" 9 | 10 | /** 11 | * Factory class of UAzureKinectDevice. 12 | * UAzureKinectDevice can be Asset Type due to this factory class. 13 | */ 14 | UCLASS(hidecategories=Object) 15 | class UAzureKinectDeviceFactory : public UFactory 16 | { 17 | GENERATED_UCLASS_BODY() 18 | public: 19 | 20 | virtual UObject* FactoryCreateNew(UClass* InClass, UObject* InParent, FName InName, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) override; 21 | virtual bool ShouldShowInNewMenu() const override; 22 | virtual uint32 GetMenuCategories() const override; 23 | }; 24 | --------------------------------------------------------------------------------