├── .gitignore ├── CaptureToDisk ├── CaptureToDisk.uproject ├── Config │ ├── DefaultEditor.ini │ ├── DefaultEngine.ini │ ├── DefaultGame.ini │ └── DefaultInput.ini ├── Content │ ├── NewWorld.umap │ └── PP_Materials │ │ ├── PP_Depth.uasset │ │ ├── PP_PixelAnnotation.uasset │ │ └── PP_Segmentation.uasset └── Source │ ├── CaptureToDisk.Target.cs │ ├── CaptureToDisk │ ├── CaptureToDisk.Build.cs │ ├── CaptureToDisk.cpp │ ├── CaptureToDisk.h │ ├── Private │ │ └── FrameCaptureManager.cpp │ └── Public │ │ └── FrameCaptureManager.h │ └── CaptureToDiskEditor.Target.cs ├── LICENSE ├── README.md └── gfx ├── Activate_Lumen.png ├── Apply_custom_depth_stencil.png ├── CaptureResult_color.jpeg ├── CaptureResult_segmentation.png ├── Debug_level_blueprint_capture.png ├── Enable_custom_depth_in_project.png ├── FrameCaptureManage_settings.png ├── pp_depth.png └── pp_segmentation.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Visual Studio 2015 user specific files 2 | .vs/ 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | 22 | # Compiled Static libraries 23 | *.lai 24 | *.la 25 | *.a 26 | *.lib 27 | 28 | # Executables 29 | *.exe 30 | *.out 31 | *.app 32 | *.ipa 33 | 34 | # These project files can be generated by the engine 35 | *.xcodeproj 36 | *.xcworkspace 37 | *.sln 38 | *.suo 39 | *.opensdf 40 | *.sdf 41 | *.VC.db 42 | *.VC.opendb 43 | 44 | # Precompiled Assets 45 | SourceArt/**/*.png 46 | SourceArt/**/*.tga 47 | 48 | # Binary Files 49 | Binaries/* 50 | Plugins/*/Binaries/* 51 | 52 | # Builds 53 | Build/* 54 | 55 | # Whitelist PakBlacklist-.txt files 56 | !Build/*/ 57 | Build/*/** 58 | !Build/*/PakBlacklist*.txt 59 | 60 | # Don't ignore icon files in Build 61 | !Build/**/*.ico 62 | 63 | # Built data for maps 64 | *_BuiltData.uasset 65 | 66 | # Configuration files generated by the Editor 67 | Saved/* 68 | 69 | # Compiled source files for the engine to use 70 | Intermediate/* 71 | Plugins/*/Intermediate/* 72 | 73 | # Cache files for the editor to use 74 | DerivedDataCache/* 75 | 76 | -------------------------------------------------------------------------------- /CaptureToDisk/CaptureToDisk.uproject: -------------------------------------------------------------------------------- 1 | { 2 | "FileVersion": 3, 3 | "EngineAssociation": "5.5", 4 | "Category": "", 5 | "Description": "", 6 | "Modules": [ 7 | { 8 | "Name": "CaptureToDisk", 9 | "Type": "Runtime", 10 | "LoadingPhase": "Default", 11 | "AdditionalDependencies": [ 12 | "Engine" 13 | ] 14 | } 15 | ], 16 | "Plugins": [ 17 | { 18 | "Name": "ModelingToolsEditorMode", 19 | "Enabled": true, 20 | "TargetAllowList": [ 21 | "Editor" 22 | ] 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /CaptureToDisk/Config/DefaultEditor.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/CaptureToDisk/Config/DefaultEditor.ini -------------------------------------------------------------------------------- /CaptureToDisk/Config/DefaultEngine.ini: -------------------------------------------------------------------------------- 1 | 2 | 3 | [/Script/EngineSettings.GameMapsSettings] 4 | GameDefaultMap=None 5 | EditorStartupMap=/Game/NewWorld.NewWorld 6 | 7 | [/Script/Engine.RendererSettings] 8 | r.AllowStaticLighting=False 9 | 10 | r.GenerateMeshDistanceFields=True 11 | 12 | r.DynamicGlobalIlluminationMethod=1 13 | 14 | r.ReflectionMethod=1 15 | 16 | r.SkinCache.CompileShaders=True 17 | 18 | r.RayTracing=True 19 | 20 | r.Shadow.Virtual.Enable=1 21 | 22 | r.DefaultFeature.AutoExposure.ExtendDefaultLuminanceRange=True 23 | 24 | r.DefaultFeature.LocalExposure.HighlightContrastScale=0.8 25 | 26 | r.DefaultFeature.LocalExposure.ShadowContrastScale=0.8 27 | r.CustomDepth=3 28 | 29 | [/Script/WindowsTargetPlatform.WindowsTargetSettings] 30 | DefaultGraphicsRHI=DefaultGraphicsRHI_DX12 31 | DefaultGraphicsRHI=DefaultGraphicsRHI_DX12 32 | -D3D12TargetedShaderFormats=PCD3D_SM5 33 | +D3D12TargetedShaderFormats=PCD3D_SM6 34 | -D3D11TargetedShaderFormats=PCD3D_SM5 35 | +D3D11TargetedShaderFormats=PCD3D_SM5 36 | Compiler=Default 37 | AudioSampleRate=48000 38 | AudioCallbackBufferFrameSize=1024 39 | AudioNumBuffersToEnqueue=1 40 | AudioMaxChannels=0 41 | AudioNumSourceWorkers=4 42 | SpatializationPlugin= 43 | SourceDataOverridePlugin= 44 | ReverbPlugin= 45 | OcclusionPlugin= 46 | CompressionOverrides=(bOverrideCompressionTimes=False,DurationThreshold=5.000000,MaxNumRandomBranches=0,SoundCueQualityIndex=0) 47 | CacheSizeKB=65536 48 | MaxChunkSizeOverrideKB=0 49 | bResampleForDevice=False 50 | MaxSampleRate=48000.000000 51 | HighSampleRate=32000.000000 52 | MedSampleRate=24000.000000 53 | LowSampleRate=12000.000000 54 | MinSampleRate=8000.000000 55 | CompressionQualityModifier=1.000000 56 | AutoStreamingThreshold=0.000000 57 | SoundCueCookQualityIndex=-1 58 | 59 | [/Script/LinuxTargetPlatform.LinuxTargetSettings] 60 | -TargetedRHIs=SF_VULKAN_SM5 61 | +TargetedRHIs=SF_VULKAN_SM6 62 | 63 | [/Script/HardwareTargeting.HardwareTargetingSettings] 64 | TargetedHardwareClass=Desktop 65 | AppliedTargetedHardwareClass=Desktop 66 | DefaultGraphicsPerformance=Maximum 67 | AppliedDefaultGraphicsPerformance=Maximum 68 | 69 | [/Script/WorldPartitionEditor.WorldPartitionEditorSettings] 70 | CommandletClass=Class'/Script/UnrealEd.WorldPartitionConvertCommandlet' 71 | 72 | [/Script/Engine.UserInterfaceSettings] 73 | bAuthorizeAutomaticWidgetVariableCreation=False 74 | FontDPIPreset=Standard 75 | FontDPI=72 76 | 77 | [/Script/Engine.Engine] 78 | +ActiveGameNameRedirects=(OldGameName="TP_Blank",NewGameName="/Script/CaptureToDisk") 79 | +ActiveGameNameRedirects=(OldGameName="/Script/TP_Blank",NewGameName="/Script/CaptureToDisk") 80 | 81 | [/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings] 82 | bEnablePlugin=True 83 | bAllowNetworkConnection=True 84 | SecurityToken=C073144A4B99B66C517BCE84ED67D96E 85 | bIncludeInShipping=False 86 | bAllowExternalStartInShipping=False 87 | bCompileAFSProject=False 88 | bUseCompression=False 89 | bLogFiles=False 90 | bReportStats=False 91 | ConnectionType=USBOnly 92 | bUseManualIPAddress=False 93 | ManualIPAddress= 94 | 95 | -------------------------------------------------------------------------------- /CaptureToDisk/Config/DefaultGame.ini: -------------------------------------------------------------------------------- 1 | 2 | [/Script/EngineSettings.GeneralProjectSettings] 3 | ProjectID=367F1D8A41791D7E0556B6BBFEE3C31B 4 | -------------------------------------------------------------------------------- /CaptureToDisk/Config/DefaultInput.ini: -------------------------------------------------------------------------------- 1 | [/Script/Engine.InputSettings] 2 | -AxisConfig=(AxisKeyName="Gamepad_LeftX",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 3 | -AxisConfig=(AxisKeyName="Gamepad_LeftY",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 4 | -AxisConfig=(AxisKeyName="Gamepad_RightX",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 5 | -AxisConfig=(AxisKeyName="Gamepad_RightY",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 6 | -AxisConfig=(AxisKeyName="MouseX",AxisProperties=(DeadZone=0.f,Exponent=1.f,Sensitivity=0.07f)) 7 | -AxisConfig=(AxisKeyName="MouseY",AxisProperties=(DeadZone=0.f,Exponent=1.f,Sensitivity=0.07f)) 8 | -AxisConfig=(AxisKeyName="Mouse2D",AxisProperties=(DeadZone=0.f,Exponent=1.f,Sensitivity=0.07f)) 9 | +AxisConfig=(AxisKeyName="Gamepad_LeftX",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 10 | +AxisConfig=(AxisKeyName="Gamepad_LeftY",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 11 | +AxisConfig=(AxisKeyName="Gamepad_RightX",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 12 | +AxisConfig=(AxisKeyName="Gamepad_RightY",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 13 | +AxisConfig=(AxisKeyName="MouseX",AxisProperties=(DeadZone=0.000000,Sensitivity=0.070000,Exponent=1.000000,bInvert=False)) 14 | +AxisConfig=(AxisKeyName="MouseY",AxisProperties=(DeadZone=0.000000,Sensitivity=0.070000,Exponent=1.000000,bInvert=False)) 15 | +AxisConfig=(AxisKeyName="Mouse2D",AxisProperties=(DeadZone=0.000000,Sensitivity=0.070000,Exponent=1.000000,bInvert=False)) 16 | +AxisConfig=(AxisKeyName="MouseWheelAxis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 17 | +AxisConfig=(AxisKeyName="Gamepad_LeftTriggerAxis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 18 | +AxisConfig=(AxisKeyName="Gamepad_RightTriggerAxis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 19 | +AxisConfig=(AxisKeyName="Gamepad_Special_Left_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 20 | +AxisConfig=(AxisKeyName="Gamepad_Special_Left_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 21 | +AxisConfig=(AxisKeyName="Vive_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 22 | +AxisConfig=(AxisKeyName="Vive_Left_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 23 | +AxisConfig=(AxisKeyName="Vive_Left_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 24 | +AxisConfig=(AxisKeyName="Vive_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 25 | +AxisConfig=(AxisKeyName="Vive_Right_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 26 | +AxisConfig=(AxisKeyName="Vive_Right_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 27 | +AxisConfig=(AxisKeyName="MixedReality_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 28 | +AxisConfig=(AxisKeyName="MixedReality_Left_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 29 | +AxisConfig=(AxisKeyName="MixedReality_Left_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 30 | +AxisConfig=(AxisKeyName="MixedReality_Left_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 31 | +AxisConfig=(AxisKeyName="MixedReality_Left_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 32 | +AxisConfig=(AxisKeyName="MixedReality_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 33 | +AxisConfig=(AxisKeyName="MixedReality_Right_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 34 | +AxisConfig=(AxisKeyName="MixedReality_Right_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 35 | +AxisConfig=(AxisKeyName="MixedReality_Right_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 36 | +AxisConfig=(AxisKeyName="MixedReality_Right_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 37 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 38 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 39 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 40 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 41 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 42 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 43 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 44 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 45 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 46 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Grip_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 47 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 48 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 49 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 50 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 51 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 52 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trackpad_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 53 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 54 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Grip_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 55 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 56 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 57 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 58 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 59 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 60 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trackpad_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 61 | bAltEnterTogglesFullscreen=True 62 | bF11TogglesFullscreen=True 63 | bUseMouseForTouch=False 64 | bEnableMouseSmoothing=True 65 | bEnableFOVScaling=True 66 | bCaptureMouseOnLaunch=True 67 | bEnableLegacyInputScales=True 68 | bEnableMotionControls=True 69 | bFilterInputByPlatformUser=False 70 | bShouldFlushPressedKeysOnViewportFocusLost=True 71 | bAlwaysShowTouchInterface=False 72 | bShowConsoleOnFourFingerTap=True 73 | bEnableGestureRecognizer=False 74 | bUseAutocorrect=False 75 | DefaultViewportMouseCaptureMode=CapturePermanently_IncludingInitialMouseDown 76 | DefaultViewportMouseLockMode=LockOnCapture 77 | FOVScale=0.011110 78 | DoubleClickTime=0.200000 79 | DefaultPlayerInputClass=/Script/EnhancedInput.EnhancedPlayerInput 80 | DefaultInputComponentClass=/Script/EnhancedInput.EnhancedInputComponent 81 | DefaultTouchInterface=/Engine/MobileResources/HUD/DefaultVirtualJoysticks.DefaultVirtualJoysticks 82 | -ConsoleKeys=Tilde 83 | +ConsoleKeys=Tilde 84 | 85 | -------------------------------------------------------------------------------- /CaptureToDisk/Content/NewWorld.umap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/CaptureToDisk/Content/NewWorld.umap -------------------------------------------------------------------------------- /CaptureToDisk/Content/PP_Materials/PP_Depth.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/CaptureToDisk/Content/PP_Materials/PP_Depth.uasset -------------------------------------------------------------------------------- /CaptureToDisk/Content/PP_Materials/PP_PixelAnnotation.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/CaptureToDisk/Content/PP_Materials/PP_PixelAnnotation.uasset -------------------------------------------------------------------------------- /CaptureToDisk/Content/PP_Materials/PP_Segmentation.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/CaptureToDisk/Content/PP_Materials/PP_Segmentation.uasset -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDisk.Target.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | using System.Collections.Generic; 5 | 6 | public class CaptureToDiskTarget : TargetRules 7 | { 8 | public CaptureToDiskTarget(TargetInfo Target) : base(Target) 9 | { 10 | Type = TargetType.Game; 11 | DefaultBuildSettings = BuildSettingsVersion.V5; 12 | IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_5; 13 | ExtraModuleNames.Add("CaptureToDisk"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDisk/CaptureToDisk.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | 5 | public class CaptureToDisk : ModuleRules 6 | { 7 | public CaptureToDisk(ReadOnlyTargetRules Target) : base(Target) 8 | { 9 | PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; 10 | 11 | PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "EnhancedInput" , "ImageWrapper", "RenderCore", "Renderer", "RHI"}); 12 | 13 | PrivateDependencyModuleNames.AddRange(new string[] { }); 14 | 15 | // Uncomment if you are using Slate UI 16 | // PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" }); 17 | 18 | // Uncomment if you are using online features 19 | // PrivateDependencyModuleNames.Add("OnlineSubsystem"); 20 | 21 | // To include OnlineSubsystemSteam, add it to the plugins section in your uproject file with the Enabled attribute set to true 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDisk/CaptureToDisk.cpp: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #include "CaptureToDisk.h" 4 | #include "Modules/ModuleManager.h" 5 | 6 | IMPLEMENT_PRIMARY_GAME_MODULE( FDefaultGameModuleImpl, CaptureToDisk, "CaptureToDisk" ); 7 | -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDisk/CaptureToDisk.h: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | 7 | -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDisk/Private/FrameCaptureManager.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | 4 | #include "FrameCaptureManager.h" 5 | 6 | #include "Engine/SceneCapture2D.h" 7 | #include "Components/SceneCaptureComponent2D.h" 8 | #include "ShowFlags.h" 9 | 10 | #include "Engine/TextureRenderTarget2D.h" 11 | 12 | #include "Materials/Material.h" 13 | 14 | #include "RHICommandList.h" 15 | 16 | #include "IImageWrapper.h" 17 | #include "IImageWrapperModule.h" 18 | 19 | #include "ImageUtils.h" 20 | 21 | #include "Modules/ModuleManager.h" 22 | 23 | #include "Misc/FileHelper.h" 24 | 25 | // Sets default values 26 | AFrameCaptureManager::AFrameCaptureManager() 27 | { 28 | // Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it. 29 | PrimaryActorTick.bCanEverTick = true; 30 | 31 | } 32 | 33 | // Called when the game starts or when spawned 34 | void AFrameCaptureManager::BeginPlay() 35 | { 36 | Super::BeginPlay(); 37 | 38 | if(CaptureComponent){ // nullptr check 39 | SetupCaptureComponent(); 40 | } else{ 41 | UE_LOG(LogTemp, Error, TEXT("No CaptureComponent set!")); 42 | } 43 | } 44 | 45 | // Called every frame 46 | void AFrameCaptureManager::Tick(float DeltaTime) 47 | { 48 | Super::Tick(DeltaTime); 49 | 50 | if(!RenderRequestQueue.IsEmpty()){ 51 | // Peek the next RenderRequest from queue 52 | TSharedPtr nextRenderRequest = *RenderRequestQueue.Peek(); 53 | 54 | if(nextRenderRequest){ //nullptr check 55 | if(nextRenderRequest->RenderFence.IsFenceComplete() && nextRenderRequest->Readback.IsReady()){ 56 | // Load the image wrapper module 57 | //IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); 58 | 59 | // Get Data from Readback 60 | nextRenderRequest->RawSize = nextRenderRequest->ImageSize.X * nextRenderRequest->ImageSize.Y * sizeof(FColor); 61 | if(ImageFormat == ECustomImageFormat::EXR){ // handle float case 62 | nextRenderRequest->RawSize = nextRenderRequest->ImageSize.X * nextRenderRequest->ImageSize.Y * sizeof(FFloat16Color); 63 | } 64 | 65 | int32 RowPitchInPixels; 66 | nextRenderRequest->RawData = nextRenderRequest->Readback.Lock(RowPitchInPixels, nullptr); // Pass RowPitchInPixels and no buffer size 67 | 68 | // Generate image name 69 | FString fileName = ""; 70 | fileName = FPaths::ProjectSavedDir() + SubDirectoryName + "/img" + "_" + ToStringWithLeadingZeros(ImgCounter, NumDigits); 71 | fileName += GetFileEnding(ImageFormat); 72 | 73 | // Pass the raw data, filename, width, height, and format to the async task 74 | RunAsyncImageSaveTask(nextRenderRequest, fileName, FrameWidth, FrameHeight, GetRGBFormatFromImageFormat(ImageFormat), ConvertImageFormat(ImageFormat)); 75 | 76 | // Increase ImgCounter for file names 77 | ImgCounter += 1; 78 | 79 | // Release RenderRequest form the queue 80 | RenderRequestQueue.Pop(); // Delete the first element from RenderQueue 81 | // Put it into the queue of outsourced threads 82 | InThreadRenderRequestQueue.Enqueue(nextRenderRequest); //Push to InThreadRenderRequestQueue 83 | } 84 | } 85 | } 86 | if(!InThreadRenderRequestQueue.IsEmpty()){ 87 | UE_LOG(LogTemp, Log, TEXT("InThreadRenderRequestQueue not empty.")); 88 | 89 | // Get next element of the InThreadRenderRequestQueue 90 | TSharedPtr nextRenderRequest = *InThreadRenderRequestQueue.Peek(); 91 | 92 | if(nextRenderRequest){ //nullptr check 93 | if(nextRenderRequest->bIsComplete){ //check if complete 94 | InThreadRenderRequestQueue.Pop(); // Remove from queue 95 | } 96 | } 97 | } 98 | } 99 | 100 | void AFrameCaptureManager::SetupCaptureComponent(){ 101 | if(!IsValid(CaptureComponent)){ 102 | UE_LOG(LogTemp, Error, TEXT("SetupCaptureComponent: CaptureComponent is not valid!")); 103 | return; 104 | } 105 | 106 | // Create RenderTargets 107 | UTextureRenderTarget2D* renderTarget2D = NewObject(); 108 | renderTarget2D->InitAutoFormat(256, 256); // some random format, got crashing otherwise 109 | 110 | // Float Capture 111 | if(ImageFormat == ECustomImageFormat::EXR){ // handle float case 112 | renderTarget2D->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA32f; 113 | renderTarget2D->InitCustomFormat(FrameWidth, FrameHeight, PF_FloatRGBA, true); // PF_B8G8R8A8 disables HDR which will boost storing to disk due to less image information 114 | } 115 | // Color Capture 116 | else{ 117 | renderTarget2D->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8_SRGB; //ETextureRenderTargetFormat::RTF_RGBA8; //8-bit color format 118 | renderTarget2D->InitCustomFormat(FrameWidth, FrameHeight, PF_R8G8B8A8, true); // PF_R8G8B8A8 //PF_B8G8R8A8 // PF... disables HDR, which is most important since HDR gives gigantic overhead, and is not needed! 119 | renderTarget2D->bForceLinearGamma = true; // Important for viewport-like color reproduction. 120 | } 121 | renderTarget2D->bGPUSharedFlag = true; // demand buffer on GPU 122 | 123 | // Assign RenderTarget 124 | CaptureComponent->GetCaptureComponent2D()->TextureTarget = renderTarget2D; 125 | // Set Camera Properties 126 | CaptureComponent->GetCaptureComponent2D()->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR; 127 | CaptureComponent->GetCaptureComponent2D()->TextureTarget->TargetGamma = GEngine->GetDisplayGamma(); 128 | CaptureComponent->GetCaptureComponent2D()->ShowFlags.SetTemporalAA(true); 129 | // lookup more showflags in documentation 130 | 131 | // Assign PostProcess Material if assigned 132 | if(PostProcessMaterial){ // check nullptr 133 | CaptureComponent->GetCaptureComponent2D()->AddOrUpdateBlendable(PostProcessMaterial); 134 | } else { 135 | UE_LOG(LogTemp, Log, TEXT("No PostProcessMaterial is assigend")); 136 | } 137 | UE_LOG(LogTemp, Warning, TEXT("Initialized RenderTarget!")); 138 | } 139 | 140 | 141 | 142 | void AFrameCaptureManager::CaptureNonBlocking(){ 143 | if(!IsValid(CaptureComponent)){ 144 | UE_LOG(LogTemp, Error, TEXT("CaptureColorNonBlocking: CaptureComponent was not valid!")); 145 | return; 146 | } 147 | CaptureComponent->GetCaptureComponent2D()->TextureTarget->TargetGamma = GEngine->GetDisplayGamma(); // 1.2f; 148 | 149 | // Get RenderConterxt 150 | FTextureRenderTargetResource* renderTargetResource = CaptureComponent->GetCaptureComponent2D()->TextureTarget->GameThread_GetRenderTargetResource(); 151 | 152 | TSharedPtr renderRequest = 153 | MakeShared( 154 | renderTargetResource->GetSizeXY(), 155 | FRHIGPUTextureReadback(TEXT("CameraCaptureManagerReadback") 156 | ) 157 | ); 158 | 159 | ENQUEUE_RENDER_COMMAND(SceneDrawCompletion)( 160 | [renderRequest, renderTargetResource](FRHICommandListImmediate& RHICmdList) { 161 | FTextureRHIRef Target = renderTargetResource->GetRenderTargetTexture(); 162 | renderRequest->Readback.EnqueueCopy(RHICmdList, Target); 163 | }); 164 | 165 | // Notifiy new task in RenderQueue 166 | RenderRequestQueue.Enqueue(renderRequest); 167 | 168 | // Set RenderCommandFence 169 | renderRequest->RenderFence.BeginFence(); 170 | } 171 | 172 | 173 | 174 | FString AFrameCaptureManager::ToStringWithLeadingZeros(int32 Integer, int32 MaxDigits){ 175 | FString result = FString::FromInt(Integer); 176 | int32 stringSize = result.Len(); 177 | int32 stringDelta = MaxDigits - stringSize; 178 | if(stringDelta < 0){ 179 | UE_LOG(LogTemp, Error, TEXT("MaxDigits of ImageCounter Overflow!")); 180 | return result; 181 | } 182 | //FIXME: Smarter function for this.. 183 | FString leadingZeros = ""; 184 | for(size_t i=0;i RenderRequest, 194 | FString ImageName, 195 | int32 Width, 196 | int32 Height, 197 | ERGBFormat ActualRGBFormat, 198 | EImageFormat ActualImageFormat) 199 | { 200 | UE_LOG(LogTemp, Warning, TEXT("Running Async Task")); 201 | (new FAutoDeleteAsyncTask(RenderRequest, ImageName, Width, Height, ActualRGBFormat, ActualImageFormat))->StartBackgroundTask(); 202 | } 203 | 204 | 205 | 206 | /* 207 | ******************************************************************* 208 | */ 209 | 210 | AsyncSaveImageToDiskTask::AsyncSaveImageToDiskTask( 211 | TSharedPtr InRenderRequest, 212 | FString ImageName, 213 | int32 Width, 214 | int32 Height, 215 | ERGBFormat RGBFormat, 216 | EImageFormat ImageFormat 217 | ): 218 | RenderRequest(InRenderRequest), 219 | FileName(ImageName), 220 | Width(Width), 221 | Height(Height), 222 | RGBFormat(RGBFormat), 223 | ImageFormat(ImageFormat) 224 | { 225 | } 226 | 227 | AsyncSaveImageToDiskTask::~AsyncSaveImageToDiskTask(){ 228 | UE_LOG(LogTemp, Warning, TEXT("AsyncTaskDone")); 229 | } 230 | 231 | 232 | void AsyncSaveImageToDiskTask::DoWork(){ 233 | UE_LOG(LogTemp, Warning, TEXT("Starting Work")); 234 | 235 | // Load the image wrapper module (if not already loaded) 236 | IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); 237 | 238 | // Create an image wrapper 239 | TSharedPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper(ImageFormat); 240 | 241 | // Error handling 242 | if (!ImageWrapper.IsValid()) { 243 | UE_LOG(LogTemp, Error, TEXT("Failed to create IImageWrapper!")); 244 | return; 245 | } 246 | 247 | // Set the raw data 248 | int32 PixelDepth = 8; 249 | if(ImageFormat == EImageFormat::EXR){ // Adjust pixel depth for EXR (float) data // Has to be EImageFormat because already converted.. 250 | PixelDepth = 16; 251 | } 252 | UE_LOG(LogTemp, Warning, TEXT("PixelDepth: %d"), PixelDepth); 253 | ImageWrapper->SetRaw(RenderRequest->RawData, RenderRequest->RawSize, Width, Height, RGBFormat, PixelDepth); 254 | 255 | // Compress the image 256 | /** from /Engine/Source/Runtime/ImageWrapper/Public/IImageWrapper.h 257 | * Enumerates available image compression qualities. 258 | * 259 | * JPEG interprets Quality as 1-100 260 | * JPEG default quality is 85 , Uncompressed means 100 261 | * 262 | * for PNG: 263 | * Negative qualities in [-1,-9] set PNG zlib level 264 | * PNG interprets "Uncompressed" as zlib level 0 (none) 265 | * otherwise default zlib level 3 is used. 266 | * 267 | * EXR respects the "Uncompressed" flag to turn off compression; otherwise ZIP_COMPRESSION is used. 268 | */ 269 | const TArray64& CompressedData = ImageWrapper->GetCompressed(100); 270 | 271 | // Save the compressed image to disk 272 | FFileHelper::SaveArrayToFile(CompressedData, *FileName); 273 | 274 | //unlock the readback after the processing is done 275 | RenderRequest->Readback.Unlock(); 276 | 277 | // Indicate that the processing is complete (using a flag) 278 | RenderRequest->bIsComplete = true; 279 | 280 | //UE_LOG(LogTemp, Log, TEXT("Stored Image: %s"), *FileName); 281 | } -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDisk/Public/FrameCaptureManager.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | class ASceneCapture2D; 6 | class UMaterial; 7 | 8 | #include "CoreMinimal.h" 9 | #include "IImageWrapper.h" // Include the header that defines ERGBFormat 10 | #include "GameFramework/Actor.h" 11 | #include "FrameCaptureManager.generated.h" 12 | 13 | 14 | UENUM(BlueprintType) 15 | enum class ECustomImageFormat : uint8 16 | { 17 | //Invalid UMETA(DisplayName = "Invalid"), 18 | PNG UMETA(DisplayName = "PNG"), 19 | JPEG UMETA(DisplayName = "JPEG"), 20 | //GrayscaleJPEG UMETA(DisplayName = "GrayscaleJPEG"), 21 | //BMP UMETA(DisplayName = "BMP"), 22 | //ICO UMETA(DisplayName = "ICO"), 23 | EXR UMETA(DisplayName = "EXR"), 24 | //ICNS UMETA(DisplayName = "ICNS"), 25 | //TGA UMETA(DisplayName = "TGA"), 26 | //HDR UMETA(DisplayName = "HDR"), 27 | //TIFF UMETA(DisplayName = "TIFF"), 28 | //DDS UMETA(DisplayName = "DDS"), 29 | //UEJPEG UMETA(DisplayName = "UEJPEG"), 30 | //GrayscaleUEJPEG UMETA(DisplayName = "GrayscaleUEJPEG") 31 | }; 32 | 33 | struct FRenderRequestStruct{ 34 | FIntPoint ImageSize; 35 | FRHIGPUTextureReadback Readback; 36 | FRenderCommandFence RenderFence; 37 | 38 | void* RawData = nullptr; 39 | int64 RawSize = 0; 40 | 41 | bool bIsComplete = false; 42 | 43 | FRenderRequestStruct( 44 | const FIntPoint& ImageSize, 45 | const FRHIGPUTextureReadback& Readback) : 46 | ImageSize(ImageSize), 47 | Readback(Readback) {} 48 | }; 49 | 50 | 51 | UCLASS() 52 | class CAPTURETODISK_API AFrameCaptureManager : public AActor 53 | { 54 | GENERATED_BODY() 55 | 56 | public: 57 | // Sets default values for this actor's properties 58 | AFrameCaptureManager(); 59 | 60 | // Captured Data Sub-Directory Name 61 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Capture File") 62 | FString SubDirectoryName = "/Color/"; 63 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Capture File") 64 | int NumDigits = 6; 65 | 66 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Capture Format") 67 | ECustomImageFormat ImageFormat = ECustomImageFormat::PNG; 68 | 69 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Capture Format") 70 | int FrameWidth = 640; 71 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Capture Format") 72 | int FrameHeight = 480; 73 | 74 | // Color Capture Components 75 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Capture Source") 76 | ASceneCapture2D* CaptureComponent; 77 | 78 | // PostProcessMaterial used for segmentation 79 | UPROPERTY(EditAnywhere, Category="Capture Source") 80 | UMaterial* PostProcessMaterial = nullptr; 81 | 82 | protected: 83 | // Called when the game starts or when spawned 84 | virtual void BeginPlay() override; 85 | 86 | // RenderRequest Queue 87 | TQueue> RenderRequestQueue; 88 | TQueue> InThreadRenderRequestQueue; // holding the RenderRequests currently storing to disk 89 | 90 | // Counter for file names 91 | int ImgCounter = 0; 92 | 93 | // Prepares SceneCapture2D component for capturing 94 | void SetupCaptureComponent(); 95 | 96 | // Creates an async task that will save the captured image to disk 97 | void RunAsyncImageSaveTask( 98 | TSharedPtr RenderRequest, 99 | FString ImageName, 100 | int32 Width, 101 | int32 Height, 102 | ERGBFormat ActualRGBFormat, 103 | EImageFormat ActualImageFormat 104 | ); 105 | 106 | FString ToStringWithLeadingZeros(int32 Integer, int32 MaxDigits); 107 | 108 | FString GetFileEnding(ECustomImageFormat Format) 109 | { 110 | switch (Format) 111 | { 112 | case ECustomImageFormat::PNG: return ".png"; 113 | case ECustomImageFormat::JPEG: return ".jpeg"; 114 | case ECustomImageFormat::EXR: return ".exr"; 115 | default: return ""; 116 | } 117 | }; 118 | ERGBFormat GetRGBFormatFromImageFormat(ECustomImageFormat MyFormat) 119 | { 120 | { 121 | switch (MyFormat) 122 | { 123 | case ECustomImageFormat::PNG: return ERGBFormat::RGBA; 124 | case ECustomImageFormat::JPEG: return ERGBFormat::RGBA; 125 | case ECustomImageFormat::EXR: return ERGBFormat::RGBAF; 126 | default: return ERGBFormat::RGBA; 127 | } 128 | } 129 | }; 130 | EImageFormat ConvertImageFormat(ECustomImageFormat MyFormat) 131 | { 132 | switch (MyFormat) 133 | { 134 | case ECustomImageFormat::PNG: return EImageFormat::PNG; 135 | case ECustomImageFormat::JPEG: return EImageFormat::JPEG; 136 | //case ECustomImageFormat::GrayscaleJPEG: return EImageFormat::GrayscaleJPEG; 137 | //case ECustomImageFormat::BMP: return EImageFormat::BMP; 138 | //case ECustomImageFormat::ICO: return EImageFormat::ICO; 139 | case ECustomImageFormat::EXR: return EImageFormat::EXR; 140 | //case ECustomImageFormat::ICNS: return EImageFormat::ICNS; 141 | //case ECustomImageFormat::TGA: return EImageFormat::TGA; 142 | //case ECustomImageFormat::HDR: return EImageFormat::HDR; 143 | //case ECustomImageFormat::TIFF: return EImageFormat::TIFF; 144 | //case ECustomImageFormat::DDS: return EImageFormat::DDS; 145 | //case ECustomImageFormat::UEJPEG: return EImageFormat::UEJPEG; 146 | //case ECustomImageFormat::GrayscaleUEJPEG: return EImageFormat::GrayscaleUEJPEG; 147 | default: return EImageFormat::PNG; 148 | } 149 | }; 150 | 151 | public: 152 | // Called every frame 153 | virtual void Tick(float DeltaTime) override; 154 | 155 | UFUNCTION(BlueprintCallable, Category = "ImageCapture") 156 | void CaptureNonBlocking(); 157 | }; 158 | 159 | 160 | class AsyncSaveImageToDiskTask : public FNonAbandonableTask 161 | { 162 | public: 163 | AsyncSaveImageToDiskTask( 164 | TSharedPtr InRenderRequest, 165 | FString ImageName, 166 | int32 Width, 167 | int32 Height, 168 | ERGBFormat RGBFormat, 169 | EImageFormat ImageFormat); 170 | ~AsyncSaveImageToDiskTask(); 171 | 172 | void DoWork(); 173 | 174 | FORCEINLINE TStatId GetStatId() const 175 | { 176 | RETURN_QUICK_DECLARE_CYCLE_STAT(AsyncSaveImageToDiskTask, STATGROUP_ThreadPoolAsyncTasks); 177 | } 178 | 179 | private: 180 | TSharedPtr RenderRequest; // Hold the shared pointer 181 | FString FileName; 182 | int32 Width; 183 | int32 Height; 184 | ERGBFormat RGBFormat; 185 | EImageFormat ImageFormat; 186 | }; 187 | -------------------------------------------------------------------------------- /CaptureToDisk/Source/CaptureToDiskEditor.Target.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | using System.Collections.Generic; 5 | 6 | public class CaptureToDiskEditorTarget : TargetRules 7 | { 8 | public CaptureToDiskEditorTarget( TargetInfo Target) : base(Target) 9 | { 10 | Type = TargetType.Editor; 11 | DefaultBuildSettings = BuildSettingsVersion.V5; 12 | IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_5; 13 | ExtraModuleNames.Add("CaptureToDisk"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Timm Hess 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Image Capturing With UnrealEngine 5.5 (for deep learning) 2 | 3 | 4 | | Color | Segmentation | 5 | |---------|---------| 6 | | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/CaptureResult_color.jpeg) | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/CaptureResult_segmentation.png) | 7 | 8 | # Changelog 9 | - Updated to UE5.5.4 10 | - Streamlined capture code to not block the *render thread*, and further reduce the load on the *main game thread*. 11 | - Fixed alpha channel for pixel annotation in the post-process material. Images are no longer appearing as "empty" because of their alpha mask being 0. 12 | - Removed plugins from repository - reduce the mess of files 13 | - Slightly updated the tutorial text 14 | 15 | # Outline 16 | - [Introduction](#a-small-introduction) 17 | - [MAIN: How to Save Images to Disk (without blocking the main threads)](#how-to-save-images-to-disk-in-ue5-without-blocking-the-rendering-or-main-thread) 18 | - [Capturing Object Pixel-Annotations (segmentation mask)](#capturing-annotations) 19 | - [Capturing Scene Depth](#setup-a-depth-capture) 20 | - [Enable Lumen on SceneCapture2D](#enable-lumen-on-scenecapture2d) 21 | - [Known Issues](#known-issues) 22 | 23 | # TLDR 24 | Use these links to the [FrameCaptureManger.h](https://github.com/TimmHess/UnrealImageCapture/blob/master/CaptureToDisk/Source/CaptureToDisk/Public/FrameCaptureManager.h) and [FrameCaptureManger.cpp](https://github.com/TimmHess/UnrealImageCapture/blob/master/CaptureToDisk/Source/CaptureToDisk/Private/FrameCaptureManager.cpp) file. They are the only source needed. 25 | Plus, make sure to link the correct unreal-libs to your project - check the [prerequisite](#prerequisite) or [CaptureToDisk.Build.cs](https://github.com/TimmHess/UnrealImageCapture/blob/master/CaptureToDisk/Source/CaptureToDisk/CaptureToDisk.Build.cs). 26 | 27 | **Kudos to the UE4 and UE5 community!** 28 | 29 | **Special thanks to [Panakotta00](https://github.com/Panakotta00), for pointing to an even better GPU readback!** 30 | 31 | Using the source as is you should get an `AFrameCaptureManager` (`Actor`) in your scene with the following settings. 32 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/FrameCaptureManage_settings.png) 33 | 34 | Maybe most important is the `ImageFormat` setting. You want to use `PNG` for lossless compression when storing [object annotation](#capturing-object-segmentation-masks). For color rendered images most likely `JPEG` gives you better file-sizes. `EXR` is the float format - the code automatically stores .exr images when used. 35 | 36 | # Known Issues 37 | Capturing per-pixel annotations is done using the `CustomDepth` feature. The `CustomDepthStencil` is of type `uint8` which allows a range of 0-255, i.e. **we can handle at most 256** different annotations per image! 38 | 39 | # 40 | # A Small Introduction 41 | In this repository I condense my findings on how to implement a component to capture images to disk from an arbitrary UE5 (former UE4) scene **from scratch** lowering the bar for UE novices (and potentially bypassing the need for large frameworks that don't fit ones own particular needs). This will include: 42 | 1. Capturing rendered images to disk at high FPS, without blocking the UE rendering thread or the main game thread 43 | 2. Rendering pixel annotations (or other graphics buffers, such as depth) at the same time 44 | 45 | UnrealEngine (UE) is a powerful tool to create virtual worlds capable of AAA productions. Generating temporally consistent data with automatic pixel-wise annotations from complex scenes, such as traffic scenarios, is a capability worth leveraging. Especially for training and validation of machine learning- or deep learning applications it has been explored in a variety of projects. Already, there are plugins available that allow rendering images from UE to disk at runtime, such as prominently [Carla](https://carla-ue5.readthedocs.io/en/latest/), [UnrealCV](https://unrealcv.org/), or [AirSim](https://github.com/microsoft/AirSim). This repository aims to be a tutorial that demonstrates such an 'image capturing' mechanism in detail for you to understand its inner workings, and in turn enable you to reuse it in a custom fashions that suit the needs of your project. 46 | 47 | When I was setting up scenes for my research the plugins mentioned above were just not yet supporting the latest engine versions that I wanted/needed to use. Also, I was missing a place where the knowledge of how to render images to disk was explains for non-advanced graphics-programmers. Of course, there are lots of sources for code available online and also there are community blog-entries scattered across multiple platforms explaining parts of the problem and possible solutions, even though they typically are targeting very particular scenarios. 48 | 49 | **Disclaimer: I do not claim to own any of the code. Merely, I condensed the sources already available online for easier use and provide an overview to the general functionality of this particular approach!** 50 | 51 | 52 | 53 | # 54 | # How to Save Images to Disk In UE5 (without blocking the rendering or main thread) 55 | I will go through the main components of the code step-by-step so that hopefully it will be easier to implement each step as you are following along. However, I recommend looking at the source that is merely a single class ([here]()). 56 | 57 | *In the explanations I skip certain quality-of-life-like aspects for sakes of readability, for example exposing image resolution settings to the editor instead of hardcoding them. Make sure to check out the sources linked in [TLDR](#tldr)* 58 | 59 | 60 | # 61 | ## Prerequisite 62 | You will need a UE5 C++ project. 63 | 64 | Also, you will have to add a few packages to your `'YourProjectName'.Build.cs` file. These are part of UnrealEngine, however, sometimes they are not added automatically resulting in unpleasant (linker) errors. Find the `'YourProjectName'.Build.cs` file in the `Source/'YourProjectName/` directory, and add or extend it to include the modules: `"ImageWrapper", "RenderCore", "Renderer", "RHI"`, for example like this: 65 | 66 | ``` cpp 67 | PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "EnhancedInput" , "ImageWrapper", "RenderCore", "Renderer", "RHI"}); 68 | ``` 69 | 70 | # 71 | ## Setup A FrameCapture Component 72 | I am using ```SceneCaptureComponent2D``` as the basis for capturing images. Placing one of these into your scene will give you an ```ASceneCaptureComponent``` which is its `Actor` instance. It basically behaves like any other camera component, but its viewport is not restricted by your computer's monitor or main camera viewport. This provides us the possibility to render images of arbitrary resolution independent from the actual screen resolution. 73 | 74 | > Add a ```FrameCaptureManager``` class of type `Actor` to your project. 75 | 76 | All functionality to request the capturing of a frame, as well as receiving the rendered image back, and storing the frame to disk will be handled by the `FrameCaptureManager`. 77 | 78 | In the ```FrameCaptureManager.h``` we add the following:\ 79 | **FrameCaptureManager.h** 80 | ``` cpp 81 | #pragma once 82 | class ASceneCapture2D; // forward declaration 83 | 84 | #include ... // the stuff that is already there 85 | ``` 86 | and to our public variables: 87 | ``` cpp 88 | // Color Capture Components 89 | UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="Capture") 90 | ASceneCapture2D* CaptureComponent; 91 | ``` 92 | 93 | This enables you to assign a ```CaptureComponent2d``` to your ```FrameCaptureManager``` inside the UE5 Editor. 94 | 95 | > Compile and place a ```FrameCaptureManager``` in your scene. 96 | 97 | As it does not have any primitive to render you will only see it in the editor's outline. In the details panel of the placed ```FrameCaptureManager``` you can now see the ```CaptureComponent``` assigned to ```None```. From the drop down menu select the ```CaptureComponent2D``` you already placed in the scene. 98 | 99 | Back to code: We will now prepare our yet "naked" `CaptureComponent2D` class for capturing images. This includes creating and assigning a `RenderTarget` - which is basically a `Texture` to store our image data to - and setting the camera properties. 100 | 101 | *Note: You could also do this in the Editor but if you deal with, i.e. multiple capture components, you may find it handy not to worry about creating and assigning all the components by hand!* 102 | 103 | > Create a setup function to put all your setup code for the CaptureComponents in the CaptureManger: 104 | 105 | **FrameCaptureManager.h** 106 | ``` cpp 107 | protected: 108 | 109 | void SetupCaptureComponent(); 110 | ``` 111 | **FrameCaptureManager.cpp** 112 | ``` cpp 113 | #include ... 114 | 115 | // A bunch of includes we need 116 | #include "Engine/SceneCapture2D.h" 117 | #include "Components/SceneCaptureComponent2D.h" 118 | #include "ShowFlags.h" 119 | 120 | #include "Engine/TextureRenderTarget2D.h" 121 | #include "Materials/Material.h" 122 | #include "RHICommandList.h" 123 | #include "IImageWrapper.h" 124 | #include "IImageWrapperModule.h" 125 | #include "ImageUtils.h" 126 | 127 | #include "Modules/ModuleManager.h" 128 | #include "Misc/FileHelper.h" 129 | 130 | 131 | void AFrameCaptureManager::SetupCaptureComponent(){ 132 | if(!IsValid(CaptureComponent)){ 133 | UE_LOG(LogTemp, Error, TEXT("SetupCaptureComponent: CaptureComponent is not valid!")); 134 | return; 135 | } 136 | 137 | // Create RenderTargets 138 | UTextureRenderTarget2D* renderTarget2D = NewObject(); 139 | renderTarget2D->InitAutoFormat(256, 256); // some random format, got crashing otherwise 140 | 141 | renderTarget2D->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8_SRGB; //ETextureRenderTargetFormat::RTF_RGBA8; //8-bit color format 142 | renderTarget2D->InitCustomFormat(FrameWidth, FrameHeight, PF_R8G8B8A8, true); // PF... disables HDR, which is most important since HDR gives gigantic overhead, and is not needed! 143 | renderTarget2D->bForceLinearGamma = true; // Important for viewport-like color reproduction. 144 | 145 | renderTarget2D->bGPUSharedFlag = true; // demand buffer on GPU 146 | 147 | // Assign RenderTarget 148 | CaptureComponent->GetCaptureComponent2D()->TextureTarget = renderTarget2D; 149 | // Set Camera Properties 150 | CaptureComponent->GetCaptureComponent2D()->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR; 151 | CaptureComponent->GetCaptureComponent2D()->TextureTarget->TargetGamma = GEngine->GetDisplayGamma(); 152 | CaptureComponent->GetCaptureComponent2D()->ShowFlags.SetTemporalAA(true); 153 | // lookup additional showflags in documentation 154 | } 155 | ``` 156 | > Call the code during `BeginPlay` of the `FrameCaptureManager` 157 | 158 | **FrameCaptureManager.cpp** 159 | ``` cpp 160 | // Called when the game starts or when spawned 161 | void AFrameCaptureManager::BeginPlay() 162 | { 163 | Super::BeginPlay(); 164 | 165 | // Setup CaptureComponent 166 | if(CaptureComponent){ // nullptr check 167 | SetupCaptureComponent(); 168 | } else{ 169 | UE_LOG(LogTemp, Error, TEXT("No CaptureComponent set!")); 170 | } 171 | } 172 | 173 | ``` 174 | Now that because we have a `RenderTarget` applied to our `CaptureComponent` we can read its data and store it to disk. 175 | 176 | 177 | # 178 | ## Organize RenderRequests 179 | We do this by basically re-implementing UE's code for taking screenshots. Importantly, with the addition of not flushing our rendering pipeline. This prevents rendering *hiccups* that drop the framerate to 3 - 5 FPS. 180 | 181 | This addition will come at the price of needing to handle 'waiting times' before an image is done and copied from GPU. This is important to prevent reading old or uninitialized buffers (remember that `RenderThread` and `GameThread` are asynchronous). We do this by keeping a queue of ```RenderRequest``` that we can probe for being completed. 182 | 183 | > We add the following ```struct``` to our `FrameCaptureManager.h` above the `UCLASS()` definition: 184 | 185 | **FrameCaptureManager.h** 186 | ``` cpp 187 | #include ... 188 | 189 | [...] 190 | 191 | struct FRenderRequestStruct{ 192 | FIntPoint ImageSize; 193 | FRHIGPUTextureReadback Readback; 194 | FRenderCommandFence RenderFence; 195 | 196 | void* RawData = nullptr; 197 | int64 RawSize = 0; 198 | 199 | bool bIsComplete = false; 200 | 201 | FRenderRequestStruct( 202 | const FIntPoint& ImageSize, 203 | const FRHIGPUTextureReadback& Readback) : 204 | ImageSize(ImageSize), 205 | Readback(Readback) {} 206 | }; 207 | 208 | [...] 209 | UCLASS() 210 | class ... 211 | [...] 212 | ``` 213 | The ```FRHIGPUTextureReadback``` will hold the rendered results, e.g. color or depth values. The ```RenderFence``` is a neat feature of UE, letting you put a 'fence' into the render pipeline that can be checked to notify when it has passed the full rendering-pipeline. This gives a way to determine whether our render request is done and the buffers are safe to read. 214 | 215 | > We need to add a ```TQueue``` as a data structure to keep track of our render requests: 216 | 217 | **CaptureManger.h** 218 | ``` cpp 219 | protected: 220 | // RenderRequest Queue 221 | TQueue> RenderRequestQueue; 222 | TQueue> InThreadRenderRequestQueue; 223 | ``` 224 | 225 | # 226 | ## Implement placing render requests: 227 | This function will place a render request on the UE rendering pipeline asking the data captured from our `CaptureComponent` to be copied in our image buffer so that we can further process it. 228 | 229 | **CaptureManger.h** 230 | ``` cpp 231 | public: 232 | UFUNCTION(BlueprintCallable, Category = "ImageCapture") 233 | void CaptureNonBlocking(); 234 | ``` 235 | 236 | **CaptureManger.cpp** 237 | ``` cpp 238 | void AFrameCaptureManager::CaptureNonBlocking(){ 239 | if(!IsValid(CaptureComponent)){ 240 | UE_LOG(LogTemp, Error, TEXT("CaptureColorNonBlocking: CaptureComponent was not valid!")); 241 | return; 242 | } 243 | CaptureComponent->GetCaptureComponent2D()->TextureTarget->TargetGamma = GEngine->GetDisplayGamma(); 244 | 245 | // Get RenderConterxt 246 | FTextureRenderTargetResource* renderTargetResource = CaptureComponent->GetCaptureComponent2D()->TextureTarget->GameThread_GetRenderTargetResource(); 247 | 248 | TSharedPtr renderRequest = 249 | MakeShared( 250 | renderTargetResource->GetSizeXY(), 251 | FRHIGPUTextureReadback(TEXT("CameraCaptureManagerReadback") 252 | ) 253 | ); 254 | 255 | ENQUEUE_RENDER_COMMAND(SceneDrawCompletion)( 256 | [renderRequest, renderTargetResource](FRHICommandListImmediate& RHICmdList) { 257 | FTextureRHIRef Target = renderTargetResource->GetRenderTargetTexture(); 258 | renderRequest->Readback.EnqueueCopy(RHICmdList, Target); 259 | }); 260 | 261 | // Notifiy new task in RenderQueue 262 | RenderRequestQueue.Enqueue(renderRequest); 263 | 264 | // Set RenderCommandFence 265 | renderRequest->RenderFence.BeginFence(); 266 | } 267 | ``` 268 | With this, the image data will be stored in our queue of requests. Now we can think of storing it to disk. 269 | 270 | *Note: UFUNCTION(BlueprintCallable, Category = "ImageCapture") exposes this function to blueprint, so that you can easily test it* 271 | 272 | 273 | # 274 | ## Save Image Data to Disk 275 | In each tick of the `FrameCaptureManager` we look up the first element of the `RenderQueue`. If it's `RenderFence` is completed and the data is ready to read, we proceed with saving the image to disk. 276 | 277 | We need a procedure to write the data to disk, preferably without blocking our `GameThread`. 278 | We implement an [asynchronous](https://wiki.unrealengine.com/Using_AsyncTasks) procedure storing the data to disk. 279 | 280 | **FrameCaptureManager.h** 281 | ``` cpp 282 | UCLASS() 283 | class ... { 284 | [...] 285 | }; 286 | 287 | // Below the AFrameCaptureManager class definition 288 | 289 | class AsyncSaveImageToDiskTask : public FNonAbandonableTask 290 | { 291 | public: 292 | AsyncSaveImageToDiskTask( 293 | TSharedPtr InRenderRequest, 294 | FString ImageName, 295 | int32 Width, 296 | int32 Height, 297 | ERGBFormat RGBFormat, 298 | EImageFormat ImageFormat); 299 | ~AsyncSaveImageToDiskTask(); 300 | 301 | void DoWork(); 302 | 303 | FORCEINLINE TStatId GetStatId() const 304 | { 305 | RETURN_QUICK_DECLARE_CYCLE_STAT(AsyncSaveImageToDiskTask, STATGROUP_ThreadPoolAsyncTasks); 306 | } 307 | 308 | private: 309 | TSharedPtr RenderRequest; // Hold the shared pointer 310 | FString FileName; 311 | int32 Width; 312 | int32 Height; 313 | ERGBFormat RGBFormat; 314 | EImageFormat ImageFormat; 315 | }; 316 | ``` 317 | 318 | **FrameCaptureManager.cpp** 319 | ``` cpp 320 | #include ... 321 | 322 | AsyncSaveImageToDiskTask::AsyncSaveImageToDiskTask( 323 | TSharedPtr InRenderRequest, 324 | FString ImageName, 325 | int32 Width, 326 | int32 Height, 327 | ERGBFormat RGBFormat, 328 | EImageFormat ImageFormat 329 | ): 330 | RenderRequest(InRenderRequest), 331 | FileName(ImageName), 332 | Width(Width), 333 | Height(Height), 334 | RGBFormat(RGBFormat), 335 | ImageFormat(ImageFormat) 336 | { 337 | } 338 | 339 | AsyncSaveImageToDiskTask::~AsyncSaveImageToDiskTask(){ 340 | UE_LOG(LogTemp, Warning, TEXT("AsyncTaskDone")); 341 | } 342 | 343 | 344 | void AsyncSaveImageToDiskTask::DoWork(){ 345 | UE_LOG(LogTemp, Warning, TEXT("Starting Work")); 346 | 347 | // Load the image wrapper module (if not already loaded) 348 | IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); 349 | 350 | // Create an image wrapper 351 | TSharedPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper(ImageFormat); 352 | 353 | // Error handling 354 | if (!ImageWrapper.IsValid()) { 355 | UE_LOG(LogTemp, Error, TEXT("Failed to create IImageWrapper!")); 356 | return; 357 | } 358 | 359 | // Set the raw data 360 | int32 PixelDepth = 8; 361 | ImageWrapper->SetRaw(RenderRequest->RawData, RenderRequest->RawSize, Width, Height, RGBFormat, PixelDepth); 362 | 363 | // Compress the image 364 | const TArray64& CompressedData = ImageWrapper->GetCompressed(100); 365 | 366 | // Save the compressed image to disk 367 | FFileHelper::SaveArrayToFile(CompressedData, *FileName); 368 | 369 | //unlock the readback after the processing is done 370 | RenderRequest->Readback.Unlock(); 371 | 372 | // Indicate that the processing is complete (using a flag) 373 | RenderRequest->bIsComplete = true; 374 | } 375 | ``` 376 | We offload the entire image processing into `DoWork()`, from applying compression encoding to finally storing it to disk, to the asynchronous thread. 377 | 378 | *Note that this requires our ``RenderRequest`, more precisely its `RawData` image-buffer to stay available while the data is being stored to disk. Otherwise we encounter segmentation faults and the engine will crash!* 379 | 380 | 381 | # 382 | ## Override the `Tick` function of the `FrameCaptureManager`: 383 | Finally, we put everything together in the `Tick` function of our `FrameCaptureManager`. We wait for a `RenderRequest` to become available, we hand it over to an asynchronous thread for being stored to disk, and we monitor the progress of the asynchronous thread to finally release the `RenderRequest`'s buffers to garbage collection. 384 | 385 | 386 | **FrameCaptureManager.h** 387 | ``` cpp 388 | public: 389 | 390 | // Called every frame 391 | virtual void Tick(float DeltaTime) override; 392 | ``` 393 | **FrameCaptureManager.cpp** 394 | ``` cpp 395 | // Called every frame 396 | void AFrameCaptureManager::Tick(float DeltaTime) 397 | { 398 | Super::Tick(DeltaTime); 399 | 400 | if(!RenderRequestQueue.IsEmpty()){ 401 | // Peek the next RenderRequest from queue 402 | TSharedPtr nextRenderRequest = *RenderRequestQueue.Peek(); 403 | 404 | if(nextRenderRequest){ //nullptr check 405 | if(nextRenderRequest->RenderFence.IsFenceComplete() && nextRenderRequest->Readback.IsReady()){ 406 | // Load the image wrapper module 407 | IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); 408 | 409 | // Get Data from Readback 410 | nextRenderRequest->RawSize = nextRenderRequest->ImageSize.X * nextRenderRequest->ImageSize.Y * sizeof(FColor); 411 | 412 | int32 RowPitchInPixels; 413 | nextRenderRequest->RawData = nextRenderRequest->Readback.Lock(RowPitchInPixels, nullptr); // Pass RowPitchInPixels and no buffer size 414 | 415 | // Generate image name 416 | FString fileName = ""; 417 | fileName = FPaths::ProjectSavedDir() + SubDirectoryName + "/img" + "_" + ToStringWithLeadingZeros(ImgCounter, NumDigits); 418 | fileName += GetFileEnding(ImageFormat); 419 | 420 | // Pass the raw data, filename, width, height, and format to the async task 421 | RunAsyncImageSaveTask(nextRenderRequest, fileName, 1920, 1080, ERGBFormat::RGBA, EImageFormat::PNG); 422 | 423 | // Security check 424 | if(VerboseLogging && !fileName.IsEmpty()){ 425 | UE_LOG(LogTemp, Warning, TEXT("%s"), *fileName); 426 | } 427 | 428 | // Increase ImgCounter for file names 429 | ImgCounter += 1; 430 | 431 | // Release RenderRequest form the queue 432 | RenderRequestQueue.Pop(); // Delete the first element from RenderQueue 433 | // Put it into the queue of outsourced threads 434 | InThreadRenderRequestQueue.Enqueue(nextRenderRequest); //Push to InThreadRenderRequestQueue 435 | } 436 | } 437 | } 438 | if(!InThreadRenderRequestQueue.IsEmpty()){ 439 | UE_LOG(LogTemp, Log, TEXT("InThreadRenderRequestQueue not empty.")); 440 | 441 | // Get next element of the InThreadRenderRequestQueue 442 | TSharedPtr nextRenderRequest = *InThreadRenderRequestQueue.Peek(); 443 | 444 | if(nextRenderRequest){ //nullptr check 445 | if(nextRenderRequest->bIsComplete){ //check if complete 446 | InThreadRenderRequestQueue.Pop(); // Remove from queue 447 | } 448 | } 449 | } 450 | } 451 | ``` 452 | 453 | For testing purposes we can call the `CaptureColorBlocking()` from the `LevelBlueprint` by attaching it to a button pressed event. 454 | 455 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/Debug_level_blueprint_capture.png) 456 | 457 | The captured images will now be saved into your project's `Saved` directory. 458 | 459 | 460 | # 461 | # Capturing Annotations 462 | To be able to render color and segmentation at the same time, we need one additional `SceneCapture2D` component in our scene for each *type* of capture you want. Attach it to the `SceneCapture2D` already placed in your scene (from earlier in this tutorial). Make sure both have exactly the same location, rotation, and perspective settings. Otherwise your annotation will not match. 463 | 464 | # Capturing Object Segmentation Masks 465 | To get labels for our images we will add a second `CaptureComponent` equipped with a `PostProcessMaterial` that visualizes `CustomDepth`. The `CustomDepthStencil` is settable for each actor in the scene, effectively letting us label and visualize categories of, as well as individual, actors. 466 | 467 | ## 1. Enable and Set-Up Custom Depth Stencils 468 | Find the **ProjectSettings** in your editor and search for *stencil* which will bring up `Custom Depth-Stencil Pass`. Switch this option from `Enabled` to `Enabled with Stencil`. 469 | 470 | You can set the custom depth in editor or from code. For simplicity I chose the editor. Place an arbitrary object(MeshActor) into the scene, and search for `custom depth` in its details panel. Under `Rendering` enable `Render CustomDepth Pass`, and set `CustomDepth Stencil Value` to whatever you like. For illustration purposes set it to 200. 471 | 472 | *Note: Make sure you have custom depth enabled with stencils in your project settings.* 473 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/Enable_custom_depth_in_project.png) 474 | 475 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/Apply_custom_depth_stencil.png) 476 | 477 | 478 | ## 2. Setting Up The PostProcess Material 479 | Add a new `Material` to your project content. (I will call it `PP_Segmentation`) 480 | 481 | Click on the material's output node and switch `MaterialDomain` from `Surface` to `PostProcess`. 482 | 483 | In the same panel search for "alpha" and activate `Output Alpha`. Set this value to `1.0` in the Material node. 484 | 485 | Right-click to open the node search and type `SceneTexture`, select the node from `Texture`-Category. 486 | 487 | In the details of this node, select `CustomStencil` as `SceneTextureId`. 488 | 489 | Add a `Division` node and connect the `SceneTexture`'s `Color` output to the division node. Set the division to be by 255. 490 | 491 | *Note: This is needed because the image buffer seems to be float valued, leading to values > 1 having no meaning, as image information ranges from 0.0 to 1.0.* 492 | 493 | Apply and save the material. 494 | 495 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/pp_segmentation.png) 496 | 497 | 498 | **FrameCaptureManager.h** 499 | ``` cpp 500 | public: 501 | // PostProcessMaterial used for segmentation 502 | UPROPERTY(EditAnywhere, Category="Segmentation Setup") 503 | UMaterial* PostProcessMaterial = nullptr; 504 | ``` 505 | **FrameCaptureManager.cpp** 506 | ``` cpp 507 | void AFrameCaptureManager::SetupCaptureComponent(){ 508 | [...] // previous function code 509 | 510 | // Assign PostProcess Material if assigned 511 | if(PostProcessMaterial){ // check nullptr 512 | CaptureComponent->GetCaptureComponent2D()->AddOrUpdateBlendable(PostProcessMaterial); 513 | } else { 514 | UE_LOG(LogTemp, Log, TEXT("No PostProcessMaterial is assigend")); 515 | } 516 | } 517 | ``` 518 | 519 | You can now reference the ``PostProcessMaterial` in the details panel of the `FrameCaptureManager` in the editor just like before the `SceneCapture2D`. 520 | 521 | 522 | # 523 | # Setup a Depth Capture 524 | Capturing `SceneDepth` information has one important caveat - it requires storing float images (.exr). 525 | Every thing else follows like [Object Segmentation](#capturing-object-segmentation-masks). We create a `PostProcessMaterial` to access the respectie GPU buffer. 526 | 527 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/pp_depth.png) 528 | 529 | Luckily UnrealEngine is perfectly capable of storing float images and even provides the .exr file format in its `IImageWrapper`. However, there are three places in our code where we need to take care to handle the float format correctly. Missing any of those will result in segmentation faults of the engine. 530 | 531 | ## 1. Accounting for the RawSize correctly 532 | We adjust the code that allocates the RawSize of our captured images to accomodate the float values using 16 bits. 533 | 534 | *Note that I decide for the RawSize to used based on the ImageFormat. This I do not explain in the tutorial - please check the source.* 535 | 536 | **FrameCaptureManager.cpp** 537 | ``` cpp 538 | void AFrameCaptureManager::SetupCaptureComponent(){ 539 | [...] 540 | 541 | // Get Data from Readback 542 | nextRenderRequest->RawSize = nextRenderRequest->ImageSize.X * nextRenderRequest->ImageSize.Y * sizeof(FColor); 543 | if(ImageFormat == ECustomImageFormat::EXR){ // handle float case 544 | nextRenderRequest->RawSize = nextRenderRequest->ImageSize.X * nextRenderRequest->ImageSize.Y * sizeof(FFloat16Color); //FLOAT 545 | } 546 | 547 | [...] 548 | } 549 | ``` 550 | 551 | ## 2. Adjust the correct RenderTarget format 552 | **FrameCaptureManager.cpp** 553 | ``` cpp 554 | void AFrameCaptureManager::SetupCaptureComponent(){ 555 | [...] 556 | 557 | // Create RenderTargets 558 | UTextureRenderTarget2D* renderTarget2D = NewObject(); 559 | renderTarget2D->InitAutoFormat(256, 256); // some random format, got crashing otherwise 560 | 561 | // Float Capture 562 | if(ImageFormat == ECustomImageFormat::EXR){ // handle float case 563 | renderTarget2D->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA32f; 564 | renderTarget2D->InitCustomFormat(FrameWidth, FrameHeight, PF_FloatRGBA, true); // PF_B8G8R8A8 disables HDR which will boost storing to disk due to less image information 565 | } 566 | // Color Capture 567 | else{ 568 | renderTarget2D->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8_SRGB; //ETextureRenderTargetFormat::RTF_RGBA8; //8-bit color format 569 | renderTarget2D->InitCustomFormat(FrameWidth, FrameHeight, PF_R8G8B8A8, true); // PF_R8G8B8A8 //PF_B8G8R8A8 // PF... disables HDR, which is most important since HDR gives gigantic overhead, and is not needed! 570 | renderTarget2D->bForceLinearGamma = true; // Important for viewport-like color reproduction. 571 | } 572 | renderTarget2D->bGPUSharedFlag = true; // demand buffer on GPU 573 | 574 | [...] 575 | } 576 | ``` 577 | 578 | ## 3. Adjust the PixelDepth when storing 579 | **FrameCaptureManager.cpp** 580 | ``` cpp 581 | void AsyncSaveImageToDiskTask::DoWork(){ 582 | [...] 583 | 584 | // Set the raw data 585 | int32 PixelDepth = 8; 586 | if(ImageFormat == EImageFormat::EXR){ // Adjust pixel depth for EXR (float) data // Has to be EImageFormat because already converted.. 587 | PixelDepth = 16; 588 | } 589 | UE_LOG(LogTemp, Warning, TEXT("PixelDepth: %d"), PixelDepth); 590 | ImageWrapper->SetRaw(RenderRequest->RawData, RenderRequest->RawSize, Width, Height, RGBFormat, PixelDepth); 591 | 592 | [...] 593 | } 594 | ``` 595 | 596 | 597 | # 598 | # Enable Lumen on SceneCapture2D 599 | In my tests using UE5.5.3+ the `SceneCapture2D` was fully capable of rendering scenes with Lumen. However you need might need to actiate it in the `SceneCapture2D` itself as it was not listening to the `PostProcessVolume` in my scene. 600 | 601 | ![](https://github.com/TimmHess/UnrealImageCapture/blob/master/gfx/Activate_Lumen.png) 602 | 603 | 604 | -------------------------------------------------------------------------------- /gfx/Activate_Lumen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/Activate_Lumen.png -------------------------------------------------------------------------------- /gfx/Apply_custom_depth_stencil.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/Apply_custom_depth_stencil.png -------------------------------------------------------------------------------- /gfx/CaptureResult_color.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/CaptureResult_color.jpeg -------------------------------------------------------------------------------- /gfx/CaptureResult_segmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/CaptureResult_segmentation.png -------------------------------------------------------------------------------- /gfx/Debug_level_blueprint_capture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/Debug_level_blueprint_capture.png -------------------------------------------------------------------------------- /gfx/Enable_custom_depth_in_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/Enable_custom_depth_in_project.png -------------------------------------------------------------------------------- /gfx/FrameCaptureManage_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/FrameCaptureManage_settings.png -------------------------------------------------------------------------------- /gfx/pp_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/pp_depth.png -------------------------------------------------------------------------------- /gfx/pp_segmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimmHess/UnrealImageCapture/20447f1453e0915ba1d391722fd00bd9eb648ead/gfx/pp_segmentation.png --------------------------------------------------------------------------------