├── .gitignore ├── Config ├── DefaultEditor.ini ├── DefaultEngine.ini ├── DefaultGame.ini └── DefaultInput.ini ├── Content ├── Maps │ └── RtspDisplayMap.umap ├── Materials │ ├── M_Display.uasset │ └── M_Display_Inst.uasset ├── RtspDisplay.uasset └── UI │ └── W_RtspDisplay.uasset ├── LICENSE ├── README.md ├── RtspDisplay.uproject └── Source ├── RtspDisplay.Target.cs ├── RtspDisplay ├── MyRunnable.cpp ├── MyRunnable.h ├── RtspClientComponent.cpp ├── RtspClientComponent.h ├── RtspDisplay.Build.cs ├── RtspDisplay.cpp ├── RtspDisplay.h ├── RtspDisplayGameModeBase.cpp ├── RtspDisplayGameModeBase.h ├── jpeg_frame.hpp ├── jpeg_header.hpp ├── rtp_jpeg_packet.hpp ├── rtp_packet.cpp └── rtp_packet.hpp └── RtspDisplayEditor.Target.cs /.gitignore: -------------------------------------------------------------------------------- 1 | # Visual Studio 2015 user specific files 2 | .vs/ 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | 22 | # Compiled Static libraries 23 | *.lai 24 | *.la 25 | *.a 26 | *.lib 27 | 28 | # Executables 29 | *.exe 30 | *.out 31 | *.app 32 | *.ipa 33 | 34 | # These project files can be generated by the engine 35 | *.xcodeproj 36 | *.xcworkspace 37 | *.sln 38 | *.suo 39 | *.opensdf 40 | *.sdf 41 | *.VC.db 42 | *.VC.opendb 43 | 44 | # Precompiled Assets 45 | SourceArt/**/*.png 46 | SourceArt/**/*.tga 47 | 48 | # Binary Files 49 | Binaries/* 50 | Plugins/*/Binaries/* 51 | 52 | # Builds 53 | Build/* 54 | 55 | # Whitelist PakBlacklist-.txt files 56 | !Build/*/ 57 | Build/*/** 58 | !Build/*/PakBlacklist*.txt 59 | 60 | # Don't ignore icon files in Build 61 | !Build/**/*.ico 62 | 63 | # Built data for maps 64 | *_BuiltData.uasset 65 | 66 | # Configuration files generated by the Editor 67 | Saved/* 68 | 69 | # Compiled source files for the engine to use 70 | Intermediate/* 71 | Plugins/*/Intermediate/* 72 | 73 | # Cache files for the editor to use 74 | DerivedDataCache/* 75 | .DS_Store 76 | packaged/ 77 | -------------------------------------------------------------------------------- /Config/DefaultEditor.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/finger563/unreal-rtsp-display/e723afdf0f4846c20463efca50ed6ba016b4e45a/Config/DefaultEditor.ini -------------------------------------------------------------------------------- /Config/DefaultEngine.ini: -------------------------------------------------------------------------------- 1 | 2 | 3 | [/Script/EngineSettings.GameMapsSettings] 4 | GameDefaultMap=/Game/Maps/RtspDisplayMap.RtspDisplayMap 5 | EditorStartupMap=/Game/Maps/RtspDisplayMap.RtspDisplayMap 6 | 7 | [/Script/WindowsTargetPlatform.WindowsTargetSettings] 8 | DefaultGraphicsRHI=DefaultGraphicsRHI_DX12 9 | -D3D12TargetedShaderFormats=PCD3D_SM5 10 | +D3D12TargetedShaderFormats=PCD3D_SM6 11 | -D3D11TargetedShaderFormats=PCD3D_SM5 12 | +D3D11TargetedShaderFormats=PCD3D_SM5 13 | Compiler=Default 14 | AudioSampleRate=48000 15 | AudioCallbackBufferFrameSize=1024 16 | AudioNumBuffersToEnqueue=1 17 | AudioMaxChannels=0 18 | AudioNumSourceWorkers=4 19 | SpatializationPlugin= 20 | SourceDataOverridePlugin= 21 | ReverbPlugin= 22 | OcclusionPlugin= 23 | CompressionOverrides=(bOverrideCompressionTimes=False,DurationThreshold=5.000000,MaxNumRandomBranches=0,SoundCueQualityIndex=0) 24 | CacheSizeKB=65536 25 | MaxChunkSizeOverrideKB=0 26 | bResampleForDevice=False 27 | MaxSampleRate=48000.000000 28 | HighSampleRate=32000.000000 29 | MedSampleRate=24000.000000 30 | LowSampleRate=12000.000000 31 | MinSampleRate=8000.000000 32 | CompressionQualityModifier=1.000000 33 | AutoStreamingThreshold=0.000000 34 | SoundCueCookQualityIndex=-1 35 | 36 | [/Script/HardwareTargeting.HardwareTargetingSettings] 37 | TargetedHardwareClass=Desktop 38 | AppliedTargetedHardwareClass=Desktop 39 | DefaultGraphicsPerformance=Maximum 40 | AppliedDefaultGraphicsPerformance=Maximum 41 | 42 | [/Script/Engine.RendererSettings] 43 | r.GenerateMeshDistanceFields=True 44 | r.DynamicGlobalIlluminationMethod=1 45 | r.ReflectionMethod=1 46 | r.Shadow.Virtual.Enable=1 47 | r.DefaultFeature.AutoExposure.ExtendDefaultLuminanceRange=True 48 | 49 | [/Script/WorldPartitionEditor.WorldPartitionEditorSettings] 50 | CommandletClass=Class'/Script/UnrealEd.WorldPartitionConvertCommandlet' 51 | 52 | [/Script/Engine.UserInterfaceSettings] 53 | bAuthorizeAutomaticWidgetVariableCreation=False 54 | 55 | [/Script/Engine.Engine] 56 | +ActiveGameNameRedirects=(OldGameName="TP_Blank",NewGameName="/Script/RtspDisplay") 57 | +ActiveGameNameRedirects=(OldGameName="/Script/TP_Blank",NewGameName="/Script/RtspDisplay") 58 | +ActiveClassRedirects=(OldClassName="TP_BlankGameModeBase",NewClassName="RtspDisplayGameModeBase") 59 | 60 | [/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings] 61 | bEnablePlugin=True 62 | bAllowNetworkConnection=True 63 | SecurityToken=923282B71F46FB99CAE943978824D512 64 | bIncludeInShipping=False 65 | bAllowExternalStartInShipping=False 66 | bCompileAFSProject=False 67 | bUseCompression=False 68 | bLogFiles=False 69 | bReportStats=False 70 | ConnectionType=USBOnly 71 | bUseManualIPAddress=False 72 | ManualIPAddress= 73 | 74 | [/Script/AndroidRuntimeSettings.AndroidRuntimeSettings] 75 | PackageName=com.finger563.[PROJECT] 76 | 77 | -------------------------------------------------------------------------------- /Config/DefaultGame.ini: -------------------------------------------------------------------------------- 1 | 2 | [/Script/EngineSettings.GeneralProjectSettings] 3 | ProjectID=F9FF1A47064501E7F3CBAA83D2572898 4 | -------------------------------------------------------------------------------- /Config/DefaultInput.ini: -------------------------------------------------------------------------------- 1 | [/Script/Engine.InputSettings] 2 | -AxisConfig=(AxisKeyName="Gamepad_LeftX",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 3 | -AxisConfig=(AxisKeyName="Gamepad_LeftY",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 4 | -AxisConfig=(AxisKeyName="Gamepad_RightX",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 5 | -AxisConfig=(AxisKeyName="Gamepad_RightY",AxisProperties=(DeadZone=0.25,Exponent=1.f,Sensitivity=1.f)) 6 | -AxisConfig=(AxisKeyName="MouseX",AxisProperties=(DeadZone=0.f,Exponent=1.f,Sensitivity=0.07f)) 7 | -AxisConfig=(AxisKeyName="MouseY",AxisProperties=(DeadZone=0.f,Exponent=1.f,Sensitivity=0.07f)) 8 | -AxisConfig=(AxisKeyName="Mouse2D",AxisProperties=(DeadZone=0.f,Exponent=1.f,Sensitivity=0.07f)) 9 | +AxisConfig=(AxisKeyName="Gamepad_LeftX",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 10 | +AxisConfig=(AxisKeyName="Gamepad_LeftY",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 11 | +AxisConfig=(AxisKeyName="Gamepad_RightX",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 12 | +AxisConfig=(AxisKeyName="Gamepad_RightY",AxisProperties=(DeadZone=0.250000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 13 | +AxisConfig=(AxisKeyName="MouseX",AxisProperties=(DeadZone=0.000000,Sensitivity=0.070000,Exponent=1.000000,bInvert=False)) 14 | +AxisConfig=(AxisKeyName="MouseY",AxisProperties=(DeadZone=0.000000,Sensitivity=0.070000,Exponent=1.000000,bInvert=False)) 15 | +AxisConfig=(AxisKeyName="Mouse2D",AxisProperties=(DeadZone=0.000000,Sensitivity=0.070000,Exponent=1.000000,bInvert=False)) 16 | +AxisConfig=(AxisKeyName="MouseWheelAxis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 17 | +AxisConfig=(AxisKeyName="Gamepad_LeftTriggerAxis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 18 | +AxisConfig=(AxisKeyName="Gamepad_RightTriggerAxis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 19 | +AxisConfig=(AxisKeyName="Gamepad_Special_Left_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 20 | +AxisConfig=(AxisKeyName="Gamepad_Special_Left_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 21 | +AxisConfig=(AxisKeyName="Vive_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 22 | +AxisConfig=(AxisKeyName="Vive_Left_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 23 | +AxisConfig=(AxisKeyName="Vive_Left_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 24 | +AxisConfig=(AxisKeyName="Vive_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 25 | +AxisConfig=(AxisKeyName="Vive_Right_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 26 | +AxisConfig=(AxisKeyName="Vive_Right_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 27 | +AxisConfig=(AxisKeyName="MixedReality_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 28 | +AxisConfig=(AxisKeyName="MixedReality_Left_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 29 | +AxisConfig=(AxisKeyName="MixedReality_Left_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 30 | +AxisConfig=(AxisKeyName="MixedReality_Left_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 31 | +AxisConfig=(AxisKeyName="MixedReality_Left_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 32 | +AxisConfig=(AxisKeyName="MixedReality_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 33 | +AxisConfig=(AxisKeyName="MixedReality_Right_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 34 | +AxisConfig=(AxisKeyName="MixedReality_Right_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 35 | +AxisConfig=(AxisKeyName="MixedReality_Right_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 36 | +AxisConfig=(AxisKeyName="MixedReality_Right_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 37 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 38 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 39 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 40 | +AxisConfig=(AxisKeyName="OculusTouch_Left_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 41 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 42 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 43 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 44 | +AxisConfig=(AxisKeyName="OculusTouch_Right_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 45 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 46 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Grip_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 47 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 48 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 49 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 50 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 51 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 52 | +AxisConfig=(AxisKeyName="ValveIndex_Left_Trackpad_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 53 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Grip_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 54 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Grip_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 55 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trigger_Axis",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 56 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Thumbstick_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 57 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Thumbstick_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 58 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trackpad_X",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 59 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trackpad_Y",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 60 | +AxisConfig=(AxisKeyName="ValveIndex_Right_Trackpad_Force",AxisProperties=(DeadZone=0.000000,Sensitivity=1.000000,Exponent=1.000000,bInvert=False)) 61 | bAltEnterTogglesFullscreen=True 62 | bF11TogglesFullscreen=True 63 | bUseMouseForTouch=False 64 | bEnableMouseSmoothing=True 65 | bEnableFOVScaling=True 66 | bCaptureMouseOnLaunch=True 67 | bEnableLegacyInputScales=True 68 | bEnableMotionControls=True 69 | bFilterInputByPlatformUser=False 70 | bShouldFlushPressedKeysOnViewportFocusLost=True 71 | bAlwaysShowTouchInterface=False 72 | bShowConsoleOnFourFingerTap=True 73 | bEnableGestureRecognizer=False 74 | bUseAutocorrect=False 75 | DefaultViewportMouseCaptureMode=CapturePermanently_IncludingInitialMouseDown 76 | DefaultViewportMouseLockMode=LockOnCapture 77 | FOVScale=0.011110 78 | DoubleClickTime=0.200000 79 | DefaultPlayerInputClass=/Script/EnhancedInput.EnhancedPlayerInput 80 | DefaultInputComponentClass=/Script/EnhancedInput.EnhancedInputComponent 81 | DefaultTouchInterface=/Engine/MobileResources/HUD/DefaultVirtualJoysticks.DefaultVirtualJoysticks 82 | -ConsoleKeys=Tilde 83 | +ConsoleKeys=Tilde 84 | +ConsoleKeys=Caret 85 | 86 | -------------------------------------------------------------------------------- /Content/Maps/RtspDisplayMap.umap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/finger563/unreal-rtsp-display/e723afdf0f4846c20463efca50ed6ba016b4e45a/Content/Maps/RtspDisplayMap.umap -------------------------------------------------------------------------------- /Content/Materials/M_Display.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/finger563/unreal-rtsp-display/e723afdf0f4846c20463efca50ed6ba016b4e45a/Content/Materials/M_Display.uasset -------------------------------------------------------------------------------- /Content/Materials/M_Display_Inst.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/finger563/unreal-rtsp-display/e723afdf0f4846c20463efca50ed6ba016b4e45a/Content/Materials/M_Display_Inst.uasset -------------------------------------------------------------------------------- /Content/RtspDisplay.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/finger563/unreal-rtsp-display/e723afdf0f4846c20463efca50ed6ba016b4e45a/Content/RtspDisplay.uasset -------------------------------------------------------------------------------- /Content/UI/W_RtspDisplay.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/finger563/unreal-rtsp-display/e723afdf0f4846c20463efca50ed6ba016b4e45a/Content/UI/W_RtspDisplay.uasset -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 William Emfinger 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # unreal-rtsp-display 2 | Example code for receiving video streams over RTSP using Unreal Engine FSockets 3 | and displaying them in real time. 4 | 5 | https://github.com/finger563/unreal-rtsp-display/assets/213467/64270d95-f223-43ef-885b-599e607d48b1 6 | 7 | When running this example, you will need to configure the address, port, and 8 | path of your RTSP server on the actor: 9 | 10 | ![CleanShot 2023-07-18 at 13 42 36](https://github.com/finger563/unreal-rtsp-display/assets/213467/8884b601-5fa0-4b29-89db-1c271c8055cc) 11 | 12 | Note: this example currently only supports MJPEG streams over RTSP, which are 13 | parsed with the RtpJpegPacket class into JpegFrames. This example uses Unreal 14 | Engine's built-in image decoding to decode the jpeg frames into uncompressed 15 | image data for use in a UTexture2D. 16 | 17 | This example contains a few components: 18 | 19 | 1. The `RtspClientComponent` class: This component can be added to an actor and 20 | exposes some functions for connecting to an RTSP server and configuring / 21 | controlling the stream. Inside its TickComponent function, it waits for new 22 | images (decompressed) to be available and if so, creates a new transient 23 | UTexture2D and writes the decompressed data to the new texture. It then 24 | broadcasts this new texture using the multicast delegate to any registered 25 | listeners. 26 | 2. The `RtpPacket`, `RtpJpegPacket`, `JpegHeader`, and `JpegFrame` classes which 27 | handle the parsing of the media data (as RTP over UDP from the server to the 28 | client) and reassembling of multiple networks packets into a single jpeg 29 | frame. 30 | 3. The `MyRunnable` class: used by the RtspClientComponent when it connects to a 31 | server it spawns two runnables (Unreal Engine threads) for receiving data 32 | from the server on the RTP/UDP socket and the RTCP/UDP socket. The RTP 33 | runnable runs a bound function from the RtspClientComponent class which 34 | receives the raw data, parses it into jpeg frames, decompresses the jpeg 35 | frames, and then updates some mutex-protected data to inform the game thread 36 | (RtspClientComponent::TickComponent) that new data is available. This class 37 | is also used to allow the FSocket::Connect (TCP connection from RTSP Client 38 | to RTSP Server) to run without blocking the main / game thread. 39 | 4. `M_Display` and `M_Display_Inst`: these assets in the Content/Materials 40 | directory are simple materials which render a texture parameter with optional 41 | configuration for the UV mapping of the texture. This material instance is 42 | the base for the dynamic material instance that is created at runtime in the 43 | RtspDisplay blueprint. 44 | 5. `RtspDisplay`: This blueprint actor contains an RtspClientComponent and a 45 | Plane static mesh component. On BeginPlay it creates a dynamic material 46 | instance of the M_Display_Inst which it stores a reference to so that it can 47 | dynamically update the texture parameter that the material is rendering. It 48 | sets this material on the plane static mesh component. It binds an event to 49 | the OnFrameReceived event from the RtspClientComponent and when it receives a 50 | message from that event, it sets the new texture to be the dynamic material 51 | instance's texture parameter. 52 | 6. `W_RtspDisplay`: This user widget contains the UI (2D) for interacting with a 53 | RtspClientComponent. It is configured by the `RtspDisplayMap`'s level 54 | blueprint to be added as the UI to the viewport for the first player 55 | controller and to control the RtspClientComponent of the RtspDisplay actor in 56 | the level. It provides the UI (display image, URL textbox, and buttons for 57 | connect, disconnect, play, and pause) for the RtspClientComponent. 58 | 59 | Image of the running example in the editor: 60 | 61 | Disconnected (With text box to write URI and connect button): 62 | ![CleanShot 2023-07-18 at 13 40 12](https://github.com/finger563/unreal-rtsp-display/assets/213467/88722e5d-f8fa-4852-b55b-3ba9be8da057) 63 | Connected: 64 | ![CleanShot 2023-07-18 at 13 40 27](https://github.com/finger563/unreal-rtsp-display/assets/213467/9271463d-55eb-47bc-aedc-0aea512df317) 65 | Playing: 66 | ![CleanShot 2023-07-18 at 13 40 42](https://github.com/finger563/unreal-rtsp-display/assets/213467/885ee177-535e-4da9-a843-aa2342e79ee0) 67 | 68 | ### Details 69 | 70 | #### RtspDisplay Actor Blueprint 71 | 72 | ![CleanShot 2023-07-18 at 13 44 33](https://github.com/finger563/unreal-rtsp-display/assets/213467/6d7109b6-fd43-46af-b526-889ab9237294) 73 | 74 | #### M_Display Material 75 | 76 | CleanShot 2023-07-08 at 10 51 50@2x 77 | 78 | #### RtspDisplay User Widget 79 | 80 | ![CleanShot 2023-07-18 at 13 45 26](https://github.com/finger563/unreal-rtsp-display/assets/213467/ecab159c-0201-4ee0-8fdd-90ee3e997023) 81 | Note the image has been set with angle 180 and X scale of -1 so that it matches the camera image that I'm currently sending. May need to be changed for other streams depending on camera orientation. 82 | 83 | Its blueprint: 84 | ![CleanShot 2023-07-18 at 13 48 23](https://github.com/finger563/unreal-rtsp-display/assets/213467/bbea4667-841b-4004-8afa-b12e4b667da2) 85 | 86 | #### Level Blueprint 87 | 88 | ![CleanShot 2023-07-18 at 13 48 56](https://github.com/finger563/unreal-rtsp-display/assets/213467/c97d9954-a887-4773-8a3b-54104b102e31) 89 | 90 | 91 | ### Setup for Android App 92 | 93 | Follow the setup instructions 94 | [here](https://docs.unrealengine.com/5.2/en-US/how-to-set-up-android-sdk-and-ndk-for-your-unreal-engine-development-environment/). 95 | 96 | Note: you will likely have to modify the `/Users/Shared/Epic\ 97 | Games/UE_5.2/Engine/Extras/Android/SetupAndroid.command` file - possibly to 98 | point to the right `JAVA_HOME` directory. In my case I had to modify the 99 | JAVA_HOME export in the `SetupAndroid.command` file to point to 100 | `/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home` and had 101 | to install jdk8 specifically. 102 | 103 | You will need to set the environment variables (under `Android SDK`) 104 | appropriately, e.g.: 105 | 106 | - `Android SDK` : `/Users/bob/Library/Android/sdk` 107 | - `Android NDK` : `/Users/bob/Library/Android/sdk/ndk/25.1.8937393` 108 | - `Location of JAVA` : `/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home` 109 | - `SDK API Level` : `android-32` 110 | - `NDK API Level` : `android-32` 111 | 112 | For that version of java (jdk 8) which is required to successfully build for 113 | android, you can (on macos) install it via: 114 | 115 | ``` sh 116 | brew install --cask adoptopenjdk8 117 | ``` 118 | 119 | -------------------------------------------------------------------------------- /RtspDisplay.uproject: -------------------------------------------------------------------------------- 1 | { 2 | "FileVersion": 3, 3 | "EngineAssociation": "5.2", 4 | "Category": "", 5 | "Description": "", 6 | "Modules": [ 7 | { 8 | "Name": "RtspDisplay", 9 | "Type": "Runtime", 10 | "LoadingPhase": "Default" 11 | } 12 | ], 13 | "Plugins": [ 14 | { 15 | "Name": "ModelingToolsEditorMode", 16 | "Enabled": true, 17 | "TargetAllowList": [ 18 | "Editor" 19 | ] 20 | }, 21 | { 22 | "Name": "Fab", 23 | "Enabled": false, 24 | "SupportedTargetPlatforms": [ 25 | "Win64", 26 | "Mac", 27 | "Linux" 28 | ] 29 | }, 30 | { 31 | "Name": "EditorTelemetry", 32 | "Enabled": false 33 | }, 34 | { 35 | "Name": "EditorPerformance", 36 | "Enabled": false 37 | }, 38 | { 39 | "Name": "StudioTelemetry", 40 | "Enabled": false 41 | } 42 | ] 43 | } -------------------------------------------------------------------------------- /Source/RtspDisplay.Target.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | using System.Collections.Generic; 5 | 6 | public class RtspDisplayTarget : TargetRules 7 | { 8 | public RtspDisplayTarget( TargetInfo Target) : base(Target) 9 | { 10 | Type = TargetType.Game; 11 | DefaultBuildSettings = BuildSettingsVersion.V5; 12 | // CppStandard = CppStandardVersion.Cpp20; 13 | // IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_5; 14 | 15 | ExtraModuleNames.AddRange( new string[] { "RtspDisplay" } ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Source/RtspDisplay/MyRunnable.cpp: -------------------------------------------------------------------------------- 1 | #include "MyRunnable.h" 2 | 3 | #pragma region Main Thread Code 4 | // This code will be run on the thread that invoked this thread (i.e. game thread) 5 | 6 | FMyRunnable::FMyRunnable(FMyRunnable::callback_t callback) 7 | : bRunThread(true) 8 | , Callback(callback) 9 | { 10 | // Link to the thread that created this object 11 | Thread = FRunnableThread::Create(this, TEXT("FMyRunnable")); 12 | } 13 | 14 | FMyRunnable::~FMyRunnable() 15 | { 16 | if (Thread != NULL) 17 | { 18 | // blocking call until thread has completed 19 | Thread->Kill(); 20 | delete Thread; 21 | } 22 | } 23 | 24 | void FMyRunnable::Stop() 25 | { 26 | bRunThread = false; 27 | } 28 | 29 | #pragma endregion 30 | // the code below will run on the new thread 31 | 32 | bool FMyRunnable::Init() 33 | { 34 | // This code will not run until the thread has been created 35 | // This code will run on the thread that created this object 36 | 37 | // This is where you can do any thread specific initialization that needs to be done. 38 | // This is not the place to start your thread - that should be done in the constructor or Start() method. 39 | // Return true if initialization was successful, false otherwise 40 | return true; 41 | } 42 | 43 | uint32 FMyRunnable::Run() 44 | { 45 | // This code will not run until the thread has been created 46 | // This code will run on the thread that created this object 47 | 48 | // This is the loop that will run on the new thread 49 | while (bRunThread) 50 | { 51 | // do some work 52 | if (Callback == nullptr) 53 | break; 54 | 55 | bool ShouldStop = Callback(); 56 | if (ShouldStop) 57 | break; 58 | } 59 | 60 | return 0; 61 | } 62 | 63 | void FMyRunnable::Exit() 64 | { 65 | // This code will not run until the thread has been created 66 | // This code will run on the thread that created this object 67 | 68 | // This is where you can do any thread specific cleanup that needs to be done. 69 | // This is not the place to kill your thread - that should be done in the destructor or Stop() method. 70 | } 71 | -------------------------------------------------------------------------------- /Source/RtspDisplay/MyRunnable.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "CoreMinimal.h" 6 | #include "HAL/Runnable.h" 7 | 8 | class FMyRunnable : public FRunnable 9 | { 10 | public: 11 | typedef std::function callback_t; 12 | 13 | FMyRunnable(callback_t callback); 14 | 15 | virtual ~FMyRunnable() override; 16 | 17 | virtual bool Init() override; 18 | virtual uint32 Run() override; 19 | virtual void Exit() override; 20 | virtual void Stop() override; 21 | 22 | private: 23 | bool bRunThread = true; 24 | callback_t Callback = nullptr; 25 | FRunnableThread *Thread = nullptr; 26 | }; 27 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspClientComponent.cpp: -------------------------------------------------------------------------------- 1 | #include "RtspClientComponent.h" 2 | 3 | #include "Async/Async.h" 4 | #include "Common/TcpSocketBuilder.h" 5 | #include "Common/UdpSocketBuilder.h" 6 | #include "GenericPlatform/GenericPlatformHttp.h" 7 | #include "IImageWrapper.h" 8 | #include "IImageWrapperModule.h" 9 | #include "Interfaces/IPv4/IPv4Address.h" 10 | #include "Sockets.h" 11 | #include "SocketSubsystem.h" 12 | #include "SocketTypes.h" 13 | 14 | #include "MyRunnable.h" 15 | 16 | #include "jpeg_frame.hpp" 17 | 18 | URtspClientComponent::URtspClientComponent() { 19 | PrimaryComponentTick.bCanEverTick = true; 20 | } 21 | 22 | void URtspClientComponent::BeginPlay() { 23 | Super::BeginPlay(); 24 | } 25 | 26 | void URtspClientComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) { 27 | disconnect(); 28 | Super::EndPlay(EndPlayReason); 29 | } 30 | 31 | void URtspClientComponent::TickComponent(float DeltaTime, enum ELevelTick TickType, 32 | FActorComponentTickFunction *ThisTickFunction) { 33 | Super::TickComponent(DeltaTime, TickType, ThisTickFunction); 34 | // if there's not a new frame, return 35 | if (!image_data_ready_) { 36 | return; 37 | } 38 | // if we've got a new frame, let's get it 39 | std::vector data; 40 | size_t width = 0; 41 | size_t height = 0; 42 | { 43 | // copy the latest frame, which was set by the worker thread 44 | std::unique_lock lock(image_mutex_); 45 | data.assign(image_data_.begin(), image_data_.end()); 46 | image_data_ready_ = false; 47 | width = image_width_; 48 | height = image_height_; 49 | } 50 | 51 | UE_LOG(LogTemp, Log, TEXT("URtspClientComponent::TickComponent: Got a new frame, size = %d"), data.size()); 52 | 53 | // now convert the jpeg frame into a texture and broadcast it 54 | // create a texture 55 | auto texture = UTexture2D::CreateTransient(width, height, PF_B8G8R8A8); 56 | // lock the texture 57 | uint8 *mip_data = static_cast(texture->GetPlatformData()->Mips[0].BulkData.Lock(LOCK_READ_WRITE)); 58 | // copy the jpeg data into the texture 59 | std::copy(data.data(), data.data() + data.size(), mip_data); 60 | // unlock the texture 61 | texture->GetPlatformData()->Mips[0].BulkData.Unlock(); 62 | // update the texture 63 | texture->UpdateResource(); 64 | // broadcast the texture 65 | OnFrameReceived.Broadcast(texture); 66 | } 67 | 68 | std::string URtspClientComponent::send_request(const std::string& method, const std::string& path, 69 | const std::unordered_map& extra_headers, 70 | std::error_code& ec) { 71 | // send the request 72 | std::string request = method + " " + path + " RTSP/1.0\r\n"; 73 | request += "CSeq: " + std::to_string(cseq_) + "\r\n"; 74 | if (session_id_.size() > 0) { 75 | request += "Session: " + session_id_ + "\r\n"; 76 | } 77 | for (auto &[key, value] : extra_headers) { 78 | request += key + ": " + value + "\r\n"; 79 | } 80 | request += "User-Agent: rtsp-client\r\n"; 81 | request += "Accept: application/sdp\r\n"; 82 | request += "\r\n"; 83 | std::string response; 84 | 85 | uint8_t buffer[1024]; 86 | int bytes_sent = 0; 87 | int bytes_received = 0; 88 | bool did_send = rtsp_socket_->Send((uint8_t *)request.c_str(), request.size(), bytes_sent); 89 | if (!did_send) { 90 | ec = std::make_error_code(std::errc::io_error); 91 | UE_LOG(LogTemp, Error, TEXT("Failed to send request")); 92 | return {}; 93 | } 94 | if (bytes_sent <= 0) { 95 | ec = std::make_error_code(std::errc::io_error); 96 | UE_LOG(LogTemp, Error, TEXT("Failed to send request, bytes_sent = %d"), bytes_sent); 97 | return {}; 98 | } 99 | rtsp_socket_->Recv(buffer, sizeof(buffer), bytes_received, ESocketReceiveFlags::None); 100 | if (bytes_received <= 0) { 101 | ec = std::make_error_code(std::errc::io_error); 102 | UE_LOG(LogTemp, Error, TEXT("Failed to receive response")); 103 | return {}; 104 | } 105 | response.assign(buffer, buffer + bytes_received); 106 | 107 | // TODO: how to keep receiving until we get the full response? 108 | // if (response.find("\r\n\r\n") != std::string::npos) { 109 | // break; 110 | // } 111 | 112 | // parse the response 113 | UE_LOG(LogTemp, Log, TEXT("Response:\n%s"), *FString(response.c_str())); 114 | if (!parse_response(response)) { 115 | ec = std::make_error_code(std::errc::io_error); 116 | UE_LOG(LogTemp, Error, TEXT("Failed to parse response")); 117 | return {}; 118 | } 119 | return response; 120 | } 121 | 122 | bool URtspClientComponent::connect() { 123 | return connect_to_address(Address, Port, Path); 124 | } 125 | 126 | bool URtspClientComponent::connect_to_uri(FString uri) { 127 | // parse the uri 128 | auto address = FGenericPlatformHttp::GetUrlDomain(uri); 129 | int port = 8554; 130 | auto maybe_port = FGenericPlatformHttp::GetUrlPort(uri); 131 | if (maybe_port.IsSet()) { 132 | port = maybe_port.GetValue(); 133 | } 134 | auto path = FGenericPlatformHttp::GetUrlPath(uri); 135 | return connect_to_address(address, port, path); 136 | } 137 | 138 | bool URtspClientComponent::connect_to_address(FString address, int port, FString path) { 139 | if (IsConnected) { 140 | UE_LOG(LogTemp, Warning, TEXT("Already connected, disconnecting first")); 141 | disconnect(); 142 | } 143 | 144 | UE_LOG(LogTemp, Log, TEXT("Connecting to RTSP server at %s:%d%s"), *address, port, *path); 145 | 146 | FString socket_name = FString::Printf(TEXT("RTSP Socket %s:%d"), *address, port); 147 | rtsp_socket_ = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(NAME_Stream, address, false); 148 | 149 | // save the rtsp address and port 150 | Address = address; 151 | Port = port; 152 | Path = path; 153 | path_ = TCHAR_TO_UTF8(*path); 154 | 155 | // make the server addr 156 | FIPv4Address ip; 157 | bool did_parse = FIPv4Address::Parse(address, ip); 158 | if (!did_parse) { 159 | UE_LOG(LogTemp, Error, TEXT("Failed to parse IP address")); 160 | return false; 161 | } 162 | 163 | rtsp_addr_ = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateInternetAddr(); 164 | bool valid = false; 165 | rtsp_addr_->SetIp(ip.Value); 166 | rtsp_addr_->SetPort(port); 167 | 168 | // make a thread to receive rtp packets using the rtp_socket 169 | connect_thread_ = new FMyRunnable(std::bind(&URtspClientComponent::connect_thread_func, this)); 170 | 171 | return true; 172 | } 173 | 174 | void URtspClientComponent::disconnect() { 175 | UE_LOG(LogTemp, Log, TEXT("Disconnecting from RTSP server")); 176 | // make sure we stop the connect thread if it's still runnning (and didn't 177 | // connect) 178 | if (connect_thread_) { 179 | connect_thread_->Stop(); 180 | delete connect_thread_; 181 | connect_thread_ = nullptr; 182 | } 183 | if (!IsConnected) { 184 | UE_LOG(LogTemp, Warning, TEXT("Not connected, nothing to disconnect")); 185 | return; 186 | } 187 | // try to send the teardown request, but don't care if it fails 188 | teardown(); 189 | IsConnected = false; 190 | IsPlaying = false; 191 | // stop the main socket 192 | UE_LOG(LogTemp, Log, TEXT("Stopping RTSP socket")); 193 | if (rtsp_socket_) { 194 | rtsp_socket_->Close(); 195 | delete rtsp_socket_; 196 | rtsp_socket_ = nullptr; 197 | } 198 | // stop the threads 199 | UE_LOG(LogTemp, Log, TEXT("Stopping RTP/RTCP threads")); 200 | if (rtp_thread_) { 201 | rtp_thread_->Stop(); 202 | delete rtp_thread_; 203 | } 204 | if (rtcp_thread_) { 205 | rtcp_thread_->Stop(); 206 | delete rtcp_thread_; 207 | } 208 | // stop the sockets 209 | UE_LOG(LogTemp, Log, TEXT("Stopping RTP/RTCP sockets")); 210 | if (rtp_socket_) { 211 | rtp_socket_->Close(); 212 | delete rtp_socket_; 213 | } 214 | if (rtcp_socket_) { 215 | rtcp_socket_->Close(); 216 | delete rtcp_socket_; 217 | } 218 | // Broadcast to the listeners 219 | OnDisconnected.Broadcast(); 220 | } 221 | 222 | bool URtspClientComponent::describe() { 223 | if (!IsConnected) { 224 | UE_LOG(LogTemp, Error, TEXT("Cannot describe: not connected")); 225 | return false; 226 | } 227 | std::error_code ec; 228 | // send the describe request 229 | auto response = send_request("DESCRIBE", path_, {}, ec); 230 | if (ec) { 231 | return false; 232 | } 233 | // sdp response is of the form: 234 | // std::regex sdp_regex("m=video (\\d+) RTP/AVP (\\d+)"); 235 | // parse the sdp response and get the video port without using regex 236 | // this is a very simple sdp parser that only works for this specific case 237 | auto sdp_start = response.find("m=video"); 238 | if (sdp_start == std::string::npos) { 239 | UE_LOG(LogTemp, Error, TEXT("Invalid sdp")); 240 | return false; 241 | } 242 | auto sdp_end = response.find("\r\n", sdp_start); 243 | if (sdp_end == std::string::npos) { 244 | UE_LOG(LogTemp, Error, TEXT("Incomplete sdp")); 245 | return false; 246 | } 247 | auto sdp = response.substr(sdp_start, sdp_end - sdp_start); 248 | auto port_start = sdp.find(" "); 249 | if (port_start == std::string::npos) { 250 | UE_LOG(LogTemp, Error, TEXT("Could not find port start")); 251 | return false; 252 | } 253 | auto port_end = sdp.find(" ", port_start + 1); 254 | if (port_end == std::string::npos) { 255 | UE_LOG(LogTemp, Error, TEXT("Could not find port end")); 256 | return false; 257 | } 258 | auto port = sdp.substr(port_start + 1, port_end - port_start - 1); 259 | video_port_ = std::stoi(port); 260 | UE_LOG(LogTemp, Log, TEXT("Video port: %d"), video_port_); 261 | auto payload_type_start = sdp.find(" ", port_end + 1); 262 | if (payload_type_start == std::string::npos) { 263 | UE_LOG(LogTemp, Error, TEXT("Could not find payload type start")); 264 | return false; 265 | } 266 | auto payload_type = sdp.substr(payload_type_start + 1, sdp.size() - payload_type_start - 1); 267 | video_payload_type_ = std::stoi(payload_type); 268 | UE_LOG(LogTemp, Log, TEXT("Video payload type: %d"), video_payload_type_); 269 | return true; 270 | } 271 | 272 | bool URtspClientComponent::setup(int rtp_port, int rtcp_port) { 273 | if (!IsConnected) { 274 | UE_LOG(LogTemp, Error, TEXT("Cannot setup: not connected")); 275 | return false; 276 | } 277 | UE_LOG(LogTemp, Log, TEXT("Setting up RTSP session on ports %d-%d"), rtp_port, rtcp_port); 278 | std::error_code ec; 279 | // send the setup request 280 | std::unordered_map extra_headers = { 281 | {"Transport", "RTP/AVP;unicast;client_port=" + std::to_string(rtp_port) + "-" + std::to_string(rtcp_port)}}; 282 | auto response = send_request("SETUP", path_, extra_headers, ec); 283 | if (ec) { 284 | UE_LOG(LogTemp, Error, TEXT("Failed to setup")); 285 | return false; 286 | } 287 | 288 | init_rtp(rtp_port); 289 | init_rtcp(rtcp_port); 290 | 291 | return true; 292 | } 293 | 294 | bool URtspClientComponent::play() { 295 | if (!IsConnected) { 296 | UE_LOG(LogTemp, Error, TEXT("Cannot play: not connected")); 297 | return false; 298 | } 299 | UE_LOG(LogTemp, Log, TEXT("Playing RTSP session")); 300 | std::error_code ec; 301 | // send the play request 302 | auto response = send_request("PLAY", path_, {}, ec); 303 | IsPlaying = ec ? false : true; 304 | if (IsPlaying) { 305 | OnPlay.Broadcast(); 306 | } 307 | return !ec; 308 | } 309 | 310 | bool URtspClientComponent::pause() { 311 | if (!IsConnected) { 312 | UE_LOG(LogTemp, Error, TEXT("Cannot pause: not connected")); 313 | return false; 314 | } 315 | UE_LOG(LogTemp, Log, TEXT("Pausing RTSP session")); 316 | std::error_code ec; 317 | // send the pause request 318 | auto response = send_request("PAUSE", path_, {}, ec); 319 | IsPlaying = ec ? true : false; 320 | if (!IsPlaying) { 321 | OnPause.Broadcast(); 322 | } 323 | return !ec; 324 | } 325 | 326 | bool URtspClientComponent::teardown() { 327 | if (!IsConnected) { 328 | UE_LOG(LogTemp, Error, TEXT("Cannot teardown: not connected")); 329 | return false; 330 | } 331 | UE_LOG(LogTemp, Log, TEXT("Tearing down RTSP session")); 332 | std::error_code ec; 333 | // send the teardown request 334 | auto response = send_request("TEARDOWN", path_, {}, ec); 335 | IsPlaying = false; 336 | return !ec; 337 | } 338 | 339 | bool URtspClientComponent::parse_response(const std::string &response) { 340 | if (response.empty()) { 341 | UE_LOG(LogTemp, Error, TEXT("Empty response")); 342 | return false; 343 | } 344 | 345 | auto response_start = response.find("RTSP/1.0 "); 346 | if (response_start == std::string::npos) { 347 | UE_LOG(LogTemp, Error, TEXT("Invalid response")); 348 | return false; 349 | } 350 | // parse the status code and message 351 | auto response_end = response.find("\r\n", response_start); 352 | if (response_end == std::string::npos) { 353 | UE_LOG(LogTemp, Error, TEXT("Incomplete response")); 354 | return false; 355 | } 356 | auto response_code = response.substr(response_start + 9, response_end - response_start - 9); 357 | if (response_code != "200 OK") { 358 | UE_LOG(LogTemp, Error, TEXT("Invalid response code: %s"), *FString(response_code.c_str())); 359 | return false; 360 | } 361 | // parse the session id if present 362 | auto session_start = response.find("Session: "); 363 | if (session_start != std::string::npos) { 364 | session_id_ = response.substr(session_start + 9, response.find("\r\n", session_start) - session_start - 9); 365 | } 366 | // increment the cseq 367 | cseq_++; 368 | return true; 369 | } 370 | 371 | void URtspClientComponent::init_rtp(size_t rtp_port) { 372 | if (rtp_socket_) { 373 | rtp_socket_->Close(); 374 | delete rtp_socket_; 375 | } 376 | FString socket_name = FString::Printf(TEXT("RTP Socket %d"), rtp_port); 377 | rtp_socket_ = FUdpSocketBuilder(*socket_name) 378 | .AsReusable() 379 | .BoundToPort(rtp_port) 380 | .WithReceiveBufferSize(6 * 1024) 381 | .WithSendBufferSize(6 * 1024) 382 | .Build(); 383 | UE_LOG(LogTemp, Log, TEXT("RTP port: %d"), rtp_port); 384 | // make a thread to receive rtp packets using the rtp_socket 385 | rtp_thread_ = new FMyRunnable(std::bind(&URtspClientComponent::rtp_thread_func, this)); 386 | } 387 | 388 | void URtspClientComponent::init_rtcp(size_t rtcp_port) { 389 | if (rtcp_socket_) { 390 | rtcp_socket_->Close(); 391 | delete rtcp_socket_; 392 | } 393 | FString socket_name = FString::Printf(TEXT("RTCP Socket %d"), rtcp_port); 394 | rtcp_socket_ = FUdpSocketBuilder(*socket_name) 395 | .AsReusable() 396 | .BoundToPort(rtcp_port) 397 | .WithReceiveBufferSize(6 * 1024) 398 | .WithSendBufferSize(6 * 1024) 399 | .Build(); 400 | UE_LOG(LogTemp, Log, TEXT("RTCP port: %d"), rtcp_port); 401 | // make a thread to receive rtcp packets using the rtcp_socket 402 | rtcp_thread_ = new FMyRunnable(std::bind(&URtspClientComponent::rtcp_thread_func, this)); 403 | } 404 | 405 | bool URtspClientComponent::connect_thread_func() { 406 | // now connect 407 | if (!rtsp_socket_->Connect(*rtsp_addr_)) { 408 | UE_LOG(LogTemp, Error, TEXT("Failed to connect to RTSP server")); 409 | // go ahead and stop the thread early 410 | return true; 411 | } 412 | 413 | auto conn_state = rtsp_socket_->GetConnectionState(); 414 | switch (conn_state) { 415 | case ESocketConnectionState::SCS_NotConnected: 416 | UE_LOG(LogTemp, Error, TEXT("RTSP socket not connected")); 417 | // go ahead and stop the thread early 418 | return true; 419 | case ESocketConnectionState::SCS_Connected: 420 | UE_LOG(LogTemp, Log, TEXT("RTSP socket connected")); 421 | break; 422 | case ESocketConnectionState::SCS_ConnectionError: 423 | UE_LOG(LogTemp, Error, TEXT("RTSP socket connection error")); 424 | // go ahead and stop the thread early 425 | return true; 426 | } 427 | 428 | // update the state 429 | IsConnected = true; 430 | 431 | // can only broadcast events from the game thread, so use async with lambda 432 | AsyncTask(ENamedThreads::GameThread, [this]() { 433 | // notify the listeners 434 | OnConnected.Broadcast(); 435 | }); 436 | 437 | // send the OPTIONS request 438 | std::error_code ec; 439 | send_request("OPTIONS", "*", {}, ec); 440 | if (ec) { 441 | UE_LOG(LogTemp, Error, TEXT("Failed to send OPTIONS request")); 442 | } 443 | 444 | // we're done, so stop the thread 445 | return true; 446 | } 447 | 448 | bool URtspClientComponent::rtp_thread_func() { 449 | // receive the rtp packet 450 | size_t max_packet_size = 65536; 451 | uint8_t *data = new uint8_t[max_packet_size]; 452 | int32 bytes_read = 0; 453 | rtp_socket_->Recv(data, max_packet_size, bytes_read, ESocketReceiveFlags::None); 454 | if (bytes_read > 0) { 455 | std::vector packet(data, data + bytes_read); 456 | handle_rtp_packet(packet); 457 | } 458 | delete[] data; 459 | 460 | // Sleep the thread for a bit 461 | FPlatformProcess::Sleep(0.005f); 462 | 463 | // don't want to stop the thread 464 | return false; 465 | } 466 | 467 | bool URtspClientComponent::rtcp_thread_func() { 468 | // receive the rtcp packet 469 | size_t max_packet_size = 65536; 470 | uint8_t *data = new uint8_t[max_packet_size]; 471 | int32 bytes_read = 0; 472 | rtcp_socket_->Recv(data, max_packet_size, bytes_read, ESocketReceiveFlags::None); 473 | if (bytes_read > 0) { 474 | std::vector packet(data, data + bytes_read); 475 | handle_rtcp_packet(packet); 476 | } 477 | delete[] data; 478 | 479 | // Sleep the thread for a bit 480 | FPlatformProcess::Sleep(0.005f); 481 | 482 | // don't want to stop the thread 483 | return false; 484 | } 485 | 486 | void URtspClientComponent::handle_rtp_packet(std::vector &data) { 487 | // parse the rtp packet 488 | // jpeg frame that we are building 489 | static std::unique_ptr jpeg_frame; 490 | 491 | UE_LOG(LogTemp, Log, TEXT("Got RTP packet of size: %d"), data.size()); 492 | 493 | std::string_view packet(reinterpret_cast(data.data()), data.size()); 494 | // parse the rtp packet 495 | espp::RtpJpegPacket rtp_jpeg_packet(packet); 496 | auto frag_offset = rtp_jpeg_packet.get_offset(); 497 | if (frag_offset == 0) { 498 | // first fragment 499 | UE_LOG(LogTemp, Log, TEXT("Received first fragment, size: %d, sequence number: %d"), 500 | rtp_jpeg_packet.get_data().size(), rtp_jpeg_packet.get_sequence_number()); 501 | if (jpeg_frame) { 502 | // we already have a frame, this is an error 503 | UE_LOG(LogTemp, Warning, TEXT("Received first fragment but already have a frame")); 504 | jpeg_frame.reset(); 505 | } 506 | jpeg_frame = std::make_unique(rtp_jpeg_packet); 507 | } else if (jpeg_frame) { 508 | UE_LOG(LogTemp, Log, TEXT("Received middle fragment, size: %d, sequence number: %d"), 509 | rtp_jpeg_packet.get_data().size(), rtp_jpeg_packet.get_sequence_number()); 510 | // middle fragment 511 | jpeg_frame->append(rtp_jpeg_packet); 512 | } else { 513 | // we don't have a frame to append to but we got a middle fragment 514 | // this is an error 515 | UE_LOG(LogTemp, Warning, TEXT("Received middle fragment without a frame")); 516 | return; 517 | } 518 | 519 | // check if this is the last packet of the frame (the last packet will have 520 | // the marker bit set) 521 | if (jpeg_frame && jpeg_frame->is_complete()) { 522 | // get the jpeg data 523 | auto jpeg_data = jpeg_frame->get_data(); 524 | UE_LOG(LogTemp, Log, TEXT("Received jpeg frame of size: %d B (%d x %d pixels)"), 525 | jpeg_data.size(), jpeg_frame->get_width(), jpeg_frame->get_height()); 526 | 527 | IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); 528 | auto image_format = ImageWrapperModule.DetectImageFormat(jpeg_data.data(), jpeg_data.size()); 529 | if (image_format == EImageFormat::Invalid) { 530 | UE_LOG(LogTemp, Error, TEXT("Failed to detect image format")); 531 | return; 532 | } 533 | 534 | TSharedPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper(image_format); 535 | if (!ImageWrapper.IsValid()) { 536 | UE_LOG(LogTemp, Error, TEXT("Failed to create image wrapper")); 537 | return; 538 | } 539 | 540 | // decompress the jpeg data 541 | if (!ImageWrapper->SetCompressed(jpeg_data.data(), jpeg_data.size())) { 542 | UE_LOG(LogTemp, Error, TEXT("Failed to set compressed data")); 543 | return; 544 | } 545 | // Get the decompressed data 546 | TArray UncompressedBGRA; 547 | if (!ImageWrapper->GetRaw(ERGBFormat::BGRA, 8, UncompressedBGRA)) { 548 | UE_LOG(LogTemp, Error, TEXT("Failed to get raw data")); 549 | return; 550 | } 551 | size_t width = ImageWrapper->GetWidth(); 552 | size_t height = ImageWrapper->GetHeight(); 553 | auto rgb_data = UncompressedBGRA.GetData(); 554 | auto rgb_data_size = UncompressedBGRA.Num(); 555 | 556 | std::unique_lock lock(image_mutex_); 557 | image_data_.clear(); 558 | image_data_.reserve(rgb_data_size); 559 | image_data_.assign(rgb_data, rgb_data + rgb_data_size); 560 | image_width_ = jpeg_frame->get_width(); 561 | image_height_ = jpeg_frame->get_height(); 562 | image_data_ready_ = true; 563 | // now reset the jpeg_frame 564 | jpeg_frame.reset(); 565 | } 566 | } 567 | 568 | void URtspClientComponent::handle_rtcp_packet(std::vector &data) { 569 | UE_LOG(LogTemp, Log, TEXT("Got RTCP packet of size: %d"), data.size()); 570 | // parse the rtcp packet 571 | // send the packet to the decoder 572 | } 573 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspClientComponent.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "CoreMinimal.h" 12 | #include "Components/ActorComponent.h" 13 | #include "IPAddress.h" 14 | #include "RtspClientComponent.generated.h" 15 | 16 | class FMyRunnable; 17 | class FSocket; 18 | class UTexture2D; 19 | 20 | // Blueprints can bind to this to update the UI 21 | DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnConnected); 22 | DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnDisconnected); 23 | DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPlay); 24 | DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPause); 25 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnFrameReceived, UTexture2D*, Texture); 26 | 27 | /** 28 | * @brief This class is used to connect to a RTSP server and receive the video 29 | * stream. 30 | * 31 | * @details This class is used to connect to a RTSP server and receive the video 32 | * stream. It currently supports only MJPEG streams (which are simply a 33 | * sequence of JPEG images). It uses a TCP socket (FSocket with 34 | * ESocketType::SOCKTYPE_Streaming) to connect to the RTSP server send 35 | * RTSP requests and receive RTSP responses. It uses a UDP socket 36 | * (FSocket with ESocketType::SOCKTYPE_Datagram) to receive the RTP and 37 | * RTCP packets, from which it extracts the JPEG images. It will 38 | * convert the JPEG images to UTexture2D and broadcast them to the 39 | * any registered listeners. 40 | */ 41 | UCLASS(ClassGroup = (Custom), meta = (BlueprintSpawnableComponent)) 42 | class RTSPDISPLAY_API URtspClientComponent : public UActorComponent 43 | { 44 | GENERATED_BODY() 45 | public: 46 | 47 | URtspClientComponent(); 48 | 49 | // Connect to the RTSP server using the address, port, and path that were 50 | // previously set. 51 | UFUNCTION(BlueprintCallable, Category = "RTSP") 52 | bool connect(); 53 | 54 | // Connect to the RTSP server using the specified URI. Parses the URI and 55 | // overrides the previously set address, port, and path. 56 | UFUNCTION(BlueprintCallable, Category = "RTSP") 57 | bool connect_to_uri(FString uri); 58 | 59 | // Connect to the RTSP server using the specified address, port, and path. 60 | // Overrides the previously set address, port, and path. 61 | UFUNCTION(BlueprintCallable, Category = "RTSP") 62 | bool connect_to_address(FString rtsp_address, int rtsp_port = 8554, FString path = TEXT("/mjpeg/1")); 63 | 64 | UFUNCTION(BlueprintCallable, Category = "RTSP") 65 | void disconnect(); 66 | 67 | UFUNCTION(BlueprintCallable, Category = "RTSP") 68 | bool describe(); 69 | 70 | UFUNCTION(BlueprintCallable, Category = "RTSP") 71 | bool setup(int rtp_port = 5000, int rtcp_port = 5001); 72 | 73 | UFUNCTION(BlueprintCallable, Category = "RTSP") 74 | bool play(); 75 | 76 | UFUNCTION(BlueprintCallable, Category = "RTSP") 77 | bool pause(); 78 | 79 | UFUNCTION(BlueprintCallable, Category = "RTSP") 80 | bool teardown(); 81 | 82 | UPROPERTY(BlueprintAssignable, Category = "RTSP") 83 | FOnConnected OnConnected; 84 | 85 | UPROPERTY(BlueprintAssignable, Category = "RTSP") 86 | FOnDisconnected OnDisconnected; 87 | 88 | UPROPERTY(BlueprintAssignable, Category = "RTSP") 89 | FOnPause OnPause; 90 | 91 | UPROPERTY(BlueprintAssignable, Category = "RTSP") 92 | FOnPlay OnPlay; 93 | 94 | UPROPERTY(BlueprintAssignable, Category = "RTSP") 95 | FOnFrameReceived OnFrameReceived; 96 | 97 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "RTSP") 98 | FString Address; 99 | 100 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "RTSP") 101 | int Port = 8554; 102 | 103 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "RTSP") 104 | FString Path = TEXT("/mjpeg/1"); 105 | 106 | UPROPERTY(BlueprintReadOnly, Category = "RTSP") 107 | bool IsConnected = false; 108 | 109 | UPROPERTY(BlueprintReadOnly, Category = "RTSP") 110 | bool IsPlaying = false; 111 | 112 | protected: 113 | 114 | std::string send_request(const std::string& method, const std::string& path, 115 | const std::unordered_map &extra_headers, std::error_code &ec); 116 | 117 | bool parse_response(const std::string &response_data); 118 | 119 | void init_rtp(size_t rtp_port); 120 | 121 | void init_rtcp(size_t rtcp_port); 122 | 123 | bool connect_thread_func(); 124 | 125 | bool rtp_thread_func(); 126 | 127 | bool rtcp_thread_func(); 128 | 129 | void handle_rtp_packet(std::vector &data); 130 | 131 | void handle_rtcp_packet(std::vector &data); 132 | 133 | FSocket *rtsp_socket_ = nullptr; 134 | FSocket *rtp_socket_ = nullptr; 135 | FSocket *rtcp_socket_ = nullptr; 136 | 137 | FMyRunnable *connect_thread_ = nullptr; 138 | FMyRunnable *rtp_thread_ = nullptr; 139 | FMyRunnable *rtcp_thread_ = nullptr; 140 | 141 | std::string path_; 142 | int cseq_ = 0; 143 | int video_port_ = 0; 144 | int video_payload_type_ = 0; 145 | std::string session_id_; 146 | 147 | std::mutex image_mutex_; 148 | std::vector image_data_; 149 | std::atomic image_data_ready_ = false; 150 | std::atomic image_width_ = 0; 151 | std::atomic image_height_ = 0; 152 | 153 | TSharedPtr rtsp_addr_; 154 | 155 | public: 156 | 157 | void BeginPlay() override; 158 | void EndPlay(const EEndPlayReason::Type EndPlayReason) override; 159 | void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction *ThisTickFunction) override; 160 | }; 161 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspDisplay.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | 5 | public class RtspDisplay : ModuleRules 6 | { 7 | public RtspDisplay(ReadOnlyTargetRules Target) : base(Target) 8 | { 9 | PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; 10 | 11 | PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "ImageWrapper", "HTTP", "Sockets", "Networking" }); 12 | 13 | PrivateDependencyModuleNames.AddRange(new string[] { }); 14 | 15 | // Uncomment if you are using Slate UI 16 | // PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" }); 17 | 18 | // Uncomment if you are using online features 19 | // PrivateDependencyModuleNames.Add("OnlineSubsystem"); 20 | 21 | // To include OnlineSubsystemSteam, add it to the plugins section in your uproject file with the Enabled attribute set to true 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspDisplay.cpp: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #include "RtspDisplay.h" 4 | #include "Modules/ModuleManager.h" 5 | 6 | IMPLEMENT_PRIMARY_GAME_MODULE( FDefaultGameModuleImpl, RtspDisplay, "RtspDisplay" ); 7 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspDisplay.h: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | 7 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspDisplayGameModeBase.cpp: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | 4 | #include "RtspDisplayGameModeBase.h" 5 | 6 | -------------------------------------------------------------------------------- /Source/RtspDisplay/RtspDisplayGameModeBase.h: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "GameFramework/GameModeBase.h" 7 | #include "RtspDisplayGameModeBase.generated.h" 8 | 9 | /** 10 | * 11 | */ 12 | UCLASS() 13 | class RTSPDISPLAY_API ARtspDisplayGameModeBase : public AGameModeBase 14 | { 15 | GENERATED_BODY() 16 | 17 | }; 18 | -------------------------------------------------------------------------------- /Source/RtspDisplay/jpeg_frame.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "jpeg_header.hpp" 4 | #include "rtp_jpeg_packet.hpp" 5 | 6 | namespace espp { 7 | /// A class that represents a complete JPEG frame. 8 | /// 9 | /// This class is used to collect the JPEG scans that are received in RTP 10 | /// packets and to serialize them into a complete JPEG frame. 11 | class JpegFrame { 12 | public: 13 | /// Construct a JpegFrame from a RtpJpegPacket. 14 | /// 15 | /// This constructor will parse the header of the packet and add the JPEG 16 | /// data to the frame. 17 | /// 18 | /// @param packet The packet to parse. 19 | explicit JpegFrame(const RtpJpegPacket &packet) 20 | : header_(packet.get_width(), packet.get_height(), packet.get_q_table(0), 21 | packet.get_q_table(1)) { 22 | // add the jpeg header 23 | serialize_header(); 24 | // add the jpeg data 25 | add_scan(packet); 26 | } 27 | 28 | /// Construct a JpegFrame from buffer of jpeg data 29 | /// @param data The buffer containing the jpeg data. 30 | /// @param size The size of the buffer. 31 | explicit JpegFrame(const char *data, size_t size) 32 | : data_(data, data + size), header_(std::string_view((const char*) data_.data(), size)) {} 33 | 34 | /// Get a reference to the header. 35 | /// @return A reference to the header. 36 | const JpegHeader &get_header() const { return header_; } 37 | 38 | /// Get the width of the frame. 39 | /// @return The width of the frame. 40 | int get_width() const { return header_.get_width(); } 41 | 42 | /// Get the height of the frame. 43 | /// @return The height of the frame. 44 | int get_height() const { return header_.get_height(); } 45 | 46 | /// Check if the frame is complete. 47 | /// @return True if the frame is complete, false otherwise. 48 | bool is_complete() const { return finalized_; } 49 | 50 | /// Append a RtpJpegPacket to the frame. 51 | /// This will add the JPEG data to the frame. 52 | /// @param packet The packet containing the scan to append. 53 | void append(const RtpJpegPacket &packet) { add_scan(packet); } 54 | 55 | /// Append a JPEG scan to the frame. 56 | /// This will add the JPEG data to the frame. 57 | /// @note If the packet contains the EOI marker, the frame will be 58 | /// finalized, and no further scans can be added. 59 | /// @param packet The packet containing the scan to append. 60 | void add_scan(const RtpJpegPacket &packet) { 61 | add_scan(packet.get_jpeg_data()); 62 | if (packet.get_marker()) { 63 | finalize(); 64 | } 65 | } 66 | 67 | /// Get the serialized data. 68 | /// This will return the serialized data. 69 | /// @return The serialized data. 70 | std::string_view get_data() const { return std::string_view((const char*)data_.data(), data_.size()); } 71 | 72 | /// Get the scan data. 73 | /// This will return the scan data. 74 | /// @return The scan data. 75 | std::string_view get_scan_data() const { 76 | auto header_data = header_.get_data(); 77 | size_t header_size = header_data.size(); 78 | return std::string_view((const char*)data_.data() + header_size, data_.size() - header_size); 79 | } 80 | 81 | protected: 82 | /// Serialize the header. 83 | void serialize_header() { 84 | auto header_data = header_.get_data(); 85 | data_.resize(header_data.size()); 86 | memcpy(data_.data(), header_data.data(), header_data.size()); 87 | } 88 | 89 | /// Append a JPEG scan to the frame. 90 | /// This will add the JPEG data to the frame. 91 | /// @param scan The jpeg scan to append. 92 | void add_scan(std::string_view scan) { 93 | if (finalized_) { 94 | // TODO: handle this error 95 | return; 96 | } 97 | data_.insert(std::end(data_), std::begin(scan), std::end(scan)); 98 | } 99 | 100 | /// Add the EOI marker to the frame. 101 | /// This will add the EOI marker to the frame. This must be called before 102 | /// calling get_data(). 103 | /// @note This will prevent any further scans from being added to the frame. 104 | void finalize() { 105 | if (!finalized_) { 106 | finalized_ = true; 107 | // add_eoi(); 108 | } else { 109 | // TODO: handle this error 110 | // already finalized 111 | } 112 | } 113 | 114 | /// Add the EOI marker to the frame. 115 | void add_eoi() { 116 | data_.push_back(0xFF); 117 | data_.push_back(0xD9); 118 | } 119 | 120 | std::vector data_; 121 | JpegHeader header_; 122 | bool finalized_ = false; 123 | }; 124 | } // namespace espp 125 | -------------------------------------------------------------------------------- /Source/RtspDisplay/jpeg_header.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace espp { 8 | /// A class to generate a JPEG header for a given image size and quantization tables. 9 | /// The header is generated once and then cached for future use. 10 | /// The header is generated according to the JPEG standard and is compatible with 11 | /// the ESP32 camera driver. 12 | class JpegHeader { 13 | public: 14 | /// Create a JPEG header for a given image size and quantization tables. 15 | /// @param width The image width in pixels. 16 | /// @param height The image height in pixels. 17 | /// @param q0_table The quantization table for the Y channel. 18 | /// @param q1_table The quantization table for the Cb and Cr channels. 19 | explicit JpegHeader(int width, int height, std::string_view q0_table, std::string_view q1_table) 20 | : width_(width), height_(height), q0_table_(q0_table), q1_table_(q1_table) { 21 | serialize(); 22 | } 23 | 24 | /// Create a JPEG header from a given JPEG header data. 25 | explicit JpegHeader(std::string_view data) : data_(data.data(), data.data() + data.size()) { 26 | parse(); 27 | } 28 | 29 | ~JpegHeader() {} 30 | 31 | /// Get the image width. 32 | /// @return The image width in pixels. 33 | int get_width() const { return width_; } 34 | 35 | /// Get the image height. 36 | /// @return The image height in pixels. 37 | int get_height() const { return height_; } 38 | 39 | /// Get the JPEG header data. 40 | /// @return The JPEG header data. 41 | std::string_view get_data() const { return std::string_view((const char*) data_.data(), data_.size()); } 42 | 43 | /// Get the Quantization table at the index. 44 | /// @param index The index of the quantization table. 45 | /// @return The quantization table. 46 | std::string_view get_quantization_table(int index) const { 47 | return index == 0 ? q0_table_ : q1_table_; 48 | } 49 | 50 | protected: 51 | static constexpr int SOF0_SIZE = 19; 52 | static constexpr int DQT_HEADER_SIZE = 5; 53 | 54 | // JFIF APP0 Marker for version 1.2 with 72 DPI and no thumbnail 55 | static constexpr uint8_t JFIF_APP0_DATA[] = { 56 | 0xFF, 0xE0, // APP0 marker 57 | 0x00, 0x10, // Length of APP0 data (16 bytes) 58 | 0x4A, 0x46, 0x49, 0x46, 0x00, // Identifier: ASCII "JFIF\0" 59 | 0x01, 0x01, // Version number (1.1) 60 | 0x01, // Units: 1 = dots per inch 61 | 0x00, 0x00, // X density 62 | 0x00, 0x00, // Y density 63 | 0x00, 0x00 // No thumbnail 64 | }; 65 | 66 | static constexpr uint8_t HUFFMAN_TABLES[] = { 67 | // Huffman table DC (luminance) 68 | 0xff, 69 | 0xc4, 70 | 0x00, 71 | 0x1f, 72 | 0x00, 73 | 0x00, 74 | 0x01, 75 | 0x05, 76 | 0x01, 77 | 0x01, 78 | 0x01, 79 | 0x01, 80 | 0x01, 81 | 0x01, 82 | 0x00, 83 | 0x00, 84 | 0x00, 85 | 0x00, 86 | 0x00, 87 | 0x00, 88 | 0x00, 89 | 0x00, 90 | 0x01, 91 | 0x02, 92 | 0x03, 93 | 0x04, 94 | 0x05, 95 | 0x06, 96 | 0x07, 97 | 0x08, 98 | 0x09, 99 | 0x0a, 100 | 0x0b, 101 | // Huffman table AC (luminance) 102 | 0xff, 103 | 0xc4, 104 | 0x00, 105 | 0xb5, 106 | 0x10, 107 | 0x00, 108 | 0x02, 109 | 0x01, 110 | 0x03, 111 | 0x03, 112 | 0x02, 113 | 0x04, 114 | 0x03, 115 | 0x05, 116 | 0x05, 117 | 0x04, 118 | 0x04, 119 | 0x00, 120 | 0x00, 121 | 0x01, 122 | 0x7d, 123 | 0x01, 124 | 0x02, 125 | 0x03, 126 | 0x00, 127 | 0x04, 128 | 0x11, 129 | 0x05, 130 | 0x12, 131 | 0x21, 132 | 0x31, 133 | 0x41, 134 | 0x06, 135 | 0x13, 136 | 0x51, 137 | 0x61, 138 | 0x07, 139 | 0x22, 140 | 0x71, 141 | 0x14, 142 | 0x32, 143 | 0x81, 144 | 0x91, 145 | 0xa1, 146 | 0x08, 147 | 0x23, 148 | 0x42, 149 | 0xb1, 150 | 0xc1, 151 | 0x15, 152 | 0x52, 153 | 0xd1, 154 | 0xf0, 155 | 0x24, 156 | 0x33, 157 | 0x62, 158 | 0x72, 159 | 0x82, 160 | 0x09, 161 | 0x0a, 162 | 0x16, 163 | 0x17, 164 | 0x18, 165 | 0x19, 166 | 0x1a, 167 | 0x25, 168 | 0x26, 169 | 0x27, 170 | 0x28, 171 | 0x29, 172 | 0x2a, 173 | 0x34, 174 | 0x35, 175 | 0x36, 176 | 0x37, 177 | 0x38, 178 | 0x39, 179 | 0x3a, 180 | 0x43, 181 | 0x44, 182 | 0x45, 183 | 0x46, 184 | 0x47, 185 | 0x48, 186 | 0x49, 187 | 0x4a, 188 | 0x53, 189 | 0x54, 190 | 0x55, 191 | 0x56, 192 | 0x57, 193 | 0x58, 194 | 0x59, 195 | 0x5a, 196 | 0x63, 197 | 0x64, 198 | 0x65, 199 | 0x66, 200 | 0x67, 201 | 0x68, 202 | 0x69, 203 | 0x6a, 204 | 0x73, 205 | 0x74, 206 | 0x75, 207 | 0x76, 208 | 0x77, 209 | 0x78, 210 | 0x79, 211 | 0x7a, 212 | 0x83, 213 | 0x84, 214 | 0x85, 215 | 0x86, 216 | 0x87, 217 | 0x88, 218 | 0x89, 219 | 0x8a, 220 | 0x92, 221 | 0x93, 222 | 0x94, 223 | 0x95, 224 | 0x96, 225 | 0x97, 226 | 0x98, 227 | 0x99, 228 | 0x9a, 229 | 0xa2, 230 | 0xa3, 231 | 0xa4, 232 | 0xa5, 233 | 0xa6, 234 | 0xa7, 235 | 0xa8, 236 | 0xa9, 237 | 0xaa, 238 | 0xb2, 239 | 0xb3, 240 | 0xb4, 241 | 0xb5, 242 | 0xb6, 243 | 0xb7, 244 | 0xb8, 245 | 0xb9, 246 | 0xba, 247 | 0xc2, 248 | 0xc3, 249 | 0xc4, 250 | 0xc5, 251 | 0xc6, 252 | 0xc7, 253 | 0xc8, 254 | 0xc9, 255 | 0xca, 256 | 0xd2, 257 | 0xd3, 258 | 0xd4, 259 | 0xd5, 260 | 0xd6, 261 | 0xd7, 262 | 0xd8, 263 | 0xd9, 264 | 0xda, 265 | 0xe1, 266 | 0xe2, 267 | 0xe3, 268 | 0xe4, 269 | 0xe5, 270 | 0xe6, 271 | 0xe7, 272 | 0xe8, 273 | 0xe9, 274 | 0xea, 275 | 0xf1, 276 | 0xf2, 277 | 0xf3, 278 | 0xf4, 279 | 0xf5, 280 | 0xf6, 281 | 0xf7, 282 | 0xf8, 283 | 0xf9, 284 | 0xfa, 285 | // Huffman table DC (chrominance) 286 | 0xff, 287 | 0xc4, 288 | 0x00, 289 | 0x1f, 290 | 0x01, 291 | 0x00, 292 | 0x03, 293 | 0x01, 294 | 0x01, 295 | 0x01, 296 | 0x01, 297 | 0x01, 298 | 0x01, 299 | 0x01, 300 | 0x01, 301 | 0x01, 302 | 0x00, 303 | 0x00, 304 | 0x00, 305 | 0x00, 306 | 0x00, 307 | 0x00, 308 | 0x01, 309 | 0x02, 310 | 0x03, 311 | 0x04, 312 | 0x05, 313 | 0x06, 314 | 0x07, 315 | 0x08, 316 | 0x09, 317 | 0x0a, 318 | 0x0b, 319 | // Huffman table AC (chrominance) 320 | 0xff, 321 | 0xc4, 322 | 0x00, 323 | 0xb5, 324 | 0x11, 325 | 0x00, 326 | 0x02, 327 | 0x01, 328 | 0x02, 329 | 0x04, 330 | 0x04, 331 | 0x03, 332 | 0x04, 333 | 0x07, 334 | 0x05, 335 | 0x04, 336 | 0x04, 337 | 0x00, 338 | 0x01, 339 | 0x02, 340 | 0x77, 341 | 0x00, 342 | 0x01, 343 | 0x02, 344 | 0x03, 345 | 0x11, 346 | 0x04, 347 | 0x05, 348 | 0x21, 349 | 0x31, 350 | 0x06, 351 | 0x12, 352 | 0x41, 353 | 0x51, 354 | 0x07, 355 | 0x61, 356 | 0x71, 357 | 0x13, 358 | 0x22, 359 | 0x32, 360 | 0x81, 361 | 0x08, 362 | 0x14, 363 | 0x42, 364 | 0x91, 365 | 0xa1, 366 | 0xb1, 367 | 0xc1, 368 | 0x09, 369 | 0x23, 370 | 0x33, 371 | 0x52, 372 | 0xf0, 373 | 0x15, 374 | 0x62, 375 | 0x72, 376 | 0xd1, 377 | 0x0a, 378 | 0x16, 379 | 0x24, 380 | 0x34, 381 | 0xe1, 382 | 0x25, 383 | 0xf1, 384 | 0x17, 385 | 0x18, 386 | 0x19, 387 | 0x1a, 388 | 0x26, 389 | 0x27, 390 | 0x28, 391 | 0x29, 392 | 0x2a, 393 | 0x35, 394 | 0x36, 395 | 0x37, 396 | 0x38, 397 | 0x39, 398 | 0x3a, 399 | 0x43, 400 | 0x44, 401 | 0x45, 402 | 0x46, 403 | 0x47, 404 | 0x48, 405 | 0x49, 406 | 0x4a, 407 | 0x53, 408 | 0x54, 409 | 0x55, 410 | 0x56, 411 | 0x57, 412 | 0x58, 413 | 0x59, 414 | 0x5a, 415 | 0x63, 416 | 0x64, 417 | 0x65, 418 | 0x66, 419 | 0x67, 420 | 0x68, 421 | 0x69, 422 | 0x6a, 423 | 0x73, 424 | 0x74, 425 | 0x75, 426 | 0x76, 427 | 0x77, 428 | 0x78, 429 | 0x79, 430 | 0x7a, 431 | 0x82, 432 | 0x83, 433 | 0x84, 434 | 0x85, 435 | 0x86, 436 | 0x87, 437 | 0x88, 438 | 0x89, 439 | 0x8a, 440 | 0x92, 441 | 0x93, 442 | 0x94, 443 | 0x95, 444 | 0x96, 445 | 0x97, 446 | 0x98, 447 | 0x99, 448 | 0x9a, 449 | 0xa2, 450 | 0xa3, 451 | 0xa4, 452 | 0xa5, 453 | 0xa6, 454 | 0xa7, 455 | 0xa8, 456 | 0xa9, 457 | 0xaa, 458 | 0xb2, 459 | 0xb3, 460 | 0xb4, 461 | 0xb5, 462 | 0xb6, 463 | 0xb7, 464 | 0xb8, 465 | 0xb9, 466 | 0xba, 467 | 0xc2, 468 | 0xc3, 469 | 0xc4, 470 | 0xc5, 471 | 0xc6, 472 | 0xc7, 473 | 0xc8, 474 | 0xc9, 475 | 0xca, 476 | 0xd2, 477 | 0xd3, 478 | 0xd4, 479 | 0xd5, 480 | 0xd6, 481 | 0xd7, 482 | 0xd8, 483 | 0xd9, 484 | 0xda, 485 | 0xe2, 486 | 0xe3, 487 | 0xe4, 488 | 0xe5, 489 | 0xe6, 490 | 0xe7, 491 | 0xe8, 492 | 0xe9, 493 | 0xea, 494 | 0xf2, 495 | 0xf3, 496 | 0xf4, 497 | 0xf5, 498 | 0xf6, 499 | 0xf7, 500 | 0xf8, 501 | 0xf9, 502 | 0xfa, 503 | }; 504 | 505 | // Scan header (SOS) 506 | static constexpr uint8_t SOS[] = { 507 | 0xFF, 0xDA, // SOS marker 508 | 0x00, 0x0C, // length 509 | 0x03, // number of components 510 | 0x01, 0x00, // component IDs and Huffman tables 511 | 0x02, 0x11, // component IDs and Huffman tables 512 | 0x03, 0x11, // component IDs and Huffman tables 513 | 0x00, 0x3F, 0x00 // Ss, Se, Ah/Al 514 | }; 515 | 516 | int add_sof0(int offset) { 517 | // add the SOF0 marker 518 | data_[offset++] = 0xFF; 519 | data_[offset++] = 0xC0; 520 | // add the length of the marker 521 | data_[offset++] = 0x00; 522 | data_[offset++] = 0x11; 523 | // add the precision 524 | data_[offset++] = 0x08; 525 | // add the height 526 | data_[offset++] = (height_ >> 8) & 0xFF; 527 | data_[offset++] = height_ & 0xFF; 528 | // add the width 529 | data_[offset++] = (width_ >> 8) & 0xFF; 530 | data_[offset++] = width_ & 0xFF; 531 | // add the number of components 532 | data_[offset++] = 0x03; 533 | // add the Y component 534 | data_[offset++] = 0x01; 535 | data_[offset++] = 0x21; 536 | data_[offset++] = 0x00; 537 | // add the Cb component 538 | data_[offset++] = 0x02; 539 | data_[offset++] = 0x11; 540 | data_[offset++] = 0x01; 541 | // add the Cr component 542 | data_[offset++] = 0x03; 543 | data_[offset++] = 0x11; 544 | data_[offset++] = 0x01; 545 | return offset; 546 | } 547 | 548 | void serialize() { 549 | int header_size = 2 + sizeof(JFIF_APP0_DATA) + DQT_HEADER_SIZE + q0_table_.size() + 550 | DQT_HEADER_SIZE + q1_table_.size() + sizeof(HUFFMAN_TABLES) + SOF0_SIZE + 551 | sizeof(SOS); 552 | // serialize the jpeg header to the data_ vector 553 | data_.resize(header_size); 554 | int offset = 0; 555 | 556 | // add the SOI marker 557 | data_[offset++] = 0xFF; 558 | data_[offset++] = 0xD8; 559 | 560 | // add the JFIF APP0 marker 561 | memcpy(data_.data() + offset, JFIF_APP0_DATA, sizeof(JFIF_APP0_DATA)); 562 | offset += sizeof(JFIF_APP0_DATA); 563 | 564 | // add the DQT marker for luminance 565 | data_[offset++] = 0xFF; 566 | data_[offset++] = 0xDB; 567 | data_[offset++] = 0x00; 568 | data_[offset++] = 0x43; 569 | data_[offset++] = 0x00; 570 | memcpy(data_.data() + offset, q0_table_.data(), q0_table_.size()); 571 | offset += q0_table_.size(); 572 | 573 | // add the DQT marker for chrominance 574 | data_[offset++] = 0xFF; 575 | data_[offset++] = 0xDB; 576 | data_[offset++] = 0x00; 577 | data_[offset++] = 0x43; 578 | data_[offset++] = 0x01; 579 | memcpy(data_.data() + offset, q1_table_.data(), q1_table_.size()); 580 | offset += q1_table_.size(); 581 | 582 | // add huffman tables 583 | memcpy(data_.data() + offset, HUFFMAN_TABLES, sizeof(HUFFMAN_TABLES)); 584 | offset += sizeof(HUFFMAN_TABLES); 585 | 586 | // add the SOF0 587 | offset = add_sof0(offset); 588 | 589 | // add the SOS marker 590 | memcpy(data_.data() + offset, SOS, sizeof(SOS)); 591 | offset += sizeof(SOS); 592 | } 593 | 594 | void parse() { 595 | // parse the jpeg header from the data_ vector 596 | int offset = 0; 597 | // check the SOI marker 598 | if (data_[offset++] != 0xFF || data_[offset++] != 0xD8) { 599 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOI marker\n")); 600 | return; 601 | } 602 | // check the JFIF APP0 marker 603 | if (memcmp(data_.data() + offset, JFIF_APP0_DATA, sizeof(JFIF_APP0_DATA)) != 0) { 604 | // UE_LOG(LogTemp, Error, TEXT("Invalid JFIF APP0 marker\n")); 605 | return; 606 | } 607 | offset += sizeof(JFIF_APP0_DATA); 608 | // check the DQT marker for luminance 609 | if (data_[offset++] != 0xFF || data_[offset++] != 0xDB) { 610 | // UE_LOG(LogTemp, Error, TEXT("Invalid DQT marker\n")); 611 | return; 612 | } 613 | if (data_[offset++] != 0x00 || data_[offset++] != 0x43) { 614 | // UE_LOG(LogTemp, Error, TEXT("Invalid DQT marker\n")); 615 | return; 616 | } 617 | if (data_[offset++] != 0x00) { 618 | // UE_LOG(LogTemp, Error, TEXT("Invalid DQT marker\n")); 619 | return; 620 | } 621 | q0_table_ = std::string_view((const char*) data_.data() + offset, 64); 622 | offset += 64; 623 | // check the DQT marker for chrominance 624 | if (data_[offset++] != 0xFF || data_[offset++] != 0xDB) { 625 | // UE_LOG(LogTemp, Error, TEXT("Invalid DQT marker\n")); 626 | return; 627 | } 628 | if (data_[offset++] != 0x00 || data_[offset++] != 0x43) { 629 | // UE_LOG(LogTemp, Error, TEXT("Invalid DQT marker\n")); 630 | return; 631 | } 632 | if (data_[offset++] != 0x01) { 633 | // UE_LOG(LogTemp, Error, TEXT("Invalid DQT marker\n")); 634 | return; 635 | } 636 | q1_table_ = std::string_view((const char*) data_.data() + offset, 64); 637 | offset += 64; 638 | // check huffman tables 639 | if (data_[offset++] != 0xFF || data_[offset++] != 0xC4) { 640 | // UE_LOG(LogTemp, Error, TEXT("Invalid huffman tables marker\n")); 641 | return; 642 | } 643 | if (data_[offset++] != 0x00 || data_[offset++] != 0x1f || data_[offset++] != 0x00) { 644 | // UE_LOG(LogTemp, Error, TEXT("Invalid huffman tables marker\n")); 645 | return; 646 | } 647 | offset += sizeof(HUFFMAN_TABLES) - 5; 648 | // check the SOF0 marker 649 | if (data_[offset++] != 0xFF || data_[offset++] != 0xC0) { 650 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 651 | return; 652 | } 653 | if (data_[offset++] != 0x00 || data_[offset++] != 0x11) { 654 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 655 | return; 656 | } 657 | // skip the precision 658 | offset++; 659 | // get the height and width 660 | height_ = (data_[offset] << 8) | data_[offset + 1]; 661 | offset += 2; 662 | width_ = (data_[offset] << 8) | data_[offset + 1]; 663 | offset += 2; 664 | if (data_[offset++] != 0x03) { 665 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 666 | return; 667 | } 668 | if (data_[offset++] != 0x01) { 669 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 670 | return; 671 | } 672 | if (data_[offset++] != 0x21) { 673 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 674 | return; 675 | } 676 | if (data_[offset++] != 0x00) { 677 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 678 | return; 679 | } 680 | if (data_[offset++] != 0x02) { 681 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 682 | return; 683 | } 684 | if (data_[offset++] != 0x11) { 685 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 686 | return; 687 | } 688 | if (data_[offset++] != 0x01) { 689 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 690 | return; 691 | } 692 | if (data_[offset++] != 0x03) { 693 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 694 | return; 695 | } 696 | if (data_[offset++] != 0x11) { 697 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 698 | return; 699 | } 700 | if (data_[offset++] != 0x01) { 701 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOF0 marker\n")); 702 | return; 703 | } 704 | // check the SOS marker 705 | if (data_[offset++] != 0xFF || data_[offset++] != 0xDA) { 706 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 707 | return; 708 | } 709 | if (data_[offset++] != 0x00 || data_[offset++] != 0x0C) { 710 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 711 | return; 712 | } 713 | if (data_[offset++] != 0x03) { 714 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 715 | return; 716 | } 717 | if (data_[offset++] != 0x01) { 718 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 719 | return; 720 | } 721 | if (data_[offset++] != 0x00) { 722 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 723 | return; 724 | } 725 | if (data_[offset++] != 0x02) { 726 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 727 | return; 728 | } 729 | if (data_[offset++] != 0x11) { 730 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 731 | return; 732 | } 733 | if (data_[offset++] != 0x03) { 734 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 735 | return; 736 | } 737 | if (data_[offset++] != 0x11) { 738 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 739 | return; 740 | } 741 | if (data_[offset++] != 0x00) { 742 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 743 | return; 744 | } 745 | if (data_[offset++] != 0x3F) { 746 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 747 | return; 748 | } 749 | if (data_[offset++] != 0x00) { 750 | // UE_LOG(LogTemp, Error, TEXT("Invalid SOS marker\n")); 751 | return; 752 | } 753 | data_.resize(offset); 754 | } 755 | 756 | int width_; 757 | int height_; 758 | std::string_view q0_table_; 759 | std::string_view q1_table_; 760 | 761 | std::vector data_; 762 | }; 763 | } // namespace espp 764 | -------------------------------------------------------------------------------- /Source/RtspDisplay/rtp_jpeg_packet.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "rtp_packet.hpp" 4 | 5 | namespace espp { 6 | /// RTP packet for JPEG video. 7 | /// The RTP payload for JPEG is defined in RFC 2435. 8 | class RtpJpegPacket : public RtpPacket { 9 | public: 10 | /// Construct an RTP packet from a buffer. 11 | /// @param data The buffer containing the RTP packet. 12 | explicit RtpJpegPacket(std::string_view data) : RtpPacket(data) { parse_mjpeg_header(); } 13 | 14 | /// Construct an RTP packet from fields 15 | /// @details This will construct a packet with quantization tables, so it 16 | /// can only be used for the first packet in a frame. 17 | /// @param type_specific The type-specific field. 18 | /// @param frag_type The fragment type field. 19 | /// @param q The q field. 20 | /// @param width The width field. 21 | /// @param height The height field. 22 | /// @param q0 The first quantization table. 23 | /// @param q1 The second quantization table. 24 | /// @param scan_data The scan data. 25 | explicit RtpJpegPacket(const int type_specific, const int frag_type, const int q, const int width, 26 | const int height, std::string_view q0, std::string_view q1, 27 | std::string_view scan_data) 28 | : RtpPacket(PAYLOAD_OFFSET_WITH_QUANT + scan_data.size()), type_specific_(type_specific), 29 | offset_(0), frag_type_(frag_type), q_(q), width_(width), height_(height) { 30 | 31 | jpeg_data_start_ = PAYLOAD_OFFSET_WITH_QUANT; 32 | jpeg_data_size_ = scan_data.size(); 33 | 34 | serialize_mjpeg_header(); 35 | serialize_q_tables(q0, q1); 36 | 37 | auto &packet = get_packet(); 38 | size_t jpeg_offset = jpeg_data_start_ + get_rtp_header_size(); 39 | memcpy(packet.data() + jpeg_offset, scan_data.data(), scan_data.size()); 40 | } 41 | 42 | /// Construct an RTP packet from fields 43 | /// @details This will construct a packet without quantization tables, so it 44 | /// cannot be used for the first packet in a frame. 45 | /// @param type_specific The type-specific field. 46 | /// @param offset The offset field. 47 | /// @param frag_type The fragment type field. 48 | /// @param q The q field. 49 | /// @param width The width field. 50 | /// @param height The height field. 51 | /// @param scan_data The scan data. 52 | explicit RtpJpegPacket(const int type_specific, const int offset, const int frag_type, 53 | const int q, const int width, const int height, std::string_view scan_data) 54 | : RtpPacket(PAYLOAD_OFFSET_NO_QUANT + scan_data.size()), type_specific_(type_specific), 55 | offset_(offset), frag_type_(frag_type), q_(q), width_(width), height_(height) { 56 | jpeg_data_start_ = PAYLOAD_OFFSET_NO_QUANT; 57 | jpeg_data_size_ = scan_data.size(); 58 | 59 | serialize_mjpeg_header(); 60 | 61 | auto &packet = get_packet(); 62 | size_t jpeg_offset = jpeg_data_start_ + get_rtp_header_size(); 63 | memcpy(packet.data() + jpeg_offset, scan_data.data(), scan_data.size()); 64 | } 65 | 66 | ~RtpJpegPacket() {} 67 | 68 | /// Get the type-specific field. 69 | /// @return The type-specific field. 70 | int get_type_specific() const { return type_specific_; } 71 | 72 | /// Get the offset field. 73 | /// @return The offset field. 74 | int get_offset() const { return offset_; } 75 | 76 | /// Get the fragment type field. 77 | /// @return The fragment type field. 78 | int get_q() const { return q_; } 79 | 80 | /// Get the fragment type field. 81 | /// @return The fragment type field. 82 | int get_width() const { return width_; } 83 | 84 | /// Get the fragment type field. 85 | /// @return The fragment type field. 86 | int get_height() const { return height_; } 87 | 88 | /// Get the mjepg header. 89 | /// @return The mjepg header. 90 | std::string_view get_mjpeg_header() { 91 | return std::string_view((char*)get_payload().data(), MJPEG_HEADER_SIZE); 92 | } 93 | 94 | /// Get whether the packet contains quantization tables. 95 | /// @note The quantization tables are optional. If they are present, the 96 | /// number of quantization tables is always 2. 97 | /// @note This check is based on the value of the q field. If the q field 98 | /// is 128-256, the packet contains quantization tables. 99 | /// @return Whether the packet contains quantization tables. 100 | bool has_q_tables() const { return q_ >= 128 && q_ <= 256; } 101 | 102 | /// Get the number of quantization tables. 103 | /// @note The quantization tables are optional. If they are present, the 104 | /// number of quantization tables is always 2. 105 | /// @note Only the first packet in a frame contains quantization tables. 106 | /// @return The number of quantization tables. 107 | int get_num_q_tables() const { return q_tables_.size(); } 108 | 109 | /// Get the quantization table at the specified index. 110 | /// @param index The index of the quantization table. 111 | /// @return The quantization table at the specified index. 112 | std::string_view get_q_table(int index) const { 113 | if (index < get_num_q_tables()) { 114 | return q_tables_[index]; 115 | } 116 | return {}; 117 | } 118 | 119 | void set_q_table(int index, std::string_view q_table) { 120 | if (index < get_num_q_tables()) { 121 | q_tables_[index] = q_table; 122 | } 123 | } 124 | 125 | /// Get the JPEG data. 126 | /// The jpeg data is the payload minus the mjpeg header and quantization 127 | /// tables. 128 | /// @return The JPEG data. 129 | std::string_view get_jpeg_data() const { 130 | auto payload = get_payload(); 131 | return std::string_view((char*)payload.data() + jpeg_data_start_, jpeg_data_size_); 132 | } 133 | 134 | protected: 135 | static constexpr int MJPEG_HEADER_SIZE = 8; 136 | static constexpr int QUANT_HEADER_SIZE = 4; 137 | static constexpr int NUM_Q_TABLES = 2; 138 | static constexpr int Q_TABLE_SIZE = 64; 139 | 140 | static constexpr int PAYLOAD_OFFSET_NO_QUANT = MJPEG_HEADER_SIZE; 141 | static constexpr int PAYLOAD_OFFSET_WITH_QUANT = 142 | MJPEG_HEADER_SIZE + QUANT_HEADER_SIZE + (NUM_Q_TABLES * Q_TABLE_SIZE); 143 | 144 | void parse_mjpeg_header() { 145 | auto payload = get_payload(); 146 | type_specific_ = payload[0]; 147 | offset_ = (payload[1] << 16) | (payload[2] << 8) | payload[3]; 148 | frag_type_ = payload[4]; 149 | q_ = payload[5]; 150 | width_ = payload[6] * 8; 151 | height_ = payload[7] * 8; 152 | 153 | size_t offset = MJPEG_HEADER_SIZE; 154 | 155 | if (has_q_tables()) { 156 | uint8_t num_quant_bytes = payload[11]; 157 | int expected_num_quant_bytes = NUM_Q_TABLES * Q_TABLE_SIZE; 158 | if (num_quant_bytes == expected_num_quant_bytes) { 159 | q_tables_.resize(NUM_Q_TABLES); 160 | offset += QUANT_HEADER_SIZE; 161 | for (int i = 0; i < NUM_Q_TABLES; i++) { 162 | q_tables_[i] = std::string_view((char*)payload.data() + offset, Q_TABLE_SIZE); 163 | offset += Q_TABLE_SIZE; 164 | } 165 | } 166 | } 167 | 168 | jpeg_data_start_ = offset; 169 | jpeg_data_size_ = payload.size() - jpeg_data_start_; 170 | } 171 | 172 | void serialize_mjpeg_header() { 173 | auto &packet = get_packet(); 174 | size_t offset = get_rtp_header_size(); 175 | 176 | packet[offset++] = type_specific_; 177 | packet[offset++] = (offset_ >> 16) & 0xff; 178 | packet[offset++] = (offset_ >> 8) & 0xff; 179 | packet[offset++] = offset_ & 0xff; 180 | packet[offset++] = frag_type_; 181 | packet[offset++] = q_; 182 | packet[offset++] = width_ / 8; 183 | packet[offset++] = height_ / 8; 184 | } 185 | 186 | void serialize_q_tables(std::string_view q0, std::string_view q1) { 187 | q_tables_.resize(NUM_Q_TABLES); 188 | auto &packet = get_packet(); 189 | int offset = get_rtp_header_size() + MJPEG_HEADER_SIZE; 190 | packet[offset++] = 0; 191 | packet[offset++] = 0; 192 | packet[offset++] = 0; 193 | packet[offset++] = NUM_Q_TABLES * Q_TABLE_SIZE; 194 | 195 | memcpy(packet.data() + offset, q0.data(), Q_TABLE_SIZE); 196 | q_tables_[0] = std::string_view((char*)packet.data() + offset, Q_TABLE_SIZE); 197 | offset += Q_TABLE_SIZE; 198 | 199 | memcpy(packet.data() + offset, q1.data(), Q_TABLE_SIZE); 200 | q_tables_[1] = std::string_view((char*)packet.data() + offset, Q_TABLE_SIZE); 201 | offset += Q_TABLE_SIZE; 202 | } 203 | 204 | uint8_t type_specific_{0}; 205 | uint32_t offset_{0}; 206 | uint8_t frag_type_{0}; 207 | uint8_t q_{0}; 208 | uint32_t width_{0}; 209 | uint32_t height_{0}; 210 | int jpeg_data_start_{0}; 211 | int jpeg_data_size_{0}; 212 | std::vector q_tables_; 213 | }; 214 | } // namespace espp 215 | -------------------------------------------------------------------------------- /Source/RtspDisplay/rtp_packet.cpp: -------------------------------------------------------------------------------- 1 | #include "rtp_packet.hpp" 2 | 3 | using namespace espp; 4 | 5 | RtpPacket::RtpPacket() { 6 | // ensure that the packet_ vector is at least RTP_HEADER_SIZE bytes long 7 | packet_.resize(RTP_HEADER_SIZE); 8 | } 9 | 10 | RtpPacket::RtpPacket(size_t payload_size) 11 | : payload_size_(payload_size) { 12 | // ensure that the packet_ vector is at least RTP_HEADER_SIZE + payload_size bytes long 13 | packet_.resize(RTP_HEADER_SIZE + payload_size); 14 | } 15 | 16 | RtpPacket::RtpPacket(std::string_view data) { 17 | packet_.assign(data.begin(), data.end()); 18 | payload_size_ = packet_.size() - RTP_HEADER_SIZE; 19 | if (packet_.size() >= RTP_HEADER_SIZE) 20 | parse_rtp_header(); 21 | } 22 | 23 | RtpPacket::~RtpPacket() {} 24 | 25 | /// Getters for the RTP header fields. 26 | int RtpPacket::get_version() const { return version_; } 27 | bool RtpPacket::get_padding() const { return padding_; } 28 | bool RtpPacket::get_extension() const { return extension_; } 29 | int RtpPacket::get_csrc_count() const { return csrc_count_; } 30 | bool RtpPacket::get_marker() const { return marker_; } 31 | int RtpPacket::get_payload_type() const { return payload_type_; } 32 | int RtpPacket::get_sequence_number() const { return sequence_number_; } 33 | int RtpPacket::get_timestamp() const { return timestamp_; } 34 | int RtpPacket::get_ssrc() const { return ssrc_; } 35 | 36 | /// Setters for the RTP header fields. 37 | void RtpPacket::set_version(int version) { version_ = version; } 38 | void RtpPacket::set_padding(bool padding) { padding_ = padding; } 39 | void RtpPacket::set_extension(bool extension) { extension_ = extension; } 40 | void RtpPacket::set_csrc_count(int csrc_count) { csrc_count_ = csrc_count; } 41 | void RtpPacket::set_marker(bool marker) { marker_ = marker; } 42 | void RtpPacket::set_payload_type(int payload_type) { payload_type_ = payload_type; } 43 | void RtpPacket::set_sequence_number(int sequence_number) { sequence_number_ = sequence_number; } 44 | void RtpPacket::set_timestamp(int timestamp) { timestamp_ = timestamp; } 45 | void RtpPacket::set_ssrc(int ssrc) { ssrc_ = ssrc; } 46 | 47 | void RtpPacket::serialize() { serialize_rtp_header(); } 48 | 49 | std::string_view RtpPacket::get_data() const { 50 | return std::string_view((char *)packet_.data(), packet_.size()); 51 | } 52 | 53 | size_t RtpPacket::get_rtp_header_size() const { return RTP_HEADER_SIZE; } 54 | 55 | std::string_view RtpPacket::get_rpt_header() const { 56 | return std::string_view((char *)packet_.data(), RTP_HEADER_SIZE); 57 | } 58 | 59 | std::vector &RtpPacket::get_packet() { return packet_; } 60 | 61 | std::string_view RtpPacket::get_payload() const { 62 | return std::string_view((char *)packet_.data() + RTP_HEADER_SIZE, payload_size_); 63 | } 64 | 65 | void RtpPacket::set_payload(std::string_view payload) { 66 | packet_.resize(RTP_HEADER_SIZE + payload.size()); 67 | std::copy(payload.begin(), payload.end(), packet_.begin() + RTP_HEADER_SIZE); 68 | payload_size_ = payload.size(); 69 | } 70 | 71 | void RtpPacket::parse_rtp_header() { 72 | version_ = (packet_[0] & 0xC0) >> 6; 73 | padding_ = (packet_[0] & 0x20) >> 5; 74 | extension_ = (packet_[0] & 0x10) >> 4; 75 | csrc_count_ = packet_[0] & 0x0F; 76 | marker_ = (packet_[1] & 0x80) >> 7; 77 | payload_type_ = packet_[1] & 0x7F; 78 | sequence_number_ = (packet_[2] << 8) | packet_[3]; 79 | timestamp_ = (packet_[4] << 24) | (packet_[5] << 16) | (packet_[6] << 8) | packet_[7]; 80 | ssrc_ = (packet_[8] << 24) | (packet_[9] << 16) | (packet_[10] << 8) | packet_[11]; 81 | } 82 | 83 | void RtpPacket::serialize_rtp_header() { 84 | packet_[0] = (version_ << 6) | (padding_ << 5) | (extension_ << 4) | csrc_count_; 85 | packet_[1] = (marker_ << 7) | payload_type_; 86 | packet_[2] = sequence_number_ >> 8; 87 | packet_[3] = sequence_number_ & 0xFF; 88 | packet_[4] = timestamp_ >> 24; 89 | packet_[5] = (timestamp_ >> 16) & 0xFF; 90 | packet_[6] = (timestamp_ >> 8) & 0xFF; 91 | packet_[7] = timestamp_ & 0xFF; 92 | packet_[8] = ssrc_ >> 24; 93 | packet_[9] = (ssrc_ >> 16) & 0xFF; 94 | packet_[10] = (ssrc_ >> 8) & 0xFF; 95 | packet_[11] = ssrc_ & 0xFF; 96 | } 97 | -------------------------------------------------------------------------------- /Source/RtspDisplay/rtp_packet.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace espp { 9 | /// RtpPacket is a class to parse RTP packet. 10 | /// It can be used to parse and serialize RTP packets. 11 | /// The RTP header fields are stored in the class and can be modified. 12 | /// The payload is stored in the packet_ vector and can be modified. 13 | class RtpPacket { 14 | public: 15 | /// Construct an empty RtpPacket. 16 | /// The packet_ vector is empty and the header fields are set to 0. 17 | RtpPacket(); 18 | 19 | /// Construct an RtpPacket with a payload of size payload_size. 20 | /// The packet_ vector is resized to RTP_HEADER_SIZE + payload_size. 21 | explicit RtpPacket(size_t payload_size); 22 | 23 | /// Construct an RtpPacket from a string_view. 24 | /// Store the string_view in the packet_ vector and parses the header. 25 | /// @param data The string_view to parse. 26 | explicit RtpPacket(std::string_view data); 27 | 28 | /// Destructor. 29 | ~RtpPacket(); 30 | 31 | // ----------------------------------------------------------------- 32 | // Getters for the RTP header fields. 33 | // ----------------------------------------------------------------- 34 | 35 | /// Get the RTP version. 36 | /// @return The RTP version. 37 | int get_version() const; 38 | 39 | /// Get the padding flag. 40 | /// @return The padding flag. 41 | bool get_padding() const; 42 | 43 | /// Get the extension flag. 44 | /// @return The extension flag. 45 | bool get_extension() const; 46 | 47 | /// Get the CSRC count. 48 | /// @return The CSRC count. 49 | int get_csrc_count() const; 50 | 51 | /// Get the marker flag. 52 | /// @return The marker flag. 53 | bool get_marker() const; 54 | 55 | /// Get the payload type. 56 | /// @return The payload type. 57 | int get_payload_type() const; 58 | 59 | /// Get the sequence number. 60 | /// @return The sequence number. 61 | int get_sequence_number() const; 62 | 63 | /// Get the timestamp. 64 | /// @return The timestamp. 65 | int get_timestamp() const; 66 | 67 | /// Get the SSRC. 68 | /// @return The SSRC. 69 | int get_ssrc() const; 70 | 71 | // ----------------------------------------------------------------- 72 | // Setters for the RTP header fields. 73 | // ----------------------------------------------------------------- 74 | 75 | /// Set the RTP version. 76 | /// @param version The RTP version to set. 77 | void set_version(int version); 78 | 79 | /// Set the padding flag. 80 | /// @param padding The padding flag to set. 81 | void set_padding(bool padding); 82 | 83 | /// Set the extension flag. 84 | /// @param extension The extension flag to set. 85 | void set_extension(bool extension); 86 | 87 | /// Set the CSRC count. 88 | /// @param csrc_count The CSRC count to set. 89 | void set_csrc_count(int csrc_count); 90 | 91 | /// Set the marker flag. 92 | /// @param marker The marker flag to set. 93 | void set_marker(bool marker); 94 | 95 | /// Set the payload type. 96 | /// @param payload_type The payload type to set. 97 | void set_payload_type(int payload_type); 98 | 99 | /// Set the sequence number. 100 | /// @param sequence_number The sequence number to set. 101 | void set_sequence_number(int sequence_number); 102 | 103 | /// Set the timestamp. 104 | /// @param timestamp The timestamp to set. 105 | void set_timestamp(int timestamp); 106 | 107 | /// Set the SSRC. 108 | /// @param ssrc The SSRC to set. 109 | void set_ssrc(int ssrc); 110 | 111 | // ----------------------------------------------------------------- 112 | // Utility methods. 113 | // ----------------------------------------------------------------- 114 | 115 | /// Serialize the RTP header. 116 | /// @note This method should be called after modifying the RTP header fields. 117 | /// @note This method does not serialize the payload. To set the payload, use 118 | /// set_payload(). 119 | /// To get the payload, use get_payload(). 120 | void serialize(); 121 | 122 | /// Get a string_view of the whole packet. 123 | /// @note The string_view is valid as long as the packet_ vector is not modified. 124 | /// @note If you manually build the packet_ vector, you should make sure that you 125 | /// call serialize() before calling this method. 126 | /// @return A string_view of the whole packet. 127 | std::string_view get_data() const; 128 | 129 | /// Get the size of the RTP header. 130 | /// @return The size of the RTP header. 131 | size_t get_rtp_header_size() const; 132 | 133 | /// Get a string_view of the RTP header. 134 | /// @return A string_view of the RTP header. 135 | std::string_view get_rpt_header() const; 136 | 137 | /// Get a reference to the packet_ vector. 138 | /// @return A reference to the packet_ vector. 139 | std::vector &get_packet(); 140 | 141 | /// Get a string_view of the payload. 142 | /// @return A string_view of the payload. 143 | std::string_view get_payload() const; 144 | 145 | /// Set the payload. 146 | /// @param payload The payload to set. 147 | void set_payload(std::string_view payload); 148 | 149 | protected: 150 | static constexpr int RTP_HEADER_SIZE = 12; 151 | 152 | void parse_rtp_header(); 153 | void serialize_rtp_header(); 154 | 155 | std::vector packet_; 156 | int version_{2}; 157 | bool padding_{false}; 158 | bool extension_{false}; 159 | int csrc_count_{0}; 160 | bool marker_{false}; 161 | int payload_type_{0}; 162 | int sequence_number_{0}; 163 | int timestamp_{0}; 164 | int ssrc_{0}; 165 | int payload_size_{0}; 166 | }; 167 | } // namespace espp 168 | -------------------------------------------------------------------------------- /Source/RtspDisplayEditor.Target.cs: -------------------------------------------------------------------------------- 1 | // Copyright Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | using System.Collections.Generic; 5 | 6 | public class RtspDisplayEditorTarget : TargetRules 7 | { 8 | public RtspDisplayEditorTarget( TargetInfo Target) : base(Target) 9 | { 10 | Type = TargetType.Editor; 11 | DefaultBuildSettings = BuildSettingsVersion.V5; 12 | // CppStandard = CppStandardVersion.Cpp20; 13 | // IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_5; 14 | 15 | ExtraModuleNames.AddRange( new string[] { "RtspDisplay" } ); 16 | } 17 | } 18 | --------------------------------------------------------------------------------