├── IVideoSource.cpp ├── IVideoSource.h ├── LICENSE ├── OpenCVVideoSource.cpp ├── OpenCVVideoSource.h ├── README.md ├── VideoDisplaySurface.cpp └── VideoDisplaySurface.h /IVideoSource.cpp: -------------------------------------------------------------------------------- 1 | /***************************** 2 | Copyright 2015 (c) Leonardo Malave. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are 5 | permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of 8 | conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 11 | of conditions and the following disclaimer in the documentation and/or other materials 12 | provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY Leonardo Malave ''AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Rafael Muñoz Salinas OR 17 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those of the 25 | authors and should not be interpreted as representing official policies, either expressed 26 | or implied, of Leonardo Malave. 27 | ********************************/ 28 | 29 | #include "OculusARPOC.h" 30 | #include "IVideoSource.h" 31 | 32 | 33 | IVideoSource::IVideoSource() 34 | { 35 | } 36 | 37 | IVideoSource::~IVideoSource() 38 | { 39 | } 40 | -------------------------------------------------------------------------------- /IVideoSource.h: -------------------------------------------------------------------------------- 1 | /***************************** 2 | Copyright 2015 (c) Leonardo Malave. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are 5 | permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of 8 | conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 11 | of conditions and the following disclaimer in the documentation and/or other materials 12 | provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY Leonardo Malave ''AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Rafael Muñoz Salinas OR 17 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those of the 25 | authors and should not be interpreted as representing official policies, either expressed 26 | or implied, of Leonardo Malave. 27 | ********************************/ 28 | 29 | #pragma once 30 | 31 | /** 32 | * This is an interface representing a video source to be used with a VideoDisplaySurface 33 | */ 34 | class IVideoSource 35 | { 36 | public: 37 | 38 | IVideoSource(); 39 | ~IVideoSource(); 40 | 41 | virtual void GetFrameImage(uint8* DestinationImageBuffer) = 0; 42 | 43 | virtual uint16 GetVideoWidth() = 0; 44 | 45 | virtual void SetVideoWidth(uint16 Width) = 0; 46 | 47 | virtual uint16 GetVideoHeight() = 0; 48 | 49 | virtual void SetVideoHeight(uint16 Height) = 0; 50 | 51 | virtual float GetWidthToDistanceRatio() = 0; 52 | 53 | virtual float GetHeightToDistanceRatio() = 0; 54 | 55 | virtual void Init() = 0; 56 | 57 | virtual void Close() = 0; 58 | 59 | }; 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 lmalave 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /OpenCVVideoSource.cpp: -------------------------------------------------------------------------------- 1 | /***************************** 2 | Copyright 2015 (c) Leonardo Malave. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are 5 | permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of 8 | conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 11 | of conditions and the following disclaimer in the documentation and/or other materials 12 | provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY Leonardo Malave ''AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Rafael Muñoz Salinas OR 17 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those of the 25 | authors and should not be interpreted as representing official policies, either expressed 26 | or implied, of Leonardo Malave. 27 | ********************************/ 28 | 29 | #include "OculusARPOC.h" 30 | #include "Engine.h" 31 | #include "OpenCVVideoSource.h" 32 | #include "opencv2/core/core.hpp" 33 | #include "opencv2/highgui/highgui.hpp" 34 | #include "opencv2/imgproc/imgproc.hpp" 35 | 36 | OpenCVVideoSource::OpenCVVideoSource(uint8 cameraIndex, uint16 videoWidth, uint16 videoHeight) 37 | { 38 | this->CameraIndex = cameraIndex; 39 | this->VideoWidth = videoWidth; 40 | this->VideoHeight = videoHeight; 41 | } 42 | 43 | OpenCVVideoSource::~OpenCVVideoSource() 44 | { 45 | } 46 | 47 | uint16 OpenCVVideoSource::GetVideoWidth() { 48 | return VideoWidth; 49 | } 50 | 51 | void OpenCVVideoSource::SetVideoWidth(uint16 Width) { 52 | this->VideoWidth = Width; 53 | } 54 | 55 | uint16 OpenCVVideoSource::GetVideoHeight() { 56 | return VideoHeight; 57 | } 58 | 59 | void OpenCVVideoSource::SetVideoHeight(uint16 Height) { 60 | this->VideoHeight = Height; 61 | } 62 | 63 | 64 | float OpenCVVideoSource::GetWidthToDistanceRatio() { 65 | return WidthToDistanceRatio; 66 | } 67 | 68 | float OpenCVVideoSource::GetHeightToDistanceRatio() { 69 | return HeightToDistanceRatio; 70 | } 71 | 72 | void OpenCVVideoSource::Init() { 73 | cv::VideoCapture VidCap(CameraIndex); 74 | this->VideoCapture = VidCap; 75 | if (VideoCapture.isOpened()) { 76 | VideoCapture.set(CV_CAP_PROP_FRAME_WIDTH, VideoWidth); 77 | VideoCapture.set(CV_CAP_PROP_FRAME_HEIGHT, VideoHeight); 78 | } 79 | } 80 | 81 | void OpenCVVideoSource::Close() { 82 | VideoCapture.release(); 83 | } 84 | 85 | void OpenCVVideoSource::GetFrameImage(uint8* DestinationFrameBuffer) { 86 | cv::Mat Frame; 87 | VideoCapture >> Frame; // get a new frame from camera 88 | uint16 Width = Frame.size().width; 89 | uint16 Height = Frame.size().height; 90 | uint8* RawFrameBuffer = (uint8*) Frame.data; 91 | uint8* DestinationPointer = NULL; 92 | uint8* SourcePointer = NULL; 93 | 94 | uint8 RedChannel; 95 | uint8 GreenChannel; 96 | uint8 BlueChannel; 97 | 98 | bool CameraUpsideDown = true; 99 | if (RawFrameBuffer != NULL) { 100 | if (CameraUpsideDown) { // draw bottom to top and right to left to flip image 101 | SourcePointer = RawFrameBuffer; 102 | for (int32 y = 0; y < VideoHeight; y++) 103 | { 104 | DestinationPointer = &DestinationFrameBuffer[(VideoHeight - y) * VideoWidth * sizeof(FColor) - 1]; 105 | for (int32 x = 0; x < VideoWidth; x++) 106 | { 107 | BlueChannel = *SourcePointer++; 108 | GreenChannel = *SourcePointer++; 109 | RedChannel = *SourcePointer++; 110 | *DestinationPointer-- = 0xFF; 111 | *DestinationPointer-- = RedChannel; 112 | *DestinationPointer-- = GreenChannel; 113 | *DestinationPointer-- = BlueChannel; 114 | } 115 | } 116 | } else { 117 | DestinationPointer = DestinationFrameBuffer; 118 | SourcePointer = RawFrameBuffer; 119 | for (int32 y = 0; y < VideoHeight; y++) 120 | { 121 | for (int32 x = 0; x < VideoWidth; x++) 122 | { 123 | *DestinationPointer++ = *SourcePointer++; 124 | *DestinationPointer++ = *SourcePointer++; 125 | *DestinationPointer++ = *SourcePointer++; 126 | *DestinationPointer++ = 0xFF; 127 | } 128 | } 129 | } 130 | } 131 | } 132 | 133 | -------------------------------------------------------------------------------- /OpenCVVideoSource.h: -------------------------------------------------------------------------------- 1 | /***************************** 2 | Copyright 2015 (c) Leonardo Malave. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are 5 | permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of 8 | conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 11 | of conditions and the following disclaimer in the documentation and/or other materials 12 | provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY Leonardo Malave ''AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Rafael Muñoz Salinas OR 17 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those of the 25 | authors and should not be interpreted as representing official policies, either expressed 26 | or implied, of Leonardo Malave. 27 | ********************************/ 28 | 29 | #include "IVideoSource.h" 30 | #include 31 | 32 | #pragma once 33 | 34 | /** 35 | * This represents an interface to a webcam with OpenCV 36 | */ 37 | class OpenCVVideoSource : public IVideoSource 38 | { 39 | public: 40 | OpenCVVideoSource(uint8 cameraIndex, uint16 videoWidth, uint16 videoHeight); 41 | ~OpenCVVideoSource(); 42 | 43 | void GetFrameImage(uint8* DestinationImageBuffer) override; 44 | 45 | uint16 GetVideoWidth() override; 46 | 47 | void SetVideoWidth(uint16 Width) override; 48 | 49 | uint16 GetVideoHeight() override; 50 | 51 | void SetVideoHeight(uint16 Height) override; 52 | 53 | float GetWidthToDistanceRatio() override; 54 | 55 | float GetHeightToDistanceRatio() override; 56 | 57 | void Init() override; 58 | 59 | void Close() override; 60 | 61 | 62 | protected: 63 | 64 | uint8 CameraIndex; 65 | 66 | uint16 VideoWidth; 67 | 68 | uint16 VideoHeight; 69 | 70 | float WidthToDistanceRatio; 71 | 72 | float HeightToDistanceRatio; 73 | 74 | cv::VideoCapture VideoCapture; 75 | 76 | 77 | }; 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # unreal-augmented-reality-toolkit 2 | This is a collection of classes to enable augmented reality development on Unreal Engine 3 | 4 | For now the classes just provide a means to draw a webcam video to the background, but the idea is that over time more classes will be added related to computer vision, detecting markers, etc. 5 | 6 | Let's go straight to the video - here is a demo of drawing objects (in this case Coherent UI browser views) on top of a video that is in the background: 7 | 8 | https://www.youtube.com/watch?v=I0IZ_G_TWIo 9 | 10 | ## How to integrate into Unreal Engine C++ project 11 | 12 | * To use the OpenCVVideo source, will need to install the OpenCV SDK and add the "core" and "highgui" libraries to the libpath. For OpenCV you will also need to know the index of your webcam. If you have only one webcam then it is index 0. If you are on a laptop with a built-in webcam and you are using a second USB webcam (for example mounted to the Oculus), then the USB webcam will probably be index 1. 13 | 14 | * Create a Material called "VideoMaterial" at the path /Game/Materials/ in the Content Browser, using a TextureSample input for the BaseColor, and then convert the TextureSample to a parameter with the name "VideoTexture" 15 | 16 | * Create a Blueprint using VideoDisplaySurface as the base class 17 | 18 | add to your Character class constructor: 19 | 20 | static ConstructorHelpers::FObjectFinder VideoSurfaceBlueprint(TEXT("Blueprint'/Game/Blueprints/VideoDisplaySurfaceBlueprint.VideoDisplaySurfaceBlueprint'")); 21 | if (VideoSurfaceBlueprint.Object){ 22 | VideoSurfaceBlueprintClass = (UClass*)VideoSurfaceBlueprint.Object->GeneratedClass; 23 | } 24 | 25 | BackgroundVideoSurface = ObjectInitializer.CreateDefaultSubobject(this, TEXT("BackgroundVideoSurface")); 26 | BackgroundVideoSurface->AttachParent = FirstPersonCameraComponent; 27 | BackgroundVideoSurface->ChildActorClass = VideoSurfaceBlueprintClass; 28 | 29 | add to BeginPlay(): 30 | 31 | AVideoDisplaySurface* BackgroundVideoDisplaySurface = (AVideoDisplaySurface*)BackgroundVideoSurface->ChildActor; 32 | OpenCVVideoSource* VideoSource = new OpenCVVideoSource(1, 1280, 720); 33 | VideoSource->Init(); 34 | BackgroundVideoDisplaySurface->Init(VideoSource); 35 | BackgroundVideoSurface->RelativeLocation = FVector(500.f, -0.f, 0.f); // place video surface 5 meters away in virtual space 36 | BackgroundVideoSurface->RelativeRotation = FRotator(0.f, 90.f, 90.f); 37 | BackgroundVideoSurface->RelativeScale3D = FVector(8.89, 5.00, 1.0); // This is for 1280x720 and assuming a 5 meter distance away 38 | 39 | That's it! The VideoDisplaySurface and OpenCV integration is intended to be simple and flexible. 40 | 41 | The actual implementation with OpenCV involves the 3 classes documented below. 42 | 43 | ## C++ Class Design and Implementation 44 | 45 | The basic idea is to capture video frames from a webcam attached to the HMD, and then draw the video frames to a dynamic texture. In order to support various video sources, an IVideoSource interface is defined. For example, OpenCVVideoSource is an implementation for OpenCV, but I have also implemented using the Leap SDK Image API as a source, and also the OvrVision camera as a source (which has its own SDK). 46 | 47 | For now there are 3 classes provided 48 | 49 | * IVideoSource - an interface representing a video source that can be displayed on a VideoDisplaySurface 50 | * OpenCVVideoSource - an implementation of the IVideoSource that uses OpenCV to read data from a webcam 51 | * VideoDisplaySurface - is a ShapePlane mesh with a dynamic material that can draw video frames to the dynamic texture 52 | 53 | ### IVideoSource interface 54 | 55 | IVideoSource is an interface intended to encapsulate a video source that can be used to provide raw video frame data to be displayed by the VideoDisplaySurface. The implementation class needs implement the GetFrameImage() function, which takes a destination frame image buffer as a parameter. The VideoSource should read the video frame and write the data to this buffer in RGBA format. 56 | 57 | ### OpenCVVideoSource class 58 | 59 | OpenCVVideoSource is an OpenCV based implementation of the IVideoSource interface. This class uses OpenCV to read video frames from a standard webcam, and then reformats the data from RGB into RGBA. The class also supports the camera being mounted upside down (as I am doing with my Logitech C615 mounted updside down on my Oculus) 60 | 61 | ### VideoDisplaySurface class 62 | 63 | The job of the VideoDisplaySurface is to draw the raw RGBA video frame data to a dynamic texture. The core of the implementation is the UpdateTextureRegions helpfully provided by Epic Games on their wiki: https://wiki.unrealengine.com/Dynamic_Textures and is the reason why the implemenation seems to perform quite well (I'm testing with 1280x720 resolution on a Mac laptop with integrated graphics) 64 | 65 | ## Roadmap 66 | 67 | * Decouple frame update from Tick() so that the game FPS isn't pinned by the webcam FPS. For augmented reality applications 30 FPS seems to be fine, though. 68 | * Support for other video sources (I will post Leap Motion and OvrVision implementations soon) 69 | * Additional augmented reality functionality like detecting markers 70 | 71 | -------------------------------------------------------------------------------- /VideoDisplaySurface.cpp: -------------------------------------------------------------------------------- 1 | /***************************** 2 | Copyright 2015 (c) Leonardo Malave. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are 5 | permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of 8 | conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 11 | of conditions and the following disclaimer in the documentation and/or other materials 12 | provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY Leonardo Malave ''AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Rafael Muñoz Salinas OR 17 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those of the 25 | authors and should not be interpreted as representing official policies, either expressed 26 | or implied, of Leonardo Malave. 27 | ********************************/ 28 | 29 | #include "OculusARPOC.h" 30 | #include "Engine.h" 31 | #include "VideoDisplaySurface.h" 32 | 33 | 34 | AVideoDisplaySurface::AVideoDisplaySurface(const class FPostConstructInitializeProperties& PCIP) 35 | : Super(PCIP) 36 | { 37 | VideoSurfaceMesh = PCIP.CreateDefaultSubobject(this, TEXT("VideoSurfaceMesh")); 38 | RootComponent = VideoSurfaceMesh; 39 | 40 | static ConstructorHelpers::FObjectFinderShapePlaneMesh(TEXT("StaticMesh'/Game/StarterContent/Shapes/Shape_Plane.Shape_Plane'")); 41 | static ConstructorHelpers::FObjectFinder LeapVideoMaterial(TEXT("Material'/Game/Materials/VideoMaterial.VideoMaterial'")); 42 | VideoSurfaceMesh->SetStaticMesh(ShapePlaneMesh.Object); 43 | VideoSurfaceMesh->SetMaterial(0, LeapVideoMaterial.Object); 44 | PrimaryActorTick.bCanEverTick = true; 45 | 46 | PreferredDistanceInMeters = 10.0; 47 | } 48 | 49 | ////////////////////////////////////////////////////////////////////////// 50 | // Texture 51 | 52 | void AVideoDisplaySurface::UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData) 53 | { 54 | if (Texture && Texture->Resource) 55 | { 56 | 57 | struct FUpdateTextureRegionsData 58 | { 59 | FTexture2DResource* Texture2DResource; 60 | int32 MipIndex; 61 | uint32 NumRegions; 62 | FUpdateTextureRegion2D* Regions; 63 | uint32 SrcPitch; 64 | uint32 SrcBpp; 65 | uint8* SrcData; 66 | }; 67 | 68 | FUpdateTextureRegionsData* RegionData = new FUpdateTextureRegionsData; 69 | 70 | RegionData->Texture2DResource = (FTexture2DResource*)Texture->Resource; 71 | RegionData->MipIndex = MipIndex; 72 | RegionData->NumRegions = NumRegions; 73 | RegionData->Regions = Regions; 74 | RegionData->SrcPitch = SrcPitch; 75 | RegionData->SrcBpp = SrcBpp; 76 | RegionData->SrcData = SrcData; 77 | 78 | ENQUEUE_UNIQUE_RENDER_COMMAND_TWOPARAMETER( 79 | UpdateTextureRegionsData, 80 | FUpdateTextureRegionsData*, RegionData, RegionData, 81 | bool, bFreeData, bFreeData, 82 | { 83 | for (uint32 RegionIndex = 0; RegionIndex < RegionData->NumRegions; ++RegionIndex) 84 | { 85 | int32 CurrentFirstMip = RegionData->Texture2DResource->GetCurrentFirstMip(); 86 | if (RegionData->MipIndex >= CurrentFirstMip) 87 | { 88 | RHIUpdateTexture2D( 89 | RegionData->Texture2DResource->GetTexture2DRHI(), 90 | RegionData->MipIndex - CurrentFirstMip, 91 | RegionData->Regions[RegionIndex], 92 | RegionData->SrcPitch, 93 | RegionData->SrcData 94 | + RegionData->Regions[RegionIndex].SrcY * RegionData->SrcPitch 95 | + RegionData->Regions[RegionIndex].SrcX * RegionData->SrcBpp 96 | ); 97 | } 98 | } 99 | if (bFreeData) 100 | { 101 | FMemory::Free(RegionData->Regions); 102 | FMemory::Free(RegionData->SrcData); 103 | } 104 | delete RegionData; 105 | 106 | }); 107 | 108 | } 109 | 110 | } 111 | 112 | void AVideoDisplaySurface::UpdateVideoFrame() 113 | { 114 | uint8* DestinationImageBuffer = (uint8*)VideoFrameData.GetData(); 115 | VideoSource->GetFrameImage(DestinationImageBuffer); 116 | UpdateTextureRegions(VideoTexture, (int32)0, (uint32)1, VideoTextureRegion, (uint32)(4 * VideoSource->GetVideoWidth()), (uint32)4, DestinationImageBuffer, false); 117 | 118 | } 119 | 120 | FVector AVideoDisplaySurface::GetWorldLocationFromPixelCoordinates(FVector2D PixelCoordinates) { 121 | FVector2D NormalizedCoordinates = FVector2D(PixelCoordinates.X / VideoSource->GetVideoWidth() - 0.5, PixelCoordinates.Y / VideoSource->GetVideoHeight() - 0.5); 122 | return this->GetActorLocation() - NormalizedCoordinates.X * this->GetActorForwardVector() * this->GetActorScale3D().X * 50 - NormalizedCoordinates.Y * this->GetActorRightVector() * this->GetActorScale3D().Y * 50; 123 | } 124 | 125 | void AVideoDisplaySurface::BeginPlay() 126 | { 127 | } 128 | 129 | void AVideoDisplaySurface::Init(IVideoSource* videoSource) 130 | { 131 | this->VideoSource = videoSource; 132 | VideoFrameData.Init(FColor::MakeRandomColor(), VideoSource->GetVideoWidth() * VideoSource->GetVideoHeight()); // TODO: avoid hardcoding of image height/width 133 | InitVideoMaterialTexture(); 134 | } 135 | 136 | void AVideoDisplaySurface::EndPlay(const EEndPlayReason::Type EndPlayReason) 137 | { 138 | VideoSource->Close(); 139 | } 140 | 141 | void AVideoDisplaySurface::Tick(float DeltaTime) 142 | { 143 | UpdateVideoFrame(); 144 | } 145 | 146 | void AVideoDisplaySurface::InitVideoMaterialTexture() 147 | { 148 | // First find the dynamic material 149 | TArray Primitives; 150 | this->GetComponents(Primitives); 151 | uint16 PrimitivesCount = Primitives.Num(); 152 | for (uint16 i = 0; i != PrimitivesCount; ++i) 153 | { 154 | UPrimitiveComponent* Primitive = Primitives[i]; 155 | uint16 MaterialsCount = Primitive->GetNumMaterials(); 156 | for (uint16 m = 0; m != MaterialsCount; ++m) 157 | { 158 | UMaterialInterface* Material = Primitive->GetMaterial(m); 159 | UTexture* Texture = nullptr; 160 | if (!Material) 161 | { 162 | continue; 163 | } 164 | 165 | if (Material->GetTextureParameterValue("VideoTexture", Texture)) 166 | { 167 | UMaterialInstanceDynamic* DynamicMaterialInstance = Cast(Material); 168 | if (!DynamicMaterialInstance) 169 | { 170 | DynamicMaterialInstance = UMaterialInstanceDynamic::Create(Material, Primitive); 171 | Primitive->SetMaterial(m, DynamicMaterialInstance); 172 | } 173 | VideoMaterial = DynamicMaterialInstance; 174 | // now that we have dynamic video material, create transient texture to draw video to, and set the material VideoTexture parameter 175 | VideoTexture = UTexture2D::CreateTransient(VideoSource->GetVideoWidth(), VideoSource->GetVideoHeight()); 176 | VideoTexture->UpdateResource(); 177 | VideoMaterial->SetTextureParameterValue(FName("VideoTexture"), VideoTexture); 178 | VideoTextureRegion = new FUpdateTextureRegion2D(0, 0, 0, 0, VideoSource->GetVideoWidth(), VideoSource->GetVideoHeight()); // Note: This never gets freed 179 | break; 180 | } 181 | } 182 | } 183 | } 184 | 185 | uint16 AVideoDisplaySurface::GetWidthToDistanceRatio() { 186 | return VideoSource->GetWidthToDistanceRatio(); 187 | } 188 | 189 | uint16 AVideoDisplaySurface::GetHeightToDistanceRatio() { 190 | return VideoSource->GetHeightToDistanceRatio(); 191 | } 192 | 193 | -------------------------------------------------------------------------------- /VideoDisplaySurface.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | /***************************** 3 | Copyright 2015 (c) Leonardo Malave. All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are 6 | permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this list of 9 | conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 12 | of conditions and the following disclaimer in the documentation and/or other materials 13 | provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY Leonardo Malave ''AS IS'' AND ANY EXPRESS OR IMPLIED 16 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 17 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Rafael Muñoz Salinas OR 18 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 21 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 22 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | The views and conclusions contained in the software and documentation are those of the 26 | authors and should not be interpreted as representing official policies, either expressed 27 | or implied, of Leonardo Malave. 28 | ********************************/ 29 | 30 | #pragma once 31 | 32 | #include "GameFramework/Actor.h" 33 | #include "IVideoSource.h" 34 | #include "VideoDisplaySurface.generated.h" 35 | 36 | /** 37 | * This is a surface (ShapePlane mesh) that can display video from a camera 38 | */ 39 | UCLASS() 40 | class AVideoDisplaySurface : public AActor 41 | { 42 | GENERATED_BODY() 43 | 44 | 45 | public: 46 | AVideoDisplaySurface(const class FPostConstructInitializeProperties& PCIP); 47 | 48 | // mesh to use 49 | UPROPERTY(VisibleDefaultsOnly, BlueprintReadOnly, Category = AugmentedReality) 50 | TSubobjectPtr VideoSurfaceMesh; 51 | 52 | UPROPERTY(VisibleAnywhere, BlueprintReadWrite, Category = AugmentedReality) 53 | float PreferredDistanceInMeters; 54 | 55 | 56 | UFUNCTION(BlueprintCallable, Category = AugmentedReality) 57 | uint16 GetWidthToDistanceRatio(); 58 | 59 | UFUNCTION(BlueprintCallable, Category = AugmentedReality) 60 | uint16 GetHeightToDistanceRatio(); 61 | 62 | void Init(IVideoSource* VideoSource); 63 | 64 | protected: 65 | 66 | UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = AugmentedReality) 67 | UTexture2D* VideoTexture; 68 | 69 | /** Update texture region from https://wiki.unrealengine.com/Dynamic_Textures */ 70 | void UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData); 71 | 72 | //UFUNCTION(BlueprintCallable, Category = AugmentedReality) 73 | //void CreateVideoTexture(); 74 | 75 | UFUNCTION(BlueprintCallable, Category = AugmentedReality) 76 | void UpdateVideoFrame(); 77 | 78 | virtual void Tick(float DeltaTime) override; 79 | 80 | virtual void BeginPlay() override; 81 | 82 | virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; 83 | 84 | void InitVideoMaterialTexture(); 85 | 86 | FVector GetWorldLocationFromPixelCoordinates(FVector2D PixelCoordinates); 87 | 88 | UPROPERTY() 89 | TArray VideoFrameData; 90 | 91 | FUpdateTextureRegion2D *VideoTextureRegion; 92 | 93 | UMaterialInstanceDynamic *VideoMaterial; 94 | 95 | IVideoSource* VideoSource; 96 | 97 | private: 98 | 99 | 100 | }; 101 | --------------------------------------------------------------------------------