diff --git a/Config/DefaultEngine.ini b/Config/DefaultEngine.ini index 48382cfb..cd216709 100644 --- a/Config/DefaultEngine.ini +++ b/Config/DefaultEngine.ini @@ -49,6 +49,7 @@ gc.PendingKillEnabled=False ;LogEOSSdk=VeryVerbose ;LogHandshake=VeryVerbose LogHotfixManager=Log +LogNeuralNetworkInference=VeryVerbose [/Script/Engine.Player] ; These numbers should match TotalNetBandwidth @@ -313,3 +314,9 @@ ManualIPAddress= [Internationalization] +LocalizationPaths=%GAMEDIR%Content/Localization/EngineOverrides + + +[Engine.BufferVisualizationMaterials] +ViewTangent=(Material="/StyleTransfer/ViewTangent.ViewTangent", Name=LOCTEXT("ViewTangentMat", "View Tangent")) +ViewNormal=(Material="/StyleTransfer/ViewNormal.ViewNormal", Name=LOCTEXT("ViewNormalMat", "View Normal")) +ShadowMask=(Material="/StyleTransfer/ShadowMask.ShadowMask", Name=LOCTEXT("ShadowMaskMat", "Shadow Mask")) \ No newline at end of file diff --git a/Lyra.uproject b/Lyra.uproject index c58664b4..6c9f5e56 100644 --- a/Lyra.uproject +++ b/Lyra.uproject @@ -149,7 +149,8 @@ "SupportedTargetPlatforms": [ "Win64", "Linux", - "Android" + "Android", + "HoloLens" ] }, { diff --git a/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset b/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset index 1cd0d566..f127784f 100644 --- a/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset +++ b/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b7cb605bc4872bcfc4805f09a3072211029114bd2001a70bb431c2757d4261d4 -size 5128074 +oid sha256:f5a9eefb2868e25ca28e258c0a1203352e1a28b19536e161b0675f186b5a1884 +size 5128046 diff --git a/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset b/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset index ecbbba0e..ce79e311 100644 --- a/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset +++ b/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c244fa1e3b61fdd0a8b1ece6e772770bbad766e0a11408734b725811225a6c6b -size 6888931 +oid sha256:6de1ba831645f534c67cee88ba1b38b6704fd7226172e5ee215daf2512601786 +size 6828095 diff --git a/Plugins/StyleTransfer/Content/ShadowMask.uasset b/Plugins/StyleTransfer/Content/ShadowMask.uasset new file mode 100644 index 00000000..7f42f93f --- /dev/null +++ b/Plugins/StyleTransfer/Content/ShadowMask.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af7fcc272c8b8ac4f276f4eb9a1d2755af119b66ae6b19a297a4e6b70467e74d +size 6547 diff --git a/Plugins/StyleTransfer/Content/ViewNormal.uasset b/Plugins/StyleTransfer/Content/ViewNormal.uasset new file mode 100644 index 00000000..f89fb31b --- /dev/null +++ b/Plugins/StyleTransfer/Content/ViewNormal.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3854ba3c1b3aebe3c667c21e0527b9389b2f804922cfc96e036a8aaf59107df9 +size 11846 diff --git a/Plugins/StyleTransfer/Content/ViewTangent.uasset b/Plugins/StyleTransfer/Content/ViewTangent.uasset new file mode 100644 index 00000000..d1c51ca5 --- /dev/null +++ b/Plugins/StyleTransfer/Content/ViewTangent.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae4c4649dada7a39151409539b4dce3c1a2d050f28efa4a5be0a08fef47e6aae +size 11837 diff --git a/Plugins/StyleTransfer/Shaders/Private/OutputTensorToSceneColor.usf b/Plugins/StyleTransfer/Shaders/Private/OutputTensorToSceneColor.usf index e5d8f5cf..450f3d58 100644 --- a/Plugins/StyleTransfer/Shaders/Private/OutputTensorToSceneColor.usf +++ b/Plugins/StyleTransfer/Shaders/Private/OutputTensorToSceneColor.usf @@ -1,11 +1,11 @@ // Copyright 2022 Manuel Wagner - All rights reserved -// this assumes that the OutputTexture has -// the exact same dimensions as InputTensor! -uint TensorVolume; -uint2 TextureSize; RWTexture2D OutputTexture; Buffer InputTensor; +uint TensorVolume; +// this assumes that the OutputTexture has +// the exact same dimensions as InputTensor! +uint2 TextureSize; // DispatchThreadID corresponds to InputTensor shape dimensions not texture XY -> DispatchThreadID.X = Texture.Y [numthreads(THREADGROUP_SIZE_X, THREADGROUP_SIZE_Y, THREADGROUP_SIZE_Z)] diff --git a/Plugins/StyleTransfer/Shaders/Private/SceneColorToInputTensor.usf b/Plugins/StyleTransfer/Shaders/Private/SceneColorToInputTensor.usf index 3f1881ab..cacdc668 100644 --- a/Plugins/StyleTransfer/Shaders/Private/SceneColorToInputTensor.usf +++ b/Plugins/StyleTransfer/Shaders/Private/SceneColorToInputTensor.usf @@ -15,7 +15,7 @@ void SceneColorToInputTensorCS(in const uint3 DispatchThreadID : SV_DispatchThre return; } - const uint GlobalIndex = ((OutputUAVTexelCoordinate.x * OutputDimensions.y + OutputUAVTexelCoordinate.y)*3); + const uint GlobalIndex = ((OutputUAVTexelCoordinate.x * OutputDimensions.y + OutputUAVTexelCoordinate.y)*3); // note that the OutputUAV has shape (1, Y, X, C) // which is why we need to flip the indexing diff --git a/Plugins/StyleTransfer/Shaders/Private/ShadowMaskToInputTensor.usf b/Plugins/StyleTransfer/Shaders/Private/ShadowMaskToInputTensor.usf index cf3aa12a..31b0177e 100644 --- a/Plugins/StyleTransfer/Shaders/Private/ShadowMaskToInputTensor.usf +++ b/Plugins/StyleTransfer/Shaders/Private/ShadowMaskToInputTensor.usf @@ -15,7 +15,7 @@ void ShadowMaskToInputTensorCS(in const uint3 DispatchThreadID : SV_DispatchThre return; } - const uint GlobalIndex = ((OutputUAVTexelCoordinate.x * OutputDimensions.y + OutputUAVTexelCoordinate.y)); + const uint GlobalIndex = OutputUAVTexelCoordinate.x * OutputDimensions.y + OutputUAVTexelCoordinate.y; // note that the OutputUAV has shape (1, Y, X, C) // which is why we need to flip the indexing diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp index e3ae5ed9..d07fc786 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp @@ -38,11 +38,11 @@ TAutoConsoleVariable CVarAutoCaptureStyleTransfer( template OutType CastNarrowingSafe(InType InValue) { - if (InValue > TNumericLimits::Max()) + if (!ensure(InValue <= TNumericLimits::Max())) { return TNumericLimits::Max(); } - if (InValue < TNumericLimits::Min()) + if (!ensure(InValue >= TNumericLimits::Min())) { return TNumericLimits::Min(); } @@ -247,54 +247,79 @@ FRDGTexture* TensorToTexture(FRDGBuilder& GraphBuilder, const FRDGTextureDesc& B return OutputTexture; } -template -void TextureToTensor(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) +void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) { const FIntVector InputTensorDimensions = { CastNarrowingSafe(DestinationTensor.GetSize(1)), CastNarrowingSafe(DestinationTensor.GetSize(2)), CastNarrowingSafe(DestinationTensor.GetSize(3)), }; - const FIntPoint SceneColorRenderTargetDimensions = SourceTexture->Desc.Extent; + const FIntPoint RgbRenderTargetDimensions = SourceTexture->Desc.Extent; - FRDGBufferRef StyleTransferContentInputBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); - typename ShaderType::FParameters* SceneColorToInputTensorParameters = GraphBuilder.AllocParameters(); - SceneColorToInputTensorParameters->TensorVolume = CastNarrowingSafe(DestinationTensor.Num()); - SceneColorToInputTensorParameters->InputTexture = SourceTexture; - SceneColorToInputTensorParameters->InputTextureSampler = TStaticSamplerState::GetRHI(); - SceneColorToInputTensorParameters->OutputUAV = GraphBuilder.CreateUAV(StyleTransferContentInputBuffer); - SceneColorToInputTensorParameters->OutputDimensions = {InputTensorDimensions.X, InputTensorDimensions.Y}; - SceneColorToInputTensorParameters->HalfPixelUV = FVector2f(0.5f / SceneColorRenderTargetDimensions.X, 0.5 / SceneColorRenderTargetDimensions.Y); - FIntVector SceneColorToInputTensorGroupCount = FComputeShaderUtils::GetGroupCount( + const FRDGBufferRef DestinationTensorBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); + FSceneColorToInputTensorCS::FParameters* RgbToInputTensorParameters = GraphBuilder.AllocParameters(); + RgbToInputTensorParameters->TensorVolume = CastNarrowingSafe(DestinationTensor.Num()); + RgbToInputTensorParameters->InputTexture = SourceTexture; + RgbToInputTensorParameters->InputTextureSampler = TStaticSamplerState::GetRHI(); + RgbToInputTensorParameters->OutputUAV = GraphBuilder.CreateUAV(DestinationTensorBuffer); + RgbToInputTensorParameters->OutputDimensions = {InputTensorDimensions.X, InputTensorDimensions.Y}; + RgbToInputTensorParameters->HalfPixelUV = FVector2f(0.5f / RgbRenderTargetDimensions.X, 0.5 / RgbRenderTargetDimensions.Y); + FIntVector ComputeGroupCount = FComputeShaderUtils::GetGroupCount( {InputTensorDimensions.X, InputTensorDimensions.Y, 1}, - ShaderType::ThreadGroupSize + FSceneColorToInputTensorCS::ThreadGroupSize ); - TShaderMapRef SceneColorToInputTensorCS(GetGlobalShaderMap(GMaxRHIFeatureLevel)); - + TShaderMapRef RgbToInputTensorCS(GetGlobalShaderMap(GMaxRHIFeatureLevel)); GraphBuilder.AddPass( - RDG_EVENT_NAME("TextureToTensor(%s)", ShaderType::StaticType.GetName()), - SceneColorToInputTensorParameters, + RDG_EVENT_NAME("TextureToTensorRGB(%s)", FSceneColorToInputTensorCS::StaticType.GetName()), + RgbToInputTensorParameters, ERDGPassFlags::Compute, - [SceneColorToInputTensorCS, SceneColorToInputTensorParameters, SceneColorToInputTensorGroupCount](FRHICommandList& RHICommandList) + [RgbToInputTensorCS, RgbToInputTensorParameters, ComputeGroupCount](FRHICommandList& RHICommandList) { - FComputeShaderUtils::Dispatch(RHICommandList, SceneColorToInputTensorCS, - *SceneColorToInputTensorParameters, SceneColorToInputTensorGroupCount); + FComputeShaderUtils::Dispatch(RHICommandList, RgbToInputTensorCS, + *RgbToInputTensorParameters, ComputeGroupCount); } ); } -template -void TextureToTensor(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); -template <> -void TextureToTensor<1>(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor){return TextureToTensor(GraphBuilder, SourceTexture, DestinationTensor);} -template <> -void TextureToTensor<3>(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor){return TextureToTensor(GraphBuilder, SourceTexture, DestinationTensor);} - - -void FStyleTransferSceneViewExtension::TextureToTensor(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) +void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) { - ::TextureToTensor<3>(GraphBuilder, SourceTexture, DestinationTensor); + const FIntVector InputTensorDimensions = { + CastNarrowingSafe(DestinationTensor.GetSize(1)), + CastNarrowingSafe(DestinationTensor.GetSize(2)), + CastNarrowingSafe(DestinationTensor.GetSize(3)), + }; + const FIntPoint GrayscaleRenderTargetDimensions = SourceTexture->Desc.Extent; + + const FRDGBufferRef DestinationTensorBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); + FShadowMaskToInputTensorCS::FParameters* GrayscaleToInputTensorParameters = GraphBuilder.AllocParameters(); + GrayscaleToInputTensorParameters->TensorVolume = CastNarrowingSafe(DestinationTensor.Num()); + GrayscaleToInputTensorParameters->InputTexture = SourceTexture; + GrayscaleToInputTensorParameters->InputTextureSampler = TStaticSamplerState::GetRHI(); + GrayscaleToInputTensorParameters->OutputUAV = GraphBuilder.CreateUAV(DestinationTensorBuffer); + GrayscaleToInputTensorParameters->OutputDimensions = {InputTensorDimensions.X, InputTensorDimensions.Y}; + GrayscaleToInputTensorParameters->HalfPixelUV = FVector2f(0.5f / GrayscaleRenderTargetDimensions.X, 0.5 / GrayscaleRenderTargetDimensions.Y); + FIntVector ComputeGroupCount = FComputeShaderUtils::GetGroupCount( + {InputTensorDimensions.X, InputTensorDimensions.Y, 1}, + FShadowMaskToInputTensorCS::ThreadGroupSize + ); + + TShaderMapRef GrayscaleToInputTensorCS(GetGlobalShaderMap(GMaxRHIFeatureLevel)); + GraphBuilder.AddPass( + RDG_EVENT_NAME("TextureToTensorGrayscale(%s)", FShadowMaskToInputTensorCS::StaticType.GetName()), + GrayscaleToInputTensorParameters, + ERDGPassFlags::Compute, + [GrayscaleToInputTensorCS, GrayscaleToInputTensorParameters, ComputeGroupCount](FRHICommandList& RHICommandList) + { + FComputeShaderUtils::Dispatch(RHICommandList, GrayscaleToInputTensorCS, + *GrayscaleToInputTensorParameters, ComputeGroupCount); + } + ); +} + +void FStyleTransferSceneViewExtension::TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) +{ + ::TextureToTensorRGB(GraphBuilder, SourceTexture, DestinationTensor); } void FStyleTransferSceneViewExtension::InterpolateTensors(FRDGBuilder& GraphBuilder, const FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha) @@ -359,10 +384,9 @@ FScreenPassTexture FStyleTransferSceneViewExtension::PostProcessPassAfterTonemap const FNeuralTensor& StyleTransferStyleWeightsInputTensor = StyleTransferNetwork->GetInputTensorForContext(*InferenceContext, StyleWeightsInputTensorIndex); check(GScreenShadowMaskTexture); - ::TextureToTensor<1>(GraphBuilder, GScreenShadowMaskTexture, StyleTransferStyleWeightsInputTensor); - GScreenShadowMaskTexture = nullptr; + ::TextureToTensorGrayscale(GraphBuilder, GScreenShadowMaskTexture, StyleTransferStyleWeightsInputTensor); - ::TextureToTensor<3>(GraphBuilder, SceneColor.Texture, StyleTransferContentInputTensor); + ::TextureToTensorRGB(GraphBuilder, SceneColor.Texture, StyleTransferContentInputTensor); StyleTransferNetwork->Run(GraphBuilder, *InferenceContext); diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp index 56ecb378..0ab7689a 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp @@ -144,7 +144,7 @@ void UStyleTransferSubsystem::UpdateStyle(UTexture2D* StyleTexture, uint32 Style const FNeuralTensor& InputStyleImageTensor = StylePredictionNetwork->GetInputTensorForContext(StylePredictionInferenceContext, 0); FTextureResource* StyleTextureResource = StyleTexture->GetResource(); FRDGTextureRef RDGStyleTexture = GraphBuilder.RegisterExternalTexture(CreateRenderTarget(StyleTextureResource->TextureRHI, TEXT("StyleInputTexture"))); - FStyleTransferSceneViewExtension::TextureToTensor(GraphBuilder, RDGStyleTexture, InputStyleImageTensor); + FStyleTransferSceneViewExtension::TextureToTensorRGB(GraphBuilder, RDGStyleTexture, InputStyleImageTensor); StylePredictionNetwork->Run(GraphBuilder, StylePredictionInferenceContext); diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h index e90eb7c5..5fadbc37 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h @@ -42,7 +42,8 @@ public: static void AddRescalingTextureCopy(FRDGBuilder& GraphBuilder, FRDGTexture& RDGSourceTexture, FScreenPassRenderTarget& DestinationRenderTarget); static FRDGTexture* TensorToTexture(FRDGBuilder& GraphBuilder, const FRDGTextureDesc& BaseDestinationDesc, const FNeuralTensor& SourceTensor); - static void TextureToTensor(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); + static void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); + static void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); static void InterpolateTensors(FRDGBuilder& GraphBuilder, const FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha); private: