diff --git a/Config/DefaultGame.ini b/Config/DefaultGame.ini index 875bc64b..45f237cd 100644 --- a/Config/DefaultGame.ini +++ b/Config/DefaultGame.ini @@ -222,9 +222,8 @@ VoiceChatVolumeControlBus=/Game/Audio/Modulation/ControlBuses/CB_VoiceChat.CB_Vo LoadingScreenControlBusMix=/Game/Audio/Modulation/ControlBusMixes/CBM_LoadingScreenMix.CBM_LoadingScreenMix [/Script/StyleTransfer.StyleTransferSettings] -StyleTransferNetwork=/StyleTransfer/NN_TransferVGG.NN_TransferVGG -StylePredictionNetwork=/StyleTransfer/NN_PredictorVGG.NN_PredictorVGG -+StyleTextures=/StyleTransfer/T_StyleImage.T_StyleImage -+StyleTextures=/StyleTransfer/T_StyleImage3.T_StyleImage3 +StyleTransferNetwork=/StyleTransfer/rstnet-960-32-3_transfer.rstnet-960-32-3_transfer +StylePredictionNetwork=/StyleTransfer/rstnet-960-32-3_predictor.rstnet-960-32-3_predictor ++StyleTextures=/StyleTransfer/Styles/T_StyleImage1.T_StyleImage1 InterpolationCurve=(EditorCurveData=(Keys=((InterpMode=RCIM_Cubic,TangentMode=RCTM_User),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=2.241476,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=3.908512,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=5.831762),(Time=7.971708)),DefaultValue=340282346638528859811704183484516925440.000000,PreInfinityExtrap=RCCE_Constant,PostInfinityExtrap=RCCE_Constant),ExternalCurve=None) diff --git a/Plugins/StyleTransfer/Content/Styles/00059bb14dffa5cb836f35e2f207f39946d25ff6.uasset b/Plugins/StyleTransfer/Content/Styles/00059bb14dffa5cb836f35e2f207f39946d25ff6.uasset new file mode 100644 index 00000000..5f7152c7 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/00059bb14dffa5cb836f35e2f207f39946d25ff6.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc37db87a7e53ebab6ae171cb7ca25b9ffc41ef4bcc6b47c4ab44028ca3811f0 +size 128402 diff --git a/Plugins/StyleTransfer/Content/Styles/0011fc013403f895146b783b2dd338d639883ec8.uasset b/Plugins/StyleTransfer/Content/Styles/0011fc013403f895146b783b2dd338d639883ec8.uasset new file mode 100644 index 00000000..e7fdf410 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/0011fc013403f895146b783b2dd338d639883ec8.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82a35670b287a564a18163010931b5d6d5685ad806adec33c158236eaf7b4d5c +size 75579 diff --git a/Plugins/StyleTransfer/Content/Styles/00217cb856d9d97865cd3c59669155d3b5c5c0cd.uasset b/Plugins/StyleTransfer/Content/Styles/00217cb856d9d97865cd3c59669155d3b5c5c0cd.uasset new file mode 100644 index 00000000..1fb52970 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/00217cb856d9d97865cd3c59669155d3b5c5c0cd.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cc2c44b8a0c981c3dfe336bdfc91001eea8cfc40c5de392db72d9c9296296c3 +size 53805 diff --git a/Plugins/StyleTransfer/Content/Styles/00218b41b034d31c8129a328879680c0771cd95d.uasset b/Plugins/StyleTransfer/Content/Styles/00218b41b034d31c8129a328879680c0771cd95d.uasset new file mode 100644 index 00000000..6996e072 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/00218b41b034d31c8129a328879680c0771cd95d.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bd5f7bba7e2aec049a4bb813d0d883e8d689fced11abe3e5d67797c6027a7a2 +size 113907 diff --git a/Plugins/StyleTransfer/Content/Styles/00230bf26630ee75ed5dfc8861047e1f13fbf38a.uasset b/Plugins/StyleTransfer/Content/Styles/00230bf26630ee75ed5dfc8861047e1f13fbf38a.uasset new file mode 100644 index 00000000..dcfb1d21 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/00230bf26630ee75ed5dfc8861047e1f13fbf38a.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0235c20236ea1e1f886e1c8a2ca07b10aaec9706ef15e78c7cb3459271895e95 +size 67732 diff --git a/Plugins/StyleTransfer/Content/Styles/T_StyleImage.uasset b/Plugins/StyleTransfer/Content/Styles/T_StyleImage.uasset new file mode 100644 index 00000000..61bf1890 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/T_StyleImage.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ec8ca0cf2079a3e350273817c81d7019ed09e48a53eb20d16dbf25873d0336d +size 2714570 diff --git a/Plugins/StyleTransfer/Content/Styles/T_StyleImage1.uasset b/Plugins/StyleTransfer/Content/Styles/T_StyleImage1.uasset new file mode 100644 index 00000000..f3000743 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/T_StyleImage1.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aa4678b74d868770be2df2e2a3a23af160d8a83567ad7da434b35c0791d0a73 +size 57385 diff --git a/Plugins/StyleTransfer/Content/Styles/T_StyleImage2.uasset b/Plugins/StyleTransfer/Content/Styles/T_StyleImage2.uasset new file mode 100644 index 00000000..c7b0d248 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/T_StyleImage2.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1573033e2f09b56d7bb17719f07201cb7d1003c199a86e031728542150ab0c0 +size 261452 diff --git a/Plugins/StyleTransfer/Content/Styles/T_StyleImage3.uasset b/Plugins/StyleTransfer/Content/Styles/T_StyleImage3.uasset new file mode 100644 index 00000000..94444c69 --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/T_StyleImage3.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8604757df7d819ea44cf974267a2f8c11e519023c00371892f039d192532e589 +size 224307 diff --git a/Plugins/StyleTransfer/Content/Styles/T_StyleImage4.uasset b/Plugins/StyleTransfer/Content/Styles/T_StyleImage4.uasset new file mode 100644 index 00000000..4c2962af --- /dev/null +++ b/Plugins/StyleTransfer/Content/Styles/T_StyleImage4.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc4f68e3f7e3bb842f4df58d244790511bb0e673b08357cf7c5e97204dace21c +size 53792 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage.uasset b/Plugins/StyleTransfer/Content/T_StyleImage.uasset index 0cb846b9..17f5efaf 100644 --- a/Plugins/StyleTransfer/Content/T_StyleImage.uasset +++ b/Plugins/StyleTransfer/Content/T_StyleImage.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:acbfb62d054055fd0d80e5f664a1525994296d41dfe3da89527deac01f2341bd -size 2714563 +oid sha256:227985c6eba3c1b58822fb5c3d49ad7ed56d07701cc28217dd4ff370505e4b94 +size 1382 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage1.uasset b/Plugins/StyleTransfer/Content/T_StyleImage1.uasset index 103232b9..2e2d88f3 100644 --- a/Plugins/StyleTransfer/Content/T_StyleImage1.uasset +++ b/Plugins/StyleTransfer/Content/T_StyleImage1.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:55cef5d1a3f9a44217d194f065be0078a44d7e50f6eaea2405470334231ac94a -size 57283 +oid sha256:18a6f98f6e4130d271fcecf86981a43e92c46ce3d76b4c0c865d8397052a10e2 +size 1389 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage2.uasset b/Plugins/StyleTransfer/Content/T_StyleImage2.uasset index 7f0c779d..f47e3b3b 100644 --- a/Plugins/StyleTransfer/Content/T_StyleImage2.uasset +++ b/Plugins/StyleTransfer/Content/T_StyleImage2.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44a327852dbeb78d075ed78f887d246dd0ef7b948d917a36f01a9a91b43b9076 -size 261350 +oid sha256:85cb8d0aeacae169534c57bdcbdd2a325c4b4ff1b2f309b45ce3d3ae19bd3df4 +size 1389 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage3.uasset b/Plugins/StyleTransfer/Content/T_StyleImage3.uasset index 33acf0f0..9e6fb32e 100644 --- a/Plugins/StyleTransfer/Content/T_StyleImage3.uasset +++ b/Plugins/StyleTransfer/Content/T_StyleImage3.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:21c65178a53ca1c298198c838810a6ec59c7c6bc7e49bd5caf340f5c3e9d0b9c -size 224205 +oid sha256:f0898935f5f7b922d5795e4b597de5c4ea58059469675eab5011fd5e7c964d73 +size 1389 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage4.uasset b/Plugins/StyleTransfer/Content/T_StyleImage4.uasset index 244d7af8..784c5b09 100644 --- a/Plugins/StyleTransfer/Content/T_StyleImage4.uasset +++ b/Plugins/StyleTransfer/Content/T_StyleImage4.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bea15c4a65a84b51de6f30970a42d8932c8370d2895af36878ef60a31f990a09 -size 53690 +oid sha256:19d898256a66a5106051ae1ec2deca1d5d174c9e857ef6a75cf4f55e858738ad +size 1389 diff --git a/Plugins/StyleTransfer/Content/rstnet-960-32-3_predictor.uasset b/Plugins/StyleTransfer/Content/rstnet-960-32-3_predictor.uasset new file mode 100644 index 00000000..9f5b2d7d --- /dev/null +++ b/Plugins/StyleTransfer/Content/rstnet-960-32-3_predictor.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f11391bf171ee1807eba0e08382d4c389795033f34d757cdc9b886fc2b31d3e5 +size 4313804 diff --git a/Plugins/StyleTransfer/Content/rstnet-960-32-3_transfer.uasset b/Plugins/StyleTransfer/Content/rstnet-960-32-3_transfer.uasset new file mode 100644 index 00000000..e7e2b433 --- /dev/null +++ b/Plugins/StyleTransfer/Content/rstnet-960-32-3_transfer.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39c56c1908629d7f0542990ceab0ce1ade48d0cef345448f7363d8920677d6e +size 602971 diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp index e9ed8add..847bb953 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp @@ -173,10 +173,8 @@ FRDGTexture* FStyleTransferSceneViewExtension::TensorToTexture(FRDGBuilder& Grap FRDGTexture* OutputTexture = GraphBuilder.CreateTexture( DestinationDesc, TEXT("OutputTexture")); - FRDGBufferRef SourceTensorBuffer = GraphBuilder.RegisterExternalBuffer(SourceTensor.GetPooledBuffer()); - auto OutputTensorToSceneColorParameters = GraphBuilder.AllocParameters(); - OutputTensorToSceneColorParameters->InputTensor = GraphBuilder.CreateSRV(SourceTensorBuffer, EPixelFormat::PF_R32_FLOAT); + OutputTensorToSceneColorParameters->InputTensor = SourceTensor.GetBufferSRVRef(); OutputTensorToSceneColorParameters->OutputTexture = GraphBuilder.CreateUAV(OutputTexture); OutputTensorToSceneColorParameters->TensorVolume = SourceTensor.Num(); OutputTensorToSceneColorParameters->TextureSize = DestinationDesc.Extent; @@ -219,10 +217,9 @@ FRDGTexture* TensorToTexture(FRDGBuilder& GraphBuilder, const FRDGTextureDesc& B FRDGTexture* OutputTexture = GraphBuilder.CreateTexture( DestinationDesc, TEXT("OutputTexture")); - FRDGBufferRef SourceTensorBuffer = GraphBuilder.RegisterExternalBuffer(SourceTensor.GetPooledBuffer()); auto OutputTensorToSceneColorParameters = GraphBuilder.AllocParameters(); - OutputTensorToSceneColorParameters->InputTensor = GraphBuilder.CreateSRV(SourceTensorBuffer, EPixelFormat::PF_R32_FLOAT); + OutputTensorToSceneColorParameters->InputTensor = SourceTensor.GetBufferSRVRef(); OutputTensorToSceneColorParameters->OutputTexture = GraphBuilder.CreateUAV(OutputTexture); OutputTensorToSceneColorParameters->TensorVolume = SourceTensor.Num(); OutputTensorToSceneColorParameters->TextureSize = DestinationDesc.Extent; @@ -246,7 +243,7 @@ FRDGTexture* TensorToTexture(FRDGBuilder& GraphBuilder, const FRDGTextureDesc& B return OutputTexture; } -void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) +FRDGPassRef TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, FNeuralTensor& DestinationTensor) { const FIntVector InputTensorDimensions = { CastNarrowingSafe(DestinationTensor.GetSize(1)), @@ -255,12 +252,11 @@ void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, }; const FIntPoint RgbRenderTargetDimensions = SourceTexture->Desc.Extent; - const FRDGBufferRef DestinationTensorBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); FSceneColorToInputTensorCS::FParameters* RgbToInputTensorParameters = GraphBuilder.AllocParameters(); RgbToInputTensorParameters->TensorVolume = CastNarrowingSafe(DestinationTensor.Num()); RgbToInputTensorParameters->InputTexture = SourceTexture; RgbToInputTensorParameters->InputTextureSampler = TStaticSamplerState::GetRHI(); - RgbToInputTensorParameters->OutputUAV = GraphBuilder.CreateUAV(DestinationTensorBuffer); + RgbToInputTensorParameters->OutputUAV = DestinationTensor.GetBufferUAVRef(); RgbToInputTensorParameters->OutputDimensions = {InputTensorDimensions.X, InputTensorDimensions.Y}; RgbToInputTensorParameters->HalfPixelUV = FVector2f(0.5f / RgbRenderTargetDimensions.X, 0.5 / RgbRenderTargetDimensions.Y); FIntVector ComputeGroupCount = FComputeShaderUtils::GetGroupCount( @@ -269,7 +265,7 @@ void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, ); TShaderMapRef RgbToInputTensorCS(GetGlobalShaderMap(GMaxRHIFeatureLevel)); - GraphBuilder.AddPass( + return GraphBuilder.AddPass( RDG_EVENT_NAME("TextureToTensorRGB(%s)", FSceneColorToInputTensorCS::StaticType.GetName()), RgbToInputTensorParameters, ERDGPassFlags::Compute, @@ -281,7 +277,7 @@ void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, ); } -void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) +FRDGPassRef TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, FNeuralTensor& DestinationTensor) { const FIntVector InputTensorDimensions = { CastNarrowingSafe(DestinationTensor.GetSize(1)), @@ -290,12 +286,11 @@ void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTe }; const FIntPoint GrayscaleRenderTargetDimensions = SourceTexture->Desc.Extent; - const FRDGBufferRef DestinationTensorBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); FShadowMaskToInputTensorCS::FParameters* GrayscaleToInputTensorParameters = GraphBuilder.AllocParameters(); GrayscaleToInputTensorParameters->TensorVolume = CastNarrowingSafe(DestinationTensor.Num()); GrayscaleToInputTensorParameters->InputTexture = SourceTexture; GrayscaleToInputTensorParameters->InputTextureSampler = TStaticSamplerState::GetRHI(); - GrayscaleToInputTensorParameters->OutputUAV = GraphBuilder.CreateUAV(DestinationTensorBuffer); + GrayscaleToInputTensorParameters->OutputUAV = DestinationTensor.GetBufferUAVRef(); GrayscaleToInputTensorParameters->OutputDimensions = {InputTensorDimensions.X, InputTensorDimensions.Y}; GrayscaleToInputTensorParameters->HalfPixelUV = FVector2f(0.5f / GrayscaleRenderTargetDimensions.X, 0.5 / GrayscaleRenderTargetDimensions.Y); FIntVector ComputeGroupCount = FComputeShaderUtils::GetGroupCount( @@ -304,7 +299,7 @@ void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTe ); TShaderMapRef GrayscaleToInputTensorCS(GetGlobalShaderMap(GMaxRHIFeatureLevel)); - GraphBuilder.AddPass( + return GraphBuilder.AddPass( RDG_EVENT_NAME("TextureToTensorGrayscale(%s)", FShadowMaskToInputTensorCS::StaticType.GetName()), GrayscaleToInputTensorParameters, ERDGPassFlags::Compute, @@ -316,24 +311,19 @@ void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTe ); } -void FStyleTransferSceneViewExtension::TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor) +void FStyleTransferSceneViewExtension::TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, FNeuralTensor& DestinationTensor) { ::TextureToTensorRGB(GraphBuilder, SourceTexture, DestinationTensor); } -void FStyleTransferSceneViewExtension::InterpolateTensors(FRDGBuilder& GraphBuilder, const FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha) +void FStyleTransferSceneViewExtension::InterpolateTensors(FRDGBuilder& GraphBuilder, FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha) { RDG_EVENT_SCOPE(GraphBuilder, "InterpolateTensors"); - const FRDGBufferRef DestinationBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); - const FRDGBufferRef InputBufferA = GraphBuilder.RegisterExternalBuffer(InputTensorA.GetPooledBuffer()); - const FRDGBufferRef InputBufferB = GraphBuilder.RegisterExternalBuffer(InputTensorB.GetPooledBuffer()); - - auto InterpolateTensorsParameters = GraphBuilder.AllocParameters(); - InterpolateTensorsParameters->InputSrvA = GraphBuilder.CreateSRV(InputBufferA, EPixelFormat::PF_R32_FLOAT); - InterpolateTensorsParameters->InputSrvB = GraphBuilder.CreateSRV(InputBufferB, EPixelFormat::PF_R32_FLOAT); - InterpolateTensorsParameters->OutputUAV = GraphBuilder.CreateUAV(DestinationBuffer); + InterpolateTensorsParameters->InputSrvA = InputTensorA.GetBufferSRVRef(); + InterpolateTensorsParameters->InputSrvB = InputTensorB.GetBufferSRVRef(); + InterpolateTensorsParameters->OutputUAV = DestinationTensor.GetBufferUAVRef(); InterpolateTensorsParameters->Alpha = Alpha; InterpolateTensorsParameters->TensorVolume = DestinationTensor.Num(); FIntVector InterpolateTensorsThreadGroupCount = FComputeShaderUtils::GetGroupCount( @@ -379,21 +369,27 @@ FScreenPassTexture FStyleTransferSceneViewExtension::PostProcessPassAfterTonemap checkSlow(View.bIsViewInfo); const FViewInfo& ViewInfo = static_cast(View); - const FNeuralTensor& StyleTransferContentInputTensor = StyleTransferNetwork->GetInputTensorForContext(*InferenceContext, ContentInputTensorIndex); + FNeuralTensor& StyleTransferContentInputTensor = StyleTransferNetwork->GetInputTensorForContextMutable(*InferenceContext, ContentInputTensorIndex); + FNeuralTensor& StyleTransferStyleParamsInputTensor = StyleTransferNetwork->GetInputTensorForContextMutable(*InferenceContext, StyleParamsInputTensorIndex); + + StyleTransferContentInputTensor.GPUToRDGBuilder_RenderThread(&GraphBuilder); + StyleTransferStyleParamsInputTensor.GPUToRDGBuilder_RenderThread(&GraphBuilder); if (StyleWeightsInputTensorIndex != INDEX_NONE) { - const FNeuralTensor& StyleTransferStyleWeightsInputTensor = StyleTransferNetwork->GetInputTensorForContext(*InferenceContext, StyleWeightsInputTensorIndex); + FNeuralTensor& StyleTransferStyleWeightsInputTensor = StyleTransferNetwork->GetInputTensorForContextMutable(*InferenceContext, StyleWeightsInputTensorIndex); + StyleTransferStyleWeightsInputTensor.GPUToRDGBuilder_RenderThread(&GraphBuilder); check(GScreenShadowMaskTexture); ::TextureToTensorGrayscale(GraphBuilder, GScreenShadowMaskTexture, StyleTransferStyleWeightsInputTensor); } - + ::TextureToTensorRGB(GraphBuilder, SceneColor.Texture, StyleTransferContentInputTensor); StyleTransferNetwork->Run(GraphBuilder, *InferenceContext); - const FNeuralTensor& StyleTransferContentOutputTensor = StyleTransferNetwork->GetOutputTensorForContext(*InferenceContext, 0); + + FNeuralTensor& StyleTransferContentOutputTensor = StyleTransferNetwork->GetOutputTensorForContextMutable(*InferenceContext, 0); FRDGTexture* StyleTransferRenderTargetTexture = TensorToTexture(GraphBuilder, SceneColor.Texture->Desc, StyleTransferContentOutputTensor); TSharedPtr StyleTransferOutputTarget = MakeShared(StyleTransferRenderTargetTexture, SceneColor.ViewRect, diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp index 0ab7689a..bb4b1f31 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp @@ -92,6 +92,7 @@ void UStyleTransferSubsystem::StartStylizingViewport(FViewportClient* ViewportCl FTextureCompilingManager::Get().FinishCompilation({StyleTexture}); #endif UpdateStyle(StyleTexture, i, StylePredictionInferenceContext); + //UpdateStyle(StyleTexture, i, StylePredictionInferenceContext); } //UpdateStyle(FPaths::GetPath("C:\\projects\\realtime-style-transfer\\temp\\style_params_tensor.bin")); UE_LOG(LogStyleTransfer, Log, TEXT("Creating FStyleTransferSceneViewExtension")); @@ -136,23 +137,26 @@ void UStyleTransferSubsystem::UpdateStyle(UTexture2D* StyleTexture, uint32 Style ENQUEUE_RENDER_COMMAND(StylePrediction)([this, StyleTexture, StylePredictionInferenceContext, StyleIndex](FRHICommandListImmediate& RHICommandList) { IRenderCaptureProvider* RenderCaptureProvider = ConditionalBeginRenderCapture(RHICommandList); - FRDGBuilder GraphBuilder(RHICommandList); { RDG_EVENT_SCOPE(GraphBuilder, "StylePrediction"); - const FNeuralTensor& InputStyleImageTensor = StylePredictionNetwork->GetInputTensorForContext(StylePredictionInferenceContext, 0); + FNeuralTensor& InputStyleImageTensor = StylePredictionNetwork->GetInputTensorForContextMutable(StylePredictionInferenceContext, 0); + FNeuralTensor& InputStyleParams = StyleTransferNetwork->GetInputTensorForContextMutable(*StyleTransferInferenceContext, StyleTransferStyleParamsInputIndex); + InputStyleImageTensor.GPUToRDGBuilder_RenderThread(&GraphBuilder); + InputStyleParams.GPUToRDGBuilder_RenderThread(&GraphBuilder); FTextureResource* StyleTextureResource = StyleTexture->GetResource(); FRDGTextureRef RDGStyleTexture = GraphBuilder.RegisterExternalTexture(CreateRenderTarget(StyleTextureResource->TextureRHI, TEXT("StyleInputTexture"))); FStyleTransferSceneViewExtension::TextureToTensorRGB(GraphBuilder, RDGStyleTexture, InputStyleImageTensor); + + StylePredictionNetwork->Run(GraphBuilder, StylePredictionInferenceContext); - const FNeuralTensor& OutputStyleParams = StylePredictionNetwork->GetOutputTensorForContext(StylePredictionInferenceContext, 0); - const FNeuralTensor& InputStyleParams = StyleTransferNetwork->GetInputTensorForContext(*StyleTransferInferenceContext, StyleTransferStyleParamsInputIndex); - FRDGBufferRef OutputStyleParamsBuffer = GraphBuilder.RegisterExternalBuffer(OutputStyleParams.GetPooledBuffer()); - FRDGBufferRef InputStyleParamsBuffer = GraphBuilder.RegisterExternalBuffer(InputStyleParams.GetPooledBuffer()); + FNeuralTensor& OutputStyleParams = StylePredictionNetwork->GetOutputTensorForContextMutable(StylePredictionInferenceContext, 0); + FRDGBufferRef OutputStyleParamsBuffer = OutputStyleParams.GetBufferSRVRef()->GetParent(); + FRDGBufferRef InputStyleParamsBuffer = InputStyleParams.GetBufferUAVRef()->GetParent(); const uint64 NumBytes = OutputStyleParams.NumInBytes(); const uint64 DstOffset = StyleIndex * NumBytes; @@ -162,12 +166,12 @@ void UStyleTransferSubsystem::UpdateStyle(UTexture2D* StyleTexture, uint32 Style Parameters->DstBuffer = InputStyleParamsBuffer; GraphBuilder.AddPass( - RDG_EVENT_NAME("CopyBuffer(%s Size=%ubytes)", Parameters->SrcBuffer, Parameters->SrcBuffer->Desc.GetSize()), + RDG_EVENT_NAME("CopyBuffer(%s Size=%ubytes)", Parameters->SrcBuffer->Name, Parameters->SrcBuffer->Desc.GetSize()), Parameters, ERDGPassFlags::Copy, [Parameters, NumBytes, DstOffset](FRHICommandList& RHICmdList) { - RHICmdList.CopyBufferRegion(Parameters->DstBuffer->GetRHI(), DstOffset, Parameters->SrcBuffer->GetRHI(), 0, NumBytes); + RHICmdList.CopyBufferRegion(Parameters->DstBuffer->GetRHI(), /*DstOffset*/ 0, Parameters->SrcBuffer->GetRHI(), 0, NumBytes); }); } GraphBuilder.Execute(); @@ -185,7 +189,7 @@ void UStyleTransferSubsystem::UpdateStyle(FString StyleTensorDataPath) { FArchive& FileReader = *IFileManager::Get().CreateFileReader(*StyleTensorDataPath); TArray StyleParams; - StyleParams.SetNumUninitialized(2758); + StyleParams.SetNumUninitialized(2758); // hardcoded for brevity reasons FileReader << StyleParams; @@ -268,9 +272,12 @@ void UStyleTransferSubsystem::InterpolateStyles(int32 StylePredictionInferenceCo { RDG_EVENT_SCOPE(GraphBuilder, "StylePrediction"); - const FNeuralTensor& InputStyleImageTensorA = StylePredictionNetwork->GetOutputTensorForContext(StylePredictionInferenceContextA, 0); - const FNeuralTensor& InputStyleImageTensorB = StylePredictionNetwork->GetOutputTensorForContext(StylePredictionInferenceContextB, 0); - const FNeuralTensor& OutputStyleParamsTensor = StyleTransferNetwork->GetInputTensorForContext(*StyleTransferInferenceContext, StyleTransferStyleParamsInputIndex); + FNeuralTensor& InputStyleImageTensorA = StylePredictionNetwork->GetOutputTensorForContextMutable(StylePredictionInferenceContextA, 0); + FNeuralTensor& InputStyleImageTensorB = StylePredictionNetwork->GetOutputTensorForContextMutable(StylePredictionInferenceContextB, 0); + FNeuralTensor& OutputStyleParamsTensor = StyleTransferNetwork->GetInputTensorForContextMutable(*StyleTransferInferenceContext, StyleTransferStyleParamsInputIndex); + InputStyleImageTensorA.GPUToRDGBuilder_RenderThread(&GraphBuilder); + InputStyleImageTensorB.GPUToRDGBuilder_RenderThread(&GraphBuilder); + OutputStyleParamsTensor.GPUToRDGBuilder_RenderThread(&GraphBuilder); FStyleTransferSceneViewExtension::InterpolateTensors(GraphBuilder, OutputStyleParamsTensor, InputStyleImageTensorA, InputStyleImageTensorB, Alpha); } GraphBuilder.Execute(); diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h index 5fadbc37..26e4f700 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h @@ -42,9 +42,9 @@ public: static void AddRescalingTextureCopy(FRDGBuilder& GraphBuilder, FRDGTexture& RDGSourceTexture, FScreenPassRenderTarget& DestinationRenderTarget); static FRDGTexture* TensorToTexture(FRDGBuilder& GraphBuilder, const FRDGTextureDesc& BaseDestinationDesc, const FNeuralTensor& SourceTensor); - static void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); - static void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); - static void InterpolateTensors(FRDGBuilder& GraphBuilder, const FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha); + static void TextureToTensorRGB(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, FNeuralTensor& DestinationTensor); + static void TextureToTensorGrayscale(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, FNeuralTensor& DestinationTensor); + static void InterpolateTensors(FRDGBuilder& GraphBuilder, FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha); private: /** The actual Network pointer is not tracked so we need a WeakPtr too so we can check its validity on the game thread. */