diff --git a/Config/DefaultEngine.ini b/Config/DefaultEngine.ini index 7ce63a6a..48382cfb 100644 --- a/Config/DefaultEngine.ini +++ b/Config/DefaultEngine.ini @@ -36,7 +36,7 @@ ScriptStackOnWarnings=true GlobalDefaultGameMode=/Game/B_LyraGameMode.B_LyraGameMode_C GameInstanceClass=/Game/B_LyraGameInstance.B_LyraGameInstance_C GameDefaultMap=/Game/System/FrontEnd/Maps/L_LyraFrontEnd.L_LyraFrontEnd -EditorStartupMap=/Game/System/DefaultEditorMap/L_DefaultEditorOverview.L_DefaultEditorOverview +EditorStartupMap=/ShooterMaps/Maps/L_Expanse.L_Expanse [/Script/Hotfix.OnlineHotfixManager] HotfixManagerClassName=/Script/LyraGame.LyraHotfixManager diff --git a/Config/DefaultGame.ini b/Config/DefaultGame.ini index 089336bf..875bc64b 100644 --- a/Config/DefaultGame.ini +++ b/Config/DefaultGame.ini @@ -221,3 +221,10 @@ VoiceChatVolumeControlBus=/Game/Audio/Modulation/ControlBuses/CB_VoiceChat.CB_Vo +LDRAudioSubmixEffectChain=(Submix="/Game/Audio/Submixes/MainSubmix.MainSubmix",SubmixEffectChain=("/Game/Audio/DYN_LowMultibandDynamics.DYN_LowMultibandDynamics","/Game/Audio/Effects/SubmixEffects/DYN_LowDynamics.DYN_LowDynamics")) LoadingScreenControlBusMix=/Game/Audio/Modulation/ControlBusMixes/CBM_LoadingScreenMix.CBM_LoadingScreenMix +[/Script/StyleTransfer.StyleTransferSettings] +StyleTransferNetwork=/StyleTransfer/NN_TransferVGG.NN_TransferVGG +StylePredictionNetwork=/StyleTransfer/NN_PredictorVGG.NN_PredictorVGG ++StyleTextures=/StyleTransfer/T_StyleImage.T_StyleImage ++StyleTextures=/StyleTransfer/T_StyleImage3.T_StyleImage3 +InterpolationCurve=(EditorCurveData=(Keys=((InterpMode=RCIM_Cubic,TangentMode=RCTM_User),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=2.241476,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=3.908512,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=5.831762),(Time=7.971708)),DefaultValue=340282346638528859811704183484516925440.000000,PreInfinityExtrap=RCCE_Constant,PostInfinityExtrap=RCCE_Constant),ExternalCurve=None) + diff --git a/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset b/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset new file mode 100644 index 00000000..10863975 --- /dev/null +++ b/Plugins/StyleTransfer/Content/NN_PredictorVGG.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e8c479db0bc4d2c91f7a17ba9a82e544068365c77054a15d53ae47a2e52b825 +size 5128046 diff --git a/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset b/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset new file mode 100644 index 00000000..44b85ead --- /dev/null +++ b/Plugins/StyleTransfer/Content/NN_TransferVGG.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ca79b4a94ae300ca98a699ce3ee2e1d6025b418cc83ff6fe48671e564b2b2ee +size 6797742 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage1.uasset b/Plugins/StyleTransfer/Content/T_StyleImage1.uasset new file mode 100644 index 00000000..103232b9 --- /dev/null +++ b/Plugins/StyleTransfer/Content/T_StyleImage1.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55cef5d1a3f9a44217d194f065be0078a44d7e50f6eaea2405470334231ac94a +size 57283 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage2.uasset b/Plugins/StyleTransfer/Content/T_StyleImage2.uasset new file mode 100644 index 00000000..7f0c779d --- /dev/null +++ b/Plugins/StyleTransfer/Content/T_StyleImage2.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44a327852dbeb78d075ed78f887d246dd0ef7b948d917a36f01a9a91b43b9076 +size 261350 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage3.uasset b/Plugins/StyleTransfer/Content/T_StyleImage3.uasset new file mode 100644 index 00000000..33acf0f0 --- /dev/null +++ b/Plugins/StyleTransfer/Content/T_StyleImage3.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21c65178a53ca1c298198c838810a6ec59c7c6bc7e49bd5caf340f5c3e9d0b9c +size 224205 diff --git a/Plugins/StyleTransfer/Content/T_StyleImage4.uasset b/Plugins/StyleTransfer/Content/T_StyleImage4.uasset new file mode 100644 index 00000000..244d7af8 --- /dev/null +++ b/Plugins/StyleTransfer/Content/T_StyleImage4.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bea15c4a65a84b51de6f30970a42d8932c8370d2895af36878ef60a31f990a09 +size 53690 diff --git a/Plugins/StyleTransfer/Shaders/Private/InterpolateTensors.usf b/Plugins/StyleTransfer/Shaders/Private/InterpolateTensors.usf new file mode 100644 index 00000000..6ea14b8a --- /dev/null +++ b/Plugins/StyleTransfer/Shaders/Private/InterpolateTensors.usf @@ -0,0 +1,21 @@ +// Copyright 2022 Manuel Wagner - All rights reserved + +RWBuffer OutputUAV; +Buffer InputSrvA; +Buffer InputSrvB; +uint TensorVolume; +float Alpha; + +[numthreads(THREADGROUP_SIZE_X, THREADGROUP_SIZE_Y, THREADGROUP_SIZE_Z)] +void InterpolateTensorsCS(in const uint3 DispatchThreadID : SV_DispatchThreadID) +{ + const uint Index = DispatchThreadID.x; + if (Index >= TensorVolume) + { + return; + } + + OutputUAV[Index] = lerp(InputSrvA[Index], InputSrvB[Index], Alpha); +} + +#include "/Engine/Public/Platform.ush" \ No newline at end of file diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferModule.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferModule.cpp index 8623dfe7..3df6b3e7 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferModule.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferModule.cpp @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. #include "StyleTransferModule.h" diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp index d3c13d56..b2aed579 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSceneViewExtension.cpp @@ -12,6 +12,7 @@ #include "SceneView.h" #include "ScreenPass.h" #include "CommonRenderResources.h" +#include "InterpolateTensorsCS.h" #include "IPixWinPlugin.h" #include "IPixWinPlugin.h" #include "IRenderCaptureProvider.h" @@ -215,6 +216,39 @@ void FStyleTransferSceneViewExtension::TextureToTensor(FRDGBuilder& GraphBuilder ); } +void FStyleTransferSceneViewExtension::InterpolateTensors(FRDGBuilder& GraphBuilder, const FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha) +{ + RDG_EVENT_SCOPE(GraphBuilder, "InterpolateTensors"); + + const FRDGBufferRef DestinationBuffer = GraphBuilder.RegisterExternalBuffer(DestinationTensor.GetPooledBuffer()); + const FRDGBufferRef InputBufferA = GraphBuilder.RegisterExternalBuffer(InputTensorA.GetPooledBuffer()); + const FRDGBufferRef InputBufferB = GraphBuilder.RegisterExternalBuffer(InputTensorB.GetPooledBuffer()); + + + auto InterpolateTensorsParameters = GraphBuilder.AllocParameters(); + InterpolateTensorsParameters->InputSrvA = GraphBuilder.CreateSRV(InputBufferA, EPixelFormat::PF_R32_FLOAT); + InterpolateTensorsParameters->InputSrvB = GraphBuilder.CreateSRV(InputBufferB, EPixelFormat::PF_R32_FLOAT); + InterpolateTensorsParameters->OutputUAV = GraphBuilder.CreateUAV(DestinationBuffer); + InterpolateTensorsParameters->Alpha = Alpha; + InterpolateTensorsParameters->TensorVolume = DestinationTensor.Num(); + FIntVector InterpolateTensorsThreadGroupCount = FComputeShaderUtils::GetGroupCount( + {CastNarrowingSafe(DestinationTensor.Num()), 1, 1}, + FInterpolateTensorsCS::ThreadGroupSize + ); + + TShaderMapRef InterpolateTensorsCS(GetGlobalShaderMap(GMaxRHIFeatureLevel)); + GraphBuilder.AddPass( + RDG_EVENT_NAME("InterpolateTensors"), + InterpolateTensorsParameters, + ERDGPassFlags::Compute, + [InterpolateTensorsCS, InterpolateTensorsParameters, InterpolateTensorsThreadGroupCount](FRHICommandList& RHICommandList) + { + FComputeShaderUtils::Dispatch(RHICommandList, InterpolateTensorsCS, + *InterpolateTensorsParameters, InterpolateTensorsThreadGroupCount); + } + ); +} + FScreenPassTexture FStyleTransferSceneViewExtension::PostProcessPassAfterTonemap_RenderThread(FRDGBuilder& GraphBuilder, const FSceneView& View, const FPostProcessMaterialInputs& InOutInputs) { const FScreenPassTexture& SceneColor = InOutInputs.Textures[(uint32)EPostProcessMaterialInput::SceneColor]; diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSettings.h b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSettings.h index bc30c3aa..acb6af92 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSettings.h +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSettings.h @@ -24,5 +24,8 @@ public: TSoftObjectPtr StylePredictionNetwork = nullptr; UPROPERTY(EditAnywhere, Config) - TSoftObjectPtr StyleTexture = nullptr; + TArray> StyleTextures; + + UPROPERTY(EditAnywhere, Config) + FRuntimeFloatCurve InterpolationCurve; }; diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp index 5cdc2000..1b21b4a1 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp @@ -19,6 +19,12 @@ TAutoConsoleVariable CVarStyleTransferEnabled( TEXT("Set to true to enable style transfer") ); +TAutoConsoleVariable CVarAutoCaptureStylePrediction( + TEXT("r.StyleTransfer.AutoCapturePrediction"), + false, + TEXT("Set to true to enable style transfer auto capture for profiling in PIX etc.") +); + void UStyleTransferSubsystem::Initialize(FSubsystemCollectionBase& Collection) { @@ -34,6 +40,26 @@ void UStyleTransferSubsystem::Deinitialize() Super::Deinitialize(); } +bool UStyleTransferSubsystem::Tick(float DeltaTime) +{ + if (!GetWorld()) + return true; + + + if (StylePredictionInferenceContexts.Num() > 1) + { + const UStyleTransferSettings* StyleTransferSettings = GetDefault(); + const FRichCurve* InterpCurve = StyleTransferSettings->InterpolationCurve.GetRichCurveConst(); + float MinTime, MaxTime; + InterpCurve->GetTimeRange(MinTime, MaxTime); + const double Time = MinTime + FMath::Fmod(GetWorld()->GetTimeSeconds(), static_cast(MaxTime - MinTime)); + const float Alpha = InterpCurve->Eval(Time); + UE_LOG(LogStyleTransfer, VeryVerbose, TEXT("Alpha is %0.4f"), Alpha); + InterpolateStyles(StylePredictionInferenceContexts[0], StylePredictionInferenceContexts[1], Alpha); + } + return true; +} + void UStyleTransferSubsystem::StartStylizingViewport(FViewportClient* ViewportClient) { if (!StylePredictionNetwork->IsLoaded() || !StyleTransferNetwork->IsLoaded()) @@ -45,20 +71,27 @@ void UStyleTransferSubsystem::StartStylizingViewport(FViewportClient* ViewportCl if (!StyleTransferSceneViewExtension) { const UStyleTransferSettings* StyleTransferSettings = GetDefault(); - StylePredictionInferenceContext = StylePredictionNetwork->CreateInferenceContext(); - checkf(StylePredictionInferenceContext != INDEX_NONE, TEXT("Could not create inference context for StylePredictionNetwork")); - StyleTransferInferenceContext = MakeShared(StyleTransferNetwork->CreateInferenceContext()); - checkf(*StyleTransferInferenceContext != INDEX_NONE, TEXT("Could not create inference context for StyleTransferNetwork")); - UTexture2D* StyleTexture = StyleTransferSettings->StyleTexture.LoadSynchronous(); - //UTexture2D* StyleTexture = LoadObject(this, TEXT("/Script/Engine.Texture2D'/StyleTransfer/T_StyleImage.T_StyleImage'")); + if (!StyleTransferInferenceContext || *StyleTransferInferenceContext == INDEX_NONE) + { + StyleTransferInferenceContext = MakeShared(StyleTransferNetwork->CreateInferenceContext()); + checkf(*StyleTransferInferenceContext != INDEX_NONE, TEXT("Could not create inference context for StyleTransferNetwork")); + } + + for (int32 i = 0; i < FMath::Min(2, StyleTransferSettings->StyleTextures.Num()); ++i) + { + const int32& StylePredictionInferenceContext = StylePredictionInferenceContexts.Emplace_GetRef(StylePredictionNetwork->CreateInferenceContext()); + checkf(StylePredictionInferenceContext != INDEX_NONE, TEXT("Could not create inference context for StylePredictionNetwork")); + + UTexture2D* StyleTexture = StyleTransferSettings->StyleTextures[i].LoadSynchronous(); + //UTexture2D* StyleTexture = LoadObject(this, TEXT("/Script/Engine.Texture2D'/StyleTransfer/T_StyleImage.T_StyleImage'")); #if WITH_EDITOR - FTextureCompilingManager::Get().FinishCompilation({StyleTexture}); + FTextureCompilingManager::Get().FinishCompilation({StyleTexture}); #endif - UpdateStyle(StyleTexture); + UpdateStyle(StyleTexture, StylePredictionInferenceContext); + } //UpdateStyle(FPaths::GetPath("C:\\projects\\realtime-style-transfer\\temp\\style_params_tensor.bin")); StyleTransferSceneViewExtension = FSceneViewExtensions::NewExtension(ViewportClient->GetWorld(), ViewportClient, StyleTransferNetwork, StyleTransferInferenceContext.ToSharedRef()); - } StyleTransferSceneViewExtension->SetEnabled(true); } @@ -67,10 +100,13 @@ void UStyleTransferSubsystem::StopStylizingViewport() { FlushRenderingCommands(); StyleTransferSceneViewExtension.Reset(); - if (StylePredictionInferenceContext != INDEX_NONE) + if (StylePredictionInferenceContexts.Num()) { - StylePredictionNetwork->DestroyInferenceContext(StylePredictionInferenceContext); - StylePredictionInferenceContext = INDEX_NONE; + for (auto It = StylePredictionInferenceContexts.CreateIterator(); It; ++It) + { + StylePredictionNetwork->DestroyInferenceContext(*It); + It.RemoveCurrent(); + } } if (StyleTransferInferenceContext && *StyleTransferInferenceContext != INDEX_NONE) { @@ -80,20 +116,14 @@ void UStyleTransferSubsystem::StopStylizingViewport() } } -void UStyleTransferSubsystem::UpdateStyle(UTexture2D* StyleTexture) +void UStyleTransferSubsystem::UpdateStyle(UTexture2D* StyleTexture, int32 StylePredictionInferenceContext) { checkf(StyleTransferInferenceContext.IsValid() && (*StyleTransferInferenceContext) != INDEX_NONE, TEXT("Can not infer style without inference context")); checkf(StylePredictionInferenceContext != INDEX_NONE, TEXT("Can not update style without inference context")); FlushRenderingCommands(); - ENQUEUE_RENDER_COMMAND(StylePrediction)([this, StyleTexture](FRHICommandListImmediate& RHICommandList) + ENQUEUE_RENDER_COMMAND(StylePrediction)([this, StyleTexture, StylePredictionInferenceContext](FRHICommandListImmediate& RHICommandList) { - IRenderCaptureProvider* RenderCaptureProvider = nullptr; - const FName RenderCaptureProviderType = IRenderCaptureProvider::GetModularFeatureName(); - if(IModularFeatures::Get().IsModularFeatureAvailable(RenderCaptureProviderType)) - { - RenderCaptureProvider = &IModularFeatures::Get().GetModularFeature(RenderCaptureProviderType); - RenderCaptureProvider->BeginCapture(&RHICommandList); - } + IRenderCaptureProvider* RenderCaptureProvider = ConditionalBeginRenderCapture(RHICommandList); FRDGBuilder GraphBuilder(RHICommandList); { @@ -120,7 +150,7 @@ void UStyleTransferSubsystem::UpdateStyle(UTexture2D* StyleTexture) } GraphBuilder.Execute(); - if(RenderCaptureProvider) + if (RenderCaptureProvider) { RenderCaptureProvider->EndCapture(&RHICommandList); } @@ -161,7 +191,7 @@ void UStyleTransferSubsystem::HandleConsoleVariableChanged(IConsoleVariable* Con if (CVarStyleTransferEnabled->GetBool()) { - if(!(StyleTransferNetwork || StylePredictionNetwork)) + if (!(StyleTransferNetwork || StylePredictionNetwork)) { LoadNetworks(); } @@ -203,3 +233,40 @@ void UStyleTransferSubsystem::LoadNetworks() UE_LOG(LogStyleTransfer, Error, TEXT("StylePredictionNetwork could not be loaded.")); } } + +void UStyleTransferSubsystem::InterpolateStyles(int32 StylePredictionInferenceContextA, int32 StylePredictionInferenceContextB, float Alpha) +{ + checkf(StyleTransferInferenceContext.IsValid() && (*StyleTransferInferenceContext) != INDEX_NONE, TEXT("Can not transfer style without inference context")); + checkf(StylePredictionInferenceContexts.Contains(StylePredictionInferenceContextA), TEXT("Can not update style without inference context A")); + checkf(StylePredictionInferenceContexts.Contains(StylePredictionInferenceContextB), TEXT("Can not update style without inference context B")); + ENQUEUE_RENDER_COMMAND(StylePrediction)([this, StylePredictionInferenceContextA, StylePredictionInferenceContextB, Alpha](FRHICommandListImmediate& RHICommandList) + { + IRenderCaptureProvider* RenderCaptureProvider = ConditionalBeginRenderCapture(RHICommandList); + FRDGBuilder GraphBuilder(RHICommandList); + { + RDG_EVENT_SCOPE(GraphBuilder, "StylePrediction"); + + const FNeuralTensor& InputStyleImageTensorA = StylePredictionNetwork->GetOutputTensorForContext(StylePredictionInferenceContextA, 0); + const FNeuralTensor& InputStyleImageTensorB = StylePredictionNetwork->GetOutputTensorForContext(StylePredictionInferenceContextB, 0); + const FNeuralTensor& OutputStyleParamsTensor = StyleTransferNetwork->GetInputTensorForContext(*StyleTransferInferenceContext, StyleTransferStyleParamsInputIndex); + FStyleTransferSceneViewExtension::InterpolateTensors(GraphBuilder, OutputStyleParamsTensor, InputStyleImageTensorA, InputStyleImageTensorB, Alpha); + } + GraphBuilder.Execute(); + if(RenderCaptureProvider) RenderCaptureProvider->EndCapture(&RHICommandList); + }); +} + +IRenderCaptureProvider* UStyleTransferSubsystem::ConditionalBeginRenderCapture(FRHICommandListImmediate& RHICommandList) +{ + IRenderCaptureProvider* RenderCaptureProvider = nullptr; + if (CVarAutoCaptureStylePrediction.GetValueOnRenderThread()) + { + const FName RenderCaptureProviderType = IRenderCaptureProvider::GetModularFeatureName(); + if (IModularFeatures::Get().IsModularFeatureAvailable(RenderCaptureProviderType)) + { + RenderCaptureProvider = &IModularFeatures::Get().GetModularFeature(RenderCaptureProviderType); + RenderCaptureProvider->BeginCapture(&RHICommandList); + } + } + return RenderCaptureProvider; +} diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferModule.h b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferModule.h index 20a73f5a..ec911d30 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferModule.h +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferModule.h @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. #pragma once diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h index d37d0dd2..e03531d8 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSceneViewExtension.h @@ -43,6 +43,7 @@ public: static void AddRescalingTextureCopy(FRDGBuilder& GraphBuilder, FRDGTexture& RDGSourceTexture, FScreenPassRenderTarget& DestinationRenderTarget); static FRDGTexture* TensorToTexture(FRDGBuilder& GraphBuilder, const FRDGTextureDesc& BaseDestinationDesc, const FNeuralTensor& SourceTensor); static void TextureToTensor(FRDGBuilder& GraphBuilder, FRDGTextureRef SourceTexture, const FNeuralTensor& DestinationTensor); + static void InterpolateTensors(FRDGBuilder& GraphBuilder, const FNeuralTensor& DestinationTensor, const FNeuralTensor& InputTensorA, const FNeuralTensor& InputTensorB, float Alpha); private: /** The actual Network pointer is not tracked so we need a WeakPtr too so we can check its validity on the game thread. */ diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSubsystem.h b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSubsystem.h index 098a06f8..4295d443 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSubsystem.h +++ b/Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSubsystem.h @@ -3,6 +3,7 @@ #pragma once #include "CoreMinimal.h" +#include "IRenderCaptureProvider.h" #include "StyleTransferSceneViewExtension.h" #include "Subsystems/GameInstanceSubsystem.h" #include "UObject/Object.h" @@ -12,7 +13,7 @@ * */ UCLASS() -class STYLETRANSFER_API UStyleTransferSubsystem : public UGameInstanceSubsystem +class STYLETRANSFER_API UStyleTransferSubsystem : public UGameInstanceSubsystem, public FTSTickerObjectBase { GENERATED_BODY() @@ -22,11 +23,16 @@ public: virtual void Deinitialize() override; // -- + // - FTSTickerObjectBase + virtual bool Tick(float DeltaTime) override final; + // -- + void StartStylizingViewport(FViewportClient* ViewportClient); void StopStylizingViewport(); - void UpdateStyle(UTexture2D* StyleTexture); + void UpdateStyle(UTexture2D* StyleTexture, int32 StylePredictionInferenceContext); void UpdateStyle(FString StyleTensorDataPath); + void InterpolateStyles(int32 StylePredictionInferenceContextA, int32 StylePredictionInferenceContextB, float Alpha); private: FStyleTransferSceneViewExtension::Ptr StyleTransferSceneViewExtension; @@ -37,7 +43,7 @@ private: UPROPERTY() TObjectPtr StylePredictionNetwork; - int32 StylePredictionInferenceContext = INDEX_NONE; + TArray StylePredictionInferenceContexts; TSharedPtr StyleTransferInferenceContext; @@ -46,4 +52,6 @@ private: void HandleConsoleVariableChanged(IConsoleVariable*); void LoadNetworks(); + + IRenderCaptureProvider* ConditionalBeginRenderCapture(FRHICommandListImmediate& RHICommandList); }; diff --git a/Plugins/StyleTransfer/Source/StyleTransfer/StyleTransfer.Build.cs b/Plugins/StyleTransfer/Source/StyleTransfer/StyleTransfer.Build.cs index 495abe44..56394da4 100644 --- a/Plugins/StyleTransfer/Source/StyleTransfer/StyleTransfer.Build.cs +++ b/Plugins/StyleTransfer/Source/StyleTransfer/StyleTransfer.Build.cs @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. using System.IO; using UnrealBuildTool; diff --git a/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/InterpolateTensorsCS.cpp b/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/InterpolateTensorsCS.cpp new file mode 100644 index 00000000..655ad8a0 --- /dev/null +++ b/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/InterpolateTensorsCS.cpp @@ -0,0 +1,19 @@ +// Copyright Manuel Wagner All Rights Reserved. + +#include "InterpolateTensorsCS.h" + +const FIntVector FInterpolateTensorsCS::ThreadGroupSize{64, 1, 1}; + + +void FInterpolateTensorsCS::ModifyCompilationEnvironment(const FGlobalShaderPermutationParameters& Parameters, FShaderCompilerEnvironment& OutEnvironment) +{ + FGlobalShader::ModifyCompilationEnvironment(Parameters, OutEnvironment); + + OutEnvironment.SetDefine(TEXT("THREADGROUP_SIZE_X"), ThreadGroupSize.X); + OutEnvironment.SetDefine(TEXT("THREADGROUP_SIZE_Y"), ThreadGroupSize.Y); + OutEnvironment.SetDefine(TEXT("THREADGROUP_SIZE_Z"), ThreadGroupSize.Z); +} + +IMPLEMENT_GLOBAL_SHADER(FInterpolateTensorsCS, + "/Plugins/StyleTransfer/Shaders/Private/InterpolateTensors.usf", + "InterpolateTensorsCS", SF_Compute); // Path defined in StyleTransferModule.cpp diff --git a/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/OutputTensorToSceneColorCS.cpp b/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/OutputTensorToSceneColorCS.cpp index e54bb785..84a6587c 100644 --- a/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/OutputTensorToSceneColorCS.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/OutputTensorToSceneColorCS.cpp @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. #include "OutputTensorToSceneColorCS.h" diff --git a/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/SceneColorToInputTensorCS.cpp b/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/SceneColorToInputTensorCS.cpp index c2d7069d..31e4d1fc 100644 --- a/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/SceneColorToInputTensorCS.cpp +++ b/Plugins/StyleTransfer/Source/StyleTransferShaders/Private/SceneColorToInputTensorCS.cpp @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. #include "SceneColorToInputTensorCS.h" diff --git a/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/InterpolateTensorsCS.h b/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/InterpolateTensorsCS.h new file mode 100644 index 00000000..a4558fe0 --- /dev/null +++ b/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/InterpolateTensorsCS.h @@ -0,0 +1,41 @@ +// Copyright Manuel Wagner All Rights Reserved. + +#pragma once + +// GPU/RHI/shaders +#include "GlobalShader.h" +#include "RHI.h" +#include "ProfilingDebugging/RealtimeGPUProfiler.h" +#include "RenderGraphUtils.h" +#include "ShaderParameterUtils.h" + + + +class STYLETRANSFERSHADERS_API FInterpolateTensorsCS : public FGlobalShader +{ + DECLARE_GLOBAL_SHADER(FInterpolateTensorsCS); + SHADER_USE_PARAMETER_STRUCT(FInterpolateTensorsCS, FGlobalShader) + + + static const FIntVector ThreadGroupSize; + + + BEGIN_SHADER_PARAMETER_STRUCT(FParameters, ) + SHADER_PARAMETER_RDG_BUFFER_UAV(RWBuffer, OutputUAV) + // Unreal sadly does not support arrays of Buffers in shader compiler + SHADER_PARAMETER_RDG_BUFFER_SRV(Buffer, InputSrvA) + SHADER_PARAMETER_RDG_BUFFER_SRV(Buffer, InputSrvB) + SHADER_PARAMETER(float, Alpha) + SHADER_PARAMETER(uint32, TensorVolume) + END_SHADER_PARAMETER_STRUCT() + + // - FShader + static bool ShouldCompilePermutation(const FGlobalShaderPermutationParameters& Parameters) { + return GetMaxSupportedFeatureLevel(Parameters.Platform) >= ERHIFeatureLevel::SM5; + } + + static void ModifyCompilationEnvironment(const FGlobalShaderPermutationParameters& Parameters, FShaderCompilerEnvironment& OutEnvironment); + // -- + +private: +}; diff --git a/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/OutputTensorToSceneColorCS.h b/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/OutputTensorToSceneColorCS.h index 8766c0e6..de18f613 100644 --- a/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/OutputTensorToSceneColorCS.h +++ b/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/OutputTensorToSceneColorCS.h @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. #pragma once diff --git a/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/SceneColorToInputTensorCS.h b/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/SceneColorToInputTensorCS.h index 6e86d50a..616e7851 100644 --- a/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/SceneColorToInputTensorCS.h +++ b/Plugins/StyleTransfer/Source/StyleTransferShaders/Public/SceneColorToInputTensorCS.h @@ -1,4 +1,4 @@ -// Copyright Epic Games, Inc. All Rights Reserved. +// Copyright Manuel Wagner All Rights Reserved. #pragma once diff --git a/Source/LyraGame/GameFeatures/GameFeatureAction_StyleTransfer.cpp b/Source/LyraGame/GameFeatures/GameFeatureAction_StyleTransfer.cpp deleted file mode 100644 index b9533d8f..00000000 --- a/Source/LyraGame/GameFeatures/GameFeatureAction_StyleTransfer.cpp +++ /dev/null @@ -1,25 +0,0 @@ - - -#include "GameFeatureAction_StyleTransfer.h" - -#include "StyleTransferSubsystem.h" - -void UGameFeatureAction_StyleTransfer::OnGameFeatureActivating(FGameFeatureActivatingContext& Context) -{ - Super::OnGameFeatureActivating(Context); -} - -void UGameFeatureAction_StyleTransfer::OnGameFeatureDeactivating(FGameFeatureDeactivatingContext& Context) -{ - Super::OnGameFeatureDeactivating(Context); - - // @todo shutdown cleanly -} - -void UGameFeatureAction_StyleTransfer::AddToWorld(const FWorldContext& WorldContext, const FGameFeatureStateChangeContext& ChangeContext) -{ - auto* StyleTransferSubsystem = WorldContext.OwningGameInstance->GetSubsystem(); - - UGameViewportClient* GameViewportClient = WorldContext.GameViewport; - StyleTransferSubsystem->StartStylizingViewport(GameViewportClient); -} diff --git a/Source/LyraGame/GameFeatures/GameFeatureAction_StyleTransfer.h b/Source/LyraGame/GameFeatures/GameFeatureAction_StyleTransfer.h deleted file mode 100644 index 000ae52b..00000000 --- a/Source/LyraGame/GameFeatures/GameFeatureAction_StyleTransfer.h +++ /dev/null @@ -1,22 +0,0 @@ -// Fill out your copyright notice in the Description page of Project Settings. - -#pragma once - -#include "CoreMinimal.h" -#include "GameFeatureAction_WorldActionBase.h" -#include "UObject/Object.h" -#include "GameFeatureAction_StyleTransfer.generated.h" - -/** - * - */ -UCLASS() -class LYRAGAME_API UGameFeatureAction_StyleTransfer : public UGameFeatureAction_WorldActionBase -{ - GENERATED_BODY() -public: - virtual void OnGameFeatureActivating(FGameFeatureActivatingContext& Context) override; - virtual void OnGameFeatureDeactivating(FGameFeatureDeactivatingContext& Context) override; -private: - virtual void AddToWorld(const FWorldContext& WorldContext, const FGameFeatureStateChangeContext& ChangeContext) override; -};