singinwhale
/
RealtimeStyleTransferRuntime
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added automated style switching and display in L_expanse with own experience

This commit is contained in:
singinwhale 2022-12-05 20:07:00 +01:00
parent 36c097e26e
commit ed7daa5a6a
20 changed files with 358 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Config/DefaultGame.ini
View File

@ -224,7 +224,6 @@ LoadingScreenControlBusMix=/Game/Audio/Modulation/ControlBusMixes/CBM_LoadingScr
[/Script/StyleTransfer.StyleTransferSettings] [/Script/StyleTransfer.StyleTransferSettings]
StyleTransferNetwork=/StyleTransfer/rstnet-960-32-3_transfer.rstnet-960-32-3_transfer StyleTransferNetwork=/StyleTransfer/rstnet-960-32-3_transfer.rstnet-960-32-3_transfer
StylePredictionNetwork=/StyleTransfer/rstnet-960-32-3_predictor.rstnet-960-32-3_predictor StylePredictionNetwork=/StyleTransfer/rstnet-960-32-3_predictor.rstnet-960-32-3_predictor
+StyleTextures=/StyleTransfer/Styles/002af7d2cf8317f84d629a0b12cdb4dc10899f1d.002af7d2cf8317f84d629a0b12cdb4dc10899f1d
+StyleTextures=/StyleTransfer/Styles/00059bb14dffa5cb836f35e2f207f39946d25ff6.00059bb14dffa5cb836f35e2f207f39946d25ff6 +StyleTextures=/StyleTransfer/Styles/00059bb14dffa5cb836f35e2f207f39946d25ff6.00059bb14dffa5cb836f35e2f207f39946d25ff6
+StyleTextures=/StyleTransfer/Styles/0011fc013403f895146b783b2dd338d639883ec8.0011fc013403f895146b783b2dd338d639883ec8 +StyleTextures=/StyleTransfer/Styles/0011fc013403f895146b783b2dd338d639883ec8.0011fc013403f895146b783b2dd338d639883ec8
+StyleTextures=/StyleTransfer/Styles/00217cb856d9d97865cd3c59669155d3b5c5c0cd.00217cb856d9d97865cd3c59669155d3b5c5c0cd +StyleTextures=/StyleTransfer/Styles/00217cb856d9d97865cd3c59669155d3b5c5c0cd.00217cb856d9d97865cd3c59669155d3b5c5c0cd
@ -235,5 +234,6 @@ StylePredictionNetwork=/StyleTransfer/rstnet-960-32-3_predictor.rstnet-960-32-3_
+StyleTextures=/StyleTransfer/Styles/T_StyleImage2.T_StyleImage2 +StyleTextures=/StyleTransfer/Styles/T_StyleImage2.T_StyleImage2
+StyleTextures=/StyleTransfer/Styles/T_StyleImage3.T_StyleImage3 +StyleTextures=/StyleTransfer/Styles/T_StyleImage3.T_StyleImage3
+StyleTextures=/StyleTransfer/Styles/T_StyleImage4.T_StyleImage4 +StyleTextures=/StyleTransfer/Styles/T_StyleImage4.T_StyleImage4
+StyleTextures=/StyleTransfer/Styles/002af7d2cf8317f84d629a0b12cdb4dc10899f1d.002af7d2cf8317f84d629a0b12cdb4dc10899f1d
InterpolationCurve=(EditorCurveData=(Keys=((InterpMode=RCIM_Cubic,TangentMode=RCTM_User),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=2.241476,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=3.908512,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=5.831762),(Time=7.971708)),DefaultValue=340282346638528859811704183484516925440.000000,PreInfinityExtrap=RCCE_Constant,PostInfinityExtrap=RCCE_Constant),ExternalCurve=None) InterpolationCurve=(EditorCurveData=(Keys=((InterpMode=RCIM_Cubic,TangentMode=RCTM_User),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=2.241476,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=3.908512,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=5.831762),(Time=7.971708)),DefaultValue=340282346638528859811704183484516925440.000000,PreInfinityExtrap=RCCE_Constant,PostInfinityExtrap=RCCE_Constant),ExternalCurve=None)

600
Lyra.uproject
View File

@ -1,300 +1,304 @@
{ {
"FileVersion": 3, "FileVersion": 3,
"EngineAssociation": "{67760C9E-4E78-132C-8B45-828F7E94A5A9}", "EngineAssociation": "{0CEC6B5D-4712-B2FC-F019-9BBC776471A8}",
"Category": "", "Category": "",
"Description": "", "Description": "",
"Modules": [ "Modules": [
{ {
"Name": "LyraGame", "Name": "LyraGame",
"Type": "Runtime", "Type": "Runtime",
"LoadingPhase": "Default", "LoadingPhase": "Default",
"AdditionalDependencies": [ "AdditionalDependencies": [
"DeveloperSettings", "DeveloperSettings",
"Engine" "Engine"
] ]
}, },
{ {
"Name": "LyraEditor", "Name": "LyraEditor",
"Type": "Editor", "Type": "Editor",
"LoadingPhase": "Default" "LoadingPhase": "Default"
} }
], ],
"Plugins": [ "Plugins": [
{ {
"Name": "ActorPalette", "Name": "ActorPalette",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GameplayAbilities", "Name": "GameplayAbilities",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "Gauntlet", "Name": "Gauntlet",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "CommonLoadingScreen", "Name": "CommonLoadingScreen",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "CommonConversation", "Name": "CommonConversation",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GameFeatures", "Name": "GameFeatures",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ModularGameplay", "Name": "ModularGameplay",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ModularGameplayActors", "Name": "ModularGameplayActors",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "EnhancedInput", "Name": "EnhancedInput",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "WinDualShock", "Name": "WinDualShock",
"Enabled": true, "Enabled": true,
"SupportedTargetPlatforms": [ "SupportedTargetPlatforms": [
"Win64" "Win64"
] ]
}, },
{ {
"Name": "Volumetrics", "Name": "Volumetrics",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "DataRegistry", "Name": "DataRegistry",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ReplicationGraph", "Name": "ReplicationGraph",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "SignificanceManager", "Name": "SignificanceManager",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "Niagara", "Name": "Niagara",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "Water", "Name": "Water",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "CommonUI", "Name": "CommonUI",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ControlFlows", "Name": "ControlFlows",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GameSettings", "Name": "GameSettings",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "CommonUser", "Name": "CommonUser",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "CommonGame", "Name": "CommonGame",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GameSubtitles", "Name": "GameSubtitles",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "PocketWorlds", "Name": "PocketWorlds",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "UIExtension", "Name": "UIExtension",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AsyncMixin", "Name": "AsyncMixin",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "Metasound", "Name": "Metasound",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "MagicLeap", "Name": "MagicLeap",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "MagicLeapMedia", "Name": "MagicLeapMedia",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "MagicLeapPassableWorld", "Name": "MagicLeapPassableWorld",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "OculusVR", "Name": "OculusVR",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "OpenXR", "Name": "OpenXR",
"Enabled": true, "Enabled": true,
"SupportedTargetPlatforms": [ "SupportedTargetPlatforms": [
"Win64", "Win64",
"Linux", "Linux",
"Android", "Android",
"HoloLens" "HoloLens"
] ]
}, },
{ {
"Name": "OpenXREyeTracker", "Name": "OpenXREyeTracker",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "OpenXRHandTracking", "Name": "OpenXRHandTracking",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "OpenXRHMD", "Name": "OpenXRHMD",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "SteamVR", "Name": "SteamVR",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "GearVR", "Name": "GearVR",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "LuminPlatformFeatures", "Name": "LuminPlatformFeatures",
"Enabled": false, "Enabled": false,
"SupportedTargetPlatforms": [ "SupportedTargetPlatforms": [
"Lumin" "Lumin"
] ]
}, },
{ {
"Name": "MLSDK", "Name": "MLSDK",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "OnlineFramework", "Name": "OnlineFramework",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "OnlineSubsystemEOS", "Name": "OnlineSubsystemEOS",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "OnlineServicesEOS", "Name": "OnlineServicesEOS",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "OnlineServicesNull", "Name": "OnlineServicesNull",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "OnlineServicesOSSAdapter", "Name": "OnlineServicesOSSAdapter",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "OnlineSubsystemSteam", "Name": "OnlineSubsystemSteam",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GameplayMessageRouter", "Name": "GameplayMessageRouter",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "SteamSockets", "Name": "SteamSockets",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AssetReferenceRestrictions", "Name": "AssetReferenceRestrictions",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ModelingToolsEditorMode", "Name": "ModelingToolsEditorMode",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GeometryScripting", "Name": "GeometryScripting",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AnimationLocomotionLibrary", "Name": "AnimationLocomotionLibrary",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AudioModulation", "Name": "AudioModulation",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AudioGameplayVolume", "Name": "AudioGameplayVolume",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AudioGameplay", "Name": "AudioGameplay",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "SoundUtilities", "Name": "SoundUtilities",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AnimationWarping", "Name": "AnimationWarping",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "MovieRenderPipeline", "Name": "MovieRenderPipeline",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "MoviePipelineMaskRenderPass", "Name": "MoviePipelineMaskRenderPass",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "AssetSearch", "Name": "AssetSearch",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "GameplayInsights", "Name": "GameplayInsights",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ResonanceAudio", "Name": "ResonanceAudio",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "RuntimePhysXCooking", "Name": "RuntimePhysXCooking",
"Enabled": false "Enabled": false
}, },
{ {
"Name": "Spatialization", "Name": "Spatialization",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ShooterCore", "Name": "ShooterCore",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "ShooterMaps", "Name": "ShooterMaps",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "TopDownArena", "Name": "TopDownArena",
"Enabled": true "Enabled": true
}, },
{ {
"Name": "FunctionalTestingEditor", "Name": "FunctionalTestingEditor",
"Enabled": true "Enabled": true
} },
] {
"Name": "StyleTransfer",
"Enabled": true
}
]
} }

BIN
Plugins/GameFeatures/ShooterCore/Content/Experiences/B_ShooterGame_Elimination.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/Experiences/LAS_ShooterGame_StandardComponents.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/StyleTransfer/B_StyleTransferComponent.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/StyleTransfer/LAS_StyleTransfer.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/StyleTransfer/W_StyleSwitcher.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterMaps/Content/B_ShooterGame_Elimination_StyleTransfer.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterMaps/Content/Maps/L_Expanse.umap (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage1.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage2.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage3.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage4.uasset (Stored with Git LFS)
View File

Binary file not shown.

1
Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferModule.cpp
View File

@ -2,7 +2,6 @@
#include "StyleTransferModule.h" #include "StyleTransferModule.h"
#include "ShaderCore.h"
#include "Interfaces/IPluginManager.h" #include "Interfaces/IPluginManager.h"
#include "Logging/LogMacros.h" #include "Logging/LogMacros.h"

8
Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSettings.h
View File

@ -17,15 +17,15 @@ class STYLETRANSFER_API UStyleTransferSettings : public UDeveloperSettings
public: public:
UStyleTransferSettings(); UStyleTransferSettings();
UPROPERTY(EditAnywhere, Config) UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
TSoftObjectPtr<UNeuralNetwork> StyleTransferNetwork = nullptr; TSoftObjectPtr<UNeuralNetwork> StyleTransferNetwork = nullptr;
UPROPERTY(EditAnywhere, Config) UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
TSoftObjectPtr<UNeuralNetwork> StylePredictionNetwork = nullptr; TSoftObjectPtr<UNeuralNetwork> StylePredictionNetwork = nullptr;
UPROPERTY(EditAnywhere, Config) UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
TArray<TSoftObjectPtr<UTexture2D>> StyleTextures; TArray<TSoftObjectPtr<UTexture2D>> StyleTextures;
UPROPERTY(EditAnywhere, Config) UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
FRuntimeFloatCurve InterpolationCurve; FRuntimeFloatCurve InterpolationCurve;
}; };

41
Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp
View File

@ -40,27 +40,6 @@ void UStyleTransferSubsystem::Deinitialize()
Super::Deinitialize(); Super::Deinitialize();
} }
bool UStyleTransferSubsystem::Tick(float DeltaTime)
{
return false;
if (!GetWorld())
return true;
if (StylePredictionInferenceContexts.Num() > 1)
{
const UStyleTransferSettings* StyleTransferSettings = GetDefault<UStyleTransferSettings>();
const FRichCurve* InterpCurve = StyleTransferSettings->InterpolationCurve.GetRichCurveConst();
float MinTime, MaxTime;
InterpCurve->GetTimeRange(MinTime, MaxTime);
const double Time = MinTime + FMath::Fmod(GetWorld()->GetTimeSeconds(), static_cast<double>(MaxTime - MinTime));
const float Alpha = InterpCurve->Eval(Time);
UE_LOG(LogStyleTransfer, VeryVerbose, TEXT("Alpha is %0.4f"), Alpha);
InterpolateStyles(StylePredictionInferenceContexts[0], StylePredictionInferenceContexts[1], Alpha);
}
return true;
}
void UStyleTransferSubsystem::StartStylizingViewport(FViewportClient* ViewportClient) void UStyleTransferSubsystem::StartStylizingViewport(FViewportClient* ViewportClient)
{ {
if (!StylePredictionNetwork->IsLoaded() || !StyleTransferNetwork->IsLoaded()) if (!StylePredictionNetwork->IsLoaded() || !StyleTransferNetwork->IsLoaded())
@ -284,6 +263,26 @@ void UStyleTransferSubsystem::InterpolateStyles(int32 StylePredictionInferenceCo
}); });
} }
float UStyleTransferSubsystem::GetStyleAlpha() const
{
return StyleAlpha;
}
void UStyleTransferSubsystem::SetStyleAlpha(float Alpha)
{
Alpha = FMath::Clamp(Alpha, 0.0f, float(StylePredictionInferenceContexts.Num() - 1));
StyleAlpha = Alpha;
if(!StyleTransferSceneViewExtension.IsValid())
return;
const int32 FromStyle = FMath::FloorToInt32(StyleAlpha);
const int32 ToStyle = FMath::CeilToInt32(StyleAlpha);
InterpolateStyles(StylePredictionInferenceContexts[FromStyle], StylePredictionInferenceContexts[ToStyle], StyleAlpha - FromStyle);
}
IRenderCaptureProvider* BeginRenderCapture(FRHICommandListImmediate& RHICommandList) IRenderCaptureProvider* BeginRenderCapture(FRHICommandListImmediate& RHICommandList)
{ {
IRenderCaptureProvider* RenderCaptureProvider = nullptr; IRenderCaptureProvider* RenderCaptureProvider = nullptr;

15
Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSubsystem.h
View File

@ -13,7 +13,7 @@
* *
*/ */
UCLASS() UCLASS()
class STYLETRANSFER_API UStyleTransferSubsystem : public UGameInstanceSubsystem, public FTSTickerObjectBase class STYLETRANSFER_API UStyleTransferSubsystem : public UGameInstanceSubsystem
{ {
GENERATED_BODY() GENERATED_BODY()
@ -23,10 +23,6 @@ public:
virtual void Deinitialize() override; virtual void Deinitialize() override;
// -- // --
// - FTSTickerObjectBase
virtual bool Tick(float DeltaTime) override final;
// --
void StartStylizingViewport(FViewportClient* ViewportClient); void StartStylizingViewport(FViewportClient* ViewportClient);
void StopStylizingViewport(); void StopStylizingViewport();
@ -34,6 +30,12 @@ public:
void UpdateStyle(FString StyleTensorDataPath); void UpdateStyle(FString StyleTensorDataPath);
void InterpolateStyles(int32 StylePredictionInferenceContextA, int32 StylePredictionInferenceContextB, float Alpha); void InterpolateStyles(int32 StylePredictionInferenceContextA, int32 StylePredictionInferenceContextB, float Alpha);
UFUNCTION(BlueprintPure)
float GetStyleAlpha() const;
UFUNCTION(BlueprintCallable)
void SetStyleAlpha(float Alpha);
private: private:
FStyleTransferSceneViewExtension::Ptr StyleTransferSceneViewExtension; FStyleTransferSceneViewExtension::Ptr StyleTransferSceneViewExtension;
@ -46,9 +48,10 @@ private:
TArray<int32> StylePredictionInferenceContexts; TArray<int32> StylePredictionInferenceContexts;
TSharedPtr<int32, ESPMode::ThreadSafe> StyleTransferInferenceContext; TSharedPtr<int32, ESPMode::ThreadSafe> StyleTransferInferenceContext;
int32 StyleTransferStyleParamsInputIndex = INDEX_NONE; int32 StyleTransferStyleParamsInputIndex = INDEX_NONE;
float StyleAlpha = 0.0f;
void HandleConsoleVariableChanged(IConsoleVariable*); void HandleConsoleVariableChanged(IConsoleVariable*);
void LoadNetworks(); void LoadNetworks();

3
Plugins/StyleTransfer/StyleTransfer.uplugin
View File

@ -13,6 +13,9 @@
"IsBetaVersion": false, "IsBetaVersion": false,
"IsExperimentalVersion": false, "IsExperimentalVersion": false,
"Installed": false, "Installed": false,
"ExplicitlyLoaded": false,
"EnabledByDefault": true,
"BuiltInInitialFeatureState": "Registered",
"Modules": [ "Modules": [
{ {
"Name": "StyleTransfer", "Name": "StyleTransfer",