singinwhale
/
RealtimeStyleTransferRuntime
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added automated style switching and display in L_expanse with own experience

This commit is contained in:
singinwhale 2022-12-05 20:07:00 +01:00
parent 36c097e26e
commit ed7daa5a6a
20 changed files with 358 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Config/DefaultGame.ini
View File

@ -224,7 +224,6 @@ LoadingScreenControlBusMix=/Game/Audio/Modulation/ControlBusMixes/CBM_LoadingScr
[/Script/StyleTransfer.StyleTransferSettings]
StyleTransferNetwork=/StyleTransfer/rstnet-960-32-3_transfer.rstnet-960-32-3_transfer
StylePredictionNetwork=/StyleTransfer/rstnet-960-32-3_predictor.rstnet-960-32-3_predictor
+StyleTextures=/StyleTransfer/Styles/002af7d2cf8317f84d629a0b12cdb4dc10899f1d.002af7d2cf8317f84d629a0b12cdb4dc10899f1d
+StyleTextures=/StyleTransfer/Styles/00059bb14dffa5cb836f35e2f207f39946d25ff6.00059bb14dffa5cb836f35e2f207f39946d25ff6
+StyleTextures=/StyleTransfer/Styles/0011fc013403f895146b783b2dd338d639883ec8.0011fc013403f895146b783b2dd338d639883ec8
+StyleTextures=/StyleTransfer/Styles/00217cb856d9d97865cd3c59669155d3b5c5c0cd.00217cb856d9d97865cd3c59669155d3b5c5c0cd
@ -235,5 +234,6 @@ StylePredictionNetwork=/StyleTransfer/rstnet-960-32-3_predictor.rstnet-960-32-3_
+StyleTextures=/StyleTransfer/Styles/T_StyleImage2.T_StyleImage2
+StyleTextures=/StyleTransfer/Styles/T_StyleImage3.T_StyleImage3
+StyleTextures=/StyleTransfer/Styles/T_StyleImage4.T_StyleImage4
+StyleTextures=/StyleTransfer/Styles/002af7d2cf8317f84d629a0b12cdb4dc10899f1d.002af7d2cf8317f84d629a0b12cdb4dc10899f1d
InterpolationCurve=(EditorCurveData=(Keys=((InterpMode=RCIM_Cubic,TangentMode=RCTM_User),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=2.241476,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=3.908512,Value=1.000000),(InterpMode=RCIM_Cubic,TangentMode=RCTM_User,Time=5.831762),(Time=7.971708)),DefaultValue=340282346638528859811704183484516925440.000000,PreInfinityExtrap=RCCE_Constant,PostInfinityExtrap=RCCE_Constant),ExternalCurve=None)

600
Lyra.uproject
View File

@ -1,300 +1,304 @@
{
"FileVersion": 3,
"EngineAssociation": "{67760C9E-4E78-132C-8B45-828F7E94A5A9}",
"Category": "",
"Description": "",
"Modules": [
{
"Name": "LyraGame",
"Type": "Runtime",
"LoadingPhase": "Default",
"AdditionalDependencies": [
"DeveloperSettings",
"Engine"
]
},
{
"Name": "LyraEditor",
"Type": "Editor",
"LoadingPhase": "Default"
}
],
"Plugins": [
{
"Name": "ActorPalette",
"Enabled": true
},
{
"Name": "GameplayAbilities",
"Enabled": true
},
{
"Name": "Gauntlet",
"Enabled": true
},
{
"Name": "CommonLoadingScreen",
"Enabled": true
},
{
"Name": "CommonConversation",
"Enabled": true
},
{
"Name": "GameFeatures",
"Enabled": true
},
{
"Name": "ModularGameplay",
"Enabled": true
},
{
"Name": "ModularGameplayActors",
"Enabled": true
},
{
"Name": "EnhancedInput",
"Enabled": true
},
{
"Name": "WinDualShock",
"Enabled": true,
"SupportedTargetPlatforms": [
"Win64"
]
},
{
"Name": "Volumetrics",
"Enabled": true
},
{
"Name": "DataRegistry",
"Enabled": true
},
{
"Name": "ReplicationGraph",
"Enabled": true
},
{
"Name": "SignificanceManager",
"Enabled": true
},
{
"Name": "Niagara",
"Enabled": true
},
{
"Name": "Water",
"Enabled": true
},
{
"Name": "CommonUI",
"Enabled": true
},
{
"Name": "ControlFlows",
"Enabled": true
},
{
"Name": "GameSettings",
"Enabled": true
},
{
"Name": "CommonUser",
"Enabled": true
},
{
"Name": "CommonGame",
"Enabled": true
},
{
"Name": "GameSubtitles",
"Enabled": true
},
{
"Name": "PocketWorlds",
"Enabled": true
},
{
"Name": "UIExtension",
"Enabled": true
},
{
"Name": "AsyncMixin",
"Enabled": true
},
{
"Name": "Metasound",
"Enabled": true
},
{
"Name": "MagicLeap",
"Enabled": false
},
{
"Name": "MagicLeapMedia",
"Enabled": false
},
{
"Name": "MagicLeapPassableWorld",
"Enabled": false
},
{
"Name": "OculusVR",
"Enabled": false
},
{
"Name": "OpenXR",
"Enabled": true,
"SupportedTargetPlatforms": [
"Win64",
"Linux",
"Android",
"HoloLens"
]
},
{
"Name": "OpenXREyeTracker",
"Enabled": false
},
{
"Name": "OpenXRHandTracking",
"Enabled": false
},
{
"Name": "OpenXRHMD",
"Enabled": false
},
{
"Name": "SteamVR",
"Enabled": false
},
{
"Name": "GearVR",
"Enabled": false
},
{
"Name": "LuminPlatformFeatures",
"Enabled": false,
"SupportedTargetPlatforms": [
"Lumin"
]
},
{
"Name": "MLSDK",
"Enabled": false
},
{
"Name": "OnlineFramework",
"Enabled": true
},
{
"Name": "OnlineSubsystemEOS",
"Enabled": true
},
{
"Name": "OnlineServicesEOS",
"Enabled": true
},
{
"Name": "OnlineServicesNull",
"Enabled": true
},
{
"Name": "OnlineServicesOSSAdapter",
"Enabled": true
},
{
"Name": "OnlineSubsystemSteam",
"Enabled": true
},
{
"Name": "GameplayMessageRouter",
"Enabled": true
},
{
"Name": "SteamSockets",
"Enabled": true
},
{
"Name": "AssetReferenceRestrictions",
"Enabled": true
},
{
"Name": "ModelingToolsEditorMode",
"Enabled": true
},
{
"Name": "GeometryScripting",
"Enabled": true
},
{
"Name": "AnimationLocomotionLibrary",
"Enabled": true
},
{
"Name": "AudioModulation",
"Enabled": true
},
{
"Name": "AudioGameplayVolume",
"Enabled": true
},
{
"Name": "AudioGameplay",
"Enabled": true
},
{
"Name": "SoundUtilities",
"Enabled": true
},
{
"Name": "AnimationWarping",
"Enabled": true
},
{
"Name": "MovieRenderPipeline",
"Enabled": true
},
{
"Name": "MoviePipelineMaskRenderPass",
"Enabled": true
},
{
"Name": "AssetSearch",
"Enabled": true
},
{
"Name": "GameplayInsights",
"Enabled": true
},
{
"Name": "ResonanceAudio",
"Enabled": false
},
{
"Name": "RuntimePhysXCooking",
"Enabled": false
},
{
"Name": "Spatialization",
"Enabled": true
},
{
"Name": "ShooterCore",
"Enabled": true
},
{
"Name": "ShooterMaps",
"Enabled": true
},
{
"Name": "TopDownArena",
"Enabled": true
},
{
"Name": "FunctionalTestingEditor",
"Enabled": true
}
]
"FileVersion": 3,
"EngineAssociation": "{0CEC6B5D-4712-B2FC-F019-9BBC776471A8}",
"Category": "",
"Description": "",
"Modules": [
{
"Name": "LyraGame",
"Type": "Runtime",
"LoadingPhase": "Default",
"AdditionalDependencies": [
"DeveloperSettings",
"Engine"
]
},
{
"Name": "LyraEditor",
"Type": "Editor",
"LoadingPhase": "Default"
}
],
"Plugins": [
{
"Name": "ActorPalette",
"Enabled": true
},
{
"Name": "GameplayAbilities",
"Enabled": true
},
{
"Name": "Gauntlet",
"Enabled": true
},
{
"Name": "CommonLoadingScreen",
"Enabled": true
},
{
"Name": "CommonConversation",
"Enabled": true
},
{
"Name": "GameFeatures",
"Enabled": true
},
{
"Name": "ModularGameplay",
"Enabled": true
},
{
"Name": "ModularGameplayActors",
"Enabled": true
},
{
"Name": "EnhancedInput",
"Enabled": true
},
{
"Name": "WinDualShock",
"Enabled": true,
"SupportedTargetPlatforms": [
"Win64"
]
},
{
"Name": "Volumetrics",
"Enabled": true
},
{
"Name": "DataRegistry",
"Enabled": true
},
{
"Name": "ReplicationGraph",
"Enabled": true
},
{
"Name": "SignificanceManager",
"Enabled": true
},
{
"Name": "Niagara",
"Enabled": true
},
{
"Name": "Water",
"Enabled": true
},
{
"Name": "CommonUI",
"Enabled": true
},
{
"Name": "ControlFlows",
"Enabled": true
},
{
"Name": "GameSettings",
"Enabled": true
},
{
"Name": "CommonUser",
"Enabled": true
},
{
"Name": "CommonGame",
"Enabled": true
},
{
"Name": "GameSubtitles",
"Enabled": true
},
{
"Name": "PocketWorlds",
"Enabled": true
},
{
"Name": "UIExtension",
"Enabled": true
},
{
"Name": "AsyncMixin",
"Enabled": true
},
{
"Name": "Metasound",
"Enabled": true
},
{
"Name": "MagicLeap",
"Enabled": false
},
{
"Name": "MagicLeapMedia",
"Enabled": false
},
{
"Name": "MagicLeapPassableWorld",
"Enabled": false
},
{
"Name": "OculusVR",
"Enabled": false
},
{
"Name": "OpenXR",
"Enabled": true,
"SupportedTargetPlatforms": [
"Win64",
"Linux",
"Android",
"HoloLens"
]
},
{
"Name": "OpenXREyeTracker",
"Enabled": false
},
{
"Name": "OpenXRHandTracking",
"Enabled": false
},
{
"Name": "OpenXRHMD",
"Enabled": false
},
{
"Name": "SteamVR",
"Enabled": false
},
{
"Name": "GearVR",
"Enabled": false
},
{
"Name": "LuminPlatformFeatures",
"Enabled": false,
"SupportedTargetPlatforms": [
"Lumin"
]
},
{
"Name": "MLSDK",
"Enabled": false
},
{
"Name": "OnlineFramework",
"Enabled": true
},
{
"Name": "OnlineSubsystemEOS",
"Enabled": true
},
{
"Name": "OnlineServicesEOS",
"Enabled": true
},
{
"Name": "OnlineServicesNull",
"Enabled": true
},
{
"Name": "OnlineServicesOSSAdapter",
"Enabled": true
},
{
"Name": "OnlineSubsystemSteam",
"Enabled": true
},
{
"Name": "GameplayMessageRouter",
"Enabled": true
},
{
"Name": "SteamSockets",
"Enabled": true
},
{
"Name": "AssetReferenceRestrictions",
"Enabled": true
},
{
"Name": "ModelingToolsEditorMode",
"Enabled": true
},
{
"Name": "GeometryScripting",
"Enabled": true
},
{
"Name": "AnimationLocomotionLibrary",
"Enabled": true
},
{
"Name": "AudioModulation",
"Enabled": true
},
{
"Name": "AudioGameplayVolume",
"Enabled": true
},
{
"Name": "AudioGameplay",
"Enabled": true
},
{
"Name": "SoundUtilities",
"Enabled": true
},
{
"Name": "AnimationWarping",
"Enabled": true
},
{
"Name": "MovieRenderPipeline",
"Enabled": true
},
{
"Name": "MoviePipelineMaskRenderPass",
"Enabled": true
},
{
"Name": "AssetSearch",
"Enabled": true
},
{
"Name": "GameplayInsights",
"Enabled": true
},
{
"Name": "ResonanceAudio",
"Enabled": false
},
{
"Name": "RuntimePhysXCooking",
"Enabled": false
},
{
"Name": "Spatialization",
"Enabled": true
},
{
"Name": "ShooterCore",
"Enabled": true
},
{
"Name": "ShooterMaps",
"Enabled": true
},
{
"Name": "TopDownArena",
"Enabled": true
},
{
"Name": "FunctionalTestingEditor",
"Enabled": true
},
{
"Name": "StyleTransfer",
"Enabled": true
}
]
}

BIN
Plugins/GameFeatures/ShooterCore/Content/Experiences/B_ShooterGame_Elimination.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/Experiences/LAS_ShooterGame_StandardComponents.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/StyleTransfer/B_StyleTransferComponent.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/StyleTransfer/LAS_StyleTransfer.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterCore/Content/StyleTransfer/W_StyleSwitcher.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterMaps/Content/B_ShooterGame_Elimination_StyleTransfer.uasset (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
Plugins/GameFeatures/ShooterMaps/Content/Maps/L_Expanse.umap (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage1.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage2.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage3.uasset (Stored with Git LFS)
View File

Binary file not shown.

BIN
Plugins/StyleTransfer/Content/T_StyleImage4.uasset (Stored with Git LFS)
View File

Binary file not shown.

1
Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferModule.cpp
View File

@ -2,7 +2,6 @@
#include "StyleTransferModule.h"
#include "ShaderCore.h"
#include "Interfaces/IPluginManager.h"
#include "Logging/LogMacros.h"

8
Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSettings.h
View File

@ -17,15 +17,15 @@ class STYLETRANSFER_API UStyleTransferSettings : public UDeveloperSettings
public:
UStyleTransferSettings();
UPROPERTY(EditAnywhere, Config)
UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
TSoftObjectPtr<UNeuralNetwork> StyleTransferNetwork = nullptr;
UPROPERTY(EditAnywhere, Config)
UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
TSoftObjectPtr<UNeuralNetwork> StylePredictionNetwork = nullptr;
UPROPERTY(EditAnywhere, Config)
UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
TArray<TSoftObjectPtr<UTexture2D>> StyleTextures;
UPROPERTY(EditAnywhere, Config)
UPROPERTY(EditAnywhere, BlueprintReadOnly, Config)
FRuntimeFloatCurve InterpolationCurve;
};

41
Plugins/StyleTransfer/Source/StyleTransfer/Private/StyleTransferSubsystem.cpp
View File

@ -40,27 +40,6 @@ void UStyleTransferSubsystem::Deinitialize()
Super::Deinitialize();
}
bool UStyleTransferSubsystem::Tick(float DeltaTime)
{
return false;
if (!GetWorld())
return true;
if (StylePredictionInferenceContexts.Num() > 1)
{
const UStyleTransferSettings* StyleTransferSettings = GetDefault<UStyleTransferSettings>();
const FRichCurve* InterpCurve = StyleTransferSettings->InterpolationCurve.GetRichCurveConst();
float MinTime, MaxTime;
InterpCurve->GetTimeRange(MinTime, MaxTime);
const double Time = MinTime + FMath::Fmod(GetWorld()->GetTimeSeconds(), static_cast<double>(MaxTime - MinTime));
const float Alpha = InterpCurve->Eval(Time);
UE_LOG(LogStyleTransfer, VeryVerbose, TEXT("Alpha is %0.4f"), Alpha);
InterpolateStyles(StylePredictionInferenceContexts[0], StylePredictionInferenceContexts[1], Alpha);
}
return true;
}
void UStyleTransferSubsystem::StartStylizingViewport(FViewportClient* ViewportClient)
{
if (!StylePredictionNetwork->IsLoaded() || !StyleTransferNetwork->IsLoaded())
@ -284,6 +263,26 @@ void UStyleTransferSubsystem::InterpolateStyles(int32 StylePredictionInferenceCo
});
}
float UStyleTransferSubsystem::GetStyleAlpha() const
{
return StyleAlpha;
}
void UStyleTransferSubsystem::SetStyleAlpha(float Alpha)
{
Alpha = FMath::Clamp(Alpha, 0.0f, float(StylePredictionInferenceContexts.Num() - 1));
StyleAlpha = Alpha;
if(!StyleTransferSceneViewExtension.IsValid())
return;
const int32 FromStyle = FMath::FloorToInt32(StyleAlpha);
const int32 ToStyle = FMath::CeilToInt32(StyleAlpha);
InterpolateStyles(StylePredictionInferenceContexts[FromStyle], StylePredictionInferenceContexts[ToStyle], StyleAlpha - FromStyle);
}
IRenderCaptureProvider* BeginRenderCapture(FRHICommandListImmediate& RHICommandList)
{
IRenderCaptureProvider* RenderCaptureProvider = nullptr;

15
Plugins/StyleTransfer/Source/StyleTransfer/Public/StyleTransferSubsystem.h
View File

@ -13,7 +13,7 @@
*
*/
UCLASS()
class STYLETRANSFER_API UStyleTransferSubsystem : public UGameInstanceSubsystem, public FTSTickerObjectBase
class STYLETRANSFER_API UStyleTransferSubsystem : public UGameInstanceSubsystem
{
GENERATED_BODY()
@ -23,10 +23,6 @@ public:
virtual void Deinitialize() override;
// --
// - FTSTickerObjectBase
virtual bool Tick(float DeltaTime) override final;
// --
void StartStylizingViewport(FViewportClient* ViewportClient);
void StopStylizingViewport();
@ -34,6 +30,12 @@ public:
void UpdateStyle(FString StyleTensorDataPath);
void InterpolateStyles(int32 StylePredictionInferenceContextA, int32 StylePredictionInferenceContextB, float Alpha);
UFUNCTION(BlueprintPure)
float GetStyleAlpha() const;
UFUNCTION(BlueprintCallable)
void SetStyleAlpha(float Alpha);
private:
FStyleTransferSceneViewExtension::Ptr StyleTransferSceneViewExtension;
@ -46,9 +48,10 @@ private:
TArray<int32> StylePredictionInferenceContexts;
TSharedPtr<int32, ESPMode::ThreadSafe> StyleTransferInferenceContext;
int32 StyleTransferStyleParamsInputIndex = INDEX_NONE;
float StyleAlpha = 0.0f;
void HandleConsoleVariableChanged(IConsoleVariable*);
void LoadNetworks();

3
Plugins/StyleTransfer/StyleTransfer.uplugin
View File

@ -13,6 +13,9 @@
"IsBetaVersion": false,
"IsExperimentalVersion": false,
"Installed": false,
"ExplicitlyLoaded": false,
"EnabledByDefault": true,
"BuiltInInitialFeatureState": "Registered",
"Modules": [
{
"Name": "StyleTransfer",