Skip to content

RS Local texture parameters latency improvements #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
egorovaanastasiia wants to merge 2 commits into
base: RS3.0-UE5.3
Choose a base branch
from
Open

RS Local texture parameters latency improvements #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bafaf78
textureparam: moved registerTexture for RS Local to a different API e…
egorovaanastasiia Mar 12, 2026
4c5866d
rs: log message if plugin fails to load a function instead of faling …
egorovaanastasiia Mar 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 27 additions & 5 deletions Source/RenderStream/Private/RenderStream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,16 +183,38 @@ void FRenderStreamModule::StartupModule()

// This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module

if (!RenderStreamLink::instance().loadExplicit())
FString loadError;
if (!RenderStreamLink::instance().loadExplicit(loadError))
{
UE_LOG(LogRenderStream, Error, TEXT ("Failed to load RenderStream DLL - d3 not installed?"));
if (loadError.IsEmpty())
{
UE_LOG(LogRenderStream, Error, TEXT("Failed to load RenderStream DLL - d3 not installed?"));
}
else
{
if (RenderStreamLink::instance().rs_initialise)
{
int errCode = RenderStreamLink::instance().rs_initialise(RENDER_STREAM_VERSION_MAJOR, RENDER_STREAM_VERSION_MINOR);
if (errCode == RenderStreamLink::RS_ERROR_SUCCESS)
{
FString msg = FString::Printf(TEXT("Failed to load %s from RenderStream DLL - d3 version is incompatible with this plugin version. Please update d3 or use an older plugin version"), *loadError);
if (RenderStreamLink::instance().rs_logToD3)
RenderStreamLink::instance().rs_logToD3(TCHAR_TO_ANSI(*msg));
}
}
if (!GIsEditor)
{
FPlatformMisc::RequestExit(true);
Copy link
Copy Markdown
Contributor

@ivaylo-disguise ivaylo-disguise Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if this will suffer from the same exit infinite loop issue as the one seen in the UE ghost process problem. Try this in an nDisplay session with 2 RXs.

Copy link
Copy Markdown
Contributor Author

@egorovaanastasiia egorovaanastasiia Mar 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, both RXs in this case kill the UE process. All good

}
return;
}
}
else
{
m_logDevice = MakeShared<FRenderStreamLogOutputDevice, ESPMode::ThreadSafe>();

int errCode = RenderStreamLink::instance().rs_initialise(RENDER_STREAM_VERSION_MAJOR, RENDER_STREAM_VERSION_MINOR);

if (errCode != RenderStreamLink::RS_ERROR_SUCCESS)
{
if (errCode == RenderStreamLink::RS_ERROR_INCOMPATIBLE_VERSION)
Expand All @@ -206,7 +228,7 @@ void FRenderStreamModule::StartupModule()
RenderStreamLink::instance().unloadExplicit();
return;
}

FCoreDelegates::OnHandleSystemError.AddRaw(this, &FRenderStreamModule::OnSystemError);

FCoreUObjectDelegates::PostLoadMapWithWorld.AddRaw(this, &FRenderStreamModule::OnPostLoadMapWithWorld);
Expand Down
26 changes: 17 additions & 9 deletions Source/RenderStream/Private/RenderStreamLink.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ namespace {

RenderStreamLink::RenderStreamLink()
{
loadExplicit();
FString unused;
Copy link
Copy Markdown
Contributor

@josharms-disguise josharms-disguise Mar 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unused ? Why do we need this ?

loadExplicit(unused);
}

RenderStreamLink::~RenderStreamLink()
Expand All @@ -87,8 +88,9 @@ bool RenderStreamLink::isAvailable()
return m_dll && m_loaded;
}

bool RenderStreamLink::loadExplicit()
bool RenderStreamLink::loadExplicit(FString& outError)
{
outError.Empty();
if (isAvailable())
return true;

Expand Down Expand Up @@ -157,7 +159,7 @@ bool RenderStreamLink::loadExplicit()
FString osMsg = e.message().c_str();
UE_LOG(LogRenderStream, Error, TEXT("Failed to get function %s from DLL. %s (%i)"), fnName, *osMsg, e.value());
m_loaded = false;
LogFatalIfNotInEditor("A function failed to load from the RenderStream DLL, this suggests an incompatible version is installed.");
outError = fnName;
return false;
}
return true;
Expand All @@ -166,6 +168,8 @@ bool RenderStreamLink::loadExplicit()
#define LOAD_FN(FUNC) \
if (!loadFn(FUNC, TEXT(#FUNC))) \
return false;



LOAD_FN(rs_initialise);
LOAD_FN(rs_initialiseGpGpuWithDX11Device);
Expand All @@ -174,6 +178,10 @@ bool RenderStreamLink::loadExplicit()
LOAD_FN(rs_initialiseGpGpuWithVulkanDevice);
LOAD_FN(rs_shutdown);

LOAD_FN(rs_logToD3);
LOAD_FN(rs_sendProfilingData);
LOAD_FN(rs_setNewStatusMessage);

LOAD_FN(rs_registerLoggingFunc);
LOAD_FN(rs_registerErrorLoggingFunc);
LOAD_FN(rs_registerVerboseLoggingFunc);
Expand All @@ -182,6 +190,10 @@ bool RenderStreamLink::loadExplicit()
LOAD_FN(rs_unregisterErrorLoggingFunc);
LOAD_FN(rs_unregisterVerboseLoggingFunc);

rs_registerLoggingFunc(&log_default);
rs_registerErrorLoggingFunc(&log_error);
rs_registerVerboseLoggingFunc(&log_verbose);

LOAD_FN(rs_useDX12SharedHeapFlag);

LOAD_FN(rs_setSchema);
Expand All @@ -197,6 +209,8 @@ bool RenderStreamLink::loadExplicit()
LOAD_FN(rs_getFrameParameters);
LOAD_FN(rs_getFrameImageData);
LOAD_FN(rs_getFrameImage2);
LOAD_FN(rs_registerTextureParams);

LOAD_FN(rs_getFrameText);

LOAD_FN(rs_getSkeletonLayout);
Expand All @@ -208,15 +222,9 @@ bool RenderStreamLink::loadExplicit()

LOAD_FN(rs_releaseImage2);

LOAD_FN(rs_logToD3);
LOAD_FN(rs_sendProfilingData);
LOAD_FN(rs_setNewStatusMessage);

m_loaded = true;

rs_registerLoggingFunc(&log_default);
rs_registerErrorLoggingFunc(&log_error);
rs_registerVerboseLoggingFunc(&log_verbose);

#endif

Expand Down
8 changes: 8 additions & 0 deletions Source/RenderStream/Private/RenderStreamSceneSelector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "ProfilingDebugging/RealtimeGPUProfiler.h"
#include <Kismet/KismetRenderingLibrary.h>
#include "OpenColorIOBlueprintLibrary.h"
#include "RenderingThread.h"

RenderStreamSceneSelector::~RenderStreamSceneSelector()
{
Expand Down Expand Up @@ -526,6 +527,13 @@ void RenderStreamSceneSelector::ApplyParameters(uint32_t sceneId, const TArray<A
UE_LOG(LogRenderStream, Error, TEXT("Unable to get float frame parameters - %d"), res);
return;
}
if (nImageParams > 0)
{
ENQUEUE_RENDER_COMMAND(RegisterTextureParams)([](FRHICommandListImmediate& RHICmdList) {
RenderStreamLink::instance().rs_registerTextureParams();
});
FlushRenderingCommands();
Copy link
Copy Markdown
Contributor

@ivaylo-disguise ivaylo-disguise Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm worried about the performance impact of synchronizing the GPU and CPU threads with the flush. If we really need the flush, please check performance is not impacted too much.

Copy link
Copy Markdown
Contributor Author

@egorovaanastasiia egorovaanastasiia Mar 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might be right. Seems like with a heavier scene with 9 TPs the fps of the streams dropped by 2-3 frames

}
res = RenderStreamLink::instance().rs_getFrameImageData(params.hash, imageValues.data(), imageValues.size());
if (res != RenderStreamLink::RS_ERROR_SUCCESS)
{
Expand Down
4 changes: 3 additions & 1 deletion Source/RenderStream/Public/RenderStreamLink.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,7 @@ class RenderStreamLink
typedef RS_ERROR rs_getFrameParametersFn(uint64_t schemaHash, /*Out*/void* outParameterData, uint64_t outParameterDataSize); // returns the remote parameters for this frame.
typedef RS_ERROR rs_getFrameImageDataFn(uint64_t schemaHash, /*Out*/ImageFrameData* outParameterData, uint64_t outParameterDataCount); // returns the remote image data for this frame.
typedef RS_ERROR rs_getFrameImageFn(int64_t imageId, /*InOut*/const SenderFrame* data); // fills in (data) with the remote image
typedef RS_ERROR rs_registerTextureParamsFn(); // processes texture parameter registrations, must be called from a render thread
typedef RS_ERROR rs_getFrameTextFn(uint64_t schemaHash, uint32_t textParamIndex, /*Out*/const char** outTextPtr); // // returns the remote text data (pointer only valid until next rs_awaitFrameData)

typedef RS_ERROR rs_getSkeletonLayoutFn(uint64_t schemaHash, uint64_t id, /*Out*/SkeletonLayout* layout, /*Out*/int* numJoints);
Expand All @@ -467,7 +468,7 @@ class RenderStreamLink
public:
RENDERSTREAM_API bool isAvailable();

bool loadExplicit();
bool loadExplicit(FString& outError);
bool unloadExplicit();

struct ScopedSchema
Expand Down Expand Up @@ -581,6 +582,7 @@ class RenderStreamLink
rs_getFrameParametersFn* rs_getFrameParameters = nullptr;
rs_getFrameImageDataFn* rs_getFrameImageData = nullptr;
rs_getFrameImageFn* rs_getFrameImage2 = nullptr;
rs_registerTextureParamsFn* rs_registerTextureParams = nullptr;
rs_getFrameTextFn* rs_getFrameText = nullptr;
rs_getSkeletonLayoutFn* rs_getSkeletonLayout = nullptr;
rs_getSkeletonJointNamesFn* rs_getSkeletonJointNames = nullptr;
Expand Down