Stat Tracking and Test Result reporting

.github/workflows/BuildAndRunTests.yml

@@ -27,18 +27,43 @@ on:
       - '**.cmake'
       - '**BuildAndRunTests.yml'
 jobs:
-    Build-And-Test:
-        runs-on: windows-latest
-        steps:
-          - name: Checkout the repo
-            uses: actions/checkout@v4
+  Build-And-Test:
 
-          - name: Setup CMake
-            uses: threeal/cmake-action@v1.3.0
+    permissions:
+      checks: write
+      pull-requests: write
 
-          - name: Build the project
-            run: cmake --build build
+    runs-on: windows-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
 
-          - name: Test
-            run: ctest --test-dir build -C Debug --output-on-failure
+      - name: Print CMake version
+        shell: cmake -P {0}
+        run: |
+          message(STATUS "Host CMake version: ${CMAKE_VERSION}")
+
+      - name: Setup CMake
+        uses: threeal/cmake-action@v1.3.0
+
+      - name: Build the project
+        run: cmake --build build
+
+      - name: Test
+        run: ctest --test-dir build -C Debug --output-on-failure
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action/windows@v2
+        if: always()
+        with:
+          files: |
+            build\test\Debug\Reports\*.xml
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4.3.1
+        if: always()
+        with:
+          report_paths: build\test\Debug\Reports\*.xml
+          detailed_summary: true
+
 

cmake/ExternalProjects.cmake

@@ -15,7 +15,13 @@ function(add_catch2 IN_TARGET)
     list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
 
     include(Catch)
-    catch_discover_tests(${IN_TARGET} WORKING_DIRECTORY "$<TARGET_FILE_DIR:${IN_TARGET}>")
+    catch_discover_tests(${IN_TARGET}
+        EXTRA_ARGS "--durations yes"
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:${IN_TARGET}>"
+        REPORTER junit
+        OUTPUT_DIR Reports
+        OUTPUT_SUFFIX ".xml"
+        )
     return()
 endfunction()
 

src/ShaderTestFramework/CMakeLists.txt

@@ -51,6 +51,8 @@ set(SOURCES
     Public/Framework/TestDataBufferProcessor.h
     Public/Framework/TestDataBufferStructs.h
     Public/Platform.h
+    Public/Stats/ScopedObject.h
+    Public/Stats/StatSystem.h
     Public/Utility/Algorithm.h
     Public/Utility/Concepts.h
     Public/Utility/EnumReflection.h
@@ -64,6 +66,7 @@ set(SOURCES
     Public/Utility/MoveOnly.h
     Public/Utility/OverloadSet.h
     Public/Utility/Pointer.h
+    Public/Utility/Time.h
     Public/Utility/Tuple.h
     Public/Utility/Type.h
     Public/Utility/TypeList.h
@@ -89,6 +92,7 @@ set(SOURCES
     Private/Framework/TestDataBufferLayout.cpp
     Private/Framework/TestDataBufferProcessor.cpp
     Private/Framework/TestDataBufferStructs.cpp
+    Private/Stats/StatSystem.cpp
 )
 
 set(SHADER_SOURCES

src/ShaderTestFramework/Private/Framework/ShaderTestFixture.cpp

@@ -25,6 +25,9 @@ namespace
     }
 }
 
+StatSystem ShaderTestFixture::statSystem;
+std::vector<TimedStat> ShaderTestFixture::cachedStats;
+
 ShaderTestFixture::ShaderTestFixture(Desc InParams)
 	: m_Device()
 	, m_Compiler()
@@ -36,6 +39,9 @@ ShaderTestFixture::ShaderTestFixture(Desc InParams)
     , m_TestDataLayout(InParams.TestDataLayout)
 	, m_IsWarp(InParams.GPUDeviceParams.DeviceType == GPUDevice::EDeviceType::Software)
 {
+    ScopedDuration scope("Shader Test Fixture construction");
+    cachedStats.clear();
+
     fs::path shaderDir = std::filesystem::current_path();
     shaderDir += "/";
     shaderDir += SHADER_SRC;
@@ -47,13 +53,19 @@ ShaderTestFixture::ShaderTestFixture(Desc InParams)
     PopulateDefaultByteReaders();
 }
 
+ShaderTestFixture::~ShaderTestFixture() noexcept
+{
+    cachedStats = statSystem.FlushTimedStats();
+}
+
 void ShaderTestFixture::TakeCapture()
 {
     m_CaptureRequested = true;
 }
 
 STF::Results ShaderTestFixture::RunTest(const std::string_view InName, u32 InX, u32 InY, u32 InZ)
 {
+    ScopedDuration fullTest(std::format("ShaderTestFixture::RunTest: {}", InName));
 	const auto compileResult = CompileShader(InName, EShaderType::Compute);
 
 	if (!compileResult.has_value())
@@ -90,6 +102,7 @@ STF::Results ShaderTestFixture::RunTest(const std::string_view InName, u32 InX,
     const auto allocationUAV = CreateAssertBufferUAV(allocationBuffer, resourceHeap, 1);
 
     {
+        ScopedDuration testExecution("ShaderTestFixture::RunTest Test Execution");
         const auto capturer = PIXCapturer(InName, ShouldTakeCapture());
 
         engine.Execute(InName,
@@ -149,11 +162,15 @@ STF::Results ShaderTestFixture::RunTest(const std::string_view InName, u32 InX,
         engine.Flush();
     }
 
-    return ReadbackResults(readBackAllocationBuffer, readBackBuffer, uint3(dimX, dimY, dimZ));
+    {
+        ScopedDuration readbackScope(std::format("ShaderTestFixture::ReadbackResults: {}", InName));
+        return ReadbackResults(readBackAllocationBuffer, readBackBuffer, uint3(dimX, dimY, dimZ));
+    }
 }
 
 STF::Results ShaderTestFixture::RunCompileTimeTest()
 {
+    ScopedDuration scope("ShaderTestFixture::RunCompileTimeTest");
     const auto compileResult = CompileShader("", EShaderType::Lib);
 
     if (!compileResult.has_value())
@@ -179,8 +196,14 @@ bool ShaderTestFixture::IsUsingAgilitySDK() const
 	return m_Device.GetHardwareInfo().FeatureInfo.EnhancedBarriersSupport;
 }
 
+std::vector<TimedStat> ShaderTestFixture::GetTestStats()
+{
+    return cachedStats;
+}
+
 CompilationResult ShaderTestFixture::CompileShader(const std::string_view InName, const EShaderType InType) const
 {
+    ScopedDuration scope(std::format("ShaderTestFixture::CompileShader: {}", InName));
     ShaderCompilationJobDesc job;
     job.AdditionalFlags = m_CompilationFlags;
     job.AdditionalFlags.emplace_back(L"-enable-16bit-types");

src/ShaderTestFramework/Private/Stats/StatSystem.cpp

@@ -0,0 +1,45 @@
+#include "Stats/StatSystem.h"
+#include <ranges>
+
+StatSystem::Handle StatSystem::BeginStat(std::string InName)
+{
+    if (m_RawTimedStats.size() > Handle::MaxIds)
+    {
+        return Handle{ 0u, 0u, false };
+    }
+    const u32 id = static_cast<u32>(m_RawTimedStats.size());
+
+    m_RawTimedStats.push_back(RawTimedStat{ std::move(InName), Clock::now() });
+
+    return Handle{id, m_CurrentGen, true};
+}
+
+void StatSystem::EndStat(const Handle InHandle)
+{
+    if (InHandle && InHandle.GetGeneration() == m_CurrentGen && InHandle.GetId() < m_RawTimedStats.size())
+    {
+        auto& rawStat = m_RawTimedStats[InHandle.GetId()];
+        rawStat.EndTime = Clock::now();
+    }
+}
+
+std::vector<TimedStat> StatSystem::FlushTimedStats()
+{
+    std::vector<TimedStat> ret;
+
+    for (auto&& completed : std::views::all(m_RawTimedStats)
+                            | std::views::filter(
+                                [](const RawTimedStat& In) 
+                                { 
+                                    return In.EndTime.time_since_epoch().count() != 0; 
+                                }))
+    {
+        ret.push_back(TimedStat{ std::move(completed.Name), completed.EndTime - completed.StartTime });
+    }
+
+    m_RawTimedStats.clear();
+    m_CurrentGen = (m_CurrentGen + 1) % Handle::MaxGens;
+
+    return ret;
+}
+

src/ShaderTestFramework/Public/Framework/ShaderTestFixture.h

@@ -7,6 +7,7 @@
 #include "Framework/HLSLTypes.h"
 #include "Framework/TestDataBufferLayout.h"
 #include "Framework/TestDataBufferProcessor.h"
+#include "Stats/StatSystem.h"
 #include <optional>
 #include <vector>
 
@@ -30,6 +31,7 @@ class ShaderTestFixture
     };
 
     ShaderTestFixture(Desc InParams);
+    ~ShaderTestFixture() noexcept;
 
     void TakeCapture();
     STF::Results RunTest(const std::string_view InName, u32 InX, u32 InY, u32 InZ);
@@ -41,8 +43,15 @@ class ShaderTestFixture
 
     bool IsUsingAgilitySDK() const;
 
+    static std::vector<TimedStat> GetTestStats();
+
 private:
 
+    static StatSystem statSystem;
+    static constexpr auto StatSystemGetter = []() -> StatSystem& { return statSystem; };
+    using ScopedDuration = ScopedCPUDurationStat<StatSystemGetter>;
+    static std::vector<TimedStat> cachedStats;
+
     CompilationResult CompileShader(const std::string_view InName,  const EShaderType InType) const;
     CommandEngine CreateCommandEngine() const;
     DescriptorHeap CreateDescriptorHeap() const;

src/ShaderTestFramework/Public/Stats/ScopedObject.h

@@ -0,0 +1,37 @@
+
+#include "Utility/Concepts.h"
+
+template<CallableType CallOnDestruction>
+class ScopedObject
+{
+public:
+    template<CallableType CallOnConstruction, CallableType OtherCallOnDestruction>
+    ScopedObject(CallOnConstruction&& OnConstruction, OtherCallOnDestruction&& OnDestruction)
+        : m_OnDestruction(std::forward<OtherCallOnDestruction>(OnDestruction))
+    {
+        OnConstruction();
+    }
+
+    ~ScopedObject() noexcept
+    {
+        m_OnDestruction();
+    }
+
+    ScopedObject(const ScopedObject&) = delete;
+    ScopedObject(ScopedObject&&) = delete;
+    ScopedObject& operator=(const ScopedObject&) = delete;
+    ScopedObject& operator=(ScopedObject&&) = delete;
+
+    template<typename... Ts>
+    static void* operator new(std::size_t, Ts&&...) = delete;
+
+    template<typename... Ts>
+    static void* operator new[](std::size_t, Ts&&...) = delete;
+
+private:
+
+    CallOnDestruction m_OnDestruction;
+};
+
+template<typename OnConstruct, typename OnDestruct>
+ScopedObject(OnConstruct, OnDestruct) -> ScopedObject<OnDestruct>;