update E2E test to work with polling

Apps/Find/src/SUI.Find.Application/Dtos/SearchResultEntry.cs

@@ -1,22 +1,14 @@
+using SUI.Find.Application.Models;
+
 namespace SUI.Find.Application.Dtos;
 
-public class SearchResultEntry
+public record SearchResultEntry : SearchResultItem
 {
     /// <summary>
     /// Submitting custodian's Org ID
     /// </summary>
     public required string CustodianId { get; init; }
 
-    public required string SystemId { get; init; }
-
-    /// <summary>
-    /// Submitting custodian's Org Name
-    /// </summary>
-    public required string CustodianName { get; init; }
-
-    public required string RecordType { get; init; }
-    public required string RecordUrl { get; init; }
-    public string? RecordId { get; init; }
     public DateTimeOffset SubmittedAtUtc { get; init; }
     public required string JobId { get; init; }
     public required string WorkItemId { get; init; }

Apps/Find/src/SUI.Find.Application/Models/SearchResultItem.cs

@@ -2,7 +2,7 @@
 
 namespace SUI.Find.Application.Models;
 
-public sealed record SearchResultItem
+public record SearchResultItem
 {
     public required string RecordType { get; init; }
     public required string RecordUrl { get; init; }

Apps/Find/src/SUI.Find.Application/Models/SearchResultsV2Dto.cs

@@ -1,13 +1,7 @@
-using SUI.Find.Application.Dtos;
-using SUI.Find.Application.Enums;
-
 namespace SUI.Find.Application.Models;
 
-public record SearchResultsV2Dto
+public record SearchResultsV2Dto : SearchResultsDto
 {
     public string WorkItemId { get; init; } = string.Empty;
-    public string Suid { get; init; } = string.Empty;
-    public SearchStatus Status { get; init; }
-    public IReadOnlyList<SearchResultEntry> Items { get; init; } = [];
     public int CompletenessPercentage { get; init; }
 }

Apps/Find/src/SUI.Find.FindApi/Models/SearchResults.cs

@@ -1,20 +1,14 @@
 using System.Text.Json.Serialization;
-using SUI.Find.Application.Enums;
 using SUI.Find.Application.Models;
 
 namespace SUI.Find.FindApi.Models;
 
-public record SearchResults
+public record SearchResults : SearchResultsBase
 {
-    public required string JobId { get; set; } = string.Empty;
-    public required string Suid { get; set; } = string.Empty;
-
-    [JsonConverter(typeof(JsonStringEnumConverter))]
-    public required SearchStatus Status { get; set; }
-    public required SearchResultItem[] Items { get; set; } = [];
+    public required string JobId { get; init; } = string.Empty;
 
     [JsonPropertyName("_links")]
-    public required Dictionary<string, HalLink> Links { get; set; } = [];
+    public required Dictionary<string, HalLink> Links { get; init; } = [];
 
     public static SearchResults FromDto(SearchResultsDto dto)
     {
@@ -24,7 +18,7 @@ public static SearchResults FromDto(SearchResultsDto dto)
             Suid = dto.Suid,
             Status = dto.Status,
             Items = dto.Items,
-            Links = new Dictionary<string, HalLink>()
+            Links = new Dictionary<string, HalLink>
             {
                 { "self", new HalLink($"/search/{dto.JobId}/results", "GET") },
                 { "job", new HalLink($"/search/{dto.JobId}", "GET") },

Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsBase.cs

@@ -0,0 +1,17 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json.Serialization;
+using SUI.Find.Application.Enums;
+using SUI.Find.Application.Models;
+
+namespace SUI.Find.FindApi.Models;
+
+public record SearchResultsBase
+{
+    [ExcludeFromCodeCoverage]
+    public required string Suid { get; init; } = string.Empty;
+
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public required SearchStatus Status { get; init; }
+
+    public SearchResultItem[] Items { get; init; } = [];
+}

Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsV2.cs

@@ -1,20 +1,12 @@
-using System.Text.Json.Serialization;
-using SUI.Find.Application.Dtos;
-using SUI.Find.Application.Enums;
 using SUI.Find.Application.Models;
 
 namespace SUI.Find.FindApi.Models;
 
-public record SearchResultsV2
+public record SearchResultsV2 : SearchResultsBase
 {
     public required string WorkItemId { get; set; } = string.Empty;
-    public required string Suid { get; set; } = string.Empty;
-
-    [JsonConverter(typeof(JsonStringEnumConverter))]
-    public required SearchStatus Status { get; set; }
 
     public required int CompletenessPercentage { get; set; }
-    public required IReadOnlyList<SearchResultEntry> Items { get; set; } = [];
 
     public static SearchResultsV2 FromDto(SearchResultsV2Dto dto)
     {

Apps/Find/src/SUI.Find.Infrastructure/Services/JobSearchService.cs

@@ -73,7 +73,7 @@ await workItemJobCountRepository.GetByWorkItemIdAndJobTypeAsync(
             WorkItemId = workItemId,
             Suid = payload?.Sui ?? string.Empty,
             Status = status,
-            Items = completedRecords,
+            Items = completedRecords.Select(SearchResultItem (x) => x).ToArray(),
             CompletenessPercentage = completenessPercentage,
         };
 

Apps/Find/tests/SUI.Find.E2ETests/StartANewSearchTests.cs

@@ -35,7 +35,7 @@ public class StartANewSearchTests(FunctionTestFixture fixture, ITestOutputHelper
         "fetch-record.read",
     ];
 
-    private record SearchStatusResponse(string? Status);
+    private record SearchStatusResponse(string? Status, int? CompletenessPercentage);
 
     public async Task InitializeAsync()
     {
@@ -47,6 +47,8 @@ await ResetAzureTablesAsync([
                 "ResultsUrlMappings",
                 "TestHubNameHistory",
                 "TestHubNameInstances",
+                "Jobs",
+                "WorkItemJobCounts",
             ]);
         }
     }
@@ -149,7 +151,24 @@ private async Task ResetAzureTablesAsync(string[] tableNames)
         "xUnit1045",
         Justification = "The `TestData` is a C# record, and the default string serialization of records provides distinct text for the purposes of test exploration, identification and results."
     )]
-    public async Task Should_PersistSearchData_When_OrchestrationCompletes(TestData testData)
+    public async Task Should_PersistSearchData_When_OrchestrationCompletesV1(TestData testData)
+    {
+        await RunTest(testData, false);
+    }
+
+    [Theory]
+    [MemberData(nameof(TestData))]
+    [SuppressMessage(
+        "Usage",
+        "xUnit1045",
+        Justification = "The `TestData` is a C# record, and the default string serialization of records provides distinct text for the purposes of test exploration, identification and results."
+    )]
+    public async Task Should_PersistSearchData_When_OrchestrationCompletesV2(TestData testData)
+    {
+        await RunTest(testData, true);
+    }
+
+    private async Task RunTest(TestData testData, bool usePolling)
     {
         var authToken = await GetAuthTokenAsync(
             testData.TestClientId,
@@ -162,33 +181,40 @@ public async Task Should_PersistSearchData_When_OrchestrationCompletes(TestData
         );
 
         // Step 1, start a new search
-        var newSearchJob = await RunAndAssertNewSearchEndpoint(
-            Fixture.Config.UseEncryptedIds ? testData.EncryptedSui : testData.Sui
+        var searchJobLinks = await RunAndAssertNewSearchEndpoint(
+            Fixture.Config.UseEncryptedIds ? testData.EncryptedSui : testData.Sui,
+            usePolling
         );
 
-        var hasStatusLink = newSearchJob.Links.TryGetValue("status", out var statusLink);
-        Assert.True(hasStatusLink);
+        var hasStatusLink = searchJobLinks.TryGetValue("status", out var statusLink);
+        if (!usePolling)
+        {
+            Assert.True(hasStatusLink);
+        }
 
-        var hasResultsLink = newSearchJob.Links.TryGetValue("results", out var resultsLink);
+        var hasResultsLink = searchJobLinks.TryGetValue("results", out var resultsLink);
         Assert.True(hasResultsLink);
 
         // Step 2, check the status
-        var statusResult = await RunAndAwaitAndAssertSearchStatusCompletion(
-            statusLink!.Href,
+        await RunAndAwaitAndAssertSearchStatusCompletion(
+            statusLink != null ? statusLink.Href : resultsLink!.Href,
             resultsLink!.Href,
-            testData
+            testData,
+            usePolling
         );
-        Assert.NotNull(statusResult);
 
         // Step 3, Get the results from fetch and assert
         await RunAndAssertFetchEndpoints(resultsLink.Href, testData);
 
         // Fin
     }
 
-    private async Task<SearchJob> RunAndAssertNewSearchEndpoint(string suid)
+    private async Task<Dictionary<string, HalLink>> RunAndAssertNewSearchEndpoint(
+        string suid,
+        bool usePolling
+    )
     {
-        const string startSearchUrl = "v1/searches";
+        var startSearchUrl = usePolling ? "v2/searches" : "v1/searches";
 
         TestOutputHelper.WriteLine(
             $"Starting new search to find records for SUI: {suid} ({Fixture.Client.BaseAddress}{startSearchUrl})"
@@ -204,8 +230,28 @@ private async Task<SearchJob> RunAndAssertNewSearchEndpoint(string suid)
         // We then want to assert the returned body and status code
         Assert.Equal(HttpStatusCode.Accepted, newSearchJobResult.StatusCode);
         var searchJobContent = await newSearchJobResult.Content.ReadAsStringAsync();
-        var searchJob = JsonSerializer.Deserialize<SearchJob>(searchJobContent);
-        Assert.False(string.IsNullOrEmpty(searchJob!.JobId));
+        var searchId = string.Empty;
+        var links = new Dictionary<string, HalLink>();
+        if (usePolling)
+        {
+            var searchJob = JsonSerializer.Deserialize<SearchJobV2>(searchJobContent);
+            if (searchJob != null)
+            {
+                searchId = searchJob.WorkItemId;
+                links = searchJob.Links;
+            }
+        }
+        else
+        {
+            var searchJob = JsonSerializer.Deserialize<SearchJob>(searchJobContent);
+            if (searchJob != null)
+            {
+                searchId = searchJob.JobId;
+                links = searchJob.Links;
+            }
+        }
+
+        Assert.False(string.IsNullOrEmpty(searchId));
 
         var traceId =
             (
@@ -221,8 +267,9 @@ private async Task<SearchJob> RunAndAssertNewSearchEndpoint(string suid)
                     : null
             ) ?? "Unknown";
 
+        var topLevelTaskName = usePolling ? "workItemId" : "jobId";
         TestOutputHelper.WriteLine(
-            $"Search started: {new { traceId, invocationId, jobId = searchJob.JobId }}"
+            $"Search started: traceId={traceId}, invocationId={invocationId}, {topLevelTaskName}={searchId}"
         );
 
         var observabilityLink = Fixture.Config.IsLocal
@@ -233,7 +280,7 @@ private async Task<SearchJob> RunAndAssertNewSearchEndpoint(string suid)
         TestOutputHelper.WriteLine($"Trace observability: {observabilityLink}");
         TestOutputHelper.WriteLine("");
 
-        return searchJob;
+        return links;
     }
 
     private async Task RunAndAssertPartialSearchResults(string url, TestData testData)
@@ -246,7 +293,8 @@ private async Task RunAndAssertPartialSearchResults(string url, TestData testDat
 
         var searchResults = await Fixture.Client.GetAsync(url);
         var searchResultContent = await searchResults.Content.ReadAsStringAsync();
-        var searchResultTypedContent = JsonSerializer.Deserialize<SearchResults>(
+
+        var searchResultTypedContent = JsonSerializer.Deserialize<SearchResultsBase>(
             searchResultContent
         );
 
@@ -281,7 +329,7 @@ private async Task RunAndAssertFetchEndpoints(string url, TestData testData)
         var searchResults = await Fixture.Client.GetAsync(url);
         // Look for URL link and save as variable
         var searchResultContent = await searchResults.Content.ReadAsStringAsync();
-        var searchResultTypedContent = JsonSerializer.Deserialize<SearchResults>(
+        var searchResultTypedContent = JsonSerializer.Deserialize<SearchResultsBase>(
             searchResultContent
         );
         Assert.True(
@@ -369,18 +417,22 @@ private async Task RunAndAssertFetchEndpoints(string url, TestData testData)
     /// <param name="statusUrl">URL of the Search Status endpoint</param>
     /// <param name="resultsUrl">URL of the Search Results endpoint</param>
     /// <param name="testData">The test data for this search run</param>
-    private async Task<SearchJob> RunAndAwaitAndAssertSearchStatusCompletion(
+    /// <param name="usePolling">If true, specifies the tests are being run for the Polling Architecture, rather than Fan-out.</param>
+    private async Task RunAndAwaitAndAssertSearchStatusCompletion(
         string statusUrl,
         string resultsUrl,
-        TestData testData
+        TestData testData,
+        bool usePolling
     )
     {
         statusUrl = RemoveLeadingSlashFromUrl(statusUrl);
 
-        const int retryCount = 150;
+        var timeout = TimeSpan.FromMinutes(usePolling ? 10 : 5);
+        var retryDelay = TimeSpan.FromSeconds(2);
+        var retryCount = (int)Math.Round(timeout / retryDelay);
 
         var isCompleted = false;
-        var status = "unknown";
+        var statusMessage = "unknown";
 
         TestOutputHelper.WriteLine(
             $"Checking for search completion: {Fixture.Client.BaseAddress}{statusUrl}"
@@ -391,13 +443,14 @@ TestData testData
             {
                 if (!r.IsSuccessStatusCode)
                 {
-                    status = $"{r.StatusCode} - {r.Content.ReadAsStringAsync().Result}";
+                    statusMessage = $"{r.StatusCode} - {r.Content.ReadAsStringAsync().Result}";
                     return true; // i.e. retry
                 }
 
                 // Read response, if status is NOT "Completed", keep retrying
                 var content = r.Content.ReadFromJsonAsync<SearchStatusResponse>().Result;
-                status = content?.Status;
+                var status = content?.Status;
+
                 switch (status)
                 {
                     case "Completed":
@@ -410,29 +463,38 @@ TestData testData
                 }
 
                 var retry = status is "Queued" or "Running";
+
+                statusMessage =
+                    content?.CompletenessPercentage != null
+                        ? $"{status} ({content.CompletenessPercentage}%)"
+                        : status;
+
                 return retry;
             })
             .WaitAndRetryAsync(
                 retryCount,
                 retryAttempt =>
                 {
                     TestOutputHelper.WriteLine(
-                        $"Still checking for search completion, status was {status}, retry {retryAttempt} / {retryCount}..."
+                        $"Still checking for search completion, status was {statusMessage}, retry {retryAttempt} / {retryCount}..."
                     );
-                    return TimeSpan.FromSeconds(2);
+                    return retryDelay;
                 }
             );
 
         await retryPolicy.ExecuteAsync(() => Fixture.Client.GetAsync(statusUrl));
 
         TestOutputHelper.WriteLine(
-            $"Finished checking for search completion, final status was {status}, is completed: {isCompleted}"
+            $"Finished checking for search completion, final status was {statusMessage}, is completed: {isCompleted}"
         );
 
         Assert.True(isCompleted);
 
-        var typedResult = await Fixture.Client.GetFromJsonAsync<SearchJob>(statusUrl);
-        return typedResult!;
+        object? typedResult = usePolling
+            ? await Fixture.Client.GetFromJsonAsync<SearchJobV2>(statusUrl)
+            : await Fixture.Client.GetFromJsonAsync<SearchJob>(statusUrl);
+
+        Assert.NotNull(typedResult);
     }
 
     private static string RemoveLeadingSlashFromUrl(string url) =>