diff --git a/Apps/Find/src/SUI.Find.Application/Dtos/SearchResultEntry.cs b/Apps/Find/src/SUI.Find.Application/Dtos/SearchResultEntry.cs index 3c0ce28bc..22f37d169 100644 --- a/Apps/Find/src/SUI.Find.Application/Dtos/SearchResultEntry.cs +++ b/Apps/Find/src/SUI.Find.Application/Dtos/SearchResultEntry.cs @@ -1,22 +1,14 @@ +using SUI.Find.Application.Models; + namespace SUI.Find.Application.Dtos; -public class SearchResultEntry +public record SearchResultEntry : SearchResultItem { /// /// Submitting custodian's Org ID /// public required string CustodianId { get; init; } - public required string SystemId { get; init; } - - /// - /// Submitting custodian's Org Name - /// - public required string CustodianName { get; init; } - - public required string RecordType { get; init; } - public required string RecordUrl { get; init; } - public string? RecordId { get; init; } public DateTimeOffset SubmittedAtUtc { get; init; } public required string JobId { get; init; } public required string WorkItemId { get; init; } diff --git a/Apps/Find/src/SUI.Find.Application/Models/SearchResultItem.cs b/Apps/Find/src/SUI.Find.Application/Models/SearchResultItem.cs index 74b40aff0..643ccc08f 100644 --- a/Apps/Find/src/SUI.Find.Application/Models/SearchResultItem.cs +++ b/Apps/Find/src/SUI.Find.Application/Models/SearchResultItem.cs @@ -2,7 +2,7 @@ namespace SUI.Find.Application.Models; -public sealed record SearchResultItem +public record SearchResultItem { public required string RecordType { get; init; } public required string RecordUrl { get; init; } diff --git a/Apps/Find/src/SUI.Find.Application/Models/SearchResultsV2Dto.cs b/Apps/Find/src/SUI.Find.Application/Models/SearchResultsV2Dto.cs index bec307d76..df8c3111d 100644 --- a/Apps/Find/src/SUI.Find.Application/Models/SearchResultsV2Dto.cs +++ b/Apps/Find/src/SUI.Find.Application/Models/SearchResultsV2Dto.cs @@ -1,13 +1,7 @@ -using SUI.Find.Application.Dtos; -using SUI.Find.Application.Enums; - namespace SUI.Find.Application.Models; -public record SearchResultsV2Dto +public record SearchResultsV2Dto : SearchResultsDto { public string WorkItemId { get; init; } = string.Empty; - public string Suid { get; init; } = string.Empty; - public SearchStatus Status { get; init; } - public IReadOnlyList Items { get; init; } = []; public int CompletenessPercentage { get; init; } } diff --git a/Apps/Find/src/SUI.Find.FindApi/Models/SearchResults.cs b/Apps/Find/src/SUI.Find.FindApi/Models/SearchResults.cs index b34892335..ae002486f 100644 --- a/Apps/Find/src/SUI.Find.FindApi/Models/SearchResults.cs +++ b/Apps/Find/src/SUI.Find.FindApi/Models/SearchResults.cs @@ -1,20 +1,14 @@ using System.Text.Json.Serialization; -using SUI.Find.Application.Enums; using SUI.Find.Application.Models; namespace SUI.Find.FindApi.Models; -public record SearchResults +public record SearchResults : SearchResultsBase { - public required string JobId { get; set; } = string.Empty; - public required string Suid { get; set; } = string.Empty; - - [JsonConverter(typeof(JsonStringEnumConverter))] - public required SearchStatus Status { get; set; } - public required SearchResultItem[] Items { get; set; } = []; + public required string JobId { get; init; } = string.Empty; [JsonPropertyName("_links")] - public required Dictionary Links { get; set; } = []; + public required Dictionary Links { get; init; } = []; public static SearchResults FromDto(SearchResultsDto dto) { @@ -24,7 +18,7 @@ public static SearchResults FromDto(SearchResultsDto dto) Suid = dto.Suid, Status = dto.Status, Items = dto.Items, - Links = new Dictionary() + Links = new Dictionary { { "self", new HalLink($"/search/{dto.JobId}/results", "GET") }, { "job", new HalLink($"/search/{dto.JobId}", "GET") }, diff --git a/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsBase.cs b/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsBase.cs new file mode 100644 index 000000000..970783b48 --- /dev/null +++ b/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsBase.cs @@ -0,0 +1,17 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using SUI.Find.Application.Enums; +using SUI.Find.Application.Models; + +namespace SUI.Find.FindApi.Models; + +public record SearchResultsBase +{ + [ExcludeFromCodeCoverage] + public required string Suid { get; init; } = string.Empty; + + [JsonConverter(typeof(JsonStringEnumConverter))] + public required SearchStatus Status { get; init; } + + public SearchResultItem[] Items { get; init; } = []; +} diff --git a/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsV2.cs b/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsV2.cs index 9beba6d92..386ad34ec 100644 --- a/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsV2.cs +++ b/Apps/Find/src/SUI.Find.FindApi/Models/SearchResultsV2.cs @@ -1,20 +1,12 @@ -using System.Text.Json.Serialization; -using SUI.Find.Application.Dtos; -using SUI.Find.Application.Enums; using SUI.Find.Application.Models; namespace SUI.Find.FindApi.Models; -public record SearchResultsV2 +public record SearchResultsV2 : SearchResultsBase { public required string WorkItemId { get; set; } = string.Empty; - public required string Suid { get; set; } = string.Empty; - - [JsonConverter(typeof(JsonStringEnumConverter))] - public required SearchStatus Status { get; set; } public required int CompletenessPercentage { get; set; } - public required IReadOnlyList Items { get; set; } = []; public static SearchResultsV2 FromDto(SearchResultsV2Dto dto) { diff --git a/Apps/Find/src/SUI.Find.Infrastructure/Services/JobSearchService.cs b/Apps/Find/src/SUI.Find.Infrastructure/Services/JobSearchService.cs index 5480c0cf4..5482e451a 100644 --- a/Apps/Find/src/SUI.Find.Infrastructure/Services/JobSearchService.cs +++ b/Apps/Find/src/SUI.Find.Infrastructure/Services/JobSearchService.cs @@ -73,7 +73,7 @@ await workItemJobCountRepository.GetByWorkItemIdAndJobTypeAsync( WorkItemId = workItemId, Suid = payload?.Sui ?? string.Empty, Status = status, - Items = completedRecords, + Items = completedRecords.Select(SearchResultItem (x) => x).ToArray(), CompletenessPercentage = completenessPercentage, }; diff --git a/Apps/Find/tests/SUI.Find.E2ETests/StartANewSearchTests.cs b/Apps/Find/tests/SUI.Find.E2ETests/StartANewSearchTests.cs index 4999d1825..b8d4cf5cf 100644 --- a/Apps/Find/tests/SUI.Find.E2ETests/StartANewSearchTests.cs +++ b/Apps/Find/tests/SUI.Find.E2ETests/StartANewSearchTests.cs @@ -35,7 +35,7 @@ public class StartANewSearchTests(FunctionTestFixture fixture, ITestOutputHelper "fetch-record.read", ]; - private record SearchStatusResponse(string? Status); + private record SearchStatusResponse(string? Status, int? CompletenessPercentage); public async Task InitializeAsync() { @@ -47,6 +47,8 @@ await ResetAzureTablesAsync([ "ResultsUrlMappings", "TestHubNameHistory", "TestHubNameInstances", + "Jobs", + "WorkItemJobCounts", ]); } } @@ -149,7 +151,24 @@ private async Task ResetAzureTablesAsync(string[] tableNames) "xUnit1045", Justification = "The `TestData` is a C# record, and the default string serialization of records provides distinct text for the purposes of test exploration, identification and results." )] - public async Task Should_PersistSearchData_When_OrchestrationCompletes(TestData testData) + public async Task Should_PersistSearchData_When_OrchestrationCompletesV1(TestData testData) + { + await RunTest(testData, false); + } + + [Theory] + [MemberData(nameof(TestData))] + [SuppressMessage( + "Usage", + "xUnit1045", + Justification = "The `TestData` is a C# record, and the default string serialization of records provides distinct text for the purposes of test exploration, identification and results." + )] + public async Task Should_PersistSearchData_When_OrchestrationCompletesV2(TestData testData) + { + await RunTest(testData, true); + } + + private async Task RunTest(TestData testData, bool usePolling) { var authToken = await GetAuthTokenAsync( testData.TestClientId, @@ -162,23 +181,27 @@ public async Task Should_PersistSearchData_When_OrchestrationCompletes(TestData ); // Step 1, start a new search - var newSearchJob = await RunAndAssertNewSearchEndpoint( - Fixture.Config.UseEncryptedIds ? testData.EncryptedSui : testData.Sui + var searchJobLinks = await RunAndAssertNewSearchEndpoint( + Fixture.Config.UseEncryptedIds ? testData.EncryptedSui : testData.Sui, + usePolling ); - var hasStatusLink = newSearchJob.Links.TryGetValue("status", out var statusLink); - Assert.True(hasStatusLink); + var hasStatusLink = searchJobLinks.TryGetValue("status", out var statusLink); + if (!usePolling) + { + Assert.True(hasStatusLink); + } - var hasResultsLink = newSearchJob.Links.TryGetValue("results", out var resultsLink); + var hasResultsLink = searchJobLinks.TryGetValue("results", out var resultsLink); Assert.True(hasResultsLink); // Step 2, check the status - var statusResult = await RunAndAwaitAndAssertSearchStatusCompletion( - statusLink!.Href, + await RunAndAwaitAndAssertSearchStatusCompletion( + statusLink != null ? statusLink.Href : resultsLink!.Href, resultsLink!.Href, - testData + testData, + usePolling ); - Assert.NotNull(statusResult); // Step 3, Get the results from fetch and assert await RunAndAssertFetchEndpoints(resultsLink.Href, testData); @@ -186,9 +209,12 @@ public async Task Should_PersistSearchData_When_OrchestrationCompletes(TestData // Fin } - private async Task RunAndAssertNewSearchEndpoint(string suid) + private async Task> RunAndAssertNewSearchEndpoint( + string suid, + bool usePolling + ) { - const string startSearchUrl = "v1/searches"; + var startSearchUrl = usePolling ? "v2/searches" : "v1/searches"; TestOutputHelper.WriteLine( $"Starting new search to find records for SUI: {suid} ({Fixture.Client.BaseAddress}{startSearchUrl})" @@ -204,8 +230,28 @@ private async Task RunAndAssertNewSearchEndpoint(string suid) // We then want to assert the returned body and status code Assert.Equal(HttpStatusCode.Accepted, newSearchJobResult.StatusCode); var searchJobContent = await newSearchJobResult.Content.ReadAsStringAsync(); - var searchJob = JsonSerializer.Deserialize(searchJobContent); - Assert.False(string.IsNullOrEmpty(searchJob!.JobId)); + var searchId = string.Empty; + var links = new Dictionary(); + if (usePolling) + { + var searchJob = JsonSerializer.Deserialize(searchJobContent); + if (searchJob != null) + { + searchId = searchJob.WorkItemId; + links = searchJob.Links; + } + } + else + { + var searchJob = JsonSerializer.Deserialize(searchJobContent); + if (searchJob != null) + { + searchId = searchJob.JobId; + links = searchJob.Links; + } + } + + Assert.False(string.IsNullOrEmpty(searchId)); var traceId = ( @@ -221,8 +267,9 @@ private async Task RunAndAssertNewSearchEndpoint(string suid) : null ) ?? "Unknown"; + var topLevelTaskName = usePolling ? "workItemId" : "jobId"; TestOutputHelper.WriteLine( - $"Search started: {new { traceId, invocationId, jobId = searchJob.JobId }}" + $"Search started: traceId={traceId}, invocationId={invocationId}, {topLevelTaskName}={searchId}" ); var observabilityLink = Fixture.Config.IsLocal @@ -233,7 +280,7 @@ private async Task RunAndAssertNewSearchEndpoint(string suid) TestOutputHelper.WriteLine($"Trace observability: {observabilityLink}"); TestOutputHelper.WriteLine(""); - return searchJob; + return links; } private async Task RunAndAssertPartialSearchResults(string url, TestData testData) @@ -246,7 +293,8 @@ private async Task RunAndAssertPartialSearchResults(string url, TestData testDat var searchResults = await Fixture.Client.GetAsync(url); var searchResultContent = await searchResults.Content.ReadAsStringAsync(); - var searchResultTypedContent = JsonSerializer.Deserialize( + + var searchResultTypedContent = JsonSerializer.Deserialize( searchResultContent ); @@ -281,7 +329,7 @@ private async Task RunAndAssertFetchEndpoints(string url, TestData testData) var searchResults = await Fixture.Client.GetAsync(url); // Look for URL link and save as variable var searchResultContent = await searchResults.Content.ReadAsStringAsync(); - var searchResultTypedContent = JsonSerializer.Deserialize( + var searchResultTypedContent = JsonSerializer.Deserialize( searchResultContent ); Assert.True( @@ -369,18 +417,22 @@ private async Task RunAndAssertFetchEndpoints(string url, TestData testData) /// URL of the Search Status endpoint /// URL of the Search Results endpoint /// The test data for this search run - private async Task RunAndAwaitAndAssertSearchStatusCompletion( + /// If true, specifies the tests are being run for the Polling Architecture, rather than Fan-out. + private async Task RunAndAwaitAndAssertSearchStatusCompletion( string statusUrl, string resultsUrl, - TestData testData + TestData testData, + bool usePolling ) { statusUrl = RemoveLeadingSlashFromUrl(statusUrl); - const int retryCount = 150; + var timeout = TimeSpan.FromMinutes(usePolling ? 10 : 5); + var retryDelay = TimeSpan.FromSeconds(2); + var retryCount = (int)Math.Round(timeout / retryDelay); var isCompleted = false; - var status = "unknown"; + var statusMessage = "unknown"; TestOutputHelper.WriteLine( $"Checking for search completion: {Fixture.Client.BaseAddress}{statusUrl}" @@ -391,13 +443,14 @@ TestData testData { if (!r.IsSuccessStatusCode) { - status = $"{r.StatusCode} - {r.Content.ReadAsStringAsync().Result}"; + statusMessage = $"{r.StatusCode} - {r.Content.ReadAsStringAsync().Result}"; return true; // i.e. retry } // Read response, if status is NOT "Completed", keep retrying var content = r.Content.ReadFromJsonAsync().Result; - status = content?.Status; + var status = content?.Status; + switch (status) { case "Completed": @@ -410,6 +463,12 @@ TestData testData } var retry = status is "Queued" or "Running"; + + statusMessage = + content?.CompletenessPercentage != null + ? $"{status} ({content.CompletenessPercentage}%)" + : status; + return retry; }) .WaitAndRetryAsync( @@ -417,22 +476,25 @@ TestData testData retryAttempt => { TestOutputHelper.WriteLine( - $"Still checking for search completion, status was {status}, retry {retryAttempt} / {retryCount}..." + $"Still checking for search completion, status was {statusMessage}, retry {retryAttempt} / {retryCount}..." ); - return TimeSpan.FromSeconds(2); + return retryDelay; } ); await retryPolicy.ExecuteAsync(() => Fixture.Client.GetAsync(statusUrl)); TestOutputHelper.WriteLine( - $"Finished checking for search completion, final status was {status}, is completed: {isCompleted}" + $"Finished checking for search completion, final status was {statusMessage}, is completed: {isCompleted}" ); Assert.True(isCompleted); - var typedResult = await Fixture.Client.GetFromJsonAsync(statusUrl); - return typedResult!; + object? typedResult = usePolling + ? await Fixture.Client.GetFromJsonAsync(statusUrl) + : await Fixture.Client.GetFromJsonAsync(statusUrl); + + Assert.NotNull(typedResult); } private static string RemoveLeadingSlashFromUrl(string url) => diff --git a/Apps/Find/tests/SUI.Find.FindApi.UnitTests/Models/SearchResultsBaseTests.cs b/Apps/Find/tests/SUI.Find.FindApi.UnitTests/Models/SearchResultsBaseTests.cs new file mode 100644 index 000000000..1bd332758 --- /dev/null +++ b/Apps/Find/tests/SUI.Find.FindApi.UnitTests/Models/SearchResultsBaseTests.cs @@ -0,0 +1,43 @@ +using System.Text.Json; +using Shouldly; +using SUI.Find.Application.Enums; +using SUI.Find.FindApi.Models; + +namespace SUI.Find.FindApi.UnitTests.Models; + +public class SearchResultsBaseTests +{ + [Fact] + public void Ctor_Test() + { + var sut = new SearchResultsBase { Suid = "9991234566", Status = SearchStatus.Completed }; + + // ASSERT + sut.Suid.ShouldBe("9991234566"); + sut.Status.ShouldBe(SearchStatus.Completed); + } + + [Fact] + public void SearchResultsBase_Does_JsonDeserialize_AsExpected() + { + const string json = """ + { + "Suid": "9691292211", + "Status": "Running" + } + """; + + // ACT + var sut = JsonSerializer.Deserialize(json); + + // ASSERT + sut.ShouldBeEquivalentTo( + new SearchResultsBase + { + Suid = "9691292211", + Status = SearchStatus.Running, + Items = [], + } + ); + } +} diff --git a/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Handlers/JobResultHandlerTests.cs b/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Handlers/JobResultHandlerTests.cs index f6f358dcf..3903e1f3a 100644 --- a/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Handlers/JobResultHandlerTests.cs +++ b/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Handlers/JobResultHandlerTests.cs @@ -334,7 +334,13 @@ await _maskUrlService .Received(1) .CreateAsync( Arg.Is>(x => x.Count == 2), - Arg.Any(), + Arg.Is(x => + x.WorkItemId == message.WorkItemId + && x.RequestingOrg == searchingOrganisationId + && x.JobId == message.JobId + && x.Suid == "sui1" + && x.Provider.OrgId == message.CustodianId + ), Arg.Any() ); } diff --git a/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Services/JobSearchServiceTests.cs b/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Services/JobSearchServiceTests.cs index d82eb82bc..f3b02b09a 100644 --- a/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Services/JobSearchServiceTests.cs +++ b/Apps/Find/tests/SUI.Find.Infrastructure.UnitTests/Services/JobSearchServiceTests.cs @@ -30,7 +30,9 @@ public class JobSearchServiceTests public JobSearchServiceTests() { _logger.IsEnabled(LogLevel.Information).Returns(true); + _jobWindowStartService.GetWindowStart().Returns(_dateTime.AddHours(-72)); + _sut = new JobSearchService( _searchResultEntryRepository, _workItemJobCountRepository, diff --git a/Apps/StubCustodians/src/SUI.StubCustodians.Application/Extensions/ConfigExtensions.cs b/Apps/StubCustodians/src/SUI.StubCustodians.Application/Extensions/ConfigExtensions.cs index c162dddfe..8ff740171 100644 --- a/Apps/StubCustodians/src/SUI.StubCustodians.Application/Extensions/ConfigExtensions.cs +++ b/Apps/StubCustodians/src/SUI.StubCustodians.Application/Extensions/ConfigExtensions.cs @@ -8,7 +8,7 @@ public static bool UseEncryptedId(this IConfiguration config) { var value = !string.IsNullOrWhiteSpace(config["UseEncryptedId"]) ? config["UseEncryptedId"] - : "false"; - return bool.Parse(value); + : null; + return bool.Parse(value ?? "false"); } }