[DMS-981] Core Emits Concrete JSON Locations for Document References

src/dms/backend/EdFi.DataManagementService.Backend.RelationalModel.Tests.Unit/KeyUnificationPassTests.cs

@@ -341,9 +341,9 @@ public void It_should_emit_one_descriptor_fk_per_storage_column_after_unificatio
             .ToArray();
         var dedup = _resourceModel.DescriptorForeignKeyDeduplications.Should().ContainSingle().Subject;
 
-        descriptorForeignKeys.Should().ContainSingle();
-        descriptorForeignKeys[0].Columns.Should().Equal(keyUnificationClass.CanonicalColumn);
-        descriptorForeignKeys[0].TargetColumns.Should().Equal(RelationalNameConventions.DocumentIdColumnName);
+        var descriptorFk = descriptorForeignKeys.Should().ContainSingle().Subject;
+        descriptorFk.Columns.Should().Equal(keyUnificationClass.CanonicalColumn);
+        descriptorFk.TargetColumns.Should().Equal(RelationalNameConventions.DocumentIdColumnName);
         dedup.Table.Should().Be(_rootTable.Table);
         dedup.StorageColumn.Should().Be(keyUnificationClass.CanonicalColumn);
         dedup.BindingColumns.Should().Equal(expectedBindingColumns);

src/dms/backend/EdFi.DataManagementService.Old.Postgresql.Tests.Integration/DatabaseTest.cs

@@ -448,7 +448,8 @@ protected static DocumentReference CreateDocumentReference(Reference reference)
         return new(
             ResourceInfo: CreateResourceInfo(reference.ResourceName),
             DocumentIdentity: new([]),
-            ReferentialId: new ReferentialId(reference.ReferentialIdGuid)
+            ReferentialId: new ReferentialId(reference.ReferentialIdGuid),
+            Path: new JsonPath("$")
         );
     }
 
@@ -467,7 +468,8 @@ protected static SuperclassIdentity CreateSuperclassIdentity(string resourceName
         return new(
             ResourceInfo: CreateResourceInfo(resourceName),
             DocumentIdentity: new([]),
-            ReferentialId: new ReferentialId(referentialIdGuid)
+            ReferentialId: new ReferentialId(referentialIdGuid),
+            Path: new JsonPath("$")
         );
     }
 

src/dms/core/EdFi.DataManagementService.Core.External/Model/DocumentReference.cs

@@ -20,5 +20,9 @@ public record DocumentReference(
     /// <summary>
     /// The referentialId derived from the DocumentIdentity
     /// </summary>
-    ReferentialId ReferentialId
+    ReferentialId ReferentialId,
+    /// <summary>
+    /// The concrete JsonPath to the reference object in the document, including numeric indices
+    /// </summary>
+    JsonPath Path
 );

src/dms/core/EdFi.DataManagementService.Core.External/Model/SuperclassIdentity.cs

@@ -40,5 +40,9 @@ public record SuperclassIdentity(
     /// <summary>
     /// The referentialId derived from the DocumentIdentity
     /// </summary>
-    ReferentialId ReferentialId
-) : DocumentReference(ResourceInfo, DocumentIdentity, ReferentialId);
+    ReferentialId ReferentialId,
+    /// <summary>
+    /// The concrete JsonPath to the reference object in the document, including numeric indices
+    /// </summary>
+    JsonPath Path
+) : DocumentReference(ResourceInfo, DocumentIdentity, ReferentialId, Path);

src/dms/core/EdFi.DataManagementService.Core/Extraction/IdentityExtractor.cs

@@ -100,7 +100,8 @@ ILogger logger
             return new(
                 superclassResourceInfo,
                 documentIdentity,
-                ReferentialIdFrom(superclassResourceInfo, documentIdentity)
+                ReferentialIdFrom(superclassResourceInfo, documentIdentity),
+                new JsonPath("$")
             );
         }
 
@@ -112,7 +113,8 @@ ILogger logger
         return new(
             superclassResourceInfo,
             superclassIdentity,
-            ReferentialIdFrom(superclassResourceInfo, superclassIdentity)
+            ReferentialIdFrom(superclassResourceInfo, superclassIdentity),
+            new JsonPath("$")
         );
     }
 

src/dms/core/EdFi.DataManagementService.Core/Extraction/ReferenceExtractor.cs

@@ -3,7 +3,6 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
-using System.Diagnostics;
 using System.Text.Json.Nodes;
 using EdFi.DataManagementService.Core.ApiSchema;
 using EdFi.DataManagementService.Core.ApiSchema.Helpers;
@@ -18,6 +17,16 @@ namespace EdFi.DataManagementService.Core.Extraction;
 /// </summary>
 internal static class ReferenceExtractor
 {
+    /// <summary>
+    /// Strips the last dot-separated segment from a JSONPath string.
+    /// e.g. "$.classPeriods[*].classPeriodReference.classPeriodName" -> "$.classPeriods[*].classPeriodReference"
+    /// </summary>
+    private static string StripLastJsonPathSegment(string jsonPath)
+    {
+        int lastDotIndex = jsonPath.LastIndexOf('.');
+        return lastDotIndex > 0 ? jsonPath[..lastDotIndex] : jsonPath;
+    }
+
     /// <summary>
     /// Takes an API JSON body for the resource and extracts the document reference information from the JSON body.
     /// </summary>
@@ -44,36 +53,65 @@ ILogger logger
                 continue;
             }
 
-            // Extract the reference values from the document
-            IntermediateReferenceElement[] intermediateReferenceElements = documentPath
-                .ReferenceJsonPathsElements.Select(
-                    referenceJsonPathsElement => new IntermediateReferenceElement(
-                        referenceJsonPathsElement.IdentityJsonPath,
-                        documentBody
-                            .SelectNodesFromArrayPathCoerceToStrings(
-                                referenceJsonPathsElement.ReferenceJsonPath.Value,
-                                logger
-                            )
-                            .ToArray()
-                    )
-                )
-                .ToArray();
+            var identityElements = documentPath.ReferenceJsonPathsElements.ToArray();
 
-            int valueSliceLength = intermediateReferenceElements[0].ValueSlice.Length;
+            if (identityElements.Length == 0)
+            {
+                throw new InvalidOperationException(
+                    $"Reference '{documentPath.ResourceName.Value}' has no identity elements in ReferenceJsonPathsElements"
+                );
+            }
 
-            // Number of document values from resolved JsonPaths should all be the same, otherwise something is very wrong
-            Trace.Assert(
-                Array.TrueForAll(intermediateReferenceElements, x => x.ValueSlice.Length == valueSliceLength),
-                "Length of document value slices are not equal",
-                ""
-            );
+            // Group identity values by their concrete reference-object path.
+            var referenceGroups =
+                new List<(string parentPath, List<DocumentIdentityElement> identityElements)>();
+            var pathToGroupIndex = new Dictionary<string, int>();
+
+            foreach (var element in identityElements)
+            {
+                var pathValues = documentBody
+                    .SelectNodesAndLocationFromArrayPathCoerceToStrings(
+                        element.ReferenceJsonPath.Value,
+                        logger
+                    )
+                    .ToArray();
+
+                foreach (var pv in pathValues)
+                {
+                    string concreteParentPath = StripLastJsonPathSegment(pv.jsonPath);
+
+                    if (!pathToGroupIndex.TryGetValue(concreteParentPath, out int groupIndex))
+                    {
+                        groupIndex = referenceGroups.Count;
+                        pathToGroupIndex[concreteParentPath] = groupIndex;
+                        referenceGroups.Add((concreteParentPath, []));
+                    }
+
+                    referenceGroups[groupIndex]
+                        .identityElements.Add(
+                            new DocumentIdentityElement(element.IdentityJsonPath, pv.value)
+                        );
+                }
+            }
 
-            // If a JsonPath selection had no results, we can assume an optional reference wasn't there
-            if (valueSliceLength == 0)
+            // If no matches, assume an optional reference wasn't there
+            if (referenceGroups.Count == 0)
             {
                 continue;
             }
 
+            // Each reference group must have exactly one value per identity element
+            foreach (var (path, elements) in referenceGroups)
+            {
+                if (elements.Count != identityElements.Length)
+                {
+                    throw new InvalidOperationException(
+                        $"Reference '{documentPath.ResourceName.Value}' at '{path}': "
+                            + $"expected {identityElements.Length} identity elements but found {elements.Count}"
+                    );
+                }
+            }
+
             BaseResourceInfo resourceInfo = new(
                 documentPath.ProjectName,
                 documentPath.ResourceName,
@@ -82,39 +120,26 @@ ILogger logger
 
             List<DocumentReference> documentReferencesForThisArray = [];
 
-            // Reorient intermediateReferenceElements into actual DocumentReferences
-            for (int index = 0; index < valueSliceLength; index += 1)
+            foreach (var (parentPath, identityParts) in referenceGroups)
             {
-                List<DocumentIdentityElement> documentIdentityElements = [];
-
-                foreach (
-                    IntermediateReferenceElement intermediateReferenceElement in intermediateReferenceElements
-                )
-                {
-                    documentIdentityElements.Add(
-                        new DocumentIdentityElement(
-                            intermediateReferenceElement.IdentityJsonPath,
-                            intermediateReferenceElement.ValueSlice[index]
-                        )
-                    );
-                }
-
-                DocumentIdentity documentIdentity = new(documentIdentityElements.ToArray());
+                DocumentIdentity documentIdentity = new(identityParts.ToArray());
                 documentReferencesForThisArray.Add(
-                    new(resourceInfo, documentIdentity, ReferentialIdFrom(resourceInfo, documentIdentity))
+                    new(
+                        resourceInfo,
+                        documentIdentity,
+                        ReferentialIdFrom(resourceInfo, documentIdentity),
+                        new JsonPath(parentPath)
+                    )
                 );
             }
 
-            // Get the parent path from the first ReferenceJsonPathsElement
-            string firstReferenceJsonPath = documentPath
-                .ReferenceJsonPathsElements.First()
-                .ReferenceJsonPath.Value;
-            int lastDotIndex = firstReferenceJsonPath.LastIndexOf('.');
-            string parentPath =
-                lastDotIndex > 0 ? firstReferenceJsonPath.Substring(0, lastDotIndex) : firstReferenceJsonPath;
+            // Derive the wildcard parent path by stripping the last segment from the first identity element's path
+            string wildcardParentPath = StripLastJsonPathSegment(identityElements[0].ReferenceJsonPath.Value);
 
             documentReferences.AddRange(documentReferencesForThisArray);
-            documentReferenceArrays.Add(new(new(parentPath), documentReferencesForThisArray.ToArray()));
+            documentReferenceArrays.Add(
+                new(new(wildcardParentPath), documentReferencesForThisArray.ToArray())
+            );
         }
 
         return (documentReferences.ToArray(), documentReferenceArrays.ToArray());