Resolve collection identity during strict composition

internal/engine/compose.go

@@ -6,16 +6,17 @@ import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
 // precedence layers. Claims remain ordered by source and declaration traversal;
 // origins retain every source that contributes to a composite mapping.
 type composedDocument struct {
-	root           *document.Node
-	acquisition    []acquisitionScope
-	importGraph    importInclusionGraph
-	claims         []compositionClaim
-	sequenceRanges []sequenceRange
-	origins        map[string]Origin
-	overlays       []overlayRecord
-	applications   []historyApplication
-	inputBytes     int64
-	inputNodes     int
+	root               *document.Node
+	acquisition        []acquisitionScope
+	importGraph        importInclusionGraph
+	collectionPolicies collectionPolicyScope
+	claims             []compositionClaim
+	sequenceRanges     []sequenceRange
+	origins            map[string]Origin
+	overlays           []overlayRecord
+	applications       []historyApplication
+	inputBytes         int64
+	inputNodes         int
 }
 
 type sequenceHistoryMode uint8
@@ -54,10 +55,11 @@ func cloneComposedDocument(base *composedDocument) *composedDocument {
 	}
 	return &composedDocument{
 		root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims),
-		acquisition:    cloneAcquisitionScopes(base.acquisition),
-		importGraph:    base.importGraph.clone(),
-		sequenceRanges: cloneSequenceRanges(base.sequenceRanges),
-		origins:        cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays),
+		acquisition:        cloneAcquisitionScopes(base.acquisition),
+		importGraph:        base.importGraph.clone(),
+		collectionPolicies: cloneCollectionPolicyScope(base.collectionPolicies),
+		sequenceRanges:     cloneSequenceRanges(base.sequenceRanges),
+		origins:            cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays),
 		applications: cloneHistoryApplications(base.applications),
 		inputBytes:   base.inputBytes, inputNodes: base.inputNodes,
 	}
@@ -138,6 +140,17 @@ func composeDocuments(inputs ...*document.Node) (*composedDocument, []Diagnostic
 // sequence rules when policy contains registrations. The ordinary adapter above
 // deliberately preserves the existing unregistered behavior for current callers.
 func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) (*composedDocument, []Diagnostic) {
+	return composeDocumentsWithCollectionPolicy(policy, collectionPolicyScope{}, nil, inputs...)
+}
+
+// composeDocumentsWithCollectionPolicy resolves collection declarations and
+// eligible lower policy before strictly composing completed peer documents.
+func composeDocumentsWithCollectionPolicy(
+	policy sequencePolicy,
+	lower collectionPolicyScope,
+	declarations []sequenceDeclarationSpec,
+	inputs ...*document.Node,
+) (*composedDocument, []Diagnostic) {
 	if len(inputs) == 0 {
 		return nil, []Diagnostic{compositionInvalidDiagnostic("strict composition requires at least one document", nil)}
 	}
@@ -155,6 +168,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 		collected.append(input, Path{}, sourceIndex)
 	}
 
+	identityFieldsByPath, collectionPolicies, diagnostics := resolveStrictCollectionPolicies(
+		collected.ordered, policy, lower, declarations,
+	)
+	if len(diagnostics) != 0 {
+		return nil, diagnostics
+	}
+
 	checked := make(map[string]struct{}, len(collected.byPath))
 	for _, claim := range collected.ordered {
 		pointer := claim.path.String()
@@ -163,13 +183,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 		}
 		checked[pointer] = struct{}{}
 		claims := collected.byPath[pointer]
-		if len(claims) < 2 || allMappingClaims(claims) || (allSequenceClaims(claims) && policy.registered(claim.path)) {
+		if len(claims) < 2 || allMappingClaims(claims) || (allSequenceClaims(claims) && len(identityFieldsByPath[pointer]) != 0) {
 			continue
 		}
 		return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)}
 	}
 
-	validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, policy)
+	validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, identityFieldsByPath)
 	if len(diagnostics) != 0 {
 		return nil, diagnostics
 	}
@@ -177,7 +197,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 	root := document.Clone(inputs[0])
 	index := new(mappingIndex)
 	for sourceIndex, input := range inputs[1:] {
-		mergeComposedMappings(root, input, Path{}, index, sourceIndex+1, policy, validated)
+		mergeComposedMappings(root, input, Path{}, index, sourceIndex+1, identityFieldsByPath, validated)
 	}
 	applications := make([]historyApplication, 0, len(inputs))
 	seenSequences := make(map[string]bool)
@@ -204,11 +224,12 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 	origins := projectedCompositionOrigins(inputs, ranges)
 	completeCompositionOrigins(root, Path{}, origins)
 	return &composedDocument{
-		root:           root,
-		claims:         cloneCompositionClaims(collected.ordered),
-		sequenceRanges: cloneSequenceRanges(ranges),
-		origins:        origins,
-		applications:   applications,
+		root:               root,
+		collectionPolicies: collectionPolicies,
+		claims:             cloneCompositionClaims(collected.ordered),
+		sequenceRanges:     cloneSequenceRanges(ranges),
+		origins:            origins,
+		applications:       applications,
 	}, nil
 }
 
@@ -264,106 +285,140 @@ type validatedSequence struct {
 	items []*document.Node
 }
 
-func validateRegisteredSequences(claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
+func resolveStrictCollectionPolicies(
+	claims []compositionClaim,
+	applicationPolicy sequencePolicy,
+	lower collectionPolicyScope,
+	declarations []sequenceDeclarationSpec,
+) (map[string][]string, collectionPolicyScope, []Diagnostic) {
+	paths := make([]Path, 0, len(claims)+len(declarations))
+	seen := make(map[string]bool, len(claims)+len(declarations))
+	sequencePresent := make(map[string]bool)
+	declarationsByPath := make(map[string][]sequenceDeclarationSpec, len(declarations))
+	for _, claim := range claims {
+		if claim.kind != document.Sequence {
+			continue
+		}
+		key := claim.path.String()
+		sequencePresent[key] = true
+		if !seen[key] {
+			seen[key] = true
+			paths = append(paths, claim.path.clone())
+		}
+	}
+	for _, declaration := range declarations {
+		key := declaration.destination.String()
+		declarationsByPath[key] = append(declarationsByPath[key], declaration)
+		if !seen[key] {
+			seen[key] = true
+			paths = append(paths, declaration.destination.clone())
+		}
+	}
+
+	identityFieldsByPath := make(map[string][]string)
+	collectionPolicies := cloneCollectionPolicyScope(lower)
+	var diagnostics []Diagnostic
+	for _, path := range paths {
+		key := path.String()
+		currentDeclarations := declarationsByPath[key]
+
+		lowerEntry, hasLower := lower.collections[key]
+		var lowerPolicy *collectionPolicy
+		if hasLower {
+			copy := cloneCollectionPolicy(lowerEntry.policy)
+			lowerPolicy = &copy
+		}
+		applicationIdentity, _ := applicationPolicy.identityFieldsAt(path)
+
+		// A replacing value drops the old YAML identity before this tier's
+		// peers are checked. The explicit operation itself remains applicable.
+		selectedMode := sequenceMode(0)
+		for _, declaration := range currentDeclarations {
+			if declaration.modePresent {
+				selectedMode = declaration.mode
+				break
+			}
+		}
+		if selectedMode == 0 && lowerPolicy != nil && lowerPolicy.explicitModePresent {
+			selectedMode = lowerPolicy.explicitMode
+		}
+		if sequencePresent[key] && selectedMode == sequenceModeReplace && lowerPolicy != nil {
+			lowerPolicy.identityFields = nil
+			lowerPolicy.identityPresent = false
+			lowerPolicy.identityLocation = nil
+		}
+
+		resolution, found := resolveCollectionPolicy(path, currentDeclarations, applicationIdentity, lowerPolicy)
+		if len(found) != 0 {
+			diagnostics = append(diagnostics, found...)
+			continue
+		}
+		if len(resolution.effectiveIdentityFields) != 0 {
+			identityFieldsByPath[key] = cloneStrings(resolution.effectiveIdentityFields)
+		}
+
+		policyForPath := resolution.policy
+		if hasLower && !sequencePresent[key] {
+			// A declaration without a new value changes policy metadata but
+			// leaves retained record ownership untouched.
+			policyForPath = cloneCollectionPolicy(lowerEntry.policy)
+			policyForPath.identityFields = cloneStrings(resolution.policy.identityFields)
+			policyForPath.identityPresent = resolution.policy.identityPresent
+			policyForPath.identityLocation = cloneLocation(resolution.policy.identityLocation)
+			policyForPath.explicitMode = resolution.policy.explicitMode
+			policyForPath.explicitModePresent = resolution.policy.explicitModePresent
+			policyForPath.modeLocation = cloneLocation(resolution.policy.modeLocation)
+		}
+		if collectionPolicies.collections == nil {
+			collectionPolicies.collections = make(map[string]collectionPolicyEntry)
+		}
+		collectionPolicies.collections[key] = collectionPolicyEntry{destination: path.clone(), policy: policyForPath}
+	}
+	return identityFieldsByPath, collectionPolicies, canonicalDiagnostics(diagnostics)
+}
+
+func validateRegisteredSequences(claims []compositionClaim, identityFieldsByPath map[string][]string) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
 	itemsByPath := make(map[string]map[int]validatedSequence)
-	identitiesByPath := make(map[string]map[string]Location)
+	identitiesByPath := make(map[string]map[string]document.Location)
 	lengthsByPath := make(map[string]int)
 	var ranges []sequenceRange
 	for _, claim := range claims {
 		if claim.kind != document.Sequence {
 			continue
 		}
-		identityFields, registered := policy.identityFieldsAt(claim.path)
-		if !registered {
+		identityFields := identityFieldsByPath[claim.path.String()]
+		if len(identityFields) == 0 {
 			continue
 		}
 		node := claim.node
 		pathKey := claim.path.String()
 		identities := identitiesByPath[pathKey]
 		if identities == nil {
-			identities = make(map[string]Location, len(node.Sequence))
+			identities = make(map[string]document.Location, len(node.Sequence))
 			identitiesByPath[pathKey] = identities
 		}
 		start := lengthsByPath[pathKey]
 		contribution := validatedSequence{items: node.Sequence}
-		for itemIndex, item := range node.Sequence {
-			itemPath := claim.path.Index(start + itemIndex)
-			if item == nil || item.Kind != document.Mapping {
-				location := (*Location)(nil)
-				if item != nil {
-					converted := publicLocation(item.Location)
-					location = &converted
-				}
-				return nil, nil, []Diagnostic{{
-					Code: CodeComposeInvalid, Severity: SeverityError,
-					Path: itemPath, Message: "registered sequence items must be mappings", Primary: location,
-				}}
-			}
-			var identity string
-			var identityLocation document.Location
-			if legacyField, legacy := policy.identityFields[claim.path.String()]; legacy {
-				fieldIndex := documentEntryIndex(item.Mapping, legacyField)
-				if fieldIndex < 0 {
-					location := publicLocation(item.Location)
-					return nil, nil, []Diagnostic{{
-						Code: CodeComposeInvalid, Severity: SeverityError,
-						Path: itemPath, Message: "registered sequence item is missing its identity field",
-						Primary: &location,
-					}}
-				}
-				field := item.Mapping[fieldIndex].Value
-				if field == nil || field.Kind != document.String {
-					location := publicLocation(item.Mapping[fieldIndex].ValueLocation)
-					if field != nil {
-						location = publicLocation(field.Location)
-					}
-					return nil, nil, []Diagnostic{{
-						Code: CodeComposeInvalid, Severity: SeverityError,
-						Path: itemPath.Key(legacyField), Message: "registered sequence identity must be a string",
-						Primary: &location,
-					}}
-				}
-				identity, identityLocation = field.Text, field.Location
-			} else {
-				mapping := newSequenceMappingIndex(item)
-				var locations []document.Location
-				var fieldName string
-				var fieldIndex int
-				identity, locations, fieldName, fieldIndex = mapping.identity(identityFields)
-				if fieldIndex == -1 {
-					location := publicLocation(item.Location)
-					return nil, nil, []Diagnostic{{
-						Code: CodeComposeInvalid, Severity: SeverityError,
-						Path: itemPath.Key(fieldName), Message: "registered sequence item is missing an identity field",
-						Primary: &location,
-					}}
-				}
-				if fieldIndex == -2 || fieldIndex >= 0 {
-					location := publicLocation(item.Location)
-					fieldPath := itemPath.Key(fieldName)
-					if fieldIndex >= 0 {
-						location = publicLocation(item.Mapping[fieldIndex].ValueLocation)
-						if field := item.Mapping[fieldIndex].Value; field != nil {
-							location = publicLocation(field.Location)
-						}
-					}
-					return nil, nil, []Diagnostic{{
-						Code: CodeComposeInvalid, Severity: SeverityError,
-						Path: fieldPath, Message: "registered sequence identity fields must be scalar values",
-						Primary: &location,
-					}}
-				}
-				identityLocation = locations[0]
-			}
-			if first, duplicate := identities[identity]; duplicate {
-				current := publicLocation(identityLocation)
+		records, diagnostics := validateNormalizedSequenceIdentityFrom(node, claim.path, identityFields, sequenceIdentityDiagnosticCodes{
+			invalid: CodeComposeInvalid, conflict: CodeComposeConflict,
+		}, start)
+		if len(diagnostics) != 0 {
+			return nil, nil, diagnostics
+		}
+		byIndex := make([]sequenceIdentityRecord, len(node.Sequence))
+		for _, record := range records {
+			byIndex[record.index] = record
+		}
+		for _, record := range byIndex {
+			if first, duplicate := identities[record.tuple]; duplicate {
+				current := publicLocation(record.location)
 				return nil, nil, []Diagnostic{{
 					Code: CodeComposeConflict, Severity: SeverityError,
 					Path: claim.path.clone(), Message: "registered sequence contains duplicate identities",
-					Primary: &first, Related: []Location{current},
+					Primary: ptrLocation(publicLocation(first)), Related: []Location{current},
 				}}
 			}
-			identities[identity] = publicLocation(identityLocation)
+			identities[record.tuple] = record.location
 		}
 		if itemsByPath[pathKey] == nil {
 			itemsByPath[pathKey] = make(map[int]validatedSequence)
@@ -377,7 +432,7 @@ func validateRegisteredSequences(claims []compositionClaim, policy sequencePolic
 	return itemsByPath, ranges, nil
 }
 
-func mergeComposedMappings(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, policy sequencePolicy, validated map[string]map[int]validatedSequence) {
+func mergeComposedMappings(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, identityFieldsByPath map[string][]string, validated map[string]map[int]validatedSequence) {
 	for _, incomingEntry := range incoming.Mapping {
 		entryPath := path.Key(incomingEntry.Key)
 		position := index.find(target, incomingEntry.Key)
@@ -388,7 +443,7 @@ func mergeComposedMappings(target, incoming *document.Node, path Path, index *ma
 			continue
 		}
 		current := target.Mapping[position].Value
-		if current.Kind == document.Sequence && incomingEntry.Value.Kind == document.Sequence && policy.registered(entryPath) {
+		if current.Kind == document.Sequence && incomingEntry.Value.Kind == document.Sequence && len(identityFieldsByPath[entryPath.String()]) != 0 {
 			contribution := validated[entryPath.String()][sourceIndex]
 			if contribution.items != nil {
 				for _, item := range contribution.items {
@@ -397,15 +452,15 @@ func mergeComposedMappings(target, incoming *document.Node, path Path, index *ma
 			}
 			continue
 		}
-		mergeDisjointMappingsAtPath(current, incomingEntry.Value, entryPath, index, sourceIndex, policy, validated)
+		mergeDisjointMappingsAtPath(current, incomingEntry.Value, entryPath, index, sourceIndex, identityFieldsByPath, validated)
 	}
 }
 
-func mergeDisjointMappingsAtPath(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, policy sequencePolicy, validated map[string]map[int]validatedSequence) {
+func mergeDisjointMappingsAtPath(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, identityFieldsByPath map[string][]string, validated map[string]map[int]validatedSequence) {
 	if target.Kind != document.Mapping || incoming.Kind != document.Mapping {
 		return
 	}
-	mergeComposedMappings(target, incoming, path, index, sourceIndex, policy, validated)
+	mergeComposedMappings(target, incoming, path, index, sourceIndex, identityFieldsByPath, validated)
 }
 
 func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange {