diff --git a/docs/config.md b/docs/config.md index cd91b06..6dbf3b1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -824,12 +824,13 @@ headers are consumed before strict composition and cannot supply a registered record identity. Final typed decoding and application validation still run on the completed configuration. -Ordinary source documents may also declare collection behavior under the root -`configkit.sequences` key. Each entry has a required literal `destination` and may -specify `key` as one string or a nonempty list of strings, plus a `mode` of -`replace`, `append`, or `upsert`. These controls are removed before application -decoding. A declaration destination is relative to that source's placement, so a -mounted fragment's declaration is mounted with its values. Declarations from +Ordinary sources and selected overlays may declare collection behavior under +`configkit.sequences` at the document root. Each entry has a required literal +`destination` and may specify `key` as one string or a nonempty list of strings, +plus a `mode` of `replace`, `append`, or `upsert`. These controls are removed +before application decoding. A declaration destination is relative to that +source's placement, so a mounted fragment's declaration is mounted with its +values. Declarations from strict peers must agree on identity fields and on any explicitly supplied mode; they apply even when the declaring source has no value at that destination. Singleton sequence contributions are validated as well as peers. A key supplies @@ -839,9 +840,11 @@ a keyed collection uses `upsert`, while an unkeyed collection uses `replace`. replaces a complete matched record at its existing position and appends new identities in incoming order; it does not merge record fields. -At this boundary, declarations govern ordinary strict source composition, -including loads with imports disabled. They do not change selected-overlay or -layered precedence, inheritance, or CLI behavior. A root `configkit.imports` +Declarations govern ordinary strict source composition, including loads with +imports disabled, and selected-overlay precedence. Overlay declarations apply +after the base source graph is composed and must agree with retained collection +identity and any application registration. They do not change layered +precedence, inheritance, or CLI behavior. A root `configkit.imports` declaration still requires `Options.Imports`; `NoSource` has no declaration scope. An empty mapping contributes no child definitions. Two empty mappings compose, and @@ -885,10 +888,15 @@ origin data; history retains the displaced definitions and identifies the parent replacement. Surviving overlay paths have overlay origins, while recursively shared mapping containers retain their contributing locations. -An overlay may not contain `configkit` anywhere, contribute the configured schema -version, or replace an ancestor of that version with a scalar or null. These rules -apply even if the supplied version text would be unchanged. Overlays cannot mask a -base parse, composition, or version failure because those checks run first. +An overlay may contain only root `configkit.sequences` declarations; overlay +imports and nested directives are rejected. Sequence declarations are removed +before application decoding. At declared or registered sequence destinations, +`replace`, `append`, and `upsert` use the same identity and policy rules as source +composition. Other overlay sequences remain atomic replacements. An overlay may +not contribute the configured schema version or replace an ancestor of that +version with a scalar or null. These rules apply even if the supplied version +text would be unchanged. Overlays cannot mask a base parse, composition, or +version failure because those checks run first. ## Explicit CLI overrides diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 835777a..7640fdb 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Sequence Identity And Combination Implementation Plan -Status: implementation in progress; Stages 1–13 complete. +Status: implementation in progress; Stages 1–14 complete. Audience: a Luna coding agent implementing one numbered stage per prompt, in order. This plan starts at commit @@ -519,6 +519,10 @@ using the source/declaration handoff already active in Stage 13. Validation: ordinary-load, overlay, composition, policy, and metadata tests plus the everyday check. +**Completion:** Implemented and verified with ordinary overlay-operation, +sequence-policy, source-version, and metadata tests. Validation commands: +`go test ./internal/engine -count=1` and `./scripts/check.sh`. + ## Stage 15 — Integrate Lower Policy Across Layered Assembly Scope: source-backed underlay coordination and the layered selected-overlay path, diff --git a/internal/engine/directives.go b/internal/engine/directives.go index 71cd408..effd1f3 100644 --- a/internal/engine/directives.go +++ b/internal/engine/directives.go @@ -218,24 +218,28 @@ func nodeImportDiagnostic(code Code, path Path, message string, node *document.N return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location} } -func overlayDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { +func overlayDirectiveDiagnostics(node *document.Node, current Path, allowSequences ...bool) []Diagnostic { + sequencesAllowed := len(allowSequences) != 0 && allowSequences[0] if node.Kind == document.Mapping { for _, entry := range node.Mapping { childPath := current.Key(entry.Key) if entry.Key == directiveKey { + if sequencesAllowed && len(current.segments) == 0 && entry.Value != nil && entry.Value.Kind == document.Mapping && len(entry.Value.Mapping) == 1 && entry.Value.Mapping[0].Key == "sequences" { + continue + } location := publicLocation(entry.KeyLocation) return []Diagnostic{{ Code: CodeOverlayDirective, Severity: SeverityError, Path: childPath, - Message: "overlays cannot contain configuration directives", Primary: &location, + Message: "overlays may contain only root collection sequence declarations", Primary: &location, }} } - if diagnostics := overlayDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 { + if diagnostics := overlayDirectiveDiagnostics(entry.Value, childPath, sequencesAllowed); len(diagnostics) != 0 { return diagnostics } } } else if node.Kind == document.Sequence { for index, child := range node.Sequence { - if diagnostics := overlayDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 { + if diagnostics := overlayDirectiveDiagnostics(child, current.Index(index), sequencesAllowed); len(diagnostics) != 0 { return diagnostics } } diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index 78bd041..74b8d16 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -271,7 +271,7 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI if activeOverlay(options.Overlay) { input.sources, diagnostics = loadSelectedOverlayIntoWithVersion( source, options.Filesystem, input.parsed, assembly, input.sources, - options.Overlay, options.Definition.Version, defaultImportLimits(), input.versionState, + options.Overlay, options.Definition.Version, defaultImportLimits(), input.versionState, options.sequencePolicy, ) if len(diagnostics) != 0 { return input, diagnostics diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index a0ae32b..c70af75 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -146,7 +146,11 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a selection := Source{kind: sourceBytes} selected := parsedSource{document: &document.Node{Kind: document.Mapping}, source: &SourceInfo{}} var diagnostics []Diagnostic - input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.protectionVersion(), defaultImportLimits(), input.versionState) + var overlayPolicy []sequencePolicy + if request.ordinary { + overlayPolicy = []sequencePolicy{request.sequencePolicy} + } + input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.protectionVersion(), defaultImportLimits(), input.versionState, overlayPolicy...) if len(diagnostics) != 0 { return input, diagnostics } diff --git a/internal/engine/overlay.go b/internal/engine/overlay.go index 81e8d0e..66ee7ae 100644 --- a/internal/engine/overlay.go +++ b/internal/engine/overlay.go @@ -52,8 +52,194 @@ func (a *assemblyAccumulator) applyOverlay(overlay *document.Node, incomingOrigi return nil } +// applyOverlayWithCollectionPolicy is the shared recursive-precedence entry for +// overlays. The caller supplies only the policy and declarations owned by this +// contribution; layered loading can use the same operation when its lifecycle +// reaches this boundary. +func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy( + overlay *document.Node, + incomingOrigins map[string]Origin, + applications []historyApplication, + layer Layer, + sequencePolicy sequencePolicy, + declarations []sequenceDeclarationSpec, +) []Diagnostic { + if a == nil || a.current == nil || a.current.root == nil || a.current.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping { + return invalidOverlayDocument() + } + mergeOriginContribution(a.current.origins, Path{}, incomingOrigin(incomingOrigins, Path{}, overlay, layer)) + sequences, diagnostics := mergeOverlayMappingsWithCollectionPolicy(a, overlay, incomingOrigins, layer, sequencePolicy, declarations) + if len(diagnostics) != 0 { + return diagnostics + } + if len(applications) == 0 { + application := historyApplication{layer: layer, value: overlay, sequences: sequences} + application.sequencesByPath = indexSequenceHistoryContributions(application.sequences) + a.current.applications = append(a.current.applications, application) + } else { + for _, application := range applications { + application.layer = layer + a.current.applications = append(a.current.applications, application) + } + } + return nil +} + +func mergeOverlayMappingsWithCollectionPolicy( + assembly *assemblyAccumulator, + overlay *document.Node, + incomingOrigins map[string]Origin, + layer Layer, + sequencePolicy sequencePolicy, + declarations []sequenceDeclarationSpec, +) ([]sequenceHistoryContribution, []Diagnostic) { + result := assembly.current + claims := compositionClaims{byPath: make(map[string][]compositionClaim)} + claims.append(overlay, Path{}, 0) + identityFieldsByPath, resolvedPolicies, diagnostics := resolveStrictCollectionPolicies( + claims.ordered, sequencePolicy, result.collectionPolicies, declarations, + ) + if len(diagnostics) != 0 { + return nil, diagnostics + } + declarationsByPath := make(map[string][]sequenceDeclarationSpec, len(declarations)) + registeredPaths := make(map[string]Path) + for _, declaration := range declarations { + key := declaration.destination.String() + declarationsByPath[key] = append(declarationsByPath[key], declaration) + registeredPaths[key] = declaration.destination.clone() + } + registered := make(map[string]bool) + for _, claim := range claims.ordered { + if claim.kind == document.Sequence { + if _, ok := sequencePolicy.identityFieldsAt(claim.path); ok { + registered[claim.path.String()] = true + registeredPaths[claim.path.String()] = claim.path.clone() + } + } + } + for key := range declarationsByPath { + registered[key] = true + } + for key := range result.collectionPolicies.collections { + if node, found := documentAtPath(overlay, result.collectionPolicies.collections[key].destination); found && node != nil { + registered[key] = true + registeredPaths[key] = result.collectionPolicies.collections[key].destination.clone() + } + } + + resolutions := make(map[string]collectionPolicyResolution, len(registered)) + for key := range registered { + path := registeredPaths[key] + policy, hasPolicy := resolvedPolicies.at(path) + if !hasPolicy { + continue + } + operation := sequenceModeReplace + if policy.explicitModePresent { + operation = policy.explicitMode + } else if len(identityFieldsByPath[key]) != 0 { + operation = sequenceModeUpsert + } + resolutions[key] = collectionPolicyResolution{ + policy: policy, operation: operation, + effectiveIdentityFields: cloneStrings(identityFieldsByPath[key]), + } + } + + sequenceApplications := make(map[string]sequenceHistoryContribution) + policyValues := make(map[string]bool) + var combinationDiagnostics []Diagnostic + mergeOverlayMappingsWithHooks(result.root, overlay, Path{}, &assembly.index, + recursiveMergeHooks{ + added: func(path Path, incoming *document.Node) { + collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming, path, layer) + }, + combined: func(path Path, incoming *document.Node) { + mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, incoming, layer)) + }, + replaced: func(path Path, existing, incoming *document.Node) { + assembly.current.collectionPolicies.retireCollectionPoliciesAtOrBelow(path) + recordOverlayReplacement(result, path, existing, incoming, layer, true, incomingOrigins) + }, + merge: func(path Path, existing, incoming *document.Node) (*document.Node, bool) { + resolution, active := resolutions[path.String()] + if !active { + return nil, false + } + combined, issues := combineSequence(existing, incoming, resolution, sequenceCombinationContext{destination: path}) + if len(issues) != 0 { + combinationDiagnostics = append(combinationDiagnostics, issues...) + return existing, false + } + applyOverlaySequenceEvidence(result, path, existing, incoming, combined, resolution, incomingOrigins, layer) + policyValues[path.String()] = true + history := sequenceHistoryContribution{path: path.clone(), mode: sequenceHistoryMapped} + if resolution.operation == sequenceModeReplace { + history.mode = sequenceHistoryReplace + if incoming != nil && incoming.Kind == document.Sequence { + history.length = len(incoming.Sequence) + } + } else { + history.items = append([]sequenceItemProjection(nil), combined.higherItems...) + history.replacements = append([]sequenceItemReplacement(nil), combined.replacements...) + } + sequenceApplications[path.String()] = history + return combined.value, true + }, + }, + ) + if len(combinationDiagnostics) != 0 { + return nil, canonicalDiagnostics(combinationDiagnostics) + } + + for key, grouped := range declarationsByPath { + if policyValues[key] { + continue + } + path := grouped[0].destination + applicationIdentity, _ := sequencePolicy.identityFieldsAt(path) + current, diagnostics := resolveCollectionPolicy(path, grouped, applicationIdentity, nil) + if len(diagnostics) != 0 { + combinationDiagnostics = append(combinationDiagnostics, diagnostics...) + continue + } + lower, found := assembly.current.collectionPolicies.at(path) + var lowerPolicy *collectionPolicy + if found { + lowerPolicy = &lower + } + retained, _ := documentAtPath(assembly.current.root, path) + if retained != nil && retained.Kind == document.Sequence && len(current.effectiveIdentityFields) != 0 { + if _, issues := validateSequenceIdentity(retained, path, current.effectiveIdentityFields, sequenceIdentityDiagnosticCodes{invalid: CodeSequenceInvalid, conflict: CodeSequenceConflict}); len(issues) != 0 { + combinationDiagnostics = append(combinationDiagnostics, issues...) + continue + } + } + policy, issues := collectionPolicyWithoutValue(lowerPolicy, current.policy, retained, path) + if len(issues) != 0 { + combinationDiagnostics = append(combinationDiagnostics, issues...) + continue + } + assembly.current.collectionPolicies.set(path, policy) + } + if len(combinationDiagnostics) != 0 { + return nil, canonicalDiagnostics(combinationDiagnostics) + } + keys := make([]string, 0, len(sequenceApplications)) + for key := range sequenceApplications { + keys = append(keys, key) + } + slices.Sort(keys) + sequences := make([]sequenceHistoryContribution, 0, len(keys)) + for _, key := range keys { + sequences = append(sequences, sequenceApplications[key]) + } + return sequences, nil +} + func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool, incomingOrigins map[string]Origin, index *mappingIndex) { - mergeRecursiveMappings(target, overlay, current, index, recursiveMergeHooks{ + mergeOverlayMappingsWithHooks(target, overlay, current, index, recursiveMergeHooks{ added: func(path Path, incoming *document.Node) { collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming, path, layer) }, @@ -78,6 +264,147 @@ func mergeOverlayMappings(result *composedDocument, target, overlay *document.No }) } +func mergeOverlayMappingsWithHooks( + target, overlay *document.Node, + current Path, + index *mappingIndex, + hooks recursiveMergeHooks, +) { + mergeRecursiveMappings(target, overlay, current, index, hooks) +} + +func recordOverlayReplacement( + result *composedDocument, + path Path, + existing, incoming *document.Node, + layer Layer, + recordReplacement bool, + incomingOrigins map[string]Origin, +) { + var displaced []displacedDefinition + var previous Origin + if existing != nil { + displaced = displacedOrigins(existing, result.origins, path) + previous = cloneOrigin(result.origins[path.String()]) + deleteOriginSubtree(result.origins, path, existing) + } + collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming, path, layer) + if recordReplacement && existing != nil { + result.overlays = append(result.overlays, overlayRecord{ + change: Change{ + Path: path.clone(), Origin: incomingOrigin(incomingOrigins, path, incoming, layer), + Previous: previousOrigins(previous), + }, + displaced: displaced, + }) + } +} + +func applyOverlaySequenceEvidence( + result *composedDocument, + path Path, + lower, higher *document.Node, + combined sequenceCombinationResult, + resolution collectionPolicyResolution, + incomingOrigins map[string]Origin, + layer Layer, +) { + lowerPolicy, hadLowerPolicy := result.collectionPolicies.at(path) + if resolution.operation == sequenceModeReplace { + var lowerValue *collectionPolicy + if hadLowerPolicy { + lowerValue = &lowerPolicy + } + result.collectionPolicies.retireCollectionPoliciesAtOrBelow(path) + resolution.policy = collectionPolicyForReplacement(lowerValue, resolution.policy) + } + + if combined.value == nil || combined.value.Kind != document.Sequence { + recordOverlayReplacement(result, path, lower, higher, layer, true, incomingOrigins) + result.collectionPolicies.set(path, resolution.policy) + return + } + + if resolution.operation == sequenceModeReplace { + var displaced []displacedDefinition + var previous Origin + if lower != nil { + displaced = displacedOrigins(lower, result.origins, path) + previous = cloneOrigin(result.origins[path.String()]) + deleteOriginSubtree(result.origins, path, lower) + } + mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, higher, layer)) + for _, projection := range combined.higherItems { + if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) { + source := higher.Sequence[projection.inputIndex] + collectOverlayOriginsFrom(result.origins, incomingOrigins, source, path.Index(projection.outputIndex), layer) + } + } + if lower != nil { + result.overlays = append(result.overlays, overlayRecord{ + change: Change{ + Path: path.clone(), Origin: incomingOrigin(incomingOrigins, path, higher, layer), + Previous: previousOrigins(previous), + }, displaced: displaced, + }) + } + } else { + mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, higher, layer)) + for _, replacement := range combined.replacements { + lowerPath := path.Index(replacement.lowerIndex) + higherPath := path.Index(replacement.higherIndex) + outputPath := path.Index(replacement.outputIndex) + if lower == nil || lower.Kind != document.Sequence || replacement.lowerIndex < 0 || replacement.lowerIndex >= len(lower.Sequence) || + higher == nil || higher.Kind != document.Sequence || replacement.higherIndex < 0 || replacement.higherIndex >= len(higher.Sequence) { + continue + } + lowerItem := lower.Sequence[replacement.lowerIndex] + higherItem := higher.Sequence[replacement.higherIndex] + displaced := displacedOrigins(lowerItem, result.origins, lowerPath) + previous := cloneOrigin(result.origins[lowerPath.String()]) + deleteOriginSubtree(result.origins, lowerPath, lowerItem) + collectOverlayOriginsFrom(result.origins, incomingOrigins, higherItem, outputPath, layer) + result.overlays = append(result.overlays, overlayRecord{ + change: Change{ + Path: outputPath.clone(), Origin: incomingOrigin(incomingOrigins, higherPath, higherItem, layer), + Previous: previousOrigins(previous), + }, + displaced: displaced, + }) + } + for _, projection := range combined.higherItems { + if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) { + higherItem := higher.Sequence[projection.inputIndex] + collectOverlayOriginsFrom(result.origins, incomingOrigins, higherItem, path.Index(projection.outputIndex), layer) + } + } + } + + policy := cloneCollectionPolicy(resolution.policy) + policy.recordOrder = nil + policy.recordPolicies = nil + if hadLowerPolicy && resolution.operation != sequenceModeReplace { + lowerProjection := newSequenceItemProjectionIndex(combined.lowerItems) + higherProjection := newSequenceItemProjectionIndex(combined.higherItems) + for outputIndex := range combined.value.Sequence { + if _, replaced := higherProjection.inputForOutput(outputIndex); replaced { + policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy()) + continue + } + if lowerIndex, retained := lowerProjection.inputForOutput(outputIndex); retained { + if record, found := lowerPolicy.recordPolicyAt(lowerIndex); found { + policy.setRecordPolicyAt(outputIndex, retainedCollectionRecordPolicy(record)) + } + } + } + } else { + for outputIndex := range combined.value.Sequence { + policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy()) + } + } + result.collectionPolicies.set(path, policy) +} + func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path, layer Layer) { collectOverlayOriginsFrom(origins, nil, node, path, layer) } diff --git a/internal/engine/recursive_merge.go b/internal/engine/recursive_merge.go index a03819e..a6176b3 100644 --- a/internal/engine/recursive_merge.go +++ b/internal/engine/recursive_merge.go @@ -9,6 +9,7 @@ type recursiveMergeHooks struct { added func(Path, *document.Node) combined func(Path, *document.Node) replaced func(Path, *document.Node, *document.Node) + merge func(Path, *document.Node, *document.Node) (*document.Node, bool) } // mergeRecursiveMappings changes an owned mapping and never aliases incoming @@ -18,6 +19,14 @@ func mergeRecursiveMappings(target, incoming *document.Node, current Path, index path := current.Key(entry.Key) position := index.find(target, entry.Key) if position < 0 { + if hooks.merge != nil { + if merged, handled := hooks.merge(path, nil, entry.Value); handled { + cloned := entry + cloned.Value = document.Clone(merged) + index.append(target, cloned) + continue + } + } cloned := entry cloned.Value = document.Clone(entry.Value) index.append(target, cloned) @@ -35,6 +44,13 @@ func mergeRecursiveMappings(target, incoming *document.Node, current Path, index mergeRecursiveMappings(existing, entry.Value, path, index, hooks) continue } + if hooks.merge != nil { + if merged, handled := hooks.merge(path, existing, entry.Value); handled { + target.Mapping[position] = entry + target.Mapping[position].Value = document.Clone(merged) + continue + } + } if hooks.replaced != nil { hooks.replaced(path, existing, entry.Value) diff --git a/internal/engine/selected_overlay.go b/internal/engine/selected_overlay.go index 3d7c616..d985c0a 100644 --- a/internal/engine/selected_overlay.go +++ b/internal/engine/selected_overlay.go @@ -41,7 +41,11 @@ func loadSelectedOverlayInto(source Source, context *FilesystemContext, main par return loadSelectedOverlayIntoWithVersion(source, context, main, base, sources, name, rule, limits, nil) } -func loadSelectedOverlayIntoWithVersion(source Source, context *FilesystemContext, main parsedSource, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits, versionState *sourceVersionState) ([]SourceInfo, []Diagnostic) { +func loadSelectedOverlayIntoWithVersion(source Source, context *FilesystemContext, main parsedSource, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits, versionState *sourceVersionState, policies ...sequencePolicy) ([]SourceInfo, []Diagnostic) { + var policy sequencePolicy + if len(policies) != 0 { + policy = policies[0] + } if nativeContext(source, context) { access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context) if diagnostic != nil { @@ -50,7 +54,10 @@ func loadSelectedOverlayIntoWithVersion(source Source, context *FilesystemContex if diagnostic := validateNativeDirectories(access, basePath, mainPath); diagnostic != nil { return cloneSourceInfos(sources), []Diagnostic{*diagnostic} } - return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState) + if len(policies) == 0 { + return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState) + } + return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState, policy) } rootPath, _, _, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { @@ -62,14 +69,21 @@ func loadSelectedOverlayIntoWithVersion(source Source, context *FilesystemContex value.Primary = nil return cloneSourceInfos(sources), []Diagnostic{value} } - return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState) + if len(policies) == 0 { + return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState) + } + return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState, policy) } func loadSelectedOverlayWithAccess(access *filesystemAccess, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) { return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, nil) } -func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits, versionState *sourceVersionState) ([]SourceInfo, []Diagnostic) { +func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits, versionState *sourceVersionState, policies ...sequencePolicy) ([]SourceInfo, []Diagnostic) { + var policy sequencePolicy + if len(policies) != 0 { + policy = policies[0] + } closed := false defer func() { if !closed { @@ -131,17 +145,33 @@ func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *ass if base.current.inputNodes+nodeCount > limits.nodes { return sources, []Diagnostic{importDiagnostic(CodeOverlayLimitExceeded, "selected overlay exceeds the aggregate node limit", overlayLocation)} } - if diagnostics := overlayDirectiveDiagnostics(overlayRoot, Path{}); len(diagnostics) != 0 { + if diagnostics := overlayDirectiveDiagnostics(overlayRoot, Path{}, len(policies) != 0); len(diagnostics) != 0 { return sources, diagnostics } - overlayPayload := overlayRoot - if versionState != nil && versionState.mode != sourceVersionModeDefault { - processed, failures := versionState.process(overlayRoot, sourceVersionContribution, Path{}, sourceID, placementModeMapping) - if len(failures) != 0 { - return sources, failures + var processed sourceVersionProcessingResult + var failures []Diagnostic + if len(policies) != 0 { + if versionState != nil { + processed, failures = versionState.processWithDirectiveContext( + overlayRoot, sourceVersionContribution, Path{}, sourceID, placementModeMapping, rootDirectiveContextOverlay, + ) + } else { + processed = processSourceVersionWithDirectiveContext( + overlayRoot, rule, sourceVersionContribution, Path{}, sourceVersionModeDefault, rootDirectiveContextOverlay, + ) + failures = processed.diagnostics + } + } else { + if versionState != nil && versionState.mode != sourceVersionModeDefault { + processed, failures = versionState.process(overlayRoot, sourceVersionContribution, Path{}, sourceID, placementModeMapping) + } else { + processed = sourceVersionProcessingResult{payload: overlayRoot} } - overlayPayload = processed.payload } + if len(failures) != 0 { + return sources, failures + } + overlayPayload := processed.payload if versionNode := fragmentVersionClaim(overlayPayload, rule); versionNode != nil { location := publicLocation(versionNode.Location) return sources, []Diagnostic{{ @@ -155,8 +185,14 @@ func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *ass return sources, []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed after reading the selected overlay", nil)} } - if diagnostics := base.applyOverlay(overlayPayload, nil, nil, LayerOverlay); len(diagnostics) != 0 { - return sources, diagnostics + var overlayDiagnostics []Diagnostic + if len(policies) != 0 { + overlayDiagnostics = base.applyOverlayWithCollectionPolicy(overlayPayload, nil, nil, LayerOverlay, policy, processed.sequences) + } else { + overlayDiagnostics = base.applyOverlay(overlayPayload, nil, nil, LayerOverlay) + } + if len(overlayDiagnostics) != 0 { + return sources, overlayDiagnostics } base.current.inputBytes += int64(len(data)) base.current.inputNodes += nodeCount diff --git a/internal/engine/selected_overlay_test.go b/internal/engine/selected_overlay_test.go index 300d37c..78603c8 100644 --- a/internal/engine/selected_overlay_test.go +++ b/internal/engine/selected_overlay_test.go @@ -94,6 +94,263 @@ value: scalar } } +func TestLoadSelectedOverlayAppliesCollectionOperationsAndEvidence(t *testing.T) { + t.Parallel() + type item struct { + ID string `yaml:"id"` + Value string `yaml:"value"` + Children []item `yaml:"children"` + } + type config struct { + Items []item `yaml:"items"` + } + tests := []struct { + name string + mode string + overlay string + want []item + }{ + { + name: "append", + mode: "append", + overlay: `configkit: + sequences: + - destination: [items] + key: id + mode: append +items: [{id: c, value: added}] +`, + want: []item{{ID: "a", Value: "old"}, {ID: "b", Value: "kept"}, {ID: "c", Value: "added"}}, + }, + { + name: "upsert replaces complete records at retained positions", + mode: "upsert", + overlay: `configkit: + sequences: + - destination: [items] + key: id + mode: upsert +items: [{id: a, value: new}] +`, + want: []item{{ID: "a", Value: "new"}, {ID: "b", Value: "kept"}}, + }, + { + name: "replace", + mode: "replace", + overlay: `configkit: + sequences: + - destination: [items] + key: id + mode: replace +items: [{id: c, value: replacement}] +`, + want: []item{{ID: "c", Value: "replacement"}}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "items: [{id: a, value: old, children: [{id: child}]}, {id: b, value: kept}]\n") + writeImportFile(t, directory, "overlays/testing.yml", test.overlay) + result, err := testLoad[config](File(mainPath), Options[config]{Overlay: "testing"}) + if err != nil { + t.Fatal(err) + } + if len(result.Config.Items) != len(test.want) { + t.Fatalf("items = %#v, want %#v", result.Config.Items, test.want) + } + for index, want := range test.want { + if result.Config.Items[index].ID != want.ID || result.Config.Items[index].Value != want.Value { + t.Fatalf("items[%d] = %#v, want id %q value %q", index, result.Config.Items[index], want.ID, want.Value) + } + } + if _, found := result.Metadata().Lookup(Keys("configkit")); found { + t.Fatal("overlay collection controls reached application metadata") + } + if test.mode == "append" { + base, _ := result.Metadata().Lookup(Keys("items").Index(0).Key("value")) + added, _ := result.Metadata().Lookup(Keys("items").Index(2).Key("value")) + if base.Origin.Layer != LayerBase || added.Origin.Layer != LayerOverlay { + t.Fatalf("append origins = base %#v added %#v", base.Origin, added.Origin) + } + history := result.Metadata().History(Keys("items").Index(2).Key("value")) + if len(history) == 0 || history[len(history)-1].Origin.Layer != LayerOverlay { + t.Fatalf("append history = %#v", history) + } + } + if test.mode == "upsert" { + if _, found := result.Metadata().Lookup(Keys("items").Index(0).Key("children").Index(0)); found { + t.Fatal("upsert retained a discarded child") + } + history := result.Metadata().History(Keys("items").Index(0).Key("children").Index(0)) + if len(history) < 2 { + t.Fatalf("discarded child history = %#v", history) + } + } + }) + } +} + +func TestLoadSelectedOverlayReconcilesPendingAndRetainedCollectionPolicy(t *testing.T) { + t.Parallel() + requireCode := func(t *testing.T, main, overlay string, code Code) { + t.Helper() + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", main) + writeImportFile(t, directory, "overlays/testing.yml", overlay) + _, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Overlay: "testing"}) + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != code { + t.Fatalf("load error = %#v, want %s", err, code) + } + } + t.Run("key only validates retained records", func(t *testing.T) { + requireCode(t, "items: [{id: duplicate}, {id: duplicate}]\n", `configkit: + sequences: + - destination: [items] + key: id +`, CodeSequenceConflict) + }) + + t.Run("mode only retains lower identity", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + sequences: + - destination: [items] + key: id + mode: append +items: [{id: base}] +`) + writeImportFile(t, directory, "overlays/testing.yml", `configkit: + sequences: + - destination: [items] + mode: append +items: [{id: overlay}] +`) + result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Overlay: "testing"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Config.Items, []ordinarySequenceItem{{ID: "base"}, {ID: "overlay"}}) { + t.Fatalf("items = %#v", result.Config.Items) + } + }) + + t.Run("lower explicit mode remains active when overlay omits it", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + sequences: + - destination: [items] + key: id + mode: append +items: [{id: base}] +`) + writeImportFile(t, directory, "overlays/testing.yml", "items: [{id: overlay}]\n") + result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Overlay: "testing"}) + if err != nil || !reflect.DeepEqual(result.Config.Items, []ordinarySequenceItem{{ID: "base"}, {ID: "overlay"}}) { + t.Fatalf("retained operation = %#v, %v", result, err) + } + }) + + t.Run("overlay identity must agree with retained identity", func(t *testing.T) { + requireCode(t, `configkit: + sequences: + - destination: [items] + key: id +items: [{id: base}] +`, `configkit: + sequences: + - destination: [items] + key: slug +items: [{id: overlay}] +`, CodeSequenceConflict) + }) + + t.Run("replace accepts null and append rejects it", func(t *testing.T) { + for _, test := range []struct { + mode string + want Code + }{ + {mode: "replace"}, + {mode: "append", want: CodeSequenceInvalid}, + } { + overlay := "configkit:\n sequences:\n - destination: [items]\n key: id\n mode: " + test.mode + "\nitems: null\n" + if test.want != "" { + requireCode(t, "items: [{id: base}]\n", overlay, test.want) + continue + } + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "items: [{id: base}]\n") + writeImportFile(t, directory, "overlays/testing.yml", overlay) + result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Overlay: "testing"}) + if err != nil || result.Config.Items != nil { + t.Fatalf("replace null = %#v, %v", result, err) + } + } + }) + + t.Run("empty contributions and kind mismatches", func(t *testing.T) { + for _, test := range []struct { + name, mode, value string + want int + wantCode Code + }{ + {name: "append empty", mode: "append", value: "[]", want: 1}, + {name: "upsert empty", mode: "upsert", value: "[]", want: 1}, + {name: "replace empty", mode: "replace", value: "[]", want: 0}, + {name: "append mapping", mode: "append", value: "{id: overlay}", wantCode: CodeSequenceInvalid}, + } { + t.Run(test.name, func(t *testing.T) { + overlay := "configkit:\n sequences:\n - destination: [items]\n key: id\n mode: " + test.mode + "\nitems: " + test.value + "\n" + if test.wantCode != "" { + requireCode(t, "items: [{id: base}]\n", overlay, test.wantCode) + return + } + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "items: [{id: base}]\n") + writeImportFile(t, directory, "overlays/testing.yml", overlay) + result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Overlay: "testing"}) + if err != nil || len(result.Config.Items) != test.want { + t.Fatalf("%s result = %#v, %v", test.name, result, err) + } + }) + } + }) +} + +func TestLoadSelectedOverlayKeepsCollectionPoliciesIndependent(t *testing.T) { + t.Parallel() + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + sequences: + - destination: [items] + key: id + mode: append + - destination: [catalog, items] + key: id + mode: append +items: [{id: base}] +catalog: {items: [{id: catalog-base}]} +`) + writeImportFile(t, directory, "overlays/testing.yml", `configkit: + sequences: + - destination: [items] + mode: upsert + - destination: [catalog, items] + mode: append +items: [{id: base}] +catalog: {items: [{id: catalog-overlay}]} +`) + result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Overlay: "testing"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Config.Items, []ordinarySequenceItem{{ID: "base"}}) || + !reflect.DeepEqual(result.Config.Catalog.Items, []ordinarySequenceItem{{ID: "catalog-base"}, {ID: "catalog-overlay"}}) { + t.Fatalf("independent overlay policies = items %#v, catalog %#v", result.Config.Items, result.Config.Catalog.Items) + } +} + func TestLoadOverlaySelectionUsesOneFixedYAMLFile(t *testing.T) { directory := t.TempDir() mainPath := writeImportFile(t, directory, "main.yml", "settings: {kept: base}\n") diff --git a/internal/engine/sequence_registration_test.go b/internal/engine/sequence_registration_test.go index dab4986..7bac837 100644 --- a/internal/engine/sequence_registration_test.go +++ b/internal/engine/sequence_registration_test.go @@ -394,19 +394,16 @@ func TestRegisteredSequencePolicyAcrossLoadingLifecycle(t *testing.T) { } }) - t.Run("replacement overlay and CLI may introduce duplicates", func(t *testing.T) { + t.Run("replacement overlay validates identities and CLI may introduce duplicates", func(t *testing.T) { directory := t.TempDir() mainPath := writeImportFile(t, directory, "main.yml", "items: [{id: base}]\n") writeImportFile(t, directory, "overlays/testing.yml", "items: [{id: duplicate}, {id: duplicate}]\n") - result, err := testLoad[config](File(mainPath), Options[config]{Definition: definition, Overlay: "testing"}) - if err != nil { - t.Fatalf("replacement overlay error code = %s", loadCode(t, err)) - } - if len(result.Config.Items) != 2 { - t.Fatalf("replacement overlay = %#v, %v", result, err) + _, err := testLoad[config](File(mainPath), Options[config]{Definition: definition, Overlay: "testing"}) + if got := loadCode(t, err); got != CodeSequenceConflict { + t.Fatalf("replacement overlay error code = %s, want %s", got, CodeSequenceConflict) } - result, err = testLoad[config](Bytes("main", []byte("items: [{id: base}]\n")), Options[config]{ + result, err := testLoad[config](Bytes("main", []byte("items: [{id: base}]\n")), Options[config]{ Definition: definition, Overrides: []Override{{Path: Keys("items"), Flag: "--items", Value: []any{map[string]any{"id": "duplicate"}, map[string]any{"id": "duplicate"}}}}, }) diff --git a/internal/engine/source_version_test.go b/internal/engine/source_version_test.go index f83a646..404a1f1 100644 --- a/internal/engine/source_version_test.go +++ b/internal/engine/source_version_test.go @@ -1195,8 +1195,16 @@ func TestSourceVersionSelectedOverlayConsumesHeaderAndRetainsWarning(t *testing. t.Parallel() directory := t.TempDir() - mainPath := writeImportFile(t, directory, "main.yml", "schema: {version: main-v}\nsettings: {kept: base}\n") - overlayPath := writeImportFile(t, directory, "overlays/testing.yml", "schema: {version: overlay-v}\nsettings: {changed: overlay}\n") + mainPath := writeImportFile(t, directory, "main.yml", "schema: {version: main-v}\nsettings: {kept: base}\nitems: [{id: base}]\n") + overlayPath := writeImportFile(t, directory, "overlays/testing.yml", `schema: {version: overlay-v} +configkit: + sequences: + - destination: [items] + key: id + mode: append +settings: {changed: overlay} +items: [{id: overlay}] +`) var checked []string loaded, err := Load(File(mainPath), testSecretBinding(), Options[map[string]any]{ Overlay: "testing", SourceVersionMode: sourceVersionModeRequired, @@ -1220,6 +1228,9 @@ func TestSourceVersionSelectedOverlayConsumesHeaderAndRetainsWarning(t *testing. if loaded.Config["schema"].(map[string]any)["version"] != "main-v" || loaded.Config["settings"].(map[string]any)["changed"] != "overlay" { t.Fatalf("overlay payload = %#v", loaded.Config) } + if _, found := loaded.Config["configkit"]; found || len(loaded.Config["items"].([]any)) != 2 { + t.Fatalf("overlay declarations or collection values = %#v", loaded.Config) + } diagnostics := loaded.Inspection.Diagnostics() if len(diagnostics) != 1 || diagnostics[0].Code != CodeVersionWarning { t.Fatalf("overlay warning was not retained: %#v", diagnostics)