From 41b5dff69b588cce6e35e379879db53f8ccadf98 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 23 Oct 2026 21:37:13 +0000 Subject: [PATCH] Apply CLI overrides after typed decoding --- README.md | 8 +- docs/config.md | 96 ++++---- docs/roadmap/implementation.md | 9 +- example_test.go | 11 +- facade_inheritance_test.go | 15 +- .../engine/destination_accounting_test.go | 8 +- internal/engine/explain.go | 72 +++--- internal/engine/explain_test.go | 16 ++ internal/engine/history.go | 4 +- internal/engine/history_sequence_test.go | 6 +- internal/engine/inheritance_history_test.go | 2 +- internal/engine/load.go | 35 ++- internal/engine/load_input.go | 14 +- internal/engine/load_request.go | 9 +- internal/engine/load_result.go | 49 +++-- internal/engine/override.go | 45 ++-- internal/engine/override_apply.go | 110 ---------- internal/engine/override_context.go | 78 ++++++- internal/engine/override_lifecycle_test.go | 207 ++++++++++++++++++ internal/engine/override_test.go | 21 +- 20 files changed, 516 insertions(+), 299 deletions(-) delete mode 100644 internal/engine/override_apply.go create mode 100644 internal/engine/override_lifecycle_test.go diff --git a/README.md b/README.md index 7e63bc1..641c6a0 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ then report domain errors and warnings through read-only semantic validation. Se tags and a path predicate drive deterministic redacted effective YAML. Typed secret references resolve environment names before validation, and explicit secrets-directory bootstrap is available as a separate startup operation. Explicit -CLI override records apply above overlays. Value explanation is available through +CLI override records apply after inheritance and typed decoding, before application +resolution. Value explanation is available through detached structured records and deterministic redacted output. Built-in `extends` relationships let a mapping inherit from a named sibling during the same load, retaining explicit presence and source/derivation history. The root @@ -76,8 +77,9 @@ mappings combine, while overlapping non-mapping definitions are errors. Between precedence tiers, mappings combine recursively, retaining omitted lower-priority children; other values replace lower-priority values. The main document and its imports form one strictly composed tier. Explicit underlay groups sit below it -and above typed defaults; a selected overlay and explicit CLI overrides sit -above it. The [configuration contract](docs/config.md#strict-document-composition) +and above typed defaults; a selected overlay applies before inheritance, while +explicit CLI overrides apply to the decoded configuration before application +resolution. The [configuration contract](docs/config.md#strict-document-composition) owns the detailed combination rules and input-specific constraints. At definition-registered paths only, strict composition can append records with distinct explicit IDs; ordinary and unregistered sequences keep their conflict diff --git a/docs/config.md b/docs/config.md index 6c97a3d..8f09570 100644 --- a/docs/config.md +++ b/docs/config.md @@ -55,8 +55,8 @@ The current lifecycle is: validate options → parse the selected source → extract source controls and traverse/strictly compose enabled imports → build schema → check the optional application version → apply a selected overlay - → apply explicit CLI overrides → expand sibling inheritance - → create and clone defaults → strictly decode + → expand sibling inheritance + → create and clone defaults → strictly decode → apply explicit CLI overrides → recorded application resolution → resolve typed secret references → semantic validation → classify secrets → snapshot configuration and metadata @@ -137,13 +137,13 @@ Each group is applied once as a precedence tier. The main tier is then composed with the accumulated underlay policy, and the optional application version is checked against that composed main tier only. Underlay roots cannot claim that version path (`underlay.version`); imported fragments retain `import.version` -protection. ConfigKit then applies the selected overlay and explicit CLI -overrides. Scalar, sequence, and null definitions replace lower values between -tiers unless a collection policy defines the sequence operation; mappings -combine recursively. Syntax and strict-composition errors inside a tier cannot -be masked by a higher-priority tier. Defaults, typed decoding, resolution, -secret lookup, validation, and publication run once on the final assembled -input. +protection. ConfigKit then applies the selected overlay and resolves inheritance. +Scalar, sequence, and null definitions replace lower values between tiers unless +a collection policy defines the sequence operation; mappings combine recursively. +After successful defaults and typed decoding, CLI overrides are applied to that +typed configuration before application resolution. Syntax, strict-composition, +inheritance, and decoding errors cannot be masked by a CLI override. Resolution, +secret lookup, validation, and publication run once on the resulting values. An active layered overlay requires `LayeredOptions.OverlayFilesystem` explicitly, independent of the main source's namespace. Its name and root-relative lookup @@ -530,20 +530,20 @@ historical path. Its effective value and each chronological before/after value a parser values. `Present=false` with empty YAML means absence or deletion; explicit null is present and renders as `null`. The history starts with the canonical typed application-default or Go-zero baseline, followed by relevant underlay and main-tier -contributions in precedence order, the selected overlay, supplied CLI records, -and audited net resolution operations. Within each strict tier, replay follows +contributions in precedence order, the selected overlay, and structural inheritance +events. Typed defaulting and decoding reconcile that source history before supplied +CLI records and audited net resolution operations. Within each strict tier, replay follows [scoped assembly order](#explicit-imports), not source-record acquisition order. -Structural inheritance events appear after source replay -and before typed reconciliation, including consumed or canceled `extends` input -and equal-valued child replacements. Overlapping parent/child resolution records +Inheritance events include consumed or canceled `extends` input and equal-valued +child replacements. Overlapping parent/child resolution records are rendered as one net subtree event with combined reasons and dependencies, rather than pretending that the recorded assignments ran in lexical order. Explicit equal -writes remain visible. When input replay -differs from the final typed input, a `ChangeDecoding` event records the transition -before domain resolution. Its origin retains the decoded path's source/default -attribution; its reason identifies static-default retention and canonical decoding. -This built-in inspection operation is not accepted by `Resolution.Record`. +writes remain visible. When structural replay differs from the pre-CLI typed input, +a `ChangeDecoding` event records retained defaults and canonical decoding. CLI +contributions then retain their raw input representation while effective and +resolution values use the typed projection. This built-in inspection operation is +not accepted by `Resolution.Record`. Base, overlay, and CLI history preserves the policy-validated input representation, including a superseded value that could not have decoded into the final Go field. @@ -640,13 +640,15 @@ must resolve to a nonempty value. Secret references work in typed structs, maps, pointers, slices, and arrays. A dynamic `any` value does not establish secret semantics and cannot contain `Secret`; applications must use a typed slot. -ConfigKit resolves only the effective names after recorded application resolution -and before semantic validation. Each unique name is read with `os.LookupEnv` once -per load. Validation can inspect the credential through the deliberately named -`Value()` accessor and distinguish success with `IsResolved()`. Later loads read -the environment again, and cloned defaults are rebuilt from names so a previously -resolved result cannot retain a stale credential. `Load` never writes to the -environment. +ConfigKit resolves only the effective names after CLI assignment and recorded +application resolution, and before semantic validation. CLI overrides may supply +or replace a typed secret reference; lookup uses that effective name and retains +its effective source location for diagnostics. Each unique name is read with +`os.LookupEnv` once per load. Validation can inspect the credential through the +deliberately named `Value()` accessor and distinguish success with `IsResolved()`. +Later loads read the environment again, and cloned defaults are rebuilt from names +so a previously resolved result cannot retain a stale credential. `Load` never +writes to the environment. Credential bytes are held only in the per-load lookup state and the trusted returned configuration, and are returned only by `Value`. Secret formatting uses a safe @@ -710,18 +712,19 @@ files when appropriate; ConfigKit stores no implicit global report. ### Recorded application resolution -An optional `Definition.Resolve` callback runs after successful typed decoding. It -receives the working configuration and a per-load `Resolution` context. The context's +An optional `Definition.Resolve` callback runs after successful defaults, typed +decoding, and explicit CLI assignment. It receives the working configuration and a +per-load `Resolution` context. The context's `IsExplicit` query uses surviving input presence, so explicit false, zero, empty, and null are never confused with absence or static defaults. `Resolution.InputInfo(path)` returns detached effective metadata from the fixed -decoded entry state, before application resolution runs. It includes retained static -defaults, preserves explicit null, and uses the same exact semantic path matching as -`Metadata.Lookup`; missing, malformed, and key/index-shape-mismatched paths return -false without diagnostics. Its `Explicit` value agrees with `IsExplicit` for every -found entry path. Resolution changes, additions, removals, and moves do not alter -this entry view. +post-decode entry state, including explicit CLI overrides, before application +resolution runs. It includes retained static defaults, preserves explicit null, and +uses the same exact semantic path matching as `Metadata.Lookup`; missing, +malformed, and key/index-shape-mismatched paths return false without diagnostics. +Its `Explicit` value agrees with `IsExplicit` for every found entry path. Resolution +changes, additions, removals, and moves do not alter this entry view. `Resolution.Source(id)` returns a detached record for an already acquired source in the same load: the main source, imports, selected overlay, and supplied CLI records @@ -950,10 +953,16 @@ must never contain the raw argument or configured value. General environment variable overlays are an explicit non-goal: environment access is reserved for typed secret references. -Overrides apply after the selected overlay and before typed defaults/decoding, so -the input precedence is application defaults < composed base < selected overlay < -CLI. Recorded resolution, typed secret lookup, and semantic validation follow the -effective override tree. Overrides also work with `NoSource`. +Overrides are prepared after source assembly and inheritance, then assigned after +typed defaults and decoding and before application resolution. The order is: +defaults and decoded sources < CLI < application resolution. A CLI mapping +recursively overlays the decoded mapping; a CLI sequence replaces the complete +decoded sequence regardless of source collection operations. Inheritance and +typed defaulting do not run again after CLI assignment. Earlier parsing, +composition, inheritance, and decoding failures remain errors even if an override +would replace the failing value. Recorded resolution, typed secret lookup, and +semantic validation use the effective post-override values. Overrides also work +with `NoSource`. Values are converted directly into ConfigKit's owned tree without YAML parsing or string coercion. Supported values are `nil`, built-in strings and booleans, all @@ -964,7 +973,8 @@ false, and zero remain intentional; empty sequences replace whole lists; and map values recursively overlay existing mappings. Map keys are sorted canonically and sequence order is preserved. -All records are detached and validated before any is applied. Root and explicit +All records are detached and validated before any is applied. They cannot create +`configkit` or `extends` controls. Root and explicit sequence-index paths are invalid; parsed numeric tokens remain mapping keys. A path may create missing mapping ancestors but cannot traverse an existing scalar, null, or sequence. Duplicate and ancestor/descendant-overlapping records are rejected. @@ -981,8 +991,12 @@ unsupported types, nonfinite floats, and limit excesses fail with safe `override diagnostics. Diagnostics and provenance identify the safe `Flag` source, never raw arguments or values. -Override origins are explicit CLI origins. History retains replaced base/overlay -definitions and records equal-value replacements as well as structural changes. +Override origins are explicit CLI origins. Resolution receives CLI values with +their explicit-presence and source information. Absence-based defaults preserve +those values, while recorded normalization and derivation may change them. History +retains replaced source definitions and records equal-value replacements as well +as structural changes. Secret lookup and validation use post-resolution values +with their effective source locations. Caller-owned maps and slices are snapshotted during loading and must not be mutated concurrently with the call. The executable `ExampleOverride` shows translating a parser's explicit-set state into records while preserving an explicitly supplied diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f9b9e69..951f175 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Sequence Identity And Combination Implementation Plan -Status: implementation in progress; Stages 1–25 complete. +Status: implementation in progress; Stages 1–26 complete. Audience: a Luna coding agent implementing one numbered stage per prompt, in order. This plan starts at commit @@ -878,6 +878,13 @@ assignment, inspection, and effective-value handoffs. Validation: override/lifecycle/resolution/inheritance/history/secret tests, `go test -race ./...`, and the everyday check. +**Completion:** Activated typed CLI assignment after structural expansion and +decoding in the shared load lifecycle, including resolution, history, and +effective secret-reference context. Removed early application and replay fallback +paths, updated contracts and examples, and revised obsolete relationship-change +expectations. Verified focused lifecycle and secret tests, `go test -race ./...`, +and `./scripts/check.sh`. + ## Stage 27 — Enforce Final Application Identity Constraints Scope: the final-result guarantee, not another combination or inheritance phase. diff --git a/example_test.go b/example_test.go index b296084..e68cdc4 100644 --- a/example_test.go +++ b/example_test.go @@ -176,8 +176,15 @@ func ExampleOverride() { }) } result, err := configkit.Load(configkit.NoSource(), configkit.Options[Config]{ - Definition: configkit.Definition[Config]{Defaults: func() Config { return Config{Workers: 4} }}, - Overrides: overrides, + Definition: configkit.Definition[Config]{ + Defaults: func() Config { return Config{Workers: 4} }, + Resolve: func(configuration *Config, resolution *configkit.Resolution) error { + configkit.Default(resolution, configkit.Keys("workers"), &configuration.Workers, 100, + "preserve an explicitly supplied worker count") + return nil + }, + }, + Overrides: overrides, }) fmt.Println(result.Config.Workers, result.Sources()[0].Label, err) // Output: 0 --workers diff --git a/facade_inheritance_test.go b/facade_inheritance_test.go index 9d67d1e..0972ea3 100644 --- a/facade_inheritance_test.go +++ b/facade_inheritance_test.go @@ -414,27 +414,24 @@ func TestFacadeInheritanceFailureStopsLaterCallbacks(t *testing.T) { } } -func TestFacadeInheritanceResolvesAfterImportsAndCLIReferenceChange(t *testing.T) { +func TestFacadeCLIOverridesCannotChangeInheritanceRelationship(t *testing.T) { filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{ "main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}}\n")}, "parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}, second: {count: 2}}\n")}, }) - result, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[inheritedConfiguration]{ + _, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[inheritedConfiguration]{ Imports: true, Overrides: []configkit.Override{{Path: configkit.Keys("entries", "child", "extends"), Value: "second", Flag: "--extends"}}, }) - if err != nil || result.Config.Entries["child"].Count != 2 { - t.Fatalf("import and CLI reference = %#v, %v", result, err) - } - history := result.Metadata().History(configkit.Keys("entries", "child", "extends")) - if len(history) < 3 || history[len(history)-1].Origin.Operation != configkit.ChangeInheritance { - t.Fatalf("superseded reference history = %#v", history) + loadError, ok := err.(*configkit.LoadError) + if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeOverrideDirective { + t.Fatalf("CLI inheritance relationship change = %v", err) } conflicting := configkit.NewFilesystem("bundle", fstest.MapFS{ "main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}, first: {count: 3}}\n")}, "parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}}\n")}, }) _, err = configkit.Load[inheritedConfiguration](conflicting.File("main.yml"), configkit.Options[inheritedConfiguration]{Imports: true}) - loadError, ok := err.(*configkit.LoadError) + loadError, ok = err.(*configkit.LoadError) if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict { t.Fatalf("strict conflict before expansion = %v", err) } diff --git a/internal/engine/destination_accounting_test.go b/internal/engine/destination_accounting_test.go index 4ed7b28..5f7a184 100644 --- a/internal/engine/destination_accounting_test.go +++ b/internal/engine/destination_accounting_test.go @@ -431,7 +431,13 @@ func TestSelectedOverlayAndCLIUsePlacementNodeCharges(t *testing.T) { overrides: []Override{override}, } accounting := &importAccounting{nodes: limits.nodes - baseNodes - overrideNodes + 1} - _, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting) + input, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting) + if len(diagnostics) != 0 { + t.Fatalf("assembly diagnostics = %#v", diagnostics) + } + _, diagnostics = preparePostStructuralOverrides(overridePreparationInput{ + root: input.document, inputBytes: accounting.bytes, inputNodes: accounting.nodes, + }, input.sources, request.overrides, nil, limits) if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded { t.Fatalf("override diagnostics = %#v", diagnostics) } diff --git a/internal/engine/explain.go b/internal/engine/explain.go index 7ae5174..d9e6a76 100644 --- a/internal/engine/explain.go +++ b/internal/engine/explain.go @@ -76,13 +76,9 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) { // Intermediate input replacements do not permanently suppress defaults: // decoding applies them to the final input tree. Show that reconciliation - // explicitly, including canonical scalar conversions, before domain changes. - decodedInput := r.input - decodedOrigins := r.inputOrigins - if r.preCLI != nil { - decodedInput = r.preCLI - decodedOrigins = r.preCLIOrigins - } + // explicitly before typed CLI contributions and domain resolution. + decodedInput := r.preCLI + decodedOrigins := r.preCLIOrigins replayed, _ := documentAtPath(working, path) decoded, _ := documentAtPathWithLookup(decodedInput, path, r.immutableLookup) if !configurationValuesEqual(replayed, decoded) { @@ -97,41 +93,39 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) { } working = document.Clone(decodedInput) - if r.preCLI != nil { - for _, application := range r.postDecodeApps { - operationPath, relevant := applicationOperationPath(application, path, r.immutableLookup) - var before ValueSnapshot - if relevant { - before = valueSnapshot(working, path, r.secrets) - } - applyHistoryApplication(&working, application, replayIndex) - if !relevant { - continue - } - after := valueSnapshot(working, path, r.secrets) - if !before.Present && !after.Present { - continue - } - explanation.History = append(explanation.History, ValueChange{ - Before: before, - After: after, - Origin: historyApplicationOrigin(application, operationPath, r.immutableLookup), - }) + for _, application := range r.postDecodeApps { + operationPath, relevant := applicationOperationPath(application, path, r.immutableLookup) + var before ValueSnapshot + if relevant { + before = valueSnapshot(working, path, r.secrets) } - replayed, _ = documentAtPath(working, path) - effectiveInput, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup) - if !configurationValuesEqual(replayed, effectiveInput) { - origin := cloneOrigin(r.inputOrigins[path.String()]) - origin.Operation = ChangeDecoding - explanation.History = append(explanation.History, ValueChange{ - Before: valueSnapshot(working, path, r.secrets), - After: valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup), - Origin: origin, - Reasons: []string{"applied the typed CLI assignment"}, - }) + applyHistoryApplication(&working, application, replayIndex) + if !relevant { + continue } - working = document.Clone(r.input) + after := valueSnapshot(working, path, r.secrets) + if !before.Present && !after.Present { + continue + } + explanation.History = append(explanation.History, ValueChange{ + Before: before, + After: after, + Origin: historyApplicationOrigin(application, operationPath, r.immutableLookup), + }) } + replayed, _ = documentAtPath(working, path) + effectiveInput, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup) + if !configurationValuesEqual(replayed, effectiveInput) { + origin := cloneOrigin(r.inputOrigins[path.String()]) + origin.Operation = ChangeDecoding + explanation.History = append(explanation.History, ValueChange{ + Before: valueSnapshot(working, path, r.secrets), + After: valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup), + Origin: origin, + Reasons: []string{"applied the typed CLI assignment"}, + }) + } + working = document.Clone(r.input) for _, snapshot := range r.resolutionReplay { relevant := pathsIntersect(snapshot.change.Path, path) var before ValueSnapshot diff --git a/internal/engine/explain_test.go b/internal/engine/explain_test.go index 5b80204..2094856 100644 --- a/internal/engine/explain_test.go +++ b/internal/engine/explain_test.go @@ -351,6 +351,10 @@ func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) { opts.Overlay = "p" } result, err := testLoad(File(main), opts) + if cli && base == "replaced-text" { + requireSecretLoadCode(t, err, CodeDecodeType) + return + } if err != nil { t.Fatal(err) } @@ -360,6 +364,18 @@ func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) { t.Fatal("missing explanation") } last := explanation.History[len(explanation.History)-1] + if cli && path.String() == "/m" { + if last.After != explanation.Effective || last.Origin.Layer != LayerCLI || last.Origin.Operation != "" { + t.Fatalf("CLI assignment was not the final history event: %#v", explanation) + } + continue + } + if cli && path.String() == "/m/a" { + if explanation.Effective.Present { + t.Fatalf("CLI map override retained descendants discarded by source null: %#v", explanation) + } + continue + } if last.After != explanation.Effective || last.Origin.Operation != ChangeDecoding { t.Fatalf("history does not reconcile final value: %#v", explanation) } diff --git a/internal/engine/history.go b/internal/engine/history.go index b8f45ec..dec21a6 100644 --- a/internal/engine/history.go +++ b/internal/engine/history.go @@ -42,8 +42,8 @@ func (c *retainedNodeCounter) add(root *document.Node) bool { // History applications borrow immutable policy-validated input trees. Metadata // records answer presence and origin queries; replay reconstructs value changes. -// Replay reconciles with canonical decoding before domain resolution so both -// contracts agree on the effective result without retaining a tree per field. +// Replay reconciles with canonical decoding, applies supplied CLI contributions, +// then records domain resolution without retaining a tree per field. type historyApplication struct { layer Layer path Path diff --git a/internal/engine/history_sequence_test.go b/internal/engine/history_sequence_test.go index dc41ca6..aa18a38 100644 --- a/internal/engine/history_sequence_test.go +++ b/internal/engine/history_sequence_test.go @@ -171,7 +171,7 @@ func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T inspection := &Inspection{ metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, - input: composed.root, final: composed.root, applications: composed.applications, + input: composed.root, preCLI: composed.root, final: composed.root, applications: composed.applications, } inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool { return reflect.DeepEqual(candidate, path) @@ -218,7 +218,7 @@ func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) { finalizeMetadataPaths(&metadata, final, applications) inspection := &Inspection{ metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, - input: final, final: final, applications: applications, + input: final, preCLI: final, final: final, applications: applications, } inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool { return reflect.DeepEqual(candidate, path) @@ -290,7 +290,7 @@ func TestMappedUpsertProjectsSensitiveReplacementEvidence(t *testing.T) { finalizeMetadataPaths(&metadata, finalRoot, applications) inspection := &Inspection{ metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, - input: finalRoot, final: finalRoot, applications: applications, + input: finalRoot, preCLI: finalRoot, final: finalRoot, applications: applications, } secretRule := func(candidate Path) bool { return reflect.DeepEqual(candidate, secretPath) } inspection.secrets = classifySecretPaths(nil, secretRule, metadata, finalRoot, applications) diff --git a/internal/engine/inheritance_history_test.go b/internal/engine/inheritance_history_test.go index 0f408bc..627e3a9 100644 --- a/internal/engine/inheritance_history_test.go +++ b/internal/engine/inheritance_history_test.go @@ -126,7 +126,7 @@ func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocume applyInheritanceMetadata(&metadata, composed.root, composed.origins, composed.overlays, expanded) finalizeMetadataPaths(&metadata, expanded.root, composed.applications, baseline, composed.root) return &Inspection{ - metadata: metadata, baseline: baseline, input: expanded.root, + metadata: metadata, baseline: baseline, input: expanded.root, preCLI: expanded.root, inputOrigins: expanded.origins, final: expanded.root, applications: composed.applications, inheritance: expanded.operations, }, expanded diff --git a/internal/engine/load.go b/internal/engine/load.go index 3a9a1cf..d1a531f 100644 --- a/internal/engine/load.go +++ b/internal/engine/load.go @@ -176,7 +176,6 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo mainOptions.deferAssembly = true mainOptions.SourceVersionMode = request.sourceVersionMode mainOptions.Definition.Version = request.protectionVersion() - mainOptions.Overrides = request.overrides if request.ordinary { mainOptions.Overlay = request.overlay mainOptions.directiveContext = rootDirectiveContextSource @@ -278,6 +277,19 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo input.document = expanded.root } + base := overridePreparationInput{root: input.document} + if input.composition != nil { + base.inputBytes = input.composition.inputBytes + base.inputNodes = input.composition.inputNodes + } + preparedOverrides, diagnostics := preparePostStructuralOverrides(base, input.sources, request.overrides, request.protectionVersion(), defaultImportLimits()) + accumulator.add(diagnostics...) + if hasErrors(diagnostics) { + return nil, accumulator.loadError() + } + input.sources = preparedOverrides.sources + accumulator.sources = input.sources + decodeDefinition := request.definition decodeDefinition.Version = request.protectionVersion() decoded, diagnostics := decodeLoadInput(schema, decodeDefinition, input) @@ -285,12 +297,25 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo if len(diagnostics) != 0 { return nil, accumulator.loadError() } - resolved, diagnostics := resolveLoadInput(schema, request.definition, input, decoded) + decodeContext := decodeContext{} + if decodeDefinition.Version != nil { + decodeContext.versionPath = &decodeDefinition.Version.Path + } + assigned, issues := applyDecodedOverrides(schema, decoded, preparedOverrides, decodeContext) + if len(issues) != 0 { + for _, issue := range issues { + accumulator.add(diagnosticFromDecodeIssue(issue)) + } + return nil, accumulator.loadError() + } + postDecode := preparePostDecodeOverrideContext(input, decoded, assigned) + resolved, diagnostics := resolveLoadInput(schema, request.definition, input, postDecode) accumulator.add(diagnostics...) if hasErrors(diagnostics) { return nil, accumulator.loadError() } - if !historyNodesWithinLimit(decoded.baseline, decoded.final, resolved.final, resolved.applications, resolved.snapshots, input.inheritance) { + historyApplications := append(cloneHistoryApplications(resolved.applications), cloneHistoryApplications(resolved.postDecodeApps)...) + if !historyNodesWithinLimit(decoded.baseline, decoded.final, resolved.final, historyApplications, resolved.snapshots, input.inheritance) { accumulator.add(Diagnostic{Code: CodeExplainLimitExceeded, Severity: SeverityError, Message: "retained configuration history exceeds the node limit"}) return nil, accumulator.loadError() } @@ -301,13 +326,13 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo accumulator.add(*ownershipIssue) return nil, accumulator.loadError() } - secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, input.document, lookup) + secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, resolved.referenceInput, lookup) accumulator.add(secretResolution.diagnostics...) if hasErrors(secretResolution.diagnostics) { return nil, accumulator.loadError() } if request.definition.Validate != nil { - diagnostics = runValidation(&configuration, request.definition.Validate, schema, resolved.final, input.document, secretResolution.lookups) + diagnostics = runValidation(&configuration, request.definition.Validate, schema, resolved.final, resolved.referenceInput, secretResolution.lookups) accumulator.add(diagnostics...) if hasErrors(diagnostics) { return nil, accumulator.loadError() diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index e87a2bf..8e1eed7 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -287,7 +287,7 @@ func registerAcquiredSource(source Source, context *FilesystemContext, info *Sou } func applyInputLayers[T any](source Source, options Options[T], input assembledInput) (assembledInput, []Diagnostic) { - if !activeOverlay(options.Overlay) && len(options.Overrides) == 0 { + if !activeOverlay(options.Overlay) { return input, nil } composition, diagnostics := compositionForInputWithPolicy(input, options.sequencePolicy) @@ -309,21 +309,13 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI } mergeResourceContexts(&input.contexts, contexts) } - if len(options.Overrides) != 0 { - input.sources, diagnostics = assembly.applyOverrides( - input.sources, options.Overrides, options.Definition.Version, defaultImportLimits(), - ) - if len(diagnostics) != 0 { - return input, diagnostics - } - } input.composition = assembly.current input.document = assembly.current.root return input, nil } -// compositionForInput initializes one handoff for overlays and CLI, including -// defaults-only CLI input. Raw source bytes and mapping keys retain their budgets. +// compositionForInput initializes the structural handoff used by inheritance +// expansion. Raw source bytes and mapping keys retain their budgets. func compositionForInput(input assembledInput) (*composedDocument, []Diagnostic) { return compositionForInputWithPolicy(input, sequencePolicy{}) } diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index 858482b..f943fec 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -167,7 +167,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a } } } - if assembly == nil && (activeOverlay(request.overlay) || len(request.overrides) != 0) { + if assembly == nil && activeOverlay(request.overlay) { composition, _ := compositionForInput(assembledInput{}) assembly = newAssemblyAccumulator(composition) } @@ -193,13 +193,6 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a } mergeResourceContexts(&input.contexts, contexts) } - if len(request.overrides) != 0 { - var diagnostics []Diagnostic - input.sources, diagnostics = assembly.applyOverrides(input.sources, request.overrides, request.protectionVersion(), defaultImportLimits()) - if len(diagnostics) != 0 { - return input, diagnostics - } - } input.composition = assembly.current input.document = assembly.current.root return input, nil diff --git a/internal/engine/load_result.go b/internal/engine/load_result.go index 6af60a1..18048e2 100644 --- a/internal/engine/load_result.go +++ b/internal/engine/load_result.go @@ -15,6 +15,11 @@ type resolvedLoad[T any] struct { final *document.Node snapshots []resolutionSnapshot applications []historyApplication + postDecodeApps []historyApplication + structuralInput *document.Node + preCLI *document.Node + preCLIOrigins map[string]Origin + referenceInput *document.Node versionEvidence []*sourceVersionEvidence inheritance *inheritanceScope contexts map[string]ResourceContext @@ -46,24 +51,16 @@ func decodeLoadInput[T any](schema *schemaDescriptor, definition Definition[T], return decoded, nil } -func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput, decoded decodedConfiguration) (resolvedLoad[T], []Diagnostic) { +func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput, prepared postDecodeOverrideContext) (resolvedLoad[T], []Diagnostic) { var diagnostics []Diagnostic + decoded := prepared.decoded configuration := decoded.value.Interface().(T) - metadata := metadataFromDecoded(decoded) - if input.inheritance != nil { - applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, input.composition.overlays, input.inheritance) - } else if input.composition != nil { - applyOverlayMetadata(&metadata, input.composition.overlays) - } + metadata := prepared.metadata final := decoded.final + referenceInput := prepared.referenceInput var resolutionSnapshots []resolutionSnapshot if definition.Resolve != nil { - sources := input.sources - if input.versionState != nil { - sources = sourceInfosWithVersionEvidence(sources, input.versionState.evidence, nil, true) - } - entry := newResolutionInput(metadata, decoded.final, sources, input.contexts) - resolved := runResolution(&configuration, definition.Resolve, schema, decoded, input.document, entry) + resolved := runResolution(&configuration, definition.Resolve, schema, decoded, referenceInput, prepared.resolution) diagnostics = append(diagnostics, resolved.diagnostics...) if hasErrors(resolved.diagnostics) { return resolvedLoad[T]{}, diagnostics @@ -71,6 +68,7 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], final = resolved.final resolutionSnapshots = resolved.snapshots applyResolutionMetadata(&metadata, resolved) + referenceInput = documentWithMetadataLocations(final, metadata, input.document) } applications := []historyApplication(nil) if input.composition != nil { @@ -82,7 +80,13 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], if input.versionState != nil { versionEvidence = append(versionEvidence, input.versionState.evidence...) } - return resolvedLoad[T]{configuration: configuration, decoded: decoded, metadata: metadata, final: final, snapshots: resolutionSnapshots, applications: applications, versionEvidence: versionEvidence, inheritance: input.inheritance, contexts: input.contexts}, diagnostics + return resolvedLoad[T]{ + configuration: configuration, decoded: decoded, metadata: metadata, final: final, + snapshots: resolutionSnapshots, applications: applications, + postDecodeApps: cloneHistoryApplications(prepared.postDecodeApps), structuralInput: document.Clone(prepared.structuralInput), + preCLI: document.Clone(prepared.preCLI), preCLIOrigins: cloneOrigins(prepared.preCLIOrigins), referenceInput: referenceInput, + versionEvidence: versionEvidence, inheritance: input.inheritance, contexts: input.contexts, + }, diagnostics } func detachLoadConfiguration[T any](schema *schemaDescriptor, configuration T, mode secretCloneMode) (T, *Diagnostic) { @@ -105,12 +109,15 @@ func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T] } decoded, metadata := resolved.decoded, resolved.metadata final, applications, resolutionSnapshots := resolved.final, resolved.applications, resolved.snapshots - finalizeMetadataPaths(&metadata, final, applications, decoded.baseline, decoded.final) - secrets := classifySecretPathsWithEvidence(schema, definition.Secret, metadata, final, applications, resolved.versionEvidence, decoded.baseline, decoded.final) + structuralInput := document.Clone(resolved.structuralInput) + preCLI := document.Clone(resolved.preCLI) + allApplications := append(cloneHistoryApplications(applications), cloneHistoryApplications(resolved.postDecodeApps)...) + finalizeMetadataPaths(&metadata, final, allApplications, decoded.baseline, resolved.structuralInput, resolved.preCLI, decoded.final) + secrets := classifySecretPathsWithEvidence(schema, definition.Secret, metadata, final, allApplications, resolved.versionEvidence, decoded.baseline, resolved.structuralInput, resolved.preCLI, decoded.final) sources := sourceInfosWithVersionEvidence(accumulator.sources, resolved.versionEvidence, secrets, false) - retainedTrees := make([]*document.Node, 0, len(applications)+2) - retainedTrees = append(retainedTrees, decoded.final, final) - for _, application := range applications { + retainedTrees := make([]*document.Node, 0, len(allApplications)+5) + retainedTrees = append(retainedTrees, decoded.final, final, structuralInput, preCLI) + for _, application := range allApplications { retainedTrees = append(retainedTrees, application.value) } inspection := &Inspection{ @@ -121,6 +128,10 @@ func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T] baseline: decoded.baseline, input: decoded.final, inputOrigins: cloneOriginIndex(decoded.origins), + structuralInput: structuralInput, + preCLI: preCLI, + preCLIOrigins: cloneOrigins(resolved.preCLIOrigins), + postDecodeApps: cloneHistoryApplications(resolved.postDecodeApps), final: final, immutableLookup: buildDocumentLookup(retainedTrees...), resolution: resolutionSnapshots, diff --git a/internal/engine/override.go b/internal/engine/override.go index d9bd8f5..e4e5f51 100644 --- a/internal/engine/override.go +++ b/internal/engine/override.go @@ -32,25 +32,13 @@ type overridePreparationInput struct { inputNodes int } -func overridePreparationInputFor(base *composedDocument) overridePreparationInput { - if base == nil { - return overridePreparationInput{} - } - return overridePreparationInput{root: base.root, inputBytes: base.inputBytes, inputNodes: base.inputNodes} -} - -func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (preparedOverrideBatch, []Diagnostic) { - return prepareOverrideBatch(overridePreparationInputFor(base), sources, overrides, rule, limits, false) -} - -// preparePostStructuralOverrides prepares the same detached batch for the -// lifecycle after structural expansion, where record inheritance directives -// are no longer valid CLI input. +// preparePostStructuralOverrides snapshots CLI input for typed assignment after +// structural expansion, when inheritance directives can no longer be supplied. func preparePostStructuralOverrides(base overridePreparationInput, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (preparedOverrideBatch, []Diagnostic) { - return prepareOverrideBatch(base, sources, overrides, rule, limits, true) + return prepareOverrideBatch(base, sources, overrides, rule, limits) } -func prepareOverrideBatch(base overridePreparationInput, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits, rejectInheritanceDirectives bool) (preparedOverrideBatch, []Diagnostic) { +func prepareOverrideBatch(base overridePreparationInput, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (preparedOverrideBatch, []Diagnostic) { if len(overrides) > maxOverrides || len(sources)+len(overrides) > limits.sources { return preparedOverrideBatch{sources: cloneSourceInfos(sources)}, []Diagnostic{{ Code: CodeOverrideLimitExceeded, Severity: SeverityError, @@ -120,7 +108,7 @@ func prepareOverrideBatch(base overridePreparationInput, sources []SourceInfo, o }) continue } - if pathContainsDirective(override.Path) || (rejectInheritanceDirectives && pathContainsInheritanceDirective(override.Path)) { + if pathContainsDirective(override.Path) || pathContainsInheritanceDirective(override.Path) { diagnostics = append(diagnostics, Diagnostic{ Code: CodeOverrideDirective, Severity: SeverityError, Path: override.Path.clone(), Message: "configuration overrides cannot create configuration directives", Primary: location, @@ -137,11 +125,7 @@ func prepareOverrideBatch(base overridePreparationInput, sources []SourceInfo, o diagnostics = append(diagnostics, diagnostic) continue } - findDirective := overrideDirectivePath - if rejectInheritanceDirectives { - findDirective = overrideStructuralDirectivePath - } - if relative, ok := findDirective(value, Path{}); ok { + if relative, ok := overrideStructuralDirectivePath(value, Path{}); ok { diagnostics = append(diagnostics, Diagnostic{ Code: CodeOverrideDirective, Severity: SeverityError, Path: appendOverridePath(override.Path, relative), Message: "configuration overrides cannot create configuration directives", Primary: location, @@ -241,28 +225,27 @@ func appendOverridePath(base, relative Path) Path { return result } -func overrideDirectivePath(node *document.Node, path Path) (Path, bool) { - return overrideDirectivePathWithInheritance(node, path, false) -} - func overrideStructuralDirectivePath(node *document.Node, path Path) (Path, bool) { - return overrideDirectivePathWithInheritance(node, path, true) + return overrideStructuralDirectivePathAt(node, path) } -func overrideDirectivePathWithInheritance(node *document.Node, path Path, rejectInheritanceDirectives bool) (Path, bool) { +func overrideStructuralDirectivePathAt(node *document.Node, path Path) (Path, bool) { + if node == nil { + return Path{}, false + } if node.Kind == document.Mapping { for _, entry := range node.Mapping { child := path.Key(entry.Key) - if entry.Key == directiveKey || (rejectInheritanceDirectives && entry.Key == inheritanceKey) { + if entry.Key == directiveKey || entry.Key == inheritanceKey { return child, true } - if found, ok := overrideDirectivePathWithInheritance(entry.Value, child, rejectInheritanceDirectives); ok { + if found, ok := overrideStructuralDirectivePathAt(entry.Value, child); ok { return found, true } } } else if node.Kind == document.Sequence { for index, child := range node.Sequence { - if found, ok := overrideDirectivePathWithInheritance(child, path.Index(index), rejectInheritanceDirectives); ok { + if found, ok := overrideStructuralDirectivePathAt(child, path.Index(index)); ok { return found, true } } diff --git a/internal/engine/override_apply.go b/internal/engine/override_apply.go deleted file mode 100644 index beba4c4..0000000 --- a/internal/engine/override_apply.go +++ /dev/null @@ -1,110 +0,0 @@ -package engine - -import ( - "gitea.maximumdirect.net/eric/ConfigKit/internal/document" -) - -func applyOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { - if len(overrides) == 0 { - return base, cloneSourceInfos(sources), nil - } - assembly := newAssemblyAccumulator(cloneComposedDocument(base)) - allSources, diagnostics := assembly.applyOverrides(sources, overrides, rule, limits) - if len(diagnostics) != 0 { - return nil, allSources, diagnostics - } - return assembly.current, allSources, nil -} - -func (a *assemblyAccumulator) applyOverrides(sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) { - if len(overrides) == 0 { - return cloneSourceInfos(sources), nil - } - batch, diagnostics := prepareOverrides(a.current, sources, overrides, rule, limits) - if len(diagnostics) != 0 { - return batch.sources, diagnostics - } - for _, override := range batch.overrides { - applyPreparedOverride(a.current, override, &a.index) - a.current.inputBytes += override.bytes - a.current.inputNodes += override.nodes - } - return batch.sources, nil -} - -func applyPreparedOverride(result *composedDocument, override preparedOverride, index *mappingIndex) { - location := document.Location{SourceID: override.source.ID} - mergeOverlayOrigin(result.origins, Path{}, location, LayerCLI) - current := result.root - currentPath := Path{} - for segmentIndex, segment := range override.path.segments { - path := currentPath.Key(segment.key) - position := index.find(current, segment.key) - if segmentIndex < len(override.path.segments)-1 { - if position < 0 { - child := &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{}} - index.append(current, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: child}) - collectOverlayOrigins(result.origins, child, path, LayerCLI) - current = child - } else { - current = current.Mapping[position].Value - mergeOverlayOrigin(result.origins, path, location, LayerCLI) - } - currentPath = path - continue - } - - previous := cloneOrigin(result.origins[path.String()]) - var displaced []displacedDefinition - if position < 0 { - incoming := document.Clone(override.value) - index.append(current, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: incoming}) - collectOverlayOrigins(result.origins, incoming, path, LayerCLI) - } else { - existing := current.Mapping[position].Value - displaced = affectedOverrideOrigins(existing, override.value, result.origins, path, index) - if existing.Kind == document.Mapping && override.value.Kind == document.Mapping { - mergeOverlayOrigin(result.origins, path, location, LayerCLI) - mergeOverlayMappings(result, existing, override.value, path, LayerCLI, false, nil, index) - } else { - deleteOriginSubtree(result.origins, path, existing) - current.Mapping[position] = document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: document.Clone(override.value)} - collectOverlayOrigins(result.origins, current.Mapping[position].Value, path, LayerCLI) - } - } - result.overlays = append(result.overlays, overlayRecord{change: Change{ - Path: path.clone(), Origin: overlayOrigin(LayerCLI, location), Previous: previousOrigins(previous), - }, displaced: displaced}) - result.applications = append(result.applications, historyApplication{ - layer: LayerCLI, path: path.clone(), value: override.value, - origin: overlayOrigin(LayerCLI, location), - }) - } -} - -func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) []displacedDefinition { - var displaced []displacedDefinition - collectAffectedOverrideOrigins(&displaced, existing, incoming, origins, path, index) - return sortDisplacedDefinitions(displaced) -} - -func collectAffectedOverrideOrigins(displaced *[]displacedDefinition, existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) { - if existing.Kind != document.Mapping || incoming.Kind != document.Mapping { - collectDisplacedOrigins(displaced, existing, origins, path, Origin{}) - return - } - *displaced = append(*displaced, displacedDefinition{path: path.clone(), origin: cloneOrigin(origins[path.String()])}) - for _, entry := range incoming.Mapping { - position := index.find(existing, entry.Key) - if position < 0 { - continue - } - childPath := path.Key(entry.Key) - childExisting := existing.Mapping[position].Value - if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping { - collectAffectedOverrideOrigins(displaced, childExisting, entry.Value, origins, childPath, index) - } else { - collectDisplacedOrigins(displaced, childExisting, origins, childPath, Origin{}) - } - } -} diff --git a/internal/engine/override_context.go b/internal/engine/override_context.go index 80cf7b0..04c743f 100644 --- a/internal/engine/override_context.go +++ b/internal/engine/override_context.go @@ -25,10 +25,16 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig after.final = document.Clone(assigned.final) metadata := metadataFromDecoded(after) + cliRecords := overrideMetadataRecords(before, assigned.overrides) + var structuralOverlays []overlayRecord if input.inheritance != nil && input.composition != nil { - applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, input.composition.overlays, input.inheritance) + structuralOverlays = input.composition.overlays + applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, append(cloneOverlayRecords(structuralOverlays), cliRecords...), input.inheritance) } else if input.composition != nil { - applyOverlayMetadata(&metadata, input.composition.overlays) + structuralOverlays = input.composition.overlays + applyOverlayMetadata(&metadata, append(cloneOverlayRecords(structuralOverlays), cliRecords...)) + } else if len(cliRecords) != 0 { + applyOverlayMetadata(&metadata, cliRecords) } sources := cloneSourceInfos(input.sources) @@ -42,6 +48,9 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig knownSources[override.source.ID] = true } } + if input.versionState != nil { + sources = sourceInfosWithVersionEvidence(sources, input.versionState.evidence, nil, true) + } resolution := newResolutionInput(metadata, assigned.final, sources, input.contexts) applications := structuralHistoryApplications(input) @@ -67,7 +76,54 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig sources: sources, applications: applications, postDecodeApps: postDecodeApps, - referenceInput: documentWithMetadataLocations(assigned.final, metadata), + referenceInput: documentWithMetadataLocations(assigned.final, metadata, input.document), + } +} + +func overrideMetadataRecords(before decodedConfiguration, overrides []preparedOverride) []overlayRecord { + if len(overrides) == 0 { + return nil + } + lookup := buildDocumentLookup(before.final) + records := make([]overlayRecord, 0, len(overrides)) + for _, override := range overrides { + location := document.Location{SourceID: override.source.ID} + previous := cloneOrigin(before.origins[override.path.String()]) + var displaced []displacedDefinition + if existing, ok := documentAtPathWithLookup(before.final, override.path, lookup); ok { + displaced = affectedOverrideOrigins(existing, override.value, before.origins, override.path, new(mappingIndex)) + } + records = append(records, overlayRecord{change: Change{ + Path: override.path.clone(), Origin: overlayOrigin(LayerCLI, location), Previous: previousOrigins(previous), + }, displaced: displaced}) + } + return records +} + +func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) []displacedDefinition { + var displaced []displacedDefinition + collectAffectedOverrideOrigins(&displaced, existing, incoming, origins, path, index) + return sortDisplacedDefinitions(displaced) +} + +func collectAffectedOverrideOrigins(displaced *[]displacedDefinition, existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) { + if existing.Kind != document.Mapping || incoming.Kind != document.Mapping { + collectDisplacedOrigins(displaced, existing, origins, path, Origin{}) + return + } + *displaced = append(*displaced, displacedDefinition{path: path.clone(), origin: cloneOrigin(origins[path.String()])}) + for _, entry := range incoming.Mapping { + position := index.find(existing, entry.Key) + if position < 0 { + continue + } + childPath := path.Key(entry.Key) + childExisting := existing.Mapping[position].Value + if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping { + collectAffectedOverrideOrigins(displaced, childExisting, entry.Value, origins, childPath, index) + } else { + collectDisplacedOrigins(displaced, childExisting, origins, childPath, Origin{}) + } } } @@ -88,18 +144,26 @@ func structuralHistoryApplications(input assembledInput) []historyApplication { // documentWithMetadataLocations attaches effective origin coordinates to a // canonical typed projection for secret and validation diagnostics. -func documentWithMetadataLocations(root *document.Node, metadata Metadata) *document.Node { +func documentWithMetadataLocations(root *document.Node, metadata Metadata, fallback ...*document.Node) *document.Node { located := document.Clone(root) if located == nil { return nil } + var fallbackRoot *document.Node + if len(fallback) != 0 { + fallbackRoot = fallback[0] + } var visit func(*document.Node, Path) visit = func(node *document.Node, path Path) { pointer := path.String() if info, exists := metadata.fields[pointer]; exists { if location, ok := metadataFieldLocation(metadata, pointer, info.Origin); ok { node.Location = location + } else if location, ok := nearestDocumentLocation(fallbackRoot, path); ok { + node.Location = location } + } else if location, ok := nearestDocumentLocation(fallbackRoot, path); ok { + node.Location = location } switch node.Kind { case document.Mapping: @@ -110,7 +174,13 @@ func documentWithMetadataLocations(root *document.Node, metadata Metadata) *docu if location, ok := metadataFieldLocation(metadata, childPath.String(), info.Origin); ok { entry.KeyLocation = location entry.ValueLocation = location + } else if location, ok := nearestDocumentLocation(fallbackRoot, childPath); ok { + entry.KeyLocation = location + entry.ValueLocation = location } + } else if location, ok := nearestDocumentLocation(fallbackRoot, childPath); ok { + entry.KeyLocation = location + entry.ValueLocation = location } visit(entry.Value, childPath) } diff --git a/internal/engine/override_lifecycle_test.go b/internal/engine/override_lifecycle_test.go new file mode 100644 index 0000000..7a77cfb --- /dev/null +++ b/internal/engine/override_lifecycle_test.go @@ -0,0 +1,207 @@ +package engine + +import ( + "reflect" + "testing" +) + +func TestPostDecodeOverrideLifecycleMatchesOrdinaryAndLayeredLoads(t *testing.T) { + t.Parallel() + type config struct { + Count int `yaml:"count"` + Derived int `yaml:"derived"` + } + input := Bytes("settings.yml", []byte("count: 4\n")) + defaultsCalls := 0 + definition := Definition[config]{ + Defaults: func() config { + defaultsCalls++ + return config{Count: 9, Derived: 1} + }, + Resolve: func(value *config, resolution *Resolution) error { + info, ok := resolution.InputInfo(Keys("count")) + if !ok || !info.Explicit || info.Origin.Layer != LayerCLI { + t.Errorf("resolution count info = %#v, %v", info, ok) + } + if source, ok := resolution.Source("source-2"); !ok || source.Layer != LayerCLI { + t.Errorf("resolution CLI source = %#v, %v", source, ok) + } + Default(resolution, Keys("count"), &value.Count, 99, "preserve explicit count") + Default(resolution, Keys("derived"), &value.Derived, value.Count*2, "derive from CLI count", Keys("count")) + Normalize(resolution, Keys("count"), &value.Count, value.Count+1, "normalize after CLI") + return nil + }, + } + options := Options[config]{Definition: definition, Overrides: []Override{{Path: Keys("count"), Value: 4, Flag: "--count"}}} + ordinary, err := testLoad(input, options) + if err != nil { + t.Fatal(err) + } + layered, err := testGroupedLoad(loadRequest[config]{ + main: loadInput{source: input}, definition: definition, + overrides: []Override{{Path: Keys("count"), Value: 4, Flag: "--count"}}, + }) + if err != nil { + t.Fatal(err) + } + if defaultsCalls != 2 { + t.Fatalf("defaults factory calls = %d, want once per load", defaultsCalls) + } + for name, result := range map[string]*testResult[config]{"ordinary": ordinary, "layered": layered} { + if result.Config.Count != 5 || result.Config.Derived != 8 { + t.Errorf("%s resolved configuration = %#v", name, result.Config) + } + if got := historyLayers(result.Metadata().History(Keys("count"))); !reflect.DeepEqual(got, []Layer{LayerDefault, LayerBase, LayerCLI, LayerResolution}) { + t.Errorf("%s count history layers = %#v", name, got) + } + info, ok := result.Metadata().Lookup(Keys("count")) + if !ok || info.Origin.Layer != LayerResolution { + t.Errorf("%s final count origin = %#v, %v", name, info, ok) + } + } + if !reflect.DeepEqual(ordinary.Config, layered.Config) || + !reflect.DeepEqual(ordinary.Metadata().History(Keys("count")), layered.Metadata().History(Keys("count"))) { + t.Fatalf("ordinary and layered results differ: %#v / %#v", ordinary.Config, layered.Config) + } +} + +func TestNoSourceOverridesReachResolutionAfterDefaults(t *testing.T) { + t.Parallel() + type config struct { + Count int `yaml:"count"` + Derived int `yaml:"derived"` + } + calls := 0 + result, err := testLoad(NoSource(), Options[config]{ + Definition: Definition[config]{ + Defaults: func() config { + calls++ + return config{Count: 2} + }, + Resolve: func(value *config, resolution *Resolution) error { + info, ok := resolution.InputInfo(Keys("count")) + if !ok || !info.Explicit || info.Origin.Layer != LayerCLI { + t.Errorf("resolution count info = %#v, %v", info, ok) + } + Default(resolution, Keys("count"), &value.Count, 10, "preserve CLI value") + Default(resolution, Keys("derived"), &value.Derived, value.Count*3, "derive from CLI count") + return nil + }, + }, + Overrides: []Override{{Path: Keys("count"), Value: 4, Flag: "--count"}}, + }) + if err != nil { + t.Fatal(err) + } + if calls != 1 || result.Config.Count != 4 || result.Config.Derived != 12 { + t.Fatalf("defaults-only CLI result = %#v, defaults calls %d", result.Config, calls) + } + if sources := result.Sources(); len(sources) != 1 || sources[0].Layer != LayerCLI { + t.Fatalf("defaults-only CLI sources = %#v", sources) + } +} + +func TestCLISequenceAssignmentReplacesAppendAndUpsertResults(t *testing.T) { + t.Parallel() + type item struct { + ID string `yaml:"id"` + } + type config struct { + Items []item `yaml:"items"` + } + declaration := func(mode string, id string) Source { + return Bytes(id, []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: "+mode+"\nitems: [{id: "+id+"}]\n")) + } + result, err := testGroupedLoad(loadRequest[config]{ + main: loadInput{source: declaration("upsert", "main")}, + underlays: []loadGroup{{inputs: []loadInput{{source: declaration("append", "underlay")}}}}, + overrides: []Override{{Path: Keys("items"), Value: []any{map[string]any{"id": "cli"}}, Flag: "--items"}}, + }) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Config.Items, []item{{ID: "cli"}}) { + t.Fatalf("CLI sequence assignment = %#v, want complete replacement", result.Config.Items) + } +} + +func TestCLISecretReferenceUsesEffectiveLookupAndLocation(t *testing.T) { + t.Parallel() + type config struct { + APIKey Secret `yaml:"api_key"` + } + options := Options[config]{ + Definition: Definition[config]{Defaults: func() config { return config{APIKey: SecretEnv("DEFAULT_SECRET")} }}, + Overrides: []Override{{Path: Keys("api_key"), Value: "CLI_SECRET", Flag: "--api-key"}}, + } + source := Bytes("secret-input", []byte("api_key: BASE_SECRET\n")) + t.Run("effective reference is looked up", func(t *testing.T) { + calls := 0 + result, err := testLoadWithSecretLookup(source, func(name string) (string, bool) { + calls++ + if name != "CLI_SECRET" { + t.Errorf("secret lookup name = %q", name) + } + return "effective-credential", true + }, options) + if err != nil || calls != 1 || result.Config.APIKey.Value() != "effective-credential" { + t.Fatalf("CLI secret result = %#v, calls %d, error %v", result, calls, err) + } + info, ok := result.Metadata().Lookup(Keys("api_key")) + if !ok || info.Origin.Layer != LayerCLI { + t.Fatalf("CLI secret origin = %#v, %v", info, ok) + } + }) + t.Run("missing effective reference uses CLI location", func(t *testing.T) { + result, err := testLoadWithSecretLookup(source, func(name string) (string, bool) { + if name != "CLI_SECRET" { + t.Errorf("secret lookup name = %q", name) + } + return "", false + }, options) + if result != nil { + t.Fatalf("missing secret returned a result: %#v", result) + } + loadErr, ok := err.(*LoadError) + if !ok || len(loadErr.Diagnostics()) != 1 { + t.Fatalf("missing secret error = %v", err) + } + diagnostic := loadErr.Diagnostics()[0] + if diagnostic.Code != CodeSecretUnset || diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" { + t.Fatalf("missing secret diagnostic = %#v", diagnostic) + } + }) +} + +func TestCLIOverridesCannotHideEarlierFailuresOrAddInheritance(t *testing.T) { + t.Parallel() + type config struct { + Count int `yaml:"count"` + Child map[string]any `yaml:"child"` + Parent map[string]any `yaml:"parent"` + } + t.Run("decode error", func(t *testing.T) { + _, err := testLoad(Bytes("bad-count", []byte("count: invalid\n")), Options[config]{ + Overrides: []Override{{Path: Keys("count"), Value: 3, Flag: "--count"}}, + }) + requireSecretLoadCode(t, err, CodeDecodeType) + }) + t.Run("inheritance error", func(t *testing.T) { + _, err := testLoad(Bytes("missing-parent", []byte("child: {extends: missing, value: source}\n")), Options[config]{ + Overrides: []Override{{Path: Keys("child"), Value: map[string]any{"value": "cli"}, Flag: "--child"}}, + }) + requireSecretLoadCode(t, err, CodeInheritanceMissing) + }) + t.Run("cannot add relationship", func(t *testing.T) { + _, err := testLoad(Bytes("new-relationship", []byte("child: {value: source}\nparent: {value: parent}\n")), Options[config]{ + Overrides: []Override{{Path: Keys("child"), Value: map[string]any{"extends": "parent"}, Flag: "--child"}}, + }) + requireSecretLoadCode(t, err, CodeOverrideDirective) + }) + t.Run("cannot cancel relationship", func(t *testing.T) { + _, err := testLoad(Bytes("cancel-relationship", []byte("child: {extends: parent}\nparent: {value: parent}\n")), Options[config]{ + Overrides: []Override{{Path: Keys("child", "extends"), Value: nil, Flag: "--extends"}}, + }) + requireSecretLoadCode(t, err, CodeOverrideDirective) + }) +} diff --git a/internal/engine/override_test.go b/internal/engine/override_test.go index daf3cb3..27633b6 100644 --- a/internal/engine/override_test.go +++ b/internal/engine/override_test.go @@ -440,9 +440,11 @@ func TestOverrideAggregateLimitsFailBeforeApplication(t *testing.T) { t.Fatal(diagnostics) } base.inputNodes = 1 - result, sources, diagnostics := applyOverrides(base, nil, []Override{{Path: Keys("value"), Value: "x", Flag: "--value"}}, nil, importLimits{sources: 1, bytes: 1, nodes: 2}) - if result != nil || len(sources) != 1 || len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded || len(base.root.Mapping) != 0 { - t.Fatalf("bounded apply = %#v, %#v, %#v; base %#v", result, sources, diagnostics, base.root) + _, diagnostics = preparePostStructuralOverrides(overridePreparationInput{ + root: base.root, inputBytes: base.inputBytes, inputNodes: base.inputNodes, + }, nil, []Override{{Path: Keys("value"), Value: "x", Flag: "--value"}}, nil, importLimits{sources: 1, bytes: 1, nodes: 2}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded || len(base.root.Mapping) != 0 { + t.Fatalf("bounded preparation = %#v; base %#v", diagnostics, base.root) } } @@ -454,12 +456,14 @@ func TestPreparedOverrideBatchDetachesInputsAndAllocatesSourcesInOrder(t *testin sources := []SourceInfo{{ID: "source-1", Label: "base", Layer: LayerBase, Kind: SourceKindMapping}} firstValue := map[string]any{"nested": map[string]any{"name": "original"}} secondValue := []any{map[string]any{"enabled": true}} - batch, diagnostics := prepareOverrides(base, sources, []Override{ + batch, diagnostics := preparePostStructuralOverrides(overridePreparationInput{ + root: base.root, inputBytes: base.inputBytes, inputNodes: base.inputNodes, + }, sources, []Override{ {Path: Keys("settings", "first"), Value: firstValue, Flag: "--first"}, {Path: Keys("settings", "second"), Value: secondValue, Flag: "--second"}, }, nil, defaultImportLimits()) if len(diagnostics) != 0 { - t.Fatalf("prepare overrides: %#v", diagnostics) + t.Fatalf("prepare post-structural overrides: %#v", diagnostics) } if len(batch.overrides) != 2 || len(batch.sources) != 3 { t.Fatalf("prepared counts = %d overrides, %d sources", len(batch.overrides), len(batch.sources)) @@ -514,10 +518,9 @@ func TestPostStructuralOverridePreparationRejectsInheritanceDirectives(t *testin } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if _, diagnostics := prepareOverrides(base, nil, []Override{test.override}, nil, defaultImportLimits()); len(diagnostics) != 0 { - t.Fatalf("legacy preparation changed before lifecycle activation: %#v", diagnostics) - } - _, diagnostics := preparePostStructuralOverrides(overridePreparationInputFor(base), nil, []Override{test.override}, nil, defaultImportLimits()) + _, diagnostics := preparePostStructuralOverrides(overridePreparationInput{ + root: base.root, inputBytes: base.inputBytes, inputNodes: base.inputNodes, + }, nil, []Override{test.override}, nil, defaultImportLimits()) if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideDirective || diagnostics[0].Path.String() != test.path { t.Fatalf("post-structural diagnostics = %#v, want directive at %s", diagnostics, test.path) }