From d4ec8607d19eb1f47af026bfbd23c89663259549 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 6 Sep 2026 21:55:35 +0000 Subject: [PATCH] Apply explicit CLI configuration overrides --- README.md | 5 +- docs/config.md | 65 ++++- docs/development.md | 3 +- docs/internal/overview.md | 4 +- docs/roadmap/implementation.md | 36 ++- example_test.go | 19 ++ load.go | 38 ++- load_metadata.go | 126 ++++++--- load_test.go | 1 - overlay.go | 51 ++-- override.go | 457 +++++++++++++++++++++++++++++++++ override_test.go | 307 ++++++++++++++++++++++ records.go | 9 + 13 files changed, 1049 insertions(+), 72 deletions(-) create mode 100644 override.go create mode 100644 override_test.go diff --git a/README.md b/README.md index 402718e..0eb9f4d 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ Applications may perform audited defaulting and normalization through a resolver then report domain errors and warnings through read-only semantic validation. Secret tags and a path predicate drive deterministic redacted effective YAML. Typed secret references resolve environment names before validation, and explicit -secrets-directory bootstrap is available as a separate startup operation. Overrides -and value explanation are still under development. +secrets-directory bootstrap is available as a separate startup operation. Explicit +CLI override records apply above profiles. Value explanation is still under +development. ```go type Config struct { diff --git a/docs/config.md b/docs/config.md index 34214ec..5471fdb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -41,7 +41,8 @@ The current lifecycle is: ```text validate options → parse the selected source → traverse/compose enabled imports → build schema → check the optional application version → apply a selected profile - → create and clone defaults → strictly decode → recorded application resolution + → apply explicit CLI overrides → create and clone defaults → strictly decode + → recorded application resolution → resolve typed secret references → semantic validation → classify secrets → snapshot configuration and metadata ``` @@ -54,11 +55,11 @@ missing file remains an error; it is never interpreted as no input. The current `Options` boundary accepts a defaults factory, `VersionRule`, resolver, semantic validator, explicit path and glob imports, one selected profile, and -filesystem context. It also accepts static secret tags and a dynamic path predicate. -Overrides return `options.unavailable` because their lifecycle stage is not -implemented. `NoSource` rejects imports, active profiles, and filesystem context as -invalid combinations. Input containing a reserved `_configkit` mapping key is -rejected while imports are disabled. +filesystem context. It also accepts explicit CLI override records, static secret +tags, and a dynamic path predicate. `NoSource` rejects imports, active profiles, +and filesystem context as invalid combinations but accepts overrides. Input +containing a reserved `_configkit` mapping key is rejected while imports are +disabled. Every source, options, schema, version, defaults, decode, resolution, or validation failure returns a nil result and a `*LoadError`. Successful results contain caller-owned `Config`, detached @@ -106,8 +107,8 @@ by the profile location; an empty mapping is still an explicit container even th it contributes no children. Resolution origins are synthetic: they have layer `resolution`, no fabricated file location, and a `derived_default` or `normalization` operation. Reasons and dependency paths retain first occurrence order with duplicates -removed. Source records remain ordered as selected source, depth-first imports, then -the selected profile. +removed. Source records remain ordered as selected source, depth-first imports, the +selected profile, then synthetic CLI records in supplied order. ### Secret classification and effective rendering @@ -354,6 +355,54 @@ version, or replace an ancestor of that version with a scalar or null. These rul apply even if the supplied version text would be unchanged. Profiles cannot mask a base parse, composition, or version failure because those checks run first. +## Explicit CLI overrides + +`Options.Overrides` accepts only records the application knows were explicitly set +by its CLI parser. Each `Override` supplies a non-root mapping `Path`, a detached +`Value`, and a required safe display `Flag` such as `--workers`; ConfigKit has no CLI +parser dependency. Parser defaults must not be submitted as overrides, and `Flag` +must never contain the raw argument or configured value. General environment +variable overlays are an explicit non-goal: environment access is reserved for +typed secret references. + +Overrides apply after the selected profile and before typed defaults/decoding, so +the input precedence is application defaults < composed base < selected profile < +CLI. Recorded resolution, typed secret lookup, and semantic validation follow the +effective override tree. Overrides also work with `NoSource`. + +Values are converted directly into ConfigKit's owned tree without YAML parsing or +string coercion. Supported values are `nil`, built-in strings and booleans, all +built-in signed and unsigned integer widths except `uintptr`, finite `float32` and +`float64`, and recursive `[]any`/`map[string]any`. Strings stay strings; duration and +secret fields therefore receive string values. Nil is explicit null; empty strings, +false, and zero remain intentional; empty sequences replace whole lists; and mapping +values recursively overlay existing mappings. Map keys are sorted canonically and +sequence order is preserved. + +All records are detached and validated before any is applied. Root and explicit +sequence-index paths are invalid; parsed numeric tokens remain mapping keys. A path +may create missing mapping ancestors but cannot traverse an existing scalar, null, +or sequence. Duplicate and ancestor/descendant-overlapping records are rejected. +Neither paths nor values may create `_configkit`, and no record may overlap the +configured schema-version path. Unknown fields and effective type mismatches remain +ordinary strict-decoder errors. + +At most 1,024 records are accepted, and each record creates one synthetic CLI +source after file/profile sources in supplied order. Override sources share the +1,024-source ceiling. A record has an 8 MiB string-byte limit; override strings and +source bytes share the 64 MiB aggregate limit. Synthetic mapping keys and values +count toward the existing 1,000,000-node and depth-256 ceilings. Cycles, named or +unsupported types, nonfinite floats, and limit excesses fail with safe `override.*` +diagnostics. Diagnostics and provenance identify the safe `Flag` source, never raw +arguments or values. + +Override origins are explicit CLI origins. History retains replaced base/profile +definitions and records equal-value replacements as well as structural changes. +Caller-owned maps and slices are snapshotted during loading and must not be mutated +concurrently with the call. The executable `ExampleOverride` shows translating a +parser's explicit-set state into records while preserving an explicitly supplied +zero. + ## Explicit imports Imports are opt-in. With `Options.Imports` enabled, a document may reserve its root diff --git a/docs/development.md b/docs/development.md index bce5176..b6c3805 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,7 +4,8 @@ This is the contributor entry point for ConfigKit. Read the task-specific owners below before making changes. The repository is in early implementation: its public loader currently covers strict selected sources, explicit path/glob import graphs, composition, selected profile overlays, typed defaults, schema-version checks, -recorded application resolution, semantic validation, layered metadata, secret +explicit CLI overrides, recorded application resolution, semantic validation, +layered metadata, secret classification, typed environment references, explicit secrets bootstrap, and redacted effective rendering. The [component overview](internal/overview.md) owns the implementation inventory. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index febe28b..a27f066 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -15,6 +15,8 @@ module and package documentation boundary. Its implemented public primitives are conflicting location, and never aliases an input tree; - fixed selected-profile loading in `profile.go` and recursive, non-aliasing overlay with displacement records in `overlay.go`; +- detached parser-neutral CLI override validation and synthetic-source application + in `override.go`, sharing overlay, source, byte, node, and depth policies; - a per-operation private reflection descriptor graph in `schema.go` for supported types, YAML field binding, inlining, secret tags, recursion, and semantic paths; - descriptor-driven canonical projection, detached typed cloning, cycle/limit @@ -41,7 +43,7 @@ module and package documentation boundary. Its implemented public primitives are only upstream dumper integration isolated in `internal/yaml/write.go`. The [configuration contract](../config.md) owns implemented input, schema, decoding, -loading lifecycle, path imports, composition, profile overlay, version checking, +loading lifecycle, path imports, composition, profile/CLI overlay, version checking, recorded resolution, typed secret references/bootstrap, semantic validation, metadata, secret classification, redacted effective rendering, and ownership behavior. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2b4d42b..c7fec71 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1301,7 +1301,41 @@ Acceptance: ## Stage 18 — Apply explicit CLI overrides above profiles -Status: pending. +Status: complete. + +Completion evidence (2026-09-06): + +- Activated the existing `Override`/`Options.Overrides` surface through the new + parser-neutral implementation in [`override.go`](../../override.go). It accepts + only built-in scalar values, `nil`, `[]any`, and `map[string]any`; preserves + scalar kinds and sequence order; sorts maps; rejects named/unsupported values, + nonfinite floats and cycles; and snapshots all accepted caller-owned state without + YAML reparsing or textual inference. +- Added full-record preflight for safe flag labels, non-root mapping paths, + sequence traversal, existing non-mapping ancestors, duplicates/overlaps, + `_configkit`, schema-version overlap, and fixed record/source/string-byte/node/ + depth budgets. Valid records create deterministic synthetic CLI sources after + file and profile sources; failures identify those safe flag sources without raw + arguments or values. +- Generalized the existing non-aliasing overlay machinery across profile and CLI + layers. Overrides apply after the selected profile and before defaults/typed + decode, create missing mapping ancestors, recursively overlay mappings, replace + sequences atomically, and retain detached before/after snapshots plus displaced + origins even for equal-value operations. Layered history reconstruction now + preserves default/base/profile/CLI order and parent operations for descendants. +- Added lifecycle and boundary coverage in + [`override_test.go`](../../override_test.go) for defaults < imports/base < profile + < CLI < recorded normalization, followed by secret lookup and validation; explicit + zero/false/empty/list/null; defaults-only loads; map merge, whole-list replacement, + parsed numeric map keys, caller mutation, equal replacements, safe decode source + attribution, invalid paths/ancestors/overlaps/directives/version writes, value + types/cycles/nonfinite values, and bounds. Added executable `ExampleOverride` and + moved the normative behavior into the + [current override contract](../config.md#explicit-cli-overrides), explicitly + retaining general environment overlays as a non-goal. Verified `go test ./...`, + `go test -race ./...`, `go vet ./...`, `go build ./...`, `go mod tidy -diff`, and + `git diff --check` with Go 1.26.7 on linux/amd64. No deviations were required; + safe historical-value explanation remains later work. Read: [override contract](initial.md#cli-overrides), overlay/presence/version rules, source ordering, and the secret-reference boundary. diff --git a/example_test.go b/example_test.go index fa1d999..4f31d1f 100644 --- a/example_test.go +++ b/example_test.go @@ -115,3 +115,22 @@ func ExampleSecretEnv() { fmt.Println(result.Config.APIKey.EnvName(), result.Config.APIKey.IsResolved(), err) // Output: CONFIGKIT_EXAMPLE_API_KEY true } + +func ExampleOverride() { + type Config struct { + Workers int `yaml:"workers"` + } + workers, workersWasSet := 0, true // Supplied by the application's CLI parser. + var overrides []configkit.Override + if workersWasSet { + overrides = append(overrides, configkit.Override{ + Path: configkit.Keys("workers"), Value: workers, Flag: "--workers", + }) + } + result, err := configkit.Load(configkit.NoSource(), configkit.Options[Config]{ + Definition: configkit.Definition[Config]{Defaults: func() Config { return Config{Workers: 4} }}, + Overrides: overrides, + }) + fmt.Println(result.Config.Workers, result.Sources()[0].Label, err) + // Output: 0 --workers +} diff --git a/load.go b/load.go index 1651322..0c2d141 100644 --- a/load.go +++ b/load.go @@ -20,7 +20,7 @@ type Definition[T any] struct { // Options contains the application definition and per-load input selections. // Imports enables explicit path and glob imports; Profile selects one fixed profile -// overlay. Explicit overrides remain unavailable until implemented. +// overlay; Overrides contains only values explicitly supplied by an application CLI. type Options[T any] struct { Definition Definition[T] Imports bool @@ -35,7 +35,9 @@ type FilesystemContext struct { Root string } -// Override describes one explicitly supplied application override. +// Override describes one explicitly supplied application override. Value uses +// parser-neutral built-in scalars, nil, []any, or map[string]any; Flag is a safe +// display name and must not contain a raw argument or value. type Override struct { Path Path Value any @@ -70,6 +72,7 @@ type Result[T any] struct { diagnostics []Diagnostic final *document.Node resolution []resolutionSnapshot + overlays []overlayRecord secrets map[string]bool } @@ -180,6 +183,29 @@ func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied .. } inputDocument = composition.root } + if len(options.Overrides) != 0 { + if composition == nil { + if inputDocument == nil { + composition = &composedDocument{root: &document.Node{Kind: document.Mapping}, origins: make(map[string]Origin)} + } else { + composition, diagnostics = composeDocuments(inputDocument) + if len(diagnostics) != 0 { + accumulator.add(diagnostics...) + return nil, accumulator.loadError() + } + composition.inputBytes = parsed.bytes + composition.inputNodes = countInputNodes(inputDocument) + } + } + composition, accumulator.sources, diagnostics = applyOverrides( + composition, accumulator.sources, options.Overrides, options.Definition.Version, defaultImportLimits(), + ) + if len(diagnostics) != 0 { + accumulator.add(diagnostics...) + return nil, accumulator.loadError() + } + inputDocument = composition.root + } defaults := reflect.Zero(reflect.TypeFor[T]()) if options.Definition.Defaults != nil { @@ -226,6 +252,10 @@ func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied .. } } secrets := classifySecretPaths(schema, options.Definition.Secret, metadata, final) + var overlaySnapshots []overlayRecord + if composition != nil { + overlaySnapshots = cloneOverlayRecords(composition.overlays) + } return &Result[T]{ Config: configuration, metadata: metadata, @@ -233,6 +263,7 @@ func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied .. diagnostics: canonicalDiagnostics(accumulator.diagnostics), final: document.Clone(final), resolution: cloneResolutionSnapshots(resolutionSnapshots), + overlays: overlaySnapshots, secrets: secrets, }, nil } @@ -253,9 +284,6 @@ func selectOptions[T any](source Source, supplied []Options[T]) (Options[T], []D if options.Filesystem != nil { diagnostics = append(diagnostics, filesystemOptionDiagnostics(source, options.Filesystem)...) } - if len(options.Overrides) != 0 { - diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsUnavailable, "configuration overrides are not available in this release boundary")) - } if source.kind == sourceNone && options.Imports { diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource cannot enable imports")) } diff --git a/load_metadata.go b/load_metadata.go index 4d67966..7f5833d 100644 --- a/load_metadata.go +++ b/load_metadata.go @@ -39,40 +39,104 @@ func metadataFromDecoded(decoded decodedConfiguration) Metadata { } func applyOverlayMetadata(metadata *Metadata, records []overlayRecord) { - for _, record := range records { - for _, displaced := range record.displaced { - pointer := displaced.path.String() - history := metadata.history[pointer] - profileIndex := -1 - for index := len(history) - 1; index >= 0; index-- { - if history[index].Origin.Layer == LayerProfile { - profileIndex = index - break - } + baseline := make(map[string][]Change, len(metadata.history)) + for pointer, history := range metadata.history { + for _, change := range history { + if change.Origin.Layer == LayerDefault { + baseline[pointer] = append(baseline[pointer], cloneChange(change)) } - baseChange := Change{Path: displaced.path.clone(), Origin: cloneOrigin(displaced.origin)} - if len(history) != 0 && history[len(history)-1].Origin.Layer == LayerDefault { - baseChange.Previous = []Origin{cloneOrigin(history[len(history)-1].Origin)} - } - if profileIndex >= 0 { - history = append(history[:profileIndex], append([]Change{baseChange}, history[profileIndex:]...)...) - } else { - history = append(history, baseChange) - } - - change := cloneChange(record.change) - change.Previous = []Origin{cloneOrigin(displaced.origin)} - if profileIndex >= 0 { - // The effective-input projection created a leaf-shaped profile - // record. Replace it with the actual parent operation. - profileIndex++ - history[profileIndex] = change - } else { - history = append(history, change) - } - metadata.history[pointer] = history } } + events := make(map[string][]Change) + for _, record := range records { + exact := false + for _, displaced := range record.displaced { + pointer := displaced.path.String() + appendInputOriginEvent(events, baseline, pointer, displaced.path, displaced.origin) + appendInputChangeEvent(events, baseline, pointer, record.change) + if pointer == record.change.Path.String() { + exact = true + } + } + if !exact { + pointer := record.change.Path.String() + for _, previous := range record.change.Previous { + appendInputOriginEvent(events, baseline, pointer, record.change.Path, previous) + } + appendInputChangeEvent(events, baseline, pointer, record.change) + } + } + for pointer, info := range metadata.fields { + if !info.Explicit { + continue + } + path, err := ParsePath(pointer) + if err != nil { + continue + } + if history := events[pointer]; len(history) != 0 && history[len(history)-1].Origin.Layer == info.Origin.Layer && history[len(history)-1].Origin.Operation == info.Origin.Operation { + continue + } + appendInputOriginEvent(events, baseline, pointer, path, info.Origin) + } + metadata.history = make(map[string][]Change, len(baseline)+len(events)) + for pointer, history := range baseline { + metadata.history[pointer] = cloneChanges(history) + } + for pointer, history := range events { + metadata.history[pointer] = append(metadata.history[pointer], cloneChanges(history)...) + } +} + +func appendInputOriginEvent(events map[string][]Change, baseline map[string][]Change, pointer string, path Path, origin Origin) { + if origin.Layer == "" { + return + } + history := events[pointer] + if len(history) != 0 && originContainedBy(history[len(history)-1].Origin, origin) { + return + } + appendInputChangeEvent(events, baseline, pointer, Change{Path: path.clone(), Origin: cloneOrigin(origin)}) +} + +func appendInputChangeEvent(events map[string][]Change, baseline map[string][]Change, pointer string, value Change) { + change := cloneChange(value) + history := events[pointer] + if len(history) != 0 { + change.Previous = []Origin{cloneOrigin(history[len(history)-1].Origin)} + } else if defaults := baseline[pointer]; len(defaults) != 0 { + change.Previous = []Origin{cloneOrigin(defaults[len(defaults)-1].Origin)} + } else { + change.Previous = nil + } + events[pointer] = append(history, change) +} + +func originContainedBy(candidate, container Origin) bool { + if candidate.Layer != container.Layer || candidate.Operation != container.Operation { + return false + } + for _, location := range candidate.Locations { + found := false + for _, available := range container.Locations { + if location == available { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func cloneChanges(changes []Change) []Change { + cloned := make([]Change, len(changes)) + for index, change := range changes { + cloned[index] = cloneChange(change) + } + return cloned } func collectFieldInfo(node *document.Node, path Path, decoded decodedConfiguration, fields map[string]FieldInfo) { diff --git a/load_test.go b/load_test.go index b7169a2..33b3789 100644 --- a/load_test.go +++ b/load_test.go @@ -224,7 +224,6 @@ func TestLoadRejectsUnavailableOptionsAndDisabledDirectives(t *testing.T) { {name: "multiple options", source: validSource(), options: []Options[config]{{}, {}}, code: CodeOptionsInvalid}, {name: "imports without filesystem context", source: validSource(), options: []Options[config]{{Imports: true}}, code: CodeOptionsInvalid}, {name: "incomplete filesystem", source: validSource(), options: []Options[config]{{Filesystem: &FilesystemContext{}}}, code: CodeOptionsInvalid}, - {name: "overrides", source: validSource(), options: []Options[config]{{Overrides: []Override{{Path: Keys("name"), Value: "other", Flag: "--name"}}}}, code: CodeOptionsUnavailable}, {name: "missing version checker", source: validSource(), options: []Options[config]{{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version")}}}}, code: CodeOptionsInvalid}, {name: "no-source filesystem", source: NoSource(), options: []Options[config]{{Filesystem: &FilesystemContext{}}}, code: CodeOptionsInvalid}, } diff --git a/overlay.go b/overlay.go index 21c20af..80855cb 100644 --- a/overlay.go +++ b/overlay.go @@ -10,6 +10,8 @@ import ( type overlayRecord struct { change Change displaced []displacedDefinition + before *document.Node + after *document.Node } type displacedDefinition struct { @@ -32,12 +34,12 @@ func overlayDocuments(base *composedDocument, overlay *document.Node) (*composed inputBytes: base.inputBytes, inputNodes: base.inputNodes, } - mergeOverlayOrigin(result.origins, Path{}, overlay.Location) - mergeOverlayMappings(result, result.root, overlay, Path{}) + mergeOverlayOrigin(result.origins, Path{}, overlay.Location, LayerProfile) + mergeOverlayMappings(result, result.root, overlay, Path{}, LayerProfile, true) return result, nil } -func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path) { +func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool) { for _, incoming := range overlay.Mapping { path := current.Key(incoming.Key) index := documentEntryIndex(target.Mapping, incoming.Key) @@ -45,14 +47,14 @@ func mergeOverlayMappings(result *composedDocument, target, overlay *document.No cloned := incoming cloned.Value = document.Clone(incoming.Value) target.Mapping = append(target.Mapping, cloned) - collectOverlayOrigins(result.origins, incoming.Value, path) + collectOverlayOrigins(result.origins, incoming.Value, path, layer) continue } existing := target.Mapping[index].Value if existing.Kind == document.Mapping && incoming.Value.Kind == document.Mapping { - mergeOverlayOrigin(result.origins, path, incoming.Value.Location) - mergeOverlayMappings(result, existing, incoming.Value, path) + mergeOverlayOrigin(result.origins, path, incoming.Value.Location, layer) + mergeOverlayMappings(result, existing, incoming.Value, path, layer, recordReplacements) continue } @@ -61,41 +63,44 @@ func mergeOverlayMappings(result *composedDocument, target, overlay *document.No deleteOriginSubtree(result.origins, path) target.Mapping[index] = incoming target.Mapping[index].Value = document.Clone(incoming.Value) - collectOverlayOrigins(result.origins, incoming.Value, path) - result.overlays = append(result.overlays, overlayRecord{ - change: Change{ - Path: path.clone(), Origin: profileOrigin(incoming.Value.Location), - Previous: previousOrigins(previous), - }, - displaced: displaced, - }) + collectOverlayOrigins(result.origins, incoming.Value, path, layer) + if recordReplacements { + result.overlays = append(result.overlays, overlayRecord{ + change: Change{ + Path: path.clone(), Origin: overlayOrigin(layer, incoming.Value.Location), + Previous: previousOrigins(previous), + }, + displaced: displaced, + before: document.Clone(existing), after: document.Clone(incoming.Value), + }) + } } } -func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path) { - origins[path.String()] = profileOrigin(node.Location) +func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path, layer Layer) { + origins[path.String()] = overlayOrigin(layer, node.Location) switch node.Kind { case document.Mapping: for _, entry := range node.Mapping { - collectOverlayOrigins(origins, entry.Value, path.Key(entry.Key)) + collectOverlayOrigins(origins, entry.Value, path.Key(entry.Key), layer) } case document.Sequence: for index, child := range node.Sequence { - collectOverlayOrigins(origins, child, path.Index(index)) + collectOverlayOrigins(origins, child, path.Index(index), layer) } } } -func mergeOverlayOrigin(origins map[string]Origin, path Path, location document.Location) { +func mergeOverlayOrigin(origins map[string]Origin, path Path, location document.Location, layer Layer) { pointer := path.String() origin := cloneOrigin(origins[pointer]) - origin.Layer = LayerProfile + origin.Layer = layer origin.Locations = append(origin.Locations, publicLocation(location)) origins[pointer] = origin } -func profileOrigin(location document.Location) Origin { - return Origin{Layer: LayerProfile, Locations: []Location{publicLocation(location)}} +func overlayOrigin(layer Layer, location document.Location) Origin { + return Origin{Layer: layer, Locations: []Location{publicLocation(location)}} } func previousOrigins(origin Origin) []Origin { @@ -154,6 +159,8 @@ func cloneOverlayRecords(records []overlayRecord) []overlayRecord { cloned := make([]overlayRecord, len(records)) for index, record := range records { cloned[index].change = cloneChange(record.change) + cloned[index].before = document.Clone(record.before) + cloned[index].after = document.Clone(record.after) cloned[index].displaced = make([]displacedDefinition, len(record.displaced)) for displacedIndex, displaced := range record.displaced { cloned[index].displaced[displacedIndex] = displacedDefinition{ diff --git a/override.go b/override.go new file mode 100644 index 0000000..2a90543 --- /dev/null +++ b/override.go @@ -0,0 +1,457 @@ +package configkit + +import ( + "math" + "reflect" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +const maxOverrides = 1024 + +type preparedOverride struct { + path Path + value *document.Node + source SourceInfo + bytes int64 + nodes int +} + +type overrideValueBuilder struct { + sourceID string + active map[valueVisit]bool + bytes int64 + nodes int +} + +func applyOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { + if len(overrides) == 0 { + return base, cloneSourceInfos(sources), nil + } + if len(overrides) > maxOverrides || len(sources)+len(overrides) > limits.sources { + return nil, cloneSourceInfos(sources), []Diagnostic{{ + Code: CodeOverrideLimitExceeded, Severity: SeverityError, + Message: "configuration overrides exceed the record or source-count limit", + }} + } + + var diagnostics []Diagnostic + for index, override := range overrides { + if !validOverrideLabel(override.Flag) { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideInvalidLabel, Severity: SeverityError, + Message: "override record " + strconv.Itoa(index+1) + " requires a safe nonempty flag name", + }) + } + } + if len(diagnostics) != 0 { + return nil, cloneSourceInfos(sources), canonicalDiagnostics(diagnostics) + } + + prepared := make([]preparedOverride, len(overrides)) + allSources := cloneSourceInfos(sources) + for index, override := range overrides { + sourceID := "source-" + strconv.Itoa(len(sources)+index+1) + info := SourceInfo{ID: sourceID, Label: override.Flag, Layer: LayerCLI} + allSources = append(allSources, info) + prepared[index].source = info + } + + seen := make(map[string]int, len(overrides)) + pathAccepted := make([]bool, len(overrides)) + var usedBytes int64 + usedNodes := 0 + for index, override := range overrides { + location := &Location{SourceID: prepared[index].source.ID} + if issue := validateOverridePath(override.Path, base.root); issue != nil { + issue.Primary = location + diagnostics = append(diagnostics, *issue) + continue + } + pointer := override.Path.String() + if previous, duplicate := seen[pointer]; duplicate { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideDuplicate, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration override path is repeated", Primary: location, + Related: []Location{{SourceID: prepared[previous].source.ID}}, + }) + continue + } + seen[pointer] = index + if conflicting := overlappingOverride(overrides[:index], pathAccepted[:index], override.Path); conflicting >= 0 { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideOverlap, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration override paths overlap", Primary: location, + Related: []Location{{SourceID: prepared[conflicting].source.ID}}, + }) + continue + } + if rule != nil && (pathContains(override.Path, rule.Path) || pathContains(rule.Path, override.Path)) { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideVersion, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides cannot replace the schema-version path", Primary: location, + }) + continue + } + if pathContainsDirective(override.Path) { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideDirective, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides cannot create configuration directives", Primary: location, + }) + continue + } + pathAccepted[index] = true + + builder := overrideValueBuilder{sourceID: prepared[index].source.ID, active: make(map[valueVisit]bool)} + value, issue := builder.build(override.Value, override.Path, len(override.Path.segments)) + if issue != nil { + issue.Primary = location + diagnostics = append(diagnostics, *issue) + continue + } + if relative, ok := overrideDirectivePath(value, Path{}); ok { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideDirective, Severity: SeverityError, Path: appendOverridePath(override.Path, relative), + Message: "configuration overrides cannot create configuration directives", Primary: location, + }) + continue + } + nodes := builder.nodes + len(override.Path.segments)*2 + if nodes > limits.nodes || base.inputNodes+usedNodes+nodes > limits.nodes { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideLimitExceeded, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides exceed the aggregate node limit", Primary: location, + }) + continue + } + if builder.bytes > maxSourceBytes || base.inputBytes+usedBytes+builder.bytes > limits.bytes { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideLimitExceeded, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides exceed a record or aggregate string-byte limit", Primary: location, + }) + continue + } + prepared[index].path = override.Path.clone() + prepared[index].value = value + prepared[index].bytes = builder.bytes + prepared[index].nodes = nodes + usedBytes += builder.bytes + usedNodes += nodes + } + if len(diagnostics) != 0 { + return nil, allSources, canonicalDiagnostics(diagnostics) + } + + result := cloneComposedDocument(base) + for _, override := range prepared { + applyPreparedOverride(result, override) + } + result.inputBytes += usedBytes + result.inputNodes += usedNodes + return result, allSources, nil +} + +func validOverrideLabel(label string) bool { + if label == "" || strings.TrimSpace(label) != label || !utf8.ValidString(label) { + return false + } + for _, character := range label { + if unicode.IsControl(character) { + return false + } + } + return true +} + +func validateOverridePath(path Path, root *document.Node) *Diagnostic { + if !path.valid() || len(path.segments) == 0 { + return &Diagnostic{Code: CodeOverrideInvalidPath, Severity: SeverityError, Path: path.clone(), Message: "configuration override path must be a valid non-root mapping path"} + } + current := root + for index, segment := range path.segments { + if segment.kind == pathSegmentIndex { + return &Diagnostic{Code: CodeOverrideInvalidPath, Severity: SeverityError, Path: path.clone(), Message: "configuration override paths cannot contain sequence indices"} + } + if current == nil { + continue + } + if current.Kind != document.Mapping { + return &Diagnostic{Code: CodeOverrideInvalidPath, Severity: SeverityError, Path: path.clone(), Message: "configuration override path traverses a non-mapping value"} + } + position := documentEntryIndex(current.Mapping, segment.key) + if index == len(path.segments)-1 { + continue + } + if position < 0 { + current = nil + continue + } + current = current.Mapping[position].Value + } + return nil +} + +func overlappingOverride(previous []Override, valid []bool, path Path) int { + for index, override := range previous { + if valid[index] && (pathContains(override.Path, path) || pathContains(path, override.Path)) { + return index + } + } + return -1 +} + +func pathContainsDirective(path Path) bool { + for _, segment := range path.segments { + if segment.key == "_configkit" { + return true + } + } + return false +} + +func appendOverridePath(base, relative Path) Path { + result := base.clone() + result.segments = append(result.segments, clonePathSegments(relative.segments, 0)...) + return result +} + +func overrideDirectivePath(node *document.Node, path Path) (Path, bool) { + if node.Kind == document.Mapping { + for _, entry := range node.Mapping { + child := path.Key(entry.Key) + if entry.Key == "_configkit" { + return child, true + } + if found, ok := overrideDirectivePath(entry.Value, child); ok { + return found, true + } + } + } else if node.Kind == document.Sequence { + for index, child := range node.Sequence { + if found, ok := overrideDirectivePath(child, path.Index(index)); ok { + return found, true + } + } + } + return Path{}, false +} + +func (b *overrideValueBuilder) build(value any, path Path, depth int) (*document.Node, *Diagnostic) { + if depth > maxDocumentDepth { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the depth limit") + } + b.nodes++ + if b.nodes > maxDocumentNodes { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the node limit") + } + location := document.Location{SourceID: b.sourceID} + switch value := value.(type) { + case nil: + return &document.Node{Kind: document.Null, Location: location}, nil + case string: + if !b.addBytes(len(value)) { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the record string-byte limit") + } + return &document.Node{Kind: document.String, Text: value, Location: location}, nil + case bool: + return &document.Node{Kind: document.Boolean, Text: strconv.FormatBool(value), Location: location}, nil + case int: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int8: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int16: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int32: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int64: + return overrideNumber(strconv.FormatInt(value, 10), location), nil + case uint: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint8: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint16: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint32: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint64: + return overrideNumber(strconv.FormatUint(value, 10), location), nil + case float32: + if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) { + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override float must be finite") + } + return overrideNumber(strconv.FormatFloat(float64(value), 'g', -1, 32), location), nil + case float64: + if math.IsNaN(value) || math.IsInf(value, 0) { + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override float must be finite") + } + return overrideNumber(strconv.FormatFloat(value, 'g', -1, 64), location), nil + case []any: + if value == nil { + return &document.Node{Kind: document.Null, Location: location}, nil + } + leave, diagnostic := b.enter(reflect.ValueOf(value), path) + if diagnostic != nil { + return nil, diagnostic + } + defer leave() + if len(value) > maxDocumentNodes-b.nodes { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the node limit") + } + node := &document.Node{Kind: document.Sequence, Location: location, Sequence: make([]*document.Node, len(value))} + for index, child := range value { + built, diagnostic := b.build(child, path.Index(index), depth+1) + if diagnostic != nil { + return nil, diagnostic + } + node.Sequence[index] = built + } + return node, nil + case map[string]any: + if value == nil { + return &document.Node{Kind: document.Null, Location: location}, nil + } + leave, diagnostic := b.enter(reflect.ValueOf(value), path) + if diagnostic != nil { + return nil, diagnostic + } + defer leave() + if len(value) > (maxDocumentNodes-b.nodes)/2 { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the node limit") + } + keys := make([]string, 0, len(value)) + for key := range value { + keys = append(keys, key) + } + slices.Sort(keys) + node := &document.Node{Kind: document.Mapping, Location: location, Mapping: make([]document.Entry, 0, len(keys))} + for _, key := range keys { + b.nodes++ // mapping key + if !b.addBytes(len(key)) { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path.Key(key), "configuration override exceeds the record string-byte limit") + } + built, diagnostic := b.build(value[key], path.Key(key), depth+1) + if diagnostic != nil { + return nil, diagnostic + } + node.Mapping = append(node.Mapping, document.Entry{Key: key, KeyLocation: location, ValueLocation: location, Value: built}) + } + return node, nil + default: + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override value has an unsupported type") + } +} + +func (b *overrideValueBuilder) addBytes(count int) bool { + if count < 0 || int64(count) > maxSourceBytes-b.bytes { + return false + } + b.bytes += int64(count) + return true +} + +func (b *overrideValueBuilder) enter(value reflect.Value, path Path) (func(), *Diagnostic) { + visit, ok := visitFor(value) + if !ok { + return func() {}, nil + } + if b.active[visit] { + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override values must not contain cycles") + } + b.active[visit] = true + return func() { delete(b.active, visit) }, nil +} + +func overrideValueDiagnostic(code Code, path Path, message string) *Diagnostic { + return &Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message} +} + +func overrideNumber(text string, location document.Location) *document.Node { + return &document.Node{Kind: document.Number, Text: text, Location: location} +} + +func cloneComposedDocument(base *composedDocument) *composedDocument { + return &composedDocument{ + root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims), + origins: cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays), + inputBytes: base.inputBytes, inputNodes: base.inputNodes, + } +} + +func applyPreparedOverride(result *composedDocument, override preparedOverride) { + location := document.Location{SourceID: override.source.ID} + mergeOverlayOrigin(result.origins, Path{}, location, LayerCLI) + current := result.root + currentPath := Path{} + for index, segment := range override.path.segments { + path := currentPath.Key(segment.key) + position := documentEntryIndex(current.Mapping, segment.key) + if index < len(override.path.segments)-1 { + if position < 0 { + child := &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{}} + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: child}) + collectOverlayOrigins(result.origins, child, path, LayerCLI) + current = child + } else { + current = current.Mapping[position].Value + mergeOverlayOrigin(result.origins, path, location, LayerCLI) + } + currentPath = path + continue + } + + previous := cloneOrigin(result.origins[path.String()]) + var displaced []displacedDefinition + var before *document.Node + if position < 0 { + incoming := document.Clone(override.value) + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: incoming}) + collectOverlayOrigins(result.origins, incoming, path, LayerCLI) + } else { + existing := current.Mapping[position].Value + before = document.Clone(existing) + displaced = affectedOverrideOrigins(existing, override.value, result.origins, path) + if existing.Kind == document.Mapping && override.value.Kind == document.Mapping { + mergeOverlayOrigin(result.origins, path, location, LayerCLI) + mergeOverlayMappings(result, existing, override.value, path, LayerCLI, false) + } else { + deleteOriginSubtree(result.origins, path) + current.Mapping[position] = document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: document.Clone(override.value)} + collectOverlayOrigins(result.origins, current.Mapping[position].Value, path, LayerCLI) + } + } + position = documentEntryIndex(current.Mapping, segment.key) + result.overlays = append(result.overlays, overlayRecord{change: Change{ + Path: path.clone(), Origin: overlayOrigin(LayerCLI, location), Previous: previousOrigins(previous), + }, displaced: displaced, before: before, after: document.Clone(current.Mapping[position].Value)}) + } +} + +func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path) []displacedDefinition { + displaced := []displacedDefinition{{path: path.clone(), origin: cloneOrigin(origins[path.String()])}} + if existing.Kind == document.Mapping && incoming.Kind == document.Mapping { + for _, entry := range incoming.Mapping { + position := documentEntryIndex(existing.Mapping, entry.Key) + if position < 0 { + continue + } + childPath := path.Key(entry.Key) + childExisting := existing.Mapping[position].Value + if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping { + displaced = append(displaced, affectedOverrideOrigins(childExisting, entry.Value, origins, childPath)...) + } else { + displaced = append(displaced, displacedOrigins(childExisting, origins, childPath)...) + } + } + } else { + displaced = displacedOrigins(existing, origins, path) + } + slices.SortFunc(displaced, func(left, right displacedDefinition) int { + return strings.Compare(left.path.String(), right.path.String()) + }) + return displaced +} diff --git a/override_test.go b/override_test.go new file mode 100644 index 0000000..2b4023b --- /dev/null +++ b/override_test.go @@ -0,0 +1,307 @@ +package configkit + +import ( + "errors" + "math" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type overrideLifecycleSettings struct { + Target string `yaml:"target"` + Count int `yaml:"count"` + Enabled bool `yaml:"enabled"` + Text string `yaml:"text"` + Items []string `yaml:"items"` + Maybe *string `yaml:"maybe"` + Labels map[string]string `yaml:"labels"` + APIKey Secret `yaml:"api_key"` +} + +type overrideLifecycleConfig struct { + Version string `yaml:"version"` + Settings overrideLifecycleSettings `yaml:"settings"` +} + +func TestOverridesRunAtHighestInputPrecedenceBeforeResolutionAndValidation(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit:\n imports: [{path: base.yml}]\nversion: v1\n") + writeImportFile(t, directory, "base.yml", `settings: + target: base + count: 4 + enabled: true + text: base + items: [base] + maybe: base + labels: {base: retained} + api_key: BASE_SECRET +`) + writeImportFile(t, directory, filepath.Join("profiles", "testing.yml"), `settings: + target: profile + labels: {profile: retained} +`) + validated := false + overrides := []Override{ + {Path: Keys("settings", "target"), Value: " cli ", Flag: "--target"}, + {Path: Keys("settings", "count"), Value: 0, Flag: "--count"}, + {Path: Keys("settings", "enabled"), Value: false, Flag: "--enabled"}, + {Path: Keys("settings", "text"), Value: "", Flag: "--text"}, + {Path: Keys("settings", "items"), Value: []any{}, Flag: "--items"}, + {Path: Keys("settings", "maybe"), Value: nil, Flag: "--maybe"}, + {Path: Keys("settings", "labels"), Value: map[string]any{"cli": "selected"}, Flag: "--label"}, + {Path: Keys("settings", "api_key"), Value: "CLI_SECRET", Flag: "--api-key-env"}, + } + result, err := loadWithSecretLookup(File(mainPath), func(name string) (string, bool) { + if name != "CLI_SECRET" { + t.Fatalf("looked up superseded secret name %q", name) + } + return "credential-value", true + }, Options[overrideLifecycleConfig]{ + Imports: true, Profile: "testing", Overrides: overrides, + Definition: Definition[overrideLifecycleConfig]{ + Defaults: func() overrideLifecycleConfig { + return overrideLifecycleConfig{Settings: overrideLifecycleSettings{Target: "default", Count: 9, Text: "default", Items: []string{"default"}}} + }, + Version: &VersionRule{Path: Keys("version"), Check: func(version Version, validation *Validation) error { + if version.Text != "v1" { + validation.Error(Keys("version"), "unsupported version") + } + return nil + }}, + Resolve: func(configuration *overrideLifecycleConfig, resolution *Resolution) error { + Normalize(resolution, Keys("settings", "target"), &configuration.Settings.Target, strings.TrimSpace(configuration.Settings.Target), "trimmed target") + return nil + }, + Validate: func(configuration *overrideLifecycleConfig, validation *Validation) { + validated = configuration.Settings.Target == "cli" && configuration.Settings.APIKey.Value() == "credential-value" + }, + }, + }) + if err != nil { + t.Fatalf("loadWithSecretLookup() error = %v", err) + } + want := overrideLifecycleSettings{ + Target: "cli", Labels: map[string]string{"base": "retained", "profile": "retained", "cli": "selected"}, + APIKey: result.Config.Settings.APIKey, + } + if !validated || result.Config.Settings.Count != 0 || result.Config.Settings.Enabled || result.Config.Settings.Text != "" || + len(result.Config.Settings.Items) != 0 || result.Config.Settings.Items == nil || result.Config.Settings.Maybe != nil || + result.Config.Settings.Target != want.Target || !reflect.DeepEqual(result.Config.Settings.Labels, want.Labels) { + t.Fatalf("effective settings = %#v", result.Config.Settings) + } + sources := result.Sources() + if len(sources) != 3+len(overrides) || sources[0].Layer != LayerBase || sources[1].Layer != LayerBase || sources[2].Layer != LayerProfile { + t.Fatalf("sources = %#v", sources) + } + for index, override := range overrides { + if sources[index+3].Layer != LayerCLI || sources[index+3].Label != override.Flag || sources[index+3].Path != "" { + t.Fatalf("CLI source %d = %#v", index, sources[index+3]) + } + } + history := result.Metadata().History(Keys("settings", "target")) + wantLayers := []Layer{LayerDefault, LayerBase, LayerProfile, LayerCLI, LayerResolution} + if got := historyLayers(history); !reflect.DeepEqual(got, wantLayers) { + t.Fatalf("target history layers = %#v, want %#v; history %#v", got, wantLayers, history) + } +} + +func TestOverridesSupportDefaultsOnlyMapMergeListReplacementAndNumericKeys(t *testing.T) { + type config struct { + Labels map[string]string `yaml:"labels"` + Items []int `yaml:"items"` + Numbers map[string]int `yaml:"numbers"` + Dynamic any `yaml:"dynamic"` + } + labels := map[string]any{"0": "zero", "new": "value"} + items := []any{0, 2} + dynamic := map[string]any{ + "int": int8(1), "uint": uint16(2), "float": float32(3.5), "bool": false, + "null": nil, "list": []any{"value"}, "map": map[string]any{"key": "value"}, + } + result, err := Load(NoSource(), Options[config]{ + Definition: Definition[config]{Defaults: func() config { + return config{Labels: map[string]string{"kept": "default"}, Items: []int{9}} + }}, + Overrides: []Override{ + {Path: parseOverrideTestPath(t, "/labels"), Value: labels, Flag: "--labels"}, + {Path: Keys("items"), Value: items, Flag: "--items"}, + {Path: parseOverrideTestPath(t, "/numbers/0"), Value: 7, Flag: "--number"}, + {Path: Keys("dynamic"), Value: dynamic, Flag: "--dynamic"}, + }, + }) + labels["new"] = "mutated" + items[0] = 99 + dynamic["int"] = 99 + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Config.Labels, map[string]string{"kept": "default", "0": "zero", "new": "value"}) || !reflect.DeepEqual(result.Config.Items, []int{0, 2}) || result.Config.Numbers["0"] != 7 || result.Config.Dynamic.(map[string]any)["int"] != int64(1) { + t.Fatalf("defaults-only overrides = %#v", result.Config) + } + var effective strings.Builder + if err := result.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + if strings.Contains(effective.String(), "mutated") || !strings.Contains(effective.String(), "new: value") { + t.Fatalf("detached effective output = %q", effective.String()) + } + if info, ok := result.Metadata().Lookup(Keys("labels", "0")); !ok || !info.Explicit || info.Origin.Layer != LayerCLI { + t.Fatalf("numeric mapping key metadata = %#v, %v", info, ok) + } +} + +func TestOverrideValidationRejectsPathsOverlapValuesAndProtectedTargets(t *testing.T) { + type config struct { + Version string `yaml:"version"` + Scalar string `yaml:"scalar"` + Items []string `yaml:"items"` + Nested map[string]any `yaml:"nested"` + Unknown string `yaml:"unknown"` + } + base := Bytes("override-errors.yml", []byte("version: v1\nscalar: value\nitems: [one]\nnested: {}\n")) + version := &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }} + cycle := map[string]any{} + cycle["self"] = cycle + deep := any("leaf") + for range maxDocumentDepth + 1 { + deep = []any{deep} + } + for _, test := range []struct { + name string + overrides []Override + code Code + }{ + {name: "root", overrides: []Override{{Path: Keys(), Value: "x", Flag: "--root"}}, code: CodeOverrideInvalidPath}, + {name: "index", overrides: []Override{{Path: Keys("items").Index(0), Value: "x", Flag: "--item"}}, code: CodeOverrideInvalidPath}, + {name: "parsed sequence index", overrides: []Override{{Path: parseOverrideTestPath(t, "/items/0"), Value: "x", Flag: "--item"}}, code: CodeOverrideInvalidPath}, + {name: "scalar ancestor", overrides: []Override{{Path: Keys("scalar", "child"), Value: "x", Flag: "--child"}}, code: CodeOverrideInvalidPath}, + {name: "duplicate", overrides: []Override{{Path: Keys("scalar"), Value: "a", Flag: "--first"}, {Path: Keys("scalar"), Value: "b", Flag: "--second"}}, code: CodeOverrideDuplicate}, + {name: "overlap", overrides: []Override{{Path: Keys("nested"), Value: map[string]any{}, Flag: "--nested"}, {Path: Keys("nested", "child"), Value: "x", Flag: "--child"}}, code: CodeOverrideOverlap}, + {name: "unsupported", overrides: []Override{{Path: Keys("scalar"), Value: time.Second, Flag: "--scalar"}}, code: CodeOverrideInvalidValue}, + {name: "cycle", overrides: []Override{{Path: Keys("nested"), Value: cycle, Flag: "--nested"}}, code: CodeOverrideInvalidValue}, + {name: "nonfinite", overrides: []Override{{Path: Keys("scalar"), Value: math.Inf(1), Flag: "--scalar"}}, code: CodeOverrideInvalidValue}, + {name: "depth", overrides: []Override{{Path: Keys("nested"), Value: deep, Flag: "--nested"}}, code: CodeOverrideLimitExceeded}, + {name: "record bytes", overrides: []Override{{Path: Keys("scalar"), Value: strings.Repeat("x", maxSourceBytes+1), Flag: "--scalar"}}, code: CodeOverrideLimitExceeded}, + {name: "directive path", overrides: []Override{{Path: Keys("_configkit", "x"), Value: true, Flag: "--directive"}}, code: CodeOverrideDirective}, + {name: "directive value", overrides: []Override{{Path: Keys("nested"), Value: map[string]any{"_configkit": map[string]any{}}, Flag: "--nested"}}, code: CodeOverrideDirective}, + {name: "version", overrides: []Override{{Path: Keys("version"), Value: "v2", Flag: "--version"}}, code: CodeOverrideVersion}, + {name: "version ancestor", overrides: []Override{{Path: Keys("version"), Value: map[string]any{"child": "v2"}, Flag: "--version"}}, code: CodeOverrideVersion}, + {name: "label", overrides: []Override{{Path: Keys("scalar"), Value: "x", Flag: "\nCANARY"}}, code: CodeOverrideInvalidLabel}, + } { + t.Run(test.name, func(t *testing.T) { + result, err := Load(base, Options[config]{Definition: Definition[config]{Version: version}, Overrides: test.overrides}) + if result != nil { + t.Fatalf("result = %#v, want nil", result) + } + loadError := requireSecretLoadCode(t, err, test.code) + if strings.Contains(fmtLoadError(loadError), strings.Repeat("x", 64)) { + t.Fatal("override diagnostic leaked a raw value") + } + }) + } +} + +func TestOverrideCannotReplaceSchemaVersionAncestor(t *testing.T) { + type schemaConfig struct { + Version string `yaml:"version"` + } + type config struct { + Schema schemaConfig `yaml:"schema"` + } + _, err := Load(Bytes("version.yml", []byte("schema: {version: v1}\n")), Options[config]{ + Definition: Definition[config]{Version: &VersionRule{Path: Keys("schema", "version"), Check: func(Version, *Validation) error { return nil }}}, + Overrides: []Override{{Path: Keys("schema"), Value: map[string]any{"other": "value"}, Flag: "--schema"}}, + }) + requireSecretLoadCode(t, err, CodeOverrideVersion) +} + +func TestOverrideDecodeFailureUsesFlagSourceWithoutRawValue(t *testing.T) { + type config struct { + Count int `yaml:"count"` + } + result, err := Load(Bytes("base.yml", []byte("count: 1\n")), Options[config]{Overrides: []Override{{ + Path: Keys("missing"), Value: "RAW-ARG-CANARY", Flag: "--missing", + }}}) + if result != nil { + t.Fatalf("result = %#v", result) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error = %T", err) + } + diagnostics := loadError.Diagnostics() + sources := loadError.Sources() + if len(diagnostics) != 1 || diagnostics[0].Code != CodeDecodeUnknownField || diagnostics[0].Primary == nil || + len(sources) != 2 || sources[1].Label != "--missing" || diagnostics[0].Primary.SourceID != sources[1].ID { + t.Fatalf("diagnostics/sources = %#v / %#v", diagnostics, sources) + } + if strings.Contains(fmtLoadError(loadError), "RAW-ARG-CANARY") { + t.Fatal("decode failure leaked override value") + } +} + +func TestOverrideTypeMismatchUsesNormalStrictDecode(t *testing.T) { + type config struct { + Count int `yaml:"count"` + } + _, err := Load(Bytes("base.yml", []byte("count: 1\n")), Options[config]{Overrides: []Override{{ + Path: Keys("count"), Value: "not-an-integer", Flag: "--count", + }}}) + loadError := requireSecretLoadCode(t, err, CodeDecodeType) + if loadError.Diagnostics()[0].Primary == nil || loadError.Sources()[1].Label != "--count" || strings.Contains(fmtLoadError(loadError), "not-an-integer") { + t.Fatalf("type mismatch diagnostics/sources = %#v / %#v", loadError.Diagnostics(), loadError.Sources()) + } +} + +func TestOverrideEqualReplacementStillAppearsInHistory(t *testing.T) { + type config struct { + Value string `yaml:"value"` + } + result, err := Load(Bytes("equal.yml", []byte("value: same\n")), Options[config]{Overrides: []Override{{Path: Keys("value"), Value: "same", Flag: "--value"}}}) + if err != nil { + t.Fatal(err) + } + if got := historyLayers(result.Metadata().History(Keys("value"))); !reflect.DeepEqual(got, []Layer{LayerDefault, LayerBase, LayerCLI}) { + t.Fatalf("equal override history = %#v", result.Metadata().History(Keys("value"))) + } +} + +func TestOverrideAggregateLimitsFailBeforeApplication(t *testing.T) { + base, diagnostics := composeDocuments(&document.Node{Kind: document.Mapping}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + base.inputNodes = 1 + result, sources, diagnostics := applyOverrides(base, nil, []Override{{Path: Keys("value"), Value: "x", Flag: "--value"}}, nil, importLimits{sources: 1, bytes: 1, nodes: 2}) + if result != nil || len(sources) != 1 || len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded || len(base.root.Mapping) != 0 { + t.Fatalf("bounded apply = %#v, %#v, %#v; base %#v", result, sources, diagnostics, base.root) + } +} + +func historyLayers(history []Change) []Layer { + layers := make([]Layer, len(history)) + for index, change := range history { + layers[index] = change.Origin.Layer + } + return layers +} + +func parseOverrideTestPath(t *testing.T, value string) Path { + t.Helper() + path, err := ParsePath(value) + if err != nil { + t.Fatal(err) + } + return path +} + +func fmtLoadError(loadError *LoadError) string { + var output strings.Builder + _ = WriteDiagnostics(&output, loadError.Diagnostics()) + return loadError.Error() + output.String() +} diff --git a/records.go b/records.go index 69d0c98..7d76eac 100644 --- a/records.go +++ b/records.go @@ -132,6 +132,15 @@ const ( CodeSecretInvalidName Code = "secret.invalid_name" CodeSecretUnset Code = "secret.unset" CodeSecretEmpty Code = "secret.empty" + + CodeOverrideInvalidPath Code = "override.invalid_path" + CodeOverrideInvalidLabel Code = "override.invalid_label" + CodeOverrideInvalidValue Code = "override.invalid_value" + CodeOverrideDuplicate Code = "override.duplicate" + CodeOverrideOverlap Code = "override.overlap" + CodeOverrideDirective Code = "override.directive" + CodeOverrideVersion Code = "override.version" + CodeOverrideLimitExceeded Code = "override.limit_exceeded" ) // Location identifies a position in a configuration source. Zero line and column