From 9b4f9ac50e9701b4b3180790545f9c5952d283e1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 12 Sep 2026 15:58:06 +0000 Subject: [PATCH] Add internal engine relocation candidate --- docs/internal/overview.md | 4 + docs/roadmap/implementation.md | 20 + internal/engine/callback_ownership_test.go | 52 ++ internal/engine/clone.go | 159 ++++++ internal/engine/compose.go | 190 +++++++ internal/engine/compose_test.go | 194 +++++++ internal/engine/composition_fuzz_test.go | 68 +++ internal/engine/decode.go | 470 ++++++++++++++++ internal/engine/decode_test.go | 312 +++++++++++ internal/engine/diagnostic.go | 248 +++++++++ internal/engine/diagnostic_test.go | 143 +++++ internal/engine/directives.go | 229 ++++++++ internal/engine/document_compare.go | 110 ++++ internal/engine/document_walk.go | 86 +++ internal/engine/explain.go | 122 +++++ internal/engine/explain_render.go | 123 +++++ internal/engine/explain_test.go | 516 ++++++++++++++++++ internal/engine/filesystem.go | 121 +++++ internal/engine/glob.go | 205 +++++++ internal/engine/glob_test.go | 273 ++++++++++ internal/engine/history.go | 276 ++++++++++ internal/engine/imports.go | 324 +++++++++++ internal/engine/imports_test.go | 476 ++++++++++++++++ internal/engine/independence_test.go | 116 ++++ internal/engine/load.go | 185 +++++++ internal/engine/load_input.go | 88 +++ internal/engine/load_metadata.go | 330 +++++++++++ internal/engine/load_result.go | 120 ++++ internal/engine/load_test.go | 604 +++++++++++++++++++++ internal/engine/metadata.go | 149 +++++ internal/engine/metadata_test.go | 274 ++++++++++ internal/engine/options.go | 41 ++ internal/engine/overlay.go | 161 ++++++ internal/engine/overlay_test.go | 78 +++ internal/engine/override.go | 223 ++++++++ internal/engine/override_apply.go | 100 ++++ internal/engine/override_test.go | 307 +++++++++++ internal/engine/override_value.go | 150 +++++ internal/engine/parse.go | 62 +++ internal/engine/parse_test.go | 38 ++ internal/engine/path.go | 148 +++++ internal/engine/path_fuzz_test.go | 27 + internal/engine/path_test.go | 88 +++ internal/engine/profile.go | 131 +++++ internal/engine/profile_test.go | 267 +++++++++ internal/engine/projection.go | 248 +++++++++ internal/engine/projection_test.go | 394 ++++++++++++++ internal/engine/provenance_test.go | 510 +++++++++++++++++ internal/engine/records.go | 286 ++++++++++ internal/engine/render.go | 46 ++ internal/engine/render_test.go | 250 +++++++++ internal/engine/resolution.go | 367 +++++++++++++ internal/engine/resolution_test.go | 457 ++++++++++++++++ internal/engine/schema.go | 438 +++++++++++++++ internal/engine/schema_test.go | 342 ++++++++++++ internal/engine/secret.go | 153 ++++++ internal/engine/secret_reference.go | 225 ++++++++ internal/engine/secret_reference_test.go | 377 +++++++++++++ internal/engine/secrets.go | 371 +++++++++++++ internal/engine/secrets_test.go | 377 +++++++++++++ internal/engine/snapshot.go | 63 +++ internal/engine/source.go | 216 ++++++++ internal/engine/source_fifo_test.go | 71 +++ internal/engine/source_file.go | 75 +++ internal/engine/source_file_test.go | 48 ++ internal/engine/source_test.go | 202 +++++++ internal/engine/validation.go | 248 +++++++++ internal/engine/validation_test.go | 214 ++++++++ internal/engine/value_operation.go | 126 +++++ internal/engine/version_path_test.go | 90 +++ 70 files changed, 14602 insertions(+) create mode 100644 internal/engine/callback_ownership_test.go create mode 100644 internal/engine/clone.go create mode 100644 internal/engine/compose.go create mode 100644 internal/engine/compose_test.go create mode 100644 internal/engine/composition_fuzz_test.go create mode 100644 internal/engine/decode.go create mode 100644 internal/engine/decode_test.go create mode 100644 internal/engine/diagnostic.go create mode 100644 internal/engine/diagnostic_test.go create mode 100644 internal/engine/directives.go create mode 100644 internal/engine/document_compare.go create mode 100644 internal/engine/document_walk.go create mode 100644 internal/engine/explain.go create mode 100644 internal/engine/explain_render.go create mode 100644 internal/engine/explain_test.go create mode 100644 internal/engine/filesystem.go create mode 100644 internal/engine/glob.go create mode 100644 internal/engine/glob_test.go create mode 100644 internal/engine/history.go create mode 100644 internal/engine/imports.go create mode 100644 internal/engine/imports_test.go create mode 100644 internal/engine/independence_test.go create mode 100644 internal/engine/load.go create mode 100644 internal/engine/load_input.go create mode 100644 internal/engine/load_metadata.go create mode 100644 internal/engine/load_result.go create mode 100644 internal/engine/load_test.go create mode 100644 internal/engine/metadata.go create mode 100644 internal/engine/metadata_test.go create mode 100644 internal/engine/options.go create mode 100644 internal/engine/overlay.go create mode 100644 internal/engine/overlay_test.go create mode 100644 internal/engine/override.go create mode 100644 internal/engine/override_apply.go create mode 100644 internal/engine/override_test.go create mode 100644 internal/engine/override_value.go create mode 100644 internal/engine/parse.go create mode 100644 internal/engine/parse_test.go create mode 100644 internal/engine/path.go create mode 100644 internal/engine/path_fuzz_test.go create mode 100644 internal/engine/path_test.go create mode 100644 internal/engine/profile.go create mode 100644 internal/engine/profile_test.go create mode 100644 internal/engine/projection.go create mode 100644 internal/engine/projection_test.go create mode 100644 internal/engine/provenance_test.go create mode 100644 internal/engine/records.go create mode 100644 internal/engine/render.go create mode 100644 internal/engine/render_test.go create mode 100644 internal/engine/resolution.go create mode 100644 internal/engine/resolution_test.go create mode 100644 internal/engine/schema.go create mode 100644 internal/engine/schema_test.go create mode 100644 internal/engine/secret.go create mode 100644 internal/engine/secret_reference.go create mode 100644 internal/engine/secret_reference_test.go create mode 100644 internal/engine/secrets.go create mode 100644 internal/engine/secrets_test.go create mode 100644 internal/engine/snapshot.go create mode 100644 internal/engine/source.go create mode 100644 internal/engine/source_fifo_test.go create mode 100644 internal/engine/source_file.go create mode 100644 internal/engine/source_file_test.go create mode 100644 internal/engine/source_test.go create mode 100644 internal/engine/validation.go create mode 100644 internal/engine/validation_test.go create mode 100644 internal/engine/value_operation.go create mode 100644 internal/engine/version_path_test.go diff --git a/docs/internal/overview.md b/docs/internal/overview.md index b8f5e01..897aedc 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -3,6 +3,10 @@ ConfigKit exposes the root `configkit` package and keeps its document and YAML representations private. The implemented components and their owners are: +- a temporary `internal/engine` relocation candidate containing a mechanical + copy of the root loading, decoding, resolution, provenance, inspection, and + rendering mechanisms with their behavior tests; it imports no root code and + is not yet used by the root package or consumers; - immutable semantic `Path` construction and strict JSON Pointer parsing; - detached effective presence/origin and chronological history queries in `metadata.go`, with provenance and diagnostic record types; diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3fe1a0f..9d8915d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -233,6 +233,26 @@ no root code, existing internal/parser boundaries remain intact, and root consum still execute the unchanged original implementation. This stage is deliberately mechanical; do not mix ownership redesign into the relocation diff. +### Completion + +Status: complete (2026-09-12). + +Copied the 39 root production files from the Stage 1 baseline `f81def9` into +`internal/engine`, excluding only root package orientation in `doc.go`, and +changed package declarations mechanically. Copied the 29 applicable root +behavior and fuzz tests. Repository documentation, release-script and +import-boundary checks, root external-package/API tests, and maintained consumer +integration tests remain at their existing owners; root has not been changed to +delegate to the candidate. No root testdata fixtures exist to copy. + +`docs/internal/overview.md` now labels the candidate as temporary and unused. +`go list -deps` confirms that the engine has no dependency on the root module +package; its existing dependencies remain the private document and YAML +boundaries. Linux validation with Go 1.26.7 passed `go test ./internal/engine` +and `./scripts/check.sh`. There were no behavioral deviations or package-renamed +diagnostics. The next handoff replaces the copied engine's concrete public +secret assumption with an explicit per-operation exact-type binding. + ## Stage 3 — Decouple reflection from the public secret type Status: not started. diff --git a/internal/engine/callback_ownership_test.go b/internal/engine/callback_ownership_test.go new file mode 100644 index 0000000..ede2318 --- /dev/null +++ b/internal/engine/callback_ownership_test.go @@ -0,0 +1,52 @@ +package engine + +import "testing" + +func TestCallbackAssignmentsAreDetachedAcrossLoads(t *testing.T) { + type object struct{ Value string } + type cfg struct { + Map map[string][]*object + Pointer *object + Secret Secret + } + for _, validatorAssignment := range []bool{false, true} { + name := "resolver" + if validatorAssignment { + name = "validator" + } + t.Run(name, func(t *testing.T) { + external := &object{Value: "original"} + shared := map[string][]*object{"key": {external}} + definition := Definition[cfg]{Resolve: func(c *cfg, r *Resolution) error { + Normalize(r, Keys("map"), &c.Map, shared, "populate") + c.Pointer = external + r.Record(Keys("pointer"), ChangeNormalization, "populate") + c.Secret = SecretEnv("CONFIGKIT_CALLBACK_SECRET") + r.Record(Keys("secret"), ChangeNormalization, "reference") + return nil + }} + if validatorAssignment { + definition.Validate = func(c *cfg, v *Validation) { c.Map = shared; c.Pointer = external } + } + calls := 0 + lookup := func(string) (string, bool) { calls++; return "credential", true } + first, err := loadWithSecretLookup(NoSource(), lookup, Options[cfg]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + second, err := loadWithSecretLookup(NoSource(), lookup, Options[cfg]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + first.Config.Map["key"][0].Value = "changed" + first.Config.Pointer.Value = "changed pointer" + external.Value = "external change" + if second.Config.Map["key"][0].Value != "original" || second.Config.Pointer.Value != "original" { + t.Fatal("published configuration shares callback state") + } + if second.Config.Secret.Value() != "credential" || !second.Config.Secret.IsResolved() || calls != 2 { + t.Fatal("publication changed secret state or repeated environment lookup") + } + }) + } +} diff --git a/internal/engine/clone.go b/internal/engine/clone.go new file mode 100644 index 0000000..e8a9828 --- /dev/null +++ b/internal/engine/clone.go @@ -0,0 +1,159 @@ +package engine + +import ( + "encoding" + "math" + "reflect" +) + +// secretCloneMode distinguishes fresh unresolved references from publication of +// already validated private state, which must not trigger another lookup. +type secretCloneMode uint8 + +const ( + cloneUnresolvedSecrets secretCloneMode = iota + cloneValidatedSecrets +) + +func cloneConfiguration(schema *schemaDescriptor, value reflect.Value, limits projectionLimits) (reflect.Value, *projectionIssue) { + return cloneConfigurationWithMode(schema, value, limits, cloneUnresolvedSecrets) +} + +func cloneConfigurationWithMode(schema *schemaDescriptor, value reflect.Value, limits projectionLimits, mode secretCloneMode) (reflect.Value, *projectionIssue) { + if schema == nil || schema.root == nil || !value.IsValid() || value.Type() != schema.root.typeOf { + return reflect.Value{}, &projectionIssue{code: CodeSchemaUnsupportedType, message: "configuration value does not match its schema"} + } + operation := valueOperation{schema: schema, limits: limits, active: make(map[valueVisit]bool), cloneMode: mode} + return operation.clone(value, schema.root, Path{}, 0) +} + +func (o *valueOperation) clone(value reflect.Value, descriptor *typeDescriptor, path Path, depth int) (reflect.Value, *projectionIssue) { + if issue := o.checkDepth(path, depth); issue != nil { + return reflect.Value{}, issue + } + if descriptor.kind == descriptorPointer { + if value.IsNil() { + return reflect.Zero(value.Type()), nil + } + leave, issue := o.enter(value, path) + if issue != nil { + return reflect.Value{}, issue + } + defer leave() + child, issue := o.clone(value.Elem(), descriptor.element, path, depth) + if issue != nil { + return reflect.Value{}, issue + } + result := reflect.New(value.Type().Elem()) + result.Elem().Set(child) + return result, nil + } + + switch descriptor.kind { + case descriptorStruct: + result := reflect.New(value.Type()).Elem() + for _, member := range descriptor.members { + memberPath, memberDepth := path, depth + if !member.inline { + memberPath, memberDepth = path.Key(member.name), depth+1 + } + cloned, issue := o.clone(value.Field(member.index), member.descriptor, memberPath, memberDepth) + if issue != nil { + return reflect.Value{}, issue + } + result.Field(member.index).Set(cloned) + } + return result, nil + case descriptorMap: + if value.IsNil() { + return reflect.Zero(value.Type()), nil + } + leave, issue := o.enter(value, path) + if issue != nil { + return reflect.Value{}, issue + } + defer leave() + result := reflect.MakeMapWithSize(value.Type(), value.Len()) + for _, key := range value.MapKeys() { + cloned, issue := o.clone(value.MapIndex(key), descriptor.element, path.Key(key.String()), depth+1) + if issue != nil { + return reflect.Value{}, issue + } + result.SetMapIndex(key, cloned) + } + return result, nil + case descriptorSlice: + if value.IsNil() { + return reflect.Zero(value.Type()), nil + } + leave, issue := o.enter(value, path) + if issue != nil { + return reflect.Value{}, issue + } + defer leave() + result := reflect.MakeSlice(value.Type(), value.Len(), value.Len()) + for i := 0; i < value.Len(); i++ { + cloned, issue := o.clone(value.Index(i), descriptor.element, path.Index(i), depth+1) + if issue != nil { + return reflect.Value{}, issue + } + result.Index(i).Set(cloned) + } + return result, nil + case descriptorArray: + result := reflect.New(value.Type()).Elem() + for i := 0; i < value.Len(); i++ { + cloned, issue := o.clone(value.Index(i), descriptor.element, path.Index(i), depth+1) + if issue != nil { + return reflect.Value{}, issue + } + result.Index(i).Set(cloned) + } + return result, nil + case descriptorDynamic: + if value.IsNil() { + return reflect.Zero(value.Type()), nil + } + concrete := value.Elem() + dynamic, issue := o.dynamicDescriptor(concrete.Type(), path) + if issue != nil { + return reflect.Value{}, issue + } + cloned, issue := o.clone(concrete, dynamic, path, depth) + if issue != nil { + return reflect.Value{}, issue + } + result := reflect.New(value.Type()).Elem() + result.Set(cloned) + return result, nil + case descriptorSecret: + result := reflect.New(value.Type()).Elem() + reference := value.Interface().(Secret) + if o.cloneMode == cloneValidatedSecrets { + result.Set(value) + } else if reference.specified { + result.Set(reflect.ValueOf(SecretEnv(reference.EnvName()))) + } + return result, nil + case descriptorText: + text, issue := marshalCanonicalText(value, path) + if issue != nil { + return reflect.Value{}, issue + } + result := reflect.New(value.Type()).Elem() + unmarshaler := result.Addr().Interface().(encoding.TextUnmarshaler) + if err := unmarshaler.UnmarshalText([]byte(text)); err != nil { + return reflect.Value{}, &projectionIssue{code: CodeSchemaCodec, path: path.clone(), message: "custom text scalar could not be reconstructed"} + } + return result, nil + case descriptorFloat: + if math.IsNaN(value.Float()) || math.IsInf(value.Float(), 0) { + return reflect.Value{}, &projectionIssue{code: CodeSchemaNonFinite, path: path.clone(), message: "configuration float must be finite"} + } + fallthrough + default: + result := reflect.New(value.Type()).Elem() + result.Set(value) + return result, nil + } +} diff --git a/internal/engine/compose.go b/internal/engine/compose.go new file mode 100644 index 0000000..2ca828f --- /dev/null +++ b/internal/engine/compose.go @@ -0,0 +1,190 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +// composedDocument is the private handoff between source assembly and later +// precedence layers. Claims remain ordered by source and declaration traversal; +// origins retain every source that contributes to a composite mapping. +type composedDocument struct { + root *document.Node + claims []compositionClaim + origins map[string]Origin + overlays []overlayRecord + applications []historyApplication + inputBytes int64 + inputNodes int +} + +// cloneComposedDocument detaches mutable composition and provenance state; +// history application trees are immutable and retain their existing sharing. +func cloneComposedDocument(base *composedDocument) *composedDocument { + return &composedDocument{ + root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims), + origins: cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays), + applications: cloneHistoryApplications(base.applications), + inputBytes: base.inputBytes, inputNodes: base.inputNodes, + } +} + +type compositionClaim struct { + path Path + kind document.Kind + location Location +} + +type compositionClaims struct { + ordered []compositionClaim + byPath map[string][]compositionClaim +} + +// composeDocuments recursively joins disjoint mapping entries. Source order +// controls only output traversal and diagnostics: every non-mapping overlap is +// rejected before a result is assembled. +func composeDocuments(inputs ...*document.Node) (*composedDocument, []Diagnostic) { + if len(inputs) == 0 { + return nil, []Diagnostic{compositionInvalidDiagnostic("strict composition requires at least one document", nil)} + } + + collected := compositionClaims{byPath: make(map[string][]compositionClaim)} + for _, input := range inputs { + if input == nil || input.Kind != document.Mapping { + var location *Location + if input != nil { + converted := publicLocation(input.Location) + location = &converted + } + return nil, []Diagnostic{compositionInvalidDiagnostic("strict composition requires mapping document roots", location)} + } + collected.append(input, Path{}) + } + + for _, claim := range collected.ordered { + claims := collected.byPath[claim.path.String()] + if len(claims) < 2 || allMappingClaims(claims) { + continue + } + return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)} + } + + root := document.Clone(inputs[0]) + for _, input := range inputs[1:] { + mergeDisjointMappings(root, input) + } + applications := make([]historyApplication, 0, len(inputs)) + for _, input := range inputs { + applications = append(applications, historyApplication{layer: LayerBase, value: input}) + } + return &composedDocument{ + root: root, + claims: cloneCompositionClaims(collected.ordered), + origins: compositionOrigins(collected.ordered), + applications: applications, + }, nil +} + +func (c *compositionClaims) append(node *document.Node, path Path) { + claim := compositionClaim{ + path: path.clone(), + kind: node.Kind, + location: publicLocation(node.Location), + } + c.ordered = append(c.ordered, claim) + pointer := path.String() + c.byPath[pointer] = append(c.byPath[pointer], claim) + + if node.Kind != document.Mapping { + return + } + for _, entry := range node.Mapping { + c.append(entry.Value, path.Key(entry.Key)) + } +} + +func allMappingClaims(claims []compositionClaim) bool { + for _, claim := range claims { + if claim.kind != document.Mapping { + return false + } + } + return true +} + +func mergeDisjointMappings(target, incoming *document.Node) { + for _, incomingEntry := range incoming.Mapping { + index := documentEntryIndex(target.Mapping, incomingEntry.Key) + if index < 0 { + cloned := incomingEntry + cloned.Value = document.Clone(incomingEntry.Value) + target.Mapping = append(target.Mapping, cloned) + continue + } + // Claim validation guarantees that the only remaining overlap is a + // mapping pair. Empty mappings naturally contribute no child entries. + mergeDisjointMappings(target.Mapping[index].Value, incomingEntry.Value) + } +} + +func documentEntryIndex(entries []document.Entry, key string) int { + for index := range entries { + if entries[index].Key == key { + return index + } + } + return -1 +} + +func compositionOrigins(claims []compositionClaim) map[string]Origin { + origins := make(map[string]Origin, len(claims)) + for _, claim := range claims { + pointer := claim.path.String() + origin := origins[pointer] + origin.Layer = LayerBase + origin.Locations = append(origin.Locations, claim.location) + origins[pointer] = origin + } + return origins +} + +func applyCompositionOrigins(decoded *decodedConfiguration, origins map[string]Origin) { + for pointer, origin := range origins { + if _, present := decoded.presence[pointer]; present { + decoded.origins[pointer] = cloneOrigin(origin) + } + } +} + +func cloneCompositionClaims(claims []compositionClaim) []compositionClaim { + cloned := make([]compositionClaim, len(claims)) + for index, claim := range claims { + cloned[index] = claim + cloned[index].path = claim.path.clone() + } + return cloned +} + +func compositionInvalidDiagnostic(message string, location *Location) Diagnostic { + return Diagnostic{ + Code: CodeComposeInvalid, Severity: SeverityError, Message: message, + Primary: cloneLocation(location), + } +} + +func compositionConflictDiagnostic(path Path, claims []compositionClaim) Diagnostic { + primary := claims[0].location + related := make([]Location, 0, len(claims)-1) + for _, claim := range claims[1:] { + related = append(related, claim.location) + } + return Diagnostic{ + Code: CodeComposeConflict, + Severity: SeverityError, + Path: path.clone(), + Message: "configuration path is defined by multiple sources", + Primary: &primary, + Related: related, + } +} + +func publicLocation(location document.Location) Location { + return Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column} +} diff --git a/internal/engine/compose_test.go b/internal/engine/compose_test.go new file mode 100644 index 0000000..cd063f6 --- /dev/null +++ b/internal/engine/compose_test.go @@ -0,0 +1,194 @@ +package engine + +import ( + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +func TestComposeDocumentsJoinsDisjointMappingsAndTracksClaims(t *testing.T) { + base := parseCompositionDocument(t, "source-1", `services: + api: + enabled: true +empty: {} +`) + fragment := parseCompositionDocument(t, "source-2", `services: + worker: + enabled: false +values: [one, two] +`) + baseBefore := document.Clone(base) + fragmentBefore := document.Clone(fragment) + + composed, diagnostics := composeDocuments(base, fragment) + if len(diagnostics) != 0 { + t.Fatalf("composeDocuments() diagnostics = %#v", diagnostics) + } + for _, path := range []Path{ + Keys("services", "api", "enabled"), + Keys("services", "worker", "enabled"), + Keys("empty"), + Keys("values"), + } { + if _, ok := documentAtPath(composed.root, path); !ok { + t.Errorf("composed document is missing %s", path.String()) + } + } + + serviceOrigin := composed.origins[Keys("services").String()] + wantLocations := []Location{ + {SourceID: "source-1", Line: 2, Column: 3}, + {SourceID: "source-2", Line: 2, Column: 3}, + } + if !reflect.DeepEqual(serviceOrigin, Origin{Layer: LayerBase, Locations: wantLocations}) { + t.Fatalf("services origin = %#v, want composite origin", serviceOrigin) + } + wantClaimPaths := []string{"", "/services", "/services/api", "/services/api/enabled", "/empty", "", "/services", "/services/worker", "/services/worker/enabled", "/values"} + gotClaimPaths := make([]string, len(composed.claims)) + for index, claim := range composed.claims { + gotClaimPaths[index] = claim.path.String() + } + if !reflect.DeepEqual(gotClaimPaths, wantClaimPaths) { + t.Fatalf("claim paths = %#v, want %#v", gotClaimPaths, wantClaimPaths) + } + reversed, reversedDiagnostics := composeDocuments(fragment, base) + if len(reversedDiagnostics) != 0 { + t.Fatalf("reversed composeDocuments() diagnostics = %#v", reversedDiagnostics) + } + if deltas := compareConfigurationValues(composed.root, reversed.root); len(deltas) != 0 { + t.Fatalf("input order changed effective values: %#v", deltas) + } + + if !reflect.DeepEqual(base, baseBefore) || !reflect.DeepEqual(fragment, fragmentBefore) { + t.Fatal("composition mutated an input document") + } + composed.root.Mapping[0].Value.Mapping[0].Value.Mapping[0].Value.Text = "changed" + if !reflect.DeepEqual(base, baseBefore) || !reflect.DeepEqual(fragment, fragmentBefore) { + t.Fatal("composed document aliases an input document") + } +} + +func TestComposeDocumentsRejectsEveryPresentNonMappingOverlap(t *testing.T) { + tests := []struct { + name string + left string + right string + path string + }{ + {name: "equal scalar", left: "value: true\n", right: "value: true\n", path: "/value"}, + {name: "different scalar", left: "value: true\n", right: "value: false\n", path: "/value"}, + {name: "null", left: "value: null\n", right: "value: null\n", path: "/value"}, + {name: "sequence", left: "value: [one]\n", right: "value: [two]\n", path: "/value"}, + {name: "mapping and scalar", left: "value:\n nested: true\n", right: "value: text\n", path: "/value"}, + {name: "scalar and mapping", left: "value: text\n", right: "value:\n nested: true\n", path: "/value"}, + {name: "nested keyed entry", left: "items:\n shared:\n left: 1\n", right: "items:\n shared:\n left: 2\n", path: "/items/shared/left"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + left := parseCompositionDocument(t, "source-1", test.left) + right := parseCompositionDocument(t, "source-2", test.right) + _, diagnostics := composeDocuments(left, right) + assertCompositionConflict(t, diagnostics, test.path, []string{"source-1", "source-2"}) + }) + } +} + +func TestComposeDocumentsReportsAllClaimantsAtExactLocations(t *testing.T) { + first := parseCompositionDocument(t, "source-1", "group:\n value: 1\n") + second := parseCompositionDocument(t, "source-2", "group:\n value: 2\n") + third := parseCompositionDocument(t, "source-3", "group:\n value: 3\n") + + _, diagnostics := composeDocuments(first, second, third) + assertCompositionConflict(t, diagnostics, "/group/value", []string{"source-1", "source-2", "source-3"}) + locations := append([]Location{*diagnostics[0].Primary}, diagnostics[0].Related...) + for _, location := range locations { + if location.Line != 2 || location.Column != 10 { + t.Fatalf("claim location = %#v, want line 2 column 10", location) + } + } + + _, diagnostics = composeDocuments( + parseCompositionDocument(t, "source-1", "group:\n left: 1\n"), + parseCompositionDocument(t, "source-2", "group:\n right: 2\n"), + parseCompositionDocument(t, "source-3", "group: blocked\n"), + ) + assertCompositionConflict(t, diagnostics, "/group", []string{"source-1", "source-2", "source-3"}) +} + +func TestComposeDocumentsTreatsEmptyMappingsAsChildlessContributions(t *testing.T) { + empty, diagnostics := composeDocuments( + parseCompositionDocument(t, "source-1", "{}\n"), + parseCompositionDocument(t, "source-2", "{}\n"), + ) + if len(diagnostics) != 0 || len(empty.root.Mapping) != 0 || len(empty.origins[""].Locations) != 2 { + t.Fatalf("empty root composition = (%#v, %#v)", empty, diagnostics) + } + + tests := []struct { + name string + left string + right string + }{ + {name: "two empty mappings", left: "value: {}\n", right: "value: {}\n"}, + {name: "empty then populated", left: "value: {}\n", right: "value: {nested: true}\n"}, + {name: "populated then empty", left: "value: {nested: true}\n", right: "value: {}\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + composed, diagnostics := composeDocuments( + parseCompositionDocument(t, "source-1", test.left), + parseCompositionDocument(t, "source-2", test.right), + ) + if len(diagnostics) != 0 { + t.Fatalf("composeDocuments() diagnostics = %#v", diagnostics) + } + if origin := composed.origins["/value"]; len(origin.Locations) != 2 { + t.Fatalf("value origin = %#v, want both mapping locations", origin) + } + if test.name != "two empty mappings" { + if _, ok := documentAtPath(composed.root, Keys("value", "nested")); !ok { + t.Fatal("populated mapping contribution was not retained") + } + } + }) + } +} + +func TestComposeDocumentsRejectsInvalidBoundaries(t *testing.T) { + if composed, diagnostics := composeDocuments(); composed != nil || len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid { + t.Fatalf("empty composition = (%#v, %#v)", composed, diagnostics) + } + nonMapping := &document.Node{Kind: document.String, Location: document.Location{SourceID: "source-1", Line: 4, Column: 2}} + if composed, diagnostics := composeDocuments(nonMapping); composed != nil || len(diagnostics) != 1 || diagnostics[0].Primary == nil || diagnostics[0].Primary.Line != 4 { + t.Fatalf("non-mapping composition = (%#v, %#v)", composed, diagnostics) + } +} + +func parseCompositionDocument(t *testing.T, sourceID, input string) *document.Node { + t.Helper() + parsed, diagnostics := parseSource(Bytes(sourceID, []byte(input)), sourceID, defaultParseLimits()) + if len(diagnostics) != 0 { + t.Fatalf("parse diagnostics = %#v", diagnostics) + } + return parsed.document +} + +func assertCompositionConflict(t *testing.T, diagnostics []Diagnostic, path string, sources []string) { + t.Helper() + if len(diagnostics) != 1 { + t.Fatalf("diagnostics = %#v, want one conflict", diagnostics) + } + diagnostic := diagnostics[0] + if diagnostic.Code != CodeComposeConflict || diagnostic.Path.String() != path || diagnostic.Primary == nil { + t.Fatalf("diagnostic = %#v, want compose conflict at %s", diagnostic, path) + } + locations := append([]Location{*diagnostic.Primary}, diagnostic.Related...) + gotSources := make([]string, len(locations)) + for index, location := range locations { + gotSources[index] = location.SourceID + } + if !reflect.DeepEqual(gotSources, sources) { + t.Fatalf("claim sources = %#v, want %#v", gotSources, sources) + } +} diff --git a/internal/engine/composition_fuzz_test.go b/internal/engine/composition_fuzz_test.go new file mode 100644 index 0000000..2c6e655 --- /dev/null +++ b/internal/engine/composition_fuzz_test.go @@ -0,0 +1,68 @@ +package engine + +import ( + "bytes" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +func FuzzCompositionAndOverlay(f *testing.F) { + for _, seed := range [][2]string{ + {"{}\n", "{}\n"}, + {"left: {value: one}\n", "right: [two]\n"}, + {"shared: {left: one}\n", "shared: {right: two}\n"}, + {"value: one\n", "value: two\n"}, + } { + f.Add([]byte(seed[0]), []byte(seed[1])) + } + + limits := yamladapter.Limits{Depth: 12, Nodes: 256, ParserDepth: 24} + f.Fuzz(func(t *testing.T, leftInput, rightInput []byte) { + if len(leftInput) > 4<<10 || len(rightInput) > 4<<10 { + return + } + left, leftIssues := yamladapter.Parse(bytes.NewReader(leftInput), "left", limits) + right, rightIssues := yamladapter.Parse(bytes.NewReader(rightInput), "right", limits) + if len(leftIssues) != 0 || len(rightIssues) != 0 { + return + } + leftBefore, rightBefore := document.Clone(left), document.Clone(right) + + composed, diagnostics := composeDocuments(left, right) + if !reflect.DeepEqual(left, leftBefore) || !reflect.DeepEqual(right, rightBefore) { + t.Fatal("composition mutated an input document") + } + if len(diagnostics) == 0 { + reversed, reversedDiagnostics := composeDocuments(right, left) + if len(reversedDiagnostics) != 0 { + t.Fatalf("reversed composition failed: %#v", reversedDiagnostics) + } + if differences := compareConfigurationValues(composed.root, reversed.root); len(differences) != 0 { + t.Fatalf("composition order changed effective values: %#v", differences) + } + composed.root.Mapping = append(composed.root.Mapping, document.Entry{Key: "fuzz-detachment"}) + if !reflect.DeepEqual(left, leftBefore) || !reflect.DeepEqual(right, rightBefore) { + t.Fatal("composition result aliases an input document") + } + } + + base, baseDiagnostics := composeDocuments(left) + if len(baseDiagnostics) != 0 { + t.Fatalf("single accepted mapping did not compose: %#v", baseDiagnostics) + } + overlaid, overlayDiagnostics := overlayDocuments(base, right) + if len(overlayDiagnostics) != 0 { + t.Fatalf("accepted mappings did not overlay: %#v", overlayDiagnostics) + } + if !reflect.DeepEqual(left, leftBefore) || !reflect.DeepEqual(right, rightBefore) { + t.Fatal("overlay mutated an input document") + } + overlaid.root.Mapping = append(overlaid.root.Mapping, document.Entry{Key: "fuzz-detachment"}) + if !reflect.DeepEqual(left, leftBefore) || !reflect.DeepEqual(right, rightBefore) { + t.Fatal("overlay result aliases an input document") + } + }) +} diff --git a/internal/engine/decode.go b/internal/engine/decode.go new file mode 100644 index 0000000..cd61475 --- /dev/null +++ b/internal/engine/decode.go @@ -0,0 +1,470 @@ +package engine + +import ( + "encoding" + "math" + "reflect" + "regexp" + "strconv" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +var integerPattern = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`) +var unsignedIntegerPattern = regexp.MustCompile(`^(0|[1-9][0-9]*)$`) + +type decodeIssue struct { + code Code + path Path + message string + location document.Location +} + +type inputPresence struct { + null bool + location document.Location +} + +type decodedConfiguration struct { + value reflect.Value + presence map[string]inputPresence + origins map[string]Origin + baseline *document.Node + final *document.Node +} + +type decodeContext struct { + versionPath *Path +} + +func decodeConfiguration(schema *schemaDescriptor, defaults reflect.Value, input *document.Node) (decodedConfiguration, []decodeIssue) { + return decodeConfigurationWithContext(schema, defaults, input, decodeContext{}) +} + +func decodeConfigurationWithContext(schema *schemaDescriptor, defaults reflect.Value, input *document.Node, context decodeContext) (decodedConfiguration, []decodeIssue) { + if schema == nil || schema.root == nil || !defaults.IsValid() || defaults.Type() != schema.root.typeOf { + return decodedConfiguration{}, []decodeIssue{{code: CodeSchemaUnsupportedType, message: "configuration defaults do not match the schema"}} + } + baseline, projectionIssue := projectConfiguration(schema, defaults, defaultProjectionLimits()) + if projectionIssue != nil { + return decodedConfiguration{}, []decodeIssue{decodeIssueFromProjection(projectionIssue)} + } + working, cloneIssue := cloneConfiguration(schema, defaults, defaultProjectionLimits()) + if cloneIssue != nil { + return decodedConfiguration{}, []decodeIssue{decodeIssueFromProjection(cloneIssue)} + } + + presence := make(map[string]inputPresence) + if input != nil { + collectPresence(input, Path{}, presence) + decoded, issues := decodeValue(input, working, schema.root, Path{}, schema, context) + if len(issues) != 0 { + return decodedConfiguration{}, issues + } + working = decoded + } + final, projectionIssue := projectConfiguration(schema, working, defaultProjectionLimits()) + if projectionIssue != nil { + return decodedConfiguration{}, []decodeIssue{decodeIssueFromProjection(projectionIssue)} + } + origins := make(map[string]Origin) + collectOrigins(final, Path{}, presence, origins) + return decodedConfiguration{ + value: working, + presence: presence, + origins: origins, + baseline: baseline, + final: final, + }, nil +} + +func decodeValue(node *document.Node, base reflect.Value, descriptor *typeDescriptor, path Path, schema *schemaDescriptor, context decodeContext) (reflect.Value, []decodeIssue) { + if node.Kind == document.Null { + switch descriptor.kind { + case descriptorPointer, descriptorMap, descriptorSlice, descriptorDynamic: + return reflect.Zero(descriptor.typeOf), nil + case descriptorSecret: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeSecretRequired, path, node, "required secret reference cannot be null")} + default: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeNull, path, node, "null is not allowed for this configuration type")} + } + } + if descriptor.kind == descriptorPointer { + var elementBase reflect.Value + if base.IsNil() { + elementBase = reflect.Zero(descriptor.typeOf.Elem()) + } else { + elementBase = base.Elem() + } + decoded, issues := decodeValue(node, elementBase, descriptor.element, path, schema, context) + if len(issues) != 0 { + return reflect.Value{}, issues + } + result := reflect.New(descriptor.typeOf.Elem()) + result.Elem().Set(decoded) + return result, nil + } + + switch descriptor.kind { + case descriptorStruct: + return decodeStruct(node, base, descriptor, path, schema, context) + case descriptorMap: + return decodeMap(node, base, descriptor, path, schema, context) + case descriptorSlice: + return decodeSlice(node, descriptor, path, schema, context) + case descriptorArray: + return decodeArray(node, descriptor, path, schema, context) + case descriptorDynamic: + return decodeDynamic(node, base, path, schema, context) + case descriptorString: + if node.Kind != document.String && !(pathMatches(context.versionPath, path) && isIntegerVersion(node)) { + return typeMismatch(path, node, "string") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetString(node.Text) + return result, nil + case descriptorBoolean: + if node.Kind != document.Boolean { + return typeMismatch(path, node, "boolean") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetBool(node.Text == "true") + return result, nil + case descriptorSigned: + if node.Kind != document.Number || !integerPattern.MatchString(node.Text) { + return typeMismatch(path, node, "integer") + } + value, err := strconv.ParseInt(node.Text, 10, descriptor.typeOf.Bits()) + if err != nil { + return rangeFailure(path, node, "integer") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetInt(value) + return result, nil + case descriptorUnsigned: + if node.Kind != document.Number || !unsignedIntegerPattern.MatchString(node.Text) { + return typeMismatch(path, node, "unsigned integer") + } + value, err := strconv.ParseUint(node.Text, 10, descriptor.typeOf.Bits()) + if err != nil { + return rangeFailure(path, node, "unsigned integer") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetUint(value) + return result, nil + case descriptorFloat: + if node.Kind != document.Number { + return typeMismatch(path, node, "number") + } + value, err := strconv.ParseFloat(node.Text, descriptor.typeOf.Bits()) + if err != nil || math.IsNaN(value) || math.IsInf(value, 0) { + return rangeFailure(path, node, "number") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetFloat(value) + return result, nil + case descriptorDuration: + if node.Kind != document.String { + return typeMismatch(path, node, "duration string") + } + value, err := time.ParseDuration(node.Text) + if err != nil { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeDuration, path, node, "duration string is invalid")} + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetInt(int64(value)) + return result, nil + case descriptorSecret: + if node.Kind != document.String { + return typeMismatch(path, node, "secret environment name") + } + return reflect.ValueOf(SecretEnv(node.Text)), nil + case descriptorText: + if node.Kind != document.String { + return typeMismatch(path, node, "text string") + } + result := reflect.New(descriptor.typeOf).Elem() + unmarshaler := result.Addr().Interface().(encoding.TextUnmarshaler) + if err := unmarshaler.UnmarshalText([]byte(node.Text)); err != nil { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeText, path, node, "text value is invalid for its configuration type")} + } + return result, nil + default: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeSchemaUnsupportedType, path, node, "configuration type is not supported")} + } +} + +func decodeStruct(node *document.Node, base reflect.Value, descriptor *typeDescriptor, path Path, schema *schemaDescriptor, context decodeContext) (reflect.Value, []decodeIssue) { + if node.Kind != document.Mapping { + return typeMismatch(path, node, "mapping") + } + result := reflect.New(descriptor.typeOf).Elem() + result.Set(base) + var issues []decodeIssue + for _, entry := range node.Mapping { + childPath := path.Key(entry.Key) + if position, ok := descriptor.fieldByName[entry.Key]; ok { + field := descriptor.fields[position] + target, ok := fieldForDecode(result, field.index, true) + if !ok { + issues = append(issues, decodeIssue{code: CodeSchemaUnsupportedType, path: childPath, message: "configuration field could not be addressed", location: entry.ValueLocation}) + continue + } + decoded, fieldIssues := decodeValue(entry.Value, target, field.descriptor, childPath, schema, context) + if len(fieldIssues) != 0 { + issues = append(issues, fieldIssues...) + continue + } + target.Set(decoded) + continue + } + if descriptor.inlineMap != nil { + if issue := decodeInlineMapEntry(result, *descriptor.inlineMap, entry, childPath, schema, context); issue != nil { + issues = append(issues, issue...) + } + continue + } + issues = append(issues, decodeIssue{code: CodeDecodeUnknownField, path: childPath, message: "configuration field is not declared", location: entry.KeyLocation}) + } + return result, issues +} + +func decodeInlineMapEntry(target reflect.Value, field fieldDescriptor, entry document.Entry, path Path, schema *schemaDescriptor, context decodeContext) []decodeIssue { + mapValue, ok := fieldForDecode(target, field.index, true) + if !ok || mapValue.Kind() != reflect.Map { + return []decodeIssue{{code: CodeSchemaInline, path: path, message: "inline map could not be addressed", location: entry.KeyLocation}} + } + if mapValue.IsNil() { + mapValue.Set(reflect.MakeMap(mapValue.Type())) + } + key := reflect.New(mapValue.Type().Key()).Elem() + key.SetString(entry.Key) + base := mapValue.MapIndex(key) + if !base.IsValid() { + base = reflect.Zero(mapValue.Type().Elem()) + } + decoded, issues := decodeValue(entry.Value, base, field.descriptor.element, path, schema, context) + if len(issues) == 0 { + mapValue.SetMapIndex(key, decoded) + } + return issues +} + +func decodeMap(node *document.Node, base reflect.Value, descriptor *typeDescriptor, path Path, schema *schemaDescriptor, context decodeContext) (reflect.Value, []decodeIssue) { + if node.Kind != document.Mapping { + return typeMismatch(path, node, "mapping") + } + result := base + if result.IsNil() { + result = reflect.MakeMap(descriptor.typeOf) + } + var issues []decodeIssue + for _, entry := range node.Mapping { + key := reflect.New(descriptor.typeOf.Key()).Elem() + key.SetString(entry.Key) + entryBase := result.MapIndex(key) + if !entryBase.IsValid() { + entryBase = reflect.Zero(descriptor.typeOf.Elem()) + } + decoded, entryIssues := decodeValue(entry.Value, entryBase, descriptor.element, path.Key(entry.Key), schema, context) + if len(entryIssues) != 0 { + issues = append(issues, entryIssues...) + continue + } + result.SetMapIndex(key, decoded) + } + return result, issues +} + +func decodeSlice(node *document.Node, descriptor *typeDescriptor, path Path, schema *schemaDescriptor, context decodeContext) (reflect.Value, []decodeIssue) { + if node.Kind != document.Sequence { + return typeMismatch(path, node, "sequence") + } + result := reflect.MakeSlice(descriptor.typeOf, len(node.Sequence), len(node.Sequence)) + var issues []decodeIssue + for i, child := range node.Sequence { + decoded, childIssues := decodeValue(child, reflect.Zero(descriptor.typeOf.Elem()), descriptor.element, path.Index(i), schema, context) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + result.Index(i).Set(decoded) + } + return result, issues +} + +func decodeArray(node *document.Node, descriptor *typeDescriptor, path Path, schema *schemaDescriptor, context decodeContext) (reflect.Value, []decodeIssue) { + if node.Kind != document.Sequence { + return typeMismatch(path, node, "sequence") + } + if len(node.Sequence) != descriptor.length { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeArrayLength, path, node, "configuration array has the wrong length")} + } + result := reflect.New(descriptor.typeOf).Elem() + var issues []decodeIssue + for i, child := range node.Sequence { + decoded, childIssues := decodeValue(child, reflect.Zero(descriptor.typeOf.Elem()), descriptor.element, path.Index(i), schema, context) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + result.Index(i).Set(decoded) + } + return result, issues +} + +func decodeDynamic(node *document.Node, base reflect.Value, path Path, schema *schemaDescriptor, context decodeContext) (reflect.Value, []decodeIssue) { + if node.Kind == document.Mapping && base.IsValid() && !base.IsNil() { + concrete := base.Elem() + operation := valueOperation{schema: schema} + descriptor, issue := operation.dynamicDescriptor(concrete.Type(), path) + if issue == nil && acceptsMapping(descriptor) { + decoded, issues := decodeValue(node, concrete, descriptor, path, schema, context) + if len(issues) != 0 { + return reflect.Value{}, issues + } + result := reflect.New(base.Type()).Elem() + result.Set(decoded) + return result, nil + } + } + var value any + switch node.Kind { + case document.Null: + return reflect.Zero(base.Type()), nil + case document.String: + value = node.Text + case document.Boolean: + value = node.Text == "true" + case document.Number: + if integerPattern.MatchString(node.Text) { + parsed, err := strconv.ParseInt(node.Text, 10, 64) + if err != nil { + return rangeFailure(path, node, "dynamic integer") + } + value = parsed + } else { + parsed, err := strconv.ParseFloat(node.Text, 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return rangeFailure(path, node, "dynamic number") + } + value = parsed + } + case document.Sequence: + sequence := make([]any, len(node.Sequence)) + var issues []decodeIssue + for i, child := range node.Sequence { + decoded, childIssues := decodeDynamic(child, reflect.Zero(reflect.TypeFor[any]()), path.Index(i), schema, context) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + sequence[i] = decoded.Interface() + } + if len(issues) != 0 { + return reflect.Value{}, issues + } + value = sequence + case document.Mapping: + mapping := make(map[string]any, len(node.Mapping)) + var issues []decodeIssue + for _, entry := range node.Mapping { + decoded, childIssues := decodeDynamic(entry.Value, reflect.Zero(reflect.TypeFor[any]()), path.Key(entry.Key), schema, context) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + mapping[entry.Key] = decoded.Interface() + } + if len(issues) != 0 { + return reflect.Value{}, issues + } + value = mapping + default: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeType, path, node, "dynamic configuration value has an unsupported kind")} + } + result := reflect.New(base.Type()).Elem() + if value != nil { + result.Set(reflect.ValueOf(value)) + } + return result, nil +} + +func acceptsMapping(descriptor *typeDescriptor) bool { + for descriptor != nil && descriptor.kind == descriptorPointer { + descriptor = descriptor.element + } + return descriptor != nil && (descriptor.kind == descriptorStruct || descriptor.kind == descriptorMap || descriptor.kind == descriptorDynamic) +} + +func fieldForDecode(root reflect.Value, index []int, allocate bool) (reflect.Value, bool) { + current := root + for position, fieldIndex := range index { + if current.Kind() != reflect.Struct { + return reflect.Value{}, false + } + current = current.Field(fieldIndex) + if position == len(index)-1 { + return current, current.CanSet() + } + for current.Kind() == reflect.Pointer { + if current.IsNil() { + if !allocate || !current.CanSet() { + return reflect.Value{}, false + } + current.Set(reflect.New(current.Type().Elem())) + } + current = current.Elem() + } + } + return reflect.Value{}, false +} + +func collectPresence(node *document.Node, path Path, presence map[string]inputPresence) { + presence[path.String()] = inputPresence{null: node.Kind == document.Null, location: node.Location} + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectPresence(entry.Value, path.Key(entry.Key), presence) + } + case document.Sequence: + for i, child := range node.Sequence { + collectPresence(child, path.Index(i), presence) + } + } +} + +func collectOrigins(node *document.Node, path Path, presence map[string]inputPresence, origins map[string]Origin) { + if input, ok := presence[path.String()]; ok { + origins[path.String()] = Origin{Layer: LayerBase, Locations: []Location{{SourceID: input.location.SourceID, Line: input.location.Line, Column: input.location.Column}}} + } else { + origins[path.String()] = Origin{Layer: LayerDefault} + } + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectOrigins(entry.Value, path.Key(entry.Key), presence, origins) + } + case document.Sequence: + for i, child := range node.Sequence { + collectOrigins(child, path.Index(i), presence, origins) + } + } +} + +func decodeIssueFromProjection(issue *projectionIssue) decodeIssue { + return decodeIssue{code: issue.code, path: issue.path.clone(), message: issue.message} +} + +func newDecodeIssue(code Code, path Path, node *document.Node, message string) decodeIssue { + return decodeIssue{code: code, path: path.clone(), message: message, location: node.Location} +} + +func typeMismatch(path Path, node *document.Node, expected string) (reflect.Value, []decodeIssue) { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeType, path, node, "configuration value must be a "+expected)} +} + +func rangeFailure(path Path, node *document.Node, expected string) (reflect.Value, []decodeIssue) { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeRange, path, node, expected+" is outside the supported range")} +} diff --git a/internal/engine/decode_test.go b/internal/engine/decode_test.go new file mode 100644 index 0000000..91acf85 --- /dev/null +++ b/internal/engine/decode_test.go @@ -0,0 +1,312 @@ +package engine + +import ( + "errors" + "math" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type namedSmall int8 + +type validatedText string + +func (value validatedText) MarshalText() ([]byte, error) { + return []byte(value), nil +} + +func (value *validatedText) UnmarshalText(text []byte) error { + if string(text) == "bad-secret-canary" { + return errors.New("CANARY_TEXT_ERROR: bad-secret-canary") + } + *value = validatedText(text) + return nil +} + +func TestDecodeAppliesInputOverDetachedDefaults(t *testing.T) { + t.Parallel() + + type nested struct { + A string `yaml:"a"` + B string `yaml:"b"` + } + type config struct { + Enabled bool `yaml:"enabled"` + Workers int `yaml:"workers"` + Label string `yaml:"label"` + Nested nested `yaml:"nested"` + EmptyMapping nested `yaml:"empty_mapping"` + Objects map[string]nested `yaml:"objects"` + Sequence []string `yaml:"sequence"` + Replacement []string `yaml:"replacement"` + Pointer *nested `yaml:"pointer"` + NilPointer *nested `yaml:"nil_pointer"` + Nullable map[string]string `yaml:"nullable"` + Dynamic any `yaml:"dynamic"` + DynamicShape any `yaml:"dynamic_shape"` + DynamicNull any `yaml:"dynamic_null"` + Fixed [2]int `yaml:"fixed"` + Small namedSmall `yaml:"small"` + Delay time.Duration `yaml:"delay"` + Custom validatedText `yaml:"custom"` + } + defaults := config{ + Enabled: true, + Workers: 4, + Label: "default", + Nested: nested{A: "keep", B: "before"}, + EmptyMapping: nested{A: "keep-empty", B: "keep-empty"}, + Objects: map[string]nested{ + "existing": {A: "keep-map", B: "before"}, + }, + Sequence: []string{"default"}, + Replacement: []string{"default"}, + Pointer: &nested{A: "keep-pointer", B: "before"}, + Nullable: map[string]string{"default": "value"}, + Dynamic: map[string]any{"keep": "default", "count": int64(4)}, + DynamicShape: nested{A: "keep-dynamic", B: "before"}, + DynamicNull: "default", + Fixed: [2]int{1, 2}, + Small: 7, + Delay: time.Second, + Custom: "default", + } + input := ` +enabled: false +workers: 0 +label: "" +nested: {b: changed} +empty_mapping: {} +objects: + existing: {b: changed} + new: {b: added} +sequence: [] +replacement: [one, two] +pointer: {b: changed} +nil_pointer: {b: added} +nullable: null +dynamic: {count: 0, fraction: 1.5} +dynamic_shape: {b: changed} +dynamic_null: null +fixed: [3, 4] +small: -8 +delay: 1.5s +custom: accepted +` + schema := mustSchemaFor[config](t) + documentRoot := parseDocumentForDecode(t, input) + decoded, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), documentRoot) + if len(issues) != 0 { + t.Fatalf("decode issues = %#v", issues) + } + got := decoded.value.Interface().(config) + if got.Enabled || got.Workers != 0 || got.Label != "" { + t.Fatalf("explicit zero values lost: %#v", got) + } + if got.Nested != (nested{A: "keep", B: "changed"}) || got.EmptyMapping != defaults.EmptyMapping { + t.Fatalf("mapping defaults = %#v / %#v", got.Nested, got.EmptyMapping) + } + if got.Objects["existing"] != (nested{A: "keep-map", B: "changed"}) || got.Objects["new"] != (nested{B: "added"}) { + t.Fatalf("map defaults = %#v", got.Objects) + } + if got.Sequence == nil || len(got.Sequence) != 0 || !reflect.DeepEqual(got.Replacement, []string{"one", "two"}) { + t.Fatalf("sequence replacement = %#v / %#v", got.Sequence, got.Replacement) + } + if got.Pointer == defaults.Pointer || *got.Pointer != (nested{A: "keep-pointer", B: "changed"}) || got.NilPointer == nil || *got.NilPointer != (nested{B: "added"}) { + t.Fatalf("pointer defaults = %#v / %#v", got.Pointer, got.NilPointer) + } + if got.Nullable != nil || got.DynamicNull != nil { + t.Fatalf("null values = %#v / %#v", got.Nullable, got.DynamicNull) + } + dynamic := got.Dynamic.(map[string]any) + if dynamic["keep"] != "default" || dynamic["count"] != int64(0) || dynamic["fraction"] != float64(1.5) { + t.Fatalf("dynamic mapping = %#v", dynamic) + } + if got.DynamicShape != (nested{A: "keep-dynamic", B: "changed"}) { + t.Fatalf("dynamic concrete mapping = %#v", got.DynamicShape) + } + if got.Fixed != [2]int{3, 4} || got.Small != -8 || got.Delay != 1500*time.Millisecond || got.Custom != "accepted" { + t.Fatalf("scalar decoding = %#v", got) + } + + for _, path := range []Path{Keys(), Keys("enabled"), Keys("nested"), Keys("nested", "b"), Keys("empty_mapping")} { + if _, ok := decoded.presence[path.String()]; !ok { + t.Errorf("explicit path %q missing", path.String()) + } + } + if _, ok := decoded.presence[Keys("nested", "a").String()]; ok { + t.Fatal("parent mapping made omitted child explicit") + } + if decoded.origins[Keys("nested", "a").String()].Layer != LayerDefault || decoded.origins[Keys("nested", "b").String()].Layer != LayerBase { + t.Fatalf("origins = %#v", decoded.origins) + } + if decoded.origins[Keys("nested", "b").String()].Locations[0].Line == 0 { + t.Fatal("input origin lost source coordinates") + } + + defaults.Objects["existing"] = nested{A: "mutated"} + defaults.Pointer.B = "mutated" + defaults.Sequence[0] = "mutated" + if got.Objects["existing"].A != "keep-map" || got.Pointer.B != "changed" || len(got.Sequence) != 0 { + t.Fatal("decoded value aliases defaults") + } + got.Pointer.B = "caller mutation" + if mappingValue(t, mappingValue(t, decoded.final, "pointer"), "b").Text != "changed" { + t.Fatal("final snapshot aliases returned typed value") + } + if mappingValue(t, mappingValue(t, decoded.baseline, "nested"), "b").Text != "before" { + t.Fatal("baseline snapshot was mutated by decode") + } +} + +func TestDecodeDefaultsOnlyAndRepeatedResultsAreIndependent(t *testing.T) { + t.Parallel() + + type config struct { + Values map[string][]string `yaml:"values"` + } + defaults := config{Values: map[string][]string{"one": {"a"}}} + schema := mustSchemaFor[config](t) + first, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), nil) + if len(issues) != 0 { + t.Fatal(issues) + } + second, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), nil) + if len(issues) != 0 { + t.Fatal(issues) + } + firstValue := first.value.Interface().(config) + secondValue := second.value.Interface().(config) + firstValue.Values["one"][0] = "changed" + if secondValue.Values["one"][0] != "a" || defaults.Values["one"][0] != "a" { + t.Fatal("repeated defaults-only results share aliases") + } + if len(first.presence) != 0 || first.origins[Keys("values", "one").String()].Layer != LayerDefault { + t.Fatalf("defaults-only metadata = %#v / %#v", first.presence, first.origins) + } +} + +func TestDecodeCollectsIndependentTypeAndShapeFailures(t *testing.T) { + t.Parallel() + + type nested struct { + Known int `yaml:"known"` + } + type config struct { + Small int8 `yaml:"small"` + Unsigned uint8 `yaml:"unsigned"` + Number float32 `yaml:"number"` + Text string `yaml:"text"` + Flag bool `yaml:"flag"` + Delay time.Duration `yaml:"delay"` + Fixed [2]int `yaml:"fixed"` + Nested nested `yaml:"nested"` + Custom validatedText `yaml:"custom"` + Dynamic any `yaml:"dynamic"` + } + input := ` +small: 128 +unsigned: -1 +number: 1e999 +text: 3 +flag: "true" +delay: 10 +fixed: [1] +nested: {first_unknown: 1, second_unknown: 2} +custom: bad-secret-canary +dynamic: 9223372036854775808 +` + schema := mustSchemaFor[config](t) + _, issues := decodeConfiguration(schema, reflect.ValueOf(config{Custom: "ok"}), parseDocumentForDecode(t, input)) + if len(issues) != 11 { + t.Fatalf("issues = %d: %#v", len(issues), issues) + } + want := map[string]Code{ + "/small": CodeDecodeRange, + "/unsigned": CodeDecodeType, + "/number": CodeDecodeRange, + "/text": CodeDecodeType, + "/flag": CodeDecodeType, + "/delay": CodeDecodeType, + "/fixed": CodeDecodeArrayLength, + "/nested/first_unknown": CodeDecodeUnknownField, + "/nested/second_unknown": CodeDecodeUnknownField, + "/custom": CodeDecodeText, + "/dynamic": CodeDecodeRange, + } + for _, issue := range issues { + if wantCode, ok := want[issue.path.String()]; !ok || wantCode != issue.code { + t.Errorf("unexpected issue %#v", issue) + } + if issue.location.Line == 0 { + t.Errorf("issue lost input location: %#v", issue) + } + if strings.Contains(issue.message, "CANARY") || strings.Contains(issue.message, "bad-secret") { + t.Errorf("issue exposed value: %#v", issue) + } + } +} + +func TestDecodeNullabilityAndNestedMapStrictness(t *testing.T) { + t.Parallel() + + type nested struct { + Value int `yaml:"value"` + } + type config struct { + Scalar int `yaml:"scalar"` + Object nested `yaml:"object"` + Array [1]int `yaml:"array"` + Map map[string]nested `yaml:"map"` + } + input := "scalar: null\nobject: null\narray: null\nmap:\n item: {unknown: 1}\n" + _, issues := decodeConfiguration(mustSchemaFor[config](t), reflect.ValueOf(config{}), parseDocumentForDecode(t, input)) + want := map[string]Code{ + "/scalar": CodeDecodeNull, + "/object": CodeDecodeNull, + "/array": CodeDecodeNull, + "/map/item/unknown": CodeDecodeUnknownField, + } + if len(issues) != len(want) { + t.Fatalf("issues = %#v", issues) + } + for _, issue := range issues { + if want[issue.path.String()] != issue.code { + t.Errorf("issue = %#v", issue) + } + } +} + +func TestDecodeRejectsInvalidDefaultsBeforeApplyingInput(t *testing.T) { + t.Parallel() + + type config struct { + Name string `yaml:"name"` + Extras map[string]any `yaml:",inline"` + Float float64 `yaml:"float"` + } + schema := mustSchemaFor[config](t) + for name, defaults := range map[string]config{ + "inline collision": {Name: "declared", Extras: map[string]any{"name": "duplicate"}}, + "non-finite": {Float: math.NaN()}, + } { + t.Run(name, func(t *testing.T) { + if _, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), parseDocumentForDecode(t, "{}\n")); len(issues) != 1 || !strings.HasPrefix(string(issues[0].code), "schema.") { + t.Fatalf("issues = %#v", issues) + } + }) + } +} + +func parseDocumentForDecode(t *testing.T, input string) *document.Node { + t.Helper() + parsed, diagnostics := parseSource(Bytes("decode.yml", []byte(input)), "source-1", defaultParseLimits()) + if len(diagnostics) != 0 { + t.Fatalf("parse diagnostics = %#v", diagnostics) + } + return parsed.document +} diff --git a/internal/engine/diagnostic.go b/internal/engine/diagnostic.go new file mode 100644 index 0000000..dd25d33 --- /dev/null +++ b/internal/engine/diagnostic.go @@ -0,0 +1,248 @@ +package engine + +import ( + "cmp" + "fmt" + "io" + "slices" + "strconv" + "strings" + "unicode" +) + +// LoadError reports safe structured diagnostics for a failed configuration load. +// It intentionally does not expose or unwrap parser and application errors. +type LoadError struct { + diagnostics []Diagnostic + sources []SourceInfo +} + +// Error implements error without rendering diagnostic messages or input values. +func (e *LoadError) Error() string { + if e == nil { + return "configkit: configuration load failed" + } + count := 0 + for _, diagnostic := range e.diagnostics { + if diagnostic.Severity == SeverityError { + count++ + } + } + if count == 1 { + return "configkit: configuration load failed with 1 error" + } + return fmt.Sprintf("configkit: configuration load failed with %d errors", count) +} + +// Diagnostics returns a deep copy of the error's canonical diagnostics. +func (e *LoadError) Diagnostics() []Diagnostic { + if e == nil { + return nil + } + return cloneDiagnostics(e.diagnostics) +} + +// Sources returns a deep copy of sources discovered before loading failed. +func (e *LoadError) Sources() []SourceInfo { + if e == nil { + return nil + } + return cloneSourceInfos(e.sources) +} + +// WriteDiagnostics writes diagnostics in canonical order. It escapes terminal +// control characters and propagates writer errors, including short writes. +func WriteDiagnostics(output io.Writer, diagnostics []Diagnostic) error { + canonical := canonicalDiagnostics(diagnostics) + for _, diagnostic := range canonical { + if err := writeDiagnostic(output, diagnostic); err != nil { + return err + } + } + return nil +} + +type diagnosticAccumulator struct { + diagnostics []Diagnostic + sources []SourceInfo +} + +func (a *diagnosticAccumulator) add(diagnostics ...Diagnostic) { + for _, diagnostic := range diagnostics { + a.diagnostics = append(a.diagnostics, cloneDiagnostic(diagnostic)) + } +} + +func (a *diagnosticAccumulator) loadError() *LoadError { + return &LoadError{ + diagnostics: canonicalDiagnostics(a.diagnostics), + sources: cloneSourceInfos(a.sources), + } +} + +func canonicalDiagnostics(diagnostics []Diagnostic) []Diagnostic { + canonical := cloneDiagnostics(diagnostics) + for i := range canonical { + slices.SortFunc(canonical[i].Related, compareLocation) + } + slices.SortFunc(canonical, compareDiagnostic) + return canonical +} + +func compareDiagnostic(left, right Diagnostic) int { + if order := compareOptionalLocation(left.Primary, right.Primary); order != 0 { + return order + } + if order := cmp.Compare(left.Path.String(), right.Path.String()); order != 0 { + return order + } + if order := cmp.Compare(left.Code, right.Code); order != 0 { + return order + } + if order := cmp.Compare(left.Severity, right.Severity); order != 0 { + return order + } + if order := cmp.Compare(left.Message, right.Message); order != 0 { + return order + } + return compareLocations(left.Related, right.Related) +} + +func compareOptionalLocation(left, right *Location) int { + if left == nil && right == nil { + return 0 + } + if left == nil { + return -1 + } + if right == nil { + return 1 + } + return compareLocation(*left, *right) +} + +func compareLocations(left, right []Location) int { + for i := 0; i < min(len(left), len(right)); i++ { + if order := compareLocation(left[i], right[i]); order != 0 { + return order + } + } + return cmp.Compare(len(left), len(right)) +} + +func compareLocation(left, right Location) int { + if order := compareSourceID(left.SourceID, right.SourceID); order != 0 { + return order + } + if order := cmp.Compare(left.Line, right.Line); order != 0 { + return order + } + return cmp.Compare(left.Column, right.Column) +} + +func compareSourceID(left, right string) int { + leftOrder, leftOK := sourceSequence(left) + rightOrder, rightOK := sourceSequence(right) + if leftOK && rightOK { + return cmp.Compare(leftOrder, rightOrder) + } + return cmp.Compare(left, right) +} + +func sourceSequence(sourceID string) (int, bool) { + const prefix = "source-" + if !strings.HasPrefix(sourceID, prefix) { + return 0, false + } + value, err := strconv.Atoi(strings.TrimPrefix(sourceID, prefix)) + if err != nil || value < 1 { + return 0, false + } + return value, true +} + +func writeDiagnostic(output io.Writer, diagnostic Diagnostic) error { + severity := diagnostic.Severity + if severity == "" { + severity = SeverityError + } + path := diagnostic.Path.String() + if path == "" { + path = "" + } + + var b strings.Builder + b.WriteString(escapeControls(string(severity))) + if diagnostic.Code != "" { + b.WriteString(" [") + b.WriteString(escapeControls(string(diagnostic.Code))) + b.WriteByte(']') + } + b.WriteByte(' ') + b.WriteString(escapeControls(path)) + if diagnostic.Primary != nil { + b.WriteString(" (") + writeLocationText(&b, *diagnostic.Primary) + b.WriteByte(')') + } + if diagnostic.Message != "" { + b.WriteString(": ") + b.WriteString(escapeControls(diagnostic.Message)) + } + b.WriteByte('\n') + for _, related := range diagnostic.Related { + b.WriteString(" related: ") + writeLocationText(&b, related) + b.WriteByte('\n') + } + return writeString(output, b.String()) +} + +func writeLocationText(output *strings.Builder, location Location) { + label := location.SourceID + if label == "" { + label = "" + } + output.WriteString(escapeControls(label)) + if location.Line > 0 { + output.WriteByte(':') + output.WriteString(strconv.Itoa(location.Line)) + if location.Column > 0 { + output.WriteByte(':') + output.WriteString(strconv.Itoa(location.Column)) + } + } +} + +func writeString(output io.Writer, value string) error { + written, err := io.WriteString(output, value) + if err != nil { + return err + } + if written != len(value) { + return io.ErrShortWrite + } + return nil +} + +func escapeControls(value string) string { + var b strings.Builder + for _, r := range value { + if unicode.IsControl(r) { + quoted := strconv.QuoteRune(r) + b.WriteString(quoted[1 : len(quoted)-1]) + continue + } + b.WriteRune(r) + } + return b.String() +} + +func hasErrors(diagnostics []Diagnostic) bool { + for _, diagnostic := range diagnostics { + if diagnostic.Severity == SeverityError { + return true + } + } + return false +} diff --git a/internal/engine/diagnostic_test.go b/internal/engine/diagnostic_test.go new file mode 100644 index 0000000..0200745 --- /dev/null +++ b/internal/engine/diagnostic_test.go @@ -0,0 +1,143 @@ +package engine + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" +) + +func TestDiagnosticsAreCanonicalAndDetached(t *testing.T) { + t.Parallel() + + diagnostics := []Diagnostic{ + { + Code: "decode.type", + Severity: SeverityError, + Path: Keys("later"), + Message: "later", + Primary: &Location{SourceID: "source-10", Line: 1, Column: 1}, + }, + { + Code: "schema.invalid", + Severity: SeverityError, + Path: Keys("root"), + Message: "source-less", + }, + { + Code: "decode.type", + Severity: SeverityError, + Path: Keys("earlier"), + Message: "earlier", + Primary: &Location{SourceID: "source-2", Line: 3, Column: 4}, + Related: []Location{ + {SourceID: "source-10", Line: 1, Column: 1}, + {SourceID: "source-2", Line: 9, Column: 1}, + }, + }, + } + + var forward bytes.Buffer + if err := WriteDiagnostics(&forward, diagnostics); err != nil { + t.Fatal(err) + } + reversed := []Diagnostic{diagnostics[2], diagnostics[1], diagnostics[0]} + var backward bytes.Buffer + if err := WriteDiagnostics(&backward, reversed); err != nil { + t.Fatal(err) + } + if forward.String() != backward.String() { + t.Fatalf("rendering depends on submission order:\n%s\n---\n%s", forward.String(), backward.String()) + } + lines := strings.Split(forward.String(), "\n") + if !strings.Contains(lines[0], "source-less") || !strings.Contains(lines[1], "source-2:3:4") { + t.Fatalf("unexpected canonical order:\n%s", forward.String()) + } + if !strings.Contains(lines[2], "source-2:9:1") || !strings.Contains(lines[3], "source-10:1:1") { + t.Fatalf("related locations not canonical:\n%s", forward.String()) + } + + accumulator := diagnosticAccumulator{ + sources: []SourceInfo{{ + ID: "source-1", + Label: "config.yml", + ImportedFrom: &Location{SourceID: "source-0", Line: 1}, + }}, + } + accumulator.add(diagnostics...) + loadErr := accumulator.loadError() + var found *LoadError + if !errors.As(error(loadErr), &found) || found != loadErr { + t.Fatal("errors.As did not find LoadError") + } + + first := loadErr.Diagnostics() + first[1].Primary.SourceID = "mutated" + first[1].Path.segments[0].key = "mutated" + first[1].Related[0].SourceID = "mutated" + second := loadErr.Diagnostics() + if second[1].Primary.SourceID != "source-2" || second[1].Path.String() != "/earlier" || second[1].Related[0].SourceID != "source-2" { + t.Fatal("Diagnostics returned aliased records") + } + sources := loadErr.Sources() + sources[0].ImportedFrom.SourceID = "mutated" + if loadErr.Sources()[0].ImportedFrom.SourceID != "source-0" { + t.Fatal("Sources returned aliased records") + } + if got := loadErr.Error(); got != "configkit: configuration load failed with 3 errors" { + t.Fatalf("Error() = %q", got) + } +} + +func TestWriteDiagnosticsEscapesControls(t *testing.T) { + t.Parallel() + + diagnostic := Diagnostic{ + Code: "validate.error", + Severity: SeverityError, + Path: Keys("line\nbreak"), + Message: "message\r\nnext", + Primary: &Location{SourceID: "source\tlabel", Line: 1}, + } + var output bytes.Buffer + if err := WriteDiagnostics(&output, []Diagnostic{diagnostic}); err != nil { + t.Fatal(err) + } + got := output.String() + if strings.Count(got, "\n") != 1 { + t.Fatalf("control characters created output lines: %q", got) + } + for _, escaped := range []string{`line\nbreak`, `message\r\nnext`, `source\tlabel`} { + if !strings.Contains(got, escaped) { + t.Errorf("output %q does not contain %q", got, escaped) + } + } +} + +func TestWriteDiagnosticsPropagatesWriterFailures(t *testing.T) { + t.Parallel() + + diagnostics := []Diagnostic{{Severity: SeverityError, Message: "safe"}} + want := errors.New("write failed") + if err := WriteDiagnostics(errorWriter{err: want}, diagnostics); !errors.Is(err, want) { + t.Fatalf("error writer returned %v", err) + } + if err := WriteDiagnostics(shortWriter{}, diagnostics); !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("short writer returned %v", err) + } +} + +type errorWriter struct { + err error +} + +func (w errorWriter) Write(p []byte) (int, error) { + return min(3, len(p)), w.err +} + +type shortWriter struct{} + +func (shortWriter) Write(p []byte) (int, error) { + return max(0, len(p)-1), nil +} diff --git a/internal/engine/directives.go b/internal/engine/directives.go new file mode 100644 index 0000000..738acee --- /dev/null +++ b/internal/engine/directives.go @@ -0,0 +1,229 @@ +package engine + +import ( + "path" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +const directiveKey = "configkit" + +type importDeclaration struct { + path string + glob string + optional bool + location Location +} + +func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { + body := document.Clone(root) + var declarations []importDeclaration + directiveIndex := -1 + for index, entry := range body.Mapping { + if entry.Key != directiveKey { + continue + } + directiveIndex = index + parsed, diagnostics := parseImportDirective(entry.Value, Keys(directiveKey)) + if len(diagnostics) != 0 { + return nil, nil, diagnostics + } + declarations = parsed + break + } + if directiveIndex >= 0 { + body.Mapping = append(body.Mapping[:directiveIndex], body.Mapping[directiveIndex+1:]...) + } + if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 { + return nil, nil, diagnostics + } + return body, declarations, nil +} + +func parseImportDirective(node *document.Node, directivePath Path) ([]importDeclaration, []Diagnostic) { + if node.Kind != document.Mapping { + return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)} + } + var declarations []importDeclaration + for _, entry := range node.Mapping { + fieldPath := directivePath.Key(entry.Key) + if entry.Key != "imports" { + return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value)} + } + if entry.Value.Kind != document.Sequence { + return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value)} + } + for index, item := range entry.Value.Sequence { + declaration, diagnostics := parseImportDeclaration(item, fieldPath.Index(index)) + if len(diagnostics) != 0 { + return nil, diagnostics + } + declarations = append(declarations, declaration) + } + } + return declarations, nil +} + +func parseImportDeclaration(node *document.Node, declarationPath Path) (importDeclaration, []Diagnostic) { + if node.Kind != document.Mapping { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)} + } + declaration := importDeclaration{location: publicLocation(node.Location)} + seenTarget := false + for _, entry := range node.Mapping { + fieldPath := declarationPath.Key(entry.Key) + switch entry.Key { + case "path", "glob": + if seenTarget || entry.Value.Kind != document.String || entry.Value.Text == "" || strings.TrimSpace(entry.Value.Text) != entry.Value.Text { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration requires exactly one nonempty path or glob string", entry.Value)} + } + seenTarget = true + declaration.location = publicLocation(entry.Value.Location) + if entry.Key == "path" { + declaration.path = entry.Value.Text + } else { + declaration.glob = entry.Value.Text + } + case "optional": + if entry.Value.Kind != document.Boolean { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "optional must be a boolean", entry.Value)} + } + declaration.optional = entry.Value.Text == "true" + default: + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} + } + } + if !seenTarget { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path or glob", node)} + } + return declaration, nil +} + +func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { + if node.Kind == document.Mapping { + for _, entry := range node.Mapping { + childPath := current.Key(entry.Key) + if entry.Key == directiveKey { + location := publicLocation(entry.KeyLocation) + return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: childPath, Message: "configuration directives are permitted only at the document root", Primary: &location}} + } + if diagnostics := nestedDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 { + return diagnostics + } + } + } else if node.Kind == document.Sequence { + for index, child := range node.Sequence { + if diagnostics := nestedDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 { + return diagnostics + } + } + } + return nil +} + +func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node { + if rule == nil { + return nil + } + current := root + if len(rule.Path.segments) == 0 { + return current + } + for _, segment := range rule.Path.segments { + if current.Kind != document.Mapping { + return current + } + if segment.kind == pathSegmentIndex { + return nil + } + key := segment.key + index := documentEntryIndex(current.Mapping, key) + if index < 0 { + return nil + } + current = current.Mapping[index].Value + } + return current +} + +func validImportPath(value string) bool { + if value == "" || strings.TrimSpace(value) != value || path.IsAbs(value) || strings.HasPrefix(value, "//") { + return false + } + return !hasImportScheme(value) +} + +func hasImportScheme(value string) bool { + colon := strings.IndexByte(value, ':') + slash := strings.IndexByte(value, '/') + if colon <= 0 || (slash >= 0 && slash < colon) { + return false + } + for index := 0; index < colon; index++ { + character := value[index] + if index == 0 { + if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') { + return false + } + continue + } + if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && character != '+' && character != '-' && character != '.' { + return false + } + } + return true +} + +func nodeImportDiagnostic(code Code, path Path, message string, node *document.Node) Diagnostic { + location := publicLocation(node.Location) + return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location} +} + +func profileDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { + if node.Kind == document.Mapping { + for _, entry := range node.Mapping { + childPath := current.Key(entry.Key) + if entry.Key == directiveKey { + location := publicLocation(entry.KeyLocation) + return []Diagnostic{{ + Code: CodeProfileDirective, Severity: SeverityError, Path: childPath, + Message: "profiles cannot contain configuration directives", Primary: &location, + }} + } + if diagnostics := profileDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 { + return diagnostics + } + } + } else if node.Kind == document.Sequence { + for index, child := range node.Sequence { + if diagnostics := profileDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 { + return diagnostics + } + } + } + return nil +} + +func disabledDirectiveDiagnostics(node *document.Node, path Path) []Diagnostic { + var diagnostics []Diagnostic + if node.Kind == document.Mapping { + for _, entry := range node.Mapping { + childPath := path.Key(entry.Key) + if entry.Key == directiveKey { + location := Location{SourceID: entry.KeyLocation.SourceID, Line: entry.KeyLocation.Line, Column: entry.KeyLocation.Column} + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeImportDisabled, Severity: SeverityError, Path: childPath, + Message: "configuration directives require imports to be enabled", Primary: &location, + }) + } + diagnostics = append(diagnostics, disabledDirectiveDiagnostics(entry.Value, childPath)...) + } + } else if node.Kind == document.Sequence { + for index, child := range node.Sequence { + diagnostics = append(diagnostics, disabledDirectiveDiagnostics(child, path.Index(index))...) + } + } + return diagnostics +} diff --git a/internal/engine/document_compare.go b/internal/engine/document_compare.go new file mode 100644 index 0000000..68286d3 --- /dev/null +++ b/internal/engine/document_compare.go @@ -0,0 +1,110 @@ +package engine + +import ( + "cmp" + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + "slices" +) + +type documentDeltaKind uint8 + +const ( + documentAdded documentDeltaKind = iota + 1 + documentRemoved + documentReplaced +) + +type documentDelta struct { + path Path + kind documentDeltaKind +} + +// compareConfigurationValues ignores locations and sensitivity metadata. Mapping +// order is immaterial; sequences are atomic for change reporting. +func compareConfigurationValues(before, after *document.Node) []documentDelta { + var deltas []documentDelta + compareConfigurationValue(before, after, Path{}, &deltas) + slices.SortFunc(deltas, func(left, right documentDelta) int { + if order := cmp.Compare(left.path.String(), right.path.String()); order != 0 { + return order + } + return int(left.kind) - int(right.kind) + }) + return deltas +} + +func compareConfigurationValue(before, after *document.Node, path Path, deltas *[]documentDelta) { + if before == nil && after == nil { + return + } + if before == nil { + *deltas = append(*deltas, documentDelta{path: path.clone(), kind: documentAdded}) + return + } + if after == nil { + *deltas = append(*deltas, documentDelta{path: path.clone(), kind: documentRemoved}) + return + } + if before.Kind != after.Kind { + *deltas = append(*deltas, documentDelta{path: path.clone(), kind: documentReplaced}) + return + } + switch before.Kind { + case document.Mapping: + beforeEntries := make(map[string]*document.Node, len(before.Mapping)) + afterEntries := make(map[string]*document.Node, len(after.Mapping)) + keys := make(map[string]bool, len(before.Mapping)+len(after.Mapping)) + for _, entry := range before.Mapping { + beforeEntries[entry.Key] = entry.Value + keys[entry.Key] = true + } + for _, entry := range after.Mapping { + afterEntries[entry.Key] = entry.Value + keys[entry.Key] = true + } + ordered := make([]string, 0, len(keys)) + for key := range keys { + ordered = append(ordered, key) + } + slices.Sort(ordered) + for _, key := range ordered { + compareConfigurationValue(beforeEntries[key], afterEntries[key], path.Key(key), deltas) + } + case document.Sequence: + if !configurationValuesEqual(before, after) { + *deltas = append(*deltas, documentDelta{path: path.clone(), kind: documentReplaced}) + } + default: + if before.Text != after.Text { + *deltas = append(*deltas, documentDelta{path: path.clone(), kind: documentReplaced}) + } + } +} + +// configurationValuesEqual compares only configuration values, not provenance or +// sensitivity. Callers auditing validation must check sensitivity separately. +func configurationValuesEqual(left, right *document.Node) bool { + if left == nil || right == nil { + return left == right + } + if left.Kind != right.Kind || left.Text != right.Text || len(left.Mapping) != len(right.Mapping) || len(left.Sequence) != len(right.Sequence) { + return false + } + if left.Kind == document.Mapping { + leftMap := make(map[string]*document.Node, len(left.Mapping)) + for _, entry := range left.Mapping { + leftMap[entry.Key] = entry.Value + } + for _, entry := range right.Mapping { + if !configurationValuesEqual(leftMap[entry.Key], entry.Value) { + return false + } + } + } + for i := range left.Sequence { + if !configurationValuesEqual(left.Sequence[i], right.Sequence[i]) { + return false + } + } + return true +} diff --git a/internal/engine/document_walk.go b/internal/engine/document_walk.go new file mode 100644 index 0000000..b2c93c9 --- /dev/null +++ b/internal/engine/document_walk.go @@ -0,0 +1,86 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +// walkDocument visits value nodes in preorder, preserving stored mapping order +// and sequence index order. Mapping keys are not separate nodes in this walk. +// The callback borrows paths and nodes; clone any mutable state it retains. +// Specialized origin, pruning, and budget traversals retain their own semantics. +func walkDocument(node *document.Node, path Path, visit func(Path, *document.Node)) { + if node == nil { + return + } + visit(path, node) + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + walkDocument(entry.Value, path.Key(entry.Key), visit) + } + case document.Sequence: + for index, child := range node.Sequence { + walkDocument(child, path.Index(index), visit) + } + } +} + +// indexDocument borrows nodes from its caller-owned tree. +func indexDocument(node *document.Node, path Path, index map[string]*document.Node) { + walkDocument(node, path, func(path Path, node *document.Node) { + index[path.String()] = node + }) +} + +func documentAtPath(root *document.Node, path Path) (*document.Node, bool) { + if root == nil || !path.valid() { + return nil, false + } + current := root + for _, segment := range path.segments { + switch current.Kind { + case document.Mapping: + if segment.kind == pathSegmentIndex { + return nil, false + } + found := false + for _, entry := range current.Mapping { + if entry.Key == segment.key { + current = entry.Value + found = true + break + } + } + if !found { + return nil, false + } + case document.Sequence: + index, ok := schemaIndex(segment) + if !ok || index >= len(current.Sequence) { + return nil, false + } + current = current.Sequence[index] + default: + return nil, false + } + } + return current, true +} + +// countInputNodes includes mapping-key nodes, unlike walkDocument's value-only +// traversal. Aggregate source budgets charge every input occurrence. +func countInputNodes(node *document.Node) int { + if node == nil { + return 0 + } + count := 1 + if node.Kind == document.Mapping { + count += len(node.Mapping) + for _, entry := range node.Mapping { + count += countInputNodes(entry.Value) + } + } else if node.Kind == document.Sequence { + for _, child := range node.Sequence { + count += countInputNodes(child) + } + } + return count +} diff --git a/internal/engine/explain.go b/internal/engine/explain.go new file mode 100644 index 0000000..86e28bc --- /dev/null +++ b/internal/engine/explain.go @@ -0,0 +1,122 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +// ValueSnapshot is a detached, deterministic YAML representation of one value. +// Present distinguishes absence or deletion from an explicit null value. +type ValueSnapshot struct { + Present bool + Redacted bool + YAML string +} + +// ValueChange describes one explicit definition or audited domain operation. +type ValueChange struct { + Before ValueSnapshot + After ValueSnapshot + Origin Origin + Reasons []string + Dependencies []Path +} + +// SecretReferenceInfo describes a successfully resolved effective reference. +// Name is the environment-variable name, never its value. +type SecretReferenceInfo struct { + Name string +} + +// Explanation is a detached safe account of one effective or historical path. +type Explanation struct { + Path Path + Effective ValueSnapshot + Origin Origin + History []ValueChange + SecretReference *SecretReferenceInfo +} + +// Explain returns safe effective and chronological values for a known final or +// historical path. Unknown and malformed paths return false. +func (r *Result[T]) Explain(path Path) (Explanation, bool) { + if r == nil || r.final == nil || r.baseline == nil || !r.metadata.hasHistoricalPath(path) { + return Explanation{}, false + } + explanation := Explanation{Path: path.clone(), Effective: valueSnapshot(r.final, path, r.secrets)} + if info, ok := r.metadata.Lookup(path); ok { + explanation.Origin = cloneOrigin(info.Origin) + } + + working := document.Clone(r.baseline) + if _, present := documentAtPath(working, path); present { + explanation.History = append(explanation.History, ValueChange{ + After: valueSnapshot(working, path, r.secrets), Origin: Origin{Layer: LayerDefault}, + }) + } + for _, application := range r.applications { + operationPath, relevant := applicationOperationPath(application, path) + var before ValueSnapshot + if relevant { + before = valueSnapshot(working, path, r.secrets) + } + applyHistoryApplication(&working, application) + if !relevant { + continue + } + after := valueSnapshot(working, path, r.secrets) + if !before.Present && !after.Present { + continue + } + explanation.History = append(explanation.History, ValueChange{ + Before: before, + After: after, + Origin: historyApplicationOrigin(application, operationPath), + }) + } + + // Intermediate input replacements do not permanently suppress defaults: + // decoding applies them to the final input tree. Show that reconciliation + // explicitly, including canonical scalar conversions, before domain changes. + replayed, _ := documentAtPath(working, path) + decoded, _ := documentAtPath(r.input, path) + if !configurationValuesEqual(replayed, decoded) { + origin := cloneOrigin(r.inputOrigins[path.String()]) + origin.Operation = ChangeDecoding + explanation.History = append(explanation.History, ValueChange{ + Before: valueSnapshot(working, path, r.secrets), + After: valueSnapshot(r.input, path, r.secrets), + Origin: origin, + Reasons: []string{"applied retained defaults and canonical typed decoding"}, + }) + } + + working = document.Clone(r.input) + for _, snapshot := range coalescedResolutionSnapshots(r.resolution) { + relevant := pathsIntersect(snapshot.change.Path, path) + var before ValueSnapshot + if relevant { + before = valueSnapshot(working, path, r.secrets) + } + replaceHistoryPath(&working, snapshot.change.Path, snapshot.after) + if !relevant { + continue + } + after := valueSnapshot(working, path, r.secrets) + if !before.Present && !after.Present { + continue + } + explanation.History = append(explanation.History, ValueChange{ + Before: before, + After: after, + Origin: cloneOrigin(snapshot.change.Origin), + Reasons: append([]string(nil), snapshot.change.Reasons...), + Dependencies: clonePaths(snapshot.change.Dependencies), + }) + } + + if !explanation.Effective.Present && len(explanation.History) == 0 { + return Explanation{}, false + } + if name, ok := r.secretReferences[path.String()]; ok && !pathIsSecret(r.secrets, path) { + explanation.SecretReference = &SecretReferenceInfo{Name: name} + } + return explanation, true +} diff --git a/internal/engine/explain_render.go b/internal/engine/explain_render.go new file mode 100644 index 0000000..8c62790 --- /dev/null +++ b/internal/engine/explain_render.go @@ -0,0 +1,123 @@ +package engine + +import ( + "errors" + "io" + "strconv" + "strings" +) + +var ( + errExplanationUnavailable = errors.New("configkit: configuration explanation is unavailable") + errExplanationUnknownPath = errors.New("configkit: configuration explanation path is unknown") +) + +// WriteExplanation writes a deterministic human-readable rendering of Explain. +// Unknown paths produce no output. Writer failures may leave partial output. +func (r *Result[T]) WriteExplanation(output io.Writer, path Path) error { + if r == nil || output == nil { + return errExplanationUnavailable + } + explanation, ok := r.Explain(path) + if !ok { + return errExplanationUnknownPath + } + var rendered strings.Builder + rendered.WriteString("path: ") + rendered.WriteString(strconv.Quote(explanation.Path.String())) + rendered.WriteByte('\n') + writeExplanationOrigin(&rendered, "effective source", explanation.Origin, r.sources) + writeExplanationSnapshot(&rendered, "effective (canonical typed)", explanation.Effective, 0) + if explanation.SecretReference != nil { + rendered.WriteString("secret reference: ") + rendered.WriteString(strconv.Quote(explanation.SecretReference.Name)) + rendered.WriteString(" (resolved from process environment)\n") + } + rendered.WriteString("history:\n") + for _, change := range explanation.History { + rendered.WriteString(" - representation: ") + if change.Origin.Operation == ChangeDecoding { + rendered.WriteString("input to canonical typed\n") + } else if change.Origin.Layer == LayerBase || change.Origin.Layer == LayerProfile || change.Origin.Layer == LayerCLI { + rendered.WriteString("policy-validated input\n") + } else { + rendered.WriteString("canonical typed\n") + } + writeExplanationOrigin(&rendered, " source", change.Origin, r.sources) + writeExplanationSnapshot(&rendered, " before", change.Before, 4) + writeExplanationSnapshot(&rendered, " after", change.After, 4) + for _, reason := range change.Reasons { + rendered.WriteString(" reason: ") + rendered.WriteString(strconv.Quote(reason)) + rendered.WriteByte('\n') + } + for _, dependency := range change.Dependencies { + rendered.WriteString(" dependency: ") + rendered.WriteString(strconv.Quote(dependency.String())) + rendered.WriteByte('\n') + } + } + data := []byte(rendered.String()) + written, err := output.Write(data) + if err == nil && written != len(data) { + return io.ErrShortWrite + } + return err +} + +func writeExplanationOrigin(output *strings.Builder, label string, origin Origin, sources []SourceInfo) { + output.WriteString(label) + output.WriteString(": layer=") + output.WriteString(strconv.Quote(string(origin.Layer))) + if origin.Operation != "" { + output.WriteString(" operation=") + output.WriteString(strconv.Quote(string(origin.Operation))) + } + output.WriteByte('\n') + for _, location := range origin.Locations { + output.WriteString(" location: source=") + output.WriteString(strconv.Quote(location.SourceID)) + if label := sourceLabel(sources, location.SourceID); label != "" { + output.WriteString(" label=") + output.WriteString(strconv.Quote(label)) + } + output.WriteString(" line=") + output.WriteString(strconv.Itoa(location.Line)) + output.WriteString(" column=") + output.WriteString(strconv.Itoa(location.Column)) + output.WriteByte('\n') + } +} + +func sourceLabel(sources []SourceInfo, sourceID string) string { + for _, source := range sources { + if source.ID == sourceID { + return source.Label + } + } + return "" +} + +func writeExplanationSnapshot(output *strings.Builder, label string, snapshot ValueSnapshot, indent int) { + if !snapshot.Present { + output.WriteString(label) + output.WriteString(": absent\n") + return + } + output.WriteString(label) + if snapshot.Redacted { + output.WriteString(" (redacted)") + } + output.WriteString(":\n") + prefix := strings.Repeat(" ", indent+2) + for _, line := range strings.SplitAfter(snapshot.YAML, "\n") { + if line == "" { + continue + } + output.WriteString(prefix) + output.WriteString(line) + if !strings.HasSuffix(line, "\n") { + output.WriteByte('\n') + } + } +} diff --git a/internal/engine/explain_test.go b/internal/engine/explain_test.go new file mode 100644 index 0000000..d155d4f --- /dev/null +++ b/internal/engine/explain_test.go @@ -0,0 +1,516 @@ +package engine + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type explanationConcurrency struct { + Transcription struct { + Concurrency int `yaml:"concurrency"` + } `yaml:"transcription"` +} + +func TestExplainShowsDefaultBaseAndEffectiveProfileValue(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "config.yml", "configkit:\n imports: [{path: transcription.yml}]\n") + writeImportFile(t, directory, "transcription.yml", "transcription:\n concurrency: 8\n") + writeImportFile(t, directory, filepath.Join("profiles", "testing.yml"), "transcription:\n concurrency: 1\n") + result, err := Load[explanationConcurrency](File(mainPath), Options[explanationConcurrency]{ + Imports: true, Profile: "testing", + Definition: Definition[explanationConcurrency]{Defaults: func() explanationConcurrency { + value := explanationConcurrency{} + value.Transcription.Concurrency = 4 + return value + }}, + }) + if err != nil { + t.Fatal(err) + } + explanation, ok := result.Explain(Keys("transcription", "concurrency")) + if !ok { + t.Fatal("known path was not explained") + } + if got := explanationValues(explanation); !reflect.DeepEqual(got, []string{"4\n", "8\n", "1\n"}) { + t.Fatalf("history values = %#v", got) + } + if explanation.Effective.YAML != "1\n" || explanation.Origin.Layer != LayerProfile || + len(explanation.Origin.Locations) != 1 || explanation.Origin.Locations[0].SourceID != "source-3" || + explanation.Origin.Locations[0].Line != 2 { + t.Fatalf("effective explanation = %#v", explanation) + } +} + +type explanationConfig struct { + Concurrency int `yaml:"concurrency"` + Equal string `yaml:"equal"` + Limit int `yaml:"limit"` + Items []string `yaml:"items"` + Labels map[string]string `yaml:"labels"` + Dynamic any `yaml:"dynamic"` + Maybe *string `yaml:"maybe"` + Empty []string `yaml:"empty"` + Credential Secret `yaml:"credential"` + Classified Secret `yaml:"classified"` + DerivedA string `yaml:"derived_a"` + DerivedB string `yaml:"derived_b"` +} + +func TestExplainReplaysCLIResolutionContainersAndSafeHistoricalValues(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "config.yml", `concurrency: 8 +equal: same +limit: wrong-before-profile +items: [base, removed] +labels: {base: retained} +dynamic: {retired: HISTORICAL_EXPLAIN_CANARY, public: visible} +credential: APP_SECRET +classified: APP_SECRET +`) + writeImportFile(t, directory, filepath.Join("profiles", "testing.yml"), `concurrency: 1 +equal: same +limit: 7 +items: [profile] +labels: {profile: retained} +dynamic: selected +`) + result, err := loadWithSecretLookup(File(mainPath), func(name string) (string, bool) { + if name != "APP_SECRET" { + t.Fatalf("secret lookup = %q", name) + } + return "RESOLVED_EXPLAIN_CANARY", true + }, Options[explanationConfig]{ + Profile: "testing", + Overrides: []Override{ + {Path: Keys("concurrency"), Value: 2, Flag: "--concurrency"}, + {Path: Keys("equal"), Value: "same", Flag: "--equal"}, + {Path: Keys("items"), Value: []any{"cli", "restored"}, Flag: "--items"}, + {Path: Keys("labels"), Value: map[string]any{"cli": "retained"}, Flag: "--label"}, + }, + Definition: Definition[explanationConfig]{ + Defaults: func() explanationConfig { + return explanationConfig{ + Concurrency: 4, Equal: "same", Labels: map[string]string{"default": "retained"}, + Maybe: nil, Empty: []string{}, Credential: SecretEnv("APP_SECRET"), Classified: SecretEnv("APP_SECRET"), + } + }, + Secret: func(path Path) bool { + return path.String() == "/dynamic/retired" || path.String() == "/classified" + }, + Resolve: func(configuration *explanationConfig, resolution *Resolution) error { + Normalize(resolution, Keys("concurrency"), &configuration.Concurrency, 3, + "normalized\nconcurrency", Keys("labels")) + Normalize(resolution, Keys("equal"), &configuration.Equal, "same", "record equal value") + Normalize(resolution, Keys("derived_a"), &configuration.DerivedA, "DERIVED_EXPLAIN_A", + "derived cycle a", Keys("derived_b")) + Normalize(resolution, Keys("derived_b"), &configuration.DerivedB, "DERIVED_EXPLAIN_B", + "derived cycle b", Keys("derived_a"), Keys("dynamic")) + return nil + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + concurrency, ok := result.Explain(Keys("concurrency")) + if !ok { + t.Fatal("concurrency was not explained") + } + if got := explanationValues(concurrency); !reflect.DeepEqual(got, []string{"4\n", "8\n", "1\n", "2\n", "3\n"}) { + t.Fatalf("concurrency history = %#v", got) + } + if got := historyValueLayers(concurrency); !reflect.DeepEqual(got, []Layer{LayerDefault, LayerBase, LayerProfile, LayerCLI, LayerResolution}) { + t.Fatalf("concurrency layers = %#v", got) + } + if concurrency.History[4].Before.YAML != "2\n" || concurrency.History[4].Reasons[0] != "normalized\nconcurrency" || + concurrency.History[4].Dependencies[0].String() != "/labels" { + t.Fatalf("resolution history = %#v", concurrency.History[4]) + } + + equal, _ := result.Explain(Keys("equal")) + if len(equal.History) != 5 { + t.Fatalf("equal writes were lost: %#v", equal.History) + } + for _, change := range equal.History[1:] { + if change.Before.YAML != "same\n" || change.After.YAML != "same\n" { + t.Fatalf("equal replacement = %#v", change) + } + } + + limit, _ := result.Explain(Keys("limit")) + if got := explanationValues(limit); !reflect.DeepEqual(got, []string{"0\n", "wrong-before-profile\n", "7\n"}) { + t.Fatalf("overridden invalid input was not retained: %#v", got) + } + removed, ok := result.Explain(Keys("dynamic", "retired")) + if !ok || removed.Effective.Present || len(removed.History) < 2 { + t.Fatalf("removed path = %#v, %v", removed, ok) + } + for _, change := range removed.History { + if strings.Contains(change.Before.YAML+change.After.YAML, "HISTORICAL_EXPLAIN_CANARY") { + t.Fatalf("historical secret escaped redaction: %#v", change) + } + } + if !removed.History[0].After.Redacted || removed.History[len(removed.History)-1].After.Present { + t.Fatalf("removed secret history = %#v", removed.History) + } + dynamic, _ := result.Explain(Keys("dynamic")) + if len(dynamic.History) < 3 || !dynamic.History[1].After.Redacted || + strings.Contains(dynamic.History[1].After.YAML, "HISTORICAL_EXPLAIN_CANARY") { + t.Fatalf("container redaction history = %#v", dynamic.History) + } + + items, _ := result.Explain(Keys("items")) + if len(items.History) != 4 || !strings.Contains(items.History[1].After.YAML, "removed") || + strings.Contains(items.History[2].After.YAML, "removed") || !strings.Contains(items.History[3].After.YAML, "restored") { + t.Fatalf("atomic sequence history = %#v", items.History) + } + removedIndex, ok := result.Explain(Keys("items").Index(1)) + if !ok || !removedIndex.History[1].Before.Present || removedIndex.History[1].After.Present || + !removedIndex.History[2].After.Present { + t.Fatalf("sequence descendant history = %#v", removedIndex.History) + } + labels, _ := result.Explain(Keys("labels")) + if len(labels.History) != 4 || !strings.Contains(labels.Effective.YAML, "default: retained") || + !strings.Contains(labels.Effective.YAML, "profile: retained") || !strings.Contains(labels.Effective.YAML, "cli: retained") { + t.Fatalf("composite mapping history = %#v", labels) + } + if labels.Origin.Layer != LayerCLI || len(labels.Origin.Locations) != 3 { + t.Fatalf("composite effective origin = %#v", labels.Origin) + } + + maybe, _ := result.Explain(Keys("maybe")) + empty, _ := result.Explain(Keys("empty")) + if !maybe.Effective.Present || maybe.Effective.YAML != "null\n" || !empty.Effective.Present || empty.Effective.YAML != "[]\n" { + t.Fatalf("null/empty snapshots = maybe %#v empty %#v", maybe.Effective, empty.Effective) + } + credential, _ := result.Explain(Keys("credential")) + if credential.SecretReference == nil || credential.SecretReference.Name != "APP_SECRET" || credential.Effective.YAML != "APP_SECRET\n" { + t.Fatalf("secret reference = %#v", credential) + } + classified, _ := result.Explain(Keys("classified")) + if classified.SecretReference != nil || !classified.Effective.Redacted || + strings.Contains(classified.Effective.YAML, "APP_SECRET") { + t.Fatalf("classified secret reference = %#v", classified) + } + var derivedExplanations []Explanation + for _, path := range []Path{Keys("derived_a"), Keys("derived_b")} { + derived, _ := result.Explain(path) + if !derived.Effective.Redacted || strings.Contains(derived.Effective.YAML, "DERIVED_EXPLAIN") { + t.Fatalf("dependency closure at %s = %#v", path.String(), derived) + } + derivedExplanations = append(derivedExplanations, derived) + } + + formatted := fmt.Sprintf("%v %#v %v", removed, credential, derivedExplanations) + encoded, marshalErr := json.Marshal(append([]Explanation{removed, credential, classified}, derivedExplanations...)) + if marshalErr != nil || strings.Contains(formatted+string(encoded), "RESOLVED_EXPLAIN_CANARY") || + strings.Contains(formatted+string(encoded), "HISTORICAL_EXPLAIN_CANARY") || + strings.Contains(formatted+string(encoded), "DERIVED_EXPLAIN") { + t.Fatalf("public records disclosed a canary: %s %s", formatted, encoded) + } + + concurrency.Path.segments[0].key = "mutated" + concurrency.History[4].Reasons[0] = "mutated" + concurrency.History[4].Dependencies[0].segments[0].key = "mutated" + result.Config.Concurrency = 99 + again, _ := result.Explain(Keys("concurrency")) + if again.Path.String() != "/concurrency" || again.Effective.YAML != "3\n" || + again.History[4].Reasons[0] != "normalized\nconcurrency" || + again.History[4].Dependencies[0].String() != "/labels" { + t.Fatalf("returned explanation aliased internal history: %#v", again) + } + + var first, second bytes.Buffer + if err := result.WriteExplanation(&first, Keys("concurrency")); err != nil { + t.Fatal(err) + } + if err := result.WriteExplanation(&second, Keys("concurrency")); err != nil || first.String() != second.String() { + t.Fatalf("nondeterministic rendering: %v\n%s\n%s", err, first.String(), second.String()) + } + if strings.Contains(first.String(), "\nconcurrency\"") || !strings.Contains(first.String(), `"normalized\nconcurrency"`) { + t.Fatalf("reason controls were not escaped: %q", first.String()) + } + unknownOutput := bytes.NewBufferString("unchanged") + if err := result.WriteExplanation(unknownOutput, Keys("unknown")); !errors.Is(err, errExplanationUnknownPath) || unknownOutput.String() != "unchanged" { + t.Fatalf("unknown writer = %q, %v", unknownOutput.String(), err) + } + if _, ok := result.Explain(Keys("items").Index(-1)); ok { + t.Fatal("malformed path was explained") + } + wantWriteError := errors.New("write failed") + if err := result.WriteExplanation(errorWriter{err: wantWriteError}, Keys("concurrency")); !errors.Is(err, wantWriteError) { + t.Fatalf("writer error = %v", err) + } + if err := result.WriteExplanation(shortWriter{}, Keys("concurrency")); !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("short writer error = %v", err) + } + var invalidInput bytes.Buffer + if err := result.WriteExplanation(&invalidInput, Keys("limit")); err != nil || + !strings.Contains(invalidInput.String(), "policy-validated input") || + strings.Contains(invalidInput.String(), "wrong-before-profile\n source: layer=\"base\"\n before (canonical typed)") { + t.Fatalf("historical representation rendering = %v\n%s", err, invalidInput.String()) + } +} + +func TestSecretsBootstrapLoadAndExplanationRemainSeparateAndSafe(t *testing.T) { + const name = "CONFIGKIT_EXPLAIN_BOOTSTRAP_TEST" + previous, present := os.LookupEnv(name) + if err := os.Unsetenv(name); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if present { + _ = os.Setenv(name, previous) + } else { + _ = os.Unsetenv(name) + } + }) + directory := t.TempDir() + writeSecretTestFile(t, directory, name, "BOOTSTRAP_EXPLAIN_CANARY\n") + report, err := LoadSecretsDir(directory) + if err != nil || len(report.Entries()) != 1 || report.Entries()[0].Status != SecretFileLoaded { + t.Fatalf("bootstrap = %#v, %v", report, err) + } + type config struct { + Credential Secret `yaml:"credential"` + } + result, err := Load[config](Bytes("config.yml", []byte("credential: "+name+"\n"))) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + if err := result.WriteExplanation(&output, Keys("credential")); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), name) || strings.Contains(output.String(), "BOOTSTRAP_EXPLAIN_CANARY") { + t.Fatalf("explanation mixed bootstrap value into reference report: %s", output.String()) + } +} + +func TestHistoryNodeBudgetCountsSharedTreesOnce(t *testing.T) { + leaf := &document.Node{Kind: document.String, Text: "value"} + root := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{Key: "value", Value: leaf}}} + applications := []historyApplication{{layer: LayerBase, value: root}} + if !historyNodesWithin(2, root, root, root, applications, nil) { + t.Fatal("structurally shared history was counted more than once") + } + other := document.Clone(root) + if historyNodesWithin(3, root, other, root, applications, nil) { + t.Fatal("distinct retained nodes did not consume the budget") + } +} + +func explanationValues(explanation Explanation) []string { + values := make([]string, len(explanation.History)) + for index, change := range explanation.History { + values[index] = change.After.YAML + } + return values +} + +func historyValueLayers(explanation Explanation) []Layer { + layers := make([]Layer, len(explanation.History)) + for index, change := range explanation.History { + layers[index] = change.Origin.Layer + } + return layers +} + +func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) { + type cfg struct { + M map[string]int `yaml:"m"` + } + for _, base := range []string{"null", "replaced-text"} { + for _, cli := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/cli=%v", base, cli), func(t *testing.T) { + root := t.TempDir() + main := writeImportFile(t, root, "main.yml", "m: "+base+"\n") + opts := Options[cfg]{Definition: Definition[cfg]{Defaults: func() cfg { return cfg{M: map[string]int{"a": 1}} }}} + if cli { + opts.Overrides = []Override{{Path: Keys("m"), Value: map[string]any{"b": 2}, Flag: "--m"}} + } else { + writeImportFile(t, root, "profiles/p.yml", "m: {b: 2}\n") + opts.Profile = "p" + } + result, err := Load(File(main), opts) + if err != nil { + t.Fatal(err) + } + for _, path := range []Path{Keys("m"), Keys("m", "a")} { + explanation, ok := result.Explain(path) + if !ok { + t.Fatal("missing explanation") + } + last := explanation.History[len(explanation.History)-1] + if last.After != explanation.Effective || last.Origin.Operation != ChangeDecoding { + t.Fatalf("history does not reconcile final value: %#v", explanation) + } + info, _ := result.Metadata().Lookup(path) + if last.Origin.Layer != info.Origin.Layer { + t.Fatal("decoding lost effective origin") + } + } + }) + } + } +} + +func TestIntegerVersionInspectionRetainsInputAndReconcilesTypedValue(t *testing.T) { + type config struct { + Version string `yaml:"version"` + } + rule := &VersionRule{Path: Keys("version"), Check: func(version Version, _ *Validation) error { + if version.Text != "4" { + t.Errorf("version = %#v", version) + } + return nil + }} + options := Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Version: "4"} }, + Version: rule, + }} + first, err := Load(Bytes("version.yml", []byte("version: 4\n")), options) + if err != nil { + t.Fatal(err) + } + second, err := Load(Bytes("version.yml", []byte("version: 4\n")), options) + if err != nil { + t.Fatal(err) + } + if first.Config.Version != "4" || second.Config.Version != "4" { + t.Fatalf("configurations = %#v / %#v", first.Config, second.Config) + } + inputVersion, inputOK := documentAtPath(first.applications[0].value, Keys("version")) + finalVersion, finalOK := documentAtPath(first.final, Keys("version")) + if !inputOK || inputVersion.Kind != document.Number || inputVersion.Text != "4" || + !finalOK || finalVersion.Kind != document.String || finalVersion.Text != "4" { + t.Fatalf("input/final = %#v / %#v", inputVersion, finalVersion) + } + + info, ok := first.Metadata().Lookup(Keys("version")) + if !ok || !info.Explicit || info.Null || info.Origin.Layer != LayerBase || len(info.Origin.Locations) != 1 || + info.Origin.Locations[0].SourceID != "source-1" || info.Origin.Locations[0].Line != 1 { + t.Fatalf("version metadata = %#v, %v", info, ok) + } + explanation, ok := first.Explain(Keys("version")) + if !ok || explanation.Effective.YAML != "'4'\n" { + t.Fatalf("version explanation = %#v, %v", explanation, ok) + } + foundInput, foundDecoding := false, false + for _, change := range explanation.History { + if change.After.YAML == "4\n" && change.Origin.Layer == LayerBase { + foundInput = true + } + if change.Before.YAML == "4\n" && change.After.YAML == "'4'\n" && change.Origin.Operation == ChangeDecoding { + foundDecoding = true + } + if change.Origin.Layer == LayerResolution { + t.Fatalf("conversion recorded as resolution: %#v", change) + } + } + if !foundInput || !foundDecoding { + t.Fatalf("version history = %#v", explanation.History) + } + paths := first.Metadata().HistoricalPaths() + foundPath := false + for _, path := range paths { + if path.String() == "/version" { + foundPath = true + } + } + if !foundPath || len(first.Metadata().History(Keys("version"))) < 2 { + t.Fatalf("historical version paths/history = %#v / %#v", paths, first.Metadata().History(Keys("version"))) + } + + var effective strings.Builder + if err := first.WriteEffective(&effective); err != nil || effective.String() != "version: '4'\n" { + t.Fatalf("effective output = %q, %v", effective.String(), err) + } + first.Config.Version = "mutated" + effective.Reset() + if err := first.WriteEffective(&effective); err != nil || effective.String() != "version: '4'\n" || second.Config.Version != "4" { + t.Fatalf("detached results = %q / %#v, %v", effective.String(), second.Config, err) + } +} + +func TestSensitiveIntegerVersionStaysRedactedWithoutSecretLookup(t *testing.T) { + type config struct { + Version string `yaml:"version"` + } + lookups := 0 + result, err := loadWithSecretLookup(Bytes("version.yml", []byte("version: 987654321\n")), func(string) (string, bool) { + lookups++ + return "CREDENTIAL_CANARY", true + }, Options[config]{Definition: Definition[config]{ + Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }}, + Secret: func(path Path) bool { return path.String() == "/version" }, + }}) + if err != nil || result.Config.Version != "987654321" || lookups != 0 { + t.Fatalf("result = %#v, lookups = %d, err = %v", result, lookups, err) + } + var effective strings.Builder + if err := result.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + explanation, ok := result.Explain(Keys("version")) + formatted := effective.String() + fmt.Sprintf("%#v %#v", explanation, result.Metadata().History(Keys("version"))) + if !ok || !explanation.Effective.Redacted || strings.Contains(formatted, "987654321") || strings.Contains(formatted, "CREDENTIAL_CANARY") { + t.Fatalf("sensitive inspection = %q / %#v", effective.String(), explanation) + } +} + +func TestExplanationReplaysResolutionAdditionsAndOverlappingIntent(t *testing.T) { + type cfg struct { + M map[string]any `yaml:"m"` + } + for _, overlap := range []bool{false, true} { + t.Run(fmt.Sprint(overlap), func(t *testing.T) { + result, err := Load(Bytes("base", []byte("m: {old: 1}\n")), Options[cfg]{Definition: Definition[cfg]{Resolve: func(c *cfg, r *Resolution) error { + c.M["new"] = map[string]any{"child": int64(7)} + delete(c.M, "old") + r.Record(Keys("m", "new"), ChangeDerivedDefault, "add nested", Keys("m", "old")) + r.Record(Keys("m", "old"), ChangeNormalization, "remove old") + if overlap { + r.Record(Keys("m"), ChangeNormalization, "update container") + } + return nil + }}}) + if err != nil { + t.Fatal(err) + } + for _, path := range []Path{Keys("m"), Keys("m", "new"), Keys("m", "new", "child"), Keys("m", "old")} { + x, ok := result.Explain(path) + if !ok { + t.Fatal("missing known history") + } + var count int + for _, v := range x.History { + if v.Origin.Layer == LayerResolution { + count++ + if overlap && len(v.Reasons) != 3 { + t.Fatal("lost overlapping intent") + } + } + } + if overlap && count != 1 { + t.Fatalf("overlapping records applied %d times", count) + } + last := x.History[len(x.History)-1] + if last.After != x.Effective { + t.Fatalf("incorrect replay for %s: %#v", path, x) + } + } + }) + } +} diff --git a/internal/engine/filesystem.go b/internal/engine/filesystem.go new file mode 100644 index 0000000..8b99b52 --- /dev/null +++ b/internal/engine/filesystem.go @@ -0,0 +1,121 @@ +package engine + +import ( + "os" + "path/filepath" + "strings" +) + +func validateFilesystemContext(source Source, context *FilesystemContext, main parsedSource) *Diagnostic { + rootPath, _, _, diagnostic := importFilesystem(source, context, main) + if diagnostic != nil { + return diagnostic + } + root, err := os.OpenRoot(rootPath) + if err != nil { + value := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) + value.Primary = nil + return &value + } + if err := root.Close(); err != nil { + value := importDiagnostic(CodeImportRead, "configuration root could not be closed", nil) + return &value + } + return nil +} + +func filesystemOptionDiagnostics(source Source, context *FilesystemContext) []Diagnostic { + switch source.kind { + case sourceNone: + return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource cannot use filesystem context")} + case sourceFile: + if context.BaseDir != "" { + return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory")} + } + case sourceBytes, sourceReader: + if context.BaseDir == "" || context.Root == "" { + return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "byte and reader filesystem context requires base and root directories")} + } + if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) { + return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "byte and reader filesystem base and root directories must be absolute")} + } + } + return nil +} + +func importFilesystem(source Source, context *FilesystemContext, main parsedSource) (rootPath, basePath, mainPath string, diagnostic *Diagnostic) { + if main.source == nil || main.document == nil { + value := importDiagnostic(CodeImportInvalid, "imports require a selected configuration source", nil) + return "", "", "", &value + } + + if source.kind == sourceFile { + if context != nil && context.BaseDir != "" { + value := optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory") + return "", "", "", &value + } + canonical, err := canonicalExistingPath(source.path) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "selected source path could not be canonicalized", nil) + return "", "", "", &value + } + mainPath = canonical + basePath = filepath.Dir(canonical) + rootPath = basePath + if context != nil && context.Root != "" { + rootPath, err = canonicalExistingPath(context.Root) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil) + return "", "", "", &value + } + } + } else { + if context == nil || context.BaseDir == "" || context.Root == "" { + value := optionDiagnostic(CodeOptionsInvalid, "byte and reader imports require absolute base and root directories") + return "", "", "", &value + } + if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) { + value := optionDiagnostic(CodeOptionsInvalid, "byte and reader import base and root directories must be absolute") + return "", "", "", &value + } + var err error + basePath, err = canonicalExistingPath(context.BaseDir) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil) + return "", "", "", &value + } + rootPath, err = canonicalExistingPath(context.Root) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil) + return "", "", "", &value + } + } + if !containedPath(rootPath, basePath) || (mainPath != "" && !containedPath(rootPath, mainPath)) { + value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil) + return "", "", "", &value + } + for _, candidate := range []string{rootPath, basePath} { + info, err := os.Stat(candidate) + if err != nil || !info.IsDir() { + value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil) + return "", "", "", &value + } + } + return rootPath, basePath, mainPath, nil +} + +func canonicalExistingPath(name string) (string, error) { + absolute, err := filepath.Abs(name) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(filepath.Clean(absolute)) +} + +func containedPath(root, target string) bool { + relative, err := filepath.Rel(root, target) + if err != nil || filepath.IsAbs(relative) { + return false + } + return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} diff --git a/internal/engine/glob.go b/internal/engine/glob.go new file mode 100644 index 0000000..468be6a --- /dev/null +++ b/internal/engine/glob.go @@ -0,0 +1,205 @@ +package engine + +import ( + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" +) + +const globReadBatch = 128 + +type globCandidate struct { + absolute string + relative string +} + +func (s *importTraversal) expandImportGlob(basePath string, declaration importDeclaration) ([]string, []Diagnostic) { + location := declaration.location + segments, valid := validGlobPattern(declaration.glob) + if !valid { + return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid or uses recursive ** syntax", &location)} + } + + candidates := []globCandidate{{absolute: basePath}} + for index, segment := range segments { + last := index == len(segments)-1 + literal, patterned := globLiteralSegment(segment) + var next []globCandidate + for _, candidate := range candidates { + var names []string + if patterned { + matched, diagnostics := s.readGlobMatches(candidate.absolute, segment, location) + if len(diagnostics) != 0 { + return nil, diagnostics + } + names = matched + } else { + names = []string{literal} + } + for _, name := range names { + absolute := filepath.Join(candidate.absolute, filepath.FromSlash(name)) + relative := path.Join(candidate.relative, name) + if !containedPath(s.rootPath, filepath.Clean(absolute)) { + return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob traversal is outside the configuration root", &location)} + } + canonical, err := canonicalExistingPath(absolute) + if err != nil { + if errors.Is(err, fs.ErrNotExist) && !patterned { + continue + } + return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob match could not be canonicalized", location)} + } + if !containedPath(s.rootPath, canonical) { + return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob traversal resolves outside the configuration root", &location)} + } + if last { + next = append(next, globCandidate{absolute: canonical, relative: relative}) + if len(s.sources)+len(next) > s.limits.sources { + return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob matches exceed the remaining source-count limit", &location)} + } + continue + } + info, err := os.Stat(canonical) + if err != nil { + return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob traversal target could not be inspected", location)} + } + if info.IsDir() { + next = append(next, globCandidate{absolute: canonical, relative: relative}) + } + } + } + candidates = next + if len(candidates) == 0 { + return nil, nil + } + } + + matches := make([]string, len(candidates)) + for index, candidate := range candidates { + matches[index] = candidate.relative + } + slices.Sort(matches) + return matches, nil +} + +func (s *importTraversal) readGlobMatches(directory, pattern string, location Location) ([]string, []Diagnostic) { + canonical, err := canonicalExistingPath(directory) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be canonicalized", location)} + } + if !containedPath(s.rootPath, canonical) { + return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory resolves outside the configuration root", &location)} + } + relative, err := filepath.Rel(s.rootPath, canonical) + if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)} + } + directoryFile, err := s.root.Open(relative) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be opened", location)} + } + + var matches []string + for { + entries, readErr := directoryFile.ReadDir(globReadBatch) + for _, entry := range entries { + s.usedEntries++ + if s.usedEntries > s.limits.entries { + _ = directoryFile.Close() + return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)} + } + matched, matchErr := path.Match(pattern, entry.Name()) + if matchErr != nil { + _ = directoryFile.Close() + return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)} + } + if matched { + matches = append(matches, entry.Name()) + } + } + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + _ = directoryFile.Close() + return nil, []Diagnostic{globFilesystemDiagnostic(readErr, "glob directory could not be read", location)} + } + } + if err := directoryFile.Close(); err != nil { + return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be closed", location)} + } + slices.Sort(matches) + return matches, nil +} + +func validGlobPattern(pattern string) ([]string, bool) { + if !validImportPath(pattern) { + return nil, false + } + segments := strings.Split(pattern, "/") + for _, segment := range segments { + if segment == "" || hasRecursiveStars(segment) { + return nil, false + } + if _, err := path.Match(segment, ""); err != nil { + return nil, false + } + } + return segments, true +} + +func hasRecursiveStars(segment string) bool { + inClass := false + for index := 0; index < len(segment); index++ { + switch segment[index] { + case '\\': + index++ + case '[': + inClass = true + case ']': + inClass = false + case '*': + if !inClass && index+1 < len(segment) && segment[index+1] == '*' { + return true + } + } + } + return false +} + +func globLiteralSegment(segment string) (string, bool) { + var literal strings.Builder + for index := 0; index < len(segment); index++ { + switch segment[index] { + case '*', '?', '[': + return "", true + case '\\': + index++ + if index < len(segment) { + literal.WriteByte(segment[index]) + } + default: + literal.WriteByte(segment[index]) + } + } + return literal.String(), false +} + +func globFilesystemDiagnostic(err error, message string, location Location) Diagnostic { + code := CodeImportRead + if errors.Is(err, fs.ErrPermission) { + code = CodeImportPermission + } + return importDiagnostic(code, message, &location) +} diff --git a/internal/engine/glob_test.go b/internal/engine/glob_test.go new file mode 100644 index 0000000..d575536 --- /dev/null +++ b/internal/engine/glob_test.go @@ -0,0 +1,273 @@ +package engine + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +func TestLoadGlobImportsUseLexicalDeclarationOrderAndNestedDFS(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - glob: fragments/*.yml + - path: after.yml +items: {root: root} +`) + aPath := writeImportFile(t, directory, "fragments/a.yml", `configkit: + imports: [{path: nested/child.yml}] +items: {alpha: alpha} +`) + childPath := writeImportFile(t, directory, "fragments/nested/child.yml", "items: {child: child}\n") + bPath := writeImportFile(t, directory, "fragments/b.yml", "items: {beta: beta}\n") + afterPath := writeImportFile(t, directory, "after.yml", "items: {after: after}\n") + + var previous []SourceInfo + for run := 0; run < 2; run++ { + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil { + t.Fatal(err) + } + wantItems := map[string]string{"root": "root", "alpha": "alpha", "child": "child", "beta": "beta", "after": "after"} + if !reflect.DeepEqual(result.Config.Items, wantItems) { + t.Fatalf("items = %#v, want %#v", result.Config.Items, wantItems) + } + sources := result.Sources() + wantPaths := []string{ + canonicalTestPath(t, mainPath), canonicalTestPath(t, aPath), + canonicalTestPath(t, childPath), canonicalTestPath(t, bPath), + canonicalTestPath(t, afterPath), + } + gotPaths := make([]string, len(sources)) + for index, source := range sources { + gotPaths[index] = source.Path + } + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("source paths = %#v, want %#v", gotPaths, wantPaths) + } + if run != 0 && !reflect.DeepEqual(sources, previous) { + t.Fatalf("repeated traversal changed sources: %#v != %#v", sources, previous) + } + previous = sources + } +} + +func TestLoadGlobImportsSupportEscapesClassesAndExplicitConfD(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - glob: 'conf.d/literal\*.yml' + - glob: 'conf.d/item-[ab].yml' + - glob: 'conf.d/question?.yml' +items: {root: root} +`) + writeImportFile(t, directory, "conf.d/literal*.yml", "items: {literal: literal}\n") + writeImportFile(t, directory, "conf.d/item-b.yml", "items: {b: b}\n") + writeImportFile(t, directory, "conf.d/item-a.yml", "items: {a: a}\n") + writeImportFile(t, directory, "conf.d/question1.yml", "items: {question: question}\n") + + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil { + t.Fatal(err) + } + if want := map[string]string{"root": "root", "literal": "literal", "a": "a", "b": "b", "question": "question"}; !reflect.DeepEqual(result.Config.Items, want) { + t.Fatalf("items = %#v, want %#v", result.Config.Items, want) + } + + ignoredPath := writeImportFile(t, directory, "conf.d/not-discovered.yml", "unknown: value\n") + _ = ignoredPath + plainPath := writeImportFile(t, directory, "plain.yml", "items: {root: only}\n") + plain, err := Load[importTestConfig](File(plainPath), Options[importTestConfig]{Imports: true}) + if err != nil || len(plain.Sources()) != 1 || plain.Config.Items["root"] != "only" { + t.Fatalf("implicit discovery result = %#v, %v", plain, err) + } +} + +func TestLoadGlobImportsRejectInvalidAndUnmatchedPatterns(t *testing.T) { + tests := []struct { + name string + pattern string + code Code + }{ + {name: "bad class", pattern: "empty/[", code: CodeImportInvalid}, + {name: "recursive star segment", pattern: "empty/**/value.yml", code: CodeImportInvalid}, + {name: "recursive adjacent stars", pattern: "empty/value**.yml", code: CodeImportInvalid}, + {name: "trailing separator", pattern: "empty/", code: CodeImportInvalid}, + {name: "absolute", pattern: "/tmp/*.yml", code: CodeImportInvalid}, + {name: "URL", pattern: "https://example.invalid/*.yml", code: CodeImportInvalid}, + {name: "missing parent escape", pattern: "../missing/*.yml", code: CodeImportOutsideRoot}, + {name: "unmatched", pattern: "empty/*.yml", code: CodeImportMissing}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { + t.Fatal(err) + } + mainPath := writeGlobRoot(t, directory, test.pattern, false) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) + }) + } + + directory := t.TempDir() + if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { + t.Fatal(err) + } + mainPath := writeGlobRoot(t, directory, "empty/*.yml", true) + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil || len(result.Sources()) != 1 { + t.Fatalf("optional unmatched glob = %#v, %v", result, err) + } +} + +func TestLoadGlobImportsPreserveIdentityAndFileRules(t *testing.T) { + t.Run("overlapping declarations", func(t *testing.T) { + directory := t.TempDir() + writeImportFile(t, directory, "conf/value.yml", "items: {value: value}\n") + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - glob: conf/*.yml + - glob: conf/value.* +`) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportRepeated) + }) + + t.Run("matched directory", func(t *testing.T) { + directory := t.TempDir() + if err := os.MkdirAll(filepath.Join(directory, "conf/matched.yml"), 0o755); err != nil { + t.Fatal(err) + } + mainPath := writeGlobRoot(t, directory, "conf/*.yml", true) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourceNotRegular) + }) + + t.Run("hard links stay distinct", func(t *testing.T) { + directory := t.TempDir() + one := writeImportFile(t, directory, "conf/one.yml", "items: {same: value}\n") + if err := os.Link(one, filepath.Join(directory, "conf/two.yml")); err != nil { + t.Skipf("hard links unavailable: %v", err) + } + mainPath := writeGlobRoot(t, directory, "conf/*.yml", false) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeComposeConflict) + }) +} + +func TestLoadGlobImportsConfineSymlinkDirectories(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("initial filesystem contract covers Linux and macOS") + } + t.Run("contained absolute target", func(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target") + writeImportFile(t, target, "value.yml", "items: {linked: linked}\n") + if err := os.Symlink(target, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeGlobRoot(t, root, "linked/*.yml", false) + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil || result.Config.Items["linked"] != "linked" { + t.Fatalf("contained directory symlink = %#v, %v", result, err) + } + }) + + t.Run("escaping target", func(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + writeImportFile(t, outside, "value.yml", "items: {escaped: escaped}\n") + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeGlobRoot(t, root, "linked/*.yml", true) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) + }) +} + +func TestOptionalGlobDoesNotSuppressDirectoryPermissionFailure(t *testing.T) { + directory := t.TempDir() + locked := filepath.Join(directory, "locked") + if err := os.Mkdir(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0); err != nil { + t.Skipf("cannot create unreadable directory fixture: %v", err) + } + defer func() { _ = os.Chmod(locked, 0o700) }() + if opened, err := os.Open(locked); err == nil { + _ = opened.Close() + t.Skip("filesystem does not enforce the unreadable-directory fixture") + } + mainPath := writeGlobRoot(t, directory, "locked/*.yml", true) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportPermission) +} + +func TestGlobTraversalBoundsNonmatchesAndAcceptedCandidates(t *testing.T) { + directory := t.TempDir() + for _, name := range []string{"one.txt", "two.txt", "three.txt"} { + writeImportFile(t, directory, "many/"+name, "ignored\n") + } + mainPath := writeGlobRoot(t, directory, "many/*.yml", true) + main := parseImportMain(t, mainPath) + limits := defaultImportLimits() + limits.entries = 2 + _, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("entry-budget diagnostics = %#v", diagnostics) + } + + for _, name := range []string{"a.yml", "b.yml"} { + writeImportFile(t, directory, "matches/"+name, "items: {"+name[:1]+": value}\n") + } + mainPath = writeGlobRoot(t, directory, "matches/*.yml", false) + main = parseImportMain(t, mainPath) + limits = defaultImportLimits() + limits.sources = 2 + _, _, diagnostics = loadImportedDocuments(File(mainPath), nil, main, nil, limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("candidate-budget diagnostics = %#v", diagnostics) + } +} + +func TestFilesystemContextIsValidatedWithoutImportsOrActiveProfile(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "items: {root: root}\n") + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{ + Profile: "default", Filesystem: &FilesystemContext{Root: directory}, + }) + if err != nil || result.Config.Items["root"] != "root" { + t.Fatalf("file filesystem context = %#v, %v", result, err) + } + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Filesystem: &FilesystemContext{BaseDir: directory, Root: directory}}, CodeOptionsInvalid) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Filesystem: &FilesystemContext{Root: filepath.Join(directory, "missing")}}, CodeImportInvalid) + + input := Bytes("memory.yml", []byte("items: {root: memory}\n")) + result, err = Load[importTestConfig](input, Options[importTestConfig]{Filesystem: &FilesystemContext{BaseDir: directory, Root: directory}}) + if err != nil || result.Config.Items["root"] != "memory" { + t.Fatalf("byte filesystem context = %#v, %v", result, err) + } + requireLoadCode(t, Bytes("memory.yml", []byte("{}\n")), Options[importTestConfig]{Filesystem: &FilesystemContext{BaseDir: directory}}, CodeOptionsInvalid) + defaultsOnly, err := Load[importTestConfig](NoSource(), Options[importTestConfig]{Profile: "default"}) + if err != nil || defaultsOnly == nil { + t.Fatalf("default profile with NoSource = %#v, %v", defaultsOnly, err) + } + + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { + selectedLink := filepath.Join(directory, "selected-link.yml") + if err := os.Symlink(mainPath, selectedLink); err == nil { + linked, err := Load[importTestConfig](File(selectedLink), Options[importTestConfig]{Filesystem: &FilesystemContext{Root: directory}}) + if err != nil || linked.Sources()[0].Path != canonicalTestPath(t, mainPath) { + t.Fatalf("selected symlink identity = %#v, %v", linked, err) + } + } + } +} + +func writeGlobRoot(t *testing.T, directory, pattern string, optional bool) string { + t.Helper() + optionalLine := "" + if optional { + optionalLine = "\n optional: true" + } + return writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - glob: '"+pattern+"'"+optionalLine+"\n") +} diff --git a/internal/engine/history.go b/internal/engine/history.go new file mode 100644 index 0000000..4f42892 --- /dev/null +++ b/internal/engine/history.go @@ -0,0 +1,276 @@ +package engine + +import ( + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +const historyNodeLimit = 4_000_000 + +// History applications borrow immutable policy-validated input trees. Metadata +// records answer presence and origin queries; replay reconstructs value changes. +// Replay reconciles with canonical decoding before domain resolution so both +// contracts agree on the effective result without retaining a tree per field. +type historyApplication struct { + layer Layer + path Path + value *document.Node + origin Origin +} + +func cloneHistoryApplications(values []historyApplication) []historyApplication { + if values == nil { + return nil + } + cloned := make([]historyApplication, len(values)) + for index, value := range values { + cloned[index] = historyApplication{ + layer: value.layer, path: value.path.clone(), value: value.value, + origin: cloneOrigin(value.origin), + } + } + return cloned +} + +func applicationOperationPath(application historyApplication, query Path) (Path, bool) { + if !application.path.valid() || application.value == nil { + return Path{}, false + } + if application.path.String() != "" { + if pathsIntersect(application.path, query) { + return application.path.clone(), true + } + return Path{}, false + } + if len(query.segments) == 0 { + return Path{}, true + } + for length := 1; length <= len(query.segments); length++ { + prefix := Path{segments: clonePathSegments(query.segments[:length], 0)} + node, present := documentAtPath(application.value, prefix) + if !present { + return Path{}, false + } + if length == len(query.segments) || node.Kind != document.Mapping { + return prefix, true + } + } + return Path{}, false +} + +func historyApplicationOrigin(application historyApplication, operationPath Path) Origin { + if application.origin.Layer != "" { + return cloneOrigin(application.origin) + } + origin := Origin{Layer: application.layer} + relative, ok := relativeHistoryPath(application.path, operationPath) + if !ok { + return origin + } + if node, present := documentAtPath(application.value, relative); present { + origin.Locations = []Location{publicLocation(node.Location)} + } + return origin +} + +func relativeHistoryPath(root, path Path) (Path, bool) { + if !pathContains(root, path) { + return Path{}, false + } + return Path{segments: clonePathSegments(path.segments[len(root.segments):], 0)}, true +} + +func pathsIntersect(left, right Path) bool { + return pathContains(left, right) || pathContains(right, left) +} + +func applyHistoryApplication(root **document.Node, application historyApplication) { + if application.path.String() == "" { + mergeHistoryDocument(root, application.value) + return + } + mergeHistoryAtPath(root, application.path, application.value) +} + +func mergeHistoryDocument(target **document.Node, incoming *document.Node) { + if incoming == nil { + return + } + if *target == nil || (*target).Kind != document.Mapping || incoming.Kind != document.Mapping { + *target = document.Clone(incoming) + return + } + for _, entry := range incoming.Mapping { + position := documentEntryIndex((*target).Mapping, entry.Key) + if position < 0 { + cloned := entry + cloned.Value = document.Clone(entry.Value) + (*target).Mapping = append((*target).Mapping, cloned) + continue + } + mergeHistoryDocument(&(*target).Mapping[position].Value, entry.Value) + } +} + +func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node) { + if root == nil || *root == nil || len(path.segments) == 0 { + mergeHistoryDocument(root, incoming) + return + } + current := *root + for index, segment := range path.segments { + if current.Kind != document.Mapping || segment.kind == pathSegmentIndex { + return + } + position := documentEntryIndex(current.Mapping, segment.key) + if index == len(path.segments)-1 { + if position < 0 { + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, Value: document.Clone(incoming)}) + } else { + mergeHistoryDocument(¤t.Mapping[position].Value, incoming) + } + return + } + if position < 0 { + child := &document.Node{Kind: document.Mapping} + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, Value: child}) + current = child + } else { + current = current.Mapping[position].Value + } + } +} + +// Resolution records describe one before/final transformation, not assignment +// order. A parent already carries its descendants' final values; merge overlapping +// intent into that event rather than replay a child assignment a second time. +func coalescedResolutionSnapshots(snapshots []resolutionSnapshot) []resolutionSnapshot { + groups := make([]resolutionSnapshot, 0, len(snapshots)) + for _, snapshot := range snapshots { + merged := false + for i := range groups { + if !pathContains(groups[i].change.Path, snapshot.change.Path) { + continue + } + for _, reason := range snapshot.change.Reasons { + groups[i].change.Reasons = appendUniqueString(groups[i].change.Reasons, reason) + } + for _, dependency := range snapshot.change.Dependencies { + groups[i].change.Dependencies = appendUniquePath(groups[i].change.Dependencies, dependency) + } + if snapshot.change.Origin.Operation == ChangeNormalization { + groups[i].change.Origin.Operation = ChangeNormalization + } + merged = true + break + } + if !merged { + snapshot.change = cloneChange(snapshot.change) + groups = append(groups, snapshot) + } + } + return groups +} + +func replaceHistoryPath(root **document.Node, path Path, replacement *document.Node) { + if root == nil || *root == nil { + return + } + if len(path.segments) == 0 { + *root = document.Clone(replacement) + return + } + current := *root + for index, segment := range path.segments { + last := index == len(path.segments)-1 + switch current.Kind { + case document.Mapping: + if segment.kind == pathSegmentIndex { + return + } + position := documentEntryIndex(current.Mapping, segment.key) + if position < 0 { + if replacement == nil { + return + } + child := &document.Node{Kind: document.Mapping} + if last { + child = document.Clone(replacement) + } + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, Value: child}) + if last { + return + } + current = child + continue + } + if last { + if replacement == nil { + current.Mapping = append(current.Mapping[:position], current.Mapping[position+1:]...) + } else { + current.Mapping[position].Value = document.Clone(replacement) + } + return + } + current = current.Mapping[position].Value + case document.Sequence: + position, ok := schemaIndex(segment) + if !ok || position >= len(current.Sequence) { + return + } + if last { + if replacement != nil { + current.Sequence[position] = document.Clone(replacement) + } + return + } + current = current.Sequence[position] + default: + return + } + } +} + +func historyNodesWithinLimit(baseline, input, final *document.Node, applications []historyApplication, resolutions []resolutionSnapshot) bool { + return historyNodesWithin(historyNodeLimit, baseline, input, final, applications, resolutions) +} + +func historyNodesWithin(limit int, baseline, input, final *document.Node, applications []historyApplication, resolutions []resolutionSnapshot) bool { + seen := make(map[*document.Node]bool) + count := 0 + var visit func(*document.Node) bool + visit = func(node *document.Node) bool { + if node == nil || seen[node] { + return true + } + seen[node] = true + count++ + if count > limit { + return false + } + for _, entry := range node.Mapping { + if !visit(entry.Value) { + return false + } + } + for _, child := range node.Sequence { + if !visit(child) { + return false + } + } + return true + } + if !visit(baseline) || !visit(input) || !visit(final) { + return false + } + for _, application := range applications { + if !visit(application.value) { + return false + } + } + for _, resolution := range resolutions { + if !visit(resolution.before) || !visit(resolution.after) { + return false + } + } + return true +} diff --git a/internal/engine/imports.go b/internal/engine/imports.go new file mode 100644 index 0000000..28d2428 --- /dev/null +++ b/internal/engine/imports.go @@ -0,0 +1,324 @@ +package engine + +import ( + "bytes" + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "strconv" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +const ( + maxImportDepth = 64 + maxImportSources = 1024 + maxGlobEntries = 1_000_000 +) + +type importLimits struct { + depth int + sources int + bytes int64 + nodes int + entries int +} + +func defaultImportLimits() importLimits { + return importLimits{ + depth: maxImportDepth, sources: maxImportSources, + bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries, + } +} + +type importRoute struct { + canonical string + locations []Location +} + +type importTraversal struct { + rootPath string + root *os.Root + limits importLimits + rule *VersionRule + + documents []*document.Node + sources []SourceInfo + active map[string]int + visited map[string]importRoute + stack []importRoute + usedBytes int64 + usedNodes int + usedEntries int +} + +func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { + rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) + if diagnostic != nil { + return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} + } + root, err := os.OpenRoot(rootPath) + if err != nil { + diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) + diagnostic.Primary = nil + return nil, sourceInfos(main.source), []Diagnostic{diagnostic} + } + + mainInfo := cloneSourceInfo(*main.source) + if mainPath != "" { + mainInfo.Path = mainPath + } + state := importTraversal{ + rootPath: rootPath, root: root, limits: limits, rule: rule, + documents: make([]*document.Node, 0, 1), + sources: []SourceInfo{mainInfo}, + active: make(map[string]int), visited: make(map[string]importRoute), + usedBytes: main.bytes, usedNodes: countInputNodes(main.document), + } + if diagnostic := state.checkInitialLimits(); diagnostic != nil { + _ = root.Close() + return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} + } + + mainIdentity := mainPath + if mainIdentity == "" { + mainIdentity = "" + } + mainRoute := importRoute{canonical: mainIdentity} + state.active[mainIdentity] = 0 + state.visited[mainIdentity] = mainRoute + state.stack = append(state.stack, mainRoute) + body, declarations, diagnostics := splitImportDirectives(main.document) + if len(diagnostics) == 0 { + state.documents = append(state.documents, body) + diagnostics = state.walkDeclarations(basePath, declarations, 0) + } + + closeErr := root.Close() + if len(diagnostics) != 0 { + return nil, cloneSourceInfos(state.sources), diagnostics + } + if closeErr != nil { + return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} + } + composed, diagnostics := composeDocuments(state.documents...) + if composed != nil { + composed.inputBytes = state.usedBytes + composed.inputNodes = state.usedNodes + } + return composed, cloneSourceInfos(state.sources), diagnostics +} + +func (s *importTraversal) checkInitialLimits() *Diagnostic { + if s.limits.sources < 1 { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) + return &value + } + if s.usedBytes > s.limits.bytes { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) + return &value + } + if s.usedNodes > s.limits.nodes { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) + return &value + } + return nil +} + +func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int) []Diagnostic { + for _, declaration := range declarations { + if declaration.glob != "" { + matches, diagnostics := s.expandImportGlob(basePath, declaration) + if len(diagnostics) != 0 { + return diagnostics + } + if len(matches) == 0 { + if declaration.optional { + continue + } + location := declaration.location + return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)} + } + for _, match := range matches { + matched := declaration + matched.path = match + matched.glob = "" + if diagnostics := s.visitPath(basePath, matched, depth+1); len(diagnostics) != 0 { + return diagnostics + } + } + continue + } + if diagnostics := s.visitPath(basePath, declaration, depth+1); len(diagnostics) != 0 { + return diagnostics + } + } + return nil +} + +func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, depth int) []Diagnostic { + location := declaration.location + if depth > s.limits.depth { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} + } + if !validImportPath(declaration.path) { + return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)} + } + + candidate := filepath.Join(basePath, filepath.FromSlash(path.Clean(declaration.path))) + if !containedPath(s.rootPath, filepath.Clean(candidate)) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} + } + canonical, err := canonicalExistingPath(candidate) + if err != nil { + if declaration.optional && errors.Is(err, fs.ErrNotExist) { + return nil + } + code := CodeImportInvalid + message := "import target could not be canonicalized" + if errors.Is(err, fs.ErrNotExist) { + code, message = CodeImportMissing, "required import target does not exist" + } else if errors.Is(err, fs.ErrPermission) { + message = "import target cannot be accessed" + } + return []Diagnostic{importDiagnostic(code, message, &location)} + } + if !containedPath(s.rootPath, canonical) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} + } + + currentRoute := s.currentRoute(location) + if activeIndex, ok := s.active[canonical]; ok { + cycle := append([]importRoute(nil), s.stack[activeIndex:]...) + return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} + } + if previous, ok := s.visited[canonical]; ok { + return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} + } + if len(s.sources)+1 > s.limits.sources { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} + } + + relative, err := filepath.Rel(s.rootPath, canonical) + if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} + } + sourceID := "source-" + strconv.Itoa(len(s.sources)+1) + info := SourceInfo{ID: sourceID, Label: declaration.path, Path: canonical, Layer: LayerBase, ImportedFrom: &location} + s.sources = append(s.sources, info) + data, skipped, diagnostic := s.readImport(relative, sourceID, declaration.optional) + if diagnostic != nil { + return []Diagnostic{*diagnostic} + } + if skipped { + s.sources = s.sources[:len(s.sources)-1] + return nil + } + + root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ + Depth: maxDocumentDepth, Nodes: maxDocumentNodes, + }) + if len(issues) != 0 { + return parserDiagnostics(sourceID, issues) + } + nodeCount := countInputNodes(root) + if s.usedNodes+nodeCount > s.limits.nodes { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} + } + s.usedNodes += nodeCount + + body, declarations, diagnostics := splitImportDirectives(root) + if len(diagnostics) != 0 { + return diagnostics + } + if versionNode := fragmentVersionClaim(body, s.rule); versionNode != nil { + versionLocation := publicLocation(versionNode.Location) + return []Diagnostic{{ + Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(), + Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation, + }} + } + + route := importRoute{canonical: canonical, locations: currentRoute.locations} + s.documents = append(s.documents, body) + s.active[canonical] = len(s.stack) + s.visited[canonical] = route + s.stack = append(s.stack, route) + diagnostics = s.walkDeclarations(filepath.Dir(canonical), declarations, depth) + delete(s.active, canonical) + s.stack = s.stack[:len(s.stack)-1] + return diagnostics +} + +func (s *importTraversal) readImport(relative, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { + remaining := s.limits.bytes - s.usedBytes + limit := min(int64(maxSourceBytes), remaining) + if limit < 0 { + limit = 0 + } + data, failure := readRegularFile(relative, s.root.Stat, s.root.Open, limit) + if failure != nil { + if optional && failure.missingTarget { + return nil, true, nil + } + value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") + return nil, false, &value + } + s.usedBytes += int64(len(data)) + return data, false, nil +} + +func (s *importTraversal) currentRoute(location Location) importRoute { + locations := make([]Location, 0, len(s.stack)+1) + for _, route := range s.stack { + if len(route.locations) != 0 { + locations = append(locations, route.locations[len(route.locations)-1]) + } + } + locations = append(locations, location) + return importRoute{locations: locations} +} + +func sourceInfos(source *SourceInfo) []SourceInfo { + if source == nil { + return nil + } + return []SourceInfo{cloneSourceInfo(*source)} +} + +func importDiagnostic(code Code, message string, location *Location) Diagnostic { + return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)} +} + +func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic { + paths := make([]string, 0, len(cycle)+1) + related := make([]Location, 0, len(cycle)) + for _, route := range cycle { + paths = append(paths, route.canonical) + if len(route.locations) != 0 { + related = append(related, route.locations[len(route.locations)-1]) + } + } + paths = append(paths, target) + return Diagnostic{ + Code: CodeImportCycle, Severity: SeverityError, + Message: "import cycle detected: " + strings.Join(paths, " -> "), + Primary: &location, Related: related, + } +} + +func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic { + related := append([]Location(nil), previous.locations...) + if len(current.locations) > 1 { + related = append(related, current.locations[:len(current.locations)-1]...) + } + return Diagnostic{ + Code: CodeImportRepeated, Severity: SeverityError, + Message: "configuration source is included by more than one import route", + Primary: &location, Related: related, + } +} diff --git a/internal/engine/imports_test.go b/internal/engine/imports_test.go new file mode 100644 index 0000000..973c24e --- /dev/null +++ b/internal/engine/imports_test.go @@ -0,0 +1,476 @@ +package engine + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "testing" +) + +type importTestConfig struct { + Settings struct { + Root string `yaml:"root"` + First string `yaml:"first"` + Nested string `yaml:"nested"` + Second string `yaml:"second"` + } `yaml:"settings"` + Version string `yaml:"version"` + Items map[string]string `yaml:"items"` +} + +func TestLoadFormerDirectiveKeyIsApplicationData(t *testing.T) { + type config struct { + Legacy struct { + Imports []struct { + Path string `yaml:"path"` + } `yaml:"imports"` + } `yaml:"_configkit"` + } + for _, imports := range []bool{false, true} { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit:\n imports:\n - path: nonexistent.yml\n") + result, err := Load[config](File(mainPath), Options[config]{Imports: imports}) + if err != nil { + t.Fatalf("Imports=%v: %v", imports, err) + } + if len(result.Config.Legacy.Imports) != 1 || result.Config.Legacy.Imports[0].Path != "nonexistent.yml" || len(result.Sources()) != 1 { + t.Fatalf("Imports=%v: former directive was not retained as application data: %#v", imports, result) + } + } +} + +func TestLoadTraversesPathImportsDepthFirstAndComposesOrigins(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: first.yml + - path: second.yml +settings: + root: root +`) + firstPath := writeImportFile(t, directory, "first.yml", `configkit: + imports: + - path: nested/third.yml +settings: + first: first +`) + thirdPath := writeImportFile(t, directory, "nested/third.yml", "settings:\n nested: nested\n") + secondPath := writeImportFile(t, directory, "second.yml", "settings:\n second: second\n") + + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil { + t.Fatal(err) + } + if got := result.Config.Settings; got.Root != "root" || got.First != "first" || got.Nested != "nested" || got.Second != "second" { + t.Fatalf("composed settings = %#v", got) + } + sources := result.Sources() + wantPaths := []string{canonicalTestPath(t, mainPath), canonicalTestPath(t, firstPath), canonicalTestPath(t, thirdPath), canonicalTestPath(t, secondPath)} + gotPaths := make([]string, len(sources)) + for index, source := range sources { + gotPaths[index] = source.Path + if source.ID != "source-"+strconv.Itoa(index+1) { + t.Errorf("source %d ID = %q", index, source.ID) + } + if index > 0 && source.ImportedFrom == nil { + t.Errorf("source %d has no import location", index) + } + } + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("source paths = %#v, want depth-first %#v", gotPaths, wantPaths) + } + info, ok := result.Metadata().Lookup(Keys("settings")) + if !ok || !info.Explicit || len(info.Origin.Locations) != 4 { + t.Fatalf("settings metadata = %#v, %v", info, ok) + } +} + +func TestLoadRejectsImportConflictsAndFragmentVersions(t *testing.T) { + t.Run("all duplicate claimants", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: one.yml + - path: two.yml + - path: three.yml +`) + for index, name := range []string{"one.yml", "two.yml", "three.yml"} { + writeImportFile(t, directory, name, "settings:\n root: "+string(rune('a'+index))+"\n") + } + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeComposeConflict) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Path.String() != "/settings/root" || diagnostic.Primary == nil || len(diagnostic.Related) != 2 { + t.Fatalf("conflict diagnostic = %#v", diagnostic) + } + }) + + versionRule := &VersionRule{Path: Keys("settings", "version"), Check: func(Version, *Validation) error { return nil }} + for _, test := range []struct { + name string + fragment string + }{ + {name: "exact integer", fragment: "settings:\n version: 4\n"}, + {name: "ancestor scalar", fragment: "settings: blocked\n"}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: fragment.yml\n") + writeImportFile(t, directory, "fragment.yml", test.fragment) + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true, Definition: Definition[importTestConfig]{Version: versionRule}}, CodeImportVersion) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Path.String() != "/settings/version" || diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" { + t.Fatalf("version placement diagnostic = %#v", diagnostic) + } + }) + } +} + +func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) { + tests := []struct { + name string + body string + code Code + }{ + {name: "directive kind", body: "configkit: []\n", code: CodeImportInvalid}, + {name: "unknown directive field", body: "configkit: {unknown: true}\n", code: CodeImportInvalid}, + {name: "imports kind", body: "configkit: {imports: {}}\n", code: CodeImportInvalid}, + {name: "declaration kind", body: "configkit: {imports: [file.yml]}\n", code: CodeImportInvalid}, + {name: "missing target", body: "configkit: {imports: [{optional: true}]}\n", code: CodeImportInvalid}, + {name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid}, + {name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid}, + {name: "optional kind", body: "configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid}, + {name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", test.body) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) + }) + } +} + +func TestLoadImportMissingContainmentAndRegularFileRules(t *testing.T) { + t.Run("optional missing", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: missing.yml + optional: true +settings: {root: retained} +`) + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil || result.Config.Settings.Root != "retained" || len(result.Sources()) != 1 { + t.Fatalf("optional result = %#v, %v", result, err) + } + }) + + tests := []struct { + name string + path string + setup func(*testing.T, string) + code Code + }{ + {name: "required missing", path: "missing.yml", code: CodeImportMissing}, + {name: "parent traversal", path: "../outside.yml", setup: func(t *testing.T, directory string) { + writeImportFile(t, filepath.Dir(directory), "outside.yml", "settings: {root: outside}\n") + }, code: CodeImportOutsideRoot}, + {name: "directory", path: "directory", setup: func(t *testing.T, directory string) { + if err := os.Mkdir(filepath.Join(directory, "directory"), 0o755); err != nil { + t.Fatal(err) + } + }, code: CodeSourceNotRegular}, + {name: "absolute", path: "/tmp/configkit.yml", code: CodeImportInvalid}, + {name: "network URL", path: "https://example.invalid/config.yml", code: CodeImportInvalid}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + if test.setup != nil { + test.setup(t, directory) + } + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: \""+test.path+"\"\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) + }) + } +} + +func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) { + t.Run("cycle", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports: [{path: a.yml}]\n") + writeImportFile(t, directory, "a.yml", "configkit:\n imports: [{path: b.yml}]\n") + writeImportFile(t, directory, "b.yml", "configkit:\n imports: [{path: main.yml}]\n") + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportCycle) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Primary == nil || len(diagnostic.Related) != 2 || !strings.Contains(diagnostic.Message, "main.yml") || !strings.Contains(diagnostic.Message, "a.yml") || !strings.Contains(diagnostic.Message, "b.yml") { + t.Fatalf("cycle diagnostic = %#v", diagnostic) + } + }) + + t.Run("diamond", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports: [{path: left.yml}, {path: right.yml}]\n") + writeImportFile(t, directory, "left.yml", "configkit:\n imports: [{path: shared.yml}]\n") + writeImportFile(t, directory, "right.yml", "configkit:\n imports: [{path: shared.yml}]\n") + writeImportFile(t, directory, "shared.yml", "settings: {root: shared}\n") + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportRepeated) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Primary == nil || len(diagnostic.Related) < 3 { + t.Fatalf("repeated-route diagnostic = %#v", diagnostic) + } + }) + + t.Run("symlink aliases", func(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("initial filesystem contract covers Linux and macOS") + } + directory := t.TempDir() + writeImportFile(t, directory, "target.yml", "settings: {root: target}\n") + if err := os.Symlink("target.yml", filepath.Join(directory, "first-link.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + if err := os.Symlink("target.yml", filepath.Join(directory, "second-link.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: first-link.yml}, {path: second-link.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportRepeated) + }) +} + +func TestLoadImportsCanonicalSymlinksAndKeepsHardLinksDistinct(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("initial filesystem contract covers Linux and macOS") + } + t.Run("contained absolute symlink", func(t *testing.T) { + directory := t.TempDir() + target := writeImportFile(t, directory, "target.yml", "settings: {first: linked}\n") + if err := os.Symlink(target, filepath.Join(directory, "link.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: link.yml}]}\n") + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil || result.Config.Settings.First != "linked" || result.Sources()[1].Path != canonicalTestPath(t, target) { + t.Fatalf("symlink result = %#v, %v", result, err) + } + }) + + t.Run("symlink escape", func(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + target := writeImportFile(t, outside, "outside.yml", "settings: {root: outside}\n") + if err := os.Symlink(target, filepath.Join(root, "escape.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, root, "main.yml", "configkit: {imports: [{path: escape.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) + }) + + t.Run("hard links are path identities", func(t *testing.T) { + directory := t.TempDir() + one := writeImportFile(t, directory, "one.yml", "settings: {first: one}\n") + if err := os.Link(one, filepath.Join(directory, "two.yml")); err != nil { + t.Skipf("hard links unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: one.yml}, {path: two.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeComposeConflict) + }) + + t.Run("canonicalization failure", func(t *testing.T) { + directory := t.TempDir() + if err := os.Symlink("loop.yml", filepath.Join(directory, "loop.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: loop.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportInvalid) + }) +} + +func TestLoadByteAndReaderImportsRequireExplicitFilesystemContext(t *testing.T) { + directory := t.TempDir() + writeImportFile(t, directory, "fragment.yml", "settings: {first: imported}\n") + input := []byte("configkit: {imports: [{path: fragment.yml}]}\n") + context := &FilesystemContext{BaseDir: directory, Root: directory} + for _, test := range []struct { + name string + source func() Source + }{ + {name: "bytes URI label", source: func() Source { return Bytes("s3://bucket/main.yml", input) }}, + {name: "reader URI label", source: func() Source { return Reader("https://example.invalid/main.yml", strings.NewReader(string(input))) }}, + } { + t.Run(test.name, func(t *testing.T) { + result, err := Load[importTestConfig](test.source(), Options[importTestConfig]{Imports: true, Filesystem: context}) + if err != nil || result.Config.Settings.First != "imported" { + t.Fatalf("explicit filesystem result = %#v, %v", result, err) + } + }) + } + requireLoadCode(t, Bytes("memory.yml", input), Options[importTestConfig]{Imports: true}, CodeOptionsInvalid) + requireLoadCode(t, Bytes("memory.yml", input), Options[importTestConfig]{Imports: true, Filesystem: &FilesystemContext{BaseDir: "relative", Root: directory}}, CodeOptionsInvalid) + requireLoadCode(t, Bytes("memory.yml", input), Options[importTestConfig]{Imports: true, Filesystem: &FilesystemContext{BaseDir: directory, Root: t.TempDir()}}, CodeImportOutsideRoot) +} + +func TestFileImportRootCanContainTheSelectedFileDirectory(t *testing.T) { + root := t.TempDir() + mainPath := writeImportFile(t, root, "configs/main.yml", "configkit: {imports: [{path: ../shared.yml}]}\n") + writeImportFile(t, root, "shared.yml", "settings: {root: shared}\n") + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{ + Imports: true, Filesystem: &FilesystemContext{Root: root}, + }) + if err != nil || result.Config.Settings.Root != "shared" { + t.Fatalf("explicit-root result = %#v, %v", result, err) + } + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true, Filesystem: &FilesystemContext{BaseDir: filepath.Dir(mainPath), Root: root}}, CodeOptionsInvalid) +} + +func TestImportedParseAndDecodeDiagnosticsKeepTheirSource(t *testing.T) { + t.Run("parse", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: broken.yml}]}\n") + writeImportFile(t, directory, "broken.yml", "settings: [\n") + loadError := requireAnyLoadError(t, File(mainPath), Options[importTestConfig]{Imports: true}) + diagnostics := loadError.Diagnostics() + if diagnostics[0].Code != CodeYAMLInvalid || diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != "source-2" || len(loadError.Sources()) != 2 { + t.Fatalf("parse failure = diagnostics %#v, sources %#v", diagnostics, loadError.Sources()) + } + }) + + t.Run("decode", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: unknown.yml}]}\n") + writeImportFile(t, directory, "unknown.yml", "unknown: value\n") + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeDecodeUnknownField) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" { + t.Fatalf("decode diagnostic = %#v", diagnostic) + } + }) +} + +func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "outside.yml", "configkit: {imports: [{path: ../missing.yml, optional: true}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) + + locked := writeImportFile(t, directory, "locked.yml", "settings: {root: locked}\n") + if err := os.Chmod(locked, 0); err != nil { + t.Skipf("cannot create unreadable fixture: %v", err) + } + defer func() { _ = os.Chmod(locked, 0o600) }() + if file, err := os.Open(locked); err == nil { + _ = file.Close() + t.Skip("filesystem does not enforce the unreadable-file fixture") + } + mainPath = writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: locked.yml, optional: true}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourcePermission) +} + +func TestImportTraversalEnforcesPrivateBudgets(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: one.yml}, {path: two.yml}]}\n") + writeImportFile(t, directory, "one.yml", "settings: {first: one}\n") + writeImportFile(t, directory, "two.yml", "settings: {second: two}\n") + main := parseImportMain(t, mainPath) + for _, test := range []struct { + name string + limits importLimits + }{ + {name: "source count", limits: importLimits{depth: 8, sources: 2, bytes: maxTotalBytes, nodes: maxDocumentNodes}}, + {name: "aggregate bytes", limits: importLimits{depth: 8, sources: 8, bytes: main.bytes + 1, nodes: maxDocumentNodes}}, + {name: "aggregate nodes", limits: importLimits{depth: 8, sources: 8, bytes: maxTotalBytes, nodes: countInputNodes(main.document) + 1}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, test.limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("budget diagnostics = %#v", diagnostics) + } + }) + } + + writeImportFile(t, directory, "one.yml", "configkit: {imports: [{path: nested.yml}]}\n") + writeImportFile(t, directory, "nested.yml", "settings: {nested: nested}\n") + main = parseImportMain(t, mainPath) + _, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, importLimits{depth: 1, sources: 8, bytes: maxTotalBytes, nodes: maxDocumentNodes}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("depth diagnostics = %#v", diagnostics) + } +} + +func TestImportTraversalAcceptsExactPrivateBudgets(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: imported.yml}]}\n") + importedPath := writeImportFile(t, directory, "imported.yml", "settings: {first: imported}\n") + main := parseImportMain(t, mainPath) + imported := parseImportMain(t, importedPath) + data, err := os.ReadFile(importedPath) + if err != nil { + t.Fatal(err) + } + limits := importLimits{ + depth: 1, + sources: 2, + bytes: main.bytes + int64(len(data)), + nodes: countInputNodes(main.document) + countInputNodes(imported.document), + } + composed, sources, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, limits) + if len(diagnostics) != 0 || composed == nil || len(sources) != 2 { + t.Fatalf("exact-limit composition = (%#v, %#v, %#v)", composed, sources, diagnostics) + } +} + +func writeImportFile(t *testing.T, root, relative, contents string) string { + t.Helper() + filename := filepath.Join(root, filepath.FromSlash(relative)) + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + return filename +} + +func canonicalTestPath(t *testing.T, filename string) string { + t.Helper() + canonical, err := canonicalExistingPath(filename) + if err != nil { + t.Fatal(err) + } + return canonical +} + +func requireLoadCode(t *testing.T, source Source, options Options[importTestConfig], code Code) *LoadError { + t.Helper() + loadError := requireAnyLoadError(t, source, options) + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 0 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v, want %s", diagnostics, code) + } + return loadError +} + +func requireAnyLoadError(t *testing.T, source Source, options Options[importTestConfig]) *LoadError { + t.Helper() + result, err := Load[importTestConfig](source, options) + if result != nil || err == nil { + t.Fatalf("result, error = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + return loadError +} + +func parseImportMain(t *testing.T, filename string) parsedSource { + t.Helper() + parsed, diagnostics := parseSource(File(filename), "source-1", defaultParseLimits()) + if len(diagnostics) != 0 { + t.Fatalf("parse diagnostics = %#v", diagnostics) + } + return parsed +} diff --git a/internal/engine/independence_test.go b/internal/engine/independence_test.go new file mode 100644 index 0000000..5da002c --- /dev/null +++ b/internal/engine/independence_test.go @@ -0,0 +1,116 @@ +package engine + +import ( + "bytes" + "errors" + "fmt" + "strings" + "sync" + "testing" +) + +type independentText string + +func (value independentText) MarshalText() ([]byte, error) { + return []byte(value), nil +} + +func (value *independentText) UnmarshalText(text []byte) error { + *value = independentText(text) + return nil +} + +type independentConfig struct { + Labels map[string]string `yaml:"labels"` + Items []string `yaml:"items"` + Custom independentText `yaml:"custom"` +} + +func TestParallelLoadsShareRulesButNotMutableValues(t *testing.T) { + definition := Definition[independentConfig]{ + Defaults: func() independentConfig { + return independentConfig{ + Labels: map[string]string{"default": "kept"}, + Items: []string{"default"}, + Custom: "default", + } + }, + } + + const count = 32 + results := make([]*Result[independentConfig], count) + errorsByIndex := make([]error, count) + var group sync.WaitGroup + for index := range count { + group.Add(1) + go func(index int) { + defer group.Done() + input := fmt.Sprintf("custom: value-%d\n", index) + results[index], errorsByIndex[index] = Load(Bytes("parallel.yml", []byte(input)), Options[independentConfig]{Definition: definition}) + }(index) + } + group.Wait() + + for index, err := range errorsByIndex { + if err != nil { + t.Fatalf("Load(%d) failed: %v", index, err) + } + } + results[0].Config.Labels["default"] = "mutated" + results[0].Config.Items[0] = "mutated" + results[0].Config.Custom = "mutated" + for index := 1; index < count; index++ { + if results[index].Config.Labels["default"] != "kept" || results[index].Config.Items[0] != "default" || results[index].Config.Custom != independentText(fmt.Sprintf("value-%d", index)) { + t.Fatalf("result %d shares mutable state: %#v", index, results[index].Config) + } + } + var effective bytes.Buffer + if err := results[0].WriteEffective(&effective); err != nil { + t.Fatal(err) + } + if strings.Contains(effective.String(), "mutated") || !strings.Contains(effective.String(), "default: kept") || !strings.Contains(effective.String(), "value-0") { + t.Fatalf("frozen effective snapshot = %q", effective.String()) + } +} + +func TestConcurrentLoadsOfCopiedReaderSourceHaveOneClaim(t *testing.T) { + source := Reader("shared", strings.NewReader("value: one\n")) + copied := source + type config struct { + Value string `yaml:"value"` + } + + start := make(chan struct{}) + errorsFound := make(chan error, 2) + var group sync.WaitGroup + for _, candidate := range []Source{source, copied} { + group.Add(1) + go func(candidate Source) { + defer group.Done() + <-start + _, err := Load[config](candidate) + errorsFound <- err + }(candidate) + } + close(start) + group.Wait() + close(errorsFound) + + successes, used := 0, 0 + for err := range errorsFound { + if err == nil { + successes++ + continue + } + var loadError *LoadError + if errors.As(err, &loadError) { + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 1 && diagnostics[0].Code == CodeSourceReaderUsed { + used++ + } + } + } + if successes != 1 || used != 1 { + t.Fatalf("successes=%d reader-used=%d", successes, used) + } +} diff --git a/internal/engine/load.go b/internal/engine/load.go new file mode 100644 index 0000000..545254b --- /dev/null +++ b/internal/engine/load.go @@ -0,0 +1,185 @@ +package engine + +import ( + "os" + "reflect" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// Definition contains reusable application rules for a configuration type. +// Secret is a pure predicate that can add sensitivity to semantic paths. +// Definitions should be treated as read-only after being shared. +type Definition[T any] struct { + Defaults func() T + Resolve func(*T, *Resolution) error + Validate func(*T, *Validation) + Version *VersionRule + Secret func(Path) bool +} + +// Options contains the application definition and per-load input selections. +// Imports enables explicit path and glob imports; Profile selects one fixed profile +// overlay; Overrides contains only values explicitly supplied by an application CLI. +type Options[T any] struct { + Definition Definition[T] + Imports bool + Profile string + Filesystem *FilesystemContext + Overrides []Override +} + +// FilesystemContext supplies filesystem roots for imports and profiles. +type FilesystemContext struct { + BaseDir string + Root string +} + +// Override describes one explicitly supplied application override. Value uses +// parser-neutral built-in scalars, nil, []any, or map[string]any; Flag is a safe +// display name and must not contain a raw argument or value. +type Override struct { + Path Path + Value any + Flag string +} + +// Resolution exposes entry-state input provenance, surviving-input presence, and +// audited application resolution changes. A resolver must not retain or +// concurrently use the context. +type Resolution struct { + state *resolutionState +} + +// VersionRule declares the input path and application checker for a schema version. +// At that path, string and decimal-integer input is supplied as Version.Text. +type VersionRule struct { + Path Path + Check func(Version, *Validation) error +} + +// Version preserves the absent, null, and textual states of a schema-version input. +// Text is the exact spelling of accepted string or decimal-integer input. +type Version struct { + Present bool + Null bool + Text string +} + +// Result is a successful independently owned configuration load. +type Result[T any] struct { + Config T + + metadata Metadata + sources []SourceInfo + diagnostics []Diagnostic + baseline *document.Node + input *document.Node + inputOrigins map[string]Origin + final *document.Node + resolution []resolutionSnapshot + applications []historyApplication + secrets map[string]bool + secretReferences map[string]string +} + +// Metadata returns detached effective presence, origin, and history records. +func (r *Result[T]) Metadata() Metadata { + if r == nil { + return Metadata{} + } + return cloneMetadata(r.metadata) +} + +// Sources returns a detached list of sources used by the load. +func (r *Result[T]) Sources() []SourceInfo { + if r == nil { + return nil + } + return cloneSourceInfos(r.sources) +} + +// Diagnostics returns detached nonfatal diagnostics from the load. +func (r *Result[T]) Diagnostics() []Diagnostic { + if r == nil { + return nil + } + return cloneDiagnostics(r.diagnostics) +} + +// Load parses and strictly decodes zero or one selected source, explicitly enabled +// imports, and one selected profile. It accepts at most one Options value and +// returns no result when any error is reported. +func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { + return loadWithSecretLookup(source, os.LookupEnv, supplied...) +} + +func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied ...Options[T]) (*Result[T], error) { + accumulator := diagnosticAccumulator{} + options, diagnostics := selectOptions(source, supplied) + accumulator.add(diagnostics...) + if len(diagnostics) != 0 { + return nil, accumulator.loadError() + } + + input, diagnostics := assembleBaseInput(source, options) + accumulator.sources = input.sources + accumulator.add(diagnostics...) + if len(diagnostics) != 0 { + return nil, accumulator.loadError() + } + schema, schemaIssue := buildSchema(reflect.TypeFor[T]()) + if schemaIssue != nil { + accumulator.add(diagnosticFromSchemaIssue(schemaIssue)) + return nil, accumulator.loadError() + } + versionDiagnostics := checkVersion(options.Definition.Version, input.document) + accumulator.add(versionDiagnostics...) + if hasErrors(versionDiagnostics) { + return nil, accumulator.loadError() + } + input, diagnostics = applyInputLayers(source, options, input) + accumulator.sources = input.sources + accumulator.add(diagnostics...) + if len(diagnostics) != 0 { + return nil, accumulator.loadError() + } + + decoded, diagnostics := decodeLoadInput(schema, options.Definition, input) + accumulator.add(diagnostics...) + if len(diagnostics) != 0 { + return nil, accumulator.loadError() + } + resolved, diagnostics := resolveLoadInput(schema, options.Definition, input, decoded) + accumulator.add(diagnostics...) + if hasErrors(diagnostics) { + return nil, accumulator.loadError() + } + if !historyNodesWithinLimit(decoded.baseline, decoded.final, resolved.final, resolved.applications, resolved.snapshots) { + accumulator.add(Diagnostic{Code: CodeExplainLimitExceeded, Severity: SeverityError, Message: "retained configuration history exceeds the node limit"}) + return nil, accumulator.loadError() + } + + // Detach resolver-borrowed values before reference mutation or validation. + configuration, ownershipIssue := detachLoadConfiguration(schema, resolved.configuration, cloneUnresolvedSecrets) + if ownershipIssue != nil { + accumulator.add(*ownershipIssue) + return nil, accumulator.loadError() + } + secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, input.document, lookup) + accumulator.add(secretResolution.diagnostics...) + if hasErrors(secretResolution.diagnostics) { + return nil, accumulator.loadError() + } + if options.Definition.Validate != nil { + diagnostics = runValidation(&configuration, options.Definition.Validate, schema, resolved.final, input.document, secretResolution.lookups) + accumulator.add(diagnostics...) + if hasErrors(diagnostics) { + return nil, accumulator.loadError() + } + } + resolved.configuration = configuration + // Publication detaches equal-valued validator replacements, preserving the + // validated secret state, then finalizes metadata and sensitivity. + return publishLoadResult(schema, options.Definition, resolved, secretResolution.references, &accumulator) +} diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go new file mode 100644 index 0000000..c20cc47 --- /dev/null +++ b/internal/engine/load_input.go @@ -0,0 +1,88 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +// assembledInput is the source/document handoff across schema and version checks. +type assembledInput struct { + parsed parsedSource + document *document.Node + composition *composedDocument + sources []SourceInfo +} + +func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) { + parsed, diagnostics := parseSource(source, "source-1", defaultParseLimits()) + input := assembledInput{parsed: parsed, document: parsed.document} + if len(diagnostics) != 0 { + input.sources = sourceInfos(parsed.source) + return input, diagnostics + } + if options.Imports { + input.composition, input.sources, diagnostics = loadImportedDocuments(source, options.Filesystem, parsed, options.Definition.Version, defaultImportLimits()) + if len(diagnostics) != 0 { + return input, diagnostics + } + input.document = input.composition.root + } else { + input.sources = sourceInfos(parsed.source) + if options.Filesystem != nil && !activeProfile(options.Profile) { + if diagnostic := validateFilesystemContext(source, options.Filesystem, parsed); diagnostic != nil { + return input, []Diagnostic{*diagnostic} + } + } + } + if !options.Imports && input.document != nil { + if diagnostics := disabledDirectiveDiagnostics(input.document, Path{}); len(diagnostics) != 0 { + return input, diagnostics + } + } + return input, nil +} + +func applyInputLayers[T any](source Source, options Options[T], input assembledInput) (assembledInput, []Diagnostic) { + if !activeProfile(options.Profile) && len(options.Overrides) == 0 { + return input, nil + } + composition, diagnostics := compositionForInput(input) + if len(diagnostics) != 0 { + return input, diagnostics + } + if activeProfile(options.Profile) { + composition, input.sources, diagnostics = loadProfileDocument( + source, options.Filesystem, input.parsed, composition, input.sources, + options.Profile, options.Definition.Version, defaultImportLimits(), + ) + if len(diagnostics) != 0 { + return input, diagnostics + } + } + if len(options.Overrides) != 0 { + composition, input.sources, diagnostics = applyOverrides( + composition, input.sources, options.Overrides, options.Definition.Version, defaultImportLimits(), + ) + if len(diagnostics) != 0 { + return input, diagnostics + } + } + input.composition = composition + input.document = composition.root + return input, nil +} + +// compositionForInput initializes one handoff for profiles and CLI, including +// defaults-only CLI input. Raw source bytes and mapping keys retain their budgets. +func compositionForInput(input assembledInput) (*composedDocument, []Diagnostic) { + if input.composition != nil { + return input.composition, nil + } + if input.document == nil { + return &composedDocument{root: &document.Node{Kind: document.Mapping}, origins: make(map[string]Origin)}, nil + } + composition, diagnostics := composeDocuments(input.document) + if len(diagnostics) != 0 { + return nil, diagnostics + } + composition.inputBytes = input.parsed.bytes + composition.inputNodes = countInputNodes(input.document) + return composition, nil +} diff --git a/internal/engine/load_metadata.go b/internal/engine/load_metadata.go new file mode 100644 index 0000000..df2400b --- /dev/null +++ b/internal/engine/load_metadata.go @@ -0,0 +1,330 @@ +package engine + +import ( + "slices" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// resolutionInput is the immutable entry-state provenance available to a +// resolution callback. It owns detached metadata and source records so later +// resolution updates cannot change the entry view. +type resolutionInput struct { + metadata Metadata + sources map[string]SourceInfo +} + +func newResolutionInput(metadata Metadata, final *document.Node, sources []SourceInfo) resolutionInput { + frozen := Metadata{ + fields: make(map[string]FieldInfo, len(metadata.fields)), + effectivePaths: effectivePathIndex(final), + } + for pointer, info := range metadata.fields { + frozen.fields[pointer] = cloneFieldInfo(info) + } + indexed := make(map[string]SourceInfo, len(sources)) + for _, source := range sources { + if source.ID == "" { + panic("configkit: source identity is missing") + } + if _, exists := indexed[source.ID]; exists { + panic("configkit: source identity is duplicated") + } + indexed[source.ID] = cloneSourceInfo(source) + } + return resolutionInput{metadata: frozen, sources: indexed} +} + +func (input resolutionInput) lookup(path Path) (FieldInfo, bool) { + return input.metadata.Lookup(path) +} + +func (input resolutionInput) source(id string) (SourceInfo, bool) { + source, ok := input.sources[id] + if !ok { + return SourceInfo{}, false + } + return cloneSourceInfo(source), true +} + +func metadataFromDecoded(decoded decodedConfiguration) Metadata { + metadata := Metadata{ + fields: make(map[string]FieldInfo), + history: make(map[string][]Change), + } + baseline := make(map[string]*document.Node) + indexDocument(decoded.baseline, Path{}, baseline) + for pointer := range baseline { + path, err := ParsePath(pointer) + if err != nil { + continue + } + metadata.history[pointer] = append(metadata.history[pointer], Change{ + Path: path, Origin: Origin{Layer: LayerDefault}, + }) + } + for pointer := range decoded.presence { + path, err := ParsePath(pointer) + if err != nil { + continue + } + origin := cloneOrigin(decoded.origins[pointer]) + change := Change{Path: path, Origin: origin} + if _, ok := baseline[pointer]; ok { + change.Previous = []Origin{{Layer: LayerDefault}} + } + metadata.history[pointer] = append(metadata.history[pointer], change) + } + collectFieldInfo(decoded.final, Path{}, decoded, metadata.fields) + return metadata +} + +func applyOverlayMetadata(metadata *Metadata, records []overlayRecord) { + baseline := make(map[string][]Change, len(metadata.history)) + for pointer, history := range metadata.history { + for _, change := range history { + if change.Origin.Layer == LayerDefault { + baseline[pointer] = append(baseline[pointer], cloneChange(change)) + } + } + } + events := make(map[string][]Change) + for _, record := range records { + exact := false + for _, displaced := range record.displaced { + pointer := displaced.path.String() + appendInputOriginEvent(events, baseline, pointer, displaced.path, displaced.origin) + appendInputChangeEvent(events, baseline, pointer, record.change) + if pointer == record.change.Path.String() { + exact = true + } + } + if !exact { + pointer := record.change.Path.String() + for _, previous := range record.change.Previous { + appendInputOriginEvent(events, baseline, pointer, record.change.Path, previous) + } + appendInputChangeEvent(events, baseline, pointer, record.change) + } + } + for pointer, info := range metadata.fields { + if !info.Explicit { + continue + } + path, err := ParsePath(pointer) + if err != nil { + continue + } + if history := events[pointer]; len(history) != 0 && history[len(history)-1].Origin.Layer == info.Origin.Layer && history[len(history)-1].Origin.Operation == info.Origin.Operation { + continue + } + appendInputOriginEvent(events, baseline, pointer, path, info.Origin) + } + metadata.history = make(map[string][]Change, len(baseline)+len(events)) + for pointer, history := range baseline { + metadata.history[pointer] = cloneChanges(history) + } + for pointer, history := range events { + metadata.history[pointer] = append(metadata.history[pointer], cloneChanges(history)...) + } +} + +func appendInputOriginEvent(events map[string][]Change, baseline map[string][]Change, pointer string, path Path, origin Origin) { + if origin.Layer == "" { + return + } + history := events[pointer] + if len(history) != 0 && originContainedBy(history[len(history)-1].Origin, origin) { + return + } + appendInputChangeEvent(events, baseline, pointer, Change{Path: path.clone(), Origin: cloneOrigin(origin)}) +} + +func appendInputChangeEvent(events map[string][]Change, baseline map[string][]Change, pointer string, value Change) { + change := cloneChange(value) + history := events[pointer] + if len(history) != 0 { + change.Previous = []Origin{cloneOrigin(history[len(history)-1].Origin)} + } else if defaults := baseline[pointer]; len(defaults) != 0 { + change.Previous = []Origin{cloneOrigin(defaults[len(defaults)-1].Origin)} + } else { + change.Previous = nil + } + events[pointer] = append(history, change) +} + +func originContainedBy(candidate, container Origin) bool { + if candidate.Layer != container.Layer || candidate.Operation != container.Operation { + return false + } + for _, location := range candidate.Locations { + found := false + for _, available := range container.Locations { + if location == available { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func cloneChanges(changes []Change) []Change { + cloned := make([]Change, len(changes)) + for index, change := range changes { + cloned[index] = cloneChange(change) + } + return cloned +} + +func collectFieldInfo(node *document.Node, path Path, decoded decodedConfiguration, fields map[string]FieldInfo) { + pointer := path.String() + _, explicit := decoded.presence[pointer] + fields[pointer] = FieldInfo{ + Explicit: explicit, + Null: node.Kind == document.Null, + Origin: cloneOrigin(decoded.origins[pointer]), + } + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectFieldInfo(entry.Value, path.Key(entry.Key), decoded, fields) + } + case document.Sequence: + for index, child := range node.Sequence { + collectFieldInfo(child, path.Index(index), decoded, fields) + } + } +} + +func cloneMetadata(metadata Metadata) Metadata { + if metadata.fields == nil && metadata.history == nil { + return Metadata{} + } + cloned := Metadata{ + fields: make(map[string]FieldInfo, len(metadata.fields)), + history: make(map[string][]Change, len(metadata.history)), + } + cloned.effectivePaths = make(map[string]Path, len(metadata.effectivePaths)) + for pointer, path := range metadata.effectivePaths { + cloned.effectivePaths[pointer] = path.clone() + } + cloned.historicalPaths = make(map[string][]Path, len(metadata.historicalPaths)) + for pointer, paths := range metadata.historicalPaths { + cloned.historicalPaths[pointer] = clonePaths(paths) + } + for path, info := range metadata.fields { + cloned.fields[path] = cloneFieldInfo(info) + } + for path, changes := range metadata.history { + clonedChanges := make([]Change, len(changes)) + for index, change := range changes { + clonedChanges[index] = cloneChange(change) + } + cloned.history[path] = clonedChanges + } + return cloned +} + +func applyResolutionMetadata(metadata *Metadata, resolved resolutionResult) { + for _, snapshot := range resolved.snapshots { + pointer := snapshot.change.Path.String() + change := cloneChange(snapshot.change) + if info, ok := metadata.fields[pointer]; ok { + change.Previous = []Origin{cloneOrigin(info.Origin)} + info.Origin = cloneOrigin(change.Origin) + metadata.fields[pointer] = info + } + metadata.history[pointer] = append(metadata.history[pointer], change) + } + appendResolutionDescendantHistory(metadata, resolved) + for _, delta := range resolved.deltas { + record := coveringResolutionSnapshot(resolved.snapshots, delta.path) + if record == nil { + continue + } + for pointer, info := range metadata.fields { + path, err := ParsePath(pointer) + if err != nil || !pathContains(delta.path, path) { + continue + } + info.Origin = cloneOrigin(record.change.Origin) + metadata.fields[pointer] = info + } + } + refreshResolutionFields(metadata.fields, resolved.final, resolved.snapshots) +} + +func appendResolutionDescendantHistory(metadata *Metadata, resolved resolutionResult) { + finalPaths := make(map[string]*document.Node) + indexDocument(resolved.final, Path{}, finalPaths) + for _, delta := range resolved.deltas { + record := coveringResolutionSnapshot(resolved.snapshots, delta.path) + if record == nil { + continue + } + affected := make(map[string]bool) + for pointer := range metadata.history { + if path, err := ParsePath(pointer); err == nil && pathContains(delta.path, path) { + affected[pointer] = true + } + } + for pointer := range finalPaths { + if path, err := ParsePath(pointer); err == nil && pathContains(delta.path, path) { + affected[pointer] = true + } + } + pointers := make([]string, 0, len(affected)) + for pointer := range affected { + if pointer != record.change.Path.String() { + pointers = append(pointers, pointer) + } + } + slices.Sort(pointers) + for _, pointer := range pointers { + change := cloneChange(record.change) + if info, ok := metadata.fields[pointer]; ok { + change.Previous = []Origin{cloneOrigin(info.Origin)} + } + metadata.history[pointer] = append(metadata.history[pointer], change) + } + } +} + +func coveringResolutionSnapshot(snapshots []resolutionSnapshot, path Path) *resolutionSnapshot { + var covering *resolutionSnapshot + for index := range snapshots { + snapshot := &snapshots[index] + if pathContains(snapshot.change.Path, path) && (covering == nil || len(snapshot.change.Path.segments) > len(covering.change.Path.segments)) { + covering = snapshot + } + } + return covering +} + +func refreshResolutionFields(fields map[string]FieldInfo, final *document.Node, snapshots []resolutionSnapshot) { + finalPaths := make(map[string]*document.Node) + indexDocument(final, Path{}, finalPaths) + for pointer := range fields { + if _, ok := finalPaths[pointer]; !ok { + delete(fields, pointer) + } + } + for pointer, node := range finalPaths { + if info, ok := fields[pointer]; ok { + info.Null = node.Kind == document.Null + fields[pointer] = info + continue + } + origin := Origin{Layer: LayerResolution} + if path, err := ParsePath(pointer); err == nil { + if covering := coveringResolutionSnapshot(snapshots, path); covering != nil { + origin = cloneOrigin(covering.change.Origin) + } + } + fields[pointer] = FieldInfo{Null: node.Kind == document.Null, Origin: origin} + } +} diff --git a/internal/engine/load_result.go b/internal/engine/load_result.go new file mode 100644 index 0000000..abc72cc --- /dev/null +++ b/internal/engine/load_result.go @@ -0,0 +1,120 @@ +package engine + +import ( + "reflect" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// resolvedLoad carries typed phase outputs needed for validation and publication; +// it owns no source access, options, or long-lived schema state. +type resolvedLoad[T any] struct { + configuration T + decoded decodedConfiguration + metadata Metadata + final *document.Node + snapshots []resolutionSnapshot + applications []historyApplication +} + +func decodeLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput) (decodedConfiguration, []Diagnostic) { + defaults := reflect.Zero(reflect.TypeFor[T]()) + if definition.Defaults != nil { + defaults = reflect.ValueOf(definition.Defaults()) + } + context := decodeContext{} + if definition.Version != nil { + context.versionPath = &definition.Version.Path + } + decoded, decodeIssues := decodeConfigurationWithContext(schema, defaults, input.document, context) + if len(decodeIssues) != 0 { + diagnostics := make([]Diagnostic, len(decodeIssues)) + for i, issue := range decodeIssues { + diagnostics[i] = diagnosticFromDecodeIssue(issue) + } + return decodedConfiguration{}, diagnostics + } + if input.composition != nil { + applyCompositionOrigins(&decoded, input.composition.origins) + } + + return decoded, nil +} + +func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput, decoded decodedConfiguration) (resolvedLoad[T], []Diagnostic) { + var diagnostics []Diagnostic + configuration := decoded.value.Interface().(T) + metadata := metadataFromDecoded(decoded) + if input.composition != nil { + applyOverlayMetadata(&metadata, input.composition.overlays) + } + final := decoded.final + var resolutionSnapshots []resolutionSnapshot + if definition.Resolve != nil { + entry := newResolutionInput(metadata, decoded.final, input.sources) + resolved := runResolution(&configuration, definition.Resolve, schema, decoded, input.document, entry) + diagnostics = append(diagnostics, resolved.diagnostics...) + if hasErrors(resolved.diagnostics) { + return resolvedLoad[T]{}, diagnostics + } + final = resolved.final + resolutionSnapshots = resolved.snapshots + applyResolutionMetadata(&metadata, resolved) + } + applications := []historyApplication(nil) + if input.composition != nil { + applications = cloneHistoryApplications(input.composition.applications) + } else if input.document != nil { + applications = []historyApplication{{layer: LayerBase, value: input.document}} + } + return resolvedLoad[T]{configuration: configuration, decoded: decoded, metadata: metadata, final: final, snapshots: resolutionSnapshots, applications: applications}, diagnostics +} + +func detachLoadConfiguration[T any](schema *schemaDescriptor, configuration T, mode secretCloneMode) (T, *Diagnostic) { + detached, issue := cloneConfigurationWithMode(schema, reflect.ValueOf(configuration), defaultProjectionLimits(), mode) + if issue != nil { + diagnostic := diagnosticFromDecodeIssue(decodeIssueFromProjection(issue)) + var zero T + return zero, &diagnostic + } + return detached.Interface().(T), nil +} + +func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T], resolved resolvedLoad[T], references map[string]string, accumulator *diagnosticAccumulator) (*Result[T], error) { + // A read-only validator can replace a value with an equal borrowed object. + // Detach without re-reading the environment or discarding validated state. + configuration, issue := detachLoadConfiguration(schema, resolved.configuration, cloneValidatedSecrets) + if issue != nil { + accumulator.add(*issue) + return nil, accumulator.loadError() + } + decoded, metadata := resolved.decoded, resolved.metadata + final, applications, resolutionSnapshots := resolved.final, resolved.applications, resolved.snapshots + finalizeMetadataPaths(&metadata, final, applications, decoded.baseline, decoded.final) + secrets := classifySecretPaths(schema, definition.Secret, metadata, final, applications, decoded.baseline, decoded.final) + return &Result[T]{ + Config: configuration, + metadata: metadata, + sources: cloneSourceInfos(accumulator.sources), + diagnostics: canonicalDiagnostics(accumulator.diagnostics), + baseline: decoded.baseline, + input: decoded.final, + inputOrigins: cloneOriginIndex(decoded.origins), + final: final, + resolution: resolutionSnapshots, + applications: applications, + secrets: secrets, + secretReferences: cloneSecretReferences(references), + }, nil +} + +func diagnosticFromSchemaIssue(issue *schemaIssue) Diagnostic { + return Diagnostic{Code: issue.code, Severity: SeverityError, Path: issue.path.clone(), Message: issue.message} +} + +func diagnosticFromDecodeIssue(issue decodeIssue) Diagnostic { + return Diagnostic{ + Code: issue.code, Severity: SeverityError, Path: issue.path.clone(), Message: issue.message, + Primary: documentLocationPointer(issue.location), + } +} diff --git a/internal/engine/load_test.go b/internal/engine/load_test.go new file mode 100644 index 0000000..bf0f787 --- /dev/null +++ b/internal/engine/load_test.go @@ -0,0 +1,604 @@ +package engine + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestLoadSingleSourceKindsAndDefaultsOnly(t *testing.T) { + t.Parallel() + + type config struct { + Name string `yaml:"name"` + } + input := []byte("name: configured\n") + temporary := t.TempDir() + filename := filepath.Join(temporary, "config.yml") + if err := os.WriteFile(filename, input, 0o600); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + source func() Source + want string + wantSources int + }{ + {name: "file", source: func() Source { return File(filename) }, want: "configured", wantSources: 1}, + {name: "bytes", source: func() Source { return Bytes("memory.yml", input) }, want: "configured", wantSources: 1}, + {name: "reader", source: func() Source { return Reader("reader.yml", strings.NewReader(string(input))) }, want: "configured", wantSources: 1}, + {name: "no source", source: NoSource, want: "default", wantSources: 0}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var defaultsCalls atomic.Int32 + result, err := Load(test.source(), Options[config]{Definition: Definition[config]{ + Defaults: func() config { + defaultsCalls.Add(1) + return config{Name: "default"} + }, + }}) + if err != nil { + t.Fatal(err) + } + if result.Config.Name != test.want || defaultsCalls.Load() != 1 || len(result.Sources()) != test.wantSources { + t.Fatalf("result = %#v, calls = %d, sources = %#v", result.Config, defaultsCalls.Load(), result.Sources()) + } + }) + } +} + +func TestLoadVersionStatesWarningsAndSafeFailures(t *testing.T) { + t.Parallel() + + type config struct { + SchemaVersion *string `yaml:"schema_version"` + } + versionPath := Keys("schema_version") + tests := []struct { + name string + source Source + want Version + wantLine int + wantErr Code + returnOnly bool + }{ + {name: "absent", source: Bytes("absent.yml", []byte("{}\n")), want: Version{}, wantLine: 1}, + {name: "no source absent", source: NoSource(), want: Version{}}, + {name: "null", source: Bytes("null.yml", []byte("schema_version: null\n")), want: Version{Present: true, Null: true}, wantLine: 1}, + {name: "string", source: Bytes("string.yml", []byte("schema_version: app.v1\n")), want: Version{Present: true, Text: "app.v1"}, wantLine: 1}, + {name: "integer", source: Bytes("integer.yml", []byte("schema_version: 1\n")), want: Version{Present: true, Text: "1"}, wantLine: 1}, + {name: "boolean", source: Bytes("boolean.yml", []byte("schema_version: true\n")), wantErr: CodeVersionType}, + {name: "sequence", source: Bytes("sequence.yml", []byte("schema_version: []\n")), wantErr: CodeVersionType}, + {name: "returned error", source: Bytes("return.yml", []byte("{}\n")), wantErr: CodeVersionHookFailed, returnOnly: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + called := false + rule := &VersionRule{Path: versionPath, Check: func(version Version, validation *Validation) error { + called = true + if test.returnOnly { + return errors.New("CANARY_VERSION_ERROR") + } + if version != test.want { + t.Errorf("version = %#v, want %#v", version, test.want) + } + validation.Warning(versionPath, "review schema compatibility", Keys("missing_related")) + return nil + }} + result, err := Load(test.source, Options[config]{Definition: Definition[config]{Version: rule}}) + if test.wantErr != "" { + if result != nil || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) != 1 || diagnostics[0].Code != test.wantErr || strings.Contains(diagnostics[0].Message, "CANARY") { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if test.wantErr == CodeVersionType && called { + t.Fatal("checker ran for an unsupported version kind") + } + return + } + if err != nil { + t.Fatal(err) + } + diagnostics := result.Diagnostics() + if len(diagnostics) != 1 || diagnostics[0].Code != CodeVersionWarning || diagnostics[0].Severity != SeverityWarning { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if test.wantLine == 0 && (diagnostics[0].Primary != nil || len(diagnostics[0].Related) != 0) { + t.Fatalf("synthetic diagnostic locations = %#v", diagnostics[0]) + } + if test.wantLine != 0 && (diagnostics[0].Primary == nil || diagnostics[0].Primary.Line != test.wantLine || len(diagnostics[0].Related) != 1) { + t.Fatalf("diagnostic locations = %#v", diagnostics[0]) + } + }) + } +} + +func TestLoadDecodesIntegerVersionOnlyAtConfiguredStringPath(t *testing.T) { + t.Parallel() + + type stringConfig struct { + Version string `yaml:"version"` + } + type pointerConfig struct { + Version *string `yaml:"version"` + } + versionPath := Keys("version") + + for _, test := range []struct { + name string + source string + load func(Source, *VersionRule) (string, error) + }{ + {name: "integer into string", source: "version: 4\n", load: func(source Source, rule *VersionRule) (string, error) { + result, err := Load(source, Options[stringConfig]{Definition: Definition[stringConfig]{Version: rule}}) + if err != nil { + return "", err + } + return result.Config.Version, nil + }}, + {name: "string into string", source: "version: \"4\"\n", load: func(source Source, rule *VersionRule) (string, error) { + result, err := Load(source, Options[stringConfig]{Definition: Definition[stringConfig]{Version: rule}}) + if err != nil { + return "", err + } + return result.Config.Version, nil + }}, + {name: "integer into pointer", source: "version: 4\n", load: func(source Source, rule *VersionRule) (string, error) { + result, err := Load(source, Options[pointerConfig]{Definition: Definition[pointerConfig]{Version: rule}}) + if err != nil { + return "", err + } + if result.Config.Version == nil { + t.Fatal("version pointer is nil") + } + return *result.Config.Version, nil + }}, + {name: "string into pointer", source: "version: \"4\"\n", load: func(source Source, rule *VersionRule) (string, error) { + result, err := Load(source, Options[pointerConfig]{Definition: Definition[pointerConfig]{Version: rule}}) + if err != nil { + return "", err + } + if result.Config.Version == nil { + t.Fatal("version pointer is nil") + } + return *result.Config.Version, nil + }}, + } { + t.Run(test.name, func(t *testing.T) { + var checked Version + rule := &VersionRule{Path: versionPath, Check: func(version Version, _ *Validation) error { + checked = version + return nil + }} + got, err := test.load(Bytes("version.yml", []byte(test.source)), rule) + if err != nil { + t.Fatal(err) + } + if got != "4" || checked != (Version{Present: true, Text: "4"}) { + t.Fatalf("value = %q, checked = %#v", got, checked) + } + }) + } + + if result, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[stringConfig]{}); result != nil { + t.Fatalf("load without rule = %#v, %v", result, err) + } else { + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeDecodeType { + t.Fatalf("load without rule error = %v", err) + } + } + type unrelatedConfig struct { + Version string `yaml:"version"` + Label string `yaml:"label"` + } + rule := &VersionRule{Path: versionPath, Check: func(Version, *Validation) error { return nil }} + if result, err := Load(Bytes("version.yml", []byte("version: 4\nlabel: 5\n")), Options[unrelatedConfig]{Definition: Definition[unrelatedConfig]{Version: rule}}); result != nil { + t.Fatalf("unrelated coercion = %#v, %v", result, err) + } else { + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeDecodeType || loadError.Diagnostics()[0].Path.String() != "/label" { + t.Fatalf("unrelated coercion error = %v", err) + } + } +} + +func TestLoadVersionScalarAndPathBoundaries(t *testing.T) { + t.Parallel() + + type config struct { + Version string `yaml:"version"` + } + for _, test := range []struct { + name string + value string + want string + }{ + {name: "zero", value: "0", want: "0"}, + {name: "negative", value: "-7", want: "-7"}, + {name: "negative zero", value: "-0", want: "-0"}, + {name: "beyond int64", value: "123456789012345678901234567890", want: "123456789012345678901234567890"}, + {name: "empty string", value: `""`, want: ""}, + } { + t.Run(test.name, func(t *testing.T) { + var checked Version + rule := &VersionRule{Path: Keys("version"), Check: func(version Version, _ *Validation) error { + checked = version + return nil + }} + result, err := Load(Bytes("version.yml", []byte("version: "+test.value+"\n")), Options[config]{Definition: Definition[config]{Version: rule}}) + if err != nil { + t.Fatal(err) + } + if result.Config.Version != test.want || checked.Text != test.want || !checked.Present || checked.Null { + t.Fatalf("configuration = %#v, checked = %#v", result.Config, checked) + } + }) + } + + for _, test := range []struct { + name string + value string + }{ + {name: "fraction", value: "4.0"}, + {name: "exponent", value: "4e0"}, + {name: "boolean", value: "true"}, + {name: "mapping", value: "{}"}, + {name: "sequence", value: "[]"}, + } { + t.Run(test.name, func(t *testing.T) { + called := false + rule := &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { + called = true + return nil + }} + result, err := Load(Bytes("version.yml", []byte("version: "+test.value+"\n")), Options[config]{Definition: Definition[config]{Version: rule}}) + if result != nil || called { + t.Fatalf("result = %#v, checker called = %v", result, called) + } + requireDiagnostic(t, err, CodeVersionType, "/version") + }) + } + + type namedVersion string + type nested struct { + Versions map[string]**namedVersion `yaml:"versions"` + Sibling namedVersion `yaml:"sibling"` + } + parsed, err := ParsePath("/versions/0") + if err != nil { + t.Fatal(err) + } + rule := &VersionRule{Path: parsed, Check: func(version Version, _ *Validation) error { + if version.Text != "4" { + t.Errorf("version = %#v", version) + } + return nil + }} + result, err := Load(Bytes("nested.yml", []byte("versions:\n \"0\": 4\nsibling: quoted\n")), Options[nested]{Definition: Definition[nested]{Version: rule}}) + if err != nil { + t.Fatal(err) + } + if result.Config.Versions["0"] == nil || *result.Config.Versions["0"] == nil || **result.Config.Versions["0"] != "4" || result.Config.Sibling != "quoted" { + t.Fatalf("nested configuration = %#v", result.Config) + } + type sequenceConfig struct { + Versions []namedVersion `yaml:"versions"` + } + sequenceResult, err := Load(Bytes("sequence.yml", []byte("versions: [6]\n")), Options[sequenceConfig]{Definition: Definition[sequenceConfig]{Version: &VersionRule{Path: parsed, Check: func(Version, *Validation) error { return nil }}}}) + if err != nil || len(sequenceResult.Config.Versions) != 1 || sequenceResult.Config.Versions[0] != "6" { + t.Fatalf("sequence configuration = %#v, %v", sequenceResult, err) + } + + type escaped struct { + Versions map[string]namedVersion `yaml:"versions"` + } + escapedPath, err := ParsePath("/versions/a~1b") + if err != nil { + t.Fatal(err) + } + escapedResult, err := Load(Bytes("escaped.yml", []byte("versions:\n a/b: 9\n")), Options[escaped]{Definition: Definition[escaped]{Version: &VersionRule{Path: escapedPath, Check: func(Version, *Validation) error { return nil }}}}) + if err != nil || escapedResult.Config.Versions["a/b"] != "9" { + t.Fatalf("escaped configuration = %#v, %v", escapedResult, err) + } + + siblingRule := &VersionRule{Path: Keys("versions", "target"), Check: func(Version, *Validation) error { return nil }} + if result, err := Load(Bytes("sibling.yml", []byte("versions:\n target: 4\n other: 5\n")), Options[escaped]{Definition: Definition[escaped]{Version: siblingRule}}); result != nil { + t.Fatalf("same-type sibling result = %#v, %v", result, err) + } else { + requireDiagnostic(t, err, CodeDecodeType, "/versions/other") + } +} + +func TestLoadVersionDestinationTypesKeepOrdinaryDecoding(t *testing.T) { + t.Parallel() + + check := func(Version, *Validation) error { return nil } + t.Run("integer destination", func(t *testing.T) { + type config struct { + Version int8 `yaml:"version"` + } + result, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + if err != nil || result.Config.Version != 4 { + t.Fatalf("result = %#v, %v", result, err) + } + result, err = Load(Bytes("version.yml", []byte("version: 999\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + if result != nil { + t.Fatalf("range result = %#v", result) + } + requireDiagnostic(t, err, CodeDecodeRange, "/version") + result, err = Load(Bytes("version.yml", []byte("version: \"4\"\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + if result != nil { + t.Fatalf("quoted integer result = %#v", result) + } + requireDiagnostic(t, err, CodeDecodeType, "/version") + }) + + t.Run("dynamic destination", func(t *testing.T) { + type config struct { + Version any `yaml:"version"` + } + result, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + if err != nil || result.Config.Version != int64(4) { + t.Fatalf("result = %#v, %v", result, err) + } + result, err = Load(Bytes("version.yml", []byte("version: 123456789012345678901234567890\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + if result != nil { + t.Fatalf("range result = %#v", result) + } + requireDiagnostic(t, err, CodeDecodeRange, "/version") + }) + + for _, test := range []struct { + name string + load func() error + }{ + {name: "text codec", load: func() error { + type config struct { + Version validatedText `yaml:"version"` + } + _, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + return err + }}, + {name: "duration", load: func() error { + type config struct { + Version time.Duration `yaml:"version"` + } + _, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + return err + }}, + {name: "secret", load: func() error { + type config struct { + Version Secret `yaml:"version"` + } + _, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: check}}}) + return err + }}, + } { + t.Run(test.name, func(t *testing.T) { + requireDiagnostic(t, test.load(), CodeDecodeType, "/version") + }) + } +} + +func TestLoadVersionPolicyStopsLaterHooks(t *testing.T) { + t.Parallel() + + type config struct { + Version string `yaml:"version"` + } + var defaultsCalled, resolutionCalled, validationCalled bool + rule := &VersionRule{Path: Keys("version"), Check: func(version Version, validation *Validation) error { + if version.Text != "4" { + t.Errorf("version = %#v", version) + } + validation.Error(Keys("version"), "unsupported configuration version") + return nil + }} + result, err := Load(Bytes("version.yml", []byte("version: 4\n")), Options[config]{Definition: Definition[config]{ + Defaults: func() config { defaultsCalled = true; return config{} }, + Version: rule, + Resolve: func(*config, *Resolution) error { resolutionCalled = true; return nil }, + Validate: func(*config, *Validation) { validationCalled = true }, + }}) + if result != nil || defaultsCalled || resolutionCalled || validationCalled { + t.Fatalf("result = %#v, hooks = %v/%v/%v", result, defaultsCalled, resolutionCalled, validationCalled) + } + requireDiagnostic(t, err, CodeVersionError, "/version") +} + +func requireDiagnostic(t *testing.T, err error, code Code, path string) { + t.Helper() + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error = %T %v", err, err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) != 1 || diagnostics[0].Code != code || diagnostics[0].Path.String() != path || diagnostics[0].Primary == nil { + t.Fatalf("diagnostics = %#v", diagnostics) + } +} + +func TestLoadVersionReportSuppressesReturnedErrorTextAndDefaults(t *testing.T) { + t.Parallel() + + type config struct { + Name string `yaml:"name"` + } + var defaultsCalls atomic.Int32 + result, err := Load(Bytes("version.yml", []byte("name: value\n")), Options[config]{Definition: Definition[config]{ + Defaults: func() config { + defaultsCalls.Add(1) + return config{Name: "default"} + }, + Version: &VersionRule{Path: Keys("schema_version"), Check: func(_ Version, validation *Validation) error { + validation.Error(Keys("schema_version"), "schema_version is required") + return errors.New("CANARY_RETURNED_ERROR") + }}, + }}) + if result != nil || err == nil || defaultsCalls.Load() != 0 { + t.Fatalf("result = %#v, err = %v, calls = %d", result, err, defaultsCalls.Load()) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) != 1 || diagnostics[0].Code != CodeVersionError || strings.Contains(diagnostics[0].Message, "CANARY") { + t.Fatalf("diagnostics = %#v", diagnostics) + } +} + +func TestLoadMetadataReflectsDefaultsInputAndDetachedRecords(t *testing.T) { + t.Parallel() + + type nested struct { + Kept string `yaml:"kept"` + Set string `yaml:"set"` + } + type config struct { + Nested nested `yaml:"nested"` + Nullable map[string]string `yaml:"nullable"` + } + result, err := Load(Bytes("metadata.yml", []byte("nested: {set: configured}\nnullable: null\n")), Options[config]{Definition: Definition[config]{ + Defaults: func() config { + return config{Nested: nested{Kept: "default", Set: "default"}, Nullable: map[string]string{"key": "value"}} + }, + }}) + if err != nil { + t.Fatal(err) + } + metadata := result.Metadata() + kept, ok := metadata.Lookup(Keys("nested", "kept")) + if !ok || kept.Explicit || kept.Origin.Layer != LayerDefault { + t.Fatalf("kept = %#v, %v", kept, ok) + } + set, ok := metadata.Lookup(Keys("nested", "set")) + if !ok || !set.Explicit || set.Origin.Layer != LayerBase || len(set.Origin.Locations) != 1 { + t.Fatalf("set = %#v, %v", set, ok) + } + nullable, ok := metadata.Lookup(Keys("nullable")) + if !ok || !nullable.Explicit || !nullable.Null || nullable.Origin.Layer != LayerBase { + t.Fatalf("nullable = %#v, %v", nullable, ok) + } + if metadata.IsExplicit(Keys("nested", "kept")) || !metadata.IsExplicit(Keys("nested", "set")) { + t.Fatal("explicit presence disagrees with lookup") + } + if history := metadata.History(Keys("nested", "set")); len(history) != 2 || history[0].Origin.Layer != LayerDefault || history[1].Origin.Layer != LayerBase || len(history[1].Previous) != 1 { + t.Fatalf("history = %#v", history) + } + + set.Origin.Locations[0].Line = 99 + history := metadata.History(Keys("nested", "set")) + history[1].Origin.Locations[0].Line = 99 + sources := result.Sources() + sources[0].Label = "changed" + diagnostics := result.Diagnostics() + if diagnostics != nil { + diagnostics[0].Message = "changed" + } + fresh, _ := result.Metadata().Lookup(Keys("nested", "set")) + if fresh.Origin.Locations[0].Line == 99 || result.Sources()[0].Label != "metadata.yml" || result.Diagnostics() != nil { + t.Fatal("result inspection aliases internal records") + } +} + +func TestLoadRejectsUnavailableOptionsAndDisabledDirectives(t *testing.T) { + t.Parallel() + + type config struct { + Name string `yaml:"name"` + } + validSource := func() Source { return Bytes("options.yml", []byte("name: value\n")) } + tests := []struct { + name string + source Source + options []Options[config] + code Code + }{ + {name: "multiple options", source: validSource(), options: []Options[config]{{}, {}}, code: CodeOptionsInvalid}, + {name: "imports without filesystem context", source: validSource(), options: []Options[config]{{Imports: true}}, code: CodeOptionsInvalid}, + {name: "incomplete filesystem", source: validSource(), options: []Options[config]{{Filesystem: &FilesystemContext{}}}, code: CodeOptionsInvalid}, + {name: "missing version checker", source: validSource(), options: []Options[config]{{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version")}}}}, code: CodeOptionsInvalid}, + {name: "no-source filesystem", source: NoSource(), options: []Options[config]{{Filesystem: &FilesystemContext{}}}, code: CodeOptionsInvalid}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := Load(test.source, test.options...) + if result != nil || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != test.code { + t.Fatalf("error = %#v", err) + } + }) + } + + result, err := Load[config](Bytes("directive.yml", []byte("nested:\n configkit: {}\n"))) + if result != nil || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatal(err) + } + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Code != CodeImportDisabled || diagnostic.Path.String() != "/nested/configkit" || diagnostic.Primary == nil || diagnostic.Primary.Line != 2 { + t.Fatalf("diagnostic = %#v", diagnostic) + } +} + +func TestLoadReturnsNilForSourceSchemaAndDecodeFailures(t *testing.T) { + t.Parallel() + + type invalidSchema struct { + Channel chan int `yaml:"channel"` + } + type validConfig struct { + Known int `yaml:"known"` + } + tests := []struct { + name string + run func() (any, error) + code Code + }{ + {name: "source", run: func() (any, error) { return Load[validConfig](File(filepath.Join(t.TempDir(), "missing.yml"))) }, code: CodeSourceMissing}, + {name: "schema", run: func() (any, error) { return Load[invalidSchema](Bytes("schema.yml", []byte("{}\n"))) }, code: CodeSchemaUnsupportedType}, + {name: "decode", run: func() (any, error) { return Load[validConfig](Bytes("decode.yml", []byte("unknown: 1\n"))) }, code: CodeDecodeUnknownField}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := test.run() + if !isNilInterface(result) || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != test.code { + t.Fatalf("error = %#v", err) + } + }) + } +} + +func isNilInterface(value any) bool { + if value == nil { + return true + } + reflected := reflect.ValueOf(value) + return reflected.Kind() == reflect.Pointer && reflected.IsNil() +} diff --git a/internal/engine/metadata.go b/internal/engine/metadata.go new file mode 100644 index 0000000..06b3625 --- /dev/null +++ b/internal/engine/metadata.go @@ -0,0 +1,149 @@ +package engine + +import ( + "sort" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// Metadata provides detached effective presence/origin and chronological history +// queries for a load. Its zero value contains no paths. +type Metadata struct { + effectivePaths map[string]Path + historicalPaths map[string][]Path + fields map[string]FieldInfo + history map[string][]Change +} + +// Lookup returns information for an effective path. Unknown and invalid paths +// return false. +func (m Metadata) Lookup(path Path) (FieldInfo, bool) { + if !path.valid() || !matchesPathShape(path, m.effectivePaths[path.String()]) { + return FieldInfo{}, false + } + info, ok := m.fields[path.String()] + if !ok { + return FieldInfo{}, false + } + return cloneFieldInfo(info), true +} + +// IsExplicit reports whether a surviving input contribution explicitly defined +// path. +func (m Metadata) IsExplicit(path Path) bool { + info, ok := m.Lookup(path) + return ok && info.Explicit +} + +// History returns detached definition and resolution records for path. A record's +// Path can be an ancestor when that container operation replaced or removed path. +// Removed historical paths can have history even though Lookup returns false. +// Unknown and invalid paths return nil. +func (m Metadata) History(path Path) []Change { + if !m.hasHistoricalPath(path) { + return nil + } + history, ok := m.history[path.String()] + if !ok { + return nil + } + cloned := make([]Change, len(history)) + for i, change := range history { + cloned[i] = cloneChange(change) + } + return cloned +} + +// HistoricalPaths returns the paths observed during loading, including effective +// paths and paths removed by overlays or recorded resolution. It includes default +// paths and the root, but not paths that never existed. The zero Metadata returns +// an empty slice. +// +// Paths are detached, unique by canonical JSON Pointer, and sorted by that +// spelling. Like ParsePath results, numeric tokens are resolved contextually; +// one pointer can identify a mapping key in an earlier layer and a sequence index +// in a later one. Use History to inspect definitions and Lookup to check whether +// a path remains effective. Enumeration exposes no configuration values and does +// not perform another load or resolve secrets. +func (m Metadata) HistoricalPaths() []Path { + pointers := make([]string, 0, len(m.historicalPaths)) + for pointer := range m.historicalPaths { + pointers = append(pointers, pointer) + } + sort.Strings(pointers) + paths := make([]Path, 0, len(pointers)) + for _, pointer := range pointers { + // The index contains only canonical pointers produced by Path.String. + path, _ := ParsePath(pointer) + paths = append(paths, path) + } + return paths +} + +// Textual pointers are lookup keys, but explicit key/index kinds remain part of +// the query. Candidate paths are constructed from actual document containers. +func matchesPathShape(query, candidate Path) bool { + if !query.valid() || query.String() != candidate.String() || len(query.segments) != len(candidate.segments) { + return false + } + for i, segment := range query.segments { + actual := candidate.segments[i] + if actual.kind == pathSegmentIndex { + index, ok := schemaIndex(segment) + if !ok || index != actual.index { + return false + } + } else if segment.kind == pathSegmentIndex { + return false + } + } + return true +} + +func (m Metadata) hasHistoricalPath(path Path) bool { + if !path.valid() { + return false + } + for _, candidate := range m.historicalPaths[path.String()] { + if matchesPathShape(path, candidate) { + return true + } + } + return false +} + +func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []historyApplication, earlier ...*document.Node) { + m.effectivePaths = effectivePathIndex(final) + m.historicalPaths = make(map[string][]Path) + add := func(path Path, _ *document.Node) { + pointer := path.String() + for _, prior := range m.historicalPaths[pointer] { + if matchesPathShape(path, prior) { + return + } + } + m.historicalPaths[pointer] = append(m.historicalPaths[pointer], path.clone()) + } + walkDocument(final, Path{}, add) + for _, tree := range earlier { + walkDocument(tree, Path{}, add) + } + for _, application := range applications { + // Overrides have mapping-only prefixes; parsed tokens are keys in this layer. + prefix := Path{} + for _, segment := range application.path.segments { + prefix = prefix.Key(segment.key) + } + walkDocument(application.value, prefix, add) + } +} + +// effectivePathIndex records document-derived paths with their contextual segment +// kinds. Callers retain no aliases to the walked tree or its paths. +func effectivePathIndex(final *document.Node) map[string]Path { + paths := make(map[string]Path) + walkDocument(final, Path{}, func(path Path, _ *document.Node) { + paths[path.String()] = path.clone() + }) + return paths +} diff --git a/internal/engine/metadata_test.go b/internal/engine/metadata_test.go new file mode 100644 index 0000000..013a239 --- /dev/null +++ b/internal/engine/metadata_test.go @@ -0,0 +1,274 @@ +package engine + +import ( + "bytes" + "reflect" + "testing" +) + +func TestMetadataHistoricalPaths(t *testing.T) { + t.Parallel() + result, err := Load[map[string]any](Bytes("base", []byte("retired: {child: 1}\nshape: {'0': base}\n'a/b~': {'': 2}\n")), Options[map[string]any]{ + Overrides: []Override{ + {Path: Keys("retired"), Value: nil, Flag: "--retired"}, + {Path: Keys("shape"), Value: []any{"replacement"}, Flag: "--shape"}, + }, + }) + if err != nil { + t.Fatal(err) + } + metadata := result.Metadata() + paths := metadata.HistoricalPaths() + var got []string + for _, path := range paths { + got = append(got, path.String()) + if len(metadata.History(path)) == 0 { + t.Fatalf("enumerated path %s has no history", path) + } + } + want := []string{"", "/a~1b~0", "/a~1b~0/", "/retired", "/retired/child", "/shape", "/shape/0"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("historical paths = %v, want %v", got, want) + } + if _, ok := metadata.Lookup(paths[4]); ok { + t.Fatal("removed descendant remains effective") + } + if _, ok := metadata.Lookup(paths[6]); !ok { + t.Fatal("enumerated numeric path cannot query the new sequence shape") + } + // Returned slices and nested paths must not alias retained metadata. + paths[1].segments[0].key = "mutated" + paths[2] = Keys("replacement") + if again := metadata.HistoricalPaths(); again[1].String() != want[1] || again[2].String() != want[2] { + t.Fatal("enumeration returned aliased paths") + } + if len((Metadata{}).HistoricalPaths()) != 0 { + t.Fatal("zero metadata contains paths") + } +} + +func TestHistoricalPathsExposeHiddenBaseOwnershipWithoutResolvingBaseSecrets(t *testing.T) { + t.Setenv("CONFIGKIT_HISTORY_UNUSED_PROD_KEY", "") + t.Setenv("CONFIGKIT_HISTORY_TEST_KEY", "synthetic-test-key") + type config struct { + Credential *Secret `yaml:"credential"` + Definitions map[string]map[string]string `yaml:"definitions"` + } + for _, credential := range []string{"CONFIGKIT_HISTORY_TEST_KEY", "null"} { + t.Run(credential, func(t *testing.T) { + dir := t.TempDir() + root := writeImportFile(t, dir, "main.yml", "configkit:\n imports: [{path: first.yml}, {path: second.yml}]\ncredential: CONFIGKIT_HISTORY_UNUSED_PROD_KEY\n") + writeImportFile(t, dir, "first.yml", "definitions: {recap: {prompt: recap}}\n") + writeImportFile(t, dir, "second.yml", "definitions: {recap: {output: recap.md}}\n") + writeImportFile(t, dir, "profiles/testing.yml", "definitions: null\ncredential: "+credential+"\n") + result, err := Load[config](File(root), Options[config]{Imports: true, Profile: "testing"}) + if err != nil { + t.Fatal(err) + } + metadata := result.Metadata() + found := false + for _, path := range metadata.HistoricalPaths() { + if path.String() != "/definitions/recap" { + continue + } + found = true + owners := map[string]bool{} + for _, change := range metadata.History(path) { + if change.Origin.Layer == LayerBase { + for _, location := range change.Origin.Locations { + owners[location.SourceID] = true + } + } + } + if len(owners) != 2 { + t.Fatalf("hidden declaration owners = %v", owners) + } + } + if !found { + t.Fatal("hidden base declaration was not discoverable") + } + }) + } +} + +func TestMetadataQueriesReturnDetachedRecords(t *testing.T) { + t.Parallel() + + path := Keys("service", "workers") + metadata := Metadata{ + effectivePaths: map[string]Path{path.String(): path}, + historicalPaths: map[string][]Path{path.String(): {path}}, + fields: map[string]FieldInfo{ + path.String(): { + Explicit: true, + Origin: Origin{ + Layer: LayerBase, + Locations: []Location{{SourceID: "source-1", Line: 2, Column: 3}}, + }, + }, + }, + history: map[string][]Change{ + path.String(): {{ + Path: path, + Origin: Origin{Layer: LayerResolution, Locations: []Location{{SourceID: "source-1"}}}, + Previous: []Origin{{Layer: LayerDefault, Locations: []Location{{SourceID: "default"}}}}, + Reasons: []string{"bounded worker count"}, + Dependencies: []Path{Keys("limits")}, + }}, + }, + } + + info, ok := metadata.Lookup(path) + if !ok || !info.Explicit || !metadata.IsExplicit(path) { + t.Fatalf("Lookup() = %#v, %v", info, ok) + } + info.Origin.Locations[0].SourceID = "mutated" + again, _ := metadata.Lookup(path) + if again.Origin.Locations[0].SourceID != "source-1" { + t.Fatal("Lookup returned aliased locations") + } + + history := metadata.History(path) + history[0].Reasons[0] = "mutated" + history[0].Previous[0].Locations[0].SourceID = "mutated" + history[0].Dependencies[0].segments[0].key = "mutated" + againHistory := metadata.History(path) + if againHistory[0].Reasons[0] != "bounded worker count" || + againHistory[0].Previous[0].Locations[0].SourceID != "default" || + againHistory[0].Dependencies[0].String() != "/limits" { + t.Fatal("History returned aliased nested records") + } + + missing := Keys("missing") + if _, ok := metadata.Lookup(missing); ok || metadata.IsExplicit(missing) || metadata.History(missing) != nil { + t.Fatal("unknown path produced metadata") + } + if _, ok := metadata.Lookup(Path{}.Index(-1)); ok { + t.Fatal("invalid path produced metadata") + } +} + +func TestResolutionInputKeepsDetachedEntryMetadata(t *testing.T) { + t.Parallel() + + type configuration struct { + Default string `yaml:"default"` + Items []string `yaml:"items"` + Values map[string]string `yaml:"values"` + Nullable *string `yaml:"nullable"` + } + schema := mustSchemaFor[configuration](t) + document := parseDocumentForDecode(t, "items: [first]\nvalues: {'0': key, 'a/b~': escaped}\nnullable: null\n") + decoded, issues := decodeConfiguration(schema, reflect.ValueOf(configuration{Default: "retained"}), document) + if len(issues) != 0 { + t.Fatalf("decode issues = %#v", issues) + } + metadata := metadataFromDecoded(decoded) + input := newResolutionInput(metadata, decoded.final, []SourceInfo{{ + ID: "source-1", Label: "decode.yml", Layer: LayerBase, + ImportedFrom: &Location{SourceID: "parent", Line: 1, Column: 1}, + }}) + + parsedItem, err := ParsePath("/items/0") + if err != nil { + t.Fatal(err) + } + parsedKey, err := ParsePath("/values/0") + if err != nil { + t.Fatal(err) + } + for _, path := range []Path{Path{}, Keys("default"), Keys("nullable"), parsedItem, parsedKey, Keys("values", "a/b~")} { + if _, ok := input.lookup(path); !ok { + t.Fatalf("entry path %s is missing", path) + } + } + defaultInfo, _ := input.lookup(Keys("default")) + nullInfo, _ := input.lookup(Keys("nullable")) + if defaultInfo.Explicit || !nullInfo.Explicit || !nullInfo.Null { + t.Fatalf("default = %#v, nullable = %#v", defaultInfo, nullInfo) + } + for _, path := range []Path{ + Keys("default", "child"), + Keys("items", "0"), + Keys("values").Index(0), + Keys("items").Index(1), + } { + if _, ok := input.lookup(path); ok { + t.Fatalf("entry lookup accepted %s", path) + } + } + + item, _ := input.lookup(parsedItem) + item.Origin.Locations[0].SourceID = "mutated" + source, ok := input.source("source-1") + if !ok { + t.Fatal("source is missing") + } + source.ImportedFrom.SourceID = "mutated" + + configurationAtEntry := decoded.value.Interface().(configuration) + resolved := runResolution(&configurationAtEntry, func(value *configuration, resolution *Resolution) error { + value.Items[0] = "normalized" + resolution.Record(Keys("items"), ChangeNormalization, "normalized item") + return nil + }, schema, decoded, document, input) + if len(resolved.diagnostics) != 0 { + t.Fatalf("resolution diagnostics = %#v", resolved.diagnostics) + } + applyResolutionMetadata(&metadata, resolved) + metadata.effectivePaths = effectivePathIndex(resolved.final) + + again, ok := input.lookup(parsedItem) + if !ok || again.Origin.Locations[0].SourceID != "source-1" || again.Origin.Operation != "" { + t.Fatalf("entry metadata changed after resolution = %#v, %v", again, ok) + } + againSource, ok := input.source("source-1") + if !ok || againSource.ImportedFrom == nil || againSource.ImportedFrom.SourceID != "parent" { + t.Fatalf("entry source changed = %#v, %v", againSource, ok) + } + if _, ok := input.source(""); ok { + t.Fatal("empty source identity resolved") + } +} + +func TestQueriesRespectActualHistoricalAndEffectiveContainers(t *testing.T) { + result, err := Load[map[string]any](Bytes("base", []byte("items: [1]\nm: {'0': 2}\nx: 3\nretired: {child: 4}\n")), Options[map[string]any]{Overrides: []Override{{Path: Keys("retired"), Value: "removed", Flag: "--retired"}}}) + if err != nil { + t.Fatal(err) + } + for _, path := range []Path{Keys("items", "0"), Keys("m").Index(0), Keys("x", "missing"), Keys("items").Index(8)} { + if _, ok := result.Metadata().Lookup(path); ok { + t.Fatalf("invalid lookup %s", path) + } + if result.Metadata().History(path) != nil { + t.Fatalf("invalid history %s", path) + } + if _, ok := result.Explain(path); ok { + t.Fatalf("invalid explanation %s", path) + } + var out bytes.Buffer + if err := result.WriteExplanation(&out, path); err == nil || out.Len() != 0 { + t.Fatalf("invalid path wrote output: %s", path) + } + } + for _, pointer := range []string{"/items/0", "/m/0"} { + path, _ := ParsePath(pointer) + if _, ok := result.Metadata().Lookup(path); !ok { + t.Fatal("parsed path lost lookup") + } + if _, ok := result.Explain(path); !ok { + t.Fatal("parsed path lost explanation") + } + } + retired := Keys("retired", "child") + if _, ok := result.Metadata().Lookup(retired); ok { + t.Fatal("removed path remains effective") + } + if len(result.Metadata().History(retired)) == 0 { + t.Fatal("removed history missing") + } + x, ok := result.Explain(retired) + if !ok || x.Effective.Present || len(x.History) == 0 { + t.Fatal("removed explanation missing") + } +} diff --git a/internal/engine/options.go b/internal/engine/options.go new file mode 100644 index 0000000..dea8980 --- /dev/null +++ b/internal/engine/options.go @@ -0,0 +1,41 @@ +package engine + +func selectOptions[T any](source Source, supplied []Options[T]) (Options[T], []Diagnostic) { + if len(supplied) > 1 { + return Options[T]{}, []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "Load accepts at most one options value")} + } + var options Options[T] + if len(supplied) == 1 { + options = supplied[0] + } + var diagnostics []Diagnostic + definition := options.Definition + if !validProfileName(options.Profile) { + diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "profile name must contain only ASCII letters, digits, underscore, or hyphen")) + } + if options.Filesystem != nil { + diagnostics = append(diagnostics, filesystemOptionDiagnostics(source, options.Filesystem)...) + } + if source.kind == sourceNone && options.Imports { + diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource cannot enable imports")) + } + if source.kind == sourceNone && options.Profile != "" && options.Profile != "default" { + diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource cannot select an active profile")) + } + if options.Filesystem == nil && (source.kind == sourceBytes || source.kind == sourceReader) && (options.Imports || activeProfile(options.Profile)) { + diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "byte and reader imports or profiles require filesystem context")) + } + if definition.Version != nil { + if !definition.Version.Path.valid() { + diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "schema-version path is invalid")) + } + if definition.Version.Check == nil { + diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "schema-version rule requires a checker")) + } + } + return options, canonicalDiagnostics(diagnostics) +} + +func optionDiagnostic(code Code, message string) Diagnostic { + return Diagnostic{Code: code, Severity: SeverityError, Message: message} +} diff --git a/internal/engine/overlay.go b/internal/engine/overlay.go new file mode 100644 index 0000000..9c7c20e --- /dev/null +++ b/internal/engine/overlay.go @@ -0,0 +1,161 @@ +package engine + +import ( + "slices" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type overlayRecord struct { + change Change + displaced []displacedDefinition +} + +type displacedDefinition struct { + path Path + origin Origin +} + +func overlayDocuments(base *composedDocument, overlay *document.Node) (*composedDocument, []Diagnostic) { + if base == nil || base.root == nil || base.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping { + return nil, []Diagnostic{{ + Code: CodeProfileInvalid, Severity: SeverityError, + Message: "profile overlay requires mapping document roots", + }} + } + result := cloneComposedDocument(base) + mergeOverlayOrigin(result.origins, Path{}, overlay.Location, LayerProfile) + mergeOverlayMappings(result, result.root, overlay, Path{}, LayerProfile, true) + result.applications = append(result.applications, historyApplication{layer: LayerProfile, value: overlay}) + return result, nil +} + +func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool) { + for _, incoming := range overlay.Mapping { + path := current.Key(incoming.Key) + index := documentEntryIndex(target.Mapping, incoming.Key) + if index < 0 { + cloned := incoming + cloned.Value = document.Clone(incoming.Value) + target.Mapping = append(target.Mapping, cloned) + collectOverlayOrigins(result.origins, incoming.Value, path, layer) + continue + } + + existing := target.Mapping[index].Value + if existing.Kind == document.Mapping && incoming.Value.Kind == document.Mapping { + mergeOverlayOrigin(result.origins, path, incoming.Value.Location, layer) + mergeOverlayMappings(result, existing, incoming.Value, path, layer, recordReplacements) + continue + } + + displaced := displacedOrigins(existing, result.origins, path) + previous := cloneOrigin(result.origins[path.String()]) + deleteOriginSubtree(result.origins, path) + target.Mapping[index] = incoming + target.Mapping[index].Value = document.Clone(incoming.Value) + collectOverlayOrigins(result.origins, incoming.Value, path, layer) + if recordReplacements { + result.overlays = append(result.overlays, overlayRecord{ + change: Change{ + Path: path.clone(), Origin: overlayOrigin(layer, incoming.Value.Location), + Previous: previousOrigins(previous), + }, + displaced: displaced, + }) + } + } +} + +func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path, layer Layer) { + origins[path.String()] = overlayOrigin(layer, node.Location) + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectOverlayOrigins(origins, entry.Value, path.Key(entry.Key), layer) + } + case document.Sequence: + for index, child := range node.Sequence { + collectOverlayOrigins(origins, child, path.Index(index), layer) + } + } +} + +func mergeOverlayOrigin(origins map[string]Origin, path Path, location document.Location, layer Layer) { + pointer := path.String() + origin := cloneOrigin(origins[pointer]) + origin.Layer = layer + origin.Locations = append(origin.Locations, publicLocation(location)) + origins[pointer] = origin +} + +func overlayOrigin(layer Layer, location document.Location) Origin { + return Origin{Layer: layer, Locations: []Location{publicLocation(location)}} +} + +func previousOrigins(origin Origin) []Origin { + if origin.Layer == "" { + return nil + } + return []Origin{origin} +} + +func displacedOrigins(node *document.Node, origins map[string]Origin, path Path) []displacedDefinition { + var displaced []displacedDefinition + collectDisplacedOrigins(&displaced, node, origins, path, Origin{}) + slices.SortFunc(displaced, func(left, right displacedDefinition) int { + return strings.Compare(left.path.String(), right.path.String()) + }) + return displaced +} + +func collectDisplacedOrigins(displaced *[]displacedDefinition, node *document.Node, origins map[string]Origin, path Path, inherited Origin) { + origin, ok := origins[path.String()] + if !ok { + origin = cloneOrigin(inherited) + origin.Locations = []Location{publicLocation(node.Location)} + } + *displaced = append(*displaced, displacedDefinition{path: path.clone(), origin: cloneOrigin(origin)}) + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectDisplacedOrigins(displaced, entry.Value, origins, path.Key(entry.Key), origin) + } + case document.Sequence: + for index, child := range node.Sequence { + collectDisplacedOrigins(displaced, child, origins, path.Index(index), origin) + } + } +} + +func deleteOriginSubtree(origins map[string]Origin, path Path) { + prefix := path.String() + for pointer := range origins { + if pointer == prefix || strings.HasPrefix(pointer, prefix+"/") { + delete(origins, pointer) + } + } +} + +func cloneOriginIndex(origins map[string]Origin) map[string]Origin { + cloned := make(map[string]Origin, len(origins)) + for pointer, origin := range origins { + cloned[pointer] = cloneOrigin(origin) + } + return cloned +} + +func cloneOverlayRecords(records []overlayRecord) []overlayRecord { + cloned := make([]overlayRecord, len(records)) + for index, record := range records { + cloned[index].change = cloneChange(record.change) + cloned[index].displaced = make([]displacedDefinition, len(record.displaced)) + for displacedIndex, displaced := range record.displaced { + cloned[index].displaced[displacedIndex] = displacedDefinition{ + path: displaced.path.clone(), origin: cloneOrigin(displaced.origin), + } + } + } + return cloned +} diff --git a/internal/engine/overlay_test.go b/internal/engine/overlay_test.go new file mode 100644 index 0000000..50aeb0f --- /dev/null +++ b/internal/engine/overlay_test.go @@ -0,0 +1,78 @@ +package engine + +import ( + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +func TestOverlayDocumentsRecursesMappingsAndReplacesAtomicSubtrees(t *testing.T) { + baseRoot := parseCompositionDocument(t, "source-1", `shared: + kept: base + changed: old +sequence: [one, two] +nullable: value +to_map: scalar +to_scalar: {child: old} +`) + base, diagnostics := composeDocuments(baseRoot) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + overlay := parseCompositionDocument(t, "source-2", `shared: + changed: new + added: profile +sequence: [] +nullable: null +to_map: {child: new} +to_scalar: scalar +`) + baseBefore := document.Clone(base.root) + overlayBefore := document.Clone(overlay) + + result, diagnostics := overlayDocuments(base, overlay) + if len(diagnostics) != 0 { + t.Fatalf("overlayDocuments() diagnostics = %#v", diagnostics) + } + checks := []struct { + path Path + kind document.Kind + text string + }{ + {path: Keys("shared", "kept"), kind: document.String, text: "base"}, + {path: Keys("shared", "changed"), kind: document.String, text: "new"}, + {path: Keys("shared", "added"), kind: document.String, text: "profile"}, + {path: Keys("sequence"), kind: document.Sequence}, + {path: Keys("nullable"), kind: document.Null}, + {path: Keys("to_map"), kind: document.Mapping}, + {path: Keys("to_map", "child"), kind: document.String, text: "new"}, + {path: Keys("to_scalar"), kind: document.String, text: "scalar"}, + } + for _, check := range checks { + node, ok := documentAtPath(result.root, check.path) + if !ok || node.Kind != check.kind || node.Text != check.text { + t.Errorf("node %s = %#v, %v", check.path.String(), node, ok) + } + } + if _, exists := result.origins["/sequence/0"]; exists { + t.Fatal("replaced empty sequence retained a stale index origin") + } + if _, exists := result.origins["/to_scalar/child"]; exists { + t.Fatal("parent replacement retained a stale child origin") + } + shared := result.origins["/shared"] + if shared.Layer != LayerProfile || len(shared.Locations) != 2 { + t.Fatalf("shared mapping origin = %#v", shared) + } + if len(result.overlays) != 5 { + t.Fatalf("overlay records = %#v, want five replacements", result.overlays) + } + if !reflect.DeepEqual(base.root, baseBefore) || !reflect.DeepEqual(overlay, overlayBefore) { + t.Fatal("overlay mutated an input document") + } + result.root.Mapping[0].Value.Mapping[0].Value.Text = "changed" + if !reflect.DeepEqual(base.root, baseBefore) || !reflect.DeepEqual(overlay, overlayBefore) { + t.Fatal("overlay result aliases an input document") + } +} diff --git a/internal/engine/override.go b/internal/engine/override.go new file mode 100644 index 0000000..8ba93e9 --- /dev/null +++ b/internal/engine/override.go @@ -0,0 +1,223 @@ +package engine + +import ( + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +const maxOverrides = 1024 + +type preparedOverride struct { + path Path + value *document.Node + source SourceInfo + bytes int64 + nodes int +} + +func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) ([]preparedOverride, []SourceInfo, []Diagnostic) { + if len(overrides) > maxOverrides || len(sources)+len(overrides) > limits.sources { + return nil, cloneSourceInfos(sources), []Diagnostic{{ + Code: CodeOverrideLimitExceeded, Severity: SeverityError, + Message: "configuration overrides exceed the record or source-count limit", + }} + } + + var diagnostics []Diagnostic + for index, override := range overrides { + if !validOverrideLabel(override.Flag) { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideInvalidLabel, Severity: SeverityError, + Message: "override record " + strconv.Itoa(index+1) + " requires a safe nonempty flag name", + }) + } + } + if len(diagnostics) != 0 { + return nil, cloneSourceInfos(sources), canonicalDiagnostics(diagnostics) + } + + prepared := make([]preparedOverride, len(overrides)) + allSources := cloneSourceInfos(sources) + for index, override := range overrides { + sourceID := "source-" + strconv.Itoa(len(sources)+index+1) + info := SourceInfo{ID: sourceID, Label: override.Flag, Layer: LayerCLI} + allSources = append(allSources, info) + prepared[index].source = info + } + + seen := make(map[string]int, len(overrides)) + pathAccepted := make([]bool, len(overrides)) + var usedBytes int64 + usedNodes := 0 + for index, override := range overrides { + location := &Location{SourceID: prepared[index].source.ID} + if issue := validateOverridePath(override.Path, base.root); issue != nil { + issue.Primary = location + diagnostics = append(diagnostics, *issue) + continue + } + pointer := override.Path.String() + if previous, duplicate := seen[pointer]; duplicate { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideDuplicate, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration override path is repeated", Primary: location, + Related: []Location{{SourceID: prepared[previous].source.ID}}, + }) + continue + } + seen[pointer] = index + if conflicting := overlappingOverride(overrides[:index], pathAccepted[:index], override.Path); conflicting >= 0 { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideOverlap, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration override paths overlap", Primary: location, + Related: []Location{{SourceID: prepared[conflicting].source.ID}}, + }) + continue + } + if rule != nil && (pathContains(override.Path, rule.Path) || pathContains(rule.Path, override.Path)) { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideVersion, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides cannot replace the schema-version path", Primary: location, + }) + continue + } + if pathContainsDirective(override.Path) { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideDirective, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides cannot create configuration directives", Primary: location, + }) + continue + } + pathAccepted[index] = true + + builder := overrideValueBuilder{sourceID: prepared[index].source.ID, active: make(map[valueVisit]bool)} + value, issue := builder.build(override.Value, override.Path, len(override.Path.segments)) + if issue != nil { + issue.Primary = location + diagnostics = append(diagnostics, *issue) + continue + } + if relative, ok := overrideDirectivePath(value, Path{}); ok { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideDirective, Severity: SeverityError, Path: appendOverridePath(override.Path, relative), + Message: "configuration overrides cannot create configuration directives", Primary: location, + }) + continue + } + nodes := builder.nodes + len(override.Path.segments)*2 + if nodes > limits.nodes || base.inputNodes+usedNodes+nodes > limits.nodes { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideLimitExceeded, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides exceed the aggregate node limit", Primary: location, + }) + continue + } + if builder.bytes > maxSourceBytes || base.inputBytes+usedBytes+builder.bytes > limits.bytes { + diagnostics = append(diagnostics, Diagnostic{ + Code: CodeOverrideLimitExceeded, Severity: SeverityError, Path: override.Path.clone(), + Message: "configuration overrides exceed a record or aggregate string-byte limit", Primary: location, + }) + continue + } + prepared[index].path = override.Path.clone() + prepared[index].value = value + prepared[index].bytes = builder.bytes + prepared[index].nodes = nodes + usedBytes += builder.bytes + usedNodes += nodes + } + if len(diagnostics) != 0 { + return nil, allSources, canonicalDiagnostics(diagnostics) + } + + return prepared, allSources, nil +} + +func validOverrideLabel(label string) bool { + if label == "" || strings.TrimSpace(label) != label || !utf8.ValidString(label) { + return false + } + for _, character := range label { + if unicode.IsControl(character) { + return false + } + } + return true +} + +func validateOverridePath(path Path, root *document.Node) *Diagnostic { + if !path.valid() || len(path.segments) == 0 { + return &Diagnostic{Code: CodeOverrideInvalidPath, Severity: SeverityError, Path: path.clone(), Message: "configuration override path must be a valid non-root mapping path"} + } + current := root + for index, segment := range path.segments { + if segment.kind == pathSegmentIndex { + return &Diagnostic{Code: CodeOverrideInvalidPath, Severity: SeverityError, Path: path.clone(), Message: "configuration override paths cannot contain sequence indices"} + } + if current == nil { + continue + } + if current.Kind != document.Mapping { + return &Diagnostic{Code: CodeOverrideInvalidPath, Severity: SeverityError, Path: path.clone(), Message: "configuration override path traverses a non-mapping value"} + } + position := documentEntryIndex(current.Mapping, segment.key) + if index == len(path.segments)-1 { + continue + } + if position < 0 { + current = nil + continue + } + current = current.Mapping[position].Value + } + return nil +} + +func overlappingOverride(previous []Override, valid []bool, path Path) int { + for index, override := range previous { + if valid[index] && (pathContains(override.Path, path) || pathContains(path, override.Path)) { + return index + } + } + return -1 +} + +func pathContainsDirective(path Path) bool { + for _, segment := range path.segments { + if segment.key == directiveKey { + return true + } + } + return false +} + +func appendOverridePath(base, relative Path) Path { + result := base.clone() + result.segments = append(result.segments, clonePathSegments(relative.segments, 0)...) + return result +} + +func overrideDirectivePath(node *document.Node, path Path) (Path, bool) { + if node.Kind == document.Mapping { + for _, entry := range node.Mapping { + child := path.Key(entry.Key) + if entry.Key == directiveKey { + return child, true + } + if found, ok := overrideDirectivePath(entry.Value, child); ok { + return found, true + } + } + } else if node.Kind == document.Sequence { + for index, child := range node.Sequence { + if found, ok := overrideDirectivePath(child, path.Index(index)); ok { + return found, true + } + } + } + return Path{}, false +} diff --git a/internal/engine/override_apply.go b/internal/engine/override_apply.go new file mode 100644 index 0000000..0add545 --- /dev/null +++ b/internal/engine/override_apply.go @@ -0,0 +1,100 @@ +package engine + +import ( + "slices" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +func applyOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { + if len(overrides) == 0 { + return base, cloneSourceInfos(sources), nil + } + prepared, allSources, diagnostics := prepareOverrides(base, sources, overrides, rule, limits) + if len(diagnostics) != 0 { + return nil, allSources, diagnostics + } + result := cloneComposedDocument(base) + for _, override := range prepared { + applyPreparedOverride(result, override) + result.inputBytes += override.bytes + result.inputNodes += override.nodes + } + return result, allSources, nil +} + +func applyPreparedOverride(result *composedDocument, override preparedOverride) { + location := document.Location{SourceID: override.source.ID} + mergeOverlayOrigin(result.origins, Path{}, location, LayerCLI) + current := result.root + currentPath := Path{} + for index, segment := range override.path.segments { + path := currentPath.Key(segment.key) + position := documentEntryIndex(current.Mapping, segment.key) + if index < len(override.path.segments)-1 { + if position < 0 { + child := &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{}} + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: child}) + collectOverlayOrigins(result.origins, child, path, LayerCLI) + current = child + } else { + current = current.Mapping[position].Value + mergeOverlayOrigin(result.origins, path, location, LayerCLI) + } + currentPath = path + continue + } + + previous := cloneOrigin(result.origins[path.String()]) + var displaced []displacedDefinition + if position < 0 { + incoming := document.Clone(override.value) + current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: incoming}) + collectOverlayOrigins(result.origins, incoming, path, LayerCLI) + } else { + existing := current.Mapping[position].Value + displaced = affectedOverrideOrigins(existing, override.value, result.origins, path) + if existing.Kind == document.Mapping && override.value.Kind == document.Mapping { + mergeOverlayOrigin(result.origins, path, location, LayerCLI) + mergeOverlayMappings(result, existing, override.value, path, LayerCLI, false) + } else { + deleteOriginSubtree(result.origins, path) + current.Mapping[position] = document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: document.Clone(override.value)} + collectOverlayOrigins(result.origins, current.Mapping[position].Value, path, LayerCLI) + } + } + result.overlays = append(result.overlays, overlayRecord{change: Change{ + Path: path.clone(), Origin: overlayOrigin(LayerCLI, location), Previous: previousOrigins(previous), + }, displaced: displaced}) + result.applications = append(result.applications, historyApplication{ + layer: LayerCLI, path: path.clone(), value: override.value, + origin: overlayOrigin(LayerCLI, location), + }) + } +} + +func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path) []displacedDefinition { + displaced := []displacedDefinition{{path: path.clone(), origin: cloneOrigin(origins[path.String()])}} + if existing.Kind == document.Mapping && incoming.Kind == document.Mapping { + for _, entry := range incoming.Mapping { + position := documentEntryIndex(existing.Mapping, entry.Key) + if position < 0 { + continue + } + childPath := path.Key(entry.Key) + childExisting := existing.Mapping[position].Value + if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping { + displaced = append(displaced, affectedOverrideOrigins(childExisting, entry.Value, origins, childPath)...) + } else { + displaced = append(displaced, displacedOrigins(childExisting, origins, childPath)...) + } + } + } else { + displaced = displacedOrigins(existing, origins, path) + } + slices.SortFunc(displaced, func(left, right displacedDefinition) int { + return strings.Compare(left.path.String(), right.path.String()) + }) + return displaced +} diff --git a/internal/engine/override_test.go b/internal/engine/override_test.go new file mode 100644 index 0000000..9ea6dd7 --- /dev/null +++ b/internal/engine/override_test.go @@ -0,0 +1,307 @@ +package engine + +import ( + "errors" + "math" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type overrideLifecycleSettings struct { + Target string `yaml:"target"` + Count int `yaml:"count"` + Enabled bool `yaml:"enabled"` + Text string `yaml:"text"` + Items []string `yaml:"items"` + Maybe *string `yaml:"maybe"` + Labels map[string]string `yaml:"labels"` + APIKey Secret `yaml:"api_key"` +} + +type overrideLifecycleConfig struct { + Version string `yaml:"version"` + Settings overrideLifecycleSettings `yaml:"settings"` +} + +func TestOverridesRunAtHighestInputPrecedenceBeforeResolutionAndValidation(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports: [{path: base.yml}]\nversion: v1\n") + writeImportFile(t, directory, "base.yml", `settings: + target: base + count: 4 + enabled: true + text: base + items: [base] + maybe: base + labels: {base: retained} + api_key: BASE_SECRET +`) + writeImportFile(t, directory, filepath.Join("profiles", "testing.yml"), `settings: + target: profile + labels: {profile: retained} +`) + validated := false + overrides := []Override{ + {Path: Keys("settings", "target"), Value: " cli ", Flag: "--target"}, + {Path: Keys("settings", "count"), Value: 0, Flag: "--count"}, + {Path: Keys("settings", "enabled"), Value: false, Flag: "--enabled"}, + {Path: Keys("settings", "text"), Value: "", Flag: "--text"}, + {Path: Keys("settings", "items"), Value: []any{}, Flag: "--items"}, + {Path: Keys("settings", "maybe"), Value: nil, Flag: "--maybe"}, + {Path: Keys("settings", "labels"), Value: map[string]any{"cli": "selected"}, Flag: "--label"}, + {Path: Keys("settings", "api_key"), Value: "CLI_SECRET", Flag: "--api-key-env"}, + } + result, err := loadWithSecretLookup(File(mainPath), func(name string) (string, bool) { + if name != "CLI_SECRET" { + t.Fatalf("looked up superseded secret name %q", name) + } + return "credential-value", true + }, Options[overrideLifecycleConfig]{ + Imports: true, Profile: "testing", Overrides: overrides, + Definition: Definition[overrideLifecycleConfig]{ + Defaults: func() overrideLifecycleConfig { + return overrideLifecycleConfig{Settings: overrideLifecycleSettings{Target: "default", Count: 9, Text: "default", Items: []string{"default"}}} + }, + Version: &VersionRule{Path: Keys("version"), Check: func(version Version, validation *Validation) error { + if version.Text != "v1" { + validation.Error(Keys("version"), "unsupported version") + } + return nil + }}, + Resolve: func(configuration *overrideLifecycleConfig, resolution *Resolution) error { + Normalize(resolution, Keys("settings", "target"), &configuration.Settings.Target, strings.TrimSpace(configuration.Settings.Target), "trimmed target") + return nil + }, + Validate: func(configuration *overrideLifecycleConfig, validation *Validation) { + validated = configuration.Settings.Target == "cli" && configuration.Settings.APIKey.Value() == "credential-value" + }, + }, + }) + if err != nil { + t.Fatalf("loadWithSecretLookup() error = %v", err) + } + want := overrideLifecycleSettings{ + Target: "cli", Labels: map[string]string{"base": "retained", "profile": "retained", "cli": "selected"}, + APIKey: result.Config.Settings.APIKey, + } + if !validated || result.Config.Settings.Count != 0 || result.Config.Settings.Enabled || result.Config.Settings.Text != "" || + len(result.Config.Settings.Items) != 0 || result.Config.Settings.Items == nil || result.Config.Settings.Maybe != nil || + result.Config.Settings.Target != want.Target || !reflect.DeepEqual(result.Config.Settings.Labels, want.Labels) { + t.Fatalf("effective settings = %#v", result.Config.Settings) + } + sources := result.Sources() + if len(sources) != 3+len(overrides) || sources[0].Layer != LayerBase || sources[1].Layer != LayerBase || sources[2].Layer != LayerProfile { + t.Fatalf("sources = %#v", sources) + } + for index, override := range overrides { + if sources[index+3].Layer != LayerCLI || sources[index+3].Label != override.Flag || sources[index+3].Path != "" { + t.Fatalf("CLI source %d = %#v", index, sources[index+3]) + } + } + history := result.Metadata().History(Keys("settings", "target")) + wantLayers := []Layer{LayerDefault, LayerBase, LayerProfile, LayerCLI, LayerResolution} + if got := historyLayers(history); !reflect.DeepEqual(got, wantLayers) { + t.Fatalf("target history layers = %#v, want %#v; history %#v", got, wantLayers, history) + } +} + +func TestOverridesSupportDefaultsOnlyMapMergeListReplacementAndNumericKeys(t *testing.T) { + type config struct { + Labels map[string]string `yaml:"labels"` + Items []int `yaml:"items"` + Numbers map[string]int `yaml:"numbers"` + Dynamic any `yaml:"dynamic"` + } + labels := map[string]any{"0": "zero", "new": "value"} + items := []any{0, 2} + dynamic := map[string]any{ + "int": int8(1), "uint": uint16(2), "float": float32(3.5), "bool": false, + "null": nil, "list": []any{"value"}, "map": map[string]any{"key": "value"}, + } + result, err := Load(NoSource(), Options[config]{ + Definition: Definition[config]{Defaults: func() config { + return config{Labels: map[string]string{"kept": "default"}, Items: []int{9}} + }}, + Overrides: []Override{ + {Path: parseOverrideTestPath(t, "/labels"), Value: labels, Flag: "--labels"}, + {Path: Keys("items"), Value: items, Flag: "--items"}, + {Path: parseOverrideTestPath(t, "/numbers/0"), Value: 7, Flag: "--number"}, + {Path: Keys("dynamic"), Value: dynamic, Flag: "--dynamic"}, + }, + }) + labels["new"] = "mutated" + items[0] = 99 + dynamic["int"] = 99 + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Config.Labels, map[string]string{"kept": "default", "0": "zero", "new": "value"}) || !reflect.DeepEqual(result.Config.Items, []int{0, 2}) || result.Config.Numbers["0"] != 7 || result.Config.Dynamic.(map[string]any)["int"] != int64(1) { + t.Fatalf("defaults-only overrides = %#v", result.Config) + } + var effective strings.Builder + if err := result.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + if strings.Contains(effective.String(), "mutated") || !strings.Contains(effective.String(), "new: value") { + t.Fatalf("detached effective output = %q", effective.String()) + } + if info, ok := result.Metadata().Lookup(Keys("labels", "0")); !ok || !info.Explicit || info.Origin.Layer != LayerCLI { + t.Fatalf("numeric mapping key metadata = %#v, %v", info, ok) + } +} + +func TestOverrideValidationRejectsPathsOverlapValuesAndProtectedTargets(t *testing.T) { + type config struct { + Version string `yaml:"version"` + Scalar string `yaml:"scalar"` + Items []string `yaml:"items"` + Nested map[string]any `yaml:"nested"` + Unknown string `yaml:"unknown"` + } + base := Bytes("override-errors.yml", []byte("version: 4\nscalar: value\nitems: [one]\nnested: {}\n")) + version := &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }} + cycle := map[string]any{} + cycle["self"] = cycle + deep := any("leaf") + for range maxDocumentDepth + 1 { + deep = []any{deep} + } + for _, test := range []struct { + name string + overrides []Override + code Code + }{ + {name: "root", overrides: []Override{{Path: Keys(), Value: "x", Flag: "--root"}}, code: CodeOverrideInvalidPath}, + {name: "index", overrides: []Override{{Path: Keys("items").Index(0), Value: "x", Flag: "--item"}}, code: CodeOverrideInvalidPath}, + {name: "parsed sequence index", overrides: []Override{{Path: parseOverrideTestPath(t, "/items/0"), Value: "x", Flag: "--item"}}, code: CodeOverrideInvalidPath}, + {name: "scalar ancestor", overrides: []Override{{Path: Keys("scalar", "child"), Value: "x", Flag: "--child"}}, code: CodeOverrideInvalidPath}, + {name: "duplicate", overrides: []Override{{Path: Keys("scalar"), Value: "a", Flag: "--first"}, {Path: Keys("scalar"), Value: "b", Flag: "--second"}}, code: CodeOverrideDuplicate}, + {name: "overlap", overrides: []Override{{Path: Keys("nested"), Value: map[string]any{}, Flag: "--nested"}, {Path: Keys("nested", "child"), Value: "x", Flag: "--child"}}, code: CodeOverrideOverlap}, + {name: "unsupported", overrides: []Override{{Path: Keys("scalar"), Value: time.Second, Flag: "--scalar"}}, code: CodeOverrideInvalidValue}, + {name: "cycle", overrides: []Override{{Path: Keys("nested"), Value: cycle, Flag: "--nested"}}, code: CodeOverrideInvalidValue}, + {name: "nonfinite", overrides: []Override{{Path: Keys("scalar"), Value: math.Inf(1), Flag: "--scalar"}}, code: CodeOverrideInvalidValue}, + {name: "depth", overrides: []Override{{Path: Keys("nested"), Value: deep, Flag: "--nested"}}, code: CodeOverrideLimitExceeded}, + {name: "record bytes", overrides: []Override{{Path: Keys("scalar"), Value: strings.Repeat("x", maxSourceBytes+1), Flag: "--scalar"}}, code: CodeOverrideLimitExceeded}, + {name: "directive path", overrides: []Override{{Path: Keys("configkit", "x"), Value: true, Flag: "--directive"}}, code: CodeOverrideDirective}, + {name: "directive value", overrides: []Override{{Path: Keys("nested"), Value: map[string]any{"configkit": map[string]any{}}, Flag: "--nested"}}, code: CodeOverrideDirective}, + {name: "version", overrides: []Override{{Path: Keys("version"), Value: "v2", Flag: "--version"}}, code: CodeOverrideVersion}, + {name: "version ancestor", overrides: []Override{{Path: Keys("version"), Value: map[string]any{"child": "v2"}, Flag: "--version"}}, code: CodeOverrideVersion}, + {name: "label", overrides: []Override{{Path: Keys("scalar"), Value: "x", Flag: "\nCANARY"}}, code: CodeOverrideInvalidLabel}, + } { + t.Run(test.name, func(t *testing.T) { + result, err := Load(base, Options[config]{Definition: Definition[config]{Version: version}, Overrides: test.overrides}) + if result != nil { + t.Fatalf("result = %#v, want nil", result) + } + loadError := requireSecretLoadCode(t, err, test.code) + if strings.Contains(fmtLoadError(loadError), strings.Repeat("x", 64)) { + t.Fatal("override diagnostic leaked a raw value") + } + }) + } +} + +func TestOverrideCannotReplaceSchemaVersionAncestor(t *testing.T) { + type schemaConfig struct { + Version string `yaml:"version"` + } + type config struct { + Schema schemaConfig `yaml:"schema"` + } + _, err := Load(Bytes("version.yml", []byte("schema: {version: 4}\n")), Options[config]{ + Definition: Definition[config]{Version: &VersionRule{Path: Keys("schema", "version"), Check: func(Version, *Validation) error { return nil }}}, + Overrides: []Override{{Path: Keys("schema"), Value: map[string]any{"other": "value"}, Flag: "--schema"}}, + }) + requireSecretLoadCode(t, err, CodeOverrideVersion) +} + +func TestOverrideDecodeFailureUsesFlagSourceWithoutRawValue(t *testing.T) { + type config struct { + Count int `yaml:"count"` + } + result, err := Load(Bytes("base.yml", []byte("count: 1\n")), Options[config]{Overrides: []Override{{ + Path: Keys("missing"), Value: "RAW-ARG-CANARY", Flag: "--missing", + }}}) + if result != nil { + t.Fatalf("result = %#v", result) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error = %T", err) + } + diagnostics := loadError.Diagnostics() + sources := loadError.Sources() + if len(diagnostics) != 1 || diagnostics[0].Code != CodeDecodeUnknownField || diagnostics[0].Primary == nil || + len(sources) != 2 || sources[1].Label != "--missing" || diagnostics[0].Primary.SourceID != sources[1].ID { + t.Fatalf("diagnostics/sources = %#v / %#v", diagnostics, sources) + } + if strings.Contains(fmtLoadError(loadError), "RAW-ARG-CANARY") { + t.Fatal("decode failure leaked override value") + } +} + +func TestOverrideTypeMismatchUsesNormalStrictDecode(t *testing.T) { + type config struct { + Count int `yaml:"count"` + } + _, err := Load(Bytes("base.yml", []byte("count: 1\n")), Options[config]{Overrides: []Override{{ + Path: Keys("count"), Value: "not-an-integer", Flag: "--count", + }}}) + loadError := requireSecretLoadCode(t, err, CodeDecodeType) + if loadError.Diagnostics()[0].Primary == nil || loadError.Sources()[1].Label != "--count" || strings.Contains(fmtLoadError(loadError), "not-an-integer") { + t.Fatalf("type mismatch diagnostics/sources = %#v / %#v", loadError.Diagnostics(), loadError.Sources()) + } +} + +func TestOverrideEqualReplacementStillAppearsInHistory(t *testing.T) { + type config struct { + Value string `yaml:"value"` + } + result, err := Load(Bytes("equal.yml", []byte("value: same\n")), Options[config]{Overrides: []Override{{Path: Keys("value"), Value: "same", Flag: "--value"}}}) + if err != nil { + t.Fatal(err) + } + if got := historyLayers(result.Metadata().History(Keys("value"))); !reflect.DeepEqual(got, []Layer{LayerDefault, LayerBase, LayerCLI}) { + t.Fatalf("equal override history = %#v", result.Metadata().History(Keys("value"))) + } +} + +func TestOverrideAggregateLimitsFailBeforeApplication(t *testing.T) { + base, diagnostics := composeDocuments(&document.Node{Kind: document.Mapping}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + base.inputNodes = 1 + result, sources, diagnostics := applyOverrides(base, nil, []Override{{Path: Keys("value"), Value: "x", Flag: "--value"}}, nil, importLimits{sources: 1, bytes: 1, nodes: 2}) + if result != nil || len(sources) != 1 || len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded || len(base.root.Mapping) != 0 { + t.Fatalf("bounded apply = %#v, %#v, %#v; base %#v", result, sources, diagnostics, base.root) + } +} + +func historyLayers(history []Change) []Layer { + layers := make([]Layer, len(history)) + for index, change := range history { + layers[index] = change.Origin.Layer + } + return layers +} + +func parseOverrideTestPath(t *testing.T, value string) Path { + t.Helper() + path, err := ParsePath(value) + if err != nil { + t.Fatal(err) + } + return path +} + +func fmtLoadError(loadError *LoadError) string { + var output strings.Builder + _ = WriteDiagnostics(&output, loadError.Diagnostics()) + return loadError.Error() + output.String() +} diff --git a/internal/engine/override_value.go b/internal/engine/override_value.go new file mode 100644 index 0000000..3f3ecfc --- /dev/null +++ b/internal/engine/override_value.go @@ -0,0 +1,150 @@ +package engine + +import ( + "math" + "reflect" + "slices" + "strconv" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type overrideValueBuilder struct { + sourceID string + active map[valueVisit]bool + bytes int64 + nodes int +} + +func (b *overrideValueBuilder) build(value any, path Path, depth int) (*document.Node, *Diagnostic) { + if depth > maxDocumentDepth { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the depth limit") + } + b.nodes++ + if b.nodes > maxDocumentNodes { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the node limit") + } + location := document.Location{SourceID: b.sourceID} + switch value := value.(type) { + case nil: + return &document.Node{Kind: document.Null, Location: location}, nil + case string: + if !b.addBytes(len(value)) { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the record string-byte limit") + } + return &document.Node{Kind: document.String, Text: value, Location: location}, nil + case bool: + return &document.Node{Kind: document.Boolean, Text: strconv.FormatBool(value), Location: location}, nil + case int: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int8: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int16: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int32: + return overrideNumber(strconv.FormatInt(int64(value), 10), location), nil + case int64: + return overrideNumber(strconv.FormatInt(value, 10), location), nil + case uint: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint8: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint16: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint32: + return overrideNumber(strconv.FormatUint(uint64(value), 10), location), nil + case uint64: + return overrideNumber(strconv.FormatUint(value, 10), location), nil + case float32: + if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) { + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override float must be finite") + } + return overrideNumber(strconv.FormatFloat(float64(value), 'g', -1, 32), location), nil + case float64: + if math.IsNaN(value) || math.IsInf(value, 0) { + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override float must be finite") + } + return overrideNumber(strconv.FormatFloat(value, 'g', -1, 64), location), nil + case []any: + if value == nil { + return &document.Node{Kind: document.Null, Location: location}, nil + } + leave, diagnostic := b.enter(reflect.ValueOf(value), path) + if diagnostic != nil { + return nil, diagnostic + } + defer leave() + if len(value) > maxDocumentNodes-b.nodes { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the node limit") + } + node := &document.Node{Kind: document.Sequence, Location: location, Sequence: make([]*document.Node, len(value))} + for index, child := range value { + built, diagnostic := b.build(child, path.Index(index), depth+1) + if diagnostic != nil { + return nil, diagnostic + } + node.Sequence[index] = built + } + return node, nil + case map[string]any: + if value == nil { + return &document.Node{Kind: document.Null, Location: location}, nil + } + leave, diagnostic := b.enter(reflect.ValueOf(value), path) + if diagnostic != nil { + return nil, diagnostic + } + defer leave() + if len(value) > (maxDocumentNodes-b.nodes)/2 { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path, "configuration override exceeds the node limit") + } + keys := make([]string, 0, len(value)) + for key := range value { + keys = append(keys, key) + } + slices.Sort(keys) + node := &document.Node{Kind: document.Mapping, Location: location, Mapping: make([]document.Entry, 0, len(keys))} + for _, key := range keys { + b.nodes++ // mapping key + if !b.addBytes(len(key)) { + return nil, overrideValueDiagnostic(CodeOverrideLimitExceeded, path.Key(key), "configuration override exceeds the record string-byte limit") + } + built, diagnostic := b.build(value[key], path.Key(key), depth+1) + if diagnostic != nil { + return nil, diagnostic + } + node.Mapping = append(node.Mapping, document.Entry{Key: key, KeyLocation: location, ValueLocation: location, Value: built}) + } + return node, nil + default: + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override value has an unsupported type") + } +} + +func (b *overrideValueBuilder) addBytes(count int) bool { + if count < 0 || int64(count) > maxSourceBytes-b.bytes { + return false + } + b.bytes += int64(count) + return true +} + +func (b *overrideValueBuilder) enter(value reflect.Value, path Path) (func(), *Diagnostic) { + visit, ok := visitFor(value) + if !ok { + return func() {}, nil + } + if b.active[visit] { + return nil, overrideValueDiagnostic(CodeOverrideInvalidValue, path, "configuration override values must not contain cycles") + } + b.active[visit] = true + return func() { delete(b.active, visit) }, nil +} + +func overrideValueDiagnostic(code Code, path Path, message string) *Diagnostic { + return &Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message} +} + +func overrideNumber(text string, location document.Location) *document.Node { + return &document.Node{Kind: document.Number, Text: text, Location: location} +} diff --git a/internal/engine/parse.go b/internal/engine/parse.go new file mode 100644 index 0000000..6f162c6 --- /dev/null +++ b/internal/engine/parse.go @@ -0,0 +1,62 @@ +package engine + +import ( + "bytes" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +const ( + maxDocumentDepth = 256 + maxDocumentNodes = 1_000_000 +) + +type parseLimits struct { + source sourceLimits + document yamladapter.Limits +} + +func defaultParseLimits() parseLimits { + return parseLimits{ + source: defaultSourceLimits(), + document: yamladapter.Limits{ + Depth: maxDocumentDepth, + Nodes: maxDocumentNodes, + }, + } +} + +type parsedSource struct { + document *document.Node + source *SourceInfo + bytes int64 +} + +// parserDiagnostics translates adapter issues into detached, ordered public records. +func parserDiagnostics(sourceID string, issues []yamladapter.Issue) []Diagnostic { + diagnostics := make([]Diagnostic, len(issues)) + for index, issue := range issues { + diagnostics[index] = Diagnostic{ + Code: Code(issue.Code), Severity: SeverityError, Message: issue.Message, + Primary: &Location{SourceID: sourceID, Line: issue.Line, Column: issue.Column}, + } + for _, related := range issue.Related { + diagnostics[index].Related = append(diagnostics[index].Related, publicLocation(related)) + } + } + return canonicalDiagnostics(diagnostics) +} + +func parseSource(source Source, sourceID string, limits parseLimits) (parsedSource, []Diagnostic) { + read, diagnostics := readSource(source, sourceID, limits.source) + if len(diagnostics) != 0 || read.info == nil { + return parsedSource{source: read.info}, diagnostics + } + + root, issues := yamladapter.Parse(bytes.NewReader(read.data), sourceID, limits.document) + if len(issues) != 0 { + return parsedSource{source: read.info, bytes: int64(len(read.data))}, parserDiagnostics(sourceID, issues) + } + return parsedSource{document: root, source: read.info, bytes: int64(len(read.data))}, nil +} diff --git a/internal/engine/parse_test.go b/internal/engine/parse_test.go new file mode 100644 index 0000000..03e9e26 --- /dev/null +++ b/internal/engine/parse_test.go @@ -0,0 +1,38 @@ +package engine + +import ( + "strings" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +func TestParseSourceOwnsDocumentAndSafeDiagnostics(t *testing.T) { + t.Parallel() + + limits := defaultParseLimits() + parsed, diagnostics := parseSource(Bytes("memory.yml", []byte("name: config\n")), "source-1", limits) + if len(diagnostics) != 0 || parsed.source == nil || parsed.source.Label != "memory.yml" || parsed.document.Kind != document.Mapping { + t.Fatalf("parse result = %#v, diagnostics = %#v", parsed, diagnostics) + } + + const canary = "CANARY_PRIVATE_INPUT" + failed, diagnostics := parseSource(Bytes("unsafe\nlabel", []byte("name: ["+canary+"\n")), "source-1", limits) + if failed.document != nil || failed.source == nil || len(diagnostics) != 1 || diagnostics[0].Code != CodeYAMLInvalid { + t.Fatalf("failed parse = %#v, diagnostics = %#v", failed, diagnostics) + } + var output strings.Builder + if err := WriteDiagnostics(&output, diagnostics); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), canary) || strings.Count(output.String(), "\n") != 1 { + t.Fatalf("unsafe rendered diagnostic = %q", output.String()) + } + + limits.document = yaml.Limits{Depth: 1, Nodes: 20, ParserDepth: 20} + _, diagnostics = parseSource(Bytes("deep", []byte("a:\n b: value\n")), "source-1", limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeYAMLDepthLimit || diagnostics[0].Primary.Line == 0 { + t.Fatalf("depth diagnostics = %#v", diagnostics) + } +} diff --git a/internal/engine/path.go b/internal/engine/path.go new file mode 100644 index 0000000..faff696 --- /dev/null +++ b/internal/engine/path.go @@ -0,0 +1,148 @@ +package engine + +import ( + "errors" + "strconv" + "strings" +) + +// Path identifies a configuration value using YAML field names. Its textual form +// is a JSON Pointer: the root is empty, ~ escapes as ~0, and / escapes as ~1. +// Paths are semantic rather than filesystem or URL paths: empty, dot, dot-dot, +// percent-containing, and numeric keys retain their literal meaning. Parsed +// numeric tokens are resolved only when traversal knows their containing type. +// +// Paths are immutable. The zero value is the root path. +type Path struct { + segments []pathSegment + invalid bool +} + +type pathSegmentKind uint8 + +const ( + pathSegmentKey pathSegmentKind = iota + pathSegmentIndex + pathSegmentParsed +) + +type pathSegment struct { + kind pathSegmentKind + key string + index int +} + +// Keys constructs a path of mapping keys. Keys with no arguments returns the +// root path. +func Keys(keys ...string) Path { + p := Path{} + for _, key := range keys { + p = p.Key(key) + } + return p +} + +// ParsePath parses a JSON Pointer. Numeric tokens remain unresolved until schema +// or document traversal determines whether the containing value is a mapping or +// sequence. +func ParsePath(pointer string) (Path, error) { + if pointer == "" { + return Path{}, nil + } + if pointer[0] != '/' { + return Path{}, errors.New("configkit: path must be empty or begin with slash") + } + + parts := strings.Split(pointer[1:], "/") + segments := make([]pathSegment, 0, len(parts)) + for _, part := range parts { + key, ok := unescapePathToken(part) + if !ok { + return Path{}, errors.New("configkit: path contains an invalid tilde escape") + } + segments = append(segments, pathSegment{kind: pathSegmentParsed, key: key}) + } + return Path{segments: segments}, nil +} + +// Key returns a new path with key appended as a mapping-key segment. +func (p Path) Key(key string) Path { + segments := clonePathSegments(p.segments, 1) + segments = append(segments, pathSegment{kind: pathSegmentKey, key: key}) + return Path{segments: segments, invalid: p.invalid} +} + +// Index returns a new path with index appended as a sequence-index segment. +// A negative index produces an invalid path that configuration operations reject. +func (p Path) Index(index int) Path { + segments := clonePathSegments(p.segments, 1) + segments = append(segments, pathSegment{kind: pathSegmentIndex, index: index}) + return Path{segments: segments, invalid: p.invalid || index < 0} +} + +// String returns the path's canonical JSON Pointer spelling. +func (p Path) String() string { + if len(p.segments) == 0 { + return "" + } + + var b strings.Builder + for _, segment := range p.segments { + b.WriteByte('/') + if segment.kind == pathSegmentIndex { + b.WriteString(strconv.Itoa(segment.index)) + continue + } + b.WriteString(escapePathToken(segment.key)) + } + return b.String() +} + +func (p Path) clone() Path { + return Path{segments: clonePathSegments(p.segments, 0), invalid: p.invalid} +} + +func (p Path) valid() bool { + return !p.invalid +} + +func clonePathSegments(segments []pathSegment, extra int) []pathSegment { + if len(segments) == 0 && extra == 0 { + return nil + } + cloned := make([]pathSegment, len(segments), len(segments)+extra) + copy(cloned, segments) + return cloned +} + +func escapePathToken(token string) string { + token = strings.ReplaceAll(token, "~", "~0") + return strings.ReplaceAll(token, "/", "~1") +} + +func unescapePathToken(token string) (string, bool) { + if !strings.Contains(token, "~") { + return token, true + } + + var b strings.Builder + for i := 0; i < len(token); i++ { + if token[i] != '~' { + b.WriteByte(token[i]) + continue + } + if i+1 >= len(token) { + return "", false + } + i++ + switch token[i] { + case '0': + b.WriteByte('~') + case '1': + b.WriteByte('/') + default: + return "", false + } + } + return b.String(), true +} diff --git a/internal/engine/path_fuzz_test.go b/internal/engine/path_fuzz_test.go new file mode 100644 index 0000000..53f253d --- /dev/null +++ b/internal/engine/path_fuzz_test.go @@ -0,0 +1,27 @@ +package engine + +import "testing" + +func FuzzParsePath(f *testing.F) { + for _, seed := range []string{"", "/", "/service/port", "/a~1b/~0key", "/0", "relative", "/bad~2escape"} { + f.Add(seed) + } + + f.Fuzz(func(t *testing.T, input string) { + if len(input) > 4096 { + return + } + path, err := ParsePath(input) + if err != nil { + return + } + canonical := path.String() + reparsed, err := ParsePath(canonical) + if err != nil { + t.Fatalf("canonical path %q did not parse: %v", canonical, err) + } + if reparsed.String() != canonical { + t.Fatalf("path round trip = %q, want %q", reparsed.String(), canonical) + } + }) +} diff --git a/internal/engine/path_test.go b/internal/engine/path_test.go new file mode 100644 index 0000000..e47d60b --- /dev/null +++ b/internal/engine/path_test.go @@ -0,0 +1,88 @@ +package engine + +import "testing" + +func TestPathRoundTripsSemanticKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + pointer string + }{ + {name: "root", pointer: ""}, + {name: "slash and tilde", pointer: "/a~1b/c~0d"}, + {name: "empty key", pointer: "/"}, + {name: "dotted key", pointer: "/outputs/a.b"}, + {name: "numeric key", pointer: "/outputs/0"}, + {name: "dot keys", pointer: "/./.."}, + {name: "percent text", pointer: "/%2F/%7E"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + parsed, err := ParsePath(test.pointer) + if err != nil { + t.Fatalf("ParsePath() error = %v", err) + } + if got := parsed.String(); got != test.pointer { + t.Fatalf("String() = %q, want %q", got, test.pointer) + } + }) + } + + if got := Keys("a/b", "c~d", "", "..").String(); got != "/a~1b/c~0d//.." { + t.Fatalf("Keys().String() = %q", got) + } + if got := Keys("items").Index(12).Key("name").String(); got != "/items/12/name" { + t.Fatalf("indexed path = %q", got) + } +} + +func TestPathPreservesSegmentIntentAndImmutability(t *testing.T) { + t.Parallel() + + key := Keys("items", "0") + index := Keys("items").Index(0) + parsed, err := ParsePath("/items/0") + if err != nil { + t.Fatal(err) + } + if key.String() != index.String() || index.String() != parsed.String() { + t.Fatal("equivalent textual paths should share a JSON Pointer spelling") + } + if key.segments[1].kind != pathSegmentKey { + t.Fatal("Keys numeric segment lost mapping-key intent") + } + if index.segments[1].kind != pathSegmentIndex { + t.Fatal("Index segment lost sequence intent") + } + if parsed.segments[1].kind != pathSegmentParsed { + t.Fatal("parsed numeric segment was resolved before traversal") + } + + base := Keys("parent") + child := base.Key("child") + child.segments[0].key = "changed" + if got := base.String(); got != "/parent" { + t.Fatalf("builder mutated parent: %q", got) + } + + negative := Path{}.Index(-1) + if negative.valid() { + t.Fatal("negative index must produce an invalid path") + } +} + +func TestParsePathRejectsMalformedPointers(t *testing.T) { + t.Parallel() + + for _, pointer := range []string{"relative", "~0", "/bad~", "/bad~2", "/bad~01~"} { + if _, err := ParsePath(pointer); err == nil { + t.Errorf("ParsePath(%q) succeeded", pointer) + } + } + + if parsed, err := ParsePath("/missing"); err != nil || !parsed.valid() { + t.Fatalf("valid missing path rejected: path=%v error=%v", parsed, err) + } +} diff --git a/internal/engine/profile.go b/internal/engine/profile.go new file mode 100644 index 0000000..a617fc0 --- /dev/null +++ b/internal/engine/profile.go @@ -0,0 +1,131 @@ +package engine + +import ( + "bytes" + "errors" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +func activeProfile(name string) bool { + return name != "" && name != "default" +} + +func validProfileName(name string) bool { + if !activeProfile(name) { + return true + } + for _, character := range name { + if (character >= 'a' && character <= 'z') || + (character >= 'A' && character <= 'Z') || + (character >= '0' && character <= '9') || character == '_' || character == '-' { + continue + } + return false + } + return true +} + +func loadProfileDocument(source Source, context *FilesystemContext, main parsedSource, base *composedDocument, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { + rootPath, _, _, diagnostic := importFilesystem(source, context, main) + if diagnostic != nil { + return nil, cloneSourceInfos(sources), []Diagnostic{*diagnostic} + } + profilePath := filepath.Join(rootPath, "profiles", name+".yml") + canonical, err := canonicalExistingPath(profilePath) + if err != nil { + code := CodeProfileInvalid + message := "selected profile could not be resolved" + if errors.Is(err, fs.ErrNotExist) { + code, message = CodeProfileMissing, "selected profile does not exist" + } else if errors.Is(err, fs.ErrPermission) { + code, message = CodeImportPermission, "selected profile cannot be accessed" + } + return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(code, message, nil)} + } + if !containedPath(rootPath, canonical) { + return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeProfileOutsideRoot, "selected profile resolves outside the configuration root", nil)} + } + for _, existing := range sources { + if existing.Path != "" && existing.Path == canonical { + return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeProfileRepeated, "selected profile is already present in the base source graph", nil)} + } + } + if len(sources)+1 > limits.sources { + return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeProfileLimitExceeded, "selected profile exceeds the source-count limit", nil)} + } + + root, err := os.OpenRoot(rootPath) + if err != nil { + value := globFilesystemDiagnostic(err, "configuration root could not be opened for the selected profile", Location{}) + value.Primary = nil + return nil, cloneSourceInfos(sources), []Diagnostic{value} + } + rootClosed := false + defer func() { + if !rootClosed { + _ = root.Close() + } + }() + relative, err := filepath.Rel(rootPath, canonical) + if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeProfileOutsideRoot, "selected profile could not be addressed inside the configuration root", nil)} + } + + sourceID := "source-" + strconv.Itoa(len(sources)+1) + profileLocation := &Location{SourceID: sourceID} + profileSource := SourceInfo{ID: sourceID, Label: filepath.ToSlash(filepath.Join("profiles", name+".yml")), Path: canonical, Layer: LayerProfile} + sources = append(cloneSourceInfos(sources), profileSource) + data, diagnostic := readProfileFile(root, relative, sourceID, limits.bytes-base.inputBytes) + if diagnostic != nil { + return nil, sources, []Diagnostic{*diagnostic} + } + profileRoot, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{Depth: maxDocumentDepth, Nodes: maxDocumentNodes}) + if len(issues) != 0 { + return nil, sources, parserDiagnostics(sourceID, issues) + } + nodeCount := countInputNodes(profileRoot) + if base.inputNodes+nodeCount > limits.nodes { + return nil, sources, []Diagnostic{importDiagnostic(CodeProfileLimitExceeded, "selected profile exceeds the aggregate node limit", profileLocation)} + } + if diagnostics := profileDirectiveDiagnostics(profileRoot, Path{}); len(diagnostics) != 0 { + return nil, sources, diagnostics + } + if versionNode := fragmentVersionClaim(profileRoot, rule); versionNode != nil { + location := publicLocation(versionNode.Location) + return nil, sources, []Diagnostic{{ + Code: CodeProfileVersion, Severity: SeverityError, Path: rule.Path.clone(), + Message: "profiles cannot contribute to or replace the schema-version path", Primary: &location, + }} + } + closeErr := root.Close() + rootClosed = true + if closeErr != nil { + return nil, sources, []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed after reading the selected profile", nil)} + } + + profiled, diagnostics := overlayDocuments(base, profileRoot) + if profiled != nil { + profiled.inputBytes = base.inputBytes + int64(len(data)) + profiled.inputNodes = base.inputNodes + nodeCount + } + return profiled, sources, diagnostics +} + +func readProfileFile(root *os.Root, relative, sourceID string, remaining int64) ([]byte, *Diagnostic) { + limit := min(int64(maxSourceBytes), remaining) + if limit < 0 { + limit = 0 + } + data, failure := readRegularFile(relative, root.Stat, root.Open, limit) + if failure != nil { + value := sourceReadDiagnostic(failure, sourceID, CodeProfileLimitExceeded, "selected profile exceeds a source or aggregate byte limit") + return nil, &value + } + return data, nil +} diff --git a/internal/engine/profile_test.go b/internal/engine/profile_test.go new file mode 100644 index 0000000..90935e5 --- /dev/null +++ b/internal/engine/profile_test.go @@ -0,0 +1,267 @@ +package engine + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +type profileTestConfig struct { + Settings struct { + Kept string `yaml:"kept"` + Changed string `yaml:"changed"` + Added string `yaml:"added"` + } `yaml:"settings"` + Values []string `yaml:"values"` + Nullable *string `yaml:"nullable"` + Value any `yaml:"value"` + Schema map[string]string `yaml:"schema"` +} + +func TestLoadAppliesSelectedProfileAfterBaseAndBeforeDefaults(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: [{path: base.yml}] +settings: + kept: main + changed: base +values: [base, stale] +nullable: base +value: {child: base} +`) + basePath := writeImportFile(t, directory, "base.yml", "settings: {added: imported}\n") + profilePath := writeImportFile(t, directory, "profiles/testing.yml", `settings: + changed: profile +values: [profile] +nullable: null +value: scalar +`) + + result, err := Load[profileTestConfig](File(mainPath), Options[profileTestConfig]{ + Imports: true, + Profile: "testing", + Definition: Definition[profileTestConfig]{Defaults: func() profileTestConfig { + defaultValue := "default" + return profileTestConfig{ + Values: []string{"default"}, Nullable: &defaultValue, + Settings: struct { + Kept string `yaml:"kept"` + Changed string `yaml:"changed"` + Added string `yaml:"added"` + }{Kept: "default", Changed: "default", Added: "default"}, + } + }}, + }) + if err != nil { + t.Fatal(err) + } + if result.Config.Settings.Kept != "main" || result.Config.Settings.Changed != "profile" || result.Config.Settings.Added != "imported" { + t.Fatalf("settings = %#v", result.Config.Settings) + } + if !reflect.DeepEqual(result.Config.Values, []string{"profile"}) || result.Config.Nullable != nil || result.Config.Value != "scalar" { + t.Fatalf("profile values = values %#v nullable %#v value %#v", result.Config.Values, result.Config.Nullable, result.Config.Value) + } + sources := result.Sources() + wantPaths := []string{canonicalTestPath(t, mainPath), canonicalTestPath(t, basePath), canonicalTestPath(t, profilePath)} + gotPaths := make([]string, len(sources)) + for index, source := range sources { + gotPaths[index] = source.Path + } + if !reflect.DeepEqual(gotPaths, wantPaths) || sources[2].Layer != LayerProfile { + t.Fatalf("sources = %#v", sources) + } + changed, _ := result.Metadata().Lookup(Keys("settings", "changed")) + kept, _ := result.Metadata().Lookup(Keys("settings", "kept")) + if changed.Origin.Layer != LayerProfile || changed.Origin.Locations[0].SourceID != "source-3" || kept.Origin.Layer != LayerBase { + t.Fatalf("origins = changed %#v kept %#v", changed, kept) + } + if _, ok := result.Metadata().Lookup(Keys("value", "child")); ok { + t.Fatal("removed child remains effective") + } + childHistory := result.Metadata().History(Keys("value", "child")) + if len(childHistory) < 2 || childHistory[len(childHistory)-1].Path.String() != "/value" || childHistory[len(childHistory)-1].Origin.Layer != LayerProfile { + t.Fatalf("removed child history = %#v", childHistory) + } + if _, ok := result.Metadata().Lookup(Keys("values").Index(1)); ok { + t.Fatal("removed sequence index remains effective") + } + indexHistory := result.Metadata().History(Keys("values").Index(1)) + if len(indexHistory) < 2 || indexHistory[len(indexHistory)-1].Path.String() != "/values" { + t.Fatalf("removed sequence history = %#v", indexHistory) + } +} + +func TestLoadProfileSelectionUsesOneFixedYAMLFile(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "settings: {kept: base}\n") + writeImportFile(t, directory, "profiles/testing.yml", "settings: {changed: selected}\n") + writeImportFile(t, directory, "profiles/default.yml", "settings: {changed: forbidden}\n") + + for _, name := range []string{"", "default"} { + result, err := Load[profileTestConfig](File(mainPath), Options[profileTestConfig]{Profile: name}) + if err != nil || result.Config.Settings.Changed != "" || len(result.Sources()) != 1 { + t.Fatalf("inactive profile %q = %#v, %v", name, result, err) + } + } + selected, err := Load[profileTestConfig](File(mainPath), Options[profileTestConfig]{Profile: "testing"}) + if err != nil || selected.Config.Settings.Changed != "selected" || len(selected.Sources()) != 2 { + t.Fatalf("selected profile = %#v, %v", selected, err) + } + + for _, name := range []string{"../testing", "testing.yml", "one,two", "with space", "é", "bad\x00name"} { + t.Run(name, func(t *testing.T) { + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: name}, CodeOptionsInvalid) + }) + } + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: "missing"}, CodeProfileMissing) + writeImportFile(t, directory, "profiles/fallback.yaml", "settings: {changed: fallback}\n") + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: "fallback"}, CodeProfileMissing) +} + +func TestLoadProfileRejectsDirectivesVersionsAndBaseAliases(t *testing.T) { + versionRule := &VersionRule{Path: Keys("schema", "version"), Check: func(version Version, _ *Validation) error { + if !version.Present || version.Text != "4" { + return errors.New("unexpected version") + } + return nil + }} + for _, test := range []struct { + name string + profile string + code Code + }{ + {name: "root directive", profile: "configkit: {}\n", code: CodeProfileDirective}, + {name: "nested directive", profile: "settings: {configkit: {}}\n", code: CodeProfileDirective}, + {name: "equal integer version", profile: "schema: {version: 4}\n", code: CodeProfileVersion}, + {name: "ancestor scalar", profile: "schema: null\n", code: CodeProfileVersion}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "schema: {version: 4}\n") + writeImportFile(t, directory, "profiles/testing.yml", test.profile) + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: "testing", Definition: Definition[profileTestConfig]{Version: versionRule}}, test.code) + }) + } + + t.Run("mapping sibling is allowed", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "schema: {version: 4}\n") + writeImportFile(t, directory, "profiles/testing.yml", "schema: {other: allowed}\n") + result, err := Load[profileTestConfig](File(mainPath), Options[profileTestConfig]{Profile: "testing", Definition: Definition[profileTestConfig]{Version: versionRule}}) + if err != nil || result.Config.Schema["other"] != "allowed" { + t.Fatalf("mapping sibling = %#v, %v", result, err) + } + }) + + t.Run("profile aliases imported base", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: profiles/testing.yml}]}\n") + writeImportFile(t, directory, "profiles/testing.yml", "settings: {changed: value}\n") + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Imports: true, Profile: "testing"}, CodeProfileRepeated) + }) +} + +func TestLoadProfileKeepsBaseFailuresAndFilesystemContainment(t *testing.T) { + t.Run("base parse precedes profile", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "settings: [\n") + writeImportFile(t, directory, "profiles/testing.yml", "settings: {changed: profile}\n") + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: "testing"}, CodeYAMLInvalid) + }) + + t.Run("profile parse", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "{}\n") + writeImportFile(t, directory, "profiles/testing.yml", "settings: [\n") + loadError := requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: "testing"}, CodeYAMLInvalid) + if loadError.Diagnostics()[0].Primary.SourceID != "source-2" || len(loadError.Sources()) != 2 { + t.Fatalf("profile parse attribution = %#v / %#v", loadError.Diagnostics(), loadError.Sources()) + } + }) + + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { + t.Run("symlink escape", func(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + outsideProfile := writeImportFile(t, outside, "testing.yml", "settings: {changed: outside}\n") + if err := os.Mkdir(filepath.Join(root, "profiles"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outsideProfile, filepath.Join(root, "profiles/testing.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, root, "main.yml", "{}\n") + requireProfileLoadCode(t, File(mainPath), Options[profileTestConfig]{Profile: "testing"}, CodeProfileOutsideRoot) + }) + } + + t.Run("byte source context", func(t *testing.T) { + directory := t.TempDir() + writeImportFile(t, directory, "profiles/testing.yml", "settings: {changed: selected}\n") + result, err := Load[profileTestConfig](Bytes("s3://bucket/main.yml", []byte("{}\n")), Options[profileTestConfig]{ + Profile: "testing", Filesystem: &FilesystemContext{BaseDir: directory, Root: directory}, + }) + if err != nil || result.Config.Settings.Changed != "selected" { + t.Fatalf("byte profile = %#v, %v", result, err) + } + requireProfileLoadCode(t, Bytes("memory.yml", []byte("{}\n")), Options[profileTestConfig]{Profile: "testing"}, CodeOptionsInvalid) + }) +} + +func TestLoadProfileCountsAgainstAggregateBudgets(t *testing.T) { + for _, test := range []struct { + name string + limits func(parsedSource, int) importLimits + }{ + {name: "source count", limits: func(_ parsedSource, _ int) importLimits { + return importLimits{sources: 1, bytes: maxTotalBytes, nodes: maxDocumentNodes} + }}, + {name: "aggregate bytes", limits: func(main parsedSource, profileBytes int) importLimits { + return importLimits{sources: 2, bytes: main.bytes + int64(profileBytes) - 1, nodes: maxDocumentNodes} + }}, + {name: "aggregate nodes", limits: func(main parsedSource, _ int) importLimits { + return importLimits{sources: 2, bytes: maxTotalBytes, nodes: countInputNodes(main.document) + 1} + }}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "{}\n") + profileData := "settings: {changed: selected}\n" + writeImportFile(t, directory, "profiles/testing.yml", profileData) + main, diagnostics := parseSource(File(mainPath), "source-1", defaultParseLimits()) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + base, diagnostics := composeDocuments(main.document) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + base.inputBytes = main.bytes + base.inputNodes = countInputNodes(main.document) + _, _, diagnostics = loadProfileDocument(File(mainPath), nil, main, base, sourceInfos(main.source), "testing", nil, test.limits(main, len(profileData))) + if len(diagnostics) == 0 || diagnostics[0].Code != CodeProfileLimitExceeded { + t.Fatalf("diagnostics = %#v", diagnostics) + } + }) + } +} + +func requireProfileLoadCode(t *testing.T, source Source, options Options[profileTestConfig], code Code) *LoadError { + t.Helper() + result, err := Load[profileTestConfig](source, options) + if result != nil || err == nil { + t.Fatalf("result, error = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 0 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v, want %s", diagnostics, code) + } + return loadError +} diff --git a/internal/engine/projection.go b/internal/engine/projection.go new file mode 100644 index 0000000..1c3a7c7 --- /dev/null +++ b/internal/engine/projection.go @@ -0,0 +1,248 @@ +package engine + +import ( + "cmp" + "encoding" + "math" + "reflect" + "slices" + "strconv" + "time" + "unicode/utf8" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +func projectConfiguration(schema *schemaDescriptor, value reflect.Value, limits projectionLimits) (*document.Node, *projectionIssue) { + if schema == nil || schema.root == nil || !value.IsValid() || value.Type() != schema.root.typeOf { + return nil, &projectionIssue{code: CodeSchemaUnsupportedType, message: "configuration value does not match its schema"} + } + operation := valueOperation{schema: schema, limits: limits, active: make(map[valueVisit]bool)} + return operation.project(value, schema.root, Path{}, 0) +} + +func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor, path Path, depth int) (*document.Node, *projectionIssue) { + if issue := o.checkDepth(path, depth); issue != nil { + return nil, issue + } + if descriptor.kind == descriptorPointer { + if value.IsNil() { + return o.node(document.Null, path, depth) + } + leave, issue := o.enter(value, path) + if issue != nil { + return nil, issue + } + defer leave() + return o.project(value.Elem(), descriptor.element, path, depth) + } + + switch descriptor.kind { + case descriptorStruct: + result, issue := o.node(document.Mapping, path, depth) + if issue != nil { + return nil, issue + } + result.Mapping = make([]document.Entry, 0, len(descriptor.fields)) + seen := make(map[string]bool, len(descriptor.fields)) + for _, member := range descriptor.members { + memberValue := value.Field(member.index) + if member.inline { + if isNilValue(memberValue) { + continue + } + inline, issue := o.project(memberValue, member.descriptor, path, depth) + if issue != nil { + return nil, issue + } + if inline.Kind != document.Mapping { + return nil, &projectionIssue{code: CodeSchemaUnsupportedType, path: path.clone(), message: "inline value did not project to a mapping"} + } + o.nodes-- // The flattened container is not retained in the output tree. + for _, entry := range inline.Mapping { + if seen[entry.Key] { + return nil, &projectionIssue{code: CodeSchemaAmbiguousField, path: path.Key(entry.Key), message: "inline value conflicts with a declared field"} + } + seen[entry.Key] = true + result.Mapping = append(result.Mapping, entry) + } + continue + } + childPath := path.Key(member.name) + if seen[member.name] { + return nil, &projectionIssue{code: CodeSchemaAmbiguousField, path: childPath, message: "inline value conflicts with a declared field"} + } + if issue := o.charge(childPath); issue != nil { // mapping key + return nil, issue + } + child, issue := o.project(memberValue, member.descriptor, childPath, depth+1) + if issue != nil { + return nil, issue + } + seen[member.name] = true + result.Mapping = append(result.Mapping, document.Entry{Key: member.name, Value: child}) + } + for _, entry := range result.Mapping { + field := descriptor.inlineMap + if index, ok := descriptor.fieldByName[entry.Key]; ok { + field = &descriptor.fields[index] + } + if field != nil && field.secret { + entry.Value.Sensitive = true + } + } + return result, nil + case descriptorMap: + if value.IsNil() { + return o.node(document.Null, path, depth) + } + leave, issue := o.enter(value, path) + if issue != nil { + return nil, issue + } + defer leave() + result, issue := o.node(document.Mapping, path, depth) + if issue != nil { + return nil, issue + } + result.Mapping = make([]document.Entry, 0, value.Len()) + keys := value.MapKeys() + slices.SortFunc(keys, func(left, right reflect.Value) int { + return cmp.Compare(left.String(), right.String()) + }) + for _, key := range keys { + childPath := path.Key(key.String()) + if issue := o.charge(childPath); issue != nil { + return nil, issue + } + child, issue := o.project(value.MapIndex(key), descriptor.element, childPath, depth+1) + if issue != nil { + return nil, issue + } + result.Mapping = append(result.Mapping, document.Entry{Key: key.String(), Value: child}) + } + return result, nil + case descriptorSlice: + if value.IsNil() { + return o.node(document.Null, path, depth) + } + leave, issue := o.enter(value, path) + if issue != nil { + return nil, issue + } + defer leave() + fallthrough + case descriptorArray: + result, issue := o.node(document.Sequence, path, depth) + if issue != nil { + return nil, issue + } + result.Sequence = make([]*document.Node, 0, value.Len()) + for i := 0; i < value.Len(); i++ { + childPath := path.Index(i) + child, issue := o.project(value.Index(i), descriptor.element, childPath, depth+1) + if issue != nil { + return nil, issue + } + result.Sequence = append(result.Sequence, child) + } + return result, nil + case descriptorDynamic: + if value.IsNil() { + return o.node(document.Null, path, depth) + } + concrete := value.Elem() + dynamic, issue := o.dynamicDescriptor(concrete.Type(), path) + if issue != nil { + return nil, issue + } + return o.project(concrete, dynamic, path, depth) + case descriptorString: + node, issue := o.node(document.String, path, depth) + if issue == nil { + node.Text = value.String() + } + return node, issue + case descriptorBoolean: + node, issue := o.node(document.Boolean, path, depth) + if issue == nil { + node.Text = strconv.FormatBool(value.Bool()) + } + return node, issue + case descriptorSigned: + node, issue := o.node(document.Number, path, depth) + if issue == nil { + node.Text = strconv.FormatInt(value.Int(), 10) + } + return node, issue + case descriptorUnsigned: + node, issue := o.node(document.Number, path, depth) + if issue == nil { + node.Text = strconv.FormatUint(value.Uint(), 10) + } + return node, issue + case descriptorFloat: + if math.IsNaN(value.Float()) || math.IsInf(value.Float(), 0) { + return nil, &projectionIssue{code: CodeSchemaNonFinite, path: path.clone(), message: "configuration float must be finite"} + } + node, issue := o.node(document.Number, path, depth) + if issue == nil { + node.Text = strconv.FormatFloat(value.Float(), 'g', -1, value.Type().Bits()) + } + return node, issue + case descriptorDuration: + node, issue := o.node(document.String, path, depth) + if issue == nil { + node.Text = time.Duration(value.Int()).String() + } + return node, issue + case descriptorSecret: + node, issue := o.node(document.String, path, depth) + if issue == nil { + node.Text = value.Interface().(Secret).EnvName() + } + return node, issue + case descriptorText: + text, issue := marshalCanonicalText(value, path) + if issue != nil { + return nil, issue + } + node, nodeIssue := o.node(document.String, path, depth) + if nodeIssue == nil { + node.Text = text + } + return node, nodeIssue + default: + return nil, &projectionIssue{code: CodeSchemaUnsupportedType, path: path.clone(), message: "configuration type is not supported"} + } +} + +func (o *valueOperation) node(kind document.Kind, path Path, depth int) (*document.Node, *projectionIssue) { + if issue := o.checkDepth(path, depth); issue != nil { + return nil, issue + } + if issue := o.charge(path); issue != nil { + return nil, issue + } + return &document.Node{Kind: kind}, nil +} + +func marshalCanonicalText(value reflect.Value, path Path) (string, *projectionIssue) { + var marshaler encoding.TextMarshaler + if value.CanInterface() { + marshaler, _ = value.Interface().(encoding.TextMarshaler) + } + if marshaler == nil { + addressable := reflect.New(value.Type()) + addressable.Elem().Set(value) + marshaler, _ = addressable.Interface().(encoding.TextMarshaler) + } + if marshaler == nil { + return "", &projectionIssue{code: CodeSchemaCodec, path: path.clone(), message: "custom text scalar has no usable text marshaler"} + } + text, err := marshaler.MarshalText() + if err != nil || !utf8.Valid(text) { + return "", &projectionIssue{code: CodeSchemaCodec, path: path.clone(), message: "custom text scalar could not be projected"} + } + return string(text), nil +} diff --git a/internal/engine/projection_test.go b/internal/engine/projection_test.go new file mode 100644 index 0000000..a38782f --- /dev/null +++ b/internal/engine/projection_test.go @@ -0,0 +1,394 @@ +package engine + +import ( + "errors" + "math" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type mutableText struct { + data []byte +} + +func (value mutableText) MarshalText() ([]byte, error) { + return value.data, nil +} + +func (value *mutableText) UnmarshalText(text []byte) error { + value.data = append(value.data[:0], text...) + return nil +} + +type failingText struct { + failUnmarshal bool +} + +func (value failingText) MarshalText() ([]byte, error) { + return []byte("CANARY_CODEC_TEXT"), errors.New("CANARY_MARSHAL_ERROR") +} + +func (value *failingText) UnmarshalText([]byte) error { + if value.failUnmarshal { + return errors.New("CANARY_UNMARSHAL_ERROR") + } + return nil +} + +type reconstructFailText string + +func (value reconstructFailText) MarshalText() ([]byte, error) { + return []byte(value), nil +} + +func (value *reconstructFailText) UnmarshalText([]byte) error { + return errors.New("CANARY_RECONSTRUCT_ERROR") +} + +func TestProjectionPreservesCanonicalTypesAndNilEmptyDistinctions(t *testing.T) { + t.Parallel() + + type nested struct { + Value string `yaml:"value"` + } + type config struct { + Zero int `yaml:"zero"` + Wide uint64 `yaml:"wide"` + Delay time.Duration `yaml:"delay"` + Pointer *nested `yaml:"pointer"` + NilPointer *nested `yaml:"nil_pointer"` + EmptyMap map[string]string `yaml:"empty_map"` + NilMap map[string]string `yaml:"nil_map"` + EmptySlice []string `yaml:"empty_slice"` + NilSlice []string `yaml:"nil_slice"` + Objects map[string]*nested `yaml:"objects"` + Fixed [2]int `yaml:"fixed"` + Custom mutableText `yaml:"custom"` + Dynamic any `yaml:"dynamic"` + Ignored *nested `yaml:"-"` + private *nested + } + value := config{ + Wide: math.MaxUint64, + Delay: 1500 * time.Millisecond, + Pointer: &nested{Value: "pointer"}, + EmptyMap: map[string]string{}, + EmptySlice: []string{}, + Objects: map[string]*nested{ + "z": {Value: "last"}, + "a": {Value: "first"}, + }, + Fixed: [2]int{1, 2}, + Custom: mutableText{data: []byte("canonical")}, + Dynamic: map[string]any{"count": int64(9), "items": []any{"x", false}}, + Ignored: &nested{Value: "ignored"}, + private: &nested{Value: "private"}, + } + schema := mustSchemaFor[config](t) + projected, issue := projectConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()) + if issue != nil { + t.Fatalf("projectConfiguration() issue = %#v", issue) + } + + assertDocument(t, mappingValue(t, projected, "zero"), document.Number, "0") + assertDocument(t, mappingValue(t, projected, "wide"), document.Number, "18446744073709551615") + assertDocument(t, mappingValue(t, projected, "delay"), document.String, "1.5s") + assertDocument(t, mappingValue(t, projected, "custom"), document.String, "canonical") + if mappingValue(t, projected, "nil_pointer").Kind != document.Null || mappingValue(t, projected, "nil_map").Kind != document.Null || mappingValue(t, projected, "nil_slice").Kind != document.Null { + t.Fatal("nil values did not project as null") + } + if empty := mappingValue(t, projected, "empty_map"); empty.Kind != document.Mapping || empty.Mapping == nil || len(empty.Mapping) != 0 { + t.Fatalf("empty map = %#v", empty) + } + if empty := mappingValue(t, projected, "empty_slice"); empty.Kind != document.Sequence || empty.Sequence == nil || len(empty.Sequence) != 0 { + t.Fatalf("empty slice = %#v", empty) + } + objects := mappingValue(t, projected, "objects") + if len(objects.Mapping) != 2 || objects.Mapping[0].Key != "a" || objects.Mapping[1].Key != "z" { + t.Fatalf("map order = %#v", objects.Mapping) + } + if hasMappingKey(projected, "ignored") || hasMappingKey(projected, "private") { + t.Fatal("ignored field entered projection") + } + clonedValue, cloneIssue := cloneConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()) + if cloneIssue != nil { + t.Fatal(cloneIssue.message) + } + cloned := clonedValue.Interface().(config) + if cloned.NilMap != nil || cloned.NilSlice != nil || cloned.NilPointer != nil || cloned.EmptyMap == nil || cloned.EmptySlice == nil { + t.Fatalf("clone lost nil/empty distinctions: %#v", cloned) + } +} + +func TestCloneConfigurationDetachesAllSupportedMutableValues(t *testing.T) { + t.Parallel() + + type nested struct { + Values []string `yaml:"values"` + } + type config struct { + First *nested `yaml:"first"` + Second *nested `yaml:"second"` + Map map[string]*nested `yaml:"map"` + Slice []map[string]int `yaml:"slice"` + Custom mutableText `yaml:"custom"` + Dynamic any `yaml:"dynamic"` + Ignored *nested `yaml:"-"` + } + shared := &nested{Values: []string{"original"}} + value := config{ + First: shared, + Second: shared, + Map: map[string]*nested{"entry": {Values: []string{"map"}}}, + Slice: []map[string]int{{"count": 1}}, + Custom: mutableText{data: []byte("text")}, + Dynamic: map[string]any{ + "items": []any{map[string]any{"name": "dynamic"}}, + }, + Ignored: &nested{Values: []string{"ignored"}}, + } + schema := mustSchemaFor[config](t) + clonedValue, issue := cloneConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()) + if issue != nil { + t.Fatalf("cloneConfiguration() issue = %#v", issue) + } + cloned := clonedValue.Interface().(config) + if cloned.First == cloned.Second || cloned.First == shared || cloned.Custom.data == nil || &cloned.Custom.data[0] == &value.Custom.data[0] { + t.Fatal("clone retained mutable identity") + } + if cloned.Ignored != nil { + t.Fatal("ignored field was retained") + } + + value.First.Values[0] = "changed" + value.Map["entry"].Values[0] = "changed" + value.Slice[0]["count"] = 2 + value.Custom.data[0] = 'X' + value.Dynamic.(map[string]any)["items"].([]any)[0].(map[string]any)["name"] = "changed" + if cloned.First.Values[0] != "original" || cloned.Map["entry"].Values[0] != "map" || cloned.Slice[0]["count"] != 1 || string(cloned.Custom.data) != "text" || cloned.Dynamic.(map[string]any)["items"].([]any)[0].(map[string]any)["name"] != "dynamic" { + t.Fatalf("clone changed after source mutation: %#v", cloned) + } +} + +func TestProjectionAndCloneRejectCyclesAndInvalidDynamicValues(t *testing.T) { + t.Parallel() + + type recursive struct { + Next *recursive `yaml:"next"` + } + type config struct { + Tree *recursive `yaml:"tree"` + Dynamic any `yaml:"dynamic"` + } + cyclic := &recursive{} + cyclic.Next = cyclic + schema := mustSchemaFor[config](t) + acyclic := config{Tree: &recursive{Next: &recursive{}}} + if _, issue := projectConfiguration(schema, reflect.ValueOf(acyclic), defaultProjectionLimits()); issue != nil { + t.Fatalf("acyclic recursive projection issue = %#v", issue) + } + if _, issue := cloneConfiguration(schema, reflect.ValueOf(acyclic), defaultProjectionLimits()); issue != nil { + t.Fatalf("acyclic recursive clone issue = %#v", issue) + } + value := config{Tree: cyclic} + if _, issue := projectConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaCycle || issue.path.String() != "/tree/next" { + t.Fatalf("projection cycle issue = %#v", issue) + } + if _, issue := cloneConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaCycle { + t.Fatalf("clone cycle issue = %#v", issue) + } + + dynamicCycle := map[string]any{} + dynamicCycle["self"] = dynamicCycle + value = config{Dynamic: dynamicCycle} + if _, issue := projectConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaCycle { + t.Fatalf("dynamic cycle issue = %#v", issue) + } + value = config{Dynamic: make(chan int)} + if _, issue := projectConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaDynamicType { + t.Fatalf("dynamic type issue = %#v", issue) + } + if _, issue := cloneConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaDynamicType { + t.Fatalf("dynamic clone issue = %#v", issue) + } +} + +func TestProjectionRejectsNonFiniteLimitsAndCodecFailuresSafely(t *testing.T) { + t.Parallel() + + type numbers struct { + Value float64 `yaml:"value"` + } + numberSchema := mustSchemaFor[numbers](t) + for _, value := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + if _, issue := projectConfiguration(numberSchema, reflect.ValueOf(numbers{Value: value}), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaNonFinite { + t.Fatalf("non-finite issue = %#v", issue) + } + } + + type codecConfig struct { + Value failingText `yaml:"value"` + } + codecSchema := mustSchemaFor[codecConfig](t) + if _, issue := projectConfiguration(codecSchema, reflect.ValueOf(codecConfig{}), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaCodec || strings.Contains(issue.message, "CANARY") { + t.Fatalf("codec projection issue = %#v", issue) + } + + type reconstructConfig struct { + Value reconstructFailText `yaml:"value"` + } + reconstructSchema := mustSchemaFor[reconstructConfig](t) + if _, issue := cloneConfiguration(reconstructSchema, reflect.ValueOf(reconstructConfig{Value: "safe-name"}), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaCodec || strings.Contains(issue.message, "CANARY") { + t.Fatalf("codec clone issue = %#v", issue) + } + + type simple struct { + Value string `yaml:"value"` + } + simpleSchema := mustSchemaFor[simple](t) + if _, issue := projectConfiguration(simpleSchema, reflect.ValueOf(simple{}), projectionLimits{depth: 1, nodes: 3}); issue != nil { + t.Fatalf("exact node limit rejected: %#v", issue) + } + if _, issue := projectConfiguration(simpleSchema, reflect.ValueOf(simple{}), projectionLimits{depth: 1, nodes: 2}); issue == nil || issue.code != CodeSchemaProjectionLimit { + t.Fatalf("node overflow issue = %#v", issue) + } + if _, issue := projectConfiguration(simpleSchema, reflect.ValueOf(simple{}), projectionLimits{depth: 0, nodes: 3}); issue == nil || issue.code != CodeSchemaProjectionLimit { + t.Fatalf("depth overflow issue = %#v", issue) + } +} + +func TestInlineProjectionAndCloneUsePhysicalMembers(t *testing.T) { + t.Parallel() + + type Embedded struct { + Name string `yaml:"name"` + Items map[string]string `yaml:"items"` + } + type config struct { + *Embedded `yaml:",inline"` + Extras map[string]any `yaml:",inline"` + } + value := config{ + Embedded: &Embedded{Name: "value", Items: map[string]string{"a": "b"}}, + Extras: map[string]any{"extra": []string{"one"}}, + } + schema := mustSchemaFor[config](t) + projected, issue := projectConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()) + if issue != nil || !hasMappingKey(projected, "name") || !hasMappingKey(projected, "items") || !hasMappingKey(projected, "extra") { + t.Fatalf("inline projection = %#v, issue=%#v", projected, issue) + } + clonedValue, issue := cloneConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()) + if issue != nil { + t.Fatal(issue.message) + } + cloned := clonedValue.Interface().(config) + value.Items["a"] = "changed" + value.Extras["extra"].([]string)[0] = "changed" + if cloned.Items["a"] != "b" || cloned.Extras["extra"].([]string)[0] != "one" { + t.Fatal("inline clone retained aliases") + } + + value.Extras["name"] = "collision" + if _, issue := projectConfiguration(schema, reflect.ValueOf(value), defaultProjectionLimits()); issue == nil || issue.code != CodeSchemaAmbiguousField || issue.path.String() != "/name" { + t.Fatalf("inline collision issue = %#v", issue) + } +} + +func TestCompareDocumentTreesTracksSemanticChanges(t *testing.T) { + t.Parallel() + + type nested struct { + Value string `yaml:"value"` + } + type config struct { + Nested nested `yaml:"nested"` + Mapping map[string]int `yaml:"mapping"` + Sequence []string `yaml:"sequence"` + Maybe map[string]int `yaml:"maybe"` + } + schema := mustSchemaFor[config](t) + before, issue := projectConfiguration(schema, reflect.ValueOf(config{ + Nested: nested{Value: "before"}, + Mapping: map[string]int{"remove": 1, "same": 2}, + Sequence: []string{"a", "b"}, + }), defaultProjectionLimits()) + if issue != nil { + t.Fatal(issue.message) + } + after, issue := projectConfiguration(schema, reflect.ValueOf(config{ + Nested: nested{Value: "after"}, + Mapping: map[string]int{"add": 3, "same": 2}, + Sequence: []string{"a", "changed"}, + Maybe: map[string]int{}, + }), defaultProjectionLimits()) + if issue != nil { + t.Fatal(issue.message) + } + deltas := compareConfigurationValues(before, after) + want := []struct { + path string + kind documentDeltaKind + }{ + {path: "/mapping/add", kind: documentAdded}, + {path: "/mapping/remove", kind: documentRemoved}, + {path: "/maybe", kind: documentReplaced}, + {path: "/nested/value", kind: documentReplaced}, + {path: "/sequence", kind: documentReplaced}, + } + if len(deltas) != len(want) { + t.Fatalf("deltas = %#v", deltas) + } + for i := range want { + if deltas[i].path.String() != want[i].path || deltas[i].kind != want[i].kind { + t.Fatalf("delta[%d] = %#v, want %#v", i, deltas[i], want[i]) + } + } + for _, delta := range deltas { + if strings.HasPrefix(delta.path.String(), "/sequence/") { + t.Fatal("sequence comparison created index correspondence") + } + } + if equal := compareConfigurationValues(after, document.Clone(after)); len(equal) != 0 { + t.Fatalf("equal trees changed = %#v", equal) + } +} + +func mustSchemaFor[T any](t *testing.T) *schemaDescriptor { + t.Helper() + schema, issue := buildSchema(reflect.TypeFor[T]()) + if issue != nil { + t.Fatalf("buildSchema() issue = %#v", issue) + } + return schema +} + +func mappingValue(t *testing.T, node *document.Node, key string) *document.Node { + t.Helper() + for _, entry := range node.Mapping { + if entry.Key == key { + return entry.Value + } + } + t.Fatalf("mapping key %q missing", key) + return nil +} + +func hasMappingKey(node *document.Node, key string) bool { + for _, entry := range node.Mapping { + if entry.Key == key { + return true + } + } + return false +} + +func assertDocument(t *testing.T, node *document.Node, kind document.Kind, text string) { + t.Helper() + if node.Kind != kind || node.Text != text { + t.Fatalf("node = %#v, want kind=%v text=%q", node, kind, text) + } +} diff --git a/internal/engine/provenance_test.go b/internal/engine/provenance_test.go new file mode 100644 index 0000000..18412fa --- /dev/null +++ b/internal/engine/provenance_test.go @@ -0,0 +1,510 @@ +package engine + +import ( + "errors" + "reflect" + "strings" + "sync" + "testing" +) + +type layeredItem struct { + Value string `yaml:"value"` +} + +type layeredConfig struct { + Shared struct { + Equal string `yaml:"equal"` + Imported string `yaml:"imported"` + Profile string `yaml:"profile"` + } `yaml:"shared"` + Values []int `yaml:"values"` + Items map[string]layeredItem `yaml:"items"` + Value any `yaml:"value"` + Nullable *string `yaml:"nullable"` + Derived int `yaml:"derived"` + Empty map[string]string `yaml:"empty"` +} + +func TestLayeredProvenanceAgreesWithResolvedSnapshot(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: [{path: imported.yml}] +shared: {equal: same} +values: [1, 2] +items: + keep: {value: 'yes'} + remove: {value: 'no'} +value: {child: base} +nullable: base +empty: {} +`) + importPath := writeImportFile(t, directory, "imported.yml", "shared: {imported: base}\n") + profilePath := writeImportFile(t, directory, "profiles/testing.yml", `shared: {profile: selected} +values: [3, 4] +value: selected +nullable: null +empty: {} +`) + defaultNullable := "default" + definition := Definition[layeredConfig]{ + Defaults: func() layeredConfig { + defaults := layeredConfig{ + Values: []int{0, 0}, Value: map[string]any{"child": "default"}, + Nullable: &defaultNullable, Empty: map[string]string{"retained": "default"}, + } + defaults.Shared.Equal = "same" + return defaults + }, + Resolve: func(configuration *layeredConfig, resolution *Resolution) error { + Normalize(resolution, Keys("values"), &configuration.Values, []int{5}, + "bounded values", Keys("shared", "imported"), Keys("shared", "imported")) + Normalize(resolution, Keys("values"), &configuration.Values, []int{5}, + "bounded values", Keys("shared", "imported")) + delete(configuration.Items, "remove") + resolution.Record(Keys("items"), ChangeNormalization, "removed disabled items") + Default(resolution, Keys("derived"), &configuration.Derived, 7, + "derived from equal", Keys("shared", "equal"), Keys("shared", "equal")) + resolution.Warning(Keys("values"), "values were bounded", Keys("shared", "imported")) + return nil + }, + } + + result, err := Load[layeredConfig](File(mainPath), Options[layeredConfig]{ + Imports: true, Profile: "testing", Definition: definition, + }) + if err != nil { + var loadError *LoadError + _ = errors.As(err, &loadError) + t.Fatalf("%v: %#v", err, loadError.Diagnostics()) + } + wantSources := []string{canonicalTestPath(t, mainPath), canonicalTestPath(t, importPath), canonicalTestPath(t, profilePath)} + sources := result.Sources() + gotSources := make([]string, len(sources)) + for index, source := range sources { + gotSources[index] = source.Path + } + if !reflect.DeepEqual(gotSources, wantSources) || sources[2].Layer != LayerProfile { + t.Fatalf("sources = %#v", sources) + } + if result.Config.Shared.Equal != "same" || result.Config.Shared.Imported != "base" || result.Config.Shared.Profile != "selected" || + !reflect.DeepEqual(result.Config.Values, []int{5}) || result.Config.Nullable != nil || result.Config.Derived != 7 || + result.Config.Empty["retained"] != "default" { + t.Fatalf("configuration = %#v", result.Config) + } + + metadata := result.Metadata() + equal, _ := metadata.Lookup(Keys("shared", "equal")) + derived, _ := metadata.Lookup(Keys("derived")) + nullable, _ := metadata.Lookup(Keys("nullable")) + empty, _ := metadata.Lookup(Keys("empty")) + firstValue, _ := metadata.Lookup(Keys("values").Index(0)) + if !equal.Explicit || equal.Origin.Layer != LayerBase || derived.Explicit || derived.Origin.Operation != ChangeDerivedDefault || + !nullable.Explicit || !nullable.Null || nullable.Origin.Layer != LayerProfile || + !empty.Explicit || empty.Origin.Layer != LayerProfile || len(empty.Origin.Locations) != 2 || + !firstValue.Explicit || firstValue.Origin.Operation != ChangeNormalization { + t.Fatalf("metadata = equal %#v derived %#v nullable %#v empty %#v value %#v", equal, derived, nullable, empty, firstValue) + } + if _, ok := metadata.Lookup(Keys("value", "child")); ok { + t.Fatal("profile-removed child remains effective") + } + if _, ok := metadata.Lookup(Keys("values").Index(1)); ok { + t.Fatal("resolution-removed sequence index remains effective") + } + if _, ok := metadata.Lookup(Keys("items", "remove", "value")); ok { + t.Fatal("resolution-removed mapping child remains effective") + } + + assertHistoryLayersAndPaths(t, metadata.History(Keys("values").Index(1)), + []Layer{LayerDefault, LayerBase, LayerProfile, LayerResolution}, + []string{"/values/1", "/values/1", "/values", "/values"}) + assertHistoryLayersAndPaths(t, metadata.History(Keys("value", "child")), + []Layer{LayerDefault, LayerBase, LayerProfile}, + []string{"/value/child", "/value/child", "/value"}) + removedHistory := metadata.History(Keys("items", "remove", "value")) + assertHistoryLayersAndPaths(t, removedHistory, + []Layer{LayerBase, LayerResolution}, + []string{"/items/remove/value", "/items"}) + if !reflect.DeepEqual(removedHistory[1].Reasons, []string{"removed disabled items"}) { + t.Fatalf("removed history = %#v", removedHistory) + } + valuesHistory := metadata.History(Keys("values").Index(0)) + lastValue := valuesHistory[len(valuesHistory)-1] + if !reflect.DeepEqual(lastValue.Reasons, []string{"bounded values"}) || len(lastValue.Dependencies) != 1 || lastValue.Dependencies[0].String() != "/shared/imported" { + t.Fatalf("canonical resolution history = %#v", lastValue) + } + + schema, issue := buildSchema(reflect.TypeFor[layeredConfig]()) + if issue != nil { + t.Fatal(issue) + } + projected, projectionIssue := projectConfiguration(schema, reflect.ValueOf(result.Config), defaultProjectionLimits()) + if projectionIssue != nil || !configurationValuesEqual(projected, result.final) { + t.Fatalf("typed projection and private snapshot disagree: %#v", projectionIssue) + } + + metadata.history[Keys("values").Index(0).String()][0].Origin.Layer = "mutated" + sources[1].ImportedFrom.SourceID = "mutated" + diagnostics := result.Diagnostics() + diagnostics[0].Related[0].SourceID = "mutated" + result.Config.Values[0] = 99 + freshHistory := result.Metadata().History(Keys("values").Index(0)) + if freshHistory[0].Origin.Layer == "mutated" || result.Sources()[1].ImportedFrom.SourceID == "mutated" || + result.Diagnostics()[0].Related[0].SourceID == "mutated" { + t.Fatal("returned inspection records alias private state") + } + snapshotValue, ok := documentAtPath(result.final, Keys("values").Index(0)) + if !ok || snapshotValue.Text != "5" { + t.Fatal("caller configuration mutation changed the private final snapshot") + } +} + +func TestSelectedProfileLoadsRemainIndependentConcurrently(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "values: [1, 2]\n") + writeImportFile(t, directory, "profiles/testing.yml", "values: [3, 4]\n") + options := Options[layeredConfig]{Profile: "testing"} + + const loads = 16 + results := make(chan *Result[layeredConfig], loads) + errors := make(chan error, loads) + var group sync.WaitGroup + for index := 0; index < loads; index++ { + group.Add(1) + go func() { + defer group.Done() + result, err := Load[layeredConfig](File(mainPath), options) + if err != nil { + errors <- err + return + } + results <- result + }() + } + group.Wait() + close(results) + close(errors) + for err := range errors { + t.Fatal(err) + } + var collected []*Result[layeredConfig] + for result := range results { + collected = append(collected, result) + } + if len(collected) != loads { + t.Fatalf("completed loads = %d", len(collected)) + } + collected[0].Config.Values[0] = 99 + collected[0].metadata.fields[Keys("values").String()] = FieldInfo{} + for _, result := range collected[1:] { + if !reflect.DeepEqual(result.Config.Values, []int{3, 4}) { + t.Fatalf("concurrent result contaminated = %#v", result.Config.Values) + } + if info, ok := result.Metadata().Lookup(Keys("values")); !ok || info.Origin.Layer != LayerProfile { + t.Fatalf("concurrent metadata contaminated = %#v, %v", info, ok) + } + } +} + +func TestResolutionInputKeepsLayeredOriginsAndSources(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: [{path: first.yml}, {path: second.yml}] +main: main +equal: same +cli_equal: same +composite: {main: main} +`) + firstPath := writeImportFile(t, directory, "first.yml", `imported: imported +sequence: [first, second] +composite: {first: first} +`) + secondPath := writeImportFile(t, directory, "second.yml", "composite: {second: second}\n") + profilePath := writeImportFile(t, directory, "profiles/testing.yml", "equal: same\nprofile: profile\n") + compositeSources := map[string]bool{ + canonicalTestPath(t, mainPath): true, + canonicalTestPath(t, firstPath): true, + canonicalTestPath(t, secondPath): true, + } + type config struct { + Main string `yaml:"main"` + Imported string `yaml:"imported"` + Sequence []string `yaml:"sequence"` + Composite map[string]string `yaml:"composite"` + Equal string `yaml:"equal"` + Profile string `yaml:"profile"` + CLIEqual string `yaml:"cli_equal"` + } + var callbackErr error + result, err := Load[config](File(mainPath), Options[config]{ + Imports: true, Profile: "testing", Overrides: []Override{{Path: Keys("cli_equal"), Value: "same", Flag: "--cli-equal"}}, + Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + check := func(ok bool, message string) { + if !ok && callbackErr == nil { + callbackErr = errors.New(message) + } + } + fileOwner := func(path Path, want string) FieldInfo { + info, found := resolution.InputInfo(path) + check(found && len(info.Origin.Locations) == 1, "scalar has no exact origin") + if !found || len(info.Origin.Locations) != 1 { + return FieldInfo{} + } + source, exists := resolution.Source(info.Origin.Locations[0].SourceID) + check(exists && source.Path == want, "scalar source does not match its declaration") + return info + } + fileOwner(Keys("main"), canonicalTestPath(t, mainPath)) + fileOwner(Keys("imported"), canonicalTestPath(t, firstPath)) + fileOwner(Keys("sequence").Index(1), canonicalTestPath(t, firstPath)) + profile := fileOwner(Keys("equal"), canonicalTestPath(t, profilePath)) + fileOwner(Keys("profile"), canonicalTestPath(t, profilePath)) + check(profile.Origin.Layer == LayerProfile, "profile replacement retained base ownership") + cli, found := resolution.InputInfo(Keys("cli_equal")) + check(found && cli.Origin.Layer == LayerCLI && len(cli.Origin.Locations) == 1, "CLI replacement lost ownership") + if found && len(cli.Origin.Locations) == 1 { + source, exists := resolution.Source(cli.Origin.Locations[0].SourceID) + check(exists && source.Layer == LayerCLI && source.Path == "", "CLI source has a file path or is missing") + } + + composite, found := resolution.InputInfo(Keys("composite")) + check(found && len(composite.Origin.Locations) == 3, "composite mapping did not retain every contributor") + for _, location := range composite.Origin.Locations { + source, exists := resolution.Source(location.SourceID) + check(exists && compositeSources[source.Path], "composite location has no expected source record") + delete(compositeSources, source.Path) + } + check(len(compositeSources) == 0, "composite sources are incomplete") + + imported, _ := resolution.InputInfo(Keys("imported")) + importedSourceID := imported.Origin.Locations[0].SourceID + imported.Origin.Locations[0].SourceID = "mutated" + importSource, _ := resolution.Source(importedSourceID) + check(importSource.ImportedFrom != nil, "import source is missing its import location") + if importSource.ImportedFrom != nil { + importSource.ImportedFrom.SourceID = "mutated" + } + again, _ := resolution.InputInfo(Keys("imported")) + againSource, _ := resolution.Source(importedSourceID) + check(again.Origin.Locations[0].SourceID == importedSourceID, "input record is aliased") + check(againSource.ImportedFrom != nil && againSource.ImportedFrom.SourceID == "source-1", "source record is aliased") + return callbackErr + }}, + }) + if callbackErr != nil { + t.Fatal(callbackErr) + } + if err != nil { + t.Fatal(err) + } + if result.Config.Main != "main" || result.Config.Imported != "imported" || !reflect.DeepEqual(result.Config.Sequence, []string{"first", "second"}) || + result.Config.Equal != "same" || result.Config.Profile != "profile" || result.Config.CLIEqual != "same" { + t.Fatalf("configuration = %#v", result.Config) + } + + resultField, _ := result.Metadata().Lookup(Keys("imported")) + resultSource := result.Sources()[1] + if resultField.Origin.Locations[0].SourceID != "source-2" || resultSource.ImportedFrom == nil || resultSource.ImportedFrom.SourceID != "source-1" { + t.Fatal("callback record mutation changed the completed result") + } +} + +func TestResolutionInputUsesEntryShapesAcrossTypedAndDynamicValues(t *testing.T) { + t.Parallel() + + type Embedded struct { + Name string `yaml:"name"` + } + type config struct { + Embedded `yaml:",inline"` + Mappings map[string]string `yaml:"mappings"` + Array [2]string `yaml:"array"` + Dynamic any `yaml:"dynamic"` + } + var callbackErr error + _, err := Load[config](Bytes("shapes.yml", []byte("name: inlined\nmappings: {'-0': key, 'a/b~': escaped}\narray: [zero, one]\ndynamic: {nested: [dynamic]}\n")), Options[config]{Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + check := func(path Path, want bool) { + _, found := resolution.InputInfo(path) + if found != want && callbackErr == nil { + callbackErr = errors.New("entry shape lookup disagrees with its document container") + } + } + parsedMapping, _ := ParsePath("/mappings/-0") + escapedMapping, _ := ParsePath("/mappings/a~1b~0") + check(Keys("name"), true) + check(parsedMapping, true) + check(escapedMapping, true) + check(Keys("mappings").Index(0), false) + check(Keys("array").Index(1), true) + check(Keys("array", "1"), false) + check(Keys("array").Index(2), false) + check(Keys("dynamic", "nested").Index(0), true) + check(Keys("dynamic", "nested", "0"), false) + negative, _ := ParsePath("/array/-1") + check(negative, false) + return callbackErr + }}}) + if callbackErr != nil { + t.Fatal(callbackErr) + } + if err != nil { + t.Fatal(err) + } +} + +func TestResolutionInputDistinguishesNonFileRootsAndImportedFiles(t *testing.T) { + directory := t.TempDir() + importPath := writeImportFile(t, directory, "fragment.yml", "imported: file\n") + type config struct { + Root string `yaml:"root"` + Imported string `yaml:"imported"` + Default string `yaml:"default"` + } + input := "configkit: {imports: [{path: fragment.yml}]}\nroot: memory\n" + context := &FilesystemContext{BaseDir: directory, Root: directory} + for _, test := range []struct { + name string + source func() Source + }{ + {name: "bytes", source: func() Source { return Bytes("memory://config.yml", []byte(input)) }}, + {name: "reader", source: func() Source { return Reader("memory://config.yml", strings.NewReader(input)) }}, + } { + t.Run(test.name, func(t *testing.T) { + var callbackErr error + _, err := Load[config](test.source(), Options[config]{Imports: true, Filesystem: context, Definition: Definition[config]{ + Defaults: func() config { return config{Default: "retained"} }, + Resolve: func(_ *config, resolution *Resolution) error { + root, _ := resolution.InputInfo(Keys("root")) + imported, _ := resolution.InputInfo(Keys("imported")) + defaulted, _ := resolution.InputInfo(Keys("default")) + rootSource, rootFound := resolution.Source(root.Origin.Locations[0].SourceID) + importSource, importFound := resolution.Source(imported.Origin.Locations[0].SourceID) + if !rootFound || rootSource.Path != "" || !importFound || importSource.Path != canonicalTestPath(t, importPath) || defaulted.Explicit { + callbackErr = errors.New("non-file root or imported file provenance is wrong") + } + if _, found := resolution.Source("unknown"); found { + callbackErr = errors.New("unknown source was found") + } + return callbackErr + }, + }}) + if callbackErr != nil { + t.Fatal(callbackErr) + } + if err != nil { + t.Fatal(err) + } + }) + } + + var callbackErr error + _, err := Load[config](Bytes("memory://standalone.yml", []byte("root: memory\n")), Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Default: "retained"} }, + Resolve: func(_ *config, resolution *Resolution) error { + root, rootFound := resolution.InputInfo(Keys("root")) + defaulted, defaultFound := resolution.InputInfo(Keys("default")) + source, sourceFound := resolution.Source(root.Origin.Locations[0].SourceID) + if !rootFound || !defaultFound || !sourceFound || source.Path != "" || defaulted.Explicit { + callbackErr = errors.New("standalone non-file provenance is wrong") + } + return callbackErr + }, + }}) + if callbackErr != nil { + t.Fatal(callbackErr) + } + if err != nil { + t.Fatal(err) + } +} + +func TestResolutionInputRemainsFixedAcrossRecordedMutations(t *testing.T) { + t.Parallel() + + type config struct { + Existing string `yaml:"existing"` + Values map[string]string `yaml:"values"` + } + var callbackErr error + result, err := Load[config](Bytes("mutations.yml", []byte("existing: original\nvalues: {original: move}\n")), Options[config]{Definition: Definition[config]{Resolve: func(value *config, resolution *Resolution) error { + original, originalFound := resolution.InputInfo(Keys("values", "original")) + _, destinationFound := resolution.InputInfo(Keys("values", "destination")) + existing, existingFound := resolution.InputInfo(Keys("existing")) + Normalize(resolution, Keys("existing"), &value.Existing, "normalized", "normalized existing value") + value.Values["destination"] = value.Values["original"] + delete(value.Values, "original") + resolution.Record(Keys("values"), ChangeNormalization, "moved configured value") + againOriginal, againOriginalFound := resolution.InputInfo(Keys("values", "original")) + againDestination, againDestinationFound := resolution.InputInfo(Keys("values", "destination")) + againExisting, againExistingFound := resolution.InputInfo(Keys("existing")) + if !originalFound || destinationFound || !existingFound || !againOriginalFound || againDestinationFound || !againExistingFound || + !reflect.DeepEqual(original, againOriginal) || !reflect.DeepEqual(existing, againExisting) || !reflect.DeepEqual(againDestination, FieldInfo{}) { + callbackErr = errors.New("entry view changed during resolution") + } + return callbackErr + }}}) + if callbackErr != nil { + t.Fatal(callbackErr) + } + if err != nil { + t.Fatal(err) + } + if result.Config.Existing != "normalized" || result.Config.Values["destination"] != "move" { + t.Fatalf("configuration = %#v", result.Config) + } + if _, found := result.Metadata().Lookup(Keys("values", "original")); found { + t.Fatal("final metadata retained a removed path") + } + destination, found := result.Metadata().Lookup(Keys("values", "destination")) + existing, existingFound := result.Metadata().Lookup(Keys("existing")) + if !found || destination.Origin.Layer != LayerResolution || !existingFound || existing.Origin.Layer != LayerResolution { + t.Fatalf("final metadata = destination %#v existing %#v", destination, existing) + } +} + +func TestResolutionInputQueriesRemainIndependentAcrossLoads(t *testing.T) { + t.Parallel() + + type config struct { + Value string `yaml:"value"` + } + const loads = 16 + failures := make(chan error, loads) + var group sync.WaitGroup + for index := 0; index < loads; index++ { + group.Add(1) + go func() { + defer group.Done() + _, err := Load[config](Bytes("parallel.yml", []byte("value: stable\n")), Options[config]{Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + info, found := resolution.InputInfo(Keys("value")) + source, sourceFound := resolution.Source(info.Origin.Locations[0].SourceID) + if !found || !sourceFound || source.ID != "source-1" { + return errors.New("independent entry lookup failed") + } + info.Origin.Locations[0].SourceID = "mutated" + source.Label = "mutated" + return nil + }}}) + if err != nil { + failures <- err + } + }() + } + group.Wait() + close(failures) + for err := range failures { + t.Fatal(err) + } +} + +func assertHistoryLayersAndPaths(t *testing.T, history []Change, layers []Layer, paths []string) { + t.Helper() + if len(history) != len(layers) || len(history) != len(paths) { + t.Fatalf("history = %#v", history) + } + for index, change := range history { + if change.Origin.Layer != layers[index] || change.Path.String() != paths[index] { + t.Fatalf("history[%d] = %#v", index, change) + } + } +} diff --git a/internal/engine/records.go b/internal/engine/records.go new file mode 100644 index 0000000..b0b3528 --- /dev/null +++ b/internal/engine/records.go @@ -0,0 +1,286 @@ +package engine + +// Layer identifies the precedence layer responsible for a configuration value. +type Layer string + +const ( + LayerDefault Layer = "default" + LayerBase Layer = "base" + LayerProfile Layer = "profile" + LayerCLI Layer = "cli" + LayerResolution Layer = "resolution" +) + +// ChangeKind identifies a recorded operation. The zero value is +// used for input and default origins. +type ChangeKind string + +const ( + // ChangeDecoding records the built-in transition to canonical typed values. + // It is inspection metadata, not an operation accepted by Resolution.Record. + ChangeDecoding ChangeKind = "decoding" + ChangeDerivedDefault ChangeKind = "derived_default" + ChangeNormalization ChangeKind = "normalization" +) + +// Severity classifies a diagnostic. +type Severity string + +const ( + SeverityError Severity = "error" + SeverityWarning Severity = "warning" +) + +// Code is a stable, machine-readable diagnostic classification. +type Code string + +const ( + CodeSourceInvalid Code = "source.invalid" + CodeSourceMissing Code = "source.missing" + CodeSourcePermission Code = "source.permission" + CodeSourceRead Code = "source.read" + CodeSourceNotRegular Code = "source.not_regular" + CodeSourceLimitExceeded Code = "source.limit_exceeded" + CodeSourceReaderUsed Code = "source.reader_used" + + CodeYAMLInvalid Code = "yaml.invalid" + CodeYAMLEmpty Code = "yaml.empty" + CodeYAMLMultipleDocuments Code = "yaml.multiple_documents" + CodeYAMLRootType Code = "yaml.root_type" + CodeYAMLDuplicateKey Code = "yaml.duplicate_key" + CodeYAMLTag Code = "yaml.tag" + CodeYAMLAnchor Code = "yaml.anchor" + CodeYAMLAlias Code = "yaml.alias" + CodeYAMLMergeKey Code = "yaml.merge_key" + CodeYAMLScalar Code = "yaml.scalar" + CodeYAMLDepthLimit Code = "yaml.depth_limit" + CodeYAMLNodeLimit Code = "yaml.node_limit" + + CodeSchemaRoot Code = "schema.root" + CodeSchemaUnsupportedType Code = "schema.unsupported_type" + CodeSchemaAmbiguousField Code = "schema.ambiguous_field" + CodeSchemaTag Code = "schema.tag" + CodeSchemaTextCodec Code = "schema.text_codec" + CodeSchemaInline Code = "schema.inline" + CodeSchemaCycle Code = "schema.cycle" + CodeSchemaProjectionLimit Code = "schema.projection_limit" + CodeSchemaCodec Code = "schema.codec" + CodeSchemaNonFinite Code = "schema.non_finite" + CodeSchemaDynamicType Code = "schema.dynamic_type" + + CodeDecodeType Code = "decode.type" + CodeDecodeNull Code = "decode.null" + CodeDecodeRange Code = "decode.range" + CodeDecodeDuration Code = "decode.duration" + CodeDecodeText Code = "decode.text" + CodeDecodeUnknownField Code = "decode.unknown_field" + CodeDecodeArrayLength Code = "decode.array_length" + + CodeOptionsInvalid Code = "options.invalid" + CodeOptionsUnavailable Code = "options.unavailable" + + CodeImportDisabled Code = "import.disabled" + CodeImportInvalid Code = "import.invalid" + CodeImportMissing Code = "import.missing" + CodeImportPermission Code = "import.permission" + CodeImportRead Code = "import.read" + CodeImportOutsideRoot Code = "import.outside_root" + CodeImportCycle Code = "import.cycle" + CodeImportRepeated Code = "import.repeated" + CodeImportLimitExceeded Code = "import.limit_exceeded" + CodeImportVersion Code = "import.version" + + CodeProfileInvalid Code = "profile.invalid" + CodeProfileMissing Code = "profile.missing" + CodeProfileOutsideRoot Code = "profile.outside_root" + CodeProfileRepeated Code = "profile.repeated" + CodeProfileDirective Code = "profile.directive" + CodeProfileVersion Code = "profile.version" + CodeProfileLimitExceeded Code = "profile.limit_exceeded" + + CodeComposeInvalid Code = "compose.invalid" + CodeComposeConflict Code = "compose.conflict" + + CodeVersionType Code = "version.type" + CodeVersionError Code = "version.error" + CodeVersionWarning Code = "version.warning" + CodeVersionHookFailed Code = "version.hook_failed" + + CodeResolveError Code = "resolve.error" + CodeResolveWarning Code = "resolve.warning" + CodeResolveHookFailed Code = "resolve.hook_failed" + CodeResolveInvalidPath Code = "resolve.invalid_path" + CodeResolveInvalidKind Code = "resolve.invalid_kind" + CodeResolveInvalidRecord Code = "resolve.invalid_record" + CodeResolveInvalidTarget Code = "resolve.invalid_target" + CodeResolveInvalidValue Code = "resolve.invalid_value" + CodeResolveUnrecordedChange Code = "resolve.unrecorded_change" + + CodeValidateError Code = "validate.error" + CodeValidateWarning Code = "validate.warning" + CodeValidateInvalidPath Code = "validate.invalid_path" + CodeValidateMutation Code = "validate.mutation" + + CodeSecretsInvalidPath Code = "secrets.invalid_path" + CodeSecretsMissing Code = "secrets.missing" + CodeSecretsPermission Code = "secrets.permission" + CodeSecretsRead Code = "secrets.read" + CodeSecretsNotRegular Code = "secrets.not_regular" + CodeSecretsInvalidName Code = "secrets.invalid_name" + CodeSecretsLimitExceeded Code = "secrets.limit_exceeded" + CodeSecretsInvalidValue Code = "secrets.invalid_value" + CodeSecretsApplyFailed Code = "secrets.apply_failed" + + CodeSecretRequired Code = "secret.required" + CodeSecretInvalidName Code = "secret.invalid_name" + CodeSecretUnset Code = "secret.unset" + CodeSecretEmpty Code = "secret.empty" + + CodeOverrideInvalidPath Code = "override.invalid_path" + CodeOverrideInvalidLabel Code = "override.invalid_label" + CodeOverrideInvalidValue Code = "override.invalid_value" + CodeOverrideDuplicate Code = "override.duplicate" + CodeOverrideOverlap Code = "override.overlap" + CodeOverrideDirective Code = "override.directive" + CodeOverrideVersion Code = "override.version" + CodeOverrideLimitExceeded Code = "override.limit_exceeded" + + CodeExplainLimitExceeded Code = "explain.limit_exceeded" +) + +// Location identifies a position in a configuration source. Zero line and column +// values mean that coordinates are unavailable or synthetic. +type Location struct { + SourceID string + Line int + Column int +} + +// SourceInfo describes a source discovered during a load. Path is a canonical +// local path when one exists. ImportedFrom identifies the declaration that +// included this source. +type SourceInfo struct { + ID string + Label string + Path string + Layer Layer + ImportedFrom *Location +} + +// Diagnostic is a safe structured problem report. Message and source labels must +// never contain configuration values or other secrets. +type Diagnostic struct { + Code Code + Severity Severity + Path Path + Message string + Primary *Location + Related []Location +} + +// Origin describes where an effective definition came from. Multiple locations +// represent a composite origin; resolution origins are synthetic and identify their +// operation instead of a file location. +type Origin struct { + Layer Layer + Locations []Location + Operation ChangeKind +} + +// FieldInfo describes an effective configuration path. +type FieldInfo struct { + Explicit bool + Null bool + Origin Origin +} + +// Change describes a definition or application-resolution operation. Path can name +// an ancestor operation affecting the queried descendant. Historical values are +// intentionally not exposed by this record. +type Change struct { + Path Path + Origin Origin + Previous []Origin + Reasons []string + Dependencies []Path +} + +func cloneLocation(location *Location) *Location { + if location == nil { + return nil + } + cloned := *location + return &cloned +} + +func cloneLocations(locations []Location) []Location { + if locations == nil { + return nil + } + return append([]Location(nil), locations...) +} + +func cloneSourceInfo(source SourceInfo) SourceInfo { + source.ImportedFrom = cloneLocation(source.ImportedFrom) + return source +} + +func cloneSourceInfos(sources []SourceInfo) []SourceInfo { + if sources == nil { + return nil + } + cloned := make([]SourceInfo, len(sources)) + for i, source := range sources { + cloned[i] = cloneSourceInfo(source) + } + return cloned +} + +func cloneOrigin(origin Origin) Origin { + origin.Locations = cloneLocations(origin.Locations) + return origin +} + +func cloneFieldInfo(info FieldInfo) FieldInfo { + info.Origin = cloneOrigin(info.Origin) + return info +} + +func cloneChange(change Change) Change { + change.Path = change.Path.clone() + change.Origin = cloneOrigin(change.Origin) + if change.Previous != nil { + previous := make([]Origin, len(change.Previous)) + for i, origin := range change.Previous { + previous[i] = cloneOrigin(origin) + } + change.Previous = previous + } + change.Reasons = append([]string(nil), change.Reasons...) + if change.Dependencies != nil { + dependencies := make([]Path, len(change.Dependencies)) + for i, path := range change.Dependencies { + dependencies[i] = path.clone() + } + change.Dependencies = dependencies + } + return change +} + +func cloneDiagnostic(diagnostic Diagnostic) Diagnostic { + diagnostic.Path = diagnostic.Path.clone() + diagnostic.Primary = cloneLocation(diagnostic.Primary) + diagnostic.Related = cloneLocations(diagnostic.Related) + return diagnostic +} + +func cloneDiagnostics(diagnostics []Diagnostic) []Diagnostic { + if diagnostics == nil { + return nil + } + cloned := make([]Diagnostic, len(diagnostics)) + for i, diagnostic := range diagnostics { + cloned[i] = cloneDiagnostic(diagnostic) + } + return cloned +} diff --git a/internal/engine/render.go b/internal/engine/render.go new file mode 100644 index 0000000..a7cc52f --- /dev/null +++ b/internal/engine/render.go @@ -0,0 +1,46 @@ +package engine + +import ( + "errors" + "io" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +var errEffectiveUnavailable = errors.New("configkit: effective configuration is unavailable") + +// WriteEffective writes deterministic redacted YAML from the frozen resolved +// snapshot. The output is intended for inspection and is not promised reloadable. +// Writer errors, including short writes, are returned and may leave partial output. +func (r *Result[T]) WriteEffective(output io.Writer) error { + if r == nil || r.final == nil || output == nil { + return errEffectiveUnavailable + } + safe := redactedDocument(r.final, r.secrets, Path{}) + return yamladapter.Write(output, safe) +} + +func redactedDocument(node *document.Node, secret map[string]bool, path Path) *document.Node { + result, _ := safeSnapshotDocument(node, secret, path) + return result +} + +func pathIsSecret(secret map[string]bool, path Path) bool { + if secret[""] { + return true + } + pointer := path.String() + for pointer != "" { + if secret[pointer] { + return true + } + position := strings.LastIndexByte(pointer, '/') + if position <= 0 { + break + } + pointer = pointer[:position] + } + return false +} diff --git a/internal/engine/render_test.go b/internal/engine/render_test.go new file mode 100644 index 0000000..560eed9 --- /dev/null +++ b/internal/engine/render_test.go @@ -0,0 +1,250 @@ +package engine + +import ( + "bytes" + "errors" + "fmt" + "io" + "strings" + "sync/atomic" + "testing" +) + +const ( + taggedCanary = "TAGGED_CANARY_7c65" + dynamicCanary = "DYNAMIC_CANARY_6a12" + historicalCanary = "HISTORICAL_CANARY_91ef" + derivedACanary = "DERIVED_A_CANARY_4c20" + derivedBCanary = "DERIVED_B_CANARY_831d" +) + +var renderTextMarshalCalls atomic.Int64 +var renderYAMLMarshalCalls atomic.Int64 + +type renderText string + +func (value renderText) MarshalText() ([]byte, error) { + renderTextMarshalCalls.Add(1) + return []byte(value), nil +} + +func (value *renderText) UnmarshalText(text []byte) error { + *value = renderText(text) + return nil +} + +type renderCredentials struct { + Token string `yaml:"token"` + Nested []string `yaml:"nested"` +} + +type renderConfig struct { + Credentials renderCredentials `yaml:"credentials" configkit:"secret"` + Custom renderText `yaml:"custom"` + DerivedA string `yaml:"derived_a"` + DerivedB string `yaml:"derived_b"` + Dynamic any `yaml:"dynamic"` + EmptyMap map[string]string `yaml:"empty_map"` + EmptySequence []string `yaml:"empty_sequence"` + Labels map[string]string `yaml:"labels"` + NilMap map[string]string `yaml:"nil_map"` + NilSequence []string `yaml:"nil_sequence"` + Sequence []string `yaml:"sequence"` +} + +func (renderConfig) MarshalYAML() (any, error) { + renderYAMLMarshalCalls.Add(1) + return map[string]string{"forbidden": "application YAML hook ran"}, nil +} + +func TestWriteEffectiveRedactsTagsDynamicHistoryAndDependencyCycles(t *testing.T) { + renderTextMarshalCalls.Store(0) + renderYAMLMarshalCalls.Store(0) + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `credentials: + token: TAGGED_CANARY_7c65 + nested: [nested-secret] +custom: custom value +dynamic: {retired: HISTORICAL_CANARY_91ef} +empty_map: {} +empty_sequence: [] +labels: {hidden: DYNAMIC_CANARY_6a12, public: visible} +sequence: [second, first] +`) + writeImportFile(t, directory, "profiles/testing.yml", "dynamic: selected\n") + definition := Definition[renderConfig]{ + Secret: func(path Path) bool { + return path.String() == "/labels/hidden" || path.String() == "/dynamic/retired" + }, + Resolve: func(configuration *renderConfig, resolution *Resolution) error { + Normalize(resolution, Keys("derived_a"), &configuration.DerivedA, derivedACanary, + "derived through cycle", Keys("derived_b")) + Normalize(resolution, Keys("derived_b"), &configuration.DerivedB, derivedBCanary, + "derived from retired secret", Keys("derived_a"), Keys("dynamic")) + resolution.Warning(Keys("derived_a"), "derived output is configured") + return nil + }, + } + result, err := Load[renderConfig](File(mainPath), Options[renderConfig]{Definition: definition, Profile: "testing"}) + if err != nil { + t.Fatal(err) + } + if result.Config.Credentials.Token != taggedCanary || result.Config.Labels["hidden"] != dynamicCanary || + result.Config.DerivedA != derivedACanary || result.Config.DerivedB != derivedBCanary { + t.Fatal("typed configuration did not retain trusted values") + } + if !result.secrets["/dynamic/retired"] || + !result.secrets["/derived_a"] || !result.secrets["/derived_b"] { + t.Fatalf("secret closure = %#v", result.secrets) + } + if renderYAMLMarshalCalls.Load() != 0 { + t.Fatal("loading called an application YAML marshal hook") + } + textCalls := renderTextMarshalCalls.Load() + + var output bytes.Buffer + if err := result.WriteEffective(&output); err != nil { + t.Fatal(err) + } + want := "credentials: '[REDACTED]'\ncustom: custom value\nderived_a: '[REDACTED]'\nderived_b: '[REDACTED]'\ndynamic: selected\nempty_map: {}\nempty_sequence: []\nlabels:\n hidden: '[REDACTED]'\n public: visible\nnil_map: null\nnil_sequence: null\nsequence:\n - second\n - first\n" + if output.String() != want { + t.Fatalf("effective output:\n%s\nwant:\n%s", output.String(), want) + } + if renderTextMarshalCalls.Load() != textCalls || renderYAMLMarshalCalls.Load() != 0 { + t.Fatal("rendering called an application marshal hook") + } + for _, canary := range []string{taggedCanary, dynamicCanary, historicalCanary, derivedACanary, derivedBCanary, "nested-secret"} { + if strings.Contains(output.String(), canary) { + t.Fatalf("effective output contains %q", canary) + } + formatted := fmt.Sprintf("%v %#v %v %#v", result.Metadata(), result.Metadata(), result.Diagnostics(), result.Diagnostics()) + if strings.Contains(formatted, canary) { + t.Fatalf("metadata formatting contains %q", canary) + } + } + + beforeMutation := output.String() + result.Config.Credentials.Token = "MUTATED_SECRET" + result.Config.Labels["hidden"] = "MUTATED_DYNAMIC_SECRET" + result.Config.Sequence[0] = "mutated" + output.Reset() + if err := result.WriteEffective(&output); err != nil || output.String() != beforeMutation { + t.Fatalf("frozen output after caller mutation = %q, %v", output.String(), err) + } + + wantWriteError := errors.New("write failed") + if err := result.WriteEffective(errorWriter{err: wantWriteError}); !errors.Is(err, wantWriteError) { + t.Fatalf("writer error = %v", err) + } + if err := result.WriteEffective(shortWriter{}); !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("short writer error = %v", err) + } + var nilResult *Result[renderConfig] + if err := nilResult.WriteEffective(io.Discard); !errors.Is(err, errEffectiveUnavailable) { + t.Fatalf("nil result error = %v", err) + } +} + +func TestSecretValuesStayOutOfLoadFailuresAndWarnings(t *testing.T) { + tests := []struct { + name string + input string + definition Definition[renderConfig] + }{ + {name: "parse", input: "credentials: [" + taggedCanary + "\n"}, + {name: "decode", input: "credentials: " + taggedCanary + "\n"}, + {name: "returned hook error", input: "credentials: {token: " + taggedCanary + "}\n", definition: Definition[renderConfig]{ + Resolve: func(*renderConfig, *Resolution) error { return errors.New(taggedCanary) }, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.definition.Secret = func(path Path) bool { return path.String() == "/credentials" } + result, err := Load[renderConfig](Bytes("secret-failure.yml", []byte(test.input)), Options[renderConfig]{Definition: test.definition}) + if result != nil || err == nil { + t.Fatalf("result, error = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + formatted := fmt.Sprintf("%v %#v %v %#v %v %#v", err, err, loadError.Diagnostics(), loadError.Diagnostics(), loadError.Sources(), loadError.Sources()) + if strings.Contains(formatted, taggedCanary) { + t.Fatalf("failure formatting disclosed a secret: %s", formatted) + } + }) + } +} + +func TestDynamicConcreteTagsRemainSecretThroughHistory(t *testing.T) { + type sensitive struct { + Token string `yaml:"token" configkit:"secret"` + } + type cfg struct { + Payload any `yaml:"payload"` + Derived string `yaml:"derived"` + } + for _, change := range []string{"retain", "replace", "delete", "introduce"} { + t.Run(change, func(t *testing.T) { + const canary = "DYNAMIC_TAG_CANARY" + definition := Definition[cfg]{Defaults: func() cfg { return cfg{Payload: map[string]any{"nested": []any{sensitive{Token: canary}}}} }} + path := Keys("payload", "nested").Index(0).Key("token") + if change == "introduce" { + definition.Defaults = nil + } + definition.Resolve = func(c *cfg, r *Resolution) error { + if change == "introduce" { + Normalize(r, Keys("payload"), &c.Payload, any(map[string]any{"nested": []any{sensitive{Token: canary}}}), "create") + } + if change == "replace" { + Normalize(r, Keys("payload"), &c.Payload, any("public"), "replace") + } + if change == "delete" { + Normalize(r, Keys("payload"), &c.Payload, any(nil), "remove") + } + Normalize(r, Keys("derived"), &c.Derived, canary, "derive", path) + return nil + } + result, err := Load(NoSource(), Options[cfg]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + if err := result.WriteEffective(&output); err != nil { + t.Fatal(err) + } + for _, query := range []Path{Keys(), path, Keys("derived")} { + explanation, _ := result.Explain(query) + output.WriteString(fmt.Sprintf("%#v", explanation)) + } + if strings.Contains(output.String(), canary) { + t.Fatal("dynamic secret escaped inspection") + } + if !result.secrets[path.String()] || !result.secrets["/derived"] { + t.Fatal("dynamic tag or dependency classification missing") + } + }) + } +} + +func TestValidatorCannotChangeDynamicSensitivityWithoutChangingText(t *testing.T) { + type sensitive struct { + Token string `yaml:"token" configkit:"secret"` + } + type cfg struct { + Payload any `yaml:"payload"` + } + _, err := Load(Bytes("base", []byte("payload: {token: disposable}\n")), Options[cfg]{Definition: Definition[cfg]{Validate: func(c *cfg, v *Validation) { + c.Payload = sensitive{Token: c.Payload.(map[string]any)["token"].(string)} + }}}) + var failure *LoadError + if !errors.As(err, &failure) { + t.Fatal("validator changed inspection sensitivity without a mutation error") + } + for _, d := range failure.Diagnostics() { + if d.Code == CodeValidateMutation { + return + } + } + t.Fatal("missing validation mutation diagnostic") +} diff --git a/internal/engine/resolution.go b/internal/engine/resolution.go new file mode 100644 index 0000000..d8b5016 --- /dev/null +++ b/internal/engine/resolution.go @@ -0,0 +1,367 @@ +package engine + +import ( + "reflect" + "slices" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +const closedCallbackContextMessage = "configkit: callback context is no longer active" + +type resolutionState struct { + active bool + schema *schemaDescriptor + presence map[string]inputPresence + document *document.Node + input resolutionInput + records map[string]*resolutionRecord + diagnostics []Diagnostic +} + +type resolutionRecord struct { + path Path + kind ChangeKind + reasons []string + dependencies []Path + helperExpected *document.Node +} + +type resolutionSnapshot struct { + change Change + before *document.Node + after *document.Node +} + +type resolutionResult struct { + final *document.Node + deltas []documentDelta + snapshots []resolutionSnapshot + diagnostics []Diagnostic +} + +// IsExplicit reports whether surviving input explicitly defined path. +func (r *Resolution) IsExplicit(path Path) bool { + state := r.activeState() + if !state.validPath(path) { + return false + } + _, explicit := state.presence[path.String()] + return explicit +} + +// InputInfo returns detached entry-state metadata for path. The entry state is +// the decoded configuration, including retained static defaults, immediately +// before resolution starts. It is fixed while resolution records mutate the +// working configuration; unknown or shape-mismatched paths return false. +// +// InputInfo is valid only during the resolution callback. See the configuration +// contract for source and provenance semantics. +func (r *Resolution) InputInfo(path Path) (FieldInfo, bool) { + return r.activeState().input.lookup(path) +} + +// Source returns a detached record for a source acquired by this load. Empty or +// unknown IDs return false and do not perform input, filesystem, or credential +// access. A source path is present only when that acquired source has one. +// +// Source is valid only during the resolution callback. See the configuration +// contract for source and provenance semantics. +func (r *Resolution) Source(id string) (SourceInfo, bool) { + return r.activeState().input.source(id) +} + +// Record declares the intent for a resolver mutation. A container path covers +// mutations to its descendants. +func (r *Resolution) Record(path Path, kind ChangeKind, reason string, dependencies ...Path) { + state := r.activeState() + state.addRecord(path, kind, reason, dependencies, nil) +} + +// Error reports a safe, value-free resolution error at path. +func (r *Resolution) Error(path Path, message string, related ...Path) { + r.activeState().report(SeverityError, path, message, related) +} + +// Warning reports a safe, value-free resolution warning at path. +func (r *Resolution) Warning(path Path, message string, related ...Path) { + r.activeState().report(SeverityWarning, path, message, related) +} + +// Default assigns and records value only when path was absent from surviving +// input. Static defaults are not considered explicit. +func Default[V any](resolution *Resolution, path Path, target *V, value V, reason string, dependencies ...Path) { + state := resolution.activeState() + descriptor, ok := state.helperDescriptor(path, target) + if !ok { + return + } + if _, explicit := state.presence[path.String()]; explicit { + return + } + *target = value + expected, ok := state.projectHelperValue(path, descriptor, reflect.ValueOf(target).Elem()) + if !ok { + return + } + state.addRecord(path, ChangeDerivedDefault, reason, dependencies, expected) +} + +// Normalize assigns and records value regardless of input presence. +func Normalize[V any](resolution *Resolution, path Path, target *V, value V, reason string, dependencies ...Path) { + state := resolution.activeState() + descriptor, ok := state.helperDescriptor(path, target) + if !ok { + return + } + *target = value + expected, ok := state.projectHelperValue(path, descriptor, reflect.ValueOf(target).Elem()) + if !ok { + return + } + state.addRecord(path, ChangeNormalization, reason, dependencies, expected) +} + +func (r *Resolution) activeState() *resolutionState { + if r == nil || r.state == nil || !r.state.active { + panic(closedCallbackContextMessage) + } + return r.state +} + +func (state *resolutionState) validPath(path Path) bool { + if _, ok := resolveSchemaPath(state.schema, path); ok { + return true + } + state.addDiagnostic(CodeResolveInvalidPath, path, "resolution path is not valid for the configuration schema") + return false +} + +func (state *resolutionState) helperDescriptor(path Path, target any) (*typeDescriptor, bool) { + resolved, ok := resolveSchemaPath(state.schema, path) + if !ok { + state.addDiagnostic(CodeResolveInvalidPath, path, "resolution helper path is not valid for the configuration schema") + return nil, false + } + value := reflect.ValueOf(target) + if !value.IsValid() || value.Kind() != reflect.Pointer || value.IsNil() { + state.addDiagnostic(CodeResolveInvalidTarget, path, "resolution helper target must be a non-nil pointer") + return nil, false + } + if resolved.descriptor.kind != descriptorDynamic && value.Elem().Type() != resolved.descriptor.typeOf { + state.addDiagnostic(CodeResolveInvalidTarget, path, "resolution helper target type does not match its configuration path") + return nil, false + } + return resolved.descriptor, true +} + +func (state *resolutionState) projectHelperValue(path Path, descriptor *typeDescriptor, value reflect.Value) (*document.Node, bool) { + if descriptor.kind == descriptorDynamic { + wrapped := reflect.New(descriptor.typeOf).Elem() + if value.IsValid() { + wrapped.Set(value) + } + value = wrapped + } + operation := valueOperation{ + schema: state.schema, limits: defaultProjectionLimits(), active: make(map[valueVisit]bool), + } + node, issue := operation.project(value, descriptor, path, 0) + if issue != nil { + state.addDiagnostic(CodeResolveInvalidValue, issue.path, "resolution helper value is not a valid configuration value") + return nil, false + } + return node, true +} + +func (state *resolutionState) addRecord(path Path, kind ChangeKind, reason string, dependencies []Path, expected *document.Node) { + if !state.validPath(path) { + return + } + if kind != ChangeDerivedDefault && kind != ChangeNormalization { + state.addDiagnostic(CodeResolveInvalidKind, path, "resolution record has an invalid change kind") + return + } + if reason == "" { + state.addDiagnostic(CodeResolveInvalidRecord, path, "resolution record requires a safe reason") + return + } + for _, dependency := range dependencies { + if !state.validPath(dependency) { + return + } + } + pointer := path.String() + record := state.records[pointer] + if record == nil { + record = &resolutionRecord{path: path.clone(), kind: kind} + state.records[pointer] = record + } + if kind == ChangeNormalization { + record.kind = ChangeNormalization + } + record.reasons = appendUniqueString(record.reasons, reason) + for _, dependency := range dependencies { + record.dependencies = appendUniquePath(record.dependencies, dependency) + } + if expected != nil { + record.helperExpected = document.Clone(expected) + } +} + +func (state *resolutionState) report(severity Severity, path Path, message string, related []Path) { + code := CodeResolveError + if severity == SeverityWarning { + code = CodeResolveWarning + } + if !state.validPath(path) { + return + } + diagnostic := Diagnostic{Code: code, Severity: severity, Path: path.clone(), Message: message} + if location, ok := nearestDocumentLocation(state.document, path); ok { + diagnostic.Primary = documentLocationPointer(location) + } + for _, relatedPath := range related { + if !state.validPath(relatedPath) { + continue + } + if location, ok := nearestDocumentLocation(state.document, relatedPath); ok { + diagnostic.Related = append(diagnostic.Related, Location{ + SourceID: location.SourceID, Line: location.Line, Column: location.Column, + }) + } + } + state.diagnostics = append(state.diagnostics, diagnostic) +} + +func (state *resolutionState) addDiagnostic(code Code, path Path, message string) { + diagnostic := Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message} + if location, ok := nearestDocumentLocation(state.document, path); ok { + diagnostic.Primary = documentLocationPointer(location) + } + state.diagnostics = append(state.diagnostics, diagnostic) +} + +func runResolution[T any](configuration *T, resolver func(*T, *Resolution) error, schema *schemaDescriptor, decoded decodedConfiguration, input *document.Node, entry resolutionInput) resolutionResult { + state := &resolutionState{ + active: true, schema: schema, presence: decoded.presence, document: input, + input: entry, + records: make(map[string]*resolutionRecord), + } + context := &Resolution{state: state} + err := resolver(configuration, context) + state.active = false + if err != nil && !hasErrors(state.diagnostics) { + state.addDiagnostic(CodeResolveHookFailed, Path{}, "application resolution failed") + } + if hasErrors(state.diagnostics) { + return resolutionResult{diagnostics: canonicalDiagnostics(state.diagnostics)} + } + + after, issue := projectConfiguration(schema, reflect.ValueOf(*configuration), defaultProjectionLimits()) + if issue != nil { + state.addDiagnostic(CodeResolveInvalidValue, issue.path, "application resolution produced an invalid configuration value") + return resolutionResult{diagnostics: canonicalDiagnostics(state.diagnostics)} + } + deltas := compareConfigurationValues(decoded.final, after) + ordered := orderedResolutionRecords(state.records) + for _, delta := range deltas { + if coveringResolutionRecord(ordered, delta.path) == nil { + state.addDiagnostic(CodeResolveUnrecordedChange, delta.path, "application resolution changed a value without a covering record") + } + } + for _, record := range ordered { + _, beforePresent := documentAtPath(decoded.final, record.path) + afterNode, afterPresent := documentAtPath(after, record.path) + if !beforePresent && !afterPresent { + state.addDiagnostic(CodeResolveInvalidRecord, record.path, "resolution record does not identify an existing or removed value") + continue + } + if record.helperExpected != nil && (!afterPresent || !configurationValuesEqual(record.helperExpected, afterNode)) { + state.addDiagnostic(CodeResolveInvalidTarget, record.path, "resolution helper assignment does not match its declared configuration path") + } + } + if hasErrors(state.diagnostics) { + return resolutionResult{diagnostics: canonicalDiagnostics(state.diagnostics)} + } + + snapshots := make([]resolutionSnapshot, 0, len(ordered)) + for _, record := range ordered { + beforeNode, _ := documentAtPath(decoded.final, record.path) + afterNode, _ := documentAtPath(after, record.path) + snapshots = append(snapshots, resolutionSnapshot{ + change: Change{ + Path: record.path.clone(), + Origin: Origin{Layer: LayerResolution, Operation: record.kind}, + Reasons: append([]string(nil), record.reasons...), + Dependencies: clonePaths(record.dependencies), + }, + before: document.Clone(beforeNode), after: document.Clone(afterNode), + }) + } + return resolutionResult{ + final: after, deltas: deltas, snapshots: snapshots, + diagnostics: canonicalDiagnostics(state.diagnostics), + } +} + +func orderedResolutionRecords(records map[string]*resolutionRecord) []*resolutionRecord { + ordered := make([]*resolutionRecord, 0, len(records)) + for _, record := range records { + ordered = append(ordered, record) + } + slices.SortFunc(ordered, func(left, right *resolutionRecord) int { + return strings.Compare(left.path.String(), right.path.String()) + }) + return ordered +} + +func coveringResolutionRecord(records []*resolutionRecord, path Path) *resolutionRecord { + var covering *resolutionRecord + for _, record := range records { + if pathContains(record.path, path) && (covering == nil || len(record.path.segments) > len(covering.path.segments)) { + covering = record + } + } + return covering +} + +func pathContains(container, descendant Path) bool { + if !container.valid() || !descendant.valid() || len(container.segments) > len(descendant.segments) { + return false + } + if container.String() == "" { + return true + } + prefix := container.String() + value := descendant.String() + return value == prefix || strings.HasPrefix(value, prefix+"/") +} + +func appendUniqueString(values []string, value string) []string { + if !slices.Contains(values, value) { + return append(values, value) + } + return values +} + +func appendUniquePath(paths []Path, path Path) []Path { + for _, existing := range paths { + if existing.String() == path.String() { + return paths + } + } + return append(paths, path.clone()) +} + +func clonePaths(paths []Path) []Path { + if paths == nil { + return nil + } + cloned := make([]Path, len(paths)) + for index, path := range paths { + cloned[index] = path.clone() + } + return cloned +} diff --git a/internal/engine/resolution_test.go b/internal/engine/resolution_test.go new file mode 100644 index 0000000..28f3210 --- /dev/null +++ b/internal/engine/resolution_test.go @@ -0,0 +1,457 @@ +package engine + +import ( + "errors" + "math" + "reflect" + "strings" + "testing" + "time" +) + +func TestLoadRunsRecordedResolutionWithMapWriteback(t *testing.T) { + t.Parallel() + + type artifact struct { + Output string `yaml:"output"` + Timeout time.Duration `yaml:"timeout"` + } + type config struct { + PollInterval time.Duration `yaml:"poll_interval"` + Timeout time.Duration `yaml:"timeout"` + Artifacts map[string]artifact `yaml:"artifacts"` + } + definition := Definition[config]{ + Defaults: func() config { + return config{PollInterval: time.Minute, Timeout: 5 * time.Second} + }, + Resolve: func(configuration *config, resolution *Resolution) error { + Default(resolution, Keys("timeout"), &configuration.Timeout, + configuration.PollInterval/10, "derived from poll interval", Keys("poll_interval")) + for name, current := range configuration.Artifacts { + path := Keys("artifacts", name) + Default(resolution, path.Key("timeout"), ¤t.Timeout, + configuration.Timeout, "inherited configuration timeout", Keys("timeout")) + Normalize(resolution, path.Key("output"), ¤t.Output, + strings.TrimSpace(current.Output), "trimmed surrounding whitespace") + configuration.Artifacts[name] = current + } + resolution.Warning(Keys("timeout"), "using derived timeout", Keys("poll_interval")) + return nil + }, + } + input := "poll_interval: 1m\nartifacts:\n second: {output: ' second '}\n first: {output: ' first '}\n" + result, err := Load(Bytes("resolution.yml", []byte(input)), Options[config]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + if result.Config.Timeout != 6*time.Second { + t.Fatalf("timeout = %s", result.Config.Timeout) + } + for name, want := range map[string]string{"first": "first", "second": "second"} { + current := result.Config.Artifacts[name] + if current.Timeout != 6*time.Second || current.Output != want { + t.Errorf("artifact %q = %#v", name, current) + } + } + if diagnostics := result.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != CodeResolveWarning || diagnostics[0].Primary == nil || len(diagnostics[0].Related) != 1 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + + metadata := result.Metadata() + timeout, ok := metadata.Lookup(Keys("timeout")) + if !ok || timeout.Explicit || timeout.Origin.Layer != LayerResolution || timeout.Origin.Operation != ChangeDerivedDefault { + t.Fatalf("timeout metadata = %#v, %v", timeout, ok) + } + output, ok := metadata.Lookup(Keys("artifacts", "first", "output")) + if !ok || !output.Explicit || output.Origin.Operation != ChangeNormalization { + t.Fatalf("output metadata = %#v, %v", output, ok) + } + history := metadata.History(Keys("timeout")) + last := history[len(history)-1] + if last.Origin.Operation != ChangeDerivedDefault || !reflect.DeepEqual(last.Reasons, []string{"derived from poll interval"}) || len(last.Dependencies) != 1 || last.Dependencies[0].String() != "/poll_interval" { + t.Fatalf("timeout history = %#v", history) + } + wantOrder := []string{ + "/artifacts/first/output", "/artifacts/first/timeout", + "/artifacts/second/output", "/artifacts/second/timeout", "/timeout", + } + gotOrder := make([]string, len(result.resolution)) + for index, snapshot := range result.resolution { + gotOrder[index] = snapshot.change.Path.String() + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Fatalf("resolution order = %#v", gotOrder) + } +} + +func TestResolutionPreservesExplicitNullAndZeroWhileReplacingAbsentBaseline(t *testing.T) { + t.Parallel() + + type config struct { + Optional *int `yaml:"optional"` + Count int `yaml:"count"` + Absent int `yaml:"absent"` + } + five := 5 + result, err := Load(Bytes("explicit.yml", []byte("optional: null\ncount: 0\n")), Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Optional: &five, Count: 5, Absent: 5} }, + Resolve: func(configuration *config, resolution *Resolution) error { + if !resolution.IsExplicit(Keys("optional")) || !resolution.IsExplicit(Keys("count")) || resolution.IsExplicit(Keys("absent")) { + return errors.New("unexpected presence") + } + Default(resolution, Keys("optional"), &configuration.Optional, &five, "would derive optional") + Default(resolution, Keys("count"), &configuration.Count, 9, "would derive count") + Default(resolution, Keys("absent"), &configuration.Absent, 9, "derived absent value") + return nil + }, + }}) + if err != nil { + var loadError *LoadError + _ = errors.As(err, &loadError) + t.Fatalf("%v: %#v", err, loadError.Diagnostics()) + } + if result.Config.Optional != nil || result.Config.Count != 0 || result.Config.Absent != 9 { + t.Fatalf("config = %#v", result.Config) + } + if len(result.Metadata().History(Keys("optional"))) != 2 || len(result.Metadata().History(Keys("count"))) != 2 || len(result.Metadata().History(Keys("absent"))) != 2 { + t.Fatal("resolution history disagrees with explicitness") + } +} + +func TestResolutionCoalescesEqualAndRepeatedHelperIntent(t *testing.T) { + t.Parallel() + + type config struct { + Value int `yaml:"value"` + Other int `yaml:"other"` + } + result, err := Load(NoSource(), Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Value: 5} }, + Resolve: func(configuration *config, resolution *Resolution) error { + Default(resolution, Keys("value"), &configuration.Value, 5, "equal derived intent", Keys("other")) + Default(resolution, Keys("value"), &configuration.Value, 7, "updated derived intent", Keys("other")) + Normalize(resolution, Keys("value"), &configuration.Value, 7, "canonicalized value", Keys("other")) + return nil + }, + }}) + if err != nil { + t.Fatal(err) + } + history := result.Metadata().History(Keys("value")) + if len(history) != 2 { + t.Fatalf("history = %#v", history) + } + change := history[1] + if change.Origin.Operation != ChangeNormalization || !reflect.DeepEqual(change.Reasons, []string{"equal derived intent", "updated derived intent", "canonicalized value"}) || len(change.Dependencies) != 1 { + t.Fatalf("coalesced change = %#v", change) + } +} + +func TestResolutionContainerRecordCoversSubtreeRemovalAndSnapshotsAreDetached(t *testing.T) { + t.Parallel() + + type item struct { + Value string `yaml:"value"` + } + type config struct { + Items map[string]item `yaml:"items"` + } + result, err := Load(Bytes("remove.yml", []byte("items:\n keep: {value: 'yes'}\n remove: {value: 'no'}\n")), Options[config]{Definition: Definition[config]{ + Resolve: func(configuration *config, resolution *Resolution) error { + delete(configuration.Items, "remove") + resolution.Record(Keys("items"), ChangeNormalization, "removed disabled entries") + return nil + }, + }}) + if err != nil { + t.Fatal(err) + } + if _, exists := result.Config.Items["remove"]; exists { + t.Fatal("removed entry remains") + } + if _, ok := result.Metadata().Lookup(Keys("items", "remove")); ok { + t.Fatal("removed entry remains effective metadata") + } + if len(result.resolution) != 1 || result.resolution[0].before == nil || result.resolution[0].after == nil { + t.Fatalf("snapshots = %#v", result.resolution) + } + result.Config.Items["keep"] = item{Value: "changed"} + if mappingValue(t, result.resolution[0].after, "keep").Mapping[0].Value.Text != "yes" { + t.Fatal("caller mutation changed private resolution snapshot") + } +} + +func TestResolutionSequenceReplacementUpdatesDescendantOrigins(t *testing.T) { + t.Parallel() + + type config struct { + Values []int `yaml:"values"` + } + result, err := Load(Bytes("sequence.yml", []byte("values: [1, 2]\n")), Options[config]{Definition: Definition[config]{ + Resolve: func(configuration *config, resolution *Resolution) error { + Normalize(resolution, Keys("values"), &configuration.Values, []int{3}, "replaced values") + return nil + }, + }}) + if err != nil { + t.Fatal(err) + } + info, ok := result.Metadata().Lookup(Keys("values").Index(0)) + if !ok || !info.Explicit || info.Origin.Layer != LayerResolution || info.Origin.Operation != ChangeNormalization { + t.Fatalf("element metadata = %#v, %v", info, ok) + } + if _, ok := result.Metadata().Lookup(Keys("values").Index(1)); ok { + t.Fatal("removed sequence element remains effective") + } +} + +func TestResolutionRejectsUnrecordedAndMismatchedMutations(t *testing.T) { + t.Parallel() + + type config struct { + Value int `yaml:"value"` + } + tests := []struct { + name string + resolve func(*config, *Resolution) error + code Code + }{ + {name: "unrecorded", resolve: func(configuration *config, _ *Resolution) error { + configuration.Value = 2 + return nil + }, code: CodeResolveUnrecordedChange}, + {name: "unrelated target", resolve: func(_ *config, resolution *Resolution) error { + unrelated := 1 + Normalize(resolution, Keys("value"), &unrelated, 2, "changed value") + return nil + }, code: CodeResolveInvalidTarget}, + {name: "invalid record kind", resolve: func(_ *config, resolution *Resolution) error { + resolution.Record(Keys("value"), ChangeKind("other"), "invalid kind") + return nil + }, code: CodeResolveInvalidKind}, + {name: "invalid path", resolve: func(_ *config, resolution *Resolution) error { + resolution.Record(Keys("unknown"), ChangeNormalization, "unknown field") + return nil + }, code: CodeResolveInvalidPath}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := Load(Bytes("invalid-resolution.yml", []byte("value: 1\n")), Options[config]{Definition: Definition[config]{Resolve: test.resolve}}) + assertLoadDiagnostic(t, result, err, test.code) + }) + } +} + +func TestResolutionRejectsInvalidProjectedValuesAndSuppressesReturnedErrors(t *testing.T) { + t.Parallel() + + type config struct { + Number float64 `yaml:"number"` + Value int `yaml:"value"` + } + tests := []struct { + name string + resolve func(*config, *Resolution) error + code Code + }{ + {name: "invalid helper value", resolve: func(configuration *config, resolution *Resolution) error { + Normalize(resolution, Keys("number"), &configuration.Number, math.NaN(), "invalid number") + return nil + }, code: CodeResolveInvalidValue}, + {name: "reported error", resolve: func(_ *config, resolution *Resolution) error { + resolution.Error(Keys("value"), "value cannot be resolved") + return errors.New("CANARY_RESOLUTION_ERROR") + }, code: CodeResolveError}, + {name: "returned error", resolve: func(_ *config, _ *Resolution) error { + return errors.New("CANARY_RESOLUTION_ERROR") + }, code: CodeResolveHookFailed}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := Load(Bytes("resolution-errors.yml", []byte("value: 1\nnumber: 1\n")), Options[config]{Definition: Definition[config]{Resolve: test.resolve}}) + loadError := assertLoadDiagnostic(t, result, err, test.code) + for _, diagnostic := range loadError.Diagnostics() { + if strings.Contains(diagnostic.Message, "CANARY") { + t.Fatalf("diagnostic exposed returned error: %#v", diagnostic) + } + } + }) + } +} + +func TestResolutionEntryQueriesExposeDetachedLoadInput(t *testing.T) { + t.Parallel() + + type config struct { + Declared string `yaml:"declared"` + Default string `yaml:"default"` + Optional *string `yaml:"optional"` + } + filename := writeImportFile(t, t.TempDir(), "config.yml", "declared: file\noptional: null\n") + var savedField FieldInfo + var savedSource SourceInfo + result, err := Load[config](File(filename), Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Default: "retained"} }, + Resolve: func(_ *config, resolution *Resolution) error { + for _, path := range []Path{Keys("declared"), Keys("default"), Keys("optional")} { + info, found := resolution.InputInfo(path) + if !found || info.Explicit != resolution.IsExplicit(path) { + return errors.New("entry metadata does not agree with explicitness") + } + } + defaultInfo, _ := resolution.InputInfo(Keys("default")) + optionalInfo, _ := resolution.InputInfo(Keys("optional")) + if defaultInfo.Explicit || defaultInfo.Origin.Layer != LayerDefault || !optionalInfo.Explicit || !optionalInfo.Null { + return errors.New("entry metadata lost default or null state") + } + if _, found := resolution.InputInfo(Keys("missing")); found { + return errors.New("missing entry path was found") + } + declared, _ := resolution.InputInfo(Keys("declared")) + if len(declared.Origin.Locations) != 1 { + return errors.New("declared source location is missing") + } + source, found := resolution.Source(declared.Origin.Locations[0].SourceID) + if !found || source.Path != canonicalTestPath(t, filename) || source.ID != "source-1" { + return errors.New("declared source record is missing") + } + if _, found := resolution.Source(""); found { + return errors.New("empty source identity was found") + } + if _, found := resolution.Source("unknown"); found { + return errors.New("unknown source identity was found") + } + savedField, savedSource = declared, source + return nil + }, + }}) + if err != nil { + t.Fatal(err) + } + if result.Config.Declared != "file" || result.Config.Default != "retained" || result.Config.Optional != nil { + t.Fatalf("configuration = %#v", result.Config) + } + savedField.Origin.Locations[0].SourceID = "mutated" + savedSource.Label = "mutated" + field, _ := result.Metadata().Lookup(Keys("declared")) + if field.Origin.Locations[0].SourceID != "source-1" || result.Sources()[0].Label == "mutated" { + t.Fatal("saved resolution records alias a completed result") + } + + noSource, err := Load[config](NoSource(), Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Default: "retained"} }, + Resolve: func(_ *config, resolution *Resolution) error { + info, found := resolution.InputInfo(Keys("default")) + if !found || info.Explicit || resolution.IsExplicit(Keys("default")) { + return errors.New("defaults-only entry metadata is wrong") + } + if _, found := resolution.Source("source-1"); found { + return errors.New("defaults-only load invented a source") + } + return nil + }, + }}) + if err != nil || noSource.Config.Default != "retained" { + t.Fatalf("defaults-only load = %#v, %v", noSource, err) + } +} + +func TestResolutionContextPanicsAfterCallback(t *testing.T) { + t.Parallel() + + type config struct { + Value int `yaml:"value"` + } + var retained *Resolution + if _, err := Load(NoSource(), Options[config]{Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + retained = resolution + return nil + }}}); err != nil { + t.Fatal(err) + } + defer func() { + if recovered := recover(); recovered != closedCallbackContextMessage { + t.Fatalf("panic = %#v", recovered) + } + }() + retained.IsExplicit(Keys("value")) +} + +func TestResolutionEntryQueriesRequireAnActiveCallback(t *testing.T) { + t.Parallel() + + type config struct { + Value int `yaml:"value"` + } + requireClosed := func(t *testing.T, call func()) { + t.Helper() + defer func() { + if recovered := recover(); recovered != closedCallbackContextMessage { + t.Fatalf("panic = %#v", recovered) + } + }() + call() + } + requireClosed(t, func() { (*Resolution)(nil).InputInfo(Keys("value")) }) + requireClosed(t, func() { (&Resolution{}).Source("source-1") }) + + var successful *Resolution + if _, err := Load(NoSource(), Options[config]{Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + successful = resolution + return nil + }}}); err != nil { + t.Fatal(err) + } + requireClosed(t, func() { successful.InputInfo(Keys("value")) }) + requireClosed(t, func() { successful.Source("source-1") }) + + var failed *Resolution + if _, err := Load(NoSource(), Options[config]{Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + failed = resolution + return errors.New("resolver failed") + }}}); err == nil { + t.Fatal("failed resolver succeeded") + } + requireClosed(t, func() { failed.InputInfo(Keys("value")) }) + requireClosed(t, func() { failed.Source("source-1") }) +} + +func TestValidationContextPanicsAfterVersionCallback(t *testing.T) { + t.Parallel() + + type config struct { + Version string `yaml:"version"` + } + var retained *Validation + if _, err := Load(Bytes("version-context.yml", []byte("version: one\n")), Options[config]{Definition: Definition[config]{ + Version: &VersionRule{Path: Keys("version"), Check: func(_ Version, validation *Validation) error { + retained = validation + return nil + }}, + }}); err != nil { + t.Fatal(err) + } + defer func() { + if recovered := recover(); recovered != closedCallbackContextMessage { + t.Fatalf("panic = %#v", recovered) + } + }() + retained.Warning(Keys("version"), "too late") +} + +func assertLoadDiagnostic[T any](t *testing.T, result *Result[T], err error, code Code) *LoadError { + t.Helper() + if result != nil || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 0 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v", diagnostics) + } + return loadError +} diff --git a/internal/engine/schema.go b/internal/engine/schema.go new file mode 100644 index 0000000..abfc595 --- /dev/null +++ b/internal/engine/schema.go @@ -0,0 +1,438 @@ +package engine + +import ( + "encoding" + "reflect" + "strconv" + "strings" + "time" +) + +type descriptorKind uint8 + +const ( + descriptorInvalid descriptorKind = iota + descriptorStruct + descriptorMap + descriptorSlice + descriptorArray + descriptorPointer + descriptorDynamic + descriptorString + descriptorBoolean + descriptorSigned + descriptorUnsigned + descriptorFloat + descriptorDuration + descriptorSecret + descriptorText +) + +type schemaDescriptor struct { + root *typeDescriptor + types map[reflect.Type]*typeDescriptor +} + +type typeDescriptor struct { + typeOf reflect.Type + kind descriptorKind + element *typeDescriptor + length int + members []structMember + fields []fieldDescriptor + fieldByName map[string]int + inlineMap *fieldDescriptor +} + +type structMember struct { + index int + name string + descriptor *typeDescriptor + inline bool +} + +type fieldDescriptor struct { + name string + index []int + descriptor *typeDescriptor + secret bool + inlineMap bool +} + +type schemaIssue struct { + code Code + path Path + message string +} + +var ( + durationType = reflect.TypeFor[time.Duration]() + secretType = reflect.TypeFor[Secret]() + textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]() + textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]() +) + +func buildSchema(root reflect.Type) (*schemaDescriptor, *schemaIssue) { + if root == nil || (root.Kind() != reflect.Struct && !isStringMap(root)) { + return nil, &schemaIssue{code: CodeSchemaRoot, message: "configuration root must be a struct or string-keyed map"} + } + builder := schemaBuilder{ + cache: make(map[reflect.Type]*typeDescriptor), + building: make(map[reflect.Type]bool), + } + descriptor, issue := builder.describe(root, Path{}) + if issue != nil { + return nil, issue + } + if descriptor.kind != descriptorStruct && descriptor.kind != descriptorMap { + return nil, &schemaIssue{code: CodeSchemaRoot, message: "configuration root must be a structural struct or string-keyed map"} + } + return &schemaDescriptor{root: descriptor, types: builder.cache}, nil +} + +type schemaBuilder struct { + cache map[reflect.Type]*typeDescriptor + building map[reflect.Type]bool +} + +func (b *schemaBuilder) describe(current reflect.Type, path Path) (*typeDescriptor, *schemaIssue) { + if descriptor, ok := b.cache[current]; ok { + return descriptor, nil + } + descriptor := &typeDescriptor{typeOf: current} + b.cache[current] = descriptor + b.building[current] = true + defer delete(b.building, current) + + fail := func(code Code, message string) (*typeDescriptor, *schemaIssue) { + delete(b.cache, current) + return nil, &schemaIssue{code: code, path: path.clone(), message: message} + } + + if current.Kind() == reflect.Pointer { + descriptor.kind = descriptorPointer + element, issue := b.describe(current.Elem(), path) + if issue != nil { + return nil, issue + } + descriptor.element = element + return descriptor, nil + } + if current == durationType { + descriptor.kind = descriptorDuration + return descriptor, nil + } + if current == secretType { + descriptor.kind = descriptorSecret + return descriptor, nil + } + marshals, unmarshals := textCodecSupport(current) + if marshals || unmarshals { + if !marshals || !unmarshals { + return fail(CodeSchemaTextCodec, "custom text scalar must implement both text marshal and unmarshal") + } + descriptor.kind = descriptorText + return descriptor, nil + } + + switch current.Kind() { + case reflect.Struct: + descriptor.kind = descriptorStruct + descriptor.fieldByName = make(map[string]int) + if issue := b.describeFields(descriptor, current, path); issue != nil { + delete(b.cache, current) + return nil, issue + } + case reflect.Map: + if current.Key().Kind() != reflect.String { + return fail(CodeSchemaUnsupportedType, "configuration maps must have string keys") + } + descriptor.kind = descriptorMap + element, issue := b.describe(current.Elem(), path) + if issue != nil { + return nil, issue + } + descriptor.element = element + case reflect.Slice: + descriptor.kind = descriptorSlice + element, issue := b.describe(current.Elem(), path.Index(0)) + if issue != nil { + return nil, issue + } + descriptor.element = element + case reflect.Array: + descriptor.kind = descriptorArray + descriptor.length = current.Len() + element, issue := b.describe(current.Elem(), path.Index(0)) + if issue != nil { + return nil, issue + } + descriptor.element = element + case reflect.Interface: + if current.NumMethod() != 0 { + return fail(CodeSchemaUnsupportedType, "only the empty interface is supported for dynamic values") + } + descriptor.kind = descriptorDynamic + case reflect.String: + descriptor.kind = descriptorString + case reflect.Bool: + descriptor.kind = descriptorBoolean + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + descriptor.kind = descriptorSigned + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + descriptor.kind = descriptorUnsigned + case reflect.Float32, reflect.Float64: + descriptor.kind = descriptorFloat + default: + return fail(CodeSchemaUnsupportedType, "configuration type is not supported") + } + return descriptor, nil +} + +func (b *schemaBuilder) describeFields(descriptor *typeDescriptor, current reflect.Type, path Path) *schemaIssue { + for i := 0; i < current.NumField(); i++ { + field := current.Field(i) + if !field.IsExported() { + continue + } + name, inline, ignored, issue := parseYAMLField(field, path) + if issue != nil { + return issue + } + if ignored { + continue + } + secret, issue := parseConfigKitField(field, path) + if issue != nil { + return issue + } + + if inline { + if name != "" { + return &schemaIssue{code: CodeSchemaTag, path: path.clone(), message: "inline field cannot declare a YAML name"} + } + inlineType := field.Type + seenPointers := make(map[reflect.Type]bool) + for inlineType.Kind() == reflect.Pointer { + if seenPointers[inlineType] { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline pointer chain must terminate in a struct"} + } + seenPointers[inlineType] = true + inlineType = inlineType.Elem() + } + if inlineType.Kind() == reflect.Map && field.Type.Kind() == reflect.Pointer { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline maps cannot be pointers"} + } + switch inlineType.Kind() { + case reflect.Struct: + if b.building[inlineType] { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "recursive inline struct is not supported"} + } + child, childIssue := b.describe(inlineType, path) + if childIssue != nil { + return childIssue + } + if child.kind != descriptorStruct { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline field must be a structural struct or string-keyed map"} + } + for _, nested := range child.fields { + nested.index = prependIndex(i, nested.index) + nested.secret = secret || nested.secret + if issue := addField(descriptor, nested, path); issue != nil { + return issue + } + } + if child.inlineMap != nil { + nested := *child.inlineMap + nested.index = prependIndex(i, nested.index) + nested.secret = secret || nested.secret + if descriptor.inlineMap != nil { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "configuration object has more than one inline map"} + } + descriptor.inlineMap = &nested + } + memberType, memberIssue := b.describe(field.Type, path) + if memberIssue != nil { + return memberIssue + } + descriptor.members = append(descriptor.members, structMember{index: i, descriptor: memberType, inline: true}) + case reflect.Map: + if inlineType.Key().Kind() != reflect.String { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline map must have string keys"} + } + if descriptor.inlineMap != nil { + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "configuration object has more than one inline map"} + } + fieldPath := path.Key("*") + fieldType, childIssue := b.describe(inlineType, fieldPath) + if childIssue != nil { + return childIssue + } + descriptor.inlineMap = &fieldDescriptor{index: []int{i}, descriptor: fieldType, secret: secret, inlineMap: true} + descriptor.members = append(descriptor.members, structMember{index: i, descriptor: fieldType, inline: true}) + default: + return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline field must be a struct, pointer to struct, or string-keyed map"} + } + continue + } + + fieldPath := path.Key(name) + fieldType, childIssue := b.describe(field.Type, fieldPath) + if childIssue != nil { + return childIssue + } + if issue := addField(descriptor, fieldDescriptor{name: name, index: []int{i}, descriptor: fieldType, secret: secret}, path); issue != nil { + return issue + } + descriptor.members = append(descriptor.members, structMember{index: i, name: name, descriptor: fieldType}) + } + return nil +} + +func addField(owner *typeDescriptor, field fieldDescriptor, path Path) *schemaIssue { + if _, exists := owner.fieldByName[field.name]; exists { + return &schemaIssue{code: CodeSchemaAmbiguousField, path: path.Key(field.name), message: "configuration field name is ambiguous"} + } + owner.fieldByName[field.name] = len(owner.fields) + owner.fields = append(owner.fields, field) + return nil +} + +func parseYAMLField(field reflect.StructField, path Path) (name string, inline, ignored bool, issue *schemaIssue) { + name = strings.ToLower(field.Name) + tag, present := field.Tag.Lookup("yaml") + if !present { + return name, false, false, nil + } + parts := strings.Split(tag, ",") + if parts[0] == "-" { + if len(parts) != 1 { + return "", false, false, &schemaIssue{code: CodeSchemaTag, path: path.clone(), message: "ignored YAML field cannot declare options"} + } + return "", false, true, nil + } + if parts[0] != "" { + name = parts[0] + } + seen := make(map[string]bool, len(parts)-1) + for _, option := range parts[1:] { + if option == "" || seen[option] { + return "", false, false, &schemaIssue{code: CodeSchemaTag, path: path.Key(name), message: "YAML field tag is malformed"} + } + seen[option] = true + switch option { + case "inline": + inline = true + case "omitempty": + // ConfigKit accepts the spelling but never omits projected values. + default: + return "", false, false, &schemaIssue{code: CodeSchemaTag, path: path.Key(name), message: "YAML field tag option is not supported"} + } + } + if inline && parts[0] == "" { + name = "" + } + return name, inline, false, nil +} + +func parseConfigKitField(field reflect.StructField, path Path) (bool, *schemaIssue) { + tag, present := field.Tag.Lookup("configkit") + if !present || tag == "" { + return false, nil + } + if tag != "secret" { + return false, &schemaIssue{code: CodeSchemaTag, path: path.clone(), message: "ConfigKit field tag is not supported"} + } + return true, nil +} + +func textCodecSupport(current reflect.Type) (marshals, unmarshals bool) { + marshals = current.Implements(textMarshalerType) + unmarshals = current.Implements(textUnmarshalerType) + if current.Kind() != reflect.Pointer { + pointer := reflect.PointerTo(current) + marshals = marshals || pointer.Implements(textMarshalerType) + unmarshals = unmarshals || pointer.Implements(textUnmarshalerType) + } + return marshals, unmarshals +} + +func isStringMap(current reflect.Type) bool { + return current.Kind() == reflect.Map && current.Key().Kind() == reflect.String +} + +func prependIndex(index int, nested []int) []int { + result := make([]int, 1, len(nested)+1) + result[0] = index + return append(result, nested...) +} + +type schemaPath struct { + descriptor *typeDescriptor + field *fieldDescriptor + dynamic bool +} + +func resolveSchemaPath(schema *schemaDescriptor, path Path) (schemaPath, bool) { + if schema == nil || schema.root == nil || !path.valid() { + return schemaPath{}, false + } + current := schema.root + var field *fieldDescriptor + for _, segment := range path.segments { + for current.kind == descriptorPointer { + current = current.element + } + switch current.kind { + case descriptorStruct: + if segment.kind == pathSegmentIndex { + return schemaPath{}, false + } + position, ok := current.fieldByName[segment.key] + if ok { + field = ¤t.fields[position] + current = field.descriptor + continue + } + if current.inlineMap != nil { + field = current.inlineMap + current = current.inlineMap.descriptor.element + continue + } + return schemaPath{}, false + case descriptorMap: + if segment.kind == pathSegmentIndex { + return schemaPath{}, false + } + field = nil + current = current.element + case descriptorSlice, descriptorArray: + index, ok := schemaIndex(segment) + if !ok || (current.kind == descriptorArray && index >= current.length) { + return schemaPath{}, false + } + field = nil + current = current.element + case descriptorDynamic: + return schemaPath{descriptor: current, field: field, dynamic: true}, true + default: + return schemaPath{}, false + } + } + dynamic := current + for dynamic.kind == descriptorPointer { + dynamic = dynamic.element + } + return schemaPath{descriptor: current, field: field, dynamic: dynamic.kind == descriptorDynamic}, true +} + +func schemaIndex(segment pathSegment) (int, bool) { + if segment.kind == pathSegmentIndex { + return segment.index, segment.index >= 0 + } + if segment.kind != pathSegmentParsed || segment.key == "" || (len(segment.key) > 1 && segment.key[0] == '0') { + return 0, false + } + index, err := strconv.Atoi(segment.key) + return index, err == nil && index >= 0 && strconv.Itoa(index) == segment.key +} diff --git a/internal/engine/schema_test.go b/internal/engine/schema_test.go new file mode 100644 index 0000000..7b16c95 --- /dev/null +++ b/internal/engine/schema_test.go @@ -0,0 +1,342 @@ +package engine + +import ( + "errors" + "reflect" + "testing" + "time" +) + +type namedCount int32 + +type textValue struct { + value string +} + +func (v textValue) MarshalText() ([]byte, error) { + return []byte(v.value), nil +} + +func (v *textValue) UnmarshalText(data []byte) error { + v.value = string(data) + return nil +} + +type marshalOnly string + +func (v marshalOnly) MarshalText() ([]byte, error) { + return []byte(v), nil +} + +type unmarshalOnly string + +func (v *unmarshalOnly) UnmarshalText(data []byte) error { + *v = unmarshalOnly(data) + return nil +} + +type recursiveConfig struct { + Name string `yaml:"name"` + Next *recursiveConfig `yaml:"next"` + Children map[string]recursiveConfig `yaml:"children"` +} + +func TestBuildSchemaDescribesSupportedTypes(t *testing.T) { + t.Parallel() + + type Embedded struct { + InlineName string `yaml:"inline_name"` + } + type nested struct { + Enabled bool `yaml:"enabled"` + } + type config struct { + HTTPServer string `yaml:"server,omitempty" configkit:"secret"` + Count namedCount `yaml:"count"` + Delay time.Duration `yaml:"delay"` + Custom textValue `yaml:"custom"` + Optional *textValue `yaml:"optional"` + Nested nested `yaml:"nested"` + Objects map[string]nested `yaml:"objects"` + Items []nested `yaml:"items"` + Fixed [2]string `yaml:"fixed"` + Dynamic any `yaml:"dynamic"` + *Embedded `yaml:",inline"` + Extras map[string]int `yaml:",inline"` + UntaggedField bool + Ignored chan int `yaml:"-"` + private chan int + } + + schema, issue := buildSchema(reflect.TypeFor[config]()) + if issue != nil { + t.Fatalf("buildSchema() issue = %#v", issue) + } + root := schema.root + checks := map[string]descriptorKind{ + "server": descriptorString, + "count": descriptorSigned, + "delay": descriptorDuration, + "custom": descriptorText, + "optional": descriptorPointer, + "nested": descriptorStruct, + "objects": descriptorMap, + "items": descriptorSlice, + "fixed": descriptorArray, + "dynamic": descriptorDynamic, + "inline_name": descriptorString, + "untaggedfield": descriptorBoolean, + } + for name, want := range checks { + position, ok := root.fieldByName[name] + if !ok { + t.Errorf("field %q missing", name) + continue + } + got := root.fields[position].descriptor.kind + if got != want { + t.Errorf("field %q kind = %v, want %v", name, got, want) + } + } + server := root.fields[root.fieldByName["server"]] + if !server.secret { + t.Fatal("secret tag was not retained") + } + if root.inlineMap == nil || root.inlineMap.descriptor.kind != descriptorMap { + t.Fatalf("inline map = %#v", root.inlineMap) + } + if mapped, mapIssue := buildSchema(reflect.TypeFor[map[string]nested]()); mapIssue != nil || mapped.root.kind != descriptorMap { + t.Fatalf("string-keyed root map = %#v, issue=%#v", mapped, mapIssue) + } + if _, exists := root.fieldByName["ignored"]; exists { + t.Fatal("ignored field was bound") + } +} + +func TestBuildSchemaSupportsRecursiveDescriptorsWithoutSharingLoads(t *testing.T) { + t.Parallel() + + first, issue := buildSchema(reflect.TypeFor[recursiveConfig]()) + if issue != nil { + t.Fatal(issue.message) + } + next := first.root.fields[first.root.fieldByName["next"]].descriptor + if next.kind != descriptorPointer || next.element != first.root { + t.Fatal("recursive pointer did not refer to the existing descriptor") + } + children := first.root.fields[first.root.fieldByName["children"]].descriptor + if children.kind != descriptorMap || children.element != first.root { + t.Fatal("recursive map did not refer to the existing descriptor") + } + second, issue := buildSchema(reflect.TypeFor[recursiveConfig]()) + if issue != nil { + t.Fatal(issue.message) + } + if first.root == second.root { + t.Fatal("descriptor graph was cached across loads") + } +} + +func TestBuildSchemaRejectsInvalidRootsTypesTagsAndBindings(t *testing.T) { + t.Parallel() + + type InlineA struct { + Value string `yaml:"same"` + } + type InlineB struct { + Value string `yaml:"same"` + } + type ambiguous struct { + InlineA `yaml:",inline"` + InlineB `yaml:",inline"` + } + type directConflict struct { + Same string `yaml:"same"` + InlineA `yaml:",inline"` + } + type twoMaps struct { + First map[string]string `yaml:",inline"` + Second map[string]string `yaml:",inline"` + } + type invalidInline struct { + Value string `yaml:",inline"` + } + type invalidMap struct { + Values map[int]string `yaml:"values"` + } + type unsupported struct { + Value complex64 `yaml:"value"` + } + type badOption struct { + Value string `yaml:"value,flow"` + } + type badConfigTag struct { + Value string `configkit:"private"` + } + type missingMarshal struct { + Value unmarshalOnly `yaml:"value"` + } + type missingUnmarshal struct { + Value marshalOnly `yaml:"value"` + } + type RecursiveInline struct { + *RecursiveInline `yaml:",inline"` + } + + tests := []struct { + name string + typeOf reflect.Type + code Code + }{ + {name: "nil root", typeOf: nil, code: CodeSchemaRoot}, + {name: "pointer root", typeOf: reflect.TypeFor[*recursiveConfig](), code: CodeSchemaRoot}, + {name: "slice root", typeOf: reflect.TypeFor[[]string](), code: CodeSchemaRoot}, + {name: "non-string root map", typeOf: reflect.TypeFor[map[int]string](), code: CodeSchemaRoot}, + {name: "text scalar root", typeOf: reflect.TypeFor[textValue](), code: CodeSchemaRoot}, + {name: "ambiguous fields", typeOf: reflect.TypeFor[ambiguous](), code: CodeSchemaAmbiguousField}, + {name: "direct inline conflict", typeOf: reflect.TypeFor[directConflict](), code: CodeSchemaAmbiguousField}, + {name: "multiple inline maps", typeOf: reflect.TypeFor[twoMaps](), code: CodeSchemaInline}, + {name: "invalid inline", typeOf: reflect.TypeFor[invalidInline](), code: CodeSchemaInline}, + {name: "invalid nested map", typeOf: reflect.TypeFor[invalidMap](), code: CodeSchemaUnsupportedType}, + {name: "unsupported kind", typeOf: reflect.TypeFor[unsupported](), code: CodeSchemaUnsupportedType}, + {name: "unsupported YAML option", typeOf: reflect.TypeFor[badOption](), code: CodeSchemaTag}, + {name: "unsupported ConfigKit tag", typeOf: reflect.TypeFor[badConfigTag](), code: CodeSchemaTag}, + {name: "missing text marshal", typeOf: reflect.TypeFor[missingMarshal](), code: CodeSchemaTextCodec}, + {name: "missing text unmarshal", typeOf: reflect.TypeFor[missingUnmarshal](), code: CodeSchemaTextCodec}, + {name: "recursive inline", typeOf: reflect.TypeFor[RecursiveInline](), code: CodeSchemaInline}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, issue := buildSchema(test.typeOf); issue == nil || issue.code != test.code { + t.Fatalf("issue = %#v, want %s", issue, test.code) + } + }) + } +} + +func TestBuildSchemaDoesNotExecuteApplicationMethods(t *testing.T) { + t.Parallel() + + type config struct { + Value panicText `yaml:"value"` + Structural yamlHookStruct `yaml:"structural"` + } + if _, issue := buildSchema(reflect.TypeFor[config]()); issue != nil { + t.Fatal(issue.message) + } +} + +type panicText struct{} + +type yamlHookStruct struct { + Value string `yaml:"value"` +} + +func (*yamlHookStruct) UnmarshalYAML(any) error { + panic("descriptor construction called parser hook") +} + +func (panicText) MarshalText() ([]byte, error) { + panic("descriptor construction called MarshalText") +} + +func (*panicText) UnmarshalText([]byte) error { + panic("descriptor construction called UnmarshalText") +} + +func (panicText) UnmarshalYAML(any) error { + panic("descriptor construction called parser hook") +} + +func TestResolveSchemaPathUsesContainerSemantics(t *testing.T) { + t.Parallel() + + type item struct { + Name string `yaml:"name"` + } + type config struct { + Items []item `yaml:"items"` + Values map[string]item `yaml:"values"` + Fixed [2]string `yaml:"fixed"` + Dynamic any `yaml:"dynamic"` + Extras map[string]string `yaml:",inline"` + } + schema, issue := buildSchema(reflect.TypeFor[config]()) + if issue != nil { + t.Fatal(issue.message) + } + + valid := []Path{ + Keys("items").Index(0).Key("name"), + mustPath(t, "/items/0/name"), + Keys("values", "0", "name"), + mustPath(t, "/values/0/name"), + Keys("fixed").Index(1), + Keys("dynamic", "anything").Index(9), + Keys("application_key"), + } + for _, path := range valid { + if _, ok := resolveSchemaPath(schema, path); !ok { + t.Errorf("valid path %q rejected", path.String()) + } + } + invalid := []Path{ + Keys("missing", "field"), + Keys("items", "0", "name"), + mustPath(t, "/items/00/name"), + Keys("fixed").Index(2), + Keys("values").Index(0), + Path{}.Index(-1), + } + for _, path := range invalid { + if _, ok := resolveSchemaPath(schema, path); ok { + t.Errorf("invalid path %q accepted", path.String()) + } + } +} + +func mustPath(t *testing.T, pointer string) Path { + t.Helper() + path, err := ParsePath(pointer) + if err != nil { + t.Fatal(err) + } + return path +} + +func TestInlineMapPointersFailBeforeDefaults(t *testing.T) { + type cfg struct { + Extra *map[string]int `yaml:",inline"` + } + for _, nonNil := range []bool{false, true} { + called := false + _, err := Load(NoSource(), Options[cfg]{Definition: Definition[cfg]{Defaults: func() cfg { + called = true + if nonNil { + m := map[string]int{"value": 1} + return cfg{Extra: &m} + } + return cfg{} + }}}) + var loadErr *LoadError + if !errors.As(err, &loadErr) || called { + t.Fatalf("invalid inline schema reached defaults: called=%v error=%v", called, err) + } + if len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeSchemaInline { + t.Fatal("missing inline schema diagnostic") + } + } + type pointer *pointer + type recursive struct { + Extra pointer `yaml:",inline"` + } + if _, err := Load[recursive](NoSource()); err == nil { + t.Fatal("recursive inline pointer was accepted") + } + type double struct { + Extra **map[string]int `yaml:",inline"` + } + if _, err := Load[double](NoSource()); err == nil { + t.Fatal("double inline map pointer was accepted") + } +} diff --git a/internal/engine/secret.go b/internal/engine/secret.go new file mode 100644 index 0000000..51ac30b --- /dev/null +++ b/internal/engine/secret.go @@ -0,0 +1,153 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +type secretDependency struct { + target Path + dependency Path +} + +func classifySecretPaths(schema *schemaDescriptor, predicate func(Path) bool, metadata Metadata, final *document.Node, applications []historyApplication, projections ...*document.Node) map[string]bool { + paths := make(map[string]Path) + finalPaths := make(map[string]*document.Node) + indexDocument(final, Path{}, finalPaths) + for pointer := range finalPaths { + addSecretPath(paths, pointer) + } + for _, application := range applications { + applicationPaths := make(map[string]*document.Node) + indexDocument(application.value, application.path, applicationPaths) + for pointer := range applicationPaths { + addSecretPath(paths, pointer) + } + } + var dependencies []secretDependency + for pointer, history := range metadata.history { + addSecretPath(paths, pointer) + for _, change := range history { + paths[change.Path.String()] = change.Path.clone() + for _, dependency := range change.Dependencies { + paths[dependency.String()] = dependency.clone() + dependencies = append(dependencies, secretDependency{ + target: change.Path.clone(), dependency: dependency.clone(), + }) + } + } + } + + secret := make(map[string]bool) + for _, tree := range append(projections, final) { + projected := make(map[string]*document.Node) + indexDocument(tree, Path{}, projected) + for pointer, node := range projected { + addSecretPath(paths, pointer) + if node.Sensitive { + secret[pointer] = true + } + } + } + for pointer, path := range paths { + if root, ok := schemaSecretRoot(schema, path); ok { + secret[root.String()] = true + } + if predicate != nil && predicate(path.clone()) { + secret[pointer] = true + } + } + for { + changed := false + for _, dependency := range dependencies { + if secretIntersectsPath(paths, secret, dependency.dependency) && !secret[dependency.target.String()] { + secret[dependency.target.String()] = true + changed = true + } + } + if !changed { + return secret + } + } +} + +func addSecretPath(paths map[string]Path, pointer string) { + if path, err := ParsePath(pointer); err == nil { + paths[pointer] = path + } +} + +func secretIntersectsPath(paths map[string]Path, secret map[string]bool, candidate Path) bool { + for pointer := range secret { + classified, ok := paths[pointer] + if !ok { + continue + } + if pathContains(classified, candidate) || pathContains(candidate, classified) { + return true + } + } + return false +} + +func schemaSecretRoot(schema *schemaDescriptor, path Path) (Path, bool) { + if schema == nil || schema.root == nil || !path.valid() { + return Path{}, false + } + current := schema.root + walked := Path{} + for _, segment := range path.segments { + walked.segments = append(walked.segments, segment) + for current.kind == descriptorPointer { + current = current.element + } + switch current.kind { + case descriptorStruct: + if segment.kind == pathSegmentIndex { + return Path{}, false + } + position, ok := current.fieldByName[segment.key] + var field *fieldDescriptor + if ok { + field = ¤t.fields[position] + } else { + field = current.inlineMap + } + if field == nil { + return Path{}, false + } + if field.secret { + return walked.clone(), true + } + if field.inlineMap { + current = field.descriptor.element + } else { + current = field.descriptor + } + case descriptorMap: + if segment.kind == pathSegmentIndex { + return Path{}, false + } + current = current.element + case descriptorSlice, descriptorArray: + if _, ok := schemaIndex(segment); !ok { + return Path{}, false + } + current = current.element + case descriptorDynamic: + return Path{}, false + default: + return Path{}, false + } + } + return Path{}, false +} + +// Dynamic type replacement can change classification even when projected text +// is equal. Validators must preserve this inspection metadata as well as values. +func projectedSensitivity(root *document.Node) map[string]Path { + result := make(map[string]Path) + walkDocument(root, Path{}, func(path Path, node *document.Node) { + if node.Sensitive { + result[path.String()] = path.clone() + } + }) + return result +} diff --git a/internal/engine/secret_reference.go b/internal/engine/secret_reference.go new file mode 100644 index 0000000..b1a2705 --- /dev/null +++ b/internal/engine/secret_reference.go @@ -0,0 +1,225 @@ +package engine + +import ( + "cmp" + "encoding/json" + "fmt" + "io" + "reflect" + "slices" + "strconv" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// Secret is an opaque reference to an environment variable. Its name is public +// configuration; its resolved value is available only through Value. +type Secret struct { + name string + value string + specified bool + resolved bool +} + +// SecretEnv constructs an unresolved environment-backed secret reference. +// Invalid names are reported if the reference survives a configuration load. +func SecretEnv(name string) Secret { + return Secret{name: name, specified: true} +} + +// EnvName returns the public environment-variable name. +func (s Secret) EnvName() string { + return s.name +} + +// Value returns the resolved credential, or an empty string when unresolved. +func (s Secret) Value() string { + if !s.resolved { + return "" + } + return s.value +} + +// IsResolved reports whether this reference resolved to a nonempty value during +// its configuration load. +func (s Secret) IsResolved() bool { + return s.resolved +} + +// Format renders only the public reference name for every formatting verb. +func (s Secret) Format(state fmt.State, _ rune) { + _, _ = io.WriteString(state, "configkit.SecretEnv("+strconv.Quote(s.name)+")") +} + +// MarshalJSON emits only the public environment-variable name. +func (s Secret) MarshalJSON() ([]byte, error) { + return json.Marshal(s.name) +} + +type secretLookup func(string) (string, bool) + +type secretLookupResult struct { + value string + present bool +} + +type secretResolutionResult struct { + lookups map[string]secretLookupResult + references map[string]string + diagnostics []Diagnostic +} + +func resolveSecretReferences(root reflect.Value, schema *schemaDescriptor, input *document.Node, lookup secretLookup) secretResolutionResult { + result := secretResolutionResult{ + lookups: make(map[string]secretLookupResult), references: make(map[string]string), + } + if schema == nil || schema.root == nil || !root.IsValid() || root.Type() != schema.root.typeOf { + result.diagnostics = []Diagnostic{{Code: CodeSecretRequired, Severity: SeverityError, Message: "configuration value does not match its secret-reference schema"}} + return result + } + if lookup == nil { + lookup = func(string) (string, bool) { return "", false } + } + resolveSecretValue(root, schema.root, Path{}, false, input, lookup, &result) + result.diagnostics = canonicalDiagnostics(result.diagnostics) + return result +} + +func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, path Path, optional bool, input *document.Node, lookup secretLookup, result *secretResolutionResult) { + if descriptor.kind == descriptorPointer { + if value.IsNil() { + return + } + resolveSecretValue(value.Elem(), descriptor.element, path, descriptor.element.kind == descriptorSecret, input, lookup, result) + return + } + + switch descriptor.kind { + case descriptorSecret: + reference := value.Interface().(Secret) + name := reference.EnvName() + if name == "" { + code := CodeSecretRequired + message := "required secret reference is absent" + if optional || reference.specified { + code = CodeSecretInvalidName + message = "secret environment name is empty" + } + result.diagnostics = append(result.diagnostics, secretDiagnostic(code, path, message, input)) + return + } + if !validEnvironmentName(name) { + result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretInvalidName, path, "secret environment name is invalid", input)) + return + } + found, exists := result.lookups[name] + if !exists { + found.value, found.present = lookup(name) + result.lookups[name] = found + } + if !found.present { + result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretUnset, path, "secret environment variable is not set", input)) + return + } + if found.value == "" { + result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretEmpty, path, "secret environment variable is empty", input)) + return + } + value.Set(reflect.ValueOf(Secret{name: name, value: found.value, specified: true, resolved: true})) + result.references[path.String()] = name + case descriptorStruct: + for _, member := range descriptor.members { + memberPath := path + if !member.inline { + memberPath = path.Key(member.name) + } + resolveSecretValue(value.Field(member.index), member.descriptor, memberPath, false, input, lookup, result) + } + case descriptorMap: + if value.IsNil() { + return + } + keys := value.MapKeys() + slices.SortFunc(keys, func(left, right reflect.Value) int { return cmp.Compare(left.String(), right.String()) }) + for _, key := range keys { + entry := reflect.New(value.Type().Elem()).Elem() + entry.Set(value.MapIndex(key)) + resolveSecretValue(entry, descriptor.element, path.Key(key.String()), false, input, lookup, result) + value.SetMapIndex(key, entry) + } + case descriptorSlice, descriptorArray: + if descriptor.kind == descriptorSlice && value.IsNil() { + return + } + for index := 0; index < value.Len(); index++ { + resolveSecretValue(value.Index(index), descriptor.element, path.Index(index), false, input, lookup, result) + } + } +} + +func secretDiagnostic(code Code, path Path, message string, input *document.Node) Diagnostic { + diagnostic := Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message} + if location, ok := nearestDocumentLocation(input, path); ok { + diagnostic.Primary = documentLocationPointer(location) + } + return diagnostic +} + +func changedSecretStates(root reflect.Value, descriptor *typeDescriptor, lookups map[string]secretLookupResult) []Path { + var changed []Path + collectChangedSecretStates(root, descriptor, Path{}, lookups, &changed) + slices.SortFunc(changed, func(left, right Path) int { return cmp.Compare(left.String(), right.String()) }) + return changed +} + +func collectChangedSecretStates(value reflect.Value, descriptor *typeDescriptor, path Path, lookups map[string]secretLookupResult, changed *[]Path) { + if descriptor.kind == descriptorPointer { + if !value.IsNil() { + collectChangedSecretStates(value.Elem(), descriptor.element, path, lookups, changed) + } + return + } + switch descriptor.kind { + case descriptorSecret: + reference := value.Interface().(Secret) + expected, ok := lookups[reference.EnvName()] + if ok && (!expected.present || expected.value == "" || !reference.resolved || reference.value != expected.value) { + *changed = append(*changed, path.clone()) + } + case descriptorStruct: + for _, member := range descriptor.members { + memberPath := path + if !member.inline { + memberPath = path.Key(member.name) + } + collectChangedSecretStates(value.Field(member.index), member.descriptor, memberPath, lookups, changed) + } + case descriptorMap: + if value.IsNil() { + return + } + keys := value.MapKeys() + slices.SortFunc(keys, func(left, right reflect.Value) int { return cmp.Compare(left.String(), right.String()) }) + for _, key := range keys { + collectChangedSecretStates(value.MapIndex(key), descriptor.element, path.Key(key.String()), lookups, changed) + } + case descriptorSlice, descriptorArray: + if descriptor.kind == descriptorSlice && value.IsNil() { + return + } + for index := 0; index < value.Len(); index++ { + collectChangedSecretStates(value.Index(index), descriptor.element, path.Index(index), lookups, changed) + } + } +} + +func cloneSecretReferences(values map[string]string) map[string]string { + if values == nil { + return nil + } + cloned := make(map[string]string, len(values)) + for path, name := range values { + cloned[path] = name + } + return cloned +} diff --git a/internal/engine/secret_reference_test.go b/internal/engine/secret_reference_test.go new file mode 100644 index 0000000..d883306 --- /dev/null +++ b/internal/engine/secret_reference_test.go @@ -0,0 +1,377 @@ +package engine + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "strings" + "testing" +) + +type secretReferenceConfig struct { + Required Secret `yaml:"required"` + Optional *Secret `yaml:"optional"` + Map map[string]Secret `yaml:"map"` + Pointers map[string]*Secret `yaml:"pointers"` + Sequence []Secret `yaml:"sequence"` +} + +type resolutionQueryReader struct { + input *strings.Reader + reads int +} + +func (reader *resolutionQueryReader) Read(data []byte) (int, error) { + reader.reads++ + return reader.input.Read(data) +} + +func TestSecretReferencesResolveTypedShapesOncePerName(t *testing.T) { + input := `required: SHARED_SECRET +optional: OPTIONAL_SECRET +map: + first: SHARED_SECRET +pointers: + nil: null + value: POINTER_SECRET +sequence: [SHARED_SECRET, SEQUENCE_SECRET] +` + values := map[string]string{ + "SHARED_SECRET": "shared-value", + "OPTIONAL_SECRET": "optional-value", + "POINTER_SECRET": "pointer-value", + "SEQUENCE_SECRET": "sequence-value", + } + counts := map[string]int{} + lookup := func(name string) (string, bool) { + counts[name]++ + value, present := values[name] + return value, present + } + validationCalled := false + result, err := loadWithSecretLookup(Bytes("secrets.yml", []byte(input)), lookup, Options[secretReferenceConfig]{ + Definition: Definition[secretReferenceConfig]{Validate: func(configuration *secretReferenceConfig, validation *Validation) { + validationCalled = true + if configuration.Required.Value() != "shared-value" || !configuration.Required.IsResolved() || + configuration.Optional.Value() != "optional-value" || configuration.Pointers["value"].Value() != "pointer-value" || + configuration.Sequence[1].Value() != "sequence-value" { + validation.Error(Keys("required"), "references were not resolved before validation") + } + }}, + }) + if err != nil { + t.Fatalf("loadWithSecretLookup() error = %v", err) + } + if !validationCalled || result.Config.Optional == nil || result.Config.Pointers["nil"] != nil { + t.Fatalf("resolved config = %#v", result.Config) + } + if result.Config.Required.EnvName() != "SHARED_SECRET" || result.Config.Map["first"].Value() != "shared-value" { + t.Fatalf("reference names/values = %#v", result.Config) + } + if !reflect.DeepEqual(counts, map[string]int{"SHARED_SECRET": 1, "OPTIONAL_SECRET": 1, "POINTER_SECRET": 1, "SEQUENCE_SECRET": 1}) { + t.Fatalf("lookup counts = %#v", counts) + } +} + +func TestSecretReferenceErrorsAreSafeAndPathSpecific(t *testing.T) { + for _, test := range []struct { + name string + input string + lookup secretLookup + code Code + }{ + {name: "absent", input: "{}\n", code: CodeSecretRequired}, + {name: "null", input: "required: null\n", code: CodeSecretRequired}, + {name: "empty name", input: "required: ''\n", code: CodeSecretInvalidName}, + {name: "invalid name", input: "required: INVALID-NAME\n", code: CodeSecretInvalidName}, + {name: "unset", input: "required: MISSING_SECRET\n", code: CodeSecretUnset}, + {name: "empty environment", input: "required: EMPTY_SECRET\n", lookup: func(string) (string, bool) { return "", true }, code: CodeSecretEmpty}, + } { + t.Run(test.name, func(t *testing.T) { + lookup := test.lookup + if lookup == nil { + lookup = func(string) (string, bool) { return "", false } + } + _, err := loadWithSecretLookup(Bytes("secret-errors.yml", []byte(test.input)), lookup, Options[secretReferenceConfig]{}) + loadError := requireSecretLoadCode(t, err, test.code) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Path.String() != "/required" || strings.Contains(fmt.Sprintf("%v %#v", err, err), "CREDENTIAL-CANARY") { + t.Fatalf("diagnostic/error = %#v / %v", diagnostic, err) + } + }) + } + + t.Run("non-nil optional is required to resolve", func(t *testing.T) { + type config struct { + Optional *Secret `yaml:"optional"` + } + _, err := loadWithSecretLookup(Bytes("optional.yml", []byte("optional: ''\n")), func(string) (string, bool) { return "", false }, Options[config]{}) + requireSecretLoadCode(t, err, CodeSecretInvalidName) + result, err := loadWithSecretLookup(Bytes("optional.yml", []byte("optional: null\n")), func(string) (string, bool) { return "", false }, Options[config]{}) + if err != nil || result.Config.Optional != nil { + t.Fatalf("nil optional result = %#v, %v", result, err) + } + }) + + t.Run("failed lookup is cached", func(t *testing.T) { + type config struct { + Secrets []Secret `yaml:"secrets"` + } + lookups := 0 + _, err := loadWithSecretLookup(Bytes("missing-twice.yml", []byte("secrets: [MISSING_SHARED, MISSING_SHARED]\n")), func(string) (string, bool) { + lookups++ + return "", false + }, Options[config]{}) + loadError := requireSecretLoadCode(t, err, CodeSecretUnset) + if lookups != 1 || len(loadError.Diagnostics()) != 2 { + t.Fatalf("lookups = %d, diagnostics = %#v", lookups, loadError.Diagnostics()) + } + }) +} + +func TestResolutionSelectsEffectiveSecretNameBeforeLookup(t *testing.T) { + type config struct { + Name string `yaml:"name"` + Secret Secret `yaml:"secret"` + } + lookedUp := []string{} + validated := "" + result, err := loadWithSecretLookup(Bytes("resolved-secret.yml", []byte("name: SELECTED_SECRET\nsecret: OLD_SECRET\n")), func(name string) (string, bool) { + lookedUp = append(lookedUp, name) + return "resolved-value", true + }, Options[config]{Definition: Definition[config]{ + Resolve: func(configuration *config, resolution *Resolution) error { + Normalize(resolution, Keys("secret"), &configuration.Secret, SecretEnv(configuration.Name), "selected named credential", Keys("name")) + return nil + }, + Validate: func(configuration *config, validation *Validation) { + validated = configuration.Secret.Value() + }, + }}) + if err != nil { + t.Fatalf("loadWithSecretLookup() error = %v", err) + } + if !reflect.DeepEqual(lookedUp, []string{"SELECTED_SECRET"}) || validated != "resolved-value" || result.Config.Secret.EnvName() != "SELECTED_SECRET" { + t.Fatalf("lookups = %#v, validated = %q, config = %#v", lookedUp, validated, result.Config) + } + history := result.Metadata().History(Keys("secret")) + if len(history) != 3 || history[len(history)-1].Origin.Operation != ChangeNormalization { + t.Fatalf("secret history = %#v", history) + } +} + +func TestResolutionInputQueriesDoNotReadOrResolveSecrets(t *testing.T) { + t.Parallel() + + type config struct { + Value string `yaml:"value"` + Secret Secret `yaml:"secret"` + } + reader := &resolutionQueryReader{input: strings.NewReader("value: source\nsecret: QUERY_SECRET\n")} + lookups := 0 + var callbackErr error + result, err := loadWithSecretLookup(Reader("query.yml", reader), func(name string) (string, bool) { + lookups++ + if name != "QUERY_SECRET" { + t.Fatalf("secret lookup name = %q", name) + } + return "query-credential", true + }, Options[config]{Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error { + reads := reader.reads + value, valueFound := resolution.InputInfo(Keys("value")) + secret, secretFound := resolution.InputInfo(Keys("secret")) + if !valueFound || !secretFound || len(value.Origin.Locations) != 1 || len(secret.Origin.Locations) != 1 { + callbackErr = errors.New("entry metadata is incomplete") + return callbackErr + } + if source, found := resolution.Source(value.Origin.Locations[0].SourceID); !found || source.Path != "" { + callbackErr = errors.New("entry source is incomplete") + return callbackErr + } + if reader.reads != reads || lookups != 0 { + callbackErr = errors.New("entry query performed input or credential access") + } + return callbackErr + }}}) + if callbackErr != nil { + t.Fatal(callbackErr) + } + if err != nil { + t.Fatal(err) + } + if lookups != 1 || result.Config.Secret.Value() != "query-credential" { + t.Fatalf("secret resolution = lookups %d config %#v", lookups, result.Config) + } +} + +func TestValidationCannotReplaceResolvedSecretWithSameName(t *testing.T) { + type config struct { + Secret Secret `yaml:"secret"` + } + _, err := loadWithSecretLookup(Bytes("validator.yml", []byte("secret: VALIDATOR_SECRET\n")), func(string) (string, bool) { + return "CREDENTIAL-CANARY", true + }, Options[config]{Definition: Definition[config]{Validate: func(configuration *config, validation *Validation) { + configuration.Secret = SecretEnv(configuration.Secret.EnvName()) + }}}) + loadError := requireSecretLoadCode(t, err, CodeValidateMutation) + if loadError.Diagnostics()[0].Path.String() != "/secret" || strings.Contains(fmt.Sprintf("%v %#v", err, err), "CREDENTIAL-CANARY") { + t.Fatalf("validator mutation error = %v %#v", err, err) + } +} + +func TestResolvedDefaultsAreRebuiltAndLaterLoadsRelookup(t *testing.T) { + type config struct { + Secret Secret `yaml:"secret"` + } + first, err := loadWithSecretLookup(NoSource(), func(string) (string, bool) { return "first-value", true }, Options[config]{ + Definition: Definition[config]{Defaults: func() config { return config{Secret: SecretEnv("DEFAULT_SECRET")} }}, + }) + if err != nil { + t.Fatal(err) + } + second, err := loadWithSecretLookup(NoSource(), func(string) (string, bool) { return "second-value", true }, Options[config]{ + Definition: Definition[config]{Defaults: func() config { return first.Config }}, + }) + if err != nil { + t.Fatal(err) + } + if first.Config.Secret.Value() != "first-value" || second.Config.Secret.Value() != "second-value" { + t.Fatalf("resolved defaults retained stale state: %q, %q", first.Config.Secret.Value(), second.Config.Secret.Value()) + } + + t.Setenv("CONFIGKIT_RELOOKUP_SECRET", "one") + options := Options[config]{Definition: Definition[config]{Defaults: func() config { return config{Secret: SecretEnv("CONFIGKIT_RELOOKUP_SECRET")} }}} + realFirst, err := Load(NoSource(), options) + if err != nil { + t.Fatal(err) + } + if err := os.Setenv("CONFIGKIT_RELOOKUP_SECRET", "two"); err != nil { + t.Fatal(err) + } + realSecond, err := Load(NoSource(), options) + if err != nil { + t.Fatal(err) + } + if realFirst.Config.Secret.Value() != "one" || realSecond.Config.Secret.Value() != "two" { + t.Fatalf("later loads = %q, %q", realFirst.Config.Secret.Value(), realSecond.Config.Secret.Value()) + } +} + +func TestSecretSafeFormattingJSONProjectionAndHistory(t *testing.T) { + const credential = "CREDENTIAL-CANARY" + secret := Secret{name: "SAFE_SECRET_NAME", value: credential, specified: true, resolved: true} + for _, format := range []string{"%v", "%+v", "%#v", "%s", "%q", "%x", "%d"} { + formatted := fmt.Sprintf(format, secret) + if strings.Contains(formatted, credential) || !strings.Contains(formatted, "SAFE_SECRET_NAME") { + t.Fatalf("format %q produced %q", format, formatted) + } + } + encoded, err := json.Marshal(secret) + if err != nil || string(encoded) != `"SAFE_SECRET_NAME"` || strings.Contains(string(encoded), credential) { + t.Fatalf("MarshalJSON() = %s, %v", encoded, err) + } + + type config struct { + Secret Secret `yaml:"secret"` + } + result, err := loadWithSecretLookup(Bytes("safe-secret.yml", []byte("secret: SAFE_SECRET_NAME\n")), func(string) (string, bool) { + return credential, true + }, Options[config]{}) + if err != nil { + t.Fatal(err) + } + encodedConfig, err := json.Marshal(result.Config) + if err != nil || strings.Contains(string(encodedConfig), credential) || !strings.Contains(string(encodedConfig), "SAFE_SECRET_NAME") { + t.Fatalf("configuration JSON = %s, %v", encodedConfig, err) + } + var effective strings.Builder + if err := result.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + outputs := []string{effective.String(), fmt.Sprintf("%#v", result.Metadata()), fmt.Sprintf("%#v", result.Metadata().History(Keys("secret"))), fmt.Sprintf("%#v", result.Config)} + for _, output := range outputs { + if strings.Contains(output, credential) { + t.Fatalf("inspection leaked credential: %q", output) + } + } + if !strings.Contains(effective.String(), "SAFE_SECRET_NAME") { + t.Fatalf("effective reference name missing: %q", effective.String()) + } +} + +func TestBootstrapExistingEmptyWinsAndReferenceFails(t *testing.T) { + const name = "CONFIGKIT_BLOCKED_EMPTY_SECRET" + t.Setenv(name, "") + directory := t.TempDir() + writeSecretTestFile(t, directory, name, "directory-value") + report, err := LoadSecretsDir(directory) + if err != nil { + t.Fatal(err) + } + entries := report.Entries() + if len(entries) != 1 || entries[0].Status != SecretFileSkippedExisting || !entries[0].ExistingEmpty { + t.Fatalf("bootstrap report = %#v", entries) + } + type config struct { + Secret Secret `yaml:"secret"` + } + _, err = Load[config](Bytes("blocked.yml", []byte("secret: "+name+"\n"))) + requireSecretLoadCode(t, err, CodeSecretEmpty) + if value, present := os.LookupEnv(name); !present || value != "" { + t.Fatalf("environment changed to %q, %v", value, present) + } +} + +func TestDynamicValuesDoNotEstablishSecretReferences(t *testing.T) { + type dynamic struct { + Secret Secret `yaml:"secret"` + } + type config struct { + Dynamic any `yaml:"dynamic"` + } + schema := mustSchemaFor[config](t) + for _, value := range []any{SecretEnv("DYNAMIC_SECRET"), dynamic{Secret: SecretEnv("DYNAMIC_SECRET")}} { + _, issue := projectConfiguration(schema, reflect.ValueOf(config{Dynamic: value}), defaultProjectionLimits()) + if issue == nil || issue.code != CodeSchemaDynamicType { + t.Fatalf("dynamic Secret projection issue = %#v", issue) + } + } + lookups := 0 + result, err := loadWithSecretLookup(Bytes("dynamic.yml", []byte("dynamic: ORDINARY_STRING\n")), func(string) (string, bool) { + lookups++ + return "unexpected", true + }, Options[config]{}) + if err != nil || result.Config.Dynamic != "ORDINARY_STRING" || lookups != 0 { + t.Fatalf("dynamic string result = %#v, lookups = %d, error = %v", result, lookups, err) + } +} + +func TestLoadDoesNotMutateSecretEnvironment(t *testing.T) { + const name = "CONFIGKIT_UNCHANGED_SECRET" + t.Setenv(name, "original") + type config struct { + Secret Secret `yaml:"secret"` + } + result, err := Load[config](Bytes("unchanged.yml", []byte("secret: "+name+"\n"))) + if err != nil || result.Config.Secret.Value() != "original" { + t.Fatalf("Load() = %#v, %v", result, err) + } + if got := os.Getenv(name); got != "original" { + t.Fatalf("environment = %q", got) + } +} + +func requireSecretLoadCode(t *testing.T, err error, code Code) *LoadError { + t.Helper() + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error = %T %v, want *LoadError", err, err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 0 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v, want first code %q", diagnostics, code) + } + return loadError +} diff --git a/internal/engine/secrets.go b/internal/engine/secrets.go new file mode 100644 index 0000000..effb3ec --- /dev/null +++ b/internal/engine/secrets.go @@ -0,0 +1,371 @@ +package engine + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +const ( + secretsEntryLimit = 1024 + secretFileByteLimit = int64(1 << 20) + secretsByteLimit = int64(8 << 20) +) + +// SecretFileStatus describes the outcome for one file considered by +// LoadSecretsDir. +type SecretFileStatus string + +const ( + SecretFileLoaded SecretFileStatus = "loaded" + SecretFileSkippedExisting SecretFileStatus = "skipped_existing" + SecretFileNotApplied SecretFileStatus = "not_applied" + SecretFileFailed SecretFileStatus = "failed" +) + +// SecretFileResult is the safe, value-free bootstrap result for one directory +// entry. ExistingEmpty is meaningful when Status is SecretFileSkippedExisting. +type SecretFileResult struct { + Name string + Path string + Status SecretFileStatus + ExistingEmpty bool +} + +// SecretsReport is a detached snapshot of an explicit secrets-directory +// bootstrap. It does not retain file contents or environment values. +type SecretsReport struct { + entries []SecretFileResult +} + +// Entries returns a copy of the report's filename-ordered entries. +func (r *SecretsReport) Entries() []SecretFileResult { + if r == nil { + return nil + } + return slices.Clone(r.entries) +} + +// WriteTo renders one safe, deterministic line per entry. Names and paths are +// quoted so control characters cannot affect terminal output. +func (r *SecretsReport) WriteTo(output io.Writer) (int64, error) { + if output == nil { + return 0, errors.New("configkit: secrets report output is nil") + } + if r == nil { + return 0, nil + } + var total int64 + for _, entry := range r.entries { + line := "status=" + strconv.Quote(string(entry.Status)) + + " name=" + strconv.Quote(entry.Name) + + " path=" + strconv.Quote(entry.Path) + + " existing_empty=" + strconv.FormatBool(entry.ExistingEmpty) + "\n" + written, err := io.WriteString(output, line) + total += int64(written) + if err != nil { + return total, err + } + if written != len(line) { + return total, io.ErrShortWrite + } + } + return total, nil +} + +// SecretsError reports safe structured diagnostics for a failed explicit +// secrets-directory bootstrap. It intentionally does not unwrap filesystem or +// environment errors. +type SecretsError struct { + diagnostics []Diagnostic +} + +// Error implements error without rendering filenames, paths, or file contents. +func (e *SecretsError) Error() string { + if e == nil { + return "configkit: secrets bootstrap failed" + } + count := 0 + for _, diagnostic := range e.diagnostics { + if diagnostic.Severity == SeverityError { + count++ + } + } + if count == 1 { + return "configkit: secrets bootstrap failed with 1 error" + } + return fmt.Sprintf("configkit: secrets bootstrap failed with %d errors", count) +} + +// Diagnostics returns a deep copy of the bootstrap diagnostics. +func (e *SecretsError) Diagnostics() []Diagnostic { + if e == nil { + return nil + } + return cloneDiagnostics(e.diagnostics) +} + +type secretsEnvironment struct { + lookup func(string) (string, bool) + set func(string, string) error +} + +type secretsAccess struct { + openDirectory func(string) (*os.File, error) + lstat func(string) (fs.FileInfo, error) + openFile func(string) (*os.File, error) +} + +type stagedSecret struct { + entry int + value string +} + +// LoadSecretsDir explicitly loads immediate regular files from directory into +// previously unset process environment variables. Applications should call it +// once during startup, before concurrent work; Load never calls it. +func LoadSecretsDir(directory string) (*SecretsReport, error) { + return loadSecretsDir(directory, secretsEnvironment{lookup: os.LookupEnv, set: os.Setenv}, secretsAccess{ + openDirectory: os.Open, + lstat: os.Lstat, + openFile: os.Open, + }) +} + +func loadSecretsDir(directory string, environment secretsEnvironment, access secretsAccess) (*SecretsReport, error) { + report := &SecretsReport{} + if strings.TrimSpace(directory) == "" { + return report, secretsFailure(CodeSecretsInvalidPath, "secrets directory path is required") + } + + canonical, err := canonicalExistingPath(directory) + if err != nil { + return report, secretsFailure(classifySecretsFilesystemError(err), "secrets directory could not be canonicalized: "+strconv.Quote(cleanAbsolutePath(directory))) + } + info, err := os.Stat(canonical) + if err != nil { + return report, secretsFailure(classifySecretsFilesystemError(err), "secrets directory could not be inspected: "+strconv.Quote(canonical)) + } + if !info.IsDir() { + return report, secretsFailure(CodeSecretsInvalidPath, "secrets directory is not a directory: "+strconv.Quote(canonical)) + } + + directoryFile, err := access.openDirectory(canonical) + if err != nil { + return report, secretsFailure(classifySecretsFilesystemError(err), "secrets directory could not be opened: "+strconv.Quote(canonical)) + } + entries, readErr := readSecretsEntries(directoryFile) + closeErr := directoryFile.Close() + if readErr != nil { + return report, secretsFailure(classifySecretsFilesystemError(readErr), "secrets directory could not be enumerated: "+strconv.Quote(canonical)) + } + if closeErr != nil { + return report, secretsFailure(CodeSecretsRead, "secrets directory could not be closed: "+strconv.Quote(canonical)) + } + slices.SortFunc(entries, func(left, right fs.DirEntry) int { return strings.Compare(left.Name(), right.Name()) }) + report.entries = make([]SecretFileResult, len(entries)) + for index, entry := range entries { + name := entry.Name() + report.entries[index] = SecretFileResult{Name: name, Path: filepath.Join(canonical, name), Status: SecretFileNotApplied} + } + if len(entries) > secretsEntryLimit { + return report, secretsFailure(CodeSecretsLimitExceeded, "secrets directory exceeds the 1024-entry limit: "+strconv.Quote(canonical)) + } + + for index := range report.entries { + entry := &report.entries[index] + if !validEnvironmentName(entry.Name) { + entry.Status = SecretFileFailed + return report, secretsFailure(CodeSecretsInvalidName, "secret filename is not a valid environment name: "+strconv.Quote(entry.Path)) + } + info, err := access.lstat(entry.Path) + if err != nil { + entry.Status = SecretFileFailed + return report, secretsFailure(classifySecretsFilesystemError(err), "secret file could not be inspected: "+strconv.Quote(entry.Path)) + } + if info.Mode()&fs.ModeSymlink != 0 || !info.Mode().IsRegular() { + entry.Status = SecretFileFailed + return report, secretsFailure(CodeSecretsNotRegular, "secret entry is not a regular file: "+strconv.Quote(entry.Path)) + } + } + + staged := make([]stagedSecret, 0, len(report.entries)) + var consumed int64 + for index := range report.entries { + entry := &report.entries[index] + if existing, present := environment.lookup(entry.Name); present { + entry.Status = SecretFileSkippedExisting + entry.ExistingEmpty = existing == "" + continue + } + remaining := secretsByteLimit - consumed + value, used, err := readSecretValue(entry.Path, remaining, access) + if err != nil { + entry.Status = SecretFileFailed + return report, secretsFailure(secretReadCode(err), secretReadMessage(err)+": "+strconv.Quote(entry.Path)) + } + consumed += used + staged = append(staged, stagedSecret{entry: index, value: value}) + } + + for _, secret := range staged { + entry := &report.entries[secret.entry] + if existing, present := environment.lookup(entry.Name); present { + entry.Status = SecretFileSkippedExisting + entry.ExistingEmpty = existing == "" + continue + } + if err := environment.set(entry.Name, secret.value); err != nil { + entry.Status = SecretFileFailed + return report, secretsFailure(CodeSecretsApplyFailed, "secret environment variable could not be set for file: "+strconv.Quote(entry.Path)) + } + entry.Status = SecretFileLoaded + } + return report, nil +} + +func readSecretsEntries(directory *os.File) ([]fs.DirEntry, error) { + entries := make([]fs.DirEntry, 0) + for len(entries) <= secretsEntryLimit { + batch, err := directory.ReadDir(secretsEntryLimit + 1 - len(entries)) + entries = append(entries, batch...) + if err == io.EOF { + return entries, nil + } + if err != nil { + return entries, err + } + if len(entries) > secretsEntryLimit { + return entries, nil + } + } + return entries, nil +} + +type secretReadFailure int + +const ( + secretReadFilesystem secretReadFailure = iota + secretReadFileLimit + secretReadTotalLimit + secretReadNUL +) + +type secretReadError struct { + kind secretReadFailure + err error +} + +func (e *secretReadError) Error() string { return "secret read failed" } + +func readSecretValue(path string, remaining int64, access secretsAccess) (string, int64, error) { + if remaining < 0 { + return "", 0, &secretReadError{kind: secretReadTotalLimit} + } + limit := min(secretFileByteLimit, remaining) + file, err := access.openFile(path) + if err != nil { + return "", 0, &secretReadError{kind: secretReadFilesystem, err: err} + } + data, readErr := io.ReadAll(io.LimitReader(file, limit+1)) + closeErr := file.Close() + if readErr != nil { + return "", 0, &secretReadError{kind: secretReadFilesystem, err: readErr} + } + if closeErr != nil { + return "", 0, &secretReadError{kind: secretReadFilesystem, err: closeErr} + } + if int64(len(data)) > limit { + kind := secretReadFileLimit + if remaining < secretFileByteLimit { + kind = secretReadTotalLimit + } + return "", 0, &secretReadError{kind: kind} + } + if slices.Contains(data, byte(0)) { + return "", 0, &secretReadError{kind: secretReadNUL} + } + value := string(data) + if strings.HasSuffix(value, "\r\n") { + value = strings.TrimSuffix(value, "\r\n") + } else { + value = strings.TrimSuffix(value, "\n") + } + return value, int64(len(data)), nil +} + +func validEnvironmentName(name string) bool { + if name == "" || !asciiEnvironmentFirst(name[0]) { + return false + } + for index := 1; index < len(name); index++ { + character := name[index] + if !asciiEnvironmentFirst(character) && (character < '0' || character > '9') { + return false + } + } + return true +} + +func asciiEnvironmentFirst(character byte) bool { + return character == '_' || character >= 'A' && character <= 'Z' || character >= 'a' && character <= 'z' +} + +func cleanAbsolutePath(path string) string { + absolute, err := filepath.Abs(path) + if err != nil { + return filepath.Clean(path) + } + return filepath.Clean(absolute) +} + +func classifySecretsFilesystemError(err error) Code { + switch { + case errors.Is(err, fs.ErrNotExist): + return CodeSecretsMissing + case errors.Is(err, fs.ErrPermission): + return CodeSecretsPermission + default: + return CodeSecretsRead + } +} + +func secretReadCode(err error) Code { + var failure *secretReadError + if errors.As(err, &failure) { + switch failure.kind { + case secretReadFileLimit, secretReadTotalLimit: + return CodeSecretsLimitExceeded + case secretReadNUL: + return CodeSecretsInvalidValue + case secretReadFilesystem: + return classifySecretsFilesystemError(failure.err) + } + } + return CodeSecretsRead +} + +func secretReadMessage(err error) string { + var failure *secretReadError + if errors.As(err, &failure) { + switch failure.kind { + case secretReadFileLimit: + return "secret file exceeds the 1 MiB limit" + case secretReadTotalLimit: + return "eligible secret files exceed the 8 MiB total limit" + case secretReadNUL: + return "secret file contains a NUL byte" + } + } + return "secret file could not be read" +} + +func secretsFailure(code Code, message string) *SecretsError { + return &SecretsError{diagnostics: []Diagnostic{{Code: code, Severity: SeverityError, Message: message}}} +} diff --git a/internal/engine/secrets_test.go b/internal/engine/secrets_test.go new file mode 100644 index 0000000..9b9a3b4 --- /dev/null +++ b/internal/engine/secrets_test.go @@ -0,0 +1,377 @@ +package engine + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestLoadSecretsDirLoadsCanonicalSortedValues(t *testing.T) { + directory := t.TempDir() + writeSecretTestFile(t, directory, "CONFIGKIT_SECRET_Z", "last\n\n") + writeSecretTestFile(t, directory, "CONFIGKIT_SECRET_A", "first\r\n") + writeSecretTestFile(t, directory, "CONFIGKIT_SECRET_EMPTY", "") + + selected := directory + if runtime.GOOS != "windows" { + selected = filepath.Join(t.TempDir(), "selected") + if err := os.Symlink(directory, selected); err != nil { + t.Fatalf("Symlink() error = %v", err) + } + } + for _, name := range []string{"CONFIGKIT_SECRET_A", "CONFIGKIT_SECRET_EMPTY", "CONFIGKIT_SECRET_Z"} { + t.Setenv(name, "temporary") + if err := os.Unsetenv(name); err != nil { + t.Fatalf("Unsetenv(%q) error = %v", name, err) + } + } + + report, err := LoadSecretsDir(selected) + if err != nil { + t.Fatalf("LoadSecretsDir() error = %v", err) + } + if got := os.Getenv("CONFIGKIT_SECRET_A"); got != "first" { + t.Fatalf("LF value = %q, want first", got) + } + if got := os.Getenv("CONFIGKIT_SECRET_Z"); got != "last\n" { + t.Fatalf("two-LF value = %q, want one retained LF", got) + } + if value, present := os.LookupEnv("CONFIGKIT_SECRET_EMPTY"); !present || value != "" { + t.Fatalf("empty value = %q, %v; want present empty", value, present) + } + + canonical := canonicalTestPath(t, directory) + want := []SecretFileResult{ + {Name: "CONFIGKIT_SECRET_A", Path: filepath.Join(canonical, "CONFIGKIT_SECRET_A"), Status: SecretFileLoaded}, + {Name: "CONFIGKIT_SECRET_EMPTY", Path: filepath.Join(canonical, "CONFIGKIT_SECRET_EMPTY"), Status: SecretFileLoaded}, + {Name: "CONFIGKIT_SECRET_Z", Path: filepath.Join(canonical, "CONFIGKIT_SECRET_Z"), Status: SecretFileLoaded}, + } + if got := report.Entries(); !reflect.DeepEqual(got, want) { + t.Fatalf("Entries() = %#v, want %#v", got, want) + } + entries := report.Entries() + entries[0].Name = "mutated" + if report.Entries()[0].Name != want[0].Name { + t.Fatal("Entries() did not return a detached slice") + } +} + +func TestLoadSecretsDirPreservesExistingWithoutReading(t *testing.T) { + directory := t.TempDir() + writeSecretTestFile(t, directory, "EMPTY_WINNER", strings.Repeat("CANARY", int(secretFileByteLimit))) + writeSecretTestFile(t, directory, "VALUE_WINNER", "CANARY-CONTENT") + environment := map[string]string{"EMPTY_WINNER": "", "VALUE_WINNER": "existing"} + opened := 0 + access := testSecretsAccess() + access.openFile = func(string) (*os.File, error) { + opened++ + return nil, errors.New("must not open") + } + + report, err := loadSecretsDir(directory, mapSecretsEnvironment(environment), access) + if err != nil { + t.Fatalf("loadSecretsDir() error = %v", err) + } + if opened != 0 { + t.Fatalf("opened skipped files %d times", opened) + } + want := []SecretFileResult{ + {Name: "EMPTY_WINNER", Path: filepath.Join(canonicalTestPath(t, directory), "EMPTY_WINNER"), Status: SecretFileSkippedExisting, ExistingEmpty: true}, + {Name: "VALUE_WINNER", Path: filepath.Join(canonicalTestPath(t, directory), "VALUE_WINNER"), Status: SecretFileSkippedExisting}, + } + if got := report.Entries(); !reflect.DeepEqual(got, want) { + t.Fatalf("Entries() = %#v, want %#v", got, want) + } +} + +func TestLoadSecretsDirValidatesAllMetadataBeforeReadingOrWriting(t *testing.T) { + for _, test := range []struct { + name string + make func(*testing.T, string) + code Code + }{ + {name: "invalid name", make: func(t *testing.T, directory string) { writeSecretTestFile(t, directory, "1INVALID", "CANARY") }, code: CodeSecretsInvalidName}, + {name: "directory", make: func(t *testing.T, directory string) { + if err := os.Mkdir(filepath.Join(directory, "BROKEN"), 0o700); err != nil { + t.Fatal(err) + } + }, code: CodeSecretsNotRegular}, + {name: "symlink", make: func(t *testing.T, directory string) { + if runtime.GOOS == "windows" { + t.Skip("symlink setup differs on Windows") + } + writeSecretTestFile(t, directory, "TARGET", "CANARY") + if err := os.Symlink(filepath.Join(directory, "TARGET"), filepath.Join(directory, "BROKEN")); err != nil { + t.Fatal(err) + } + }, code: CodeSecretsNotRegular}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + writeSecretTestFile(t, directory, "A_VALID", "first") + test.make(t, directory) + writes := 0 + access := testSecretsAccess() + opened := 0 + access.openFile = func(path string) (*os.File, error) { opened++; return os.Open(path) } + report, err := loadSecretsDir(directory, secretsEnvironment{ + lookup: func(string) (string, bool) { return "", false }, + set: func(string, string) error { writes++; return nil }, + }, access) + requireSecretsCode(t, err, test.code) + if writes != 0 || opened != 0 { + t.Fatalf("writes = %d, opens = %d; want no preflight activity", writes, opened) + } + if report == nil || len(report.Entries()) < 2 { + t.Fatalf("report = %#v, want enumerated entries", report) + } + }) + } +} + +func TestLoadSecretsDirPreflightFailuresDoNotWrite(t *testing.T) { + for _, test := range []struct { + name string + body string + code Code + }{ + {name: "file limit", body: strings.Repeat("x", int(secretFileByteLimit)+1), code: CodeSecretsLimitExceeded}, + {name: "NUL", body: "before\x00CANARY", code: CodeSecretsInvalidValue}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + writeSecretTestFile(t, directory, "A_FIRST", "first") + writeSecretTestFile(t, directory, "Z_FAILED", test.body) + writes := 0 + report, err := loadSecretsDir(directory, secretsEnvironment{ + lookup: func(string) (string, bool) { return "", false }, + set: func(string, string) error { writes++; return nil }, + }, testSecretsAccess()) + requireSecretsCode(t, err, test.code) + if writes != 0 { + t.Fatalf("writes = %d, want zero", writes) + } + entries := report.Entries() + if entries[0].Status != SecretFileNotApplied || entries[1].Status != SecretFileFailed { + t.Fatalf("statuses = %#v", entries) + } + }) + } +} + +func TestLoadSecretsDirClassifiesReadFailuresWithoutLeakingThem(t *testing.T) { + for _, test := range []struct { + name string + err error + code Code + }{ + {name: "permission", err: fs.ErrPermission, code: CodeSecretsPermission}, + {name: "missing", err: fs.ErrNotExist, code: CodeSecretsMissing}, + {name: "other", err: errors.New("READ-ERROR-CANARY"), code: CodeSecretsRead}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + writeSecretTestFile(t, directory, "SECRET", "CREDENTIAL-CANARY") + access := testSecretsAccess() + access.openFile = func(string) (*os.File, error) { return nil, test.err } + writes := 0 + report, err := loadSecretsDir(directory, secretsEnvironment{ + lookup: func(string) (string, bool) { return "", false }, + set: func(string, string) error { writes++; return nil }, + }, access) + requireSecretsCode(t, err, test.code) + if writes != 0 || report.Entries()[0].Status != SecretFileFailed { + t.Fatalf("writes = %d, entries = %#v", writes, report.Entries()) + } + if strings.Contains(fmt.Sprintf("%v %#v", err, err), "CANARY") { + t.Fatalf("error leaked underlying failure: %v %#v", err, err) + } + }) + } +} + +func TestLoadSecretsDirEnforcesEnumerationAndTotalBounds(t *testing.T) { + t.Run("entries", func(t *testing.T) { + directory := t.TempDir() + for index := 0; index <= secretsEntryLimit; index++ { + writeSecretTestFile(t, directory, fmt.Sprintf("SECRET_%04d", index), "") + } + report, err := loadSecretsDir(directory, mapSecretsEnvironment(map[string]string{}), testSecretsAccess()) + requireSecretsCode(t, err, CodeSecretsLimitExceeded) + if len(report.Entries()) != secretsEntryLimit+1 { + t.Fatalf("entries = %d, want %d bounded enumerated entries", len(report.Entries()), secretsEntryLimit+1) + } + }) + + t.Run("total bytes", func(t *testing.T) { + directory := t.TempDir() + for index := 0; index < 9; index++ { + writeSecretTestFile(t, directory, fmt.Sprintf("SECRET_%02d", index), strings.Repeat("x", int(secretFileByteLimit))) + } + writes := 0 + report, err := loadSecretsDir(directory, secretsEnvironment{ + lookup: func(string) (string, bool) { return "", false }, + set: func(string, string) error { writes++; return nil }, + }, testSecretsAccess()) + requireSecretsCode(t, err, CodeSecretsLimitExceeded) + if writes != 0 || report.Entries()[8].Status != SecretFileFailed { + t.Fatalf("writes = %d, final status = %q", writes, report.Entries()[8].Status) + } + }) +} + +func TestLoadSecretsDirApplyReportsPartialFailureAndPresenceRace(t *testing.T) { + directory := t.TempDir() + for _, name := range []string{"A_LOADED", "B_RACED", "C_FAILED", "D_WAITING"} { + writeSecretTestFile(t, directory, name, "CANARY-"+name) + } + values := map[string]string{} + lookups := map[string]int{} + environment := secretsEnvironment{ + lookup: func(name string) (string, bool) { + lookups[name]++ + if name == "B_RACED" && lookups[name] == 2 { + values[name] = "" + } + value, present := values[name] + return value, present + }, + set: func(name, value string) error { + if name == "C_FAILED" { + return errors.New("CANARY set error") + } + values[name] = value + return nil + }, + } + + report, err := loadSecretsDir(directory, environment, testSecretsAccess()) + requireSecretsCode(t, err, CodeSecretsApplyFailed) + entries := report.Entries() + wantStatuses := []SecretFileStatus{SecretFileLoaded, SecretFileSkippedExisting, SecretFileFailed, SecretFileNotApplied} + for index, want := range wantStatuses { + if entries[index].Status != want { + t.Fatalf("entry %d status = %q, want %q", index, entries[index].Status, want) + } + } + if !entries[1].ExistingEmpty || values["A_LOADED"] != "CANARY-A_LOADED" { + t.Fatalf("partial report/state = %#v, %#v", entries, values) + } + if _, present := values["D_WAITING"]; present { + t.Fatal("entry after apply failure was written") + } +} + +func TestLoadSecretsDirFailuresAreSafeAndDetached(t *testing.T) { + directory := t.TempDir() + writeSecretTestFile(t, directory, "SECRET", "CREDENTIAL-CANARY\x00") + report, err := loadSecretsDir(directory, mapSecretsEnvironment(map[string]string{}), testSecretsAccess()) + secretsErr := requireSecretsCode(t, err, CodeSecretsInvalidValue) + diagnostics := secretsErr.Diagnostics() + diagnostics[0].Message = "mutated" + if secretsErr.Diagnostics()[0].Message == "mutated" { + t.Fatal("Diagnostics() did not return a detached slice") + } + var rendered strings.Builder + if _, writeErr := report.WriteTo(&rendered); writeErr != nil { + t.Fatal(writeErr) + } + jsonReport, marshalErr := json.Marshal(report) + if marshalErr != nil { + t.Fatal(marshalErr) + } + outputs := []string{err.Error(), fmt.Sprintf("%v %#v", err, err), fmt.Sprintf("%v %#v", report, report), rendered.String(), string(jsonReport)} + for _, output := range outputs { + if strings.Contains(output, "CREDENTIAL-CANARY") { + t.Fatalf("safe output leaked content: %q", output) + } + } + if secretsErr.Diagnostics()[0].Primary != nil || secretsErr.Diagnostics()[0].Path.String() != "" { + t.Fatalf("bootstrap diagnostic fabricated source location/path: %#v", secretsErr.Diagnostics()[0]) + } +} + +func TestSecretsReportWriteToPropagatesWriterFailures(t *testing.T) { + report := &SecretsReport{entries: []SecretFileResult{{Name: "A\nB", Path: "/tmp/A\nB", Status: SecretFileLoaded}}} + var output strings.Builder + written, err := report.WriteTo(&output) + if err != nil || written != int64(output.Len()) || strings.Contains(output.String(), "A\nB") || !strings.Contains(output.String(), `A\nB`) { + t.Fatalf("WriteTo() = (%d, %v), output %q", written, err, output.String()) + } + failure := errors.New("writer CANARY") + written, err = report.WriteTo(errorAfterWriter{limit: 5, err: failure}) + if written != 5 || !errors.Is(err, failure) { + t.Fatalf("failing WriteTo() = (%d, %v)", written, err) + } + written, err = report.WriteTo(shortWriter{}) + if written == 0 || !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("short WriteTo() = (%d, %v)", written, err) + } +} + +func TestLoadSecretsDirRejectsMissingAndInvalidDirectory(t *testing.T) { + report, err := LoadSecretsDir("") + requireSecretsCode(t, err, CodeSecretsInvalidPath) + if report == nil || len(report.Entries()) != 0 { + t.Fatalf("empty path report = %#v", report) + } + report, err = LoadSecretsDir(filepath.Join(t.TempDir(), "missing")) + requireSecretsCode(t, err, CodeSecretsMissing) + if report == nil || len(report.Entries()) != 0 { + t.Fatalf("missing path report = %#v", report) + } + file := filepath.Join(t.TempDir(), "FILE") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + _, err = LoadSecretsDir(file) + requireSecretsCode(t, err, CodeSecretsInvalidPath) +} + +func testSecretsAccess() secretsAccess { + return secretsAccess{openDirectory: os.Open, lstat: os.Lstat, openFile: os.Open} +} + +func mapSecretsEnvironment(values map[string]string) secretsEnvironment { + return secretsEnvironment{ + lookup: func(name string) (string, bool) { value, present := values[name]; return value, present }, + set: func(name, value string) error { values[name] = value; return nil }, + } +} + +func writeSecretTestFile(t *testing.T, directory, name, value string) { + t.Helper() + if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil { + t.Fatalf("WriteFile(%q) error = %v", name, err) + } +} + +func requireSecretsCode(t *testing.T, err error, code Code) *SecretsError { + t.Helper() + var secretsErr *SecretsError + if !errors.As(err, &secretsErr) { + t.Fatalf("error = %T %v, want *SecretsError", err, err) + } + diagnostics := secretsErr.Diagnostics() + if len(diagnostics) != 1 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v, want code %q", diagnostics, code) + } + return secretsErr +} + +type errorAfterWriter struct { + limit int + err error +} + +func (w errorAfterWriter) Write(data []byte) (int, error) { + return min(w.limit, len(data)), w.err +} diff --git a/internal/engine/snapshot.go b/internal/engine/snapshot.go new file mode 100644 index 0000000..7a97c9f --- /dev/null +++ b/internal/engine/snapshot.go @@ -0,0 +1,63 @@ +package engine + +import ( + "bytes" + "slices" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +// Snapshot projection is the shared redaction boundary for effective YAML +// and explanation values. It borrows frozen trees and returns a detached tree. +func valueSnapshot(root *document.Node, path Path, secrets map[string]bool) ValueSnapshot { + node, present := documentAtPath(root, path) + if !present { + return ValueSnapshot{} + } + safe, redacted := safeSnapshotDocument(node, secrets, path) + var output bytes.Buffer + if err := yamladapter.Write(&output, safe); err != nil { + return ValueSnapshot{Present: true, Redacted: true, YAML: "'[REDACTED]'\n"} + } + return ValueSnapshot{Present: true, Redacted: redacted, YAML: output.String()} +} + +func safeSnapshotDocument(node *document.Node, secrets map[string]bool, path Path) (*document.Node, bool) { + if node == nil { + return nil, false + } + if pathIsSecret(secrets, path) { + return &document.Node{Kind: document.String, Text: "[REDACTED]"}, true + } + result := *node + redacted := false + switch node.Kind { + case document.Mapping: + result.Mapping = make([]document.Entry, len(node.Mapping)) + copy(result.Mapping, node.Mapping) + sortDocumentMapping(result.Mapping) + for index := range result.Mapping { + entry := &result.Mapping[index] + entry.Value, redacted = safeSnapshotChild(entry.Value, secrets, path.Key(entry.Key), redacted) + } + case document.Sequence: + result.Sequence = make([]*document.Node, len(node.Sequence)) + for index, child := range node.Sequence { + result.Sequence[index], redacted = safeSnapshotChild(child, secrets, path.Index(index), redacted) + } + } + return &result, redacted +} + +func safeSnapshotChild(node *document.Node, secrets map[string]bool, path Path, prior bool) (*document.Node, bool) { + safe, redacted := safeSnapshotDocument(node, secrets, path) + return safe, prior || redacted +} + +func sortDocumentMapping(entries []document.Entry) { + slices.SortFunc(entries, func(left, right document.Entry) int { + return strings.Compare(left.Key, right.Key) + }) +} diff --git a/internal/engine/source.go b/internal/engine/source.go new file mode 100644 index 0000000..82a8008 --- /dev/null +++ b/internal/engine/source.go @@ -0,0 +1,216 @@ +package engine + +import ( + "bytes" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "reflect" + "sync/atomic" +) + +const ( + maxSourceBytes = 8 << 20 + maxTotalBytes = 64 << 20 +) + +// Source is an opaque configuration input. Construct sources with File, Bytes, +// Reader, or NoSource; the zero value is invalid. +type Source struct { + kind sourceKind + path string + label string + data []byte + reader *readerSource + createErr sourceCreationError +} + +type sourceKind uint8 + +const ( + sourceInvalid sourceKind = iota + sourceFile + sourceBytes + sourceReader + sourceNone +) + +type sourceCreationError uint8 + +const ( + creationValid sourceCreationError = iota + creationInvalidPath + creationInvalidLabel + creationInvalidReader + creationTooLarge +) + +type readerSource struct { + input io.Reader + used atomic.Bool +} + +// File returns a source that reopens path for every load attempt. +func File(path string) Source { + if path == "" { + return Source{kind: sourceFile, createErr: creationInvalidPath} + } + return Source{kind: sourceFile, path: path, label: path} +} + +// Bytes snapshots data immediately. Label must be nonempty and safe to display; +// constructor problems are reported when the source is consumed. +func Bytes(label string, data []byte) Source { + source := Source{kind: sourceBytes, label: label} + if label == "" { + source.createErr = creationInvalidLabel + return source + } + if len(data) > maxSourceBytes { + source.createErr = creationTooLarge + return source + } + source.data = bytes.Clone(data) + return source +} + +// Reader returns a single-use source. ConfigKit consumes input but never closes it. +// Copies of the returned Source share the same single-use claim. +func Reader(label string, input io.Reader) Source { + source := Source{kind: sourceReader, label: label} + if label == "" { + source.createErr = creationInvalidLabel + return source + } + if nilReader(input) { + source.createErr = creationInvalidReader + return source + } + source.reader = &readerSource{input: input} + return source +} + +// NoSource explicitly selects a defaults-only input. It creates no source record. +func NoSource() Source { + return Source{kind: sourceNone} +} + +type sourceLimits struct { + perSource int64 + remaining int64 +} + +func defaultSourceLimits() sourceLimits { + return sourceLimits{perSource: maxSourceBytes, remaining: maxTotalBytes} +} + +type sourceRead struct { + data []byte + info *SourceInfo +} + +func readSource(source Source, sourceID string, limits sourceLimits) (sourceRead, []Diagnostic) { + if diagnostic := sourceConstructorDiagnostic(source); diagnostic != nil { + return sourceRead{}, []Diagnostic{*diagnostic} + } + if source.kind == sourceNone { + return sourceRead{}, nil + } + + limit := min(limits.perSource, limits.remaining) + if limit < 0 { + limit = 0 + } + info := &SourceInfo{ID: sourceID, Label: source.label, Layer: LayerBase} + + var input io.Reader + switch source.kind { + case sourceFile: + absolute, err := filepath.Abs(source.path) + if err != nil { + return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceRead, sourceID, "source path could not be resolved")} + } + info.Path = filepath.Clean(absolute) + canonical, err := filepath.EvalSymlinks(info.Path) + if err != nil { + return sourceRead{info: info}, []Diagnostic{classifySourceError(err, sourceID)} + } + info.Path = canonical + data, failure := readRegularFile(canonical, os.Stat, os.Open, limit) + if failure != nil { + return sourceRead{info: info}, []Diagnostic{sourceReadDiagnostic(failure, sourceID, CodeSourceLimitExceeded, "source exceeds the configured byte limit")} + } + return sourceRead{data: bytes.Clone(data), info: info}, nil + case sourceBytes: + input = bytes.NewReader(source.data) + case sourceReader: + if !source.reader.used.CompareAndSwap(false, true) { + return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceReaderUsed, sourceID, "reader source has already been consumed")} + } + input = source.reader.input + default: + return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceInvalid, sourceID, "source is invalid")} + } + data, failure := readBoundedSource(input, limit) + if failure != nil { + return sourceRead{info: info}, []Diagnostic{sourceReadDiagnostic(failure, sourceID, CodeSourceLimitExceeded, "source exceeds the configured byte limit")} + } + return sourceRead{data: bytes.Clone(data), info: info}, nil +} + +func sourceConstructorDiagnostic(source Source) *Diagnostic { + var message string + code := CodeSourceInvalid + switch source.createErr { + case creationValid: + if source.kind != sourceInvalid { + return nil + } + message = "source is invalid" + case creationInvalidPath: + message = "source path must not be empty" + case creationInvalidLabel: + message = "source label must not be empty" + case creationInvalidReader: + message = "source reader must not be nil" + case creationTooLarge: + code = CodeSourceLimitExceeded + message = "source exceeds the configured byte limit" + } + diagnostic := sourceDiagnostic(code, "", message) + return &diagnostic +} + +func classifySourceError(err error, sourceID string) Diagnostic { + switch { + case errors.Is(err, fs.ErrNotExist): + return sourceDiagnostic(CodeSourceMissing, sourceID, "source does not exist") + case errors.Is(err, fs.ErrPermission): + return sourceDiagnostic(CodeSourcePermission, sourceID, "source cannot be accessed") + default: + return sourceDiagnostic(CodeSourceRead, sourceID, "source could not be read") + } +} + +func sourceDiagnostic(code Code, sourceID, message string) Diagnostic { + diagnostic := Diagnostic{Code: code, Severity: SeverityError, Message: message} + if sourceID != "" { + diagnostic.Primary = &Location{SourceID: sourceID} + } + return diagnostic +} + +func nilReader(input io.Reader) bool { + if input == nil { + return true + } + value := reflect.ValueOf(input) + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return value.IsNil() + default: + return false + } +} diff --git a/internal/engine/source_fifo_test.go b/internal/engine/source_fifo_test.go new file mode 100644 index 0000000..2f698f6 --- /dev/null +++ b/internal/engine/source_fifo_test.go @@ -0,0 +1,71 @@ +//go:build darwin || linux + +package engine + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestNonregularSourcesDoNotBlock(t *testing.T) { + if mode := os.Getenv("CONFIGKIT_FIFO_TEST_MODE"); mode != "" { + root := os.Getenv("CONFIGKIT_FIFO_TEST_ROOT") + source := File(filepath.Join(root, "config.yml")) + opts := Options[map[string]any]{} + switch mode { + case "file": + source = File(filepath.Join(root, "pipe.yml")) + case "import": + opts.Imports = true + case "profile": + opts.Profile = "fifo" + } + _, err := Load(source, opts) + var loadErr *LoadError + if !errors.As(err, &loadErr) { + t.Fatalf("expected safe load error, got %v", err) + } + for _, d := range loadErr.Diagnostics() { + if d.Code == CodeSourceNotRegular { + return + } + } + t.Fatalf("expected nonregular classification, got %v", loadErr.Diagnostics()) + } + for _, mode := range []string{"file", "import", "profile"} { + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + input := "value: 1\n" + if mode == "import" { + input = "configkit:\n imports:\n - path: pipe.yml\n" + } + if err := os.WriteFile(filepath.Join(root, "config.yml"), []byte(input), 0600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "profiles"), 0700); err != nil { + t.Fatal(err) + } + fifo := filepath.Join(root, "pipe.yml") + if mode == "profile" { + fifo = filepath.Join(root, "profiles", "fifo.yml") + } + if err := syscall.Mkfifo(fifo, 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestNonregularSourcesDoNotBlock$") + cmd.Env = append(os.Environ(), "CONFIGKIT_FIFO_TEST_MODE="+mode, "CONFIGKIT_FIFO_TEST_ROOT="+root) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("FIFO load failed or blocked: %v, %s", err, out) + } + }) + } +} diff --git a/internal/engine/source_file.go b/internal/engine/source_file.go new file mode 100644 index 0000000..ab15bba --- /dev/null +++ b/internal/engine/source_file.go @@ -0,0 +1,75 @@ +package engine + +import ( + "errors" + "io" + "io/fs" + "os" +) + +type sourceReadFailureKind uint8 + +const ( + sourceReadIO sourceReadFailureKind = iota + sourceReadNotRegular + sourceReadLimit +) + +// sourceReadFailure never escapes the private source boundary. missingTarget is +// set only for preflight/open failures; a later missing-file error is not optional. +type sourceReadFailure struct { + kind sourceReadFailureKind + cause error + missingTarget bool +} + +// readRegularFile borrows stat/open operations so root-bound callers retain their +// containment policy. Preflight rejects FIFOs before open can block. The opened +// handle is checked again and always closed; read-only close errors are ignored. +func readRegularFile(name string, stat func(string) (fs.FileInfo, error), open func(string) (*os.File, error), limit int64) ([]byte, *sourceReadFailure) { + info, err := stat(name) + if err != nil { + return nil, &sourceReadFailure{cause: err, missingTarget: errors.Is(err, fs.ErrNotExist)} + } + if !info.Mode().IsRegular() { + return nil, &sourceReadFailure{kind: sourceReadNotRegular} + } + file, err := open(name) + if err != nil { + return nil, &sourceReadFailure{cause: err, missingTarget: errors.Is(err, fs.ErrNotExist)} + } + defer func() { _ = file.Close() }() + info, err = file.Stat() + if err != nil { + return nil, &sourceReadFailure{cause: err} + } + if !info.Mode().IsRegular() { + return nil, &sourceReadFailure{kind: sourceReadNotRegular} + } + if info.Size() > limit { + return nil, &sourceReadFailure{kind: sourceReadLimit} + } + return readBoundedSource(file, limit) +} + +func readBoundedSource(input io.Reader, limit int64) ([]byte, *sourceReadFailure) { + data, err := io.ReadAll(io.LimitReader(input, limit+1)) + if err != nil { + return nil, &sourceReadFailure{cause: err} + } + if int64(len(data)) > limit { + return nil, &sourceReadFailure{kind: sourceReadLimit} + } + return data, nil +} + +func sourceReadDiagnostic(failure *sourceReadFailure, sourceID string, limitCode Code, limitMessage string) Diagnostic { + switch failure.kind { + case sourceReadNotRegular: + return sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file") + case sourceReadLimit: + return sourceDiagnostic(limitCode, sourceID, limitMessage) + default: + return classifySourceError(failure.cause, sourceID) + } +} diff --git a/internal/engine/source_file_test.go b/internal/engine/source_file_test.go new file mode 100644 index 0000000..cdcba40 --- /dev/null +++ b/internal/engine/source_file_test.go @@ -0,0 +1,48 @@ +package engine + +import ( + "io/fs" + "os" + "path/filepath" + "testing" +) + +// Files disappearing between filesystem operations are not deterministic with +// real concurrent deletion. Exercise the private seam that controls optionality. +func TestRegularFileMissingFailurePhases(t *testing.T) { + name := filepath.Join(t.TempDir(), "input.yml") + if err := os.WriteFile(name, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + stat func(string) (fs.FileInfo, error) + open func(string) (*os.File, error) + missing bool + }{ + {"preflight", func(string) (fs.FileInfo, error) { return nil, fs.ErrNotExist }, os.Open, true}, + {"open", os.Stat, func(string) (*os.File, error) { return nil, fs.ErrNotExist }, true}, + {"post-open stat", os.Stat, func(name string) (*os.File, error) { + file, err := os.Open(name) + if err == nil { + err = file.Close() + } + return file, err + }, false}, + } { + t.Run(test.name, func(t *testing.T) { + data, failure := readRegularFile(name, test.stat, test.open, 16) + if data != nil || failure == nil || failure.kind != sourceReadIO || failure.missingTarget != test.missing { + t.Fatalf("data = %v, failure = %#v", data, failure) + } + }) + } + _, failure := readBoundedSource(missingDuringRead{}, 16) + if failure == nil || failure.kind != sourceReadIO || failure.missingTarget { + t.Fatalf("late read failure = %#v", failure) + } +} + +type missingDuringRead struct{} + +func (missingDuringRead) Read([]byte) (int, error) { return 0, fs.ErrNotExist } diff --git a/internal/engine/source_test.go b/internal/engine/source_test.go new file mode 100644 index 0000000..9e0d440 --- /dev/null +++ b/internal/engine/source_test.go @@ -0,0 +1,202 @@ +package engine + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestSourceConstructorsAndOwnership(t *testing.T) { + t.Parallel() + + data := []byte("value: before\n") + source := Bytes("memory", data) + copy(data, "value: after!\n") + read, diagnostics := readSource(source, "source-1", defaultSourceLimits()) + if len(diagnostics) != 0 || string(read.data) != "value: before\n" { + t.Fatalf("byte snapshot = %q, diagnostics = %#v", read.data, diagnostics) + } + read.data[0] = 'X' + again, diagnostics := readSource(source, "source-1", defaultSourceLimits()) + if len(diagnostics) != 0 || string(again.data) != "value: before\n" { + t.Fatal("returned bytes alias source snapshot") + } + + closer := &observedReadCloser{Reader: strings.NewReader("value: reader\n")} + readerSource := Reader("stream", closer) + read, diagnostics = readSource(readerSource, "source-1", defaultSourceLimits()) + if len(diagnostics) != 0 || string(read.data) != "value: reader\n" || closer.closed { + t.Fatalf("reader result = %q, closed=%v diagnostics=%#v", read.data, closer.closed, diagnostics) + } + _, diagnostics = readSource(readerSource, "source-1", defaultSourceLimits()) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceReaderUsed { + t.Fatalf("second reader diagnostics = %#v", diagnostics) + } + + if read, diagnostics := readSource(NoSource(), "source-1", defaultSourceLimits()); len(diagnostics) != 0 || read.info != nil || read.data != nil { + t.Fatalf("NoSource read = %#v, %#v", read, diagnostics) + } +} + +func TestFileSourcesReopenAndClassifyFailures(t *testing.T) { + t.Parallel() + + directory := t.TempDir() + path := filepath.Join(directory, "config.yml") + if err := os.WriteFile(path, []byte("value: first\n"), 0o600); err != nil { + t.Fatal(err) + } + source := File(path) + first, diagnostics := readSource(source, "source-1", defaultSourceLimits()) + if len(diagnostics) != 0 || string(first.data) != "value: first\n" || first.info.Path != canonicalTestPath(t, path) || first.info.Label != path { + t.Fatalf("first read = %#v, %#v", first, diagnostics) + } + if err := os.WriteFile(path, []byte("value: second\n"), 0o600); err != nil { + t.Fatal(err) + } + second, diagnostics := readSource(source, "source-1", defaultSourceLimits()) + if len(diagnostics) != 0 || string(second.data) != "value: second\n" { + t.Fatalf("second read = %#v, %#v", second, diagnostics) + } + + missing, diagnostics := readSource(File(filepath.Join(directory, "missing.yml")), "source-2", defaultSourceLimits()) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceMissing || missing.info == nil { + t.Fatalf("missing diagnostics = %#v, info=%#v", diagnostics, missing.info) + } + _, diagnostics = readSource(File(directory), "source-3", defaultSourceLimits()) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceNotRegular { + t.Fatalf("directory diagnostics = %#v", diagnostics) + } + + permissionPath := filepath.Join(directory, "private.yml") + if err := os.WriteFile(permissionPath, []byte("value: private\n"), 0); err != nil { + t.Fatal(err) + } + _, diagnostics = readSource(File(permissionPath), "source-4", defaultSourceLimits()) + if os.Geteuid() != 0 { + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourcePermission { + t.Fatalf("permission diagnostics = %#v", diagnostics) + } + } +} + +func TestSourceReadLimitsAndErrors(t *testing.T) { + t.Parallel() + + limits := sourceLimits{perSource: 5, remaining: 20} + read, diagnostics := readSource(Reader("exact", &fragmentReader{data: []byte("12345"), width: 1}), "source-1", limits) + if len(diagnostics) != 0 || string(read.data) != "12345" { + t.Fatalf("exact limit read = %q, %#v", read.data, diagnostics) + } + _, diagnostics = readSource(Reader("over", strings.NewReader("123456")), "source-2", limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceLimitExceeded { + t.Fatalf("over-limit diagnostics = %#v", diagnostics) + } + _, diagnostics = readSource(Reader("total", strings.NewReader("1234")), "source-3", sourceLimits{perSource: 10, remaining: 3}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceLimitExceeded { + t.Fatalf("total-limit diagnostics = %#v", diagnostics) + } + want := errors.New("read canary: must not escape") + _, diagnostics = readSource(Reader("failing", &dataErrorReader{data: []byte("partial"), err: want}), "source-4", limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceRead || strings.Contains(diagnostics[0].Message, "canary") { + t.Fatalf("read-error diagnostics = %#v", diagnostics) + } +} + +func TestReaderSourceCopiesCompeteForOneClaim(t *testing.T) { + t.Parallel() + + source := Reader("shared", strings.NewReader("value: one\n")) + copyOfSource := source + start := make(chan struct{}) + results := make(chan []Diagnostic, 2) + var group sync.WaitGroup + for _, candidate := range []Source{source, copyOfSource} { + group.Add(1) + go func(candidate Source) { + defer group.Done() + <-start + _, diagnostics := readSource(candidate, "source-1", defaultSourceLimits()) + results <- diagnostics + }(candidate) + } + close(start) + group.Wait() + close(results) + + successes, used := 0, 0 + for diagnostics := range results { + if len(diagnostics) == 0 { + successes++ + } else if len(diagnostics) == 1 && diagnostics[0].Code == CodeSourceReaderUsed { + used++ + } + } + if successes != 1 || used != 1 { + t.Fatalf("successes=%d reader-used=%d", successes, used) + } +} + +func TestInvalidSourcesFailWithoutConsumption(t *testing.T) { + t.Parallel() + + var typedNil *strings.Reader + for name, source := range map[string]Source{ + "zero": {}, + "empty file": File(""), + "empty bytes": Bytes("", nil), + "nil reader": Reader("reader", nil), + "typed nil": Reader("reader", typedNil), + "large bytes": Bytes("large", make([]byte, maxSourceBytes+1)), + } { + t.Run(name, func(t *testing.T) { + _, diagnostics := readSource(source, "source-1", defaultSourceLimits()) + if len(diagnostics) != 1 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + }) + } +} + +type observedReadCloser struct { + io.Reader + closed bool +} + +func (r *observedReadCloser) Close() error { + r.closed = true + return nil +} + +type fragmentReader struct { + data []byte + width int +} + +func (r *fragmentReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + n := min(len(p), r.width, len(r.data)) + copy(p, r.data[:n]) + r.data = r.data[n:] + return n, nil +} + +type dataErrorReader struct { + data []byte + err error +} + +func (r *dataErrorReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, r.err +} diff --git a/internal/engine/validation.go b/internal/engine/validation.go new file mode 100644 index 0000000..44bba41 --- /dev/null +++ b/internal/engine/validation.go @@ -0,0 +1,248 @@ +package engine + +import ( + "reflect" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// Validation collects safe application diagnostics during a supported callback. +// A callback must not retain the context or use it concurrently. +type Validation struct { + state *validationState +} + +type validationState struct { + active bool + errorCode Code + warningCode Code + invalidPath Code + schema *schemaDescriptor + document *document.Node + diagnostics []Diagnostic +} + +// Error reports a value-free application error at path. Related paths are +// resolved to source locations when possible. +func (v *Validation) Error(path Path, message string, related ...Path) { + v.report(SeverityError, path, message, related) +} + +// Warning reports a value-free application warning at path. Related paths are +// resolved to source locations when possible. +func (v *Validation) Warning(path Path, message string, related ...Path) { + v.report(SeverityWarning, path, message, related) +} + +func (v *Validation) report(severity Severity, path Path, message string, related []Path) { + if v == nil || v.state == nil || !v.state.active { + panic(closedCallbackContextMessage) + } + code := v.state.errorCode + if severity == SeverityWarning { + code = v.state.warningCode + } + diagnostic := Diagnostic{Code: code, Severity: severity, Path: path.clone(), Message: message} + if !path.valid() { + invalidCode := v.state.invalidPath + if invalidCode == "" { + invalidCode = v.state.errorCode + } + diagnostic.Severity = SeverityError + diagnostic.Code = invalidCode + diagnostic.Path = Path{} + diagnostic.Message = "application reported an invalid configuration path" + } else if v.state.schema != nil { + if _, ok := resolveSchemaPath(v.state.schema, path); !ok { + diagnostic.Severity = SeverityError + diagnostic.Code = v.state.invalidPath + diagnostic.Message = "application reported a path outside the configuration schema" + v.state.diagnostics = append(v.state.diagnostics, diagnostic) + return + } + if location, ok := nearestDocumentLocation(v.state.document, path); ok { + diagnostic.Primary = documentLocationPointer(location) + } + } else if location, ok := nearestDocumentLocation(v.state.document, path); ok { + diagnostic.Primary = documentLocationPointer(location) + } + for _, relatedPath := range related { + if !relatedPath.valid() { + invalidCode := v.state.invalidPath + if invalidCode == "" { + invalidCode = v.state.errorCode + } + v.state.diagnostics = append(v.state.diagnostics, Diagnostic{ + Code: invalidCode, Severity: SeverityError, + Message: "application reported an invalid related configuration path", + }) + continue + } + if v.state.schema != nil { + if _, ok := resolveSchemaPath(v.state.schema, relatedPath); !ok { + v.state.diagnostics = append(v.state.diagnostics, Diagnostic{ + Code: v.state.invalidPath, Severity: SeverityError, Path: relatedPath.clone(), + Message: "application reported a related path outside the configuration schema", + }) + continue + } + } + if location, ok := nearestDocumentLocation(v.state.document, relatedPath); ok { + diagnostic.Related = append(diagnostic.Related, Location{ + SourceID: location.SourceID, Line: location.Line, Column: location.Column, + }) + } + } + v.state.diagnostics = append(v.state.diagnostics, diagnostic) +} + +func runValidation[T any](configuration *T, validator func(*T, *Validation), schema *schemaDescriptor, before, input *document.Node, lookups map[string]secretLookupResult) []Diagnostic { + state := &validationState{ + active: true, errorCode: CodeValidateError, warningCode: CodeValidateWarning, + invalidPath: CodeValidateInvalidPath, schema: schema, document: input, + } + context := &Validation{state: state} + validator(configuration, context) + state.active = false + after, issue := projectConfiguration(schema, reflect.ValueOf(*configuration), defaultProjectionLimits()) + if issue != nil { + state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(issue.path)) + return canonicalDiagnostics(state.diagnostics) + } + state.diagnostics = append(state.diagnostics, validationValueMutations(before, after)...) + for _, path := range changedSecretStates(reflect.ValueOf(*configuration), schema.root, lookups) { + state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(path)) + } + return canonicalDiagnostics(state.diagnostics) +} + +func validationMutationDiagnostic(path Path) Diagnostic { + return Diagnostic{ + Code: CodeValidateMutation, Severity: SeverityError, Path: path.clone(), + Message: "application validation mutated the configuration", + } +} + +func nearestDocumentLocation(root *document.Node, path Path) (document.Location, bool) { + if root == nil || !path.valid() { + return document.Location{}, false + } + current := root + nearest := current.Location + for _, segment := range path.segments { + switch current.Kind { + case document.Mapping: + if segment.kind == pathSegmentIndex { + return nearest, locationKnown(nearest) + } + found := false + for _, entry := range current.Mapping { + if entry.Key == segment.key { + current = entry.Value + nearest = current.Location + found = true + break + } + } + if !found { + return nearest, locationKnown(nearest) + } + case document.Sequence: + index, ok := schemaIndex(segment) + if !ok || index >= len(current.Sequence) { + return nearest, locationKnown(nearest) + } + current = current.Sequence[index] + nearest = current.Location + default: + return nearest, locationKnown(nearest) + } + } + return nearest, locationKnown(nearest) +} + +func locationKnown(location document.Location) bool { + return location.SourceID != "" || location.Line != 0 || location.Column != 0 +} + +// validationValueMutations combines value and sensitivity changes; resolved +// secret state is audited separately because it never enters projected trees. +func validationValueMutations(before, after *document.Node) []Diagnostic { + var diagnostics []Diagnostic + for _, delta := range compareConfigurationValues(before, after) { + diagnostics = append(diagnostics, validationMutationDiagnostic(delta.path)) + } + beforeSensitivity := projectedSensitivity(before) + afterSensitivity := projectedSensitivity(after) + for pointer, path := range beforeSensitivity { + if _, exists := afterSensitivity[pointer]; !exists { + diagnostics = append(diagnostics, validationMutationDiagnostic(path)) + } + } + for pointer, path := range afterSensitivity { + if _, exists := beforeSensitivity[pointer]; !exists { + diagnostics = append(diagnostics, validationMutationDiagnostic(path)) + } + } + return diagnostics +} + +func checkVersion(rule *VersionRule, root *document.Node) []Diagnostic { + if rule == nil { + return nil + } + version := Version{} + node, present := documentAtPath(root, rule.Path) + if present { + version.Present = true + switch node.Kind { + case document.Null: + version.Null = true + case document.String: + version.Text = node.Text + case document.Number: + if isIntegerVersion(node) { + version.Text = node.Text + break + } + fallthrough + default: + location, _ := nearestDocumentLocation(root, rule.Path) + primary := documentLocationPointer(location) + return []Diagnostic{{ + Code: CodeVersionType, Severity: SeverityError, Path: rule.Path.clone(), + Message: "schema version must be a string, integer, or null", Primary: primary, + }} + } + } + state := &validationState{ + active: true, errorCode: CodeVersionError, warningCode: CodeVersionWarning, + invalidPath: CodeVersionError, document: root, + } + validation := &Validation{state: state} + err := rule.Check(version, validation) + state.active = false + if err != nil && !hasErrors(state.diagnostics) { + location, _ := nearestDocumentLocation(root, rule.Path) + state.diagnostics = append(state.diagnostics, Diagnostic{ + Code: CodeVersionHookFailed, Severity: SeverityError, Path: rule.Path.clone(), + Message: "schema-version check failed", Primary: documentLocationPointer(location), + }) + } + return canonicalDiagnostics(state.diagnostics) +} + +func isIntegerVersion(node *document.Node) bool { + return node != nil && node.Kind == document.Number && integerPattern.MatchString(node.Text) +} + +func pathMatches(configured *Path, current Path) bool { + return configured != nil && current.valid() && matchesPathShape(*configured, current) +} + +func documentLocationPointer(location document.Location) *Location { + if !locationKnown(location) { + return nil + } + return &Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column} +} diff --git a/internal/engine/validation_test.go b/internal/engine/validation_test.go new file mode 100644 index 0000000..cce1b83 --- /dev/null +++ b/internal/engine/validation_test.go @@ -0,0 +1,214 @@ +package engine + +import ( + "errors" + "reflect" + "testing" +) + +func TestValidationCollectsSchemaAwareReportsAndNearestLocations(t *testing.T) { + t.Parallel() + + type config struct { + Primary string `yaml:"primary"` + Missing string `yaml:"missing"` + Dynamic map[string]any `yaml:"dynamic"` + } + result, err := Load(Bytes("validation.yml", []byte("primary: configured\ndynamic: {present: 1}\n")), Options[config]{Definition: Definition[config]{ + Validate: func(_ *config, validation *Validation) { + validation.Error(Keys("missing"), "is required", Keys("primary")) + validation.Error(Keys("dynamic", "missing"), "dynamic entry is required") + validation.Warning(Keys("primary"), "primary value is deprecated") + }, + }}) + if result != nil || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) != 3 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + byPath := make(map[string]Diagnostic) + for _, diagnostic := range diagnostics { + byPath[diagnostic.Path.String()] = diagnostic + } + missing := byPath["/missing"] + if missing.Code != CodeValidateError || missing.Primary == nil || missing.Primary.Line != 1 || len(missing.Related) != 1 || missing.Related[0].Line != 1 { + t.Fatalf("missing diagnostic = %#v", missing) + } + dynamic := byPath["/dynamic/missing"] + if dynamic.Code != CodeValidateError || dynamic.Primary == nil || dynamic.Primary.Line != 2 { + t.Fatalf("dynamic diagnostic = %#v", dynamic) + } + if warning := byPath["/primary"]; warning.Code != CodeValidateWarning || warning.Severity != SeverityWarning { + t.Fatalf("warning = %#v", warning) + } + if len(loadError.Sources()) != 1 { + t.Fatalf("sources = %#v", loadError.Sources()) + } +} + +func TestValidationRejectsUnknownStructPaths(t *testing.T) { + t.Parallel() + + type config struct { + Known string `yaml:"known"` + } + result, err := Load(Bytes("validation-path.yml", []byte("known: value\n")), Options[config]{Definition: Definition[config]{ + Validate: func(_ *config, validation *Validation) { + validation.Error(Keys("unknown"), "must be configured") + }, + }}) + assertLoadDiagnostic(t, result, err, CodeValidateInvalidPath) +} + +func TestValidationMutationFailsWithoutPublishingResult(t *testing.T) { + t.Parallel() + + type config struct { + Value int `yaml:"value"` + Items map[string]int `yaml:"items"` + } + result, err := Load(Bytes("mutation.yml", []byte("value: 1\nitems: {one: 1}\n")), Options[config]{Definition: Definition[config]{ + Validate: func(configuration *config, _ *Validation) { + configuration.Value = 2 + configuration.Items["one"] = 2 + }, + }}) + if result != nil || err == nil { + t.Fatalf("result, err = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatal(err) + } + var paths []string + for _, diagnostic := range loadError.Diagnostics() { + if diagnostic.Code != CodeValidateMutation || diagnostic.Message != "application validation mutated the configuration" { + t.Fatalf("diagnostic = %#v", diagnostic) + } + paths = append(paths, diagnostic.Path.String()) + } + if !reflect.DeepEqual(paths, []string{"/items/one", "/value"}) { + t.Fatalf("mutation paths = %#v", paths) + } +} + +func TestHookOrderStopsAfterErrorsAndContinuesAfterWarnings(t *testing.T) { + t.Parallel() + + type config struct { + Version string `yaml:"version"` + Value int `yaml:"value"` + } + t.Run("warnings continue", func(t *testing.T) { + var events []string + definition := Definition[config]{ + Version: &VersionRule{Path: Keys("version"), Check: func(_ Version, validation *Validation) error { + events = append(events, "version") + validation.Warning(Keys("version"), "version warning") + return nil + }}, + Defaults: func() config { + events = append(events, "defaults") + return config{Value: 1} + }, + Resolve: func(_ *config, resolution *Resolution) error { + events = append(events, "resolve") + resolution.Warning(Keys("value"), "resolution warning") + return nil + }, + Validate: func(_ *config, validation *Validation) { + events = append(events, "validate") + validation.Warning(Keys("value"), "validation warning") + }, + } + result, err := Load(Bytes("order.yml", []byte("version: one\n")), Options[config]{Definition: definition}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(events, []string{"version", "defaults", "resolve", "validate"}) || len(result.Diagnostics()) != 3 { + t.Fatalf("events = %#v, diagnostics = %#v", events, result.Diagnostics()) + } + }) + + t.Run("version error stops later hooks", func(t *testing.T) { + var events []string + definition := Definition[config]{ + Version: &VersionRule{Path: Keys("version"), Check: func(_ Version, validation *Validation) error { + events = append(events, "version") + validation.Error(Keys("version"), "unsupported version") + return nil + }}, + Defaults: func() config { events = append(events, "defaults"); return config{} }, + Resolve: func(_ *config, _ *Resolution) error { events = append(events, "resolve"); return nil }, + Validate: func(_ *config, _ *Validation) { events = append(events, "validate") }, + } + result, err := Load(Bytes("order-error.yml", []byte("version: one\n")), Options[config]{Definition: definition}) + assertLoadDiagnostic(t, result, err, CodeVersionError) + if !reflect.DeepEqual(events, []string{"version"}) { + t.Fatalf("events = %#v", events) + } + }) + + t.Run("resolution error stops validation", func(t *testing.T) { + var validated bool + result, err := Load(Bytes("resolution-order.yml", []byte("value: 1\nversion: one\n")), Options[config]{Definition: Definition[config]{ + Resolve: func(_ *config, resolution *Resolution) error { + resolution.Error(Keys("value"), "cannot resolve") + return nil + }, + Validate: func(_ *config, _ *Validation) { validated = true }, + }}) + assertLoadDiagnostic(t, result, err, CodeResolveError) + if validated { + t.Fatal("validation ran after resolution error") + } + }) + + t.Run("decode error stops application hooks", func(t *testing.T) { + var resolved, validated bool + result, err := Load(Bytes("decode-order.yml", []byte("value: wrong\nversion: one\n")), Options[config]{Definition: Definition[config]{ + Resolve: func(_ *config, _ *Resolution) error { resolved = true; return nil }, + Validate: func(_ *config, _ *Validation) { validated = true }, + }}) + assertLoadDiagnostic(t, result, err, CodeDecodeType) + if resolved || validated { + t.Fatalf("later hooks ran: resolve=%v validate=%v", resolved, validated) + } + }) +} + +func TestCompleteSingleSourceLifecycleKeepsMetadataAgreement(t *testing.T) { + t.Parallel() + + type config struct { + Input int `yaml:"input"` + Derived int `yaml:"derived"` + } + result, err := Load(Bytes("complete.yml", []byte("input: 3\n")), Options[config]{Definition: Definition[config]{ + Defaults: func() config { return config{Derived: 1} }, + Resolve: func(configuration *config, resolution *Resolution) error { + Default(resolution, Keys("derived"), &configuration.Derived, configuration.Input*2, "derived from input", Keys("input")) + return nil + }, + Validate: func(configuration *config, validation *Validation) { + if configuration.Derived != 6 { + validation.Error(Keys("derived"), "derived value is inconsistent", Keys("input")) + } + }, + }}) + if err != nil { + t.Fatal(err) + } + input, _ := result.Metadata().Lookup(Keys("input")) + derived, _ := result.Metadata().Lookup(Keys("derived")) + history := result.Metadata().History(Keys("derived")) + if result.Config != (config{Input: 3, Derived: 6}) || !input.Explicit || input.Origin.Layer != LayerBase || derived.Explicit || derived.Origin.Operation != ChangeDerivedDefault || history[len(history)-1].Dependencies[0].String() != "/input" { + t.Fatalf("config/metadata = %#v / %#v / %#v / %#v", result.Config, input, derived, history) + } +} diff --git a/internal/engine/value_operation.go b/internal/engine/value_operation.go new file mode 100644 index 0000000..08661df --- /dev/null +++ b/internal/engine/value_operation.go @@ -0,0 +1,126 @@ +package engine + +import "reflect" + +type projectionLimits struct { + depth int + nodes int +} + +func defaultProjectionLimits() projectionLimits { + return projectionLimits{depth: maxDocumentDepth, nodes: maxDocumentNodes} +} + +type projectionIssue struct { + code Code + path Path + message string +} + +type valueVisit struct { + typeOf reflect.Type + kind reflect.Kind + ptr uintptr +} + +// valueOperation owns one traversal's cycle and budget state and reuses the +// current load's descriptor cache for dynamic values. +type valueOperation struct { + cloneMode secretCloneMode + schema *schemaDescriptor + limits projectionLimits + active map[valueVisit]bool + nodes int +} + +func (o *valueOperation) dynamicDescriptor(current reflect.Type, path Path) (*typeDescriptor, *projectionIssue) { + if current == secretType { + return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration values cannot establish secret-reference semantics"} + } + builder := schemaBuilder{cache: o.schema.types, building: make(map[reflect.Type]bool)} + descriptor, issue := builder.describe(current, path) + if issue != nil { + return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration value has an unsupported type"} + } + if descriptorContainsSecret(descriptor, make(map[*typeDescriptor]bool)) { + return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration values cannot establish secret-reference semantics"} + } + return descriptor, nil +} + +func descriptorContainsSecret(descriptor *typeDescriptor, visited map[*typeDescriptor]bool) bool { + if descriptor == nil || visited[descriptor] { + return false + } + if descriptor.kind == descriptorSecret { + return true + } + visited[descriptor] = true + if descriptorContainsSecret(descriptor.element, visited) { + return true + } + for _, member := range descriptor.members { + if descriptorContainsSecret(member.descriptor, visited) { + return true + } + } + return false +} + +func (o *valueOperation) charge(path Path) *projectionIssue { + o.nodes++ + if o.limits.nodes >= 0 && o.nodes > o.limits.nodes { + return &projectionIssue{code: CodeSchemaProjectionLimit, path: path.clone(), message: "configuration projection exceeds the node limit"} + } + return nil +} + +func (o *valueOperation) checkDepth(path Path, depth int) *projectionIssue { + if depth > o.limits.depth { + return &projectionIssue{code: CodeSchemaProjectionLimit, path: path.clone(), message: "configuration projection exceeds the depth limit"} + } + return nil +} + +func (o *valueOperation) enter(value reflect.Value, path Path) (func(), *projectionIssue) { + visit, ok := visitFor(value) + if !ok { + return func() {}, nil + } + if o.active[visit] { + return nil, &projectionIssue{code: CodeSchemaCycle, path: path.clone(), message: "configuration values must not contain cycles"} + } + o.active[visit] = true + return func() { delete(o.active, visit) }, nil +} + +func visitFor(value reflect.Value) (valueVisit, bool) { + if isNilValue(value) { + return valueVisit{}, false + } + visit := valueVisit{typeOf: value.Type(), kind: value.Kind()} + switch value.Kind() { + case reflect.Pointer, reflect.Slice: + if value.Kind() == reflect.Slice && value.Len() == 0 { + return valueVisit{}, false + } + visit.ptr = value.Pointer() + case reflect.Map: + if value.Len() == 0 { + return valueVisit{}, false + } + visit.ptr = uintptr(value.UnsafePointer()) + default: + return valueVisit{}, false + } + return visit, visit.ptr != 0 +} + +func isNilValue(value reflect.Value) bool { + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return value.IsNil() + default: + return false + } +} diff --git a/internal/engine/version_path_test.go b/internal/engine/version_path_test.go new file mode 100644 index 0000000..d266938 --- /dev/null +++ b/internal/engine/version_path_test.go @@ -0,0 +1,90 @@ +package engine + +import "testing" + +func TestVersionPathsShareCanonicalSequenceSemantics(t *testing.T) { + t.Parallel() + type config struct { + Versions []string `yaml:"versions"` + } + for _, token := range []string{"0", "+0", "-0", "00", "+1"} { + for _, value := range []string{"4", "'4'"} { + t.Run(token+"/"+value, func(t *testing.T) { + path, err := ParsePath("/versions/" + token) + if err != nil { + t.Fatal(err) + } + var checked Version + result, err := Load[config](Bytes("version.yml", []byte("versions: ["+value+"]\n")), Options[config]{Definition: Definition[config]{ + Version: &VersionRule{Path: path, Check: func(v Version, _ *Validation) error { checked = v; return nil }}, + }}) + want := Version{} + if token == "0" { + want = Version{Present: true, Text: "4"} + } + if checked != want { + t.Fatalf("checked = %#v, want %#v", checked, want) + } + if token != "0" && value == "4" { + requireDiagnostic(t, err, CodeDecodeType, "/versions/0") + return + } + if err != nil || result.Config.Versions[0] != "4" { + t.Fatalf("result = %#v, err = %v", result, err) + } + _, found := result.Metadata().Lookup(path) + if found != (token == "0") { + t.Fatalf("metadata lookup = %v", found) + } + }) + } + } +} + +func TestVersionPathsPreserveLiteralMappingKeysAndSegmentIntent(t *testing.T) { + t.Parallel() + for _, key := range []string{"0", "+0", "-0", "00", "a/b~"} { + t.Run(key, func(t *testing.T) { + path, err := ParsePath(Keys(key).String()) + if err != nil { + t.Fatal(err) + } + result, err := Load[map[string]string](Bytes("version.yml", []byte("'"+key+"': 4\n")), Options[map[string]string]{Definition: Definition[map[string]string]{ + Version: &VersionRule{Path: path, Check: func(v Version, _ *Validation) error { + if !v.Present || v.Text != "4" { + t.Errorf("checked = %#v", v) + } + return nil + }}, + }}) + if err != nil || result.Config[key] != "4" { + t.Fatalf("result = %#v, err = %v", result, err) + } + if _, ok := result.Metadata().Lookup(path); !ok { + t.Fatal("literal key missing from metadata") + } + }) + } + for _, test := range []struct { + name string + path Path + input string + }{ + {"key cannot address sequence", Keys("versions", "0"), "versions: [4]\n"}, + {"index cannot address mapping", Keys("versions").Index(0), "versions: {'0': 4}\n"}, + } { + t.Run(test.name, func(t *testing.T) { + type config struct { + Versions any `yaml:"versions"` + } + _, err := Load[config](Bytes("version.yml", []byte(test.input)), Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: test.path, Check: func(v Version, validation *Validation) error { + if v.Present { + t.Errorf("wrong segment kind found version: %#v", v) + } + validation.Error(Keys("versions"), "version absent") + return nil + }}}}) + requireDiagnostic(t, err, CodeVersionError, "/versions") + }) + } +}