From d6e32d88073d57327ee5a1f5e2b9a9e4c30fd931 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 6 Sep 2026 21:40:15 +0000 Subject: [PATCH] Add typed environment secret references --- README.md | 8 +- decode.go | 7 + docs/config.md | 53 +++++- docs/development.md | 3 +- docs/internal/overview.md | 10 +- docs/roadmap/implementation.md | 34 +++- example_test.go | 21 +++ load.go | 12 +- projection.go | 38 ++++ records.go | 5 + schema.go | 6 + secret_reference.go | 211 +++++++++++++++++++++ secret_reference_test.go | 323 +++++++++++++++++++++++++++++++++ validation.go | 5 +- 14 files changed, 723 insertions(+), 13 deletions(-) create mode 100644 secret_reference.go create mode 100644 secret_reference_test.go diff --git a/README.md b/README.md index 67c23ed..402718e 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,10 @@ can overlay one selected fixed profile and check an application schema version. returns detached metadata and safe diagnostics. Applications may perform audited defaulting and normalization through a resolver, then report domain errors and warnings through read-only semantic validation. Secret -tags and a path predicate drive deterministic redacted effective YAML. Explicit -secrets-directory bootstrap is available as a separate startup operation. -Overrides, typed secret references, and value explanation are still under -development. +tags and a path predicate drive deterministic redacted effective YAML. Typed secret +references resolve environment names before validation, and explicit +secrets-directory bootstrap is available as a separate startup operation. Overrides +and value explanation are still under development. ```go type Config struct { diff --git a/decode.go b/decode.go index 86f00cd..427e1a0 100644 --- a/decode.go +++ b/decode.go @@ -76,6 +76,8 @@ func decodeValue(node *document.Node, base reflect.Value, descriptor *typeDescri switch descriptor.kind { case descriptorPointer, descriptorMap, descriptorSlice, descriptorDynamic: return reflect.Zero(descriptor.typeOf), nil + case descriptorSecret: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeSecretRequired, path, node, "required secret reference cannot be null")} default: return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeNull, path, node, "null is not allowed for this configuration type")} } @@ -165,6 +167,11 @@ func decodeValue(node *document.Node, base reflect.Value, descriptor *typeDescri result := reflect.New(descriptor.typeOf).Elem() result.SetInt(int64(value)) return result, nil + case descriptorSecret: + if node.Kind != document.String { + return typeMismatch(path, node, "secret environment name") + } + return reflect.ValueOf(SecretEnv(node.Text)), nil case descriptorText: if node.Kind != document.String { return typeMismatch(path, node, "text string") diff --git a/docs/config.md b/docs/config.md index 8dd1f45..34214ec 100644 --- a/docs/config.md +++ b/docs/config.md @@ -42,7 +42,8 @@ The current lifecycle is: validate options → parse the selected source → traverse/compose enabled imports → build schema → check the optional application version → apply a selected profile → create and clone defaults → strictly decode → recorded application resolution - → semantic validation → classify secrets → snapshot configuration and metadata + → resolve typed secret references → semantic validation → classify secrets + → snapshot configuration and metadata ``` `File`, `Bytes`, and `Reader` each contribute the first base source with deterministic @@ -144,6 +145,50 @@ configuration values. Applications must still keep source labels, resolution reasons, and diagnostic messages value-free because ConfigKit cannot infer and sanitize secrets embedded in arbitrary application prose. +## Typed secret references + +Credential-bearing fields use the opaque `Secret` type and store an environment +variable name in YAML, such as `api_key: OPENROUTER_API_KEY`. `SecretEnv(name)` +constructs the same unresolved reference for application defaults or recorded +resolution. Names must match `[A-Za-z_][A-Za-z0-9_]*`; they are public +configuration and remain visible through `EnvName`, canonical projection, history, +and effective output unless the path is separately classified for redaction. + +A non-pointer `Secret` is required. An absent, null, or invalid reference, or an +unset or empty environment value, fails with a safe `secret.*` diagnostic at the +configuration path. `*Secret` is optional only when nil; every non-nil reference +must resolve to a nonempty value. Secret references work in typed structs, maps, +pointers, slices, and arrays. A dynamic `any` value does not establish secret +semantics and cannot contain `Secret`; applications must use a typed slot. + +ConfigKit resolves only the effective names after recorded application resolution +and before semantic validation. Each unique name is read with `os.LookupEnv` once +per load. Validation can inspect the credential through the deliberately named +`Value()` accessor and distinguish success with `IsResolved()`. Later loads read +the environment again, and cloned defaults are rebuilt from names so a previously +resolved result cannot retain a stale credential. `Load` never writes to the +environment. + +Credential bytes are held only in the per-load lookup state and the trusted returned +configuration, and are returned only by `Value`. Secret formatting uses a safe +public-name representation, JSON marshals only the name, and projection, metadata, +histories, diagnostics, effective output, +and resolution snapshots never retain the credential. Validation remains read-only: +in addition to ordinary projected-value comparison, ConfigKit checks the private +resolved state against this load's lookup result. Applications must keep their +validation messages value-free and must not include `Value()` in diagnostics. + +For example, a typed pipeline field is declared as: + +```go +type PipelineConfig struct { + APIKey configkit.Secret `yaml:"api_key"` +} +``` + +The executable `ExampleSecretEnv` demonstrates loading and inspecting the safe +reference state without printing its value. + ## Explicit secrets-directory bootstrap `LoadSecretsDir(directory)` is a separate, opt-in startup operation. It is never @@ -410,7 +455,8 @@ it keeps no global mutable descriptor cache. A configuration root must be a stru or string-keyed map. Nested supported shapes are: - booleans, strings, signed and unsigned integers, and finite floating-point types; -- `time.Duration` and paired `encoding.TextMarshaler`/`TextUnmarshaler` scalars; +- `time.Duration`, typed `Secret` references, and paired + `encoding.TextMarshaler`/`TextUnmarshaler` scalars; - pointers, structs, string-keyed maps, slices, fixed arrays, and `any`; and - recursive type definitions whose eventual application values are acyclic. @@ -434,7 +480,8 @@ Schema path traversal uses these same bindings. Explicit key segments address mappings, explicit index segments address sequences, and parsed numeric pointer tokens become indices only in a sequence. Sequence indices cannot have leading zeroes; fixed-array indices are bounds checked. Dynamic `any` and inline-map entries -remain schema-valid for application-defined descendants. +remain schema-valid for application-defined descendants, but dynamic values cannot +introduce typed secret references. Descriptors are reused by typed decoding, projection, and cloning so those operations cannot acquire different field bindings. diff --git a/docs/development.md b/docs/development.md index 1972038..bce5176 100644 --- a/docs/development.md +++ b/docs/development.md @@ -5,7 +5,8 @@ below before making changes. The repository is in early implementation: its publ loader currently covers strict selected sources, explicit path/glob import graphs, composition, selected profile overlays, typed defaults, schema-version checks, recorded application resolution, semantic validation, layered metadata, secret -classification, and redacted effective rendering. The +classification, typed environment references, explicit secrets bootstrap, and +redacted effective rendering. The [component overview](internal/overview.md) owns the implementation inventory. ## Initial orientation diff --git a/docs/internal/overview.md b/docs/internal/overview.md index a25fb5d..febe28b 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -31,14 +31,20 @@ module and package documentation boundary. Its implemented public primitives are intent, private before/after snapshots, and resolution metadata integration; and - post-resolution schema-aware report collection and validator mutation detection in `validation.go`, completing the public callback lifecycle after composed input; +- opaque typed environment references and per-load name resolution in + `secret_reference.go`, including credential-free projection/cloning and private + validator-state auditing; +- explicit bounded startup environment bootstrap and safe partial reporting in + `secrets.go`; - fixed-point tag/predicate/dependency sensitivity classification in `secret.go`; and - frozen-snapshot redaction and canonical effective output in `render.go`, with the only upstream dumper integration isolated in `internal/yaml/write.go`. The [configuration contract](../config.md) owns implemented input, schema, decoding, loading lifecycle, path imports, composition, profile overlay, version checking, -recorded resolution, semantic validation, metadata, secret classification, redacted -effective rendering, and ownership behavior. +recorded resolution, typed secret references/bootstrap, semantic validation, +metadata, secret classification, redacted effective rendering, and ownership +behavior. The source-extraction inventory records the application foundations assigned to future implementation work: diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 7cb0b80..2b4d42b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1233,7 +1233,39 @@ Acceptance: ## Stage 17 — Add environment-backed typed secret references -Status: pending. +Status: complete. + +Completion evidence (2026-09-06): + +- Added the opaque `Secret` and `SecretEnv` API in + [`secret_reference.go`](../../secret_reference.go), with name/value/resolution + accessors, name-only JSON, and credential-free formatting for every `fmt` verb. + The zero value remains an absent reference and no public constructor can inject + resolved credential state. +- Extended the shared descriptor, decode, projection, clone, comparison, resolution, + and validation paths with a first-class secret-reference descriptor. References + project only their public names, cloned defaults are rebuilt unresolved, dynamic + `any` values cannot introduce secret semantics, and existing tag/predicate + classification continues to hide additionally sensitive names or derived data. +- Inserted deterministic per-load environment resolution after recorded application + resolution and before semantic validation. Only effective valid names are read, + each unique success/unset/empty outcome is cached for that load, required and + optional shapes receive path-specific `secret.*` diagnostics, and `Load` performs + no environment writes. Validation compares private reference state with the lookup + cache in addition to the ordinary projected mutation audit. +- Added focused coverage in + [`secret_reference_test.go`](../../secret_reference_test.go) for structs, maps, + pointers and sequences; successful and failed lookup deduplication; absent/null/ + empty/invalid/unset cases; resolver-selected names; resolved-value validation; + same-name unresolved validator replacement; later-load environment changes; + stale resolved defaults; dynamic values; bootstrap blocked-empty integration; and + non-disclosure through fmt, JSON, effective projection, history, and diagnostics. + Added executable `ExampleSecretEnv` and updated the + [current secret-reference contract](../config.md#typed-secret-references), component + inventory, developer guide, and README. Verified `go test ./...`, + `go test -race ./...`, `go vet ./...`, `go build ./...`, `go mod tidy -diff`, and + `git diff --check` with Go 1.26.7 on linux/amd64. No deviations were required; + CLI overrides remain later work. Read: [secret reference contract](initial.md#secret-references-and-explicit-bootstrap), shared descriptor/projection rules and lifecycle, Stage 16 report boundary. diff --git a/example_test.go b/example_test.go index 3cdaac0..fa1d999 100644 --- a/example_test.go +++ b/example_test.go @@ -3,6 +3,7 @@ package configkit_test import ( "errors" "fmt" + "os" "strings" configkit "gitea.maximumdirect.net/eric/ConfigKit" @@ -94,3 +95,23 @@ func ExampleResolution() { fmt.Println(strings.Join(order, " then "), result.Config.Derived, err) // Output: resolve then validate 6 } + +func ExampleSecretEnv() { + const name = "CONFIGKIT_EXAMPLE_API_KEY" + previous, present := os.LookupEnv(name) + _ = os.Setenv(name, "disposable-example-value") + defer func() { + if present { + _ = os.Setenv(name, previous) + } else { + _ = os.Unsetenv(name) + } + }() + + type Config struct { + APIKey configkit.Secret `yaml:"api_key"` + } + result, err := configkit.Load[Config](configkit.Bytes("secret.yml", []byte("api_key: "+name+"\n"))) + fmt.Println(result.Config.APIKey.EnvName(), result.Config.APIKey.IsResolved(), err) + // Output: CONFIGKIT_EXAMPLE_API_KEY true +} diff --git a/load.go b/load.go index 9e97567..1651322 100644 --- a/load.go +++ b/load.go @@ -1,6 +1,7 @@ package configkit import ( + "os" "reflect" "gitea.maximumdirect.net/eric/ConfigKit/internal/document" @@ -100,6 +101,10 @@ func (r *Result[T]) Diagnostics() []Diagnostic { // imports, and one selected profile. It accepts at most one Options value and // returns no result when any error is reported. func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { + return loadWithSecretLookup(source, os.LookupEnv, supplied...) +} + +func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied ...Options[T]) (*Result[T], error) { accumulator := diagnosticAccumulator{} options, diagnostics := selectOptions(source, supplied) if len(diagnostics) != 0 { @@ -208,8 +213,13 @@ func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { resolutionSnapshots = resolved.snapshots applyResolutionMetadata(&metadata, resolved) } + secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, inputDocument, lookup) + accumulator.add(secretResolution.diagnostics...) + if hasErrors(secretResolution.diagnostics) { + return nil, accumulator.loadError() + } if options.Definition.Validate != nil { - validationDiagnostics := runValidation(&configuration, options.Definition.Validate, schema, final, inputDocument) + validationDiagnostics := runValidation(&configuration, options.Definition.Validate, schema, final, inputDocument, secretResolution.lookups) accumulator.add(validationDiagnostics...) if hasErrors(validationDiagnostics) { return nil, accumulator.loadError() diff --git a/projection.go b/projection.go index 03f293e..c653861 100644 --- a/projection.go +++ b/projection.go @@ -222,6 +222,12 @@ func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor node.Text = time.Duration(value.Int()).String() } return node, issue + case descriptorSecret: + node, issue := o.node(document.String, path, depth) + if issue == nil { + node.Text = value.Interface().(Secret).EnvName() + } + return node, issue case descriptorText: text, issue := marshalCanonicalText(value, path) if issue != nil { @@ -336,6 +342,13 @@ func (o *valueOperation) clone(value reflect.Value, descriptor *typeDescriptor, result := reflect.New(value.Type()).Elem() result.Set(cloned) return result, nil + case descriptorSecret: + result := reflect.New(value.Type()).Elem() + reference := value.Interface().(Secret) + if reference.specified { + result.Set(reflect.ValueOf(SecretEnv(reference.EnvName()))) + } + return result, nil case descriptorText: text, issue := marshalCanonicalText(value, path) if issue != nil { @@ -360,14 +373,39 @@ func (o *valueOperation) clone(value reflect.Value, descriptor *typeDescriptor, } func (o *valueOperation) dynamicDescriptor(current reflect.Type, path Path) (*typeDescriptor, *projectionIssue) { + if current == secretType { + return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration values cannot establish secret-reference semantics"} + } builder := schemaBuilder{cache: o.schema.types, building: make(map[reflect.Type]bool)} descriptor, issue := builder.describe(current, path) if issue != nil { return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration value has an unsupported type"} } + if descriptorContainsSecret(descriptor, make(map[*typeDescriptor]bool)) { + return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration values cannot establish secret-reference semantics"} + } return descriptor, nil } +func descriptorContainsSecret(descriptor *typeDescriptor, visited map[*typeDescriptor]bool) bool { + if descriptor == nil || visited[descriptor] { + return false + } + if descriptor.kind == descriptorSecret { + return true + } + visited[descriptor] = true + if descriptorContainsSecret(descriptor.element, visited) { + return true + } + for _, member := range descriptor.members { + if descriptorContainsSecret(member.descriptor, visited) { + return true + } + } + return false +} + func (o *valueOperation) node(kind document.Kind, path Path, depth int) (*document.Node, *projectionIssue) { if issue := o.checkDepth(path, depth); issue != nil { return nil, issue diff --git a/records.go b/records.go index 03ddedb..69d0c98 100644 --- a/records.go +++ b/records.go @@ -127,6 +127,11 @@ const ( CodeSecretsLimitExceeded Code = "secrets.limit_exceeded" CodeSecretsInvalidValue Code = "secrets.invalid_value" CodeSecretsApplyFailed Code = "secrets.apply_failed" + + CodeSecretRequired Code = "secret.required" + CodeSecretInvalidName Code = "secret.invalid_name" + CodeSecretUnset Code = "secret.unset" + CodeSecretEmpty Code = "secret.empty" ) // Location identifies a position in a configuration source. Zero line and column diff --git a/schema.go b/schema.go index a89b2b2..9eda711 100644 --- a/schema.go +++ b/schema.go @@ -24,6 +24,7 @@ const ( descriptorUnsigned descriptorFloat descriptorDuration + descriptorSecret descriptorText ) @@ -66,6 +67,7 @@ type schemaIssue struct { var ( durationType = reflect.TypeFor[time.Duration]() + secretType = reflect.TypeFor[Secret]() textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]() textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]() ) @@ -120,6 +122,10 @@ func (b *schemaBuilder) describe(current reflect.Type, path Path) (*typeDescript descriptor.kind = descriptorDuration return descriptor, nil } + if current == secretType { + descriptor.kind = descriptorSecret + return descriptor, nil + } marshals, unmarshals := textCodecSupport(current) if marshals || unmarshals { if !marshals || !unmarshals { diff --git a/secret_reference.go b/secret_reference.go new file mode 100644 index 0000000..695fbe4 --- /dev/null +++ b/secret_reference.go @@ -0,0 +1,211 @@ +package configkit + +import ( + "encoding/json" + "fmt" + "io" + "reflect" + "slices" + "strconv" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +// Secret is an opaque reference to an environment variable. Its name is public +// configuration; its resolved value is available only through Value. +type Secret struct { + name string + value string + specified bool + resolved bool +} + +// SecretEnv constructs an unresolved environment-backed secret reference. +// Invalid names are reported if the reference survives a configuration load. +func SecretEnv(name string) Secret { + return Secret{name: name, specified: true} +} + +// EnvName returns the public environment-variable name. +func (s Secret) EnvName() string { + return s.name +} + +// Value returns the resolved credential, or an empty string when unresolved. +func (s Secret) Value() string { + if !s.resolved { + return "" + } + return s.value +} + +// IsResolved reports whether this reference resolved to a nonempty value during +// its configuration load. +func (s Secret) IsResolved() bool { + return s.resolved +} + +// Format renders only the public reference name for every formatting verb. +func (s Secret) Format(state fmt.State, _ rune) { + _, _ = io.WriteString(state, "configkit.SecretEnv("+strconv.Quote(s.name)+")") +} + +// MarshalJSON emits only the public environment-variable name. +func (s Secret) MarshalJSON() ([]byte, error) { + return json.Marshal(s.name) +} + +type secretLookup func(string) (string, bool) + +type secretLookupResult struct { + value string + present bool +} + +type secretResolutionResult struct { + lookups map[string]secretLookupResult + diagnostics []Diagnostic +} + +func resolveSecretReferences(root reflect.Value, schema *schemaDescriptor, input *document.Node, lookup secretLookup) secretResolutionResult { + result := secretResolutionResult{ + lookups: make(map[string]secretLookupResult), + } + if schema == nil || schema.root == nil || !root.IsValid() || root.Type() != schema.root.typeOf { + result.diagnostics = []Diagnostic{{Code: CodeSecretRequired, Severity: SeverityError, Message: "configuration value does not match its secret-reference schema"}} + return result + } + if lookup == nil { + lookup = func(string) (string, bool) { return "", false } + } + resolveSecretValue(root, schema.root, Path{}, false, input, lookup, &result) + result.diagnostics = canonicalDiagnostics(result.diagnostics) + return result +} + +func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, path Path, optional bool, input *document.Node, lookup secretLookup, result *secretResolutionResult) { + if descriptor.kind == descriptorPointer { + if value.IsNil() { + return + } + resolveSecretValue(value.Elem(), descriptor.element, path, descriptor.element.kind == descriptorSecret, input, lookup, result) + return + } + + switch descriptor.kind { + case descriptorSecret: + reference := value.Interface().(Secret) + name := reference.EnvName() + if name == "" { + code := CodeSecretRequired + message := "required secret reference is absent" + if optional || reference.specified { + code = CodeSecretInvalidName + message = "secret environment name is empty" + } + result.diagnostics = append(result.diagnostics, secretDiagnostic(code, path, message, input)) + return + } + if !validEnvironmentName(name) { + result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretInvalidName, path, "secret environment name is invalid", input)) + return + } + found, exists := result.lookups[name] + if !exists { + found.value, found.present = lookup(name) + result.lookups[name] = found + } + if !found.present { + result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretUnset, path, "secret environment variable is not set", input)) + return + } + if found.value == "" { + result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretEmpty, path, "secret environment variable is empty", input)) + return + } + value.Set(reflect.ValueOf(Secret{name: name, value: found.value, specified: true, resolved: true})) + case descriptorStruct: + for _, member := range descriptor.members { + memberPath := path + if !member.inline { + memberPath = path.Key(member.name) + } + resolveSecretValue(value.Field(member.index), member.descriptor, memberPath, false, input, lookup, result) + } + case descriptorMap: + if value.IsNil() { + return + } + keys := value.MapKeys() + slices.SortFunc(keys, func(left, right reflect.Value) int { return cmpString(left.String(), right.String()) }) + for _, key := range keys { + entry := reflect.New(value.Type().Elem()).Elem() + entry.Set(value.MapIndex(key)) + resolveSecretValue(entry, descriptor.element, path.Key(key.String()), false, input, lookup, result) + value.SetMapIndex(key, entry) + } + case descriptorSlice, descriptorArray: + if descriptor.kind == descriptorSlice && value.IsNil() { + return + } + for index := 0; index < value.Len(); index++ { + resolveSecretValue(value.Index(index), descriptor.element, path.Index(index), false, input, lookup, result) + } + } +} + +func secretDiagnostic(code Code, path Path, message string, input *document.Node) Diagnostic { + diagnostic := Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message} + if location, ok := nearestDocumentLocation(input, path); ok { + diagnostic.Primary = documentLocationPointer(location) + } + return diagnostic +} + +func changedSecretStates(root reflect.Value, descriptor *typeDescriptor, lookups map[string]secretLookupResult) []Path { + var changed []Path + collectChangedSecretStates(root, descriptor, Path{}, lookups, &changed) + slices.SortFunc(changed, func(left, right Path) int { return cmpString(left.String(), right.String()) }) + return changed +} + +func collectChangedSecretStates(value reflect.Value, descriptor *typeDescriptor, path Path, lookups map[string]secretLookupResult, changed *[]Path) { + if descriptor.kind == descriptorPointer { + if !value.IsNil() { + collectChangedSecretStates(value.Elem(), descriptor.element, path, lookups, changed) + } + return + } + switch descriptor.kind { + case descriptorSecret: + reference := value.Interface().(Secret) + expected, ok := lookups[reference.EnvName()] + if ok && (!expected.present || expected.value == "" || !reference.resolved || reference.value != expected.value) { + *changed = append(*changed, path.clone()) + } + case descriptorStruct: + for _, member := range descriptor.members { + memberPath := path + if !member.inline { + memberPath = path.Key(member.name) + } + collectChangedSecretStates(value.Field(member.index), member.descriptor, memberPath, lookups, changed) + } + case descriptorMap: + if value.IsNil() { + return + } + keys := value.MapKeys() + slices.SortFunc(keys, func(left, right reflect.Value) int { return cmpString(left.String(), right.String()) }) + for _, key := range keys { + collectChangedSecretStates(value.MapIndex(key), descriptor.element, path.Key(key.String()), lookups, changed) + } + case descriptorSlice, descriptorArray: + if descriptor.kind == descriptorSlice && value.IsNil() { + return + } + for index := 0; index < value.Len(); index++ { + collectChangedSecretStates(value.Index(index), descriptor.element, path.Index(index), lookups, changed) + } + } +} diff --git a/secret_reference_test.go b/secret_reference_test.go new file mode 100644 index 0000000..e8e9a46 --- /dev/null +++ b/secret_reference_test.go @@ -0,0 +1,323 @@ +package configkit + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "strings" + "testing" +) + +type secretReferenceConfig struct { + Required Secret `yaml:"required"` + Optional *Secret `yaml:"optional"` + Map map[string]Secret `yaml:"map"` + Pointers map[string]*Secret `yaml:"pointers"` + Sequence []Secret `yaml:"sequence"` +} + +func TestSecretReferencesResolveTypedShapesOncePerName(t *testing.T) { + input := `required: SHARED_SECRET +optional: OPTIONAL_SECRET +map: + first: SHARED_SECRET +pointers: + nil: null + value: POINTER_SECRET +sequence: [SHARED_SECRET, SEQUENCE_SECRET] +` + values := map[string]string{ + "SHARED_SECRET": "shared-value", + "OPTIONAL_SECRET": "optional-value", + "POINTER_SECRET": "pointer-value", + "SEQUENCE_SECRET": "sequence-value", + } + counts := map[string]int{} + lookup := func(name string) (string, bool) { + counts[name]++ + value, present := values[name] + return value, present + } + validationCalled := false + result, err := loadWithSecretLookup(Bytes("secrets.yml", []byte(input)), lookup, Options[secretReferenceConfig]{ + Definition: Definition[secretReferenceConfig]{Validate: func(configuration *secretReferenceConfig, validation *Validation) { + validationCalled = true + if configuration.Required.Value() != "shared-value" || !configuration.Required.IsResolved() || + configuration.Optional.Value() != "optional-value" || configuration.Pointers["value"].Value() != "pointer-value" || + configuration.Sequence[1].Value() != "sequence-value" { + validation.Error(Keys("required"), "references were not resolved before validation") + } + }}, + }) + if err != nil { + t.Fatalf("loadWithSecretLookup() error = %v", err) + } + if !validationCalled || result.Config.Optional == nil || result.Config.Pointers["nil"] != nil { + t.Fatalf("resolved config = %#v", result.Config) + } + if result.Config.Required.EnvName() != "SHARED_SECRET" || result.Config.Map["first"].Value() != "shared-value" { + t.Fatalf("reference names/values = %#v", result.Config) + } + if !reflect.DeepEqual(counts, map[string]int{"SHARED_SECRET": 1, "OPTIONAL_SECRET": 1, "POINTER_SECRET": 1, "SEQUENCE_SECRET": 1}) { + t.Fatalf("lookup counts = %#v", counts) + } +} + +func TestSecretReferenceErrorsAreSafeAndPathSpecific(t *testing.T) { + for _, test := range []struct { + name string + input string + lookup secretLookup + code Code + }{ + {name: "absent", input: "{}\n", code: CodeSecretRequired}, + {name: "null", input: "required: null\n", code: CodeSecretRequired}, + {name: "empty name", input: "required: ''\n", code: CodeSecretInvalidName}, + {name: "invalid name", input: "required: INVALID-NAME\n", code: CodeSecretInvalidName}, + {name: "unset", input: "required: MISSING_SECRET\n", code: CodeSecretUnset}, + {name: "empty environment", input: "required: EMPTY_SECRET\n", lookup: func(string) (string, bool) { return "", true }, code: CodeSecretEmpty}, + } { + t.Run(test.name, func(t *testing.T) { + lookup := test.lookup + if lookup == nil { + lookup = func(string) (string, bool) { return "", false } + } + _, err := loadWithSecretLookup(Bytes("secret-errors.yml", []byte(test.input)), lookup, Options[secretReferenceConfig]{}) + loadError := requireSecretLoadCode(t, err, test.code) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Path.String() != "/required" || strings.Contains(fmt.Sprintf("%v %#v", err, err), "CREDENTIAL-CANARY") { + t.Fatalf("diagnostic/error = %#v / %v", diagnostic, err) + } + }) + } + + t.Run("non-nil optional is required to resolve", func(t *testing.T) { + type config struct { + Optional *Secret `yaml:"optional"` + } + _, err := loadWithSecretLookup(Bytes("optional.yml", []byte("optional: ''\n")), func(string) (string, bool) { return "", false }, Options[config]{}) + requireSecretLoadCode(t, err, CodeSecretInvalidName) + result, err := loadWithSecretLookup(Bytes("optional.yml", []byte("optional: null\n")), func(string) (string, bool) { return "", false }, Options[config]{}) + if err != nil || result.Config.Optional != nil { + t.Fatalf("nil optional result = %#v, %v", result, err) + } + }) + + t.Run("failed lookup is cached", func(t *testing.T) { + type config struct { + Secrets []Secret `yaml:"secrets"` + } + lookups := 0 + _, err := loadWithSecretLookup(Bytes("missing-twice.yml", []byte("secrets: [MISSING_SHARED, MISSING_SHARED]\n")), func(string) (string, bool) { + lookups++ + return "", false + }, Options[config]{}) + loadError := requireSecretLoadCode(t, err, CodeSecretUnset) + if lookups != 1 || len(loadError.Diagnostics()) != 2 { + t.Fatalf("lookups = %d, diagnostics = %#v", lookups, loadError.Diagnostics()) + } + }) +} + +func TestResolutionSelectsEffectiveSecretNameBeforeLookup(t *testing.T) { + type config struct { + Name string `yaml:"name"` + Secret Secret `yaml:"secret"` + } + lookedUp := []string{} + validated := "" + result, err := loadWithSecretLookup(Bytes("resolved-secret.yml", []byte("name: SELECTED_SECRET\nsecret: OLD_SECRET\n")), func(name string) (string, bool) { + lookedUp = append(lookedUp, name) + return "resolved-value", true + }, Options[config]{Definition: Definition[config]{ + Resolve: func(configuration *config, resolution *Resolution) error { + Normalize(resolution, Keys("secret"), &configuration.Secret, SecretEnv(configuration.Name), "selected named credential", Keys("name")) + return nil + }, + Validate: func(configuration *config, validation *Validation) { + validated = configuration.Secret.Value() + }, + }}) + if err != nil { + t.Fatalf("loadWithSecretLookup() error = %v", err) + } + if !reflect.DeepEqual(lookedUp, []string{"SELECTED_SECRET"}) || validated != "resolved-value" || result.Config.Secret.EnvName() != "SELECTED_SECRET" { + t.Fatalf("lookups = %#v, validated = %q, config = %#v", lookedUp, validated, result.Config) + } + history := result.Metadata().History(Keys("secret")) + if len(history) != 3 || history[len(history)-1].Origin.Operation != ChangeNormalization { + t.Fatalf("secret history = %#v", history) + } +} + +func TestValidationCannotReplaceResolvedSecretWithSameName(t *testing.T) { + type config struct { + Secret Secret `yaml:"secret"` + } + _, err := loadWithSecretLookup(Bytes("validator.yml", []byte("secret: VALIDATOR_SECRET\n")), func(string) (string, bool) { + return "CREDENTIAL-CANARY", true + }, Options[config]{Definition: Definition[config]{Validate: func(configuration *config, validation *Validation) { + configuration.Secret = SecretEnv(configuration.Secret.EnvName()) + }}}) + loadError := requireSecretLoadCode(t, err, CodeValidateMutation) + if loadError.Diagnostics()[0].Path.String() != "/secret" || strings.Contains(fmt.Sprintf("%v %#v", err, err), "CREDENTIAL-CANARY") { + t.Fatalf("validator mutation error = %v %#v", err, err) + } +} + +func TestResolvedDefaultsAreRebuiltAndLaterLoadsRelookup(t *testing.T) { + type config struct { + Secret Secret `yaml:"secret"` + } + first, err := loadWithSecretLookup(NoSource(), func(string) (string, bool) { return "first-value", true }, Options[config]{ + Definition: Definition[config]{Defaults: func() config { return config{Secret: SecretEnv("DEFAULT_SECRET")} }}, + }) + if err != nil { + t.Fatal(err) + } + second, err := loadWithSecretLookup(NoSource(), func(string) (string, bool) { return "second-value", true }, Options[config]{ + Definition: Definition[config]{Defaults: func() config { return first.Config }}, + }) + if err != nil { + t.Fatal(err) + } + if first.Config.Secret.Value() != "first-value" || second.Config.Secret.Value() != "second-value" { + t.Fatalf("resolved defaults retained stale state: %q, %q", first.Config.Secret.Value(), second.Config.Secret.Value()) + } + + t.Setenv("CONFIGKIT_RELOOKUP_SECRET", "one") + options := Options[config]{Definition: Definition[config]{Defaults: func() config { return config{Secret: SecretEnv("CONFIGKIT_RELOOKUP_SECRET")} }}} + realFirst, err := Load(NoSource(), options) + if err != nil { + t.Fatal(err) + } + if err := os.Setenv("CONFIGKIT_RELOOKUP_SECRET", "two"); err != nil { + t.Fatal(err) + } + realSecond, err := Load(NoSource(), options) + if err != nil { + t.Fatal(err) + } + if realFirst.Config.Secret.Value() != "one" || realSecond.Config.Secret.Value() != "two" { + t.Fatalf("later loads = %q, %q", realFirst.Config.Secret.Value(), realSecond.Config.Secret.Value()) + } +} + +func TestSecretSafeFormattingJSONProjectionAndHistory(t *testing.T) { + const credential = "CREDENTIAL-CANARY" + secret := Secret{name: "SAFE_SECRET_NAME", value: credential, specified: true, resolved: true} + for _, format := range []string{"%v", "%+v", "%#v", "%s", "%q", "%x", "%d"} { + formatted := fmt.Sprintf(format, secret) + if strings.Contains(formatted, credential) || !strings.Contains(formatted, "SAFE_SECRET_NAME") { + t.Fatalf("format %q produced %q", format, formatted) + } + } + encoded, err := json.Marshal(secret) + if err != nil || string(encoded) != `"SAFE_SECRET_NAME"` || strings.Contains(string(encoded), credential) { + t.Fatalf("MarshalJSON() = %s, %v", encoded, err) + } + + type config struct { + Secret Secret `yaml:"secret"` + } + result, err := loadWithSecretLookup(Bytes("safe-secret.yml", []byte("secret: SAFE_SECRET_NAME\n")), func(string) (string, bool) { + return credential, true + }, Options[config]{}) + if err != nil { + t.Fatal(err) + } + encodedConfig, err := json.Marshal(result.Config) + if err != nil || strings.Contains(string(encodedConfig), credential) || !strings.Contains(string(encodedConfig), "SAFE_SECRET_NAME") { + t.Fatalf("configuration JSON = %s, %v", encodedConfig, err) + } + var effective strings.Builder + if err := result.WriteEffective(&effective); err != nil { + t.Fatal(err) + } + outputs := []string{effective.String(), fmt.Sprintf("%#v", result.Metadata()), fmt.Sprintf("%#v", result.Metadata().History(Keys("secret"))), fmt.Sprintf("%#v", result.Config)} + for _, output := range outputs { + if strings.Contains(output, credential) { + t.Fatalf("inspection leaked credential: %q", output) + } + } + if !strings.Contains(effective.String(), "SAFE_SECRET_NAME") { + t.Fatalf("effective reference name missing: %q", effective.String()) + } +} + +func TestBootstrapExistingEmptyWinsAndReferenceFails(t *testing.T) { + const name = "CONFIGKIT_BLOCKED_EMPTY_SECRET" + t.Setenv(name, "") + directory := t.TempDir() + writeSecretTestFile(t, directory, name, "directory-value") + report, err := LoadSecretsDir(directory) + if err != nil { + t.Fatal(err) + } + entries := report.Entries() + if len(entries) != 1 || entries[0].Status != SecretFileSkippedExisting || !entries[0].ExistingEmpty { + t.Fatalf("bootstrap report = %#v", entries) + } + type config struct { + Secret Secret `yaml:"secret"` + } + _, err = Load[config](Bytes("blocked.yml", []byte("secret: "+name+"\n"))) + requireSecretLoadCode(t, err, CodeSecretEmpty) + if value, present := os.LookupEnv(name); !present || value != "" { + t.Fatalf("environment changed to %q, %v", value, present) + } +} + +func TestDynamicValuesDoNotEstablishSecretReferences(t *testing.T) { + type dynamic struct { + Secret Secret `yaml:"secret"` + } + type config struct { + Dynamic any `yaml:"dynamic"` + } + schema := mustSchemaFor[config](t) + for _, value := range []any{SecretEnv("DYNAMIC_SECRET"), dynamic{Secret: SecretEnv("DYNAMIC_SECRET")}} { + _, issue := projectConfiguration(schema, reflect.ValueOf(config{Dynamic: value}), defaultProjectionLimits()) + if issue == nil || issue.code != CodeSchemaDynamicType { + t.Fatalf("dynamic Secret projection issue = %#v", issue) + } + } + lookups := 0 + result, err := loadWithSecretLookup(Bytes("dynamic.yml", []byte("dynamic: ORDINARY_STRING\n")), func(string) (string, bool) { + lookups++ + return "unexpected", true + }, Options[config]{}) + if err != nil || result.Config.Dynamic != "ORDINARY_STRING" || lookups != 0 { + t.Fatalf("dynamic string result = %#v, lookups = %d, error = %v", result, lookups, err) + } +} + +func TestLoadDoesNotMutateSecretEnvironment(t *testing.T) { + const name = "CONFIGKIT_UNCHANGED_SECRET" + t.Setenv(name, "original") + type config struct { + Secret Secret `yaml:"secret"` + } + result, err := Load[config](Bytes("unchanged.yml", []byte("secret: "+name+"\n"))) + if err != nil || result.Config.Secret.Value() != "original" { + t.Fatalf("Load() = %#v, %v", result, err) + } + if got := os.Getenv(name); got != "original" { + t.Fatalf("environment = %q", got) + } +} + +func requireSecretLoadCode(t *testing.T, err error, code Code) *LoadError { + t.Helper() + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error = %T %v, want *LoadError", err, err) + } + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 0 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v, want first code %q", diagnostics, code) + } + return loadError +} diff --git a/validation.go b/validation.go index 9cc32bb..6857123 100644 --- a/validation.go +++ b/validation.go @@ -96,7 +96,7 @@ func (v *Validation) report(severity Severity, path Path, message string, relate v.state.diagnostics = append(v.state.diagnostics, diagnostic) } -func runValidation[T any](configuration *T, validator func(*T, *Validation), schema *schemaDescriptor, before, input *document.Node) []Diagnostic { +func runValidation[T any](configuration *T, validator func(*T, *Validation), schema *schemaDescriptor, before, input *document.Node, lookups map[string]secretLookupResult) []Diagnostic { state := &validationState{ active: true, errorCode: CodeValidateError, warningCode: CodeValidateWarning, invalidPath: CodeValidateInvalidPath, schema: schema, document: input, @@ -112,6 +112,9 @@ func runValidation[T any](configuration *T, validator func(*T, *Validation), sch for _, delta := range compareDocumentTrees(before, after) { state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(delta.path)) } + for _, path := range changedSecretStates(reflect.ValueOf(*configuration), schema.root, lookups) { + state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(path)) + } return canonicalDiagnostics(state.diagnostics) }