Add typed environment secret references

This commit is contained in:
2026-09-06 21:40:15 +00:00
parent 16f4212770
commit d6e32d8807
14 changed files with 723 additions and 13 deletions

View File

@@ -9,10 +9,10 @@ can overlay one selected fixed profile and check an application schema version.
returns detached metadata and safe diagnostics. returns detached metadata and safe diagnostics.
Applications may perform audited defaulting and normalization through a resolver, Applications may perform audited defaulting and normalization through a resolver,
then report domain errors and warnings through read-only semantic validation. Secret then report domain errors and warnings through read-only semantic validation. Secret
tags and a path predicate drive deterministic redacted effective YAML. Explicit tags and a path predicate drive deterministic redacted effective YAML. Typed secret
secrets-directory bootstrap is available as a separate startup operation. references resolve environment names before validation, and explicit
Overrides, typed secret references, and value explanation are still under secrets-directory bootstrap is available as a separate startup operation. Overrides
development. and value explanation are still under development.
```go ```go
type Config struct { type Config struct {

View File

@@ -76,6 +76,8 @@ func decodeValue(node *document.Node, base reflect.Value, descriptor *typeDescri
switch descriptor.kind { switch descriptor.kind {
case descriptorPointer, descriptorMap, descriptorSlice, descriptorDynamic: case descriptorPointer, descriptorMap, descriptorSlice, descriptorDynamic:
return reflect.Zero(descriptor.typeOf), nil return reflect.Zero(descriptor.typeOf), nil
case descriptorSecret:
return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeSecretRequired, path, node, "required secret reference cannot be null")}
default: default:
return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeNull, path, node, "null is not allowed for this configuration type")} return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeNull, path, node, "null is not allowed for this configuration type")}
} }
@@ -165,6 +167,11 @@ func decodeValue(node *document.Node, base reflect.Value, descriptor *typeDescri
result := reflect.New(descriptor.typeOf).Elem() result := reflect.New(descriptor.typeOf).Elem()
result.SetInt(int64(value)) result.SetInt(int64(value))
return result, nil return result, nil
case descriptorSecret:
if node.Kind != document.String {
return typeMismatch(path, node, "secret environment name")
}
return reflect.ValueOf(SecretEnv(node.Text)), nil
case descriptorText: case descriptorText:
if node.Kind != document.String { if node.Kind != document.String {
return typeMismatch(path, node, "text string") return typeMismatch(path, node, "text string")

View File

@@ -42,7 +42,8 @@ The current lifecycle is:
validate options → parse the selected source → traverse/compose enabled imports validate options → parse the selected source → traverse/compose enabled imports
→ build schema → check the optional application version → apply a selected profile → build schema → check the optional application version → apply a selected profile
→ create and clone defaults → strictly decode → recorded application resolution → create and clone defaults → strictly decode → recorded application resolution
→ semantic validation → classify secrets → snapshot configuration and metadata → resolve typed secret references → semantic validation → classify secrets
→ snapshot configuration and metadata
``` ```
`File`, `Bytes`, and `Reader` each contribute the first base source with deterministic `File`, `Bytes`, and `Reader` each contribute the first base source with deterministic
@@ -144,6 +145,50 @@ configuration values. Applications must still keep source labels, resolution
reasons, and diagnostic messages value-free because ConfigKit cannot infer and reasons, and diagnostic messages value-free because ConfigKit cannot infer and
sanitize secrets embedded in arbitrary application prose. sanitize secrets embedded in arbitrary application prose.
## Typed secret references
Credential-bearing fields use the opaque `Secret` type and store an environment
variable name in YAML, such as `api_key: OPENROUTER_API_KEY`. `SecretEnv(name)`
constructs the same unresolved reference for application defaults or recorded
resolution. Names must match `[A-Za-z_][A-Za-z0-9_]*`; they are public
configuration and remain visible through `EnvName`, canonical projection, history,
and effective output unless the path is separately classified for redaction.
A non-pointer `Secret` is required. An absent, null, or invalid reference, or an
unset or empty environment value, fails with a safe `secret.*` diagnostic at the
configuration path. `*Secret` is optional only when nil; every non-nil reference
must resolve to a nonempty value. Secret references work in typed structs, maps,
pointers, slices, and arrays. A dynamic `any` value does not establish secret
semantics and cannot contain `Secret`; applications must use a typed slot.
ConfigKit resolves only the effective names after recorded application resolution
and before semantic validation. Each unique name is read with `os.LookupEnv` once
per load. Validation can inspect the credential through the deliberately named
`Value()` accessor and distinguish success with `IsResolved()`. Later loads read
the environment again, and cloned defaults are rebuilt from names so a previously
resolved result cannot retain a stale credential. `Load` never writes to the
environment.
Credential bytes are held only in the per-load lookup state and the trusted returned
configuration, and are returned only by `Value`. Secret formatting uses a safe
public-name representation, JSON marshals only the name, and projection, metadata,
histories, diagnostics, effective output,
and resolution snapshots never retain the credential. Validation remains read-only:
in addition to ordinary projected-value comparison, ConfigKit checks the private
resolved state against this load's lookup result. Applications must keep their
validation messages value-free and must not include `Value()` in diagnostics.
For example, a typed pipeline field is declared as:
```go
type PipelineConfig struct {
APIKey configkit.Secret `yaml:"api_key"`
}
```
The executable `ExampleSecretEnv` demonstrates loading and inspecting the safe
reference state without printing its value.
## Explicit secrets-directory bootstrap ## Explicit secrets-directory bootstrap
`LoadSecretsDir(directory)` is a separate, opt-in startup operation. It is never `LoadSecretsDir(directory)` is a separate, opt-in startup operation. It is never
@@ -410,7 +455,8 @@ it keeps no global mutable descriptor cache. A configuration root must be a stru
or string-keyed map. Nested supported shapes are: or string-keyed map. Nested supported shapes are:
- booleans, strings, signed and unsigned integers, and finite floating-point types; - booleans, strings, signed and unsigned integers, and finite floating-point types;
- `time.Duration` and paired `encoding.TextMarshaler`/`TextUnmarshaler` scalars; - `time.Duration`, typed `Secret` references, and paired
`encoding.TextMarshaler`/`TextUnmarshaler` scalars;
- pointers, structs, string-keyed maps, slices, fixed arrays, and `any`; and - pointers, structs, string-keyed maps, slices, fixed arrays, and `any`; and
- recursive type definitions whose eventual application values are acyclic. - recursive type definitions whose eventual application values are acyclic.
@@ -434,7 +480,8 @@ Schema path traversal uses these same bindings. Explicit key segments address
mappings, explicit index segments address sequences, and parsed numeric pointer mappings, explicit index segments address sequences, and parsed numeric pointer
tokens become indices only in a sequence. Sequence indices cannot have leading tokens become indices only in a sequence. Sequence indices cannot have leading
zeroes; fixed-array indices are bounds checked. Dynamic `any` and inline-map entries zeroes; fixed-array indices are bounds checked. Dynamic `any` and inline-map entries
remain schema-valid for application-defined descendants. remain schema-valid for application-defined descendants, but dynamic values cannot
introduce typed secret references.
Descriptors are reused by typed decoding, projection, and cloning so those operations Descriptors are reused by typed decoding, projection, and cloning so those operations
cannot acquire different field bindings. cannot acquire different field bindings.

View File

@@ -5,7 +5,8 @@ below before making changes. The repository is in early implementation: its publ
loader currently covers strict selected sources, explicit path/glob import graphs, loader currently covers strict selected sources, explicit path/glob import graphs,
composition, selected profile overlays, typed defaults, schema-version checks, composition, selected profile overlays, typed defaults, schema-version checks,
recorded application resolution, semantic validation, layered metadata, secret recorded application resolution, semantic validation, layered metadata, secret
classification, and redacted effective rendering. The classification, typed environment references, explicit secrets bootstrap, and
redacted effective rendering. The
[component overview](internal/overview.md) owns the implementation inventory. [component overview](internal/overview.md) owns the implementation inventory.
## Initial orientation ## Initial orientation

View File

@@ -31,14 +31,20 @@ module and package documentation boundary. Its implemented public primitives are
intent, private before/after snapshots, and resolution metadata integration; and intent, private before/after snapshots, and resolution metadata integration; and
- post-resolution schema-aware report collection and validator mutation detection in - post-resolution schema-aware report collection and validator mutation detection in
`validation.go`, completing the public callback lifecycle after composed input; `validation.go`, completing the public callback lifecycle after composed input;
- opaque typed environment references and per-load name resolution in
`secret_reference.go`, including credential-free projection/cloning and private
validator-state auditing;
- explicit bounded startup environment bootstrap and safe partial reporting in
`secrets.go`;
- fixed-point tag/predicate/dependency sensitivity classification in `secret.go`; and - fixed-point tag/predicate/dependency sensitivity classification in `secret.go`; and
- frozen-snapshot redaction and canonical effective output in `render.go`, with the - frozen-snapshot redaction and canonical effective output in `render.go`, with the
only upstream dumper integration isolated in `internal/yaml/write.go`. only upstream dumper integration isolated in `internal/yaml/write.go`.
The [configuration contract](../config.md) owns implemented input, schema, decoding, The [configuration contract](../config.md) owns implemented input, schema, decoding,
loading lifecycle, path imports, composition, profile overlay, version checking, loading lifecycle, path imports, composition, profile overlay, version checking,
recorded resolution, semantic validation, metadata, secret classification, redacted recorded resolution, typed secret references/bootstrap, semantic validation,
effective rendering, and ownership behavior. metadata, secret classification, redacted effective rendering, and ownership
behavior.
The source-extraction inventory records the application foundations assigned to The source-extraction inventory records the application foundations assigned to
future implementation work: future implementation work:

View File

@@ -1233,7 +1233,39 @@ Acceptance:
## Stage 17 — Add environment-backed typed secret references ## Stage 17 — Add environment-backed typed secret references
Status: pending. Status: complete.
Completion evidence (2026-09-06):
- Added the opaque `Secret` and `SecretEnv` API in
[`secret_reference.go`](../../secret_reference.go), with name/value/resolution
accessors, name-only JSON, and credential-free formatting for every `fmt` verb.
The zero value remains an absent reference and no public constructor can inject
resolved credential state.
- Extended the shared descriptor, decode, projection, clone, comparison, resolution,
and validation paths with a first-class secret-reference descriptor. References
project only their public names, cloned defaults are rebuilt unresolved, dynamic
`any` values cannot introduce secret semantics, and existing tag/predicate
classification continues to hide additionally sensitive names or derived data.
- Inserted deterministic per-load environment resolution after recorded application
resolution and before semantic validation. Only effective valid names are read,
each unique success/unset/empty outcome is cached for that load, required and
optional shapes receive path-specific `secret.*` diagnostics, and `Load` performs
no environment writes. Validation compares private reference state with the lookup
cache in addition to the ordinary projected mutation audit.
- Added focused coverage in
[`secret_reference_test.go`](../../secret_reference_test.go) for structs, maps,
pointers and sequences; successful and failed lookup deduplication; absent/null/
empty/invalid/unset cases; resolver-selected names; resolved-value validation;
same-name unresolved validator replacement; later-load environment changes;
stale resolved defaults; dynamic values; bootstrap blocked-empty integration; and
non-disclosure through fmt, JSON, effective projection, history, and diagnostics.
Added executable `ExampleSecretEnv` and updated the
[current secret-reference contract](../config.md#typed-secret-references), component
inventory, developer guide, and README. Verified `go test ./...`,
`go test -race ./...`, `go vet ./...`, `go build ./...`, `go mod tidy -diff`, and
`git diff --check` with Go 1.26.7 on linux/amd64. No deviations were required;
CLI overrides remain later work.
Read: [secret reference contract](initial.md#secret-references-and-explicit-bootstrap), Read: [secret reference contract](initial.md#secret-references-and-explicit-bootstrap),
shared descriptor/projection rules and lifecycle, Stage 16 report boundary. shared descriptor/projection rules and lifecycle, Stage 16 report boundary.

View File

@@ -3,6 +3,7 @@ package configkit_test
import ( import (
"errors" "errors"
"fmt" "fmt"
"os"
"strings" "strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit" configkit "gitea.maximumdirect.net/eric/ConfigKit"
@@ -94,3 +95,23 @@ func ExampleResolution() {
fmt.Println(strings.Join(order, " then "), result.Config.Derived, err) fmt.Println(strings.Join(order, " then "), result.Config.Derived, err)
// Output: resolve then validate 6 <nil> // Output: resolve then validate 6 <nil>
} }
func ExampleSecretEnv() {
const name = "CONFIGKIT_EXAMPLE_API_KEY"
previous, present := os.LookupEnv(name)
_ = os.Setenv(name, "disposable-example-value")
defer func() {
if present {
_ = os.Setenv(name, previous)
} else {
_ = os.Unsetenv(name)
}
}()
type Config struct {
APIKey configkit.Secret `yaml:"api_key"`
}
result, err := configkit.Load[Config](configkit.Bytes("secret.yml", []byte("api_key: "+name+"\n")))
fmt.Println(result.Config.APIKey.EnvName(), result.Config.APIKey.IsResolved(), err)
// Output: CONFIGKIT_EXAMPLE_API_KEY true <nil>
}

12
load.go
View File

@@ -1,6 +1,7 @@
package configkit package configkit
import ( import (
"os"
"reflect" "reflect"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document" "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
@@ -100,6 +101,10 @@ func (r *Result[T]) Diagnostics() []Diagnostic {
// imports, and one selected profile. It accepts at most one Options value and // imports, and one selected profile. It accepts at most one Options value and
// returns no result when any error is reported. // returns no result when any error is reported.
func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) {
return loadWithSecretLookup(source, os.LookupEnv, supplied...)
}
func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied ...Options[T]) (*Result[T], error) {
accumulator := diagnosticAccumulator{} accumulator := diagnosticAccumulator{}
options, diagnostics := selectOptions(source, supplied) options, diagnostics := selectOptions(source, supplied)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
@@ -208,8 +213,13 @@ func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) {
resolutionSnapshots = resolved.snapshots resolutionSnapshots = resolved.snapshots
applyResolutionMetadata(&metadata, resolved) applyResolutionMetadata(&metadata, resolved)
} }
secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, inputDocument, lookup)
accumulator.add(secretResolution.diagnostics...)
if hasErrors(secretResolution.diagnostics) {
return nil, accumulator.loadError()
}
if options.Definition.Validate != nil { if options.Definition.Validate != nil {
validationDiagnostics := runValidation(&configuration, options.Definition.Validate, schema, final, inputDocument) validationDiagnostics := runValidation(&configuration, options.Definition.Validate, schema, final, inputDocument, secretResolution.lookups)
accumulator.add(validationDiagnostics...) accumulator.add(validationDiagnostics...)
if hasErrors(validationDiagnostics) { if hasErrors(validationDiagnostics) {
return nil, accumulator.loadError() return nil, accumulator.loadError()

View File

@@ -222,6 +222,12 @@ func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor
node.Text = time.Duration(value.Int()).String() node.Text = time.Duration(value.Int()).String()
} }
return node, issue return node, issue
case descriptorSecret:
node, issue := o.node(document.String, path, depth)
if issue == nil {
node.Text = value.Interface().(Secret).EnvName()
}
return node, issue
case descriptorText: case descriptorText:
text, issue := marshalCanonicalText(value, path) text, issue := marshalCanonicalText(value, path)
if issue != nil { if issue != nil {
@@ -336,6 +342,13 @@ func (o *valueOperation) clone(value reflect.Value, descriptor *typeDescriptor,
result := reflect.New(value.Type()).Elem() result := reflect.New(value.Type()).Elem()
result.Set(cloned) result.Set(cloned)
return result, nil return result, nil
case descriptorSecret:
result := reflect.New(value.Type()).Elem()
reference := value.Interface().(Secret)
if reference.specified {
result.Set(reflect.ValueOf(SecretEnv(reference.EnvName())))
}
return result, nil
case descriptorText: case descriptorText:
text, issue := marshalCanonicalText(value, path) text, issue := marshalCanonicalText(value, path)
if issue != nil { if issue != nil {
@@ -360,14 +373,39 @@ func (o *valueOperation) clone(value reflect.Value, descriptor *typeDescriptor,
} }
func (o *valueOperation) dynamicDescriptor(current reflect.Type, path Path) (*typeDescriptor, *projectionIssue) { func (o *valueOperation) dynamicDescriptor(current reflect.Type, path Path) (*typeDescriptor, *projectionIssue) {
if current == secretType {
return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration values cannot establish secret-reference semantics"}
}
builder := schemaBuilder{cache: o.schema.types, building: make(map[reflect.Type]bool)} builder := schemaBuilder{cache: o.schema.types, building: make(map[reflect.Type]bool)}
descriptor, issue := builder.describe(current, path) descriptor, issue := builder.describe(current, path)
if issue != nil { if issue != nil {
return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration value has an unsupported type"} return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration value has an unsupported type"}
} }
if descriptorContainsSecret(descriptor, make(map[*typeDescriptor]bool)) {
return nil, &projectionIssue{code: CodeSchemaDynamicType, path: path.clone(), message: "dynamic configuration values cannot establish secret-reference semantics"}
}
return descriptor, nil return descriptor, nil
} }
func descriptorContainsSecret(descriptor *typeDescriptor, visited map[*typeDescriptor]bool) bool {
if descriptor == nil || visited[descriptor] {
return false
}
if descriptor.kind == descriptorSecret {
return true
}
visited[descriptor] = true
if descriptorContainsSecret(descriptor.element, visited) {
return true
}
for _, member := range descriptor.members {
if descriptorContainsSecret(member.descriptor, visited) {
return true
}
}
return false
}
func (o *valueOperation) node(kind document.Kind, path Path, depth int) (*document.Node, *projectionIssue) { func (o *valueOperation) node(kind document.Kind, path Path, depth int) (*document.Node, *projectionIssue) {
if issue := o.checkDepth(path, depth); issue != nil { if issue := o.checkDepth(path, depth); issue != nil {
return nil, issue return nil, issue

View File

@@ -127,6 +127,11 @@ const (
CodeSecretsLimitExceeded Code = "secrets.limit_exceeded" CodeSecretsLimitExceeded Code = "secrets.limit_exceeded"
CodeSecretsInvalidValue Code = "secrets.invalid_value" CodeSecretsInvalidValue Code = "secrets.invalid_value"
CodeSecretsApplyFailed Code = "secrets.apply_failed" CodeSecretsApplyFailed Code = "secrets.apply_failed"
CodeSecretRequired Code = "secret.required"
CodeSecretInvalidName Code = "secret.invalid_name"
CodeSecretUnset Code = "secret.unset"
CodeSecretEmpty Code = "secret.empty"
) )
// Location identifies a position in a configuration source. Zero line and column // Location identifies a position in a configuration source. Zero line and column

View File

@@ -24,6 +24,7 @@ const (
descriptorUnsigned descriptorUnsigned
descriptorFloat descriptorFloat
descriptorDuration descriptorDuration
descriptorSecret
descriptorText descriptorText
) )
@@ -66,6 +67,7 @@ type schemaIssue struct {
var ( var (
durationType = reflect.TypeFor[time.Duration]() durationType = reflect.TypeFor[time.Duration]()
secretType = reflect.TypeFor[Secret]()
textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]() textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]()
textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]() textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]()
) )
@@ -120,6 +122,10 @@ func (b *schemaBuilder) describe(current reflect.Type, path Path) (*typeDescript
descriptor.kind = descriptorDuration descriptor.kind = descriptorDuration
return descriptor, nil return descriptor, nil
} }
if current == secretType {
descriptor.kind = descriptorSecret
return descriptor, nil
}
marshals, unmarshals := textCodecSupport(current) marshals, unmarshals := textCodecSupport(current)
if marshals || unmarshals { if marshals || unmarshals {
if !marshals || !unmarshals { if !marshals || !unmarshals {

211
secret_reference.go Normal file
View File

@@ -0,0 +1,211 @@
package configkit
import (
"encoding/json"
"fmt"
"io"
"reflect"
"slices"
"strconv"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
// Secret is an opaque reference to an environment variable. Its name is public
// configuration; its resolved value is available only through Value.
type Secret struct {
name string
value string
specified bool
resolved bool
}
// SecretEnv constructs an unresolved environment-backed secret reference.
// Invalid names are reported if the reference survives a configuration load.
func SecretEnv(name string) Secret {
return Secret{name: name, specified: true}
}
// EnvName returns the public environment-variable name.
func (s Secret) EnvName() string {
return s.name
}
// Value returns the resolved credential, or an empty string when unresolved.
func (s Secret) Value() string {
if !s.resolved {
return ""
}
return s.value
}
// IsResolved reports whether this reference resolved to a nonempty value during
// its configuration load.
func (s Secret) IsResolved() bool {
return s.resolved
}
// Format renders only the public reference name for every formatting verb.
func (s Secret) Format(state fmt.State, _ rune) {
_, _ = io.WriteString(state, "configkit.SecretEnv("+strconv.Quote(s.name)+")")
}
// MarshalJSON emits only the public environment-variable name.
func (s Secret) MarshalJSON() ([]byte, error) {
return json.Marshal(s.name)
}
type secretLookup func(string) (string, bool)
type secretLookupResult struct {
value string
present bool
}
type secretResolutionResult struct {
lookups map[string]secretLookupResult
diagnostics []Diagnostic
}
func resolveSecretReferences(root reflect.Value, schema *schemaDescriptor, input *document.Node, lookup secretLookup) secretResolutionResult {
result := secretResolutionResult{
lookups: make(map[string]secretLookupResult),
}
if schema == nil || schema.root == nil || !root.IsValid() || root.Type() != schema.root.typeOf {
result.diagnostics = []Diagnostic{{Code: CodeSecretRequired, Severity: SeverityError, Message: "configuration value does not match its secret-reference schema"}}
return result
}
if lookup == nil {
lookup = func(string) (string, bool) { return "", false }
}
resolveSecretValue(root, schema.root, Path{}, false, input, lookup, &result)
result.diagnostics = canonicalDiagnostics(result.diagnostics)
return result
}
func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, path Path, optional bool, input *document.Node, lookup secretLookup, result *secretResolutionResult) {
if descriptor.kind == descriptorPointer {
if value.IsNil() {
return
}
resolveSecretValue(value.Elem(), descriptor.element, path, descriptor.element.kind == descriptorSecret, input, lookup, result)
return
}
switch descriptor.kind {
case descriptorSecret:
reference := value.Interface().(Secret)
name := reference.EnvName()
if name == "" {
code := CodeSecretRequired
message := "required secret reference is absent"
if optional || reference.specified {
code = CodeSecretInvalidName
message = "secret environment name is empty"
}
result.diagnostics = append(result.diagnostics, secretDiagnostic(code, path, message, input))
return
}
if !validEnvironmentName(name) {
result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretInvalidName, path, "secret environment name is invalid", input))
return
}
found, exists := result.lookups[name]
if !exists {
found.value, found.present = lookup(name)
result.lookups[name] = found
}
if !found.present {
result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretUnset, path, "secret environment variable is not set", input))
return
}
if found.value == "" {
result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretEmpty, path, "secret environment variable is empty", input))
return
}
value.Set(reflect.ValueOf(Secret{name: name, value: found.value, specified: true, resolved: true}))
case descriptorStruct:
for _, member := range descriptor.members {
memberPath := path
if !member.inline {
memberPath = path.Key(member.name)
}
resolveSecretValue(value.Field(member.index), member.descriptor, memberPath, false, input, lookup, result)
}
case descriptorMap:
if value.IsNil() {
return
}
keys := value.MapKeys()
slices.SortFunc(keys, func(left, right reflect.Value) int { return cmpString(left.String(), right.String()) })
for _, key := range keys {
entry := reflect.New(value.Type().Elem()).Elem()
entry.Set(value.MapIndex(key))
resolveSecretValue(entry, descriptor.element, path.Key(key.String()), false, input, lookup, result)
value.SetMapIndex(key, entry)
}
case descriptorSlice, descriptorArray:
if descriptor.kind == descriptorSlice && value.IsNil() {
return
}
for index := 0; index < value.Len(); index++ {
resolveSecretValue(value.Index(index), descriptor.element, path.Index(index), false, input, lookup, result)
}
}
}
func secretDiagnostic(code Code, path Path, message string, input *document.Node) Diagnostic {
diagnostic := Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message}
if location, ok := nearestDocumentLocation(input, path); ok {
diagnostic.Primary = documentLocationPointer(location)
}
return diagnostic
}
func changedSecretStates(root reflect.Value, descriptor *typeDescriptor, lookups map[string]secretLookupResult) []Path {
var changed []Path
collectChangedSecretStates(root, descriptor, Path{}, lookups, &changed)
slices.SortFunc(changed, func(left, right Path) int { return cmpString(left.String(), right.String()) })
return changed
}
func collectChangedSecretStates(value reflect.Value, descriptor *typeDescriptor, path Path, lookups map[string]secretLookupResult, changed *[]Path) {
if descriptor.kind == descriptorPointer {
if !value.IsNil() {
collectChangedSecretStates(value.Elem(), descriptor.element, path, lookups, changed)
}
return
}
switch descriptor.kind {
case descriptorSecret:
reference := value.Interface().(Secret)
expected, ok := lookups[reference.EnvName()]
if ok && (!expected.present || expected.value == "" || !reference.resolved || reference.value != expected.value) {
*changed = append(*changed, path.clone())
}
case descriptorStruct:
for _, member := range descriptor.members {
memberPath := path
if !member.inline {
memberPath = path.Key(member.name)
}
collectChangedSecretStates(value.Field(member.index), member.descriptor, memberPath, lookups, changed)
}
case descriptorMap:
if value.IsNil() {
return
}
keys := value.MapKeys()
slices.SortFunc(keys, func(left, right reflect.Value) int { return cmpString(left.String(), right.String()) })
for _, key := range keys {
collectChangedSecretStates(value.MapIndex(key), descriptor.element, path.Key(key.String()), lookups, changed)
}
case descriptorSlice, descriptorArray:
if descriptor.kind == descriptorSlice && value.IsNil() {
return
}
for index := 0; index < value.Len(); index++ {
collectChangedSecretStates(value.Index(index), descriptor.element, path.Index(index), lookups, changed)
}
}
}

323
secret_reference_test.go Normal file
View File

@@ -0,0 +1,323 @@
package configkit
import (
"encoding/json"
"errors"
"fmt"
"os"
"reflect"
"strings"
"testing"
)
type secretReferenceConfig struct {
Required Secret `yaml:"required"`
Optional *Secret `yaml:"optional"`
Map map[string]Secret `yaml:"map"`
Pointers map[string]*Secret `yaml:"pointers"`
Sequence []Secret `yaml:"sequence"`
}
func TestSecretReferencesResolveTypedShapesOncePerName(t *testing.T) {
input := `required: SHARED_SECRET
optional: OPTIONAL_SECRET
map:
first: SHARED_SECRET
pointers:
nil: null
value: POINTER_SECRET
sequence: [SHARED_SECRET, SEQUENCE_SECRET]
`
values := map[string]string{
"SHARED_SECRET": "shared-value",
"OPTIONAL_SECRET": "optional-value",
"POINTER_SECRET": "pointer-value",
"SEQUENCE_SECRET": "sequence-value",
}
counts := map[string]int{}
lookup := func(name string) (string, bool) {
counts[name]++
value, present := values[name]
return value, present
}
validationCalled := false
result, err := loadWithSecretLookup(Bytes("secrets.yml", []byte(input)), lookup, Options[secretReferenceConfig]{
Definition: Definition[secretReferenceConfig]{Validate: func(configuration *secretReferenceConfig, validation *Validation) {
validationCalled = true
if configuration.Required.Value() != "shared-value" || !configuration.Required.IsResolved() ||
configuration.Optional.Value() != "optional-value" || configuration.Pointers["value"].Value() != "pointer-value" ||
configuration.Sequence[1].Value() != "sequence-value" {
validation.Error(Keys("required"), "references were not resolved before validation")
}
}},
})
if err != nil {
t.Fatalf("loadWithSecretLookup() error = %v", err)
}
if !validationCalled || result.Config.Optional == nil || result.Config.Pointers["nil"] != nil {
t.Fatalf("resolved config = %#v", result.Config)
}
if result.Config.Required.EnvName() != "SHARED_SECRET" || result.Config.Map["first"].Value() != "shared-value" {
t.Fatalf("reference names/values = %#v", result.Config)
}
if !reflect.DeepEqual(counts, map[string]int{"SHARED_SECRET": 1, "OPTIONAL_SECRET": 1, "POINTER_SECRET": 1, "SEQUENCE_SECRET": 1}) {
t.Fatalf("lookup counts = %#v", counts)
}
}
func TestSecretReferenceErrorsAreSafeAndPathSpecific(t *testing.T) {
for _, test := range []struct {
name string
input string
lookup secretLookup
code Code
}{
{name: "absent", input: "{}\n", code: CodeSecretRequired},
{name: "null", input: "required: null\n", code: CodeSecretRequired},
{name: "empty name", input: "required: ''\n", code: CodeSecretInvalidName},
{name: "invalid name", input: "required: INVALID-NAME\n", code: CodeSecretInvalidName},
{name: "unset", input: "required: MISSING_SECRET\n", code: CodeSecretUnset},
{name: "empty environment", input: "required: EMPTY_SECRET\n", lookup: func(string) (string, bool) { return "", true }, code: CodeSecretEmpty},
} {
t.Run(test.name, func(t *testing.T) {
lookup := test.lookup
if lookup == nil {
lookup = func(string) (string, bool) { return "", false }
}
_, err := loadWithSecretLookup(Bytes("secret-errors.yml", []byte(test.input)), lookup, Options[secretReferenceConfig]{})
loadError := requireSecretLoadCode(t, err, test.code)
diagnostic := loadError.Diagnostics()[0]
if diagnostic.Path.String() != "/required" || strings.Contains(fmt.Sprintf("%v %#v", err, err), "CREDENTIAL-CANARY") {
t.Fatalf("diagnostic/error = %#v / %v", diagnostic, err)
}
})
}
t.Run("non-nil optional is required to resolve", func(t *testing.T) {
type config struct {
Optional *Secret `yaml:"optional"`
}
_, err := loadWithSecretLookup(Bytes("optional.yml", []byte("optional: ''\n")), func(string) (string, bool) { return "", false }, Options[config]{})
requireSecretLoadCode(t, err, CodeSecretInvalidName)
result, err := loadWithSecretLookup(Bytes("optional.yml", []byte("optional: null\n")), func(string) (string, bool) { return "", false }, Options[config]{})
if err != nil || result.Config.Optional != nil {
t.Fatalf("nil optional result = %#v, %v", result, err)
}
})
t.Run("failed lookup is cached", func(t *testing.T) {
type config struct {
Secrets []Secret `yaml:"secrets"`
}
lookups := 0
_, err := loadWithSecretLookup(Bytes("missing-twice.yml", []byte("secrets: [MISSING_SHARED, MISSING_SHARED]\n")), func(string) (string, bool) {
lookups++
return "", false
}, Options[config]{})
loadError := requireSecretLoadCode(t, err, CodeSecretUnset)
if lookups != 1 || len(loadError.Diagnostics()) != 2 {
t.Fatalf("lookups = %d, diagnostics = %#v", lookups, loadError.Diagnostics())
}
})
}
func TestResolutionSelectsEffectiveSecretNameBeforeLookup(t *testing.T) {
type config struct {
Name string `yaml:"name"`
Secret Secret `yaml:"secret"`
}
lookedUp := []string{}
validated := ""
result, err := loadWithSecretLookup(Bytes("resolved-secret.yml", []byte("name: SELECTED_SECRET\nsecret: OLD_SECRET\n")), func(name string) (string, bool) {
lookedUp = append(lookedUp, name)
return "resolved-value", true
}, Options[config]{Definition: Definition[config]{
Resolve: func(configuration *config, resolution *Resolution) error {
Normalize(resolution, Keys("secret"), &configuration.Secret, SecretEnv(configuration.Name), "selected named credential", Keys("name"))
return nil
},
Validate: func(configuration *config, validation *Validation) {
validated = configuration.Secret.Value()
},
}})
if err != nil {
t.Fatalf("loadWithSecretLookup() error = %v", err)
}
if !reflect.DeepEqual(lookedUp, []string{"SELECTED_SECRET"}) || validated != "resolved-value" || result.Config.Secret.EnvName() != "SELECTED_SECRET" {
t.Fatalf("lookups = %#v, validated = %q, config = %#v", lookedUp, validated, result.Config)
}
history := result.Metadata().History(Keys("secret"))
if len(history) != 3 || history[len(history)-1].Origin.Operation != ChangeNormalization {
t.Fatalf("secret history = %#v", history)
}
}
func TestValidationCannotReplaceResolvedSecretWithSameName(t *testing.T) {
type config struct {
Secret Secret `yaml:"secret"`
}
_, err := loadWithSecretLookup(Bytes("validator.yml", []byte("secret: VALIDATOR_SECRET\n")), func(string) (string, bool) {
return "CREDENTIAL-CANARY", true
}, Options[config]{Definition: Definition[config]{Validate: func(configuration *config, validation *Validation) {
configuration.Secret = SecretEnv(configuration.Secret.EnvName())
}}})
loadError := requireSecretLoadCode(t, err, CodeValidateMutation)
if loadError.Diagnostics()[0].Path.String() != "/secret" || strings.Contains(fmt.Sprintf("%v %#v", err, err), "CREDENTIAL-CANARY") {
t.Fatalf("validator mutation error = %v %#v", err, err)
}
}
func TestResolvedDefaultsAreRebuiltAndLaterLoadsRelookup(t *testing.T) {
type config struct {
Secret Secret `yaml:"secret"`
}
first, err := loadWithSecretLookup(NoSource(), func(string) (string, bool) { return "first-value", true }, Options[config]{
Definition: Definition[config]{Defaults: func() config { return config{Secret: SecretEnv("DEFAULT_SECRET")} }},
})
if err != nil {
t.Fatal(err)
}
second, err := loadWithSecretLookup(NoSource(), func(string) (string, bool) { return "second-value", true }, Options[config]{
Definition: Definition[config]{Defaults: func() config { return first.Config }},
})
if err != nil {
t.Fatal(err)
}
if first.Config.Secret.Value() != "first-value" || second.Config.Secret.Value() != "second-value" {
t.Fatalf("resolved defaults retained stale state: %q, %q", first.Config.Secret.Value(), second.Config.Secret.Value())
}
t.Setenv("CONFIGKIT_RELOOKUP_SECRET", "one")
options := Options[config]{Definition: Definition[config]{Defaults: func() config { return config{Secret: SecretEnv("CONFIGKIT_RELOOKUP_SECRET")} }}}
realFirst, err := Load(NoSource(), options)
if err != nil {
t.Fatal(err)
}
if err := os.Setenv("CONFIGKIT_RELOOKUP_SECRET", "two"); err != nil {
t.Fatal(err)
}
realSecond, err := Load(NoSource(), options)
if err != nil {
t.Fatal(err)
}
if realFirst.Config.Secret.Value() != "one" || realSecond.Config.Secret.Value() != "two" {
t.Fatalf("later loads = %q, %q", realFirst.Config.Secret.Value(), realSecond.Config.Secret.Value())
}
}
func TestSecretSafeFormattingJSONProjectionAndHistory(t *testing.T) {
const credential = "CREDENTIAL-CANARY"
secret := Secret{name: "SAFE_SECRET_NAME", value: credential, specified: true, resolved: true}
for _, format := range []string{"%v", "%+v", "%#v", "%s", "%q", "%x", "%d"} {
formatted := fmt.Sprintf(format, secret)
if strings.Contains(formatted, credential) || !strings.Contains(formatted, "SAFE_SECRET_NAME") {
t.Fatalf("format %q produced %q", format, formatted)
}
}
encoded, err := json.Marshal(secret)
if err != nil || string(encoded) != `"SAFE_SECRET_NAME"` || strings.Contains(string(encoded), credential) {
t.Fatalf("MarshalJSON() = %s, %v", encoded, err)
}
type config struct {
Secret Secret `yaml:"secret"`
}
result, err := loadWithSecretLookup(Bytes("safe-secret.yml", []byte("secret: SAFE_SECRET_NAME\n")), func(string) (string, bool) {
return credential, true
}, Options[config]{})
if err != nil {
t.Fatal(err)
}
encodedConfig, err := json.Marshal(result.Config)
if err != nil || strings.Contains(string(encodedConfig), credential) || !strings.Contains(string(encodedConfig), "SAFE_SECRET_NAME") {
t.Fatalf("configuration JSON = %s, %v", encodedConfig, err)
}
var effective strings.Builder
if err := result.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
outputs := []string{effective.String(), fmt.Sprintf("%#v", result.Metadata()), fmt.Sprintf("%#v", result.Metadata().History(Keys("secret"))), fmt.Sprintf("%#v", result.Config)}
for _, output := range outputs {
if strings.Contains(output, credential) {
t.Fatalf("inspection leaked credential: %q", output)
}
}
if !strings.Contains(effective.String(), "SAFE_SECRET_NAME") {
t.Fatalf("effective reference name missing: %q", effective.String())
}
}
func TestBootstrapExistingEmptyWinsAndReferenceFails(t *testing.T) {
const name = "CONFIGKIT_BLOCKED_EMPTY_SECRET"
t.Setenv(name, "")
directory := t.TempDir()
writeSecretTestFile(t, directory, name, "directory-value")
report, err := LoadSecretsDir(directory)
if err != nil {
t.Fatal(err)
}
entries := report.Entries()
if len(entries) != 1 || entries[0].Status != SecretFileSkippedExisting || !entries[0].ExistingEmpty {
t.Fatalf("bootstrap report = %#v", entries)
}
type config struct {
Secret Secret `yaml:"secret"`
}
_, err = Load[config](Bytes("blocked.yml", []byte("secret: "+name+"\n")))
requireSecretLoadCode(t, err, CodeSecretEmpty)
if value, present := os.LookupEnv(name); !present || value != "" {
t.Fatalf("environment changed to %q, %v", value, present)
}
}
func TestDynamicValuesDoNotEstablishSecretReferences(t *testing.T) {
type dynamic struct {
Secret Secret `yaml:"secret"`
}
type config struct {
Dynamic any `yaml:"dynamic"`
}
schema := mustSchemaFor[config](t)
for _, value := range []any{SecretEnv("DYNAMIC_SECRET"), dynamic{Secret: SecretEnv("DYNAMIC_SECRET")}} {
_, issue := projectConfiguration(schema, reflect.ValueOf(config{Dynamic: value}), defaultProjectionLimits())
if issue == nil || issue.code != CodeSchemaDynamicType {
t.Fatalf("dynamic Secret projection issue = %#v", issue)
}
}
lookups := 0
result, err := loadWithSecretLookup(Bytes("dynamic.yml", []byte("dynamic: ORDINARY_STRING\n")), func(string) (string, bool) {
lookups++
return "unexpected", true
}, Options[config]{})
if err != nil || result.Config.Dynamic != "ORDINARY_STRING" || lookups != 0 {
t.Fatalf("dynamic string result = %#v, lookups = %d, error = %v", result, lookups, err)
}
}
func TestLoadDoesNotMutateSecretEnvironment(t *testing.T) {
const name = "CONFIGKIT_UNCHANGED_SECRET"
t.Setenv(name, "original")
type config struct {
Secret Secret `yaml:"secret"`
}
result, err := Load[config](Bytes("unchanged.yml", []byte("secret: "+name+"\n")))
if err != nil || result.Config.Secret.Value() != "original" {
t.Fatalf("Load() = %#v, %v", result, err)
}
if got := os.Getenv(name); got != "original" {
t.Fatalf("environment = %q", got)
}
}
func requireSecretLoadCode(t *testing.T, err error, code Code) *LoadError {
t.Helper()
var loadError *LoadError
if !errors.As(err, &loadError) {
t.Fatalf("error = %T %v, want *LoadError", err, err)
}
diagnostics := loadError.Diagnostics()
if len(diagnostics) == 0 || diagnostics[0].Code != code {
t.Fatalf("diagnostics = %#v, want first code %q", diagnostics, code)
}
return loadError
}

View File

@@ -96,7 +96,7 @@ func (v *Validation) report(severity Severity, path Path, message string, relate
v.state.diagnostics = append(v.state.diagnostics, diagnostic) v.state.diagnostics = append(v.state.diagnostics, diagnostic)
} }
func runValidation[T any](configuration *T, validator func(*T, *Validation), schema *schemaDescriptor, before, input *document.Node) []Diagnostic { func runValidation[T any](configuration *T, validator func(*T, *Validation), schema *schemaDescriptor, before, input *document.Node, lookups map[string]secretLookupResult) []Diagnostic {
state := &validationState{ state := &validationState{
active: true, errorCode: CodeValidateError, warningCode: CodeValidateWarning, active: true, errorCode: CodeValidateError, warningCode: CodeValidateWarning,
invalidPath: CodeValidateInvalidPath, schema: schema, document: input, invalidPath: CodeValidateInvalidPath, schema: schema, document: input,
@@ -112,6 +112,9 @@ func runValidation[T any](configuration *T, validator func(*T, *Validation), sch
for _, delta := range compareDocumentTrees(before, after) { for _, delta := range compareDocumentTrees(before, after) {
state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(delta.path)) state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(delta.path))
} }
for _, path := range changedSecretStates(reflect.ValueOf(*configuration), schema.root, lookups) {
state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(path))
}
return canonicalDiagnostics(state.diagnostics) return canonicalDiagnostics(state.diagnostics)
} }