Document results of a code maintainability audit for an upcoming implementation sprint

callback_ownership_test.go

@@ -0,0 +1,52 @@
+package configkit
+
+import "testing"
+
+func TestCallbackAssignmentsAreDetachedAcrossLoads(t *testing.T) {
+	type object struct{ Value string }
+	type cfg struct {
+		Map     map[string][]*object
+		Pointer *object
+		Secret  Secret
+	}
+	for _, validatorAssignment := range []bool{false, true} {
+		name := "resolver"
+		if validatorAssignment {
+			name = "validator"
+		}
+		t.Run(name, func(t *testing.T) {
+			external := &object{Value: "original"}
+			shared := map[string][]*object{"key": {external}}
+			definition := Definition[cfg]{Resolve: func(c *cfg, r *Resolution) error {
+				Normalize(r, Keys("map"), &c.Map, shared, "populate")
+				c.Pointer = external
+				r.Record(Keys("pointer"), ChangeNormalization, "populate")
+				c.Secret = SecretEnv("CONFIGKIT_CALLBACK_SECRET")
+				r.Record(Keys("secret"), ChangeNormalization, "reference")
+				return nil
+			}}
+			if validatorAssignment {
+				definition.Validate = func(c *cfg, v *Validation) { c.Map = shared; c.Pointer = external }
+			}
+			calls := 0
+			lookup := func(string) (string, bool) { calls++; return "credential", true }
+			first, err := loadWithSecretLookup(NoSource(), lookup, Options[cfg]{Definition: definition})
+			if err != nil {
+				t.Fatal(err)
+			}
+			second, err := loadWithSecretLookup(NoSource(), lookup, Options[cfg]{Definition: definition})
+			if err != nil {
+				t.Fatal(err)
+			}
+			first.Config.Map["key"][0].Value = "changed"
+			first.Config.Pointer.Value = "changed pointer"
+			external.Value = "external change"
+			if second.Config.Map["key"][0].Value != "original" || second.Config.Pointer.Value != "original" {
+				t.Fatal("published configuration shares callback state")
+			}
+			if second.Config.Secret.Value() != "credential" || !second.Config.Secret.IsResolved() || calls != 2 {
+				t.Fatal("publication changed secret state or repeated environment lookup")
+			}
+		})
+	}
+}

docs/config.md

@@ -34,7 +34,10 @@ root `T` must satisfy the application schema contract below. With no options it
 starts from the Go zero value. A `Definition[T].Defaults` factory, when supplied,
 runs exactly once after successful source parsing, schema construction, optional
 schema-version checking, and selected-profile loading. Its result is detached before
-input is applied.
+input is applied. Resolver-provided mutable values are detached before secret lookup
+and validation, and a final detached copy prevents equal-valued aliases introduced
+by validators from escaping into the returned configuration. The publication copy
+preserves already validated secret state without rereading the environment.
 
 The current lifecycle is:
 
@@ -84,7 +87,10 @@ successful result. Callback panics remain programming failures.
 
 ### Layered metadata
 
-`Result.Metadata` describes every final canonical path. Exact surviving input paths
+`Result.Metadata` describes every final canonical path. Queries retain explicit
+key/index distinctions using actual effective and historical container shapes;
+only parsed numeric tokens are resolved contextually. A descendant that never
+existed is not historical merely because an atomic ancestor was replaced. Exact surviving input paths
 are explicit and have a base or profile origin; omitted paths have a default origin.
 An explicit parent mapping does not make omitted children explicit. Null state and
 available source coordinates are retained. `History` records the Go-zero or application
@@ -113,7 +119,9 @@ selected profile, then synthetic CLI records in supplied order.
 ### Secret classification and effective rendering
 
 Secret classification combines `configkit:"secret"` field tags with the optional
-pure `Definition.Secret(Path) bool` predicate. A tagged or predicate-classified
+pure `Definition.Secret(Path) bool` predicate. Tags on concrete structs inside
+supported dynamic values are retained in typed projections, including superseded
+defaults and pre-resolution values. A tagged or predicate-classified
 container makes all descendants sensitive. ConfigKit evaluates final and historical
 paths, so replacing or deleting a definition does not erase its classification. The
 predicate can add sensitivity but cannot declassify a tagged path.
@@ -157,11 +165,20 @@ parser values. `Present=false` with empty YAML means absence or deletion; explic
 null is present and renders as `null`. The history starts with the canonical typed
 application-default or Go-zero baseline, then includes only relevant base sources in
 traversal order, the selected profile, supplied CLI records, and audited net
-resolution operations. Explicit equal writes remain visible.
+resolution operations. Overlapping parent/child resolution records are rendered
+as one net subtree event with combined reasons and dependencies, rather than
+pretending that the recorded assignments ran in lexical order. Explicit equal
+writes remain visible. When input replay
+differs from the final typed input, a `ChangeDecoding` event records the transition
+before domain resolution. Its origin retains the decoded path's source/default
+attribution; its reason identifies static-default retention and canonical decoding.
+This built-in inspection operation is not accepted by `Resolution.Record`.
 
 Base, profile, and CLI history preserves the policy-validated input representation,
 including a superseded value that could not have decoded into the final Go field.
-Baseline, effective, and resolution values use canonical typed projection. Mapping
+Baseline, effective, and resolution values use canonical typed projection. A
+decoding event bridges replayed input and canonical typed values, so intermediate
+null/scalar replacements cannot incorrectly imply permanent default removal. Mapping
 queries reconstruct each source or override contribution, while sequences remain
 atomic: a descendant history identifies its presence before and after the containing
 sequence replacement without implying element-wise merging. Removed paths remain
@@ -226,7 +243,9 @@ public-name representation, JSON marshals only the name, and projection, metadat
 histories, diagnostics, effective output,
 and resolution snapshots never retain the credential. Validation remains read-only:
 in addition to ordinary projected-value comparison, ConfigKit checks the private
-resolved state against this load's lookup result. Applications must keep their
+resolved state against this load's lookup result. Validator mutation checks also
+reject changes to projected field sensitivity, even when a replacement dynamic
+concrete type preserves the same textual values. Applications must keep their
 validation messages value-free and must not include `Value()` in diagnostics.
 
 For example, a typed pipeline field is declared as:
@@ -488,7 +507,9 @@ source may supply `FilesystemContext.Root` to widen that root but may not supply
 `BaseDir`. Whenever byte and reader sources supply filesystem context, they must
 provide both `BaseDir` and `Root` as absolute existing directories; their diagnostic
 labels, including URI-looking labels, never imply filesystem paths. The base and
-selected file must be inside the canonical root. ConfigKit canonicalizes symlinks,
+selected file must be inside the canonical root. Nonregular targets are rejected before opening to avoid blocking on FIFOs;
+post-open regular-file checks remain in place under the stable-input assumption.
+ConfigKit canonicalizes symlinks,
 checks containment with relative paths, and opens the canonical root-relative target
 through a library-owned root handle. Contained relative or absolute symlink targets
 are accepted; escapes are rejected. Canonical paths define identity, so hard links at
@@ -570,7 +591,9 @@ Unexported fields and `yaml:"-"` are ignored. `omitempty` is accepted but does n
 change binding or projection. An explicit `yaml:",inline"` flattens a struct
 or pointer-to-struct, or designates at most one string-keyed inline map for otherwise
 unmatched names. Ambiguous names, named inline fields, multiple inline maps, and
-unsupported or malformed tag options are schema errors. Embedded fields are ordinary
+unsupported or malformed tag options are schema errors. Inline pointers must
+terminate in a struct; pointer-to-map and recursive pointer-only inline declarations
+are rejected before the defaults factory runs. Embedded fields are ordinary
 named fields unless explicitly inlined. `configkit:"secret"` is the only current
 ConfigKit field tag and marks its bound field or container for secret classification
 and redacted inspection.

docs/release.md

@@ -22,8 +22,9 @@ and macOS; cross-compilation is not native execution evidence.
 
 ## 1. Prepare access and tools
 
-The release maintainer owns this procedure. Work in a dedicated clone with Git, Bash,
-and two supported Go toolchains available. Configure private-host Git and Go module
+The release maintainer owns this procedure. Work in a dedicated clone with Git, Bash 3.2 or newer,
+and two supported Go toolchains available. The checker supports the stock macOS
+Bash; its filename enumeration is null-delimited, including names with whitespace. Configure private-host Git and Go module
 access through the normal credential manager before starting. Keep tokens, passwords,
 SSH material, and environment dumps outside the repository and command transcript.
 Verify access without displaying credentials:

docs/roadmap/remediation.md

@@ -0,0 +1,355 @@
+# Pre-release Maintainability Remediation
+
+## Status and scope
+
+Status: planned; no stages have been implemented under this roadmap.
+Audit baseline: `9b147bfd96aec25e0125dcae66e136e56dc7a670`.
+
+This roadmap owns the agreed maintainability cleanup before v0.1.0. It replaces
+the completed implementation-defect remediation record, whose findings, fixes,
+and validation evidence remain available in Git history. It does not reopen
+those defects or replace the original [implementation plan](implementation.md).
+
+The audit found worthwhile consolidation and ownership improvements, rather than
+a need to redesign ConfigKit. The offline default suite and vet passed at the
+audit baseline. That evidence is not validation of future changes or a release
+candidate.
+
+Read the [development guide](../development.md),
+[architecture policy](../policy/architecture.md),
+[testing policy](../policy/testing.md), and
+[documentation policy](../policy/documentation.md) before implementation.
+The [configuration contract](../config.md) owns current semantics; the
+[component overview](../internal/overview.md) owns implementation inventory.
+Release preparation remains governed by the [release policy](../policy/release.md)
+and [maintainer procedure](../release.md).
+
+## Execution rules and invariants
+
+Implement the numbered stages in order. Each stage is one reviewable unit of
+work: inspect its current owners and tests, make the change, validate it, update
+its status and evidence here, and commit the nonempty unit with a short plain
+English message. Do not create empty commits. Record any discovered behavioral
+defect separately with a reproduction rather than silently changing semantics
+under a refactoring stage.
+
+Keep the root public package and the existing `internal/document` and
+`internal/yaml` packages. Do not introduce dependencies, public API changes,
+provider interfaces, configurable merge policies, or a generic visitor framework.
+Private file splits do not require new packages or ADRs. Parser-specific types
+must remain behind the existing YAML adapter boundary.
+
+Preserve the implemented contracts, particularly:
+
+- Strict composition remains distinct from intentional profile/CLI overlay.
+- Input precedence, final-only typed decoding, retained defaults, canonical
+  decoding explanation events, and recorded domain resolution remain unchanged.
+- Source identity, containment, optional-import rules, limits, diagnostic codes,
+  source coordinates, ordering, and early callback termination remain intact.
+- Semantic paths retain explicit key/index distinctions and historical shapes.
+- Returned values and inspection snapshots remain detached; secret values remain
+  absent from history, rendering, and library-generated diagnostics.
+- Resolution runs before secret lookup and validation; ordinary loading never
+  bootstraps environment variables implicitly.
+
+Use focused existing behavioral tests during each stage and the baseline checks
+from the development guide before committing production changes. Add tests only
+for meaningful gaps exposed by extraction, not to enforce file locations or
+private call choreography. Update current owning documentation alongside each
+move. Planned filenames below are private implementation destinations, not new
+consumer contracts.
+
+## Stage 1 — Remove unused overlay value snapshots
+
+Status: pending.
+
+Evidence: `overlayRecord` in [overlay.go](../../overlay.go) retains `before` and
+`after` trees; profile and CLI processing construct them and
+`cloneOverlayRecords` clones them. Production consumers use the record's change
+and displaced origins, while explanation uses `historyApplication` instead.
+
+Work:
+
+1. Reconfirm all field reads before removal. Remove only the unused overlay
+   snapshot fields and their construction/cloning in `overlay.go` and
+   [override.go](../../override.go).
+2. Preserve `change`, `displaced`, input applications, and their ownership rules.
+   Do not remove resolution snapshots or retained decoded trees: those have
+   separate consumers and budget semantics.
+3. Remove locals and imports made unnecessary by the deletion. Do not replace
+   the removed state with a new cache or history representation.
+
+Acceptance: profile/CLI provenance, explanation replay, composition independence,
+and redaction tests pass unchanged in observable behavior. Source inspection
+confirms no unused overlay snapshot construction remains. An allocation benchmark
+is not required to justify removing provably unread clones.
+
+## Stage 2 — Centralize parser issue translation
+
+Status: pending.
+
+Evidence: [parse.go](../../parse.go) translates adapter issues inline, while
+`importParseDiagnostics` in [imports.go](../../imports.go) repeats the translation
+for imports and profiles.
+
+Work:
+
+1. Define one private `parserDiagnostics` helper in `parse.go` that maps adapter
+   issues to detached public diagnostics, including related locations.
+2. Route selected-source, import, and profile parsing through it. Remove the
+   import-specific translator and keep upstream parser errors private.
+3. Return canonically ordered diagnostics from the shared translation boundary.
+   Preserve existing codes, messages, primary/related coordinates, and source IDs.
+   Keep per-source parsing budgets and graph accounting at their current owners.
+
+Acceptance: malformed selected/imported/profile input retains safe, correctly
+attributed diagnostics. Use existing parser and import diagnostic tests; add a
+small cross-entry regression only if a source path lacks equivalent coverage.
+
+## Stage 3 — Share bounded regular-file read mechanics
+
+Status: pending.
+
+Evidence: `readSource`, `importTraversal.readImport`, and `readProfileFile` repeat
+preflight stat, open, post-open stat, regular-file checks, size checks, and bounded
+reads in [source.go](../../source.go), [imports.go](../../imports.go), and
+[profile.go](../../profile.go).
+
+Work:
+
+1. Introduce private helpers in `source_file.go` for regular-file opening and
+   bounded reading. Use narrow function parameters for stat/open operations so
+   callers can supply `os.Stat`/`os.Open` or root-bound methods. Do not introduce
+   an exported filesystem interface or substitute unrestricted opening for
+   `os.Root` access.
+2. Keep preflight before open and post-open verification. Close every opened
+   file on all paths. Preserve the existing treatment of read-only file close
+   errors; root-directory close diagnostics are a separate responsibility.
+3. Return private classified failures with enough distinction for callers to
+   map errors safely. Optional imports may suppress only the currently allowed
+   missing-target failures at stat/open, not later read failures. Do not expose
+   filesystem errors downstream through wrapping.
+4. Keep source metadata, canonicalization, containment, per-source/aggregate
+   budgets, byte accounting, and caller-specific limit diagnostics outside the
+   shared mechanics. Pass the computed byte limit into the reader and retain
+   both the size precheck and limit-plus-one read enforcement.
+5. Reuse bounded read mechanics for reader/byte sources only where it simplifies
+   existing code without changing single-use or ownership behavior. Keep secrets
+   bootstrap independent: its symlink, environment, and reporting policies differ.
+
+Acceptance: source, import, profile, FIFO, containment, symlink, optional-missing,
+exact-budget, and reader-ownership tests pass. Retain the deadline-isolated FIFO
+regression. Cover private failure classification only where public tests cannot
+reliably induce an important filesystem failure. Run race tests for this unit.
+
+## Stage 4 — Establish shared document traversal ownership
+
+Status: pending.
+
+Evidence: `indexDocument`, `walkDocumentPaths`, and `projectedSensitivity` repeat
+semantic tree traversal in [load_metadata.go](../../load_metadata.go),
+[metadata.go](../../metadata.go), and [secret.go](../../secret.go). General document
+lookup currently lives in [load.go](../../load.go).
+
+Work:
+
+1. Create root-private `document_walk.go` with a nil-safe preorder walker taking
+   a starting `Path` and a callback receiving both path and node. Visit mappings
+   in stored order and sequences in index order; construct explicit Key/Index
+   segments. It visits value nodes, not synthetic mapping-key nodes.
+2. Move `indexDocument` and `documentAtPath` to this owner. Implement indexing,
+   metadata shape collection, and projected sensitivity collection using the
+   walker where their traversal semantics match.
+3. Document callback borrowing: callers must clone paths or nodes when retaining
+   mutable state. Preserve existing defensive copies and avoid sharing mutable
+   metadata across results.
+4. Keep path-bearing helpers in the root package so `internal/document` does not
+   depend on the public package. Pure node representation and cloning stay in
+   `internal/document`.
+5. Leave specialized traversals explicit where they need inherited origins,
+   pruning, node identity deduplication, or different counting. In particular,
+   input limits count mapping keys and retained-history limits count shared nodes
+   once; do not implement either using the new walker without those semantics.
+
+Acceptance: numeric map keys versus sequence indices, escaped paths, unknown
+paths, historical shape changes, deleted descendants, dynamic secret tags, and
+metadata detachment remain protected by existing tests. No public path behavior
+or predicate coverage changes. Do not add tests that assert walker call counts.
+
+## Stage 5 — Separate typed projection, cloning, and comparison
+
+Status: pending.
+
+Evidence: [projection.go](../../projection.go) combines roughly 622 lines of
+projection, detached cloning, dynamic schema handling, cycle/limit enforcement,
+and structural comparison. Value equality deliberately ignores sensitivity,
+which validation must check separately.
+
+Work:
+
+1. Keep canonical typed projection in `projection.go`; move cloning into
+   `clone.go`, shared reflection operation state and cycle helpers into
+   `value_operation.go`, and value-tree comparison into `document_compare.go`.
+   Preserve the descriptor-driven approach and per-operation schema cache.
+2. Rename the private equality/comparison entry points to make configuration-value
+   semantics explicit. Document that location and sensitivity metadata are
+   excluded, mapping order is immaterial, and sequences are compared atomically
+   for change reporting. Update all callers without changing their semantics.
+3. Give validator mutation comparison a named private owner in `validation.go`.
+   It must combine projected value changes and sensitivity changes, while
+   retaining private resolved-secret-state auditing as an additional check.
+   Do not make resolver value comparison start treating sensitivity as a value.
+4. Preserve the distinction between cloning unresolved public secret references
+   and publication cloning of already validated private state. Name/document
+   the clone mode explicitly rather than leaving an unexplained boolean at
+   sensitive call sites.
+5. Replace custom `cmpString` calls with the standard library comparator and
+   remove that helper. Do not perform unrelated test-style rewrites.
+
+Acceptance: projection/cloning tests cover nil versus empty, custom text codecs,
+inline fields, dynamic types, cycles, and limits. Validator mutation regressions
+must still reject equal-text sensitivity changes and secret-state changes.
+Callback ownership and secret lookup tests must prove publication does not share
+borrowed values or reread the environment. Run race tests for this unit.
+
+## Stage 6 — Clarify source assembly and override ownership
+
+Status: pending.
+
+Evidence: [imports.go](../../imports.go) mixes filesystem context, directives,
+graph traversal, and diagnostics; [override.go](../../override.go) combines value
+construction/validation and application. Shared composition cloning is located
+inside the override feature, while profile overlay repeats it.
+
+Work:
+
+1. Keep import traversal and route diagnostics in `imports.go`. Move filesystem
+   context validation/canonicalization/containment to `filesystem.go`, and
+   directive structures/parsing/fragment restrictions to `directives.go`.
+   Give source-independent node counting an appropriate shared document owner,
+   with its mapping-key accounting documented.
+2. Keep override validation and preparation in `override.go`, move parser-neutral
+   value building to `override_value.go`, and application/displacement mechanics
+   to `override_apply.go`.
+3. Move `cloneComposedDocument` next to `composedDocument` in `compose.go`; use it
+   for both profile overlay and overrides. Ensure every retained field is copied
+   according to its existing ownership contract.
+4. Keep strict composition, overlay, and history replay separate. Share small
+   mechanics when identical; do not introduce merge strategy flags or one general
+   merge engine. Preserve the existing profile-versus-CLI recording behavior.
+5. Simplify profile root cleanup using deferred failure-path cleanup plus an
+   explicit checked success-path close. Preserve primary failure diagnostics and
+   the existing successful-read root-close error diagnostic; avoid double-close
+   effects. Import traversal cleanup may remain explicit where already clear.
+
+Acceptance: imports/globs, directive restrictions, source ordering, profiles,
+overrides, version restrictions, limits, and composition independence tests pass.
+No new cross-package dependencies appear. Root handles close on every path and
+profile/CLI provenance remains identical.
+
+## Stage 7 — Extract explicit loader lifecycle phases
+
+Status: pending.
+
+Evidence: `loadWithSecretLookup` in [load.go](../../load.go) coordinates roughly
+200 lines of source assembly, decoding, callbacks, history, secrets, and result
+publication, including repeated creation of a composition handoff.
+
+Work:
+
+1. Keep the exported entry point and a readable linear coordinator in `load.go`.
+   Move option selection to `options.go`, input assembly to `load_input.go`, and
+   typed lifecycle/result publication to `load_result.go`. Move version checking
+   beside the validation machinery rather than keeping it among document helpers.
+2. Use small private handoff structs for assembled input and necessary phase
+   outputs, not a mutable all-purpose loader object or a long list of output
+   parameters. Reuse one composition initialization helper for profile and CLI
+   processing, including NoSource override handling and source byte/node totals.
+3. Preserve the exact existing order of option validation, parsing/import checks,
+   schema validation, version checking, profile/CLI application, defaults factory,
+   typed decoding, resolution, retained-history limit enforcement, secret lookup,
+   validation, metadata/classification, and result publication.
+4. Keep diagnostic/source accumulation explicit. Warnings survive successful
+   phases; errors stop dependent work and never return a usable partial Result.
+5. Preserve both ownership barriers: detach resolver-borrowed values before secret
+   resolution and validation, then detach equal-valued validator replacements
+   before publication while preserving validated secret state. Do not remove
+   either clone as apparently redundant work.
+6. Group Result's frozen inspection state only if it clarifies this handoff; do
+   not add an additional history copy or change caller ownership of Config.
+
+Acceptance: lifecycle ordering/early-stop tests, defaults-only loading, all input
+combinations, callback ownership, secrets, warning retention, and complete
+provenance agreement pass. Keep orchestration visible enough to verify phase
+ordering by reading the coordinator. Run the complete race suite.
+
+## Stage 8 — Separate explanation replay, snapshots, and rendering
+
+Status: pending.
+
+Evidence: [explain.go](../../explain.go) combines approximately 576 lines of public
+records, replay algorithms, snapshot redaction, retained-history accounting, and
+human-readable output.
+
+Work:
+
+1. Keep public explanation records and Explain coordination in `explain.go`.
+   Move application/snapshot replay and history budgeting to `history.go`, safe
+   snapshot projection to `snapshot.go`, and WriteExplanation with its formatting
+   helpers to `explain_render.go`.
+2. Make effective rendering and explanation snapshots use the existing shared
+   redaction implementation; do not create separate sensitivity rules.
+3. Retain the canonical decoding reconciliation event between input replay and
+   domain resolution, overlapping resolution intent coalescing, added/deleted
+   paths, effective source attribution, and historical secret classification.
+4. Document the differences between metadata origin records and value replay:
+   they serve different query contracts but must agree on effective results.
+   Do not replace them with a new event-store abstraction in this cleanup.
+5. Preserve writer failure behavior and ownership of retained trees. Do not cache
+   explanations or optimize full-tree replay without measured need.
+
+Acceptance: explanation, effective rendering, history budget, provenance, secret
+redaction, and writer-failure tests pass without output changes. Container and
+leaf histories still reconcile with the frozen effective typed configuration.
+
+## Stage 9 — Complete ownership documentation and integration validation
+
+Status: pending.
+
+Work:
+
+1. Review the final code for obsolete helpers, duplicate traversal/translation,
+   unread retained state, stale comments, and imports left by earlier stages.
+   Confirm file splits reflect discrete responsibilities rather than line quotas.
+2. Update the component overview to describe the actual owners and fix its stale
+   claim that metadata queries are empty. Keep detailed consumer semantics in
+   their existing contract; repair navigation and moved-file links throughout
+   documentation, including this roadmap's evidence links if necessary.
+3. State the retained dependency direction clearly: public lifecycle coordination
+   uses private source/document/schema/inspection helpers; the owned document
+   representation has no dependency on the root package; only the private YAML
+   adapter imports the upstream YAML package.
+4. Run the full baseline and race suites, local documentation and dependency
+   boundary checks, and the complete source-candidate checker described in the
+   release procedure with available qualifying toolchains. Record exact versions,
+   host platform, results, and any unavailable platform evidence. Cross-builds
+   are not native execution evidence.
+5. Review the accumulated diff for public API/configuration/diagnostic changes.
+   Expected outcome: no consumer migration or dependency change. If any contract
+   change is discovered, resolve it explicitly before closing this roadmap.
+6. Record completion evidence and commit this final nonempty unit. Release
+   preparation must subsequently retire completed roadmap material and validate
+   the exact committed candidate under release policy. Completing this roadmap
+   does not create a tag, push refs, or authorize publication.
+
+Acceptance: all stages are complete with recorded validation, existing behavioral
+regressions remain protected, documentation matches the new ownership layout,
+and the worktree is clean after commits. Required release gates remain visible
+in the release procedure rather than being declared satisfied by this refactor.
+
+## Open Questions
+
+None required to begin implementation. The package boundaries, compatibility
+constraints, ordered units, and acceptance criteria above are decided. Routine
+private naming adjustments are acceptable when they preserve those decisions;
+new behavioral discoveries require explicit evidence and disposition.

explain.go

@@ -86,7 +86,7 @@ func cloneSecretReferences(values map[string]string) map[string]string {
 // Explain returns safe effective and chronological values for a known final or
 // historical path. Unknown and malformed paths return false.
 func (r *Result[T]) Explain(path Path) (Explanation, bool) {
-	if r == nil || r.final == nil || r.baseline == nil || !path.valid() {
+	if r == nil || r.final == nil || r.baseline == nil || !r.metadata.hasHistoricalPath(path) {
 		return Explanation{}, false
 	}
 	explanation := Explanation{Path: path.clone(), Effective: valueSnapshot(r.final, path, r.secrets)}
@@ -110,15 +110,35 @@ func (r *Result[T]) Explain(path Path) (Explanation, bool) {
 		if !relevant {
 			continue
 		}
+		after := valueSnapshot(working, path, r.secrets)
+		if !before.Present && !after.Present {
+			continue
+		}
 		explanation.History = append(explanation.History, ValueChange{
 			Before: before,
-			After:  valueSnapshot(working, path, r.secrets),
+			After:  after,
 			Origin: historyApplicationOrigin(application, operationPath),
 		})
 	}
 
+	// Intermediate input replacements do not permanently suppress defaults:
+	// decoding applies them to the final input tree. Show that reconciliation
+	// explicitly, including canonical scalar conversions, before domain changes.
+	replayed, _ := documentAtPath(working, path)
+	decoded, _ := documentAtPath(r.input, path)
+	if !documentNodesEqual(replayed, decoded) {
+		origin := cloneOrigin(r.inputOrigins[path.String()])
+		origin.Operation = ChangeDecoding
+		explanation.History = append(explanation.History, ValueChange{
+			Before:  valueSnapshot(working, path, r.secrets),
+			After:   valueSnapshot(r.input, path, r.secrets),
+			Origin:  origin,
+			Reasons: []string{"applied retained defaults and canonical typed decoding"},
+		})
+	}
+
 	working = document.Clone(r.input)
-	for _, snapshot := range r.resolution {
+	for _, snapshot := range coalescedResolutionSnapshots(r.resolution) {
 		relevant := pathsIntersect(snapshot.change.Path, path)
 		var before ValueSnapshot
 		if relevant {
@@ -128,9 +148,13 @@ func (r *Result[T]) Explain(path Path) (Explanation, bool) {
 		if !relevant {
 			continue
 		}
+		after := valueSnapshot(working, path, r.secrets)
+		if !before.Present && !after.Present {
+			continue
+		}
 		explanation.History = append(explanation.History, ValueChange{
 			Before:       before,
-			After:        valueSnapshot(working, path, r.secrets),
+			After:        after,
 			Origin:       cloneOrigin(snapshot.change.Origin),
 			Reasons:      append([]string(nil), snapshot.change.Reasons...),
 			Dependencies: clonePaths(snapshot.change.Dependencies),
@@ -255,6 +279,37 @@ func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node
 	}
 }
 
+// Resolution records describe one before/final transformation, not assignment
+// order. A parent already carries its descendants' final values; merge overlapping
+// intent into that event rather than replay a child assignment a second time.
+func coalescedResolutionSnapshots(snapshots []resolutionSnapshot) []resolutionSnapshot {
+	groups := make([]resolutionSnapshot, 0, len(snapshots))
+	for _, snapshot := range snapshots {
+		merged := false
+		for i := range groups {
+			if !pathContains(groups[i].change.Path, snapshot.change.Path) {
+				continue
+			}
+			for _, reason := range snapshot.change.Reasons {
+				groups[i].change.Reasons = appendUniqueString(groups[i].change.Reasons, reason)
+			}
+			for _, dependency := range snapshot.change.Dependencies {
+				groups[i].change.Dependencies = appendUniquePath(groups[i].change.Dependencies, dependency)
+			}
+			if snapshot.change.Origin.Operation == ChangeNormalization {
+				groups[i].change.Origin.Operation = ChangeNormalization
+			}
+			merged = true
+			break
+		}
+		if !merged {
+			snapshot.change = cloneChange(snapshot.change)
+			groups = append(groups, snapshot)
+		}
+	}
+	return groups
+}
+
 func replaceHistoryPath(root **document.Node, path Path, replacement *document.Node) {
 	if root == nil || *root == nil {
 		return
@@ -273,7 +328,19 @@ func replaceHistoryPath(root **document.Node, path Path, replacement *document.N
 			}
 			position := documentEntryIndex(current.Mapping, segment.key)
 			if position < 0 {
-				return
+				if replacement == nil {
+					return
+				}
+				child := &document.Node{Kind: document.Mapping}
+				if last {
+					child = document.Clone(replacement)
+				}
+				current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, Value: child})
+				if last {
+					return
+				}
+				current = child
+				continue
 			}
 			if last {
 				if replacement == nil {
@@ -422,7 +489,9 @@ func (r *Result[T]) WriteExplanation(output io.Writer, path Path) error {
 	rendered.WriteString("history:\n")
 	for _, change := range explanation.History {
 		rendered.WriteString("  - representation: ")
-		if change.Origin.Layer == LayerBase || change.Origin.Layer == LayerProfile || change.Origin.Layer == LayerCLI {
+		if change.Origin.Operation == ChangeDecoding {
+			rendered.WriteString("input to canonical typed\n")
+		} else if change.Origin.Layer == LayerBase || change.Origin.Layer == LayerProfile || change.Origin.Layer == LayerCLI {
 			rendered.WriteString("policy-validated input\n")
 		} else {
 			rendered.WriteString("canonical typed\n")

explain_test.go

@@ -326,3 +326,87 @@ func historyValueLayers(explanation Explanation) []Layer {
 	}
 	return layers
 }
+
+func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) {
+	type cfg struct {
+		M map[string]int `yaml:"m"`
+	}
+	for _, base := range []string{"null", "replaced-text"} {
+		for _, cli := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/cli=%v", base, cli), func(t *testing.T) {
+				root := t.TempDir()
+				main := writeImportFile(t, root, "main.yml", "m: "+base+"\n")
+				opts := Options[cfg]{Definition: Definition[cfg]{Defaults: func() cfg { return cfg{M: map[string]int{"a": 1}} }}}
+				if cli {
+					opts.Overrides = []Override{{Path: Keys("m"), Value: map[string]any{"b": 2}, Flag: "--m"}}
+				} else {
+					writeImportFile(t, root, "profiles/p.yml", "m: {b: 2}\n")
+					opts.Profile = "p"
+				}
+				result, err := Load(File(main), opts)
+				if err != nil {
+					t.Fatal(err)
+				}
+				for _, path := range []Path{Keys("m"), Keys("m", "a")} {
+					explanation, ok := result.Explain(path)
+					if !ok {
+						t.Fatal("missing explanation")
+					}
+					last := explanation.History[len(explanation.History)-1]
+					if last.After != explanation.Effective || last.Origin.Operation != ChangeDecoding {
+						t.Fatalf("history does not reconcile final value: %#v", explanation)
+					}
+					info, _ := result.Metadata().Lookup(path)
+					if last.Origin.Layer != info.Origin.Layer {
+						t.Fatal("decoding lost effective origin")
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestExplanationReplaysResolutionAdditionsAndOverlappingIntent(t *testing.T) {
+	type cfg struct {
+		M map[string]any `yaml:"m"`
+	}
+	for _, overlap := range []bool{false, true} {
+		t.Run(fmt.Sprint(overlap), func(t *testing.T) {
+			result, err := Load(Bytes("base", []byte("m: {old: 1}\n")), Options[cfg]{Definition: Definition[cfg]{Resolve: func(c *cfg, r *Resolution) error {
+				c.M["new"] = map[string]any{"child": int64(7)}
+				delete(c.M, "old")
+				r.Record(Keys("m", "new"), ChangeDerivedDefault, "add nested", Keys("m", "old"))
+				r.Record(Keys("m", "old"), ChangeNormalization, "remove old")
+				if overlap {
+					r.Record(Keys("m"), ChangeNormalization, "update container")
+				}
+				return nil
+			}}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, path := range []Path{Keys("m"), Keys("m", "new"), Keys("m", "new", "child"), Keys("m", "old")} {
+				x, ok := result.Explain(path)
+				if !ok {
+					t.Fatal("missing known history")
+				}
+				var count int
+				for _, v := range x.History {
+					if v.Origin.Layer == LayerResolution {
+						count++
+						if overlap && len(v.Reasons) != 3 {
+							t.Fatal("lost overlapping intent")
+						}
+					}
+				}
+				if overlap && count != 1 {
+					t.Fatalf("overlapping records applied %d times", count)
+				}
+				last := x.History[len(x.History)-1]
+				if last.After != x.Effective {
+					t.Fatalf("incorrect replay for %s: %#v", path, x)
+				}
+			}
+		})
+	}
+}

imports.go

@@ -382,6 +382,18 @@ func (s *importTraversal) readImport(relative, sourceID string, optional bool) (
 	if limit < 0 {
 		limit = 0
 	}
+	preflight, err := s.root.Stat(relative)
+	if err != nil {
+		if optional && errors.Is(err, fs.ErrNotExist) {
+			return nil, true, nil
+		}
+		value := classifySourceError(err, sourceID)
+		return nil, false, &value
+	}
+	if !preflight.Mode().IsRegular() {
+		value := sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file")
+		return nil, false, &value
+	}
 	file, err := s.root.Open(relative)
 	if err != nil {
 		if optional && errors.Is(err, fs.ErrNotExist) {

internal/document/document.go

@@ -32,11 +32,14 @@ type Entry struct {
 // Node is ConfigKit's owned configuration representation. Number Text retains its
 // exact validated decimal spelling; Boolean Text is "true" or "false".
 type Node struct {
-	Kind     Kind
-	Location Location
-	Text     string
-	Mapping  []Entry
-	Sequence []*Node
+	// Sensitive preserves field classification from concrete typed projections.
+	// It is metadata, not part of the configuration value or emitted YAML.
+	Sensitive bool
+	Kind      Kind
+	Location  Location
+	Text      string
+	Mapping   []Entry
+	Sequence  []*Node
 }
 
 // Clone returns a detached copy of a node tree.

load.go

@@ -72,6 +72,7 @@ type Result[T any] struct {
 	diagnostics      []Diagnostic
 	baseline         *document.Node
 	input            *document.Node
+	inputOrigins     map[string]Origin
 	final            *document.Node
 	resolution       []resolutionSnapshot
 	applications     []historyApplication
@@ -252,6 +253,14 @@ func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied ..
 		accumulator.add(Diagnostic{Code: CodeExplainLimitExceeded, Severity: SeverityError, Message: "retained configuration history exceeds the node limit"})
 		return nil, accumulator.loadError()
 	}
+	// Resolver assignments may borrow application-owned mutable values. Detach
+	// before secret resolution mutates reference fields or validation sees them.
+	detached, ownershipIssue := cloneConfiguration(schema, reflect.ValueOf(configuration), defaultProjectionLimits())
+	if ownershipIssue != nil {
+		accumulator.add(diagnosticFromDecodeIssue(decodeIssueFromProjection(ownershipIssue)))
+		return nil, accumulator.loadError()
+	}
+	configuration = detached.Interface().(T)
 	secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, inputDocument, lookup)
 	accumulator.add(secretResolution.diagnostics...)
 	if hasErrors(secretResolution.diagnostics) {
@@ -264,7 +273,16 @@ func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied ..
 			return nil, accumulator.loadError()
 		}
 	}
-	secrets := classifySecretPaths(schema, options.Definition.Secret, metadata, final, applications)
+	// A read-only validator can replace a value with an equal borrowed object.
+	// Projected mutation checks cannot detect that alias, so detach at publication.
+	published, ownershipIssue := cloneConfigurationState(schema, reflect.ValueOf(configuration), defaultProjectionLimits(), true)
+	if ownershipIssue != nil {
+		accumulator.add(diagnosticFromDecodeIssue(decodeIssueFromProjection(ownershipIssue)))
+		return nil, accumulator.loadError()
+	}
+	configuration = published.Interface().(T)
+	finalizeMetadataPaths(&metadata, final, applications, decoded.baseline, decoded.final)
+	secrets := classifySecretPaths(schema, options.Definition.Secret, metadata, final, applications, decoded.baseline, decoded.final)
 	return &Result[T]{
 		Config:           configuration,
 		metadata:         metadata,
@@ -272,6 +290,7 @@ func loadWithSecretLookup[T any](source Source, lookup secretLookup, supplied ..
 		diagnostics:      canonicalDiagnostics(accumulator.diagnostics),
 		baseline:         decoded.baseline,
 		input:            decoded.final,
+		inputOrigins:     cloneOriginIndex(decoded.origins),
 		final:            final,
 		resolution:       resolutionSnapshots,
 		applications:     applications,

load_metadata.go

@@ -184,6 +184,14 @@ func cloneMetadata(metadata Metadata) Metadata {
 		fields:  make(map[string]FieldInfo, len(metadata.fields)),
 		history: make(map[string][]Change, len(metadata.history)),
 	}
+	cloned.effectivePaths = make(map[string]Path, len(metadata.effectivePaths))
+	for pointer, path := range metadata.effectivePaths {
+		cloned.effectivePaths[pointer] = path.clone()
+	}
+	cloned.historicalPaths = make(map[string][]Path, len(metadata.historicalPaths))
+	for pointer, paths := range metadata.historicalPaths {
+		cloned.historicalPaths[pointer] = clonePaths(paths)
+	}
 	for path, info := range metadata.fields {
 		cloned.fields[path] = cloneFieldInfo(info)
 	}

metadata.go

@@ -1,16 +1,20 @@
 package configkit
 
+import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+
 // Metadata provides detached effective presence/origin and chronological history
 // queries for a load. Its zero value contains no paths.
 type Metadata struct {
-	fields  map[string]FieldInfo
-	history map[string][]Change
+	effectivePaths  map[string]Path
+	historicalPaths map[string][]Path
+	fields          map[string]FieldInfo
+	history         map[string][]Change
 }
 
 // Lookup returns information for an effective path. Unknown and invalid paths
 // return false.
 func (m Metadata) Lookup(path Path) (FieldInfo, bool) {
-	if !path.valid() {
+	if !path.valid() || !matchesPathShape(path, m.effectivePaths[path.String()]) {
 		return FieldInfo{}, false
 	}
 	info, ok := m.fields[path.String()]
@@ -32,7 +36,7 @@ func (m Metadata) IsExplicit(path Path) bool {
 // Removed historical paths can have history even though Lookup returns false.
 // Unknown and invalid paths return nil.
 func (m Metadata) History(path Path) []Change {
-	if !path.valid() {
+	if !m.hasHistoricalPath(path) {
 		return nil
 	}
 	history, ok := m.history[path.String()]
@@ -45,3 +49,75 @@ func (m Metadata) History(path Path) []Change {
 	}
 	return cloned
 }
+
+// Textual pointers are lookup keys, but explicit key/index kinds remain part of
+// the query. Candidate paths are constructed from actual document containers.
+func matchesPathShape(query, candidate Path) bool {
+	if !query.valid() || query.String() != candidate.String() || len(query.segments) != len(candidate.segments) {
+		return false
+	}
+	for i, segment := range query.segments {
+		actual := candidate.segments[i]
+		if actual.kind == pathSegmentIndex {
+			index, ok := schemaIndex(segment)
+			if !ok || index != actual.index {
+				return false
+			}
+		} else if segment.kind == pathSegmentIndex {
+			return false
+		}
+	}
+	return true
+}
+
+func (m Metadata) hasHistoricalPath(path Path) bool {
+	if !path.valid() {
+		return false
+	}
+	for _, candidate := range m.historicalPaths[path.String()] {
+		if matchesPathShape(path, candidate) {
+			return true
+		}
+	}
+	return false
+}
+
+func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []historyApplication, earlier ...*document.Node) {
+	m.effectivePaths = make(map[string]Path)
+	m.historicalPaths = make(map[string][]Path)
+	walkDocumentPaths(final, Path{}, func(path Path) { m.effectivePaths[path.String()] = path.clone() })
+	add := func(path Path) {
+		pointer := path.String()
+		for _, prior := range m.historicalPaths[pointer] {
+			if matchesPathShape(path, prior) {
+				return
+			}
+		}
+		m.historicalPaths[pointer] = append(m.historicalPaths[pointer], path.clone())
+	}
+	walkDocumentPaths(final, Path{}, add)
+	for _, tree := range earlier {
+		walkDocumentPaths(tree, Path{}, add)
+	}
+	for _, application := range applications {
+		// Overrides have mapping-only prefixes; parsed tokens are keys in this layer.
+		prefix := Path{}
+		for _, segment := range application.path.segments {
+			prefix = prefix.Key(segment.key)
+		}
+		walkDocumentPaths(application.value, prefix, add)
+	}
+}
+
+func walkDocumentPaths(node *document.Node, path Path, visit func(Path)) {
+	if node == nil {
+		return
+	}
+	visit(path)
+	for _, entry := range node.Mapping {
+		walkDocumentPaths(entry.Value, path.Key(entry.Key), visit)
+	}
+	for i, child := range node.Sequence {
+		walkDocumentPaths(child, path.Index(i), visit)
+	}
+}

metadata_test.go

@@ -1,12 +1,17 @@
 package configkit
 
-import "testing"
+import (
+	"bytes"
+	"testing"
+)
 
 func TestMetadataQueriesReturnDetachedRecords(t *testing.T) {
 	t.Parallel()
 
 	path := Keys("service", "workers")
 	metadata := Metadata{
+		effectivePaths:  map[string]Path{path.String(): path},
+		historicalPaths: map[string][]Path{path.String(): {path}},
 		fields: map[string]FieldInfo{
 			path.String(): {
 				Explicit: true,
@@ -56,3 +61,45 @@ func TestMetadataQueriesReturnDetachedRecords(t *testing.T) {
 		t.Fatal("invalid path produced metadata")
 	}
 }
+
+func TestQueriesRespectActualHistoricalAndEffectiveContainers(t *testing.T) {
+	result, err := Load[map[string]any](Bytes("base", []byte("items: [1]\nm: {'0': 2}\nx: 3\nretired: {child: 4}\n")), Options[map[string]any]{Overrides: []Override{{Path: Keys("retired"), Value: "removed", Flag: "--retired"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, path := range []Path{Keys("items", "0"), Keys("m").Index(0), Keys("x", "missing"), Keys("items").Index(8)} {
+		if _, ok := result.Metadata().Lookup(path); ok {
+			t.Fatalf("invalid lookup %s", path)
+		}
+		if result.Metadata().History(path) != nil {
+			t.Fatalf("invalid history %s", path)
+		}
+		if _, ok := result.Explain(path); ok {
+			t.Fatalf("invalid explanation %s", path)
+		}
+		var out bytes.Buffer
+		if err := result.WriteExplanation(&out, path); err == nil || out.Len() != 0 {
+			t.Fatalf("invalid path wrote output: %s", path)
+		}
+	}
+	for _, pointer := range []string{"/items/0", "/m/0"} {
+		path, _ := ParsePath(pointer)
+		if _, ok := result.Metadata().Lookup(path); !ok {
+			t.Fatal("parsed path lost lookup")
+		}
+		if _, ok := result.Explain(path); !ok {
+			t.Fatal("parsed path lost explanation")
+		}
+	}
+	retired := Keys("retired", "child")
+	if _, ok := result.Metadata().Lookup(retired); ok {
+		t.Fatal("removed path remains effective")
+	}
+	if len(result.Metadata().History(retired)) == 0 {
+		t.Fatal("removed history missing")
+	}
+	x, ok := result.Explain(retired)
+	if !ok || x.Effective.Present || len(x.History) == 0 {
+		t.Fatal("removed explanation missing")
+	}
+}

profile.go

@@ -122,6 +122,15 @@ func readProfileFile(root *os.Root, relative, sourceID string, remaining int64)
 	if limit < 0 {
 		limit = 0
 	}
+	preflight, err := root.Stat(relative)
+	if err != nil {
+		value := classifySourceError(err, sourceID)
+		return nil, &value
+	}
+	if !preflight.Mode().IsRegular() {
+		value := sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file")
+		return nil, &value
+	}
 	file, err := root.Open(relative)
 	if err != nil {
 		value := classifySourceError(err, sourceID)

projection.go

@@ -34,10 +34,11 @@ type valueVisit struct {
 }
 
 type valueOperation struct {
-	schema *schemaDescriptor
-	limits projectionLimits
-	active map[valueVisit]bool
-	nodes  int
+	preserveSecrets bool
+	schema          *schemaDescriptor
+	limits          projectionLimits
+	active          map[valueVisit]bool
+	nodes           int
 }
 
 func projectConfiguration(schema *schemaDescriptor, value reflect.Value, limits projectionLimits) (*document.Node, *projectionIssue) {
@@ -49,10 +50,15 @@ func projectConfiguration(schema *schemaDescriptor, value reflect.Value, limits
 }
 
 func cloneConfiguration(schema *schemaDescriptor, value reflect.Value, limits projectionLimits) (reflect.Value, *projectionIssue) {
+	return cloneConfigurationState(schema, value, limits, false)
+}
+
+// Publication copies preserve only already-validated private reference state.
+func cloneConfigurationState(schema *schemaDescriptor, value reflect.Value, limits projectionLimits, preserveSecrets bool) (reflect.Value, *projectionIssue) {
 	if schema == nil || schema.root == nil || !value.IsValid() || value.Type() != schema.root.typeOf {
 		return reflect.Value{}, &projectionIssue{code: CodeSchemaUnsupportedType, message: "configuration value does not match its schema"}
 	}
-	operation := valueOperation{schema: schema, limits: limits, active: make(map[valueVisit]bool)}
+	operation := valueOperation{schema: schema, limits: limits, active: make(map[valueVisit]bool), preserveSecrets: preserveSecrets}
 	return operation.clone(value, schema.root, Path{}, 0)
 }
 
@@ -117,6 +123,15 @@ func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor
 			seen[member.name] = true
 			result.Mapping = append(result.Mapping, document.Entry{Key: member.name, Value: child})
 		}
+		for _, entry := range result.Mapping {
+			field := descriptor.inlineMap
+			if index, ok := descriptor.fieldByName[entry.Key]; ok {
+				field = &descriptor.fields[index]
+			}
+			if field != nil && field.secret {
+				entry.Value.Sensitive = true
+			}
+		}
 		return result, nil
 	case descriptorMap:
 		if value.IsNil() {
@@ -345,7 +360,9 @@ func (o *valueOperation) clone(value reflect.Value, descriptor *typeDescriptor,
 	case descriptorSecret:
 		result := reflect.New(value.Type()).Elem()
 		reference := value.Interface().(Secret)
-		if reference.specified {
+		if o.preserveSecrets {
+			result.Set(value)
+		} else if reference.specified {
 			result.Set(reflect.ValueOf(SecretEnv(reference.EnvName())))
 		}
 		return result, nil

records.go

@@ -11,11 +11,14 @@ const (
 	LayerResolution Layer = "resolution"
 )
 
-// ChangeKind identifies an application resolution operation. The zero value is
+// ChangeKind identifies a recorded operation. The zero value is
 // used for input and default origins.
 type ChangeKind string
 
 const (
+	// ChangeDecoding records the built-in transition to canonical typed values.
+	// It is inspection metadata, not an operation accepted by Resolution.Record.
+	ChangeDecoding       ChangeKind = "decoding"
 	ChangeDerivedDefault ChangeKind = "derived_default"
 	ChangeNormalization  ChangeKind = "normalization"
 )

release_check_test.go

@@ -0,0 +1,100 @@
+package configkit
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestReleaseCheckerShellHandlesFilenamesAndStopsOnFailure(t *testing.T) {
+	script, err := os.ReadFile("scripts/release-check.sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, fail := range []bool{false, true} {
+		name := "success"
+		if fail {
+			name = "failed-go-check"
+		}
+		t.Run(name, func(t *testing.T) {
+			root := t.TempDir()
+			bin := t.TempDir()
+			log := filepath.Join(bin, "gofmt.args")
+			calls := filepath.Join(bin, "go.calls")
+			for _, directory := range []string{"scripts", "nested"} {
+				if err := os.Mkdir(filepath.Join(root, directory), 0700); err != nil {
+					t.Fatal(err)
+				}
+			}
+			write := func(path string, data []byte, mode os.FileMode) {
+				t.Helper()
+				if err := os.WriteFile(path, data, mode); err != nil {
+					t.Fatal(err)
+				}
+			}
+			write(filepath.Join(root, "scripts", "release-check.sh"), script, 0700)
+			write(filepath.Join(root, "go.mod"), []byte("module example.invalid/check\n"), 0600)
+			write(filepath.Join(root, "go.sum"), nil, 0600)
+			files := []string{"nested/a space.go", "nested/a\nnewline.go"}
+			for _, file := range files {
+				write(filepath.Join(root, file), []byte("package example\n"), 0600)
+			}
+			// Isolate expensive toolchain commands: this test owns shell orchestration,
+			// not Go's compiler or the production tests exercised by the real checker.
+			write(filepath.Join(bin, "go"), []byte("#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$CONFIGKIT_TEST_GO_LOG\"\nif [ \"${CONFIGKIT_TEST_FAIL-}\" = yes ] && [ \"${1-}\" = test ]; then exit 37; fi\n"), 0700)
+			write(filepath.Join(bin, "gofmt"), []byte("#!/bin/sh\nprintf '%s\\000' \"$@\" > \"$CONFIGKIT_TEST_FMT_LOG\"\n"), 0700)
+			init := exec.Command("git", "init", "-q")
+			init.Dir = root
+			if out, err := init.CombinedOutput(); err != nil {
+				t.Fatalf("git init: %v %s", err, out)
+			}
+			// HEAD is required by the whitespace gate. Commit only the module baseline.
+			for _, args := range [][]string{{"add", "go.mod", "go.sum"}, {"-c", "user.name=Test", "-c", "user.email=test@example.invalid", "-c", "commit.gpgsign=false", "commit", "-qm", "Test fixture"}} {
+				cmd := exec.Command("git", args...)
+				cmd.Dir = root
+				if out, err := cmd.CombinedOutput(); err != nil {
+					t.Fatalf("git fixture: %v %s", err, out)
+				}
+			}
+			cmd := exec.Command("/bin/bash", filepath.Join(root, "scripts", "release-check.sh"))
+			cmd.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), "CONFIGKIT_BASELINE_TOOLCHAIN=test-baseline", "CONFIGKIT_NEWER_TOOLCHAIN=test-newer", "CONFIGKIT_TEST_FMT_LOG="+log, "CONFIGKIT_TEST_GO_LOG="+calls)
+			if fail {
+				cmd.Env = append(cmd.Env, "CONFIGKIT_TEST_FAIL=yes")
+			}
+			out, err := cmd.CombinedOutput()
+			if fail {
+				if err == nil {
+					t.Fatal("checker ignored failed Go test")
+				}
+			} else if err != nil {
+				t.Fatalf("checker: %v %s", err, out)
+			}
+			arguments, err := os.ReadFile(log)
+			if err != nil {
+				t.Fatal(err)
+			}
+			parts := bytes.Split(arguments, []byte{0})
+			for _, file := range files {
+				found := false
+				for _, arg := range parts {
+					if string(arg) == file {
+						found = true
+					}
+				}
+				if !found {
+					t.Fatalf("filename was split: %q", file)
+				}
+			}
+			invoked, err := os.ReadFile(calls)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if fail && strings.Contains(string(invoked), "vet ./...") {
+				t.Fatal("checker continued after failed test")
+			}
+		})
+	}
+}

render_test.go

@@ -175,3 +175,76 @@ func TestSecretValuesStayOutOfLoadFailuresAndWarnings(t *testing.T) {
 		})
 	}
 }
+
+func TestDynamicConcreteTagsRemainSecretThroughHistory(t *testing.T) {
+	type sensitive struct {
+		Token string `yaml:"token" configkit:"secret"`
+	}
+	type cfg struct {
+		Payload any    `yaml:"payload"`
+		Derived string `yaml:"derived"`
+	}
+	for _, change := range []string{"retain", "replace", "delete", "introduce"} {
+		t.Run(change, func(t *testing.T) {
+			const canary = "DYNAMIC_TAG_CANARY"
+			definition := Definition[cfg]{Defaults: func() cfg { return cfg{Payload: map[string]any{"nested": []any{sensitive{Token: canary}}}} }}
+			path := Keys("payload", "nested").Index(0).Key("token")
+			if change == "introduce" {
+				definition.Defaults = nil
+			}
+			definition.Resolve = func(c *cfg, r *Resolution) error {
+				if change == "introduce" {
+					Normalize(r, Keys("payload"), &c.Payload, any(map[string]any{"nested": []any{sensitive{Token: canary}}}), "create")
+				}
+				if change == "replace" {
+					Normalize(r, Keys("payload"), &c.Payload, any("public"), "replace")
+				}
+				if change == "delete" {
+					Normalize(r, Keys("payload"), &c.Payload, any(nil), "remove")
+				}
+				Normalize(r, Keys("derived"), &c.Derived, canary, "derive", path)
+				return nil
+			}
+			result, err := Load(NoSource(), Options[cfg]{Definition: definition})
+			if err != nil {
+				t.Fatal(err)
+			}
+			var output bytes.Buffer
+			if err := result.WriteEffective(&output); err != nil {
+				t.Fatal(err)
+			}
+			for _, query := range []Path{Keys(), path, Keys("derived")} {
+				explanation, _ := result.Explain(query)
+				output.WriteString(fmt.Sprintf("%#v", explanation))
+			}
+			if strings.Contains(output.String(), canary) {
+				t.Fatal("dynamic secret escaped inspection")
+			}
+			if !result.secrets[path.String()] || !result.secrets["/derived"] {
+				t.Fatal("dynamic tag or dependency classification missing")
+			}
+		})
+	}
+}
+
+func TestValidatorCannotChangeDynamicSensitivityWithoutChangingText(t *testing.T) {
+	type sensitive struct {
+		Token string `yaml:"token" configkit:"secret"`
+	}
+	type cfg struct {
+		Payload any `yaml:"payload"`
+	}
+	_, err := Load(Bytes("base", []byte("payload: {token: disposable}\n")), Options[cfg]{Definition: Definition[cfg]{Validate: func(c *cfg, v *Validation) {
+		c.Payload = sensitive{Token: c.Payload.(map[string]any)["token"].(string)}
+	}}})
+	var failure *LoadError
+	if !errors.As(err, &failure) {
+		t.Fatal("validator changed inspection sensitivity without a mutation error")
+	}
+	for _, d := range failure.Diagnostics() {
+		if d.Code == CodeValidateMutation {
+			return
+		}
+	}
+	t.Fatal("missing validation mutation diagnostic")
+}

schema.go

@@ -212,9 +212,17 @@ func (b *schemaBuilder) describeFields(descriptor *typeDescriptor, current refle
 				return &schemaIssue{code: CodeSchemaTag, path: path.clone(), message: "inline field cannot declare a YAML name"}
 			}
 			inlineType := field.Type
+			seenPointers := make(map[reflect.Type]bool)
 			for inlineType.Kind() == reflect.Pointer {
+				if seenPointers[inlineType] {
+					return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline pointer chain must terminate in a struct"}
+				}
+				seenPointers[inlineType] = true
 				inlineType = inlineType.Elem()
 			}
+			if inlineType.Kind() == reflect.Map && field.Type.Kind() == reflect.Pointer {
+				return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline maps cannot be pointers"}
+			}
 			switch inlineType.Kind() {
 			case reflect.Struct:
 				if b.building[inlineType] {

schema_test.go

@@ -1,6 +1,7 @@
 package configkit
 
 import (
+	"errors"
 	"reflect"
 	"testing"
 	"time"
@@ -302,3 +303,40 @@ func mustPath(t *testing.T, pointer string) Path {
 	}
 	return path
 }
+
+func TestInlineMapPointersFailBeforeDefaults(t *testing.T) {
+	type cfg struct {
+		Extra *map[string]int `yaml:",inline"`
+	}
+	for _, nonNil := range []bool{false, true} {
+		called := false
+		_, err := Load(NoSource(), Options[cfg]{Definition: Definition[cfg]{Defaults: func() cfg {
+			called = true
+			if nonNil {
+				m := map[string]int{"value": 1}
+				return cfg{Extra: &m}
+			}
+			return cfg{}
+		}}})
+		var loadErr *LoadError
+		if !errors.As(err, &loadErr) || called {
+			t.Fatalf("invalid inline schema reached defaults: called=%v error=%v", called, err)
+		}
+		if len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeSchemaInline {
+			t.Fatal("missing inline schema diagnostic")
+		}
+	}
+	type pointer *pointer
+	type recursive struct {
+		Extra pointer `yaml:",inline"`
+	}
+	if _, err := Load[recursive](NoSource()); err == nil {
+		t.Fatal("recursive inline pointer was accepted")
+	}
+	type double struct {
+		Extra **map[string]int `yaml:",inline"`
+	}
+	if _, err := Load[double](NoSource()); err == nil {
+		t.Fatal("double inline map pointer was accepted")
+	}
+}

scripts/release-check.sh

@@ -35,7 +35,10 @@ git ls-files --cached --others --exclude-standard -z |
 diff -u go.mod "$temporary_dir/go.mod"
 diff -u go.sum "$temporary_dir/go.sum"
 
-mapfile -d '' go_files < <(git ls-files --cached --others --exclude-standard -z -- '*.go')
+go_files=()
+while IFS= read -r -d '' go_file; do
+  go_files+=("$go_file")
+done < <(git ls-files --cached --others --exclude-standard -z -- '*.go')
 if ((${#go_files[@]})); then
   unformatted=$(gofmt -l -- "${go_files[@]}")
   if [[ -n "$unformatted" ]]; then

secret.go

@@ -7,7 +7,7 @@ type secretDependency struct {
 	dependency Path
 }
 
-func classifySecretPaths(schema *schemaDescriptor, predicate func(Path) bool, metadata Metadata, final *document.Node, applications []historyApplication) map[string]bool {
+func classifySecretPaths(schema *schemaDescriptor, predicate func(Path) bool, metadata Metadata, final *document.Node, applications []historyApplication, projections ...*document.Node) map[string]bool {
 	paths := make(map[string]Path)
 	finalPaths := make(map[string]*document.Node)
 	indexDocument(final, Path{}, finalPaths)
@@ -36,6 +36,16 @@ func classifySecretPaths(schema *schemaDescriptor, predicate func(Path) bool, me
 	}
 
 	secret := make(map[string]bool)
+	for _, tree := range append(projections, final) {
+		projected := make(map[string]*document.Node)
+		indexDocument(tree, Path{}, projected)
+		for pointer, node := range projected {
+			addSecretPath(paths, pointer)
+			if node.Sensitive {
+				secret[pointer] = true
+			}
+		}
+	}
 	for pointer, path := range paths {
 		if root, ok := schemaSecretRoot(schema, path); ok {
 			secret[root.String()] = true
@@ -129,3 +139,26 @@ func schemaSecretRoot(schema *schemaDescriptor, path Path) (Path, bool) {
 	}
 	return Path{}, false
 }
+
+// Dynamic type replacement can change classification even when projected text
+// is equal. Validators must preserve this inspection metadata as well as values.
+func projectedSensitivity(root *document.Node) map[string]Path {
+	result := make(map[string]Path)
+	var visit func(*document.Node, Path)
+	visit = func(node *document.Node, path Path) {
+		if node == nil {
+			return
+		}
+		if node.Sensitive {
+			result[path.String()] = path.clone()
+		}
+		for _, entry := range node.Mapping {
+			visit(entry.Value, path.Key(entry.Key))
+		}
+		for i, child := range node.Sequence {
+			visit(child, path.Index(i))
+		}
+	}
+	visit(root, Path{})
+	return result
+}

source.go

@@ -136,7 +136,19 @@ func readSource(source Source, sourceID string, limits sourceLimits) (sourceRead
 			return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceRead, sourceID, "source path could not be resolved")}
 		}
 		info.Path = filepath.Clean(absolute)
-		file, err := os.Open(source.path)
+		canonical, err := filepath.EvalSymlinks(info.Path)
+		if err != nil {
+			return sourceRead{info: info}, []Diagnostic{classifySourceError(err, sourceID)}
+		}
+		info.Path = canonical
+		preflight, err := os.Stat(canonical)
+		if err != nil {
+			return sourceRead{info: info}, []Diagnostic{classifySourceError(err, sourceID)}
+		}
+		if !preflight.Mode().IsRegular() {
+			return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file")}
+		}
+		file, err := os.Open(canonical)
 		if err != nil {
 			return sourceRead{info: info}, []Diagnostic{classifySourceError(err, sourceID)}
 		}
@@ -150,12 +162,6 @@ func readSource(source Source, sourceID string, limits sourceLimits) (sourceRead
 			_ = file.Close()
 			return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file")}
 		}
-		canonical, err := filepath.EvalSymlinks(info.Path)
-		if err != nil {
-			_ = file.Close()
-			return sourceRead{info: info}, []Diagnostic{classifySourceError(err, sourceID)}
-		}
-		info.Path = canonical
 		if stat.Size() > limit {
 			_ = file.Close()
 			return sourceRead{info: info}, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, sourceID, "source exceeds the configured byte limit")}

source_fifo_test.go

@@ -0,0 +1,71 @@
+//go:build darwin || linux
+
+package configkit
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+)
+
+func TestNonregularSourcesDoNotBlock(t *testing.T) {
+	if mode := os.Getenv("CONFIGKIT_FIFO_TEST_MODE"); mode != "" {
+		root := os.Getenv("CONFIGKIT_FIFO_TEST_ROOT")
+		source := File(filepath.Join(root, "config.yml"))
+		opts := Options[map[string]any]{}
+		switch mode {
+		case "file":
+			source = File(filepath.Join(root, "pipe.yml"))
+		case "import":
+			opts.Imports = true
+		case "profile":
+			opts.Profile = "fifo"
+		}
+		_, err := Load(source, opts)
+		var loadErr *LoadError
+		if !errors.As(err, &loadErr) {
+			t.Fatalf("expected safe load error, got %v", err)
+		}
+		for _, d := range loadErr.Diagnostics() {
+			if d.Code == CodeSourceNotRegular {
+				return
+			}
+		}
+		t.Fatalf("expected nonregular classification, got %v", loadErr.Diagnostics())
+	}
+	for _, mode := range []string{"file", "import", "profile"} {
+		t.Run(mode, func(t *testing.T) {
+			root := t.TempDir()
+			input := "value: 1\n"
+			if mode == "import" {
+				input = "_configkit:\n  imports:\n    - path: pipe.yml\n"
+			}
+			if err := os.WriteFile(filepath.Join(root, "config.yml"), []byte(input), 0600); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Mkdir(filepath.Join(root, "profiles"), 0700); err != nil {
+				t.Fatal(err)
+			}
+			fifo := filepath.Join(root, "pipe.yml")
+			if mode == "profile" {
+				fifo = filepath.Join(root, "profiles", "fifo.yml")
+			}
+			if err := syscall.Mkfifo(fifo, 0600); err != nil {
+				t.Fatal(err)
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestNonregularSourcesDoNotBlock$")
+			cmd.Env = append(os.Environ(), "CONFIGKIT_FIFO_TEST_MODE="+mode, "CONFIGKIT_FIFO_TEST_ROOT="+root)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				t.Fatalf("FIFO load failed or blocked: %v, %s", err, out)
+			}
+		})
+	}
+}

source_test.go

@@ -52,7 +52,7 @@ func TestFileSourcesReopenAndClassifyFailures(t *testing.T) {
 	}
 	source := File(path)
 	first, diagnostics := readSource(source, "source-1", defaultSourceLimits())
-	if len(diagnostics) != 0 || string(first.data) != "value: first\n" || first.info.Path != path {
+	if len(diagnostics) != 0 || string(first.data) != "value: first\n" || first.info.Path != canonicalTestPath(t, path) || first.info.Label != path {
 		t.Fatalf("first read = %#v, %#v", first, diagnostics)
 	}
 	if err := os.WriteFile(path, []byte("value: second\n"), 0o600); err != nil {

validation.go

@@ -112,6 +112,18 @@ func runValidation[T any](configuration *T, validator func(*T, *Validation), sch
 	for _, delta := range compareDocumentTrees(before, after) {
 		state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(delta.path))
 	}
+	beforeSensitivity := projectedSensitivity(before)
+	afterSensitivity := projectedSensitivity(after)
+	for pointer, path := range beforeSensitivity {
+		if _, exists := afterSensitivity[pointer]; !exists {
+			state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(path))
+		}
+	}
+	for pointer, path := range afterSensitivity {
+		if _, exists := beforeSensitivity[pointer]; !exists {
+			state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(path))
+		}
+	}
 	for _, path := range changedSecretStates(reflect.ValueOf(*configuration), schema.root, lookups) {
 		state.diagnostics = append(state.diagnostics, validationMutationDiagnostic(path))
 	}