Prepare post-decode CLI inspection context

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Sequence Identity And Combination Implementation Plan
 
-Status: implementation in progress; Stages 1–24 complete.
+Status: implementation in progress; Stages 1–25 complete.
 
 Audience: a Luna coding agent implementing one numbered stage per prompt, in
 order. This plan starts at commit
@@ -843,6 +843,12 @@ retain the existing public CLI lifecycle until Stage 26.
 Validation: history/metadata/resolution/secret tests and the everyday check.
 Public activation must not require a later stage to repair these mechanisms.
 
+**Completion:** Prepared detached post-decode inspection, resolution metadata,
+and effective secret-reference/location handoffs using existing history
+machinery. Verified the private target pipeline and unchanged public-load
+regressions with focused history/metadata/resolution/secret tests and
+`./scripts/check.sh`.
+
 ## Stage 26 — Activate CLI After Decoding In The Shared Lifecycle
 
 Scope: activate ADR 0018 through both loading paths using Stages 23–25's complete

internal/engine/explain.go

@@ -77,20 +77,61 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) {
 	// Intermediate input replacements do not permanently suppress defaults:
 	// decoding applies them to the final input tree. Show that reconciliation
 	// explicitly, including canonical scalar conversions, before domain changes.
+	decodedInput := r.input
+	decodedOrigins := r.inputOrigins
+	if r.preCLI != nil {
+		decodedInput = r.preCLI
+		decodedOrigins = r.preCLIOrigins
+	}
 	replayed, _ := documentAtPath(working, path)
-	decoded, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup)
+	decoded, _ := documentAtPathWithLookup(decodedInput, path, r.immutableLookup)
 	if !configurationValuesEqual(replayed, decoded) {
-		origin := cloneOrigin(r.inputOrigins[path.String()])
+		origin := cloneOrigin(decodedOrigins[path.String()])
 		origin.Operation = ChangeDecoding
 		explanation.History = append(explanation.History, ValueChange{
 			Before:  valueSnapshot(working, path, r.secrets),
-			After:   valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup),
+			After:   valueSnapshotWithLookup(decodedInput, path, r.secrets, r.immutableLookup),
 			Origin:  origin,
 			Reasons: []string{"applied retained defaults and canonical typed decoding"},
 		})
 	}
 
-	working = document.Clone(r.input)
+	working = document.Clone(decodedInput)
+	if r.preCLI != nil {
+		for _, application := range r.postDecodeApps {
+			operationPath, relevant := applicationOperationPath(application, path, r.immutableLookup)
+			var before ValueSnapshot
+			if relevant {
+				before = valueSnapshot(working, path, r.secrets)
+			}
+			applyHistoryApplication(&working, application, replayIndex)
+			if !relevant {
+				continue
+			}
+			after := valueSnapshot(working, path, r.secrets)
+			if !before.Present && !after.Present {
+				continue
+			}
+			explanation.History = append(explanation.History, ValueChange{
+				Before: before,
+				After:  after,
+				Origin: historyApplicationOrigin(application, operationPath, r.immutableLookup),
+			})
+		}
+		replayed, _ = documentAtPath(working, path)
+		effectiveInput, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup)
+		if !configurationValuesEqual(replayed, effectiveInput) {
+			origin := cloneOrigin(r.inputOrigins[path.String()])
+			origin.Operation = ChangeDecoding
+			explanation.History = append(explanation.History, ValueChange{
+				Before:  valueSnapshot(working, path, r.secrets),
+				After:   valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup),
+				Origin:  origin,
+				Reasons: []string{"applied the typed CLI assignment"},
+			})
+		}
+		working = document.Clone(r.input)
+	}
 	for _, snapshot := range r.resolutionReplay {
 		relevant := pathsIntersect(snapshot.change.Path, path)
 		var before ValueSnapshot

internal/engine/load.go

@@ -93,6 +93,10 @@ type Inspection struct {
 	baseline         *document.Node
 	input            *document.Node
 	inputOrigins     map[string]Origin
+	structuralInput  *document.Node
+	preCLI           *document.Node
+	preCLIOrigins    map[string]Origin
+	postDecodeApps   []historyApplication
 	final            *document.Node
 	immutableLookup  *documentLookup
 	resolution       []resolutionSnapshot

internal/engine/override_context.go

@@ -0,0 +1,151 @@
+package engine
+
+import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+
+// postDecodeOverrideContext keeps the structural, typed pre-CLI, and effective
+// post-CLI views distinct while preparing the existing inspection machinery.
+type postDecodeOverrideContext struct {
+	structuralInput *document.Node
+	preCLI          *document.Node
+	preCLIOrigins   map[string]Origin
+	decoded         decodedConfiguration
+	metadata        Metadata
+	resolution      resolutionInput
+	sources         []SourceInfo
+	applications    []historyApplication
+	postDecodeApps  []historyApplication
+	referenceInput  *document.Node
+}
+
+func preparePostDecodeOverrideContext(input assembledInput, before decodedConfiguration, assigned decodedOverrideResult) postDecodeOverrideContext {
+	after := before
+	after.value = assigned.value
+	after.presence = cloneInputPresence(assigned.presence)
+	after.origins = cloneOrigins(assigned.origins)
+	after.final = document.Clone(assigned.final)
+
+	metadata := metadataFromDecoded(after)
+	if input.inheritance != nil && input.composition != nil {
+		applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, input.composition.overlays, input.inheritance)
+	} else if input.composition != nil {
+		applyOverlayMetadata(&metadata, input.composition.overlays)
+	}
+
+	sources := cloneSourceInfos(input.sources)
+	knownSources := make(map[string]bool, len(sources)+len(assigned.overrides))
+	for _, source := range sources {
+		knownSources[source.ID] = true
+	}
+	for _, override := range assigned.overrides {
+		if !knownSources[override.source.ID] {
+			sources = append(sources, cloneSourceInfo(override.source))
+			knownSources[override.source.ID] = true
+		}
+	}
+	resolution := newResolutionInput(metadata, assigned.final, sources, input.contexts)
+
+	applications := structuralHistoryApplications(input)
+	var postDecodeApps []historyApplication
+	for _, override := range assigned.overrides {
+		postDecodeApps = append(postDecodeApps, historyApplication{
+			layer:  LayerCLI,
+			path:   override.path.clone(),
+			value:  document.Clone(override.value),
+			origin: overlayOrigin(LayerCLI, document.Location{SourceID: override.source.ID}),
+		})
+	}
+	allApplications := append(cloneHistoryApplications(applications), cloneHistoryApplications(postDecodeApps)...)
+	finalizeMetadataPaths(&metadata, assigned.final, allApplications, input.document, before.final)
+
+	return postDecodeOverrideContext{
+		structuralInput: document.Clone(input.document),
+		preCLI:          document.Clone(before.final),
+		preCLIOrigins:   cloneOrigins(before.origins),
+		decoded:         after,
+		metadata:        metadata,
+		resolution:      resolution,
+		sources:         sources,
+		applications:    applications,
+		postDecodeApps:  postDecodeApps,
+		referenceInput:  documentWithMetadataLocations(assigned.final, metadata),
+	}
+}
+
+func (context postDecodeOverrideContext) historyApplications() []historyApplication {
+	applications := cloneHistoryApplications(context.applications)
+	return append(applications, cloneHistoryApplications(context.postDecodeApps)...)
+}
+
+func structuralHistoryApplications(input assembledInput) []historyApplication {
+	if input.composition != nil {
+		return cloneHistoryApplications(input.composition.applications)
+	}
+	if input.document == nil {
+		return nil
+	}
+	return []historyApplication{{layer: LayerBase, value: document.Clone(input.document)}}
+}
+
+// documentWithMetadataLocations attaches effective origin coordinates to a
+// canonical typed projection for secret and validation diagnostics.
+func documentWithMetadataLocations(root *document.Node, metadata Metadata) *document.Node {
+	located := document.Clone(root)
+	if located == nil {
+		return nil
+	}
+	var visit func(*document.Node, Path)
+	visit = func(node *document.Node, path Path) {
+		pointer := path.String()
+		if info, exists := metadata.fields[pointer]; exists {
+			if location, ok := metadataFieldLocation(metadata, pointer, info.Origin); ok {
+				node.Location = location
+			}
+		}
+		switch node.Kind {
+		case document.Mapping:
+			for index := range node.Mapping {
+				entry := &node.Mapping[index]
+				childPath := path.Key(entry.Key)
+				if info, exists := metadata.fields[childPath.String()]; exists {
+					if location, ok := metadataFieldLocation(metadata, childPath.String(), info.Origin); ok {
+						entry.KeyLocation = location
+						entry.ValueLocation = location
+					}
+				}
+				visit(entry.Value, childPath)
+			}
+		case document.Sequence:
+			for index, child := range node.Sequence {
+				visit(child, path.Index(index))
+			}
+		}
+	}
+	visit(located, Path{})
+	return located
+}
+
+func metadataFieldLocation(metadata Metadata, pointer string, origin Origin) (document.Location, bool) {
+	if location, ok := latestOriginDocumentLocation(origin); ok {
+		return location, true
+	}
+	for index := len(metadata.history[pointer]) - 1; index >= 0; index-- {
+		change := metadata.history[pointer][index]
+		if location, ok := latestOriginDocumentLocation(change.Origin); ok {
+			return location, true
+		}
+		for prior := len(change.Previous) - 1; prior >= 0; prior-- {
+			if location, ok := latestOriginDocumentLocation(change.Previous[prior]); ok {
+				return location, true
+			}
+		}
+	}
+	return document.Location{}, false
+}
+
+func latestOriginDocumentLocation(origin Origin) (document.Location, bool) {
+	if len(origin.Locations) == 0 {
+		return document.Location{}, false
+	}
+	location := origin.Locations[len(origin.Locations)-1]
+	return document.Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column}, true
+}

internal/engine/override_context_test.go

@@ -0,0 +1,155 @@
+package engine
+
+import (
+	"reflect"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+func TestPostDecodeOverrideContextOrdersDecodeCLIAndResolutionEvidence(t *testing.T) {
+	t.Parallel()
+	type item struct {
+		Name    string `yaml:"name"`
+		Enabled bool   `yaml:"enabled"`
+	}
+	type config struct {
+		Version string  `yaml:"version"`
+		Rate    float64 `yaml:"rate"`
+		Count   int     `yaml:"count"`
+		Items   []item  `yaml:"items"`
+		APIKey  Secret  `yaml:"api_key"`
+		Derived int     `yaml:"derived"`
+	}
+	defaults := config{Version: "default", Rate: 2, Count: 7, Items: []item{{Name: "default"}}, APIKey: SecretEnv("DEFAULT_SECRET")}
+	structural := parseDocumentForDecode(t, "version: 1\nrate: 1.20\ncount: 5\nitems: [{name: old, enabled: true}]\napi_key: BASE_SECRET\n")
+	setOverrideTestDocumentSource(structural, "source-1")
+	versionPath := Keys("version")
+	schema, schemaIssue := buildSchema(reflect.TypeFor[config](), testSecretBinding())
+	if schemaIssue != nil {
+		t.Fatalf("buildSchema() issue = %#v", schemaIssue)
+	}
+	decoded, issues := decodeConfigurationWithContext(schema, reflect.ValueOf(defaults), structural, decodeContext{versionPath: &versionPath})
+	if len(issues) != 0 {
+		t.Fatalf("decodeConfigurationWithContext() issues = %#v", issues)
+	}
+	sources := []SourceInfo{{ID: "source-1", Label: "main input", Layer: LayerBase, Kind: SourceKindMapping}}
+	contexts := map[string]ResourceContext{"source-1": {BaseDir: "/config"}}
+	input := assembledInput{document: structural, sources: sources, contexts: contexts}
+	batch, diagnostics := preparePostStructuralOverrides(overridePreparationInput{root: structural}, sources, []Override{
+		{Path: Keys("rate"), Value: 1.2, Flag: "--rate"},
+		{Path: Keys("count"), Value: 5, Flag: "--count"},
+		{Path: Keys("items"), Value: []any{map[string]any{"name": "new"}}, Flag: "--items"},
+		{Path: Keys("api_key"), Value: "CLI_SECRET", Flag: "--api-key"},
+	}, nil, defaultImportLimits())
+	if len(diagnostics) != 0 {
+		t.Fatalf("preparePostStructuralOverrides() diagnostics = %#v", diagnostics)
+	}
+	assigned, issues := applyDecodedOverrides(schema, decoded, batch, decodeContext{versionPath: &versionPath})
+	if len(issues) != 0 {
+		t.Fatalf("applyDecodedOverrides() issues = %#v", issues)
+	}
+	prepared := preparePostDecodeOverrideContext(input, decoded, assigned)
+	if prepared.structuralInput == structural || prepared.preCLI == decoded.final || prepared.decoded.final == assigned.preCLI ||
+		prepared.decoded.final == assigned.final || prepared.referenceInput == assigned.final {
+		t.Fatal("structural, pre-CLI, post-CLI, and located snapshots are not detached")
+	}
+	if raw, ok := documentAtPath(prepared.structuralInput, Keys("version")); !ok || raw.Kind != document.Number || raw.Text != "1" {
+		t.Fatalf("structural version = %#v", raw)
+	}
+	if typed, ok := documentAtPath(prepared.preCLI, Keys("version")); !ok || typed.Kind != document.String || typed.Text != "1" {
+		t.Fatalf("pre-CLI typed version = %#v", typed)
+	}
+	if prepared.metadata.fields["/count"].Origin.Layer != LayerCLI || !prepared.metadata.fields["/count"].Explicit {
+		t.Fatalf("equal-value CLI metadata = %#v", prepared.metadata.fields["/count"])
+	}
+	if prepared.decoded.origins["/count"].Layer != LayerCLI || prepared.decoded.presence["/count"].location.SourceID != batch.overrides[1].source.ID {
+		t.Fatalf("post-CLI metadata maps = %#v / %#v", prepared.decoded.origins["/count"], prepared.decoded.presence["/count"])
+	}
+	if _, explicit := prepared.decoded.presence["/items/0/enabled"]; explicit {
+		t.Fatal("replaced list retained discarded-child presence")
+	}
+	if len(prepared.applications) != 1 || len(prepared.postDecodeApps) != len(batch.overrides) || len(prepared.historyApplications()) != 1+len(batch.overrides) {
+		t.Fatalf("history applications = structural:%d CLI:%d all:%d", len(prepared.applications), len(prepared.postDecodeApps), len(prepared.historyApplications()))
+	}
+	if len(prepared.sources) != len(sources)+len(batch.overrides) || prepared.sources[len(prepared.sources)-1].Layer != LayerCLI {
+		t.Fatalf("prepared source order = %#v", prepared.sources)
+	}
+	if prepared.resolution.contexts["source-1"].BaseDir != "/config" {
+		t.Fatalf("resolution source contexts = %#v", prepared.resolution.contexts)
+	}
+	if info, ok := prepared.resolution.lookup(Keys("count")); !ok || !info.Explicit || info.Origin.Layer != LayerCLI {
+		t.Fatalf("resolution entry count = %#v, %v", info, ok)
+	}
+	cliSource := batch.overrides[3].source.ID
+	if source, ok := prepared.resolution.source(cliSource); !ok || source.Layer != LayerCLI || source.Label != "--api-key" {
+		t.Fatalf("resolution CLI source = %#v, %v", source, ok)
+	}
+
+	inspection := Inspection{
+		metadata: prepared.metadata, baseline: decoded.baseline, input: assigned.final, inputOrigins: assigned.origins,
+		structuralInput: prepared.structuralInput, preCLI: prepared.preCLI, preCLIOrigins: prepared.preCLIOrigins,
+		postDecodeApps: prepared.postDecodeApps, final: assigned.final,
+		applications: prepared.applications,
+		immutableLookup: buildDocumentLookup(prepared.structuralInput, prepared.preCLI, assigned.final,
+			prepared.applications[0].value, prepared.postDecodeApps[0].value, prepared.postDecodeApps[1].value,
+			prepared.postDecodeApps[2].value, prepared.postDecodeApps[3].value),
+	}
+	rateExplanation, ok := inspection.Explain(Keys("rate"))
+	if !ok {
+		t.Fatal("Explain(rate) did not return a history")
+	}
+	if len(rateExplanation.History) != 4 || rateExplanation.History[0].Origin.Layer != LayerDefault ||
+		rateExplanation.History[1].Origin.Layer != LayerBase || rateExplanation.History[2].Origin.Operation != ChangeDecoding ||
+		rateExplanation.History[3].Origin.Layer != LayerCLI {
+		t.Fatalf("rate history order = %#v", rateExplanation.History)
+	}
+
+	configuration := assigned.value.Interface().(config)
+	resolver := func(value *config, resolution *Resolution) error {
+		if info, found := resolution.InputInfo(Keys("count")); !found || !info.Explicit || info.Origin.Layer != LayerCLI {
+			t.Errorf("resolver input info = %#v, %v", info, found)
+		}
+		if source, found := resolution.Source(cliSource); !found || source.Layer != LayerCLI {
+			t.Errorf("resolver CLI source = %#v, %v", source, found)
+		}
+		Default(resolution, Keys("count"), &value.Count, 99, "fill missing count")
+		Default(resolution, Keys("derived"), &value.Derived, 42, "derive default")
+		value.APIKey = SecretEnv("RESOLVED_SECRET")
+		resolution.Record(Keys("api_key"), ChangeNormalization, "select effective secret reference")
+		return nil
+	}
+	resolved := runResolution(&configuration, resolver, schema, prepared.decoded, prepared.referenceInput, prepared.resolution)
+	if len(resolved.diagnostics) != 0 || configuration.Count != 5 || configuration.Derived != 42 {
+		t.Fatalf("resolution = %#v, configuration = %#v", resolved, configuration)
+	}
+	applyResolutionMetadata(&prepared.metadata, resolved)
+	effectiveReferenceInput := documentWithMetadataLocations(resolved.final, prepared.metadata)
+	if node, ok := documentAtPath(effectiveReferenceInput, Keys("api_key")); !ok || node.Text != "RESOLVED_SECRET" || node.Location.SourceID != cliSource {
+		t.Fatalf("effective reference/location = %#v", node)
+	}
+	missing := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, effectiveReferenceInput, func(name string) (string, bool) {
+		if name != "RESOLVED_SECRET" {
+			t.Errorf("secret lookup name = %q", name)
+		}
+		return "", false
+	})
+	if len(missing.diagnostics) != 1 || missing.diagnostics[0].Code != CodeSecretUnset || missing.diagnostics[0].Primary == nil || missing.diagnostics[0].Primary.SourceID != cliSource {
+		t.Fatalf("secret failure handoff = %#v", missing.diagnostics)
+	}
+}
+
+func setOverrideTestDocumentSource(node *document.Node, sourceID string) {
+	if node == nil {
+		return
+	}
+	node.Location.SourceID = sourceID
+	for index := range node.Mapping {
+		node.Mapping[index].KeyLocation.SourceID = sourceID
+		node.Mapping[index].ValueLocation.SourceID = sourceID
+		setOverrideTestDocumentSource(node.Mapping[index].Value, sourceID)
+	}
+	for _, child := range node.Sequence {
+		setOverrideTestDocumentSource(child, sourceID)
+	}
+}